Date Time Ops (#1110)

* Added datetime operators

* Added date subtract support

README.md

@@ -124,7 +124,6 @@ rm /etc/myfile.tmp
 ```
 
 ### Run with Docker or Podman
-
 #### Oneshot use:
 
 ```bash
@@ -194,7 +193,7 @@ Or, in your Dockerfile:
 FROM mikefarah/yq
 
 USER root
-RUN apk add bash
+RUN apk add --no-cache bash
 USER yq
 ```
 

pkg/yqlib/context.go

@@ -3,6 +3,7 @@ package yqlib
 import (
 	"container/list"
 	"fmt"
+	"time"
 
 	"github.com/jinzhu/copier"
 	logging "gopkg.in/op/go-logging.v1"
@@ -12,6 +13,7 @@ type Context struct {
 	MatchingNodes  *list.List
 	Variables      map[string]*list.List
 	DontAutoCreate bool
+	datetimeLayout string
 }
 
 func (n *Context) SingleReadonlyChildContext(candidate *CandidateNode) Context {
@@ -28,6 +30,17 @@ func (n *Context) SingleChildContext(candidate *CandidateNode) Context {
 	return n.ChildContext(list)
 }
 
+func (n *Context) SetDateTimeLayout(newDateTimeLayout string) {
+	n.datetimeLayout = newDateTimeLayout
+}
+
+func (n *Context) GetDateTimeLayout() string {
+	if n.datetimeLayout != "" {
+		return n.datetimeLayout
+	}
+	return time.RFC3339
+}
+
 func (n *Context) GetVariable(name string) *list.List {
 	if n.Variables == nil {
 		return nil
@@ -43,7 +56,7 @@ func (n *Context) SetVariable(name string, value *list.List) {
 }
 
 func (n *Context) ChildContext(results *list.List) Context {
-	clone := Context{DontAutoCreate: n.DontAutoCreate}
+	clone := Context{DontAutoCreate: n.DontAutoCreate, datetimeLayout: n.datetimeLayout}
 	clone.Variables = make(map[string]*list.List)
 	if len(n.Variables) > 0 {
 		err := copier.Copy(&clone.Variables, n.Variables)

pkg/yqlib/doc/operators/add.md

@@ -206,6 +206,38 @@ a: 4
 b: 6
 ```
 
+## Date addition
+You can add durations to dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.
+
+Given a sample.yml file of:
+```yaml
+a: 2021-01-01T00:00:00Z
+```
+then
+```bash
+yq '.a += "3h10m"' sample.yml
+```
+will output
+```yaml
+a: 2021-01-01T03:10:00Z
+```
+
+## Date addition - custom format
+You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.
+
+Given a sample.yml file of:
+```yaml
+a: Saturday, 15-Dec-01 at 2:59AM GMT
+```
+then
+```bash
+yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")' sample.yml
+```
+will output
+```yaml
+a: Saturday, 15-Dec-01 at 6:00AM GMT
+```
+
 ## Add to null
 Adding to null simply returns the rhs
 

pkg/yqlib/doc/operators/datetime.md

@@ -0,0 +1,203 @@
+# Date Time
+
+Various operators for parsing and manipulating dates. 
+
+## Date time formattings
+This uses the golangs built in time library for parsing and formatting date times.
+
+When not specified, the RFC3339 standard is assumed `2006-01-02T15:04:05Z07:00` for parsing.
+
+To specify a custom parsing format, use the `with_dtf` operator. The first parameter sets the datetime parsing format for the expression in the second parameter. The expression can be any valid `yq` expression tree.
+
+```bash
+yq 'with_dtf("myformat"; .a + "3h" | tz("Australia/Melbourne"))'
+```
+
+See https://pkg.go.dev/time#pkg-constants for examples of formatting options.
+
+
+## Timezones
+This uses golangs built in LoadLocation function to parse timezones strings. See https://pkg.go.dev/time#LoadLocation for more details.
+
+
+## Durations
+Durations are parsed using golangs built in [ParseDuration](https://pkg.go.dev/time#ParseDuration) function.
+
+You can durations to time using the `+` operator.
+
+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
+
+`yq e <exp> <file>`
+{% endhint %}
+
+## Format: from standard RFC3339 format
+Providing a single parameter assumes a standard RFC3339 datetime format. If the target format is not a valid yaml datetime format, the result will be a string tagged node.
+
+Given a sample.yml file of:
+```yaml
+a: 2001-12-15T02:59:43.1Z
+```
+then
+```bash
+yq '.a |= format_datetime("Monday, 02-Jan-06 at 3:04PM")' sample.yml
+```
+will output
+```yaml
+a: Saturday, 15-Dec-01 at 2:59AM
+```
+
+## Format: from custom date time
+Use with_dtf to set a custom datetime format for parsing.
+
+Given a sample.yml file of:
+```yaml
+a: Saturday, 15-Dec-01 at 2:59AM
+```
+then
+```bash
+yq '.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM"; format_datetime("2006-01-02"))' sample.yml
+```
+will output
+```yaml
+a: 2001-12-15
+```
+
+## Format: get the day of the week
+Given a sample.yml file of:
+```yaml
+a: 2001-12-15T02:59:43.1Z
+```
+then
+```bash
+yq '.a | format_datetime("Monday")' sample.yml
+```
+will output
+```yaml
+Saturday
+```
+
+## Now
+Given a sample.yml file of:
+```yaml
+a: cool
+```
+then
+```bash
+yq '.updated = now' sample.yml
+```
+will output
+```yaml
+a: cool
+updated: 2021-05-19T01:02:03Z
+```
+
+## Timezone: from standard RFC3339 format
+Returns a new datetime in the specified timezone. Specify standard IANA Time Zone format or 'utc', 'local'. When given a single parameter, this assumes the datetime is in RFC3339 format.
+
+Given a sample.yml file of:
+```yaml
+a: cool
+```
+then
+```bash
+yq '.updated = (now | tz("Australia/Sydney"))' sample.yml
+```
+will output
+```yaml
+a: cool
+updated: 2021-05-19T11:02:03+10:00
+```
+
+## Timezone: with custom format
+Specify standard IANA Time Zone format or 'utc', 'local'
+
+Given a sample.yml file of:
+```yaml
+a: Saturday, 15-Dec-01 at 2:59AM GMT
+```
+then
+```bash
+yq '.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))' sample.yml
+```
+will output
+```yaml
+a: Saturday, 15-Dec-01 at 1:59PM AEDT
+```
+
+## Add and tz custom format
+Specify standard IANA Time Zone format or 'utc', 'local'
+
+Given a sample.yml file of:
+```yaml
+a: Saturday, 15-Dec-01 at 2:59AM GMT
+```
+then
+```bash
+yq '.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))' sample.yml
+```
+will output
+```yaml
+a: Saturday, 15-Dec-01 at 1:59PM AEDT
+```
+
+## Date addition
+Given a sample.yml file of:
+```yaml
+a: 2021-01-01T00:00:00Z
+```
+then
+```bash
+yq '.a += "3h10m"' sample.yml
+```
+will output
+```yaml
+a: 2021-01-01T03:10:00Z
+```
+
+## Date subtraction
+You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.
+
+Given a sample.yml file of:
+```yaml
+a: 2021-01-01T03:10:00Z
+```
+then
+```bash
+yq '.a -= "3h10m"' sample.yml
+```
+will output
+```yaml
+a: 2021-01-01T00:00:00Z
+```
+
+## Date addition - custom format
+Given a sample.yml file of:
+```yaml
+a: Saturday, 15-Dec-01 at 2:59AM GMT
+```
+then
+```bash
+yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a += "3h1m")' sample.yml
+```
+will output
+```yaml
+a: Saturday, 15-Dec-01 at 6:00AM GMT
+```
+
+## Date script with custom format
+You can embed full expressions in with_dtf if needed.
+
+Given a sample.yml file of:
+```yaml
+a: Saturday, 15-Dec-01 at 2:59AM GMT
+```
+then
+```bash
+yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a = (.a + "3h1m" | tz("Australia/Perth")))' sample.yml
+```
+will output
+```yaml
+a: Saturday, 15-Dec-01 at 2:00PM AWST
+```
+

pkg/yqlib/doc/operators/headers/datetime.md

@@ -0,0 +1,26 @@
+# Date Time
+
+Various operators for parsing and manipulating dates. 
+
+## Date time formattings
+This uses the golangs built in time library for parsing and formatting date times.
+
+When not specified, the RFC3339 standard is assumed `2006-01-02T15:04:05Z07:00` for parsing.
+
+To specify a custom parsing format, use the `with_dtf` operator. The first parameter sets the datetime parsing format for the expression in the second parameter. The expression can be any valid `yq` expression tree.
+
+```bash
+yq 'with_dtf("myformat"; .a + "3h" | tz("Australia/Melbourne"))'
+```
+
+See https://pkg.go.dev/time#pkg-constants for examples of formatting options.
+
+
+## Timezones
+This uses golangs built in LoadLocation function to parse timezones strings. See https://pkg.go.dev/time#LoadLocation for more details.
+
+
+## Durations
+Durations are parsed using golangs built in [ParseDuration](https://pkg.go.dev/time#ParseDuration) function.
+
+You can durations to time using the `+` operator.

pkg/yqlib/doc/operators/subtract.md

@@ -117,6 +117,38 @@ a: 2
 b: 4
 ```
 
+## Date subtraction
+You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.
+
+Given a sample.yml file of:
+```yaml
+a: 2021-01-01T03:10:00Z
+```
+then
+```bash
+yq '.a -= "3h10m"' sample.yml
+```
+will output
+```yaml
+a: 2021-01-01T00:00:00Z
+```
+
+## Date subtraction - custom format
+Use with_dtf to specify your datetime format. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.
+
+Given a sample.yml file of:
+```yaml
+a: Saturday, 15-Dec-01 at 6:00AM GMT
+```
+then
+```bash
+yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a -= "3h1m")' sample.yml
+```
+will output
+```yaml
+a: Saturday, 15-Dec-01 at 2:59AM GMT
+```
+
 ## Custom types: that are really numbers
 When custom tags are encountered, yq will try to decode the underlying type.
 

pkg/yqlib/expression_tokeniser.go

@@ -324,6 +324,11 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`flatten\([0-9]+\)`), flattenWithDepth())
 	lexer.Add([]byte(`flatten`), opTokenWithPrefs(flattenOpType, nil, flattenPreferences{depth: -1}))
 
+	lexer.Add([]byte(`format_datetime`), opToken(formatDateTimeOpType))
+	lexer.Add([]byte(`now`), opToken(nowOpType))
+	lexer.Add([]byte(`tz`), opToken(tzOpType))
+	lexer.Add([]byte(`with_dtf`), opToken(withDtFormatOpType))
+
 	lexer.Add([]byte(`toyaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat))
 	lexer.Add([]byte(`to_yaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat))
 

pkg/yqlib/lib.go

@@ -84,6 +84,12 @@ var collectOpType = &operationType{Type: "COLLECT", NumArgs: 1, Precedence: 50,
 var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 50, Handler: mapOperator}
 var evalOpType = &operationType{Type: "EVAL", NumArgs: 1, Precedence: 50, Handler: evalOperator}
 var mapValuesOpType = &operationType{Type: "MAP_VALUES", NumArgs: 1, Precedence: 50, Handler: mapValuesOperator}
+
+var formatDateTimeOpType = &operationType{Type: "FORMAT_DATE_TIME", NumArgs: 1, Precedence: 50, Handler: formatDateTime}
+var withDtFormatOpType = &operationType{Type: "WITH_DATE_TIME_FORMAT", NumArgs: 1, Precedence: 50, Handler: withDateTimeFormat}
+var nowOpType = &operationType{Type: "NOW", NumArgs: 0, Precedence: 50, Handler: nowOp}
+var tzOpType = &operationType{Type: "TIMEZONE", NumArgs: 1, Precedence: 50, Handler: tzOp}
+
 var encodeOpType = &operationType{Type: "ENCODE", NumArgs: 0, Precedence: 50, Handler: encodeOperator}
 var decodeOpType = &operationType{Type: "DECODE", NumArgs: 0, Precedence: 50, Handler: decodeOperator}
 

pkg/yqlib/operator_add.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+	"time"
 
 	yaml "gopkg.in/yaml.v3"
 )
@@ -71,7 +72,7 @@ func add(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Candida
 		}
 		target.Node.Kind = yaml.ScalarNode
 		target.Node.Style = lhsNode.Style
-		if err := addScalars(target, lhsNode, rhs.Node); err != nil {
+		if err := addScalars(context, target, lhsNode, rhs.Node); err != nil {
 			return nil, err
 		}
 	}
@@ -97,7 +98,7 @@ func guessTagFromCustomType(node *yaml.Node) string {
 	return guessedTag
 }
 
-func addScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error {
+func addScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error {
 	lhsTag := lhs.Tag
 	rhsTag := rhs.Tag
 	lhsIsCustom := false
@@ -112,7 +113,18 @@ func addScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error {
 		rhsTag = guessTagFromCustomType(rhs)
 	}
 
-	if lhsTag == "!!str" {
+	isDateTime := lhs.Tag == "!!timestamp"
+
+	// if the lhs is a string, it might be a timestamp in a custom format.
+	if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 {
+		_, err := time.Parse(context.GetDateTimeLayout(), lhs.Value)
+		isDateTime = err == nil
+	}
+
+	if isDateTime {
+		return addDateTimes(context.GetDateTimeLayout(), target, lhs, rhs)
+
+	} else if lhsTag == "!!str" {
 		target.Node.Tag = lhs.Tag
 		target.Node.Value = lhs.Value + rhs.Value
 	} else if lhsTag == "!!int" && rhsTag == "!!int" {
@@ -149,6 +161,24 @@ func addScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error {
 	return nil
 }
 
+func addDateTimes(layout string, target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error {
+
+	duration, err := time.ParseDuration(rhs.Value)
+	if err != nil {
+		return fmt.Errorf("unable to parse duration [%v]: %w", rhs.Value, err)
+	}
+
+	currentTime, err := time.Parse(layout, lhs.Value)
+	if err != nil {
+		return err
+	}
+
+	newTime := currentTime.Add(duration)
+	target.Node.Value = newTime.Format(layout)
+	return nil
+
+}
+
 func addSequences(target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error {
 	target.Node.Kind = yaml.SequenceNode
 	if len(lhs.Node.Content) > 0 {

pkg/yqlib/operator_add_test.go

@@ -208,6 +208,34 @@ var addOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{a: 4, b: 6}\n",
 		},
 	},
+	{
+		description:    "Date addition",
+		subdescription: "You can add durations to dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",
+		document:       `a: 2021-01-01T00:00:00Z`,
+		expression:     `.a += "3h10m"`,
+		expected: []string{
+			"D0, P[], (doc)::a: 2021-01-01T03:10:00Z\n",
+		},
+	},
+	{
+		description:    "Date addition - custom format",
+		subdescription: "You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",
+		document:       `a: Saturday, 15-Dec-01 at 2:59AM GMT`,
+		expression:     `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")`,
+		expected: []string{
+			"D0, P[], (doc)::a: Saturday, 15-Dec-01 at 6:00AM GMT\n",
+		},
+	},
+	{
+		skipDoc:        true,
+		description:    "Date addition - custom format",
+		subdescription: "You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",
+		document:       `a: !cat Saturday, 15-Dec-01 at 2:59AM GMT`,
+		expression:     `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")`,
+		expected: []string{
+			"D0, P[], (doc)::a: !cat Saturday, 15-Dec-01 at 6:00AM GMT\n",
+		},
+	},
 	{
 		description:    "Add to null",
 		subdescription: "Adding to null simply returns the rhs",

pkg/yqlib/operator_datetime.go

@@ -0,0 +1,126 @@
+package yqlib
+
+import (
+	"container/list"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"gopkg.in/yaml.v3"
+)
+
+func getStringParamter(parameterName string, d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (string, error) {
+	result, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode)
+
+	if err != nil {
+		return "", err
+	} else if result.MatchingNodes.Len() == 0 {
+		return "", fmt.Errorf("could not find %v for format_time", parameterName)
+	}
+
+	return result.MatchingNodes.Front().Value.(*CandidateNode).Node.Value, nil
+}
+
+func withDateTimeFormat(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	if expressionNode.RHS.Operation.OperationType == blockOpType || expressionNode.RHS.Operation.OperationType == unionOpType {
+		layout, err := getStringParamter("layout", d, context, expressionNode.RHS.LHS)
+		if err != nil {
+			return Context{}, fmt.Errorf("could not get date time format: %w", err)
+		}
+		context.SetDateTimeLayout(layout)
+		return d.GetMatchingNodes(context, expressionNode.RHS.RHS)
+
+	}
+	return Context{}, errors.New(`must provide a date time format string and an expression, e.g. with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; <exp>)`)
+
+}
+
+// for unit tests
+var Now = time.Now
+
+func nowOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	node := &yaml.Node{
+		Tag:   "!!timestamp",
+		Kind:  yaml.ScalarNode,
+		Value: Now().Format(time.RFC3339),
+	}
+
+	return context.SingleChildContext(&CandidateNode{Node: node}), nil
+
+}
+
+func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	format, err := getStringParamter("format", d, context, expressionNode.RHS)
+	layout := context.GetDateTimeLayout()
+	decoder := NewYamlDecoder()
+
+	if err != nil {
+		return Context{}, err
+	}
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+
+		parsedTime, err := time.Parse(layout, candidate.Node.Value)
+		if err != nil {
+			return Context{}, fmt.Errorf("could not parse datetime of [%v]: %w", candidate.GetNicePath(), err)
+		}
+		formattedTimeStr := parsedTime.Format(format)
+		decoder.Init(strings.NewReader(formattedTimeStr))
+		var dataBucket yaml.Node
+		errorReading := decoder.Decode(&dataBucket)
+		var node *yaml.Node
+		if errorReading != nil {
+			log.Debugf("could not parse %v - lets just leave it as a string", formattedTimeStr)
+			node = &yaml.Node{
+				Kind:  yaml.ScalarNode,
+				Tag:   "!!str",
+				Value: formattedTimeStr,
+			}
+		} else {
+			node = unwrapDoc(&dataBucket)
+		}
+
+		results.PushBack(candidate.CreateReplacement(node))
+	}
+
+	return context.ChildContext(results), nil
+}
+
+func tzOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	timezoneStr, err := getStringParamter("timezone", d, context, expressionNode.RHS)
+	layout := context.GetDateTimeLayout()
+
+	if err != nil {
+		return Context{}, err
+	}
+	var results = list.New()
+
+	timezone, err := time.LoadLocation(timezoneStr)
+	if err != nil {
+		return Context{}, fmt.Errorf("could not load tz [%v]: %w", timezoneStr, err)
+	}
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+
+		parsedTime, err := time.Parse(layout, candidate.Node.Value)
+		if err != nil {
+			return Context{}, fmt.Errorf("could not parse datetime of [%v] using layout [%v]: %w", candidate.GetNicePath(), layout, err)
+		}
+		tzTime := parsedTime.In(timezone)
+
+		node := &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   candidate.Node.Tag,
+			Value: tzTime.Format(layout),
+		}
+
+		results.PushBack(candidate.CreateReplacement(node))
+	}
+
+	return context.ChildContext(results), nil
+}

pkg/yqlib/operator_datetime_test.go

@@ -0,0 +1,120 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var dateTimeOperatorScenarios = []expressionScenario{
+	{
+		description:    "Format: from standard RFC3339 format",
+		subdescription: "Providing a single parameter assumes a standard RFC3339 datetime format. If the target format is not a valid yaml datetime format, the result will be a string tagged node.",
+		document:       `a: 2001-12-15T02:59:43.1Z`,
+		expression:     `.a |= format_datetime("Monday, 02-Jan-06 at 3:04PM")`,
+		expected: []string{
+			"D0, P[], (doc)::a: Saturday, 15-Dec-01 at 2:59AM\n",
+		},
+	},
+	{
+		description:    "Format: from custom date time",
+		subdescription: "Use with_dtf to set a custom datetime format for parsing.",
+		document:       `a: Saturday, 15-Dec-01 at 2:59AM`,
+		expression:     `.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM"; format_datetime("2006-01-02"))`,
+		expected: []string{
+			"D0, P[], (doc)::a: 2001-12-15\n",
+		},
+	},
+	{
+		description: "Format: get the day of the week",
+		document:    `a: 2001-12-15T02:59:43.1Z`,
+		expression:  `.a | format_datetime("Monday")`,
+		expected: []string{
+			"D0, P[a], (!!str)::Saturday\n",
+		},
+	},
+	{
+		description: "Now",
+		document:    "a: cool",
+		expression:  `.updated = now`,
+		expected: []string{
+			"D0, P[], (doc)::a: cool\nupdated: 2021-05-19T01:02:03Z\n",
+		},
+	},
+	{
+		description:    "Timezone: from standard RFC3339 format",
+		subdescription: "Returns a new datetime in the specified timezone. Specify standard IANA Time Zone format or 'utc', 'local'. When given a single parameter, this assumes the datetime is in RFC3339 format.",
+
+		document:   "a: cool",
+		expression: `.updated = (now | tz("Australia/Sydney"))`,
+		expected: []string{
+			"D0, P[], (doc)::a: cool\nupdated: 2021-05-19T11:02:03+10:00\n",
+		},
+	},
+	{
+		description:    "Timezone: with custom format",
+		subdescription: "Specify standard IANA Time Zone format or 'utc', 'local'",
+		document:       "a: Saturday, 15-Dec-01 at 2:59AM GMT",
+		expression:     `.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))`,
+		expected: []string{
+			"D0, P[], (doc)::a: Saturday, 15-Dec-01 at 1:59PM AEDT\n",
+		},
+	},
+	{
+		description:    "Add and tz custom format",
+		subdescription: "Specify standard IANA Time Zone format or 'utc', 'local'",
+		document:       "a: Saturday, 15-Dec-01 at 2:59AM GMT",
+		expression:     `.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))`,
+		expected: []string{
+			"D0, P[], (doc)::a: Saturday, 15-Dec-01 at 1:59PM AEDT\n",
+		},
+	},
+	{
+		description: "Date addition",
+		document:    `a: 2021-01-01T00:00:00Z`,
+		expression:  `.a += "3h10m"`,
+		expected: []string{
+			"D0, P[], (doc)::a: 2021-01-01T03:10:00Z\n",
+		},
+	},
+	{
+		description:    "Date subtraction",
+		subdescription: "You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",
+		document:       `a: 2021-01-01T03:10:00Z`,
+		expression:     `.a -= "3h10m"`,
+		expected: []string{
+			"D0, P[], (doc)::a: 2021-01-01T00:00:00Z\n",
+		},
+	},
+	{
+		description: "Date addition - custom format",
+		document:    `a: Saturday, 15-Dec-01 at 2:59AM GMT`,
+		expression:  `with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a += "3h1m")`,
+		expected: []string{
+			"D0, P[], (doc)::a: Saturday, 15-Dec-01 at 6:00AM GMT\n",
+		},
+	},
+	{
+		description:    "Date script with custom format",
+		subdescription: "You can embed full expressions in with_dtf if needed.",
+		document:       `a: Saturday, 15-Dec-01 at 2:59AM GMT`,
+		expression:     `with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a = (.a + "3h1m" | tz("Australia/Perth")))`,
+		expected: []string{
+			"D0, P[], (doc)::a: Saturday, 15-Dec-01 at 2:00PM AWST\n",
+		},
+	},
+	{
+		description: "allow comma",
+		skipDoc:     true,
+		document:    "a: Saturday, 15-Dec-01 at 2:59AM GMT",
+		expression:  `.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST", tz("Australia/Sydney"))`,
+		expected: []string{
+			"D0, P[], (doc)::a: Saturday, 15-Dec-01 at 1:59PM AEDT\n",
+		},
+	},
+}
+
+func TestDatetimeOperatorScenarios(t *testing.T) {
+	for _, tt := range dateTimeOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentOperatorScenarios(t, "datetime", dateTimeOperatorScenarios)
+}

pkg/yqlib/operator_subtract.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+	"time"
 
 	"gopkg.in/yaml.v3"
 )
@@ -56,7 +57,7 @@ func subtract(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Ca
 
 	switch lhsNode.Kind {
 	case yaml.MappingNode:
-		return nil, fmt.Errorf("Maps not yet supported for subtraction")
+		return nil, fmt.Errorf("maps not yet supported for subtraction")
 	case yaml.SequenceNode:
 		if rhs.Node.Kind != yaml.SequenceNode {
 			return nil, fmt.Errorf("%v (%v) cannot be subtracted from %v", rhs.Node.Tag, rhs.Path, lhsNode.Tag)
@@ -68,13 +69,15 @@ func subtract(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Ca
 		}
 		target.Node.Kind = yaml.ScalarNode
 		target.Node.Style = lhsNode.Style
-		return subtractScalars(target, lhsNode, rhs.Node)
+		if err := subtractScalars(context, target, lhsNode, rhs.Node); err != nil {
+			return nil, err
+		}
 	}
 
 	return target, nil
 }
 
-func subtractScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) (*CandidateNode, error) {
+func subtractScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error {
 	lhsTag := lhs.Tag
 	rhsTag := rhs.Tag
 	lhsIsCustom := false
@@ -89,16 +92,25 @@ func subtractScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) (*Ca
 		rhsTag = guessTagFromCustomType(rhs)
 	}
 
-	if lhsTag == "!!str" {
+	isDateTime := lhs.Tag == "!!timestamp"
-		return nil, fmt.Errorf("strings cannot be subtracted")
+	// if the lhs is a string, it might be a timestamp in a custom format.
+	if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 {
+		_, err := time.Parse(context.GetDateTimeLayout(), lhs.Value)
+		isDateTime = err == nil
+	}
+
+	if isDateTime {
+		return subtractDateTime(context.GetDateTimeLayout(), target, lhs, rhs)
+	} else if lhsTag == "!!str" {
+		return fmt.Errorf("strings cannot be subtracted")
 	} else if lhsTag == "!!int" && rhsTag == "!!int" {
 		format, lhsNum, err := parseInt(lhs.Value)
 		if err != nil {
-			return nil, err
+			return err
 		}
 		_, rhsNum, err := parseInt(rhs.Value)
 		if err != nil {
-			return nil, err
+			return err
 		}
 		result := lhsNum - rhsNum
 		target.Node.Tag = lhs.Tag
@@ -106,11 +118,11 @@ func subtractScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) (*Ca
 	} else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") {
 		lhsNum, err := strconv.ParseFloat(lhs.Value, 64)
 		if err != nil {
-			return nil, err
+			return err
 		}
 		rhsNum, err := strconv.ParseFloat(rhs.Value, 64)
 		if err != nil {
-			return nil, err
+			return err
 		}
 		result := lhsNum - rhsNum
 		if lhsIsCustom {
@@ -120,8 +132,31 @@ func subtractScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) (*Ca
 		}
 		target.Node.Value = fmt.Sprintf("%v", result)
 	} else {
-		return nil, fmt.Errorf("%v cannot be added to %v", lhs.Tag, rhs.Tag)
+		return fmt.Errorf("%v cannot be added to %v", lhs.Tag, rhs.Tag)
 	}
 
-	return target, nil
+	return nil
+}
+
+func subtractDateTime(layout string, target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error {
+	var durationStr string
+	if strings.HasPrefix(rhs.Value, "-") {
+		durationStr = rhs.Value[1:]
+	} else {
+		durationStr = "-" + rhs.Value
+	}
+	duration, err := time.ParseDuration(durationStr)
+
+	if err != nil {
+		return fmt.Errorf("unable to parse duration [%v]: %w", rhs.Value, err)
+	}
+
+	currentTime, err := time.Parse(layout, lhs.Value)
+	if err != nil {
+		return err
+	}
+
+	newTime := currentTime.Add(duration)
+	target.Node.Value = newTime.Format(layout)
+	return nil
 }

pkg/yqlib/operator_subtract_test.go

@@ -93,6 +93,34 @@ var subtractOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{a: 2, b: 4}\n",
 		},
 	},
+	{
+		description:    "Date subtraction",
+		subdescription: "You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",
+		document:       `a: 2021-01-01T03:10:00Z`,
+		expression:     `.a -= "3h10m"`,
+		expected: []string{
+			"D0, P[], (doc)::a: 2021-01-01T00:00:00Z\n",
+		},
+	},
+	{
+		description:    "Date subtraction - custom format",
+		subdescription: "Use with_dtf to specify your datetime format. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",
+		document:       `a: Saturday, 15-Dec-01 at 6:00AM GMT`,
+		expression:     `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a -= "3h1m")`,
+		expected: []string{
+			"D0, P[], (doc)::a: Saturday, 15-Dec-01 at 2:59AM GMT\n",
+		},
+	},
+	{
+		skipDoc:        true,
+		description:    "Date subtraction - custom format",
+		subdescription: "You can subtract durations from dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",
+		document:       `a: !cat Saturday, 15-Dec-01 at 6:00AM GMT`,
+		expression:     `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a -= "3h1m")`,
+		expected: []string{
+			"D0, P[], (doc)::a: !cat Saturday, 15-Dec-01 at 2:59AM GMT\n",
+		},
+	},
 	{
 		description:    "Custom types: that are really numbers",
 		subdescription: "When custom tags are encountered, yq will try to decode the underlying type.",

pkg/yqlib/operators_test.go

@@ -10,6 +10,7 @@ import (
 	"sort"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/mikefarah/yq/v4/test"
 	"gopkg.in/op/go-logging.v1"
@@ -31,6 +32,9 @@ type expressionScenario struct {
 
 func TestMain(m *testing.M) {
 	logging.SetLevel(logging.ERROR, "")
+	Now = func() time.Time {
+		return time.Date(2021, time.May, 19, 1, 2, 3, 4, time.UTC)
+	}
 	code := m.Run()
 	os.Exit(code)
 }