diff --git a/README.md b/README.md index 2144739c..201b897f 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,6 @@ rm /etc/myfile.tmp ``` ### Run with Docker or Podman - #### Oneshot use: ```bash @@ -194,7 +193,7 @@ Or, in your Dockerfile: FROM mikefarah/yq USER root -RUN apk add bash +RUN apk add --no-cache bash USER yq ``` diff --git a/pkg/yqlib/context.go b/pkg/yqlib/context.go index 6be006f0..17cddb3c 100644 --- a/pkg/yqlib/context.go +++ b/pkg/yqlib/context.go @@ -3,6 +3,7 @@ package yqlib import ( "container/list" "fmt" + "time" "github.com/jinzhu/copier" logging "gopkg.in/op/go-logging.v1" @@ -12,6 +13,7 @@ type Context struct { MatchingNodes *list.List Variables map[string]*list.List DontAutoCreate bool + datetimeLayout string } func (n *Context) SingleReadonlyChildContext(candidate *CandidateNode) Context { @@ -28,6 +30,17 @@ func (n *Context) SingleChildContext(candidate *CandidateNode) Context { return n.ChildContext(list) } +func (n *Context) SetDateTimeLayout(newDateTimeLayout string) { + n.datetimeLayout = newDateTimeLayout +} + +func (n *Context) GetDateTimeLayout() string { + if n.datetimeLayout != "" { + return n.datetimeLayout + } + return time.RFC3339 +} + func (n *Context) GetVariable(name string) *list.List { if n.Variables == nil { return nil @@ -43,7 +56,7 @@ func (n *Context) SetVariable(name string, value *list.List) { } func (n *Context) ChildContext(results *list.List) Context { - clone := Context{DontAutoCreate: n.DontAutoCreate} + clone := Context{DontAutoCreate: n.DontAutoCreate, datetimeLayout: n.datetimeLayout} clone.Variables = make(map[string]*list.List) if len(n.Variables) > 0 { err := copier.Copy(&clone.Variables, n.Variables) diff --git a/pkg/yqlib/doc/operators/add.md b/pkg/yqlib/doc/operators/add.md index 8a61c0eb..52452ef4 100644 --- a/pkg/yqlib/doc/operators/add.md +++ b/pkg/yqlib/doc/operators/add.md @@ -206,6 +206,38 @@ a: 4 b: 6 ``` +## Date addition +You can add durations to dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information. + +Given a sample.yml file of: +```yaml +a: 2021-01-01T00:00:00Z +``` +then +```bash +yq '.a += "3h10m"' sample.yml +``` +will output +```yaml +a: 2021-01-01T03:10:00Z +``` + +## Date addition - custom format +You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information. + +Given a sample.yml file of: +```yaml +a: Saturday, 15-Dec-01 at 2:59AM GMT +``` +then +```bash +yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")' sample.yml +``` +will output +```yaml +a: Saturday, 15-Dec-01 at 6:00AM GMT +``` + ## Add to null Adding to null simply returns the rhs diff --git a/pkg/yqlib/doc/operators/datetime.md b/pkg/yqlib/doc/operators/datetime.md new file mode 100644 index 00000000..f97a37ea --- /dev/null +++ b/pkg/yqlib/doc/operators/datetime.md @@ -0,0 +1,203 @@ +# Date Time + +Various operators for parsing and manipulating dates. + +## Date time formattings +This uses the golangs built in time library for parsing and formatting date times. + +When not specified, the RFC3339 standard is assumed `2006-01-02T15:04:05Z07:00` for parsing. + +To specify a custom parsing format, use the `with_dtf` operator. The first parameter sets the datetime parsing format for the expression in the second parameter. The expression can be any valid `yq` expression tree. + +```bash +yq 'with_dtf("myformat"; .a + "3h" | tz("Australia/Melbourne"))' +``` + +See https://pkg.go.dev/time#pkg-constants for examples of formatting options. + + +## Timezones +This uses golangs built in LoadLocation function to parse timezones strings. See https://pkg.go.dev/time#LoadLocation for more details. + + +## Durations +Durations are parsed using golangs built in [ParseDuration](https://pkg.go.dev/time#ParseDuration) function. + +You can durations to time using the `+` operator. + +{% hint style="warning" %} +Note that versions prior to 4.18 require the 'eval/e' command to be specified. + +`yq e ` +{% endhint %} + +## Format: from standard RFC3339 format +Providing a single parameter assumes a standard RFC3339 datetime format. If the target format is not a valid yaml datetime format, the result will be a string tagged node. + +Given a sample.yml file of: +```yaml +a: 2001-12-15T02:59:43.1Z +``` +then +```bash +yq '.a |= format_datetime("Monday, 02-Jan-06 at 3:04PM")' sample.yml +``` +will output +```yaml +a: Saturday, 15-Dec-01 at 2:59AM +``` + +## Format: from custom date time +Use with_dtf to set a custom datetime format for parsing. + +Given a sample.yml file of: +```yaml +a: Saturday, 15-Dec-01 at 2:59AM +``` +then +```bash +yq '.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM"; format_datetime("2006-01-02"))' sample.yml +``` +will output +```yaml +a: 2001-12-15 +``` + +## Format: get the day of the week +Given a sample.yml file of: +```yaml +a: 2001-12-15T02:59:43.1Z +``` +then +```bash +yq '.a | format_datetime("Monday")' sample.yml +``` +will output +```yaml +Saturday +``` + +## Now +Given a sample.yml file of: +```yaml +a: cool +``` +then +```bash +yq '.updated = now' sample.yml +``` +will output +```yaml +a: cool +updated: 2021-05-19T01:02:03Z +``` + +## Timezone: from standard RFC3339 format +Returns a new datetime in the specified timezone. Specify standard IANA Time Zone format or 'utc', 'local'. When given a single parameter, this assumes the datetime is in RFC3339 format. + +Given a sample.yml file of: +```yaml +a: cool +``` +then +```bash +yq '.updated = (now | tz("Australia/Sydney"))' sample.yml +``` +will output +```yaml +a: cool +updated: 2021-05-19T11:02:03+10:00 +``` + +## Timezone: with custom format +Specify standard IANA Time Zone format or 'utc', 'local' + +Given a sample.yml file of: +```yaml +a: Saturday, 15-Dec-01 at 2:59AM GMT +``` +then +```bash +yq '.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))' sample.yml +``` +will output +```yaml +a: Saturday, 15-Dec-01 at 1:59PM AEDT +``` + +## Add and tz custom format +Specify standard IANA Time Zone format or 'utc', 'local' + +Given a sample.yml file of: +```yaml +a: Saturday, 15-Dec-01 at 2:59AM GMT +``` +then +```bash +yq '.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))' sample.yml +``` +will output +```yaml +a: Saturday, 15-Dec-01 at 1:59PM AEDT +``` + +## Date addition +Given a sample.yml file of: +```yaml +a: 2021-01-01T00:00:00Z +``` +then +```bash +yq '.a += "3h10m"' sample.yml +``` +will output +```yaml +a: 2021-01-01T03:10:00Z +``` + +## Date subtraction +You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information. + +Given a sample.yml file of: +```yaml +a: 2021-01-01T03:10:00Z +``` +then +```bash +yq '.a -= "3h10m"' sample.yml +``` +will output +```yaml +a: 2021-01-01T00:00:00Z +``` + +## Date addition - custom format +Given a sample.yml file of: +```yaml +a: Saturday, 15-Dec-01 at 2:59AM GMT +``` +then +```bash +yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a += "3h1m")' sample.yml +``` +will output +```yaml +a: Saturday, 15-Dec-01 at 6:00AM GMT +``` + +## Date script with custom format +You can embed full expressions in with_dtf if needed. + +Given a sample.yml file of: +```yaml +a: Saturday, 15-Dec-01 at 2:59AM GMT +``` +then +```bash +yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a = (.a + "3h1m" | tz("Australia/Perth")))' sample.yml +``` +will output +```yaml +a: Saturday, 15-Dec-01 at 2:00PM AWST +``` + diff --git a/pkg/yqlib/doc/operators/headers/datetime.md b/pkg/yqlib/doc/operators/headers/datetime.md new file mode 100644 index 00000000..39b0dd67 --- /dev/null +++ b/pkg/yqlib/doc/operators/headers/datetime.md @@ -0,0 +1,26 @@ +# Date Time + +Various operators for parsing and manipulating dates. + +## Date time formattings +This uses the golangs built in time library for parsing and formatting date times. + +When not specified, the RFC3339 standard is assumed `2006-01-02T15:04:05Z07:00` for parsing. + +To specify a custom parsing format, use the `with_dtf` operator. The first parameter sets the datetime parsing format for the expression in the second parameter. The expression can be any valid `yq` expression tree. + +```bash +yq 'with_dtf("myformat"; .a + "3h" | tz("Australia/Melbourne"))' +``` + +See https://pkg.go.dev/time#pkg-constants for examples of formatting options. + + +## Timezones +This uses golangs built in LoadLocation function to parse timezones strings. See https://pkg.go.dev/time#LoadLocation for more details. + + +## Durations +Durations are parsed using golangs built in [ParseDuration](https://pkg.go.dev/time#ParseDuration) function. + +You can durations to time using the `+` operator. diff --git a/pkg/yqlib/doc/operators/subtract.md b/pkg/yqlib/doc/operators/subtract.md index da5bfe48..a8f8d338 100644 --- a/pkg/yqlib/doc/operators/subtract.md +++ b/pkg/yqlib/doc/operators/subtract.md @@ -117,6 +117,38 @@ a: 2 b: 4 ``` +## Date subtraction +You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information. + +Given a sample.yml file of: +```yaml +a: 2021-01-01T03:10:00Z +``` +then +```bash +yq '.a -= "3h10m"' sample.yml +``` +will output +```yaml +a: 2021-01-01T00:00:00Z +``` + +## Date subtraction - custom format +Use with_dtf to specify your datetime format. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information. + +Given a sample.yml file of: +```yaml +a: Saturday, 15-Dec-01 at 6:00AM GMT +``` +then +```bash +yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a -= "3h1m")' sample.yml +``` +will output +```yaml +a: Saturday, 15-Dec-01 at 2:59AM GMT +``` + ## Custom types: that are really numbers When custom tags are encountered, yq will try to decode the underlying type. diff --git a/pkg/yqlib/expression_tokeniser.go b/pkg/yqlib/expression_tokeniser.go index aae20761..0c8d9628 100644 --- a/pkg/yqlib/expression_tokeniser.go +++ b/pkg/yqlib/expression_tokeniser.go @@ -324,6 +324,11 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`flatten\([0-9]+\)`), flattenWithDepth()) lexer.Add([]byte(`flatten`), opTokenWithPrefs(flattenOpType, nil, flattenPreferences{depth: -1})) + lexer.Add([]byte(`format_datetime`), opToken(formatDateTimeOpType)) + lexer.Add([]byte(`now`), opToken(nowOpType)) + lexer.Add([]byte(`tz`), opToken(tzOpType)) + lexer.Add([]byte(`with_dtf`), opToken(withDtFormatOpType)) + lexer.Add([]byte(`toyaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat)) lexer.Add([]byte(`to_yaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat)) diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 2e80c0b7..db91dcca 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -84,6 +84,12 @@ var collectOpType = &operationType{Type: "COLLECT", NumArgs: 1, Precedence: 50, var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 50, Handler: mapOperator} var evalOpType = &operationType{Type: "EVAL", NumArgs: 1, Precedence: 50, Handler: evalOperator} var mapValuesOpType = &operationType{Type: "MAP_VALUES", NumArgs: 1, Precedence: 50, Handler: mapValuesOperator} + +var formatDateTimeOpType = &operationType{Type: "FORMAT_DATE_TIME", NumArgs: 1, Precedence: 50, Handler: formatDateTime} +var withDtFormatOpType = &operationType{Type: "WITH_DATE_TIME_FORMAT", NumArgs: 1, Precedence: 50, Handler: withDateTimeFormat} +var nowOpType = &operationType{Type: "NOW", NumArgs: 0, Precedence: 50, Handler: nowOp} +var tzOpType = &operationType{Type: "TIMEZONE", NumArgs: 1, Precedence: 50, Handler: tzOp} + var encodeOpType = &operationType{Type: "ENCODE", NumArgs: 0, Precedence: 50, Handler: encodeOperator} var decodeOpType = &operationType{Type: "DECODE", NumArgs: 0, Precedence: 50, Handler: decodeOperator} diff --git a/pkg/yqlib/operator_add.go b/pkg/yqlib/operator_add.go index d1ba7d32..efcefdfd 100644 --- a/pkg/yqlib/operator_add.go +++ b/pkg/yqlib/operator_add.go @@ -4,6 +4,7 @@ import ( "fmt" "strconv" "strings" + "time" yaml "gopkg.in/yaml.v3" ) @@ -71,7 +72,7 @@ func add(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Candida } target.Node.Kind = yaml.ScalarNode target.Node.Style = lhsNode.Style - if err := addScalars(target, lhsNode, rhs.Node); err != nil { + if err := addScalars(context, target, lhsNode, rhs.Node); err != nil { return nil, err } } @@ -97,7 +98,7 @@ func guessTagFromCustomType(node *yaml.Node) string { return guessedTag } -func addScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error { +func addScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error { lhsTag := lhs.Tag rhsTag := rhs.Tag lhsIsCustom := false @@ -112,7 +113,18 @@ func addScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error { rhsTag = guessTagFromCustomType(rhs) } - if lhsTag == "!!str" { + isDateTime := lhs.Tag == "!!timestamp" + + // if the lhs is a string, it might be a timestamp in a custom format. + if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 { + _, err := time.Parse(context.GetDateTimeLayout(), lhs.Value) + isDateTime = err == nil + } + + if isDateTime { + return addDateTimes(context.GetDateTimeLayout(), target, lhs, rhs) + + } else if lhsTag == "!!str" { target.Node.Tag = lhs.Tag target.Node.Value = lhs.Value + rhs.Value } else if lhsTag == "!!int" && rhsTag == "!!int" { @@ -149,6 +161,24 @@ func addScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error { return nil } +func addDateTimes(layout string, target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error { + + duration, err := time.ParseDuration(rhs.Value) + if err != nil { + return fmt.Errorf("unable to parse duration [%v]: %w", rhs.Value, err) + } + + currentTime, err := time.Parse(layout, lhs.Value) + if err != nil { + return err + } + + newTime := currentTime.Add(duration) + target.Node.Value = newTime.Format(layout) + return nil + +} + func addSequences(target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error { target.Node.Kind = yaml.SequenceNode if len(lhs.Node.Content) > 0 { diff --git a/pkg/yqlib/operator_add_test.go b/pkg/yqlib/operator_add_test.go index 7f62303f..3f7206be 100644 --- a/pkg/yqlib/operator_add_test.go +++ b/pkg/yqlib/operator_add_test.go @@ -208,6 +208,34 @@ var addOperatorScenarios = []expressionScenario{ "D0, P[], (doc)::{a: 4, b: 6}\n", }, }, + { + description: "Date addition", + subdescription: "You can add durations to dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", + document: `a: 2021-01-01T00:00:00Z`, + expression: `.a += "3h10m"`, + expected: []string{ + "D0, P[], (doc)::a: 2021-01-01T03:10:00Z\n", + }, + }, + { + description: "Date addition - custom format", + subdescription: "You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", + document: `a: Saturday, 15-Dec-01 at 2:59AM GMT`, + expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")`, + expected: []string{ + "D0, P[], (doc)::a: Saturday, 15-Dec-01 at 6:00AM GMT\n", + }, + }, + { + skipDoc: true, + description: "Date addition - custom format", + subdescription: "You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", + document: `a: !cat Saturday, 15-Dec-01 at 2:59AM GMT`, + expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")`, + expected: []string{ + "D0, P[], (doc)::a: !cat Saturday, 15-Dec-01 at 6:00AM GMT\n", + }, + }, { description: "Add to null", subdescription: "Adding to null simply returns the rhs", diff --git a/pkg/yqlib/operator_datetime.go b/pkg/yqlib/operator_datetime.go new file mode 100644 index 00000000..d701fef5 --- /dev/null +++ b/pkg/yqlib/operator_datetime.go @@ -0,0 +1,126 @@ +package yqlib + +import ( + "container/list" + "errors" + "fmt" + "strings" + "time" + + "gopkg.in/yaml.v3" +) + +func getStringParamter(parameterName string, d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (string, error) { + result, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode) + + if err != nil { + return "", err + } else if result.MatchingNodes.Len() == 0 { + return "", fmt.Errorf("could not find %v for format_time", parameterName) + } + + return result.MatchingNodes.Front().Value.(*CandidateNode).Node.Value, nil +} + +func withDateTimeFormat(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { + if expressionNode.RHS.Operation.OperationType == blockOpType || expressionNode.RHS.Operation.OperationType == unionOpType { + layout, err := getStringParamter("layout", d, context, expressionNode.RHS.LHS) + if err != nil { + return Context{}, fmt.Errorf("could not get date time format: %w", err) + } + context.SetDateTimeLayout(layout) + return d.GetMatchingNodes(context, expressionNode.RHS.RHS) + + } + return Context{}, errors.New(`must provide a date time format string and an expression, e.g. with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; )`) + +} + +// for unit tests +var Now = time.Now + +func nowOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { + + node := &yaml.Node{ + Tag: "!!timestamp", + Kind: yaml.ScalarNode, + Value: Now().Format(time.RFC3339), + } + + return context.SingleChildContext(&CandidateNode{Node: node}), nil + +} + +func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { + format, err := getStringParamter("format", d, context, expressionNode.RHS) + layout := context.GetDateTimeLayout() + decoder := NewYamlDecoder() + + if err != nil { + return Context{}, err + } + var results = list.New() + + for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + + parsedTime, err := time.Parse(layout, candidate.Node.Value) + if err != nil { + return Context{}, fmt.Errorf("could not parse datetime of [%v]: %w", candidate.GetNicePath(), err) + } + formattedTimeStr := parsedTime.Format(format) + decoder.Init(strings.NewReader(formattedTimeStr)) + var dataBucket yaml.Node + errorReading := decoder.Decode(&dataBucket) + var node *yaml.Node + if errorReading != nil { + log.Debugf("could not parse %v - lets just leave it as a string", formattedTimeStr) + node = &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: formattedTimeStr, + } + } else { + node = unwrapDoc(&dataBucket) + } + + results.PushBack(candidate.CreateReplacement(node)) + } + + return context.ChildContext(results), nil +} + +func tzOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { + timezoneStr, err := getStringParamter("timezone", d, context, expressionNode.RHS) + layout := context.GetDateTimeLayout() + + if err != nil { + return Context{}, err + } + var results = list.New() + + timezone, err := time.LoadLocation(timezoneStr) + if err != nil { + return Context{}, fmt.Errorf("could not load tz [%v]: %w", timezoneStr, err) + } + + for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + + parsedTime, err := time.Parse(layout, candidate.Node.Value) + if err != nil { + return Context{}, fmt.Errorf("could not parse datetime of [%v] using layout [%v]: %w", candidate.GetNicePath(), layout, err) + } + tzTime := parsedTime.In(timezone) + + node := &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: candidate.Node.Tag, + Value: tzTime.Format(layout), + } + + results.PushBack(candidate.CreateReplacement(node)) + } + + return context.ChildContext(results), nil +} diff --git a/pkg/yqlib/operator_datetime_test.go b/pkg/yqlib/operator_datetime_test.go new file mode 100644 index 00000000..094e08a3 --- /dev/null +++ b/pkg/yqlib/operator_datetime_test.go @@ -0,0 +1,120 @@ +package yqlib + +import ( + "testing" +) + +var dateTimeOperatorScenarios = []expressionScenario{ + { + description: "Format: from standard RFC3339 format", + subdescription: "Providing a single parameter assumes a standard RFC3339 datetime format. If the target format is not a valid yaml datetime format, the result will be a string tagged node.", + document: `a: 2001-12-15T02:59:43.1Z`, + expression: `.a |= format_datetime("Monday, 02-Jan-06 at 3:04PM")`, + expected: []string{ + "D0, P[], (doc)::a: Saturday, 15-Dec-01 at 2:59AM\n", + }, + }, + { + description: "Format: from custom date time", + subdescription: "Use with_dtf to set a custom datetime format for parsing.", + document: `a: Saturday, 15-Dec-01 at 2:59AM`, + expression: `.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM"; format_datetime("2006-01-02"))`, + expected: []string{ + "D0, P[], (doc)::a: 2001-12-15\n", + }, + }, + { + description: "Format: get the day of the week", + document: `a: 2001-12-15T02:59:43.1Z`, + expression: `.a | format_datetime("Monday")`, + expected: []string{ + "D0, P[a], (!!str)::Saturday\n", + }, + }, + { + description: "Now", + document: "a: cool", + expression: `.updated = now`, + expected: []string{ + "D0, P[], (doc)::a: cool\nupdated: 2021-05-19T01:02:03Z\n", + }, + }, + { + description: "Timezone: from standard RFC3339 format", + subdescription: "Returns a new datetime in the specified timezone. Specify standard IANA Time Zone format or 'utc', 'local'. When given a single parameter, this assumes the datetime is in RFC3339 format.", + + document: "a: cool", + expression: `.updated = (now | tz("Australia/Sydney"))`, + expected: []string{ + "D0, P[], (doc)::a: cool\nupdated: 2021-05-19T11:02:03+10:00\n", + }, + }, + { + description: "Timezone: with custom format", + subdescription: "Specify standard IANA Time Zone format or 'utc', 'local'", + document: "a: Saturday, 15-Dec-01 at 2:59AM GMT", + expression: `.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))`, + expected: []string{ + "D0, P[], (doc)::a: Saturday, 15-Dec-01 at 1:59PM AEDT\n", + }, + }, + { + description: "Add and tz custom format", + subdescription: "Specify standard IANA Time Zone format or 'utc', 'local'", + document: "a: Saturday, 15-Dec-01 at 2:59AM GMT", + expression: `.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))`, + expected: []string{ + "D0, P[], (doc)::a: Saturday, 15-Dec-01 at 1:59PM AEDT\n", + }, + }, + { + description: "Date addition", + document: `a: 2021-01-01T00:00:00Z`, + expression: `.a += "3h10m"`, + expected: []string{ + "D0, P[], (doc)::a: 2021-01-01T03:10:00Z\n", + }, + }, + { + description: "Date subtraction", + subdescription: "You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", + document: `a: 2021-01-01T03:10:00Z`, + expression: `.a -= "3h10m"`, + expected: []string{ + "D0, P[], (doc)::a: 2021-01-01T00:00:00Z\n", + }, + }, + { + description: "Date addition - custom format", + document: `a: Saturday, 15-Dec-01 at 2:59AM GMT`, + expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a += "3h1m")`, + expected: []string{ + "D0, P[], (doc)::a: Saturday, 15-Dec-01 at 6:00AM GMT\n", + }, + }, + { + description: "Date script with custom format", + subdescription: "You can embed full expressions in with_dtf if needed.", + document: `a: Saturday, 15-Dec-01 at 2:59AM GMT`, + expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a = (.a + "3h1m" | tz("Australia/Perth")))`, + expected: []string{ + "D0, P[], (doc)::a: Saturday, 15-Dec-01 at 2:00PM AWST\n", + }, + }, + { + description: "allow comma", + skipDoc: true, + document: "a: Saturday, 15-Dec-01 at 2:59AM GMT", + expression: `.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST", tz("Australia/Sydney"))`, + expected: []string{ + "D0, P[], (doc)::a: Saturday, 15-Dec-01 at 1:59PM AEDT\n", + }, + }, +} + +func TestDatetimeOperatorScenarios(t *testing.T) { + for _, tt := range dateTimeOperatorScenarios { + testScenario(t, &tt) + } + documentOperatorScenarios(t, "datetime", dateTimeOperatorScenarios) +} diff --git a/pkg/yqlib/operator_subtract.go b/pkg/yqlib/operator_subtract.go index c92e8bb3..781bf03f 100644 --- a/pkg/yqlib/operator_subtract.go +++ b/pkg/yqlib/operator_subtract.go @@ -4,6 +4,7 @@ import ( "fmt" "strconv" "strings" + "time" "gopkg.in/yaml.v3" ) @@ -56,7 +57,7 @@ func subtract(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Ca switch lhsNode.Kind { case yaml.MappingNode: - return nil, fmt.Errorf("Maps not yet supported for subtraction") + return nil, fmt.Errorf("maps not yet supported for subtraction") case yaml.SequenceNode: if rhs.Node.Kind != yaml.SequenceNode { return nil, fmt.Errorf("%v (%v) cannot be subtracted from %v", rhs.Node.Tag, rhs.Path, lhsNode.Tag) @@ -68,13 +69,15 @@ func subtract(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Ca } target.Node.Kind = yaml.ScalarNode target.Node.Style = lhsNode.Style - return subtractScalars(target, lhsNode, rhs.Node) + if err := subtractScalars(context, target, lhsNode, rhs.Node); err != nil { + return nil, err + } } return target, nil } -func subtractScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) (*CandidateNode, error) { +func subtractScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error { lhsTag := lhs.Tag rhsTag := rhs.Tag lhsIsCustom := false @@ -89,16 +92,25 @@ func subtractScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) (*Ca rhsTag = guessTagFromCustomType(rhs) } - if lhsTag == "!!str" { - return nil, fmt.Errorf("strings cannot be subtracted") + isDateTime := lhs.Tag == "!!timestamp" + // if the lhs is a string, it might be a timestamp in a custom format. + if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 { + _, err := time.Parse(context.GetDateTimeLayout(), lhs.Value) + isDateTime = err == nil + } + + if isDateTime { + return subtractDateTime(context.GetDateTimeLayout(), target, lhs, rhs) + } else if lhsTag == "!!str" { + return fmt.Errorf("strings cannot be subtracted") } else if lhsTag == "!!int" && rhsTag == "!!int" { format, lhsNum, err := parseInt(lhs.Value) if err != nil { - return nil, err + return err } _, rhsNum, err := parseInt(rhs.Value) if err != nil { - return nil, err + return err } result := lhsNum - rhsNum target.Node.Tag = lhs.Tag @@ -106,11 +118,11 @@ func subtractScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) (*Ca } else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") { lhsNum, err := strconv.ParseFloat(lhs.Value, 64) if err != nil { - return nil, err + return err } rhsNum, err := strconv.ParseFloat(rhs.Value, 64) if err != nil { - return nil, err + return err } result := lhsNum - rhsNum if lhsIsCustom { @@ -120,8 +132,31 @@ func subtractScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) (*Ca } target.Node.Value = fmt.Sprintf("%v", result) } else { - return nil, fmt.Errorf("%v cannot be added to %v", lhs.Tag, rhs.Tag) + return fmt.Errorf("%v cannot be added to %v", lhs.Tag, rhs.Tag) } - return target, nil + return nil +} + +func subtractDateTime(layout string, target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error { + var durationStr string + if strings.HasPrefix(rhs.Value, "-") { + durationStr = rhs.Value[1:] + } else { + durationStr = "-" + rhs.Value + } + duration, err := time.ParseDuration(durationStr) + + if err != nil { + return fmt.Errorf("unable to parse duration [%v]: %w", rhs.Value, err) + } + + currentTime, err := time.Parse(layout, lhs.Value) + if err != nil { + return err + } + + newTime := currentTime.Add(duration) + target.Node.Value = newTime.Format(layout) + return nil } diff --git a/pkg/yqlib/operator_subtract_test.go b/pkg/yqlib/operator_subtract_test.go index 5999b1e0..f1828962 100644 --- a/pkg/yqlib/operator_subtract_test.go +++ b/pkg/yqlib/operator_subtract_test.go @@ -93,6 +93,34 @@ var subtractOperatorScenarios = []expressionScenario{ "D0, P[], (doc)::{a: 2, b: 4}\n", }, }, + { + description: "Date subtraction", + subdescription: "You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", + document: `a: 2021-01-01T03:10:00Z`, + expression: `.a -= "3h10m"`, + expected: []string{ + "D0, P[], (doc)::a: 2021-01-01T00:00:00Z\n", + }, + }, + { + description: "Date subtraction - custom format", + subdescription: "Use with_dtf to specify your datetime format. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", + document: `a: Saturday, 15-Dec-01 at 6:00AM GMT`, + expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a -= "3h1m")`, + expected: []string{ + "D0, P[], (doc)::a: Saturday, 15-Dec-01 at 2:59AM GMT\n", + }, + }, + { + skipDoc: true, + description: "Date subtraction - custom format", + subdescription: "You can subtract durations from dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", + document: `a: !cat Saturday, 15-Dec-01 at 6:00AM GMT`, + expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a -= "3h1m")`, + expected: []string{ + "D0, P[], (doc)::a: !cat Saturday, 15-Dec-01 at 2:59AM GMT\n", + }, + }, { description: "Custom types: that are really numbers", subdescription: "When custom tags are encountered, yq will try to decode the underlying type.", diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index fd5954f3..d3b8ad92 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -10,6 +10,7 @@ import ( "sort" "strings" "testing" + "time" "github.com/mikefarah/yq/v4/test" "gopkg.in/op/go-logging.v1" @@ -31,6 +32,9 @@ type expressionScenario struct { func TestMain(m *testing.M) { logging.SetLevel(logging.ERROR, "") + Now = func() time.Time { + return time.Date(2021, time.May, 19, 1, 2, 3, 4, time.UTC) + } code := m.Run() os.Exit(code) }