From 778cbd348a4c801f9e53407cb6d9bd792798b182 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Mon, 14 Feb 2022 15:29:17 +1100 Subject: [PATCH] Added date subtract support --- README.md | 3 +- pkg/yqlib/doc/operators/add.md | 2 +- pkg/yqlib/doc/operators/datetime.md | 51 ++++++++++++++---- pkg/yqlib/doc/operators/headers/datetime.md | 9 ++-- pkg/yqlib/doc/operators/subtract.md | 32 ++++++++++++ pkg/yqlib/expression_tokeniser.go | 2 +- pkg/yqlib/operator_add_test.go | 4 +- pkg/yqlib/operator_datetime.go | 2 +- pkg/yqlib/operator_datetime_test.go | 31 ++++++++--- pkg/yqlib/operator_subtract.go | 57 +++++++++++++++++---- pkg/yqlib/operator_subtract_test.go | 28 ++++++++++ 11 files changed, 181 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 2144739c..201b897f 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,6 @@ rm /etc/myfile.tmp ``` ### Run with Docker or Podman - #### Oneshot use: ```bash @@ -194,7 +193,7 @@ Or, in your Dockerfile: FROM mikefarah/yq USER root -RUN apk add bash +RUN apk add --no-cache bash USER yq ``` diff --git a/pkg/yqlib/doc/operators/add.md b/pkg/yqlib/doc/operators/add.md index ff755b91..52452ef4 100644 --- a/pkg/yqlib/doc/operators/add.md +++ b/pkg/yqlib/doc/operators/add.md @@ -231,7 +231,7 @@ a: Saturday, 15-Dec-01 at 2:59AM GMT ``` then ```bash -yq 'with_dtformat("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")' sample.yml +yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")' sample.yml ``` will output ```yaml diff --git a/pkg/yqlib/doc/operators/datetime.md b/pkg/yqlib/doc/operators/datetime.md index 09ca6e8d..f97a37ea 100644 --- a/pkg/yqlib/doc/operators/datetime.md +++ b/pkg/yqlib/doc/operators/datetime.md @@ -5,16 +5,15 @@ Various operators for parsing and manipulating dates. ## Date time formattings This uses the golangs built in time library for parsing and formatting date times. -When not specified, the RFC3339 standard is assumed `2006-01-02T15:04:05Z07:00`. +When not specified, the RFC3339 standard is assumed `2006-01-02T15:04:05Z07:00` for parsing. -To use a custom format, use the `with_dtformat` operator to set the formatting context. Expressions in the second parameter then assume that date format. +To specify a custom parsing format, use the `with_dtf` operator. The first parameter sets the datetime parsing format for the expression in the second parameter. The expression can be any valid `yq` expression tree. ```bash -yq 'with_dtformat("myformat"; .a + "3h" | tz("Australia/Melbourne"))' +yq 'with_dtf("myformat"; .a + "3h" | tz("Australia/Melbourne"))' ``` -See https://pkg.go.dev/time#pkg-constants for more examples. - +See https://pkg.go.dev/time#pkg-constants for examples of formatting options. ## Timezones @@ -49,7 +48,7 @@ a: Saturday, 15-Dec-01 at 2:59AM ``` ## Format: from custom date time -Use with_dtformat to set a custom datetime format for parsing. +Use with_dtf to set a custom datetime format for parsing. Given a sample.yml file of: ```yaml @@ -57,7 +56,7 @@ a: Saturday, 15-Dec-01 at 2:59AM ``` then ```bash -yq '.a |= with_dtformat("Monday, 02-Jan-06 at 3:04PM"; format_datetime("2006-01-02"))' sample.yml +yq '.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM"; format_datetime("2006-01-02"))' sample.yml ``` will output ```yaml @@ -119,7 +118,7 @@ a: Saturday, 15-Dec-01 at 2:59AM GMT ``` then ```bash -yq '.a |= with_dtformat("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))' sample.yml +yq '.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))' sample.yml ``` will output ```yaml @@ -135,7 +134,7 @@ a: Saturday, 15-Dec-01 at 2:59AM GMT ``` then ```bash -yq '.a |= with_dtformat("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))' sample.yml +yq '.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))' sample.yml ``` will output ```yaml @@ -156,6 +155,22 @@ will output a: 2021-01-01T03:10:00Z ``` +## Date subtraction +You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information. + +Given a sample.yml file of: +```yaml +a: 2021-01-01T03:10:00Z +``` +then +```bash +yq '.a -= "3h10m"' sample.yml +``` +will output +```yaml +a: 2021-01-01T00:00:00Z +``` + ## Date addition - custom format Given a sample.yml file of: ```yaml @@ -163,10 +178,26 @@ a: Saturday, 15-Dec-01 at 2:59AM GMT ``` then ```bash -yq 'with_dtformat("Monday, 02-Jan-06 at 3:04PM MST"; .a += "3h1m")' sample.yml +yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a += "3h1m")' sample.yml ``` will output ```yaml a: Saturday, 15-Dec-01 at 6:00AM GMT ``` +## Date script with custom format +You can embed full expressions in with_dtf if needed. + +Given a sample.yml file of: +```yaml +a: Saturday, 15-Dec-01 at 2:59AM GMT +``` +then +```bash +yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a = (.a + "3h1m" | tz("Australia/Perth")))' sample.yml +``` +will output +```yaml +a: Saturday, 15-Dec-01 at 2:00PM AWST +``` + diff --git a/pkg/yqlib/doc/operators/headers/datetime.md b/pkg/yqlib/doc/operators/headers/datetime.md index 1ce77bcc..39b0dd67 100644 --- a/pkg/yqlib/doc/operators/headers/datetime.md +++ b/pkg/yqlib/doc/operators/headers/datetime.md @@ -5,16 +5,15 @@ Various operators for parsing and manipulating dates. ## Date time formattings This uses the golangs built in time library for parsing and formatting date times. -When not specified, the RFC3339 standard is assumed `2006-01-02T15:04:05Z07:00`. +When not specified, the RFC3339 standard is assumed `2006-01-02T15:04:05Z07:00` for parsing. -To use a custom format, use the `with_dtformat` operator to set the formatting context. Expressions in the second parameter then assume that date format. +To specify a custom parsing format, use the `with_dtf` operator. The first parameter sets the datetime parsing format for the expression in the second parameter. The expression can be any valid `yq` expression tree. ```bash -yq 'with_dtformat("myformat"; .a + "3h" | tz("Australia/Melbourne"))' +yq 'with_dtf("myformat"; .a + "3h" | tz("Australia/Melbourne"))' ``` -See https://pkg.go.dev/time#pkg-constants for more examples. - +See https://pkg.go.dev/time#pkg-constants for examples of formatting options. ## Timezones diff --git a/pkg/yqlib/doc/operators/subtract.md b/pkg/yqlib/doc/operators/subtract.md index da5bfe48..a8f8d338 100644 --- a/pkg/yqlib/doc/operators/subtract.md +++ b/pkg/yqlib/doc/operators/subtract.md @@ -117,6 +117,38 @@ a: 2 b: 4 ``` +## Date subtraction +You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information. + +Given a sample.yml file of: +```yaml +a: 2021-01-01T03:10:00Z +``` +then +```bash +yq '.a -= "3h10m"' sample.yml +``` +will output +```yaml +a: 2021-01-01T00:00:00Z +``` + +## Date subtraction - custom format +Use with_dtf to specify your datetime format. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information. + +Given a sample.yml file of: +```yaml +a: Saturday, 15-Dec-01 at 6:00AM GMT +``` +then +```bash +yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a -= "3h1m")' sample.yml +``` +will output +```yaml +a: Saturday, 15-Dec-01 at 2:59AM GMT +``` + ## Custom types: that are really numbers When custom tags are encountered, yq will try to decode the underlying type. diff --git a/pkg/yqlib/expression_tokeniser.go b/pkg/yqlib/expression_tokeniser.go index 1d2cc79b..0c8d9628 100644 --- a/pkg/yqlib/expression_tokeniser.go +++ b/pkg/yqlib/expression_tokeniser.go @@ -327,7 +327,7 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`format_datetime`), opToken(formatDateTimeOpType)) lexer.Add([]byte(`now`), opToken(nowOpType)) lexer.Add([]byte(`tz`), opToken(tzOpType)) - lexer.Add([]byte(`with_dtformat`), opToken(withDtFormatOpType)) + lexer.Add([]byte(`with_dtf`), opToken(withDtFormatOpType)) lexer.Add([]byte(`toyaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat)) lexer.Add([]byte(`to_yaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat)) diff --git a/pkg/yqlib/operator_add_test.go b/pkg/yqlib/operator_add_test.go index c802b2db..3f7206be 100644 --- a/pkg/yqlib/operator_add_test.go +++ b/pkg/yqlib/operator_add_test.go @@ -221,7 +221,7 @@ var addOperatorScenarios = []expressionScenario{ description: "Date addition - custom format", subdescription: "You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", document: `a: Saturday, 15-Dec-01 at 2:59AM GMT`, - expression: `with_dtformat("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")`, + expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")`, expected: []string{ "D0, P[], (doc)::a: Saturday, 15-Dec-01 at 6:00AM GMT\n", }, @@ -231,7 +231,7 @@ var addOperatorScenarios = []expressionScenario{ description: "Date addition - custom format", subdescription: "You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", document: `a: !cat Saturday, 15-Dec-01 at 2:59AM GMT`, - expression: `with_dtformat("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")`, + expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")`, expected: []string{ "D0, P[], (doc)::a: !cat Saturday, 15-Dec-01 at 6:00AM GMT\n", }, diff --git a/pkg/yqlib/operator_datetime.go b/pkg/yqlib/operator_datetime.go index ac00ed14..d701fef5 100644 --- a/pkg/yqlib/operator_datetime.go +++ b/pkg/yqlib/operator_datetime.go @@ -32,7 +32,7 @@ func withDateTimeFormat(d *dataTreeNavigator, context Context, expressionNode *E return d.GetMatchingNodes(context, expressionNode.RHS.RHS) } - return Context{}, errors.New(`must provide a date time format string and an expression, e.g. with_dtformat("Monday, 02-Jan-06 at 3:04PM MST"; )`) + return Context{}, errors.New(`must provide a date time format string and an expression, e.g. with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; )`) } diff --git a/pkg/yqlib/operator_datetime_test.go b/pkg/yqlib/operator_datetime_test.go index b51f11ad..094e08a3 100644 --- a/pkg/yqlib/operator_datetime_test.go +++ b/pkg/yqlib/operator_datetime_test.go @@ -16,9 +16,9 @@ var dateTimeOperatorScenarios = []expressionScenario{ }, { description: "Format: from custom date time", - subdescription: "Use with_dtformat to set a custom datetime format for parsing.", + subdescription: "Use with_dtf to set a custom datetime format for parsing.", document: `a: Saturday, 15-Dec-01 at 2:59AM`, - expression: `.a |= with_dtformat("Monday, 02-Jan-06 at 3:04PM"; format_datetime("2006-01-02"))`, + expression: `.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM"; format_datetime("2006-01-02"))`, expected: []string{ "D0, P[], (doc)::a: 2001-12-15\n", }, @@ -53,7 +53,7 @@ var dateTimeOperatorScenarios = []expressionScenario{ description: "Timezone: with custom format", subdescription: "Specify standard IANA Time Zone format or 'utc', 'local'", document: "a: Saturday, 15-Dec-01 at 2:59AM GMT", - expression: `.a |= with_dtformat("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))`, + expression: `.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))`, expected: []string{ "D0, P[], (doc)::a: Saturday, 15-Dec-01 at 1:59PM AEDT\n", }, @@ -62,7 +62,7 @@ var dateTimeOperatorScenarios = []expressionScenario{ description: "Add and tz custom format", subdescription: "Specify standard IANA Time Zone format or 'utc', 'local'", document: "a: Saturday, 15-Dec-01 at 2:59AM GMT", - expression: `.a |= with_dtformat("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))`, + expression: `.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))`, expected: []string{ "D0, P[], (doc)::a: Saturday, 15-Dec-01 at 1:59PM AEDT\n", }, @@ -75,20 +75,37 @@ var dateTimeOperatorScenarios = []expressionScenario{ "D0, P[], (doc)::a: 2021-01-01T03:10:00Z\n", }, }, + { + description: "Date subtraction", + subdescription: "You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", + document: `a: 2021-01-01T03:10:00Z`, + expression: `.a -= "3h10m"`, + expected: []string{ + "D0, P[], (doc)::a: 2021-01-01T00:00:00Z\n", + }, + }, { description: "Date addition - custom format", document: `a: Saturday, 15-Dec-01 at 2:59AM GMT`, - expression: `with_dtformat("Monday, 02-Jan-06 at 3:04PM MST"; .a += "3h1m")`, + expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a += "3h1m")`, expected: []string{ "D0, P[], (doc)::a: Saturday, 15-Dec-01 at 6:00AM GMT\n", }, }, - + { + description: "Date script with custom format", + subdescription: "You can embed full expressions in with_dtf if needed.", + document: `a: Saturday, 15-Dec-01 at 2:59AM GMT`, + expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a = (.a + "3h1m" | tz("Australia/Perth")))`, + expected: []string{ + "D0, P[], (doc)::a: Saturday, 15-Dec-01 at 2:00PM AWST\n", + }, + }, { description: "allow comma", skipDoc: true, document: "a: Saturday, 15-Dec-01 at 2:59AM GMT", - expression: `.a |= with_dtformat("Monday, 02-Jan-06 at 3:04PM MST", tz("Australia/Sydney"))`, + expression: `.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST", tz("Australia/Sydney"))`, expected: []string{ "D0, P[], (doc)::a: Saturday, 15-Dec-01 at 1:59PM AEDT\n", }, diff --git a/pkg/yqlib/operator_subtract.go b/pkg/yqlib/operator_subtract.go index c92e8bb3..781bf03f 100644 --- a/pkg/yqlib/operator_subtract.go +++ b/pkg/yqlib/operator_subtract.go @@ -4,6 +4,7 @@ import ( "fmt" "strconv" "strings" + "time" "gopkg.in/yaml.v3" ) @@ -56,7 +57,7 @@ func subtract(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Ca switch lhsNode.Kind { case yaml.MappingNode: - return nil, fmt.Errorf("Maps not yet supported for subtraction") + return nil, fmt.Errorf("maps not yet supported for subtraction") case yaml.SequenceNode: if rhs.Node.Kind != yaml.SequenceNode { return nil, fmt.Errorf("%v (%v) cannot be subtracted from %v", rhs.Node.Tag, rhs.Path, lhsNode.Tag) @@ -68,13 +69,15 @@ func subtract(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Ca } target.Node.Kind = yaml.ScalarNode target.Node.Style = lhsNode.Style - return subtractScalars(target, lhsNode, rhs.Node) + if err := subtractScalars(context, target, lhsNode, rhs.Node); err != nil { + return nil, err + } } return target, nil } -func subtractScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) (*CandidateNode, error) { +func subtractScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error { lhsTag := lhs.Tag rhsTag := rhs.Tag lhsIsCustom := false @@ -89,16 +92,25 @@ func subtractScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) (*Ca rhsTag = guessTagFromCustomType(rhs) } - if lhsTag == "!!str" { - return nil, fmt.Errorf("strings cannot be subtracted") + isDateTime := lhs.Tag == "!!timestamp" + // if the lhs is a string, it might be a timestamp in a custom format. + if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 { + _, err := time.Parse(context.GetDateTimeLayout(), lhs.Value) + isDateTime = err == nil + } + + if isDateTime { + return subtractDateTime(context.GetDateTimeLayout(), target, lhs, rhs) + } else if lhsTag == "!!str" { + return fmt.Errorf("strings cannot be subtracted") } else if lhsTag == "!!int" && rhsTag == "!!int" { format, lhsNum, err := parseInt(lhs.Value) if err != nil { - return nil, err + return err } _, rhsNum, err := parseInt(rhs.Value) if err != nil { - return nil, err + return err } result := lhsNum - rhsNum target.Node.Tag = lhs.Tag @@ -106,11 +118,11 @@ func subtractScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) (*Ca } else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") { lhsNum, err := strconv.ParseFloat(lhs.Value, 64) if err != nil { - return nil, err + return err } rhsNum, err := strconv.ParseFloat(rhs.Value, 64) if err != nil { - return nil, err + return err } result := lhsNum - rhsNum if lhsIsCustom { @@ -120,8 +132,31 @@ func subtractScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) (*Ca } target.Node.Value = fmt.Sprintf("%v", result) } else { - return nil, fmt.Errorf("%v cannot be added to %v", lhs.Tag, rhs.Tag) + return fmt.Errorf("%v cannot be added to %v", lhs.Tag, rhs.Tag) } - return target, nil + return nil +} + +func subtractDateTime(layout string, target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error { + var durationStr string + if strings.HasPrefix(rhs.Value, "-") { + durationStr = rhs.Value[1:] + } else { + durationStr = "-" + rhs.Value + } + duration, err := time.ParseDuration(durationStr) + + if err != nil { + return fmt.Errorf("unable to parse duration [%v]: %w", rhs.Value, err) + } + + currentTime, err := time.Parse(layout, lhs.Value) + if err != nil { + return err + } + + newTime := currentTime.Add(duration) + target.Node.Value = newTime.Format(layout) + return nil } diff --git a/pkg/yqlib/operator_subtract_test.go b/pkg/yqlib/operator_subtract_test.go index 5999b1e0..f1828962 100644 --- a/pkg/yqlib/operator_subtract_test.go +++ b/pkg/yqlib/operator_subtract_test.go @@ -93,6 +93,34 @@ var subtractOperatorScenarios = []expressionScenario{ "D0, P[], (doc)::{a: 2, b: 4}\n", }, }, + { + description: "Date subtraction", + subdescription: "You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", + document: `a: 2021-01-01T03:10:00Z`, + expression: `.a -= "3h10m"`, + expected: []string{ + "D0, P[], (doc)::a: 2021-01-01T00:00:00Z\n", + }, + }, + { + description: "Date subtraction - custom format", + subdescription: "Use with_dtf to specify your datetime format. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", + document: `a: Saturday, 15-Dec-01 at 6:00AM GMT`, + expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a -= "3h1m")`, + expected: []string{ + "D0, P[], (doc)::a: Saturday, 15-Dec-01 at 2:59AM GMT\n", + }, + }, + { + skipDoc: true, + description: "Date subtraction - custom format", + subdescription: "You can subtract durations from dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", + document: `a: !cat Saturday, 15-Dec-01 at 6:00AM GMT`, + expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a -= "3h1m")`, + expected: []string{ + "D0, P[], (doc)::a: !cat Saturday, 15-Dec-01 at 2:59AM GMT\n", + }, + }, { description: "Custom types: that are really numbers", subdescription: "When custom tags are encountered, yq will try to decode the underlying type.",