From e59bba153b3a493baeb58f590386a071da123e7e Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 18 Mar 2022 13:22:51 +1100 Subject: [PATCH] Added support for more comparison ops --- pkg/yqlib/compare_operators.go | 50 ++- pkg/yqlib/compare_operators_test.go | 297 +++++++++++++----- pkg/yqlib/doc/operators/boolean-operators.md | 6 + pkg/yqlib/doc/operators/compare.md | 49 +++ pkg/yqlib/doc/operators/equals.md | 7 + .../operators/headers/boolean-operators.md | 6 + pkg/yqlib/doc/operators/headers/compare.md | 15 + pkg/yqlib/doc/operators/headers/equals.md | 7 + pkg/yqlib/doc/operators/headers/select.md | 6 + pkg/yqlib/doc/operators/select.md | 6 + 10 files changed, 358 insertions(+), 91 deletions(-) create mode 100644 pkg/yqlib/doc/operators/headers/compare.md diff --git a/pkg/yqlib/compare_operators.go b/pkg/yqlib/compare_operators.go index 1d7e4804..663a7a8a 100644 --- a/pkg/yqlib/compare_operators.go +++ b/pkg/yqlib/compare_operators.go @@ -3,6 +3,7 @@ package yqlib import ( "fmt" "strconv" + "time" yaml "gopkg.in/yaml.v3" ) @@ -52,18 +53,40 @@ func compare(prefs compareTypePref) func(d *dataTreeNavigator, context Context, } } +func compareDateTime(layout string, prefs compareTypePref, lhs *yaml.Node, rhs *yaml.Node) (bool, error) { + lhsTime, err := time.Parse(layout, lhs.Value) + if err != nil { + return false, err + } + + rhsTime, err := time.Parse(layout, rhs.Value) + if err != nil { + return false, err + } + + if prefs.OrEqual && lhsTime.Equal(rhsTime) { + return true, nil + } + if prefs.Greater { + return lhsTime.After(rhsTime), nil + } + return lhsTime.Before(rhsTime), nil + +} + func compareScalars(context Context, prefs compareTypePref, lhs *yaml.Node, rhs *yaml.Node) (bool, error) { lhsTag := guessTagFromCustomType(lhs) rhsTag := guessTagFromCustomType(rhs) - // isDateTime := lhs.Tag == "!!timestamp" - // // if the lhs is a string, it might be a timestamp in a custom format. - // if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 { - // _, err := time.Parse(context.GetDateTimeLayout(), lhs.Value) - // isDateTime = err == nil - // } - - if lhsTag == "!!int" && rhsTag == "!!int" { + isDateTime := lhs.Tag == "!!timestamp" + // if the lhs is a string, it might be a timestamp in a custom format. + if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 { + _, err := time.Parse(context.GetDateTimeLayout(), lhs.Value) + isDateTime = err == nil + } + if isDateTime { + return compareDateTime(context.GetDateTimeLayout(), prefs, lhs, rhs) + } else if lhsTag == "!!int" && rhsTag == "!!int" { _, lhsNum, err := parseInt(lhs.Value) if err != nil { return false, err @@ -96,6 +119,15 @@ func compareScalars(context Context, prefs compareTypePref, lhs *yaml.Node, rhs return lhsNum > rhsNum, nil } return lhsNum < rhsNum, nil + } else if lhsTag == "!!str" && rhsTag == "!!str" { + if prefs.OrEqual && lhs.Value == rhs.Value { + return true, nil + } + if prefs.Greater { + return lhs.Value > rhs.Value, nil + } + return lhs.Value < rhs.Value, nil } - return false, fmt.Errorf("not yet supported") + + return false, fmt.Errorf("%v not yet supported for comparison", lhs.Tag) } diff --git a/pkg/yqlib/compare_operators_test.go b/pkg/yqlib/compare_operators_test.go index ff8b8040..95c6b452 100644 --- a/pkg/yqlib/compare_operators_test.go +++ b/pkg/yqlib/compare_operators_test.go @@ -5,89 +5,9 @@ import ( ) var compareOperatorScenarios = []expressionScenario{ - // both null - { - description: "Both sides are null: > is false", - expression: ".a > .b", - expected: []string{ - "D0, P[], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - expression: ".a < .b", - expected: []string{ - "D0, P[], (!!bool)::false\n", - }, - }, - { - description: "Both sides are null: >= is true", - expression: ".a >= .b", - expected: []string{ - "D0, P[], (!!bool)::true\n", - }, - }, - { - skipDoc: true, - expression: ".a <= .b", - expected: []string{ - "D0, P[], (!!bool)::true\n", - }, - }, - - // one null - { - description: "One side is null: > is false", - document: `a: 5`, - expression: ".a > .b", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: `a: 5`, - expression: ".a < .b", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - { - description: "One side is null: >= is false", - document: `a: 5`, - expression: ".a >= .b", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: `a: 5`, - expression: ".a <= .b", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: `a: 5`, - expression: ".b <= .a", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: `a: 5`, - expression: ".b < .a", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - // ints, not equal { - description: "Compare integers (>)", + description: "Compare numbers (>)", document: "a: 5\nb: 4", expression: ".a > .b", expected: []string{ @@ -103,6 +23,7 @@ var compareOperatorScenarios = []expressionScenario{ }, }, { + skipDoc: true, description: "Compare integers (>=)", document: "a: 5\nb: 4", expression: ".a >= .b", @@ -121,7 +42,8 @@ var compareOperatorScenarios = []expressionScenario{ // ints, equal { - description: "Compare equal numbers", + skipDoc: true, + description: "Compare equal numbers (>)", document: "a: 5\nb: 5", expression: ".a > .b", expected: []string{ @@ -222,10 +144,221 @@ var compareOperatorScenarios = []expressionScenario{ }, // strings, not equal + { + description: "Compare strings", + subdescription: "Compares strings by their bytecode.", + document: "a: zoo\nb: apple", + expression: ".a > .b", + expected: []string{ + "D0, P[a], (!!bool)::true\n", + }, + }, + { + skipDoc: true, + document: "a: zoo\nb: apple", + expression: ".a < .b", + expected: []string{ + "D0, P[a], (!!bool)::false\n", + }, + }, + { + skipDoc: true, + document: "a: zoo\nb: apple", + expression: ".a >= .b", + expected: []string{ + "D0, P[a], (!!bool)::true\n", + }, + }, + { + skipDoc: true, + document: "a: zoo\nb: apple", + expression: ".a <= .b", + expected: []string{ + "D0, P[a], (!!bool)::false\n", + }, + }, + // strings, equal + { + skipDoc: true, + document: "a: cat\nb: cat", + expression: ".a > .b", + expected: []string{ + "D0, P[a], (!!bool)::false\n", + }, + }, + { + skipDoc: true, + document: "a: cat\nb: cat", + expression: ".a < .b", + expected: []string{ + "D0, P[a], (!!bool)::false\n", + }, + }, + { + skipDoc: true, + document: "a: cat\nb: cat", + expression: ".a >= .b", + expected: []string{ + "D0, P[a], (!!bool)::true\n", + }, + }, + { + skipDoc: true, + document: "a: cat\nb: cat", + expression: ".a <= .b", + expected: []string{ + "D0, P[a], (!!bool)::true\n", + }, + }, // datetime, not equal + { + description: "Compare date times", + subdescription: "You can compare date times. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", + document: "a: 2021-01-01T03:10:00Z\nb: 2020-01-01T03:10:00Z", + expression: ".a > .b", + expected: []string{ + "D0, P[a], (!!bool)::true\n", + }, + }, + { + skipDoc: true, + document: "a: 2021-01-01T03:10:00Z\nb: 2020-01-01T03:10:00Z", + expression: ".a < .b", + expected: []string{ + "D0, P[a], (!!bool)::false\n", + }, + }, + { + skipDoc: true, + document: "a: 2021-01-01T03:10:00Z\nb: 2020-01-01T03:10:00Z", + expression: ".a >= .b", + expected: []string{ + "D0, P[a], (!!bool)::true\n", + }, + }, + { + skipDoc: true, + document: "a: 2021-01-01T03:10:00Z\nb: 2020-01-01T03:10:00Z", + expression: ".a <= .b", + expected: []string{ + "D0, P[a], (!!bool)::false\n", + }, + }, + // datetime, equal + { + skipDoc: true, + document: "a: 2021-01-01T03:10:00Z\nb: 2021-01-01T03:10:00Z", + expression: ".a > .b", + expected: []string{ + "D0, P[a], (!!bool)::false\n", + }, + }, + { + skipDoc: true, + document: "a: 2021-01-01T03:10:00Z\nb: 2021-01-01T03:10:00Z", + expression: ".a < .b", + expected: []string{ + "D0, P[a], (!!bool)::false\n", + }, + }, + { + skipDoc: true, + document: "a: 2021-01-01T03:10:00Z\nb: 2021-01-01T03:10:00Z", + expression: ".a >= .b", + expected: []string{ + "D0, P[a], (!!bool)::true\n", + }, + }, + { + skipDoc: true, + document: "a: 2021-01-01T03:10:00Z\nb: 2021-01-01T03:10:00Z", + expression: ".a <= .b", + expected: []string{ + "D0, P[a], (!!bool)::true\n", + }, + }, + // both null + { + description: "Both sides are null: > is false", + expression: ".a > .b", + expected: []string{ + "D0, P[], (!!bool)::false\n", + }, + }, + { + skipDoc: true, + expression: ".a < .b", + expected: []string{ + "D0, P[], (!!bool)::false\n", + }, + }, + { + description: "Both sides are null: >= is true", + expression: ".a >= .b", + expected: []string{ + "D0, P[], (!!bool)::true\n", + }, + }, + { + skipDoc: true, + expression: ".a <= .b", + expected: []string{ + "D0, P[], (!!bool)::true\n", + }, + }, + + // one null + { + description: "One side is null: > is false", + document: `a: 5`, + expression: ".a > .b", + expected: []string{ + "D0, P[a], (!!bool)::false\n", + }, + }, + { + skipDoc: true, + document: `a: 5`, + expression: ".a < .b", + expected: []string{ + "D0, P[a], (!!bool)::false\n", + }, + }, + { + description: "One side is null: >= is false", + document: `a: 5`, + expression: ".a >= .b", + expected: []string{ + "D0, P[a], (!!bool)::false\n", + }, + }, + { + skipDoc: true, + document: `a: 5`, + expression: ".a <= .b", + expected: []string{ + "D0, P[a], (!!bool)::false\n", + }, + }, + { + skipDoc: true, + document: `a: 5`, + expression: ".b <= .a", + expected: []string{ + "D0, P[a], (!!bool)::false\n", + }, + }, + { + skipDoc: true, + document: `a: 5`, + expression: ".b < .a", + expected: []string{ + "D0, P[a], (!!bool)::false\n", + }, + }, } func TestCompareOperatorScenarios(t *testing.T) { diff --git a/pkg/yqlib/doc/operators/boolean-operators.md b/pkg/yqlib/doc/operators/boolean-operators.md index 70c19bd2..fb6ae394 100644 --- a/pkg/yqlib/doc/operators/boolean-operators.md +++ b/pkg/yqlib/doc/operators/boolean-operators.md @@ -10,6 +10,12 @@ The `or` and `and` operators take two parameters and return a boolean result. These are most commonly used with the `select` operator to filter particular nodes. +## Related Operators + +- equals / not equals (`==`, `!=`) operators (here)[https://mikefarah.gitbook.io/yq/operators/equals] +- comparison (`>=`, `<` etc) operators (here)[https://mikefarah.gitbook.io/yq/operators/compare] +- select operator (here)[https://mikefarah.gitbook.io/yq/operators/select] + {% hint style="warning" %} Note that versions prior to 4.18 require the 'eval/e' command to be specified. diff --git a/pkg/yqlib/doc/operators/compare.md b/pkg/yqlib/doc/operators/compare.md index 24d7d538..f59eee15 100644 --- a/pkg/yqlib/doc/operators/compare.md +++ b/pkg/yqlib/doc/operators/compare.md @@ -1,3 +1,18 @@ +# Compare Operators + +Comparison operators (`>`, `>=`, `<`, `<=`) can be used for comparing scalar values of the same time. + +The following types are currently supported: + +- numbers +- strings +- datetimes + +## Related Operators + +- equals / not equals (`==`, `!=`) operators (here)[https://mikefarah.gitbook.io/yq/operators/equals] +- boolean operators (`and`, `or`, `any` etc) (here)[https://mikefarah.gitbook.io/yq/operators/boolean-operators] +- select operator (here)[https://mikefarah.gitbook.io/yq/operators/select] {% hint style="warning" %} Note that versions prior to 4.18 require the 'eval/e' command to be specified. @@ -113,3 +128,37 @@ will output true ``` +## Compare strings +Compares strings by their bytecode. + +Given a sample.yml file of: +```yaml +a: zoo +b: apple +``` +then +```bash +yq '.a > .b' sample.yml +``` +will output +```yaml +true +``` + +## Compare date times +You can compare date times. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information. + +Given a sample.yml file of: +```yaml +a: 2021-01-01T03:10:00Z +b: 2020-01-01T03:10:00Z +``` +then +```bash +yq '.a > .b' sample.yml +``` +will output +```yaml +true +``` + diff --git a/pkg/yqlib/doc/operators/equals.md b/pkg/yqlib/doc/operators/equals.md index d108c9f0..21e0f496 100644 --- a/pkg/yqlib/doc/operators/equals.md +++ b/pkg/yqlib/doc/operators/equals.md @@ -14,6 +14,13 @@ select(.a == .b) The not equals `!=` operator returns `false` if the LHS is equal to the RHS. +## Related Operators + +- comparison (`>=`, `<` etc) operators (here)[https://mikefarah.gitbook.io/yq/operators/compare] +- boolean operators (`and`, `or`, `any` etc) (here)[https://mikefarah.gitbook.io/yq/operators/boolean-operators] +- select operator (here)[https://mikefarah.gitbook.io/yq/operators/select] + + {% hint style="warning" %} Note that versions prior to 4.18 require the 'eval/e' command to be specified. diff --git a/pkg/yqlib/doc/operators/headers/boolean-operators.md b/pkg/yqlib/doc/operators/headers/boolean-operators.md index ddf571a4..ac3072e1 100644 --- a/pkg/yqlib/doc/operators/headers/boolean-operators.md +++ b/pkg/yqlib/doc/operators/headers/boolean-operators.md @@ -9,3 +9,9 @@ The `or` and `and` operators take two parameters and return a boolean result. `any_c(condition)` and `all_c(condition)` are like `any` and `all` but they take a condition expression that is used against each element to determine if it's `true`. Note: in `jq` you can simply pass a condition to `any` or `all` and it simply works - `yq` isn't that clever..yet These are most commonly used with the `select` operator to filter particular nodes. + +## Related Operators + +- equals / not equals (`==`, `!=`) operators (here)[https://mikefarah.gitbook.io/yq/operators/equals] +- comparison (`>=`, `<` etc) operators (here)[https://mikefarah.gitbook.io/yq/operators/compare] +- select operator (here)[https://mikefarah.gitbook.io/yq/operators/select] diff --git a/pkg/yqlib/doc/operators/headers/compare.md b/pkg/yqlib/doc/operators/headers/compare.md new file mode 100644 index 00000000..a51b5b33 --- /dev/null +++ b/pkg/yqlib/doc/operators/headers/compare.md @@ -0,0 +1,15 @@ +# Compare Operators + +Comparison operators (`>`, `>=`, `<`, `<=`) can be used for comparing scalar values of the same time. + +The following types are currently supported: + +- numbers +- strings +- datetimes + +## Related Operators + +- equals / not equals (`==`, `!=`) operators (here)[https://mikefarah.gitbook.io/yq/operators/equals] +- boolean operators (`and`, `or`, `any` etc) (here)[https://mikefarah.gitbook.io/yq/operators/boolean-operators] +- select operator (here)[https://mikefarah.gitbook.io/yq/operators/select] diff --git a/pkg/yqlib/doc/operators/headers/equals.md b/pkg/yqlib/doc/operators/headers/equals.md index de32f5f0..ee47f47d 100644 --- a/pkg/yqlib/doc/operators/headers/equals.md +++ b/pkg/yqlib/doc/operators/headers/equals.md @@ -13,3 +13,10 @@ select(.a == .b) ``` The not equals `!=` operator returns `false` if the LHS is equal to the RHS. + +## Related Operators + +- comparison (`>=`, `<` etc) operators (here)[https://mikefarah.gitbook.io/yq/operators/compare] +- boolean operators (`and`, `or`, `any` etc) (here)[https://mikefarah.gitbook.io/yq/operators/boolean-operators] +- select operator (here)[https://mikefarah.gitbook.io/yq/operators/select] + diff --git a/pkg/yqlib/doc/operators/headers/select.md b/pkg/yqlib/doc/operators/headers/select.md index b1851e7e..dba72f2f 100644 --- a/pkg/yqlib/doc/operators/headers/select.md +++ b/pkg/yqlib/doc/operators/headers/select.md @@ -1,3 +1,9 @@ # Select Select is used to filter arrays and maps by a boolean expression. + +## Related Operators + +- equals / not equals (`==`, `!=`) operators (here)[https://mikefarah.gitbook.io/yq/operators/equals] +- comparison (`>=`, `<` etc) operators (here)[https://mikefarah.gitbook.io/yq/operators/compare] +- boolean operators (`and`, `or`, `any` etc) (here)[https://mikefarah.gitbook.io/yq/operators/boolean-operators] diff --git a/pkg/yqlib/doc/operators/select.md b/pkg/yqlib/doc/operators/select.md index 43a35f12..7912fb11 100644 --- a/pkg/yqlib/doc/operators/select.md +++ b/pkg/yqlib/doc/operators/select.md @@ -2,6 +2,12 @@ Select is used to filter arrays and maps by a boolean expression. +## Related Operators + +- equals / not equals (`==`, `!=`) operators (here)[https://mikefarah.gitbook.io/yq/operators/equals] +- comparison (`>=`, `<` etc) operators (here)[https://mikefarah.gitbook.io/yq/operators/compare] +- boolean operators (`and`, `or`, `any` etc) (here)[https://mikefarah.gitbook.io/yq/operators/boolean-operators] + {% hint style="warning" %} Note that versions prior to 4.18 require the 'eval/e' command to be specified.