From d58870b05604d8d8128190ce73a3bd7da506e2d5 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sun, 16 Jun 2024 10:48:01 +1000 Subject: [PATCH] Adding splat[] short hand to explode,collect,flatten,groupby,path,pivot,select and more --- pkg/yqlib/lexer_participle.go | 2 +- pkg/yqlib/lexer_participle_test.go | 2 ++ pkg/yqlib/operation.go | 24 +++++++++++----------- pkg/yqlib/operator_anchors_aliases_test.go | 10 +++++++++ pkg/yqlib/operator_collect_object_test.go | 10 +++++++++ pkg/yqlib/operator_flatten_test.go | 22 ++++++++++++++++++++ pkg/yqlib/operator_group_by_test.go | 10 +++++++++ pkg/yqlib/operator_path_test.go | 9 ++++++++ pkg/yqlib/operator_pivot_test.go | 10 +++++++++ pkg/yqlib/operator_select_test.go | 11 ++++++++++ pkg/yqlib/operator_split_document_test.go | 10 +++++++++ pkg/yqlib/operator_strings_test.go | 10 +++++++++ pkg/yqlib/operator_unique_test.go | 20 ++++++++++++++++++ pkg/yqlib/operator_with_test.go | 9 ++++++++ 14 files changed, 146 insertions(+), 13 deletions(-) diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go index af288999..1dc61741 100644 --- a/pkg/yqlib/lexer_participle.go +++ b/pkg/yqlib/lexer_participle.go @@ -328,7 +328,7 @@ func flattenWithDepth() yqAction { prefs := flattenPreferences{depth: depth} op := &Operation{OperationType: flattenOpType, Value: flattenOpType.Type, StringValue: value, Preferences: prefs} - return &token{TokenType: operationToken, Operation: op}, nil + return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: flattenOpType.CheckForPostTraverse}, nil } } diff --git a/pkg/yqlib/lexer_participle_test.go b/pkg/yqlib/lexer_participle_test.go index 9150bd47..ba3537f9 100644 --- a/pkg/yqlib/lexer_participle_test.go +++ b/pkg/yqlib/lexer_participle_test.go @@ -513,6 +513,7 @@ var participleLexerScenarios = []participleLexerScenario{ StringValue: "flatten(3)", Preferences: flattenPreferences{depth: 3}, }, + CheckForPostTraverse: flattenOpType.CheckForPostTraverse, }, }, }, @@ -527,6 +528,7 @@ var participleLexerScenarios = []participleLexerScenario{ StringValue: "flatten", Preferences: flattenPreferences{depth: -1}, }, + CheckForPostTraverse: flattenOpType.CheckForPostTraverse, }, }, }, diff --git a/pkg/yqlib/operation.go b/pkg/yqlib/operation.go index da64ce19..6ed55abc 100644 --- a/pkg/yqlib/operation.go +++ b/pkg/yqlib/operation.go @@ -117,9 +117,9 @@ var toEntriesOpType = &operationType{Type: "TO_ENTRIES", NumArgs: 0, Precedence: var fromEntriesOpType = &operationType{Type: "FROM_ENTRIES", NumArgs: 0, Precedence: 50, Handler: fromEntriesOperator} var withEntriesOpType = &operationType{Type: "WITH_ENTRIES", NumArgs: 1, Precedence: 50, Handler: withEntriesOperator} -var withOpType = &operationType{Type: "WITH", NumArgs: 1, Precedence: 50, Handler: withOperator} +var withOpType = &operationType{Type: "WITH", NumArgs: 1, Precedence: 52, Handler: withOperator, CheckForPostTraverse: true} -var splitDocumentOpType = &operationType{Type: "SPLIT_DOC", NumArgs: 0, Precedence: 50, Handler: splitDocumentOperator} +var splitDocumentOpType = &operationType{Type: "SPLIT_DOC", NumArgs: 0, Precedence: 52, Handler: splitDocumentOperator, CheckForPostTraverse: true} var getVariableOpType = &operationType{Type: "GET_VARIABLE", NumArgs: 0, Precedence: 55, Handler: getVariableOperator} var getStyleOpType = &operationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: getStyleOperator} var getTagOpType = &operationType{Type: "GET_TAG", NumArgs: 0, Precedence: 50, Handler: getTagOperator} @@ -138,10 +138,10 @@ var getFileIndexOpType = &operationType{Type: "GET_FILE_INDEX", NumArgs: 0, Prec var getPathOpType = &operationType{Type: "GET_PATH", NumArgs: 0, Precedence: 52, Handler: getPathOperator, CheckForPostTraverse: true} var setPathOpType = &operationType{Type: "SET_PATH", NumArgs: 1, Precedence: 50, Handler: setPathOperator} -var delPathsOpType = &operationType{Type: "DEL_PATHS", NumArgs: 1, Precedence: 50, Handler: delPathsOperator} +var delPathsOpType = &operationType{Type: "DEL_PATHS", NumArgs: 1, Precedence: 52, Handler: delPathsOperator, CheckForPostTraverse: true} -var explodeOpType = &operationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: explodeOperator} -var sortByOpType = &operationType{Type: "SORT_BY", NumArgs: 1, Precedence: 52, Handler: sortByOperator} +var explodeOpType = &operationType{Type: "EXPLODE", NumArgs: 1, Precedence: 52, Handler: explodeOperator, CheckForPostTraverse: true} +var sortByOpType = &operationType{Type: "SORT_BY", NumArgs: 1, Precedence: 52, Handler: sortByOperator, CheckForPostTraverse: true} var reverseOpType = &operationType{Type: "REVERSE", NumArgs: 0, Precedence: 52, Handler: reverseOperator, CheckForPostTraverse: true} var sortOpType = &operationType{Type: "SORT", NumArgs: 0, Precedence: 52, Handler: sortOperator, CheckForPostTraverse: true} var shuffleOpType = &operationType{Type: "SHUFFLE", NumArgs: 0, Precedence: 52, Handler: shuffleOperator, CheckForPostTraverse: true} @@ -153,7 +153,7 @@ var subStringOpType = &operationType{Type: "SUBSTR", NumArgs: 1, Precedence: 50, var matchOpType = &operationType{Type: "MATCH", NumArgs: 1, Precedence: 50, Handler: matchOperator} var captureOpType = &operationType{Type: "CAPTURE", NumArgs: 1, Precedence: 50, Handler: captureOperator} var testOpType = &operationType{Type: "TEST", NumArgs: 1, Precedence: 50, Handler: testOperator} -var splitStringOpType = &operationType{Type: "SPLIT", NumArgs: 1, Precedence: 50, Handler: splitStringOperator} +var splitStringOpType = &operationType{Type: "SPLIT", NumArgs: 1, Precedence: 52, Handler: splitStringOperator, CheckForPostTraverse: true} var changeCaseOpType = &operationType{Type: "CHANGE_CASE", NumArgs: 0, Precedence: 50, Handler: changeCaseOperator} var trimOpType = &operationType{Type: "TRIM", NumArgs: 0, Precedence: 50, Handler: trimSpaceOperator} var toStringOpType = &operationType{Type: "TO_STRING", NumArgs: 0, Precedence: 50, Handler: toStringOperator} @@ -185,15 +185,15 @@ var envsubstOpType = &operationType{Type: "ENVSUBST", NumArgs: 0, Precedence: 50 var recursiveDescentOpType = &operationType{Type: "RECURSIVE_DESCENT", NumArgs: 0, Precedence: 50, Handler: recursiveDescentOperator} -var selectOpType = &operationType{Type: "SELECT", NumArgs: 1, Precedence: 50, Handler: selectOperator} +var selectOpType = &operationType{Type: "SELECT", NumArgs: 1, Precedence: 52, Handler: selectOperator, CheckForPostTraverse: true} var hasOpType = &operationType{Type: "HAS", NumArgs: 1, Precedence: 50, Handler: hasOperator} -var uniqueOpType = &operationType{Type: "UNIQUE", NumArgs: 0, Precedence: 50, Handler: unique} -var uniqueByOpType = &operationType{Type: "UNIQUE_BY", NumArgs: 1, Precedence: 50, Handler: uniqueBy} -var groupByOpType = &operationType{Type: "GROUP_BY", NumArgs: 1, Precedence: 50, Handler: groupBy} -var flattenOpType = &operationType{Type: "FLATTEN_BY", NumArgs: 0, Precedence: 50, Handler: flattenOp} +var uniqueOpType = &operationType{Type: "UNIQUE", NumArgs: 0, Precedence: 52, Handler: unique, CheckForPostTraverse: true} +var uniqueByOpType = &operationType{Type: "UNIQUE_BY", NumArgs: 1, Precedence: 52, Handler: uniqueBy, CheckForPostTraverse: true} +var groupByOpType = &operationType{Type: "GROUP_BY", NumArgs: 1, Precedence: 52, Handler: groupBy, CheckForPostTraverse: true} +var flattenOpType = &operationType{Type: "FLATTEN_BY", NumArgs: 0, Precedence: 52, Handler: flattenOp, CheckForPostTraverse: true} var deleteChildOpType = &operationType{Type: "DELETE", NumArgs: 1, Precedence: 40, Handler: deleteChildOperator} -var pivotOpType = &operationType{Type: "PIVOT", NumArgs: 0, Precedence: 50, Handler: pivotOperator} +var pivotOpType = &operationType{Type: "PIVOT", NumArgs: 0, Precedence: 52, Handler: pivotOperator, CheckForPostTraverse: true} // debugging purposes only func (p *Operation) toString() string { diff --git a/pkg/yqlib/operator_anchors_aliases_test.go b/pkg/yqlib/operator_anchors_aliases_test.go index f866aca5..8b423b37 100644 --- a/pkg/yqlib/operator_anchors_aliases_test.go +++ b/pkg/yqlib/operator_anchors_aliases_test.go @@ -188,6 +188,16 @@ var anchorOperatorScenarios = []expressionScenario{ "D0, P[b], (!!str)::cat\n", }, }, + { + description: "Explode splat", + skipDoc: true, + document: `{a: &a cat, b: *a}`, + expression: `explode(.)[]`, + expected: []string{ + "D0, P[a], (!!str)::cat\n", + "D0, P[b], (!!str)::cat\n", + }, + }, { description: "Explode alias and anchor - check original parent", skipDoc: true, diff --git a/pkg/yqlib/operator_collect_object_test.go b/pkg/yqlib/operator_collect_object_test.go index ff82325b..431839e8 100644 --- a/pkg/yqlib/operator_collect_object_test.go +++ b/pkg/yqlib/operator_collect_object_test.go @@ -28,6 +28,16 @@ var collectObjectOperatorScenarios = []expressionScenario{ "D0, P[], (!!map)::dog: great\n", }, }, + { + description: "collect splat", + skipDoc: true, + document: `[{name: cat}, {name: dog}]`, + expression: `.[] | {.name: "great"}[]`, + expected: []string{ + "D0, P[cat], (!!str)::great\n", + "D0, P[dog], (!!str)::great\n", + }, + }, { skipDoc: true, expression: `({} + {}) | (.b = 3)`, diff --git a/pkg/yqlib/operator_flatten_test.go b/pkg/yqlib/operator_flatten_test.go index fc764d30..adf56d08 100644 --- a/pkg/yqlib/operator_flatten_test.go +++ b/pkg/yqlib/operator_flatten_test.go @@ -14,6 +14,17 @@ var flattenOperatorScenarios = []expressionScenario{ "D0, P[], (!!seq)::[1, 2, 3]\n", }, }, + { + description: "Flatten splat", + skipDoc: true, + document: `[1, [2], [[3]]]`, + expression: `flatten[]`, + expected: []string{ + "D0, P[0], (!!int)::1\n", + "D0, P[0], (!!int)::2\n", + "D0, P[0], (!!int)::3\n", + }, + }, { description: "Flatten with depth of one", document: `[1, [2], [[3]]]`, @@ -22,6 +33,17 @@ var flattenOperatorScenarios = []expressionScenario{ "D0, P[], (!!seq)::[1, 2, [3]]\n", }, }, + { + description: "Flatten with depth and splat", + skipDoc: true, + document: `[1, [2], [[3]]]`, + expression: `flatten(1)[]`, + expected: []string{ + "D0, P[0], (!!int)::1\n", + "D0, P[0], (!!int)::2\n", + "D0, P[0], (!!seq)::[3]\n", + }, + }, { description: "Flatten empty array", document: `[[]]`, diff --git a/pkg/yqlib/operator_group_by_test.go b/pkg/yqlib/operator_group_by_test.go index 7332bd32..b0cf3d25 100644 --- a/pkg/yqlib/operator_group_by_test.go +++ b/pkg/yqlib/operator_group_by_test.go @@ -13,6 +13,16 @@ var groupByOperatorScenarios = []expressionScenario{ "D0, P[], (!!seq)::- - {foo: 1, bar: 10}\n - {foo: 1, bar: 1}\n- - {foo: 3, bar: 100}\n", }, }, + { + description: "Group splat", + skipDoc: true, + document: `[{foo: 1, bar: 10}, {foo: 3, bar: 100}, {foo: 1, bar: 1}]`, + expression: `group_by(.foo)[]`, + expected: []string{ + "D0, P[0], (!!seq)::- {foo: 1, bar: 10}\n- {foo: 1, bar: 1}\n", + "D0, P[1], (!!seq)::- {foo: 3, bar: 100}\n", + }, + }, { description: "Group by field, with nulls", document: `[{cat: dog}, {foo: 1, bar: 10}, {foo: 3, bar: 100}, {no: foo for you}, {foo: 1, bar: 1}]`, diff --git a/pkg/yqlib/operator_path_test.go b/pkg/yqlib/operator_path_test.go index 61f2f9d8..30b5135d 100644 --- a/pkg/yqlib/operator_path_test.go +++ b/pkg/yqlib/operator_path_test.go @@ -139,6 +139,15 @@ var pathOperatorScenarios = []expressionScenario{ "D0, P[], (!!map)::a: [frog]\n", }, }, + { + description: "Delete splat", + skipDoc: true, + document: `a: [cat, frog]`, + expression: `delpaths([["a", 0]])[]`, + expected: []string{ + "D0, P[a], (!!seq)::[frog]\n", + }, + }, { description: "Delete - wrong parameter", subdescription: "delpaths does not work with a single path array", diff --git a/pkg/yqlib/operator_pivot_test.go b/pkg/yqlib/operator_pivot_test.go index d4ca51d4..8490391e 100644 --- a/pkg/yqlib/operator_pivot_test.go +++ b/pkg/yqlib/operator_pivot_test.go @@ -11,6 +11,16 @@ var pivotOperatorScenarios = []expressionScenario{ "D0, P[], ()::- - foo\n - sis\n- - bar\n - boom\n- - baz\n - bah\n", }, }, + { + description: "Pivot splat", + skipDoc: true, + document: "[[foo, bar], [sis, boom]]\n", + expression: `pivot[]`, + expected: []string{ + "D0, P[0], ()::- foo\n- sis\n", + "D0, P[1], ()::- bar\n- boom\n", + }, + }, { description: "Pivot sequence of heterogeneous sequences", subdescription: `Missing values are "padded" to null.`, diff --git a/pkg/yqlib/operator_select_test.go b/pkg/yqlib/operator_select_test.go index 249e54b4..c92808e5 100644 --- a/pkg/yqlib/operator_select_test.go +++ b/pkg/yqlib/operator_select_test.go @@ -75,6 +75,17 @@ var selectOperatorScenarios = []expressionScenario{ "D0, P[], (!!map)::b: world\n", }, }, + { + description: "select splat", + skipDoc: true, + document: "a: hello", + document2: "b: world", + expression: `select(.a == "hello" or .b == "world")[]`, + expected: []string{ + "D0, P[a], (!!str)::hello\n", + "D0, P[b], (!!str)::world\n", + }, + }, { description: "select does not update the map", skipDoc: true, diff --git a/pkg/yqlib/operator_split_document_test.go b/pkg/yqlib/operator_split_document_test.go index a2c4f906..a0e646bb 100644 --- a/pkg/yqlib/operator_split_document_test.go +++ b/pkg/yqlib/operator_split_document_test.go @@ -22,6 +22,16 @@ var splitDocOperatorScenarios = []expressionScenario{ "D1, P[1], (!!map)::{b: dog}\n", }, }, + { + description: "Split splat", + skipDoc: true, + document: `[{a: cat}, {b: dog}]`, + expression: `.[] | split_doc[]`, + expected: []string{ + "D0, P[0 a], (!!str)::cat\n", + "D1, P[1 b], (!!str)::dog\n", + }, + }, } func TestSplitDocOperatorScenarios(t *testing.T) { diff --git a/pkg/yqlib/operator_strings_test.go b/pkg/yqlib/operator_strings_test.go index a5798eda..4b68deb3 100644 --- a/pkg/yqlib/operator_strings_test.go +++ b/pkg/yqlib/operator_strings_test.go @@ -320,6 +320,16 @@ var stringsOperatorScenarios = []expressionScenario{ "D0, P[], (!!seq)::- word\n", }, }, + { + description: "Split splat", + skipDoc: true, + document: `"word; cat"`, + expression: `split("; ")[]`, + expected: []string{ + "D0, P[0], (!!str)::word\n", + "D0, P[1], (!!str)::cat\n", + }, + }, { skipDoc: true, document: `!horse "word"`, diff --git a/pkg/yqlib/operator_unique_test.go b/pkg/yqlib/operator_unique_test.go index 9ad08e6d..7b2c1629 100644 --- a/pkg/yqlib/operator_unique_test.go +++ b/pkg/yqlib/operator_unique_test.go @@ -14,6 +14,16 @@ var uniqueOperatorScenarios = []expressionScenario{ "D0, P[], (!!seq)::[2, 1, 3]\n", }, }, + { + description: "Unique splat", + skipDoc: true, + document: `[2,1,2]`, + expression: `unique[]`, + expected: []string{ + "D0, P[0], (!!int)::2\n", + "D0, P[1], (!!int)::1\n", + }, + }, { description: "Unique nulls", subdescription: "Unique works on the node value, so it considers different representations of nulls to be different", @@ -64,6 +74,16 @@ var uniqueOperatorScenarios = []expressionScenario{ "D0, P[], (!!seq)::[{name: harry, pet: cat}, {pet: fish}]\n", }, }, + { + description: "unique by splat", + skipDoc: true, + document: `[{name: harry, pet: cat}, {pet: fish}, {name: harry, pet: dog}]`, + expression: `unique_by(.name)[]`, + expected: []string{ + "D0, P[0], (!!map)::{name: harry, pet: cat}\n", + "D0, P[1], (!!map)::{pet: fish}\n", + }, + }, { skipDoc: true, document: `[{name: harry, pet: cat}, {pet: fish}, {name: harry, pet: dog}]`, diff --git a/pkg/yqlib/operator_with_test.go b/pkg/yqlib/operator_with_test.go index 36855df1..0c748446 100644 --- a/pkg/yqlib/operator_with_test.go +++ b/pkg/yqlib/operator_with_test.go @@ -11,6 +11,15 @@ var withOperatorScenarios = []expressionScenario{ "D0, P[], (!!map)::a: {deeply: {nested: 'newValue'}}\n", }, }, + { + description: "with splat", + skipDoc: true, + document: `a: {deeply: {nested: value}}`, + expression: `with(.a.deeply.nested; . = "newValue" | . style="single")[]`, + expected: []string{ + "D0, P[a], (!!map)::{deeply: {nested: 'newValue'}}\n", + }, + }, { description: "Update multiple deeply nested properties", document: `a: {deeply: {nested: value, other: thing}}`,