From 162ea5437c98cac7ab8b9ba0d4baba45e59679cf Mon Sep 17 00:00:00 2001 From: Steven WdV Date: Fri, 13 Jun 2025 19:08:36 +0200 Subject: [PATCH 01/27] Fix merge anchor traversing - Allow inline maps instead of just aliases - Disallow nested sequences - Disallow other types Closes #2386 --- pkg/yqlib/operator_traverse_path.go | 19 ++++++++++++----- pkg/yqlib/operator_traverse_path_test.go | 27 ++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/pkg/yqlib/operator_traverse_path.go b/pkg/yqlib/operator_traverse_path.go index 903ae323..0d1db100 100644 --- a/pkg/yqlib/operator_traverse_path.go +++ b/pkg/yqlib/operator_traverse_path.go @@ -295,17 +295,26 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, node *CandidateNode, wante func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, value *CandidateNode, wantedKey string, prefs traversePreferences, splat bool) error { switch value.Kind { case AliasNode: - if value.Alias.Kind != MappingNode { - return fmt.Errorf("can only use merge anchors with maps (!!map), but got %v", value.Alias.Tag) - } - return doTraverseMap(newMatches, value.Alias, wantedKey, prefs, splat) + return traverseMergeAnchor(newMatches, value.Alias, wantedKey, prefs, splat) + case MappingNode: + return doTraverseMap(newMatches, value, wantedKey, prefs, splat) case SequenceNode: for _, childValue := range value.Content { - err := traverseMergeAnchor(newMatches, childValue, wantedKey, prefs, splat) + if childValue.Kind == AliasNode { + childValue = childValue.Alias + } + if childValue.Kind != MappingNode { + return fmt.Errorf( + "can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got sequence containing %v", + childValue.Tag) + } + err := doTraverseMap(newMatches, childValue, wantedKey, prefs, splat) if err != nil { return err } } + default: + return fmt.Errorf("can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got %v", value.Tag) } return nil } diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go index a069e9f6..c81f797e 100644 --- a/pkg/yqlib/operator_traverse_path_test.go +++ b/pkg/yqlib/operator_traverse_path_test.go @@ -25,7 +25,7 @@ foobar: thing: foobar_thing ` -// cannot use merge anchors with arrays +// cannot use merge anchors with arrays of non-maps var badAliasSample = ` _common: &common-docker-file - FROM ubuntu:18.04 @@ -365,6 +365,22 @@ var traversePathOperatorScenarios = []expressionScenario{ "D0, P[0], (!!null)::null\n", }, }, + { + skipDoc: true, + document: `{<<: {a: 42}}`, + expression: `.a`, + expected: []string{ + "D0, P[<< a], (!!int)::42\n", + }, + }, + { + skipDoc: true, + document: `{<<: [{a: 42}]}`, + expression: `.a`, + expected: []string{ + "D0, P[<< 0 a], (!!int)::42\n", + }, + }, { skipDoc: true, document: mergeDocSample, @@ -553,9 +569,16 @@ var traversePathOperatorScenarios = []expressionScenario{ skipDoc: true, document: badAliasSample, expression: ".steps[]", - expectedError: "can only use merge anchors with maps (!!map), but got !!seq", + expectedError: "can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got sequence containing !!str", skipForGoccy: true, // throws an error on parsing, that's fine }, + { + skipDoc: true, + document: `{<<: 42}`, + expression: ".[]", + expectedError: "can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got !!int", + skipForGoccy: true, + }, } func TestTraversePathOperatorScenarios(t *testing.T) { From c3782799c50a23a00adf208432a9f1c51b41dab2 Mon Sep 17 00:00:00 2001 From: Steven WdV Date: Mon, 16 Jun 2025 09:56:10 +0200 Subject: [PATCH 02/27] Merge anchor traversing: add test for aliased sequence, cleanup --- pkg/yqlib/operator_traverse_path.go | 7 ++++--- pkg/yqlib/operator_traverse_path_test.go | 12 ++++++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/pkg/yqlib/operator_traverse_path.go b/pkg/yqlib/operator_traverse_path.go index 0d1db100..f3afaf41 100644 --- a/pkg/yqlib/operator_traverse_path.go +++ b/pkg/yqlib/operator_traverse_path.go @@ -293,9 +293,10 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, node *CandidateNode, wante } func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, value *CandidateNode, wantedKey string, prefs traversePreferences, splat bool) error { + if value.Kind == AliasNode { + value = value.Alias + } switch value.Kind { - case AliasNode: - return traverseMergeAnchor(newMatches, value.Alias, wantedKey, prefs, splat) case MappingNode: return doTraverseMap(newMatches, value, wantedKey, prefs, splat) case SequenceNode: @@ -313,10 +314,10 @@ func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, value *CandidateNode return err } } + return nil default: return fmt.Errorf("can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got %v", value.Tag) } - return nil } func traverseArray(candidate *CandidateNode, operation *Operation, prefs traversePreferences) (*list.List, error) { diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go index c81f797e..114c728e 100644 --- a/pkg/yqlib/operator_traverse_path_test.go +++ b/pkg/yqlib/operator_traverse_path_test.go @@ -365,7 +365,7 @@ var traversePathOperatorScenarios = []expressionScenario{ "D0, P[0], (!!null)::null\n", }, }, - { + { // Merge anchor with inline map skipDoc: true, document: `{<<: {a: 42}}`, expression: `.a`, @@ -373,7 +373,7 @@ var traversePathOperatorScenarios = []expressionScenario{ "D0, P[<< a], (!!int)::42\n", }, }, - { + { // Merge anchor with sequence with inline map skipDoc: true, document: `{<<: [{a: 42}]}`, expression: `.a`, @@ -381,6 +381,14 @@ var traversePathOperatorScenarios = []expressionScenario{ "D0, P[<< 0 a], (!!int)::42\n", }, }, + { // Merge anchor with aliased sequence with inline map + skipDoc: true, + document: `{s: &s [{a: 42}], m: {<<: *s}}`, + expression: `.m.a`, + expected: []string{ + "D0, P[s 0 a], (!!int)::42\n", + }, + }, { skipDoc: true, document: mergeDocSample, From bfcb3fc6b705264c3e84dd749f9860111fb2b698 Mon Sep 17 00:00:00 2001 From: Steven WdV Date: Mon, 16 Jun 2025 09:58:26 +0200 Subject: [PATCH 03/27] Fix merge anchor exploding - Allow inline maps instead of just aliases - Allow aliased sequences - Disallow other types - Use tag `!!merge` instead of key `<<` - Fix insertion index for sequence merge Closes #2386 --- .../operators/anchor-and-alias-operators.md | 4 +- pkg/yqlib/operator_anchors_aliases.go | 74 ++++++++++++------- pkg/yqlib/operator_anchors_aliases_test.go | 34 +++++++-- 3 files changed, 78 insertions(+), 34 deletions(-) diff --git a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md index 14b0bef2..d624b88b 100644 --- a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md +++ b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md @@ -92,8 +92,8 @@ yq '.[4] | explode(.)' sample.yml will output ```yaml r: 10 -x: 1 y: 2 +x: 1 ``` ## Get anchor @@ -293,8 +293,8 @@ bar: foobarList: b: bar_b thing: foo_thing - c: foobarList_c a: foo_a + c: foobarList_c foobar: c: foo_c a: foo_a diff --git a/pkg/yqlib/operator_anchors_aliases.go b/pkg/yqlib/operator_anchors_aliases.go index 4a2cdf4e..a33006e7 100644 --- a/pkg/yqlib/operator_anchors_aliases.go +++ b/pkg/yqlib/operator_anchors_aliases.go @@ -147,27 +147,15 @@ func reconstructAliasedMap(node *CandidateNode, context Context) error { keyNode := node.Content[index] valueNode := node.Content[index+1] log.Debugf("traversing %v", keyNode.Value) - if keyNode.Value != "<<" { + if keyNode.Tag != "!!merge" { err := overrideEntry(node, keyNode, valueNode, index, context.ChildContext(newContent)) if err != nil { return err } } else { - if valueNode.Kind == SequenceNode { - log.Debugf("an alias merge list!") - for index := len(valueNode.Content) - 1; index >= 0; index = index - 1 { - aliasNode := valueNode.Content[index] - err := applyAlias(node, aliasNode.Alias, index, context.ChildContext(newContent)) - if err != nil { - return err - } - } - } else { - log.Debugf("an alias merge!") - err := applyAlias(node, valueNode.Alias, index, context.ChildContext(newContent)) - if err != nil { - return err - } + err := applyMergeAnchor(node, valueNode, index, context, newContent) + if err != nil { + return err } } } @@ -208,7 +196,7 @@ func explodeNode(node *CandidateNode, context Context) error { hasAlias := false for index := 0; index < len(node.Content); index = index + 2 { keyNode := node.Content[index] - if keyNode.Value == "<<" { + if keyNode.Tag == "!!merge" { hasAlias = true break } @@ -237,19 +225,51 @@ func explodeNode(node *CandidateNode, context Context) error { } } -func applyAlias(node *CandidateNode, alias *CandidateNode, aliasIndex int, newContent Context) error { - log.Debug("alias is nil ?") - if alias == nil { +func applyMergeAnchor(node *CandidateNode, merge *CandidateNode, mergeIndex int, context Context, newContent *list.List) error { + if merge.Kind == AliasNode { + merge = merge.Alias + } + switch merge.Kind { + case MappingNode: + log.Debugf("a merge map!") + return applyMergeAnchorMap(node, merge, mergeIndex, context.ChildContext(newContent)) + case SequenceNode: + log.Debugf("a merge list!") + // Earlier keys take precedence + for index := len(merge.Content) - 1; index >= 0; index = index - 1 { + childValue := merge.Content[index] + if childValue.Kind == AliasNode { + childValue = childValue.Alias + } + if childValue.Kind != MappingNode { + return fmt.Errorf( + "can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got sequence containing %v", + childValue.Tag) + } + err := applyMergeAnchorMap(node, childValue, mergeIndex, context.ChildContext(newContent)) + if err != nil { + return err + } + } + return nil + default: + return fmt.Errorf("can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got %v", merge.Tag) + } +} + +func applyMergeAnchorMap(node *CandidateNode, mergeMap *CandidateNode, aliasIndex int, newContent Context) error { + log.Debug("merge map is nil ?") + if mergeMap == nil { return nil } - log.Debug("alias: %v", NodeToString(alias)) - if alias.Kind != MappingNode { - return fmt.Errorf("merge anchor only supports maps, got %v instead", alias.Tag) + log.Debug("merge map: %v", NodeToString(mergeMap)) + if mergeMap.Kind != MappingNode { + return fmt.Errorf("applyMergeAnchorMap expects !!map, got %v instead", mergeMap.Tag) } - for index := 0; index < len(alias.Content); index = index + 2 { - keyNode := alias.Content[index] - log.Debugf("applying alias key %v", keyNode.Value) - valueNode := alias.Content[index+1] + for index := 0; index < len(mergeMap.Content); index = index + 2 { + keyNode := mergeMap.Content[index] + log.Debugf("applying merge map key %v", keyNode.Value) + valueNode := mergeMap.Content[index+1] err := overrideEntry(node, keyNode, valueNode, aliasIndex, newContent) if err != nil { return err diff --git a/pkg/yqlib/operator_anchors_aliases_test.go b/pkg/yqlib/operator_anchors_aliases_test.go index a24f3fdd..9f02154e 100644 --- a/pkg/yqlib/operator_anchors_aliases_test.go +++ b/pkg/yqlib/operator_anchors_aliases_test.go @@ -45,8 +45,8 @@ bar: foobarList: b: bar_b thing: foo_thing - c: foobarList_c a: foo_a + c: foobarList_c foobar: c: foo_c a: foo_a @@ -58,7 +58,7 @@ var anchorOperatorScenarios = []expressionScenario{ skipDoc: true, description: "merge anchor not map", document: "a: &a\n - 0\nc:\n <<: [*a]\n", - expectedError: "merge anchor only supports maps, got !!seq instead", + expectedError: "can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got sequence containing !!seq", expression: "explode(.)", }, { @@ -80,7 +80,7 @@ var anchorOperatorScenarios = []expressionScenario{ subdescription: "see https://yaml.org/type/merge.html", document: specDocument + "- << : [ *BIG, *LEFT, *SMALL ]\n x: 1\n", expression: ".[4] | explode(.)", - expected: []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"}, + expected: []string{"D0, P[4], (!!map)::r: 10\ny: 2\nx: 1\n"}, }, { description: "Get anchor", @@ -254,7 +254,7 @@ var anchorOperatorScenarios = []expressionScenario{ expression: `.foo* | explode(.) | (. style="flow")`, expected: []string{ "D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n", - "D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\n", + "D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, a: foo_a, c: foobarList_c}\n", "D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\n", }, }, @@ -264,7 +264,7 @@ var anchorOperatorScenarios = []expressionScenario{ expression: `.foo* | explode(explode(.)) | (. style="flow")`, expected: []string{ "D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n", - "D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\n", + "D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, a: foo_a, c: foobarList_c}\n", "D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\n", }, }, @@ -283,6 +283,30 @@ var anchorOperatorScenarios = []expressionScenario{ expression: `.thingOne |= explode(.) * {"value": false}`, expected: []string{expectedUpdatedArrayRef}, }, + { // Merge anchor with inline map + skipDoc: true, + document: `{<<: {a: 42}}`, + expression: `explode(.)`, + expected: []string{ + "D0, P[], (!!map)::{a: 42}\n", + }, + }, + { // Merge anchor with sequence with inline map + skipDoc: true, + document: `{<<: [{a: 42}]}`, + expression: `explode(.)`, + expected: []string{ + "D0, P[], (!!map)::{a: 42}\n", + }, + }, + { // Merge anchor with aliased sequence with inline map + skipDoc: true, + document: `{s: &s [{a: 42}], m: {<<: *s}}`, + expression: `.m | explode(.)`, + expected: []string{ + "D0, P[m], (!!map)::{a: 42}\n", + }, + }, } func TestAnchorAliasOperatorScenarios(t *testing.T) { From 31628e7324fa4d4eefd84aff9c78085389a8c1a0 Mon Sep 17 00:00:00 2001 From: Steven WdV Date: Mon, 16 Jun 2025 12:23:35 +0200 Subject: [PATCH 04/27] Make merge anchor errors for traversing nonfatal --- pkg/yqlib/operator_anchors_aliases.go | 1 + pkg/yqlib/operator_traverse_path.go | 6 ++++-- pkg/yqlib/operator_traverse_path_test.go | 14 -------------- 3 files changed, 5 insertions(+), 16 deletions(-) diff --git a/pkg/yqlib/operator_anchors_aliases.go b/pkg/yqlib/operator_anchors_aliases.go index a33006e7..b9326934 100644 --- a/pkg/yqlib/operator_anchors_aliases.go +++ b/pkg/yqlib/operator_anchors_aliases.go @@ -245,6 +245,7 @@ func applyMergeAnchor(node *CandidateNode, merge *CandidateNode, mergeIndex int, return fmt.Errorf( "can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got sequence containing %v", childValue.Tag) + return nil } err := applyMergeAnchorMap(node, childValue, mergeIndex, context.ChildContext(newContent)) if err != nil { diff --git a/pkg/yqlib/operator_traverse_path.go b/pkg/yqlib/operator_traverse_path.go index f3afaf41..88c8f31d 100644 --- a/pkg/yqlib/operator_traverse_path.go +++ b/pkg/yqlib/operator_traverse_path.go @@ -305,9 +305,10 @@ func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, value *CandidateNode childValue = childValue.Alias } if childValue.Kind != MappingNode { - return fmt.Errorf( + log.Debugf( "can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got sequence containing %v", childValue.Tag) + return nil } err := doTraverseMap(newMatches, childValue, wantedKey, prefs, splat) if err != nil { @@ -316,7 +317,8 @@ func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, value *CandidateNode } return nil default: - return fmt.Errorf("can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got %v", value.Tag) + log.Debugf("can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got %v", merge.Tag) + return nil } } diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go index 114c728e..bf1cad19 100644 --- a/pkg/yqlib/operator_traverse_path_test.go +++ b/pkg/yqlib/operator_traverse_path_test.go @@ -573,20 +573,6 @@ var traversePathOperatorScenarios = []expressionScenario{ "D0, P[a 2], (!!str)::c\n", }, }, - { - skipDoc: true, - document: badAliasSample, - expression: ".steps[]", - expectedError: "can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got sequence containing !!str", - skipForGoccy: true, // throws an error on parsing, that's fine - }, - { - skipDoc: true, - document: `{<<: 42}`, - expression: ".[]", - expectedError: "can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got !!int", - skipForGoccy: true, - }, } func TestTraversePathOperatorScenarios(t *testing.T) { From 4d88d51b1b5d7d3e9513ac424fac5ef870f47e81 Mon Sep 17 00:00:00 2001 From: Steven WdV Date: Mon, 16 Jun 2025 13:36:42 +0200 Subject: [PATCH 05/27] Fix precedence of merge anchor sequence for traverse (explode was already correct) --- pkg/yqlib/doc/operators/traverse-read.md | 8 ++++---- pkg/yqlib/operator_traverse_path.go | 14 ++++++++------ pkg/yqlib/operator_traverse_path_test.go | 8 ++++---- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/pkg/yqlib/doc/operators/traverse-read.md b/pkg/yqlib/doc/operators/traverse-read.md index fefe24df..af5c965b 100644 --- a/pkg/yqlib/doc/operators/traverse-read.md +++ b/pkg/yqlib/doc/operators/traverse-read.md @@ -399,7 +399,7 @@ foobar_thing ``` ## Traversing merge anchor lists -Note that the later merge anchors override previous +Note that the keys earlier in the merge anchors sequence override later ones Given a sample.yml file of: ```yaml @@ -428,7 +428,7 @@ yq '.foobarList.thing' sample.yml ``` will output ```yaml -bar_thing +foo_thing ``` ## Splatting merge anchor lists @@ -460,9 +460,9 @@ yq '.foobarList[]' sample.yml will output ```yaml bar_b -foo_a -bar_thing +foo_thing foobarList_c +foo_a ``` ## Select multiple indices diff --git a/pkg/yqlib/operator_traverse_path.go b/pkg/yqlib/operator_traverse_path.go index 88c8f31d..ecdeafbd 100644 --- a/pkg/yqlib/operator_traverse_path.go +++ b/pkg/yqlib/operator_traverse_path.go @@ -292,15 +292,17 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, node *CandidateNode, wante return nil } -func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, value *CandidateNode, wantedKey string, prefs traversePreferences, splat bool) error { - if value.Kind == AliasNode { - value = value.Alias +func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, merge *CandidateNode, wantedKey string, prefs traversePreferences, splat bool) error { + if merge.Kind == AliasNode { + merge = merge.Alias } - switch value.Kind { + switch merge.Kind { case MappingNode: - return doTraverseMap(newMatches, value, wantedKey, prefs, splat) + return doTraverseMap(newMatches, merge, wantedKey, prefs, splat) case SequenceNode: - for _, childValue := range value.Content { + // Earlier keys take precedence + for index := len(merge.Content) - 1; index >= 0; index = index - 1 { + childValue := merge.Content[index] if childValue.Kind == AliasNode { childValue = childValue.Alias } diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go index bf1cad19..4b456d0e 100644 --- a/pkg/yqlib/operator_traverse_path_test.go +++ b/pkg/yqlib/operator_traverse_path_test.go @@ -449,11 +449,11 @@ var traversePathOperatorScenarios = []expressionScenario{ }, { description: "Traversing merge anchor lists", - subdescription: "Note that the later merge anchors override previous", + subdescription: "Note that the keys earlier in the merge anchors sequence override later ones", document: mergeDocSample, expression: `.foobarList.thing`, expected: []string{ - "D0, P[bar thing], (!!str)::bar_thing\n", + "D0, P[foo thing], (!!str)::foo_thing\n", }, }, { @@ -478,9 +478,9 @@ var traversePathOperatorScenarios = []expressionScenario{ expression: `.foobarList[]`, expected: []string{ "D0, P[bar b], (!!str)::bar_b\n", - "D0, P[foo a], (!!str)::foo_a\n", - "D0, P[bar thing], (!!str)::bar_thing\n", + "D0, P[foo thing], (!!str)::foo_thing\n", "D0, P[foobarList c], (!!str)::foobarList_c\n", + "D0, P[foo a], (!!str)::foo_a\n", }, }, { From 4734be9a4dd35cbbb3b834b0df9047e8ed117277 Mon Sep 17 00:00:00 2001 From: Steven WdV Date: Mon, 16 Jun 2025 13:53:28 +0200 Subject: [PATCH 06/27] Fix excessive exploding for merge anchor --- pkg/yqlib/operator_anchors_aliases.go | 38 ++++++++++++---------- pkg/yqlib/operator_anchors_aliases_test.go | 9 +++++ 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/pkg/yqlib/operator_anchors_aliases.go b/pkg/yqlib/operator_anchors_aliases.go index b9326934..a2d6f452 100644 --- a/pkg/yqlib/operator_anchors_aliases.go +++ b/pkg/yqlib/operator_anchors_aliases.go @@ -148,12 +148,12 @@ func reconstructAliasedMap(node *CandidateNode, context Context) error { valueNode := node.Content[index+1] log.Debugf("traversing %v", keyNode.Value) if keyNode.Tag != "!!merge" { - err := overrideEntry(node, keyNode, valueNode, index, context.ChildContext(newContent)) + err := overrideEntry(node, keyNode, valueNode, index, true, context.ChildContext(newContent)) if err != nil { return err } } else { - err := applyMergeAnchor(node, valueNode, index, context, newContent) + err := applyMergeAnchor(node, valueNode, index, context.ChildContext(newContent)) if err != nil { return err } @@ -225,14 +225,14 @@ func explodeNode(node *CandidateNode, context Context) error { } } -func applyMergeAnchor(node *CandidateNode, merge *CandidateNode, mergeIndex int, context Context, newContent *list.List) error { +func applyMergeAnchor(node *CandidateNode, merge *CandidateNode, mergeIndex int, newContent Context) error { if merge.Kind == AliasNode { merge = merge.Alias } switch merge.Kind { case MappingNode: log.Debugf("a merge map!") - return applyMergeAnchorMap(node, merge, mergeIndex, context.ChildContext(newContent)) + return applyMergeAnchorMap(node, merge, mergeIndex, newContent) case SequenceNode: log.Debugf("a merge list!") // Earlier keys take precedence @@ -245,9 +245,8 @@ func applyMergeAnchor(node *CandidateNode, merge *CandidateNode, mergeIndex int, return fmt.Errorf( "can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got sequence containing %v", childValue.Tag) - return nil } - err := applyMergeAnchorMap(node, childValue, mergeIndex, context.ChildContext(newContent)) + err := applyMergeAnchorMap(node, childValue, mergeIndex, newContent) if err != nil { return err } @@ -258,20 +257,21 @@ func applyMergeAnchor(node *CandidateNode, merge *CandidateNode, mergeIndex int, } } -func applyMergeAnchorMap(node *CandidateNode, mergeMap *CandidateNode, aliasIndex int, newContent Context) error { - log.Debug("merge map is nil ?") +func applyMergeAnchorMap(node *CandidateNode, mergeMap *CandidateNode, mergeIndex int, newContent Context) error { if mergeMap == nil { + log.Debug("merge map is nil") return nil } log.Debug("merge map: %v", NodeToString(mergeMap)) if mergeMap.Kind != MappingNode { return fmt.Errorf("applyMergeAnchorMap expects !!map, got %v instead", mergeMap.Tag) } + for index := 0; index < len(mergeMap.Content); index = index + 2 { keyNode := mergeMap.Content[index] log.Debugf("applying merge map key %v", keyNode.Value) valueNode := mergeMap.Content[index+1] - err := overrideEntry(node, keyNode, valueNode, aliasIndex, newContent) + err := overrideEntry(node, keyNode, valueNode, mergeIndex, false, newContent) if err != nil { return err } @@ -279,12 +279,12 @@ func applyMergeAnchorMap(node *CandidateNode, mergeMap *CandidateNode, aliasInde return nil } -func overrideEntry(node *CandidateNode, key *CandidateNode, value *CandidateNode, startIndex int, newContent Context) error { - - err := explodeNode(value, newContent) - - if err != nil { - return err +func overrideEntry(node *CandidateNode, key *CandidateNode, value *CandidateNode, startIndex int, explode bool, newContent Context) error { + if explode { + err := explodeNode(value, newContent) + if err != nil { + return err + } } for newEl := newContent.MatchingNodes.Front(); newEl != nil; newEl = newEl.Next() { @@ -308,9 +308,11 @@ func overrideEntry(node *CandidateNode, key *CandidateNode, value *CandidateNode } } - err = explodeNode(key, newContent) - if err != nil { - return err + if explode { + err := explodeNode(key, newContent) + if err != nil { + return err + } } log.Debugf("adding %v:%v", key.Value, value.Value) newContent.MatchingNodes.PushBack(key) diff --git a/pkg/yqlib/operator_anchors_aliases_test.go b/pkg/yqlib/operator_anchors_aliases_test.go index 9f02154e..48e3b39a 100644 --- a/pkg/yqlib/operator_anchors_aliases_test.go +++ b/pkg/yqlib/operator_anchors_aliases_test.go @@ -307,6 +307,15 @@ var anchorOperatorScenarios = []expressionScenario{ "D0, P[m], (!!map)::{a: 42}\n", }, }, + { // Exploding merge anchor should not explode neighbors + skipDoc: true, + // b must not be exploded, as `r: *a` will become invalid + document: `{b: &b {a: &a 42}, r: *a, c: {<<: *b}}`, + expression: `explode(.c)`, + expected: []string{ + "D0, P[], (!!map)::{b: &b {a: &a 42}, r: *a, c: {a: &a 42}}\n", + }, + }, } func TestAnchorAliasOperatorScenarios(t *testing.T) { From ce9a4af0df029392dade185061810bd51619b153 Mon Sep 17 00:00:00 2001 From: Steven WdV Date: Mon, 16 Jun 2025 16:09:55 +0200 Subject: [PATCH 07/27] Fix inline map exploding when it contains aliases --- pkg/yqlib/operator_anchors_aliases.go | 19 +++++++++++---- pkg/yqlib/operator_anchors_aliases_test.go | 27 ++++++++++++++++++++++ 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/pkg/yqlib/operator_anchors_aliases.go b/pkg/yqlib/operator_anchors_aliases.go index a2d6f452..9c0e9627 100644 --- a/pkg/yqlib/operator_anchors_aliases.go +++ b/pkg/yqlib/operator_anchors_aliases.go @@ -226,19 +226,23 @@ func explodeNode(node *CandidateNode, context Context) error { } func applyMergeAnchor(node *CandidateNode, merge *CandidateNode, mergeIndex int, newContent Context) error { + inline := true if merge.Kind == AliasNode { + inline = false merge = merge.Alias } switch merge.Kind { case MappingNode: log.Debugf("a merge map!") - return applyMergeAnchorMap(node, merge, mergeIndex, newContent) + return applyMergeAnchorMap(node, merge, mergeIndex, inline, newContent) case SequenceNode: log.Debugf("a merge list!") // Earlier keys take precedence for index := len(merge.Content) - 1; index >= 0; index = index - 1 { childValue := merge.Content[index] + childInline := inline if childValue.Kind == AliasNode { + childInline = false childValue = childValue.Alias } if childValue.Kind != MappingNode { @@ -246,7 +250,7 @@ func applyMergeAnchor(node *CandidateNode, merge *CandidateNode, mergeIndex int, "can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got sequence containing %v", childValue.Tag) } - err := applyMergeAnchorMap(node, childValue, mergeIndex, newContent) + err := applyMergeAnchorMap(node, childValue, mergeIndex, inline && childInline, newContent) if err != nil { return err } @@ -257,7 +261,7 @@ func applyMergeAnchor(node *CandidateNode, merge *CandidateNode, mergeIndex int, } } -func applyMergeAnchorMap(node *CandidateNode, mergeMap *CandidateNode, mergeIndex int, newContent Context) error { +func applyMergeAnchorMap(node *CandidateNode, mergeMap *CandidateNode, mergeIndex int, explode bool, newContent Context) error { if mergeMap == nil { log.Debug("merge map is nil") return nil @@ -267,11 +271,18 @@ func applyMergeAnchorMap(node *CandidateNode, mergeMap *CandidateNode, mergeInde return fmt.Errorf("applyMergeAnchorMap expects !!map, got %v instead", mergeMap.Tag) } + if explode { + err := explodeNode(mergeMap, newContent) + if err != nil { + return err + } + } + for index := 0; index < len(mergeMap.Content); index = index + 2 { keyNode := mergeMap.Content[index] log.Debugf("applying merge map key %v", keyNode.Value) valueNode := mergeMap.Content[index+1] - err := overrideEntry(node, keyNode, valueNode, mergeIndex, false, newContent) + err := overrideEntry(node, keyNode, valueNode, mergeIndex, explode, newContent) if err != nil { return err } diff --git a/pkg/yqlib/operator_anchors_aliases_test.go b/pkg/yqlib/operator_anchors_aliases_test.go index 48e3b39a..042fce2b 100644 --- a/pkg/yqlib/operator_anchors_aliases_test.go +++ b/pkg/yqlib/operator_anchors_aliases_test.go @@ -316,6 +316,33 @@ var anchorOperatorScenarios = []expressionScenario{ "D0, P[], (!!map)::{b: &b {a: &a 42}, r: *a, c: {a: &a 42}}\n", }, }, + { // Exploding sequence merge anchor should not explode neighbors + skipDoc: true, + // b must not be exploded, as `r: *a` will become invalid + document: `{b: &b {a: &a 42}, r: *a, c: {<<: [*b]}}`, + expression: `explode(.c)`, + expected: []string{ + "D0, P[], (!!map)::{b: &b {a: &a 42}, r: *a, c: {a: &a 42}}\n", + }, + }, + { // Exploding inline merge anchor + skipDoc: true, + // `<<` map must be exploded, otherwise `c: *b` will become invalid + document: `{a: {b: &b 42}, <<: {c: *b}}`, + expression: `explode(.)`, + expected: []string{ + "D0, P[], (!!map)::{a: {b: 42}, c: 42}\n", + }, + }, + { // Exploding inline merge anchor in sequence + skipDoc: true, + // `<<` map must be exploded, otherwise `c: *b` will become invalid + document: `{a: {b: &b 42}, <<: [{c: *b}]}`, + expression: `explode(.)`, + expected: []string{ + "D0, P[], (!!map)::{a: {b: 42}, c: 42}\n", + }, + }, } func TestAnchorAliasOperatorScenarios(t *testing.T) { From 78c096fa8f61eb58a79aca209bca7385c33966c3 Mon Sep 17 00:00:00 2001 From: stevenwdv Date: Tue, 17 Jun 2025 15:56:13 +0200 Subject: [PATCH 08/27] Remove redundant logic --- pkg/yqlib/operator_anchors_aliases.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/yqlib/operator_anchors_aliases.go b/pkg/yqlib/operator_anchors_aliases.go index 9c0e9627..e25d2497 100644 --- a/pkg/yqlib/operator_anchors_aliases.go +++ b/pkg/yqlib/operator_anchors_aliases.go @@ -250,7 +250,7 @@ func applyMergeAnchor(node *CandidateNode, merge *CandidateNode, mergeIndex int, "can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got sequence containing %v", childValue.Tag) } - err := applyMergeAnchorMap(node, childValue, mergeIndex, inline && childInline, newContent) + err := applyMergeAnchorMap(node, childValue, mergeIndex, childInline, newContent) if err != nil { return err } From fa6dc5c9fbdbcd5f5c9ba047357788eef596812d Mon Sep 17 00:00:00 2001 From: Steven WdV Date: Thu, 10 Jul 2025 15:02:13 +0200 Subject: [PATCH 09/27] Move new merge test comments to description field --- pkg/yqlib/operator_anchors_aliases_test.go | 71 ++++++++++++---------- pkg/yqlib/operator_traverse_path_test.go | 27 ++++---- 2 files changed, 54 insertions(+), 44 deletions(-) diff --git a/pkg/yqlib/operator_anchors_aliases_test.go b/pkg/yqlib/operator_anchors_aliases_test.go index 042fce2b..6a92eb0e 100644 --- a/pkg/yqlib/operator_anchors_aliases_test.go +++ b/pkg/yqlib/operator_anchors_aliases_test.go @@ -283,62 +283,69 @@ var anchorOperatorScenarios = []expressionScenario{ expression: `.thingOne |= explode(.) * {"value": false}`, expected: []string{expectedUpdatedArrayRef}, }, - { // Merge anchor with inline map - skipDoc: true, - document: `{<<: {a: 42}}`, - expression: `explode(.)`, + { + skipDoc: true, + description: "Merge anchor with inline map", + document: `{<<: {a: 42}}`, + expression: `explode(.)`, expected: []string{ "D0, P[], (!!map)::{a: 42}\n", }, }, - { // Merge anchor with sequence with inline map - skipDoc: true, - document: `{<<: [{a: 42}]}`, - expression: `explode(.)`, + { + skipDoc: true, + description: "Merge anchor with sequence with inline map", + document: `{<<: [{a: 42}]}`, + expression: `explode(.)`, expected: []string{ "D0, P[], (!!map)::{a: 42}\n", }, }, - { // Merge anchor with aliased sequence with inline map - skipDoc: true, - document: `{s: &s [{a: 42}], m: {<<: *s}}`, - expression: `.m | explode(.)`, + { + skipDoc: true, + description: "Merge anchor with aliased sequence with inline map", + document: `{s: &s [{a: 42}], m: {<<: *s}}`, + expression: `.m | explode(.)`, expected: []string{ "D0, P[m], (!!map)::{a: 42}\n", }, }, - { // Exploding merge anchor should not explode neighbors - skipDoc: true, - // b must not be exploded, as `r: *a` will become invalid - document: `{b: &b {a: &a 42}, r: *a, c: {<<: *b}}`, - expression: `explode(.c)`, + { + skipDoc: true, + description: "Exploding merge anchor should not explode neighbors", + subdescription: "b must not be exploded, as `r: *a` will become invalid", + document: `{b: &b {a: &a 42}, r: *a, c: {<<: *b}}`, + expression: `explode(.c)`, expected: []string{ "D0, P[], (!!map)::{b: &b {a: &a 42}, r: *a, c: {a: &a 42}}\n", }, }, - { // Exploding sequence merge anchor should not explode neighbors - skipDoc: true, - // b must not be exploded, as `r: *a` will become invalid - document: `{b: &b {a: &a 42}, r: *a, c: {<<: [*b]}}`, - expression: `explode(.c)`, + { + skipDoc: true, + description: "Exploding sequence merge anchor should not explode neighbors", + subdescription: "b must not be exploded, as `r: *a` will become invalid", + document: `{b: &b {a: &a 42}, r: *a, c: {<<: [*b]}}`, + expression: `explode(.c)`, expected: []string{ "D0, P[], (!!map)::{b: &b {a: &a 42}, r: *a, c: {a: &a 42}}\n", }, }, - { // Exploding inline merge anchor - skipDoc: true, - // `<<` map must be exploded, otherwise `c: *b` will become invalid - document: `{a: {b: &b 42}, <<: {c: *b}}`, - expression: `explode(.)`, + { + skipDoc: true, + description: "Exploding inline merge anchor", + subdescription: "`<<` map must be exploded, otherwise `c: *b` will become invalid", + document: `{a: {b: &b 42}, <<: {c: *b}}`, + expression: `explode(.)`, expected: []string{ "D0, P[], (!!map)::{a: {b: 42}, c: 42}\n", }, }, - { // Exploding inline merge anchor in sequence - skipDoc: true, - // `<<` map must be exploded, otherwise `c: *b` will become invalid - document: `{a: {b: &b 42}, <<: [{c: *b}]}`, - expression: `explode(.)`, + { + skipDoc: true, + description: "Exploding inline merge anchor in sequence", + subdescription: "`<<` map must be exploded, otherwise `c: *b` will become invalid", + document: `{a: {b: &b 42}, <<: [{c: *b}]}`, + expression: `explode(.)`, expected: []string{ "D0, P[], (!!map)::{a: {b: 42}, c: 42}\n", }, diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go index 4b456d0e..99f01a03 100644 --- a/pkg/yqlib/operator_traverse_path_test.go +++ b/pkg/yqlib/operator_traverse_path_test.go @@ -365,26 +365,29 @@ var traversePathOperatorScenarios = []expressionScenario{ "D0, P[0], (!!null)::null\n", }, }, - { // Merge anchor with inline map - skipDoc: true, - document: `{<<: {a: 42}}`, - expression: `.a`, + { + skipDoc: true, + description: "Merge anchor with inline map", + document: `{<<: {a: 42}}`, + expression: `.a`, expected: []string{ "D0, P[<< a], (!!int)::42\n", }, }, - { // Merge anchor with sequence with inline map - skipDoc: true, - document: `{<<: [{a: 42}]}`, - expression: `.a`, + { + skipDoc: true, + description: "Merge anchor with sequence with inline map", + document: `{<<: [{a: 42}]}`, + expression: `.a`, expected: []string{ "D0, P[<< 0 a], (!!int)::42\n", }, }, - { // Merge anchor with aliased sequence with inline map - skipDoc: true, - document: `{s: &s [{a: 42}], m: {<<: *s}}`, - expression: `.m.a`, + { + skipDoc: true, + description: "Merge anchor with aliased sequence with inline map", + document: `{s: &s [{a: 42}], m: {<<: *s}}`, + expression: `.m.a`, expected: []string{ "D0, P[s 0 a], (!!int)::42\n", }, From 8c06478aded8e7371a83852de4137d217483c122 Mon Sep 17 00:00:00 2001 From: Steven WdV Date: Thu, 10 Jul 2025 15:02:44 +0200 Subject: [PATCH 10/27] Remove now-unused `badAliasSample` --- pkg/yqlib/operator_traverse_path_test.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go index 99f01a03..9c83fa13 100644 --- a/pkg/yqlib/operator_traverse_path_test.go +++ b/pkg/yqlib/operator_traverse_path_test.go @@ -25,15 +25,6 @@ foobar: thing: foobar_thing ` -// cannot use merge anchors with arrays of non-maps -var badAliasSample = ` -_common: &common-docker-file - - FROM ubuntu:18.04 - -steps: - <<: *common-docker-file -` - var traversePathOperatorScenarios = []expressionScenario{ { skipDoc: true, From a47e882c8f000aa646c78402158fa6817e18856b Mon Sep 17 00:00:00 2001 From: Steven WdV Date: Wed, 16 Jul 2025 16:00:16 +0200 Subject: [PATCH 11/27] Flag for fixed list merge key traverse override behavior, and fix traversing map with merge key that would override local key (completes #2110 fix) --- .../operators/anchor-and-alias-operators.md | 32 +++++++++++ pkg/yqlib/doc/operators/traverse-read.md | 12 ++-- pkg/yqlib/operator_anchors_aliases.go | 14 +++-- pkg/yqlib/operator_anchors_aliases_test.go | 10 ++++ pkg/yqlib/operator_traverse_path.go | 37 ++++++++++-- pkg/yqlib/operator_traverse_path_test.go | 57 ++++++++++++++----- 6 files changed, 135 insertions(+), 27 deletions(-) diff --git a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md index d624b88b..e1f05b52 100644 --- a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md +++ b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md @@ -96,6 +96,38 @@ y: 2 x: 1 ``` +## Override with local key +like https://yaml.org/type/merge.html, but with x: 1 before the merge key. This is legacy behavior, see --yaml-fix-merge-anchor-to-spec + +Given a sample.yml file of: +```yaml +- &CENTER + x: 1 + y: 2 +- &LEFT + x: 0 + y: 2 +- &BIG + r: 10 +- &SMALL + r: 1 +- x: 1 + !!merge <<: + - *BIG + - *LEFT + - *SMALL +``` +then +```bash +yq '.[4] | explode(.)' sample.yml +``` +will output +```yaml +x: 0 +r: 10 +y: 2 +``` + ## Get anchor Given a sample.yml file of: ```yaml diff --git a/pkg/yqlib/doc/operators/traverse-read.md b/pkg/yqlib/doc/operators/traverse-read.md index af5c965b..3957f4c4 100644 --- a/pkg/yqlib/doc/operators/traverse-read.md +++ b/pkg/yqlib/doc/operators/traverse-read.md @@ -304,6 +304,8 @@ foo_a ``` ## Traversing merge anchors with override +This is legacy behavior, see --yaml-fix-merge-anchor-to-spec + Given a sample.yml file of: ```yaml foo: &foo @@ -399,7 +401,7 @@ foobar_thing ``` ## Traversing merge anchor lists -Note that the keys earlier in the merge anchors sequence override later ones +Note that the later merge anchors override previous, but this is legacy behavior, see --yaml-fix-merge-anchor-to-spec Given a sample.yml file of: ```yaml @@ -428,10 +430,12 @@ yq '.foobarList.thing' sample.yml ``` will output ```yaml -foo_thing +bar_thing ``` ## Splatting merge anchor lists +With legacy override behavior, see --yaml-fix-merge-anchor-to-spec + Given a sample.yml file of: ```yaml foo: &foo @@ -460,9 +464,9 @@ yq '.foobarList[]' sample.yml will output ```yaml bar_b -foo_thing -foobarList_c foo_a +bar_thing +foobarList_c ``` ## Select multiple indices diff --git a/pkg/yqlib/operator_anchors_aliases.go b/pkg/yqlib/operator_anchors_aliases.go index 1fa8ecca..937eed52 100644 --- a/pkg/yqlib/operator_anchors_aliases.go +++ b/pkg/yqlib/operator_anchors_aliases.go @@ -3,6 +3,7 @@ package yqlib import ( "container/list" "fmt" + "slices" ) func assignAliasOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { @@ -237,9 +238,13 @@ func applyMergeAnchor(node *CandidateNode, merge *CandidateNode, mergeIndex int, return applyMergeAnchorMap(node, merge, mergeIndex, inline, newContent) case SequenceNode: log.Debugf("a merge list!") - // Earlier keys take precedence - for index := len(merge.Content) - 1; index >= 0; index = index - 1 { - childValue := merge.Content[index] + // With FixMergeAnchorToSpec, we rely on overrideEntry to reject duplicates + content := slices.All(merge.Content) + if !ConfiguredYamlPreferences.FixMergeAnchorToSpec { + // Even without FixMergeAnchorToSpec, this already gave preference to earlier keys + content = slices.Backward(merge.Content) + } + for _, childValue := range content { childInline := inline if childValue.Kind == AliasNode { childInline = false @@ -303,8 +308,9 @@ func overrideEntry(node *CandidateNode, key *CandidateNode, value *CandidateNode keyNode := newEl.Value.(*CandidateNode) log.Debugf("checking new content %v:%v", keyNode.Value, valueEl.Value.(*CandidateNode).Value) if keyNode.Value == key.Value && keyNode.Alias == nil && key.Alias == nil { - log.Debugf("overridign new content") + log.Debugf("overriding new content") if !ConfiguredYamlPreferences.FixMergeAnchorToSpec { + //TODO This also fires in when an earlier element in a list merge anchor overwrites a later element, which *is* to the spec log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing the merge anchor to override the existing value at %v which isn't to the yaml spec. This flag will default to true in late 2025.", keyNode.GetNicePath()) valueEl.Value = value } diff --git a/pkg/yqlib/operator_anchors_aliases_test.go b/pkg/yqlib/operator_anchors_aliases_test.go index ec54fe6e..5f6dfacc 100644 --- a/pkg/yqlib/operator_anchors_aliases_test.go +++ b/pkg/yqlib/operator_anchors_aliases_test.go @@ -122,6 +122,7 @@ var anchorOperatorScenarios = []expressionScenario{ expression: ".[4] | explode(.)", expected: []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"}, }, + //TODO The following 2 tests warn about overwriting [3].r not being to spec while they shouldn't { description: "Override", subdescription: "see https://yaml.org/type/merge.html", @@ -129,6 +130,15 @@ var anchorOperatorScenarios = []expressionScenario{ expression: ".[4] | explode(.)", expected: []string{"D0, P[4], (!!map)::r: 10\ny: 2\nx: 1\n"}, }, + // Correctly warns about overwriting [4].x + { + description: "Override with local key", + subdescription: "like https://yaml.org/type/merge.html, but with x: 1 before the merge key. " + + "This is legacy behavior, see --yaml-fix-merge-anchor-to-spec", + document: specDocument + "- x: 1\n << : [ *BIG, *LEFT, *SMALL ]\n", + expression: ".[4] | explode(.)", + expected: []string{"D0, P[4], (!!map)::x: 0\nr: 10\ny: 2\n"}, + }, { description: "Get anchor", document: `a: &billyBob cat`, diff --git a/pkg/yqlib/operator_traverse_path.go b/pkg/yqlib/operator_traverse_path.go index ecdeafbd..d0010b42 100644 --- a/pkg/yqlib/operator_traverse_path.go +++ b/pkg/yqlib/operator_traverse_path.go @@ -3,7 +3,6 @@ package yqlib import ( "container/list" "fmt" - "github.com/elliotchance/orderedmap" ) @@ -280,11 +279,39 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, node *CandidateNode, wante log.Debug("MATCHED") if prefs.IncludeMapKeys { log.Debug("including key") - newMatches.Set(key.GetKey(), key) + keyName := key.GetKey() + if newMatches.Has(keyName) { + if ConfiguredYamlPreferences.FixMergeAnchorToSpec { + log.Debug("not overwriting existing key") + } else { + log.Warning( + "--yaml-fix-merge-anchor-to-spec is false; "+ + "causing the merge anchor to override the existing key at %v which isn't to the yaml spec. "+ + "This flag will default to true in late 2025.", key.GetNicePath()) + log.Debug("overwriting existing key") + newMatches.Set(keyName, key) + } + } else { + newMatches.Set(keyName, key) + } } if !prefs.DontIncludeMapValues { log.Debug("including value") - newMatches.Set(value.GetKey(), value) + valueName := value.GetKey() + if newMatches.Has(valueName) { + if ConfiguredYamlPreferences.FixMergeAnchorToSpec { + log.Debug("not overwriting existing value") + } else { + log.Warning( + "--yaml-fix-merge-anchor-to-spec is false; "+ + "causing the merge anchor to override the existing value at %v which isn't to the yaml spec. "+ + "This flag will default to true in late 2025.", key.GetNicePath()) + log.Debug("overwriting existing value") + newMatches.Set(valueName, value) + } + } else { + newMatches.Set(valueName, value) + } } } } @@ -300,9 +327,7 @@ func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, merge *CandidateNode case MappingNode: return doTraverseMap(newMatches, merge, wantedKey, prefs, splat) case SequenceNode: - // Earlier keys take precedence - for index := len(merge.Content) - 1; index >= 0; index = index - 1 { - childValue := merge.Content[index] + for _, childValue := range merge.Content { if childValue.Kind == AliasNode { childValue = childValue.Alias } diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go index 9c83fa13..fadbf819 100644 --- a/pkg/yqlib/operator_traverse_path_test.go +++ b/pkg/yqlib/operator_traverse_path_test.go @@ -25,6 +25,26 @@ foobar: thing: foobar_thing ` +var fixedTraversePathOperatorScenarios = []expressionScenario{ + { + description: "Traversing merge anchor lists", + subdescription: "Note that the keys earlier in the merge anchors sequence override later ones", + document: mergeDocSample, + expression: `.foobarList.thing`, + expected: []string{ + "D0, P[foo thing], (!!str)::foo_thing\n", + }, + }, + { + description: "Traversing merge anchors with override", + document: mergeDocSample, + expression: `.foobar.c`, + expected: []string{ + "D0, P[foobar c], (!!str)::foobar_c\n", + }, + }, +} + var traversePathOperatorScenarios = []expressionScenario{ { skipDoc: true, @@ -400,9 +420,10 @@ var traversePathOperatorScenarios = []expressionScenario{ }, }, { - description: "Traversing merge anchors with override", - document: mergeDocSample, - expression: `.foobar.c`, + description: "Traversing merge anchors with override", + subdescription: "This is legacy behavior, see --yaml-fix-merge-anchor-to-spec", + document: mergeDocSample, + expression: `.foobar.c`, expected: []string{ "D0, P[foo c], (!!str)::foo_c\n", }, @@ -442,12 +463,13 @@ var traversePathOperatorScenarios = []expressionScenario{ }, }, { - description: "Traversing merge anchor lists", - subdescription: "Note that the keys earlier in the merge anchors sequence override later ones", - document: mergeDocSample, - expression: `.foobarList.thing`, + description: "Traversing merge anchor lists", + subdescription: "Note that the later merge anchors override previous, " + + "but this is legacy behavior, see --yaml-fix-merge-anchor-to-spec", + document: mergeDocSample, + expression: `.foobarList.thing`, expected: []string{ - "D0, P[foo thing], (!!str)::foo_thing\n", + "D0, P[bar thing], (!!str)::bar_thing\n", }, }, { @@ -467,14 +489,15 @@ var traversePathOperatorScenarios = []expressionScenario{ }, }, { - description: "Splatting merge anchor lists", - document: mergeDocSample, - expression: `.foobarList[]`, + description: "Splatting merge anchor lists", + subdescription: "With legacy override behavior, see --yaml-fix-merge-anchor-to-spec", + document: mergeDocSample, + expression: `.foobarList[]`, expected: []string{ "D0, P[bar b], (!!str)::bar_b\n", - "D0, P[foo thing], (!!str)::foo_thing\n", - "D0, P[foobarList c], (!!str)::foobarList_c\n", "D0, P[foo a], (!!str)::foo_a\n", + "D0, P[bar thing], (!!str)::bar_thing\n", + "D0, P[foobarList c], (!!str)::foobarList_c\n", }, }, { @@ -575,3 +598,11 @@ func TestTraversePathOperatorScenarios(t *testing.T) { } documentOperatorScenarios(t, "traverse-read", traversePathOperatorScenarios) } + +func TestTraversePathOperatorAlignedToSpecScenarios(t *testing.T) { + ConfiguredYamlPreferences.FixMergeAnchorToSpec = true + for _, tt := range fixedTraversePathOperatorScenarios { + testScenario(t, &tt) + } + ConfiguredYamlPreferences.FixMergeAnchorToSpec = false +} From a5b8ef6cb111151837673639e3491d63c501d438 Mon Sep 17 00:00:00 2001 From: Steven WdV Date: Wed, 16 Jul 2025 17:56:56 +0200 Subject: [PATCH 12/27] Add some tests regarding override behavior. The one in fixedTraversePathOperatorScenarios still fails --- pkg/yqlib/operator_anchors_aliases_test.go | 31 +++++++++++++++++++++- pkg/yqlib/operator_traverse_path_test.go | 22 +++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/pkg/yqlib/operator_anchors_aliases_test.go b/pkg/yqlib/operator_anchors_aliases_test.go index 5f6dfacc..ec6dcda8 100644 --- a/pkg/yqlib/operator_anchors_aliases_test.go +++ b/pkg/yqlib/operator_anchors_aliases_test.go @@ -85,11 +85,30 @@ var fixedAnchorOperatorScenarios = []expressionScenario{ { skipDoc: true, description: "merge anchor after existing keys", - subdescription: "legacy: overrides existing keys", + subdescription: "Does not override existing keys", document: explodeWhenKeysExistDocument, expression: "explode(.)", expected: []string{explodeWhenKeysExistExpected}, }, + + // The following tests are the same as below, to verify they still works correctly with the flag: + { + description: "Override", + subdescription: "see https://yaml.org/type/merge.html", + document: specDocument + "- << : [ *BIG, *LEFT, *SMALL ]\n x: 1\n", + expression: ".[4] | explode(.)", + expected: []string{"D0, P[4], (!!map)::r: 10\ny: 2\nx: 1\n"}, + }, + { + skipDoc: true, + description: "Duplicate keys", + subdescription: "outside merge anchor", + document: `{a: 1, a: 2}`, + expression: `explode(.)`, + expected: []string{ + "D0, P[], (!!map)::{a: 1, a: 2}\n", + }, + }, } var anchorOperatorScenarios = []expressionScenario{ @@ -407,6 +426,16 @@ var anchorOperatorScenarios = []expressionScenario{ "D0, P[], (!!map)::{a: {b: 42}, c: 42}\n", }, }, + { + skipDoc: true, + description: "Duplicate keys", + subdescription: "outside merge anchor", + document: `{a: 1, a: 2}`, + expression: `explode(.)`, + expected: []string{ + "D0, P[], (!!map)::{a: 1, a: 2}\n", + }, + }, } func TestAnchorAliasOperatorScenarios(t *testing.T) { diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go index fadbf819..58ae6263 100644 --- a/pkg/yqlib/operator_traverse_path_test.go +++ b/pkg/yqlib/operator_traverse_path_test.go @@ -43,6 +43,18 @@ var fixedTraversePathOperatorScenarios = []expressionScenario{ "D0, P[foobar c], (!!str)::foobar_c\n", }, }, + + // The following tests are the same as below, to verify they still works correctly with the flag: + { + skipDoc: true, + description: "Duplicate keys", + subdescription: "outside merge anchor", + document: `{a: 1, a: 2}`, + expression: `.a`, + expected: []string{ + "D0, P[a], (!!int)::2\n", + }, + }, } var traversePathOperatorScenarios = []expressionScenario{ @@ -590,6 +602,16 @@ var traversePathOperatorScenarios = []expressionScenario{ "D0, P[a 2], (!!str)::c\n", }, }, + { + skipDoc: true, + description: "Duplicate keys", + subdescription: "outside merge anchor", + document: `{a: 1, a: 2}`, + expression: `.a`, + expected: []string{ + "D0, P[a], (!!int)::2\n", + }, + }, } func TestTraversePathOperatorScenarios(t *testing.T) { From b7aa711d94769bef6ff5f6cf94ea3b23f5ad1209 Mon Sep 17 00:00:00 2001 From: Steven WdV Date: Wed, 16 Jul 2025 18:02:51 +0200 Subject: [PATCH 13/27] Add note --- pkg/yqlib/operator_anchors_aliases_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/yqlib/operator_anchors_aliases_test.go b/pkg/yqlib/operator_anchors_aliases_test.go index ec6dcda8..988af027 100644 --- a/pkg/yqlib/operator_anchors_aliases_test.go +++ b/pkg/yqlib/operator_anchors_aliases_test.go @@ -106,6 +106,7 @@ var fixedAnchorOperatorScenarios = []expressionScenario{ document: `{a: 1, a: 2}`, expression: `explode(.)`, expected: []string{ + // {a: 2} would also be fine "D0, P[], (!!map)::{a: 1, a: 2}\n", }, }, @@ -433,6 +434,7 @@ var anchorOperatorScenarios = []expressionScenario{ document: `{a: 1, a: 2}`, expression: `explode(.)`, expected: []string{ + // {a: 2} would also be fine "D0, P[], (!!map)::{a: 1, a: 2}\n", }, }, From 08ecd39a1e47a59b581d4132cd1202feef61d003 Mon Sep 17 00:00:00 2001 From: Steven WdV Date: Wed, 16 Jul 2025 18:07:16 +0200 Subject: [PATCH 14/27] Add tests for invalid merge key handling for traverse --- pkg/yqlib/operator_traverse_path_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go index 58ae6263..65ec97f9 100644 --- a/pkg/yqlib/operator_traverse_path_test.go +++ b/pkg/yqlib/operator_traverse_path_test.go @@ -612,6 +612,25 @@ var traversePathOperatorScenarios = []expressionScenario{ "D0, P[a], (!!int)::2\n", }, }, + { + skipDoc: true, + description: "Traversing map with invalid merge anchor should not fail", + subdescription: "Otherwise code cannot do anything with it", + document: `{a: 42, <<: 37}`, + expression: `.a`, + expected: []string{ + "D0, P[a], (!!int)::42\n", + }, + }, + { + skipDoc: true, + description: "Directly accessing invalid merge anchor should not fail", + document: `{<<: 37}`, + expression: `.<<`, + expected: []string{ + "D0, P[<<], (!!int)::37\n", + }, + }, } func TestTraversePathOperatorScenarios(t *testing.T) { From 5e75db824b58b8c5a78c81cc26e270505ddb906c Mon Sep 17 00:00:00 2001 From: stevenwdv Date: Thu, 17 Jul 2025 11:43:28 +0200 Subject: [PATCH 15/27] UK spelling --- pkg/yqlib/doc/operators/anchor-and-alias-operators.md | 2 +- pkg/yqlib/doc/operators/traverse-read.md | 6 +++--- pkg/yqlib/operator_anchors_aliases_test.go | 2 +- pkg/yqlib/operator_traverse_path_test.go | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md index e1f05b52..d4a8954c 100644 --- a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md +++ b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md @@ -97,7 +97,7 @@ x: 1 ``` ## Override with local key -like https://yaml.org/type/merge.html, but with x: 1 before the merge key. This is legacy behavior, see --yaml-fix-merge-anchor-to-spec +like https://yaml.org/type/merge.html, but with x: 1 before the merge key. This is legacy behaviour, see --yaml-fix-merge-anchor-to-spec Given a sample.yml file of: ```yaml diff --git a/pkg/yqlib/doc/operators/traverse-read.md b/pkg/yqlib/doc/operators/traverse-read.md index 3957f4c4..775eeae0 100644 --- a/pkg/yqlib/doc/operators/traverse-read.md +++ b/pkg/yqlib/doc/operators/traverse-read.md @@ -304,7 +304,7 @@ foo_a ``` ## Traversing merge anchors with override -This is legacy behavior, see --yaml-fix-merge-anchor-to-spec +This is legacy behaviour, see --yaml-fix-merge-anchor-to-spec Given a sample.yml file of: ```yaml @@ -401,7 +401,7 @@ foobar_thing ``` ## Traversing merge anchor lists -Note that the later merge anchors override previous, but this is legacy behavior, see --yaml-fix-merge-anchor-to-spec +Note that the later merge anchors override previous, but this is legacy behaviour, see --yaml-fix-merge-anchor-to-spec Given a sample.yml file of: ```yaml @@ -434,7 +434,7 @@ bar_thing ``` ## Splatting merge anchor lists -With legacy override behavior, see --yaml-fix-merge-anchor-to-spec +With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec Given a sample.yml file of: ```yaml diff --git a/pkg/yqlib/operator_anchors_aliases_test.go b/pkg/yqlib/operator_anchors_aliases_test.go index 988af027..39bcc0c8 100644 --- a/pkg/yqlib/operator_anchors_aliases_test.go +++ b/pkg/yqlib/operator_anchors_aliases_test.go @@ -154,7 +154,7 @@ var anchorOperatorScenarios = []expressionScenario{ { description: "Override with local key", subdescription: "like https://yaml.org/type/merge.html, but with x: 1 before the merge key. " + - "This is legacy behavior, see --yaml-fix-merge-anchor-to-spec", + "This is legacy behaviour, see --yaml-fix-merge-anchor-to-spec", document: specDocument + "- x: 1\n << : [ *BIG, *LEFT, *SMALL ]\n", expression: ".[4] | explode(.)", expected: []string{"D0, P[4], (!!map)::x: 0\nr: 10\ny: 2\n"}, diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go index 65ec97f9..73028dc0 100644 --- a/pkg/yqlib/operator_traverse_path_test.go +++ b/pkg/yqlib/operator_traverse_path_test.go @@ -433,7 +433,7 @@ var traversePathOperatorScenarios = []expressionScenario{ }, { description: "Traversing merge anchors with override", - subdescription: "This is legacy behavior, see --yaml-fix-merge-anchor-to-spec", + subdescription: "This is legacy behaviour, see --yaml-fix-merge-anchor-to-spec", document: mergeDocSample, expression: `.foobar.c`, expected: []string{ @@ -477,7 +477,7 @@ var traversePathOperatorScenarios = []expressionScenario{ { description: "Traversing merge anchor lists", subdescription: "Note that the later merge anchors override previous, " + - "but this is legacy behavior, see --yaml-fix-merge-anchor-to-spec", + "but this is legacy behaviour, see --yaml-fix-merge-anchor-to-spec", document: mergeDocSample, expression: `.foobarList.thing`, expected: []string{ @@ -502,7 +502,7 @@ var traversePathOperatorScenarios = []expressionScenario{ }, { description: "Splatting merge anchor lists", - subdescription: "With legacy override behavior, see --yaml-fix-merge-anchor-to-spec", + subdescription: "With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec", document: mergeDocSample, expression: `.foobarList[]`, expected: []string{ From 23a7b173bf4e6dabec9bdd78557e048947ddb7bd Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sat, 19 Jul 2025 15:27:44 +1000 Subject: [PATCH 16/27] Fixing merge anchor key order --- pkg/yqlib/candidate_node.go | 12 + .../operators/anchor-and-alias-operators.md | 215 +++++++++++------- pkg/yqlib/lib.go | 11 + pkg/yqlib/operator_anchors_aliases.go | 44 +++- pkg/yqlib/operator_anchors_aliases_test.go | 183 ++++++++++----- pkg/yqlib/operators_test.go | 23 +- 6 files changed, 345 insertions(+), 143 deletions(-) diff --git a/pkg/yqlib/candidate_node.go b/pkg/yqlib/candidate_node.go index 51485f11..7dee6303 100644 --- a/pkg/yqlib/candidate_node.go +++ b/pkg/yqlib/candidate_node.go @@ -169,6 +169,18 @@ func (n *CandidateNode) getParsedKey() interface{} { } +func (n *CandidateNode) FilterMapContentByKey(keyPredicate func(*CandidateNode) bool) []*CandidateNode { + var result []*CandidateNode + for index := 0; index < len(n.Content); index = index + 2 { + keyNode := n.Content[index] + valueNode := n.Content[index+1] + if keyPredicate(keyNode) { + result = append(result, keyNode, valueNode) + } + } + return result +} + func (n *CandidateNode) GetPath() []interface{} { key := n.getParsedKey() if n.Parent != nil && key != nil { diff --git a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md index 14b0bef2..7053b53d 100644 --- a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md +++ b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md @@ -34,68 +34,6 @@ y: 2 r: 10 ``` -## Merge multiple maps -see https://yaml.org/type/merge.html - -Given a sample.yml file of: -```yaml -- &CENTER - x: 1 - y: 2 -- &LEFT - x: 0 - y: 2 -- &BIG - r: 10 -- &SMALL - r: 1 -- !!merge <<: - - *CENTER - - *BIG -``` -then -```bash -yq '.[4] | explode(.)' sample.yml -``` -will output -```yaml -r: 10 -x: 1 -y: 2 -``` - -## Override -see https://yaml.org/type/merge.html - -Given a sample.yml file of: -```yaml -- &CENTER - x: 1 - y: 2 -- &LEFT - x: 0 - y: 2 -- &BIG - r: 10 -- &SMALL - r: 1 -- !!merge <<: - - *BIG - - *LEFT - - *SMALL - x: 1 -``` -then -```bash -yq '.[4] | explode(.)' sample.yml -``` -will output -```yaml -r: 10 -x: 1 -y: 2 -``` - ## Get anchor Given a sample.yml file of: ```yaml @@ -254,7 +192,101 @@ f: cat: b ``` -## Explode with merge anchors +## Dereference and update a field +Use explode with multiply to dereference an object + +Given a sample.yml file of: +```yaml +item_value: &item_value + value: true +thingOne: + name: item_1 + !!merge <<: *item_value +thingTwo: + name: item_2 + !!merge <<: *item_value +``` +then +```bash +yq '.thingOne |= explode(.) * {"value": false}' sample.yml +``` +will output +```yaml +item_value: &item_value + value: true +thingOne: + name: item_1 + value: false +thingTwo: + name: item_2 + !!merge <<: *item_value +``` + +## Merge multiple maps +see https://yaml.org/type/merge.html + +Given a sample.yml file of: +```yaml +- &CENTER + x: 1 + y: 2 +- &LEFT + x: 0 + y: 2 +- &BIG + r: 10 +- &SMALL + r: 1 +- !!merge <<: + - *CENTER + - *BIG +``` +then +```bash +yq '.[4] | explode(.)' sample.yml +``` +will output +```yaml +r: 10 +x: 1 +y: 2 +``` + +## Override +see https://yaml.org/type/merge.html + +Given a sample.yml file of: +```yaml +- &CENTER + x: 1 + y: 2 +- &LEFT + x: 0 + y: 2 +- &BIG + r: 10 +- &SMALL + r: 1 +- !!merge <<: + - *BIG + - *LEFT + - *SMALL + x: 1 +``` +then +```bash +yq '.[4] | explode(.)' sample.yml +``` +will output +```yaml +r: 10 +x: 1 +y: 2 +``` + +## LEGACY: Explode with merge anchors +Caution: this is for when --yaml-fix-merge-anchor-to-spec=false; it's not to YAML spec because the merge anchors incorrectly override the object values. Flag will default to true in late 2025 + Given a sample.yml file of: ```yaml foo: &foo @@ -301,33 +333,52 @@ foobar: thing: foobar_thing ``` -## Dereference and update a field -Use explode with multiply to dereference an object +## FIXED: Explode with merge anchors +Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 Given a sample.yml file of: ```yaml -item_value: &item_value - value: true -thingOne: - name: item_1 - !!merge <<: *item_value -thingTwo: - name: item_2 - !!merge <<: *item_value +foo: &foo + a: foo_a + thing: foo_thing + c: foo_c +bar: &bar + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + !!merge <<: + - *foo + - *bar + c: foobarList_c +foobar: + c: foobar_c + !!merge <<: *foo + thing: foobar_thing ``` then ```bash -yq '.thingOne |= explode(.) * {"value": false}' sample.yml +yq 'explode(.)' sample.yml ``` will output ```yaml -item_value: &item_value - value: true -thingOne: - name: item_1 - value: false -thingTwo: - name: item_2 - !!merge <<: *item_value +foo: + a: foo_a + thing: foo_thing + c: foo_c +bar: + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + a: foo_a + thing: foo_thing + c: foobarList_c +foobar: + c: foobar_c + a: foo_a + thing: foobar_thing ``` diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 15561930..748b3f5b 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -28,6 +28,17 @@ func GetLogger() *logging.Logger { return log } +func getContentValueByKey(content []*CandidateNode, key string) *CandidateNode { + for index := 0; index < len(content); index = index + 2 { + keyNode := content[index] + valueNode := content[index+1] + if keyNode.Value == key { + return valueNode + } + } + return nil +} + func recurseNodeArrayEqual(lhs *CandidateNode, rhs *CandidateNode) bool { if len(lhs.Content) != len(rhs.Content) { return false diff --git a/pkg/yqlib/operator_anchors_aliases.go b/pkg/yqlib/operator_anchors_aliases.go index 50744871..84c3418d 100644 --- a/pkg/yqlib/operator_anchors_aliases.go +++ b/pkg/yqlib/operator_anchors_aliases.go @@ -138,6 +138,41 @@ func explodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expr return context, nil } +func fixedReconstructAliasedMap(node *CandidateNode) error { + var newContent = []*CandidateNode{} + + for index := 0; index < len(node.Content); index = index + 2 { + keyNode := node.Content[index] + valueNode := node.Content[index+1] + if keyNode.Value != "<<" { + // always add in plain nodes + newContent = append(newContent, keyNode, valueNode) + } else { + sequence := valueNode + if sequence.Kind != SequenceNode { + sequence = &CandidateNode{Content: []*CandidateNode{sequence}} + } + for index := 0; index < len(sequence.Content); index = index + 1 { + // for merge anchors, we only set them if the key is not already in node or the newContent + mergeNodeSeq := sequence.Content[index] + if mergeNodeSeq.Kind == AliasNode { + mergeNodeSeq = mergeNodeSeq.Alias + } + if mergeNodeSeq.Kind != MappingNode { + return fmt.Errorf("merge anchor only supports maps, got !!seq instead") + } + itemsToAdd := mergeNodeSeq.FilterMapContentByKey(func(keyNode *CandidateNode) bool { + return getContentValueByKey(node.Content, keyNode.Value) == nil && + getContentValueByKey(newContent, keyNode.Value) == nil + }) + newContent = append(newContent, itemsToAdd...) + } + } + } + node.Content = newContent + return nil +} + func reconstructAliasedMap(node *CandidateNode, context Context) error { var newContent = list.New() // can I short cut here by prechecking if there's an anchor in the map? @@ -215,6 +250,10 @@ func explodeNode(node *CandidateNode, context Context) error { } if hasAlias { + if ConfiguredYamlPreferences.FixMergeAnchorToSpec { + return fixedReconstructAliasedMap(node) + } + log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025.") // this is a slow op, which is why we want to check before running it. return reconstructAliasedMap(node, context) } @@ -272,10 +311,7 @@ func overrideEntry(node *CandidateNode, key *CandidateNode, value *CandidateNode log.Debugf("checking new content %v:%v", keyNode.Value, valueEl.Value.(*CandidateNode).Value) if keyNode.Value == key.Value && keyNode.Alias == nil && key.Alias == nil { log.Debugf("overridign new content") - if !ConfiguredYamlPreferences.FixMergeAnchorToSpec { - log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing the merge anchor to override the existing value at %v which isn't to the yaml spec. This flag will default to true in late 2025.", keyNode.GetNicePath()) - valueEl.Value = value - } + valueEl.Value = value return nil } newEl = valueEl // move forward twice diff --git a/pkg/yqlib/operator_anchors_aliases_test.go b/pkg/yqlib/operator_anchors_aliases_test.go index c5b2f74b..e47de341 100644 --- a/pkg/yqlib/operator_anchors_aliases_test.go +++ b/pkg/yqlib/operator_anchors_aliases_test.go @@ -34,6 +34,25 @@ thingTwo: !!merge <<: *item_value ` +var explodeMergeAnchorsFixedExpected = `D0, P[], (!!map)::foo: + a: foo_a + thing: foo_thing + c: foo_c +bar: + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + a: foo_a + thing: foo_thing + c: foobarList_c +foobar: + c: foobar_c + a: foo_a + thing: foobar_thing +` + var explodeMergeAnchorsExpected = `D0, P[], (!!map)::foo: a: foo_a thing: foo_thing @@ -83,23 +102,115 @@ var explodeWhenKeysExistExpected = `D0, P[], (!!map)::objects: var fixedAnchorOperatorScenarios = []expressionScenario{ { - skipDoc: true, - description: "merge anchor after existing keys", - subdescription: "legacy: overrides existing keys", - document: explodeWhenKeysExistDocument, - expression: "explode(.)", - expected: []string{explodeWhenKeysExistExpected}, + skipDoc: true, + description: "merge anchor after existing keys", + document: explodeWhenKeysExistDocument, + expression: "explode(.)", + expected: []string{explodeWhenKeysExistExpected}, + }, + { + skipDoc: true, // skip doc for now, only difference is order of keys + description: "Merge multiple maps", + subdescription: "see https://yaml.org/type/merge.html", + document: specDocument + "- << : [ *CENTER, *BIG ]\n", + expression: ".[4] | explode(.)", + expected: []string{"D0, P[4], (!!map)::x: 1\ny: 2\nr: 10\n"}, + }, + { + skipDoc: true, // skip doc for now, only difference is order of keys + description: "Override", + subdescription: "see https://yaml.org/type/merge.html", + document: specDocument + "- << : [ *BIG, *LEFT, *SMALL ]\n x: 1\n", + expression: ".[4] | explode(.)", + expected: []string{"D0, P[4], (!!map)::r: 10\ny: 2\nx: 1\n"}, + }, + { + description: "FIXED: Explode with merge anchors", + subdescription: "Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 ", + document: mergeDocSample, + expression: `explode(.)`, + expected: []string{explodeMergeAnchorsFixedExpected}, + }, + { + skipDoc: true, + document: mergeDocSample, + expression: `.foo* | explode(.) | (. style="flow")`, + expected: []string{ + "D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n", + "D0, P[foobarList], (!!map)::{b: foobarList_b, a: foo_a, thing: foo_thing, c: foobarList_c}\n", + "D0, P[foobar], (!!map)::{c: foobar_c, a: foo_a, thing: foobar_thing}\n", + }, + }, + { + skipDoc: true, + document: mergeDocSample, + expression: `.foo* | explode(explode(.)) | (. style="flow")`, + expected: []string{ + "D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n", + "D0, P[foobarList], (!!map)::{b: foobarList_b, a: foo_a, thing: foo_thing, c: foobarList_c}\n", + "D0, P[foobar], (!!map)::{c: foobar_c, a: foo_a, thing: foobar_thing}\n", + }, + }, +} + +var badAnchorOperatorScenarios = []expressionScenario{ + { + skipDoc: true, // incorrect overrides + description: "LEGACY: merge anchor after existing keys", + document: explodeWhenKeysExistDocument, + expression: "explode(.)", + expected: []string{explodeWhenKeysExistLegacy}, + }, + { + description: "Merge multiple maps", // functionally correct, but key order gets mangled + subdescription: "see https://yaml.org/type/merge.html", + document: specDocument + "- << : [ *CENTER, *BIG ]\n", + expression: ".[4] | explode(.)", + expected: []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"}, + }, + { + description: "Override", // functionally correct, but key order gets mangled + subdescription: "see https://yaml.org/type/merge.html", + document: specDocument + "- << : [ *BIG, *LEFT, *SMALL ]\n x: 1\n", + expression: ".[4] | explode(.)", + expected: []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"}, + }, + { + description: "LEGACY: Explode with merge anchors", // incorrect overrides + subdescription: "Caution: this is for when --yaml-fix-merge-anchor-to-spec=false; it's not to YAML spec because the merge anchors incorrectly override the object values. Flag will default to true in late 2025", + document: mergeDocSample, + expression: `explode(.)`, + expected: []string{explodeMergeAnchorsExpected}, + }, + { + skipDoc: true, + document: mergeDocSample, // incorrect overrides + expression: `.foo* | explode(.) | (. style="flow")`, + expected: []string{ + "D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n", + "D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\n", + "D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\n", + }, + }, + { + skipDoc: true, + document: mergeDocSample, + expression: `.foo* | explode(explode(.)) | (. style="flow")`, + expected: []string{ + "D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n", + "D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\n", + "D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\n", + }, }, } var anchorOperatorScenarios = []expressionScenario{ { - skipDoc: true, - description: "merge anchor after existing keys", - subdescription: "legacy: overrides existing keys", - document: explodeWhenKeysExistDocument, - expression: "explode(.)", - expected: []string{explodeWhenKeysExistLegacy}, + skipDoc: true, + description: "merge anchor to alias alias", + document: "b: &b 10\na: &a { k: *b }\nc:\n <<: [*a]", + expression: "explode(.)", + expected: []string{"D0, P[], (!!map)::b: 10\na: {k: 10}\nc:\n k: 10\n"}, }, { skipDoc: true, @@ -115,20 +226,6 @@ var anchorOperatorScenarios = []expressionScenario{ expression: ".[4] | explode(.)", expected: []string{expectedSpecResult}, }, - { - description: "Merge multiple maps", - subdescription: "see https://yaml.org/type/merge.html", - document: specDocument + "- << : [ *CENTER, *BIG ]\n", - expression: ".[4] | explode(.)", - expected: []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"}, - }, - { - description: "Override", - subdescription: "see https://yaml.org/type/merge.html", - document: specDocument + "- << : [ *BIG, *LEFT, *SMALL ]\n x: 1\n", - expression: ".[4] | explode(.)", - expected: []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"}, - }, { description: "Get anchor", document: `a: &billyBob cat`, @@ -289,32 +386,6 @@ var anchorOperatorScenarios = []expressionScenario{ "D0, P[], (!!map)::{f: {a: cat, cat: b}}\n", }, }, - { - description: "Explode with merge anchors", - document: mergeDocSample, - expression: `explode(.)`, - expected: []string{explodeMergeAnchorsExpected}, - }, - { - skipDoc: true, - document: mergeDocSample, - expression: `.foo* | explode(.) | (. style="flow")`, - expected: []string{ - "D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n", - "D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\n", - "D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\n", - }, - }, - { - skipDoc: true, - document: mergeDocSample, - expression: `.foo* | explode(explode(.)) | (. style="flow")`, - expected: []string{ - "D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n", - "D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\n", - "D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\n", - }, - }, { skipDoc: true, document: `{f : {a: &a cat, b: &b {foo: *a}, *a: *b}}`, @@ -333,16 +404,18 @@ var anchorOperatorScenarios = []expressionScenario{ } func TestAnchorAliasOperatorScenarios(t *testing.T) { - for _, tt := range anchorOperatorScenarios { + for _, tt := range append(anchorOperatorScenarios, badAnchorOperatorScenarios...) { testScenario(t, &tt) } - documentOperatorScenarios(t, "anchor-and-alias-operators", anchorOperatorScenarios) + documentOperatorScenarios(t, "anchor-and-alias-operators", append(anchorOperatorScenarios, badAnchorOperatorScenarios...)) } func TestAnchorAliasOperatorAlignedToSpecScenarios(t *testing.T) { ConfiguredYamlPreferences.FixMergeAnchorToSpec = true - for _, tt := range fixedAnchorOperatorScenarios { + for _, tt := range append(fixedAnchorOperatorScenarios, anchorOperatorScenarios...) { testScenario(t, &tt) + } + appendOperatorDocumentScenario(t, "anchor-and-alias-operators", fixedAnchorOperatorScenarios) ConfiguredYamlPreferences.FixMergeAnchorToSpec = false } diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index eef6da26..4c9d3d01 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -6,6 +6,7 @@ import ( "container/list" "fmt" "io" + "io/fs" "os" "sort" "strings" @@ -37,7 +38,7 @@ var goccyTesting = false var testingDecoder = NewYamlDecoder(ConfiguredYamlPreferences) func TestMain(m *testing.M) { - logging.SetLevel(logging.ERROR, "") + logging.SetLevel(logging.WARNING, "") if os.Getenv("DEBUG") == "true" { logging.SetLevel(logging.DEBUG, "") } @@ -75,6 +76,7 @@ func testScenario(t *testing.T, s *expressionScenario) { if s.skipForGoccy { return } + log.Debugf("\n\ntesting scenario %v", s.description) var err error node, err := getExpressionParser().ParseExpression(s.expression) if err != nil { @@ -220,7 +222,8 @@ func formatYaml(yaml string, filename string) string { type documentScenarioFunc func(t *testing.T, writer *bufio.Writer, scenario interface{}) func documentScenarios(t *testing.T, folder string, title string, scenarios []interface{}, documentScenario documentScenarioFunc) { - f, err := os.Create(fmt.Sprintf("doc/%v/%v.md", folder, title)) + filename := fmt.Sprintf("doc/%v/%v.md", folder, title) + f, err := os.Create(filename) if err != nil { t.Error(err) @@ -250,6 +253,22 @@ func documentScenarios(t *testing.T, folder string, title string, scenarios []in w.Flush() } +func appendOperatorDocumentScenario(t *testing.T, title string, scenarios []expressionScenario) { + filename := fmt.Sprintf("doc/%v/%v.md", "operators", title) + f, err := os.OpenFile(filename, os.O_WRONLY|os.O_APPEND, fs.ModeAppend) + if err != nil { + t.Error(err) + return + } + defer f.Close() + w := bufio.NewWriter(f) + for _, s := range scenarios { + documentOperatorScenario(t, w, s) + } + w.Flush() + +} + func documentOperatorScenarios(t *testing.T, title string, scenarios []expressionScenario) { genericScenarios := make([]interface{}, len(scenarios)) for i, s := range scenarios { From ae87394f4a66969b8a96e727a89bae56941d6d21 Mon Sep 17 00:00:00 2001 From: Steven WdV Date: Sun, 20 Jul 2025 13:24:48 +0200 Subject: [PATCH 17/27] Formatting --- pkg/yqlib/operator_traverse_path.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/yqlib/operator_traverse_path.go b/pkg/yqlib/operator_traverse_path.go index d0010b42..bad0b84f 100644 --- a/pkg/yqlib/operator_traverse_path.go +++ b/pkg/yqlib/operator_traverse_path.go @@ -3,6 +3,7 @@ package yqlib import ( "container/list" "fmt" + "github.com/elliotchance/orderedmap" ) From 3431aebb2cd825a67b02ce6dee4a9af251e42e05 Mon Sep 17 00:00:00 2001 From: Steven WdV Date: Sun, 20 Jul 2025 13:28:14 +0200 Subject: [PATCH 18/27] Add tests for accessing `!!str <<` --- pkg/yqlib/operator_anchors_aliases_test.go | 9 +++++++++ pkg/yqlib/operator_traverse_path_test.go | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/pkg/yqlib/operator_anchors_aliases_test.go b/pkg/yqlib/operator_anchors_aliases_test.go index 988af027..d66522c2 100644 --- a/pkg/yqlib/operator_anchors_aliases_test.go +++ b/pkg/yqlib/operator_anchors_aliases_test.go @@ -438,6 +438,15 @@ var anchorOperatorScenarios = []expressionScenario{ "D0, P[], (!!map)::{a: 1, a: 2}\n", }, }, + { + skipDoc: true, + description: "!!str << should not be treated as merge anchor", + document: `{!!str <<: {a: 37}}`, + expression: `explode(.).a`, + expected: []string{ + "D0, P[a], (!!null)::null\n", + }, + }, } func TestAnchorAliasOperatorScenarios(t *testing.T) { diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go index 65ec97f9..0b29376c 100644 --- a/pkg/yqlib/operator_traverse_path_test.go +++ b/pkg/yqlib/operator_traverse_path_test.go @@ -631,6 +631,15 @@ var traversePathOperatorScenarios = []expressionScenario{ "D0, P[<<], (!!int)::37\n", }, }, + { + skipDoc: true, + description: "!!str << should not be treated as merge anchor", + document: `{!!str <<: {a: 37}}`, + expression: `.a`, + expected: []string{ + "D0, P[a], (!!null)::null\n", + }, + }, } func TestTraversePathOperatorScenarios(t *testing.T) { From 9c95a9f37961ee6b647804ab5cea08eb825df078 Mon Sep 17 00:00:00 2001 From: Steven WdV Date: Sun, 20 Jul 2025 14:52:06 +0200 Subject: [PATCH 19/27] Unify reconstructAliasedMap & fixedReconstructAliasedMap --- .../operators/anchor-and-alias-operators.md | 66 +++++++++--------- pkg/yqlib/lib.go | 11 --- pkg/yqlib/operator_anchors_aliases.go | 69 ++++++------------- pkg/yqlib/operator_anchors_aliases_test.go | 41 +++++------ 4 files changed, 69 insertions(+), 118 deletions(-) diff --git a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md index 707179d2..a705f1fd 100644 --- a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md +++ b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md @@ -34,6 +34,36 @@ y: 2 r: 10 ``` +## Merge multiple maps +see https://yaml.org/type/merge.html + +Given a sample.yml file of: +```yaml +- &CENTER + x: 1 + y: 2 +- &LEFT + x: 0 + y: 2 +- &BIG + r: 10 +- &SMALL + r: 1 +- !!merge <<: + - *CENTER + - *BIG +``` +then +```bash +yq '.[4] | explode(.)' sample.yml +``` +will output +```yaml +r: 10 +x: 1 +y: 2 +``` + ## Get anchor Given a sample.yml file of: ```yaml @@ -208,7 +238,7 @@ thingTwo: ``` then ```bash -yq '.thingOne |= explode(.) * {"value": false}' sample.yml +yq '.thingOne |= (explode(.) | sort_keys(.)) * {"value": false}' sample.yml ``` will output ```yaml @@ -222,36 +252,6 @@ thingTwo: !!merge <<: *item_value ``` -## Merge multiple maps -see https://yaml.org/type/merge.html - -Given a sample.yml file of: -```yaml -- &CENTER - x: 1 - y: 2 -- &LEFT - x: 0 - y: 2 -- &BIG - r: 10 -- &SMALL - r: 1 -- !!merge <<: - - *CENTER - - *BIG -``` -then -```bash -yq '.[4] | explode(.)' sample.yml -``` -will output -```yaml -r: 10 -x: 1 -y: 2 -``` - ## LEGACY: Explode with merge anchors Caution: this is for when --yaml-fix-merge-anchor-to-spec=false; it's not to YAML spec because the merge anchors incorrectly override the object values. Flag will default to true in late 2025 @@ -341,12 +341,12 @@ bar: c: bar_c foobarList: b: foobarList_b - a: foo_a thing: foo_thing + a: foo_a c: foobarList_c foobar: - c: foobar_c a: foo_a + c: foobar_c thing: foobar_thing ``` diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 748b3f5b..15561930 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -28,17 +28,6 @@ func GetLogger() *logging.Logger { return log } -func getContentValueByKey(content []*CandidateNode, key string) *CandidateNode { - for index := 0; index < len(content); index = index + 2 { - keyNode := content[index] - valueNode := content[index+1] - if keyNode.Value == key { - return valueNode - } - } - return nil -} - func recurseNodeArrayEqual(lhs *CandidateNode, rhs *CandidateNode) bool { if len(lhs.Content) != len(rhs.Content) { return false diff --git a/pkg/yqlib/operator_anchors_aliases.go b/pkg/yqlib/operator_anchors_aliases.go index 2c8f6106..84c31709 100644 --- a/pkg/yqlib/operator_anchors_aliases.go +++ b/pkg/yqlib/operator_anchors_aliases.go @@ -139,58 +139,38 @@ func explodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expr return context, nil } -func fixedReconstructAliasedMap(node *CandidateNode) error { - var newContent = []*CandidateNode{} - - for index := 0; index < len(node.Content); index = index + 2 { - keyNode := node.Content[index] - valueNode := node.Content[index+1] - if keyNode.Value != "<<" { - // always add in plain nodes - newContent = append(newContent, keyNode, valueNode) - } else { - sequence := valueNode - if sequence.Kind != SequenceNode { - sequence = &CandidateNode{Content: []*CandidateNode{sequence}} - } - for index := 0; index < len(sequence.Content); index = index + 1 { - // for merge anchors, we only set them if the key is not already in node or the newContent - mergeNodeSeq := sequence.Content[index] - if mergeNodeSeq.Kind == AliasNode { - mergeNodeSeq = mergeNodeSeq.Alias - } - if mergeNodeSeq.Kind != MappingNode { - return fmt.Errorf("merge anchor only supports maps, got !!seq instead") - } - // Only retain keys from merge map that are not already in node.Content or newContent, - // to prevent overwriting - itemsToAdd := mergeNodeSeq.FilterMapContentByKey(func(keyNode *CandidateNode) bool { - return getContentValueByKey(node.Content, keyNode.Value) == nil && - getContentValueByKey(newContent, keyNode.Value) == nil - }) - newContent = append(newContent, itemsToAdd...) - } - } - } - node.Content = newContent - return nil -} - func reconstructAliasedMap(node *CandidateNode, context Context) error { var newContent = list.New() // can I short cut here by prechecking if there's an anchor in the map? // no it needs to recurse in overrideEntry. + if !ConfiguredYamlPreferences.FixMergeAnchorToSpec { + log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025.") + } + if ConfiguredYamlPreferences.FixMergeAnchorToSpec { + for index := len(node.Content) - 2; index >= 0; index -= 2 { + keyNode := node.Content[index] + valueNode := node.Content[index+1] + if keyNode.Tag == "!!merge" { + log.Debugf("traversing merge key %v", keyNode.Value) + err := applyMergeAnchor(node, valueNode, index, context.ChildContext(newContent)) + if err != nil { + return err + } + } + } + } for index := 0; index < len(node.Content); index = index + 2 { keyNode := node.Content[index] valueNode := node.Content[index+1] - log.Debugf("traversing %v", keyNode.Value) if keyNode.Tag != "!!merge" { + log.Debugf("traversing %v", keyNode.Value) err := overrideEntry(node, keyNode, valueNode, index, true, context.ChildContext(newContent)) if err != nil { return err } - } else { + } else if !ConfiguredYamlPreferences.FixMergeAnchorToSpec { + log.Debugf("traversing %v", keyNode.Value) err := applyMergeAnchor(node, valueNode, index, context.ChildContext(newContent)) if err != nil { return err @@ -241,10 +221,6 @@ func explodeNode(node *CandidateNode, context Context) error { } if hasAlias { - if ConfiguredYamlPreferences.FixMergeAnchorToSpec { - return fixedReconstructAliasedMap(node) - } - log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025.") // this is a slow op, which is why we want to check before running it. return reconstructAliasedMap(node, context) } @@ -279,12 +255,7 @@ func applyMergeAnchor(node *CandidateNode, merge *CandidateNode, mergeIndex int, return applyMergeAnchorMap(node, merge, mergeIndex, inline, newContent) case SequenceNode: log.Debugf("a merge list!") - // With FixMergeAnchorToSpec, we rely on overrideEntry to reject duplicates - content := slices.All(merge.Content) - if !ConfiguredYamlPreferences.FixMergeAnchorToSpec { - // Even without FixMergeAnchorToSpec, this already gave preference to earlier keys - content = slices.Backward(merge.Content) - } + content := slices.Backward(merge.Content) for _, childValue := range content { childInline := inline if childValue.Kind == AliasNode { diff --git a/pkg/yqlib/operator_anchors_aliases_test.go b/pkg/yqlib/operator_anchors_aliases_test.go index 6890e8d0..de8f526f 100644 --- a/pkg/yqlib/operator_anchors_aliases_test.go +++ b/pkg/yqlib/operator_anchors_aliases_test.go @@ -44,12 +44,12 @@ bar: c: bar_c foobarList: b: foobarList_b - a: foo_a thing: foo_thing + a: foo_a c: foobarList_c foobar: - c: foobar_c a: foo_a + c: foobar_c thing: foobar_thing ` @@ -109,14 +109,6 @@ var fixedAnchorOperatorScenarios = []expressionScenario{ expression: "explode(.)", expected: []string{explodeWhenKeysExistExpected}, }, - { - skipDoc: true, // skip doc for now, only difference is order of keys - description: "Merge multiple maps", - subdescription: "see https://yaml.org/type/merge.html", - document: specDocument + "- << : [ *CENTER, *BIG ]\n", - expression: ".[4] | explode(.)", - expected: []string{"D0, P[4], (!!map)::x: 1\ny: 2\nr: 10\n"}, - }, { description: "FIXED: Explode with merge anchors", subdescription: "Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 ", @@ -130,8 +122,8 @@ var fixedAnchorOperatorScenarios = []expressionScenario{ expression: `.foo* | explode(.) | (. style="flow")`, expected: []string{ "D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n", - "D0, P[foobarList], (!!map)::{b: foobarList_b, a: foo_a, thing: foo_thing, c: foobarList_c}\n", - "D0, P[foobar], (!!map)::{c: foobar_c, a: foo_a, thing: foobar_thing}\n", + "D0, P[foobarList], (!!map)::{b: foobarList_b, thing: foo_thing, a: foo_a, c: foobarList_c}\n", + "D0, P[foobar], (!!map)::{a: foo_a, c: foobar_c, thing: foobar_thing}\n", }, }, { @@ -140,8 +132,8 @@ var fixedAnchorOperatorScenarios = []expressionScenario{ expression: `.foo* | explode(explode(.)) | (. style="flow")`, expected: []string{ "D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n", - "D0, P[foobarList], (!!map)::{b: foobarList_b, a: foo_a, thing: foo_thing, c: foobarList_c}\n", - "D0, P[foobar], (!!map)::{c: foobar_c, a: foo_a, thing: foobar_thing}\n", + "D0, P[foobarList], (!!map)::{b: foobarList_b, thing: foo_thing, a: foo_a, c: foobarList_c}\n", + "D0, P[foobar], (!!map)::{a: foo_a, c: foobar_c, thing: foobar_thing}\n", }, }, } @@ -154,13 +146,6 @@ var badAnchorOperatorScenarios = []expressionScenario{ expression: "explode(.)", expected: []string{explodeWhenKeysExistLegacy}, }, - { - description: "Merge multiple maps", // functionally correct, but key order gets mangled - subdescription: "see https://yaml.org/type/merge.html", - document: specDocument + "- << : [ *CENTER, *BIG ]\n", - expression: ".[4] | explode(.)", - expected: []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"}, - }, { description: "LEGACY: Explode with merge anchors", // incorrect overrides subdescription: "Caution: this is for when --yaml-fix-merge-anchor-to-spec=false; it's not to YAML spec because the merge anchors incorrectly override the object values. Flag will default to true in late 2025", @@ -213,7 +198,13 @@ var anchorOperatorScenarios = []expressionScenario{ expected: []string{expectedSpecResult}, }, { - skipDoc: true, // skip doc for now, only difference is order of keys + description: "Merge multiple maps", // functionally correct, but key order gets mangled + subdescription: "see https://yaml.org/type/merge.html", + document: specDocument + "- << : [ *CENTER, *BIG ]\n", + expression: ".[4] | explode(.)", + expected: []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"}, + }, + { description: "Override", subdescription: "see https://yaml.org/type/merge.html", document: specDocument + "- << : [ *BIG, *LEFT, *SMALL ]\n x: 1\n", @@ -392,7 +383,7 @@ var anchorOperatorScenarios = []expressionScenario{ description: "Dereference and update a field", subdescription: "Use explode with multiply to dereference an object", document: simpleArrayRef, - expression: `.thingOne |= explode(.) * {"value": false}`, + expression: `.thingOne |= (explode(.) | sort_keys(.)) * {"value": false}`, expected: []string{expectedUpdatedArrayRef}, }, { @@ -447,7 +438,7 @@ var anchorOperatorScenarios = []expressionScenario{ description: "Exploding inline merge anchor", subdescription: "`<<` map must be exploded, otherwise `c: *b` will become invalid", document: `{a: {b: &b 42}, <<: {c: *b}}`, - expression: `explode(.)`, + expression: `explode(.) | sort_keys(.)`, expected: []string{ "D0, P[], (!!map)::{a: {b: 42}, c: 42}\n", }, @@ -457,7 +448,7 @@ var anchorOperatorScenarios = []expressionScenario{ description: "Exploding inline merge anchor in sequence", subdescription: "`<<` map must be exploded, otherwise `c: *b` will become invalid", document: `{a: {b: &b 42}, <<: [{c: *b}]}`, - expression: `explode(.)`, + expression: `explode(.) | sort_keys(.)`, expected: []string{ "D0, P[], (!!map)::{a: {b: 42}, c: 42}\n", }, From 904215ef4df2fa18df54c778e357861cbe0bc9bc Mon Sep 17 00:00:00 2001 From: Steven WdV Date: Sun, 20 Jul 2025 15:37:45 +0200 Subject: [PATCH 20/27] Fix key overriding in regular maps for traversing --- .../operators/anchor-and-alias-operators.md | 49 ++++++ pkg/yqlib/doc/operators/traverse-read.md | 166 +++++++++--------- pkg/yqlib/operator_anchors_aliases.go | 6 +- pkg/yqlib/operator_traverse_path.go | 68 +++---- pkg/yqlib/operator_traverse_path_test.go | 143 +++++++++------ 5 files changed, 257 insertions(+), 175 deletions(-) diff --git a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md index ea4b9b2f..eca706e5 100644 --- a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md +++ b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md @@ -382,3 +382,52 @@ foobar: thing: foobar_thing ``` +## FIXED: Explode with merge anchors +Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 + +Given a sample.yml file of: +```yaml +foo: &foo + a: foo_a + thing: foo_thing + c: foo_c +bar: &bar + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + !!merge <<: + - *foo + - *bar + c: foobarList_c +foobar: + c: foobar_c + !!merge <<: *foo + thing: foobar_thing +``` +then +```bash +yq 'explode(.)' sample.yml +``` +will output +```yaml +foo: + a: foo_a + thing: foo_thing + c: foo_c +bar: + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + thing: foo_thing + a: foo_a + c: foobarList_c +foobar: + a: foo_a + c: foobar_c + thing: foobar_thing +``` + diff --git a/pkg/yqlib/doc/operators/traverse-read.md b/pkg/yqlib/doc/operators/traverse-read.md index 775eeae0..2f85a111 100644 --- a/pkg/yqlib/doc/operators/traverse-read.md +++ b/pkg/yqlib/doc/operators/traverse-read.md @@ -303,6 +303,55 @@ will output foo_a ``` +## Traversing merge anchors with local override +Given a sample.yml file of: +```yaml +foo: &foo + a: foo_a + thing: foo_thing + c: foo_c +bar: &bar + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + !!merge <<: + - *foo + - *bar + c: foobarList_c +foobar: + c: foobar_c + !!merge <<: *foo + thing: foobar_thing +``` +then +```bash +yq '.foobar.thing' sample.yml +``` +will output +```yaml +foobar_thing +``` + +## Select multiple indices +Given a sample.yml file of: +```yaml +a: + - a + - b + - c +``` +then +```bash +yq '.a[0, 2]' sample.yml +``` +will output +```yaml +a +c +``` + ## Traversing merge anchors with override This is legacy behaviour, see --yaml-fix-merge-anchor-to-spec @@ -336,70 +385,6 @@ will output foo_c ``` -## Traversing merge anchors with local override -Given a sample.yml file of: -```yaml -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - !!merge <<: - - *foo - - *bar - c: foobarList_c -foobar: - c: foobar_c - !!merge <<: *foo - thing: foobar_thing -``` -then -```bash -yq '.foobar.thing' sample.yml -``` -will output -```yaml -foobar_thing -``` - -## Splatting merge anchors -Given a sample.yml file of: -```yaml -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - !!merge <<: - - *foo - - *bar - c: foobarList_c -foobar: - c: foobar_c - !!merge <<: *foo - thing: foobar_thing -``` -then -```bash -yq '.foobar[]' sample.yml -``` -will output -```yaml -foo_c -foo_a -foobar_thing -``` - ## Traversing merge anchor lists Note that the later merge anchors override previous, but this is legacy behaviour, see --yaml-fix-merge-anchor-to-spec @@ -433,6 +418,41 @@ will output bar_thing ``` +## Splatting merge anchors +With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec + +Given a sample.yml file of: +```yaml +foo: &foo + a: foo_a + thing: foo_thing + c: foo_c +bar: &bar + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + !!merge <<: + - *foo + - *bar + c: foobarList_c +foobar: + c: foobar_c + !!merge <<: *foo + thing: foobar_thing +``` +then +```bash +yq '.foobar[]' sample.yml +``` +will output +```yaml +foo_c +foo_a +foobar_thing +``` + ## Splatting merge anchor lists With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec @@ -469,21 +489,3 @@ bar_thing foobarList_c ``` -## Select multiple indices -Given a sample.yml file of: -```yaml -a: - - a - - b - - c -``` -then -```bash -yq '.a[0, 2]' sample.yml -``` -will output -```yaml -a -c -``` - diff --git a/pkg/yqlib/operator_anchors_aliases.go b/pkg/yqlib/operator_anchors_aliases.go index 84c31709..ab784f8a 100644 --- a/pkg/yqlib/operator_anchors_aliases.go +++ b/pkg/yqlib/operator_anchors_aliases.go @@ -144,9 +144,6 @@ func reconstructAliasedMap(node *CandidateNode, context Context) error { // can I short cut here by prechecking if there's an anchor in the map? // no it needs to recurse in overrideEntry. - if !ConfiguredYamlPreferences.FixMergeAnchorToSpec { - log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025.") - } if ConfiguredYamlPreferences.FixMergeAnchorToSpec { for index := len(node.Content) - 2; index >= 0; index -= 2 { keyNode := node.Content[index] @@ -159,7 +156,10 @@ func reconstructAliasedMap(node *CandidateNode, context Context) error { } } } + } else { + log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025.") } + for index := 0; index < len(node.Content); index = index + 2 { keyNode := node.Content[index] valueNode := node.Content[index+1] diff --git a/pkg/yqlib/operator_traverse_path.go b/pkg/yqlib/operator_traverse_path.go index bad0b84f..11bfea1b 100644 --- a/pkg/yqlib/operator_traverse_path.go +++ b/pkg/yqlib/operator_traverse_path.go @@ -3,6 +3,7 @@ package yqlib import ( "container/list" "fmt" + "slices" "github.com/elliotchance/orderedmap" ) @@ -265,53 +266,52 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, node *CandidateNode, wante // if we don't find a match directly on this node first. var contents = node.Content + + if !prefs.DontFollowAlias { + if ConfiguredYamlPreferences.FixMergeAnchorToSpec { + for index := len(node.Content) - 2; index >= 0; index -= 2 { + keyNode := node.Content[index] + valueNode := node.Content[index+1] + if keyNode.Tag == "!!merge" { + log.Debug("Merge anchor") + err := traverseMergeAnchor(newMatches, valueNode, wantedKey, prefs, splat) + if err != nil { + return err + } + } + } + } else { + log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025.") + } + } + for index := 0; index+1 < len(contents); index = index + 2 { key := contents[index] value := contents[index+1] //skip the 'merge' tag, find a direct match first - if key.Tag == "!!merge" && !prefs.DontFollowAlias && wantedKey != "<<" { - log.Debug("Merge anchor") - err := traverseMergeAnchor(newMatches, value, wantedKey, prefs, splat) - if err != nil { - return err + if key.Tag == "!!merge" && !prefs.DontFollowAlias && wantedKey != key.Value { + if !ConfiguredYamlPreferences.FixMergeAnchorToSpec { + log.Debug("Merge anchor") + err := traverseMergeAnchor(newMatches, value, wantedKey, prefs, splat) + if err != nil { + return err + } } } else if splat || keyMatches(key, wantedKey) { log.Debug("MATCHED") if prefs.IncludeMapKeys { log.Debug("including key") keyName := key.GetKey() - if newMatches.Has(keyName) { - if ConfiguredYamlPreferences.FixMergeAnchorToSpec { - log.Debug("not overwriting existing key") - } else { - log.Warning( - "--yaml-fix-merge-anchor-to-spec is false; "+ - "causing the merge anchor to override the existing key at %v which isn't to the yaml spec. "+ - "This flag will default to true in late 2025.", key.GetNicePath()) - log.Debug("overwriting existing key") - newMatches.Set(keyName, key) - } - } else { - newMatches.Set(keyName, key) + if !newMatches.Set(keyName, key) { + log.Debug("overwriting existing key") } } if !prefs.DontIncludeMapValues { log.Debug("including value") valueName := value.GetKey() - if newMatches.Has(valueName) { - if ConfiguredYamlPreferences.FixMergeAnchorToSpec { - log.Debug("not overwriting existing value") - } else { - log.Warning( - "--yaml-fix-merge-anchor-to-spec is false; "+ - "causing the merge anchor to override the existing value at %v which isn't to the yaml spec. "+ - "This flag will default to true in late 2025.", key.GetNicePath()) - log.Debug("overwriting existing value") - newMatches.Set(valueName, value) - } - } else { - newMatches.Set(valueName, value) + if !newMatches.Set(valueName, value) { + log.Debug("overwriting existing value") } } } @@ -328,7 +328,11 @@ func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, merge *CandidateNode case MappingNode: return doTraverseMap(newMatches, merge, wantedKey, prefs, splat) case SequenceNode: - for _, childValue := range merge.Content { + content := slices.All(merge.Content) + if ConfiguredYamlPreferences.FixMergeAnchorToSpec { + content = slices.Backward(merge.Content) + } + for _, childValue := range content { if childValue.Kind == AliasNode { childValue = childValue.Alias } diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go index 7574b4e0..b25da9c5 100644 --- a/pkg/yqlib/operator_traverse_path_test.go +++ b/pkg/yqlib/operator_traverse_path_test.go @@ -27,6 +27,7 @@ foobar: var fixedTraversePathOperatorScenarios = []expressionScenario{ { + skipDoc: true, description: "Traversing merge anchor lists", subdescription: "Note that the keys earlier in the merge anchors sequence override later ones", document: mergeDocSample, @@ -36,6 +37,7 @@ var fixedTraversePathOperatorScenarios = []expressionScenario{ }, }, { + skipDoc: true, description: "Traversing merge anchors with override", document: mergeDocSample, expression: `.foobar.c`, @@ -43,16 +45,89 @@ var fixedTraversePathOperatorScenarios = []expressionScenario{ "D0, P[foobar c], (!!str)::foobar_c\n", }, }, + { + skipDoc: true, + description: "Splatting merge anchors", + document: mergeDocSample, + expression: `.foobar[]`, + expected: []string{ + "D0, P[foo a], (!!str)::foo_a\n", + "D0, P[foobar thing], (!!str)::foobar_thing\n", + "D0, P[foobar c], (!!str)::foobar_c\n", + }, + }, + { + skipDoc: true, + description: "Splatting merge anchor lists", + document: mergeDocSample, + expression: `.foobarList[]`, + expected: []string{ + "D0, P[foobarList b], (!!str)::foobarList_b\n", + "D0, P[foo thing], (!!str)::foo_thing\n", + "D0, P[foobarList c], (!!str)::foobarList_c\n", + "D0, P[foo a], (!!str)::foo_a\n", + }, + }, + { + skipDoc: true, + document: mergeDocSample, + expression: `.foobarList.b`, + expected: []string{ + "D0, P[foobarList b], (!!str)::foobarList_b\n", + }, + }, +} - // The following tests are the same as below, to verify they still works correctly with the flag: +var badTraversePathOperatorScenarios = []expressionScenario{ + { + description: "Traversing merge anchors with override", + subdescription: "This is legacy behaviour, see --yaml-fix-merge-anchor-to-spec", + document: mergeDocSample, + expression: `.foobar.c`, + expected: []string{ + "D0, P[foo c], (!!str)::foo_c\n", + }, + }, + { + description: "Traversing merge anchor lists", + subdescription: "Note that the later merge anchors override previous, " + + "but this is legacy behaviour, see --yaml-fix-merge-anchor-to-spec", + document: mergeDocSample, + expression: `.foobarList.thing`, + expected: []string{ + "D0, P[bar thing], (!!str)::bar_thing\n", + }, + }, + { + description: "Splatting merge anchors", + subdescription: "With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec", + document: mergeDocSample, + expression: `.foobar[]`, + expected: []string{ + "D0, P[foo c], (!!str)::foo_c\n", + "D0, P[foo a], (!!str)::foo_a\n", + "D0, P[foobar thing], (!!str)::foobar_thing\n", + }, + }, + { + description: "Splatting merge anchor lists", + subdescription: "With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec", + document: mergeDocSample, + expression: `.foobarList[]`, + expected: []string{ + "D0, P[bar b], (!!str)::bar_b\n", + "D0, P[foo a], (!!str)::foo_a\n", + "D0, P[bar thing], (!!str)::bar_thing\n", + "D0, P[foobarList c], (!!str)::foobarList_c\n", + }, + }, { skipDoc: true, - description: "Duplicate keys", - subdescription: "outside merge anchor", - document: `{a: 1, a: 2}`, - expression: `.a`, + subdescription: "This is legacy behaviour, see --yaml-fix-merge-anchor-to-spec", + document: mergeDocSample, + expression: `.foobarList.b`, expected: []string{ - "D0, P[a], (!!int)::2\n", + "D0, P[bar b], (!!str)::bar_b\n", }, }, } @@ -431,15 +506,6 @@ var traversePathOperatorScenarios = []expressionScenario{ "D0, P[foo a], (!!str)::foo_a\n", }, }, - { - description: "Traversing merge anchors with override", - subdescription: "This is legacy behaviour, see --yaml-fix-merge-anchor-to-spec", - document: mergeDocSample, - expression: `.foobar.c`, - expected: []string{ - "D0, P[foo c], (!!str)::foo_c\n", - }, - }, { description: "Traversing merge anchors with local override", document: mergeDocSample, @@ -448,16 +514,6 @@ var traversePathOperatorScenarios = []expressionScenario{ "D0, P[foobar thing], (!!str)::foobar_thing\n", }, }, - { - description: "Splatting merge anchors", - document: mergeDocSample, - expression: `.foobar[]`, - expected: []string{ - "D0, P[foo c], (!!str)::foo_c\n", - "D0, P[foo a], (!!str)::foo_a\n", - "D0, P[foobar thing], (!!str)::foobar_thing\n", - }, - }, { skipDoc: true, document: mergeDocSample, @@ -474,16 +530,6 @@ var traversePathOperatorScenarios = []expressionScenario{ "D0, P[foo a], (!!str)::foo_a\n", }, }, - { - description: "Traversing merge anchor lists", - subdescription: "Note that the later merge anchors override previous, " + - "but this is legacy behaviour, see --yaml-fix-merge-anchor-to-spec", - document: mergeDocSample, - expression: `.foobarList.thing`, - expected: []string{ - "D0, P[bar thing], (!!str)::bar_thing\n", - }, - }, { skipDoc: true, document: mergeDocSample, @@ -492,26 +538,6 @@ var traversePathOperatorScenarios = []expressionScenario{ "D0, P[foobarList c], (!!str)::foobarList_c\n", }, }, - { - skipDoc: true, - document: mergeDocSample, - expression: `.foobarList.b`, - expected: []string{ - "D0, P[bar b], (!!str)::bar_b\n", - }, - }, - { - description: "Splatting merge anchor lists", - subdescription: "With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec", - document: mergeDocSample, - expression: `.foobarList[]`, - expected: []string{ - "D0, P[bar b], (!!str)::bar_b\n", - "D0, P[foo a], (!!str)::foo_a\n", - "D0, P[bar thing], (!!str)::bar_thing\n", - "D0, P[foobarList c], (!!str)::foobarList_c\n", - }, - }, { skipDoc: true, document: `[a,b,c]`, @@ -643,16 +669,17 @@ var traversePathOperatorScenarios = []expressionScenario{ } func TestTraversePathOperatorScenarios(t *testing.T) { - for _, tt := range traversePathOperatorScenarios { + for _, tt := range append(traversePathOperatorScenarios, badTraversePathOperatorScenarios...) { testScenario(t, &tt) } - documentOperatorScenarios(t, "traverse-read", traversePathOperatorScenarios) + documentOperatorScenarios(t, "traverse-read", append(traversePathOperatorScenarios, badTraversePathOperatorScenarios...)) } func TestTraversePathOperatorAlignedToSpecScenarios(t *testing.T) { ConfiguredYamlPreferences.FixMergeAnchorToSpec = true - for _, tt := range fixedTraversePathOperatorScenarios { + for _, tt := range append(fixedTraversePathOperatorScenarios, traversePathOperatorScenarios...) { testScenario(t, &tt) } + appendOperatorDocumentScenario(t, "anchor-and-alias-operators", fixedAnchorOperatorScenarios) ConfiguredYamlPreferences.FixMergeAnchorToSpec = false } From 70ac3d6c7aa95136db4d715371958f5bbbb82abd Mon Sep 17 00:00:00 2001 From: Steven WdV Date: Sun, 20 Jul 2025 15:46:15 +0200 Subject: [PATCH 21/27] Add override behavior comments --- pkg/yqlib/operator_anchors_aliases.go | 5 +++++ pkg/yqlib/operator_traverse_path.go | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/pkg/yqlib/operator_anchors_aliases.go b/pkg/yqlib/operator_anchors_aliases.go index ab784f8a..9ad74b2f 100644 --- a/pkg/yqlib/operator_anchors_aliases.go +++ b/pkg/yqlib/operator_anchors_aliases.go @@ -145,6 +145,9 @@ func reconstructAliasedMap(node *CandidateNode, context Context) error { // no it needs to recurse in overrideEntry. if ConfiguredYamlPreferences.FixMergeAnchorToSpec { + // First evaluate merge keys to make explicit keys take precedence, following spec + // We also iterate in reverse to make earlier merge keys take precedence, + // although normally there's just one '<<' for index := len(node.Content) - 2; index >= 0; index -= 2 { keyNode := node.Content[index] valueNode := node.Content[index+1] @@ -255,6 +258,8 @@ func applyMergeAnchor(node *CandidateNode, merge *CandidateNode, mergeIndex int, return applyMergeAnchorMap(node, merge, mergeIndex, inline, newContent) case SequenceNode: log.Debugf("a merge list!") + // Reverse to make earlier values take precedence, following spec + // Note: This was already the case before FixMergeAnchorToSpec content := slices.Backward(merge.Content) for _, childValue := range content { childInline := inline diff --git a/pkg/yqlib/operator_traverse_path.go b/pkg/yqlib/operator_traverse_path.go index 11bfea1b..85490fff 100644 --- a/pkg/yqlib/operator_traverse_path.go +++ b/pkg/yqlib/operator_traverse_path.go @@ -269,6 +269,9 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, node *CandidateNode, wante if !prefs.DontFollowAlias { if ConfiguredYamlPreferences.FixMergeAnchorToSpec { + // First evaluate merge keys to make explicit keys take precedence, following spec + // We also iterate in reverse to make earlier merge keys take precedence, + // although normally there's just one '<<' for index := len(node.Content) - 2; index >= 0; index -= 2 { keyNode := node.Content[index] valueNode := node.Content[index+1] @@ -330,6 +333,7 @@ func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, merge *CandidateNode case SequenceNode: content := slices.All(merge.Content) if ConfiguredYamlPreferences.FixMergeAnchorToSpec { + // Reverse to make earlier values take precedence, following spec content = slices.Backward(merge.Content) } for _, childValue := range content { From f35e57d901b5d961e8349c2ca522aa618663c8b5 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Tue, 22 Jul 2025 10:02:27 +1000 Subject: [PATCH 22/27] Warn less about flag --- pkg/yqlib/operator_anchors_aliases.go | 5 ++++- pkg/yqlib/operator_traverse_path.go | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/yqlib/operator_anchors_aliases.go b/pkg/yqlib/operator_anchors_aliases.go index 9ad74b2f..cd1752e9 100644 --- a/pkg/yqlib/operator_anchors_aliases.go +++ b/pkg/yqlib/operator_anchors_aliases.go @@ -6,6 +6,8 @@ import ( "slices" ) +var showMergeAnchorToSpecWarning = true + func assignAliasOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("AssignAlias operator!") @@ -159,8 +161,9 @@ func reconstructAliasedMap(node *CandidateNode, context Context) error { } } } - } else { + } else if showMergeAnchorToSpecWarning { log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025.") + showMergeAnchorToSpecWarning = false } for index := 0; index < len(node.Content); index = index + 2 { diff --git a/pkg/yqlib/operator_traverse_path.go b/pkg/yqlib/operator_traverse_path.go index 85490fff..2042828b 100644 --- a/pkg/yqlib/operator_traverse_path.go +++ b/pkg/yqlib/operator_traverse_path.go @@ -283,8 +283,6 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, node *CandidateNode, wante } } } - } else { - log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025.") } } @@ -296,6 +294,10 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, node *CandidateNode, wante if key.Tag == "!!merge" && !prefs.DontFollowAlias && wantedKey != key.Value { if !ConfiguredYamlPreferences.FixMergeAnchorToSpec { log.Debug("Merge anchor") + if showMergeAnchorToSpecWarning { + log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025.") + showMergeAnchorToSpecWarning = false + } err := traverseMergeAnchor(newMatches, value, wantedKey, prefs, splat) if err != nil { return err From 84b095bbc48674a4726ec1f22dce06554f669ad3 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Tue, 22 Jul 2025 10:18:39 +1000 Subject: [PATCH 23/27] Making examples of the merge fix flag clearer --- .../operators/anchor-and-alias-operators.md | 14 ++++-- .../headers/anchor-and-alias-operators.md | 8 ++++ .../doc/operators/headers/traverse-read.md | 9 ++++ pkg/yqlib/doc/operators/traverse-read.md | 17 +++++-- pkg/yqlib/operator_anchors_aliases.go | 2 +- pkg/yqlib/operator_anchors_aliases_test.go | 6 +-- pkg/yqlib/operator_traverse_path.go | 2 +- pkg/yqlib/operator_traverse_path_test.go | 47 +++++++++---------- 8 files changed, 69 insertions(+), 36 deletions(-) diff --git a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md index eca706e5..d2d1ed37 100644 --- a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md +++ b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md @@ -5,6 +5,14 @@ Use the `alias` and `anchor` operators to read and write yaml aliases and anchor `yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag. +## NOTE --yaml-fix-merge-anchor-to-spec flag +`yq` doesn't merge anchors `<<:` to spec, in some circumstances it incorrectly overrides existing keys when the spec documents not to do that. + +To minimise disruption while still fixing the issue, a flag has been added to toggle this behaviour. This will first default to false; and log warnings to users. Then it will default to true (and still allow users to specify false if needed) + +See examples of the flag difference below. + + ## Merge one map see https://yaml.org/type/merge.html @@ -285,7 +293,7 @@ thingTwo: ``` ## LEGACY: Explode with merge anchors -Caution: this is for when --yaml-fix-merge-anchor-to-spec=false; it's not to YAML spec because the merge anchors incorrectly override the object values. Flag will default to true in late 2025 +Caution: this is for when --yaml-fix-merge-anchor-to-spec=false; it's not to YAML spec because the merge anchors incorrectly override the object values (foobarList.b is set to bar_b when it should still be foobarList_b). Flag will default to true in late 2025 Given a sample.yml file of: ```yaml @@ -334,7 +342,7 @@ foobar: ``` ## FIXED: Explode with merge anchors -Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 +See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 Given a sample.yml file of: ```yaml @@ -383,7 +391,7 @@ foobar: ``` ## FIXED: Explode with merge anchors -Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 +See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 Given a sample.yml file of: ```yaml diff --git a/pkg/yqlib/doc/operators/headers/anchor-and-alias-operators.md b/pkg/yqlib/doc/operators/headers/anchor-and-alias-operators.md index 5c6f29e5..1e95a223 100644 --- a/pkg/yqlib/doc/operators/headers/anchor-and-alias-operators.md +++ b/pkg/yqlib/doc/operators/headers/anchor-and-alias-operators.md @@ -4,3 +4,11 @@ Use the `alias` and `anchor` operators to read and write yaml aliases and anchor `yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag. + +## NOTE --yaml-fix-merge-anchor-to-spec flag +`yq` doesn't merge anchors `<<:` to spec, in some circumstances it incorrectly overrides existing keys when the spec documents not to do that. + +To minimise disruption while still fixing the issue, a flag has been added to toggle this behaviour. This will first default to false; and log warnings to users. Then it will default to true (and still allow users to specify false if needed) + +See examples of the flag difference below. + diff --git a/pkg/yqlib/doc/operators/headers/traverse-read.md b/pkg/yqlib/doc/operators/headers/traverse-read.md index 0ab419d3..574070a5 100644 --- a/pkg/yqlib/doc/operators/headers/traverse-read.md +++ b/pkg/yqlib/doc/operators/headers/traverse-read.md @@ -1,3 +1,12 @@ # Traverse (Read) This is the simplest (and perhaps most used) operator. It is used to navigate deeply into yaml structures. + + +## NOTE --yaml-fix-merge-anchor-to-spec flag +`yq` doesn't merge anchors `<<:` to spec, in some circumstances it incorrectly overrides existing keys when the spec documents not to do that. + +To minimise disruption while still fixing the issue, a flag has been added to toggle this behaviour. This will first default to false; and log warnings to users. Then it will default to true (and still allow users to specify false if needed) + +See examples of the flag differences below. + diff --git a/pkg/yqlib/doc/operators/traverse-read.md b/pkg/yqlib/doc/operators/traverse-read.md index 2f85a111..8adbe8ff 100644 --- a/pkg/yqlib/doc/operators/traverse-read.md +++ b/pkg/yqlib/doc/operators/traverse-read.md @@ -2,6 +2,15 @@ This is the simplest (and perhaps most used) operator. It is used to navigate deeply into yaml structures. + +## NOTE --yaml-fix-merge-anchor-to-spec flag +`yq` doesn't merge anchors `<<:` to spec, in some circumstances it incorrectly overrides existing keys when the spec documents not to do that. + +To minimise disruption while still fixing the issue, a flag has been added to toggle this behaviour. This will first default to false; and log warnings to users. Then it will default to true (and still allow users to specify false if needed) + +See examples of the flag differences below. + + ## Simple map navigation Given a sample.yml file of: ```yaml @@ -352,7 +361,7 @@ a c ``` -## Traversing merge anchors with override +## LEGACY: Traversing merge anchors with override This is legacy behaviour, see --yaml-fix-merge-anchor-to-spec Given a sample.yml file of: @@ -385,7 +394,7 @@ will output foo_c ``` -## Traversing merge anchor lists +## LEGACY: Traversing merge anchor lists Note that the later merge anchors override previous, but this is legacy behaviour, see --yaml-fix-merge-anchor-to-spec Given a sample.yml file of: @@ -418,7 +427,7 @@ will output bar_thing ``` -## Splatting merge anchors +## LEGACY: Splatting merge anchors With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec Given a sample.yml file of: @@ -453,7 +462,7 @@ foo_a foobar_thing ``` -## Splatting merge anchor lists +## LEGACY: Splatting merge anchor lists With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec Given a sample.yml file of: diff --git a/pkg/yqlib/operator_anchors_aliases.go b/pkg/yqlib/operator_anchors_aliases.go index cd1752e9..326cad71 100644 --- a/pkg/yqlib/operator_anchors_aliases.go +++ b/pkg/yqlib/operator_anchors_aliases.go @@ -162,7 +162,7 @@ func reconstructAliasedMap(node *CandidateNode, context Context) error { } } } else if showMergeAnchorToSpecWarning { - log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025.") + log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025. See https://mikefarah.gitbook.io/yq/operators/anchor-and-alias-operators for more details.") showMergeAnchorToSpecWarning = false } diff --git a/pkg/yqlib/operator_anchors_aliases_test.go b/pkg/yqlib/operator_anchors_aliases_test.go index de8f526f..a09f9137 100644 --- a/pkg/yqlib/operator_anchors_aliases_test.go +++ b/pkg/yqlib/operator_anchors_aliases_test.go @@ -104,14 +104,14 @@ var fixedAnchorOperatorScenarios = []expressionScenario{ { skipDoc: true, description: "merge anchor after existing keys", - subdescription: "Does not override existing keys", + subdescription: "Does not override existing keys - note the name field in the second element is still ellipse.", document: explodeWhenKeysExistDocument, expression: "explode(.)", expected: []string{explodeWhenKeysExistExpected}, }, { description: "FIXED: Explode with merge anchors", - subdescription: "Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 ", + subdescription: "See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 ", document: mergeDocSample, expression: `explode(.)`, expected: []string{explodeMergeAnchorsFixedExpected}, @@ -148,7 +148,7 @@ var badAnchorOperatorScenarios = []expressionScenario{ }, { description: "LEGACY: Explode with merge anchors", // incorrect overrides - subdescription: "Caution: this is for when --yaml-fix-merge-anchor-to-spec=false; it's not to YAML spec because the merge anchors incorrectly override the object values. Flag will default to true in late 2025", + subdescription: "Caution: this is for when --yaml-fix-merge-anchor-to-spec=false; it's not to YAML spec because the merge anchors incorrectly override the object values (foobarList.b is set to bar_b when it should still be foobarList_b). Flag will default to true in late 2025", document: mergeDocSample, expression: `explode(.)`, expected: []string{explodeMergeAnchorsExpected}, diff --git a/pkg/yqlib/operator_traverse_path.go b/pkg/yqlib/operator_traverse_path.go index 2042828b..3cd9fd51 100644 --- a/pkg/yqlib/operator_traverse_path.go +++ b/pkg/yqlib/operator_traverse_path.go @@ -295,7 +295,7 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, node *CandidateNode, wante if !ConfiguredYamlPreferences.FixMergeAnchorToSpec { log.Debug("Merge anchor") if showMergeAnchorToSpecWarning { - log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025.") + log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025. See https://mikefarah.gitbook.io/yq/operators/traverse-read for more details.") showMergeAnchorToSpecWarning = false } err := traverseMergeAnchor(newMatches, value, wantedKey, prefs, splat) diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go index b25da9c5..4ba1c4b6 100644 --- a/pkg/yqlib/operator_traverse_path_test.go +++ b/pkg/yqlib/operator_traverse_path_test.go @@ -27,9 +27,17 @@ foobar: var fixedTraversePathOperatorScenarios = []expressionScenario{ { - skipDoc: true, - description: "Traversing merge anchor lists", - subdescription: "Note that the keys earlier in the merge anchors sequence override later ones", + description: "FIXED: Traversing merge anchors with override", + subdescription: "Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour.", + document: mergeDocSample, + expression: `.foobar.c`, + expected: []string{ + "D0, P[foobar c], (!!str)::foobar_c\n", + }, + }, + { + description: "FIXED: Traversing merge anchor lists", + subdescription: "Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Note that the keys earlier in the merge anchors sequence override later ones", document: mergeDocSample, expression: `.foobarList.thing`, expected: []string{ @@ -37,19 +45,10 @@ var fixedTraversePathOperatorScenarios = []expressionScenario{ }, }, { - skipDoc: true, - description: "Traversing merge anchors with override", - document: mergeDocSample, - expression: `.foobar.c`, - expected: []string{ - "D0, P[foobar c], (!!str)::foobar_c\n", - }, - }, - { - skipDoc: true, - description: "Splatting merge anchors", - document: mergeDocSample, - expression: `.foobar[]`, + description: "FIXED: Splatting merge anchors", + subdescription: "Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Note that the keys earlier in the merge anchors sequence override later ones", + document: mergeDocSample, + expression: `.foobar[]`, expected: []string{ "D0, P[foo a], (!!str)::foo_a\n", "D0, P[foobar thing], (!!str)::foobar_thing\n", @@ -57,10 +56,10 @@ var fixedTraversePathOperatorScenarios = []expressionScenario{ }, }, { - skipDoc: true, - description: "Splatting merge anchor lists", - document: mergeDocSample, - expression: `.foobarList[]`, + description: "FIXED: Splatting merge anchor lists", + subdescription: "Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Note that the keys earlier in the merge anchors sequence override later ones", + document: mergeDocSample, + expression: `.foobarList[]`, expected: []string{ "D0, P[foobarList b], (!!str)::foobarList_b\n", "D0, P[foo thing], (!!str)::foo_thing\n", @@ -80,7 +79,7 @@ var fixedTraversePathOperatorScenarios = []expressionScenario{ var badTraversePathOperatorScenarios = []expressionScenario{ { - description: "Traversing merge anchors with override", + description: "LEGACY: Traversing merge anchors with override", subdescription: "This is legacy behaviour, see --yaml-fix-merge-anchor-to-spec", document: mergeDocSample, expression: `.foobar.c`, @@ -89,7 +88,7 @@ var badTraversePathOperatorScenarios = []expressionScenario{ }, }, { - description: "Traversing merge anchor lists", + description: "LEGACY: Traversing merge anchor lists", subdescription: "Note that the later merge anchors override previous, " + "but this is legacy behaviour, see --yaml-fix-merge-anchor-to-spec", document: mergeDocSample, @@ -99,7 +98,7 @@ var badTraversePathOperatorScenarios = []expressionScenario{ }, }, { - description: "Splatting merge anchors", + description: "LEGACY: Splatting merge anchors", subdescription: "With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec", document: mergeDocSample, expression: `.foobar[]`, @@ -110,7 +109,7 @@ var badTraversePathOperatorScenarios = []expressionScenario{ }, }, { - description: "Splatting merge anchor lists", + description: "LEGACY: Splatting merge anchor lists", subdescription: "With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec", document: mergeDocSample, expression: `.foobarList[]`, From 3018396ed2ee6b964b7c9b08c33a91137cd7f621 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Tue, 22 Jul 2025 10:50:52 +1000 Subject: [PATCH 24/27] wip: fixing key order --- .../operators/anchor-and-alias-operators.md | 502 +++++++++++++++++- pkg/yqlib/lib.go | 11 + pkg/yqlib/operator_anchors_aliases.go | 208 ++++---- pkg/yqlib/operator_anchors_aliases_test.go | 20 +- 4 files changed, 624 insertions(+), 117 deletions(-) diff --git a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md index d2d1ed37..b1953b88 100644 --- a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md +++ b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md @@ -100,8 +100,8 @@ yq '.[4] | explode(.)' sample.yml will output ```yaml r: 10 -y: 2 x: 1 +y: 2 ``` ## Get anchor @@ -333,8 +333,8 @@ bar: foobarList: b: bar_b thing: foo_thing - a: foo_a c: foobarList_c + a: foo_a foobar: c: foo_c a: foo_a @@ -381,12 +381,12 @@ bar: c: bar_c foobarList: b: foobarList_b - thing: foo_thing a: foo_a + thing: foo_thing c: foobarList_c foobar: - a: foo_a c: foobar_c + a: foo_a thing: foobar_thing ``` @@ -430,12 +430,502 @@ bar: c: bar_c foobarList: b: foobarList_b - thing: foo_thing a: foo_a + thing: foo_thing c: foobarList_c foobar: - a: foo_a c: foobar_c + a: foo_a + thing: foobar_thing +``` + +## FIXED: Explode with merge anchors +See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 + +Given a sample.yml file of: +```yaml +foo: &foo + a: foo_a + thing: foo_thing + c: foo_c +bar: &bar + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + !!merge <<: + - *foo + - *bar + c: foobarList_c +foobar: + c: foobar_c + !!merge <<: *foo + thing: foobar_thing +``` +then +```bash +yq 'explode(.)' sample.yml +``` +will output +```yaml +foo: + a: foo_a + thing: foo_thing + c: foo_c +bar: + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + a: foo_a + thing: foo_thing + c: foobarList_c +foobar: + c: foobar_c + a: foo_a + thing: foobar_thing +``` + +## FIXED: Explode with merge anchors +See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 + +Given a sample.yml file of: +```yaml +foo: &foo + a: foo_a + thing: foo_thing + c: foo_c +bar: &bar + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + !!merge <<: + - *foo + - *bar + c: foobarList_c +foobar: + c: foobar_c + !!merge <<: *foo + thing: foobar_thing +``` +then +```bash +yq 'explode(.)' sample.yml +``` +will output +```yaml +foo: + a: foo_a + thing: foo_thing + c: foo_c +bar: + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + a: foo_a + thing: foo_thing + c: foobarList_c +foobar: + c: foobar_c + a: foo_a + thing: foobar_thing +``` + +## FIXED: Explode with merge anchors +See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 + +Given a sample.yml file of: +```yaml +foo: &foo + a: foo_a + thing: foo_thing + c: foo_c +bar: &bar + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + !!merge <<: + - *foo + - *bar + c: foobarList_c +foobar: + c: foobar_c + !!merge <<: *foo + thing: foobar_thing +``` +then +```bash +yq 'explode(.)' sample.yml +``` +will output +```yaml +foo: + a: foo_a + thing: foo_thing + c: foo_c +bar: + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + a: foo_a + thing: foo_thing + c: foobarList_c +foobar: + c: foobar_c + a: foo_a + thing: foobar_thing +``` + +## FIXED: Explode with merge anchors +See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 + +Given a sample.yml file of: +```yaml +foo: &foo + a: foo_a + thing: foo_thing + c: foo_c +bar: &bar + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + !!merge <<: + - *foo + - *bar + c: foobarList_c +foobar: + c: foobar_c + !!merge <<: *foo + thing: foobar_thing +``` +then +```bash +yq 'explode(.)' sample.yml +``` +will output +```yaml +foo: + a: foo_a + thing: foo_thing + c: foo_c +bar: + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + a: foo_a + thing: foo_thing + c: foobarList_c +foobar: + c: foobar_c + a: foo_a + thing: foobar_thing +``` + +## FIXED: Explode with merge anchors +See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 + +Given a sample.yml file of: +```yaml +foo: &foo + a: foo_a + thing: foo_thing + c: foo_c +bar: &bar + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + !!merge <<: + - *foo + - *bar + c: foobarList_c +foobar: + c: foobar_c + !!merge <<: *foo + thing: foobar_thing +``` +then +```bash +yq 'explode(.)' sample.yml +``` +will output +```yaml +foo: + a: foo_a + thing: foo_thing + c: foo_c +bar: + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + a: foo_a + thing: foo_thing + c: foobarList_c +foobar: + c: foobar_c + a: foo_a + thing: foobar_thing +``` + +## FIXED: Explode with merge anchors +See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 + +Given a sample.yml file of: +```yaml +foo: &foo + a: foo_a + thing: foo_thing + c: foo_c +bar: &bar + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + !!merge <<: + - *foo + - *bar + c: foobarList_c +foobar: + c: foobar_c + !!merge <<: *foo + thing: foobar_thing +``` +then +```bash +yq 'explode(.)' sample.yml +``` +will output +```yaml +foo: + a: foo_a + thing: foo_thing + c: foo_c +bar: + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + a: foo_a + thing: foo_thing + c: foobarList_c +foobar: + c: foobar_c + a: foo_a + thing: foobar_thing +``` + +## FIXED: Explode with merge anchors +See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 + +Given a sample.yml file of: +```yaml +foo: &foo + a: foo_a + thing: foo_thing + c: foo_c +bar: &bar + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + !!merge <<: + - *foo + - *bar + c: foobarList_c +foobar: + c: foobar_c + !!merge <<: *foo + thing: foobar_thing +``` +then +```bash +yq 'explode(.)' sample.yml +``` +will output +```yaml +foo: + a: foo_a + thing: foo_thing + c: foo_c +bar: + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + a: foo_a + thing: foo_thing + c: foobarList_c +foobar: + c: foobar_c + a: foo_a + thing: foobar_thing +``` + +## FIXED: Explode with merge anchors +See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 + +Given a sample.yml file of: +```yaml +foo: &foo + a: foo_a + thing: foo_thing + c: foo_c +bar: &bar + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + !!merge <<: + - *foo + - *bar + c: foobarList_c +foobar: + c: foobar_c + !!merge <<: *foo + thing: foobar_thing +``` +then +```bash +yq 'explode(.)' sample.yml +``` +will output +```yaml +foo: + a: foo_a + thing: foo_thing + c: foo_c +bar: + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + a: foo_a + thing: foo_thing + c: foobarList_c +foobar: + c: foobar_c + a: foo_a + thing: foobar_thing +``` + +## FIXED: Explode with merge anchors +See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 + +Given a sample.yml file of: +```yaml +foo: &foo + a: foo_a + thing: foo_thing + c: foo_c +bar: &bar + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + !!merge <<: + - *foo + - *bar + c: foobarList_c +foobar: + c: foobar_c + !!merge <<: *foo + thing: foobar_thing +``` +then +```bash +yq 'explode(.)' sample.yml +``` +will output +```yaml +foo: + a: foo_a + thing: foo_thing + c: foo_c +bar: + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + a: foo_a + thing: foo_thing + c: foobarList_c +foobar: + c: foobar_c + a: foo_a + thing: foobar_thing +``` + +## FIXED: Explode with merge anchors +See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 + +Given a sample.yml file of: +```yaml +foo: &foo + a: foo_a + thing: foo_thing + c: foo_c +bar: &bar + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + !!merge <<: + - *foo + - *bar + c: foobarList_c +foobar: + c: foobar_c + !!merge <<: *foo + thing: foobar_thing +``` +then +```bash +yq 'explode(.)' sample.yml +``` +will output +```yaml +foo: + a: foo_a + thing: foo_thing + c: foo_c +bar: + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + a: foo_a + thing: foo_thing + c: foobarList_c +foobar: + c: foobar_c + a: foo_a thing: foobar_thing ``` diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 15561930..748b3f5b 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -28,6 +28,17 @@ func GetLogger() *logging.Logger { return log } +func getContentValueByKey(content []*CandidateNode, key string) *CandidateNode { + for index := 0; index < len(content); index = index + 2 { + keyNode := content[index] + valueNode := content[index+1] + if keyNode.Value == key { + return valueNode + } + } + return nil +} + func recurseNodeArrayEqual(lhs *CandidateNode, rhs *CandidateNode) bool { if len(lhs.Content) != len(rhs.Content) { return false diff --git a/pkg/yqlib/operator_anchors_aliases.go b/pkg/yqlib/operator_anchors_aliases.go index 326cad71..720c6384 100644 --- a/pkg/yqlib/operator_anchors_aliases.go +++ b/pkg/yqlib/operator_anchors_aliases.go @@ -3,7 +3,6 @@ package yqlib import ( "container/list" "fmt" - "slices" ) var showMergeAnchorToSpecWarning = true @@ -141,45 +140,92 @@ func explodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expr return context, nil } -func reconstructAliasedMap(node *CandidateNode, context Context) error { - var newContent = list.New() - // can I short cut here by prechecking if there's an anchor in the map? - // no it needs to recurse in overrideEntry. - - if ConfiguredYamlPreferences.FixMergeAnchorToSpec { - // First evaluate merge keys to make explicit keys take precedence, following spec - // We also iterate in reverse to make earlier merge keys take precedence, - // although normally there's just one '<<' - for index := len(node.Content) - 2; index >= 0; index -= 2 { - keyNode := node.Content[index] - valueNode := node.Content[index+1] - if keyNode.Tag == "!!merge" { - log.Debugf("traversing merge key %v", keyNode.Value) - err := applyMergeAnchor(node, valueNode, index, context.ChildContext(newContent)) - if err != nil { - return err - } - } - } - } else if showMergeAnchorToSpecWarning { - log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025. See https://mikefarah.gitbook.io/yq/operators/anchor-and-alias-operators for more details.") - showMergeAnchorToSpecWarning = false - } +func fixedReconstructAliasedMap(node *CandidateNode) error { + var newContent = []*CandidateNode{} for index := 0; index < len(node.Content); index = index + 2 { keyNode := node.Content[index] valueNode := node.Content[index+1] if keyNode.Tag != "!!merge" { - log.Debugf("traversing %v", keyNode.Value) - err := overrideEntry(node, keyNode, valueNode, index, true, context.ChildContext(newContent)) + // copy to ensure exploding doesn't modify the original node + keyNodeCopy := keyNode.Copy() + valueNodeCopy := valueNode.Copy() + // always add in plain nodes + // explode both the key and value nodes + if err := explodeNode(keyNodeCopy, Context{}); err != nil { + return err + } + if err := explodeNode(valueNodeCopy, Context{}); err != nil { + return err + } + newContent = append(newContent, keyNodeCopy, valueNodeCopy) + } else { + sequence := valueNode + if sequence.Kind == AliasNode { + sequence = sequence.Alias + } + if sequence.Kind != SequenceNode { + sequence = &CandidateNode{Content: []*CandidateNode{sequence}} + } + for index := 0; index < len(sequence.Content); index = index + 1 { + // for merge anchors, we only set them if the key is not already in node or the newContent + mergeNodeSeq := sequence.Content[index] + if mergeNodeSeq.Kind == AliasNode { + mergeNodeSeq = mergeNodeSeq.Alias + } + if mergeNodeSeq.Kind != MappingNode { + return fmt.Errorf("can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got sequence containing %v", mergeNodeSeq.Tag) + } + itemsToAdd := mergeNodeSeq.FilterMapContentByKey(func(keyNode *CandidateNode) bool { + return getContentValueByKey(node.Content, keyNode.Value) == nil && + getContentValueByKey(newContent, keyNode.Value) == nil + }) + + for _, item := range itemsToAdd { + // copy to ensure exploding doesn't modify the original node + itemCopy := item.Copy() + if err := explodeNode(itemCopy, Context{}); err != nil { + return err + } + newContent = append(newContent, itemCopy) + } + } + } + } + node.Content = newContent + return nil +} + +func reconstructAliasedMap(node *CandidateNode, context Context) error { + var newContent = list.New() + // can I short cut here by prechecking if there's an anchor in the map? + // no it needs to recurse in overrideEntry. + + for index := 0; index < len(node.Content); index = index + 2 { + keyNode := node.Content[index] + valueNode := node.Content[index+1] + log.Debugf("traversing %v", keyNode.Value) + if keyNode.Value != "<<" { + err := overrideEntry(node, keyNode, valueNode, index, context.ChildContext(newContent)) if err != nil { return err } - } else if !ConfiguredYamlPreferences.FixMergeAnchorToSpec { - log.Debugf("traversing %v", keyNode.Value) - err := applyMergeAnchor(node, valueNode, index, context.ChildContext(newContent)) - if err != nil { - return err + } else { + if valueNode.Kind == SequenceNode { + log.Debugf("an alias merge list!") + for index := len(valueNode.Content) - 1; index >= 0; index = index - 1 { + aliasNode := valueNode.Content[index] + err := applyAlias(node, aliasNode.Alias, index, context.ChildContext(newContent)) + if err != nil { + return err + } + } + } else { + log.Debugf("an alias merge!") + err := applyAlias(node, valueNode.Alias, index, context.ChildContext(newContent)) + if err != nil { + return err + } } } } @@ -220,13 +266,20 @@ func explodeNode(node *CandidateNode, context Context) error { hasAlias := false for index := 0; index < len(node.Content); index = index + 2 { keyNode := node.Content[index] - if keyNode.Tag == "!!merge" { + if keyNode.Value == "<<" { hasAlias = true break } } if hasAlias { + if ConfiguredYamlPreferences.FixMergeAnchorToSpec { + return fixedReconstructAliasedMap(node) + } + if showMergeAnchorToSpecWarning { + log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025.") + showMergeAnchorToSpecWarning = false + } // this is a slow op, which is why we want to check before running it. return reconstructAliasedMap(node, context) } @@ -249,65 +302,20 @@ func explodeNode(node *CandidateNode, context Context) error { } } -func applyMergeAnchor(node *CandidateNode, merge *CandidateNode, mergeIndex int, newContent Context) error { - inline := true - if merge.Kind == AliasNode { - inline = false - merge = merge.Alias - } - switch merge.Kind { - case MappingNode: - log.Debugf("a merge map!") - return applyMergeAnchorMap(node, merge, mergeIndex, inline, newContent) - case SequenceNode: - log.Debugf("a merge list!") - // Reverse to make earlier values take precedence, following spec - // Note: This was already the case before FixMergeAnchorToSpec - content := slices.Backward(merge.Content) - for _, childValue := range content { - childInline := inline - if childValue.Kind == AliasNode { - childInline = false - childValue = childValue.Alias - } - if childValue.Kind != MappingNode { - return fmt.Errorf( - "can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got sequence containing %v", - childValue.Tag) - } - err := applyMergeAnchorMap(node, childValue, mergeIndex, childInline, newContent) - if err != nil { - return err - } - } - return nil - default: - return fmt.Errorf("can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got %v", merge.Tag) - } -} - -func applyMergeAnchorMap(node *CandidateNode, mergeMap *CandidateNode, mergeIndex int, explode bool, newContent Context) error { - if mergeMap == nil { - log.Debug("merge map is nil") +func applyAlias(node *CandidateNode, alias *CandidateNode, aliasIndex int, newContent Context) error { + log.Debug("alias is nil ?") + if alias == nil { return nil } - log.Debug("merge map: %v", NodeToString(mergeMap)) - if mergeMap.Kind != MappingNode { - return fmt.Errorf("applyMergeAnchorMap expects !!map, got %v instead", mergeMap.Tag) + log.Debug("alias: %v", NodeToString(alias)) + if alias.Kind != MappingNode { + return fmt.Errorf("can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got sequence containing %v", alias.Tag) } - - if explode { - err := explodeNode(mergeMap, newContent) - if err != nil { - return err - } - } - - for index := 0; index < len(mergeMap.Content); index = index + 2 { - keyNode := mergeMap.Content[index] - log.Debugf("applying merge map key %v", keyNode.Value) - valueNode := mergeMap.Content[index+1] - err := overrideEntry(node, keyNode, valueNode, mergeIndex, explode, newContent) + for index := 0; index < len(alias.Content); index = index + 2 { + keyNode := alias.Content[index] + log.Debugf("applying alias key %v", keyNode.Value) + valueNode := alias.Content[index+1] + err := overrideEntry(node, keyNode, valueNode, aliasIndex, newContent) if err != nil { return err } @@ -315,12 +323,12 @@ func applyMergeAnchorMap(node *CandidateNode, mergeMap *CandidateNode, mergeInde return nil } -func overrideEntry(node *CandidateNode, key *CandidateNode, value *CandidateNode, startIndex int, explode bool, newContent Context) error { - if explode { - err := explodeNode(value, newContent) - if err != nil { - return err - } +func overrideEntry(node *CandidateNode, key *CandidateNode, value *CandidateNode, startIndex int, newContent Context) error { + + err := explodeNode(value, newContent) + + if err != nil { + return err } for newEl := newContent.MatchingNodes.Front(); newEl != nil; newEl = newEl.Next() { @@ -328,7 +336,7 @@ func overrideEntry(node *CandidateNode, key *CandidateNode, value *CandidateNode keyNode := newEl.Value.(*CandidateNode) log.Debugf("checking new content %v:%v", keyNode.Value, valueEl.Value.(*CandidateNode).Value) if keyNode.Value == key.Value && keyNode.Alias == nil && key.Alias == nil { - log.Debugf("overriding new content") + log.Debugf("overridign new content") valueEl.Value = value return nil } @@ -344,11 +352,9 @@ func overrideEntry(node *CandidateNode, key *CandidateNode, value *CandidateNode } } - if explode { - err := explodeNode(key, newContent) - if err != nil { - return err - } + err = explodeNode(key, newContent) + if err != nil { + return err } log.Debugf("adding %v:%v", key.Value, value.Value) newContent.MatchingNodes.PushBack(key) diff --git a/pkg/yqlib/operator_anchors_aliases_test.go b/pkg/yqlib/operator_anchors_aliases_test.go index a09f9137..51deea27 100644 --- a/pkg/yqlib/operator_anchors_aliases_test.go +++ b/pkg/yqlib/operator_anchors_aliases_test.go @@ -44,12 +44,12 @@ bar: c: bar_c foobarList: b: foobarList_b - thing: foo_thing a: foo_a + thing: foo_thing c: foobarList_c foobar: - a: foo_a c: foobar_c + a: foo_a thing: foobar_thing ` @@ -122,8 +122,8 @@ var fixedAnchorOperatorScenarios = []expressionScenario{ expression: `.foo* | explode(.) | (. style="flow")`, expected: []string{ "D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n", - "D0, P[foobarList], (!!map)::{b: foobarList_b, thing: foo_thing, a: foo_a, c: foobarList_c}\n", - "D0, P[foobar], (!!map)::{a: foo_a, c: foobar_c, thing: foobar_thing}\n", + "D0, P[foobarList], (!!map)::{b: foobarList_b, a: foo_a, thing: foo_thing, c: foobarList_c}\n", + "D0, P[foobar], (!!map)::{c: foobar_c, a: foo_a, thing: foobar_thing}\n", }, }, { @@ -132,8 +132,8 @@ var fixedAnchorOperatorScenarios = []expressionScenario{ expression: `.foo* | explode(explode(.)) | (. style="flow")`, expected: []string{ "D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n", - "D0, P[foobarList], (!!map)::{b: foobarList_b, thing: foo_thing, a: foo_a, c: foobarList_c}\n", - "D0, P[foobar], (!!map)::{a: foo_a, c: foobar_c, thing: foobar_thing}\n", + "D0, P[foobarList], (!!map)::{b: foobarList_b, a: foo_a, thing: foo_thing, c: foobarList_c}\n", + "D0, P[foobar], (!!map)::{c: foobar_c, a: foo_a, thing: foobar_thing}\n", }, }, } @@ -198,11 +198,11 @@ var anchorOperatorScenarios = []expressionScenario{ expected: []string{expectedSpecResult}, }, { - description: "Merge multiple maps", // functionally correct, but key order gets mangled + description: "Merge multiple maps", subdescription: "see https://yaml.org/type/merge.html", document: specDocument + "- << : [ *CENTER, *BIG ]\n", expression: ".[4] | explode(.)", - expected: []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"}, + expected: []string{"D0, P[4], (!!map)::x: 1\ny: 2\nr: 10\n"}, }, { description: "Override", @@ -420,7 +420,7 @@ var anchorOperatorScenarios = []expressionScenario{ document: `{b: &b {a: &a 42}, r: *a, c: {<<: *b}}`, expression: `explode(.c)`, expected: []string{ - "D0, P[], (!!map)::{b: &b {a: &a 42}, r: *a, c: {a: &a 42}}\n", + "D0, P[], (!!map)::{b: &b {a: &a 42}, r: *a, c: {a: 42}}\n", }, }, { @@ -430,7 +430,7 @@ var anchorOperatorScenarios = []expressionScenario{ document: `{b: &b {a: &a 42}, r: *a, c: {<<: [*b]}}`, expression: `explode(.c)`, expected: []string{ - "D0, P[], (!!map)::{b: &b {a: &a 42}, r: *a, c: {a: &a 42}}\n", + "D0, P[], (!!map)::{b: &b {a: &a 42}, r: *a, c: {a: 42}}\n", }, }, { From db2a4550e50d397616a5cef1a35d893b0c0ee418 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Tue, 22 Jul 2025 11:25:02 +1000 Subject: [PATCH 25/27] Fixed tests --- .../operators/anchor-and-alias-operators.md | 650 ++++-------------- .../headers/anchor-and-alias-operators.md | 8 +- pkg/yqlib/operator_anchors_aliases.go | 9 +- pkg/yqlib/operator_anchors_aliases_test.go | 183 ++--- 4 files changed, 250 insertions(+), 600 deletions(-) diff --git a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md index b1953b88..0a745236 100644 --- a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md +++ b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md @@ -42,68 +42,6 @@ y: 2 r: 10 ``` -## Merge multiple maps -see https://yaml.org/type/merge.html - -Given a sample.yml file of: -```yaml -- &CENTER - x: 1 - y: 2 -- &LEFT - x: 0 - y: 2 -- &BIG - r: 10 -- &SMALL - r: 1 -- !!merge <<: - - *CENTER - - *BIG -``` -then -```bash -yq '.[4] | explode(.)' sample.yml -``` -will output -```yaml -r: 10 -x: 1 -y: 2 -``` - -## Override -see https://yaml.org/type/merge.html - -Given a sample.yml file of: -```yaml -- &CENTER - x: 1 - y: 2 -- &LEFT - x: 0 - y: 2 -- &BIG - r: 10 -- &SMALL - r: 1 -- !!merge <<: - - *BIG - - *LEFT - - *SMALL - x: 1 -``` -then -```bash -yq '.[4] | explode(.)' sample.yml -``` -will output -```yaml -r: 10 -x: 1 -y: 2 -``` - ## Get anchor Given a sample.yml file of: ```yaml @@ -341,53 +279,66 @@ foobar: thing: foobar_thing ``` -## FIXED: Explode with merge anchors -See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 +## Merge multiple maps +see https://yaml.org/type/merge.html Given a sample.yml file of: ```yaml -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - !!merge <<: - - *foo - - *bar - c: foobarList_c -foobar: - c: foobar_c - !!merge <<: *foo - thing: foobar_thing +- &CENTER + x: 1 + y: 2 +- &LEFT + x: 0 + y: 2 +- &BIG + r: 10 +- &SMALL + r: 1 +- !!merge <<: + - *CENTER + - *BIG ``` then ```bash -yq 'explode(.)' sample.yml +yq '.[4] | explode(.)' sample.yml ``` will output ```yaml -foo: - a: foo_a - thing: foo_thing - c: foo_c -bar: - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - a: foo_a - thing: foo_thing - c: foobarList_c -foobar: - c: foobar_c - a: foo_a - thing: foobar_thing +r: 10 +x: 1 +y: 2 +``` + +## Override +see https://yaml.org/type/merge.html + +Given a sample.yml file of: +```yaml +- &CENTER + x: 1 + y: 2 +- &LEFT + x: 0 + y: 2 +- &BIG + r: 10 +- &SMALL + r: 1 +- !!merge <<: + - *BIG + - *LEFT + - *SMALL + x: 1 +``` +then +```bash +yq '.[4] | explode(.)' sample.yml +``` +will output +```yaml +r: 10 +x: 1 +y: 2 ``` ## FIXED: Explode with merge anchors @@ -439,53 +390,66 @@ foobar: thing: foobar_thing ``` -## FIXED: Explode with merge anchors -See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 +## Merge multiple maps +see https://yaml.org/type/merge.html Given a sample.yml file of: ```yaml -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - !!merge <<: - - *foo - - *bar - c: foobarList_c -foobar: - c: foobar_c - !!merge <<: *foo - thing: foobar_thing +- &CENTER + x: 1 + y: 2 +- &LEFT + x: 0 + y: 2 +- &BIG + r: 10 +- &SMALL + r: 1 +- !!merge <<: + - *CENTER + - *BIG ``` then ```bash -yq 'explode(.)' sample.yml +yq '.[4] | explode(.)' sample.yml ``` will output ```yaml -foo: - a: foo_a - thing: foo_thing - c: foo_c -bar: - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - a: foo_a - thing: foo_thing - c: foobarList_c -foobar: - c: foobar_c - a: foo_a - thing: foobar_thing +x: 1 +y: 2 +r: 10 +``` + +## Override +see https://yaml.org/type/merge.html + +Given a sample.yml file of: +```yaml +- &CENTER + x: 1 + y: 2 +- &LEFT + x: 0 + y: 2 +- &BIG + r: 10 +- &SMALL + r: 1 +- !!merge <<: + - *BIG + - *LEFT + - *SMALL + x: 1 +``` +then +```bash +yq '.[4] | explode(.)' sample.yml +``` +will output +```yaml +r: 10 +y: 2 +x: 1 ``` ## FIXED: Explode with merge anchors @@ -537,395 +501,65 @@ foobar: thing: foobar_thing ``` -## FIXED: Explode with merge anchors -See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 +## Merge multiple maps +see https://yaml.org/type/merge.html Given a sample.yml file of: ```yaml -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - !!merge <<: - - *foo - - *bar - c: foobarList_c -foobar: - c: foobar_c - !!merge <<: *foo - thing: foobar_thing +- &CENTER + x: 1 + y: 2 +- &LEFT + x: 0 + y: 2 +- &BIG + r: 10 +- &SMALL + r: 1 +- !!merge <<: + - *CENTER + - *BIG ``` then ```bash -yq 'explode(.)' sample.yml +yq '.[4] | explode(.)' sample.yml ``` will output ```yaml -foo: - a: foo_a - thing: foo_thing - c: foo_c -bar: - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - a: foo_a - thing: foo_thing - c: foobarList_c -foobar: - c: foobar_c - a: foo_a - thing: foobar_thing +x: 1 +y: 2 +r: 10 ``` -## FIXED: Explode with merge anchors -See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 +## Override +see https://yaml.org/type/merge.html Given a sample.yml file of: ```yaml -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - !!merge <<: - - *foo - - *bar - c: foobarList_c -foobar: - c: foobar_c - !!merge <<: *foo - thing: foobar_thing +- &CENTER + x: 1 + y: 2 +- &LEFT + x: 0 + y: 2 +- &BIG + r: 10 +- &SMALL + r: 1 +- !!merge <<: + - *BIG + - *LEFT + - *SMALL + x: 1 ``` then ```bash -yq 'explode(.)' sample.yml +yq '.[4] | explode(.)' sample.yml ``` will output ```yaml -foo: - a: foo_a - thing: foo_thing - c: foo_c -bar: - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - a: foo_a - thing: foo_thing - c: foobarList_c -foobar: - c: foobar_c - a: foo_a - thing: foobar_thing -``` - -## FIXED: Explode with merge anchors -See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 - -Given a sample.yml file of: -```yaml -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - !!merge <<: - - *foo - - *bar - c: foobarList_c -foobar: - c: foobar_c - !!merge <<: *foo - thing: foobar_thing -``` -then -```bash -yq 'explode(.)' sample.yml -``` -will output -```yaml -foo: - a: foo_a - thing: foo_thing - c: foo_c -bar: - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - a: foo_a - thing: foo_thing - c: foobarList_c -foobar: - c: foobar_c - a: foo_a - thing: foobar_thing -``` - -## FIXED: Explode with merge anchors -See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 - -Given a sample.yml file of: -```yaml -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - !!merge <<: - - *foo - - *bar - c: foobarList_c -foobar: - c: foobar_c - !!merge <<: *foo - thing: foobar_thing -``` -then -```bash -yq 'explode(.)' sample.yml -``` -will output -```yaml -foo: - a: foo_a - thing: foo_thing - c: foo_c -bar: - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - a: foo_a - thing: foo_thing - c: foobarList_c -foobar: - c: foobar_c - a: foo_a - thing: foobar_thing -``` - -## FIXED: Explode with merge anchors -See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 - -Given a sample.yml file of: -```yaml -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - !!merge <<: - - *foo - - *bar - c: foobarList_c -foobar: - c: foobar_c - !!merge <<: *foo - thing: foobar_thing -``` -then -```bash -yq 'explode(.)' sample.yml -``` -will output -```yaml -foo: - a: foo_a - thing: foo_thing - c: foo_c -bar: - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - a: foo_a - thing: foo_thing - c: foobarList_c -foobar: - c: foobar_c - a: foo_a - thing: foobar_thing -``` - -## FIXED: Explode with merge anchors -See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 - -Given a sample.yml file of: -```yaml -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - !!merge <<: - - *foo - - *bar - c: foobarList_c -foobar: - c: foobar_c - !!merge <<: *foo - thing: foobar_thing -``` -then -```bash -yq 'explode(.)' sample.yml -``` -will output -```yaml -foo: - a: foo_a - thing: foo_thing - c: foo_c -bar: - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - a: foo_a - thing: foo_thing - c: foobarList_c -foobar: - c: foobar_c - a: foo_a - thing: foobar_thing -``` - -## FIXED: Explode with merge anchors -See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 - -Given a sample.yml file of: -```yaml -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - !!merge <<: - - *foo - - *bar - c: foobarList_c -foobar: - c: foobar_c - !!merge <<: *foo - thing: foobar_thing -``` -then -```bash -yq 'explode(.)' sample.yml -``` -will output -```yaml -foo: - a: foo_a - thing: foo_thing - c: foo_c -bar: - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - a: foo_a - thing: foo_thing - c: foobarList_c -foobar: - c: foobar_c - a: foo_a - thing: foobar_thing -``` - -## FIXED: Explode with merge anchors -See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 - -Given a sample.yml file of: -```yaml -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - !!merge <<: - - *foo - - *bar - c: foobarList_c -foobar: - c: foobar_c - !!merge <<: *foo - thing: foobar_thing -``` -then -```bash -yq 'explode(.)' sample.yml -``` -will output -```yaml -foo: - a: foo_a - thing: foo_thing - c: foo_c -bar: - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - a: foo_a - thing: foo_thing - c: foobarList_c -foobar: - c: foobar_c - a: foo_a - thing: foobar_thing +r: 10 +y: 2 +x: 1 ``` diff --git a/pkg/yqlib/doc/operators/headers/anchor-and-alias-operators.md b/pkg/yqlib/doc/operators/headers/anchor-and-alias-operators.md index 1e95a223..658eb579 100644 --- a/pkg/yqlib/doc/operators/headers/anchor-and-alias-operators.md +++ b/pkg/yqlib/doc/operators/headers/anchor-and-alias-operators.md @@ -8,7 +8,11 @@ Use the `alias` and `anchor` operators to read and write yaml aliases and anchor ## NOTE --yaml-fix-merge-anchor-to-spec flag `yq` doesn't merge anchors `<<:` to spec, in some circumstances it incorrectly overrides existing keys when the spec documents not to do that. -To minimise disruption while still fixing the issue, a flag has been added to toggle this behaviour. This will first default to false; and log warnings to users. Then it will default to true (and still allow users to specify false if needed) +To minimise disruption while still fixing the issue, a flag has been added to toggle this behaviour. This will first default to false; and log warnings to users. Then it will default to true (and still allow users to specify false if needed). -See examples of the flag difference below. +This flag also enables advanced merging, like inline maps, as well as fixes to ensure when exploding a particular path, neighbours are not affect ed. + +Long story short, you should be setting this flag to true. + +See examples of the flag differences below. diff --git a/pkg/yqlib/operator_anchors_aliases.go b/pkg/yqlib/operator_anchors_aliases.go index 720c6384..8dd4f84f 100644 --- a/pkg/yqlib/operator_anchors_aliases.go +++ b/pkg/yqlib/operator_anchors_aliases.go @@ -147,18 +147,15 @@ func fixedReconstructAliasedMap(node *CandidateNode) error { keyNode := node.Content[index] valueNode := node.Content[index+1] if keyNode.Tag != "!!merge" { - // copy to ensure exploding doesn't modify the original node - keyNodeCopy := keyNode.Copy() - valueNodeCopy := valueNode.Copy() // always add in plain nodes // explode both the key and value nodes - if err := explodeNode(keyNodeCopy, Context{}); err != nil { + if err := explodeNode(keyNode, Context{}); err != nil { return err } - if err := explodeNode(valueNodeCopy, Context{}); err != nil { + if err := explodeNode(valueNode, Context{}); err != nil { return err } - newContent = append(newContent, keyNodeCopy, valueNodeCopy) + newContent = append(newContent, keyNode, valueNode) } else { sequence := valueNode if sequence.Kind == AliasNode { diff --git a/pkg/yqlib/operator_anchors_aliases_test.go b/pkg/yqlib/operator_anchors_aliases_test.go index 51deea27..59c8fb8a 100644 --- a/pkg/yqlib/operator_anchors_aliases_test.go +++ b/pkg/yqlib/operator_anchors_aliases_test.go @@ -64,8 +64,8 @@ bar: foobarList: b: bar_b thing: foo_thing - a: foo_a c: foobarList_c + a: foo_a foobar: c: foo_c a: foo_a @@ -136,6 +136,87 @@ var fixedAnchorOperatorScenarios = []expressionScenario{ "D0, P[foobar], (!!map)::{c: foobar_c, a: foo_a, thing: foobar_thing}\n", }, }, + { + description: "Merge multiple maps", + subdescription: "see https://yaml.org/type/merge.html", + document: specDocument + "- << : [ *CENTER, *BIG ]\n", + expression: ".[4] | explode(.)", + expected: []string{"D0, P[4], (!!map)::x: 1\ny: 2\nr: 10\n"}, + }, + { + description: "Override", + subdescription: "see https://yaml.org/type/merge.html", + document: specDocument + "- << : [ *BIG, *LEFT, *SMALL ]\n x: 1\n", + expression: ".[4] | explode(.)", + expected: []string{"D0, P[4], (!!map)::r: 10\ny: 2\nx: 1\n"}, + }, + { + skipDoc: true, + description: "Exploding inline merge anchor", + subdescription: "`<<` map must be exploded, otherwise `c: *b` will become invalid", + document: `{a: {b: &b 42}, <<: {c: *b}}`, + expression: `explode(.) | sort_keys(.)`, + expected: []string{ + "D0, P[], (!!map)::{a: {b: 42}, c: 42}\n", + }, + }, + { + skipDoc: true, + description: "Exploding inline merge anchor in sequence", + subdescription: "`<<` map must be exploded, otherwise `c: *b` will become invalid", + document: `{a: {b: &b 42}, <<: [{c: *b}]}`, + expression: `explode(.) | sort_keys(.)`, + expected: []string{ + "D0, P[], (!!map)::{a: {b: 42}, c: 42}\n", + }, + }, + { + skipDoc: true, + description: "Exploding merge anchor should not explode neighbors", + subdescription: "b must not be exploded, as `r: *a` will become invalid", + document: `{b: &b {a: &a 42}, r: *a, c: {<<: *b}}`, + expression: `explode(.c)`, + expected: []string{ + "D0, P[], (!!map)::{b: &b {a: &a 42}, r: *a, c: {a: 42}}\n", + }, + }, + { + skipDoc: true, + description: "Exploding sequence merge anchor should not explode neighbors", + subdescription: "b must not be exploded, as `r: *a` will become invalid", + document: `{b: &b {a: &a 42}, r: *a, c: {<<: [*b]}}`, + expression: `explode(.c)`, + expected: []string{ + "D0, P[], (!!map)::{b: &b {a: &a 42}, r: *a, c: {a: 42}}\n", + }, + }, + { + skipDoc: true, + description: "Merge anchor with inline map", + document: `{<<: {a: 42}}`, + expression: `explode(.)`, + expected: []string{ + "D0, P[], (!!map)::{a: 42}\n", + }, + }, + { + skipDoc: true, + description: "Merge anchor with sequence with inline map", + document: `{<<: [{a: 42}]}`, + expression: `explode(.)`, + expected: []string{ + "D0, P[], (!!map)::{a: 42}\n", + }, + }, + { + skipDoc: true, + description: "Merge anchor with aliased sequence with inline map", + document: `{s: &s [{a: 42}], m: {<<: *s}}`, + expression: `.m | explode(.)`, + expected: []string{ + "D0, P[m], (!!map)::{a: 42}\n", + }, + }, } var badAnchorOperatorScenarios = []expressionScenario{ @@ -159,7 +240,7 @@ var badAnchorOperatorScenarios = []expressionScenario{ expression: `.foo* | explode(.) | (. style="flow")`, expected: []string{ "D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n", - "D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, a: foo_a, c: foobarList_c}\n", + "D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\n", "D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\n", }, }, @@ -169,10 +250,24 @@ var badAnchorOperatorScenarios = []expressionScenario{ expression: `.foo* | explode(explode(.)) | (. style="flow")`, expected: []string{ "D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n", - "D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, a: foo_a, c: foobarList_c}\n", + "D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\n", "D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\n", }, }, + { + description: "Merge multiple maps", + subdescription: "see https://yaml.org/type/merge.html", + document: specDocument + "- << : [ *CENTER, *BIG ]\n", + expression: ".[4] | explode(.)", + expected: []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"}, // correct data, but wrong key order + }, + { + description: "Override", + subdescription: "see https://yaml.org/type/merge.html", + document: specDocument + "- << : [ *BIG, *LEFT, *SMALL ]\n x: 1\n", + expression: ".[4] | explode(.)", + expected: []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"}, + }, } var anchorOperatorScenarios = []expressionScenario{ @@ -197,20 +292,7 @@ var anchorOperatorScenarios = []expressionScenario{ expression: ".[4] | explode(.)", expected: []string{expectedSpecResult}, }, - { - description: "Merge multiple maps", - subdescription: "see https://yaml.org/type/merge.html", - document: specDocument + "- << : [ *CENTER, *BIG ]\n", - expression: ".[4] | explode(.)", - expected: []string{"D0, P[4], (!!map)::x: 1\ny: 2\nr: 10\n"}, - }, - { - description: "Override", - subdescription: "see https://yaml.org/type/merge.html", - document: specDocument + "- << : [ *BIG, *LEFT, *SMALL ]\n x: 1\n", - expression: ".[4] | explode(.)", - expected: []string{"D0, P[4], (!!map)::r: 10\ny: 2\nx: 1\n"}, - }, + { description: "Get anchor", document: `a: &billyBob cat`, @@ -386,73 +468,6 @@ var anchorOperatorScenarios = []expressionScenario{ expression: `.thingOne |= (explode(.) | sort_keys(.)) * {"value": false}`, expected: []string{expectedUpdatedArrayRef}, }, - { - skipDoc: true, - description: "Merge anchor with inline map", - document: `{<<: {a: 42}}`, - expression: `explode(.)`, - expected: []string{ - "D0, P[], (!!map)::{a: 42}\n", - }, - }, - { - skipDoc: true, - description: "Merge anchor with sequence with inline map", - document: `{<<: [{a: 42}]}`, - expression: `explode(.)`, - expected: []string{ - "D0, P[], (!!map)::{a: 42}\n", - }, - }, - { - skipDoc: true, - description: "Merge anchor with aliased sequence with inline map", - document: `{s: &s [{a: 42}], m: {<<: *s}}`, - expression: `.m | explode(.)`, - expected: []string{ - "D0, P[m], (!!map)::{a: 42}\n", - }, - }, - { - skipDoc: true, - description: "Exploding merge anchor should not explode neighbors", - subdescription: "b must not be exploded, as `r: *a` will become invalid", - document: `{b: &b {a: &a 42}, r: *a, c: {<<: *b}}`, - expression: `explode(.c)`, - expected: []string{ - "D0, P[], (!!map)::{b: &b {a: &a 42}, r: *a, c: {a: 42}}\n", - }, - }, - { - skipDoc: true, - description: "Exploding sequence merge anchor should not explode neighbors", - subdescription: "b must not be exploded, as `r: *a` will become invalid", - document: `{b: &b {a: &a 42}, r: *a, c: {<<: [*b]}}`, - expression: `explode(.c)`, - expected: []string{ - "D0, P[], (!!map)::{b: &b {a: &a 42}, r: *a, c: {a: 42}}\n", - }, - }, - { - skipDoc: true, - description: "Exploding inline merge anchor", - subdescription: "`<<` map must be exploded, otherwise `c: *b` will become invalid", - document: `{a: {b: &b 42}, <<: {c: *b}}`, - expression: `explode(.) | sort_keys(.)`, - expected: []string{ - "D0, P[], (!!map)::{a: {b: 42}, c: 42}\n", - }, - }, - { - skipDoc: true, - description: "Exploding inline merge anchor in sequence", - subdescription: "`<<` map must be exploded, otherwise `c: *b` will become invalid", - document: `{a: {b: &b 42}, <<: [{c: *b}]}`, - expression: `explode(.) | sort_keys(.)`, - expected: []string{ - "D0, P[], (!!map)::{a: {b: 42}, c: 42}\n", - }, - }, { skipDoc: true, description: "Duplicate keys", From d5757fc82bbc65955a6c54768e39e5889a85374f Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Tue, 22 Jul 2025 11:51:33 +1000 Subject: [PATCH 26/27] Working on clarifying docs --- .../operators/anchor-and-alias-operators.md | 140 ++++-------------- pkg/yqlib/operator_anchors_aliases_test.go | 39 ++--- pkg/yqlib/operators_test.go | 2 +- 3 files changed, 51 insertions(+), 130 deletions(-) diff --git a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md index 0a745236..8c9a7381 100644 --- a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md +++ b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md @@ -8,9 +8,13 @@ Use the `alias` and `anchor` operators to read and write yaml aliases and anchor ## NOTE --yaml-fix-merge-anchor-to-spec flag `yq` doesn't merge anchors `<<:` to spec, in some circumstances it incorrectly overrides existing keys when the spec documents not to do that. -To minimise disruption while still fixing the issue, a flag has been added to toggle this behaviour. This will first default to false; and log warnings to users. Then it will default to true (and still allow users to specify false if needed) +To minimise disruption while still fixing the issue, a flag has been added to toggle this behaviour. This will first default to false; and log warnings to users. Then it will default to true (and still allow users to specify false if needed). -See examples of the flag difference below. +This flag also enables advanced merging, like inline maps, as well as fixes to ensure when exploding a particular path, neighbours are not affect ed. + +Long story short, you should be setting this flag to true. + +See examples of the flag differences below. ## Merge one map @@ -279,8 +283,8 @@ foobar: thing: foobar_thing ``` -## Merge multiple maps -see https://yaml.org/type/merge.html +## LEGACY: Merge multiple maps +see https://yaml.org/type/merge.html. This has the correct data, but the wrong key order; set --yaml-fix-merge-anchor-to-spec=true to fix the key order. Given a sample.yml file of: ```yaml @@ -309,8 +313,8 @@ x: 1 y: 2 ``` -## Override -see https://yaml.org/type/merge.html +## LEGACY: Override +see https://yaml.org/type/merge.html. This has the correct data, but the wrong key order; set --yaml-fix-merge-anchor-to-spec=true to fix the key order. Given a sample.yml file of: ```yaml @@ -342,7 +346,8 @@ y: 2 ``` ## FIXED: Explode with merge anchors -See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 +Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour (flag will default to true in late 2025). +Observe that foobarList.b property is still foobarList_b. Given a sample.yml file of: ```yaml @@ -390,8 +395,9 @@ foobar: thing: foobar_thing ``` -## Merge multiple maps -see https://yaml.org/type/merge.html +## FIXED: Merge multiple maps +Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour (flag will default to true in late 2025). +Taken from https://yaml.org/type/merge.html. Same values as legacy, but with the correct key order. Given a sample.yml file of: ```yaml @@ -420,8 +426,9 @@ y: 2 r: 10 ``` -## Override -see https://yaml.org/type/merge.html +## FIXED: Override +Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour (flag will default to true in late 2025). +Taken from https://yaml.org/type/merge.html. Same values as legacy, but with the correct key order. Given a sample.yml file of: ```yaml @@ -452,114 +459,25 @@ y: 2 x: 1 ``` -## FIXED: Explode with merge anchors -See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 +## Exploding inline merge anchor +Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour (flag will default to true in late 2025). + Given a sample.yml file of: ```yaml -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - !!merge <<: - - *foo - - *bar - c: foobarList_c -foobar: - c: foobar_c - !!merge <<: *foo - thing: foobar_thing +a: + b: &b 42 +!!merge <<: + c: *b ``` then ```bash -yq 'explode(.)' sample.yml +yq 'explode(.) | sort_keys(.)' sample.yml ``` will output ```yaml -foo: - a: foo_a - thing: foo_thing - c: foo_c -bar: - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - a: foo_a - thing: foo_thing - c: foobarList_c -foobar: - c: foobar_c - a: foo_a - thing: foobar_thing -``` - -## Merge multiple maps -see https://yaml.org/type/merge.html - -Given a sample.yml file of: -```yaml -- &CENTER - x: 1 - y: 2 -- &LEFT - x: 0 - y: 2 -- &BIG - r: 10 -- &SMALL - r: 1 -- !!merge <<: - - *CENTER - - *BIG -``` -then -```bash -yq '.[4] | explode(.)' sample.yml -``` -will output -```yaml -x: 1 -y: 2 -r: 10 -``` - -## Override -see https://yaml.org/type/merge.html - -Given a sample.yml file of: -```yaml -- &CENTER - x: 1 - y: 2 -- &LEFT - x: 0 - y: 2 -- &BIG - r: 10 -- &SMALL - r: 1 -- !!merge <<: - - *BIG - - *LEFT - - *SMALL - x: 1 -``` -then -```bash -yq '.[4] | explode(.)' sample.yml -``` -will output -```yaml -r: 10 -y: 2 -x: 1 +a: + b: 42 +c: 42 ``` diff --git a/pkg/yqlib/operator_anchors_aliases_test.go b/pkg/yqlib/operator_anchors_aliases_test.go index 59c8fb8a..0fa62905 100644 --- a/pkg/yqlib/operator_anchors_aliases_test.go +++ b/pkg/yqlib/operator_anchors_aliases_test.go @@ -111,7 +111,7 @@ var fixedAnchorOperatorScenarios = []expressionScenario{ }, { description: "FIXED: Explode with merge anchors", - subdescription: "See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 ", + subdescription: "Observe that foobarList.b property is still foobarList_b.", document: mergeDocSample, expression: `explode(.)`, expected: []string{explodeMergeAnchorsFixedExpected}, @@ -137,25 +137,24 @@ var fixedAnchorOperatorScenarios = []expressionScenario{ }, }, { - description: "Merge multiple maps", - subdescription: "see https://yaml.org/type/merge.html", + description: "FIXED: Merge multiple maps", + subdescription: "Taken from https://yaml.org/type/merge.html. Same values as legacy, but with the correct key order.", document: specDocument + "- << : [ *CENTER, *BIG ]\n", expression: ".[4] | explode(.)", expected: []string{"D0, P[4], (!!map)::x: 1\ny: 2\nr: 10\n"}, }, { - description: "Override", - subdescription: "see https://yaml.org/type/merge.html", + description: "FIXED: Override", + subdescription: "Taken from https://yaml.org/type/merge.html. Same values as legacy, but with the correct key order.", document: specDocument + "- << : [ *BIG, *LEFT, *SMALL ]\n x: 1\n", expression: ".[4] | explode(.)", expected: []string{"D0, P[4], (!!map)::r: 10\ny: 2\nx: 1\n"}, }, { - skipDoc: true, - description: "Exploding inline merge anchor", - subdescription: "`<<` map must be exploded, otherwise `c: *b` will become invalid", - document: `{a: {b: &b 42}, <<: {c: *b}}`, - expression: `explode(.) | sort_keys(.)`, + description: "Exploding inline merge anchor", + // subdescription: "`<<` map must be exploded, otherwise `c: *b` will become invalid", + document: `{a: {b: &b 42}, <<: {c: *b}}`, + expression: `explode(.) | sort_keys(.)`, expected: []string{ "D0, P[], (!!map)::{a: {b: 42}, c: 42}\n", }, @@ -255,18 +254,19 @@ var badAnchorOperatorScenarios = []expressionScenario{ }, }, { - description: "Merge multiple maps", - subdescription: "see https://yaml.org/type/merge.html", + description: "LEGACY: Merge multiple maps", + subdescription: "see https://yaml.org/type/merge.html. This has the correct data, but the wrong key order; set --yaml-fix-merge-anchor-to-spec=true to fix the key order.", document: specDocument + "- << : [ *CENTER, *BIG ]\n", expression: ".[4] | explode(.)", - expected: []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"}, // correct data, but wrong key order + expected: []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"}, }, { - description: "Override", - subdescription: "see https://yaml.org/type/merge.html", - document: specDocument + "- << : [ *BIG, *LEFT, *SMALL ]\n x: 1\n", - expression: ".[4] | explode(.)", - expected: []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"}, + description: "LEGACY: Override", + subdescription: "see https://yaml.org/type/merge.html. This has the correct data, but the wrong key order; set --yaml-fix-merge-anchor-to-spec=true to fix the key order.", + + document: specDocument + "- << : [ *BIG, *LEFT, *SMALL ]\n x: 1\n", + expression: ".[4] | explode(.)", + expected: []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"}, }, } @@ -501,7 +501,10 @@ func TestAnchorAliasOperatorAlignedToSpecScenarios(t *testing.T) { ConfiguredYamlPreferences.FixMergeAnchorToSpec = true for _, tt := range append(fixedAnchorOperatorScenarios, anchorOperatorScenarios...) { testScenario(t, &tt) + } + for i, tt := range fixedAnchorOperatorScenarios { + fixedAnchorOperatorScenarios[i].subdescription = "Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour (flag will default to true in late 2025).\n" + tt.subdescription } appendOperatorDocumentScenario(t, "anchor-and-alias-operators", fixedAnchorOperatorScenarios) ConfiguredYamlPreferences.FixMergeAnchorToSpec = false diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index 4c9d3d01..23c83c4e 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -255,7 +255,7 @@ func documentScenarios(t *testing.T, folder string, title string, scenarios []in func appendOperatorDocumentScenario(t *testing.T, title string, scenarios []expressionScenario) { filename := fmt.Sprintf("doc/%v/%v.md", "operators", title) - f, err := os.OpenFile(filename, os.O_WRONLY|os.O_APPEND, fs.ModeAppend) + f, err := os.OpenFile(filename, os.O_CREATE|os.O_RDWR|os.O_APPEND, fs.ModeAppend) if err != nil { t.Error(err) return From 92309b17a470f23bbaca69b6fbdd70706ca37ac4 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Wed, 23 Jul 2025 13:31:25 +1000 Subject: [PATCH 27/27] Fixed test doc gen --- .../operators/anchor-and-alias-operators.md | 2 +- .../headers/anchor-and-alias-operators.md | 2 +- .../doc/operators/headers/traverse-read.md | 2 +- pkg/yqlib/doc/operators/traverse-read.md | 139 +++++++++++++++++- pkg/yqlib/operator_traverse_path_test.go | 2 +- 5 files changed, 142 insertions(+), 5 deletions(-) diff --git a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md index 8c9a7381..cfdc8579 100644 --- a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md +++ b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md @@ -14,7 +14,7 @@ This flag also enables advanced merging, like inline maps, as well as fixes to e Long story short, you should be setting this flag to true. -See examples of the flag differences below. +See examples of the flag differences below, where LEGACY is with the flag off; and FIXED is with the flag on. ## Merge one map diff --git a/pkg/yqlib/doc/operators/headers/anchor-and-alias-operators.md b/pkg/yqlib/doc/operators/headers/anchor-and-alias-operators.md index 658eb579..610c364c 100644 --- a/pkg/yqlib/doc/operators/headers/anchor-and-alias-operators.md +++ b/pkg/yqlib/doc/operators/headers/anchor-and-alias-operators.md @@ -14,5 +14,5 @@ This flag also enables advanced merging, like inline maps, as well as fixes to e Long story short, you should be setting this flag to true. -See examples of the flag differences below. +See examples of the flag differences below, where LEGACY is with the flag off; and FIXED is with the flag on. diff --git a/pkg/yqlib/doc/operators/headers/traverse-read.md b/pkg/yqlib/doc/operators/headers/traverse-read.md index 574070a5..4ba14561 100644 --- a/pkg/yqlib/doc/operators/headers/traverse-read.md +++ b/pkg/yqlib/doc/operators/headers/traverse-read.md @@ -8,5 +8,5 @@ This is the simplest (and perhaps most used) operator. It is used to navigate de To minimise disruption while still fixing the issue, a flag has been added to toggle this behaviour. This will first default to false; and log warnings to users. Then it will default to true (and still allow users to specify false if needed) -See examples of the flag differences below. +See examples of the flag differences below, where LEGACY is the flag off; and FIXED is with the flag on. diff --git a/pkg/yqlib/doc/operators/traverse-read.md b/pkg/yqlib/doc/operators/traverse-read.md index 8adbe8ff..d418bc00 100644 --- a/pkg/yqlib/doc/operators/traverse-read.md +++ b/pkg/yqlib/doc/operators/traverse-read.md @@ -8,7 +8,7 @@ This is the simplest (and perhaps most used) operator. It is used to navigate de To minimise disruption while still fixing the issue, a flag has been added to toggle this behaviour. This will first default to false; and log warnings to users. Then it will default to true (and still allow users to specify false if needed) -See examples of the flag differences below. +See examples of the flag differences below, where LEGACY is the flag off; and FIXED is with the flag on. ## Simple map navigation @@ -498,3 +498,140 @@ bar_thing foobarList_c ``` +## FIXED: Traversing merge anchors with override +Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. + +Given a sample.yml file of: +```yaml +foo: &foo + a: foo_a + thing: foo_thing + c: foo_c +bar: &bar + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + !!merge <<: + - *foo + - *bar + c: foobarList_c +foobar: + c: foobar_c + !!merge <<: *foo + thing: foobar_thing +``` +then +```bash +yq '.foobar.c' sample.yml +``` +will output +```yaml +foobar_c +``` + +## FIXED: Traversing merge anchor lists +Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Note that the keys earlier in the merge anchors sequence override later ones + +Given a sample.yml file of: +```yaml +foo: &foo + a: foo_a + thing: foo_thing + c: foo_c +bar: &bar + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + !!merge <<: + - *foo + - *bar + c: foobarList_c +foobar: + c: foobar_c + !!merge <<: *foo + thing: foobar_thing +``` +then +```bash +yq '.foobarList.thing' sample.yml +``` +will output +```yaml +foo_thing +``` + +## FIXED: Splatting merge anchors +Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Note that the keys earlier in the merge anchors sequence override later ones + +Given a sample.yml file of: +```yaml +foo: &foo + a: foo_a + thing: foo_thing + c: foo_c +bar: &bar + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + !!merge <<: + - *foo + - *bar + c: foobarList_c +foobar: + c: foobar_c + !!merge <<: *foo + thing: foobar_thing +``` +then +```bash +yq '.foobar[]' sample.yml +``` +will output +```yaml +foo_a +foobar_thing +foobar_c +``` + +## FIXED: Splatting merge anchor lists +Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Note that the keys earlier in the merge anchors sequence override later ones + +Given a sample.yml file of: +```yaml +foo: &foo + a: foo_a + thing: foo_thing + c: foo_c +bar: &bar + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + !!merge <<: + - *foo + - *bar + c: foobarList_c +foobar: + c: foobar_c + !!merge <<: *foo + thing: foobar_thing +``` +then +```bash +yq '.foobarList[]' sample.yml +``` +will output +```yaml +foobarList_b +foo_thing +foobarList_c +foo_a +``` + diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go index 4ba1c4b6..2527f0e2 100644 --- a/pkg/yqlib/operator_traverse_path_test.go +++ b/pkg/yqlib/operator_traverse_path_test.go @@ -679,6 +679,6 @@ func TestTraversePathOperatorAlignedToSpecScenarios(t *testing.T) { for _, tt := range append(fixedTraversePathOperatorScenarios, traversePathOperatorScenarios...) { testScenario(t, &tt) } - appendOperatorDocumentScenario(t, "anchor-and-alias-operators", fixedAnchorOperatorScenarios) + appendOperatorDocumentScenario(t, "traverse-read", fixedTraversePathOperatorScenarios) ConfiguredYamlPreferences.FixMergeAnchorToSpec = false }