diff --git a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md index ea4b9b2f..eca706e5 100644 --- a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md +++ b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md @@ -382,3 +382,52 @@ foobar: thing: foobar_thing ``` +## FIXED: Explode with merge anchors +Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 + +Given a sample.yml file of: +```yaml +foo: &foo + a: foo_a + thing: foo_thing + c: foo_c +bar: &bar + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + !!merge <<: + - *foo + - *bar + c: foobarList_c +foobar: + c: foobar_c + !!merge <<: *foo + thing: foobar_thing +``` +then +```bash +yq 'explode(.)' sample.yml +``` +will output +```yaml +foo: + a: foo_a + thing: foo_thing + c: foo_c +bar: + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + thing: foo_thing + a: foo_a + c: foobarList_c +foobar: + a: foo_a + c: foobar_c + thing: foobar_thing +``` + diff --git a/pkg/yqlib/doc/operators/traverse-read.md b/pkg/yqlib/doc/operators/traverse-read.md index 775eeae0..2f85a111 100644 --- a/pkg/yqlib/doc/operators/traverse-read.md +++ b/pkg/yqlib/doc/operators/traverse-read.md @@ -303,6 +303,55 @@ will output foo_a ``` +## Traversing merge anchors with local override +Given a sample.yml file of: +```yaml +foo: &foo + a: foo_a + thing: foo_thing + c: foo_c +bar: &bar + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + !!merge <<: + - *foo + - *bar + c: foobarList_c +foobar: + c: foobar_c + !!merge <<: *foo + thing: foobar_thing +``` +then +```bash +yq '.foobar.thing' sample.yml +``` +will output +```yaml +foobar_thing +``` + +## Select multiple indices +Given a sample.yml file of: +```yaml +a: + - a + - b + - c +``` +then +```bash +yq '.a[0, 2]' sample.yml +``` +will output +```yaml +a +c +``` + ## Traversing merge anchors with override This is legacy behaviour, see --yaml-fix-merge-anchor-to-spec @@ -336,70 +385,6 @@ will output foo_c ``` -## Traversing merge anchors with local override -Given a sample.yml file of: -```yaml -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - !!merge <<: - - *foo - - *bar - c: foobarList_c -foobar: - c: foobar_c - !!merge <<: *foo - thing: foobar_thing -``` -then -```bash -yq '.foobar.thing' sample.yml -``` -will output -```yaml -foobar_thing -``` - -## Splatting merge anchors -Given a sample.yml file of: -```yaml -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - !!merge <<: - - *foo - - *bar - c: foobarList_c -foobar: - c: foobar_c - !!merge <<: *foo - thing: foobar_thing -``` -then -```bash -yq '.foobar[]' sample.yml -``` -will output -```yaml -foo_c -foo_a -foobar_thing -``` - ## Traversing merge anchor lists Note that the later merge anchors override previous, but this is legacy behaviour, see --yaml-fix-merge-anchor-to-spec @@ -433,6 +418,41 @@ will output bar_thing ``` +## Splatting merge anchors +With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec + +Given a sample.yml file of: +```yaml +foo: &foo + a: foo_a + thing: foo_thing + c: foo_c +bar: &bar + b: bar_b + thing: bar_thing + c: bar_c +foobarList: + b: foobarList_b + !!merge <<: + - *foo + - *bar + c: foobarList_c +foobar: + c: foobar_c + !!merge <<: *foo + thing: foobar_thing +``` +then +```bash +yq '.foobar[]' sample.yml +``` +will output +```yaml +foo_c +foo_a +foobar_thing +``` + ## Splatting merge anchor lists With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec @@ -469,21 +489,3 @@ bar_thing foobarList_c ``` -## Select multiple indices -Given a sample.yml file of: -```yaml -a: - - a - - b - - c -``` -then -```bash -yq '.a[0, 2]' sample.yml -``` -will output -```yaml -a -c -``` - diff --git a/pkg/yqlib/operator_anchors_aliases.go b/pkg/yqlib/operator_anchors_aliases.go index 84c31709..ab784f8a 100644 --- a/pkg/yqlib/operator_anchors_aliases.go +++ b/pkg/yqlib/operator_anchors_aliases.go @@ -144,9 +144,6 @@ func reconstructAliasedMap(node *CandidateNode, context Context) error { // can I short cut here by prechecking if there's an anchor in the map? // no it needs to recurse in overrideEntry. - if !ConfiguredYamlPreferences.FixMergeAnchorToSpec { - log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025.") - } if ConfiguredYamlPreferences.FixMergeAnchorToSpec { for index := len(node.Content) - 2; index >= 0; index -= 2 { keyNode := node.Content[index] @@ -159,7 +156,10 @@ func reconstructAliasedMap(node *CandidateNode, context Context) error { } } } + } else { + log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025.") } + for index := 0; index < len(node.Content); index = index + 2 { keyNode := node.Content[index] valueNode := node.Content[index+1] diff --git a/pkg/yqlib/operator_traverse_path.go b/pkg/yqlib/operator_traverse_path.go index bad0b84f..11bfea1b 100644 --- a/pkg/yqlib/operator_traverse_path.go +++ b/pkg/yqlib/operator_traverse_path.go @@ -3,6 +3,7 @@ package yqlib import ( "container/list" "fmt" + "slices" "github.com/elliotchance/orderedmap" ) @@ -265,53 +266,52 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, node *CandidateNode, wante // if we don't find a match directly on this node first. var contents = node.Content + + if !prefs.DontFollowAlias { + if ConfiguredYamlPreferences.FixMergeAnchorToSpec { + for index := len(node.Content) - 2; index >= 0; index -= 2 { + keyNode := node.Content[index] + valueNode := node.Content[index+1] + if keyNode.Tag == "!!merge" { + log.Debug("Merge anchor") + err := traverseMergeAnchor(newMatches, valueNode, wantedKey, prefs, splat) + if err != nil { + return err + } + } + } + } else { + log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025.") + } + } + for index := 0; index+1 < len(contents); index = index + 2 { key := contents[index] value := contents[index+1] //skip the 'merge' tag, find a direct match first - if key.Tag == "!!merge" && !prefs.DontFollowAlias && wantedKey != "<<" { - log.Debug("Merge anchor") - err := traverseMergeAnchor(newMatches, value, wantedKey, prefs, splat) - if err != nil { - return err + if key.Tag == "!!merge" && !prefs.DontFollowAlias && wantedKey != key.Value { + if !ConfiguredYamlPreferences.FixMergeAnchorToSpec { + log.Debug("Merge anchor") + err := traverseMergeAnchor(newMatches, value, wantedKey, prefs, splat) + if err != nil { + return err + } } } else if splat || keyMatches(key, wantedKey) { log.Debug("MATCHED") if prefs.IncludeMapKeys { log.Debug("including key") keyName := key.GetKey() - if newMatches.Has(keyName) { - if ConfiguredYamlPreferences.FixMergeAnchorToSpec { - log.Debug("not overwriting existing key") - } else { - log.Warning( - "--yaml-fix-merge-anchor-to-spec is false; "+ - "causing the merge anchor to override the existing key at %v which isn't to the yaml spec. "+ - "This flag will default to true in late 2025.", key.GetNicePath()) - log.Debug("overwriting existing key") - newMatches.Set(keyName, key) - } - } else { - newMatches.Set(keyName, key) + if !newMatches.Set(keyName, key) { + log.Debug("overwriting existing key") } } if !prefs.DontIncludeMapValues { log.Debug("including value") valueName := value.GetKey() - if newMatches.Has(valueName) { - if ConfiguredYamlPreferences.FixMergeAnchorToSpec { - log.Debug("not overwriting existing value") - } else { - log.Warning( - "--yaml-fix-merge-anchor-to-spec is false; "+ - "causing the merge anchor to override the existing value at %v which isn't to the yaml spec. "+ - "This flag will default to true in late 2025.", key.GetNicePath()) - log.Debug("overwriting existing value") - newMatches.Set(valueName, value) - } - } else { - newMatches.Set(valueName, value) + if !newMatches.Set(valueName, value) { + log.Debug("overwriting existing value") } } } @@ -328,7 +328,11 @@ func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, merge *CandidateNode case MappingNode: return doTraverseMap(newMatches, merge, wantedKey, prefs, splat) case SequenceNode: - for _, childValue := range merge.Content { + content := slices.All(merge.Content) + if ConfiguredYamlPreferences.FixMergeAnchorToSpec { + content = slices.Backward(merge.Content) + } + for _, childValue := range content { if childValue.Kind == AliasNode { childValue = childValue.Alias } diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go index 7574b4e0..b25da9c5 100644 --- a/pkg/yqlib/operator_traverse_path_test.go +++ b/pkg/yqlib/operator_traverse_path_test.go @@ -27,6 +27,7 @@ foobar: var fixedTraversePathOperatorScenarios = []expressionScenario{ { + skipDoc: true, description: "Traversing merge anchor lists", subdescription: "Note that the keys earlier in the merge anchors sequence override later ones", document: mergeDocSample, @@ -36,6 +37,7 @@ var fixedTraversePathOperatorScenarios = []expressionScenario{ }, }, { + skipDoc: true, description: "Traversing merge anchors with override", document: mergeDocSample, expression: `.foobar.c`, @@ -43,16 +45,89 @@ var fixedTraversePathOperatorScenarios = []expressionScenario{ "D0, P[foobar c], (!!str)::foobar_c\n", }, }, + { + skipDoc: true, + description: "Splatting merge anchors", + document: mergeDocSample, + expression: `.foobar[]`, + expected: []string{ + "D0, P[foo a], (!!str)::foo_a\n", + "D0, P[foobar thing], (!!str)::foobar_thing\n", + "D0, P[foobar c], (!!str)::foobar_c\n", + }, + }, + { + skipDoc: true, + description: "Splatting merge anchor lists", + document: mergeDocSample, + expression: `.foobarList[]`, + expected: []string{ + "D0, P[foobarList b], (!!str)::foobarList_b\n", + "D0, P[foo thing], (!!str)::foo_thing\n", + "D0, P[foobarList c], (!!str)::foobarList_c\n", + "D0, P[foo a], (!!str)::foo_a\n", + }, + }, + { + skipDoc: true, + document: mergeDocSample, + expression: `.foobarList.b`, + expected: []string{ + "D0, P[foobarList b], (!!str)::foobarList_b\n", + }, + }, +} - // The following tests are the same as below, to verify they still works correctly with the flag: +var badTraversePathOperatorScenarios = []expressionScenario{ + { + description: "Traversing merge anchors with override", + subdescription: "This is legacy behaviour, see --yaml-fix-merge-anchor-to-spec", + document: mergeDocSample, + expression: `.foobar.c`, + expected: []string{ + "D0, P[foo c], (!!str)::foo_c\n", + }, + }, + { + description: "Traversing merge anchor lists", + subdescription: "Note that the later merge anchors override previous, " + + "but this is legacy behaviour, see --yaml-fix-merge-anchor-to-spec", + document: mergeDocSample, + expression: `.foobarList.thing`, + expected: []string{ + "D0, P[bar thing], (!!str)::bar_thing\n", + }, + }, + { + description: "Splatting merge anchors", + subdescription: "With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec", + document: mergeDocSample, + expression: `.foobar[]`, + expected: []string{ + "D0, P[foo c], (!!str)::foo_c\n", + "D0, P[foo a], (!!str)::foo_a\n", + "D0, P[foobar thing], (!!str)::foobar_thing\n", + }, + }, + { + description: "Splatting merge anchor lists", + subdescription: "With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec", + document: mergeDocSample, + expression: `.foobarList[]`, + expected: []string{ + "D0, P[bar b], (!!str)::bar_b\n", + "D0, P[foo a], (!!str)::foo_a\n", + "D0, P[bar thing], (!!str)::bar_thing\n", + "D0, P[foobarList c], (!!str)::foobarList_c\n", + }, + }, { skipDoc: true, - description: "Duplicate keys", - subdescription: "outside merge anchor", - document: `{a: 1, a: 2}`, - expression: `.a`, + subdescription: "This is legacy behaviour, see --yaml-fix-merge-anchor-to-spec", + document: mergeDocSample, + expression: `.foobarList.b`, expected: []string{ - "D0, P[a], (!!int)::2\n", + "D0, P[bar b], (!!str)::bar_b\n", }, }, } @@ -431,15 +506,6 @@ var traversePathOperatorScenarios = []expressionScenario{ "D0, P[foo a], (!!str)::foo_a\n", }, }, - { - description: "Traversing merge anchors with override", - subdescription: "This is legacy behaviour, see --yaml-fix-merge-anchor-to-spec", - document: mergeDocSample, - expression: `.foobar.c`, - expected: []string{ - "D0, P[foo c], (!!str)::foo_c\n", - }, - }, { description: "Traversing merge anchors with local override", document: mergeDocSample, @@ -448,16 +514,6 @@ var traversePathOperatorScenarios = []expressionScenario{ "D0, P[foobar thing], (!!str)::foobar_thing\n", }, }, - { - description: "Splatting merge anchors", - document: mergeDocSample, - expression: `.foobar[]`, - expected: []string{ - "D0, P[foo c], (!!str)::foo_c\n", - "D0, P[foo a], (!!str)::foo_a\n", - "D0, P[foobar thing], (!!str)::foobar_thing\n", - }, - }, { skipDoc: true, document: mergeDocSample, @@ -474,16 +530,6 @@ var traversePathOperatorScenarios = []expressionScenario{ "D0, P[foo a], (!!str)::foo_a\n", }, }, - { - description: "Traversing merge anchor lists", - subdescription: "Note that the later merge anchors override previous, " + - "but this is legacy behaviour, see --yaml-fix-merge-anchor-to-spec", - document: mergeDocSample, - expression: `.foobarList.thing`, - expected: []string{ - "D0, P[bar thing], (!!str)::bar_thing\n", - }, - }, { skipDoc: true, document: mergeDocSample, @@ -492,26 +538,6 @@ var traversePathOperatorScenarios = []expressionScenario{ "D0, P[foobarList c], (!!str)::foobarList_c\n", }, }, - { - skipDoc: true, - document: mergeDocSample, - expression: `.foobarList.b`, - expected: []string{ - "D0, P[bar b], (!!str)::bar_b\n", - }, - }, - { - description: "Splatting merge anchor lists", - subdescription: "With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec", - document: mergeDocSample, - expression: `.foobarList[]`, - expected: []string{ - "D0, P[bar b], (!!str)::bar_b\n", - "D0, P[foo a], (!!str)::foo_a\n", - "D0, P[bar thing], (!!str)::bar_thing\n", - "D0, P[foobarList c], (!!str)::foobarList_c\n", - }, - }, { skipDoc: true, document: `[a,b,c]`, @@ -643,16 +669,17 @@ var traversePathOperatorScenarios = []expressionScenario{ } func TestTraversePathOperatorScenarios(t *testing.T) { - for _, tt := range traversePathOperatorScenarios { + for _, tt := range append(traversePathOperatorScenarios, badTraversePathOperatorScenarios...) { testScenario(t, &tt) } - documentOperatorScenarios(t, "traverse-read", traversePathOperatorScenarios) + documentOperatorScenarios(t, "traverse-read", append(traversePathOperatorScenarios, badTraversePathOperatorScenarios...)) } func TestTraversePathOperatorAlignedToSpecScenarios(t *testing.T) { ConfiguredYamlPreferences.FixMergeAnchorToSpec = true - for _, tt := range fixedTraversePathOperatorScenarios { + for _, tt := range append(fixedTraversePathOperatorScenarios, traversePathOperatorScenarios...) { testScenario(t, &tt) } + appendOperatorDocumentScenario(t, "anchor-and-alias-operators", fixedAnchorOperatorScenarios) ConfiguredYamlPreferences.FixMergeAnchorToSpec = false }