diff --git a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md index eca706e5..d2d1ed37 100644 --- a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md +++ b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md @@ -5,6 +5,14 @@ Use the `alias` and `anchor` operators to read and write yaml aliases and anchor `yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag. +## NOTE --yaml-fix-merge-anchor-to-spec flag +`yq` doesn't merge anchors `<<:` to spec, in some circumstances it incorrectly overrides existing keys when the spec documents not to do that. + +To minimise disruption while still fixing the issue, a flag has been added to toggle this behaviour. This will first default to false; and log warnings to users. Then it will default to true (and still allow users to specify false if needed) + +See examples of the flag difference below. + + ## Merge one map see https://yaml.org/type/merge.html @@ -285,7 +293,7 @@ thingTwo: ``` ## LEGACY: Explode with merge anchors -Caution: this is for when --yaml-fix-merge-anchor-to-spec=false; it's not to YAML spec because the merge anchors incorrectly override the object values. Flag will default to true in late 2025 +Caution: this is for when --yaml-fix-merge-anchor-to-spec=false; it's not to YAML spec because the merge anchors incorrectly override the object values (foobarList.b is set to bar_b when it should still be foobarList_b). Flag will default to true in late 2025 Given a sample.yml file of: ```yaml @@ -334,7 +342,7 @@ foobar: ``` ## FIXED: Explode with merge anchors -Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 +See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 Given a sample.yml file of: ```yaml @@ -383,7 +391,7 @@ foobar: ``` ## FIXED: Explode with merge anchors -Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 +See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 Given a sample.yml file of: ```yaml diff --git a/pkg/yqlib/doc/operators/headers/anchor-and-alias-operators.md b/pkg/yqlib/doc/operators/headers/anchor-and-alias-operators.md index 5c6f29e5..1e95a223 100644 --- a/pkg/yqlib/doc/operators/headers/anchor-and-alias-operators.md +++ b/pkg/yqlib/doc/operators/headers/anchor-and-alias-operators.md @@ -4,3 +4,11 @@ Use the `alias` and `anchor` operators to read and write yaml aliases and anchor `yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag. + +## NOTE --yaml-fix-merge-anchor-to-spec flag +`yq` doesn't merge anchors `<<:` to spec, in some circumstances it incorrectly overrides existing keys when the spec documents not to do that. + +To minimise disruption while still fixing the issue, a flag has been added to toggle this behaviour. This will first default to false; and log warnings to users. Then it will default to true (and still allow users to specify false if needed) + +See examples of the flag difference below. + diff --git a/pkg/yqlib/doc/operators/headers/traverse-read.md b/pkg/yqlib/doc/operators/headers/traverse-read.md index 0ab419d3..574070a5 100644 --- a/pkg/yqlib/doc/operators/headers/traverse-read.md +++ b/pkg/yqlib/doc/operators/headers/traverse-read.md @@ -1,3 +1,12 @@ # Traverse (Read) This is the simplest (and perhaps most used) operator. It is used to navigate deeply into yaml structures. + + +## NOTE --yaml-fix-merge-anchor-to-spec flag +`yq` doesn't merge anchors `<<:` to spec, in some circumstances it incorrectly overrides existing keys when the spec documents not to do that. + +To minimise disruption while still fixing the issue, a flag has been added to toggle this behaviour. This will first default to false; and log warnings to users. Then it will default to true (and still allow users to specify false if needed) + +See examples of the flag differences below. + diff --git a/pkg/yqlib/doc/operators/traverse-read.md b/pkg/yqlib/doc/operators/traverse-read.md index 2f85a111..8adbe8ff 100644 --- a/pkg/yqlib/doc/operators/traverse-read.md +++ b/pkg/yqlib/doc/operators/traverse-read.md @@ -2,6 +2,15 @@ This is the simplest (and perhaps most used) operator. It is used to navigate deeply into yaml structures. + +## NOTE --yaml-fix-merge-anchor-to-spec flag +`yq` doesn't merge anchors `<<:` to spec, in some circumstances it incorrectly overrides existing keys when the spec documents not to do that. + +To minimise disruption while still fixing the issue, a flag has been added to toggle this behaviour. This will first default to false; and log warnings to users. Then it will default to true (and still allow users to specify false if needed) + +See examples of the flag differences below. + + ## Simple map navigation Given a sample.yml file of: ```yaml @@ -352,7 +361,7 @@ a c ``` -## Traversing merge anchors with override +## LEGACY: Traversing merge anchors with override This is legacy behaviour, see --yaml-fix-merge-anchor-to-spec Given a sample.yml file of: @@ -385,7 +394,7 @@ will output foo_c ``` -## Traversing merge anchor lists +## LEGACY: Traversing merge anchor lists Note that the later merge anchors override previous, but this is legacy behaviour, see --yaml-fix-merge-anchor-to-spec Given a sample.yml file of: @@ -418,7 +427,7 @@ will output bar_thing ``` -## Splatting merge anchors +## LEGACY: Splatting merge anchors With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec Given a sample.yml file of: @@ -453,7 +462,7 @@ foo_a foobar_thing ``` -## Splatting merge anchor lists +## LEGACY: Splatting merge anchor lists With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec Given a sample.yml file of: diff --git a/pkg/yqlib/operator_anchors_aliases.go b/pkg/yqlib/operator_anchors_aliases.go index cd1752e9..326cad71 100644 --- a/pkg/yqlib/operator_anchors_aliases.go +++ b/pkg/yqlib/operator_anchors_aliases.go @@ -162,7 +162,7 @@ func reconstructAliasedMap(node *CandidateNode, context Context) error { } } } else if showMergeAnchorToSpecWarning { - log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025.") + log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025. See https://mikefarah.gitbook.io/yq/operators/anchor-and-alias-operators for more details.") showMergeAnchorToSpecWarning = false } diff --git a/pkg/yqlib/operator_anchors_aliases_test.go b/pkg/yqlib/operator_anchors_aliases_test.go index de8f526f..a09f9137 100644 --- a/pkg/yqlib/operator_anchors_aliases_test.go +++ b/pkg/yqlib/operator_anchors_aliases_test.go @@ -104,14 +104,14 @@ var fixedAnchorOperatorScenarios = []expressionScenario{ { skipDoc: true, description: "merge anchor after existing keys", - subdescription: "Does not override existing keys", + subdescription: "Does not override existing keys - note the name field in the second element is still ellipse.", document: explodeWhenKeysExistDocument, expression: "explode(.)", expected: []string{explodeWhenKeysExistExpected}, }, { description: "FIXED: Explode with merge anchors", - subdescription: "Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 ", + subdescription: "See the foobarList.b property is still foobarList_b. Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 ", document: mergeDocSample, expression: `explode(.)`, expected: []string{explodeMergeAnchorsFixedExpected}, @@ -148,7 +148,7 @@ var badAnchorOperatorScenarios = []expressionScenario{ }, { description: "LEGACY: Explode with merge anchors", // incorrect overrides - subdescription: "Caution: this is for when --yaml-fix-merge-anchor-to-spec=false; it's not to YAML spec because the merge anchors incorrectly override the object values. Flag will default to true in late 2025", + subdescription: "Caution: this is for when --yaml-fix-merge-anchor-to-spec=false; it's not to YAML spec because the merge anchors incorrectly override the object values (foobarList.b is set to bar_b when it should still be foobarList_b). Flag will default to true in late 2025", document: mergeDocSample, expression: `explode(.)`, expected: []string{explodeMergeAnchorsExpected}, diff --git a/pkg/yqlib/operator_traverse_path.go b/pkg/yqlib/operator_traverse_path.go index 2042828b..3cd9fd51 100644 --- a/pkg/yqlib/operator_traverse_path.go +++ b/pkg/yqlib/operator_traverse_path.go @@ -295,7 +295,7 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, node *CandidateNode, wante if !ConfiguredYamlPreferences.FixMergeAnchorToSpec { log.Debug("Merge anchor") if showMergeAnchorToSpecWarning { - log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025.") + log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025. See https://mikefarah.gitbook.io/yq/operators/traverse-read for more details.") showMergeAnchorToSpecWarning = false } err := traverseMergeAnchor(newMatches, value, wantedKey, prefs, splat) diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go index b25da9c5..4ba1c4b6 100644 --- a/pkg/yqlib/operator_traverse_path_test.go +++ b/pkg/yqlib/operator_traverse_path_test.go @@ -27,9 +27,17 @@ foobar: var fixedTraversePathOperatorScenarios = []expressionScenario{ { - skipDoc: true, - description: "Traversing merge anchor lists", - subdescription: "Note that the keys earlier in the merge anchors sequence override later ones", + description: "FIXED: Traversing merge anchors with override", + subdescription: "Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour.", + document: mergeDocSample, + expression: `.foobar.c`, + expected: []string{ + "D0, P[foobar c], (!!str)::foobar_c\n", + }, + }, + { + description: "FIXED: Traversing merge anchor lists", + subdescription: "Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Note that the keys earlier in the merge anchors sequence override later ones", document: mergeDocSample, expression: `.foobarList.thing`, expected: []string{ @@ -37,19 +45,10 @@ var fixedTraversePathOperatorScenarios = []expressionScenario{ }, }, { - skipDoc: true, - description: "Traversing merge anchors with override", - document: mergeDocSample, - expression: `.foobar.c`, - expected: []string{ - "D0, P[foobar c], (!!str)::foobar_c\n", - }, - }, - { - skipDoc: true, - description: "Splatting merge anchors", - document: mergeDocSample, - expression: `.foobar[]`, + description: "FIXED: Splatting merge anchors", + subdescription: "Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Note that the keys earlier in the merge anchors sequence override later ones", + document: mergeDocSample, + expression: `.foobar[]`, expected: []string{ "D0, P[foo a], (!!str)::foo_a\n", "D0, P[foobar thing], (!!str)::foobar_thing\n", @@ -57,10 +56,10 @@ var fixedTraversePathOperatorScenarios = []expressionScenario{ }, }, { - skipDoc: true, - description: "Splatting merge anchor lists", - document: mergeDocSample, - expression: `.foobarList[]`, + description: "FIXED: Splatting merge anchor lists", + subdescription: "Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Note that the keys earlier in the merge anchors sequence override later ones", + document: mergeDocSample, + expression: `.foobarList[]`, expected: []string{ "D0, P[foobarList b], (!!str)::foobarList_b\n", "D0, P[foo thing], (!!str)::foo_thing\n", @@ -80,7 +79,7 @@ var fixedTraversePathOperatorScenarios = []expressionScenario{ var badTraversePathOperatorScenarios = []expressionScenario{ { - description: "Traversing merge anchors with override", + description: "LEGACY: Traversing merge anchors with override", subdescription: "This is legacy behaviour, see --yaml-fix-merge-anchor-to-spec", document: mergeDocSample, expression: `.foobar.c`, @@ -89,7 +88,7 @@ var badTraversePathOperatorScenarios = []expressionScenario{ }, }, { - description: "Traversing merge anchor lists", + description: "LEGACY: Traversing merge anchor lists", subdescription: "Note that the later merge anchors override previous, " + "but this is legacy behaviour, see --yaml-fix-merge-anchor-to-spec", document: mergeDocSample, @@ -99,7 +98,7 @@ var badTraversePathOperatorScenarios = []expressionScenario{ }, }, { - description: "Splatting merge anchors", + description: "LEGACY: Splatting merge anchors", subdescription: "With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec", document: mergeDocSample, expression: `.foobar[]`, @@ -110,7 +109,7 @@ var badTraversePathOperatorScenarios = []expressionScenario{ }, }, { - description: "Splatting merge anchor lists", + description: "LEGACY: Splatting merge anchor lists", subdescription: "With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec", document: mergeDocSample, expression: `.foobarList[]`,