diff --git a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md index 707179d2..a705f1fd 100644 --- a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md +++ b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md @@ -34,6 +34,36 @@ y: 2 r: 10 ``` +## Merge multiple maps +see https://yaml.org/type/merge.html + +Given a sample.yml file of: +```yaml +- &CENTER + x: 1 + y: 2 +- &LEFT + x: 0 + y: 2 +- &BIG + r: 10 +- &SMALL + r: 1 +- !!merge <<: + - *CENTER + - *BIG +``` +then +```bash +yq '.[4] | explode(.)' sample.yml +``` +will output +```yaml +r: 10 +x: 1 +y: 2 +``` + ## Get anchor Given a sample.yml file of: ```yaml @@ -208,7 +238,7 @@ thingTwo: ``` then ```bash -yq '.thingOne |= explode(.) * {"value": false}' sample.yml +yq '.thingOne |= (explode(.) | sort_keys(.)) * {"value": false}' sample.yml ``` will output ```yaml @@ -222,36 +252,6 @@ thingTwo: !!merge <<: *item_value ``` -## Merge multiple maps -see https://yaml.org/type/merge.html - -Given a sample.yml file of: -```yaml -- &CENTER - x: 1 - y: 2 -- &LEFT - x: 0 - y: 2 -- &BIG - r: 10 -- &SMALL - r: 1 -- !!merge <<: - - *CENTER - - *BIG -``` -then -```bash -yq '.[4] | explode(.)' sample.yml -``` -will output -```yaml -r: 10 -x: 1 -y: 2 -``` - ## LEGACY: Explode with merge anchors Caution: this is for when --yaml-fix-merge-anchor-to-spec=false; it's not to YAML spec because the merge anchors incorrectly override the object values. Flag will default to true in late 2025 @@ -341,12 +341,12 @@ bar: c: bar_c foobarList: b: foobarList_b - a: foo_a thing: foo_thing + a: foo_a c: foobarList_c foobar: - c: foobar_c a: foo_a + c: foobar_c thing: foobar_thing ``` diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 748b3f5b..15561930 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -28,17 +28,6 @@ func GetLogger() *logging.Logger { return log } -func getContentValueByKey(content []*CandidateNode, key string) *CandidateNode { - for index := 0; index < len(content); index = index + 2 { - keyNode := content[index] - valueNode := content[index+1] - if keyNode.Value == key { - return valueNode - } - } - return nil -} - func recurseNodeArrayEqual(lhs *CandidateNode, rhs *CandidateNode) bool { if len(lhs.Content) != len(rhs.Content) { return false diff --git a/pkg/yqlib/operator_anchors_aliases.go b/pkg/yqlib/operator_anchors_aliases.go index 2c8f6106..84c31709 100644 --- a/pkg/yqlib/operator_anchors_aliases.go +++ b/pkg/yqlib/operator_anchors_aliases.go @@ -139,58 +139,38 @@ func explodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expr return context, nil } -func fixedReconstructAliasedMap(node *CandidateNode) error { - var newContent = []*CandidateNode{} - - for index := 0; index < len(node.Content); index = index + 2 { - keyNode := node.Content[index] - valueNode := node.Content[index+1] - if keyNode.Value != "<<" { - // always add in plain nodes - newContent = append(newContent, keyNode, valueNode) - } else { - sequence := valueNode - if sequence.Kind != SequenceNode { - sequence = &CandidateNode{Content: []*CandidateNode{sequence}} - } - for index := 0; index < len(sequence.Content); index = index + 1 { - // for merge anchors, we only set them if the key is not already in node or the newContent - mergeNodeSeq := sequence.Content[index] - if mergeNodeSeq.Kind == AliasNode { - mergeNodeSeq = mergeNodeSeq.Alias - } - if mergeNodeSeq.Kind != MappingNode { - return fmt.Errorf("merge anchor only supports maps, got !!seq instead") - } - // Only retain keys from merge map that are not already in node.Content or newContent, - // to prevent overwriting - itemsToAdd := mergeNodeSeq.FilterMapContentByKey(func(keyNode *CandidateNode) bool { - return getContentValueByKey(node.Content, keyNode.Value) == nil && - getContentValueByKey(newContent, keyNode.Value) == nil - }) - newContent = append(newContent, itemsToAdd...) - } - } - } - node.Content = newContent - return nil -} - func reconstructAliasedMap(node *CandidateNode, context Context) error { var newContent = list.New() // can I short cut here by prechecking if there's an anchor in the map? // no it needs to recurse in overrideEntry. + if !ConfiguredYamlPreferences.FixMergeAnchorToSpec { + log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025.") + } + if ConfiguredYamlPreferences.FixMergeAnchorToSpec { + for index := len(node.Content) - 2; index >= 0; index -= 2 { + keyNode := node.Content[index] + valueNode := node.Content[index+1] + if keyNode.Tag == "!!merge" { + log.Debugf("traversing merge key %v", keyNode.Value) + err := applyMergeAnchor(node, valueNode, index, context.ChildContext(newContent)) + if err != nil { + return err + } + } + } + } for index := 0; index < len(node.Content); index = index + 2 { keyNode := node.Content[index] valueNode := node.Content[index+1] - log.Debugf("traversing %v", keyNode.Value) if keyNode.Tag != "!!merge" { + log.Debugf("traversing %v", keyNode.Value) err := overrideEntry(node, keyNode, valueNode, index, true, context.ChildContext(newContent)) if err != nil { return err } - } else { + } else if !ConfiguredYamlPreferences.FixMergeAnchorToSpec { + log.Debugf("traversing %v", keyNode.Value) err := applyMergeAnchor(node, valueNode, index, context.ChildContext(newContent)) if err != nil { return err @@ -241,10 +221,6 @@ func explodeNode(node *CandidateNode, context Context) error { } if hasAlias { - if ConfiguredYamlPreferences.FixMergeAnchorToSpec { - return fixedReconstructAliasedMap(node) - } - log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025.") // this is a slow op, which is why we want to check before running it. return reconstructAliasedMap(node, context) } @@ -279,12 +255,7 @@ func applyMergeAnchor(node *CandidateNode, merge *CandidateNode, mergeIndex int, return applyMergeAnchorMap(node, merge, mergeIndex, inline, newContent) case SequenceNode: log.Debugf("a merge list!") - // With FixMergeAnchorToSpec, we rely on overrideEntry to reject duplicates - content := slices.All(merge.Content) - if !ConfiguredYamlPreferences.FixMergeAnchorToSpec { - // Even without FixMergeAnchorToSpec, this already gave preference to earlier keys - content = slices.Backward(merge.Content) - } + content := slices.Backward(merge.Content) for _, childValue := range content { childInline := inline if childValue.Kind == AliasNode { diff --git a/pkg/yqlib/operator_anchors_aliases_test.go b/pkg/yqlib/operator_anchors_aliases_test.go index 6890e8d0..de8f526f 100644 --- a/pkg/yqlib/operator_anchors_aliases_test.go +++ b/pkg/yqlib/operator_anchors_aliases_test.go @@ -44,12 +44,12 @@ bar: c: bar_c foobarList: b: foobarList_b - a: foo_a thing: foo_thing + a: foo_a c: foobarList_c foobar: - c: foobar_c a: foo_a + c: foobar_c thing: foobar_thing ` @@ -109,14 +109,6 @@ var fixedAnchorOperatorScenarios = []expressionScenario{ expression: "explode(.)", expected: []string{explodeWhenKeysExistExpected}, }, - { - skipDoc: true, // skip doc for now, only difference is order of keys - description: "Merge multiple maps", - subdescription: "see https://yaml.org/type/merge.html", - document: specDocument + "- << : [ *CENTER, *BIG ]\n", - expression: ".[4] | explode(.)", - expected: []string{"D0, P[4], (!!map)::x: 1\ny: 2\nr: 10\n"}, - }, { description: "FIXED: Explode with merge anchors", subdescription: "Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025 ", @@ -130,8 +122,8 @@ var fixedAnchorOperatorScenarios = []expressionScenario{ expression: `.foo* | explode(.) | (. style="flow")`, expected: []string{ "D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n", - "D0, P[foobarList], (!!map)::{b: foobarList_b, a: foo_a, thing: foo_thing, c: foobarList_c}\n", - "D0, P[foobar], (!!map)::{c: foobar_c, a: foo_a, thing: foobar_thing}\n", + "D0, P[foobarList], (!!map)::{b: foobarList_b, thing: foo_thing, a: foo_a, c: foobarList_c}\n", + "D0, P[foobar], (!!map)::{a: foo_a, c: foobar_c, thing: foobar_thing}\n", }, }, { @@ -140,8 +132,8 @@ var fixedAnchorOperatorScenarios = []expressionScenario{ expression: `.foo* | explode(explode(.)) | (. style="flow")`, expected: []string{ "D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n", - "D0, P[foobarList], (!!map)::{b: foobarList_b, a: foo_a, thing: foo_thing, c: foobarList_c}\n", - "D0, P[foobar], (!!map)::{c: foobar_c, a: foo_a, thing: foobar_thing}\n", + "D0, P[foobarList], (!!map)::{b: foobarList_b, thing: foo_thing, a: foo_a, c: foobarList_c}\n", + "D0, P[foobar], (!!map)::{a: foo_a, c: foobar_c, thing: foobar_thing}\n", }, }, } @@ -154,13 +146,6 @@ var badAnchorOperatorScenarios = []expressionScenario{ expression: "explode(.)", expected: []string{explodeWhenKeysExistLegacy}, }, - { - description: "Merge multiple maps", // functionally correct, but key order gets mangled - subdescription: "see https://yaml.org/type/merge.html", - document: specDocument + "- << : [ *CENTER, *BIG ]\n", - expression: ".[4] | explode(.)", - expected: []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"}, - }, { description: "LEGACY: Explode with merge anchors", // incorrect overrides subdescription: "Caution: this is for when --yaml-fix-merge-anchor-to-spec=false; it's not to YAML spec because the merge anchors incorrectly override the object values. Flag will default to true in late 2025", @@ -213,7 +198,13 @@ var anchorOperatorScenarios = []expressionScenario{ expected: []string{expectedSpecResult}, }, { - skipDoc: true, // skip doc for now, only difference is order of keys + description: "Merge multiple maps", // functionally correct, but key order gets mangled + subdescription: "see https://yaml.org/type/merge.html", + document: specDocument + "- << : [ *CENTER, *BIG ]\n", + expression: ".[4] | explode(.)", + expected: []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"}, + }, + { description: "Override", subdescription: "see https://yaml.org/type/merge.html", document: specDocument + "- << : [ *BIG, *LEFT, *SMALL ]\n x: 1\n", @@ -392,7 +383,7 @@ var anchorOperatorScenarios = []expressionScenario{ description: "Dereference and update a field", subdescription: "Use explode with multiply to dereference an object", document: simpleArrayRef, - expression: `.thingOne |= explode(.) * {"value": false}`, + expression: `.thingOne |= (explode(.) | sort_keys(.)) * {"value": false}`, expected: []string{expectedUpdatedArrayRef}, }, { @@ -447,7 +438,7 @@ var anchorOperatorScenarios = []expressionScenario{ description: "Exploding inline merge anchor", subdescription: "`<<` map must be exploded, otherwise `c: *b` will become invalid", document: `{a: {b: &b 42}, <<: {c: *b}}`, - expression: `explode(.)`, + expression: `explode(.) | sort_keys(.)`, expected: []string{ "D0, P[], (!!map)::{a: {b: 42}, c: 42}\n", }, @@ -457,7 +448,7 @@ var anchorOperatorScenarios = []expressionScenario{ description: "Exploding inline merge anchor in sequence", subdescription: "`<<` map must be exploded, otherwise `c: *b` will become invalid", document: `{a: {b: &b 42}, <<: [{c: *b}]}`, - expression: `explode(.)`, + expression: `explode(.) | sort_keys(.)`, expected: []string{ "D0, P[], (!!map)::{a: {b: 42}, c: 42}\n", },