suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2026-03-10 15:54:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix key overriding in regular maps for traversing

Browse Source
This commit is contained in:
Steven WdV 2025-07-20 15:37:45 +02:00
parent 41cc4fb4ac
commit 904215ef4d
No known key found for this signature in database
5 changed files with 257 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
pkg/yqlib/doc/operators/anchor-and-alias-operators.md
View File

@ -382,3 +382,52 @@ foobar:
thing: foobar_thing
```
## FIXED: Explode with merge anchors
Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Flag will default to true in late 2025
Given a sample.yml file of:
```yaml
foo: &foo
a: foo_a
thing: foo_thing
c: foo_c
bar: &bar
b: bar_b
thing: bar_thing
c: bar_c
foobarList:
b: foobarList_b
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
!!merge <<: *foo
thing: foobar_thing
```
then
```bash
yq 'explode(.)' sample.yml
```
will output
```yaml
foo:
a: foo_a
thing: foo_thing
c: foo_c
bar:
b: bar_b
thing: bar_thing
c: bar_c
foobarList:
b: foobarList_b
thing: foo_thing
a: foo_a
c: foobarList_c
foobar:
a: foo_a
c: foobar_c
thing: foobar_thing
```

166
pkg/yqlib/doc/operators/traverse-read.md
View File

@ -303,6 +303,55 @@ will output
foo_a
```
## Traversing merge anchors with local override
Given a sample.yml file of:
```yaml
foo: &foo
a: foo_a
thing: foo_thing
c: foo_c
bar: &bar
b: bar_b
thing: bar_thing
c: bar_c
foobarList:
b: foobarList_b
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
!!merge <<: *foo
thing: foobar_thing
```
then
```bash
yq '.foobar.thing' sample.yml
```
will output
```yaml
foobar_thing
```
## Select multiple indices
Given a sample.yml file of:
```yaml
a:
- a
- b
- c
```
then
```bash
yq '.a[0, 2]' sample.yml
```
will output
```yaml
a
c
```
## Traversing merge anchors with override
This is legacy behaviour, see --yaml-fix-merge-anchor-to-spec
@ -336,70 +385,6 @@ will output
foo_c
```
## Traversing merge anchors with local override
Given a sample.yml file of:
```yaml
foo: &foo
a: foo_a
thing: foo_thing
c: foo_c
bar: &bar
b: bar_b
thing: bar_thing
c: bar_c
foobarList:
b: foobarList_b
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
!!merge <<: *foo
thing: foobar_thing
```
then
```bash
yq '.foobar.thing' sample.yml
```
will output
```yaml
foobar_thing
```
## Splatting merge anchors
Given a sample.yml file of:
```yaml
foo: &foo
a: foo_a
thing: foo_thing
c: foo_c
bar: &bar
b: bar_b
thing: bar_thing
c: bar_c
foobarList:
b: foobarList_b
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
!!merge <<: *foo
thing: foobar_thing
```
then
```bash
yq '.foobar[]' sample.yml
```
will output
```yaml
foo_c
foo_a
foobar_thing
```
## Traversing merge anchor lists
Note that the later merge anchors override previous, but this is legacy behaviour, see --yaml-fix-merge-anchor-to-spec
@ -433,6 +418,41 @@ will output
bar_thing
```
## Splatting merge anchors
With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec
Given a sample.yml file of:
```yaml
foo: &foo
a: foo_a
thing: foo_thing
c: foo_c
bar: &bar
b: bar_b
thing: bar_thing
c: bar_c
foobarList:
b: foobarList_b
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
!!merge <<: *foo
thing: foobar_thing
```
then
```bash
yq '.foobar[]' sample.yml
```
will output
```yaml
foo_c
foo_a
foobar_thing
```
## Splatting merge anchor lists
With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec
@ -469,21 +489,3 @@ bar_thing
foobarList_c
```
## Select multiple indices
Given a sample.yml file of:
```yaml
a:
- a
- b
- c
```
then
```bash
yq '.a[0, 2]' sample.yml
```
will output
```yaml
a
c
```

6
pkg/yqlib/operator_anchors_aliases.go
View File

@ -144,9 +144,6 @@ func reconstructAliasedMap(node *CandidateNode, context Context) error {
// can I short cut here by prechecking if there's an anchor in the map?
// no it needs to recurse in overrideEntry.
if !ConfiguredYamlPreferences.FixMergeAnchorToSpec {
log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025.")
}
if ConfiguredYamlPreferences.FixMergeAnchorToSpec {
for index := len(node.Content) - 2; index >= 0; index -= 2 {
keyNode := node.Content[index]
@ -159,7 +156,10 @@ func reconstructAliasedMap(node *CandidateNode, context Context) error {
}
}
}
} else {
log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025.")
}
for index := 0; index < len(node.Content); index = index + 2 {
keyNode := node.Content[index]
valueNode := node.Content[index+1]

68
pkg/yqlib/operator_traverse_path.go
View File

@ -3,6 +3,7 @@ package yqlib
import (
"container/list"
"fmt"
"slices"
"github.com/elliotchance/orderedmap"
)
@ -265,53 +266,52 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, node *CandidateNode, wante
// if we don't find a match directly on this node first.
var contents = node.Content
if !prefs.DontFollowAlias {
if ConfiguredYamlPreferences.FixMergeAnchorToSpec {
for index := len(node.Content) - 2; index >= 0; index -= 2 {
keyNode := node.Content[index]
valueNode := node.Content[index+1]
if keyNode.Tag == "!!merge" {
log.Debug("Merge anchor")
err := traverseMergeAnchor(newMatches, valueNode, wantedKey, prefs, splat)
if err != nil {
return err
}
}
}
} else {
log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025.")
}
}
for index := 0; index+1 < len(contents); index = index + 2 {
key := contents[index]
value := contents[index+1]
//skip the 'merge' tag, find a direct match first
if key.Tag == "!!merge" && !prefs.DontFollowAlias && wantedKey != "<<" {
log.Debug("Merge anchor")
err := traverseMergeAnchor(newMatches, value, wantedKey, prefs, splat)
if err != nil {
return err
if key.Tag == "!!merge" && !prefs.DontFollowAlias && wantedKey != key.Value {
if !ConfiguredYamlPreferences.FixMergeAnchorToSpec {
log.Debug("Merge anchor")
err := traverseMergeAnchor(newMatches, value, wantedKey, prefs, splat)
if err != nil {
return err
}
}
} else if splat || keyMatches(key, wantedKey) {
log.Debug("MATCHED")
if prefs.IncludeMapKeys {
log.Debug("including key")
keyName := key.GetKey()
if newMatches.Has(keyName) {
if ConfiguredYamlPreferences.FixMergeAnchorToSpec {
log.Debug("not overwriting existing key")
} else {
log.Warning(
"--yaml-fix-merge-anchor-to-spec is false; "+
"causing the merge anchor to override the existing key at %v which isn't to the yaml spec. "+
"This flag will default to true in late 2025.", key.GetNicePath())
log.Debug("overwriting existing key")
newMatches.Set(keyName, key)
}
} else {
newMatches.Set(keyName, key)
if !newMatches.Set(keyName, key) {
log.Debug("overwriting existing key")
}
}
if !prefs.DontIncludeMapValues {
log.Debug("including value")
valueName := value.GetKey()
if newMatches.Has(valueName) {
if ConfiguredYamlPreferences.FixMergeAnchorToSpec {
log.Debug("not overwriting existing value")
} else {
log.Warning(
"--yaml-fix-merge-anchor-to-spec is false; "+
"causing the merge anchor to override the existing value at %v which isn't to the yaml spec. "+
"This flag will default to true in late 2025.", key.GetNicePath())
log.Debug("overwriting existing value")
newMatches.Set(valueName, value)
}
} else {
newMatches.Set(valueName, value)
if !newMatches.Set(valueName, value) {
log.Debug("overwriting existing value")
}
}
}
@ -328,7 +328,11 @@ func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, merge *CandidateNode
case MappingNode:
return doTraverseMap(newMatches, merge, wantedKey, prefs, splat)
case SequenceNode:
for _, childValue := range merge.Content {
content := slices.All(merge.Content)
if ConfiguredYamlPreferences.FixMergeAnchorToSpec {
content = slices.Backward(merge.Content)
}
for _, childValue := range content {
if childValue.Kind == AliasNode {
childValue = childValue.Alias
}

143
pkg/yqlib/operator_traverse_path_test.go
View File

@ -27,6 +27,7 @@ foobar:
var fixedTraversePathOperatorScenarios = []expressionScenario{
{
skipDoc: true,
description: "Traversing merge anchor lists",
subdescription: "Note that the keys earlier in the merge anchors sequence override later ones",
document: mergeDocSample,
@ -36,6 +37,7 @@ var fixedTraversePathOperatorScenarios = []expressionScenario{
},
},
{
skipDoc: true,
description: "Traversing merge anchors with override",
document: mergeDocSample,
expression: `.foobar.c`,
@ -43,16 +45,89 @@ var fixedTraversePathOperatorScenarios = []expressionScenario{
"D0, P[foobar c], (!!str)::foobar_c\n",
},
},
{
skipDoc: true,
description: "Splatting merge anchors",
document: mergeDocSample,
expression: `.foobar[]`,
expected: []string{
"D0, P[foo a], (!!str)::foo_a\n",
"D0, P[foobar thing], (!!str)::foobar_thing\n",
"D0, P[foobar c], (!!str)::foobar_c\n",
},
},
{
skipDoc: true,
description: "Splatting merge anchor lists",
document: mergeDocSample,
expression: `.foobarList[]`,
expected: []string{
"D0, P[foobarList b], (!!str)::foobarList_b\n",
"D0, P[foo thing], (!!str)::foo_thing\n",
"D0, P[foobarList c], (!!str)::foobarList_c\n",
"D0, P[foo a], (!!str)::foo_a\n",
},
},
{
skipDoc: true,
document: mergeDocSample,
expression: `.foobarList.b`,
expected: []string{
"D0, P[foobarList b], (!!str)::foobarList_b\n",
},
},
}
// The following tests are the same as below, to verify they still works correctly with the flag:
var badTraversePathOperatorScenarios = []expressionScenario{
{
description: "Traversing merge anchors with override",
subdescription: "This is legacy behaviour, see --yaml-fix-merge-anchor-to-spec",
document: mergeDocSample,
expression: `.foobar.c`,
expected: []string{
"D0, P[foo c], (!!str)::foo_c\n",
},
},
{
description: "Traversing merge anchor lists",
subdescription: "Note that the later merge anchors override previous, " +
"but this is legacy behaviour, see --yaml-fix-merge-anchor-to-spec",
document: mergeDocSample,
expression: `.foobarList.thing`,
expected: []string{
"D0, P[bar thing], (!!str)::bar_thing\n",
},
},
{
description: "Splatting merge anchors",
subdescription: "With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec",
document: mergeDocSample,
expression: `.foobar[]`,
expected: []string{
"D0, P[foo c], (!!str)::foo_c\n",
"D0, P[foo a], (!!str)::foo_a\n",
"D0, P[foobar thing], (!!str)::foobar_thing\n",
},
},
{
description: "Splatting merge anchor lists",
subdescription: "With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec",
document: mergeDocSample,
expression: `.foobarList[]`,
expected: []string{
"D0, P[bar b], (!!str)::bar_b\n",
"D0, P[foo a], (!!str)::foo_a\n",
"D0, P[bar thing], (!!str)::bar_thing\n",
"D0, P[foobarList c], (!!str)::foobarList_c\n",
},
},
{
skipDoc: true,
description: "Duplicate keys",
subdescription: "outside merge anchor",
document: `{a: 1, a: 2}`,
expression: `.a`,
subdescription: "This is legacy behaviour, see --yaml-fix-merge-anchor-to-spec",
document: mergeDocSample,
expression: `.foobarList.b`,
expected: []string{
"D0, P[a], (!!int)::2\n",
"D0, P[bar b], (!!str)::bar_b\n",
},
},
}
@ -431,15 +506,6 @@ var traversePathOperatorScenarios = []expressionScenario{
"D0, P[foo a], (!!str)::foo_a\n",
},
},
{
description: "Traversing merge anchors with override",
subdescription: "This is legacy behaviour, see --yaml-fix-merge-anchor-to-spec",
document: mergeDocSample,
expression: `.foobar.c`,
expected: []string{
"D0, P[foo c], (!!str)::foo_c\n",
},
},
{
description: "Traversing merge anchors with local override",
document: mergeDocSample,
@ -448,16 +514,6 @@ var traversePathOperatorScenarios = []expressionScenario{
"D0, P[foobar thing], (!!str)::foobar_thing\n",
},
},
{
description: "Splatting merge anchors",
document: mergeDocSample,
expression: `.foobar[]`,
expected: []string{
"D0, P[foo c], (!!str)::foo_c\n",
"D0, P[foo a], (!!str)::foo_a\n",
"D0, P[foobar thing], (!!str)::foobar_thing\n",
},
},
{
skipDoc: true,
document: mergeDocSample,
@ -474,16 +530,6 @@ var traversePathOperatorScenarios = []expressionScenario{
"D0, P[foo a], (!!str)::foo_a\n",
},
},
{
description: "Traversing merge anchor lists",
subdescription: "Note that the later merge anchors override previous, " +
"but this is legacy behaviour, see --yaml-fix-merge-anchor-to-spec",
document: mergeDocSample,
expression: `.foobarList.thing`,
expected: []string{
"D0, P[bar thing], (!!str)::bar_thing\n",
},
},
{
skipDoc: true,
document: mergeDocSample,
@ -492,26 +538,6 @@ var traversePathOperatorScenarios = []expressionScenario{
"D0, P[foobarList c], (!!str)::foobarList_c\n",
},
},
{
skipDoc: true,
document: mergeDocSample,
expression: `.foobarList.b`,
expected: []string{
"D0, P[bar b], (!!str)::bar_b\n",
},
},
{
description: "Splatting merge anchor lists",
subdescription: "With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec",
document: mergeDocSample,
expression: `.foobarList[]`,
expected: []string{
"D0, P[bar b], (!!str)::bar_b\n",
"D0, P[foo a], (!!str)::foo_a\n",
"D0, P[bar thing], (!!str)::bar_thing\n",
"D0, P[foobarList c], (!!str)::foobarList_c\n",
},
},
{
skipDoc: true,
document: `[a,b,c]`,
@ -643,16 +669,17 @@ var traversePathOperatorScenarios = []expressionScenario{
}
func TestTraversePathOperatorScenarios(t *testing.T) {
for _, tt := range traversePathOperatorScenarios {
for _, tt := range append(traversePathOperatorScenarios, badTraversePathOperatorScenarios...) {
testScenario(t, &tt)
}
documentOperatorScenarios(t, "traverse-read", traversePathOperatorScenarios)
documentOperatorScenarios(t, "traverse-read", append(traversePathOperatorScenarios, badTraversePathOperatorScenarios...))
}
func TestTraversePathOperatorAlignedToSpecScenarios(t *testing.T) {
ConfiguredYamlPreferences.FixMergeAnchorToSpec = true
for _, tt := range fixedTraversePathOperatorScenarios {
for _, tt := range append(fixedTraversePathOperatorScenarios, traversePathOperatorScenarios...) {
testScenario(t, &tt)
}
appendOperatorDocumentScenario(t, "anchor-and-alias-operators", fixedAnchorOperatorScenarios)
ConfiguredYamlPreferences.FixMergeAnchorToSpec = false
}