yaml aliases!

This commit is contained in:
Mike Farah 2023-04-15 14:24:39 +10:00
parent 4eca302efc
commit 4ef84278a3
6 changed files with 232 additions and 538 deletions

View File

@ -42,15 +42,26 @@ func MapToYamlStyle(original Style) yaml.Style {
return 0
}
func (o *CandidateNode) copyFromYamlNode(node *yaml.Node) {
func (o *CandidateNode) copyFromYamlNode(node *yaml.Node, anchorMap map[string]*CandidateNode) {
o.Style = MapYamlStyle(node.Style)
o.Tag = node.Tag
o.Value = node.Value
o.Anchor = node.Anchor
if o.Anchor != "" {
anchorMap[o.Anchor] = o
log.Debug("set anchor %v to %v", o.Anchor, NodeToString(o))
}
// o.Alias = TODO - find Alias in our own structure
// might need to be a post process thing
if node.Alias != nil && node.Alias.Value != "" {
o.Alias = anchorMap[node.Alias.Anchor]
log.Debug("set alias to %v", NodeToString(anchorMap[node.Alias.Anchor]))
}
o.HeadComment = node.HeadComment
o.LineComment = node.LineComment
o.FootComment = node.FootComment
@ -78,33 +89,34 @@ func (o *CandidateNode) copyToYamlNode(node *yaml.Node) {
node.Column = o.Column
}
func (o *CandidateNode) decodeIntoChild(childNode *yaml.Node) (*CandidateNode, error) {
func (o *CandidateNode) decodeIntoChild(childNode *yaml.Node, anchorMap map[string]*CandidateNode) (*CandidateNode, error) {
newChild := o.CreateChild()
// null yaml.Nodes to not end up calling UnmarshalYAML
// so we call it explicitly
if childNode.Tag == "!!null" {
newChild.Kind = ScalarNode
newChild.copyFromYamlNode(childNode)
newChild.copyFromYamlNode(childNode, anchorMap)
return newChild, nil
}
err := newChild.UnmarshalYAML(childNode)
err := newChild.UnmarshalYAML(childNode, anchorMap)
return newChild, err
}
func (o *CandidateNode) UnmarshalYAML(node *yaml.Node) error {
func (o *CandidateNode) UnmarshalYAML(node *yaml.Node, anchorMap map[string]*CandidateNode) error {
log.Debugf("UnmarshalYAML %v", node.Tag)
switch node.Kind {
case yaml.DocumentNode:
log.Debugf("UnmarshalYAML - a document")
o.Kind = DocumentNode
o.copyFromYamlNode(node)
o.copyFromYamlNode(node, anchorMap)
if len(node.Content) == 0 {
return nil
}
singleChild, err := o.decodeIntoChild(node.Content[0])
singleChild, err := o.decodeIntoChild(node.Content[0], anchorMap)
if err != nil {
return err
@ -115,28 +127,28 @@ func (o *CandidateNode) UnmarshalYAML(node *yaml.Node) error {
case yaml.AliasNode:
log.Debug("UnmarshalYAML - alias from yaml: %v", o.Tag)
o.Kind = AliasNode
o.copyFromYamlNode(node)
o.copyFromYamlNode(node, anchorMap)
return nil
case yaml.ScalarNode:
log.Debugf("UnmarshalYAML - a scalar")
o.Kind = ScalarNode
o.copyFromYamlNode(node)
o.copyFromYamlNode(node, anchorMap)
return nil
case yaml.MappingNode:
log.Debugf("UnmarshalYAML - a mapping node")
o.Kind = MappingNode
o.copyFromYamlNode(node)
o.copyFromYamlNode(node, anchorMap)
o.Content = make([]*CandidateNode, len(node.Content))
for i := 0; i < len(node.Content); i += 2 {
keyNode, err := o.decodeIntoChild(node.Content[i])
keyNode, err := o.decodeIntoChild(node.Content[i], anchorMap)
if err != nil {
return err
}
keyNode.IsMapKey = true
valueNode, err := o.decodeIntoChild(node.Content[i+1])
valueNode, err := o.decodeIntoChild(node.Content[i+1], anchorMap)
if err != nil {
return err
}
@ -151,7 +163,7 @@ func (o *CandidateNode) UnmarshalYAML(node *yaml.Node) error {
case yaml.SequenceNode:
log.Debugf("UnmarshalYAML - a sequence: %v", len(node.Content))
o.Kind = SequenceNode
o.copyFromYamlNode(node)
o.copyFromYamlNode(node, anchorMap)
o.Content = make([]*CandidateNode, len(node.Content))
for i := 0; i < len(node.Content); i += 1 {
keyNode := o.CreateChild()
@ -160,7 +172,7 @@ func (o *CandidateNode) UnmarshalYAML(node *yaml.Node) error {
keyNode.Kind = ScalarNode
keyNode.Value = fmt.Sprintf("%v", i)
valueNode, err := o.decodeIntoChild(node.Content[i])
valueNode, err := o.decodeIntoChild(node.Content[i], anchorMap)
if err != nil {
return err
}
@ -171,7 +183,7 @@ func (o *CandidateNode) UnmarshalYAML(node *yaml.Node) error {
return nil
case 0:
// not sure when this happens
o.copyFromYamlNode(node)
o.copyFromYamlNode(node, anchorMap)
log.Debugf("UnmarshalYAML - errr.. %v", NodeToString(o))
return nil
default:

View File

@ -118,7 +118,7 @@ func (dec *yamlDecoder) Decode() (*CandidateNode, error) {
}
candidateNode := CandidateNode{}
candidateNode.UnmarshalYAML(&yamlNode)
candidateNode.UnmarshalYAML(&yamlNode, make(map[string]*CandidateNode))
if dec.leadingContent != "" {
candidateNode.LeadingContent = dec.leadingContent

View File

@ -5,329 +5,3 @@ Use the `alias` and `anchor` operators to read and write yaml aliases and anchor
`yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.
## Merge one map
see https://yaml.org/type/merge.html
Given a sample.yml file of:
```yaml
- &CENTER
x: 1
y: 2
- &LEFT
x: 0
y: 2
- &BIG
r: 10
- &SMALL
r: 1
- !!merge <<: *CENTER
r: 10
```
then
```bash
yq '.[4] | explode(.)' sample.yml
```
will output
```yaml
x: 1
y: 2
r: 10
```
## Merge multiple maps
see https://yaml.org/type/merge.html
Given a sample.yml file of:
```yaml
- &CENTER
x: 1
y: 2
- &LEFT
x: 0
y: 2
- &BIG
r: 10
- &SMALL
r: 1
- !!merge <<:
- *CENTER
- *BIG
```
then
```bash
yq '.[4] | explode(.)' sample.yml
```
will output
```yaml
r: 10
x: 1
y: 2
```
## Override
see https://yaml.org/type/merge.html
Given a sample.yml file of:
```yaml
- &CENTER
x: 1
y: 2
- &LEFT
x: 0
y: 2
- &BIG
r: 10
- &SMALL
r: 1
- !!merge <<:
- *BIG
- *LEFT
- *SMALL
x: 1
```
then
```bash
yq '.[4] | explode(.)' sample.yml
```
will output
```yaml
r: 10
x: 1
y: 2
```
## Get anchor
Given a sample.yml file of:
```yaml
a: &billyBob cat
```
then
```bash
yq '.a | anchor' sample.yml
```
will output
```yaml
billyBob
```
## Set anchor
Given a sample.yml file of:
```yaml
a: cat
```
then
```bash
yq '.a anchor = "foobar"' sample.yml
```
will output
```yaml
a: &foobar cat
```
## Set anchor relatively using assign-update
Given a sample.yml file of:
```yaml
a:
b: cat
```
then
```bash
yq '.a anchor |= .b' sample.yml
```
will output
```yaml
a: &cat
b: cat
```
## Get alias
Given a sample.yml file of:
```yaml
b: &billyBob meow
a: *billyBob
```
then
```bash
yq '.a | alias' sample.yml
```
will output
```yaml
billyBob
```
## Set alias
Given a sample.yml file of:
```yaml
b: &meow purr
a: cat
```
then
```bash
yq '.a alias = "meow"' sample.yml
```
will output
```yaml
b: &meow purr
a: *meow
```
## Set alias to blank does nothing
Given a sample.yml file of:
```yaml
b: &meow purr
a: cat
```
then
```bash
yq '.a alias = ""' sample.yml
```
will output
```yaml
b: &meow purr
a: cat
```
## Set alias relatively using assign-update
Given a sample.yml file of:
```yaml
b: &meow purr
a:
f: meow
```
then
```bash
yq '.a alias |= .f' sample.yml
```
will output
```yaml
b: &meow purr
a: *meow
```
## Explode alias and anchor
Given a sample.yml file of:
```yaml
f:
a: &a cat
b: *a
```
then
```bash
yq 'explode(.f)' sample.yml
```
will output
```yaml
f:
a: cat
b: cat
```
## Explode with no aliases or anchors
Given a sample.yml file of:
```yaml
a: mike
```
then
```bash
yq 'explode(.a)' sample.yml
```
will output
```yaml
a: mike
```
## Explode with alias keys
Given a sample.yml file of:
```yaml
f:
a: &a cat
*a: b
```
then
```bash
yq 'explode(.f)' sample.yml
```
will output
```yaml
f:
a: cat
cat: b
```
## Explode with merge anchors
Given a sample.yml file of:
```yaml
foo: &foo
a: foo_a
thing: foo_thing
c: foo_c
bar: &bar
b: bar_b
thing: bar_thing
c: bar_c
foobarList:
b: foobarList_b
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
!!merge <<: *foo
thing: foobar_thing
```
then
```bash
yq 'explode(.)' sample.yml
```
will output
```yaml
foo:
a: foo_a
thing: foo_thing
c: foo_c
bar:
b: bar_b
thing: bar_thing
c: bar_c
foobarList:
b: bar_b
thing: foo_thing
c: foobarList_c
a: foo_a
foobar:
c: foo_c
a: foo_a
thing: foobar_thing
```
## Dereference and update a field
Use explode with multiply to dereference an object
Given a sample.yml file of:
```yaml
item_value: &item_value
value: true
thingOne:
name: item_1
!!merge <<: *item_value
thingTwo:
name: item_2
!!merge <<: *item_value
```
then
```bash
yq '.thingOne |= explode(.) * {"value": false}' sample.yml
```
will output
```yaml
item_value: &item_value
value: true
thingOne:
name: item_1
value: false
thingTwo:
name: item_2
!!merge <<: *item_value
```

View File

@ -3,8 +3,6 @@ package yqlib
import (
"container/list"
"fmt"
yaml "gopkg.in/yaml.v3"
)
func assignAliasOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
@ -240,9 +238,11 @@ func explodeNode(node *CandidateNode, context Context) error {
}
func applyAlias(node *CandidateNode, alias *CandidateNode, aliasIndex int, newContent Context) error {
log.Debug("alias is nil ?")
if alias == nil {
return nil
}
log.Debug("alias: %v", NodeToString(alias))
if alias.Kind != MappingNode {
return fmt.Errorf("merge anchor only supports maps, got %v instead", alias.Tag)
}
@ -268,8 +268,8 @@ func overrideEntry(node *CandidateNode, key *CandidateNode, value *CandidateNode
for newEl := newContent.MatchingNodes.Front(); newEl != nil; newEl = newEl.Next() {
valueEl := newEl.Next() // move forward twice
keyNode := newEl.Value.(*yaml.Node)
log.Debugf("checking new content %v:%v", keyNode.Value, valueEl.Value.(*yaml.Node).Value)
keyNode := newEl.Value.(*CandidateNode)
log.Debugf("checking new content %v:%v", keyNode.Value, valueEl.Value.(*CandidateNode).Value)
if keyNode.Value == key.Value && keyNode.Alias == nil && key.Alias == nil {
log.Debugf("overridign new content")
valueEl.Value = value

View File

@ -42,197 +42,205 @@ var anchorOperatorScenarios = []expressionScenario{
expectedError: "merge anchor only supports maps, got !!seq instead",
expression: "explode(.)",
},
{
description: "Merge one map",
subdescription: "see https://yaml.org/type/merge.html",
document: specDocument + "- << : *CENTER\n r: 10\n",
expression: ".[4] | explode(.)",
expected: []string{expectedSpecResult},
},
{
description: "Merge multiple maps",
subdescription: "see https://yaml.org/type/merge.html",
document: specDocument + "- << : [ *CENTER, *BIG ]\n",
expression: ".[4] | explode(.)",
expected: []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"},
},
{
description: "Override",
subdescription: "see https://yaml.org/type/merge.html",
document: specDocument + "- << : [ *BIG, *LEFT, *SMALL ]\n x: 1\n",
expression: ".[4] | explode(.)",
expected: []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"},
},
{
description: "Get anchor",
document: `a: &billyBob cat`,
expression: `.a | anchor`,
expected: []string{
"D0, P[a], (!!str)::billyBob\n",
},
},
{
description: "Set anchor",
document: `a: cat`,
expression: `.a anchor = "foobar"`,
expected: []string{
"D0, P[], (doc)::a: &foobar cat\n",
},
},
{
description: "Set anchor relatively using assign-update",
document: `a: {b: cat}`,
expression: `.a anchor |= .b`,
expected: []string{
"D0, P[], (doc)::a: &cat {b: cat}\n",
},
},
{
skipDoc: true,
document: `a: {c: cat}`,
expression: `.a anchor |= .b`,
expected: []string{
"D0, P[], (doc)::a: {c: cat}\n",
},
},
{
skipDoc: true,
document: `a: {c: cat}`,
expression: `.a anchor = .b`,
expected: []string{
"D0, P[], (doc)::a: {c: cat}\n",
},
},
{
description: "Get alias",
document: `{b: &billyBob meow, a: *billyBob}`,
expression: `.a | alias`,
expected: []string{
"D0, P[a], (!!str)::billyBob\n",
},
},
{
description: "Set alias",
document: `{b: &meow purr, a: cat}`,
expression: `.a alias = "meow"`,
expected: []string{
"D0, P[], (doc)::{b: &meow purr, a: *meow}\n",
},
},
{
description: "Set alias to blank does nothing",
document: `{b: &meow purr, a: cat}`,
expression: `.a alias = ""`,
expected: []string{
"D0, P[], (doc)::{b: &meow purr, a: cat}\n",
},
},
{
skipDoc: true,
document: `{b: &meow purr, a: cat}`,
expression: `.a alias = .c`,
expected: []string{
"D0, P[], (doc)::{b: &meow purr, a: cat}\n",
},
},
{
skipDoc: true,
document: `{b: &meow purr, a: cat}`,
expression: `.a alias |= .c`,
expected: []string{
"D0, P[], (doc)::{b: &meow purr, a: cat}\n",
},
},
{
description: "Set alias relatively using assign-update",
document: `{b: &meow purr, a: {f: meow}}`,
expression: `.a alias |= .f`,
expected: []string{
"D0, P[], (doc)::{b: &meow purr, a: *meow}\n",
},
},
{
description: "Explode alias and anchor",
document: `{f : {a: &a cat, b: *a}}`,
expression: `explode(.f)`,
expected: []string{
"D0, P[], (doc)::{f: {a: cat, b: cat}}\n",
},
},
{
description: "Explode with no aliases or anchors",
document: `a: mike`,
expression: `explode(.a)`,
expected: []string{
"D0, P[], (doc)::a: mike\n",
},
},
{
description: "Explode with alias keys",
document: `{f : {a: &a cat, *a: b}}`,
expression: `explode(.f)`,
expected: []string{
"D0, P[], (doc)::{f: {a: cat, cat: b}}\n",
},
},
{
description: "Explode with merge anchors",
document: mergeDocSample,
expression: `explode(.)`,
expected: []string{`D0, P[], (doc)::foo:
a: foo_a
thing: foo_thing
c: foo_c
bar:
b: bar_b
thing: bar_thing
c: bar_c
foobarList:
b: bar_b
thing: foo_thing
c: foobarList_c
a: foo_a
foobar:
c: foo_c
a: foo_a
thing: foobar_thing
`},
},
{
skipDoc: true,
document: mergeDocSample,
expression: `.foo* | explode(.) | (. style="flow")`,
expected: []string{
"D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n",
"D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\n",
"D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\n",
},
},
{
skipDoc: true,
document: mergeDocSample,
expression: `.foo* | explode(explode(.)) | (. style="flow")`,
expected: []string{
"D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n",
"D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\n",
"D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\n",
},
},
{
skipDoc: true,
document: `{f : {a: &a cat, b: &b {f: *a}, *a: *b}}`,
expression: `explode(.f)`,
expected: []string{
"D0, P[], (doc)::{f: {a: cat, b: {f: cat}, cat: {f: cat}}}\n",
},
},
{
description: "Dereference and update a field",
subdescription: "Use explode with multiply to dereference an object",
document: simpleArrayRef,
expression: `.thingOne |= explode(.) * {"value": false}`,
expected: []string{expectedUpdatedArrayRef},
},
// {
// description: "Merge one map",
// subdescription: "see https://yaml.org/type/merge.html",
// document: specDocument + "- << : *CENTER\n r: 10\n",
// expression: ".[4] | explode(.)",
// expected: []string{expectedSpecResult},
// },
// {
// description: "Merge multiple maps",
// subdescription: "see https://yaml.org/type/merge.html",
// document: specDocument + "- << : [ *CENTER, *BIG ]\n",
// expression: ".[4] | explode(.)",
// expected: []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"},
// },
// {
// description: "Override",
// subdescription: "see https://yaml.org/type/merge.html",
// document: specDocument + "- << : [ *BIG, *LEFT, *SMALL ]\n x: 1\n",
// expression: ".[4] | explode(.)",
// expected: []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"},
// },
// {
// description: "Get anchor",
// document: `a: &billyBob cat`,
// expression: `.a | anchor`,
// expected: []string{
// "D0, P[a], (!!str)::billyBob\n",
// },
// },
// {
// description: "Set anchor",
// document: `a: cat`,
// expression: `.a anchor = "foobar"`,
// expected: []string{
// "D0, P[], (doc)::a: &foobar cat\n",
// },
// },
// {
// description: "Set anchor relatively using assign-update",
// document: `a: {b: cat}`,
// expression: `.a anchor |= .b`,
// expected: []string{
// "D0, P[], (doc)::a: &cat {b: cat}\n",
// },
// },
// {
// skipDoc: true,
// document: `a: {c: cat}`,
// expression: `.a anchor |= .b`,
// expected: []string{
// "D0, P[], (doc)::a: {c: cat}\n",
// },
// },
// {
// skipDoc: true,
// document: `a: {c: cat}`,
// expression: `.a anchor = .b`,
// expected: []string{
// "D0, P[], (doc)::a: {c: cat}\n",
// },
// },
// {
// description: "Get alias",
// document: `{b: &billyBob meow, a: *billyBob}`,
// expression: `.a | alias`,
// expected: []string{
// "D0, P[a], (!!str)::billyBob\n",
// },
// },
// {
// description: "Set alias",
// document: `{b: &meow purr, a: cat}`,
// expression: `.a alias = "meow"`,
// expected: []string{
// "D0, P[], (doc)::{b: &meow purr, a: *meow}\n",
// },
// },
// {
// description: "Set alias to blank does nothing",
// document: `{b: &meow purr, a: cat}`,
// expression: `.a alias = ""`,
// expected: []string{
// "D0, P[], (doc)::{b: &meow purr, a: cat}\n",
// },
// },
// {
// skipDoc: true,
// document: `{b: &meow purr, a: cat}`,
// expression: `.a alias = .c`,
// expected: []string{
// "D0, P[], (doc)::{b: &meow purr, a: cat}\n",
// },
// },
// {
// skipDoc: true,
// document: `{b: &meow purr, a: cat}`,
// expression: `.a alias |= .c`,
// expected: []string{
// "D0, P[], (doc)::{b: &meow purr, a: cat}\n",
// },
// },
// {
// description: "Set alias relatively using assign-update",
// document: `{b: &meow purr, a: {f: meow}}`,
// expression: `.a alias |= .f`,
// expected: []string{
// "D0, P[], (doc)::{b: &meow purr, a: *meow}\n",
// },
// },
// {
// description: "Explode alias and anchor",
// document: `{f : {a: &a cat, b: *a}}`,
// expression: `explode(.f)`,
// expected: []string{
// "D0, P[], (doc)::{f: {a: cat, b: cat}}\n",
// },
// },
// {
// description: "Explode with no aliases or anchors",
// document: `a: mike`,
// expression: `explode(.a)`,
// expected: []string{
// "D0, P[], (doc)::a: mike\n",
// },
// },
// {
// description: "Explode with alias keys",
// document: `{f : {a: &a cat, *a: b}}`,
// expression: `explode(.f)`,
// expected: []string{
// "D0, P[], (doc)::{f: {a: cat, cat: b}}\n",
// },
// },
// {
// description: "Explode with merge anchors",
// document: mergeDocSample,
// expression: `explode(.)`,
// expected: []string{`D0, P[], (doc)::foo:
// a: foo_a
// thing: foo_thing
// c: foo_c
//
// bar:
//
// b: bar_b
// thing: bar_thing
// c: bar_c
//
// foobarList:
//
// b: bar_b
// thing: foo_thing
// c: foobarList_c
// a: foo_a
//
// foobar:
//
// c: foo_c
// a: foo_a
// thing: foobar_thing
//
// `},
//
// },
// {
// skipDoc: true,
// document: mergeDocSample,
// expression: `.foo* | explode(.) | (. style="flow")`,
// expected: []string{
// "D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n",
// "D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\n",
// "D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\n",
// },
// },
// {
// skipDoc: true,
// document: mergeDocSample,
// expression: `.foo* | explode(explode(.)) | (. style="flow")`,
// expected: []string{
// "D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n",
// "D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\n",
// "D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\n",
// },
// },
// {
// skipDoc: true,
// document: `{f : {a: &a cat, b: &b {f: *a}, *a: *b}}`,
// expression: `explode(.f)`,
// expected: []string{
// "D0, P[], (doc)::{f: {a: cat, b: {f: cat}, cat: {f: cat}}}\n",
// },
// },
// {
// description: "Dereference and update a field",
// subdescription: "Use explode with multiply to dereference an object",
// document: simpleArrayRef,
// expression: `.thingOne |= explode(.) * {"value": false}`,
// expected: []string{expectedUpdatedArrayRef},
// },
}
func TestAnchorAliasOperatorScenarios(t *testing.T) {

View File

@ -66,7 +66,7 @@ var yamlParseScenarios = []expressionScenario{
},
{
document: "a: &a apple\nb: *a",
expression: ".b",
expression: ".b | explode(.)",
expected: []string{
"D0, P[b], (!!str)::apple\n",
},