suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2025-01-24 23:35:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added merge if empty

Browse Source
This commit is contained in:
Mike Farah 2021-01-13 16:54:28 +11:00
parent 0bf3d781f6
commit 85398727ad
11 changed files with 125 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
pkg/yqlib/doc/Multiply.md
View File

@ -111,6 +111,26 @@ b:
- 5
```
## Merge, only existing fields
Given a sample.yml file of:
```yaml
a:
thing: one
cat: frog
b:
missing: two
thing: two
```
then
```bash
yq eval '.a *? .b' sample.yml
```
will output
```yaml
thing: two
cat: frog
```
## Merge, appending arrays
Given a sample.yml file of:
```yaml
@ -143,6 +163,33 @@ array:
value: banana
```
## Merge, only existing fields, appending arrays
Given a sample.yml file of:
```yaml
a:
thing:
- 1
- 2
b:
thing:
- 3
- 4
another:
- 1
```
then
```bash
yq eval '.a *?+ .b' sample.yml
```
will output
```yaml
thing:
- 1
- 2
- 3
- 4
```
## Merge to prefix an element
Given a sample.yml file of:
```yaml

5
pkg/yqlib/expression_processing_test.go
View File

@ -52,11 +52,6 @@ var pathTests = []struct {
append(make([]interface{}, 0), "[", "3 (int64)", "]"),
append(make([]interface{}, 0), "3 (int64)", "COLLECT", "SHORT_PIPE"),
},
{
`d0.a`,
append(make([]interface{}, 0), "d0", "SHORT_PIPE", "a"),
append(make([]interface{}, 0), "d0", "a", "SHORT_PIPE"),
},
{
`.a | .[].b == "apple"`,
append(make([]interface{}, 0), "a", "PIPE", "TRAVERSE_ARRAY", "[", "]", "SHORT_PIPE", "b", "EQUALS", "apple (string)"),

39
pkg/yqlib/expression_tokeniser.go
View File

@ -3,6 +3,7 @@ package yqlib
import (
"fmt"
"strconv"
"strings"
lex "github.com/timtadh/lexmachine"
"github.com/timtadh/lexmachine/machines"
@ -65,21 +66,7 @@ func pathToken(wrapped bool) lex.Action {
value = unwrap(value)
}
log.Debug("PathToken %v", value)
op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value}
return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil
}
}
func documentToken() lex.Action {
return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
var numberString = string(m.Bytes)
numberString = numberString[1:]
var number, errParsingInt = strconv.ParseInt(numberString, 10, 64) // nolint
if errParsingInt != nil {
return nil, errParsingInt
}
log.Debug("documentToken %v", string(m.Bytes))
op := &Operation{OperationType: documentFilterOpType, Value: number, StringValue: numberString}
op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: traversePreferences{}}
return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil
}
}
@ -101,6 +88,21 @@ func assignOpToken(updateAssign bool) lex.Action {
}
}
func multiplyWithPrefs() lex.Action {
return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
prefs := multiplyPreferences{}
options := string(m.Bytes)
if strings.Contains(options, "+") {
prefs.AppendArrays = true
}
if strings.Contains(options, "?") {
prefs.TraversePrefs = traversePreferences{DontAutoCreate: true}
}
op := &Operation{OperationType: multiplyOpType, Value: multiplyOpType.Type, StringValue: options, Preferences: &prefs}
return &token{TokenType: operationToken, Operation: op}, nil
}
}
func opTokenWithPrefs(op *operationType, assignOpType *operationType, preferences interface{}) lex.Action {
return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
log.Debug("opTokenWithPrefs %v", string(m.Bytes))
@ -220,10 +222,10 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`\.\[`), literalToken(traverseArrayCollect, false))
lexer.Add([]byte(`\.\.`), opTokenWithPrefs(recursiveDescentOpType, nil, &recursiveDescentPreferences{RecurseArray: true,
TraversePreferences: &traversePreferences{FollowAlias: false, IncludeMapKeys: false}}))
TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: false}}))
lexer.Add([]byte(`\.\.\.`), opTokenWithPrefs(recursiveDescentOpType, nil, &recursiveDescentPreferences{RecurseArray: true,
TraversePreferences: &traversePreferences{FollowAlias: false, IncludeMapKeys: true}}))
TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: true}}))
lexer.Add([]byte(`,`), opToken(unionOpType))
lexer.Add([]byte(`:\s*`), opToken(createMapOpType))
@ -270,7 +272,6 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte("( |\t|\n|\r)+"), skip)
lexer.Add([]byte(`d[0-9]+`), documentToken())
lexer.Add([]byte(`\."[^ "]+"`), pathToken(true))
lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=]+`), pathToken(false))
lexer.Add([]byte(`\.`), selfToken())
@ -296,7 +297,7 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`\{`), literalToken(openCollectObject, false))
lexer.Add([]byte(`\}`), literalToken(closeCollectObject, true))
lexer.Add([]byte(`\*`), opTokenWithPrefs(multiplyOpType, nil, &multiplyPreferences{AppendArrays: false}))
lexer.Add([]byte(`\*\+`), opTokenWithPrefs(multiplyOpType, nil, &multiplyPreferences{AppendArrays: true}))
lexer.Add([]byte(`\*[\+|\?]*`), multiplyWithPrefs())
lexer.Add([]byte(`\+`), opToken(addOpType))
lexer.Add([]byte(`\+=`), opToken(addAssignOpType))

3
pkg/yqlib/lib.go
View File

@ -68,7 +68,6 @@ var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Pre
var traversePathOpType = &operationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 50, Handler: traversePathOperator}
var traverseArrayOpType = &operationType{Type: "TRAVERSE_ARRAY", NumArgs: 1, Precedence: 50, Handler: traverseArrayOperator}
var documentFilterOpType = &operationType{Type: "DOCUMENT_FILTER", NumArgs: 0, Precedence: 50, Handler: traversePathOperator}
var selfReferenceOpType = &operationType{Type: "SELF", NumArgs: 0, Precedence: 50, Handler: selfOperator}
var valueOpType = &operationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: valueOperator}
var envOpType = &operationType{Type: "ENV", NumArgs: 0, Precedence: 50, Handler: envOperator}
@ -120,8 +119,6 @@ func createValueOperation(value interface{}, stringValue string) *Operation {
func (p *Operation) toString() string {
if p.OperationType == traversePathOpType {
return fmt.Sprintf("%v", p.Value)
} else if p.OperationType == documentFilterOpType {
return fmt.Sprintf("d%v", p.Value)
} else if p.OperationType == selfReferenceOpType {
return "SELF"
} else if p.OperationType == valueOpType {

2
pkg/yqlib/operator_collect_object.go
View File

@ -60,7 +60,7 @@ func collect(d *dataTreeNavigator, aggregate *list.List, remainingMatches *list.
candidate := remainingMatches.Remove(remainingMatches.Front()).(*CandidateNode)
splatted, err := splat(d, nodeToMap(candidate),
&traversePreferences{FollowAlias: false, IncludeMapKeys: false})
traversePreferences{DontFollowAlias: true, IncludeMapKeys: false})
for splatEl := splatted.Front(); splatEl != nil; splatEl = splatEl.Next() {
splatEl.Value.(*CandidateNode).Path = nil

2
pkg/yqlib/operator_delete.go
View File

@ -25,7 +25,7 @@ func deleteChildOperator(d *dataTreeNavigator, matchingNodes *list.List, express
deleteImmediateChildOpNode := &ExpressionNode{
Operation: deleteImmediateChildOp,
Rhs: createTraversalTree(candidate.Path[0 : len(candidate.Path)-1]),
Rhs: createTraversalTree(candidate.Path[0:len(candidate.Path)-1], traversePreferences{}),
}
_, err := d.GetMatchingNodes(matchingNodes, deleteImmediateChildOpNode)

22
pkg/yqlib/operator_multiply.go
View File

@ -44,7 +44,8 @@ func crossFunction(d *dataTreeNavigator, matchingNodes *list.List, expressionNod
}
type multiplyPreferences struct {
AppendArrays bool
AppendArrays bool
TraversePrefs traversePreferences
}
func multiplyOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
@ -59,29 +60,28 @@ func multiply(preferences *multiplyPreferences) func(d *dataTreeNavigator, lhs *
log.Debugf("Multipling LHS: %v", lhs.Node.Tag)
log.Debugf("- RHS: %v", rhs.Node.Tag)
shouldAppendArrays := preferences.AppendArrays
if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode ||
(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
var newBlank = lhs.CreateChild(nil, &yaml.Node{})
var newThing, err = mergeObjects(d, newBlank, lhs, false)
var newThing, err = mergeObjects(d, newBlank, lhs, &multiplyPreferences{})
if err != nil {
return nil, err
}
return mergeObjects(d, newThing, rhs, shouldAppendArrays)
return mergeObjects(d, newThing, rhs, preferences)
}
return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag)
}
}
func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode, shouldAppendArrays bool) (*CandidateNode, error) {
func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode, preferences *multiplyPreferences) (*CandidateNode, error) {
shouldAppendArrays := preferences.AppendArrays
var results = list.New()
// shouldn't recurse arrays if appending
prefs := &recursiveDescentPreferences{RecurseArray: !shouldAppendArrays,
TraversePreferences: &traversePreferences{FollowAlias: false}}
TraversePreferences: traversePreferences{DontFollowAlias: true}}
err := recursiveDecent(d, results, nodeToMap(rhs), prefs)
if err != nil {
return nil, err
@ -93,7 +93,7 @@ func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode,
}
for el := results.Front(); el != nil; el = el.Next() {
err := applyAssignment(d, pathIndexToStartFrom, lhs, el.Value.(*CandidateNode), shouldAppendArrays)
err := applyAssignment(d, pathIndexToStartFrom, lhs, el.Value.(*CandidateNode), preferences)
if err != nil {
return nil, err
}
@ -101,8 +101,8 @@ func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode,
return lhs, nil
}
func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode, shouldAppendArrays bool) error {
func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode, preferences *multiplyPreferences) error {
shouldAppendArrays := preferences.AppendArrays
log.Debugf("merge - applyAssignment lhs %v, rhs: %v", NodeToString(lhs), NodeToString(rhs))
lhsPath := rhs.Path[pathIndexToStartFrom:]
@ -116,7 +116,7 @@ func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *Candid
}
rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs}
assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath), Rhs: &ExpressionNode{Operation: rhsOp}}
assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath, preferences.TraversePrefs), Rhs: &ExpressionNode{Operation: rhsOp}}
_, err := d.GetMatchingNodes(nodeToMap(lhs), assignmentOpNode)

24
pkg/yqlib/operator_multiply_test.go
View File

@ -107,6 +107,22 @@ b:
"D0, P[a], (!!seq)::[1, 2]\n",
},
},
{
description: "Merge, only existing fields",
document: `{a: {thing: one, cat: frog}, b: {missing: two, thing: two}}`,
expression: `.a *? .b`,
expected: []string{
"D0, P[a], (!!map)::{thing: two, cat: frog}\n",
},
},
{
skipDoc: true,
document: `{a: [{thing: one}], b: [{missing: two, thing: two}]}`,
expression: `.a *? .b`,
expected: []string{
"D0, P[a], (!!seq)::[{thing: two}]\n",
},
},
{
description: "Merge, appending arrays",
document: `{a: {array: [1, 2, animal: dog], value: coconut}, b: {array: [3, 4, animal: cat], value: banana}}`,
@ -115,6 +131,14 @@ b:
"D0, P[a], (!!map)::{array: [1, 2, {animal: dog}, 3, 4, {animal: cat}], value: banana}\n",
},
},
{
description: "Merge, only existing fields, appending arrays",
document: `{a: {thing: [1,2]}, b: {thing: [3,4], another: [1]}}`,
expression: `.a *?+ .b`,
expected: []string{
"D0, P[a], (!!map)::{thing: [1, 2, 3, 4]}\n",
},
},
{
description: "Merge to prefix an element",
document: `{a: cat, b: dog}`,

2
pkg/yqlib/operator_recursive_descent.go
View File

@ -7,7 +7,7 @@ import (
)
type recursiveDescentPreferences struct {
TraversePreferences *traversePreferences
TraversePreferences traversePreferences
RecurseArray bool
}

29
pkg/yqlib/operator_traverse_path.go
View File

@ -10,11 +10,12 @@ import (
)
type traversePreferences struct {
FollowAlias bool
IncludeMapKeys bool
DontFollowAlias bool
IncludeMapKeys bool
DontAutoCreate bool // by default, we automatically create entries on the fly.
}
func splat(d *dataTreeNavigator, matches *list.List, prefs *traversePreferences) (*list.List, error) {
func splat(d *dataTreeNavigator, matches *list.List, prefs traversePreferences) (*list.List, error) {
return traverseNodesWithArrayIndices(matches, make([]*yaml.Node, 0), prefs)
}
@ -54,8 +55,7 @@ func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, operation *Oper
switch value.Kind {
case yaml.MappingNode:
log.Debug("its a map with %v entries", len(value.Content)/2)
prefs := &traversePreferences{FollowAlias: true}
return traverseMap(matchingNode, operation.StringValue, prefs, false)
return traverseMap(matchingNode, operation.StringValue, operation.Preferences.(traversePreferences), false)
case yaml.SequenceNode:
log.Debug("its a sequence of %v things!", len(value.Content))
@ -83,11 +83,10 @@ func traverseArrayOperator(d *dataTreeNavigator, matchingNodes *list.List, expre
}
var indicesToTraverse = rhs.Front().Value.(*CandidateNode).Node.Content
prefs := &traversePreferences{FollowAlias: true}
return traverseNodesWithArrayIndices(matchingNodes, indicesToTraverse, prefs)
return traverseNodesWithArrayIndices(matchingNodes, indicesToTraverse, traversePreferences{})
}
func traverseNodesWithArrayIndices(matchingNodes *list.List, indicesToTraverse []*yaml.Node, prefs *traversePreferences) (*list.List, error) {
func traverseNodesWithArrayIndices(matchingNodes *list.List, indicesToTraverse []*yaml.Node, prefs traversePreferences) (*list.List, error) {
var matchingNodeMap = list.New()
for el := matchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
@ -101,7 +100,7 @@ func traverseNodesWithArrayIndices(matchingNodes *list.List, indicesToTraverse [
return matchingNodeMap, nil
}
func traverseArrayIndices(matchingNode *CandidateNode, indicesToTraverse []*yaml.Node, prefs *traversePreferences) (*list.List, error) { // call this if doc / alias like the other traverse
func traverseArrayIndices(matchingNode *CandidateNode, indicesToTraverse []*yaml.Node, prefs traversePreferences) (*list.List, error) { // call this if doc / alias like the other traverse
node := matchingNode.Node
if node.Tag == "!!null" {
log.Debugf("OperatorArrayTraverse got a null - turning it into an empty array")
@ -124,7 +123,7 @@ func traverseArrayIndices(matchingNode *CandidateNode, indicesToTraverse []*yaml
return list.New(), nil
}
func traverseMapWithIndices(candidate *CandidateNode, indices []*yaml.Node, prefs *traversePreferences) (*list.List, error) {
func traverseMapWithIndices(candidate *CandidateNode, indices []*yaml.Node, prefs traversePreferences) (*list.List, error) {
if len(indices) == 0 {
return traverseMap(candidate, "", prefs, true)
}
@ -188,7 +187,7 @@ func keyMatches(key *yaml.Node, wantedKey string) bool {
return matchKey(key.Value, wantedKey)
}
func traverseMap(matchingNode *CandidateNode, key string, prefs *traversePreferences, splat bool) (*list.List, error) {
func traverseMap(matchingNode *CandidateNode, key string, prefs traversePreferences, splat bool) (*list.List, error) {
var newMatches = orderedmap.NewOrderedMap()
err := doTraverseMap(newMatches, matchingNode, key, prefs, splat)
@ -196,7 +195,7 @@ func traverseMap(matchingNode *CandidateNode, key string, prefs *traversePrefere
return nil, err
}
if newMatches.Len() == 0 {
if !prefs.DontAutoCreate && newMatches.Len() == 0 {
//no matches, create one automagically
valueNode := &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"}
node := matchingNode.Node
@ -214,7 +213,7 @@ func traverseMap(matchingNode *CandidateNode, key string, prefs *traversePrefere
return results, nil
}
func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode, wantedKey string, prefs *traversePreferences, splat bool) error {
func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode, wantedKey string, prefs traversePreferences, splat bool) error {
// value.Content is a concatenated array of key, value,
// so keys are in the even indexes, values in odd.
// merge aliases are defined first, but we only want to traverse them
@ -229,7 +228,7 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode,
log.Debug("checking %v (%v)", key.Value, key.Tag)
//skip the 'merge' tag, find a direct match first
if key.Tag == "!!merge" && prefs.FollowAlias {
if key.Tag == "!!merge" && !prefs.DontFollowAlias {
log.Debug("Merge anchor")
err := traverseMergeAnchor(newMatches, candidate, value, wantedKey, prefs, splat)
if err != nil {
@ -249,7 +248,7 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode,
return nil
}
func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *CandidateNode, value *yaml.Node, wantedKey string, prefs *traversePreferences, splat bool) error {
func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *CandidateNode, value *yaml.Node, wantedKey string, prefs traversePreferences, splat bool) error {
switch value.Kind {
case yaml.AliasNode:
candidateNode := originalCandidate.CreateChild(nil, value.Alias)

10
pkg/yqlib/operators.go
View File

@ -35,14 +35,16 @@ func nodeToMap(candidate *CandidateNode) *list.List {
return elMap
}
func createTraversalTree(path []interface{}) *ExpressionNode {
func createTraversalTree(path []interface{}, traversePrefs traversePreferences) *ExpressionNode {
if len(path) == 0 {
return &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
} else if len(path) == 1 {
return &ExpressionNode{Operation: &Operation{OperationType: traversePathOpType, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}}
return &ExpressionNode{Operation: &Operation{OperationType: traversePathOpType, Preferences: traversePrefs, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}}
}
return &ExpressionNode{
Operation: &Operation{OperationType: shortPipeOpType},
Lhs: createTraversalTree(path[0:1]),
Rhs: createTraversalTree(path[1:])}
Lhs: createTraversalTree(path[0:1], traversePrefs),
Rhs: createTraversalTree(path[1:], traversePrefs),
}
}