suyiiyii
/
yq
mirror of https://github.com/mikefarah/yq.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

wip

This commit is contained in:
Mike Farah 2020-11-28 10:41:09 +11:00
parent 13679e51e2
commit 3cecb4e383
8 changed files with 135 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
pkg/yqlib/doc/Multiply.md
View File

@ -66,9 +66,9 @@ b:
Given a sample.yml file of:
```yaml
a: {things: great}
b:
also: "me"
b:
also: "me"
```
then
```bash
@ -76,9 +76,6 @@ yq eval '. * {"a":.b}' sample.yml
```
will output
```yaml
a: {things: great, also: "me"}
b:
also: "me"
```
## Merge arrays
@ -109,6 +106,38 @@ b:
- 5
```
## Merge, appending arrays
Given a sample.yml file of:
```yaml
a:
array:
- 1
- 2
- animal: dog
value: coconut
b:
array:
- 3
- 4
- animal: cat
value: banana
```
then
```bash
yq eval '.a *+ .b' sample.yml
```
will output
```yaml
array:
- 1
- 2
- animal: dog
- 3
- 4
- animal: cat
value: banana
```
## Merge to prefix an element
Given a sample.yml file of:
```yaml

33
pkg/yqlib/operator_add.go
View File

@ -8,6 +8,10 @@ import (
yaml "gopkg.in/yaml.v3"
)
type AddPreferences struct {
InPlace bool
}
func toNodes(candidates *list.List) []*yaml.Node {
if candidates.Len() == 0 {
@ -41,26 +45,35 @@ func AddOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathT
return nil, err
}
preferences := pathNode.Operation.Preferences
inPlace := preferences != nil && preferences.(*AddPreferences).InPlace
for el := lhs.Front(); el != nil; el = el.Next() {
lhsCandidate := el.Value.(*CandidateNode)
lhsNode := UnwrapDoc(lhsCandidate.Node)
var newBlank = &CandidateNode{
Path: lhsCandidate.Path,
Document: lhsCandidate.Document,
Filename: lhsCandidate.Filename,
Node: &yaml.Node{},
var target *CandidateNode
if inPlace {
target = lhsCandidate
} else {
target = &CandidateNode{
Path: lhsCandidate.Path,
Document: lhsCandidate.Document,
Filename: lhsCandidate.Filename,
Node: &yaml.Node{},
}
}
switch lhsNode.Kind {
case yaml.MappingNode:
return nil, fmt.Errorf("Maps not yet supported for addition")
case yaml.SequenceNode:
newBlank.Node.Kind = yaml.SequenceNode
newBlank.Node.Style = lhsNode.Style
newBlank.Node.Tag = "!!seq"
newBlank.Node.Content = append(lhsNode.Content, toNodes(rhs)...)
results.PushBack(newBlank)
target.Node.Kind = yaml.SequenceNode
target.Node.Style = lhsNode.Style
target.Node.Tag = "!!seq"
target.Node.Content = append(lhsNode.Content, toNodes(rhs)...)
results.PushBack(target)
case yaml.ScalarNode:
return nil, fmt.Errorf("Scalars not yet supported for addition")
}

3
pkg/yqlib/operator_add_test.go
View File

@ -5,6 +5,9 @@ import (
)
var addOperatorScenarios = []expressionScenario{
{
description: "+= test and doc",
},
{
description: "Concatenate arrays",
document: `{a: [1,2], b: [3,4]}`,

2
pkg/yqlib/operator_collect_object.go
View File

@ -94,7 +94,7 @@ func collect(d *dataTreeNavigator, aggregate *list.List, remainingMatches *list.
newCandidate.Path = nil
newCandidate, err = multiply(d, newCandidate, splatCandidate)
newCandidate, err = multiply(&MultiplyPreferences{AppendArrays: false})(d, newCandidate, splatCandidate)
if err != nil {
return nil, err
}

63
pkg/yqlib/operator_multiply.go
View File

@ -43,39 +43,49 @@ func crossFunction(d *dataTreeNavigator, matchingNodes *list.List, pathNode *Pat
return results, nil
}
type MultiplyPreferences struct {
AppendArrays bool
}
func MultiplyOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
log.Debugf("-- MultiplyOperator")
return crossFunction(d, matchingNodes, pathNode, multiply)
return crossFunction(d, matchingNodes, pathNode, multiply(pathNode.Operation.Preferences.(*MultiplyPreferences)))
}
func multiply(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
lhs.Node = UnwrapDoc(lhs.Node)
rhs.Node = UnwrapDoc(rhs.Node)
log.Debugf("Multipling LHS: %v", lhs.Node.Tag)
log.Debugf("- RHS: %v", rhs.Node.Tag)
func multiply(preferences *MultiplyPreferences) func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
return func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
lhs.Node = UnwrapDoc(lhs.Node)
rhs.Node = UnwrapDoc(rhs.Node)
log.Debugf("Multipling LHS: %v", lhs.Node.Tag)
log.Debugf("- RHS: %v", rhs.Node.Tag)
if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode ||
(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
shouldAppendArrays := preferences.AppendArrays
if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode ||
(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
var newBlank = &CandidateNode{
Path: lhs.Path,
Document: lhs.Document,
Filename: lhs.Filename,
Node: &yaml.Node{},
}
var newThing, err = mergeObjects(d, newBlank, lhs, false)
if err != nil {
return nil, err
}
return mergeObjects(d, newThing, rhs, shouldAppendArrays)
var newBlank = &CandidateNode{
Path: lhs.Path,
Document: lhs.Document,
Filename: lhs.Filename,
Node: &yaml.Node{},
}
var newThing, err = mergeObjects(d, newBlank, lhs)
if err != nil {
return nil, err
}
return mergeObjects(d, newThing, rhs)
return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag)
}
return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag)
}
func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode, shouldAppendArrays bool) (*CandidateNode, error) {
var results = list.New()
err := recursiveDecent(d, results, nodeToMap(rhs))
// shouldn't recurse arrays if appending
err := recursiveDecent(d, results, nodeToMap(rhs), !shouldAppendArrays)
if err != nil {
return nil, err
}
@ -86,7 +96,7 @@ func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode)
}
for el := results.Front(); el != nil; el = el.Next() {
err := applyAssignment(d, pathIndexToStartFrom, lhs, el.Value.(*CandidateNode))
err := applyAssignment(d, pathIndexToStartFrom, lhs, el.Value.(*CandidateNode), shouldAppendArrays)
if err != nil {
return nil, err
}
@ -107,7 +117,8 @@ func createTraversalTree(path []interface{}) *PathTreeNode {
}
func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode) error {
func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode, shouldAppendArrays bool) error {
log.Debugf("merge - applyAssignment lhs %v, rhs: %v", NodeToString(lhs), NodeToString(rhs))
lhsPath := rhs.Path[pathIndexToStartFrom:]
@ -116,6 +127,10 @@ func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *Candid
if rhs.Node.Kind == yaml.ScalarNode || rhs.Node.Kind == yaml.AliasNode {
assignmentOp.OperationType = Assign
assignmentOp.Preferences = &AssignOpPreferences{false}
} else if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode {
log.Debugf("append! lhs %v, rhs: %v", NodeToString(lhs), NodeToString(rhs))
assignmentOp.OperationType = Add
assignmentOp.Preferences = &AddPreferences{InPlace: true}
}
rhsOp := &Operation{OperationType: ValueOp, CandidateNode: rhs}

31
pkg/yqlib/operator_multiply_test.go
View File

@ -5,6 +5,9 @@ import (
)
var multiplyOperatorScenarios = []expressionScenario{
{
description: "*+ doc",
},
{
skipDoc: true,
document: `{a: {also: [1]}, b: {also: me}}`,
@ -80,15 +83,15 @@ var multiplyOperatorScenarios = []expressionScenario{
description: "Merge keeps style of LHS",
dontFormatInputForDoc: true,
document: `a: {things: great}
b:
also: "me"
`,
b:
also: "me"
`,
expression: `. * {"a":.b}`,
expected: []string{
`D0, P[], (!!map)::a: {things: great, also: "me"}
b:
also: "me"
`,
b:
also: "me"
`,
},
},
{
@ -99,6 +102,22 @@ b:
"D0, P[], (!!map)::{a: [3, 4, 5], b: [3, 4, 5]}\n",
},
},
{
skipDoc: true,
document: `{a: [1], b: [2]}`,
expression: `.a *+ .b`,
expected: []string{
"D0, P[a], (!!seq)::[1, 2]\n",
},
},
{
description: "Merge, appending arrays",
document: `{a: {array: [1, 2, animal: dog], value: coconut}, b: {array: [3, 4, animal: cat], value: banana}}`,
expression: `.a *+ .b`,
expected: []string{
"D0, P[a], (!!map)::{array: [1, 2, {animal: dog}, 3, 4, {animal: cat}], value: banana}\n",
},
},
{
description: "Merge to prefix an element",
document: `{a: cat, b: dog}`,

11
pkg/yqlib/operator_recursive_descent.go
View File

@ -3,13 +3,13 @@ package yqlib
import (
"container/list"
"gopkg.in/yaml.v3"
yaml "gopkg.in/yaml.v3"
)
func RecursiveDescentOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
var results = list.New()
err := recursiveDecent(d, results, matchMap)
err := recursiveDecent(d, results, matchMap, true)
if err != nil {
return nil, err
}
@ -17,7 +17,7 @@ func RecursiveDescentOperator(d *dataTreeNavigator, matchMap *list.List, pathNod
return results, nil
}
func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.List) error {
func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.List, recurseArray bool) error {
for el := matchMap.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
@ -26,14 +26,15 @@ func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.Li
log.Debugf("Recursive Decent, added %v", NodeToString(candidate))
results.PushBack(candidate)
if candidate.Node.Kind != yaml.AliasNode && len(candidate.Node.Content) > 0 {
if candidate.Node.Kind != yaml.AliasNode && len(candidate.Node.Content) > 0 &&
(recurseArray || candidate.Node.Kind != yaml.SequenceNode) {
children, err := Splat(d, nodeToMap(candidate))
if err != nil {
return err
}
err = recursiveDecent(d, results, children)
err = recursiveDecent(d, results, children, recurseArray)
if err != nil {
return err
}

4
pkg/yqlib/path_tokeniser.go
View File

@ -251,8 +251,10 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`\]`), literalToken(CloseCollect, true))
lexer.Add([]byte(`\{`), literalToken(OpenCollectObject, false))
lexer.Add([]byte(`\}`), literalToken(CloseCollectObject, true))
lexer.Add([]byte(`\*`), opToken(Multiply))
lexer.Add([]byte(`\*`), opTokenWithPrefs(Multiply, nil, &MultiplyPreferences{AppendArrays: false}))
lexer.Add([]byte(`\*\+`), opTokenWithPrefs(Multiply, nil, &MultiplyPreferences{AppendArrays: true}))
lexer.Add([]byte(`\+`), opToken(Add))
lexer.Add([]byte(`\+=`), opTokenWithPrefs(Add, nil, &AddPreferences{InPlace: true}))
err := lexer.Compile()
if err != nil {