diff --git a/.golangci.yml b/.golangci.yml index ae6360a8..7a9f27b7 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -3,9 +3,11 @@ run: linters: enable: - errorlint + - gci - gofmt - goimports - gosec + - misspell issues: exclude-rules: - linters: diff --git a/1 b/1 deleted file mode 100644 index 377f7756..00000000 --- a/1 +++ /dev/null @@ -1 +0,0 @@ -Error: cannot pass files in when using null-input flag diff --git a/README.md b/README.md index 26717799..7c12d6b4 100644 --- a/README.md +++ b/README.md @@ -220,6 +220,7 @@ Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq) - [Update yaml inplace](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags) - [Complex expressions to select and update](https://mikefarah.gitbook.io/yq/operators/select#select-and-update-matching-values-in-map) - Keeps yaml formatting and comments when updating (though there are issues with whitespace) +- [Load content from other files](https://mikefarah.gitbook.io/yq/operators/load) - [Convert to/from json](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert) - [Convert to properties](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties) - [Pipe data in by using '-'](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate) diff --git a/cmd/utils.go b/cmd/utils.go index 80a2b295..483ea426 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -25,7 +25,7 @@ func initCommand(cmd *cobra.Command, args []string) (firstFileIndex int, err err firstFileIndex = 1 } - // backwards compatibilty + // backwards compatibility if outputToJSON { outputFormat = "json" } diff --git a/go.sum b/go.sum index 4ccfbb19..513b09cc 100644 --- a/go.sum +++ b/go.sum @@ -471,7 +471,6 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/merge-array-by-key.sh b/merge-array-by-key.sh new file mode 100755 index 00000000..09481e62 --- /dev/null +++ b/merge-array-by-key.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# ./yq ea '.[]' examples/data*.yaml + + +./yq ea ' + ((.[] | {.name: .}) as $item ireduce ({}; . * $item )) as $uniqueMap + | ( $uniqueMap | to_entries | .[]) as $item ireduce([]; . + $item.value) +' examples/data*.yaml \ No newline at end of file diff --git a/pkg/yqlib/doc/equals.md b/pkg/yqlib/doc/equals.md index d7129a79..17771963 100644 --- a/pkg/yqlib/doc/equals.md +++ b/pkg/yqlib/doc/equals.md @@ -108,7 +108,7 @@ will output a: frog ``` -## Two non existant keys are equal +## Two non existent keys are equal Given a sample.yml file of: ```yaml a: frog diff --git a/pkg/yqlib/doc/multiply-merge.md b/pkg/yqlib/doc/multiply-merge.md index ffc36b11..7f83a495 100644 --- a/pkg/yqlib/doc/multiply-merge.md +++ b/pkg/yqlib/doc/multiply-merge.md @@ -234,43 +234,60 @@ will output ``` ## Merge arrays of objects together, matching on a key -It's a complex command, the trickyness comes from needing to have the right context in the expressions. -First we save the second array into a variable '$two' which lets us reference it later. -We then need to update the first array. We will use the relative update (|=) because we need to update relative to the current element of the array in the LHS in the RHS expression. -We set the current element of the first array as $cur. Now we multiply (merge) $cur with the matching entry in $two, by passing $two through a select filter. +There are several parts of the complex expression. +The first part is doing the hard work, it creates a map from the arrays keyed by '.a', so that there are no duplicates. +Then there's another reduce that converts that map back to an array. +Finally, we set the result of the merged array back into the first doc. + +To use this, you will need to update '.myArray' to be the expression to your array (e.g. .my.array), and '.a' to be the key field of your array (e.g. '.name') + +Thanks Kev from [stackoverflow](https://stackoverflow.com/a/70109529/1168223) Given a sample.yml file of: ```yaml -- a: apple - b: appleB -- a: kiwi - b: kiwiB -- a: banana - b: bananaB +myArray: + - a: apple + b: appleB + - a: kiwi + b: kiwiB + - a: banana + b: bananaB +something: else ``` And another sample another.yml file of: ```yaml -- a: banana - c: bananaC -- a: apple - b: appleB2 -- a: dingo - c: dingoC +myArray: + - a: banana + c: bananaC + - a: apple + b: appleB2 + - a: dingo + c: dingoC ``` then ```bash -yq eval-all '(select(fi==1) | .[]) as $two | select(fi==0) | .[] |= (. as $cur | $cur * ($two | select(.a == $cur.a)))' sample.yml another.yml +yq eval-all ' +( + ((.myArray[] | {.a: .}) as $item ireduce ({}; . * $item )) as $uniqueMap + | ( $uniqueMap | to_entries | .[]) as $item ireduce([]; . + $item.value) +) as $mergedArray +| select(fi == 0) | .myArray = $mergedArray +' sample.yml another.yml ``` will output ```yaml -- a: apple - b: appleB2 -- a: kiwi - b: kiwiB -- a: banana - b: bananaB - c: bananaC +myArray: + - a: apple + b: appleB2 + - a: kiwi + b: kiwiB + - a: banana + b: bananaB + c: bananaC + - a: dingo + c: dingoC +something: else ``` ## Merge to prefix an element diff --git a/pkg/yqlib/doc/string-operators.md b/pkg/yqlib/doc/string-operators.md index dafe6f30..ec6ef53f 100644 --- a/pkg/yqlib/doc/string-operators.md +++ b/pkg/yqlib/doc/string-operators.md @@ -209,7 +209,7 @@ will output ``` ## Test using regex -Like jq'q equivalant, this works like match but only returns true/false instead of full match details +Like jq'q equivalent, this works like match but only returns true/false instead of full match details Given a sample.yml file of: ```yaml diff --git a/pkg/yqlib/expression_postfix.go b/pkg/yqlib/expression_postfix.go index e237f888..f9938680 100644 --- a/pkg/yqlib/expression_postfix.go +++ b/pkg/yqlib/expression_postfix.go @@ -68,7 +68,7 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope } // now we should have [ as the last element on the opStack, get rid of it opStack = opStack[0 : len(opStack)-1] - log.Debugf("deleteing open bracket from opstack") + log.Debugf("deleting open bracket from opstack") //and append a collect to the result // hack - see if there's the optional traverse flag diff --git a/pkg/yqlib/file_utils.go b/pkg/yqlib/file_utils.go index 7668a8e1..7a423720 100644 --- a/pkg/yqlib/file_utils.go +++ b/pkg/yqlib/file_utils.go @@ -34,7 +34,7 @@ func tryRemoveFile(filename string) { // thanks https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang func copyFileContents(src, dst string) (err error) { - // ignore CWE-22 gosec issue - that's more targetted for http based apps that run in a public directory, + // ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory, // and ensuring that it's not possible to give a path to a file outside thar directory. in, err := os.Open(src) // #nosec diff --git a/pkg/yqlib/operator_add.go b/pkg/yqlib/operator_add.go index 1473bbfc..0a141ab6 100644 --- a/pkg/yqlib/operator_add.go +++ b/pkg/yqlib/operator_add.go @@ -2,7 +2,6 @@ package yqlib import ( "fmt" - "strconv" yaml "gopkg.in/yaml.v3" diff --git a/pkg/yqlib/operator_comments.go b/pkg/yqlib/operator_comments.go index 9671bc50..ebc78b9d 100644 --- a/pkg/yqlib/operator_comments.go +++ b/pkg/yqlib/operator_comments.go @@ -1,8 +1,6 @@ package yqlib import ( - // "bufio" - // "bytes" "bufio" "bytes" "container/list" diff --git a/pkg/yqlib/operator_equals_test.go b/pkg/yqlib/operator_equals_test.go index 1064ebd3..69cd6e09 100644 --- a/pkg/yqlib/operator_equals_test.go +++ b/pkg/yqlib/operator_equals_test.go @@ -173,7 +173,7 @@ var equalsOperatorScenarios = []expressionScenario{ }, }, { - description: "Two non existant keys are equal", + description: "Two non existent keys are equal", document: "a: frog", expression: `select(.b == .c)`, expected: []string{ diff --git a/pkg/yqlib/operator_load.go b/pkg/yqlib/operator_load.go index 2c3be798..4741b09a 100644 --- a/pkg/yqlib/operator_load.go +++ b/pkg/yqlib/operator_load.go @@ -15,7 +15,7 @@ type loadPrefs struct { } func loadString(filename string) (*CandidateNode, error) { - // ignore CWE-22 gosec issue - that's more targetted for http based apps that run in a public directory, + // ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory, // and ensuring that it's not possible to give a path to a file outside that directory. filebytes, err := ioutil.ReadFile(filename) // #nosec diff --git a/pkg/yqlib/operator_multiply.go b/pkg/yqlib/operator_multiply.go index d7bec73d..ff0410e9 100644 --- a/pkg/yqlib/operator_multiply.go +++ b/pkg/yqlib/operator_multiply.go @@ -1,11 +1,10 @@ package yqlib import ( + "container/list" "fmt" "strconv" - "container/list" - "github.com/jinzhu/copier" yaml "gopkg.in/yaml.v3" ) @@ -42,7 +41,7 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex leadingContent, headComment, footComment := getComments(lhs, rhs) lhs.Node = unwrapDoc(lhs.Node) rhs.Node = unwrapDoc(rhs.Node) - log.Debugf("Multipling LHS: %v", lhs.Node.Tag) + log.Debugf("Multiplying LHS: %v", lhs.Node.Tag) log.Debugf("- RHS: %v", rhs.Node.Tag) if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode || diff --git a/pkg/yqlib/operator_multiply_test.go b/pkg/yqlib/operator_multiply_test.go index b1355075..0ad67299 100644 --- a/pkg/yqlib/operator_multiply_test.go +++ b/pkg/yqlib/operator_multiply_test.go @@ -29,10 +29,22 @@ var mergeArrayWithAnchors = `sample: - <<: *a ` -var mergeArraysObjectKeysText = `It's a complex command, the trickyness comes from needing to have the right context in the expressions. -First we save the second array into a variable '$two' which lets us reference it later. -We then need to update the first array. We will use the relative update (|=) because we need to update relative to the current element of the array in the LHS in the RHS expression. -We set the current element of the first array as $cur. Now we multiply (merge) $cur with the matching entry in $two, by passing $two through a select filter. +var mergeArraysObjectKeysText = `There are several parts of the complex expression. +The first part is doing the hard work, it creates a map from the arrays keyed by '.a', so that there are no duplicates. +Then there's another reduce that converts that map back to an array. +Finally, we set the result of the merged array back into the first doc. + +To use this, you will need to update '.myArray' to be the expression to your array (e.g. .my.array), and '.a' to be the key field of your array (e.g. '.name') + +Thanks Kev from [stackoverflow](https://stackoverflow.com/a/70109529/1168223) +` + +var mergeExpression = ` +( + ((.myArray[] | {.a: .}) as $item ireduce ({}; . * $item )) as $uniqueMap + | ( $uniqueMap | to_entries | .[]) as $item ireduce([]; . + $item.value) +) as $mergedArray +| select(fi == 0) | .myArray = $mergedArray ` var docWithHeader = `# here @@ -371,11 +383,11 @@ var multiplyOperatorScenarios = []expressionScenario{ { description: "Merge arrays of objects together, matching on a key", subdescription: mergeArraysObjectKeysText, - document: `[{a: apple, b: appleB}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB}]`, - document2: `[{a: banana, c: bananaC}, {a: apple, b: appleB2}, {a: dingo, c: dingoC}]`, - expression: `(select(fi==1) | .[]) as $two | select(fi==0) | .[] |= (. as $cur | $cur * ($two | select(.a == $cur.a)))`, + document: `{myArray: [{a: apple, b: appleB}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB}], something: else}`, + document2: `myArray: [{a: banana, c: bananaC}, {a: apple, b: appleB2}, {a: dingo, c: dingoC}]`, + expression: mergeExpression, expected: []string{ - "D0, P[], (doc)::[{a: apple, b: appleB2}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB, c: bananaC}]\n", + "D0, P[], (doc)::{myArray: [{a: apple, b: appleB2}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB, c: bananaC}, {a: dingo, c: dingoC}], something: else}\n", }, }, { diff --git a/pkg/yqlib/operator_strings_test.go b/pkg/yqlib/operator_strings_test.go index f1218010..c94a2fc8 100644 --- a/pkg/yqlib/operator_strings_test.go +++ b/pkg/yqlib/operator_strings_test.go @@ -99,7 +99,7 @@ var stringsOperatorScenarios = []expressionScenario{ }, { description: "Test using regex", - subdescription: "Like jq'q equivalant, this works like match but only returns true/false instead of full match details", + subdescription: "Like jq'q equivalent, this works like match but only returns true/false instead of full match details", document: `["cat", "dog"]`, expression: `.[] | test("at")`, expected: []string{ diff --git a/pkg/yqlib/operator_subtract.go b/pkg/yqlib/operator_subtract.go index 6a74ab27..51d68cbf 100644 --- a/pkg/yqlib/operator_subtract.go +++ b/pkg/yqlib/operator_subtract.go @@ -2,7 +2,6 @@ package yqlib import ( "fmt" - "strconv" "gopkg.in/yaml.v3" diff --git a/pkg/yqlib/operator_traverse_path.go b/pkg/yqlib/operator_traverse_path.go index e02c388e..08c36e5b 100644 --- a/pkg/yqlib/operator_traverse_path.go +++ b/pkg/yqlib/operator_traverse_path.go @@ -42,7 +42,7 @@ func traverse(d *dataTreeNavigator, context Context, matchingNode *CandidateNode if value.Tag == "!!null" && operation.Value != "[]" { log.Debugf("Guessing kind") - // we must ahve added this automatically, lets guess what it should be now + // we must have added this automatically, lets guess what it should be now switch operation.Value.(type) { case int, int64: log.Debugf("probably an array") @@ -87,7 +87,7 @@ func traverseArrayOperator(d *dataTreeNavigator, context Context, expressionNode return Context{}, err } - // rhs is a collect expression that will yield indexes to retreive of the arrays + // rhs is a collect expression that will yield indexes to retrieve of the arrays rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs) diff --git a/pkg/yqlib/utils.go b/pkg/yqlib/utils.go index f992116e..aa514770 100644 --- a/pkg/yqlib/utils.go +++ b/pkg/yqlib/utils.go @@ -17,7 +17,7 @@ func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, s if filename == "-" { reader = bufio.NewReader(os.Stdin) } else { - // ignore CWE-22 gosec issue - that's more targetted for http based apps that run in a public directory, + // ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory, // and ensuring that it's not possible to give a path to a file outside thar directory. file, err := os.Open(filename) // #nosec if err != nil { diff --git a/scripts/devtools.sh b/scripts/devtools.sh index 4e7126c2..89c895c3 100755 --- a/scripts/devtools.sh +++ b/scripts/devtools.sh @@ -1,5 +1,5 @@ #!/bin/sh set -ex -go mod download golang.org/x/tools -wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.37.1 +go mod download golang.org/x/tools@latest +wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.43.0 wget -O- -nv https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.9.1