Merge remote-tracking branch 'source/master' into carolynvs-add-log-accessor

.golangci.yml

@@ -3,9 +3,11 @@ run:
 linters:
   enable:
     - errorlint
+    - gci
     - gofmt
     - goimports
     - gosec
+    - misspell
 issues:
   exclude-rules:
   - linters:

1

@@ -1 +0,0 @@
-Error: cannot pass files in when using null-input flag

README.md

@@ -220,6 +220,7 @@ Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq)
 - [Update yaml inplace](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags)
 - [Complex expressions to select and update](https://mikefarah.gitbook.io/yq/operators/select#select-and-update-matching-values-in-map)
 - Keeps yaml formatting and comments when updating (though there are issues with whitespace)
+- [Load content from other files](https://mikefarah.gitbook.io/yq/operators/load)
 - [Convert to/from json](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert)
 - [Convert to properties](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties)
 - [Pipe data in by using '-'](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate)

cmd/utils.go

@@ -25,7 +25,7 @@ func initCommand(cmd *cobra.Command, args []string) (firstFileIndex int, err err
 		firstFileIndex = 1
 	}
 
-	// backwards compatibilty
+	// backwards compatibility
 	if outputToJSON {
 		outputFormat = "json"
 	}

go.sum

@@ -471,7 +471,6 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
-golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA=
 golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

merge-array-by-key.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+# ./yq ea '.[]'  examples/data*.yaml
+
+
+./yq ea '
+  ((.[] | {.name: .}) as $item ireduce ({}; . * $item )) as $uniqueMap
+  | ( $uniqueMap  | to_entries | .[]) as $item ireduce([]; . + $item.value)
+' examples/data*.yaml

pkg/yqlib/doc/equals.md

@@ -108,7 +108,7 @@ will output
 a: frog
 ```
 
-## Two non existant keys are equal
+## Two non existent keys are equal
 Given a sample.yml file of:
 ```yaml
 a: frog

pkg/yqlib/doc/multiply-merge.md

@@ -234,43 +234,60 @@ will output
 ```
 
 ## Merge arrays of objects together, matching on a key
-It's a complex command, the trickyness comes from needing to have the right context in the expressions.
-First we save the second array into a variable '$two' which lets us reference it later.
-We then need to update the first array. We will use the relative update (|=) because we need to update relative to the current element of the array in the LHS in the RHS expression. 
-We set the current element of the first array as $cur. Now we multiply (merge) $cur with the matching entry in $two, by passing $two through a select filter.
+There are several parts of the complex expression. 
+The first part is doing the hard work, it creates a map from the arrays keyed by '.a', so that there are no duplicates. 
+Then there's another reduce that converts that map back to an array.
+Finally, we set the result of the merged array back into the first doc.
+
+To use this, you will need to update '.myArray' to be the expression to your array (e.g. .my.array), and '.a' to be the key field of your array (e.g. '.name')
+
+Thanks Kev from [stackoverflow](https://stackoverflow.com/a/70109529/1168223)
 
 
 Given a sample.yml file of:
 ```yaml
-- a: apple
-  b: appleB
-- a: kiwi
-  b: kiwiB
-- a: banana
-  b: bananaB
+myArray:
+  - a: apple
+    b: appleB
+  - a: kiwi
+    b: kiwiB
+  - a: banana
+    b: bananaB
+something: else
 ```
 And another sample another.yml file of:
 ```yaml
-- a: banana
-  c: bananaC
-- a: apple
-  b: appleB2
-- a: dingo
-  c: dingoC
+myArray:
+  - a: banana
+    c: bananaC
+  - a: apple
+    b: appleB2
+  - a: dingo
+    c: dingoC
 ```
 then
 ```bash
-yq eval-all '(select(fi==1) | .[]) as $two | select(fi==0) | .[] |= (. as $cur |  $cur * ($two | select(.a == $cur.a)))' sample.yml another.yml
+yq eval-all '
+(
+  ((.myArray[] | {.a: .}) as $item ireduce ({}; . * $item )) as $uniqueMap
+  | ( $uniqueMap  | to_entries | .[]) as $item ireduce([]; . + $item.value)
+) as $mergedArray
+| select(fi == 0) | .myArray = $mergedArray
+' sample.yml another.yml
 ```
 will output
 ```yaml
-- a: apple
-  b: appleB2
-- a: kiwi
-  b: kiwiB
-- a: banana
-  b: bananaB
-  c: bananaC
+myArray:
+  - a: apple
+    b: appleB2
+  - a: kiwi
+    b: kiwiB
+  - a: banana
+    b: bananaB
+    c: bananaC
+  - a: dingo
+    c: dingoC
+something: else
 ```
 
 ## Merge to prefix an element

pkg/yqlib/doc/string-operators.md

@@ -209,7 +209,7 @@ will output
 ```
 
 ## Test using regex
-Like jq'q equivalant, this works like match but only returns true/false instead of full match details
+Like jq'q equivalent, this works like match but only returns true/false instead of full match details
 
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/expression_postfix.go

@@ -68,7 +68,7 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
 			}
 			// now we should have [ as the last element on the opStack, get rid of it
 			opStack = opStack[0 : len(opStack)-1]
-			log.Debugf("deleteing open bracket from opstack")
+			log.Debugf("deleting open bracket from opstack")
 
 			//and append a collect to the result
 			// hack - see if there's the optional traverse flag

pkg/yqlib/file_utils.go

@@ -34,7 +34,7 @@ func tryRemoveFile(filename string) {
 
 // thanks https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang
 func copyFileContents(src, dst string) (err error) {
-	// ignore CWE-22 gosec issue - that's more targetted for http based apps that run in a public directory,
+	// ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory,
 	// and ensuring that it's not possible to give a path to a file outside thar directory.
 
 	in, err := os.Open(src) // #nosec

pkg/yqlib/operator_add.go

@@ -2,7 +2,6 @@ package yqlib
 
 import (
 	"fmt"
-
 	"strconv"
 
 	yaml "gopkg.in/yaml.v3"

pkg/yqlib/operator_comments.go

@@ -1,8 +1,6 @@
 package yqlib
 
 import (
-	// "bufio"
-	// "bytes"
 	"bufio"
 	"bytes"
 	"container/list"

pkg/yqlib/operator_equals_test.go

@@ -173,7 +173,7 @@ var equalsOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description: "Two non existant keys are equal",
+		description: "Two non existent keys are equal",
 		document:    "a: frog",
 		expression:  `select(.b == .c)`,
 		expected: []string{

pkg/yqlib/operator_load.go

@@ -15,7 +15,7 @@ type loadPrefs struct {
 }
 
 func loadString(filename string) (*CandidateNode, error) {
-	// ignore CWE-22 gosec issue - that's more targetted for http based apps that run in a public directory,
+	// ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory,
 	// and ensuring that it's not possible to give a path to a file outside that directory.
 
 	filebytes, err := ioutil.ReadFile(filename) // #nosec

pkg/yqlib/operator_multiply.go

@@ -1,11 +1,10 @@
 package yqlib
 
 import (
+	"container/list"
 	"fmt"
 	"strconv"
 
-	"container/list"
-
 	"github.com/jinzhu/copier"
 	yaml "gopkg.in/yaml.v3"
 )
@@ -42,7 +41,7 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
 		leadingContent, headComment, footComment := getComments(lhs, rhs)
 		lhs.Node = unwrapDoc(lhs.Node)
 		rhs.Node = unwrapDoc(rhs.Node)
-		log.Debugf("Multipling LHS: %v", lhs.Node.Tag)
+		log.Debugf("Multiplying LHS: %v", lhs.Node.Tag)
 		log.Debugf("-          RHS: %v", rhs.Node.Tag)
 
 		if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode ||

pkg/yqlib/operator_multiply_test.go

@@ -29,10 +29,22 @@ var mergeArrayWithAnchors = `sample:
 - <<: *a
 `
 
-var mergeArraysObjectKeysText = `It's a complex command, the trickyness comes from needing to have the right context in the expressions.
-First we save the second array into a variable '$two' which lets us reference it later.
-We then need to update the first array. We will use the relative update (|=) because we need to update relative to the current element of the array in the LHS in the RHS expression. 
-We set the current element of the first array as $cur. Now we multiply (merge) $cur with the matching entry in $two, by passing $two through a select filter.
+var mergeArraysObjectKeysText = `There are several parts of the complex expression. 
+The first part is doing the hard work, it creates a map from the arrays keyed by '.a', so that there are no duplicates. 
+Then there's another reduce that converts that map back to an array.
+Finally, we set the result of the merged array back into the first doc.
+
+To use this, you will need to update '.myArray' to be the expression to your array (e.g. .my.array), and '.a' to be the key field of your array (e.g. '.name')
+
+Thanks Kev from [stackoverflow](https://stackoverflow.com/a/70109529/1168223)
+`
+
+var mergeExpression = `
+(
+  ((.myArray[] | {.a: .}) as $item ireduce ({}; . * $item )) as $uniqueMap
+  | ( $uniqueMap  | to_entries | .[]) as $item ireduce([]; . + $item.value)
+) as $mergedArray
+| select(fi == 0) | .myArray = $mergedArray
 `
 
 var docWithHeader = `# here
@@ -371,11 +383,11 @@ var multiplyOperatorScenarios = []expressionScenario{
 	{
 		description:    "Merge arrays of objects together, matching on a key",
 		subdescription: mergeArraysObjectKeysText,
-		document:       `[{a: apple, b: appleB}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB}]`,
-		document2:      `[{a: banana, c: bananaC}, {a: apple, b: appleB2}, {a: dingo, c: dingoC}]`,
-		expression:     `(select(fi==1) | .[]) as $two | select(fi==0) | .[] |= (. as $cur |  $cur * ($two | select(.a == $cur.a)))`,
+		document:       `{myArray: [{a: apple, b: appleB}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB}], something: else}`,
+		document2:      `myArray: [{a: banana, c: bananaC}, {a: apple, b: appleB2}, {a: dingo, c: dingoC}]`,
+		expression:     mergeExpression,
 		expected: []string{
-			"D0, P[], (doc)::[{a: apple, b: appleB2}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB, c: bananaC}]\n",
+			"D0, P[], (doc)::{myArray: [{a: apple, b: appleB2}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB, c: bananaC}, {a: dingo, c: dingoC}], something: else}\n",
 		},
 	},
 	{

pkg/yqlib/operator_strings_test.go

@@ -99,7 +99,7 @@ var stringsOperatorScenarios = []expressionScenario{
 	},
 	{
 		description:    "Test using regex",
-		subdescription: "Like jq'q equivalant, this works like match but only returns true/false instead of full match details",
+		subdescription: "Like jq'q equivalent, this works like match but only returns true/false instead of full match details",
 		document:       `["cat", "dog"]`,
 		expression:     `.[] | test("at")`,
 		expected: []string{

pkg/yqlib/operator_subtract.go

@@ -2,7 +2,6 @@ package yqlib
 
 import (
 	"fmt"
-
 	"strconv"
 
 	"gopkg.in/yaml.v3"

pkg/yqlib/operator_traverse_path.go

@@ -42,7 +42,7 @@ func traverse(d *dataTreeNavigator, context Context, matchingNode *CandidateNode
 
 	if value.Tag == "!!null" && operation.Value != "[]" {
 		log.Debugf("Guessing kind")
-		// we must ahve added this automatically, lets guess what it should be now
+		// we must have added this automatically, lets guess what it should be now
 		switch operation.Value.(type) {
 		case int, int64:
 			log.Debugf("probably an array")
@@ -87,7 +87,7 @@ func traverseArrayOperator(d *dataTreeNavigator, context Context, expressionNode
 		return Context{}, err
 	}
 
-	// rhs is a collect expression that will yield indexes to retreive of the arrays
+	// rhs is a collect expression that will yield indexes to retrieve of the arrays
 
 	rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 

pkg/yqlib/utils.go

@@ -17,7 +17,7 @@ func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, s
 	if filename == "-" {
 		reader = bufio.NewReader(os.Stdin)
 	} else {
-		// ignore CWE-22 gosec issue - that's more targetted for http based apps that run in a public directory,
+		// ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory,
 		// and ensuring that it's not possible to give a path to a file outside thar directory.
 		file, err := os.Open(filename) // #nosec
 		if err != nil {

scripts/devtools.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
 set -ex
-go mod download golang.org/x/tools
-wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.37.1
+go mod download golang.org/x/tools@latest
+wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.43.0
 wget -O- -nv https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.9.1