Lint

.golangci.yml

@@ -18,14 +18,6 @@ linters:
     - revive
     - unconvert
     - unparam
-linters-settings:
-  depguard:
-    list-type: blacklist
-    include-go-root: true
-    packages:
-      - io/ioutil
-    packages-with-error-message:
-      - io/ioutil: "The 'io/ioutil' package is deprecated. Use corresponding 'os' or 'io' functions instead."
 issues:
   exclude-rules:
   - linters:

README.md

@@ -104,9 +104,6 @@ rm /etc/myfile.tmp
 docker run --rm -v "${PWD}":/workdir mikefarah/yq <command> [flags] [expression ]FILE...
 ```
 
-Note that you can run `yq` in docker without network access and other privileges if you desire,
-namely `--security-opt=no-new-privileges --cap-drop all --network none`.
-
 ```bash
 podman run --rm -v "${PWD}":/workdir mikefarah/yq <command> [flags] [expression ]FILE...
 ```

cmd/evalute_sequence_command.go

@@ -44,11 +44,11 @@ expression and prints the result in sequence.`,
 }
 
 func processExpression(expression string) string {
-
+	var prettyPrintExp = `(... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not))  ) style=""`
 	if prettyPrint && expression == "" {
-		return yqlib.PrettyPrintExp
+		return prettyPrintExp
 	} else if prettyPrint {
-		return fmt.Sprintf("%v | %v", expression, yqlib.PrettyPrintExp)
+		return fmt.Sprintf("%v | %v", expression, prettyPrintExp)
 	}
 	return expression
 }

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.17.2"
+	Version = "4.16.2"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

go.mod

@@ -4,7 +4,7 @@ require (
 	github.com/elliotchance/orderedmap v1.4.0
 	github.com/fatih/color v1.13.0
 	github.com/goccy/go-yaml v1.9.5
-	github.com/jinzhu/copier v0.3.5
+	github.com/jinzhu/copier v0.3.4
 	github.com/magiconair/properties v1.8.5
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
 	github.com/spf13/cobra v1.3.0

go.sum

@@ -235,8 +235,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg=
-github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
+github.com/jinzhu/copier v0.3.4 h1:mfU6jI9PtCeUjkjQ322dlff9ELjGDu975C2p/nrubVI=
+github.com/jinzhu/copier v0.3.4/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=

pkg/yqlib/doc/operators/headers/multiply-merge.md

@@ -10,10 +10,9 @@ Note that when merging objects, this operator returns the merged object (not the
 ### Merge Flags
 You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`.  See examples below
 
-- `+` append arrays
-- `d` deeply merge arrays
-- `?` only merge _existing_ fields
-- `n` only merge _new_ fields
+- `+` to append arrays
+- `?` to only merge existing fields
+- `d` to deeply merge arrays
 
 ### Merging files
 Note the use of `eval-all` to ensure all documents are loaded into memory.

pkg/yqlib/doc/operators/multiply-merge.md

@@ -10,10 +10,9 @@ Note that when merging objects, this operator returns the merged object (not the
 ### Merge Flags
 You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`.  See examples below
 
-- `+` append arrays
-- `d` deeply merge arrays
-- `?` only merge _existing_ fields
-- `n` only merge _new_ fields
+- `+` to append arrays
+- `?` to only merge existing fields
+- `d` to deeply merge arrays
 
 ### Merging files
 Note the use of `eval-all` to ensure all documents are loaded into memory.
@@ -149,27 +148,6 @@ thing: two
 cat: frog
 ```
 
-## Merge, only new fields
-Given a sample.yml file of:
-```yaml
-a:
-  thing: one
-  cat: frog
-b:
-  missing: two
-  thing: two
-```
-then
-```bash
-yq eval '.a *n .b' sample.yml
-```
-will output
-```yaml
-thing: one
-cat: frog
-missing: two
-```
-
 ## Merge, appending arrays
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/operators/select.md

@@ -2,7 +2,7 @@
 
 Select is used to filter arrays and maps by a boolean expression.
 
-## Select elements from array using wildcard prefix
+## Select elements from array
 Given a sample.yml file of:
 ```yaml
 - cat
@@ -19,99 +19,7 @@ cat
 goat
 ```
 
-## Select elements from array using wildcard suffix
-Given a sample.yml file of:
-```yaml
-- go-kart
-- goat
-- dog
-```
-then
-```bash
-yq eval '.[] | select(. == "go*")' sample.yml
-```
-will output
-```yaml
-go-kart
-goat
-```
-
-## Select elements from array using wildcard prefix and suffix
-Given a sample.yml file of:
-```yaml
-- ago
-- go
-- meow
-- going
-```
-then
-```bash
-yq eval '.[] | select(. == "*go*")' sample.yml
-```
-will output
-```yaml
-ago
-go
-going
-```
-
-## Select elements from array with regular expression
-See more regular expression examples under the `string` operator docs.
-
-Given a sample.yml file of:
-```yaml
-- this_0
-- not_this
-- nor_0_this
-- thisTo_4
-```
-then
-```bash
-yq eval '.[] | select(test("[a-zA-Z]+_[0-9]$"))' sample.yml
-```
-will output
-```yaml
-this_0
-thisTo_4
-```
-
-## Select items from a map
-Given a sample.yml file of:
-```yaml
-things: cat
-bob: goat
-horse: dog
-```
-then
-```bash
-yq eval '.[] | select(. == "cat" or test("og$"))' sample.yml
-```
-will output
-```yaml
-cat
-dog
-```
-
-## Use select and with_entries to filter map keys
-Given a sample.yml file of:
-```yaml
-name: bob
-legs: 2
-game: poker
-```
-then
-```bash
-yq eval 'with_entries(select(.key | test("ame$")))' sample.yml
-```
-will output
-```yaml
-name: bob
-game: poker
-```
-
-## Select multiple items in a map and update
-Note the brackets around the entire LHS.
-
+## Select and update matching values in map
 Given a sample.yml file of:
 ```yaml
 a:
@@ -121,7 +29,7 @@ a:
 ```
 then
 ```bash
-yq eval '(.a.[] | select(. == "cat" or . == "goat")) |= "rabbit"' sample.yml
+yq eval '(.a.[] | select(. == "*at")) |= "rabbit"' sample.yml
 ```
 will output
 ```yaml

pkg/yqlib/doc/usage/convert.md

@@ -1,126 +0,0 @@
-# JSON
-
-Encode and decode to and from JSON. Note that YAML is a _superset_ of JSON - so `yq` can read any json file without doing anything special.
-
-This means you don't need to 'convert' a JSON file to YAML - however if you want idiomatic YAML styling, then you can use the `-P/--prettyPrint` flag, see examples below.
-
-## Parse json: simple
-JSON is a subset of yaml, so all you need to do is prettify the output
-
-Given a sample.json file of:
-```json
-{"cat": "meow"}
-```
-then
-```bash
-yq e -P '.' sample.json
-```
-will output
-```yaml
-cat: meow
-```
-
-## Parse json: complex
-JSON is a subset of yaml, so all you need to do is prettify the output
-
-Given a sample.json file of:
-```json
-{"a":"Easy! as one two three","b":{"c":2,"d":[3,4]}}
-```
-then
-```bash
-yq e -P '.' sample.json
-```
-will output
-```yaml
-a: Easy! as one two three
-b:
-  c: 2
-  d:
-    - 3
-    - 4
-```
-
-## Encode json: simple
-Given a sample.yml file of:
-```yaml
-cat: meow
-```
-then
-```bash
-yq e -o=json '.' sample.yml
-```
-will output
-```json
-{
-  "cat": "meow"
-}
-```
-
-## Encode json: simple - in one line
-Given a sample.yml file of:
-```yaml
-cat: meow # this is a comment, and it will be dropped.
-```
-then
-```bash
-yq e -o=json -I=0 '.' sample.yml
-```
-will output
-```json
-{"cat":"meow"}
-```
-
-## Encode json: comments
-Given a sample.yml file of:
-```yaml
-cat: meow # this is a comment, and it will be dropped.
-```
-then
-```bash
-yq e -o=json '.' sample.yml
-```
-will output
-```json
-{
-  "cat": "meow"
-}
-```
-
-## Encode json: anchors
-Anchors are dereferenced
-
-Given a sample.yml file of:
-```yaml
-cat: &ref meow
-anotherCat: *ref
-```
-then
-```bash
-yq e -o=json '.' sample.yml
-```
-will output
-```json
-{
-  "cat": "meow",
-  "anotherCat": "meow"
-}
-```
-
-## Encode json: multiple results
-Each matching node is converted into a json doc. This is best used with 0 indent (json document per line)
-
-Given a sample.yml file of:
-```yaml
-things: [{stuff: cool}, {whatever: cat}]
-```
-then
-```bash
-yq e -o=json -I=0 '.things[]' sample.yml
-```
-will output
-```json
-{"stuff":"cool"}
-{"whatever":"cat"}
-```
-

pkg/yqlib/doc/usage/headers/convert.md

@@ -1,5 +0,0 @@
-# JSON
-
-Encode and decode to and from JSON. Note that YAML is a _superset_ of JSON - so `yq` can read any json file without doing anything special.
-
-This means you don't need to 'convert' a JSON file to YAML - however if you want idiomatic YAML styling, then you can use the `-P/--prettyPrint` flag, see examples below.

pkg/yqlib/expression_tokeniser.go

@@ -110,9 +110,6 @@ func multiplyWithPrefs() lex.Action {
 		if strings.Contains(options, "?") {
 			prefs.TraversePrefs = traversePreferences{DontAutoCreate: true}
 		}
-		if strings.Contains(options, "n") {
-			prefs.AssignPrefs = assignPreferences{OnlyWriteNull: true}
-		}
 		if strings.Contains(options, "d") {
 			prefs.DeepMergeArrays = true
 		}
@@ -476,7 +473,7 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`\]\??`), literalToken(closeCollect, true))
 	lexer.Add([]byte(`\{`), literalToken(openCollectObject, false))
 	lexer.Add([]byte(`\}`), literalToken(closeCollectObject, true))
-	lexer.Add([]byte(`\*[\+|\?dn]*`), multiplyWithPrefs())
+	lexer.Add([]byte(`\*[\+|\?d]*`), multiplyWithPrefs())
 	lexer.Add([]byte(`\+`), opToken(addOpType))
 	lexer.Add([]byte(`\+=`), opToken(addAssignOpType))
 	lexer.Add([]byte(`\-`), opToken(subtractOpType))

pkg/yqlib/file_utils.go

@@ -3,6 +3,7 @@ package yqlib
 import (
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 )
 
@@ -76,7 +77,7 @@ func createTempFile() (*os.File, error) {
 		return nil, err
 	}
 
-	file, err := os.CreateTemp("", "temp")
+	file, err := ioutil.TempFile("", "temp")
 	if err != nil {
 		return nil, err
 	}

pkg/yqlib/front_matter_test.go

@@ -1,8 +1,7 @@
 package yqlib
 
 import (
-	"io"
-	"os"
+	"io/ioutil"
 	"testing"
 
 	"github.com/mikefarah/yq/v4/test"
@@ -25,7 +24,7 @@ func createTestFile(content string) string {
 }
 
 func readFile(filename string) string {
-	bytes, err := os.ReadFile(filename)
+	bytes, err := ioutil.ReadFile(filename)
 	if err != nil {
 		panic(err)
 	}
@@ -61,7 +60,7 @@ yaml: doc
 
 	test.AssertResult(t, expectedYamlFm, yamlFm)
 
-	contentBytes, err := io.ReadAll(fmHandler.GetContentReader())
+	contentBytes, err := ioutil.ReadAll(fmHandler.GetContentReader())
 	if err != nil {
 		panic(err)
 	}
@@ -98,7 +97,7 @@ yaml: doc
 
 	test.AssertResult(t, expectedYamlFm, yamlFm)
 
-	contentBytes, err := io.ReadAll(fmHandler.GetContentReader())
+	contentBytes, err := ioutil.ReadAll(fmHandler.GetContentReader())
 	if err != nil {
 		panic(err)
 	}
@@ -132,7 +131,7 @@ yaml: doc
 
 	test.AssertResult(t, expectedYamlFm, yamlFm)
 
-	contentBytes, err := io.ReadAll(fmHandler.GetContentReader())
+	contentBytes, err := ioutil.ReadAll(fmHandler.GetContentReader())
 	if err != nil {
 		panic(err)
 	}

pkg/yqlib/json_test.go

@@ -1,229 +0,0 @@
-package yqlib
-
-import (
-	"bufio"
-	"bytes"
-	"fmt"
-	"strings"
-	"testing"
-
-	"github.com/mikefarah/yq/v4/test"
-)
-
-var complexExpectYaml = `D0, P[], (!!map)::a: Easy! as one two three
-b:
-    c: 2
-    d:
-        - 3
-        - 4
-`
-
-var jsonScenarios = []formatScenario{
-	{
-		description:    "Parse json: simple",
-		subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output",
-		input:          `{"cat": "meow"}`,
-		expected:       "D0, P[], (!!map)::cat: meow\n",
-	},
-	{
-		description:    "Parse json: complex",
-		subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output",
-		input:          `{"a":"Easy! as one two three","b":{"c":2,"d":[3,4]}}`,
-		expected:       complexExpectYaml,
-	},
-	{
-		description:  "Encode json: simple",
-		input:        `cat: meow`,
-		indent:       2,
-		expected:     "{\n  \"cat\": \"meow\"\n}\n",
-		scenarioType: "encode",
-	},
-	{
-		description:  "Encode json: simple - in one line",
-		input:        `cat: meow # this is a comment, and it will be dropped.`,
-		indent:       0,
-		expected:     "{\"cat\":\"meow\"}\n",
-		scenarioType: "encode",
-	},
-	{
-		description:  "Encode json: comments",
-		input:        `cat: meow # this is a comment, and it will be dropped.`,
-		indent:       2,
-		expected:     "{\n  \"cat\": \"meow\"\n}\n",
-		scenarioType: "encode",
-	},
-	{
-		description:    "Encode json: anchors",
-		subdescription: "Anchors are dereferenced",
-		input:          "cat: &ref meow\nanotherCat: *ref",
-		indent:         2,
-		expected:       "{\n  \"cat\": \"meow\",\n  \"anotherCat\": \"meow\"\n}\n",
-		scenarioType:   "encode",
-	},
-	{
-		description:    "Encode json: multiple results",
-		subdescription: "Each matching node is converted into a json doc. This is best used with 0 indent (json document per line)",
-		input:          `things: [{stuff: cool}, {whatever: cat}]`,
-		expression:     `.things[]`,
-		indent:         0,
-		expected:       "{\"stuff\":\"cool\"}\n{\"whatever\":\"cat\"}\n",
-		scenarioType:   "encode",
-	},
-}
-
-func decodeJson(t *testing.T, jsonString string) *CandidateNode {
-	docs, err := readDocumentWithLeadingContent(jsonString, "sample.json", 0)
-
-	if err != nil {
-		t.Error(err)
-		return nil
-	}
-
-	exp, err := NewExpressionParser().ParseExpression(PrettyPrintExp)
-
-	if err != nil {
-		t.Error(err)
-		return nil
-	}
-
-	context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: docs}, exp)
-
-	if err != nil {
-		t.Error(err)
-		return nil
-	}
-
-	return context.MatchingNodes.Front().Value.(*CandidateNode)
-}
-
-func testJsonScenario(t *testing.T, s formatScenario) {
-	if s.scenarioType == "encode" || s.scenarioType == "roundtrip" {
-		test.AssertResultWithContext(t, s.expected, processJsonScenario(s), s.description)
-	} else {
-		var actual = resultToString(t, decodeJson(t, s.input))
-		test.AssertResultWithContext(t, s.expected, actual, s.description)
-	}
-}
-
-func processJsonScenario(s formatScenario) string {
-
-	var output bytes.Buffer
-	writer := bufio.NewWriter(&output)
-
-	var encoder = NewJsonEncoder(s.indent)
-
-	var decoder = NewYamlDecoder()
-
-	inputs, err := readDocuments(strings.NewReader(s.input), "sample.yml", 0, decoder)
-	if err != nil {
-		panic(err)
-	}
-
-	expression := s.expression
-	if expression == "" {
-		expression = "."
-	}
-
-	exp, err := NewExpressionParser().ParseExpression(expression)
-
-	if err != nil {
-		panic(err)
-	}
-
-	context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: inputs}, exp)
-
-	if err != nil {
-		panic(err)
-	}
-
-	printer := NewPrinter(encoder, NewSinglePrinterWriter(writer))
-	err = printer.PrintResults(context.MatchingNodes)
-	if err != nil {
-		panic(err)
-	}
-	writer.Flush()
-
-	return output.String()
-
-}
-
-func documentJsonDecodeScenario(t *testing.T, w *bufio.Writer, s formatScenario) {
-	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
-
-	if s.subdescription != "" {
-		writeOrPanic(w, s.subdescription)
-		writeOrPanic(w, "\n\n")
-	}
-
-	writeOrPanic(w, "Given a sample.json file of:\n")
-	writeOrPanic(w, fmt.Sprintf("```json\n%v\n```\n", s.input))
-
-	writeOrPanic(w, "then\n")
-	writeOrPanic(w, "```bash\nyq e -P '.' sample.json\n```\n")
-	writeOrPanic(w, "will output\n")
-
-	var output bytes.Buffer
-	printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
-
-	node := decodeJson(t, s.input)
-
-	err := printer.PrintResults(node.AsList())
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", output.String()))
-}
-
-func documentJsonScenario(t *testing.T, w *bufio.Writer, i interface{}) {
-	s := i.(formatScenario)
-
-	if s.skipDoc {
-		return
-	}
-	if s.scenarioType == "encode" {
-		documentJsonEncodeScenario(w, s)
-	} else {
-		documentJsonDecodeScenario(t, w, s)
-	}
-}
-
-func documentJsonEncodeScenario(w *bufio.Writer, s formatScenario) {
-	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
-
-	if s.subdescription != "" {
-		writeOrPanic(w, s.subdescription)
-		writeOrPanic(w, "\n\n")
-	}
-
-	writeOrPanic(w, "Given a sample.yml file of:\n")
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input))
-
-	writeOrPanic(w, "then\n")
-
-	expression := s.expression
-	if expression == "" {
-		expression = "."
-	}
-
-	if s.indent == 2 {
-		writeOrPanic(w, fmt.Sprintf("```bash\nyq e -o=json '%v' sample.yml\n```\n", expression))
-	} else {
-		writeOrPanic(w, fmt.Sprintf("```bash\nyq e -o=json -I=%v '%v' sample.yml\n```\n", s.indent, expression))
-	}
-	writeOrPanic(w, "will output\n")
-
-	writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", processJsonScenario(s)))
-}
-
-func TestJsonScenarios(t *testing.T) {
-	for _, tt := range jsonScenarios {
-		testJsonScenario(t, tt)
-	}
-	genericScenarios := make([]interface{}, len(jsonScenarios))
-	for i, s := range jsonScenarios {
-		genericScenarios[i] = s
-	}
-	documentScenarios(t, "usage", "convert", genericScenarios, documentJsonScenario)
-}

pkg/yqlib/lib.go

@@ -20,8 +20,6 @@ type xmlPreferences struct {
 
 var log = logging.MustGetLogger("yq-lib")
 
-var PrettyPrintExp = `(... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not))  ) style=""`
-
 // GetLogger returns the yq logger instance.
 func GetLogger() *logging.Logger {
 	return log

pkg/yqlib/operator_assign.go

@@ -2,15 +2,13 @@ package yqlib
 
 type assignPreferences struct {
 	DontOverWriteAnchor bool
-	OnlyWriteNull       bool
 }
 
 func assignUpdateFunc(prefs assignPreferences) crossFunctionCalculation {
 	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 		rhs.Node = unwrapDoc(rhs.Node)
-		if !prefs.OnlyWriteNull || lhs.Node.Tag == "!!null" {
-			lhs.UpdateFrom(rhs, prefs)
-		}
+
+		lhs.UpdateFrom(rhs, prefs)
 		return lhs, nil
 	}
 }
@@ -78,9 +76,7 @@ func assignAttributesOperator(d *dataTreeNavigator, context Context, expressionN
 			if expressionNode.Operation.Preferences != nil {
 				prefs = expressionNode.Operation.Preferences.(assignPreferences)
 			}
-			if !prefs.OnlyWriteNull || candidate.Node.Tag == "!!null" {
-				candidate.UpdateAttributesFrom(first.Value.(*CandidateNode), prefs)
-			}
+			candidate.UpdateAttributesFrom(first.Value.(*CandidateNode), prefs)
 		}
 	}
 	return context, nil

pkg/yqlib/operator_load.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"container/list"
 	"fmt"
+	"io/ioutil"
 	"os"
 
 	"gopkg.in/yaml.v3"
@@ -18,7 +19,7 @@ func loadString(filename string) (*CandidateNode, error) {
 	// ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory,
 	// and ensuring that it's not possible to give a path to a file outside that directory.
 
-	filebytes, err := os.ReadFile(filename) // #nosec
+	filebytes, err := ioutil.ReadFile(filename) // #nosec
 	if err != nil {
 		return nil, err
 	}

pkg/yqlib/operator_multiply.go

@@ -13,7 +13,6 @@ type multiplyPreferences struct {
 	AppendArrays    bool
 	DeepMergeArrays bool
 	TraversePrefs   traversePreferences
-	AssignPrefs     assignPreferences
 }
 
 func multiplyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
@@ -141,7 +140,7 @@ func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom
 	lhsPath := rhs.Path[pathIndexToStartFrom:]
 	log.Debugf("merge - lhsPath %v", lhsPath)
 
-	assignmentOp := &Operation{OperationType: assignAttributesOpType, Preferences: preferences.AssignPrefs}
+	assignmentOp := &Operation{OperationType: assignAttributesOpType}
 	if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode {
 		assignmentOp.OperationType = addAssignOpType
 		log.Debugf("merge - assignmentOp.OperationType = addAssignOpType")
@@ -158,11 +157,7 @@ func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom
 	}
 	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs}
 
-	assignmentOpNode := &ExpressionNode{
-		Operation: assignmentOp,
-		Lhs:       createTraversalTree(lhsPath, preferences.TraversePrefs, rhs.IsMapKey),
-		Rhs:       &ExpressionNode{Operation: rhsOp},
-	}
+	assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath, preferences.TraversePrefs, rhs.IsMapKey), Rhs: &ExpressionNode{Operation: rhsOp}}
 
 	_, err := d.GetMatchingNodes(context.SingleChildContext(lhs), assignmentOpNode)
 

pkg/yqlib/operator_multiply_test.go

@@ -344,14 +344,6 @@ var multiplyOperatorScenarios = []expressionScenario{
 			"D0, P[a], (!!map)::{thing: two, cat: frog}\n",
 		},
 	},
-	{
-		description: "Merge, only new fields",
-		document:    `{a: {thing: one, cat: frog}, b: {missing: two, thing: two}}`,
-		expression:  `.a *n .b`,
-		expected: []string{
-			"D0, P[a], (!!map)::{thing: one, cat: frog, missing: two}\n",
-		},
-	},
 	{
 		skipDoc:    true,
 		document:   `{a: [{thing: one}], b: [{missing: two, thing: two}]}`,
@@ -360,14 +352,6 @@ var multiplyOperatorScenarios = []expressionScenario{
 			"D0, P[a], (!!seq)::[{thing: two}]\n",
 		},
 	},
-	{
-		skipDoc:    true,
-		document:   `{a: [{thing: one}], b: [{missing: two, thing: two}]}`,
-		expression: `.a *nd .b`,
-		expected: []string{
-			"D0, P[a], (!!seq)::[{thing: one, missing: two}]\n",
-		},
-	},
 	{
 		skipDoc:    true,
 		document:   `{a: {array: [1]}, b: {}}`,
@@ -392,16 +376,6 @@ var multiplyOperatorScenarios = []expressionScenario{
 			"D0, P[a], (!!map)::{thing: [1, 2, 3, 4]}\n",
 		},
 	},
-	{
-		description:    "Merge, only new fields, appending arrays",
-		subdescription: "Append (+) with (n) has no effect.",
-		skipDoc:        true,
-		document:       `{a: {thing: [1,2]}, b: {thing: [3,4], another: [1]}}`,
-		expression:     `.a *n+ .b`,
-		expected: []string{
-			"D0, P[a], (!!map)::{thing: [1, 2], another: [1]}\n",
-		},
-	},
 	{
 		description:    "Merge, deeply merging arrays",
 		subdescription: "Merging arrays deeply means arrays are merge like objects, with indexes as their key. In this case, we merge the first item in the array, and do nothing with the second.",

pkg/yqlib/operator_select_test.go

@@ -28,7 +28,7 @@ var selectOperatorScenarios = []expressionScenario{
 		expected:   []string{},
 	},
 	{
-		description: "Select elements from array using wildcard prefix",
+		description: "Select elements from array",
 		document:    `[cat,goat,dog]`,
 		expression:  `.[] | select(. == "*at")`,
 		expected: []string{
@@ -36,35 +36,6 @@ var selectOperatorScenarios = []expressionScenario{
 			"D0, P[1], (!!str)::goat\n",
 		},
 	},
-	{
-		description: "Select elements from array using wildcard suffix",
-		document:    `[go-kart,goat,dog]`,
-		expression:  `.[] | select(. == "go*")`,
-		expected: []string{
-			"D0, P[0], (!!str)::go-kart\n",
-			"D0, P[1], (!!str)::goat\n",
-		},
-	},
-	{
-		description: "Select elements from array using wildcard prefix and suffix",
-		document:    `[ago, go, meow, going]`,
-		expression:  `.[] | select(. == "*go*")`,
-		expected: []string{
-			"D0, P[0], (!!str)::ago\n",
-			"D0, P[1], (!!str)::go\n",
-			"D0, P[3], (!!str)::going\n",
-		},
-	},
-	{
-		description:    "Select elements from array with regular expression",
-		subdescription: "See more regular expression examples under the `string` operator docs.",
-		document:       `[this_0, not_this, nor_0_this, thisTo_4]`,
-		expression:     `.[] | select(test("[a-zA-Z]+_[0-9]$"))`,
-		expected: []string{
-			"D0, P[0], (!!str)::this_0\n",
-			"D0, P[3], (!!str)::thisTo_4\n",
-		},
-	},
 	{
 		skipDoc:    true,
 		document:   "a: hello",
@@ -98,27 +69,9 @@ var selectOperatorScenarios = []expressionScenario{
 			"D0, P[a 1], (!!str)::goat\n"},
 	},
 	{
-		description: "Select items from a map",
-		document:    `{ things: cat, bob: goat, horse: dog }`,
-		expression:  `.[] | select(. == "cat" or test("og$"))`,
-		expected: []string{
-			"D0, P[things], (!!str)::cat\n",
-			"D0, P[horse], (!!str)::dog\n",
-		},
-	},
-	{
-		description: "Use select and with_entries to filter map keys",
-		document:    `{name: bob, legs: 2, game: poker}`,
-		expression:  `with_entries(select(.key | test("ame$")))`,
-		expected: []string{
-			"D0, P[], (!!map)::name: bob\ngame: poker\n",
-		},
-	},
-	{
-		description:    "Select multiple items in a map and update",
-		subdescription: "Note the brackets around the entire LHS.",
-		document:       `a: { things: cat, bob: goat, horse: dog }`,
-		expression:     `(.a.[] | select(. == "cat" or . == "goat")) |= "rabbit"`,
+		description: "Select and update matching values in map",
+		document:    `a: { things: cat, bob: goat, horse: dog }`,
+		expression:  `(.a.[] | select(. == "*at")) |= "rabbit"`,
 		expected: []string{
 			"D0, P[], (doc)::a: {things: rabbit, bob: rabbit, horse: dog}\n",
 		},

pkg/yqlib/operators.go

@@ -18,7 +18,7 @@ func compoundAssignFunction(d *dataTreeNavigator, context Context, expressionNod
 		return Context{}, err
 	}
 
-	assignmentOp := &Operation{OperationType: assignOpType, Preferences: expressionNode.Operation.Preferences}
+	assignmentOp := &Operation{OperationType: assignOpType}
 	valueOp := &Operation{OperationType: valueOpType}
 
 	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {

pkg/yqlib/xml_test.go

@@ -24,7 +24,7 @@ func decodeXml(t *testing.T, xml string) *CandidateNode {
 	return &CandidateNode{Node: node}
 }
 
-func processXmlScenario(s formatScenario) string {
+func processScenario(s xmlScenario) string {
 	var output bytes.Buffer
 	writer := bufio.NewWriter(&output)
 
@@ -49,10 +49,8 @@ func processXmlScenario(s formatScenario) string {
 	return output.String()
 }
 
-type formatScenario struct {
+type xmlScenario struct {
 	input          string
-	indent         int
-	expression     string
 	expected       string
 	description    string
 	subdescription string
@@ -205,7 +203,7 @@ var expectedXmlWithComments = `<!-- above_cat inline_cat --><cat><!-- above_arra
 </cat><!-- below_cat -->
 `
 
-var xmlScenarios = []formatScenario{
+var xmlScenarios = []xmlScenario{
 	{
 		description: "Parse xml: simple",
 		input:       "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat>meow</cat>",
@@ -298,9 +296,9 @@ var xmlScenarios = []formatScenario{
 	},
 }
 
-func testXmlScenario(t *testing.T, s formatScenario) {
+func testXmlScenario(t *testing.T, s xmlScenario) {
 	if s.scenarioType == "encode" || s.scenarioType == "roundtrip" {
-		test.AssertResultWithContext(t, s.expected, processXmlScenario(s), s.description)
+		test.AssertResultWithContext(t, s.expected, processScenario(s), s.description)
 	} else {
 		var actual = resultToString(t, decodeXml(t, s.input))
 		test.AssertResultWithContext(t, s.expected, actual, s.description)
@@ -308,7 +306,7 @@ func testXmlScenario(t *testing.T, s formatScenario) {
 }
 
 func documentXmlScenario(t *testing.T, w *bufio.Writer, i interface{}) {
-	s := i.(formatScenario)
+	s := i.(xmlScenario)
 
 	if s.skipDoc {
 		return
@@ -323,7 +321,7 @@ func documentXmlScenario(t *testing.T, w *bufio.Writer, i interface{}) {
 
 }
 
-func documentXmlDecodeScenario(t *testing.T, w *bufio.Writer, s formatScenario) {
+func documentXmlDecodeScenario(t *testing.T, w *bufio.Writer, s xmlScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -352,7 +350,7 @@ func documentXmlDecodeScenario(t *testing.T, w *bufio.Writer, s formatScenario)
 	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", output.String()))
 }
 
-func documentXmlEncodeScenario(w *bufio.Writer, s formatScenario) {
+func documentXmlEncodeScenario(w *bufio.Writer, s xmlScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -367,10 +365,10 @@ func documentXmlEncodeScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, "```bash\nyq e -o=xml '.' sample.yml\n```\n")
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processXmlScenario(s)))
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processScenario(s)))
 }
 
-func documentXmlRoundTripScenario(w *bufio.Writer, s formatScenario) {
+func documentXmlRoundTripScenario(w *bufio.Writer, s xmlScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -385,7 +383,7 @@ func documentXmlRoundTripScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, "```bash\nyq e -p=xml -o=xml '.' sample.xml\n```\n")
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processXmlScenario(s)))
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processScenario(s)))
 }
 
 func TestXmlScenarios(t *testing.T) {

release_instructions.txt

@@ -1,11 +1,8 @@
 - update release notes
+- check dependabot
 - increment version in version.go
 - increment version in snapcraft.yaml
 - make sure local build passes
-- check manpage 
-  - ./scripts/generate-man-page-md.sh
-  - ./scripts/generate-man-page.sh
-  - man ./yq.1
 - run ./scripts/copy-docs.sh (and commit the changed in the yq-book branch)
 - commit version update changes
 - tag git with same version number

release_notes.txt

@@ -1,13 +1,3 @@
-4.17.2:
-  - Fixed manpath issue (thanks @mr-pmillz)
-  
-4.17.1:
-  - Added XML support (#491)
-  - New merge flag (n) to only merge new fields (#1038)
-  - Fixed exit status bug for permission denied error (#1062)
-  - Fixed using multiple variables with union (,) operator (#1048)
-  - Bumped some versions of dependencies
-
 4.16.2:
   - Bumped go-lang compiler to fix CVE-2021-44717 (#1037)
   - Dependency version bumps via dependabot

scripts/generate-man-page-md.sh

@@ -3,14 +3,10 @@ set -e
 
 # note that this reqires pandoc to be installed.
 
-cat ./pkg/yqlib/doc/operators/headers/Main.md > man.md
+cat ./pkg/yqlib/doc/headers/Main.md > man.md
 printf "\n# HOW IT WORKS\n" >> man.md
 tail -n +2 how-it-works.md >> man.md
 
-for f in ./pkg/yqlib/doc/operators/*.md; do
-  cat "$f" >> man.md
-done
-
-for f in ./pkg/yqlib/doc/usage/*.md; do
+for f in ./pkg/yqlib/doc/*.md; do 
   cat "$f" >> man.md
 done

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.17.2'
+version: '4.16.2'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.
@@ -16,7 +16,7 @@ apps:
 parts:
   yq:
     plugin: go
-    go-channel: 1.17/stable
+    go-channel: 1.15/stable
     source: .
     source-type: git
     go-importpath: github.com/mikefarah/yq