Split printer!

This reverts commit 4ce9433468.

.github/workflows/release.yml

@@ -11,11 +11,37 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
-          go-version: '^1.15'
+          go-version: '^1.17.2'
+      - name: Compile man page markup
+        id: gen-man-page-md
+        run: |
+          ./scripts/generate-man-page-md.sh
+          echo "::set-output name=man-page-md::man.md"
+
+      - name: Get the version
+        id: get_version
+        run: echo ::set-output name=VERSION::${GITHUB_REF##*/}
+        
+      - name: Generate man page
+        uses: docker://pandoc/core:2.14.2
+        id: gen-man-page
+        with:
+          args: >-
+            --standalone
+            --to man
+            --variable=title:"YQ"
+            --variable=section:"1"
+            --variable=header:"yq (https://github.com/mikefarah/yq/) version ${{ steps.get_version.outputs.VERSION }}"
+            --variable=author:"Mike Farah"
+            --output=yq.1
+            ${{ steps.gen-man-page-md.outputs.man-page-md }}
+        
       - name: Cross compile
         run: |
           sudo apt-get install rhash -y
           go get github.com/mitchellh/gox
+          mkdir -p build
+          cp yq.1 build/yq.1
           ./scripts/xcompile.sh
 
       - name: Create Release

.gitignore

@@ -38,5 +38,14 @@ parts/
 prime/
 .snapcraft/
 yq*.snap
+
 test.yml
-test2.yml
+test*.yml
+0.yml
+1.yml
+2.yml
+
+
+# man page
+man.md
+yq.1

Dockerfile

@@ -10,7 +10,7 @@ RUN ./scripts/acceptance.sh
 
 # Choose alpine as a base image to make this useful for CI, as many
 # CI tools expect an interactive shell inside the container
-FROM alpine:3.13.5 as production
+FROM alpine:3 as production
 
 RUN mkdir /home/yq/
 RUN addgroup -g 1000 yq && \

README.md

@@ -260,14 +260,15 @@ Flags:
   -C, --colors                force print with colors
   -e, --exit-status           set exit status if there are no matches or null or false is returned
   -f, --front-matter string   (extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact
+      --header-preprocess     Slurp any header comments and seperators before processing expression. This is a workaround for go-yaml to persist header content (default true)
   -h, --help                  help for yq
   -I, --indent int            sets indent level for output (default 2)
   -i, --inplace               update the yaml file inplace of first yaml file given.
   -M, --no-colors             force print with no colors
   -N, --no-doc                Don't print document separators (---)
   -n, --null-input            Don't read input, simply evaluate the expression given. Useful for creating yaml docs from scratch.
+  -o, --output-format string  [yaml|y|json|j|props|p] output format type. (default "yaml")
   -P, --prettyPrint           pretty print, shorthand for '... style = ""'
-  -j, --tojson                output as json. Set indent to 0 to print json in one line.
       --unwrapScalar          unwrap scalar, print the value with no quotes, colors or comments (default true)
   -v, --verbose               verbose mode
   -V, --version               Print version information and quit

acceptance_tests/basic.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 setUp() {
-  rm -f test.yml
+  rm test*.yml || true
 }
 
 testBasicEvalRoundTrip() {

acceptance_tests/empty.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 setUp() {
+  rm test*.yml || true
   cat >test.yml <<EOL
 # comment
 EOL

acceptance_tests/front-matter.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 setUp() {
+  rm test*.yml || true
   cat >test.yml <<EOL
 ---
 a: apple

acceptance_tests/leading-seperator.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 setUp() {
+  rm test*.yml || true
   cat >test.yml <<EOL
 ---
 a: test

acceptance_tests/output-format.sh

@@ -1,5 +1,9 @@
 #!/bin/bash
 
+setUp() {
+  rm test*.yml || true
+}
+
 testOutputJsonDeprecated() {
   cat >test.yml <<EOL
 a: {b: ["cat"]}

acceptance_tests/pretty-print.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+setUp() {
+  rm test*.yml || true
+}
 
 testPrettyPrintWithBooleans() {
   cat >test.yml <<EOL

acceptance_tests/split-printer.sh

@@ -0,0 +1,157 @@
+#!/bin/bash
+
+setUp() {
+  rm test*.yml || true
+}
+
+testBasicSplitWithName() {
+  cat >test.yml <<EOL
+a: test_doc1
+--- 
+a: test_doc2
+EOL
+
+  ./yq e test.yml -s ".a"
+
+  doc1=$(cat test_doc1.yml)
+  
+  assertEquals "a: test_doc1" "$doc1"
+
+  doc2=$(cat test_doc2.yml)
+  read -r -d '' expectedDoc2 << EOM
+---
+a: test_doc2
+EOM
+  assertEquals "$expectedDoc2" "$doc2"
+}
+
+testBasicSplitWithNameEvalAll() {
+  cat >test.yml <<EOL
+a: test_doc1
+--- 
+a: test_doc2
+EOL
+
+  ./yq ea test.yml -s ".a"
+
+  doc1=$(cat test_doc1.yml)
+  
+  assertEquals "a: test_doc1" "$doc1"
+
+  doc2=$(cat test_doc2.yml)
+  read -r -d '' expectedDoc2 << EOM
+---
+a: test_doc2
+EOM
+  assertEquals "$expectedDoc2" "$doc2"
+}
+
+testBasicSplitWithIndex() {
+  cat >test.yml <<EOL
+a: test_doc1
+--- 
+a: test_doc2
+EOL
+
+  ./yq e test.yml -s '"test_" + $index'
+
+  doc1=$(cat test_0.yml)
+  
+  assertEquals "a: test_doc1" "$doc1"
+
+  doc2=$(cat test_1.yml)
+  read -r -d '' expectedDoc2 << EOM
+---
+a: test_doc2
+EOM
+  assertEquals "$expectedDoc2" "$doc2"
+}
+
+testBasicSplitWithIndexEvalAll() {
+  cat >test.yml <<EOL
+a: test_doc1
+--- 
+a: test_doc2
+EOL
+
+  ./yq ea test.yml -s '"test_" + $index'
+
+  doc1=$(cat test_0.yml)
+  
+  assertEquals "a: test_doc1" "$doc1"
+
+  doc2=$(cat test_1.yml)
+  read -r -d '' expectedDoc2 << EOM
+---
+a: test_doc2
+EOM
+  assertEquals "$expectedDoc2" "$doc2"
+}
+
+
+testArraySplitWithNameNoSeparators() {
+  cat >test.yml <<EOL
+- name: test_fred
+  age: 35
+- name: test_catherine
+  age: 37
+EOL
+
+  ./yq e --no-doc -s ".name"  ".[]" test.yml 
+
+  doc1=$(cat test_fred.yml)
+  read -r -d '' expectedDoc1 << EOM
+name: test_fred
+age: 35
+EOM
+
+  assertEquals "$expectedDoc1" "$doc1"
+
+  doc2=$(cat test_catherine.yml)
+  read -r -d '' expectedDoc2 << EOM
+name: test_catherine
+age: 37
+EOM
+  assertEquals "$expectedDoc2" "$doc2"
+}
+
+testArraySplitWithNameNoSeparatorsEvalAll() {
+  cat >test.yml <<EOL
+- name: test_fred
+  age: 35
+- name: test_catherine
+  age: 37
+EOL
+
+cat >test2.yml <<EOL
+- name: test_mike
+  age: 564
+EOL
+
+  ./yq ea --no-doc -s ".name"  ".[]" test.yml test2.yml
+
+  doc1=$(cat test_fred.yml)
+  read -r -d '' expectedDoc1 << EOM
+name: test_fred
+age: 35
+EOM
+
+  assertEquals "$expectedDoc1" "$doc1"
+
+  doc2=$(cat test_catherine.yml)
+  read -r -d '' expectedDoc2 << EOM
+name: test_catherine
+age: 37
+EOM
+  assertEquals "$expectedDoc2" "$doc2"
+
+
+  doc3=$(cat test_mike.yml)
+  read -r -d '' expectedDoc3 << EOM
+name: test_mike
+age: 564
+EOM
+  assertEquals "$expectedDoc3" "$doc3"
+}
+
+source ./scripts/shunit2

cmd/constant.go

@@ -21,4 +21,6 @@ var prettyPrint = false
 // can be either "" (off), "extract" or "process"
 var frontMatter = ""
 
+var splitFileExp = ""
+
 var completedSuccessfully = false

cmd/evaluate_all_command.go

@@ -70,6 +70,10 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("Write inplace flag only applicable when giving an expression and at least one file")
 	}
 
+	if writeInplace && splitFileExp != "" {
+		return fmt.Errorf("Write inplace cannot be used with split file")
+	}
+
 	if writeInplace {
 		// only use colors if its forced
 		colorsEnabled = forceColor
@@ -96,7 +100,9 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	printer := yqlib.NewPrinter(out, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
+	printerWriter := configurePrinterWriter(format, out)
+
+	printer := yqlib.NewPrinter(printerWriter, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
 
 	if frontMatter != "" {
 		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])

cmd/evalute_sequence_command.go

@@ -83,6 +83,10 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("Write inplace flag only applicable when giving an expression and at least one file")
 	}
 
+	if writeInplace && splitFileExp != "" {
+		return fmt.Errorf("Write inplace cannot be used with split file")
+	}
+
 	if writeInplace {
 		// only use colors if its forced
 		colorsEnabled = forceColor
@@ -106,7 +110,9 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	printer := yqlib.NewPrinter(out, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
+	printerWriter := configurePrinterWriter(format, out)
+
+	printer := yqlib.NewPrinter(printerWriter, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
 
 	streamEvaluator := yqlib.NewStreamEvaluator()
 

cmd/printer_utils.go

@@ -0,0 +1,28 @@
+package cmd
+
+import (
+	"io"
+
+	"github.com/mikefarah/yq/v4/pkg/yqlib"
+)
+
+func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) yqlib.PrinterWriter {
+
+	var printerWriter yqlib.PrinterWriter
+
+	if splitFileExp == "i" {
+		colorsEnabled = forceColor
+		printerWriter = yqlib.NewMultiPrinterWriter(nil, format)
+
+	} else if splitFileExp != "" {
+		colorsEnabled = forceColor
+		splitExp, err := yqlib.NewExpressionParser().ParseExpression(splitFileExp)
+		if err != nil {
+			return nil
+		}
+		printerWriter = yqlib.NewMultiPrinterWriter(splitExp, format)
+	} else {
+		printerWriter = yqlib.NewSinglePrinterWriter(out)
+	}
+	return printerWriter
+}

cmd/root.go

@@ -62,7 +62,10 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 	rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors")
 	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
 	rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
-	rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and seperators before processing expression. This is a workaround for go-yaml to persist header content. This flag will be removed once this feature has been out in the wild for a while.")
+	rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression. This is a workaround for go-yaml to persist header content properly.")
+
+	rootCmd.PersistentFlags().StringVarP(&splitFileExp, "split-exp", "s", "", "print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter.")
+
 	rootCmd.AddCommand(
 		createEvaluateSequenceCommand(),
 		createEvaluateAllCommand(),

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.13.3"
+	Version = "4.13.5"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

debian/rules

@@ -45,7 +45,9 @@ override_dh_auto_build:
 	(cd ${SRCDIR} && go install -buildmode=pie ./...)
 	
 override_dh_auto_test:
+ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS)))
 	(cd ${SRCDIR} && go test -v ./...)
+endif
 
 override_dh_auto_install:
 	cp ${GOBIN}/yq ${DESTDIR}/${BINDIR}

github-action/Dockerfile

@@ -1,4 +1,4 @@
-FROM mikefarah/yq:4.13.3
+FROM mikefarah/yq:4.13.5
 
 COPY entrypoint.sh /entrypoint.sh
 

go.mod

@@ -7,9 +7,7 @@ require (
 	github.com/jinzhu/copier v0.2.8
 	github.com/magiconair/properties v1.8.5
 	github.com/spf13/cobra v1.1.3
-	github.com/timtadh/data-structures v0.5.3 // indirect
 	github.com/timtadh/lexmachine v0.2.2
-	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )
@@ -19,6 +17,8 @@ require (
 	github.com/mattn/go-colorable v0.1.8 // indirect
 	github.com/mattn/go-isatty v0.0.12 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/timtadh/data-structures v0.5.3 // indirect
+	golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 )
 

go.sum

@@ -258,8 +258,8 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA=
+golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=

pkg/yqlib/candidate_node.go

@@ -1,6 +1,7 @@
 package yqlib
 
 import (
+	"container/list"
 	"fmt"
 
 	"github.com/jinzhu/copier"
@@ -28,6 +29,12 @@ func (n *CandidateNode) GetKey() string {
 	return fmt.Sprintf("%v%v - %v", keyPrefix, n.Document, n.Path)
 }
 
+func (n *CandidateNode) AsList() *list.List {
+	elMap := list.New()
+	elMap.PushBack(n)
+	return elMap
+}
+
 func (n *CandidateNode) CreateChild(path interface{}, node *yaml.Node) *CandidateNode {
 	return &CandidateNode{
 		Node:      node,

pkg/yqlib/context.go

@@ -43,11 +43,14 @@ func (n *Context) SetVariable(name string, value *list.List) {
 }
 
 func (n *Context) ChildContext(results *list.List) Context {
-	clone := Context{}
-	err := copier.Copy(&clone, n)
-	if err != nil {
-		log.Error("Error cloning context :(")
-		panic(err)
+	clone := Context{DontAutoCreate: n.DontAutoCreate}
+	clone.Variables = make(map[string]*list.List)
+	if len(n.Variables) > 0 {
+		err := copier.Copy(&clone.Variables, n.Variables)
+		if err != nil {
+			log.Error("Error cloning context :(")
+			panic(err)
+		}
 	}
 	clone.MatchingNodes = results
 	return clone

pkg/yqlib/doc/Assign (Update).md

@@ -136,7 +136,9 @@ a:
   b: frog
 ```
 
-## Update selected results
+## Update deeply selected results
+Note that the LHS is wrapped in brackets! This is to ensure we dont first filter out the yaml and then update the snippet.
+
 Given a sample.yml file of:
 ```yaml
 a:

pkg/yqlib/doc/Encoder and Decoder.md

@@ -0,0 +1,177 @@
+Encode operators will take the piped in object structure and encode it as a string in the desired format. The decode operators do the opposite, they take a formatted string and decode it into the relevant object structure.
+
+Note that you can optionally pass an indent value to the encode functions (see below).
+
+These operators are useful to process yaml documents that have stringified embeded yaml/json/props in them.
+## Encode value as yaml string
+Indent defaults to 2
+
+Given a sample.yml file of:
+```yaml
+a:
+  cool:
+    bob: dylan
+```
+then
+```bash
+yq eval '.b = (.a | to_yaml)' sample.yml
+```
+will output
+```yaml
+a:
+  cool:
+    bob: dylan
+b: |
+  cool:
+    bob: dylan
+```
+
+## Encode value as yaml string, with custom indentation
+You can specify the indentation level as the first parameter.
+
+Given a sample.yml file of:
+```yaml
+a:
+  cool:
+    bob: dylan
+```
+then
+```bash
+yq eval '.b = (.a | to_yaml(8))' sample.yml
+```
+will output
+```yaml
+a:
+  cool:
+    bob: dylan
+b: |
+  cool:
+          bob: dylan
+```
+
+## Encode value as yaml string, using toyaml
+Does the same thing as to_yaml, matching jq naming convention.
+
+Given a sample.yml file of:
+```yaml
+a:
+  cool: thing
+```
+then
+```bash
+yq eval '.b = (.a | to_yaml)' sample.yml
+```
+will output
+```yaml
+a:
+  cool: thing
+b: |
+  cool: thing
+```
+
+## Encode value as json string
+Given a sample.yml file of:
+```yaml
+a:
+  cool: thing
+```
+then
+```bash
+yq eval '.b = (.a | to_json)' sample.yml
+```
+will output
+```yaml
+a:
+  cool: thing
+b: |
+  {
+    "cool": "thing"
+  }
+```
+
+## Encode value as json string, on one line
+Pass in a 0 indent to print json on a single line.
+
+Given a sample.yml file of:
+```yaml
+a:
+  cool: thing
+```
+then
+```bash
+yq eval '.b = (.a | to_json(0))' sample.yml
+```
+will output
+```yaml
+a:
+  cool: thing
+b: '{"cool":"thing"}'
+```
+
+## Encode value as props string
+Given a sample.yml file of:
+```yaml
+a:
+  cool: thing
+```
+then
+```bash
+yq eval '.b = (.a | to_props)' sample.yml
+```
+will output
+```yaml
+a:
+  cool: thing
+b: |
+  cool = thing
+```
+
+## Decode a yaml encoded string
+Given a sample.yml file of:
+```yaml
+a: 'foo: bar'
+```
+then
+```bash
+yq eval '.b = (.a | from_yaml)' sample.yml
+```
+will output
+```yaml
+a: 'foo: bar'
+b:
+  foo: bar
+```
+
+## Update a multiline encoded yaml string
+Given a sample.yml file of:
+```yaml
+a: |
+  foo: bar
+  baz: dog
+
+```
+then
+```bash
+yq eval '.a |= (from_yaml | .foo = "cat" | to_yaml)' sample.yml
+```
+will output
+```yaml
+a: |
+  foo: cat
+  baz: dog
+```
+
+## Update a single line encoded yaml string
+Given a sample.yml file of:
+```yaml
+a: 'foo: bar'
+```
+then
+```bash
+yq eval '.a |= (from_yaml | .foo = "cat" | to_yaml)' sample.yml
+```
+will output
+```yaml
+a: 'foo: cat'
+```
+

pkg/yqlib/doc/Flatten.md

@@ -0,0 +1,70 @@
+This recursively flattens arrays.
+
+## Flatten
+Recursively flattens all arrays
+
+Given a sample.yml file of:
+```yaml
+- 1
+- - 2
+- - - 3
+```
+then
+```bash
+yq eval 'flatten' sample.yml
+```
+will output
+```yaml
+- 1
+- 2
+- 3
+```
+
+## Flatten with depth of one
+Given a sample.yml file of:
+```yaml
+- 1
+- - 2
+- - - 3
+```
+then
+```bash
+yq eval 'flatten(1)' sample.yml
+```
+will output
+```yaml
+- 1
+- 2
+- - 3
+```
+
+## Flatten empty array
+Given a sample.yml file of:
+```yaml
+- []
+```
+then
+```bash
+yq eval 'flatten' sample.yml
+```
+will output
+```yaml
+[]
+```
+
+## Flatten array of objects
+Given a sample.yml file of:
+```yaml
+- foo: bar
+- - foo: baz
+```
+then
+```bash
+yq eval 'flatten' sample.yml
+```
+will output
+```yaml
+- foo: bar
+- foo: baz
+```
+

pkg/yqlib/doc/Group By.md

@@ -0,0 +1,54 @@
+This is used to group items in an array by an expression.
+
+## Group by field
+Given a sample.yml file of:
+```yaml
+- foo: 1
+  bar: 10
+- foo: 3
+  bar: 100
+- foo: 1
+  bar: 1
+```
+then
+```bash
+yq eval 'group_by(.foo)' sample.yml
+```
+will output
+```yaml
+- - foo: 1
+    bar: 10
+  - foo: 1
+    bar: 1
+- - foo: 3
+    bar: 100
+```
+
+## Group by field, with nuls
+Given a sample.yml file of:
+```yaml
+- cat: dog
+- foo: 1
+  bar: 10
+- foo: 3
+  bar: 100
+- no: foo for you
+- foo: 1
+  bar: 1
+```
+then
+```bash
+yq eval 'group_by(.foo)' sample.yml
+```
+will output
+```yaml
+- - cat: dog
+  - no: foo for you
+- - foo: 1
+    bar: 10
+  - foo: 1
+    bar: 1
+- - foo: 3
+    bar: 100
+```
+

pkg/yqlib/doc/Multiply (Merge).md

@@ -357,7 +357,7 @@ yq eval '.foobar * .foobarList' sample.yml
 will output
 ```yaml
 c: foobarList_c
-<<:
+!!merge <<:
   - *foo
   - *bar
 thing: foobar_thing

pkg/yqlib/doc/aa.md

@@ -1,5 +1,3 @@
-# Operators
-
 In `yq` expressions are made up of operators and pipes. A context of nodes is passed through the expression and each operation takes the context as input and returns a new context as output. That output is piped in as input for the next operation in the expression. To begin with, the context is set to the first yaml document of the first yaml file (if processing in sequence using eval).
 
 Lets look at a couple of examples.

pkg/yqlib/doc/headers/Encoder and Decoder.md

@@ -0,0 +1,5 @@
+Encode operators will take the piped in object structure and encode it as a string in the desired format. The decode operators do the opposite, they take a formatted string and decode it into the relevant object structure.
+
+Note that you can optionally pass an indent value to the encode functions (see below).
+
+These operators are useful to process yaml documents that have stringified embeded yaml/json/props in them.

pkg/yqlib/doc/headers/Flatten.md

@@ -0,0 +1 @@
+This recursively flattens arrays.

pkg/yqlib/doc/headers/Group By.md

@@ -0,0 +1 @@
+This is used to group items in an array by an expression.

pkg/yqlib/doc/headers/Main.md

@@ -0,0 +1,61 @@
+# NAME
+  *yq* is a portable command-line YAML processor
+
+# SYNOPSIS 
+
+yq [eval/eval-all] [expression] files..
+
+eval/e  - Apply the expression to each document in each yaml file in sequence
+
+eval-all/ea - Loads all yaml documents of all yaml files and runs expression once
+
+# DESCRIPTION
+
+a lightweight and portable command-line YAML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.
+
+This documentation is also available at https://mikefarah.gitbook.io/yq/
+# QUICK GUIDE 
+
+## Read a value:
+```bash
+yq e '.a.b[0].c' file.yaml
+```
+
+## Pipe from STDIN:
+```bash
+cat file.yaml | yq e '.a.b[0].c' -
+```
+
+## Update a yaml file, inplace
+```bash
+yq e -i '.a.b[0].c = "cool"' file.yaml
+```
+
+## Update using environment variables
+```bash
+NAME=mike yq e -i '.a.b[0].c = strenv(NAME)' file.yaml
+```
+
+## Merge multiple files
+```
+yq ea '. as $item ireduce ({}; . * $item )' path/to/*.yml
+```
+
+## Multiple updates to a yaml file
+```bash
+yq e -i '
+  .a.b[0].c = "cool" |
+  .x.y.z = "foobar" |
+  .person.name = strenv(NAME)
+' file.yaml
+```
+
+See the [documentation](https://mikefarah.gitbook.io/yq/) for more.
+
+# KNOWN ISSUES / MISSING FEATURES
+- `yq` attempts to preserve comment positions and whitespace as much as possible, but it does not handle all scenarios (see https://github.com/go-yaml/yaml/tree/v3 for details)
+- Powershell has its own...opinions: https://mikefarah.gitbook.io/yq/usage/tips-and-tricks#quotes-in-windows-powershell
+
+# BUGS / ISSUES / FEATURE REQUESTS
+
+Please visit the GitHub page https://github.com/mikefarah/yq/.

pkg/yqlib/expression_processing_test.go

@@ -15,6 +15,16 @@ var pathTests = []struct {
 	expectedTokens  []interface{}
 	expectedPostFix []interface{}
 }{
+	{
+		`.["cat"].["dog"]`,
+		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "cat (string)", "]", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "dog (string)", "]"),
+		append(make([]interface{}, 0), "SELF", "cat (string)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SELF", "dog (string)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
+	},
+	{
+		`.["cat"]`,
+		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "cat (string)", "]"),
+		append(make([]interface{}, 0), "SELF", "cat (string)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
 	{
 		"with(.a;.=3)",
 		append(make([]interface{}, 0), "WITH", "(", "a", "BLOCK", "SELF", "ASSIGN", "3 (int64)", ")"),
@@ -127,8 +137,8 @@ var pathTests = []struct {
 	},
 	{
 		`.a.[]`,
-		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
-		append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+		append(make([]interface{}, 0), "a", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
+		append(make([]interface{}, 0), "a", "SELF", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
 	},
 	{
 		`.a[0]`,
@@ -142,8 +152,8 @@ var pathTests = []struct {
 	},
 	{
 		`.a.[0]`,
-		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
-		append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+		append(make([]interface{}, 0), "a", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
+		append(make([]interface{}, 0), "a", "SELF", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
 	},
 	{
 		`.a[].c`,
@@ -217,8 +227,8 @@ var pathTests = []struct {
 	},
 	{
 		`{.a: .c, .b.[]: .f.g[]}`,
-		append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "c", "UNION", "b", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "CREATE_MAP", "f", "SHORT_PIPE", "g", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "}"),
-		append(make([]interface{}, 0), "a", "c", "CREATE_MAP", "b", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "f", "g", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE", "CREATE_MAP", "UNION", "COLLECT_OBJECT", "SHORT_PIPE"),
+		append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "c", "UNION", "b", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "CREATE_MAP", "f", "SHORT_PIPE", "g", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "}"),
+		append(make([]interface{}, 0), "a", "c", "CREATE_MAP", "b", "SELF", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE", "f", "g", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE", "CREATE_MAP", "UNION", "COLLECT_OBJECT", "SHORT_PIPE"),
 	},
 	{
 		`explode(.a.b)`,

pkg/yqlib/expression_tokeniser.go

@@ -2,6 +2,7 @@ package yqlib
 
 import (
 	"fmt"
+	"regexp"
 	"strconv"
 	"strings"
 
@@ -111,6 +112,7 @@ func multiplyWithPrefs() lex.Action {
 		if strings.Contains(options, "d") {
 			prefs.DeepMergeArrays = true
 		}
+		prefs.TraversePrefs.DontFollowAlias = true
 		op := &Operation{OperationType: multiplyOpType, Value: multiplyOpType.Type, StringValue: options, Preferences: prefs}
 		return &token{TokenType: operationToken, Operation: op}, nil
 	}
@@ -129,6 +131,44 @@ func opTokenWithPrefs(op *operationType, assignOpType *operationType, preference
 	}
 }
 
+func extractNumberParamter(value string) (int, error) {
+	parameterParser := regexp.MustCompile(`.*\(([0-9]+)\)`)
+	matches := parameterParser.FindStringSubmatch(value)
+	var indent, errParsingInt = strconv.ParseInt(matches[1], 10, 32) // nolint
+	if errParsingInt != nil {
+		return 0, errParsingInt
+	}
+	return int(indent), nil
+}
+
+func flattenWithDepth() lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		value := string(m.Bytes)
+		var depth, errParsingInt = extractNumberParamter(value)
+		if errParsingInt != nil {
+			return nil, errParsingInt
+		}
+
+		prefs := flattenPreferences{depth: depth}
+		op := &Operation{OperationType: flattenOpType, Value: flattenOpType.Type, StringValue: value, Preferences: prefs}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+}
+
+func encodeWithIndent(outputFormat PrinterOutputFormat) lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		value := string(m.Bytes)
+		var indent, errParsingInt = extractNumberParamter(value)
+		if errParsingInt != nil {
+			return nil, errParsingInt
+		}
+
+		prefs := encoderPreferences{format: outputFormat, indent: indent}
+		op := &Operation{OperationType: encodeOpType, Value: encodeOpType.Type, StringValue: value, Preferences: prefs}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+}
+
 func assignAllCommentsOp(updateAssign bool) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		log.Debug("assignAllCommentsOp %v", string(m.Bytes))
@@ -271,11 +311,36 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`,`), opToken(unionOpType))
 	lexer.Add([]byte(`:\s*`), opToken(createMapOpType))
 	lexer.Add([]byte(`length`), opToken(lengthOpType))
+
+	lexer.Add([]byte(`flatten\([0-9]+\)`), flattenWithDepth())
+	lexer.Add([]byte(`flatten`), opTokenWithPrefs(flattenOpType, nil, flattenPreferences{depth: -1}))
+
+	lexer.Add([]byte(`toyaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat))
+	lexer.Add([]byte(`to_yaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat))
+
+	lexer.Add([]byte(`tojson\([0-9]+\)`), encodeWithIndent(JsonOutputFormat))
+	lexer.Add([]byte(`to_json\([0-9]+\)`), encodeWithIndent(JsonOutputFormat))
+
+	lexer.Add([]byte(`toyaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2}))
+	lexer.Add([]byte(`tojson`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JsonOutputFormat, indent: 2}))
+	lexer.Add([]byte(`toprops`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2}))
+
+	lexer.Add([]byte(`to_yaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2}))
+	lexer.Add([]byte(`to_json`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JsonOutputFormat, indent: 2}))
+	lexer.Add([]byte(`to_props`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2}))
+
+	lexer.Add([]byte(`fromyaml`), opToken(decodeOpType))
+	lexer.Add([]byte(`fromjson`), opToken(decodeOpType))
+
+	lexer.Add([]byte(`from_yaml`), opToken(decodeOpType))
+	lexer.Add([]byte(`from_json`), opToken(decodeOpType))
+
 	lexer.Add([]byte(`sortKeys`), opToken(sortKeysOpType))
 	lexer.Add([]byte(`select`), opToken(selectOpType))
 	lexer.Add([]byte(`has`), opToken(hasOpType))
 	lexer.Add([]byte(`unique`), opToken(uniqueOpType))
 	lexer.Add([]byte(`unique_by`), opToken(uniqueByOpType))
+	lexer.Add([]byte(`group_by`), opToken(groupByOpType))
 	lexer.Add([]byte(`explode`), opToken(explodeOpType))
 	lexer.Add([]byte(`or`), opToken(orOpType))
 	lexer.Add([]byte(`and`), opToken(andOpType))
@@ -436,17 +501,16 @@ func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postPr
 	log.Debug("processing %v", currentToken.toString(true))
 
 	if currentToken.TokenType == traverseArrayCollect {
+		// `.[exp]`` works by creating a traversal array of [self, exp] and piping that into the traverse array operator
 		//need to put a traverse array then a collect currentToken
 		// do this by adding traverse then converting currentToken to collect
 
-		if index == 0 || tokens[index-1].TokenType != operationToken ||
-			tokens[index-1].Operation.OperationType != traversePathOpType {
-			log.Debug("  adding self")
-			op := &Operation{OperationType: selfReferenceOpType, StringValue: "SELF"}
-			postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
-		}
+		log.Debug("  adding self")
+		op := &Operation{OperationType: selfReferenceOpType, StringValue: "SELF"}
+		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
+
 		log.Debug("  adding traverse array")
-		op := &Operation{OperationType: traverseArrayOpType, StringValue: "TRAVERSE_ARRAY"}
+		op = &Operation{OperationType: traverseArrayOpType, StringValue: "TRAVERSE_ARRAY"}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 
 		currentToken = &token{TokenType: openCollect}
@@ -474,20 +538,15 @@ func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postPr
 	}
 
 	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
-		tokens[index+1].TokenType == operationToken &&
-		tokens[index+1].Operation.OperationType == traversePathOpType {
+		((tokens[index+1].TokenType == operationToken && (tokens[index+1].Operation.OperationType == traversePathOpType)) ||
+			(tokens[index+1].TokenType == traverseArrayCollect)) {
 		log.Debug("  adding pipe because the next thing is traverse")
-		op := &Operation{OperationType: shortPipeOpType, Value: "PIPE"}
+		op := &Operation{OperationType: shortPipeOpType, Value: "PIPE", StringValue: "."}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 	}
 	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
 		tokens[index+1].TokenType == openCollect {
 
-		// if tokens[index].TokenType == closeCollect {
-		// 	log.Debug("  adding pipe because next is opencollect")
-		// 	op := &Operation{OperationType: shortPipeOpType, Value: "PIPE"}
-		// 	postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
-		// }
 		log.Debug("  adding traverArray because next is opencollect")
 		op := &Operation{OperationType: traverseArrayOpType}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})

pkg/yqlib/lib.go

@@ -59,6 +59,8 @@ var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence:
 
 var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: lengthOperator}
 var collectOpType = &operationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handler: collectOperator}
+var encodeOpType = &operationType{Type: "ENCODE", NumArgs: 0, Precedence: 50, Handler: encodeOperator}
+var decodeOpType = &operationType{Type: "DECODE", NumArgs: 0, Precedence: 50, Handler: decodeOperator}
 
 var anyOpType = &operationType{Type: "ANY", NumArgs: 0, Precedence: 50, Handler: anyOperator}
 var allOpType = &operationType{Type: "ALL", NumArgs: 0, Precedence: 50, Handler: allOperator}
@@ -111,6 +113,8 @@ var selectOpType = &operationType{Type: "SELECT", NumArgs: 1, Precedence: 50, Ha
 var hasOpType = &operationType{Type: "HAS", NumArgs: 1, Precedence: 50, Handler: hasOperator}
 var uniqueOpType = &operationType{Type: "UNIQUE", NumArgs: 0, Precedence: 50, Handler: unique}
 var uniqueByOpType = &operationType{Type: "UNIQUE_BY", NumArgs: 1, Precedence: 50, Handler: uniqueBy}
+var groupByOpType = &operationType{Type: "GROUP_BY", NumArgs: 1, Precedence: 50, Handler: groupBy}
+var flattenOpType = &operationType{Type: "FLATTEN_BY", NumArgs: 0, Precedence: 50, Handler: flattenOp}
 var deleteChildOpType = &operationType{Type: "DELETE", NumArgs: 1, Precedence: 40, Handler: deleteChildOperator}
 
 type Operation struct {

pkg/yqlib/matchKeyString.go

@@ -9,26 +9,50 @@ func matchKey(name string, pattern string) (matched bool) {
 		log.Debug("wild!")
 		return true
 	}
-	return deepMatch([]rune(name), []rune(pattern))
+	return deepMatch(name, pattern)
 }
 
-func deepMatch(str, pattern []rune) bool {
-	for len(pattern) > 0 {
-		switch pattern[0] {
-		default:
-			if len(str) == 0 || str[0] != pattern[0] {
-				return false
+// deepMatch reports whether the name matches the pattern in linear time.
+// Source https://research.swtch.com/glob
+func deepMatch(name, pattern string) bool {
+	px := 0
+	nx := 0
+	nextPx := 0
+	nextNx := 0
+	for px < len(pattern) || nx < len(name) {
+		if px < len(pattern) {
+			c := pattern[px]
+			switch c {
+			default: // ordinary character
+				if nx < len(name) && name[nx] == c {
+					px++
+					nx++
+					continue
+				}
+			case '?': // single-character wildcard
+				if nx < len(name) {
+					px++
+					nx++
+					continue
+				}
+			case '*': // zero-or-more-character wildcard
+				// Try to match at nx.
+				// If that doesn't work out,
+				// restart at nx+1 next.
+				nextPx = px
+				nextNx = nx + 1
+				px++
+				continue
 			}
-		case '?':
-			if len(str) == 0 {
-				return false
-			}
-		case '*':
-			return deepMatch(str, pattern[1:]) ||
-				(len(str) > 0 && deepMatch(str[1:], pattern))
 		}
-		str = str[1:]
-		pattern = pattern[1:]
+		// Mismatch. Maybe restart.
+		if 0 < nextNx && nextNx <= len(name) {
+			px = nextPx
+			nx = nextNx
+			continue
+		}
+		return false
 	}
-	return len(str) == 0 && len(pattern) == 0
+	// Matched all of pattern to all of name. Success.
+	return true
 }

pkg/yqlib/matchKeyString_test.go

@@ -0,0 +1,40 @@
+package yqlib
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestDeepMatch(t *testing.T) {
+	var tests = []struct {
+		name    string
+		pattern string
+		ok      bool
+	}{
+		{"", "", true},
+		{"", "x", false},
+		{"x", "", false},
+		{"abc", "abc", true},
+		{"abc", "*", true},
+		{"abc", "*c", true},
+		{"abc", "*b", false},
+		{"abc", "a*", true},
+		{"abc", "b*", false},
+		{"a", "a*", true},
+		{"a", "*a", true},
+		{"axbxcxdxe", "a*b*c*d*e*", true},
+		{"axbxcxdxexxx", "a*b*c*d*e*", true},
+		{"abxbbxdbxebxczzx", "a*b?c*x", true},
+		{"abxbbxdbxebxczzy", "a*b?c*x", false},
+		{strings.Repeat("a", 100), "a*a*a*a*b", false},
+		{"xxx", "*x", true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name+" "+tt.pattern, func(t *testing.T) {
+			if want, got := tt.ok, deepMatch(tt.name, tt.pattern); want != got {
+				t.Errorf("Expected %v got %v", want, got)
+			}
+		})
+	}
+}

pkg/yqlib/operator_anchors_aliases.go

@@ -141,6 +141,48 @@ func explodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expr
 	return context, nil
 }
 
+func reconstructAliasedMap(node *yaml.Node, context Context) error {
+	var newContent = list.New()
+	// can I short cut here by prechecking if there's an anchor in the map?
+	// no it needs to recurse in overrideEntry.
+
+	for index := 0; index < len(node.Content); index = index + 2 {
+		keyNode := node.Content[index]
+		valueNode := node.Content[index+1]
+		log.Debugf("traversing %v", keyNode.Value)
+		if keyNode.Value != "<<" {
+			err := overrideEntry(node, keyNode, valueNode, index, context.ChildContext(newContent))
+			if err != nil {
+				return err
+			}
+		} else {
+			if valueNode.Kind == yaml.SequenceNode {
+				log.Debugf("an alias merge list!")
+				for index := len(valueNode.Content) - 1; index >= 0; index = index - 1 {
+					aliasNode := valueNode.Content[index]
+					err := applyAlias(node, aliasNode.Alias, index, context.ChildContext(newContent))
+					if err != nil {
+						return err
+					}
+				}
+			} else {
+				log.Debugf("an alias merge!")
+				err := applyAlias(node, valueNode.Alias, index, context.ChildContext(newContent))
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+	node.Content = make([]*yaml.Node, newContent.Len())
+	index := 0
+	for newEl := newContent.Front(); newEl != nil; newEl = newEl.Next() {
+		node.Content[index] = newEl.Value.(*yaml.Node)
+		index++
+	}
+	return nil
+}
+
 func explodeNode(node *yaml.Node, context Context) error {
 	node.Anchor = ""
 	switch node.Kind {
@@ -165,43 +207,35 @@ func explodeNode(node *yaml.Node, context Context) error {
 		}
 		return nil
 	case yaml.MappingNode:
-		var newContent = list.New()
+		// //check the map has an alias in it
+		hasAlias := false
 		for index := 0; index < len(node.Content); index = index + 2 {
 			keyNode := node.Content[index]
-			valueNode := node.Content[index+1]
-			log.Debugf("traversing %v", keyNode.Value)
-			if keyNode.Value != "<<" {
-				err := overrideEntry(node, keyNode, valueNode, index, context.ChildContext(newContent))
+			if keyNode.Value == "<<" {
+				hasAlias = true
+				break
+			}
+		}
+
+		if hasAlias {
+			// this is a slow op, which is why we want to check before running it.
+			return reconstructAliasedMap(node, context)
+		} else {
+			// this map has no aliases, but it's kids might
+			for index := 0; index < len(node.Content); index = index + 2 {
+				keyNode := node.Content[index]
+				valueNode := node.Content[index+1]
+				err := explodeNode(keyNode, context)
 				if err != nil {
 					return err
 				}
-			} else {
-				if valueNode.Kind == yaml.SequenceNode {
-					log.Debugf("an alias merge list!")
-					for index := len(valueNode.Content) - 1; index >= 0; index = index - 1 {
-						aliasNode := valueNode.Content[index]
-						err := applyAlias(node, aliasNode.Alias, index, context.ChildContext(newContent))
-						if err != nil {
-							return err
-						}
-					}
-				} else {
-					log.Debugf("an alias merge!")
-					err := applyAlias(node, valueNode.Alias, index, context.ChildContext(newContent))
-					if err != nil {
-						return err
-					}
+				err = explodeNode(valueNode, context)
+				if err != nil {
+					return err
 				}
 			}
+			return nil
 		}
-		node.Content = make([]*yaml.Node, newContent.Len())
-		index := 0
-		for newEl := newContent.Front(); newEl != nil; newEl = newEl.Next() {
-			node.Content[index] = newEl.Value.(*yaml.Node)
-			index++
-		}
-
-		return nil
 	default:
 		return nil
 	}

pkg/yqlib/operator_assign_test.go

@@ -112,9 +112,10 @@ var assignOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description: "Update selected results",
-		document:    `{a: {b: apple, c: cactus}}`,
-		expression:  `(.a[] | select(. == "apple")) = "frog"`,
+		description:    "Update deeply selected results",
+		subdescription: "Note that the LHS is wrapped in brackets! This is to ensure we dont first filter out the yaml and then update the snippet.",
+		document:       `{a: {b: apple, c: cactus}}`,
+		expression:     `(.a[] | select(. == "apple")) = "frog"`,
 		expected: []string{
 			"D0, P[], (doc)::{a: {b: frog, c: cactus}}\n",
 		},

pkg/yqlib/operator_encoder_decoder.go

@@ -0,0 +1,91 @@
+package yqlib
+
+import (
+	"bufio"
+	"bytes"
+	"container/list"
+	"regexp"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+func yamlToString(candidate *CandidateNode, prefs encoderPreferences) (string, error) {
+	var output bytes.Buffer
+	log.Debug("printing with indent: %v", prefs.indent)
+
+	printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), prefs.format, true, false, prefs.indent, true)
+	err := printer.PrintResults(candidate.AsList())
+	return output.String(), err
+}
+
+type encoderPreferences struct {
+	format PrinterOutputFormat
+	indent int
+}
+
+/* encodes object as yaml string */
+
+func encodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	preferences := expressionNode.Operation.Preferences.(encoderPreferences)
+	var results = list.New()
+
+	hasOnlyOneNewLine := regexp.MustCompile("[^\n].*\n$")
+	endWithNewLine := regexp.MustCompile(".*\n$")
+	chomper := regexp.MustCompile("\n+$")
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		stringValue, err := yamlToString(candidate, preferences)
+
+		if err != nil {
+			return Context{}, err
+		}
+
+		// remove trailing newlines if needed.
+		// check if we originally decoded this path, and the original thing had a single line.
+		originalList := context.GetVariable("decoded: " + candidate.GetKey())
+		if originalList != nil && originalList.Len() > 0 && hasOnlyOneNewLine.MatchString(stringValue) {
+
+			original := originalList.Front().Value.(*CandidateNode)
+			originalNode := unwrapDoc(original.Node)
+			// original block did not have a new line at the end, get rid of this one too
+			if !endWithNewLine.MatchString(originalNode.Value) {
+				stringValue = chomper.ReplaceAllString(stringValue, "")
+			}
+		}
+
+		// dont print a new line when printing json on a single line.
+		if preferences.format == JsonOutputFormat && preferences.indent == 0 {
+			stringValue = chomper.ReplaceAllString(stringValue, "")
+		}
+
+		stringContentNode := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: stringValue}
+		results.PushBack(candidate.CreateChild(nil, stringContentNode))
+	}
+	return context.ChildContext(results), nil
+}
+
+/* takes a string and decodes it back into an object */
+func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	var results = list.New()
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+
+		context.SetVariable("decoded: "+candidate.GetKey(), candidate.AsList())
+
+		var dataBucket yaml.Node
+		log.Debugf("got: [%v]", candidate.Node.Value)
+		decoder := yaml.NewDecoder(strings.NewReader(unwrapDoc(candidate.Node).Value))
+		errorReading := decoder.Decode(&dataBucket)
+		if errorReading != nil {
+			return Context{}, errorReading
+		}
+		//first node is a doc
+		node := unwrapDoc(&dataBucket)
+
+		results.PushBack(candidate.CreateChild(nil, node))
+	}
+	return context.ChildContext(results), nil
+}

pkg/yqlib/operator_encoder_decoder_test.go

@@ -0,0 +1,117 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var prefix = "D0, P[], (doc)::a:\n    cool:\n        bob: dylan\n"
+
+var encoderDecoderOperatorScenarios = []expressionScenario{
+	{
+		description:    "Encode value as yaml string",
+		subdescription: "Indent defaults to 2",
+		document:       "a:\n  cool:\n    bob: dylan",
+		expression:     `.b = (.a | to_yaml)`,
+		expected: []string{
+			prefix + "b: |\n    cool:\n      bob: dylan\n",
+		},
+	},
+	{
+		description:    "Encode value as yaml string, with custom indentation",
+		subdescription: "You can specify the indentation level as the first parameter.",
+		document:       "a:\n  cool:\n    bob: dylan",
+		expression:     `.b = (.a | to_yaml(8))`,
+		expected: []string{
+			prefix + "b: |\n    cool:\n            bob: dylan\n",
+		},
+	},
+	{
+		description:    "Encode value as yaml string, using toyaml",
+		subdescription: "Does the same thing as to_yaml, matching jq naming convention.",
+		document:       `{a: {cool: "thing"}}`,
+		expression:     `.b = (.a | to_yaml)`,
+		expected: []string{
+			`D0, P[], (doc)::{a: {cool: "thing"}, b: "{cool: \"thing\"}\n"}
+`,
+		},
+	},
+	{
+		description: "Encode value as json string",
+		document:    `{a: {cool: "thing"}}`,
+		expression:  `.b = (.a | to_json)`,
+		expected: []string{
+			`D0, P[], (doc)::{a: {cool: "thing"}, b: "{\n  \"cool\": \"thing\"\n}\n"}
+`,
+		},
+	},
+	{
+		description:    "Encode value as json string, on one line",
+		subdescription: "Pass in a 0 indent to print json on a single line.",
+		document:       `{a: {cool: "thing"}}`,
+		expression:     `.b = (.a | to_json(0))`,
+		expected: []string{
+			`D0, P[], (doc)::{a: {cool: "thing"}, b: '{"cool":"thing"}'}
+`,
+		},
+	},
+	{
+		description: "Encode value as props string",
+		document:    `{a: {cool: "thing"}}`,
+		expression:  `.b = (.a | to_props)`,
+		expected: []string{
+			`D0, P[], (doc)::{a: {cool: "thing"}, b: "cool = thing\n"}
+`,
+		},
+	},
+	{
+		description: "Decode a yaml encoded string",
+		document:    `a: "foo: bar"`,
+		expression:  `.b = (.a | from_yaml)`,
+		expected: []string{
+			"D0, P[], (doc)::a: \"foo: bar\"\nb:\n    foo: bar\n",
+		},
+	},
+	{
+		description:           "Update a multiline encoded yaml string",
+		dontFormatInputForDoc: true,
+		document:              "a: |\n  foo: bar\n  baz: dog\n",
+		expression:            `.a |= (from_yaml | .foo = "cat" | to_yaml)`,
+		expected: []string{
+			"D0, P[], (doc)::a: |\n    foo: cat\n    baz: dog\n",
+		},
+	},
+	{
+		skipDoc:               true,
+		dontFormatInputForDoc: true,
+		document:              "a: |-\n  foo: bar\n  baz: dog\n",
+		expression:            `.a |= (from_yaml | .foo = "cat" | to_yaml)`,
+		expected: []string{
+			"D0, P[], (doc)::a: |-\n    foo: cat\n    baz: dog\n",
+		},
+	},
+	{
+		description:           "Update a single line encoded yaml string",
+		dontFormatInputForDoc: true,
+		document:              "a: 'foo: bar'",
+		expression:            `.a |= (from_yaml | .foo = "cat" | to_yaml)`,
+		expected: []string{
+			"D0, P[], (doc)::a: 'foo: cat'\n",
+		},
+	},
+	{
+		skipDoc:               true,
+		dontFormatInputForDoc: true,
+		document:              "a: \"foo: bar\"",
+		expression:            `.a |= (from_yaml | .foo = {"a": "frog"} | to_yaml)`,
+		expected: []string{
+			"D0, P[], (doc)::a: \"foo:\\n  a: frog\"\n",
+		},
+	},
+}
+
+func TestEncoderDecoderOperatorScenarios(t *testing.T) {
+	for _, tt := range encoderDecoderOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "Encoder and Decoder", encoderDecoderOperatorScenarios)
+}

pkg/yqlib/operator_flatten.go

@@ -0,0 +1,54 @@
+package yqlib
+
+import (
+	"fmt"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+type flattenPreferences struct {
+	depth int
+}
+
+func flatten(node *yaml.Node, depth int) {
+	if depth == 0 {
+		return
+	}
+	if node.Kind != yaml.SequenceNode {
+		return
+	}
+	content := node.Content
+	newSeq := make([]*yaml.Node, 0)
+
+	for i := 0; i < len(content); i++ {
+		if content[i].Kind == yaml.SequenceNode {
+			flatten(content[i], depth-1)
+			for j := 0; j < len(content[i].Content); j++ {
+				newSeq = append(newSeq, content[i].Content[j])
+			}
+		} else {
+			newSeq = append(newSeq, content[i])
+		}
+	}
+	node.Content = newSeq
+}
+
+func flattenOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	log.Debugf("-- flatten Operator")
+	depth := expressionNode.Operation.Preferences.(flattenPreferences).depth
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		candidateNode := unwrapDoc(candidate.Node)
+		if candidateNode.Kind != yaml.SequenceNode {
+			return Context{}, fmt.Errorf("Only arrays are supported for flatten")
+		}
+
+		flatten(candidateNode, depth)
+
+	}
+
+	return context, nil
+
+}

pkg/yqlib/operator_flatten_test.go

@@ -0,0 +1,48 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var flattenOperatorScenarios = []expressionScenario{
+	{
+		description:    "Flatten",
+		subdescription: "Recursively flattens all arrays",
+		document:       `[1, [2], [[3]]]`,
+		expression:     `flatten`,
+		expected: []string{
+			"D0, P[], (doc)::[1, 2, 3]\n",
+		},
+	},
+	{
+		description: "Flatten with depth of one",
+		document:    `[1, [2], [[3]]]`,
+		expression:  `flatten(1)`,
+		expected: []string{
+			"D0, P[], (doc)::[1, 2, [3]]\n",
+		},
+	},
+	{
+		description: "Flatten empty array",
+		document:    `[[]]`,
+		expression:  `flatten`,
+		expected: []string{
+			"D0, P[], (doc)::[]\n",
+		},
+	},
+	{
+		description: "Flatten array of objects",
+		document:    `[{foo: bar}, [{foo: baz}]]`,
+		expression:  `flatten`,
+		expected: []string{
+			"D0, P[], (doc)::[{foo: bar}, {foo: baz}]\n",
+		},
+	},
+}
+
+func TestFlattenOperatorScenarios(t *testing.T) {
+	for _, tt := range flattenOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "Flatten", flattenOperatorScenarios)
+}

pkg/yqlib/operator_group_by.go

@@ -0,0 +1,76 @@
+package yqlib
+
+import (
+	"container/list"
+	"fmt"
+
+	"github.com/elliotchance/orderedmap"
+	yaml "gopkg.in/yaml.v3"
+)
+
+func processIntoGroups(d *dataTreeNavigator, context Context, rhsExp *ExpressionNode, node *yaml.Node) (*orderedmap.OrderedMap, error) {
+	var newMatches = orderedmap.NewOrderedMap()
+	for _, node := range node.Content {
+		child := &CandidateNode{Node: node}
+		rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(child), rhsExp)
+
+		if err != nil {
+			return nil, err
+		}
+
+		keyValue := "null"
+
+		if rhs.MatchingNodes.Len() > 0 {
+			first := rhs.MatchingNodes.Front()
+			keyCandidate := first.Value.(*CandidateNode)
+			keyValue = keyCandidate.Node.Value
+		}
+
+		groupList, exists := newMatches.Get(keyValue)
+
+		if !exists {
+			groupList = list.New()
+			newMatches.Set(keyValue, groupList)
+		}
+		groupList.(*list.List).PushBack(node)
+	}
+	return newMatches, nil
+}
+
+func groupBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	log.Debugf("-- groupBy Operator")
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		candidateNode := unwrapDoc(candidate.Node)
+
+		if candidateNode.Kind != yaml.SequenceNode {
+			return Context{}, fmt.Errorf("Only arrays are supported for group by")
+		}
+
+		newMatches, err := processIntoGroups(d, context, expressionNode.Rhs, candidateNode)
+
+		if err != nil {
+			return Context{}, err
+		}
+
+		resultNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+		for groupEl := newMatches.Front(); groupEl != nil; groupEl = groupEl.Next() {
+			groupResultNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+			groupList := groupEl.Value.(*list.List)
+			for groupItem := groupList.Front(); groupItem != nil; groupItem = groupItem.Next() {
+				groupResultNode.Content = append(groupResultNode.Content, groupItem.Value.(*yaml.Node))
+			}
+
+			resultNode.Content = append(resultNode.Content, groupResultNode)
+		}
+
+		results.PushBack(candidate.CreateChild(nil, resultNode))
+
+	}
+
+	return context.ChildContext(results), nil
+
+}

pkg/yqlib/operator_group_by_test.go

@@ -0,0 +1,31 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var groupByOperatorScenarios = []expressionScenario{
+	{
+		description: "Group by field",
+		document:    `[{foo: 1, bar: 10}, {foo: 3, bar: 100}, {foo: 1, bar: 1}]`,
+		expression:  `group_by(.foo)`,
+		expected: []string{
+			"D0, P[], (!!seq)::- - {foo: 1, bar: 10}\n  - {foo: 1, bar: 1}\n- - {foo: 3, bar: 100}\n",
+		},
+	},
+	{
+		description: "Group by field, with nuls",
+		document:    `[{cat: dog}, {foo: 1, bar: 10}, {foo: 3, bar: 100}, {no: foo for you}, {foo: 1, bar: 1}]`,
+		expression:  `group_by(.foo)`,
+		expected: []string{
+			"D0, P[], (!!seq)::- - {cat: dog}\n  - {no: foo for you}\n- - {foo: 1, bar: 10}\n  - {foo: 1, bar: 1}\n- - {foo: 3, bar: 100}\n",
+		},
+	},
+}
+
+func TestGroupByOperatorScenarios(t *testing.T) {
+	for _, tt := range groupByOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "Group By", groupByOperatorScenarios)
+}

pkg/yqlib/operator_multiply.go

@@ -6,6 +6,7 @@ import (
 
 	"container/list"
 
+	"github.com/jinzhu/copier"
 	yaml "gopkg.in/yaml.v3"
 )
 
@@ -50,14 +51,21 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
 
 		if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode ||
 			(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
-
-			var newBlank = lhs.CreateChild(nil, newBlankNode)
-			log.Debugf("merge - merge lhs into blank")
-			var newThing, err = mergeObjects(d, context.WritableClone(), newBlank, lhs, multiplyPreferences{})
+			var newBlank = CandidateNode{}
+			err := copier.CopyWithOption(&newBlank, lhs, copier.Option{IgnoreEmpty: true, DeepCopy: true})
 			if err != nil {
 				return nil, err
 			}
-			return mergeObjects(d, context.WritableClone(), newThing, rhs, preferences)
+			newBlank.Node.HeadComment = newBlankNode.HeadComment
+			newBlank.Node.FootComment = newBlankNode.FootComment
+
+			// var newBlank = lhs.CreateChild(nil, newBlankNode)
+			// log.Debugf("merge - merge lhs into blank")
+			// var newThing, err = mergeObjects(d, context.WritableClone(), newBlank, lhs, multiplyPreferences{})
+			// if err != nil {
+			// 	return nil, err
+			// }
+			return mergeObjects(d, context.WritableClone(), &newBlank, rhs, preferences)
 		} else if lhs.Node.Tag == "!!int" && rhs.Node.Tag == "!!int" {
 			return multiplyIntegers(lhs, rhs)
 		} else if (lhs.Node.Tag == "!!int" || lhs.Node.Tag == "!!float") && (rhs.Node.Tag == "!!int" || rhs.Node.Tag == "!!float") {

pkg/yqlib/operator_multiply_test.go

@@ -60,6 +60,13 @@ var nodeWithFooter = `
 a: apple
 # footer`
 
+var document = `
+a: &cat {name: cat}
+b: {name: dog}
+c: 
+  <<: *cat
+`
+
 var multiplyOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
@@ -405,7 +412,15 @@ var multiplyOperatorScenarios = []expressionScenario{
 		document:    mergeDocSample,
 		expression:  `.foobar * .foobarList`,
 		expected: []string{
-			"D0, P[foobar], (!!map)::c: foobarList_c\n<<: [*foo, *bar]\nthing: foobar_thing\nb: foobarList_b\n",
+			"D0, P[foobar], (!!map)::c: foobarList_c\n!!merge <<: [*foo, *bar]\nthing: foobar_thing\nb: foobarList_b\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   document,
+		expression: `.b * .c`,
+		expected: []string{
+			"D0, P[b], (!!map)::{name: dog, <<: *cat}\n",
 		},
 	},
 }

pkg/yqlib/operator_select.go

@@ -17,22 +17,25 @@ func selectOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 			return Context{}, err
 		}
 
-		// grab the first value
-		first := rhs.MatchingNodes.Front()
+		// find any truthy node
+		var errDecoding error
+		includeResult := false
 
-		if first != nil {
-			result := first.Value.(*CandidateNode)
-			log.Debugf("result %v", NodeToString(result))
-			includeResult, errDecoding := isTruthy(result)
+		for resultEl := rhs.MatchingNodes.Front(); resultEl != nil; resultEl = resultEl.Next() {
+			result := resultEl.Value.(*CandidateNode)
+			includeResult, errDecoding = isTruthy(result)
 			log.Debugf("isTruthy %v", includeResult)
 			if errDecoding != nil {
 				return Context{}, errDecoding
 			}
-
 			if includeResult {
-				results.PushBack(candidate)
+				break
 			}
 		}
+
+		if includeResult {
+			results.PushBack(candidate)
+		}
 	}
 	return context.ChildContext(results), nil
 }

pkg/yqlib/operator_select_test.go

@@ -5,6 +5,28 @@ import (
 )
 
 var selectOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   `cat`,
+		expression: `select(false, true)`,
+		expected: []string{
+			"D0, P[], (doc)::cat\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `cat`,
+		expression: `select(true, false)`,
+		expected: []string{
+			"D0, P[], (doc)::cat\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `cat`,
+		expression: `select(false)`,
+		expected:   []string{},
+	},
 	{
 		description: "Select elements from array",
 		document:    `[cat,goat,dog]`,

pkg/yqlib/operators_test.go

@@ -107,7 +107,7 @@ func copyFromHeader(title string, out *os.File) error {
 
 func formatYaml(yaml string, filename string) string {
 	var output bytes.Buffer
-	printer := NewPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
 
 	node, err := NewExpressionParser().ParseExpression(".. style= \"\"")
 	if err != nil {
@@ -216,7 +216,7 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) {
 func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formattedDoc string, formattedDoc2 string) {
 	var output bytes.Buffer
 	var err error
-	printer := NewPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
 
 	node, err := NewExpressionParser().ParseExpression(s.expression)
 	if err != nil {

pkg/yqlib/printer.go

@@ -44,7 +44,7 @@ type resultsPrinter struct {
 	colorsEnabled      bool
 	indent             int
 	printDocSeparators bool
-	writer             io.Writer
+	printerWriter      PrinterWriter
 	firstTimePrinting  bool
 	previousDocIndex   uint
 	previousFileIndex  int
@@ -53,9 +53,13 @@ type resultsPrinter struct {
 	appendixReader     io.Reader
 }
 
-func NewPrinter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
+func NewPrinterWithSingleWriter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
+	return NewPrinter(NewSinglePrinterWriter(writer), outputFormat, unwrapScalar, colorsEnabled, indent, printDocSeparators)
+}
+
+func NewPrinter(printerWriter PrinterWriter, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
 	return &resultsPrinter{
-		writer:             writer,
+		printerWriter:      printerWriter,
 		outputFormat:       outputFormat,
 		unwrapScalar:       unwrapScalar,
 		colorsEnabled:      colorsEnabled,
@@ -98,15 +102,57 @@ func (p *resultsPrinter) writeString(writer io.Writer, txt string) error {
 	return errorWriting
 }
 
-func (p *resultsPrinter) safelyFlush(writer *bufio.Writer) {
-	if err := writer.Flush(); err != nil {
-		log.Error("Error flushing writer!")
-		log.Error(err.Error())
+func (p *resultsPrinter) processLeadingContent(mappedDoc *CandidateNode, writer io.Writer) error {
+	if strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$") {
+		log.Debug("headcommentwas %v", mappedDoc.Node.HeadComment)
+		log.Debug("finished headcomment")
+		reader := bufio.NewReader(strings.NewReader(mappedDoc.Node.HeadComment))
+		mappedDoc.Node.HeadComment = ""
+
+		for {
+
+			readline, errReading := reader.ReadString('\n')
+			if errReading != nil && errReading != io.EOF {
+				return errReading
+			}
+			if strings.Contains(readline, "$yqLeadingContent$") {
+				// skip this
+
+			} else if strings.Contains(readline, "$yqDocSeperator$") {
+				if p.printDocSeparators {
+					if err := p.writeString(writer, "---\n"); err != nil {
+						return err
+					}
+				}
+			} else if p.outputFormat == YamlOutputFormat {
+				if err := p.writeString(writer, readline); err != nil {
+					return err
+				}
+			}
+
+			if errReading == io.EOF {
+				if readline != "" {
+					// the last comment we read didn't have a new line, put one in
+					if err := p.writeString(writer, "\n"); err != nil {
+						return err
+					}
+				}
+				break
+			}
+		}
+
 	}
+	return nil
 }
 
 func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 	log.Debug("PrintResults for %v matches", matchingNodes.Len())
+
+	if matchingNodes.Len() == 0 {
+		log.Debug("no matching results, nothing to print")
+		return nil
+	}
+
 	if p.outputFormat != YamlOutputFormat {
 		explodeOp := Operation{OperationType: explodeOpType}
 		explodeNode := ExpressionNode{Operation: &explodeOp}
@@ -117,13 +163,6 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 		matchingNodes = context.MatchingNodes
 	}
 
-	bufferedWriter := bufio.NewWriter(p.writer)
-	defer p.safelyFlush(bufferedWriter)
-
-	if matchingNodes.Len() == 0 {
-		log.Debug("no matching results, nothing to print")
-		return nil
-	}
 	if p.firstTimePrinting {
 		node := matchingNodes.Front().Value.(*CandidateNode)
 		p.previousDocIndex = node.Document
@@ -132,72 +171,53 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 	}
 
 	for el := matchingNodes.Front(); el != nil; el = el.Next() {
+
 		mappedDoc := el.Value.(*CandidateNode)
 		log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v, printDocSeparators: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document, p.printDocSeparators)
 
+		writer, errorWriting := p.printerWriter.GetWriter(mappedDoc)
+		if errorWriting != nil {
+			return errorWriting
+		}
+
 		commentStartsWithSeparator := strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$\n$yqDocSeperator$")
 
 		if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators && !commentStartsWithSeparator {
 			log.Debug("-- writing doc sep")
-			if err := p.writeString(bufferedWriter, "---\n"); err != nil {
+			if err := p.writeString(writer, "---\n"); err != nil {
 				return err
 			}
 		}
 
-		if strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$") {
-			log.Debug("headcommentwas %v", mappedDoc.Node.HeadComment)
-			log.Debug("finished headcomment")
-			reader := bufio.NewReader(strings.NewReader(mappedDoc.Node.HeadComment))
-			mappedDoc.Node.HeadComment = ""
-
-			for {
-
-				readline, errReading := reader.ReadString('\n')
-				if errReading != nil && errReading != io.EOF {
-					return errReading
-				}
-				if strings.Contains(readline, "$yqLeadingContent$") {
-					// skip this
-
-				} else if strings.Contains(readline, "$yqDocSeperator$") {
-					if p.printDocSeparators {
-						if err := p.writeString(bufferedWriter, "---\n"); err != nil {
-							return err
-						}
-					}
-				} else if p.outputFormat == YamlOutputFormat {
-					if err := p.writeString(bufferedWriter, readline); err != nil {
-						return err
-					}
-				}
-
-				if errReading == io.EOF {
-					if readline != "" {
-						// the last comment we read didn't have a new line, put one in
-						if err := p.writeString(bufferedWriter, "\n"); err != nil {
-							return err
-						}
-					}
-					break
-				}
-			}
-
+		if err := p.processLeadingContent(mappedDoc, writer); err != nil {
+			return err
 		}
 
-		if err := p.printNode(mappedDoc.Node, bufferedWriter); err != nil {
+		if err := p.printNode(mappedDoc.Node, writer); err != nil {
 			return err
 		}
 
 		p.previousDocIndex = mappedDoc.Document
+		if err := writer.Flush(); err != nil {
+			return err
+		}
 	}
 
 	if p.appendixReader != nil && p.outputFormat == YamlOutputFormat {
+		writer, err := p.printerWriter.GetWriter(nil)
+		if err != nil {
+			return err
+		}
+
 		log.Debug("Piping appendix reader...")
 		betterReader := bufio.NewReader(p.appendixReader)
-		_, err := io.Copy(bufferedWriter, betterReader)
+		_, err = io.Copy(writer, betterReader)
 		if err != nil {
 			return err
 		}
+		if err := writer.Flush(); err != nil {
+			return err
+		}
 	}
 
 	return nil

pkg/yqlib/printer_test.go

@@ -36,7 +36,7 @@ func nodeToList(candidate *CandidateNode) *list.List {
 func TestPrinterMultipleDocsInSequence(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -74,7 +74,7 @@ func TestPrinterMultipleDocsInSequence(t *testing.T) {
 func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -116,7 +116,7 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
 func TestPrinterMultipleFilesInSequence(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -163,7 +163,7 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) {
 func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -213,7 +213,7 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
 func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -232,7 +232,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -261,7 +261,7 @@ a: coconut
 func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -287,7 +287,7 @@ a: coconut
 func TestPrinterScalarWithLeadingCont(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	node, err := NewExpressionParser().ParseExpression(".a")
 	if err != nil {
@@ -314,7 +314,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	// note printDocSeparators is true, it should still not print document separators
 	// when outputing JSON.
-	printer := NewPrinter(writer, JsonOutputFormat, true, false, 0, true)
+	printer := NewPrinterWithSingleWriter(writer, JsonOutputFormat, true, false, 0, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {

pkg/yqlib/printer_writer.go

@@ -0,0 +1,81 @@
+package yqlib
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+
+	"gopkg.in/yaml.v3"
+)
+
+type PrinterWriter interface {
+	GetWriter(node *CandidateNode) (*bufio.Writer, error)
+}
+
+type singlePrinterWriter struct {
+	bufferedWriter *bufio.Writer
+}
+
+func NewSinglePrinterWriter(writer io.Writer) PrinterWriter {
+	return &singlePrinterWriter{
+		bufferedWriter: bufio.NewWriter(writer),
+	}
+}
+
+func (sp *singlePrinterWriter) GetWriter(node *CandidateNode) (*bufio.Writer, error) {
+	return sp.bufferedWriter, nil
+}
+
+type multiPrintWriter struct {
+	treeNavigator  DataTreeNavigator
+	nameExpression *ExpressionNode
+	extension      string
+	index          int
+}
+
+func NewMultiPrinterWriter(expression *ExpressionNode, format PrinterOutputFormat) PrinterWriter {
+	extension := "yml"
+
+	switch format {
+	case JsonOutputFormat:
+		extension = "json"
+	case PropsOutputFormat:
+		extension = "properties"
+	}
+
+	return &multiPrintWriter{
+		nameExpression: expression,
+		extension:      extension,
+		treeNavigator:  NewDataTreeNavigator(),
+		index:          0,
+	}
+}
+
+func (sp *multiPrintWriter) GetWriter(node *CandidateNode) (*bufio.Writer, error) {
+	name := ""
+
+	indexVariableNode := yaml.Node{Kind: yaml.ScalarNode, Tag: "!!int", Value: fmt.Sprintf("%v", sp.index)}
+	indexVariableCandidate := CandidateNode{Node: &indexVariableNode}
+
+	context := Context{MatchingNodes: node.AsList()}
+	context.SetVariable("index", indexVariableCandidate.AsList())
+	result, err := sp.treeNavigator.GetMatchingNodes(context, sp.nameExpression)
+	if err != nil {
+		return nil, err
+	}
+	if result.MatchingNodes.Len() > 0 {
+		name = result.MatchingNodes.Front().Value.(*CandidateNode).Node.Value
+	}
+	name = fmt.Sprintf("%v.%v", name, sp.extension)
+
+	f, err := os.Create(name)
+
+	if err != nil {
+		return nil, err
+	}
+	sp.index = sp.index + 1
+
+	return bufio.NewWriter(f), nil
+
+}

release_notes.txt

@@ -1,3 +1,21 @@
+4.14.1:
+  - Added group_by operator
+  - Added encode/decode operators (toyaml, fromjson etc)
+  - Added flatten operator
+
+4.13.5:
+  - Performance improvement for deepMatch (thanks @pmatseykanets)
+  - Added manpage, included in tar.gz downloads as well as a separate tar.gz (#961)
+  - Fixed expression parsing bug #970
+  - Rebuild fixes CVE (#964)
+  - Bumped docker alpine version
+
+
+4.13.4:
+  - Fixed select bug (#958)
+  - Improved performance of `explode` (this will also speed up json conversion)
+  - Improved performance of `merge` (significantly if your merging a small file into a big one)
+
 4.13.3:
 
 - Updated go compiler to 1.17 to fix CVE (#944)

scripts/generate-man-page-md.sh

@@ -0,0 +1,19 @@
+#! /bin/bash
+set -e
+
+# note that this reqires pandoc to be installed.
+
+cat ./pkg/yqlib/doc/headers/Main.md > man.md
+printf "\n# HOW IT WORKS\n" >> man.md
+cat ./pkg/yqlib/doc/aa.md >> man.md
+
+for f in ./pkg/yqlib/doc/*.md; do 
+  docNameWithExt="${f##*/}"
+  docName="${docNameWithExt%.*}"
+  docNameCap=$(echo $docName | tr [a-z] [A-Z])
+  if [ "$docName" != "aa" ]; then
+    printf "\n\n# ${docNameCap}\n" >> man.md
+    cat "$f" >> man.md
+  fi
+  
+done

scripts/generate-man-page.sh

@@ -0,0 +1,11 @@
+#! /bin/bash
+set -e
+
+# note that this reqires pandoc to be installed.
+
+pandoc \
+  --variable=title:"YQ" \
+  --variable=section:"1" \
+  --variable=author:"Mike Farah" \
+  --variable=header:"${MAN_HEADER}" \
+  --standalone --to man man.md -o yq.1

scripts/install-man-page.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+cp yq.1 /usr/local/share/man/man1/.

scripts/xcompile.sh

@@ -8,7 +8,8 @@ CGO_ENABLED=0 gox -ldflags "${LDFLAGS}" -output="build/yq_{{.OS}}_{{.Arch}}" --o
 
 cd build
 
-find . -executable -type f | xargs -I {} tar czvf {}.tar.gz {}
+find . -executable -type f | xargs -I {} tar czvf {}.tar.gz {} yq.1 -C ../scripts install-man-page.sh
+tar czvf yq_man_page_only.tar.gz yq.1 -C ../scripts install-man-page.sh
 
 rhash -r -a . -o checksums
 
@@ -17,6 +18,7 @@ rhash --list-hashes > checksums_hashes_order
 # just in case find thinks this is executable...
 rm -f checksums_hashes_order.tar.gz 
 rm -f checksums.tar.gz
+rm yq.1 
 
 rm yq_windows_386.exe.tar.gz
 rm yq_windows_amd64.exe.tar.gz

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.13.3'
+version: '4.13.5'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.