Split printer!

.github/workflows/release-test.yml

@@ -1,95 +0,0 @@
-name: Test Release YQ
-on:
-  push:
-    tags:
-      - 'releaseTest-*'
-
-jobs:
-  publishGitRelease:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
-        with:
-          go-version: '^1.17.2'
-      - name: Compile man page markup
-        id: gen-man-page-md
-        run: |
-          ./scripts/generate-man-page-md.sh
-          echo "::set-output name=man-page-md::man.md"
-
-      - name: Get the version
-        id: get_version
-        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
-      - name: Generate man page
-        uses: docker://pandoc/core:2.14.2
-        id: gen-man-page
-        env:
-          MAN_HEADER: yq (https://github.com/mikefarah/yq/) version ${{ steps.get_version.outputs.VERSION }}
-        with:
-          args: >-
-            --standalone
-            --to man
-            --variable=title:"YQ"
-            --variable=section:"1"
-            --variable=header:"${{MAN_HEADER}}"
-            --variable=author:"Mike Farah"
-            --output=yq.1
-            ${{ steps.gen-man-page-md.outputs.man-page-md }}
-        
-      - name: Cross compile
-        run: |
-          sudo apt-get install rhash -y
-          go get github.com/mitchellh/gox
-          mkdir -p build
-          cp yq.1 build/yq.1
-          ./scripts/xcompile.sh
-
-      - name: Create Release
-        id: create_release
-        uses: actions/create-release@v1.0.0
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          tag_name: ${{ github.ref }}
-          release_name: TEST ${{ github.ref }}
-          draft: true
-          prerelease: false
-
-      - uses: shogo82148/actions-upload-release-asset@v1
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: build/*
-
-  # publishDocker:
-  #   environment: dockerhub
-  #   env:
-  #     IMAGE_NAME: mikefarah/yq
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - uses: actions/checkout@v2
-      
-  #     - name: Set up QEMU
-  #       uses: docker/setup-qemu-action@v1
-  #       with:
-  #         platforms: all
-  
-  #     - name: Set up Docker Buildx
-  #       id: buildx
-  #       uses: docker/setup-buildx-action@v1
-  #       with:
-  #         version: latest
-
-  #     - name: Available platforms
-  #       run: echo ${{ steps.buildx.outputs.platforms }} && docker version
- 
-  #     - name: Build image
-  #       run: |
-  #         IMAGE_V_VERSION="$(git describe --tags --abbrev=0)"
-  #         IMAGE_VERSION=${IMAGE_V_VERSION:1}
-
-  #         SHORT_SHA1=$(git rev-parse --short HEAD)
-  #         PLATFORMS="linux/amd64,linux/ppc64le,linux/arm64"
-  #         echo "Building version ${IMAGE_VERSION} of image ${IMAGE_NAME}"
-  #         docker buildx build --platform "${PLATFORMS}" .
-    

.github/workflows/release.yml

@@ -12,10 +12,36 @@ jobs:
       - uses: actions/setup-go@v2
         with:
           go-version: '^1.17.2'
+      - name: Compile man page markup
+        id: gen-man-page-md
+        run: |
+          ./scripts/generate-man-page-md.sh
+          echo "::set-output name=man-page-md::man.md"
+
+      - name: Get the version
+        id: get_version
+        run: echo ::set-output name=VERSION::${GITHUB_REF##*/}
+        
+      - name: Generate man page
+        uses: docker://pandoc/core:2.14.2
+        id: gen-man-page
+        with:
+          args: >-
+            --standalone
+            --to man
+            --variable=title:"YQ"
+            --variable=section:"1"
+            --variable=header:"yq (https://github.com/mikefarah/yq/) version ${{ steps.get_version.outputs.VERSION }}"
+            --variable=author:"Mike Farah"
+            --output=yq.1
+            ${{ steps.gen-man-page-md.outputs.man-page-md }}
+        
       - name: Cross compile
         run: |
           sudo apt-get install rhash -y
           go get github.com/mitchellh/gox
+          mkdir -p build
+          cp yq.1 build/yq.1
           ./scripts/xcompile.sh
 
       - name: Create Release

.gitignore

@@ -38,8 +38,12 @@ parts/
 prime/
 .snapcraft/
 yq*.snap
+
 test.yml
-test2.yml
+test*.yml
+0.yml
+1.yml
+2.yml
 
 
 # man page

acceptance_tests/basic.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 setUp() {
-  rm -f test.yml
+  rm test*.yml || true
 }
 
 testBasicEvalRoundTrip() {

acceptance_tests/empty.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 setUp() {
+  rm test*.yml || true
   cat >test.yml <<EOL
 # comment
 EOL

acceptance_tests/front-matter.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 setUp() {
+  rm test*.yml || true
   cat >test.yml <<EOL
 ---
 a: apple

acceptance_tests/leading-seperator.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 setUp() {
+  rm test*.yml || true
   cat >test.yml <<EOL
 ---
 a: test

acceptance_tests/output-format.sh

@@ -1,5 +1,9 @@
 #!/bin/bash
 
+setUp() {
+  rm test*.yml || true
+}
+
 testOutputJsonDeprecated() {
   cat >test.yml <<EOL
 a: {b: ["cat"]}

acceptance_tests/pretty-print.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+setUp() {
+  rm test*.yml || true
+}
 
 testPrettyPrintWithBooleans() {
   cat >test.yml <<EOL

acceptance_tests/split-printer.sh

@@ -0,0 +1,157 @@
+#!/bin/bash
+
+setUp() {
+  rm test*.yml || true
+}
+
+testBasicSplitWithName() {
+  cat >test.yml <<EOL
+a: test_doc1
+--- 
+a: test_doc2
+EOL
+
+  ./yq e test.yml -s ".a"
+
+  doc1=$(cat test_doc1.yml)
+  
+  assertEquals "a: test_doc1" "$doc1"
+
+  doc2=$(cat test_doc2.yml)
+  read -r -d '' expectedDoc2 << EOM
+---
+a: test_doc2
+EOM
+  assertEquals "$expectedDoc2" "$doc2"
+}
+
+testBasicSplitWithNameEvalAll() {
+  cat >test.yml <<EOL
+a: test_doc1
+--- 
+a: test_doc2
+EOL
+
+  ./yq ea test.yml -s ".a"
+
+  doc1=$(cat test_doc1.yml)
+  
+  assertEquals "a: test_doc1" "$doc1"
+
+  doc2=$(cat test_doc2.yml)
+  read -r -d '' expectedDoc2 << EOM
+---
+a: test_doc2
+EOM
+  assertEquals "$expectedDoc2" "$doc2"
+}
+
+testBasicSplitWithIndex() {
+  cat >test.yml <<EOL
+a: test_doc1
+--- 
+a: test_doc2
+EOL
+
+  ./yq e test.yml -s '"test_" + $index'
+
+  doc1=$(cat test_0.yml)
+  
+  assertEquals "a: test_doc1" "$doc1"
+
+  doc2=$(cat test_1.yml)
+  read -r -d '' expectedDoc2 << EOM
+---
+a: test_doc2
+EOM
+  assertEquals "$expectedDoc2" "$doc2"
+}
+
+testBasicSplitWithIndexEvalAll() {
+  cat >test.yml <<EOL
+a: test_doc1
+--- 
+a: test_doc2
+EOL
+
+  ./yq ea test.yml -s '"test_" + $index'
+
+  doc1=$(cat test_0.yml)
+  
+  assertEquals "a: test_doc1" "$doc1"
+
+  doc2=$(cat test_1.yml)
+  read -r -d '' expectedDoc2 << EOM
+---
+a: test_doc2
+EOM
+  assertEquals "$expectedDoc2" "$doc2"
+}
+
+
+testArraySplitWithNameNoSeparators() {
+  cat >test.yml <<EOL
+- name: test_fred
+  age: 35
+- name: test_catherine
+  age: 37
+EOL
+
+  ./yq e --no-doc -s ".name"  ".[]" test.yml 
+
+  doc1=$(cat test_fred.yml)
+  read -r -d '' expectedDoc1 << EOM
+name: test_fred
+age: 35
+EOM
+
+  assertEquals "$expectedDoc1" "$doc1"
+
+  doc2=$(cat test_catherine.yml)
+  read -r -d '' expectedDoc2 << EOM
+name: test_catherine
+age: 37
+EOM
+  assertEquals "$expectedDoc2" "$doc2"
+}
+
+testArraySplitWithNameNoSeparatorsEvalAll() {
+  cat >test.yml <<EOL
+- name: test_fred
+  age: 35
+- name: test_catherine
+  age: 37
+EOL
+
+cat >test2.yml <<EOL
+- name: test_mike
+  age: 564
+EOL
+
+  ./yq ea --no-doc -s ".name"  ".[]" test.yml test2.yml
+
+  doc1=$(cat test_fred.yml)
+  read -r -d '' expectedDoc1 << EOM
+name: test_fred
+age: 35
+EOM
+
+  assertEquals "$expectedDoc1" "$doc1"
+
+  doc2=$(cat test_catherine.yml)
+  read -r -d '' expectedDoc2 << EOM
+name: test_catherine
+age: 37
+EOM
+  assertEquals "$expectedDoc2" "$doc2"
+
+
+  doc3=$(cat test_mike.yml)
+  read -r -d '' expectedDoc3 << EOM
+name: test_mike
+age: 564
+EOM
+  assertEquals "$expectedDoc3" "$doc3"
+}
+
+source ./scripts/shunit2

cmd/constant.go

@@ -21,4 +21,6 @@ var prettyPrint = false
 // can be either "" (off), "extract" or "process"
 var frontMatter = ""
 
+var splitFileExp = ""
+
 var completedSuccessfully = false

cmd/evaluate_all_command.go

@@ -70,6 +70,10 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("Write inplace flag only applicable when giving an expression and at least one file")
 	}
 
+	if writeInplace && splitFileExp != "" {
+		return fmt.Errorf("Write inplace cannot be used with split file")
+	}
+
 	if writeInplace {
 		// only use colors if its forced
 		colorsEnabled = forceColor
@@ -96,7 +100,9 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	printer := yqlib.NewPrinter(out, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
+	printerWriter := configurePrinterWriter(format, out)
+
+	printer := yqlib.NewPrinter(printerWriter, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
 
 	if frontMatter != "" {
 		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])

cmd/evalute_sequence_command.go

@@ -83,6 +83,10 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("Write inplace flag only applicable when giving an expression and at least one file")
 	}
 
+	if writeInplace && splitFileExp != "" {
+		return fmt.Errorf("Write inplace cannot be used with split file")
+	}
+
 	if writeInplace {
 		// only use colors if its forced
 		colorsEnabled = forceColor
@@ -106,7 +110,9 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	printer := yqlib.NewPrinter(out, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
+	printerWriter := configurePrinterWriter(format, out)
+
+	printer := yqlib.NewPrinter(printerWriter, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
 
 	streamEvaluator := yqlib.NewStreamEvaluator()
 

cmd/printer_utils.go

@@ -0,0 +1,28 @@
+package cmd
+
+import (
+	"io"
+
+	"github.com/mikefarah/yq/v4/pkg/yqlib"
+)
+
+func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) yqlib.PrinterWriter {
+
+	var printerWriter yqlib.PrinterWriter
+
+	if splitFileExp == "i" {
+		colorsEnabled = forceColor
+		printerWriter = yqlib.NewMultiPrinterWriter(nil, format)
+
+	} else if splitFileExp != "" {
+		colorsEnabled = forceColor
+		splitExp, err := yqlib.NewExpressionParser().ParseExpression(splitFileExp)
+		if err != nil {
+			return nil
+		}
+		printerWriter = yqlib.NewMultiPrinterWriter(splitExp, format)
+	} else {
+		printerWriter = yqlib.NewSinglePrinterWriter(out)
+	}
+	return printerWriter
+}

cmd/root.go

@@ -63,6 +63,9 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
 	rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
 	rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression. This is a workaround for go-yaml to persist header content properly.")
+
+	rootCmd.PersistentFlags().StringVarP(&splitFileExp, "split-exp", "s", "", "print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter.")
+
 	rootCmd.AddCommand(
 		createEvaluateSequenceCommand(),
 		createEvaluateAllCommand(),

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.13.4"
+	Version = "4.13.5"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

github-action/Dockerfile

@@ -1,4 +1,4 @@
-FROM mikefarah/yq:4.13.4
+FROM mikefarah/yq:4.13.5
 
 COPY entrypoint.sh /entrypoint.sh
 

pkg/yqlib/candidate_node.go

@@ -1,6 +1,7 @@
 package yqlib
 
 import (
+	"container/list"
 	"fmt"
 
 	"github.com/jinzhu/copier"
@@ -28,6 +29,12 @@ func (n *CandidateNode) GetKey() string {
 	return fmt.Sprintf("%v%v - %v", keyPrefix, n.Document, n.Path)
 }
 
+func (n *CandidateNode) AsList() *list.List {
+	elMap := list.New()
+	elMap.PushBack(n)
+	return elMap
+}
+
 func (n *CandidateNode) CreateChild(path interface{}, node *yaml.Node) *CandidateNode {
 	return &CandidateNode{
 		Node:      node,

pkg/yqlib/doc/Assign (Update).md

@@ -136,7 +136,9 @@ a:
   b: frog
 ```
 
-## Update selected results
+## Update deeply selected results
+Note that the LHS is wrapped in brackets! This is to ensure we dont first filter out the yaml and then update the snippet.
+
 Given a sample.yml file of:
 ```yaml
 a:

pkg/yqlib/doc/Encoder and Decoder.md

@@ -0,0 +1,177 @@
+Encode operators will take the piped in object structure and encode it as a string in the desired format. The decode operators do the opposite, they take a formatted string and decode it into the relevant object structure.
+
+Note that you can optionally pass an indent value to the encode functions (see below).
+
+These operators are useful to process yaml documents that have stringified embeded yaml/json/props in them.
+## Encode value as yaml string
+Indent defaults to 2
+
+Given a sample.yml file of:
+```yaml
+a:
+  cool:
+    bob: dylan
+```
+then
+```bash
+yq eval '.b = (.a | to_yaml)' sample.yml
+```
+will output
+```yaml
+a:
+  cool:
+    bob: dylan
+b: |
+  cool:
+    bob: dylan
+```
+
+## Encode value as yaml string, with custom indentation
+You can specify the indentation level as the first parameter.
+
+Given a sample.yml file of:
+```yaml
+a:
+  cool:
+    bob: dylan
+```
+then
+```bash
+yq eval '.b = (.a | to_yaml(8))' sample.yml
+```
+will output
+```yaml
+a:
+  cool:
+    bob: dylan
+b: |
+  cool:
+          bob: dylan
+```
+
+## Encode value as yaml string, using toyaml
+Does the same thing as to_yaml, matching jq naming convention.
+
+Given a sample.yml file of:
+```yaml
+a:
+  cool: thing
+```
+then
+```bash
+yq eval '.b = (.a | to_yaml)' sample.yml
+```
+will output
+```yaml
+a:
+  cool: thing
+b: |
+  cool: thing
+```
+
+## Encode value as json string
+Given a sample.yml file of:
+```yaml
+a:
+  cool: thing
+```
+then
+```bash
+yq eval '.b = (.a | to_json)' sample.yml
+```
+will output
+```yaml
+a:
+  cool: thing
+b: |
+  {
+    "cool": "thing"
+  }
+```
+
+## Encode value as json string, on one line
+Pass in a 0 indent to print json on a single line.
+
+Given a sample.yml file of:
+```yaml
+a:
+  cool: thing
+```
+then
+```bash
+yq eval '.b = (.a | to_json(0))' sample.yml
+```
+will output
+```yaml
+a:
+  cool: thing
+b: '{"cool":"thing"}'
+```
+
+## Encode value as props string
+Given a sample.yml file of:
+```yaml
+a:
+  cool: thing
+```
+then
+```bash
+yq eval '.b = (.a | to_props)' sample.yml
+```
+will output
+```yaml
+a:
+  cool: thing
+b: |
+  cool = thing
+```
+
+## Decode a yaml encoded string
+Given a sample.yml file of:
+```yaml
+a: 'foo: bar'
+```
+then
+```bash
+yq eval '.b = (.a | from_yaml)' sample.yml
+```
+will output
+```yaml
+a: 'foo: bar'
+b:
+  foo: bar
+```
+
+## Update a multiline encoded yaml string
+Given a sample.yml file of:
+```yaml
+a: |
+  foo: bar
+  baz: dog
+
+```
+then
+```bash
+yq eval '.a |= (from_yaml | .foo = "cat" | to_yaml)' sample.yml
+```
+will output
+```yaml
+a: |
+  foo: cat
+  baz: dog
+```
+
+## Update a single line encoded yaml string
+Given a sample.yml file of:
+```yaml
+a: 'foo: bar'
+```
+then
+```bash
+yq eval '.a |= (from_yaml | .foo = "cat" | to_yaml)' sample.yml
+```
+will output
+```yaml
+a: 'foo: cat'
+```
+

pkg/yqlib/doc/Flatten.md

@@ -0,0 +1,70 @@
+This recursively flattens arrays.
+
+## Flatten
+Recursively flattens all arrays
+
+Given a sample.yml file of:
+```yaml
+- 1
+- - 2
+- - - 3
+```
+then
+```bash
+yq eval 'flatten' sample.yml
+```
+will output
+```yaml
+- 1
+- 2
+- 3
+```
+
+## Flatten with depth of one
+Given a sample.yml file of:
+```yaml
+- 1
+- - 2
+- - - 3
+```
+then
+```bash
+yq eval 'flatten(1)' sample.yml
+```
+will output
+```yaml
+- 1
+- 2
+- - 3
+```
+
+## Flatten empty array
+Given a sample.yml file of:
+```yaml
+- []
+```
+then
+```bash
+yq eval 'flatten' sample.yml
+```
+will output
+```yaml
+[]
+```
+
+## Flatten array of objects
+Given a sample.yml file of:
+```yaml
+- foo: bar
+- - foo: baz
+```
+then
+```bash
+yq eval 'flatten' sample.yml
+```
+will output
+```yaml
+- foo: bar
+- foo: baz
+```
+

pkg/yqlib/doc/Group By.md

@@ -0,0 +1,54 @@
+This is used to group items in an array by an expression.
+
+## Group by field
+Given a sample.yml file of:
+```yaml
+- foo: 1
+  bar: 10
+- foo: 3
+  bar: 100
+- foo: 1
+  bar: 1
+```
+then
+```bash
+yq eval 'group_by(.foo)' sample.yml
+```
+will output
+```yaml
+- - foo: 1
+    bar: 10
+  - foo: 1
+    bar: 1
+- - foo: 3
+    bar: 100
+```
+
+## Group by field, with nuls
+Given a sample.yml file of:
+```yaml
+- cat: dog
+- foo: 1
+  bar: 10
+- foo: 3
+  bar: 100
+- no: foo for you
+- foo: 1
+  bar: 1
+```
+then
+```bash
+yq eval 'group_by(.foo)' sample.yml
+```
+will output
+```yaml
+- - cat: dog
+  - no: foo for you
+- - foo: 1
+    bar: 10
+  - foo: 1
+    bar: 1
+- - foo: 3
+    bar: 100
+```
+

pkg/yqlib/doc/headers/Encoder and Decoder.md

@@ -0,0 +1,5 @@
+Encode operators will take the piped in object structure and encode it as a string in the desired format. The decode operators do the opposite, they take a formatted string and decode it into the relevant object structure.
+
+Note that you can optionally pass an indent value to the encode functions (see below).
+
+These operators are useful to process yaml documents that have stringified embeded yaml/json/props in them.

pkg/yqlib/doc/headers/Flatten.md

@@ -0,0 +1 @@
+This recursively flattens arrays.

pkg/yqlib/doc/headers/Group By.md

@@ -0,0 +1 @@
+This is used to group items in an array by an expression.

pkg/yqlib/expression_tokeniser.go

@@ -2,6 +2,7 @@ package yqlib
 
 import (
 	"fmt"
+	"regexp"
 	"strconv"
 	"strings"
 
@@ -130,6 +131,44 @@ func opTokenWithPrefs(op *operationType, assignOpType *operationType, preference
 	}
 }
 
+func extractNumberParamter(value string) (int, error) {
+	parameterParser := regexp.MustCompile(`.*\(([0-9]+)\)`)
+	matches := parameterParser.FindStringSubmatch(value)
+	var indent, errParsingInt = strconv.ParseInt(matches[1], 10, 32) // nolint
+	if errParsingInt != nil {
+		return 0, errParsingInt
+	}
+	return int(indent), nil
+}
+
+func flattenWithDepth() lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		value := string(m.Bytes)
+		var depth, errParsingInt = extractNumberParamter(value)
+		if errParsingInt != nil {
+			return nil, errParsingInt
+		}
+
+		prefs := flattenPreferences{depth: depth}
+		op := &Operation{OperationType: flattenOpType, Value: flattenOpType.Type, StringValue: value, Preferences: prefs}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+}
+
+func encodeWithIndent(outputFormat PrinterOutputFormat) lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		value := string(m.Bytes)
+		var indent, errParsingInt = extractNumberParamter(value)
+		if errParsingInt != nil {
+			return nil, errParsingInt
+		}
+
+		prefs := encoderPreferences{format: outputFormat, indent: indent}
+		op := &Operation{OperationType: encodeOpType, Value: encodeOpType.Type, StringValue: value, Preferences: prefs}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+}
+
 func assignAllCommentsOp(updateAssign bool) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		log.Debug("assignAllCommentsOp %v", string(m.Bytes))
@@ -272,11 +311,36 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`,`), opToken(unionOpType))
 	lexer.Add([]byte(`:\s*`), opToken(createMapOpType))
 	lexer.Add([]byte(`length`), opToken(lengthOpType))
+
+	lexer.Add([]byte(`flatten\([0-9]+\)`), flattenWithDepth())
+	lexer.Add([]byte(`flatten`), opTokenWithPrefs(flattenOpType, nil, flattenPreferences{depth: -1}))
+
+	lexer.Add([]byte(`toyaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat))
+	lexer.Add([]byte(`to_yaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat))
+
+	lexer.Add([]byte(`tojson\([0-9]+\)`), encodeWithIndent(JsonOutputFormat))
+	lexer.Add([]byte(`to_json\([0-9]+\)`), encodeWithIndent(JsonOutputFormat))
+
+	lexer.Add([]byte(`toyaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2}))
+	lexer.Add([]byte(`tojson`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JsonOutputFormat, indent: 2}))
+	lexer.Add([]byte(`toprops`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2}))
+
+	lexer.Add([]byte(`to_yaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2}))
+	lexer.Add([]byte(`to_json`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JsonOutputFormat, indent: 2}))
+	lexer.Add([]byte(`to_props`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2}))
+
+	lexer.Add([]byte(`fromyaml`), opToken(decodeOpType))
+	lexer.Add([]byte(`fromjson`), opToken(decodeOpType))
+
+	lexer.Add([]byte(`from_yaml`), opToken(decodeOpType))
+	lexer.Add([]byte(`from_json`), opToken(decodeOpType))
+
 	lexer.Add([]byte(`sortKeys`), opToken(sortKeysOpType))
 	lexer.Add([]byte(`select`), opToken(selectOpType))
 	lexer.Add([]byte(`has`), opToken(hasOpType))
 	lexer.Add([]byte(`unique`), opToken(uniqueOpType))
 	lexer.Add([]byte(`unique_by`), opToken(uniqueByOpType))
+	lexer.Add([]byte(`group_by`), opToken(groupByOpType))
 	lexer.Add([]byte(`explode`), opToken(explodeOpType))
 	lexer.Add([]byte(`or`), opToken(orOpType))
 	lexer.Add([]byte(`and`), opToken(andOpType))

pkg/yqlib/lib.go

@@ -59,6 +59,8 @@ var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence:
 
 var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: lengthOperator}
 var collectOpType = &operationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handler: collectOperator}
+var encodeOpType = &operationType{Type: "ENCODE", NumArgs: 0, Precedence: 50, Handler: encodeOperator}
+var decodeOpType = &operationType{Type: "DECODE", NumArgs: 0, Precedence: 50, Handler: decodeOperator}
 
 var anyOpType = &operationType{Type: "ANY", NumArgs: 0, Precedence: 50, Handler: anyOperator}
 var allOpType = &operationType{Type: "ALL", NumArgs: 0, Precedence: 50, Handler: allOperator}
@@ -111,6 +113,8 @@ var selectOpType = &operationType{Type: "SELECT", NumArgs: 1, Precedence: 50, Ha
 var hasOpType = &operationType{Type: "HAS", NumArgs: 1, Precedence: 50, Handler: hasOperator}
 var uniqueOpType = &operationType{Type: "UNIQUE", NumArgs: 0, Precedence: 50, Handler: unique}
 var uniqueByOpType = &operationType{Type: "UNIQUE_BY", NumArgs: 1, Precedence: 50, Handler: uniqueBy}
+var groupByOpType = &operationType{Type: "GROUP_BY", NumArgs: 1, Precedence: 50, Handler: groupBy}
+var flattenOpType = &operationType{Type: "FLATTEN_BY", NumArgs: 0, Precedence: 50, Handler: flattenOp}
 var deleteChildOpType = &operationType{Type: "DELETE", NumArgs: 1, Precedence: 40, Handler: deleteChildOperator}
 
 type Operation struct {

pkg/yqlib/operator_assign_test.go

@@ -112,9 +112,10 @@ var assignOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description: "Update selected results",
-		document:    `{a: {b: apple, c: cactus}}`,
-		expression:  `(.a[] | select(. == "apple")) = "frog"`,
+		description:    "Update deeply selected results",
+		subdescription: "Note that the LHS is wrapped in brackets! This is to ensure we dont first filter out the yaml and then update the snippet.",
+		document:       `{a: {b: apple, c: cactus}}`,
+		expression:     `(.a[] | select(. == "apple")) = "frog"`,
 		expected: []string{
 			"D0, P[], (doc)::{a: {b: frog, c: cactus}}\n",
 		},

pkg/yqlib/operator_encoder_decoder.go

@@ -0,0 +1,91 @@
+package yqlib
+
+import (
+	"bufio"
+	"bytes"
+	"container/list"
+	"regexp"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+func yamlToString(candidate *CandidateNode, prefs encoderPreferences) (string, error) {
+	var output bytes.Buffer
+	log.Debug("printing with indent: %v", prefs.indent)
+
+	printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), prefs.format, true, false, prefs.indent, true)
+	err := printer.PrintResults(candidate.AsList())
+	return output.String(), err
+}
+
+type encoderPreferences struct {
+	format PrinterOutputFormat
+	indent int
+}
+
+/* encodes object as yaml string */
+
+func encodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	preferences := expressionNode.Operation.Preferences.(encoderPreferences)
+	var results = list.New()
+
+	hasOnlyOneNewLine := regexp.MustCompile("[^\n].*\n$")
+	endWithNewLine := regexp.MustCompile(".*\n$")
+	chomper := regexp.MustCompile("\n+$")
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		stringValue, err := yamlToString(candidate, preferences)
+
+		if err != nil {
+			return Context{}, err
+		}
+
+		// remove trailing newlines if needed.
+		// check if we originally decoded this path, and the original thing had a single line.
+		originalList := context.GetVariable("decoded: " + candidate.GetKey())
+		if originalList != nil && originalList.Len() > 0 && hasOnlyOneNewLine.MatchString(stringValue) {
+
+			original := originalList.Front().Value.(*CandidateNode)
+			originalNode := unwrapDoc(original.Node)
+			// original block did not have a new line at the end, get rid of this one too
+			if !endWithNewLine.MatchString(originalNode.Value) {
+				stringValue = chomper.ReplaceAllString(stringValue, "")
+			}
+		}
+
+		// dont print a new line when printing json on a single line.
+		if preferences.format == JsonOutputFormat && preferences.indent == 0 {
+			stringValue = chomper.ReplaceAllString(stringValue, "")
+		}
+
+		stringContentNode := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: stringValue}
+		results.PushBack(candidate.CreateChild(nil, stringContentNode))
+	}
+	return context.ChildContext(results), nil
+}
+
+/* takes a string and decodes it back into an object */
+func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	var results = list.New()
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+
+		context.SetVariable("decoded: "+candidate.GetKey(), candidate.AsList())
+
+		var dataBucket yaml.Node
+		log.Debugf("got: [%v]", candidate.Node.Value)
+		decoder := yaml.NewDecoder(strings.NewReader(unwrapDoc(candidate.Node).Value))
+		errorReading := decoder.Decode(&dataBucket)
+		if errorReading != nil {
+			return Context{}, errorReading
+		}
+		//first node is a doc
+		node := unwrapDoc(&dataBucket)
+
+		results.PushBack(candidate.CreateChild(nil, node))
+	}
+	return context.ChildContext(results), nil
+}

pkg/yqlib/operator_encoder_decoder_test.go

@@ -0,0 +1,117 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var prefix = "D0, P[], (doc)::a:\n    cool:\n        bob: dylan\n"
+
+var encoderDecoderOperatorScenarios = []expressionScenario{
+	{
+		description:    "Encode value as yaml string",
+		subdescription: "Indent defaults to 2",
+		document:       "a:\n  cool:\n    bob: dylan",
+		expression:     `.b = (.a | to_yaml)`,
+		expected: []string{
+			prefix + "b: |\n    cool:\n      bob: dylan\n",
+		},
+	},
+	{
+		description:    "Encode value as yaml string, with custom indentation",
+		subdescription: "You can specify the indentation level as the first parameter.",
+		document:       "a:\n  cool:\n    bob: dylan",
+		expression:     `.b = (.a | to_yaml(8))`,
+		expected: []string{
+			prefix + "b: |\n    cool:\n            bob: dylan\n",
+		},
+	},
+	{
+		description:    "Encode value as yaml string, using toyaml",
+		subdescription: "Does the same thing as to_yaml, matching jq naming convention.",
+		document:       `{a: {cool: "thing"}}`,
+		expression:     `.b = (.a | to_yaml)`,
+		expected: []string{
+			`D0, P[], (doc)::{a: {cool: "thing"}, b: "{cool: \"thing\"}\n"}
+`,
+		},
+	},
+	{
+		description: "Encode value as json string",
+		document:    `{a: {cool: "thing"}}`,
+		expression:  `.b = (.a | to_json)`,
+		expected: []string{
+			`D0, P[], (doc)::{a: {cool: "thing"}, b: "{\n  \"cool\": \"thing\"\n}\n"}
+`,
+		},
+	},
+	{
+		description:    "Encode value as json string, on one line",
+		subdescription: "Pass in a 0 indent to print json on a single line.",
+		document:       `{a: {cool: "thing"}}`,
+		expression:     `.b = (.a | to_json(0))`,
+		expected: []string{
+			`D0, P[], (doc)::{a: {cool: "thing"}, b: '{"cool":"thing"}'}
+`,
+		},
+	},
+	{
+		description: "Encode value as props string",
+		document:    `{a: {cool: "thing"}}`,
+		expression:  `.b = (.a | to_props)`,
+		expected: []string{
+			`D0, P[], (doc)::{a: {cool: "thing"}, b: "cool = thing\n"}
+`,
+		},
+	},
+	{
+		description: "Decode a yaml encoded string",
+		document:    `a: "foo: bar"`,
+		expression:  `.b = (.a | from_yaml)`,
+		expected: []string{
+			"D0, P[], (doc)::a: \"foo: bar\"\nb:\n    foo: bar\n",
+		},
+	},
+	{
+		description:           "Update a multiline encoded yaml string",
+		dontFormatInputForDoc: true,
+		document:              "a: |\n  foo: bar\n  baz: dog\n",
+		expression:            `.a |= (from_yaml | .foo = "cat" | to_yaml)`,
+		expected: []string{
+			"D0, P[], (doc)::a: |\n    foo: cat\n    baz: dog\n",
+		},
+	},
+	{
+		skipDoc:               true,
+		dontFormatInputForDoc: true,
+		document:              "a: |-\n  foo: bar\n  baz: dog\n",
+		expression:            `.a |= (from_yaml | .foo = "cat" | to_yaml)`,
+		expected: []string{
+			"D0, P[], (doc)::a: |-\n    foo: cat\n    baz: dog\n",
+		},
+	},
+	{
+		description:           "Update a single line encoded yaml string",
+		dontFormatInputForDoc: true,
+		document:              "a: 'foo: bar'",
+		expression:            `.a |= (from_yaml | .foo = "cat" | to_yaml)`,
+		expected: []string{
+			"D0, P[], (doc)::a: 'foo: cat'\n",
+		},
+	},
+	{
+		skipDoc:               true,
+		dontFormatInputForDoc: true,
+		document:              "a: \"foo: bar\"",
+		expression:            `.a |= (from_yaml | .foo = {"a": "frog"} | to_yaml)`,
+		expected: []string{
+			"D0, P[], (doc)::a: \"foo:\\n  a: frog\"\n",
+		},
+	},
+}
+
+func TestEncoderDecoderOperatorScenarios(t *testing.T) {
+	for _, tt := range encoderDecoderOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "Encoder and Decoder", encoderDecoderOperatorScenarios)
+}

pkg/yqlib/operator_flatten.go

@@ -0,0 +1,54 @@
+package yqlib
+
+import (
+	"fmt"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+type flattenPreferences struct {
+	depth int
+}
+
+func flatten(node *yaml.Node, depth int) {
+	if depth == 0 {
+		return
+	}
+	if node.Kind != yaml.SequenceNode {
+		return
+	}
+	content := node.Content
+	newSeq := make([]*yaml.Node, 0)
+
+	for i := 0; i < len(content); i++ {
+		if content[i].Kind == yaml.SequenceNode {
+			flatten(content[i], depth-1)
+			for j := 0; j < len(content[i].Content); j++ {
+				newSeq = append(newSeq, content[i].Content[j])
+			}
+		} else {
+			newSeq = append(newSeq, content[i])
+		}
+	}
+	node.Content = newSeq
+}
+
+func flattenOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	log.Debugf("-- flatten Operator")
+	depth := expressionNode.Operation.Preferences.(flattenPreferences).depth
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		candidateNode := unwrapDoc(candidate.Node)
+		if candidateNode.Kind != yaml.SequenceNode {
+			return Context{}, fmt.Errorf("Only arrays are supported for flatten")
+		}
+
+		flatten(candidateNode, depth)
+
+	}
+
+	return context, nil
+
+}

pkg/yqlib/operator_flatten_test.go

@@ -0,0 +1,48 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var flattenOperatorScenarios = []expressionScenario{
+	{
+		description:    "Flatten",
+		subdescription: "Recursively flattens all arrays",
+		document:       `[1, [2], [[3]]]`,
+		expression:     `flatten`,
+		expected: []string{
+			"D0, P[], (doc)::[1, 2, 3]\n",
+		},
+	},
+	{
+		description: "Flatten with depth of one",
+		document:    `[1, [2], [[3]]]`,
+		expression:  `flatten(1)`,
+		expected: []string{
+			"D0, P[], (doc)::[1, 2, [3]]\n",
+		},
+	},
+	{
+		description: "Flatten empty array",
+		document:    `[[]]`,
+		expression:  `flatten`,
+		expected: []string{
+			"D0, P[], (doc)::[]\n",
+		},
+	},
+	{
+		description: "Flatten array of objects",
+		document:    `[{foo: bar}, [{foo: baz}]]`,
+		expression:  `flatten`,
+		expected: []string{
+			"D0, P[], (doc)::[{foo: bar}, {foo: baz}]\n",
+		},
+	},
+}
+
+func TestFlattenOperatorScenarios(t *testing.T) {
+	for _, tt := range flattenOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "Flatten", flattenOperatorScenarios)
+}

pkg/yqlib/operator_group_by.go

@@ -0,0 +1,76 @@
+package yqlib
+
+import (
+	"container/list"
+	"fmt"
+
+	"github.com/elliotchance/orderedmap"
+	yaml "gopkg.in/yaml.v3"
+)
+
+func processIntoGroups(d *dataTreeNavigator, context Context, rhsExp *ExpressionNode, node *yaml.Node) (*orderedmap.OrderedMap, error) {
+	var newMatches = orderedmap.NewOrderedMap()
+	for _, node := range node.Content {
+		child := &CandidateNode{Node: node}
+		rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(child), rhsExp)
+
+		if err != nil {
+			return nil, err
+		}
+
+		keyValue := "null"
+
+		if rhs.MatchingNodes.Len() > 0 {
+			first := rhs.MatchingNodes.Front()
+			keyCandidate := first.Value.(*CandidateNode)
+			keyValue = keyCandidate.Node.Value
+		}
+
+		groupList, exists := newMatches.Get(keyValue)
+
+		if !exists {
+			groupList = list.New()
+			newMatches.Set(keyValue, groupList)
+		}
+		groupList.(*list.List).PushBack(node)
+	}
+	return newMatches, nil
+}
+
+func groupBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	log.Debugf("-- groupBy Operator")
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		candidateNode := unwrapDoc(candidate.Node)
+
+		if candidateNode.Kind != yaml.SequenceNode {
+			return Context{}, fmt.Errorf("Only arrays are supported for group by")
+		}
+
+		newMatches, err := processIntoGroups(d, context, expressionNode.Rhs, candidateNode)
+
+		if err != nil {
+			return Context{}, err
+		}
+
+		resultNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+		for groupEl := newMatches.Front(); groupEl != nil; groupEl = groupEl.Next() {
+			groupResultNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+			groupList := groupEl.Value.(*list.List)
+			for groupItem := groupList.Front(); groupItem != nil; groupItem = groupItem.Next() {
+				groupResultNode.Content = append(groupResultNode.Content, groupItem.Value.(*yaml.Node))
+			}
+
+			resultNode.Content = append(resultNode.Content, groupResultNode)
+		}
+
+		results.PushBack(candidate.CreateChild(nil, resultNode))
+
+	}
+
+	return context.ChildContext(results), nil
+
+}

pkg/yqlib/operator_group_by_test.go

@@ -0,0 +1,31 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var groupByOperatorScenarios = []expressionScenario{
+	{
+		description: "Group by field",
+		document:    `[{foo: 1, bar: 10}, {foo: 3, bar: 100}, {foo: 1, bar: 1}]`,
+		expression:  `group_by(.foo)`,
+		expected: []string{
+			"D0, P[], (!!seq)::- - {foo: 1, bar: 10}\n  - {foo: 1, bar: 1}\n- - {foo: 3, bar: 100}\n",
+		},
+	},
+	{
+		description: "Group by field, with nuls",
+		document:    `[{cat: dog}, {foo: 1, bar: 10}, {foo: 3, bar: 100}, {no: foo for you}, {foo: 1, bar: 1}]`,
+		expression:  `group_by(.foo)`,
+		expected: []string{
+			"D0, P[], (!!seq)::- - {cat: dog}\n  - {no: foo for you}\n- - {foo: 1, bar: 10}\n  - {foo: 1, bar: 1}\n- - {foo: 3, bar: 100}\n",
+		},
+	},
+}
+
+func TestGroupByOperatorScenarios(t *testing.T) {
+	for _, tt := range groupByOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "Group By", groupByOperatorScenarios)
+}

pkg/yqlib/operators_test.go

@@ -107,7 +107,7 @@ func copyFromHeader(title string, out *os.File) error {
 
 func formatYaml(yaml string, filename string) string {
 	var output bytes.Buffer
-	printer := NewPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
 
 	node, err := NewExpressionParser().ParseExpression(".. style= \"\"")
 	if err != nil {
@@ -216,7 +216,7 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) {
 func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formattedDoc string, formattedDoc2 string) {
 	var output bytes.Buffer
 	var err error
-	printer := NewPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
 
 	node, err := NewExpressionParser().ParseExpression(s.expression)
 	if err != nil {

pkg/yqlib/printer.go

@@ -44,7 +44,7 @@ type resultsPrinter struct {
 	colorsEnabled      bool
 	indent             int
 	printDocSeparators bool
-	writer             io.Writer
+	printerWriter      PrinterWriter
 	firstTimePrinting  bool
 	previousDocIndex   uint
 	previousFileIndex  int
@@ -53,9 +53,13 @@ type resultsPrinter struct {
 	appendixReader     io.Reader
 }
 
-func NewPrinter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
+func NewPrinterWithSingleWriter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
+	return NewPrinter(NewSinglePrinterWriter(writer), outputFormat, unwrapScalar, colorsEnabled, indent, printDocSeparators)
+}
+
+func NewPrinter(printerWriter PrinterWriter, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
 	return &resultsPrinter{
-		writer:             writer,
+		printerWriter:      printerWriter,
 		outputFormat:       outputFormat,
 		unwrapScalar:       unwrapScalar,
 		colorsEnabled:      colorsEnabled,
@@ -98,15 +102,57 @@ func (p *resultsPrinter) writeString(writer io.Writer, txt string) error {
 	return errorWriting
 }
 
-func (p *resultsPrinter) safelyFlush(writer *bufio.Writer) {
-	if err := writer.Flush(); err != nil {
-		log.Error("Error flushing writer!")
-		log.Error(err.Error())
+func (p *resultsPrinter) processLeadingContent(mappedDoc *CandidateNode, writer io.Writer) error {
+	if strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$") {
+		log.Debug("headcommentwas %v", mappedDoc.Node.HeadComment)
+		log.Debug("finished headcomment")
+		reader := bufio.NewReader(strings.NewReader(mappedDoc.Node.HeadComment))
+		mappedDoc.Node.HeadComment = ""
+
+		for {
+
+			readline, errReading := reader.ReadString('\n')
+			if errReading != nil && errReading != io.EOF {
+				return errReading
+			}
+			if strings.Contains(readline, "$yqLeadingContent$") {
+				// skip this
+
+			} else if strings.Contains(readline, "$yqDocSeperator$") {
+				if p.printDocSeparators {
+					if err := p.writeString(writer, "---\n"); err != nil {
+						return err
+					}
+				}
+			} else if p.outputFormat == YamlOutputFormat {
+				if err := p.writeString(writer, readline); err != nil {
+					return err
+				}
+			}
+
+			if errReading == io.EOF {
+				if readline != "" {
+					// the last comment we read didn't have a new line, put one in
+					if err := p.writeString(writer, "\n"); err != nil {
+						return err
+					}
+				}
+				break
+			}
+		}
+
 	}
+	return nil
 }
 
 func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 	log.Debug("PrintResults for %v matches", matchingNodes.Len())
+
+	if matchingNodes.Len() == 0 {
+		log.Debug("no matching results, nothing to print")
+		return nil
+	}
+
 	if p.outputFormat != YamlOutputFormat {
 		explodeOp := Operation{OperationType: explodeOpType}
 		explodeNode := ExpressionNode{Operation: &explodeOp}
@@ -117,13 +163,6 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 		matchingNodes = context.MatchingNodes
 	}
 
-	bufferedWriter := bufio.NewWriter(p.writer)
-	defer p.safelyFlush(bufferedWriter)
-
-	if matchingNodes.Len() == 0 {
-		log.Debug("no matching results, nothing to print")
-		return nil
-	}
 	if p.firstTimePrinting {
 		node := matchingNodes.Front().Value.(*CandidateNode)
 		p.previousDocIndex = node.Document
@@ -132,72 +171,53 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 	}
 
 	for el := matchingNodes.Front(); el != nil; el = el.Next() {
+
 		mappedDoc := el.Value.(*CandidateNode)
 		log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v, printDocSeparators: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document, p.printDocSeparators)
 
+		writer, errorWriting := p.printerWriter.GetWriter(mappedDoc)
+		if errorWriting != nil {
+			return errorWriting
+		}
+
 		commentStartsWithSeparator := strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$\n$yqDocSeperator$")
 
 		if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators && !commentStartsWithSeparator {
 			log.Debug("-- writing doc sep")
-			if err := p.writeString(bufferedWriter, "---\n"); err != nil {
+			if err := p.writeString(writer, "---\n"); err != nil {
 				return err
 			}
 		}
 
-		if strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$") {
-			log.Debug("headcommentwas %v", mappedDoc.Node.HeadComment)
-			log.Debug("finished headcomment")
-			reader := bufio.NewReader(strings.NewReader(mappedDoc.Node.HeadComment))
-			mappedDoc.Node.HeadComment = ""
-
-			for {
-
-				readline, errReading := reader.ReadString('\n')
-				if errReading != nil && errReading != io.EOF {
-					return errReading
-				}
-				if strings.Contains(readline, "$yqLeadingContent$") {
-					// skip this
-
-				} else if strings.Contains(readline, "$yqDocSeperator$") {
-					if p.printDocSeparators {
-						if err := p.writeString(bufferedWriter, "---\n"); err != nil {
-							return err
-						}
-					}
-				} else if p.outputFormat == YamlOutputFormat {
-					if err := p.writeString(bufferedWriter, readline); err != nil {
-						return err
-					}
-				}
-
-				if errReading == io.EOF {
-					if readline != "" {
-						// the last comment we read didn't have a new line, put one in
-						if err := p.writeString(bufferedWriter, "\n"); err != nil {
-							return err
-						}
-					}
-					break
-				}
-			}
-
+		if err := p.processLeadingContent(mappedDoc, writer); err != nil {
+			return err
 		}
 
-		if err := p.printNode(mappedDoc.Node, bufferedWriter); err != nil {
+		if err := p.printNode(mappedDoc.Node, writer); err != nil {
 			return err
 		}
 
 		p.previousDocIndex = mappedDoc.Document
+		if err := writer.Flush(); err != nil {
+			return err
+		}
 	}
 
 	if p.appendixReader != nil && p.outputFormat == YamlOutputFormat {
+		writer, err := p.printerWriter.GetWriter(nil)
+		if err != nil {
+			return err
+		}
+
 		log.Debug("Piping appendix reader...")
 		betterReader := bufio.NewReader(p.appendixReader)
-		_, err := io.Copy(bufferedWriter, betterReader)
+		_, err = io.Copy(writer, betterReader)
 		if err != nil {
 			return err
 		}
+		if err := writer.Flush(); err != nil {
+			return err
+		}
 	}
 
 	return nil

pkg/yqlib/printer_test.go

@@ -36,7 +36,7 @@ func nodeToList(candidate *CandidateNode) *list.List {
 func TestPrinterMultipleDocsInSequence(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -74,7 +74,7 @@ func TestPrinterMultipleDocsInSequence(t *testing.T) {
 func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -116,7 +116,7 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
 func TestPrinterMultipleFilesInSequence(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -163,7 +163,7 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) {
 func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -213,7 +213,7 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
 func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -232,7 +232,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -261,7 +261,7 @@ a: coconut
 func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -287,7 +287,7 @@ a: coconut
 func TestPrinterScalarWithLeadingCont(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	node, err := NewExpressionParser().ParseExpression(".a")
 	if err != nil {
@@ -314,7 +314,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	// note printDocSeparators is true, it should still not print document separators
 	// when outputing JSON.
-	printer := NewPrinter(writer, JsonOutputFormat, true, false, 0, true)
+	printer := NewPrinterWithSingleWriter(writer, JsonOutputFormat, true, false, 0, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {

pkg/yqlib/printer_writer.go

@@ -0,0 +1,81 @@
+package yqlib
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+
+	"gopkg.in/yaml.v3"
+)
+
+type PrinterWriter interface {
+	GetWriter(node *CandidateNode) (*bufio.Writer, error)
+}
+
+type singlePrinterWriter struct {
+	bufferedWriter *bufio.Writer
+}
+
+func NewSinglePrinterWriter(writer io.Writer) PrinterWriter {
+	return &singlePrinterWriter{
+		bufferedWriter: bufio.NewWriter(writer),
+	}
+}
+
+func (sp *singlePrinterWriter) GetWriter(node *CandidateNode) (*bufio.Writer, error) {
+	return sp.bufferedWriter, nil
+}
+
+type multiPrintWriter struct {
+	treeNavigator  DataTreeNavigator
+	nameExpression *ExpressionNode
+	extension      string
+	index          int
+}
+
+func NewMultiPrinterWriter(expression *ExpressionNode, format PrinterOutputFormat) PrinterWriter {
+	extension := "yml"
+
+	switch format {
+	case JsonOutputFormat:
+		extension = "json"
+	case PropsOutputFormat:
+		extension = "properties"
+	}
+
+	return &multiPrintWriter{
+		nameExpression: expression,
+		extension:      extension,
+		treeNavigator:  NewDataTreeNavigator(),
+		index:          0,
+	}
+}
+
+func (sp *multiPrintWriter) GetWriter(node *CandidateNode) (*bufio.Writer, error) {
+	name := ""
+
+	indexVariableNode := yaml.Node{Kind: yaml.ScalarNode, Tag: "!!int", Value: fmt.Sprintf("%v", sp.index)}
+	indexVariableCandidate := CandidateNode{Node: &indexVariableNode}
+
+	context := Context{MatchingNodes: node.AsList()}
+	context.SetVariable("index", indexVariableCandidate.AsList())
+	result, err := sp.treeNavigator.GetMatchingNodes(context, sp.nameExpression)
+	if err != nil {
+		return nil, err
+	}
+	if result.MatchingNodes.Len() > 0 {
+		name = result.MatchingNodes.Front().Value.(*CandidateNode).Node.Value
+	}
+	name = fmt.Sprintf("%v.%v", name, sp.extension)
+
+	f, err := os.Create(name)
+
+	if err != nil {
+		return nil, err
+	}
+	sp.index = sp.index + 1
+
+	return bufio.NewWriter(f), nil
+
+}

release_notes.txt

@@ -1,3 +1,16 @@
+4.14.1:
+  - Added group_by operator
+  - Added encode/decode operators (toyaml, fromjson etc)
+  - Added flatten operator
+
+4.13.5:
+  - Performance improvement for deepMatch (thanks @pmatseykanets)
+  - Added manpage, included in tar.gz downloads as well as a separate tar.gz (#961)
+  - Fixed expression parsing bug #970
+  - Rebuild fixes CVE (#964)
+  - Bumped docker alpine version
+
+
 4.13.4:
   - Fixed select bug (#958)
   - Improved performance of `explode` (this will also speed up json conversion)

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.13.4'
+version: '4.13.5'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.