Man page release workflow wip

.github/workflows/release-test.yml

@@ -0,0 +1,95 @@
+name: Test Release YQ
+on:
+  push:
+    tags:
+      - 'releaseTest-*'
+
+jobs:
+  publishGitRelease:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '^1.17.2'
+      - name: Compile man page markup
+        id: gen-man-page-md
+        run: |
+          ./scripts/generate-man-page-md.sh
+          echo "::set-output name=man-page-md::man.md"
+
+      - name: Get the version
+        id: get_version
+        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+      - name: Generate man page
+        uses: docker://pandoc/core:2.14.2
+        id: gen-man-page
+        env:
+          MAN_HEADER: yq (https://github.com/mikefarah/yq/) version ${{ steps.get_version.outputs.VERSION }}
+        with:
+          args: >-
+            --standalone
+            --to man
+            --variable=title:"YQ"
+            --variable=section:"1"
+            --variable=header:"${MAN_HEADER}"
+            --variable=author:"Mike Farah"
+            --output=yq.1
+            ${{ steps.gen-man-page-md.outputs.man-page-md }}
+        
+      - name: Cross compile
+        run: |
+          sudo apt-get install rhash -y
+          go get github.com/mitchellh/gox
+          mkdir -p build
+          cp yq.1 build/yq.1
+          ./scripts/xcompile.sh
+
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1.0.0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: TEST ${{ github.ref }}
+          draft: true
+          prerelease: false
+
+      - uses: shogo82148/actions-upload-release-asset@v1
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: build/*
+
+  # publishDocker:
+  #   environment: dockerhub
+  #   env:
+  #     IMAGE_NAME: mikefarah/yq
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v2
+      
+  #     - name: Set up QEMU
+  #       uses: docker/setup-qemu-action@v1
+  #       with:
+  #         platforms: all
+  
+  #     - name: Set up Docker Buildx
+  #       id: buildx
+  #       uses: docker/setup-buildx-action@v1
+  #       with:
+  #         version: latest
+
+  #     - name: Available platforms
+  #       run: echo ${{ steps.buildx.outputs.platforms }} && docker version
+ 
+  #     - name: Build image
+  #       run: |
+  #         IMAGE_V_VERSION="$(git describe --tags --abbrev=0)"
+  #         IMAGE_VERSION=${IMAGE_V_VERSION:1}
+
+  #         SHORT_SHA1=$(git rev-parse --short HEAD)
+  #         PLATFORMS="linux/amd64,linux/ppc64le,linux/arm64"
+  #         echo "Building version ${IMAGE_VERSION} of image ${IMAGE_NAME}"
+  #         docker buildx build --platform "${PLATFORMS}" .
+    

.github/workflows/release.yml

@@ -12,36 +12,10 @@ jobs:
       - uses: actions/setup-go@v2
         with:
           go-version: '^1.17.2'
-      - name: Compile man page markup
-        id: gen-man-page-md
-        run: |
-          ./scripts/generate-man-page-md.sh
-          echo "::set-output name=man-page-md::man.md"
-
-      - name: Get the version
-        id: get_version
-        run: echo ::set-output name=VERSION::${GITHUB_REF##*/}
-        
-      - name: Generate man page
-        uses: docker://pandoc/core:2.14.2
-        id: gen-man-page
-        with:
-          args: >-
-            --standalone
-            --to man
-            --variable=title:"YQ"
-            --variable=section:"1"
-            --variable=header:"yq (https://github.com/mikefarah/yq/) version ${{ steps.get_version.outputs.VERSION }}"
-            --variable=author:"Mike Farah"
-            --output=yq.1
-            ${{ steps.gen-man-page-md.outputs.man-page-md }}
-        
       - name: Cross compile
         run: |
           sudo apt-get install rhash -y
           go get github.com/mitchellh/gox
-          mkdir -p build
-          cp yq.1 build/yq.1
           ./scripts/xcompile.sh
 
       - name: Create Release

.gitignore

@@ -38,12 +38,8 @@ parts/
 prime/
 .snapcraft/
 yq*.snap
-
 test.yml
-test*.yml
-0.yml
-1.yml
-2.yml
+test2.yml
 
 
 # man page

acceptance_tests/basic.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 setUp() {
-  rm test*.yml || true
+  rm -f test.yml
 }
 
 testBasicEvalRoundTrip() {

acceptance_tests/empty.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 
 setUp() {
-  rm test*.yml || true
   cat >test.yml <<EOL
 # comment
 EOL

acceptance_tests/front-matter.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 
 setUp() {
-  rm test*.yml || true
   cat >test.yml <<EOL
 ---
 a: apple

acceptance_tests/leading-seperator.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 
 setUp() {
-  rm test*.yml || true
   cat >test.yml <<EOL
 ---
 a: test

acceptance_tests/output-format.sh

@@ -1,9 +1,5 @@
 #!/bin/bash
 
-setUp() {
-  rm test*.yml || true
-}
-
 testOutputJsonDeprecated() {
   cat >test.yml <<EOL
 a: {b: ["cat"]}

acceptance_tests/pretty-print.sh

@@ -1,8 +1,5 @@
 #!/bin/bash
 
-setUp() {
-  rm test*.yml || true
-}
 
 testPrettyPrintWithBooleans() {
   cat >test.yml <<EOL

acceptance_tests/split-printer.sh

@@ -1,157 +0,0 @@
-#!/bin/bash
-
-setUp() {
-  rm test*.yml || true
-}
-
-testBasicSplitWithName() {
-  cat >test.yml <<EOL
-a: test_doc1
---- 
-a: test_doc2
-EOL
-
-  ./yq e test.yml -s ".a"
-
-  doc1=$(cat test_doc1.yml)
-  
-  assertEquals "a: test_doc1" "$doc1"
-
-  doc2=$(cat test_doc2.yml)
-  read -r -d '' expectedDoc2 << EOM
----
-a: test_doc2
-EOM
-  assertEquals "$expectedDoc2" "$doc2"
-}
-
-testBasicSplitWithNameEvalAll() {
-  cat >test.yml <<EOL
-a: test_doc1
---- 
-a: test_doc2
-EOL
-
-  ./yq ea test.yml -s ".a"
-
-  doc1=$(cat test_doc1.yml)
-  
-  assertEquals "a: test_doc1" "$doc1"
-
-  doc2=$(cat test_doc2.yml)
-  read -r -d '' expectedDoc2 << EOM
----
-a: test_doc2
-EOM
-  assertEquals "$expectedDoc2" "$doc2"
-}
-
-testBasicSplitWithIndex() {
-  cat >test.yml <<EOL
-a: test_doc1
---- 
-a: test_doc2
-EOL
-
-  ./yq e test.yml -s '"test_" + $index'
-
-  doc1=$(cat test_0.yml)
-  
-  assertEquals "a: test_doc1" "$doc1"
-
-  doc2=$(cat test_1.yml)
-  read -r -d '' expectedDoc2 << EOM
----
-a: test_doc2
-EOM
-  assertEquals "$expectedDoc2" "$doc2"
-}
-
-testBasicSplitWithIndexEvalAll() {
-  cat >test.yml <<EOL
-a: test_doc1
---- 
-a: test_doc2
-EOL
-
-  ./yq ea test.yml -s '"test_" + $index'
-
-  doc1=$(cat test_0.yml)
-  
-  assertEquals "a: test_doc1" "$doc1"
-
-  doc2=$(cat test_1.yml)
-  read -r -d '' expectedDoc2 << EOM
----
-a: test_doc2
-EOM
-  assertEquals "$expectedDoc2" "$doc2"
-}
-
-
-testArraySplitWithNameNoSeparators() {
-  cat >test.yml <<EOL
-- name: test_fred
-  age: 35
-- name: test_catherine
-  age: 37
-EOL
-
-  ./yq e --no-doc -s ".name"  ".[]" test.yml 
-
-  doc1=$(cat test_fred.yml)
-  read -r -d '' expectedDoc1 << EOM
-name: test_fred
-age: 35
-EOM
-
-  assertEquals "$expectedDoc1" "$doc1"
-
-  doc2=$(cat test_catherine.yml)
-  read -r -d '' expectedDoc2 << EOM
-name: test_catherine
-age: 37
-EOM
-  assertEquals "$expectedDoc2" "$doc2"
-}
-
-testArraySplitWithNameNoSeparatorsEvalAll() {
-  cat >test.yml <<EOL
-- name: test_fred
-  age: 35
-- name: test_catherine
-  age: 37
-EOL
-
-cat >test2.yml <<EOL
-- name: test_mike
-  age: 564
-EOL
-
-  ./yq ea --no-doc -s ".name"  ".[]" test.yml test2.yml
-
-  doc1=$(cat test_fred.yml)
-  read -r -d '' expectedDoc1 << EOM
-name: test_fred
-age: 35
-EOM
-
-  assertEquals "$expectedDoc1" "$doc1"
-
-  doc2=$(cat test_catherine.yml)
-  read -r -d '' expectedDoc2 << EOM
-name: test_catherine
-age: 37
-EOM
-  assertEquals "$expectedDoc2" "$doc2"
-
-
-  doc3=$(cat test_mike.yml)
-  read -r -d '' expectedDoc3 << EOM
-name: test_mike
-age: 564
-EOM
-  assertEquals "$expectedDoc3" "$doc3"
-}
-
-source ./scripts/shunit2

cmd/constant.go

@@ -21,6 +21,4 @@ var prettyPrint = false
 // can be either "" (off), "extract" or "process"
 var frontMatter = ""
 
-var splitFileExp = ""
-
 var completedSuccessfully = false

cmd/evaluate_all_command.go

@@ -70,10 +70,6 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("Write inplace flag only applicable when giving an expression and at least one file")
 	}
 
-	if writeInplace && splitFileExp != "" {
-		return fmt.Errorf("Write inplace cannot be used with split file")
-	}
-
 	if writeInplace {
 		// only use colors if its forced
 		colorsEnabled = forceColor
@@ -100,9 +96,7 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	printerWriter := configurePrinterWriter(format, out)
-
-	printer := yqlib.NewPrinter(printerWriter, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
+	printer := yqlib.NewPrinter(out, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
 
 	if frontMatter != "" {
 		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])

cmd/evalute_sequence_command.go

@@ -83,10 +83,6 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("Write inplace flag only applicable when giving an expression and at least one file")
 	}
 
-	if writeInplace && splitFileExp != "" {
-		return fmt.Errorf("Write inplace cannot be used with split file")
-	}
-
 	if writeInplace {
 		// only use colors if its forced
 		colorsEnabled = forceColor
@@ -110,9 +106,7 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	printerWriter := configurePrinterWriter(format, out)
-
-	printer := yqlib.NewPrinter(printerWriter, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
+	printer := yqlib.NewPrinter(out, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
 
 	streamEvaluator := yqlib.NewStreamEvaluator()
 

cmd/printer_utils.go

@@ -1,28 +0,0 @@
-package cmd
-
-import (
-	"io"
-
-	"github.com/mikefarah/yq/v4/pkg/yqlib"
-)
-
-func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) yqlib.PrinterWriter {
-
-	var printerWriter yqlib.PrinterWriter
-
-	if splitFileExp == "i" {
-		colorsEnabled = forceColor
-		printerWriter = yqlib.NewMultiPrinterWriter(nil, format)
-
-	} else if splitFileExp != "" {
-		colorsEnabled = forceColor
-		splitExp, err := yqlib.NewExpressionParser().ParseExpression(splitFileExp)
-		if err != nil {
-			return nil
-		}
-		printerWriter = yqlib.NewMultiPrinterWriter(splitExp, format)
-	} else {
-		printerWriter = yqlib.NewSinglePrinterWriter(out)
-	}
-	return printerWriter
-}

cmd/root.go

@@ -63,9 +63,6 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
 	rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
 	rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression. This is a workaround for go-yaml to persist header content properly.")
-
-	rootCmd.PersistentFlags().StringVarP(&splitFileExp, "split-exp", "s", "", "print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter.")
-
 	rootCmd.AddCommand(
 		createEvaluateSequenceCommand(),
 		createEvaluateAllCommand(),

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.13.5"
+	Version = "4.13.4"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

github-action/Dockerfile

@@ -1,4 +1,4 @@
-FROM mikefarah/yq:4.13.5
+FROM mikefarah/yq:4.13.4
 
 COPY entrypoint.sh /entrypoint.sh
 

pkg/yqlib/candidate_node.go

@@ -1,7 +1,6 @@
 package yqlib
 
 import (
-	"container/list"
 	"fmt"
 
 	"github.com/jinzhu/copier"
@@ -29,12 +28,6 @@ func (n *CandidateNode) GetKey() string {
 	return fmt.Sprintf("%v%v - %v", keyPrefix, n.Document, n.Path)
 }
 
-func (n *CandidateNode) AsList() *list.List {
-	elMap := list.New()
-	elMap.PushBack(n)
-	return elMap
-}
-
 func (n *CandidateNode) CreateChild(path interface{}, node *yaml.Node) *CandidateNode {
 	return &CandidateNode{
 		Node:      node,

pkg/yqlib/doc/Assign (Update).md

@@ -136,9 +136,7 @@ a:
   b: frog
 ```
 
-## Update deeply selected results
-Note that the LHS is wrapped in brackets! This is to ensure we dont first filter out the yaml and then update the snippet.
-
+## Update selected results
 Given a sample.yml file of:
 ```yaml
 a:

pkg/yqlib/doc/Encoder and Decoder.md

@@ -1,177 +0,0 @@
-Encode operators will take the piped in object structure and encode it as a string in the desired format. The decode operators do the opposite, they take a formatted string and decode it into the relevant object structure.
-
-Note that you can optionally pass an indent value to the encode functions (see below).
-
-These operators are useful to process yaml documents that have stringified embeded yaml/json/props in them.
-## Encode value as yaml string
-Indent defaults to 2
-
-Given a sample.yml file of:
-```yaml
-a:
-  cool:
-    bob: dylan
-```
-then
-```bash
-yq eval '.b = (.a | to_yaml)' sample.yml
-```
-will output
-```yaml
-a:
-  cool:
-    bob: dylan
-b: |
-  cool:
-    bob: dylan
-```
-
-## Encode value as yaml string, with custom indentation
-You can specify the indentation level as the first parameter.
-
-Given a sample.yml file of:
-```yaml
-a:
-  cool:
-    bob: dylan
-```
-then
-```bash
-yq eval '.b = (.a | to_yaml(8))' sample.yml
-```
-will output
-```yaml
-a:
-  cool:
-    bob: dylan
-b: |
-  cool:
-          bob: dylan
-```
-
-## Encode value as yaml string, using toyaml
-Does the same thing as to_yaml, matching jq naming convention.
-
-Given a sample.yml file of:
-```yaml
-a:
-  cool: thing
-```
-then
-```bash
-yq eval '.b = (.a | to_yaml)' sample.yml
-```
-will output
-```yaml
-a:
-  cool: thing
-b: |
-  cool: thing
-```
-
-## Encode value as json string
-Given a sample.yml file of:
-```yaml
-a:
-  cool: thing
-```
-then
-```bash
-yq eval '.b = (.a | to_json)' sample.yml
-```
-will output
-```yaml
-a:
-  cool: thing
-b: |
-  {
-    "cool": "thing"
-  }
-```
-
-## Encode value as json string, on one line
-Pass in a 0 indent to print json on a single line.
-
-Given a sample.yml file of:
-```yaml
-a:
-  cool: thing
-```
-then
-```bash
-yq eval '.b = (.a | to_json(0))' sample.yml
-```
-will output
-```yaml
-a:
-  cool: thing
-b: '{"cool":"thing"}'
-```
-
-## Encode value as props string
-Given a sample.yml file of:
-```yaml
-a:
-  cool: thing
-```
-then
-```bash
-yq eval '.b = (.a | to_props)' sample.yml
-```
-will output
-```yaml
-a:
-  cool: thing
-b: |
-  cool = thing
-```
-
-## Decode a yaml encoded string
-Given a sample.yml file of:
-```yaml
-a: 'foo: bar'
-```
-then
-```bash
-yq eval '.b = (.a | from_yaml)' sample.yml
-```
-will output
-```yaml
-a: 'foo: bar'
-b:
-  foo: bar
-```
-
-## Update a multiline encoded yaml string
-Given a sample.yml file of:
-```yaml
-a: |
-  foo: bar
-  baz: dog
-
-```
-then
-```bash
-yq eval '.a |= (from_yaml | .foo = "cat" | to_yaml)' sample.yml
-```
-will output
-```yaml
-a: |
-  foo: cat
-  baz: dog
-```
-
-## Update a single line encoded yaml string
-Given a sample.yml file of:
-```yaml
-a: 'foo: bar'
-```
-then
-```bash
-yq eval '.a |= (from_yaml | .foo = "cat" | to_yaml)' sample.yml
-```
-will output
-```yaml
-a: 'foo: cat'
-```
-

pkg/yqlib/doc/Flatten.md

@@ -1,70 +0,0 @@
-This recursively flattens arrays.
-
-## Flatten
-Recursively flattens all arrays
-
-Given a sample.yml file of:
-```yaml
-- 1
-- - 2
-- - - 3
-```
-then
-```bash
-yq eval 'flatten' sample.yml
-```
-will output
-```yaml
-- 1
-- 2
-- 3
-```
-
-## Flatten with depth of one
-Given a sample.yml file of:
-```yaml
-- 1
-- - 2
-- - - 3
-```
-then
-```bash
-yq eval 'flatten(1)' sample.yml
-```
-will output
-```yaml
-- 1
-- 2
-- - 3
-```
-
-## Flatten empty array
-Given a sample.yml file of:
-```yaml
-- []
-```
-then
-```bash
-yq eval 'flatten' sample.yml
-```
-will output
-```yaml
-[]
-```
-
-## Flatten array of objects
-Given a sample.yml file of:
-```yaml
-- foo: bar
-- - foo: baz
-```
-then
-```bash
-yq eval 'flatten' sample.yml
-```
-will output
-```yaml
-- foo: bar
-- foo: baz
-```
-

pkg/yqlib/doc/Group By.md

@@ -1,54 +0,0 @@
-This is used to group items in an array by an expression.
-
-## Group by field
-Given a sample.yml file of:
-```yaml
-- foo: 1
-  bar: 10
-- foo: 3
-  bar: 100
-- foo: 1
-  bar: 1
-```
-then
-```bash
-yq eval 'group_by(.foo)' sample.yml
-```
-will output
-```yaml
-- - foo: 1
-    bar: 10
-  - foo: 1
-    bar: 1
-- - foo: 3
-    bar: 100
-```
-
-## Group by field, with nuls
-Given a sample.yml file of:
-```yaml
-- cat: dog
-- foo: 1
-  bar: 10
-- foo: 3
-  bar: 100
-- no: foo for you
-- foo: 1
-  bar: 1
-```
-then
-```bash
-yq eval 'group_by(.foo)' sample.yml
-```
-will output
-```yaml
-- - cat: dog
-  - no: foo for you
-- - foo: 1
-    bar: 10
-  - foo: 1
-    bar: 1
-- - foo: 3
-    bar: 100
-```
-

pkg/yqlib/doc/headers/Encoder and Decoder.md

@@ -1,5 +0,0 @@
-Encode operators will take the piped in object structure and encode it as a string in the desired format. The decode operators do the opposite, they take a formatted string and decode it into the relevant object structure.
-
-Note that you can optionally pass an indent value to the encode functions (see below).
-
-These operators are useful to process yaml documents that have stringified embeded yaml/json/props in them.

pkg/yqlib/doc/headers/Flatten.md

@@ -1 +0,0 @@
-This recursively flattens arrays.

pkg/yqlib/doc/headers/Group By.md

@@ -1 +0,0 @@
-This is used to group items in an array by an expression.

pkg/yqlib/expression_tokeniser.go

@@ -2,7 +2,6 @@ package yqlib
 
 import (
 	"fmt"
-	"regexp"
 	"strconv"
 	"strings"
 
@@ -131,44 +130,6 @@ func opTokenWithPrefs(op *operationType, assignOpType *operationType, preference
 	}
 }
 
-func extractNumberParamter(value string) (int, error) {
-	parameterParser := regexp.MustCompile(`.*\(([0-9]+)\)`)
-	matches := parameterParser.FindStringSubmatch(value)
-	var indent, errParsingInt = strconv.ParseInt(matches[1], 10, 32) // nolint
-	if errParsingInt != nil {
-		return 0, errParsingInt
-	}
-	return int(indent), nil
-}
-
-func flattenWithDepth() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-		var depth, errParsingInt = extractNumberParamter(value)
-		if errParsingInt != nil {
-			return nil, errParsingInt
-		}
-
-		prefs := flattenPreferences{depth: depth}
-		op := &Operation{OperationType: flattenOpType, Value: flattenOpType.Type, StringValue: value, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func encodeWithIndent(outputFormat PrinterOutputFormat) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-		var indent, errParsingInt = extractNumberParamter(value)
-		if errParsingInt != nil {
-			return nil, errParsingInt
-		}
-
-		prefs := encoderPreferences{format: outputFormat, indent: indent}
-		op := &Operation{OperationType: encodeOpType, Value: encodeOpType.Type, StringValue: value, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
 func assignAllCommentsOp(updateAssign bool) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		log.Debug("assignAllCommentsOp %v", string(m.Bytes))
@@ -311,36 +272,11 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`,`), opToken(unionOpType))
 	lexer.Add([]byte(`:\s*`), opToken(createMapOpType))
 	lexer.Add([]byte(`length`), opToken(lengthOpType))
-
-	lexer.Add([]byte(`flatten\([0-9]+\)`), flattenWithDepth())
-	lexer.Add([]byte(`flatten`), opTokenWithPrefs(flattenOpType, nil, flattenPreferences{depth: -1}))
-
-	lexer.Add([]byte(`toyaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat))
-	lexer.Add([]byte(`to_yaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat))
-
-	lexer.Add([]byte(`tojson\([0-9]+\)`), encodeWithIndent(JsonOutputFormat))
-	lexer.Add([]byte(`to_json\([0-9]+\)`), encodeWithIndent(JsonOutputFormat))
-
-	lexer.Add([]byte(`toyaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2}))
-	lexer.Add([]byte(`tojson`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JsonOutputFormat, indent: 2}))
-	lexer.Add([]byte(`toprops`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2}))
-
-	lexer.Add([]byte(`to_yaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2}))
-	lexer.Add([]byte(`to_json`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JsonOutputFormat, indent: 2}))
-	lexer.Add([]byte(`to_props`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2}))
-
-	lexer.Add([]byte(`fromyaml`), opToken(decodeOpType))
-	lexer.Add([]byte(`fromjson`), opToken(decodeOpType))
-
-	lexer.Add([]byte(`from_yaml`), opToken(decodeOpType))
-	lexer.Add([]byte(`from_json`), opToken(decodeOpType))
-
 	lexer.Add([]byte(`sortKeys`), opToken(sortKeysOpType))
 	lexer.Add([]byte(`select`), opToken(selectOpType))
 	lexer.Add([]byte(`has`), opToken(hasOpType))
 	lexer.Add([]byte(`unique`), opToken(uniqueOpType))
 	lexer.Add([]byte(`unique_by`), opToken(uniqueByOpType))
-	lexer.Add([]byte(`group_by`), opToken(groupByOpType))
 	lexer.Add([]byte(`explode`), opToken(explodeOpType))
 	lexer.Add([]byte(`or`), opToken(orOpType))
 	lexer.Add([]byte(`and`), opToken(andOpType))

pkg/yqlib/lib.go

@@ -59,8 +59,6 @@ var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence:
 
 var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: lengthOperator}
 var collectOpType = &operationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handler: collectOperator}
-var encodeOpType = &operationType{Type: "ENCODE", NumArgs: 0, Precedence: 50, Handler: encodeOperator}
-var decodeOpType = &operationType{Type: "DECODE", NumArgs: 0, Precedence: 50, Handler: decodeOperator}
 
 var anyOpType = &operationType{Type: "ANY", NumArgs: 0, Precedence: 50, Handler: anyOperator}
 var allOpType = &operationType{Type: "ALL", NumArgs: 0, Precedence: 50, Handler: allOperator}
@@ -113,8 +111,6 @@ var selectOpType = &operationType{Type: "SELECT", NumArgs: 1, Precedence: 50, Ha
 var hasOpType = &operationType{Type: "HAS", NumArgs: 1, Precedence: 50, Handler: hasOperator}
 var uniqueOpType = &operationType{Type: "UNIQUE", NumArgs: 0, Precedence: 50, Handler: unique}
 var uniqueByOpType = &operationType{Type: "UNIQUE_BY", NumArgs: 1, Precedence: 50, Handler: uniqueBy}
-var groupByOpType = &operationType{Type: "GROUP_BY", NumArgs: 1, Precedence: 50, Handler: groupBy}
-var flattenOpType = &operationType{Type: "FLATTEN_BY", NumArgs: 0, Precedence: 50, Handler: flattenOp}
 var deleteChildOpType = &operationType{Type: "DELETE", NumArgs: 1, Precedence: 40, Handler: deleteChildOperator}
 
 type Operation struct {

pkg/yqlib/operator_assign_test.go

@@ -112,10 +112,9 @@ var assignOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description:    "Update deeply selected results",
-		subdescription: "Note that the LHS is wrapped in brackets! This is to ensure we dont first filter out the yaml and then update the snippet.",
-		document:       `{a: {b: apple, c: cactus}}`,
-		expression:     `(.a[] | select(. == "apple")) = "frog"`,
+		description: "Update selected results",
+		document:    `{a: {b: apple, c: cactus}}`,
+		expression:  `(.a[] | select(. == "apple")) = "frog"`,
 		expected: []string{
 			"D0, P[], (doc)::{a: {b: frog, c: cactus}}\n",
 		},

pkg/yqlib/operator_encoder_decoder.go

@@ -1,91 +0,0 @@
-package yqlib
-
-import (
-	"bufio"
-	"bytes"
-	"container/list"
-	"regexp"
-	"strings"
-
-	"gopkg.in/yaml.v3"
-)
-
-func yamlToString(candidate *CandidateNode, prefs encoderPreferences) (string, error) {
-	var output bytes.Buffer
-	log.Debug("printing with indent: %v", prefs.indent)
-
-	printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), prefs.format, true, false, prefs.indent, true)
-	err := printer.PrintResults(candidate.AsList())
-	return output.String(), err
-}
-
-type encoderPreferences struct {
-	format PrinterOutputFormat
-	indent int
-}
-
-/* encodes object as yaml string */
-
-func encodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	preferences := expressionNode.Operation.Preferences.(encoderPreferences)
-	var results = list.New()
-
-	hasOnlyOneNewLine := regexp.MustCompile("[^\n].*\n$")
-	endWithNewLine := regexp.MustCompile(".*\n$")
-	chomper := regexp.MustCompile("\n+$")
-
-	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
-		candidate := el.Value.(*CandidateNode)
-		stringValue, err := yamlToString(candidate, preferences)
-
-		if err != nil {
-			return Context{}, err
-		}
-
-		// remove trailing newlines if needed.
-		// check if we originally decoded this path, and the original thing had a single line.
-		originalList := context.GetVariable("decoded: " + candidate.GetKey())
-		if originalList != nil && originalList.Len() > 0 && hasOnlyOneNewLine.MatchString(stringValue) {
-
-			original := originalList.Front().Value.(*CandidateNode)
-			originalNode := unwrapDoc(original.Node)
-			// original block did not have a new line at the end, get rid of this one too
-			if !endWithNewLine.MatchString(originalNode.Value) {
-				stringValue = chomper.ReplaceAllString(stringValue, "")
-			}
-		}
-
-		// dont print a new line when printing json on a single line.
-		if preferences.format == JsonOutputFormat && preferences.indent == 0 {
-			stringValue = chomper.ReplaceAllString(stringValue, "")
-		}
-
-		stringContentNode := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: stringValue}
-		results.PushBack(candidate.CreateChild(nil, stringContentNode))
-	}
-	return context.ChildContext(results), nil
-}
-
-/* takes a string and decodes it back into an object */
-func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-
-	var results = list.New()
-	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
-		candidate := el.Value.(*CandidateNode)
-
-		context.SetVariable("decoded: "+candidate.GetKey(), candidate.AsList())
-
-		var dataBucket yaml.Node
-		log.Debugf("got: [%v]", candidate.Node.Value)
-		decoder := yaml.NewDecoder(strings.NewReader(unwrapDoc(candidate.Node).Value))
-		errorReading := decoder.Decode(&dataBucket)
-		if errorReading != nil {
-			return Context{}, errorReading
-		}
-		//first node is a doc
-		node := unwrapDoc(&dataBucket)
-
-		results.PushBack(candidate.CreateChild(nil, node))
-	}
-	return context.ChildContext(results), nil
-}

pkg/yqlib/operator_encoder_decoder_test.go

@@ -1,117 +0,0 @@
-package yqlib
-
-import (
-	"testing"
-)
-
-var prefix = "D0, P[], (doc)::a:\n    cool:\n        bob: dylan\n"
-
-var encoderDecoderOperatorScenarios = []expressionScenario{
-	{
-		description:    "Encode value as yaml string",
-		subdescription: "Indent defaults to 2",
-		document:       "a:\n  cool:\n    bob: dylan",
-		expression:     `.b = (.a | to_yaml)`,
-		expected: []string{
-			prefix + "b: |\n    cool:\n      bob: dylan\n",
-		},
-	},
-	{
-		description:    "Encode value as yaml string, with custom indentation",
-		subdescription: "You can specify the indentation level as the first parameter.",
-		document:       "a:\n  cool:\n    bob: dylan",
-		expression:     `.b = (.a | to_yaml(8))`,
-		expected: []string{
-			prefix + "b: |\n    cool:\n            bob: dylan\n",
-		},
-	},
-	{
-		description:    "Encode value as yaml string, using toyaml",
-		subdescription: "Does the same thing as to_yaml, matching jq naming convention.",
-		document:       `{a: {cool: "thing"}}`,
-		expression:     `.b = (.a | to_yaml)`,
-		expected: []string{
-			`D0, P[], (doc)::{a: {cool: "thing"}, b: "{cool: \"thing\"}\n"}
-`,
-		},
-	},
-	{
-		description: "Encode value as json string",
-		document:    `{a: {cool: "thing"}}`,
-		expression:  `.b = (.a | to_json)`,
-		expected: []string{
-			`D0, P[], (doc)::{a: {cool: "thing"}, b: "{\n  \"cool\": \"thing\"\n}\n"}
-`,
-		},
-	},
-	{
-		description:    "Encode value as json string, on one line",
-		subdescription: "Pass in a 0 indent to print json on a single line.",
-		document:       `{a: {cool: "thing"}}`,
-		expression:     `.b = (.a | to_json(0))`,
-		expected: []string{
-			`D0, P[], (doc)::{a: {cool: "thing"}, b: '{"cool":"thing"}'}
-`,
-		},
-	},
-	{
-		description: "Encode value as props string",
-		document:    `{a: {cool: "thing"}}`,
-		expression:  `.b = (.a | to_props)`,
-		expected: []string{
-			`D0, P[], (doc)::{a: {cool: "thing"}, b: "cool = thing\n"}
-`,
-		},
-	},
-	{
-		description: "Decode a yaml encoded string",
-		document:    `a: "foo: bar"`,
-		expression:  `.b = (.a | from_yaml)`,
-		expected: []string{
-			"D0, P[], (doc)::a: \"foo: bar\"\nb:\n    foo: bar\n",
-		},
-	},
-	{
-		description:           "Update a multiline encoded yaml string",
-		dontFormatInputForDoc: true,
-		document:              "a: |\n  foo: bar\n  baz: dog\n",
-		expression:            `.a |= (from_yaml | .foo = "cat" | to_yaml)`,
-		expected: []string{
-			"D0, P[], (doc)::a: |\n    foo: cat\n    baz: dog\n",
-		},
-	},
-	{
-		skipDoc:               true,
-		dontFormatInputForDoc: true,
-		document:              "a: |-\n  foo: bar\n  baz: dog\n",
-		expression:            `.a |= (from_yaml | .foo = "cat" | to_yaml)`,
-		expected: []string{
-			"D0, P[], (doc)::a: |-\n    foo: cat\n    baz: dog\n",
-		},
-	},
-	{
-		description:           "Update a single line encoded yaml string",
-		dontFormatInputForDoc: true,
-		document:              "a: 'foo: bar'",
-		expression:            `.a |= (from_yaml | .foo = "cat" | to_yaml)`,
-		expected: []string{
-			"D0, P[], (doc)::a: 'foo: cat'\n",
-		},
-	},
-	{
-		skipDoc:               true,
-		dontFormatInputForDoc: true,
-		document:              "a: \"foo: bar\"",
-		expression:            `.a |= (from_yaml | .foo = {"a": "frog"} | to_yaml)`,
-		expected: []string{
-			"D0, P[], (doc)::a: \"foo:\\n  a: frog\"\n",
-		},
-	},
-}
-
-func TestEncoderDecoderOperatorScenarios(t *testing.T) {
-	for _, tt := range encoderDecoderOperatorScenarios {
-		testScenario(t, &tt)
-	}
-	documentScenarios(t, "Encoder and Decoder", encoderDecoderOperatorScenarios)
-}

pkg/yqlib/operator_flatten.go

@@ -1,54 +0,0 @@
-package yqlib
-
-import (
-	"fmt"
-
-	yaml "gopkg.in/yaml.v3"
-)
-
-type flattenPreferences struct {
-	depth int
-}
-
-func flatten(node *yaml.Node, depth int) {
-	if depth == 0 {
-		return
-	}
-	if node.Kind != yaml.SequenceNode {
-		return
-	}
-	content := node.Content
-	newSeq := make([]*yaml.Node, 0)
-
-	for i := 0; i < len(content); i++ {
-		if content[i].Kind == yaml.SequenceNode {
-			flatten(content[i], depth-1)
-			for j := 0; j < len(content[i].Content); j++ {
-				newSeq = append(newSeq, content[i].Content[j])
-			}
-		} else {
-			newSeq = append(newSeq, content[i])
-		}
-	}
-	node.Content = newSeq
-}
-
-func flattenOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-
-	log.Debugf("-- flatten Operator")
-	depth := expressionNode.Operation.Preferences.(flattenPreferences).depth
-
-	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
-		candidate := el.Value.(*CandidateNode)
-		candidateNode := unwrapDoc(candidate.Node)
-		if candidateNode.Kind != yaml.SequenceNode {
-			return Context{}, fmt.Errorf("Only arrays are supported for flatten")
-		}
-
-		flatten(candidateNode, depth)
-
-	}
-
-	return context, nil
-
-}

pkg/yqlib/operator_flatten_test.go

@@ -1,48 +0,0 @@
-package yqlib
-
-import (
-	"testing"
-)
-
-var flattenOperatorScenarios = []expressionScenario{
-	{
-		description:    "Flatten",
-		subdescription: "Recursively flattens all arrays",
-		document:       `[1, [2], [[3]]]`,
-		expression:     `flatten`,
-		expected: []string{
-			"D0, P[], (doc)::[1, 2, 3]\n",
-		},
-	},
-	{
-		description: "Flatten with depth of one",
-		document:    `[1, [2], [[3]]]`,
-		expression:  `flatten(1)`,
-		expected: []string{
-			"D0, P[], (doc)::[1, 2, [3]]\n",
-		},
-	},
-	{
-		description: "Flatten empty array",
-		document:    `[[]]`,
-		expression:  `flatten`,
-		expected: []string{
-			"D0, P[], (doc)::[]\n",
-		},
-	},
-	{
-		description: "Flatten array of objects",
-		document:    `[{foo: bar}, [{foo: baz}]]`,
-		expression:  `flatten`,
-		expected: []string{
-			"D0, P[], (doc)::[{foo: bar}, {foo: baz}]\n",
-		},
-	},
-}
-
-func TestFlattenOperatorScenarios(t *testing.T) {
-	for _, tt := range flattenOperatorScenarios {
-		testScenario(t, &tt)
-	}
-	documentScenarios(t, "Flatten", flattenOperatorScenarios)
-}

pkg/yqlib/operator_group_by.go

@@ -1,76 +0,0 @@
-package yqlib
-
-import (
-	"container/list"
-	"fmt"
-
-	"github.com/elliotchance/orderedmap"
-	yaml "gopkg.in/yaml.v3"
-)
-
-func processIntoGroups(d *dataTreeNavigator, context Context, rhsExp *ExpressionNode, node *yaml.Node) (*orderedmap.OrderedMap, error) {
-	var newMatches = orderedmap.NewOrderedMap()
-	for _, node := range node.Content {
-		child := &CandidateNode{Node: node}
-		rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(child), rhsExp)
-
-		if err != nil {
-			return nil, err
-		}
-
-		keyValue := "null"
-
-		if rhs.MatchingNodes.Len() > 0 {
-			first := rhs.MatchingNodes.Front()
-			keyCandidate := first.Value.(*CandidateNode)
-			keyValue = keyCandidate.Node.Value
-		}
-
-		groupList, exists := newMatches.Get(keyValue)
-
-		if !exists {
-			groupList = list.New()
-			newMatches.Set(keyValue, groupList)
-		}
-		groupList.(*list.List).PushBack(node)
-	}
-	return newMatches, nil
-}
-
-func groupBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-
-	log.Debugf("-- groupBy Operator")
-	var results = list.New()
-
-	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
-		candidate := el.Value.(*CandidateNode)
-		candidateNode := unwrapDoc(candidate.Node)
-
-		if candidateNode.Kind != yaml.SequenceNode {
-			return Context{}, fmt.Errorf("Only arrays are supported for group by")
-		}
-
-		newMatches, err := processIntoGroups(d, context, expressionNode.Rhs, candidateNode)
-
-		if err != nil {
-			return Context{}, err
-		}
-
-		resultNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
-		for groupEl := newMatches.Front(); groupEl != nil; groupEl = groupEl.Next() {
-			groupResultNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
-			groupList := groupEl.Value.(*list.List)
-			for groupItem := groupList.Front(); groupItem != nil; groupItem = groupItem.Next() {
-				groupResultNode.Content = append(groupResultNode.Content, groupItem.Value.(*yaml.Node))
-			}
-
-			resultNode.Content = append(resultNode.Content, groupResultNode)
-		}
-
-		results.PushBack(candidate.CreateChild(nil, resultNode))
-
-	}
-
-	return context.ChildContext(results), nil
-
-}

pkg/yqlib/operator_group_by_test.go

@@ -1,31 +0,0 @@
-package yqlib
-
-import (
-	"testing"
-)
-
-var groupByOperatorScenarios = []expressionScenario{
-	{
-		description: "Group by field",
-		document:    `[{foo: 1, bar: 10}, {foo: 3, bar: 100}, {foo: 1, bar: 1}]`,
-		expression:  `group_by(.foo)`,
-		expected: []string{
-			"D0, P[], (!!seq)::- - {foo: 1, bar: 10}\n  - {foo: 1, bar: 1}\n- - {foo: 3, bar: 100}\n",
-		},
-	},
-	{
-		description: "Group by field, with nuls",
-		document:    `[{cat: dog}, {foo: 1, bar: 10}, {foo: 3, bar: 100}, {no: foo for you}, {foo: 1, bar: 1}]`,
-		expression:  `group_by(.foo)`,
-		expected: []string{
-			"D0, P[], (!!seq)::- - {cat: dog}\n  - {no: foo for you}\n- - {foo: 1, bar: 10}\n  - {foo: 1, bar: 1}\n- - {foo: 3, bar: 100}\n",
-		},
-	},
-}
-
-func TestGroupByOperatorScenarios(t *testing.T) {
-	for _, tt := range groupByOperatorScenarios {
-		testScenario(t, &tt)
-	}
-	documentScenarios(t, "Group By", groupByOperatorScenarios)
-}

pkg/yqlib/operators_test.go

@@ -107,7 +107,7 @@ func copyFromHeader(title string, out *os.File) error {
 
 func formatYaml(yaml string, filename string) string {
 	var output bytes.Buffer
-	printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
 
 	node, err := NewExpressionParser().ParseExpression(".. style= \"\"")
 	if err != nil {
@@ -216,7 +216,7 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) {
 func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formattedDoc string, formattedDoc2 string) {
 	var output bytes.Buffer
 	var err error
-	printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
 
 	node, err := NewExpressionParser().ParseExpression(s.expression)
 	if err != nil {

pkg/yqlib/printer.go

@@ -44,7 +44,7 @@ type resultsPrinter struct {
 	colorsEnabled      bool
 	indent             int
 	printDocSeparators bool
-	printerWriter      PrinterWriter
+	writer             io.Writer
 	firstTimePrinting  bool
 	previousDocIndex   uint
 	previousFileIndex  int
@@ -53,13 +53,9 @@ type resultsPrinter struct {
 	appendixReader     io.Reader
 }
 
-func NewPrinterWithSingleWriter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
-	return NewPrinter(NewSinglePrinterWriter(writer), outputFormat, unwrapScalar, colorsEnabled, indent, printDocSeparators)
-}
-
-func NewPrinter(printerWriter PrinterWriter, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
+func NewPrinter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
 	return &resultsPrinter{
-		printerWriter:      printerWriter,
+		writer:             writer,
 		outputFormat:       outputFormat,
 		unwrapScalar:       unwrapScalar,
 		colorsEnabled:      colorsEnabled,
@@ -102,57 +98,15 @@ func (p *resultsPrinter) writeString(writer io.Writer, txt string) error {
 	return errorWriting
 }
 
-func (p *resultsPrinter) processLeadingContent(mappedDoc *CandidateNode, writer io.Writer) error {
-	if strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$") {
-		log.Debug("headcommentwas %v", mappedDoc.Node.HeadComment)
-		log.Debug("finished headcomment")
-		reader := bufio.NewReader(strings.NewReader(mappedDoc.Node.HeadComment))
-		mappedDoc.Node.HeadComment = ""
-
-		for {
-
-			readline, errReading := reader.ReadString('\n')
-			if errReading != nil && errReading != io.EOF {
-				return errReading
-			}
-			if strings.Contains(readline, "$yqLeadingContent$") {
-				// skip this
-
-			} else if strings.Contains(readline, "$yqDocSeperator$") {
-				if p.printDocSeparators {
-					if err := p.writeString(writer, "---\n"); err != nil {
-						return err
-					}
-				}
-			} else if p.outputFormat == YamlOutputFormat {
-				if err := p.writeString(writer, readline); err != nil {
-					return err
-				}
-			}
-
-			if errReading == io.EOF {
-				if readline != "" {
-					// the last comment we read didn't have a new line, put one in
-					if err := p.writeString(writer, "\n"); err != nil {
-						return err
-					}
-				}
-				break
-			}
-		}
-
+func (p *resultsPrinter) safelyFlush(writer *bufio.Writer) {
+	if err := writer.Flush(); err != nil {
+		log.Error("Error flushing writer!")
+		log.Error(err.Error())
 	}
-	return nil
 }
 
 func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 	log.Debug("PrintResults for %v matches", matchingNodes.Len())
-
-	if matchingNodes.Len() == 0 {
-		log.Debug("no matching results, nothing to print")
-		return nil
-	}
-
 	if p.outputFormat != YamlOutputFormat {
 		explodeOp := Operation{OperationType: explodeOpType}
 		explodeNode := ExpressionNode{Operation: &explodeOp}
@@ -163,6 +117,13 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 		matchingNodes = context.MatchingNodes
 	}
 
+	bufferedWriter := bufio.NewWriter(p.writer)
+	defer p.safelyFlush(bufferedWriter)
+
+	if matchingNodes.Len() == 0 {
+		log.Debug("no matching results, nothing to print")
+		return nil
+	}
 	if p.firstTimePrinting {
 		node := matchingNodes.Front().Value.(*CandidateNode)
 		p.previousDocIndex = node.Document
@@ -171,53 +132,72 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 	}
 
 	for el := matchingNodes.Front(); el != nil; el = el.Next() {
-
 		mappedDoc := el.Value.(*CandidateNode)
 		log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v, printDocSeparators: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document, p.printDocSeparators)
 
-		writer, errorWriting := p.printerWriter.GetWriter(mappedDoc)
-		if errorWriting != nil {
-			return errorWriting
-		}
-
 		commentStartsWithSeparator := strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$\n$yqDocSeperator$")
 
 		if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators && !commentStartsWithSeparator {
 			log.Debug("-- writing doc sep")
-			if err := p.writeString(writer, "---\n"); err != nil {
+			if err := p.writeString(bufferedWriter, "---\n"); err != nil {
 				return err
 			}
 		}
 
-		if err := p.processLeadingContent(mappedDoc, writer); err != nil {
-			return err
+		if strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$") {
+			log.Debug("headcommentwas %v", mappedDoc.Node.HeadComment)
+			log.Debug("finished headcomment")
+			reader := bufio.NewReader(strings.NewReader(mappedDoc.Node.HeadComment))
+			mappedDoc.Node.HeadComment = ""
+
+			for {
+
+				readline, errReading := reader.ReadString('\n')
+				if errReading != nil && errReading != io.EOF {
+					return errReading
+				}
+				if strings.Contains(readline, "$yqLeadingContent$") {
+					// skip this
+
+				} else if strings.Contains(readline, "$yqDocSeperator$") {
+					if p.printDocSeparators {
+						if err := p.writeString(bufferedWriter, "---\n"); err != nil {
+							return err
+						}
+					}
+				} else if p.outputFormat == YamlOutputFormat {
+					if err := p.writeString(bufferedWriter, readline); err != nil {
+						return err
+					}
+				}
+
+				if errReading == io.EOF {
+					if readline != "" {
+						// the last comment we read didn't have a new line, put one in
+						if err := p.writeString(bufferedWriter, "\n"); err != nil {
+							return err
+						}
+					}
+					break
+				}
+			}
+
 		}
 
-		if err := p.printNode(mappedDoc.Node, writer); err != nil {
+		if err := p.printNode(mappedDoc.Node, bufferedWriter); err != nil {
 			return err
 		}
 
 		p.previousDocIndex = mappedDoc.Document
-		if err := writer.Flush(); err != nil {
-			return err
-		}
 	}
 
 	if p.appendixReader != nil && p.outputFormat == YamlOutputFormat {
-		writer, err := p.printerWriter.GetWriter(nil)
-		if err != nil {
-			return err
-		}
-
 		log.Debug("Piping appendix reader...")
 		betterReader := bufio.NewReader(p.appendixReader)
-		_, err = io.Copy(writer, betterReader)
+		_, err := io.Copy(bufferedWriter, betterReader)
 		if err != nil {
 			return err
 		}
-		if err := writer.Flush(); err != nil {
-			return err
-		}
 	}
 
 	return nil

pkg/yqlib/printer_test.go

@@ -36,7 +36,7 @@ func nodeToList(candidate *CandidateNode) *list.List {
 func TestPrinterMultipleDocsInSequence(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -74,7 +74,7 @@ func TestPrinterMultipleDocsInSequence(t *testing.T) {
 func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -116,7 +116,7 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
 func TestPrinterMultipleFilesInSequence(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -163,7 +163,7 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) {
 func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -213,7 +213,7 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
 func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -232,7 +232,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -261,7 +261,7 @@ a: coconut
 func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -287,7 +287,7 @@ a: coconut
 func TestPrinterScalarWithLeadingCont(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	node, err := NewExpressionParser().ParseExpression(".a")
 	if err != nil {
@@ -314,7 +314,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	// note printDocSeparators is true, it should still not print document separators
 	// when outputing JSON.
-	printer := NewPrinterWithSingleWriter(writer, JsonOutputFormat, true, false, 0, true)
+	printer := NewPrinter(writer, JsonOutputFormat, true, false, 0, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {

pkg/yqlib/printer_writer.go

@@ -1,81 +0,0 @@
-package yqlib
-
-import (
-	"bufio"
-	"fmt"
-	"io"
-	"os"
-
-	"gopkg.in/yaml.v3"
-)
-
-type PrinterWriter interface {
-	GetWriter(node *CandidateNode) (*bufio.Writer, error)
-}
-
-type singlePrinterWriter struct {
-	bufferedWriter *bufio.Writer
-}
-
-func NewSinglePrinterWriter(writer io.Writer) PrinterWriter {
-	return &singlePrinterWriter{
-		bufferedWriter: bufio.NewWriter(writer),
-	}
-}
-
-func (sp *singlePrinterWriter) GetWriter(node *CandidateNode) (*bufio.Writer, error) {
-	return sp.bufferedWriter, nil
-}
-
-type multiPrintWriter struct {
-	treeNavigator  DataTreeNavigator
-	nameExpression *ExpressionNode
-	extension      string
-	index          int
-}
-
-func NewMultiPrinterWriter(expression *ExpressionNode, format PrinterOutputFormat) PrinterWriter {
-	extension := "yml"
-
-	switch format {
-	case JsonOutputFormat:
-		extension = "json"
-	case PropsOutputFormat:
-		extension = "properties"
-	}
-
-	return &multiPrintWriter{
-		nameExpression: expression,
-		extension:      extension,
-		treeNavigator:  NewDataTreeNavigator(),
-		index:          0,
-	}
-}
-
-func (sp *multiPrintWriter) GetWriter(node *CandidateNode) (*bufio.Writer, error) {
-	name := ""
-
-	indexVariableNode := yaml.Node{Kind: yaml.ScalarNode, Tag: "!!int", Value: fmt.Sprintf("%v", sp.index)}
-	indexVariableCandidate := CandidateNode{Node: &indexVariableNode}
-
-	context := Context{MatchingNodes: node.AsList()}
-	context.SetVariable("index", indexVariableCandidate.AsList())
-	result, err := sp.treeNavigator.GetMatchingNodes(context, sp.nameExpression)
-	if err != nil {
-		return nil, err
-	}
-	if result.MatchingNodes.Len() > 0 {
-		name = result.MatchingNodes.Front().Value.(*CandidateNode).Node.Value
-	}
-	name = fmt.Sprintf("%v.%v", name, sp.extension)
-
-	f, err := os.Create(name)
-
-	if err != nil {
-		return nil, err
-	}
-	sp.index = sp.index + 1
-
-	return bufio.NewWriter(f), nil
-
-}

release_notes.txt

@@ -1,16 +1,3 @@
-4.14.1:
-  - Added group_by operator
-  - Added encode/decode operators (toyaml, fromjson etc)
-  - Added flatten operator
-
-4.13.5:
-  - Performance improvement for deepMatch (thanks @pmatseykanets)
-  - Added manpage, included in tar.gz downloads as well as a separate tar.gz (#961)
-  - Fixed expression parsing bug #970
-  - Rebuild fixes CVE (#964)
-  - Bumped docker alpine version
-
-
 4.13.4:
   - Fixed select bug (#958)
   - Improved performance of `explode` (this will also speed up json conversion)

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.13.5'
+version: '4.13.4'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.