Working around goccy

.github/workflows/codeql.yml

@@ -42,7 +42,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -53,7 +53,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v3
+      uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -67,4 +67,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
+      uses: github/codeql-action/analyze@v2

.github/workflows/docker-release.yml

@@ -45,7 +45,7 @@ jobs:
           docker buildx build \
             --label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \
             --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \
-            --label "org.opencontainers.image.description=yq is a portable command-line data file processor" \
+            --label "org.opencontainers.image.description=yq is a portable command-line YAML processor" \
             --label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \
             --label "org.opencontainers.image.licenses=MIT" \
             --label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \
@@ -65,7 +65,7 @@ jobs:
           docker buildx build \
             --label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \
             --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \
-            --label "org.opencontainers.image.description=yq is a portable command-line data file processor" \
+            --label "org.opencontainers.image.description=yq is a portable command-line YAML processor" \
             --label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \
             --label "org.opencontainers.image.licenses=MIT" \
             --label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \

.github/workflows/go.yml

@@ -11,7 +11,7 @@ jobs:
     steps:
 
     - name: Set up Go
-      uses: actions/setup-go@v5
+      uses: actions/setup-go@v4
       with:
         go-version: '^1.20'
       id: go

.github/workflows/release.yml

@@ -2,7 +2,7 @@ name: Release YQ
 on:
   push:
     tags:
-      - 'v4.*'
+      - 'v*'
       - 'draft-*'
 
 jobs:
@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-go@v5
+      - uses: actions/setup-go@v4
         with:
           go-version: '^1.20'
           check-latest: true

.golangci.yml

@@ -20,15 +20,12 @@ linters:
     - unparam
 linters-settings:
   depguard:
-    rules:
-      prevent_unmaintained_packages:
-        list-mode: lax
-        files:
-          - $all
-          - "!$test"
-        deny:
-          - pkg: io/ioutil
-            desc: "replaced by io and os packages since Go 1.16: https://tip.golang.org/doc/go1.16#ioutil"
+    list-type: blacklist
+    include-go-root: true
+    packages:
+      - io/ioutil
+    packages-with-error-message:
+      - io/ioutil: "The 'io/ioutil' package is deprecated. Use corresponding 'os' or 'io' functions instead."
 issues:
   exclude-rules:
   - linters:

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.22.1 as builder
+FROM golang:1.21.4 as builder
 
 WORKDIR /go/src/mikefarah/yq
 

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM golang:1.22.1
+FROM golang:1.21.4
 
 COPY scripts/devtools.sh /opt/devtools.sh
 

README.md

@@ -31,10 +31,6 @@ NAME=mike yq -i '.a.b[0].c = strenv(NAME)' file.yaml
 
 Merge multiple files
 ```bash
-# merge two files
-yq -n 'load("file1.yaml") * load("file2.yaml")'
-
-# merge using globs:
 # note the use of `ea` to evaluate all the files at once
 # instead of in sequence
 yq ea '. as $item ireduce ({}; . * $item )' path/to/*.yml

acceptance_tests/flags.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-
-setUp() {
-  rm test*.yml || true
-  cat >test.yml <<EOL
-# comment
-EOL
-}
-
-testStringInterpolation() {
-    X=$(./yq -n '"Mike \(3 + 4)"')
-    assertEquals "Mike 7" "$X"
-}
-
-testNoStringInterpolation() {
-    X=$(./yq --string-interpolation=f -n '"Mike \(3 + 4)"')
-    assertEquals "Mike \(3 + 4)" "$X"
-}
-
-
-source ./scripts/shunit2

acceptance_tests/inputs-format.sh

@@ -63,61 +63,6 @@ EOM
   assertEquals "$expected" "$X"
 }
 
-testInputCSVCustomSeparator() {
-  cat >test.csv <<EOL
-fruit;yumLevel
-apple;5
-banana;4
-EOL
-
-  read -r -d '' expected << EOM
-- fruit: apple
-  yumLevel: 5
-- fruit: banana
-  yumLevel: 4
-EOM
-
-  X=$(./yq -p=csv --csv-separator ";" test.csv)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea -p=csv --csv-separator ";" test.csv)
-  assertEquals "$expected" "$X"
-}
-
-testInputCSVNoAuto() {
-  cat >test.csv <<EOL
-thing1
-name: cat
-EOL
-
-  read -r -d '' expected << EOM
-- thing1: 'name: cat'
-EOM
-
-  X=$(./yq --csv-auto-parse=f test.csv -oy)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea --csv-auto-parse=f test.csv -oy)
-  assertEquals "$expected" "$X"
-}
-
-testInputTSVNoAuto() {
-  cat >test.tsv <<EOL
-thing1
-name: cat
-EOL
-
-  read -r -d '' expected << EOM
-- thing1: 'name: cat'
-EOM
-
-  X=$(./yq --tsv-auto-parse=f test.tsv -oy)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea --tsv-auto-parse=f test.tsv -oy)
-  assertEquals "$expected" "$X"
-}
-
 testInputCSVUTF8() {
   read -r -d '' expected << EOM
 - id: 1

acceptance_tests/output-format.sh

@@ -198,27 +198,6 @@ EOM
   assertEquals "$expected" "$X"
 }
 
-testOutputCSVCustomSeparator() {
-  cat >test.yml <<EOL
-- fruit: apple
-  yumLevel: 5
-- fruit: banana
-  yumLevel: 4
-EOL
-
-  read -r -d '' expected << EOM
-fruit;yumLevel
-apple;5
-banana;4
-EOM
-
-  X=$(./yq -oc --csv-separator ";" test.yml)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea -o=csv --csv-separator ";" test.yml)
-  assertEquals "$expected" "$X"
-}
-
 testOutputTSV() {
   cat >test.yml <<EOL
 - fruit: apple

acceptance_tests/shebang.sh

@@ -1,26 +0,0 @@
-#!/bin/bash
-
-setUp() {
-  rm test*.yq || true
-  cat >test.yq <<EOL
-#!./yq
-.a.b
-EOL
-chmod +x test.yq
-
-rm test*.yml || true
-  cat >test.yml <<EOL
-a: {b: apple}
-EOL
-}
-
-testCanExecYqFile() {
-  read -r -d '' expected << EOM
-apple
-EOM
-   X=$(./test.yq test.yml)
-  assertEquals "$expected" "$X"
-}
-
-source ./scripts/shunit2
-

cmd/evaluate_all_command.go

@@ -12,12 +12,6 @@ func createEvaluateAllCommand() *cobra.Command {
 		Use:     "eval-all [expression] [yaml_file1]...",
 		Aliases: []string{"ea"},
 		Short:   "Loads _all_ yaml documents of _all_ yaml files and runs expression once",
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			if len(args) == 0 {
-				return nil, cobra.ShellCompDirectiveNoFileComp
-			}
-			return nil, cobra.ShellCompDirectiveDefault
-		},
 		Example: `
 # Merge f2.yml into f1.yml (in place)
 yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' f1.yml f2.yml
@@ -32,7 +26,7 @@ yq ea '. as $item ireduce ({}; . * $item )' file1.yml file2.yml ...
 ## use '-' as a filename to pipe from STDIN
 cat file2.yml | yq ea '.a.b' file1.yml - file3.yml
 `,
-		Long: `yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) 
+		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
 See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.
 
 ## Evaluate All ##
@@ -76,7 +70,7 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 		}()
 	}
 
-	format, err := yqlib.FormatFromString(outputFormat)
+	format, err := yqlib.OutputFormatFromString(outputFormat)
 	if err != nil {
 		return err
 	}

cmd/evaluate_sequence_command.go

@@ -13,12 +13,6 @@ func createEvaluateSequenceCommand() *cobra.Command {
 		Use:     "eval [expression] [yaml_file1]...",
 		Aliases: []string{"e"},
 		Short:   "(default) Apply the expression to each document in each yaml file in sequence",
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			if len(args) == 0 {
-				return nil, cobra.ShellCompDirectiveNoFileComp
-			}
-			return nil, cobra.ShellCompDirectiveDefault
-		},
 		Example: `
 # Reads field under the given path for each file
 yq e '.a.b' f1.yml f2.yml 
@@ -37,7 +31,7 @@ yq e -n '.a.b.c = "cat"'
 # Update a file in place
 yq e '.a.b = "cool"' -i file.yaml 
 `,
-		Long: `yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) 
+		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
 See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.
 
 ## Evaluate Sequence ##
@@ -90,7 +84,7 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 		}()
 	}
 
-	format, err := yqlib.FormatFromString(outputFormat)
+	format, err := yqlib.OutputFormatFromString(outputFormat)
 	if err != nil {
 		return err
 	}

cmd/root.go

@@ -1,54 +1,21 @@
 package cmd
 
 import (
-	"fmt"
 	"os"
-	"strings"
 
-	"github.com/mikefarah/yq/v4/pkg/properties"
-	"github.com/mikefarah/yq/v4/pkg/xml"
 	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"github.com/spf13/cobra"
 	logging "gopkg.in/op/go-logging.v1"
 )
 
-type runeValue rune
-
-func newRuneVar(p *rune) *runeValue {
-	return (*runeValue)(p)
-}
-
-func (r *runeValue) String() string {
-	return string(*r)
-}
-
-func (r *runeValue) Set(rawVal string) error {
-	val := strings.ReplaceAll(rawVal, "\\n", "\n")
-	val = strings.ReplaceAll(val, "\\t", "\t")
-	val = strings.ReplaceAll(val, "\\r", "\r")
-	val = strings.ReplaceAll(val, "\\f", "\f")
-	val = strings.ReplaceAll(val, "\\v", "\v")
-	if len(val) != 1 {
-		return fmt.Errorf("[%v] is not a valid character. Must be length 1 was %v", val, len(val))
-	}
-	*r = runeValue(rune(val[0]))
-	return nil
-}
-
-func (r *runeValue) Type() string {
-	return "char"
-}
-
 func New() *cobra.Command {
 	var rootCmd = &cobra.Command{
 		Use:   "yq",
-		Short: "yq is a lightweight and portable command-line data file processor.",
-		Long: `yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) 
+		Short: "yq is a lightweight and portable command-line YAML processor.",
+		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
 See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 		Example: `
-# yq tries to auto-detect the file format based off the extension, and defaults to YAML if it's unknown (or piping through STDIN)
-# Use the '-p/--input-format' flag to specify a format type.
-cat file.xml | yq -p xml
+# yq defaults to 'eval' command if no command is specified. See "yq eval --help" for more examples.
 
 # read the "stuff" node from "myfile.yml"
 yq '.stuff' < myfile.yml
@@ -57,7 +24,7 @@ yq '.stuff' < myfile.yml
 yq -i '.stuff = "foo"' myfile.yml
 
 # print contents of sample.json as idiomatic YAML
-yq -P -oy sample.json
+yq -P sample.json
 `,
 
 		RunE: func(cmd *cobra.Command, args []string) error {
@@ -70,19 +37,18 @@ yq -P -oy sample.json
 		},
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 			cmd.SetOut(cmd.OutOrStdout())
-			level := logging.WARNING
-			stringFormat := `[%{level}] %{color}%{time:15:04:05}%{color:reset} %{message}`
 
-			if verbose {
-				level = logging.DEBUG
-				stringFormat = `[%{level:5.5s}] %{color}%{time:15:04:05}%{color:bold} %{shortfile:-33s} %{shortfunc:-25s}%{color:reset} %{message}`
-			}
-
-			var format = logging.MustStringFormatter(stringFormat)
+			var format = logging.MustStringFormatter(
+				`%{color}%{time:15:04:05} %{shortfunc} [%{level:.4s}]%{color:reset} %{message}`,
+			)
 			var backend = logging.AddModuleLevel(
 				logging.NewBackendFormatter(logging.NewLogBackend(os.Stderr, "", 0), format))
 
-			backend.SetLevel(level, "")
+			if verbose {
+				backend.SetLevel(logging.DEBUG, "")
+			} else {
+				backend.SetLevel(logging.WARNING, "")
+			}
 
 			logging.SetBackend(backend)
 			yqlib.InitExpressionParser()
@@ -99,76 +65,28 @@ yq -P -oy sample.json
 		panic(err)
 	}
 
-	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "auto", fmt.Sprintf("[auto|a|%v] output format type.", yqlib.GetAvailableOutputFormatString()))
-	var outputCompletions = []string{"auto"}
-	for _, formats := range yqlib.GetAvailableOutputFormats() {
-		outputCompletions = append(outputCompletions, formats.FormalName)
-	}
+	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "auto", "[auto|a|yaml|y|json|j|props|p|xml|x|tsv|t|csv|c] output format type.")
+	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "auto", "[auto|a|yaml|y|props|p|xml|x|tsv|t|csv|c|toml] parse format for input. Note that json is a subset of yaml.")
 
-	if err = rootCmd.RegisterFlagCompletionFunc("output-format", cobra.FixedCompletions(outputCompletions, cobra.ShellCompDirectiveNoFileComp)); err != nil {
-		panic(err)
-	}
-	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "auto", fmt.Sprintf("[auto|a|%v] parse format for input.", yqlib.GetAvailableInputFormatString()))
-
-	var inputCompletions = []string{"auto"}
-	for _, formats := range yqlib.GetAvailableInputFormats() {
-		inputCompletions = append(inputCompletions, formats.FormalName)
-	}
-
-	if err = rootCmd.RegisterFlagCompletionFunc("input-format", cobra.FixedCompletions(inputCompletions, cobra.ShellCompDirectiveNoFileComp)); err != nil {
-		panic(err)
-	}
-
-	rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", xml.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes")
-	if err = rootCmd.RegisterFlagCompletionFunc("xml-attribute-prefix", cobra.NoFileCompletions); err != nil {
-		panic(err)
-	}
-	rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.ContentName, "xml-content-name", xml.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).")
-	if err = rootCmd.RegisterFlagCompletionFunc("xml-content-name", cobra.NoFileCompletions); err != nil {
-		panic(err)
-	}
-	rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.StrictMode, "xml-strict-mode", xml.ConfiguredXMLPreferences.StrictMode, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.")
-	rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.KeepNamespace, "xml-keep-namespace", xml.ConfiguredXMLPreferences.KeepNamespace, "enables keeping namespace after parsing attributes")
-	rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.UseRawToken, "xml-raw-token", xml.ConfiguredXMLPreferences.UseRawToken, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.")
-	rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.ProcInstPrefix, "xml-proc-inst-prefix", xml.ConfiguredXMLPreferences.ProcInstPrefix, "prefix for xml processing instructions (e.g. <?xml version=\"1\"?>)")
-	if err = rootCmd.RegisterFlagCompletionFunc("xml-proc-inst-prefix", cobra.NoFileCompletions); err != nil {
-		panic(err)
-	}
-	rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.DirectiveName, "xml-directive-name", xml.ConfiguredXMLPreferences.DirectiveName, "name for xml directives (e.g. <!DOCTYPE thing cat>)")
-	if err = rootCmd.RegisterFlagCompletionFunc("xml-directive-name", cobra.NoFileCompletions); err != nil {
-		panic(err)
-	}
-	rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.SkipProcInst, "xml-skip-proc-inst", xml.ConfiguredXMLPreferences.SkipProcInst, "skip over process instructions (e.g. <?xml version=\"1\"?>)")
-	rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.SkipDirectives, "xml-skip-directives", xml.ConfiguredXMLPreferences.SkipDirectives, "skip over directives (e.g. <!DOCTYPE thing cat>)")
-
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredCsvPreferences.AutoParse, "csv-auto-parse", yqlib.ConfiguredCsvPreferences.AutoParse, "parse CSV YAML/JSON values")
-	rootCmd.PersistentFlags().Var(newRuneVar(&yqlib.ConfiguredCsvPreferences.Separator), "csv-separator", "CSV Separator character")
-
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredTsvPreferences.AutoParse, "tsv-auto-parse", yqlib.ConfiguredTsvPreferences.AutoParse, "parse TSV YAML/JSON values")
+	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", yqlib.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes")
+	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", yqlib.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.StrictMode, "xml-strict-mode", yqlib.ConfiguredXMLPreferences.StrictMode, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.KeepNamespace, "xml-keep-namespace", yqlib.ConfiguredXMLPreferences.KeepNamespace, "enables keeping namespace after parsing attributes")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.UseRawToken, "xml-raw-token", yqlib.ConfiguredXMLPreferences.UseRawToken, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.")
+	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ProcInstPrefix, "xml-proc-inst-prefix", yqlib.ConfiguredXMLPreferences.ProcInstPrefix, "prefix for xml processing instructions (e.g. <?xml version=\"1\"?>)")
+	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.DirectiveName, "xml-directive-name", yqlib.ConfiguredXMLPreferences.DirectiveName, "name for xml directives (e.g. <!DOCTYPE thing cat>)")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipProcInst, "xml-skip-proc-inst", yqlib.ConfiguredXMLPreferences.SkipProcInst, "skip over process instructions (e.g. <?xml version=\"1\"?>)")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipDirectives, "xml-skip-directives", yqlib.ConfiguredXMLPreferences.SkipDirectives, "skip over directives (e.g. <!DOCTYPE thing cat>)")
 
 	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredLuaPreferences.DocPrefix, "lua-prefix", yqlib.ConfiguredLuaPreferences.DocPrefix, "prefix")
-	if err = rootCmd.RegisterFlagCompletionFunc("lua-prefix", cobra.NoFileCompletions); err != nil {
-		panic(err)
-	}
 	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredLuaPreferences.DocSuffix, "lua-suffix", yqlib.ConfiguredLuaPreferences.DocSuffix, "suffix")
-	if err = rootCmd.RegisterFlagCompletionFunc("lua-suffix", cobra.NoFileCompletions); err != nil {
-		panic(err)
-	}
 	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.UnquotedKeys, "lua-unquoted", yqlib.ConfiguredLuaPreferences.UnquotedKeys, "output unquoted string keys (e.g. {foo=\"bar\"})")
 	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.Globals, "lua-globals", yqlib.ConfiguredLuaPreferences.Globals, "output keys as top-level global variables")
 
-	rootCmd.PersistentFlags().StringVar(&properties.ConfiguredPropertiesPreferences.KeyValueSeparator, "properties-separator", properties.ConfiguredPropertiesPreferences.KeyValueSeparator, "separator to use between keys and values")
-	rootCmd.PersistentFlags().BoolVar(&properties.ConfiguredPropertiesPreferences.UseArrayBrackets, "properties-array-brackets", properties.ConfiguredPropertiesPreferences.UseArrayBrackets, "use [x] in array paths (e.g. for SpringBoot)")
-
-	rootCmd.PersistentFlags().BoolVar(&yqlib.StringInterpolationEnabled, "string-interpolation", yqlib.StringInterpolationEnabled, "Toggles strings interpolation of \\(exp)")
-
 	rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.")
 	rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)")
 
 	rootCmd.PersistentFlags().IntVarP(&indent, "indent", "I", 2, "sets indent level for output")
-	if err = rootCmd.RegisterFlagCompletionFunc("indent", cobra.NoFileCompletions); err != nil {
-		panic(err)
-	}
 	rootCmd.Flags().BoolVarP(&version, "version", "V", false, "Print version information and quit")
 	rootCmd.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the file in place of first file given.")
 	rootCmd.PersistentFlags().VarP(unwrapScalarFlag, "unwrapScalar", "r", "unwrap scalar, print the value with no quotes, colors or comments. Defaults to true for yaml")
@@ -181,28 +99,13 @@ yq -P -oy sample.json
 	rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors")
 	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
 	rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
-	if err = rootCmd.RegisterFlagCompletionFunc("front-matter", cobra.FixedCompletions([]string{"extract", "process"}, cobra.ShellCompDirectiveNoFileComp)); err != nil {
-		panic(err)
-	}
 	rootCmd.PersistentFlags().StringVarP(&forceExpression, "expression", "", "", "forcibly set the expression argument. Useful when yq argument detection thinks your expression is a file.")
-	if err = rootCmd.RegisterFlagCompletionFunc("expression", cobra.NoFileCompletions); err != nil {
-		panic(err)
-	}
 	rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredYamlPreferences.LeadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.")
 
 	rootCmd.PersistentFlags().StringVarP(&splitFileExp, "split-exp", "s", "", "print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter.")
-	if err = rootCmd.RegisterFlagCompletionFunc("split-exp", cobra.NoFileCompletions); err != nil {
-		panic(err)
-	}
 	rootCmd.PersistentFlags().StringVarP(&splitFileExpFile, "split-exp-file", "", "", "Use a file to specify the split-exp expression.")
-	if err = rootCmd.MarkPersistentFlagFilename("split-exp-file"); err != nil {
-		panic(err)
-	}
 
 	rootCmd.PersistentFlags().StringVarP(&expressionFile, "from-file", "", "", "Load expression from specified file.")
-	if err = rootCmd.MarkPersistentFlagFilename("from-file"); err != nil {
-		panic(err)
-	}
 
 	rootCmd.AddCommand(
 		createEvaluateSequenceCommand(),

cmd/shell-completion.go

@@ -47,7 +47,7 @@ $ yq shell-completion fish > ~/.config/fish/completions/yq.fish
 		var err error = nil
 		switch args[0] {
 		case "bash":
-			err = cmd.Root().GenBashCompletionV2(os.Stdout, true)
+			err = cmd.Root().GenBashCompletion(os.Stdout)
 		case "zsh":
 			err = cmd.Root().GenZshCompletion(os.Stdout)
 		case "fish":

cmd/utils.go

@@ -6,8 +6,6 @@ import (
 	"os"
 	"strings"
 
-	"github.com/mikefarah/yq/v4/pkg/properties"
-	"github.com/mikefarah/yq/v4/pkg/xml"
 	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"github.com/spf13/cobra"
 	"gopkg.in/op/go-logging.v1"
@@ -66,9 +64,9 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 	}
 	if inputFormat == "" || inputFormat == "auto" || inputFormat == "a" {
 
-		inputFormat = yqlib.FormatStringFromFilename(inputFilename)
+		inputFormat = yqlib.FormatFromFilename(inputFilename)
 
-		_, err := yqlib.FormatFromString(inputFormat)
+		_, err := yqlib.InputFormatFromString(inputFormat)
 		if err != nil {
 			// unknown file type, default to yaml
 			yqlib.GetLogger().Debug("Unknown file format extension '%v', defaulting to yaml", inputFormat)
@@ -88,14 +86,14 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 		// before this was introduced, `yq -pcsv things.csv`
 		// would produce *yaml* output.
 		//
-		outputFormat = yqlib.FormatStringFromFilename(inputFilename)
+		outputFormat = yqlib.FormatFromFilename(inputFilename)
 		if inputFilename != "-" {
 			yqlib.GetLogger().Warning("yq default output is now 'auto' (based on the filename extension). Normally yq would output '%v', but for backwards compatibility 'yaml' has been set. Please use -oy to specify yaml, or drop the -p flag.", outputFormat)
 		}
 		outputFormat = "yaml"
 	}
 
-	outputFormatType, err := yqlib.FormatFromString(outputFormat)
+	outputFormatType, err := yqlib.OutputFormatFromString(outputFormat)
 
 	if err != nil {
 		return "", nil, err
@@ -103,32 +101,59 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 	yqlib.GetLogger().Debug("Using input format %v", inputFormat)
 	yqlib.GetLogger().Debug("Using output format %v", outputFormat)
 
-	if outputFormatType == yqlib.YamlFormat ||
-		outputFormatType == properties.PropertiesFormat {
+	if outputFormatType == yqlib.YamlOutputFormat ||
+		outputFormatType == yqlib.PropsOutputFormat {
 		unwrapScalar = true
 	}
 	if unwrapScalarFlag.IsExplicitlySet() {
 		unwrapScalar = unwrapScalarFlag.IsSet()
 	}
 
+	//copy preference form global setting
+	yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar
+
+	yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators
+
 	return expression, args, nil
 }
 
 func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
-	format, err := yqlib.FormatFromString(inputFormat)
+	yqlibInputFormat, err := yqlib.InputFormatFromString(inputFormat)
 	if err != nil {
 		return nil, err
 	}
-	yqlib.ConfiguredYamlPreferences.EvaluateTogether = evaluateTogether
-
-	yqlibDecoder := format.DecoderFactory()
+	yqlibDecoder, err := createDecoder(yqlibInputFormat, evaluateTogether)
 	if yqlibDecoder == nil {
 		return nil, fmt.Errorf("no support for %s input format", inputFormat)
 	}
-	return yqlibDecoder, nil
+	return yqlibDecoder, err
 }
 
-func configurePrinterWriter(format *yqlib.Format, out io.Writer) (yqlib.PrinterWriter, error) {
+func createDecoder(format yqlib.InputFormat, evaluateTogether bool) (yqlib.Decoder, error) {
+	switch format {
+	case yqlib.LuaInputFormat:
+		return yqlib.NewLuaDecoder(yqlib.ConfiguredLuaPreferences), nil
+	case yqlib.XMLInputFormat:
+		return yqlib.NewXMLDecoder(yqlib.ConfiguredXMLPreferences), nil
+	case yqlib.PropertiesInputFormat:
+		return yqlib.NewPropertiesDecoder(), nil
+	case yqlib.JsonInputFormat:
+		return yqlib.NewJSONDecoder(), nil
+	case yqlib.CSVObjectInputFormat:
+		return yqlib.NewCSVObjectDecoder(','), nil
+	case yqlib.TSVObjectInputFormat:
+		return yqlib.NewCSVObjectDecoder('\t'), nil
+	case yqlib.TomlInputFormat:
+		return yqlib.NewTomlDecoder(), nil
+	case yqlib.YamlInputFormat:
+		prefs := yqlib.ConfiguredYamlPreferences
+		prefs.EvaluateTogether = evaluateTogether
+		return yqlib.NewYamlDecoder(prefs), nil
+	}
+	return nil, fmt.Errorf("invalid decoder: %v", format)
+}
+
+func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yqlib.PrinterWriter, error) {
 
 	var printerWriter yqlib.PrinterWriter
 
@@ -146,29 +171,39 @@ func configurePrinterWriter(format *yqlib.Format, out io.Writer) (yqlib.PrinterW
 }
 
 func configureEncoder() (yqlib.Encoder, error) {
-	yqlibOutputFormat, err := yqlib.FormatFromString(outputFormat)
+	yqlibOutputFormat, err := yqlib.OutputFormatFromString(outputFormat)
 	if err != nil {
 		return nil, err
 	}
-	xml.ConfiguredXMLPreferences.Indent = indent
-	yqlib.ConfiguredYamlPreferences.Indent = indent
-	yqlib.ConfiguredJSONPreferences.Indent = indent
-
-	yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar
-	properties.ConfiguredPropertiesPreferences.UnwrapScalar = unwrapScalar
-	yqlib.ConfiguredJSONPreferences.UnwrapScalar = unwrapScalar
-
-	yqlib.ConfiguredYamlPreferences.ColorsEnabled = colorsEnabled
-	yqlib.ConfiguredJSONPreferences.ColorsEnabled = colorsEnabled
-
-	yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators
-
-	encoder := yqlibOutputFormat.EncoderFactory()
-
-	if encoder == nil {
+	yqlibEncoder, err := createEncoder(yqlibOutputFormat)
+	if yqlibEncoder == nil {
 		return nil, fmt.Errorf("no support for %s output format", outputFormat)
 	}
-	return encoder, err
+	return yqlibEncoder, err
+}
+
+func createEncoder(format yqlib.PrinterOutputFormat) (yqlib.Encoder, error) {
+	switch format {
+	case yqlib.JSONOutputFormat:
+		return yqlib.NewJSONEncoder(indent, colorsEnabled, unwrapScalar), nil
+	case yqlib.PropsOutputFormat:
+		return yqlib.NewPropertiesEncoder(unwrapScalar), nil
+	case yqlib.CSVOutputFormat:
+		return yqlib.NewCsvEncoder(','), nil
+	case yqlib.TSVOutputFormat:
+		return yqlib.NewCsvEncoder('\t'), nil
+	case yqlib.YamlOutputFormat:
+		return yqlib.NewYamlEncoder(indent, colorsEnabled, yqlib.ConfiguredYamlPreferences), nil
+	case yqlib.XMLOutputFormat:
+		return yqlib.NewXMLEncoder(indent, yqlib.ConfiguredXMLPreferences), nil
+	case yqlib.TomlOutputFormat:
+		return yqlib.NewTomlEncoder(), nil
+	case yqlib.ShellVariablesOutputFormat:
+		return yqlib.NewShellVariablesEncoder(), nil
+	case yqlib.LuaOutputFormat:
+		return yqlib.NewLuaEncoder(yqlib.ConfiguredLuaPreferences), nil
+	}
+	return nil, fmt.Errorf("invalid encoder: %v", format)
 }
 
 // this is a hack to enable backwards compatibility with githubactions (which pipe /dev/null into everything)
@@ -220,16 +255,6 @@ func processStdInArgs(args []string) []string {
 
 func processArgs(originalArgs []string) (string, []string, error) {
 	expression := forceExpression
-	args := processStdInArgs(originalArgs)
-	maybeFirstArgIsAFile := len(args) > 0 && maybeFile(args[0])
-
-	if expressionFile == "" && maybeFirstArgIsAFile && strings.HasSuffix(args[0], ".yq") {
-		// lets check if an expression file was given
-		yqlib.GetLogger().Debug("Assuming arg %v is an expression file", args[0])
-		expressionFile = args[0]
-		args = args[1:]
-	}
-
 	if expressionFile != "" {
 		expressionBytes, err := os.ReadFile(expressionFile)
 		if err != nil {
@@ -239,6 +264,7 @@ func processArgs(originalArgs []string) (string, []string, error) {
 		expression = strings.ReplaceAll(string(expressionBytes), "\r\n", "\n")
 	}
 
+	args := processStdInArgs(originalArgs)
 	yqlib.GetLogger().Debugf("processed args: %v", args)
 	if expression == "" && len(args) > 0 && args[0] != "-" && !maybeFile(args[0]) {
 		yqlib.GetLogger().Debug("assuming expression is '%v'", args[0])

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "v4.42.1"
+	Version = "v4.40.2"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

debian/changelog

@@ -135,7 +135,7 @@ yq (4.9.6) focal; urgency=medium
 yq (3.3.2) focal; urgency=medium
 
   * Bug fix: existStatus bug (#459)
-  * Automatically makes an os temp directory if it does not exist (#461)
+  * Automatically makes a os temp directory if it does not exist (#461)
 
  -- Roberto Mier Escandon <rmescandon@gmail.com>  Fri, 07 Aug 2020 18:53:01 +0200
 

debian/control

@@ -16,7 +16,7 @@ XSBC-Original-Maintainer: Roberto Mier Escandón <rmescandon@gmail.com>
 Package: yq
 Architecture: any
 Depends: ${shlibs:Depends}, ${misc:Depends}
-Description: lightweight and portable command-line data file processor
+Description: lightweight and portable command-line YAML processor
  .
  The aim of the project is to be the 
  [jq](https://github.com/stedolan/jq) or sed of yaml files.

examples/data1.yaml

@@ -1,11 +1,2 @@
-# block comments come through
-person: # neither do comments on maps
-    name: Mike Wazowski # comments on values appear
-    pets: 
-    - cat # comments on array values appear
-    - dog # comments on array values appear
-    - things:
-        - frog
-    food: [pizza] # comments on arrays do not
-emptyArray: []
-emptyMap: []
+a: #things
+  meow

examples/environment.yq

@@ -1,5 +0,0 @@
-#! yq
-.[] |(
-    ( select(kind == "scalar") | key + "='" + . + "'"),
-    ( select(kind == "seq") | key + "=(" + (map("'" + . + "'") | join(",")) + ")")
-)

go.mod

@@ -2,21 +2,21 @@ module github.com/mikefarah/yq/v4
 
 require (
 	github.com/a8m/envsubst v1.4.2
-	github.com/alecthomas/participle/v2 v2.1.1
-	github.com/alecthomas/repr v0.4.0
+	github.com/alecthomas/participle/v2 v2.1.0
+	github.com/alecthomas/repr v0.3.0
 	github.com/dimchansky/utfbom v1.1.1
 	github.com/elliotchance/orderedmap v1.5.1
 	github.com/fatih/color v1.16.0
 	github.com/goccy/go-json v0.10.2
-	github.com/goccy/go-yaml v1.11.3
+	github.com/goccy/go-yaml v1.11.2
 	github.com/jinzhu/copier v0.4.0
 	github.com/magiconair/properties v1.8.7
-	github.com/pelletier/go-toml/v2 v2.1.1
+	github.com/pelletier/go-toml/v2 v2.1.0
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
-	github.com/yuin/gopher-lua v1.1.1
-	golang.org/x/net v0.22.0
+	github.com/yuin/gopher-lua v1.1.0
+	golang.org/x/net v0.18.0
 	golang.org/x/text v0.14.0
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.1
@@ -26,7 +26,7 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/sys v0.14.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 )
 

go.sum

@@ -2,10 +2,10 @@ github.com/a8m/envsubst v1.4.2 h1:4yWIHXOLEJHQEFd4UjrWDrYeYlV7ncFWJOCBRLOZHQg=
 github.com/a8m/envsubst v1.4.2/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
 github.com/alecthomas/assert/v2 v2.3.0 h1:mAsH2wmvjsuvyBvAmCtm7zFsBlb8mIHx5ySLVdDZXL0=
 github.com/alecthomas/assert/v2 v2.3.0/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ=
-github.com/alecthomas/participle/v2 v2.1.1 h1:hrjKESvSqGHzRb4yW1ciisFJ4p3MGYih6icjJvbsmV8=
-github.com/alecthomas/participle/v2 v2.1.1/go.mod h1:Y1+hAs8DHPmc3YUFzqllV+eSQ9ljPTk0ZkPMtEdAx2c=
-github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
-github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/alecthomas/participle/v2 v2.1.0 h1:z7dElHRrOEEq45F2TG5cbQihMtNTv8vwldytDj7Wrz4=
+github.com/alecthomas/participle/v2 v2.1.0/go.mod h1:Y1+hAs8DHPmc3YUFzqllV+eSQ9ljPTk0ZkPMtEdAx2c=
+github.com/alecthomas/repr v0.3.0 h1:NeYzUPfjjlqHY4KtzgKJiWd6sVq2eNUPTi34PiFGjY8=
+github.com/alecthomas/repr v0.3.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -24,8 +24,8 @@ github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7a
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
-github.com/goccy/go-yaml v1.11.3 h1:B3W9IdWbvrUu2OYQGwvU1nZtvMQJPBKgBUuweJjLj6I=
-github.com/goccy/go-yaml v1.11.3/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU=
+github.com/goccy/go-yaml v1.11.2 h1:joq77SxuyIs9zzxEjgyLBugMQ9NEgTWxXfz2wVqwAaQ=
+github.com/goccy/go-yaml v1.11.2/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
@@ -43,8 +43,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
-github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
+github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
+github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -62,16 +62,16 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
-github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
-golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
-golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
-golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE=
+github.com/yuin/gopher-lua v1.1.0/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
+golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA=
+golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g=
+golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
+golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=

pkg/properties/encoder_properties_test.go

@@ -1,208 +0,0 @@
-package properties
-
-import (
-	"bufio"
-	"bytes"
-	"strings"
-	"testing"
-
-	"github.com/mikefarah/yq/v4/test"
-)
-
-type keyValuePair struct {
-	key     string
-	value   string
-	comment string
-}
-
-func (kv *keyValuePair) String(unwrap bool, sep string) string {
-	builder := strings.Builder{}
-
-	if kv.comment != "" {
-		builder.WriteString(kv.comment)
-		builder.WriteString("\n")
-	}
-
-	builder.WriteString(kv.key)
-	builder.WriteString(sep)
-
-	if unwrap {
-		builder.WriteString(kv.value)
-	} else {
-		builder.WriteString("\"")
-		builder.WriteString(kv.value)
-		builder.WriteString("\"")
-	}
-
-	return builder.String()
-}
-
-type testProperties struct {
-	pairs []keyValuePair
-}
-
-func (tp *testProperties) String(unwrap bool, sep string) string {
-	kvs := []string{}
-
-	for _, kv := range tp.pairs {
-		kvs = append(kvs, kv.String(unwrap, sep))
-	}
-
-	return strings.Join(kvs, "\n")
-}
-
-func yamlToProps(sampleYaml string, unwrapScalar bool, separator string) string {
-	var output bytes.Buffer
-	writer := bufio.NewWriter(&output)
-
-	var propsEncoder = NewPropertiesEncoder(PropertiesPreferences{KeyValueSeparator: separator, UnwrapScalar: unwrapScalar})
-	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
-	if err != nil {
-		panic(err)
-	}
-	node := inputs.Front().Value.(*CandidateNode)
-	err = propsEncoder.Encode(writer, node)
-	if err != nil {
-		panic(err)
-	}
-	writer.Flush()
-
-	return strings.TrimSuffix(output.String(), "\n")
-}
-
-func doTest(t *testing.T, sampleYaml string, props testProperties, testUnwrapped, testWrapped bool) {
-	wraps := []bool{}
-	if testUnwrapped {
-		wraps = append(wraps, true)
-	}
-	if testWrapped {
-		wraps = append(wraps, false)
-	}
-
-	for _, unwrap := range wraps {
-		for _, sep := range []string{" = ", ";", "=", " "} {
-			var actualProps = yamlToProps(sampleYaml, unwrap, sep)
-			test.AssertResult(t, props.String(unwrap, sep), actualProps)
-		}
-	}
-}
-
-func TestPropertiesEncoderSimple(t *testing.T) {
-	var sampleYaml = `a: 'bob cool'`
-
-	doTest(
-		t, sampleYaml,
-		testProperties{
-			pairs: []keyValuePair{
-				{
-					key:   "a",
-					value: "bob cool",
-				},
-			},
-		},
-		true, true,
-	)
-}
-
-func TestPropertiesEncoderSimpleWithComments(t *testing.T) {
-	var sampleYaml = `a: 'bob cool' # line`
-
-	doTest(
-		t, sampleYaml,
-		testProperties{
-			pairs: []keyValuePair{
-				{
-					key:     "a",
-					value:   "bob cool",
-					comment: "# line",
-				},
-			},
-		},
-		true, true,
-	)
-}
-
-func TestPropertiesEncoderDeep(t *testing.T) {
-	var sampleYaml = `a: 
-  b: "bob cool"
-`
-
-	doTest(
-		t, sampleYaml,
-		testProperties{
-			pairs: []keyValuePair{
-				{
-					key:   "a.b",
-					value: "bob cool",
-				},
-			},
-		},
-		true, true,
-	)
-}
-
-func TestPropertiesEncoderDeepWithComments(t *testing.T) {
-	var sampleYaml = `a:  # a thing
-  b: "bob cool" # b thing
-`
-
-	doTest(
-		t, sampleYaml,
-		testProperties{
-			pairs: []keyValuePair{
-				{
-					key:     "a.b",
-					value:   "bob cool",
-					comment: "# b thing",
-				},
-			},
-		},
-		true, true,
-	)
-}
-
-func TestPropertiesEncoderArray_Unwrapped(t *testing.T) {
-	var sampleYaml = `a: 
-  b: [{c: dog}, {c: cat}]
-`
-
-	doTest(
-		t, sampleYaml,
-		testProperties{
-			pairs: []keyValuePair{
-				{
-					key:   "a.b.0.c",
-					value: "dog",
-				},
-				{
-					key:   "a.b.1.c",
-					value: "cat",
-				},
-			},
-		},
-		true, false,
-	)
-}
-
-func TestPropertiesEncoderArray_Wrapped(t *testing.T) {
-	var sampleYaml = `a: 
-  b: [{c: dog named jim}, {c: cat named jim}]
-`
-
-	doTest(
-		t, sampleYaml,
-		testProperties{
-			pairs: []keyValuePair{
-				{
-					key:   "a.b.0.c",
-					value: "dog named jim",
-				},
-				{
-					key:   "a.b.1.c",
-					value: "cat named jim",
-				},
-			},
-		},
-		false, true,
-	)
-}

pkg/properties/properties.go

@@ -1,45 +0,0 @@
-package properties
-
-import "github.com/mikefarah/yq/v4/pkg/yqlib"
-
-type PropertiesPreferences struct {
-	UnwrapScalar      bool
-	KeyValueSeparator string
-	UseArrayBrackets  bool
-}
-
-func NewDefaultPropertiesPreferences() PropertiesPreferences {
-	return PropertiesPreferences{
-		UnwrapScalar:      true,
-		KeyValueSeparator: " = ",
-		UseArrayBrackets:  false,
-	}
-}
-
-func (p *PropertiesPreferences) Copy() PropertiesPreferences {
-	return PropertiesPreferences{
-		UnwrapScalar:      p.UnwrapScalar,
-		KeyValueSeparator: p.KeyValueSeparator,
-		UseArrayBrackets:  p.UseArrayBrackets,
-	}
-}
-
-var PropertiesFormat = &yqlib.Format{"props", []string{"p", "properties"}, "properties",
-	func() yqlib.Encoder { return NewPropertiesEncoder(ConfiguredPropertiesPreferences) },
-	func() yqlib.Decoder { return NewPropertiesDecoder() },
-	nil,
-}
-
-var propertyYqRules = []*yqlib.ParticipleYqRule{
-	{"PropertiesDecode", `from_?props|@propsd`, yqlib.CreateDecodeOpYqAction(PropertiesFormat), 0},
-	{"PropsEncode", `to_?props|@props`, yqlib.CreateEncodeOpYqAction(PropertiesFormat, 2), 0},
-	{"LoadProperties", `load_?props`, yqlib.CreateLoadOpYqAction(NewPropertiesDecoder()), 0},
-}
-
-func RegisterPropertiesFormat() {
-	yqlib.RegisterFormat(PropertiesFormat)
-	yqlib.RegisterRules(propertyYqRules)
-
-}
-
-var ConfiguredPropertiesPreferences = NewDefaultPropertiesPreferences()

pkg/xml/xml.go

@@ -1,72 +0,0 @@
-package xml
-
-import "github.com/mikefarah/yq/v4/pkg/yqlib"
-
-type XmlPreferences struct {
-	Indent          int
-	AttributePrefix string
-	ContentName     string
-	StrictMode      bool
-	KeepNamespace   bool
-	UseRawToken     bool
-	ProcInstPrefix  string
-	DirectiveName   string
-	SkipProcInst    bool
-	SkipDirectives  bool
-}
-
-func NewDefaultXmlPreferences() XmlPreferences {
-	return XmlPreferences{
-		Indent:          2,
-		AttributePrefix: "+@",
-		ContentName:     "+content",
-		StrictMode:      false,
-		KeepNamespace:   true,
-		UseRawToken:     true,
-		ProcInstPrefix:  "+p_",
-		DirectiveName:   "+directive",
-		SkipProcInst:    false,
-		SkipDirectives:  false,
-	}
-}
-
-func (p *XmlPreferences) Copy() XmlPreferences {
-	return XmlPreferences{
-		Indent:          p.Indent,
-		AttributePrefix: p.AttributePrefix,
-		ContentName:     p.ContentName,
-		StrictMode:      p.StrictMode,
-		KeepNamespace:   p.KeepNamespace,
-		UseRawToken:     p.UseRawToken,
-		ProcInstPrefix:  p.ProcInstPrefix,
-		DirectiveName:   p.DirectiveName,
-		SkipProcInst:    p.SkipProcInst,
-		SkipDirectives:  p.SkipDirectives,
-	}
-}
-
-var ConfiguredXMLPreferences = NewDefaultXmlPreferences()
-
-var XMLFormat = &yqlib.Format{"xml", []string{"x"}, "xml",
-	func() yqlib.Encoder { return NewXMLEncoder(ConfiguredXMLPreferences) },
-	func() yqlib.Decoder { return NewXMLDecoder(ConfiguredXMLPreferences) },
-	func(indent int) yqlib.Encoder {
-		prefs := ConfiguredXMLPreferences.Copy()
-		prefs.Indent = indent
-		return NewXMLEncoder(prefs)
-	},
-}
-
-var xmlYqRules = []*yqlib.ParticipleYqRule{
-	{"XMLEncodeWithIndent", `to_?xml\([0-9]+\)`, yqlib.CreateEncodeYqActionParsingIndent(XMLFormat), 0},
-	{"XmlDecode", `from_?xml|@xmld`, yqlib.CreateDecodeOpYqAction(XMLFormat), 0},
-	{"XMLEncode", `to_?xml`, yqlib.CreateEncodeOpYqAction(XMLFormat, 2), 0},
-	{"XMLEncodeNoIndent", `@xml`, yqlib.CreateEncodeOpYqAction(XMLFormat, 0), 0},
-	{"LoadXML", `load_?xml|xml_?load`, yqlib.CreateLoadOpYqAction(NewXMLDecoder(ConfiguredXMLPreferences)), 0},
-}
-
-func RegisterXmlFormat() {
-	yqlib.RegisterFormat(XMLFormat)
-	yqlib.RegisterRules(xmlYqRules)
-
-}

pkg/yqlib/all_at_once_evaluator.go

@@ -63,7 +63,7 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
 	}
 
 	if allDocuments.Len() == 0 {
-		candidateNode := CreateScalarNode(nil, "")
+		candidateNode := createScalarNode(nil, "")
 		allDocuments.PushBack(candidateNode)
 	}
 

pkg/yqlib/all_at_once_evaluator_test.go

@@ -36,7 +36,7 @@ func TestAllAtOnceEvaluateNodes(t *testing.T) {
 	var evaluator = NewAllAtOnceEvaluator()
 	// logging.SetLevel(logging.DEBUG, "")
 	for _, tt := range evaluateNodesScenario {
-		decoder := NewYamlDecoder(ConfiguredYamlPreferences)
+		decoder := NewYamlDecoder(NewDefaultYamlPreferences())
 		reader := bufio.NewReader(strings.NewReader(tt.document))
 		err := decoder.Init(reader)
 		if err != nil {

pkg/yqlib/candidate_node.go

@@ -27,14 +27,14 @@ const (
 	FlowStyle
 )
 
-func CreateStringScalarNode(stringValue string) *CandidateNode {
+func createStringScalarNode(stringValue string) *CandidateNode {
 	var node = &CandidateNode{Kind: ScalarNode}
 	node.Value = stringValue
 	node.Tag = "!!str"
 	return node
 }
 
-func CreateScalarNode(value interface{}, stringValue string) *CandidateNode {
+func createScalarNode(value interface{}, stringValue string) *CandidateNode {
 	var node = &CandidateNode{Kind: ScalarNode}
 	node.Value = stringValue
 
@@ -205,7 +205,6 @@ func (n *CandidateNode) AddKeyValueChild(rawKey *CandidateNode, rawValue *Candid
 
 	value := rawValue.Copy()
 	value.SetParent(n)
-	value.IsMapKey = false // force this, incase we are creating a value from a key
 	value.Key = key
 
 	n.Content = append(n.Content, key, value)
@@ -219,7 +218,7 @@ func (n *CandidateNode) AddChild(rawChild *CandidateNode) {
 		value.Key.SetParent(n)
 	} else {
 		index := len(n.Content)
-		keyNode := CreateScalarNode(index, fmt.Sprintf("%v", index))
+		keyNode := createScalarNode(index, fmt.Sprintf("%v", index))
 		keyNode.SetParent(n)
 		value.Key = keyNode
 	}
@@ -243,7 +242,7 @@ func (n *CandidateNode) AddChildren(children []*CandidateNode) {
 
 func (n *CandidateNode) GetValueRep() (interface{}, error) {
 	log.Debugf("GetValueRep for %v value: %v", n.GetNicePath(), n.Value)
-	realTag := n.GuessTagFromCustomType()
+	realTag := n.guessTagFromCustomType()
 
 	switch realTag {
 	case "!!int":
@@ -261,17 +260,17 @@ func (n *CandidateNode) GetValueRep() (interface{}, error) {
 	return n.Value, nil
 }
 
-func (n *CandidateNode) GuessTagFromCustomType() string {
+func (n *CandidateNode) guessTagFromCustomType() string {
 	if strings.HasPrefix(n.Tag, "!!") {
 		return n.Tag
 	} else if n.Value == "" {
-		log.Debug("GuessTagFromCustomType: node has no value to guess the type with")
+		log.Debug("guessTagFromCustomType: node has no value to guess the type with")
 		return n.Tag
 	}
 	dataBucket, errorReading := parseSnippet(n.Value)
 
 	if errorReading != nil {
-		log.Debug("GuessTagFromCustomType: could not guess underlying tag type %v", errorReading)
+		log.Debug("guessTagFromCustomType: could not guess underlying tag type %v", errorReading)
 		return n.Tag
 	}
 	guessedTag := dataBucket.Tag
@@ -376,7 +375,7 @@ func (n *CandidateNode) UpdateFrom(other *CandidateNode, prefs assignPreferences
 	// if this is an empty map or empty array, use the style of other node.
 	if (n.Kind != ScalarNode && len(n.Content) == 0) ||
 		// if the tag has changed (e.g. from str to bool)
-		(n.GuessTagFromCustomType() != other.GuessTagFromCustomType()) {
+		(n.guessTagFromCustomType() != other.guessTagFromCustomType()) {
 		n.Style = other.Style
 	}
 
@@ -428,34 +427,3 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignP
 		n.LineComment = other.LineComment
 	}
 }
-
-func (n *CandidateNode) CleanHeadAndLineComment() string {
-	return n.CleanHeadComment() + n.CleanLineComment()
-}
-
-func (n *CandidateNode) CleanHeadComment() string {
-	return strings.Replace(n.HeadComment, "#", "", 1)
-}
-
-func (n *CandidateNode) CleanLineComment() string {
-	return strings.Replace(n.LineComment, "#", "", 1)
-}
-
-func (n *CandidateNode) CleanFootComment() string {
-	return strings.Replace(n.FootComment, "#", "", 1)
-}
-
-func (n *CandidateNode) ConvertKeysToStrings() {
-
-	if n.Kind == MappingNode {
-		for index, child := range n.Content {
-			if index%2 == 0 { // its a map key
-				child.Tag = "!!str"
-			}
-		}
-	}
-
-	for _, child := range n.Content {
-		child.ConvertKeysToStrings()
-	}
-}

pkg/yqlib/candidate_node_goccy_yaml.go

@@ -16,7 +16,7 @@ func (o *CandidateNode) goccyDecodeIntoChild(childNode ast.Node, cm yaml.Comment
 	return newChild, err
 }
 
-func (o *CandidateNode) UnmarshalGoccyYAML(node ast.Node, cm yaml.CommentMap) error {
+func (o *CandidateNode) UnmarshalGoccyYAML(node ast.Node, cm yaml.CommentMap, anchorMap map[string]*CandidateNode) error {
 	log.Debugf("UnmarshalYAML %v", node)
 	log.Debugf("UnmarshalYAML %v", node.Type().String())
 	log.Debugf("UnmarshalYAML Node Value: %v", node.String())
@@ -49,6 +49,13 @@ func (o *CandidateNode) UnmarshalGoccyYAML(node ast.Node, cm yaml.CommentMap) er
 
 		}
 	}
+	o.Anchor = node.
+
+	if o.Anchor != "" {
+		anchorMap[o.Anchor] = o
+	}
+
+	if o.Ali
 
 	o.Value = node.String()
 	switch node.Type() {
@@ -92,7 +99,7 @@ func (o *CandidateNode) UnmarshalGoccyYAML(node ast.Node, cm yaml.CommentMap) er
 		// to solve the multiline > problem
 		o.Value = astLiteral.Value.Value
 	case ast.TagType:
-		if err := o.UnmarshalGoccyYAML(node.(*ast.TagNode).Value, cm); err != nil {
+		if err := o.UnmarshalGoccyYAML(node.(*ast.TagNode).Value, cm, anchorMap); err != nil {
 			return err
 		}
 		o.Tag = node.(*ast.TagNode).Start.Value

pkg/yqlib/candidate_node_test.go

@@ -64,13 +64,13 @@ func TestCandidateNodeChildWhenParentUpdated(t *testing.T) {
 	test.AssertResultWithContext(t, uint(1), child.GetDocument(), "document index")
 }
 
-type CreateScalarNodeScenario struct {
+type createScalarNodeScenario struct {
 	value       interface{}
 	stringValue string
 	expectedTag string
 }
 
-var createScalarScenarios = []CreateScalarNodeScenario{
+var createScalarScenarios = []createScalarNodeScenario{
 	{
 		value:       "mike",
 		stringValue: "mike",
@@ -100,20 +100,20 @@ var createScalarScenarios = []CreateScalarNodeScenario{
 
 func TestCreateScalarNodeScenarios(t *testing.T) {
 	for _, tt := range createScalarScenarios {
-		actual := CreateScalarNode(tt.value, tt.stringValue)
+		actual := createScalarNode(tt.value, tt.stringValue)
 		test.AssertResultWithContext(t, tt.stringValue, actual.Value, fmt.Sprintf("Value for: Value: [%v], String: %v", tt.value, tt.stringValue))
 		test.AssertResultWithContext(t, tt.expectedTag, actual.Tag, fmt.Sprintf("Value for: Value: [%v], String: %v", tt.value, tt.stringValue))
 	}
 }
 
 func TestGetKeyForMapValue(t *testing.T) {
-	key := CreateStringScalarNode("yourKey")
+	key := createStringScalarNode("yourKey")
 	n := CandidateNode{Key: key, Value: "meow", document: 3}
 	test.AssertResult(t, "3 - yourKey", n.GetKey())
 }
 
 func TestGetKeyForMapKey(t *testing.T) {
-	key := CreateStringScalarNode("yourKey")
+	key := createStringScalarNode("yourKey")
 	key.IsMapKey = true
 	key.document = 3
 	test.AssertResult(t, "key-yourKey-3 - ", key.GetKey())
@@ -125,7 +125,7 @@ func TestGetKeyForValue(t *testing.T) {
 }
 
 func TestGetParsedKeyForMapKey(t *testing.T) {
-	key := CreateStringScalarNode("yourKey")
+	key := createStringScalarNode("yourKey")
 	key.IsMapKey = true
 	key.document = 3
 	test.AssertResult(t, "yourKey", key.getParsedKey())
@@ -137,26 +137,13 @@ func TestGetParsedKeyForLooseValue(t *testing.T) {
 }
 
 func TestGetParsedKeyForMapValue(t *testing.T) {
-	key := CreateStringScalarNode("yourKey")
+	key := createStringScalarNode("yourKey")
 	n := CandidateNode{Key: key, Value: "meow", document: 3}
 	test.AssertResult(t, "yourKey", n.getParsedKey())
 }
 
 func TestGetParsedKeyForArrayValue(t *testing.T) {
-	key := CreateScalarNode(4, "4")
+	key := createScalarNode(4, "4")
 	n := CandidateNode{Key: key, Value: "meow", document: 3}
 	test.AssertResult(t, 4, n.getParsedKey())
 }
-
-func TestCandidateNodeAddKeyValueChild(t *testing.T) {
-	key := CandidateNode{Value: "cool", IsMapKey: true}
-	node := CandidateNode{}
-
-	// if we use a key in a new node as a value, it should no longer be marked as a key
-
-	_, keyIsValueNow := node.AddKeyValueChild(&CandidateNode{Value: "newKey"}, &key)
-
-	test.AssertResult(t, keyIsValueNow.IsMapKey, false)
-	test.AssertResult(t, key.IsMapKey, true)
-
-}

pkg/yqlib/candidiate_node_json.go

@@ -96,7 +96,7 @@ func (o *CandidateNode) UnmarshalJSON(data []byte) error {
 
 			if child == nil {
 				// need to represent it as a null scalar
-				child = CreateScalarNode(nil, "null")
+				child = createScalarNode(nil, "null")
 			}
 			childKey := o.CreateChild()
 			childKey.Kind = ScalarNode
@@ -162,13 +162,8 @@ func (o *CandidateNode) MarshalJSON() ([]byte, error) {
 		buf.WriteByte('}')
 		return buf.Bytes(), nil
 	case SequenceNode:
-		log.Debugf("MarshalJSON SequenceNode, %v, len: %v", o.Content, len(o.Content))
-		var err error
-		if len(o.Content) == 0 {
-			buf.WriteString("[]")
-		} else {
-			err = enc.Encode(o.Content)
-		}
+		log.Debugf("MarshalJSON SequenceNode")
+		err := enc.Encode(o.Content)
 		return buf.Bytes(), err
 	default:
 		err := enc.Encode(nil)

pkg/yqlib/csv.go

@@ -1,23 +0,0 @@
-package yqlib
-
-type CsvPreferences struct {
-	Separator rune
-	AutoParse bool
-}
-
-func NewDefaultCsvPreferences() CsvPreferences {
-	return CsvPreferences{
-		Separator: ',',
-		AutoParse: true,
-	}
-}
-
-func NewDefaultTsvPreferences() CsvPreferences {
-	return CsvPreferences{
-		Separator: '\t',
-		AutoParse: true,
-	}
-}
-
-var ConfiguredCsvPreferences = NewDefaultCsvPreferences()
-var ConfiguredTsvPreferences = NewDefaultTsvPreferences()

pkg/yqlib/csv_test.go

@@ -12,11 +12,6 @@ const csvSimple = `name,numberOfCats,likesApples,height
 Gary,1,true,168.8
 Samantha's Rabbit,2,false,-188.8
 `
-
-const csvSimpleWithObject = `name,numberOfCats,likesApples,height,facts
-Gary,1,true,168.8,cool: true
-Samantha's Rabbit,2,false,-188.8,tall: indeed
-`
 const csvMissing = `name,numberOfCats,likesApples,height
 ,null,,168.8
 `
@@ -44,31 +39,6 @@ const expectedYamlFromCSV = `- name: Gary
   likesApples: false
   height: -188.8
 `
-const expectedYamlFromCSVWithObject = `- name: Gary
-  numberOfCats: 1
-  likesApples: true
-  height: 168.8
-  facts:
-    cool: true
-- name: Samantha's Rabbit
-  numberOfCats: 2
-  likesApples: false
-  height: -188.8
-  facts:
-    tall: indeed
-`
-
-const expectedYamlFromCSVNoParsing = `- name: Gary
-  numberOfCats: 1
-  likesApples: true
-  height: 168.8
-  facts: 'cool: true'
-- name: Samantha's Rabbit
-  numberOfCats: 2
-  likesApples: false
-  height: -188.8
-  facts: 'tall: indeed'
-`
 
 const expectedYamlFromCSVMissingData = `- name: Gary
   numberOfCats: 1
@@ -94,7 +64,7 @@ const tsvTestExpectedSimpleCsv = `i	like	csv
 because	excel	is	cool
 `
 
-var csvScenarios = []FormatScenario{
+var csvScenarios = []formatScenario{
 	{
 		description:  "Encode CSV simple",
 		input:        csvTestSimpleYaml,
@@ -155,7 +125,7 @@ var csvScenarios = []FormatScenario{
 		input:        csvSimple,
 		expression:   ".[0].name | key",
 		expected:     "name\n",
-		scenarioType: "decode-csv",
+		scenarioType: "decode-csv-object",
 	},
 	{
 		description:  "decode csv parent",
@@ -163,28 +133,14 @@ var csvScenarios = []FormatScenario{
 		input:        csvSimple,
 		expression:   ".[0].name | parent | .height",
 		expected:     "168.8\n",
-		scenarioType: "decode-csv",
+		scenarioType: "decode-csv-object",
 	},
 	{
 		description:    "Parse CSV into an array of objects",
-		subdescription: "First row is assumed to be the header row. By default, entries with YAML/JSON formatting will be parsed!",
-		input:          csvSimpleWithObject,
-		expected:       expectedYamlFromCSVWithObject,
-		scenarioType:   "decode-csv",
-	},
-	{
-		description:  "Decode CSV line breaks",
-		skipDoc:      true,
-		input:        "heading1\n\"some data\nwith a line break\"\n",
-		expected:     "- heading1: |-\n    some data\n    with a line break\n",
-		scenarioType: "decode-csv",
-	},
-	{
-		description:    "Parse CSV into an array of objects, no auto-parsing",
-		subdescription: "First row is assumed to be the header row. Entries with YAML/JSON will be left as strings.",
-		input:          csvSimpleWithObject,
-		expected:       expectedYamlFromCSVNoParsing,
-		scenarioType:   "decode-csv-no-auto",
+		subdescription: "First row is assumed to be the header row.",
+		input:          csvSimple,
+		expected:       expectedYamlFromCSV,
+		scenarioType:   "decode-csv-object",
 	},
 	{
 		description:  "Scalar roundtrip",
@@ -210,26 +166,24 @@ var csvScenarios = []FormatScenario{
 	},
 }
 
-func testCSVScenario(t *testing.T, s FormatScenario) {
+func testCSVScenario(t *testing.T, s formatScenario) {
 	switch s.scenarioType {
 	case "encode-csv":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(ConfiguredCsvPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(',')), s.description)
 	case "encode-tsv":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(ConfiguredTsvPreferences)), s.description)
-	case "decode-csv":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredCsvPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
-	case "decode-csv-no-auto":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: ',', AutoParse: false}), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder('\t')), s.description)
+	case "decode-csv-object":
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(','), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
 	case "decode-tsv-object":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredTsvPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder('\t'), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
 	case "roundtrip-csv":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredCsvPreferences), NewCsvEncoder(ConfiguredCsvPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(','), NewCsvEncoder(',')), s.description)
 	default:
 		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
 	}
 }
 
-func documentCSVDecodeObjectScenario(w *bufio.Writer, s FormatScenario, formatType string) {
+func documentCSVDecodeObjectScenario(w *bufio.Writer, s formatScenario, formatType string) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -250,36 +204,11 @@ func documentCSVDecodeObjectScenario(w *bufio.Writer, s FormatScenario, formatTy
 	}
 
 	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n",
-		mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: true}), NewYamlEncoder(ConfiguredYamlPreferences))),
+		mustProcessFormatScenario(s, NewCSVObjectDecoder(separator), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))),
 	)
 }
 
-func documentCSVDecodeObjectNoAutoScenario(w *bufio.Writer, s FormatScenario, formatType string) {
-	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
-
-	if s.subdescription != "" {
-		writeOrPanic(w, s.subdescription)
-		writeOrPanic(w, "\n\n")
-	}
-
-	writeOrPanic(w, fmt.Sprintf("Given a sample.%v file of:\n", formatType))
-	writeOrPanic(w, fmt.Sprintf("```%v\n%v\n```\n", formatType, s.input))
-
-	writeOrPanic(w, "then\n")
-	writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=%v --csv-auto-parse=f sample.%v\n```\n", formatType, formatType))
-	writeOrPanic(w, "will output\n")
-
-	separator := ','
-	if formatType == "tsv" {
-		separator = '\t'
-	}
-
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n",
-		mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: false}), NewYamlEncoder(ConfiguredYamlPreferences))),
-	)
-}
-
-func documentCSVEncodeScenario(w *bufio.Writer, s FormatScenario, formatType string) {
+func documentCSVEncodeScenario(w *bufio.Writer, s formatScenario, formatType string) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -305,14 +234,13 @@ func documentCSVEncodeScenario(w *bufio.Writer, s FormatScenario, formatType str
 	if formatType == "tsv" {
 		separator = '\t'
 	}
-	csvPrefs := NewDefaultCsvPreferences()
-	csvPrefs.Separator = separator
+
 	writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
-		mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(csvPrefs))),
+		mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(separator))),
 	)
 }
 
-func documentCSVRoundTripScenario(w *bufio.Writer, s FormatScenario, formatType string) {
+func documentCSVRoundTripScenario(w *bufio.Writer, s formatScenario, formatType string) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -339,16 +267,13 @@ func documentCSVRoundTripScenario(w *bufio.Writer, s FormatScenario, formatType
 		separator = '\t'
 	}
 
-	csvPrefs := NewDefaultCsvPreferences()
-	csvPrefs.Separator = separator
-
 	writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
-		mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: true}), NewCsvEncoder(csvPrefs))),
+		mustProcessFormatScenario(s, NewCSVObjectDecoder(separator), NewCsvEncoder(separator))),
 	)
 }
 
-func documentCSVScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
-	s := i.(FormatScenario)
+func documentCSVScenario(t *testing.T, w *bufio.Writer, i interface{}) {
+	s := i.(formatScenario)
 	if s.skipDoc {
 		return
 	}
@@ -357,10 +282,8 @@ func documentCSVScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
 		documentCSVEncodeScenario(w, s, "csv")
 	case "encode-tsv":
 		documentCSVEncodeScenario(w, s, "tsv")
-	case "decode-csv":
+	case "decode-csv-object":
 		documentCSVDecodeObjectScenario(w, s, "csv")
-	case "decode-csv-no-auto":
-		documentCSVDecodeObjectNoAutoScenario(w, s, "csv")
 	case "decode-tsv-object":
 		documentCSVDecodeObjectScenario(w, s, "tsv")
 	case "roundtrip-csv":

pkg/yqlib/data_tree_navigator.go

@@ -7,13 +7,12 @@ import (
 )
 
 type DataTreeNavigator interface {
-	// given the context and an expressionNode,
+	// given the context and a expressionNode,
 	// this will process the against the given expressionNode and return
 	// a new context of matching candidates
 	GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error)
 
 	DeeplyAssign(context Context, path []interface{}, rhsNode *CandidateNode) error
-	DeeplyAssignKey(context Context, path []interface{}, rhsNode *CandidateNode) error
 }
 
 type dataTreeNavigator struct {
@@ -24,33 +23,14 @@ func NewDataTreeNavigator() DataTreeNavigator {
 }
 
 func (d *dataTreeNavigator) DeeplyAssign(context Context, path []interface{}, rhsCandidateNode *CandidateNode) error {
-	return d.deeplyAssign(context, path, rhsCandidateNode, false)
-}
-
-func (d *dataTreeNavigator) DeeplyAssignKey(context Context, path []interface{}, rhsCandidateNode *CandidateNode) error {
-	return d.deeplyAssign(context, path, rhsCandidateNode, true)
-}
-
-func (d *dataTreeNavigator) deeplyAssign(context Context, path []interface{}, rhsCandidateNode *CandidateNode, targetKey bool) error {
 
 	assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
 
-	if rhsCandidateNode.Kind == MappingNode {
-		log.Debug("DeeplyAssign: deeply merging object")
-		// if the rhs is a map, we need to deeply merge it in.
-		// otherwise we'll clobber any existing fields
-		assignmentOp = &Operation{OperationType: multiplyAssignOpType, Preferences: multiplyPreferences{
-			AppendArrays:  true,
-			TraversePrefs: traversePreferences{DontFollowAlias: true},
-			AssignPrefs:   assignPreferences{},
-		}}
-	}
-
 	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}
 
 	assignmentOpNode := &ExpressionNode{
 		Operation: assignmentOp,
-		LHS:       createTraversalTree(path, traversePreferences{}, targetKey),
+		LHS:       createTraversalTree(path, traversePreferences{}, false),
 		RHS:       &ExpressionNode{Operation: rhsOp},
 	}
 

pkg/yqlib/decoder.go

@@ -1,10 +1,66 @@
 package yqlib
 
 import (
+	"fmt"
 	"io"
+	"strings"
+)
+
+type InputFormat uint
+
+const (
+	YamlInputFormat = 1 << iota
+	XMLInputFormat
+	PropertiesInputFormat
+	Base64InputFormat
+	JsonInputFormat
+	CSVObjectInputFormat
+	TSVObjectInputFormat
+	TomlInputFormat
+	UriInputFormat
+	LuaInputFormat
 )
 
 type Decoder interface {
 	Init(reader io.Reader) error
 	Decode() (*CandidateNode, error)
 }
+
+func InputFormatFromString(format string) (InputFormat, error) {
+	switch format {
+	case "yaml", "yml", "y":
+		return YamlInputFormat, nil
+	case "xml", "x":
+		return XMLInputFormat, nil
+	case "properties", "props", "p":
+		return PropertiesInputFormat, nil
+	case "json", "ndjson", "j":
+		return JsonInputFormat, nil
+	case "csv", "c":
+		return CSVObjectInputFormat, nil
+	case "tsv", "t":
+		return TSVObjectInputFormat, nil
+	case "toml":
+		return TomlInputFormat, nil
+	case "lua", "l":
+		return LuaInputFormat, nil
+	default:
+		return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml|toml]", format)
+	}
+}
+
+func FormatFromFilename(filename string) string {
+
+	if filename != "" {
+		GetLogger().Debugf("checking file extension '%s' for auto format detection", filename)
+		nPos := strings.LastIndex(filename, ".")
+		if nPos > -1 {
+			format := filename[nPos+1:]
+			GetLogger().Debugf("detected format '%s'", format)
+			return format
+		}
+	}
+
+	GetLogger().Debugf("using default inputFormat 'yaml'")
+	return "yaml"
+}

pkg/yqlib/decoder_base64.go

@@ -68,5 +68,5 @@ func (dec *base64Decoder) Decode() (*CandidateNode, error) {
 		}
 	}
 	dec.readAnything = true
-	return CreateStringScalarNode(buf.String()), nil
+	return createStringScalarNode(buf.String()), nil
 }

pkg/yqlib/decoder_csv_object.go

@@ -9,30 +9,28 @@ import (
 )
 
 type csvObjectDecoder struct {
-	prefs    CsvPreferences
-	reader   csv.Reader
-	finished bool
+	separator rune
+	reader    csv.Reader
+	finished  bool
 }
 
-func NewCSVObjectDecoder(prefs CsvPreferences) Decoder {
-	return &csvObjectDecoder{prefs: prefs}
+func NewCSVObjectDecoder(separator rune) Decoder {
+	return &csvObjectDecoder{separator: separator}
 }
 
 func (dec *csvObjectDecoder) Init(reader io.Reader) error {
 	cleanReader, enc := utfbom.Skip(reader)
 	log.Debugf("Detected encoding: %s\n", enc)
 	dec.reader = *csv.NewReader(cleanReader)
-	dec.reader.Comma = dec.prefs.Separator
+	dec.reader.Comma = dec.separator
 	dec.finished = false
 	return nil
 }
 
 func (dec *csvObjectDecoder) convertToNode(content string) *CandidateNode {
 	node, err := parseSnippet(content)
-	// if we're not auto-parsing, then we wont put in parsed objects or arrays
-	// but we still parse scalars
-	if err != nil || (!dec.prefs.AutoParse && node.Kind != ScalarNode) {
-		return CreateScalarNode(content, content)
+	if err != nil {
+		return createScalarNode(content, content)
 	}
 	return node
 }
@@ -41,7 +39,7 @@ func (dec *csvObjectDecoder) createObject(headerRow []string, contentRow []strin
 	objectNode := &CandidateNode{Kind: MappingNode, Tag: "!!map"}
 
 	for i, header := range headerRow {
-		objectNode.AddKeyValueChild(CreateScalarNode(header, header), dec.convertToNode(contentRow[i]))
+		objectNode.AddKeyValueChild(createScalarNode(header, header), dec.convertToNode(contentRow[i]))
 	}
 	return objectNode
 }

pkg/yqlib/decoder_goccy_yaml.go

@@ -1,21 +1,23 @@
 //go:build !yq_noyaml
 
-//
-// NOTE this is still a WIP - not yet ready.
-//
-
 package yqlib
 
 import (
+	"bytes"
+	"errors"
 	"io"
+	"regexp"
 
 	yaml "github.com/goccy/go-yaml"
 	"github.com/goccy/go-yaml/ast"
 )
 
 type goccyYamlDecoder struct {
-	decoder yaml.Decoder
-	cm      yaml.CommentMap
+	decoder      yaml.Decoder
+	cm           yaml.CommentMap
+	bufferRead   bytes.Buffer
+	readAnything bool
+	anchorMap    map[string]*CandidateNode
 }
 
 func NewGoccyYAMLDecoder() Decoder {
@@ -24,21 +26,40 @@ func NewGoccyYAMLDecoder() Decoder {
 
 func (dec *goccyYamlDecoder) Init(reader io.Reader) error {
 	dec.cm = yaml.CommentMap{}
-	dec.decoder = *yaml.NewDecoder(reader, yaml.CommentToMap(dec.cm), yaml.UseOrderedMap())
+	dec.readAnything = false
+	dec.anchorMap = make(map[string]*CandidateNode)
+	readerToUse := io.TeeReader(reader, &dec.bufferRead)
+	dec.decoder = *yaml.NewDecoder(readerToUse, yaml.CommentToMap(dec.cm), yaml.UseOrderedMap())
 	return nil
 }
 
 func (dec *goccyYamlDecoder) Decode() (*CandidateNode, error) {
 
+	var commentLineRegEx = regexp.MustCompile(`^\s*#`)
+
 	var ast ast.Node
 
 	err := dec.decoder.Decode(&ast)
-	if err != nil {
+	if errors.Is(err, io.EOF) && !dec.readAnything {
+
+		content := dec.bufferRead.String()
+		// only null fix
+		if content == "null" || content == "~" {
+			dec.readAnything = true
+			return createScalarNode(nil, content), nil
+		} else if commentLineRegEx.MatchString(content) {
+			dec.readAnything = true
+			node := createScalarNode(nil, "")
+			node.LeadingContent = content
+			return node, nil
+		}
+		return nil, err
+	} else if err != nil {
 		return nil, err
 	}
 
 	candidateNode := &CandidateNode{}
-	if err := candidateNode.UnmarshalGoccyYAML(ast, dec.cm); err != nil {
+	if err := candidateNode.UnmarshalGoccyYAML(ast, dec.cm, dec.anchorMap); err != nil {
 		return nil, err
 	}
 

pkg/yqlib/decoder_properties.go

@@ -1,4 +1,4 @@
-package properties
+package yqlib
 
 import (
 	"bytes"
@@ -8,17 +8,16 @@ import (
 	"strings"
 
 	"github.com/magiconair/properties"
-	"github.com/mikefarah/yq/v4/pkg/yqlib"
 )
 
 type propertiesDecoder struct {
 	reader   io.Reader
 	finished bool
-	d        yqlib.DataTreeNavigator
+	d        DataTreeNavigator
 }
 
-func NewPropertiesDecoder() yqlib.Decoder {
-	return &propertiesDecoder{d: yqlib.NewDataTreeNavigator(), finished: false}
+func NewPropertiesDecoder() Decoder {
+	return &propertiesDecoder{d: NewDataTreeNavigator(), finished: false}
 }
 
 func (dec *propertiesDecoder) Init(reader io.Reader) error {
@@ -48,18 +47,31 @@ func (dec *propertiesDecoder) processComment(c string) string {
 	return "# " + c
 }
 
-func (dec *propertiesDecoder) applyPropertyComments(context yqlib.Context, path []interface{}, comments []string) error {
-	rhsCandidateNode := &yqlib.CandidateNode{
+func (dec *propertiesDecoder) applyPropertyComments(context Context, path []interface{}, comments []string) error {
+	assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
+
+	rhsCandidateNode := &CandidateNode{
 		Tag:         "!!str",
 		Value:       fmt.Sprintf("%v", path[len(path)-1]),
 		HeadComment: dec.processComment(strings.Join(comments, "\n")),
-		Kind:        yqlib.ScalarNode,
+		Kind:        ScalarNode,
 	}
-	rhsCandidateNode.Tag = rhsCandidateNode.GuessTagFromCustomType()
-	return dec.d.DeeplyAssignKey(context, path, rhsCandidateNode)
+
+	rhsCandidateNode.Tag = rhsCandidateNode.guessTagFromCustomType()
+
+	rhsOp := &Operation{OperationType: referenceOpType, CandidateNode: rhsCandidateNode}
+
+	assignmentOpNode := &ExpressionNode{
+		Operation: assignmentOp,
+		LHS:       createTraversalTree(path, traversePreferences{}, true),
+		RHS:       &ExpressionNode{Operation: rhsOp},
+	}
+
+	_, err := dec.d.GetMatchingNodes(context, assignmentOpNode)
+	return err
 }
 
-func (dec *propertiesDecoder) applyProperty(context yqlib.Context, properties *properties.Properties, key string) error {
+func (dec *propertiesDecoder) applyProperty(context Context, properties *properties.Properties, key string) error {
 	value, _ := properties.Get(key)
 	path := parsePropKey(key)
 
@@ -71,13 +83,13 @@ func (dec *propertiesDecoder) applyProperty(context yqlib.Context, properties *p
 		}
 	}
 
-	rhsNode := yqlib.CreateStringScalarNode(value)
-	rhsNode.Tag = rhsNode.GuessTagFromCustomType()
+	rhsNode := createStringScalarNode(value)
+	rhsNode.Tag = rhsNode.guessTagFromCustomType()
 
 	return dec.d.DeeplyAssign(context, path, rhsNode)
 }
 
-func (dec *propertiesDecoder) Decode() (*yqlib.CandidateNode, error) {
+func (dec *propertiesDecoder) Decode() (*CandidateNode, error) {
 	if dec.finished {
 		return nil, io.EOF
 	}
@@ -96,12 +108,12 @@ func (dec *propertiesDecoder) Decode() (*yqlib.CandidateNode, error) {
 	}
 	properties.DisableExpansion = true
 
-	rootMap := &yqlib.CandidateNode{
-		Kind: yqlib.MappingNode,
+	rootMap := &CandidateNode{
+		Kind: MappingNode,
 		Tag:  "!!map",
 	}
 
-	context := yqlib.Context{}
+	context := Context{}
 	context = context.SingleChildContext(rootMap)
 
 	for _, key := range properties.Keys() {

pkg/yqlib/decoder_test.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 )
 
-type FormatScenario struct {
+type formatScenario struct {
 	input          string
 	indent         int
 	expression     string
@@ -19,7 +19,7 @@ type FormatScenario struct {
 	expectedError  string
 }
 
-func processFormatScenario(s FormatScenario, decoder Decoder, encoder Encoder) (string, error) {
+func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) (string, error) {
 	var output bytes.Buffer
 	writer := bufio.NewWriter(&output)
 
@@ -64,7 +64,7 @@ func processFormatScenario(s FormatScenario, decoder Decoder, encoder Encoder) (
 	return output.String(), nil
 }
 
-func mustProcessFormatScenario(s FormatScenario, decoder Decoder, encoder Encoder) string {
+func mustProcessFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) string {
 
 	result, err := processFormatScenario(s, decoder, encoder)
 	if err != nil {

pkg/yqlib/decoder_toml.go

@@ -56,7 +56,7 @@ func (dec *tomlDecoder) getFullPath(tomlNode *toml.Node) []interface{} {
 func (dec *tomlDecoder) processKeyValueIntoMap(rootMap *CandidateNode, tomlNode *toml.Node) error {
 	value := tomlNode.Value()
 	path := dec.getFullPath(value.Next())
-	log.Debug("processKeyValueIntoMap: %v", path)
+	log.Debug("!!!processKeyValueIntoMap: %v", path)
 
 	valueNode, err := dec.decodeNode(value)
 	if err != nil {
@@ -70,14 +70,14 @@ func (dec *tomlDecoder) processKeyValueIntoMap(rootMap *CandidateNode, tomlNode
 }
 
 func (dec *tomlDecoder) decodeKeyValuesIntoMap(rootMap *CandidateNode, tomlNode *toml.Node) (bool, error) {
-	log.Debug("decodeKeyValuesIntoMap -- processing first (current) entry")
+	log.Debug("!! DECODE_KV_INTO_MAP -- processing first (current) entry")
 	if err := dec.processKeyValueIntoMap(rootMap, tomlNode); err != nil {
 		return false, err
 	}
 
 	for dec.parser.NextExpression() {
 		nextItem := dec.parser.Expression()
-		log.Debug("decodeKeyValuesIntoMap -- next exp, its a %v", nextItem.Kind)
+		log.Debug("!! DECODE_KV_INTO_MAP -- next exp, its a %v", nextItem.Kind)
 
 		if nextItem.Kind == toml.KeyValue {
 			if err := dec.processKeyValueIntoMap(rootMap, nextItem); err != nil {
@@ -85,17 +85,18 @@ func (dec *tomlDecoder) decodeKeyValuesIntoMap(rootMap *CandidateNode, tomlNode
 			}
 		} else {
 			// run out of key values
-			log.Debug("done in decodeKeyValuesIntoMap, gota a %v", nextItem.Kind)
+			log.Debug("! DECODE_KV_INTO_MAP - ok we are done in decodeKeyValuesIntoMap, gota a %v", nextItem.Kind)
+			log.Debug("! DECODE_KV_INTO_MAP - processAgainstCurrentExp = true!")
 			return true, nil
 		}
 	}
-	log.Debug("no more things to read in")
+	log.Debug("! DECODE_KV_INTO_MAP - no more things to read in")
 	return false, nil
 }
 
 func (dec *tomlDecoder) createInlineTableMap(tomlNode *toml.Node) (*CandidateNode, error) {
 	content := make([]*CandidateNode, 0)
-	log.Debug("createInlineTableMap")
+	log.Debug("!! createInlineTableMap")
 
 	iterator := tomlNode.Children()
 	for iterator.Next() {
@@ -145,30 +146,30 @@ func (dec *tomlDecoder) createArray(tomlNode *toml.Node) (*CandidateNode, error)
 
 func (dec *tomlDecoder) createStringScalar(tomlNode *toml.Node) (*CandidateNode, error) {
 	content := string(tomlNode.Data)
-	return CreateScalarNode(content, content), nil
+	return createScalarNode(content, content), nil
 }
 
 func (dec *tomlDecoder) createBoolScalar(tomlNode *toml.Node) (*CandidateNode, error) {
 	content := string(tomlNode.Data)
-	return CreateScalarNode(content == "true", content), nil
+	return createScalarNode(content == "true", content), nil
 }
 
 func (dec *tomlDecoder) createIntegerScalar(tomlNode *toml.Node) (*CandidateNode, error) {
 	content := string(tomlNode.Data)
 	_, num, err := parseInt64(content)
-	return CreateScalarNode(num, content), err
+	return createScalarNode(num, content), err
 }
 
 func (dec *tomlDecoder) createDateTimeScalar(tomlNode *toml.Node) (*CandidateNode, error) {
 	content := string(tomlNode.Data)
 	val, err := parseDateTime(time.RFC3339, content)
-	return CreateScalarNode(val, content), err
+	return createScalarNode(val, content), err
 }
 
 func (dec *tomlDecoder) createFloatScalar(tomlNode *toml.Node) (*CandidateNode, error) {
 	content := string(tomlNode.Data)
 	num, err := strconv.ParseFloat(content, 64)
-	return CreateScalarNode(num, content), err
+	return createScalarNode(num, content), err
 }
 
 func (dec *tomlDecoder) decodeNode(tomlNode *toml.Node) (*CandidateNode, error) {
@@ -248,7 +249,7 @@ func (dec *tomlDecoder) Decode() (*CandidateNode, error) {
 func (dec *tomlDecoder) processTopLevelNode(currentNode *toml.Node) (bool, error) {
 	var runAgainstCurrentExp bool
 	var err error
-	log.Debug("processTopLevelNode: Going to process %v state is current %v", currentNode.Kind, NodeToString(dec.rootMap))
+	log.Debug("!!!!!!!!!!!!Going to process %v state is current %v", currentNode.Kind, NodeToString(dec.rootMap))
 	if currentNode.Kind == toml.Table {
 		runAgainstCurrentExp, err = dec.processTable(currentNode)
 	} else if currentNode.Kind == toml.ArrayTable {
@@ -257,41 +258,38 @@ func (dec *tomlDecoder) processTopLevelNode(currentNode *toml.Node) (bool, error
 		runAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(dec.rootMap, currentNode)
 	}
 
-	log.Debug("processTopLevelNode: DONE Processing state is now %v", NodeToString(dec.rootMap))
+	log.Debug("!!!!!!!!!!!!DONE Processing state is now %v", NodeToString(dec.rootMap))
 	return runAgainstCurrentExp, err
 }
 
 func (dec *tomlDecoder) processTable(currentNode *toml.Node) (bool, error) {
-	log.Debug("Enter processTable")
+	log.Debug("!!! processing table")
 	fullPath := dec.getFullPath(currentNode.Child())
-	log.Debug("fullpath: %v", fullPath)
+	log.Debug("!!!fullpath: %v", fullPath)
+
+	hasValue := dec.parser.NextExpression()
+	if !hasValue {
+		return false, fmt.Errorf("error retrieving table %v value: %w", fullPath, dec.parser.Error())
+	}
 
 	tableNodeValue := &CandidateNode{
-		Kind:    MappingNode,
-		Tag:     "!!map",
-		Content: make([]*CandidateNode, 0),
+		Kind: MappingNode,
+		Tag:  "!!map",
 	}
 
-	var tableValue *toml.Node
-	runAgainstCurrentExp := false
-	var err error
-	hasValue := dec.parser.NextExpression()
-	// check to see if there is any table data
-	if hasValue {
-		tableValue = dec.parser.Expression()
-		// next expression is not table data, so we are done
-		if tableValue.Kind != toml.KeyValue {
-			log.Debug("got an empty table, returning")
-			return true, nil
-		}
-
-		runAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue)
-		if err != nil && !errors.Is(io.EOF, err) {
-			return false, err
-		}
+	tableValue := dec.parser.Expression()
+	if tableValue.Kind != toml.KeyValue {
+		log.Debug("got an empty table, returning")
+		return true, nil
 	}
 
+	runAgainstCurrentExp, err := dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue)
+	log.Debugf("table node err: %w", err)
+	if err != nil && !errors.Is(io.EOF, err) {
+		return false, err
+	}
 	c := Context{}
+
 	c = c.SingleChildContext(dec.rootMap)
 	err = dec.d.DeeplyAssign(c, fullPath, tableNodeValue)
 	if err != nil {
@@ -301,7 +299,6 @@ func (dec *tomlDecoder) processTable(currentNode *toml.Node) (bool, error) {
 }
 
 func (dec *tomlDecoder) arrayAppend(context Context, path []interface{}, rhsNode *CandidateNode) error {
-	log.Debug("arrayAppend to path: %v,%v", path, NodeToString(rhsNode))
 	rhsCandidateNode := &CandidateNode{
 		Kind:    SequenceNode,
 		Tag:     "!!seq",
@@ -323,9 +320,9 @@ func (dec *tomlDecoder) arrayAppend(context Context, path []interface{}, rhsNode
 }
 
 func (dec *tomlDecoder) processArrayTable(currentNode *toml.Node) (bool, error) {
-	log.Debug("Entering processArrayTable")
+	log.Debug("!!! processing table")
 	fullPath := dec.getFullPath(currentNode.Child())
-	log.Debug("Fullpath: %v", fullPath)
+	log.Debug("!!!fullpath: %v", fullPath)
 
 	// need to use the array append exp to add another entry to
 	// this array: fullpath += [ thing ]

pkg/yqlib/decoder_uri.go

@@ -48,5 +48,5 @@ func (dec *uriDecoder) Decode() (*CandidateNode, error) {
 		return nil, err
 	}
 	dec.readAnything = true
-	return CreateStringScalarNode(newValue), nil
+	return createStringScalarNode(newValue), nil
 }

pkg/yqlib/decoder_xml.go

@@ -1,6 +1,6 @@
 //go:build !yq_noxml
 
-package xml
+package yqlib
 
 import (
 	"encoding/xml"
@@ -11,7 +11,6 @@ import (
 	"strings"
 	"unicode"
 
-	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"golang.org/x/net/html/charset"
 )
 
@@ -22,7 +21,7 @@ type xmlDecoder struct {
 	prefs        XmlPreferences
 }
 
-func NewXMLDecoder(prefs XmlPreferences) yqlib.Decoder {
+func NewXMLDecoder(prefs XmlPreferences) Decoder {
 	return &xmlDecoder{
 		finished: false,
 		prefs:    prefs,
@@ -36,8 +35,8 @@ func (dec *xmlDecoder) Init(reader io.Reader) error {
 	return nil
 }
 
-func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yqlib.CandidateNode, error) {
-	yamlNode := &yqlib.CandidateNode{Kind: yqlib.SequenceNode, Tag: "!!seq"}
+func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*CandidateNode, error) {
+	yamlNode := &CandidateNode{Kind: SequenceNode, Tag: "!!seq"}
 	for _, child := range nodes {
 		yamlChild, err := dec.convertToYamlNode(child)
 		if err != nil {
@@ -64,14 +63,14 @@ func (dec *xmlDecoder) processComment(c string) string {
 	return replacement
 }
 
-func (dec *xmlDecoder) createMap(n *xmlNode) (*yqlib.CandidateNode, error) {
-	yqlib.GetLogger().Debug("createMap: headC: %v, lineC: %v, footC: %v", n.HeadComment, n.LineComment, n.FootComment)
-	yamlNode := &yqlib.CandidateNode{Kind: yqlib.MappingNode, Tag: "!!map"}
+func (dec *xmlDecoder) createMap(n *xmlNode) (*CandidateNode, error) {
+	log.Debug("createMap: headC: %v, lineC: %v, footC: %v", n.HeadComment, n.LineComment, n.FootComment)
+	yamlNode := &CandidateNode{Kind: MappingNode, Tag: "!!map"}
 
 	if len(n.Data) > 0 {
-		yqlib.GetLogger().Debugf("creating content node for map: %v", dec.prefs.ContentName)
+		log.Debugf("creating content node for map: %v", dec.prefs.ContentName)
 		label := dec.prefs.ContentName
-		labelNode := yqlib.CreateScalarNode(label, label)
+		labelNode := createScalarNode(label, label)
 		labelNode.HeadComment = dec.processComment(n.HeadComment)
 		labelNode.LineComment = dec.processComment(n.LineComment)
 		labelNode.FootComment = dec.processComment(n.FootComment)
@@ -81,19 +80,19 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yqlib.CandidateNode, error) {
 	for i, keyValuePair := range n.Children {
 		label := keyValuePair.K
 		children := keyValuePair.V
-		labelNode := yqlib.CreateScalarNode(label, label)
-		var valueNode *yqlib.CandidateNode
+		labelNode := createScalarNode(label, label)
+		var valueNode *CandidateNode
 		var err error
 
 		if i == 0 {
-			yqlib.GetLogger().Debugf("head comment here")
+			log.Debugf("head comment here")
 			labelNode.HeadComment = dec.processComment(n.HeadComment)
 
 		}
-		yqlib.GetLogger().Debugf("label=%v, i=%v, keyValuePair.FootComment: %v", label, i, keyValuePair.FootComment)
+		log.Debugf("label=%v, i=%v, keyValuePair.FootComment: %v", label, i, keyValuePair.FootComment)
 		labelNode.FootComment = dec.processComment(keyValuePair.FootComment)
 
-		yqlib.GetLogger().Debug("len of children in %v is %v", label, len(children))
+		log.Debug("len of children in %v is %v", label, len(children))
 		if len(children) > 1 {
 			valueNode, err = dec.createSequence(children)
 			if err != nil {
@@ -106,7 +105,7 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yqlib.CandidateNode, error) {
 			if len(children[0].Children) == 0 && children[0].HeadComment != "" {
 				if len(children[0].Data) > 0 {
 
-					yqlib.GetLogger().Debug("scalar comment hack, currentlabel [%v]", labelNode.HeadComment)
+					log.Debug("scalar comment hack, currentlabel [%v]", labelNode.HeadComment)
 					labelNode.HeadComment = joinComments([]string{labelNode.HeadComment, strings.TrimSpace(children[0].HeadComment)}, "\n")
 					children[0].HeadComment = ""
 				} else {
@@ -126,33 +125,33 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yqlib.CandidateNode, error) {
 	return yamlNode, nil
 }
 
-func (dec *xmlDecoder) createValueNodeFromData(values []string) *yqlib.CandidateNode {
+func (dec *xmlDecoder) createValueNodeFromData(values []string) *CandidateNode {
 	switch len(values) {
 	case 0:
-		return yqlib.CreateScalarNode(nil, "")
+		return createScalarNode(nil, "")
 	case 1:
-		return yqlib.CreateScalarNode(values[0], values[0])
+		return createScalarNode(values[0], values[0])
 	default:
-		content := make([]*yqlib.CandidateNode, 0)
+		content := make([]*CandidateNode, 0)
 		for _, value := range values {
-			content = append(content, yqlib.CreateScalarNode(value, value))
+			content = append(content, createScalarNode(value, value))
 		}
-		return &yqlib.CandidateNode{
-			Kind:    yqlib.SequenceNode,
+		return &CandidateNode{
+			Kind:    SequenceNode,
 			Tag:     "!!seq",
 			Content: content,
 		}
 	}
 }
 
-func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yqlib.CandidateNode, error) {
+func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*CandidateNode, error) {
 	if len(n.Children) > 0 {
 		return dec.createMap(n)
 	}
 
 	scalar := dec.createValueNodeFromData(n.Data)
 
-	yqlib.GetLogger().Debug("scalar (%v), headC: %v, lineC: %v, footC: %v", scalar.Tag, n.HeadComment, n.LineComment, n.FootComment)
+	log.Debug("scalar (%v), headC: %v, lineC: %v, footC: %v", scalar.Tag, n.HeadComment, n.LineComment, n.FootComment)
 	scalar.HeadComment = dec.processComment(n.HeadComment)
 	scalar.LineComment = dec.processComment(n.LineComment)
 	if scalar.Tag == "!!seq" {
@@ -165,7 +164,7 @@ func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yqlib.CandidateNode, erro
 	return scalar, nil
 }
 
-func (dec *xmlDecoder) Decode() (*yqlib.CandidateNode, error) {
+func (dec *xmlDecoder) Decode() (*CandidateNode, error) {
 	if dec.finished {
 		return nil, io.EOF
 	}
@@ -212,17 +211,17 @@ func (n *xmlNode) AddChild(s string, c *xmlNode) {
 	if n.Children == nil {
 		n.Children = make([]*xmlChildrenKv, 0)
 	}
-	yqlib.GetLogger().Debug("looking for %s", s)
+	log.Debug("looking for %s", s)
 	// see if we can find an existing entry to add to
 	for _, childEntry := range n.Children {
 		if childEntry.K == s {
-			yqlib.GetLogger().Debug("found it, appending an entry%s", s)
+			log.Debug("found it, appending an entry%s", s)
 			childEntry.V = append(childEntry.V, c)
-			yqlib.GetLogger().Debug("yay len of children in %v is %v", s, len(childEntry.V))
+			log.Debug("yay len of children in %v is %v", s, len(childEntry.V))
 			return
 		}
 	}
-	yqlib.GetLogger().Debug("not there, making a new one %s", s)
+	log.Debug("not there, making a new one %s", s)
 	n.Children = append(n.Children, &xmlChildrenKv{K: s, V: []*xmlNode{c}})
 }
 
@@ -268,7 +267,7 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
 
 		switch se := t.(type) {
 		case xml.StartElement:
-			yqlib.GetLogger().Debug("start element %v", se.Name.Local)
+			log.Debug("start element %v", se.Name.Local)
 			elem.state = "started"
 			// Build new a new current element and link it to its parent
 			elem = &element{
@@ -297,14 +296,10 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
 			if len(newBit) > 0 {
 				elem.n.Data = append(elem.n.Data, newBit)
 				elem.state = "chardata"
-				yqlib.GetLogger().Debug("chardata [%v] for %v", elem.n.Data, elem.label)
+				log.Debug("chardata [%v] for %v", elem.n.Data, elem.label)
 			}
 		case xml.EndElement:
-			if elem == nil {
-				yqlib.GetLogger().Debug("no element, probably bad xml")
-				continue
-			}
-			yqlib.GetLogger().Debug("end element %v", elem.label)
+			log.Debug("end element %v", elem.label)
 			elem.state = "finished"
 			// And add it to its parent list
 			if elem.parent != nil {
@@ -320,10 +315,10 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
 				applyFootComment(elem, commentStr)
 
 			} else if elem.state == "chardata" {
-				yqlib.GetLogger().Debug("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
+				log.Debug("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
 				elem.n.LineComment = joinComments([]string{elem.n.LineComment, commentStr}, " ")
 			} else {
-				yqlib.GetLogger().Debug("got a head comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
+				log.Debug("got a head comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
 				elem.n.HeadComment = joinComments([]string{elem.n.HeadComment, commentStr}, " ")
 			}
 
@@ -348,7 +343,7 @@ func applyFootComment(elem *element, commentStr string) {
 	if len(elem.n.Children) > 0 {
 		lastChildIndex := len(elem.n.Children) - 1
 		childKv := elem.n.Children[lastChildIndex]
-		yqlib.GetLogger().Debug("got a foot comment, putting on last child for %v: [%v]", childKv.K, commentStr)
+		log.Debug("got a foot comment, putting on last child for %v: [%v]", childKv.K, commentStr)
 		// if it's an array of scalars, put the foot comment on the scalar itself
 		if len(childKv.V) > 0 && len(childKv.V[0].Children) == 0 {
 			nodeToUpdate := childKv.V[len(childKv.V)-1]
@@ -357,7 +352,7 @@ func applyFootComment(elem *element, commentStr string) {
 			childKv.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, " ")
 		}
 	} else {
-		yqlib.GetLogger().Debug("got a foot comment for %v: [%v]", elem.label, commentStr)
+		log.Debug("got a foot comment for %v: [%v]", elem.label, commentStr)
 		elem.n.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, " ")
 	}
 }

pkg/yqlib/decoder_yaml.go

@@ -37,7 +37,7 @@ func (dec *yamlDecoder) processReadStream(reader *bufio.Reader) (io.Reader, stri
 	var yamlDirectiveLineRegEx = regexp.MustCompile(`^\s*%YA`)
 	var sb strings.Builder
 	for {
-		peekBytes, err := reader.Peek(4)
+		peekBytes, err := reader.Peek(3)
 		if errors.Is(err, io.EOF) {
 			// EOF are handled else where..
 			return reader, sb.String(), nil
@@ -51,15 +51,7 @@ func (dec *yamlDecoder) processReadStream(reader *bufio.Reader) (io.Reader, stri
 			} else if err != nil {
 				return reader, sb.String(), err
 			}
-		} else if string(peekBytes) == "--- " {
-			_, err := reader.ReadString(' ')
-			sb.WriteString("$yqDocSeparator$\n")
-			if errors.Is(err, io.EOF) {
-				return reader, sb.String(), nil
-			} else if err != nil {
-				return reader, sb.String(), err
-			}
-		} else if string(peekBytes) == "---\n" {
+		} else if string(peekBytes) == "---" {
 			_, err := reader.ReadString('\n')
 			sb.WriteString("$yqDocSeparator$\n")
 			if errors.Is(err, io.EOF) {
@@ -151,7 +143,7 @@ func (dec *yamlDecoder) Decode() (*CandidateNode, error) {
 }
 
 func (dec *yamlDecoder) blankNodeWithComment() *CandidateNode {
-	node := CreateScalarNode(nil, "")
+	node := createScalarNode(nil, "")
 	node.LeadingContent = dec.leadingContent
 	return node
 }

pkg/yqlib/doc/operators/boolean-operators.md

@@ -4,7 +4,7 @@ The `or` and `and` operators take two parameters and return a boolean result.
 
 `not` flips a boolean from true to false, or vice versa. 
 
-`any` will return `true` if there are any `true` values in an array sequence, and `all` will return true if _all_ elements in an array are true.
+`any` will return `true` if there are any `true` values in a array sequence, and `all` will return true if _all_ elements in an array are true.
 
 `any_c(condition)` and `all_c(condition)` are like `any` and `all` but they take a condition expression that is used against each element to determine if it's `true`. Note: in `jq` you can simply pass a condition to `any` or `all` and it simply works - `yq` isn't that clever..yet
 

pkg/yqlib/doc/operators/column.md

@@ -2,8 +2,6 @@
 
 Returns the column of the matching node. Starts from 1, 0 indicates there was no column data.
 
-Column is the number of characters that precede that node on the line it starts.
-
 ## Returns column of _value_ node
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/operators/headers/Main.md

@@ -1,5 +1,5 @@
 # NAME
-  *yq* is a portable command-line data file processor
+  *yq* is a portable command-line YAML processor
 
 # SYNOPSIS 
 
@@ -11,7 +11,7 @@ eval-all/ea - Loads all yaml documents of all yaml files and runs expression onc
 
 # DESCRIPTION
 
-a lightweight and portable command-line data file processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml, json, xml, csv, properties and TOML files. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.
+a lightweight and portable command-line YAML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.
 
 This documentation is also available at https://mikefarah.gitbook.io/yq/
 # QUICK GUIDE 

pkg/yqlib/doc/operators/headers/boolean-operators.md

@@ -4,7 +4,7 @@ The `or` and `and` operators take two parameters and return a boolean result.
 
 `not` flips a boolean from true to false, or vice versa. 
 
-`any` will return `true` if there are any `true` values in an array sequence, and `all` will return true if _all_ elements in an array are true.
+`any` will return `true` if there are any `true` values in a array sequence, and `all` will return true if _all_ elements in an array are true.
 
 `any_c(condition)` and `all_c(condition)` are like `any` and `all` but they take a condition expression that is used against each element to determine if it's `true`. Note: in `jq` you can simply pass a condition to `any` or `all` and it simply works - `yq` isn't that clever..yet
 

pkg/yqlib/doc/operators/headers/column.md

@@ -1,5 +1,3 @@
 # Column
 
 Returns the column of the matching node. Starts from 1, 0 indicates there was no column data.
-
-Column is the number of characters that precede that node on the line it starts.

pkg/yqlib/doc/operators/headers/multiply-merge.md

@@ -3,9 +3,7 @@
 Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently numbers, arrays and objects are supported.
 
 ## Objects and arrays - merging
-Objects are merged _deeply_ matching on matching keys. By default, array values override and are not deeply merged.
-
-You can use the add operator `+`, to shallow merge objects, see more info [here](https://mikefarah.gitbook.io/yq/operators/add).
+Objects are merged deeply matching on matching keys. By default, array values override and are not deeply merged.
 
 Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.
 
@@ -18,7 +16,6 @@ You can control how objects are merged by using one or more of the following fla
 - `n` only merge _new_ fields
 - `c` clobber custom tags
 
-To perform a shallow merge only, use the add operator `+`, see more info [here](https://mikefarah.gitbook.io/yq/operators/add).
 
 ### Merge two files together
 This uses the load operator to merge file2 into file1.

pkg/yqlib/doc/operators/headers/style.md

@@ -1,5 +1,3 @@
 # Style
 
-The style operator can be used to get or set the style of nodes (e.g. string style, yaml style).
-Use this to control the formatting of the document in yaml.
-
+The style operator can be used to get or set the style of nodes (e.g. string style, yaml style)

pkg/yqlib/doc/operators/multiply-merge.md

@@ -3,9 +3,7 @@
 Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently numbers, arrays and objects are supported.
 
 ## Objects and arrays - merging
-Objects are merged _deeply_ matching on matching keys. By default, array values override and are not deeply merged.
-
-You can use the add operator `+`, to shallow merge objects, see more info [here](https://mikefarah.gitbook.io/yq/operators/add).
+Objects are merged deeply matching on matching keys. By default, array values override and are not deeply merged.
 
 Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.
 
@@ -18,7 +16,6 @@ You can control how objects are merged by using one or more of the following fla
 - `n` only merge _new_ fields
 - `c` clobber custom tags
 
-To perform a shallow merge only, use the add operator `+`, see more info [here](https://mikefarah.gitbook.io/yq/operators/add).
 
 ### Merge two files together
 This uses the load operator to merge file2 into file1.
@@ -522,7 +519,7 @@ will output
 some: thing
 ```
 
-## Merging a null with an array
+## Merging an null with an array
 Running
 ```bash
 yq --null-input 'null * ["some"]'

pkg/yqlib/doc/operators/parent.md

@@ -37,43 +37,6 @@ fruit: banana
 name: sam
 ```
 
-## N-th parent
-You can optionally supply the number of levels to go up for the parent, the default being 1.
-
-Given a sample.yml file of:
-```yaml
-a:
-  b:
-    c: cat
-```
-then
-```bash
-yq '.a.b.c | parent(2)' sample.yml
-```
-will output
-```yaml
-b:
-  c: cat
-```
-
-## N-th parent - another level
-Given a sample.yml file of:
-```yaml
-a:
-  b:
-    c: cat
-```
-then
-```bash
-yq '.a.b.c | parent(3)' sample.yml
-```
-will output
-```yaml
-a:
-  b:
-    c: cat
-```
-
 ## No parent
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/operators/string-operators.md

@@ -56,40 +56,6 @@ IFS= read -rd '' output < <(cat my_file)
 output=$output ./yq '.data.values = strenv(output)' first.yml
 ```
 
-## Interpolation
-Given a sample.yml file of:
-```yaml
-value: things
-another: stuff
-```
-then
-```bash
-yq '.message = "I like \(.value) and \(.another)"' sample.yml
-```
-will output
-```yaml
-value: things
-another: stuff
-message: I like things and stuff
-```
-
-## Interpolation - not a string
-Given a sample.yml file of:
-```yaml
-value:
-  an: apple
-```
-then
-```bash
-yq '.message = "I like \(.value)"' sample.yml
-```
-will output
-```yaml
-value:
-  an: apple
-message: 'I like an: apple'
-```
-
 ## To up (upper) case
 Works with unicode characters
 
@@ -417,32 +383,3 @@ will output
 - word
 ```
 
-## To string
-Note that you may want to force `yq` to leave scalar values wrapped by passing in `--unwrapScalar=false` or `-r=f`
-
-Given a sample.yml file of:
-```yaml
-- 1
-- true
-- null
-- ~
-- cat
-- an: object
-- - array
-  - 2
-```
-then
-```bash
-yq '.[] |= to_string' sample.yml
-```
-will output
-```yaml
-- "1"
-- "true"
-- "null"
-- "~"
-- cat
-- "an: object"
-- "- array\n- 2"
-```
-

pkg/yqlib/doc/operators/style.md

@@ -1,8 +1,6 @@
 # Style
 
-The style operator can be used to get or set the style of nodes (e.g. string style, yaml style).
-Use this to control the formatting of the document in yaml.
-
+The style operator can be used to get or set the style of nodes (e.g. string style, yaml style)
 
 ## Update and set style of a particular node (simple)
 Given a sample.yml file of:
@@ -47,12 +45,6 @@ a: cat
 b: 5
 c: 3.2
 e: true
-f:
-  - 1
-  - 2
-  - 3
-g:
-  something: cool
 ```
 then
 ```bash
@@ -65,12 +57,6 @@ a: !!str cat
 b: !!int 5
 c: !!float 3.2
 e: !!bool true
-f: !!seq
-  - !!int 1
-  - !!int 2
-  - !!int 3
-g: !!map
-  something: !!str cool
 ```
 
 ## Set double quote style
@@ -80,12 +66,6 @@ a: cat
 b: 5
 c: 3.2
 e: true
-f:
-  - 1
-  - 2
-  - 3
-g:
-  something: cool
 ```
 then
 ```bash
@@ -97,12 +77,6 @@ a: "cat"
 b: "5"
 c: "3.2"
 e: "true"
-f:
-  - "1"
-  - "2"
-  - "3"
-g:
-  something: "cool"
 ```
 
 ## Set double quote style on map keys too
@@ -112,12 +86,6 @@ a: cat
 b: 5
 c: 3.2
 e: true
-f:
-  - 1
-  - 2
-  - 3
-g:
-  something: cool
 ```
 then
 ```bash
@@ -129,12 +97,6 @@ will output
 "b": "5"
 "c": "3.2"
 "e": "true"
-"f":
-  - "1"
-  - "2"
-  - "3"
-"g":
-  "something": "cool"
 ```
 
 ## Set single quote style
@@ -144,12 +106,6 @@ a: cat
 b: 5
 c: 3.2
 e: true
-f:
-  - 1
-  - 2
-  - 3
-g:
-  something: cool
 ```
 then
 ```bash
@@ -161,12 +117,6 @@ a: 'cat'
 b: '5'
 c: '3.2'
 e: 'true'
-f:
-  - '1'
-  - '2'
-  - '3'
-g:
-  something: 'cool'
 ```
 
 ## Set literal quote style
@@ -176,12 +126,6 @@ a: cat
 b: 5
 c: 3.2
 e: true
-f:
-  - 1
-  - 2
-  - 3
-g:
-  something: cool
 ```
 then
 ```bash
@@ -197,16 +141,6 @@ c: |-
   3.2
 e: |-
   true
-f:
-  - |-
-    1
-  - |-
-    2
-  - |-
-    3
-g:
-  something: |-
-    cool
 ```
 
 ## Set folded quote style
@@ -216,12 +150,6 @@ a: cat
 b: 5
 c: 3.2
 e: true
-f:
-  - 1
-  - 2
-  - 3
-g:
-  something: cool
 ```
 then
 ```bash
@@ -237,16 +165,6 @@ c: >-
   3.2
 e: >-
   true
-f:
-  - >-
-    1
-  - >-
-    2
-  - >-
-    3
-g:
-  something: >-
-    cool
 ```
 
 ## Set flow quote style
@@ -256,12 +174,6 @@ a: cat
 b: 5
 c: 3.2
 e: true
-f:
-  - 1
-  - 2
-  - 3
-g:
-  something: cool
 ```
 then
 ```bash
@@ -269,7 +181,7 @@ yq '.. style="flow"' sample.yml
 ```
 will output
 ```yaml
-{a: cat, b: 5, c: 3.2, e: true, f: [1, 2, 3], g: {something: cool}}
+{a: cat, b: 5, c: 3.2, e: true}
 ```
 
 ## Reset style - or pretty print
@@ -277,7 +189,10 @@ Set empty (default) quote style, note the usage of `...` to match keys too. Note
 
 Given a sample.yml file of:
 ```yaml
-{a: cat, "b": 5, 'c': 3.2, "e": true,  f: [1,2,3], "g": { something: "cool"} }
+a: cat
+"b": 5
+'c': 3.2
+"e": true
 ```
 then
 ```bash
@@ -289,12 +204,6 @@ a: cat
 b: 5
 c: 3.2
 e: true
-f:
-  - 1
-  - 2
-  - 3
-g:
-  something: cool
 ```
 
 ## Set style relatively with assign-update

pkg/yqlib/doc/usage/convert.md

@@ -36,10 +36,10 @@ will output
 ```yaml
 a: Easy! as one two three
 b:
-  c: 2
-  d:
-    - 3
-    - 4
+    c: 2
+    d:
+        - 3
+        - 4
 ```
 
 ## Encode json: simple
@@ -125,7 +125,9 @@ will output
 {"whatever":"cat"}
 ```
 
-## Roundtrip JSON Lines / NDJSON
+## Roundtrip NDJSON
+Unfortunately the json encoder strips leading spaces of values.
+
 Given a sample.json file of:
 ```json
 {"this": "is a multidoc json file"}
@@ -145,7 +147,7 @@ will output
 ```
 
 ## Roundtrip multi-document JSON
-The parser can also handle multiple multi-line json documents in a single file (despite this not being in the JSON Lines / NDJSON spec). Typically you would have one entire JSON document per line, but the parser also supports multiple multi-line json documents
+The NDJSON parser can also handle multiple multi-line json documents in a single file!
 
 Given a sample.json file of:
 ```json
@@ -227,7 +229,7 @@ will output
 {"a number":4}
 ```
 
-## Decode JSON Lines / NDJSON
+## Decode NDJSON
 Given a sample.json file of:
 ```json
 {"this": "is a multidoc json file"}
@@ -244,7 +246,7 @@ will output
 this: is a multidoc json file
 ---
 each:
-  - line is a valid json document
+    - line is a valid json document
 ---
 a number: 4
 ```

pkg/yqlib/doc/usage/csv-tsv.md

@@ -136,13 +136,13 @@ Samantha's Rabbit,,-188.8
 ```
 
 ## Parse CSV into an array of objects
-First row is assumed to be the header row. By default, entries with YAML/JSON formatting will be parsed!
+First row is assumed to be the header row.
 
 Given a sample.csv file of:
 ```csv
-name,numberOfCats,likesApples,height,facts
-Gary,1,true,168.8,cool: true
-Samantha's Rabbit,2,false,-188.8,tall: indeed
+name,numberOfCats,likesApples,height
+Gary,1,true,168.8
+Samantha's Rabbit,2,false,-188.8
 
 ```
 then
@@ -155,42 +155,10 @@ will output
   numberOfCats: 1
   likesApples: true
   height: 168.8
-  facts:
-    cool: true
 - name: Samantha's Rabbit
   numberOfCats: 2
   likesApples: false
   height: -188.8
-  facts:
-    tall: indeed
-```
-
-## Parse CSV into an array of objects, no auto-parsing
-First row is assumed to be the header row. Entries with YAML/JSON will be left as strings.
-
-Given a sample.csv file of:
-```csv
-name,numberOfCats,likesApples,height,facts
-Gary,1,true,168.8,cool: true
-Samantha's Rabbit,2,false,-188.8,tall: indeed
-
-```
-then
-```bash
-yq -p=csv --csv-auto-parse=f sample.csv
-```
-will output
-```yaml
-- name: Gary
-  numberOfCats: 1
-  likesApples: true
-  height: 168.8
-  facts: 'cool: true'
-- name: Samantha's Rabbit
-  numberOfCats: 2
-  likesApples: false
-  height: -188.8
-  facts: 'tall: indeed'
 ```
 
 ## Parse TSV into an array of objects

pkg/yqlib/doc/usage/formatting-expressions.md

@@ -1,100 +0,0 @@
-# Formatting Expressions
-
-`From version v4.41+`
-
-You can put expressions into `.yq` files, use whitespace and comments to break up complex expressions and explain what's going on.
-
-## Using expression files and comments
-Note that you can execute the file directly - but make sure you make the expression file executable.
-
-Given a sample.yaml file of:
-```yaml
-a:
-  b: old
-```
-And an 'update.yq' expression file of:
-```bash
-#! yq
-
-# This is a yq expression that updates the map
-# for several great reasons outlined here.
-
-.a.b = "new" # line comment here
-| .a.c = "frog"
-
-# Now good things will happen.
-```
-then
-```bash
-./update.yq sample.yaml
-```
-will output
-```yaml
-a:
-  b: new
-  c: frog
-```
-
-## Flags in expression files
-You can specify flags on the shebang line, this only works when executing the file directly.
-
-Given a sample.yaml file of:
-```yaml
-a:
-  b: old
-```
-And an 'update.yq' expression file of:
-```bash
-#! yq -oj
-
-# This is a yq expression that updates the map
-# for several great reasons outlined here.
-
-.a.b = "new" # line comment here
-| .a.c = "frog"
-
-# Now good things will happen.
-```
-then
-```bash
-./update.yq sample.yaml
-```
-will output
-```yaml
-{
-  "a": {
-    "b": "new",
-    "c": "frog"
-  }
-}
-```
-
-## Commenting out yq expressions
-Note that `c` is no longer set to 'frog'. In this example we're calling yq directly and passing the expression file into `--from-file`, this is no different from executing the expression file directly.
-
-Given a sample.yaml file of:
-```yaml
-a:
-  b: old
-```
-And an 'update.yq' expression file of:
-```bash
-#! yq
-# This is a yq expression that updates the map
-# for several great reasons outlined here.
-
-.a.b = "new" # line comment here
-# | .a.c = "frog"
-
-# Now good things will happen.
-```
-then
-```bash
-yq --from-file update.yq sample.yml
-```
-will output
-```yaml
-a:
-  b: new
-```
-

pkg/yqlib/doc/usage/headers/formatting-expressions.md

@@ -1,5 +0,0 @@
-# Formatting Expressions
-
-`From version v4.41+`
-
-You can put expressions into `.yq` files, use whitespace and comments to break up complex expressions and explain what's going on.

pkg/yqlib/doc/usage/properties.md

@@ -14,8 +14,6 @@ person: # neither do comments on maps
     name: Mike Wazowski # comments on values appear
     pets: 
     - cat # comments on array values appear
-    - nested:
-        - list entry
     food: [pizza] # comments on arrays do not
 emptyArray: []
 emptyMap: []
@@ -33,76 +31,9 @@ person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = cat
-person.pets.1.nested.0 = list entry
 person.food.0 = pizza
 ```
 
-## Encode properties with array brackets
-Declare the --properties-array-brackets flag to give array paths in brackets (e.g. SpringBoot).
-
-Given a sample.yml file of:
-```yaml
-# block comments come through
-person: # neither do comments on maps
-    name: Mike Wazowski # comments on values appear
-    pets: 
-    - cat # comments on array values appear
-    - nested:
-        - list entry
-    food: [pizza] # comments on arrays do not
-emptyArray: []
-emptyMap: []
-
-```
-then
-```bash
-yq -o=props --properties-array-brackets sample.yml
-```
-will output
-```properties
-# block comments come through
-# comments on values appear
-person.name = Mike Wazowski
-
-# comments on array values appear
-person.pets[0] = cat
-person.pets[1].nested[0] = list entry
-person.food[0] = pizza
-```
-
-## Encode properties - custom separator
-Use the --properties-customer-separator flag to specify your own key/value separator.
-
-Given a sample.yml file of:
-```yaml
-# block comments come through
-person: # neither do comments on maps
-    name: Mike Wazowski # comments on values appear
-    pets: 
-    - cat # comments on array values appear
-    - nested:
-        - list entry
-    food: [pizza] # comments on arrays do not
-emptyArray: []
-emptyMap: []
-
-```
-then
-```bash
-yq -o=props --properties-customer-separator=" :@ " sample.yml
-```
-will output
-```properties
-# block comments come through
-# comments on values appear
-person.name :@ Mike Wazowski
-
-# comments on array values appear
-person.pets.0 :@ cat
-person.pets.1.nested.0 :@ list entry
-person.food.0 :@ pizza
-```
-
 ## Encode properties: scalar encapsulation
 Note that string values with blank characters in them are encapsulated with double quotes
 
@@ -113,8 +44,6 @@ person: # neither do comments on maps
     name: Mike Wazowski # comments on values appear
     pets: 
     - cat # comments on array values appear
-    - nested:
-        - list entry
     food: [pizza] # comments on arrays do not
 emptyArray: []
 emptyMap: []
@@ -132,7 +61,6 @@ person.name = "Mike Wazowski"
 
 # comments on array values appear
 person.pets.0 = cat
-person.pets.1.nested.0 = "list entry"
 person.food.0 = pizza
 ```
 
@@ -144,8 +72,6 @@ person: # neither do comments on maps
     name: Mike Wazowski # comments on values appear
     pets: 
     - cat # comments on array values appear
-    - nested:
-        - list entry
     food: [pizza] # comments on arrays do not
 emptyArray: []
 emptyMap: []
@@ -159,7 +85,6 @@ will output
 ```properties
 person.name = Mike Wazowski
 person.pets.0 = cat
-person.pets.1.nested.0 = list entry
 person.food.0 = pizza
 ```
 
@@ -173,8 +98,6 @@ person: # neither do comments on maps
     name: Mike Wazowski # comments on values appear
     pets: 
     - cat # comments on array values appear
-    - nested:
-        - list entry
     food: [pizza] # comments on arrays do not
 emptyArray: []
 emptyMap: []
@@ -192,7 +115,6 @@ person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = cat
-person.pets.1.nested.0 = list entry
 person.food.0 = pizza
 emptyArray = 
 emptyMap = 
@@ -207,7 +129,6 @@ person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = cat
-person.pets.1.nested.0 = list entry
 person.food.0 = pizza
 
 ```
@@ -218,16 +139,14 @@ yq -p=props sample.properties
 will output
 ```yaml
 person:
-  # block comments come through
-  # comments on values appear
-  name: Mike Wazowski
-  pets:
-    # comments on array values appear
-    - cat
-    - nested:
-        - list entry
-  food:
-    - pizza
+    # block comments come through
+    # comments on values appear
+    name: Mike Wazowski
+    pets:
+        # comments on array values appear
+        - cat
+    food:
+        - pizza
 ```
 
 ## Decode properties - array should be a map
@@ -244,7 +163,7 @@ yq -p=props '.things |= array_to_map' sample.properties
 will output
 ```yaml
 things:
-  10: mike
+    10: mike
 ```
 
 ## Roundtrip
@@ -256,7 +175,6 @@ person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = cat
-person.pets.1.nested.0 = list entry
 person.food.0 = pizza
 
 ```
@@ -272,7 +190,6 @@ person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = dog
-person.pets.1.nested.0 = list entry
 person.food.0 = pizza
 ```
 

pkg/yqlib/doc/usage/recipes.md

@@ -59,44 +59,6 @@ will output
 - The expression `. + 1` increments the numBuckets counter.
 - See the [assign](https://mikefarah.gitbook.io/yq/operators/assign-update) and [add](https://mikefarah.gitbook.io/yq/operators/add) operators for more information.
 
-## Deeply prune a tree
-Say we are only interested in child1 and child2, and want to filter everything else out.
-
-Given a sample.yml file of:
-```yaml
-parentA:
-  - bob
-parentB:
-  child1: i am child1
-  child3: hiya
-parentC:
-  childX: cool
-  child2: me child2
-```
-then
-```bash
-yq '(
-  .. | # recurse through all the nodes
-  select(has("child1") or has("child2")) | # match parents that have either child1 or child2
-  (.child1, .child2) | # select those children
-  select(.) # filter out nulls
-) as $i ireduce({};  # using that set of nodes, create a new result map
-  setpath($i | path; $i) # and put in each node, using its original path
-)' sample.yml
-```
-will output
-```yaml
-parentB:
-  child1: i am child1
-parentC:
-  child2: me child2
-```
-
-### Explanation:
-- Find all the matching child1 and child2 nodes
-- Using ireduce, create a new map using just those nodes
-- Set each node into the new map using its original path
-
 ## Multiple or complex updates to items in an array
 We have an array and we want to _update_ the elements with a particular name in reference to its type.
 
@@ -155,7 +117,7 @@ myArray:
 - So, we use `|=` to update `.myArray`. This is the same as doing `.myArray = (.myArray | sort_by(.numBuckets))`
 
 ## Filter, flatten, sort and unique
-Lets find the unique set of names from the document.
+Lets
 
 Given a sample.yml file of:
 ```yaml
@@ -191,74 +153,3 @@ will output
 - Next we pipe `|` that through `sort` and then `unique` to get a sorted, unique list of the names!
 - See the [flatten](https://mikefarah.gitbook.io/yq/operators/flatten), [sort](https://mikefarah.gitbook.io/yq/operators/sort) and [unique](https://mikefarah.gitbook.io/yq/operators/unique) for more information and examples.
 
-## Export as environment variables (script), or any custom format
-Given a yaml document, lets output a script that will configure environment variables with that data. This same approach can be used for exporting into custom formats.
-
-Given a sample.yml file of:
-```yaml
-var0: string0
-var1: string1
-fruit:
-  - apple
-  - banana
-  - peach
-```
-then
-```bash
-yq '.[] |(
-	( select(kind == "scalar") | key + "='\''" + . + "'\''"),
-	( select(kind == "seq") | key + "=(" + (map("'\''" + . + "'\''") | join(",")) + ")")
-)' sample.yml
-```
-will output
-```yaml
-var0='string0'
-var1='string1'
-fruit=('apple','banana','peach')
-```
-
-### Explanation:
-- `.[]` matches all top level elements
-- We need a string expression for each of the different types that will produce the bash syntax, we'll use the union operator, to join them together
-- Scalars, we just need the key and quoted value: `( select(kind == "scalar") | key + "='" + . + "'")`
-- Sequences (or arrays) are trickier, we need to quote each value and `join` them with `,`: `map("'" + . + "'") | join(",")`
-
-## Custom format with nested data
-Like the previous example, but lets handle nested data structures. In this custom example, we're going to join the property paths with _. The important thing to keep in mind is that our expression is not recursive (despite the data structure being so). Instead we match _all_ elements on the tree and operate on them.
-
-Given a sample.yml file of:
-```yaml
-simple: string0
-simpleArray:
-  - apple
-  - banana
-  - peach
-deep:
-  property: value
-  array:
-    - cat
-```
-then
-```bash
-yq '.. |(
-	( select(kind == "scalar" and parent | kind != "seq") | (path | join("_")) + "='\''" + . + "'\''"),
-	( select(kind == "seq") | (path | join("_")) + "=(" + (map("'\''" + . + "'\''") | join(",")) + ")")
-)' sample.yml
-```
-will output
-```yaml
-simple='string0'
-deep_property='value'
-simpleArray=('apple','banana','peach')
-deep_array=('cat')
-```
-
-### Explanation:
-- You'll need to understand how the previous example works to understand this extension.
-- `..` matches _all_ elements, instead of `.[]` from the previous example that just matches top level elements.
-- Like before, we need a string expression for each of the different types that will produce the bash syntax, we'll use the union operator, to join them together
-- This time, however, our expression matches every node in the data structure.
-- We only want to print scalars that are not in arrays (because we handle the separately), so well add `and parent | kind != "seq"` to the select operator expression for scalars
-- We don't just want the key any more, we want the full path. So instead of `key` we have `path | join("_")`
-- The expression for sequences follows the same logic
-

pkg/yqlib/doc/usage/toml.md

@@ -104,19 +104,3 @@ owner:
       suburb: nice
 ```
 
-## Parse: Empty Table
-Given a sample.toml file of:
-```toml
-
-[dependencies]
-
-```
-then
-```bash
-yq -oy '.' sample.toml
-```
-will output
-```yaml
-dependencies: {}
-```
-

pkg/yqlib/doc/usage/xml.md

@@ -370,7 +370,7 @@ cat: purrs
 ```
 then
 ```bash
-yq -o=xml sample.yml
+yq -o=xml '.' sample.yml
 ```
 will output
 ```xml
@@ -387,7 +387,7 @@ pets:
 ```
 then
 ```bash
-yq -o=xml sample.yml
+yq -o=xml '.' sample.yml
 ```
 will output
 ```xml
@@ -409,7 +409,7 @@ cat:
 ```
 then
 ```bash
-yq -o=xml sample.yml
+yq -o=xml '.' sample.yml
 ```
 will output
 ```xml
@@ -430,7 +430,7 @@ cat:
 ```
 then
 ```bash
-yq -o=xml sample.yml
+yq -o=xml '.' sample.yml
 ```
 will output
 ```xml
@@ -457,7 +457,7 @@ cat: # inline_cat
 ```
 then
 ```bash
-yq -o=xml sample.yml
+yq -o=xml '.' sample.yml
 ```
 will output
 ```xml
@@ -487,7 +487,7 @@ apple:
 ```
 then
 ```bash
-yq -o=xml sample.yml
+yq -o=xml '.' sample.yml
 ```
 will output
 ```xml

pkg/yqlib/encoder.go

@@ -10,3 +10,18 @@ type Encoder interface {
 	PrintLeadingContent(writer io.Writer, content string) error
 	CanHandleAliases() bool
 }
+
+func mapKeysToStrings(node *CandidateNode) {
+
+	if node.Kind == MappingNode {
+		for index, child := range node.Content {
+			if index%2 == 0 { // its a map key
+				child.Tag = "!!str"
+			}
+		}
+	}
+
+	for _, child := range node.Content {
+		mapKeysToStrings(child)
+	}
+}

pkg/yqlib/encoder_base64.go

@@ -18,17 +18,17 @@ func (e *base64Encoder) CanHandleAliases() bool {
 	return false
 }
 
-func (e *base64Encoder) PrintDocumentSeparator(_ io.Writer) error {
+func (e *base64Encoder) PrintDocumentSeparator(writer io.Writer) error {
 	return nil
 }
 
-func (e *base64Encoder) PrintLeadingContent(_ io.Writer, _ string) error {
+func (e *base64Encoder) PrintLeadingContent(writer io.Writer, content string) error {
 	return nil
 }
 
 func (e *base64Encoder) Encode(writer io.Writer, node *CandidateNode) error {
-	if node.GuessTagFromCustomType() != "!!str" {
-		return fmt.Errorf("cannot encode %v as base64, can only operate on strings", node.Tag)
+	if node.guessTagFromCustomType() != "!!str" {
+		return fmt.Errorf("cannot encode %v as base64, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string", node.Tag)
 	}
 	_, err := writer.Write([]byte(e.encoding.EncodeToString([]byte(node.Value))))
 	return err

pkg/yqlib/encoder_csv.go

@@ -10,19 +10,19 @@ type csvEncoder struct {
 	separator rune
 }
 
-func NewCsvEncoder(prefs CsvPreferences) Encoder {
-	return &csvEncoder{separator: prefs.Separator}
+func NewCsvEncoder(separator rune) Encoder {
+	return &csvEncoder{separator: separator}
 }
 
 func (e *csvEncoder) CanHandleAliases() bool {
 	return false
 }
 
-func (e *csvEncoder) PrintDocumentSeparator(_ io.Writer) error {
+func (e *csvEncoder) PrintDocumentSeparator(writer io.Writer) error {
 	return nil
 }
 
-func (e *csvEncoder) PrintLeadingContent(_ io.Writer, _ string) error {
+func (e *csvEncoder) PrintLeadingContent(writer io.Writer, content string) error {
 	return nil
 }
 
@@ -65,7 +65,7 @@ func (e *csvEncoder) createChildRow(child *CandidateNode, headers []*CandidateNo
 	childRow := make([]*CandidateNode, 0)
 	for _, header := range headers {
 		keyIndex := findKeyInMap(child, header)
-		value := CreateScalarNode(nil, "")
+		value := createScalarNode(nil, "")
 		if keyIndex != -1 {
 			value = child.Content[keyIndex+1]
 		}
@@ -102,7 +102,7 @@ func (e *csvEncoder) encodeObjects(csvWriter *csv.Writer, content []*CandidateNo
 
 func (e *csvEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 	if node.Kind == ScalarNode {
-		return WriteString(writer, node.Value+"\n")
+		return writeString(writer, node.Value+"\n")
 	}
 
 	csvWriter := csv.NewWriter(writer)

pkg/yqlib/encoder_json.go

@@ -10,29 +10,30 @@ import (
 )
 
 type jsonEncoder struct {
-	prefs        JsonPreferences
 	indentString string
+	colorise     bool
+	UnwrapScalar bool
 }
 
-func NewJSONEncoder(prefs JsonPreferences) Encoder {
+func NewJSONEncoder(indent int, colorise bool, unwrapScalar bool) Encoder {
 	var indentString = ""
 
-	for index := 0; index < prefs.Indent; index++ {
+	for index := 0; index < indent; index++ {
 		indentString = indentString + " "
 	}
 
-	return &jsonEncoder{prefs, indentString}
+	return &jsonEncoder{indentString, colorise, unwrapScalar}
 }
 
 func (je *jsonEncoder) CanHandleAliases() bool {
 	return false
 }
 
-func (je *jsonEncoder) PrintDocumentSeparator(_ io.Writer) error {
+func (je *jsonEncoder) PrintDocumentSeparator(writer io.Writer) error {
 	return nil
 }
 
-func (je *jsonEncoder) PrintLeadingContent(_ io.Writer, _ string) error {
+func (je *jsonEncoder) PrintLeadingContent(writer io.Writer, content string) error {
 	return nil
 }
 
@@ -40,13 +41,13 @@ func (je *jsonEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 	log.Debugf("I need to encode %v", NodeToString(node))
 	log.Debugf("kids %v", len(node.Content))
 
-	if node.Kind == ScalarNode && je.prefs.UnwrapScalar {
-		return WriteString(writer, node.Value+"\n")
+	if node.Kind == ScalarNode && je.UnwrapScalar {
+		return writeString(writer, node.Value+"\n")
 	}
 
 	destination := writer
 	tempBuffer := bytes.NewBuffer(nil)
-	if je.prefs.ColorsEnabled {
+	if je.colorise {
 		destination = tempBuffer
 	}
 
@@ -58,7 +59,7 @@ func (je *jsonEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 	if err != nil {
 		return err
 	}
-	if je.prefs.ColorsEnabled {
+	if je.colorise {
 		return colorizeAndPrint(tempBuffer.Bytes(), writer)
 	}
 	return nil

pkg/yqlib/encoder_lua.go

@@ -72,11 +72,11 @@ func NewLuaEncoder(prefs LuaPreferences) Encoder {
 	return &luaEncoder{unescape.Replace(prefs.DocPrefix), unescape.Replace(prefs.DocSuffix), 0, "\t", prefs.UnquotedKeys, prefs.Globals, escape}
 }
 
-func (le *luaEncoder) PrintDocumentSeparator(_ io.Writer) error {
+func (le *luaEncoder) PrintDocumentSeparator(writer io.Writer) error {
 	return nil
 }
 
-func (le *luaEncoder) PrintLeadingContent(_ io.Writer, _ string) error {
+func (le *luaEncoder) PrintLeadingContent(writer io.Writer, content string) error {
 	return nil
 }
 
@@ -86,15 +86,15 @@ func (le *luaEncoder) encodeString(writer io.Writer, node *CandidateNode) error
 	case LiteralStyle, FoldedStyle, FlowStyle:
 		for i := 0; i < 10; i++ {
 			if !strings.Contains(node.Value, "]"+strings.Repeat("=", i)+"]") {
-				err := WriteString(writer, "["+strings.Repeat("=", i)+"[\n")
+				err := writeString(writer, "["+strings.Repeat("=", i)+"[\n")
 				if err != nil {
 					return err
 				}
-				err = WriteString(writer, node.Value)
+				err = writeString(writer, node.Value)
 				if err != nil {
 					return err
 				}
-				return WriteString(writer, "]"+strings.Repeat("=", i)+"]")
+				return writeString(writer, "]"+strings.Repeat("=", i)+"]")
 			}
 		}
 	case SingleQuotedStyle:
@@ -102,22 +102,22 @@ func (le *luaEncoder) encodeString(writer io.Writer, node *CandidateNode) error
 
 		// fallthrough to regular ol' string
 	}
-	return WriteString(writer, quote+le.escape.Replace(node.Value)+quote)
+	return writeString(writer, quote+le.escape.Replace(node.Value)+quote)
 }
 
 func (le *luaEncoder) writeIndent(writer io.Writer) error {
 	if le.indentStr == "" {
 		return nil
 	}
-	err := WriteString(writer, "\n")
+	err := writeString(writer, "\n")
 	if err != nil {
 		return err
 	}
-	return WriteString(writer, strings.Repeat(le.indentStr, le.indent))
+	return writeString(writer, strings.Repeat(le.indentStr, le.indent))
 }
 
 func (le *luaEncoder) encodeArray(writer io.Writer, node *CandidateNode) error {
-	err := WriteString(writer, "{")
+	err := writeString(writer, "{")
 	if err != nil {
 		return err
 	}
@@ -131,13 +131,13 @@ func (le *luaEncoder) encodeArray(writer io.Writer, node *CandidateNode) error {
 		if err != nil {
 			return err
 		}
-		err = WriteString(writer, ",")
+		err = writeString(writer, ",")
 		if err != nil {
 			return err
 		}
 		if child.LineComment != "" {
 			sansPrefix, _ := strings.CutPrefix(child.LineComment, "#")
-			err = WriteString(writer, " --"+sansPrefix)
+			err = writeString(writer, " --"+sansPrefix)
 			if err != nil {
 				return err
 			}
@@ -150,7 +150,7 @@ func (le *luaEncoder) encodeArray(writer io.Writer, node *CandidateNode) error {
 			return err
 		}
 	}
-	return WriteString(writer, "}")
+	return writeString(writer, "}")
 }
 
 func needsQuoting(s string) bool {
@@ -181,7 +181,7 @@ func needsQuoting(s string) bool {
 
 func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bool) error {
 	if !global {
-		err := WriteString(writer, "{")
+		err := writeString(writer, "{")
 		if err != nil {
 			return err
 		}
@@ -194,7 +194,7 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bo
 			if err != nil {
 				return err
 			}
-			err = WriteString(writer, ";")
+			err = writeString(writer, ";")
 			if err != nil {
 				return err
 			}
@@ -207,19 +207,19 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bo
 				}
 			}
 			if (le.unquoted || global) && child.Tag == "!!str" && !needsQuoting(child.Value) {
-				err := WriteString(writer, child.Value+" = ")
+				err := writeString(writer, child.Value+" = ")
 				if err != nil {
 					return err
 				}
 			} else {
 				if global {
 					// This only works in Lua 5.2+
-					err := WriteString(writer, "_ENV")
+					err := writeString(writer, "_ENV")
 					if err != nil {
 						return err
 					}
 				}
-				err := WriteString(writer, "[")
+				err := writeString(writer, "[")
 				if err != nil {
 					return err
 				}
@@ -227,7 +227,7 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bo
 				if err != nil {
 					return err
 				}
-				err = WriteString(writer, "] = ")
+				err = writeString(writer, "] = ")
 				if err != nil {
 					return err
 				}
@@ -235,7 +235,7 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bo
 		}
 		if child.LineComment != "" {
 			sansPrefix, _ := strings.CutPrefix(child.LineComment, "#")
-			err := WriteString(writer, strings.Repeat(" ", i%2)+"--"+sansPrefix)
+			err := writeString(writer, strings.Repeat(" ", i%2)+"--"+sansPrefix)
 			if err != nil {
 				return err
 			}
@@ -249,7 +249,7 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bo
 		}
 	}
 	if global {
-		return WriteString(writer, "\n")
+		return writeString(writer, "\n")
 	}
 	le.indent--
 	if len(node.Content) != 0 {
@@ -258,7 +258,7 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bo
 			return err
 		}
 	}
-	return WriteString(writer, "}")
+	return writeString(writer, "}")
 }
 
 func (le *luaEncoder) encodeAny(writer io.Writer, node *CandidateNode) error {
@@ -273,30 +273,30 @@ func (le *luaEncoder) encodeAny(writer io.Writer, node *CandidateNode) error {
 			return le.encodeString(writer, node)
 		case "!!null":
 			// TODO reject invalid use as a table key
-			return WriteString(writer, "nil")
+			return writeString(writer, "nil")
 		case "!!bool":
 			// Yaml 1.2 has case variation e.g. True, FALSE etc but Lua only has
 			// lower case
-			return WriteString(writer, strings.ToLower(node.Value))
+			return writeString(writer, strings.ToLower(node.Value))
 		case "!!int":
 			if strings.HasPrefix(node.Value, "0o") {
 				_, octalValue, err := parseInt64(node.Value)
 				if err != nil {
 					return err
 				}
-				return WriteString(writer, fmt.Sprintf("%d", octalValue))
+				return writeString(writer, fmt.Sprintf("%d", octalValue))
 			}
-			return WriteString(writer, strings.ToLower(node.Value))
+			return writeString(writer, strings.ToLower(node.Value))
 		case "!!float":
 			switch strings.ToLower(node.Value) {
 			case ".inf", "+.inf":
-				return WriteString(writer, "(1/0)")
+				return writeString(writer, "(1/0)")
 			case "-.inf":
-				return WriteString(writer, "(-1/0)")
+				return writeString(writer, "(-1/0)")
 			case ".nan":
-				return WriteString(writer, "(0/0)")
+				return writeString(writer, "(0/0)")
 			default:
-				return WriteString(writer, node.Value)
+				return writeString(writer, node.Value)
 			}
 		default:
 			return fmt.Errorf("Lua encoder NYI -- %s", node.Tag)
@@ -307,7 +307,7 @@ func (le *luaEncoder) encodeAny(writer io.Writer, node *CandidateNode) error {
 }
 
 func (le *luaEncoder) encodeTopLevel(writer io.Writer, node *CandidateNode) error {
-	err := WriteString(writer, le.docPrefix)
+	err := writeString(writer, le.docPrefix)
 	if err != nil {
 		return err
 	}
@@ -315,7 +315,7 @@ func (le *luaEncoder) encodeTopLevel(writer io.Writer, node *CandidateNode) erro
 	if err != nil {
 		return err
 	}
-	return WriteString(writer, le.docSuffix)
+	return writeString(writer, le.docSuffix)
 }
 
 func (le *luaEncoder) Encode(writer io.Writer, node *CandidateNode) error {

pkg/yqlib/encoder_properties.go

@@ -1,4 +1,4 @@
-package properties
+package yqlib
 
 import (
 	"bufio"
@@ -8,16 +8,15 @@ import (
 	"strings"
 
 	"github.com/magiconair/properties"
-	"github.com/mikefarah/yq/v4/pkg/yqlib"
 )
 
 type propertiesEncoder struct {
-	prefs PropertiesPreferences
+	unwrapScalar bool
 }
 
-func NewPropertiesEncoder(prefs PropertiesPreferences) yqlib.Encoder {
+func NewPropertiesEncoder(unwrapScalar bool) Encoder {
 	return &propertiesEncoder{
-		prefs: prefs,
+		unwrapScalar: unwrapScalar,
 	}
 }
 
@@ -25,7 +24,7 @@ func (pe *propertiesEncoder) CanHandleAliases() bool {
 	return false
 }
 
-func (pe *propertiesEncoder) PrintDocumentSeparator(_ io.Writer) error {
+func (pe *propertiesEncoder) PrintDocumentSeparator(writer io.Writer) error {
 	return nil
 }
 
@@ -44,7 +43,7 @@ func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content strin
 			}
 
 		} else {
-			if err := yqlib.WriteString(writer, readline); err != nil {
+			if err := writeString(writer, readline); err != nil {
 				return err
 			}
 		}
@@ -52,7 +51,7 @@ func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content strin
 		if errors.Is(errReading, io.EOF) {
 			if readline != "" {
 				// the last comment we read didn't have a newline, put one in
-				if err := yqlib.WriteString(writer, "\n"); err != nil {
+				if err := writeString(writer, "\n"); err != nil {
 					return err
 				}
 			}
@@ -62,15 +61,14 @@ func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content strin
 	return nil
 }
 
-func (pe *propertiesEncoder) Encode(writer io.Writer, node *yqlib.CandidateNode) error {
+func (pe *propertiesEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 
-	if node.Kind == yqlib.ScalarNode {
-		return yqlib.WriteString(writer, node.Value+"\n")
+	if node.Kind == ScalarNode {
+		return writeString(writer, node.Value+"\n")
 	}
 
-	node.ConvertKeysToStrings()
+	mapKeysToStrings(node)
 	p := properties.NewProperties()
-	p.WriteSeparator = pe.prefs.KeyValueSeparator
 	err := pe.doEncode(p, node, "", nil)
 	if err != nil {
 		return err
@@ -80,32 +78,32 @@ func (pe *propertiesEncoder) Encode(writer io.Writer, node *yqlib.CandidateNode)
 	return err
 }
 
-func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yqlib.CandidateNode, path string, keyNode *yqlib.CandidateNode) error {
+func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *CandidateNode, path string, keyNode *CandidateNode) error {
 
 	comments := ""
 	if keyNode != nil {
 		// include the key node comments if present
-		comments = keyNode.CleanHeadAndLineComment()
+		comments = headAndLineComment(keyNode)
 	}
-	comments = comments + node.CleanHeadAndLineComment()
+	comments = comments + headAndLineComment(node)
 	commentsWithSpaces := strings.ReplaceAll(comments, "\n", "\n ")
 	p.SetComments(path, strings.Split(commentsWithSpaces, "\n"))
 
 	switch node.Kind {
-	case yqlib.ScalarNode:
+	case ScalarNode:
 		var nodeValue string
-		if pe.prefs.UnwrapScalar || !strings.Contains(node.Value, " ") {
+		if pe.unwrapScalar || !strings.Contains(node.Value, " ") {
 			nodeValue = node.Value
 		} else {
 			nodeValue = fmt.Sprintf("%q", node.Value)
 		}
 		_, _, err := p.Set(path, nodeValue)
 		return err
-	case yqlib.SequenceNode:
+	case SequenceNode:
 		return pe.encodeArray(p, node.Content, path)
-	case yqlib.MappingNode:
+	case MappingNode:
 		return pe.encodeMap(p, node.Content, path)
-	case yqlib.AliasNode:
+	case AliasNode:
 		return pe.doEncode(p, node.Alias, path, nil)
 	default:
 		return fmt.Errorf("Unsupported node %v", node.Tag)
@@ -116,17 +114,10 @@ func (pe *propertiesEncoder) appendPath(path string, key interface{}) string {
 	if path == "" {
 		return fmt.Sprintf("%v", key)
 	}
-	switch key.(type) {
-	case int:
-		if pe.prefs.UseArrayBrackets {
-			return fmt.Sprintf("%v[%v]", path, key)
-		}
-
-	}
 	return fmt.Sprintf("%v.%v", path, key)
 }
 
-func (pe *propertiesEncoder) encodeArray(p *properties.Properties, kids []*yqlib.CandidateNode, path string) error {
+func (pe *propertiesEncoder) encodeArray(p *properties.Properties, kids []*CandidateNode, path string) error {
 	for index, child := range kids {
 		err := pe.doEncode(p, child, pe.appendPath(path, index), nil)
 		if err != nil {
@@ -136,7 +127,7 @@ func (pe *propertiesEncoder) encodeArray(p *properties.Properties, kids []*yqlib
 	return nil
 }
 
-func (pe *propertiesEncoder) encodeMap(p *properties.Properties, kids []*yqlib.CandidateNode, path string) error {
+func (pe *propertiesEncoder) encodeMap(p *properties.Properties, kids []*CandidateNode, path string) error {
 	for index := 0; index < len(kids); index = index + 2 {
 		key := kids[index]
 		value := kids[index+1]

pkg/yqlib/encoder_properties_test.go

@@ -0,0 +1,127 @@
+package yqlib
+
+import (
+	"bufio"
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+func yamlToProps(sampleYaml string, unwrapScalar bool) string {
+	var output bytes.Buffer
+	writer := bufio.NewWriter(&output)
+
+	var propsEncoder = NewPropertiesEncoder(unwrapScalar)
+	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
+	if err != nil {
+		panic(err)
+	}
+	node := inputs.Front().Value.(*CandidateNode)
+	err = propsEncoder.Encode(writer, node)
+	if err != nil {
+		panic(err)
+	}
+	writer.Flush()
+
+	return strings.TrimSuffix(output.String(), "\n")
+}
+
+func TestPropertiesEncoderSimple_Unwrapped(t *testing.T) {
+	var sampleYaml = `a: 'bob cool'`
+
+	var expectedProps = `a = bob cool`
+	var actualProps = yamlToProps(sampleYaml, true)
+	test.AssertResult(t, expectedProps, actualProps)
+}
+
+func TestPropertiesEncoderSimple_Wrapped(t *testing.T) {
+	var sampleYaml = `a: 'bob cool'`
+
+	var expectedProps = `a = "bob cool"`
+	var actualProps = yamlToProps(sampleYaml, false)
+	test.AssertResult(t, expectedProps, actualProps)
+}
+
+func TestPropertiesEncoderSimpleWithComments_Unwrapped(t *testing.T) {
+	var sampleYaml = `a: 'bob cool' # line`
+
+	var expectedProps = `# line
+a = bob cool`
+	var actualProps = yamlToProps(sampleYaml, true)
+	test.AssertResult(t, expectedProps, actualProps)
+}
+
+func TestPropertiesEncoderSimpleWithComments_Wrapped(t *testing.T) {
+	var sampleYaml = `a: 'bob cool' # line`
+
+	var expectedProps = `# line
+a = "bob cool"`
+	var actualProps = yamlToProps(sampleYaml, false)
+	test.AssertResult(t, expectedProps, actualProps)
+}
+
+func TestPropertiesEncoderDeep_Unwrapped(t *testing.T) {
+	var sampleYaml = `a: 
+  b: "bob cool"
+`
+
+	var expectedProps = `a.b = bob cool`
+	var actualProps = yamlToProps(sampleYaml, true)
+	test.AssertResult(t, expectedProps, actualProps)
+}
+
+func TestPropertiesEncoderDeep_Wrapped(t *testing.T) {
+	var sampleYaml = `a: 
+  b: "bob cool"
+`
+
+	var expectedProps = `a.b = "bob cool"`
+	var actualProps = yamlToProps(sampleYaml, false)
+	test.AssertResult(t, expectedProps, actualProps)
+}
+
+func TestPropertiesEncoderDeepWithComments_Unwrapped(t *testing.T) {
+	var sampleYaml = `a:  # a thing
+  b: "bob cool" # b thing
+`
+
+	var expectedProps = `# b thing
+a.b = bob cool`
+	var actualProps = yamlToProps(sampleYaml, true)
+	test.AssertResult(t, expectedProps, actualProps)
+}
+
+func TestPropertiesEncoderDeepWithComments_Wrapped(t *testing.T) {
+	var sampleYaml = `a:  # a thing
+  b: "bob cool" # b thing
+`
+
+	var expectedProps = `# b thing
+a.b = "bob cool"`
+	var actualProps = yamlToProps(sampleYaml, false)
+	test.AssertResult(t, expectedProps, actualProps)
+}
+
+func TestPropertiesEncoderArray_Unwrapped(t *testing.T) {
+	var sampleYaml = `a: 
+  b: [{c: dog}, {c: cat}]
+`
+
+	var expectedProps = `a.b.0.c = dog
+a.b.1.c = cat`
+	var actualProps = yamlToProps(sampleYaml, true)
+	test.AssertResult(t, expectedProps, actualProps)
+}
+
+func TestPropertiesEncoderArray_Wrapped(t *testing.T) {
+	var sampleYaml = `a: 
+  b: [{c: dog named jim}, {c: cat named jam}]
+`
+
+	var expectedProps = `a.b.0.c = "dog named jim"
+a.b.1.c = "cat named jam"`
+	var actualProps = yamlToProps(sampleYaml, false)
+	test.AssertResult(t, expectedProps, actualProps)
+}

pkg/yqlib/encoder_sh.go

@@ -21,20 +21,20 @@ func (e *shEncoder) CanHandleAliases() bool {
 	return false
 }
 
-func (e *shEncoder) PrintDocumentSeparator(_ io.Writer) error {
+func (e *shEncoder) PrintDocumentSeparator(writer io.Writer) error {
 	return nil
 }
 
-func (e *shEncoder) PrintLeadingContent(_ io.Writer, _ string) error {
+func (e *shEncoder) PrintLeadingContent(writer io.Writer, content string) error {
 	return nil
 }
 
 func (e *shEncoder) Encode(writer io.Writer, node *CandidateNode) error {
-	if node.GuessTagFromCustomType() != "!!str" {
+	if node.guessTagFromCustomType() != "!!str" {
 		return fmt.Errorf("cannot encode %v as URI, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string", node.Tag)
 	}
 
-	return WriteString(writer, e.encode(node.Value))
+	return writeString(writer, e.encode(node.Value))
 }
 
 // put any (shell-unsafe) characters into a single-quoted block, close the block lazily

pkg/yqlib/encoder_shellvariables.go

@@ -30,7 +30,7 @@ func (pe *shellVariablesEncoder) PrintLeadingContent(_ io.Writer, _ string) erro
 
 func (pe *shellVariablesEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 
-	node.ConvertKeysToStrings()
+	mapKeysToStrings(node)
 	err := pe.doEncode(&writer, node, "")
 	if err != nil {
 		return err

pkg/yqlib/encoder_test.go

@@ -16,10 +16,7 @@ func yamlToJSON(t *testing.T, sampleYaml string, indent int) string {
 	var output bytes.Buffer
 	writer := bufio.NewWriter(&output)
 
-	prefs := ConfiguredJSONPreferences.Copy()
-	prefs.Indent = indent
-	prefs.UnwrapScalar = false
-	var jsonEncoder = NewJSONEncoder(prefs)
+	var jsonEncoder = NewJSONEncoder(indent, false, false)
 	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {
 		panic(err)

pkg/yqlib/encoder_toml.go

@@ -14,16 +14,16 @@ func NewTomlEncoder() Encoder {
 
 func (te *tomlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 	if node.Kind == ScalarNode {
-		return WriteString(writer, node.Value+"\n")
+		return writeString(writer, node.Value+"\n")
 	}
 	return fmt.Errorf("only scalars (e.g. strings, numbers, booleans) are supported for TOML output at the moment. Please use yaml output format (-oy) until the encoder has been fully implemented")
 }
 
-func (te *tomlEncoder) PrintDocumentSeparator(_ io.Writer) error {
+func (te *tomlEncoder) PrintDocumentSeparator(writer io.Writer) error {
 	return nil
 }
 
-func (te *tomlEncoder) PrintLeadingContent(_ io.Writer, _ string) error {
+func (te *tomlEncoder) PrintLeadingContent(writer io.Writer, content string) error {
 	return nil
 }
 

pkg/yqlib/encoder_uri.go

@@ -17,16 +17,16 @@ func (e *uriEncoder) CanHandleAliases() bool {
 	return false
 }
 
-func (e *uriEncoder) PrintDocumentSeparator(_ io.Writer) error {
+func (e *uriEncoder) PrintDocumentSeparator(writer io.Writer) error {
 	return nil
 }
 
-func (e *uriEncoder) PrintLeadingContent(_ io.Writer, _ string) error {
+func (e *uriEncoder) PrintLeadingContent(writer io.Writer, content string) error {
 	return nil
 }
 
 func (e *uriEncoder) Encode(writer io.Writer, node *CandidateNode) error {
-	if node.GuessTagFromCustomType() != "!!str" {
+	if node.guessTagFromCustomType() != "!!str" {
 		return fmt.Errorf("cannot encode %v as URI, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string", node.Tag)
 	}
 	_, err := writer.Write([]byte(url.QueryEscape(node.Value)))

pkg/yqlib/encoder_xml.go

@@ -1,6 +1,6 @@
 //go:build !yq_noxml
 
-package xml
+package yqlib
 
 import (
 	"encoding/xml"
@@ -8,8 +8,6 @@ import (
 	"io"
 	"regexp"
 	"strings"
-
-	"github.com/mikefarah/yq/v4/pkg/yqlib"
 )
 
 type xmlEncoder struct {
@@ -19,10 +17,10 @@ type xmlEncoder struct {
 	leadingContent string
 }
 
-func NewXMLEncoder(prefs XmlPreferences) yqlib.Encoder {
+func NewXMLEncoder(indent int, prefs XmlPreferences) Encoder {
 	var indentString = ""
 
-	for index := 0; index < prefs.Indent; index++ {
+	for index := 0; index < indent; index++ {
 		indentString = indentString + " "
 	}
 	return &xmlEncoder{indentString, nil, prefs, ""}
@@ -32,16 +30,16 @@ func (e *xmlEncoder) CanHandleAliases() bool {
 	return false
 }
 
-func (e *xmlEncoder) PrintDocumentSeparator(_ io.Writer) error {
+func (e *xmlEncoder) PrintDocumentSeparator(writer io.Writer) error {
 	return nil
 }
 
-func (e *xmlEncoder) PrintLeadingContent(_ io.Writer, content string) error {
+func (e *xmlEncoder) PrintLeadingContent(writer io.Writer, content string) error {
 	e.leadingContent = content
 	return nil
 }
 
-func (e *xmlEncoder) Encode(writer io.Writer, node *yqlib.CandidateNode) error {
+func (e *xmlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 	encoder := xml.NewEncoder(writer)
 	// hack so we can manually add newlines to procInst and directives
 	e.writer = writer
@@ -61,7 +59,7 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yqlib.CandidateNode) error {
 					return err
 				}
 				if _, err := e.writer.Write([]byte("\n")); err != nil {
-					yqlib.GetLogger().Warning("Unable to write newline, skipping: %w", err)
+					log.Warning("Unable to write newline, skipping: %w", err)
 				}
 			}
 		}
@@ -81,12 +79,12 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yqlib.CandidateNode) error {
 	}
 
 	switch node.Kind {
-	case yqlib.MappingNode:
+	case MappingNode:
 		err := e.encodeTopLevelMap(encoder, node)
 		if err != nil {
 			return err
 		}
-	case yqlib.ScalarNode:
+	case ScalarNode:
 		var charData xml.CharData = []byte(node.Value)
 		err := encoder.EncodeToken(charData)
 		if err != nil {
@@ -101,8 +99,8 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yqlib.CandidateNode) error {
 
 }
 
-func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yqlib.CandidateNode) error {
-	err := e.encodeComment(encoder, node.CleanHeadAndLineComment())
+func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *CandidateNode) error {
+	err := e.encodeComment(encoder, headAndLineComment(node))
 	if err != nil {
 		return err
 	}
@@ -111,12 +109,12 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yqlib.Candida
 		value := node.Content[i+1]
 
 		start := xml.StartElement{Name: xml.Name{Local: key.Value}}
-		yqlib.GetLogger().Debugf("comments of key %v", key.Value)
-		err := e.encodeComment(encoder, key.CleanHeadAndLineComment())
+		log.Debugf("comments of key %v", key.Value)
+		err := e.encodeComment(encoder, headAndLineComment(key))
 		if err != nil {
 			return err
 		}
-		if key.CleanHeadAndLineComment() != "" {
+		if headAndLineComment(key) != "" {
 			var newLine xml.CharData = []byte("\n")
 			err = encoder.EncodeToken(newLine)
 			if err != nil {
@@ -124,7 +122,7 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yqlib.Candida
 			}
 		}
 
-		if key.Value == (e.prefs.ProcInstPrefix + "xml") { //nolint
+		if key.Value == (e.prefs.ProcInstPrefix + "xml") {
 			// dont double process these.
 		} else if strings.HasPrefix(key.Value, e.prefs.ProcInstPrefix) {
 			name := strings.Replace(key.Value, e.prefs.ProcInstPrefix, "", 1)
@@ -133,7 +131,7 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yqlib.Candida
 				return err
 			}
 			if _, err := e.writer.Write([]byte("\n")); err != nil {
-				yqlib.GetLogger().Warning("Unable to write newline, skipping: %w", err)
+				log.Warning("Unable to write newline, skipping: %w", err)
 			}
 		} else if key.Value == e.prefs.DirectiveName {
 			var directive xml.Directive = []byte(value.Value)
@@ -141,48 +139,48 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yqlib.Candida
 				return err
 			}
 			if _, err := e.writer.Write([]byte("\n")); err != nil {
-				yqlib.GetLogger().Warning("Unable to write newline, skipping: %w", err)
+				log.Warning("Unable to write newline, skipping: %w", err)
 			}
 		} else {
 
-			yqlib.GetLogger().Debugf("recursing")
+			log.Debugf("recursing")
 
 			err = e.doEncode(encoder, value, start)
 			if err != nil {
 				return err
 			}
 		}
-		err = e.encodeComment(encoder, key.CleanFootComment())
+		err = e.encodeComment(encoder, footComment(key))
 		if err != nil {
 			return err
 		}
 	}
-	return e.encodeComment(encoder, node.CleanFootComment())
+	return e.encodeComment(encoder, footComment(node))
 }
 
-func (e *xmlEncoder) encodeStart(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
+func (e *xmlEncoder) encodeStart(encoder *xml.Encoder, node *CandidateNode, start xml.StartElement) error {
 	err := encoder.EncodeToken(start)
 	if err != nil {
 		return err
 	}
-	return e.encodeComment(encoder, node.CleanHeadComment())
+	return e.encodeComment(encoder, headComment(node))
 }
 
-func (e *xmlEncoder) encodeEnd(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
+func (e *xmlEncoder) encodeEnd(encoder *xml.Encoder, node *CandidateNode, start xml.StartElement) error {
 	err := encoder.EncodeToken(start.End())
 	if err != nil {
 		return err
 	}
-	return e.encodeComment(encoder, node.CleanFootComment())
+	return e.encodeComment(encoder, footComment(node))
 }
 
-func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
+func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *CandidateNode, start xml.StartElement) error {
 	switch node.Kind {
-	case yqlib.MappingNode:
+	case MappingNode:
 		return e.encodeMap(encoder, node, start)
-	case yqlib.SequenceNode:
+	case SequenceNode:
 		return e.encodeArray(encoder, node, start)
-	case yqlib.ScalarNode:
+	case ScalarNode:
 		err := e.encodeStart(encoder, node, start)
 		if err != nil {
 			return err
@@ -194,7 +192,7 @@ func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *yqlib.CandidateNode, s
 			return err
 		}
 
-		if err = e.encodeComment(encoder, node.CleanLineComment()); err != nil {
+		if err = e.encodeComment(encoder, lineComment(node)); err != nil {
 			return err
 		}
 
@@ -209,13 +207,13 @@ var chompRegexp = regexp.MustCompile(`\n$`)
 
 func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) error {
 	if commentStr != "" {
-		yqlib.GetLogger().Debugf("got comment [%v]", commentStr)
+		log.Debugf("got comment [%v]", commentStr)
 		// multi line string
 		if len(commentStr) > 2 && strings.Contains(commentStr[1:len(commentStr)-1], "\n") {
 			commentStr = chompRegexp.ReplaceAllString(commentStr, "")
-			yqlib.GetLogger().Debugf("chompRegexp [%v]", commentStr)
+			log.Debugf("chompRegexp [%v]", commentStr)
 			commentStr = xmlEncodeMultilineCommentRegex.ReplaceAllString(commentStr, "$1$2")
-			yqlib.GetLogger().Debugf("processed multiline [%v]", commentStr)
+			log.Debugf("processed multiline [%v]", commentStr)
 			// if the first line is non blank, add a space
 			if commentStr[0] != '\n' && commentStr[0] != ' ' {
 				commentStr = " " + commentStr
@@ -227,9 +225,9 @@ func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) erro
 
 		if !strings.HasSuffix(commentStr, " ") && !strings.HasSuffix(commentStr, "\n") {
 			commentStr = commentStr + " "
-			yqlib.GetLogger().Debugf("added suffix [%v]", commentStr)
+			log.Debugf("added suffix [%v]", commentStr)
 		}
-		yqlib.GetLogger().Debugf("encoding comment [%v]", commentStr)
+		log.Debugf("encoding comment [%v]", commentStr)
 
 		var comment xml.Comment = []byte(commentStr)
 		err := encoder.EncodeToken(comment)
@@ -240,9 +238,9 @@ func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) erro
 	return nil
 }
 
-func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
+func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *CandidateNode, start xml.StartElement) error {
 
-	if err := e.encodeComment(encoder, node.CleanHeadAndLineComment()); err != nil {
+	if err := e.encodeComment(encoder, headAndLineComment(node)); err != nil {
 		return err
 	}
 
@@ -252,7 +250,7 @@ func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *yqlib.CandidateNode
 			return err
 		}
 	}
-	return e.encodeComment(encoder, node.CleanFootComment())
+	return e.encodeComment(encoder, footComment(node))
 }
 
 func (e *xmlEncoder) isAttribute(name string) bool {
@@ -262,8 +260,8 @@ func (e *xmlEncoder) isAttribute(name string) bool {
 		!strings.HasPrefix(name, e.prefs.ProcInstPrefix)
 }
 
-func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
-	yqlib.GetLogger().Debug("its a map")
+func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *CandidateNode, start xml.StartElement) error {
+	log.Debug("its a map")
 
 	//first find all the attributes and put them on the start token
 	for i := 0; i < len(node.Content); i += 2 {
@@ -271,7 +269,7 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yqlib.CandidateNode,
 		value := node.Content[i+1]
 
 		if e.isAttribute(key.Value) {
-			if value.Kind == yqlib.ScalarNode {
+			if value.Kind == ScalarNode {
 				attributeName := strings.Replace(key.Value, e.prefs.AttributePrefix, "", 1)
 				start.Attr = append(start.Attr, xml.Attr{Name: xml.Name{Local: attributeName}, Value: value.Value})
 			} else {
@@ -290,7 +288,7 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yqlib.CandidateNode,
 		key := node.Content[i]
 		value := node.Content[i+1]
 
-		err := e.encodeComment(encoder, key.CleanHeadAndLineComment())
+		err := e.encodeComment(encoder, headAndLineComment(key))
 		if err != nil {
 			return err
 		}
@@ -307,7 +305,7 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yqlib.CandidateNode,
 			}
 		} else if key.Value == e.prefs.ContentName {
 			// directly encode the contents
-			err = e.encodeComment(encoder, value.CleanHeadAndLineComment())
+			err = e.encodeComment(encoder, headAndLineComment(value))
 			if err != nil {
 				return err
 			}
@@ -316,7 +314,7 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yqlib.CandidateNode,
 			if err != nil {
 				return err
 			}
-			err = e.encodeComment(encoder, value.CleanFootComment())
+			err = e.encodeComment(encoder, footComment(value))
 			if err != nil {
 				return err
 			}
@@ -327,7 +325,7 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yqlib.CandidateNode,
 				return err
 			}
 		}
-		err = e.encodeComment(encoder, key.CleanFootComment())
+		err = e.encodeComment(encoder, footComment(key))
 		if err != nil {
 			return err
 		}

pkg/yqlib/encoder_yaml.go

@@ -12,11 +12,16 @@ import (
 )
 
 type yamlEncoder struct {
-	prefs YamlPreferences
+	indent   int
+	colorise bool
+	prefs    YamlPreferences
 }
 
-func NewYamlEncoder(prefs YamlPreferences) Encoder {
-	return &yamlEncoder{prefs}
+func NewYamlEncoder(indent int, colorise bool, prefs YamlPreferences) Encoder {
+	if indent < 0 {
+		indent = 0
+	}
+	return &yamlEncoder{indent, colorise, prefs}
 }
 
 func (ye *yamlEncoder) CanHandleAliases() bool {
@@ -25,8 +30,8 @@ func (ye *yamlEncoder) CanHandleAliases() bool {
 
 func (ye *yamlEncoder) PrintDocumentSeparator(writer io.Writer) error {
 	if ye.prefs.PrintDocSeparators {
-		log.Debug("writing doc sep")
-		if err := WriteString(writer, "---\n"); err != nil {
+		log.Debug("-- writing doc sep")
+		if err := writeString(writer, "---\n"); err != nil {
 			return err
 		}
 	}
@@ -54,7 +59,7 @@ func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) err
 			if len(readline) > 0 && readline != "\n" && readline[0] != '%' && !commentLineRegEx.MatchString(readline) {
 				readline = "# " + readline
 			}
-			if err := WriteString(writer, readline); err != nil {
+			if err := writeString(writer, readline); err != nil {
 				return err
 			}
 		}
@@ -62,7 +67,7 @@ func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) err
 		if errors.Is(errReading, io.EOF) {
 			if readline != "" {
 				// the last comment we read didn't have a newline, put one in
-				if err := WriteString(writer, "\n"); err != nil {
+				if err := writeString(writer, "\n"); err != nil {
 					return err
 				}
 			}
@@ -80,18 +85,18 @@ func (ye *yamlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 		if node.LeadingContent == "" || valueToPrint != "" {
 			valueToPrint = valueToPrint + "\n"
 		}
-		return WriteString(writer, valueToPrint)
+		return writeString(writer, valueToPrint)
 	}
 
 	destination := writer
 	tempBuffer := bytes.NewBuffer(nil)
-	if ye.prefs.ColorsEnabled {
+	if ye.colorise {
 		destination = tempBuffer
 	}
 
 	var encoder = yaml.NewEncoder(destination)
 
-	encoder.SetIndent(ye.prefs.Indent)
+	encoder.SetIndent(ye.indent)
 
 	target, err := node.MarshalYAML()
 
@@ -110,7 +115,7 @@ func (ye *yamlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 		return err
 	}
 
-	if ye.prefs.ColorsEnabled {
+	if ye.colorise {
 		return colorizeAndPrint(tempBuffer.Bytes(), writer)
 	}
 	return nil

pkg/yqlib/format.go

@@ -1,205 +0,0 @@
-package yqlib
-
-import (
-	"fmt"
-	"strings"
-)
-
-type EncoderFactoryFunction func() Encoder
-type InDocumentEncoderFactoryFunction func(indent int) Encoder
-type DecoderFactoryFunction func() Decoder
-
-type Format struct {
-	FormalName       string
-	Names            []string
-	DefaultExtension string
-	EncoderFactory   EncoderFactoryFunction
-	DecoderFactory   DecoderFactoryFunction
-
-	/**
-	* Like the Encoder Factory, but for encoding content within the document itself.
-	* Should turn off colors and other settings to ensure the content comes out right.
-	* If this function is not configured, it will default to the EncoderFactory.
-	**/
-	InDocumentEncoderFactory InDocumentEncoderFactoryFunction
-}
-
-var YamlFormat = &Format{"yaml", []string{"y", "yml"}, "yml",
-	func() Encoder { return NewYamlEncoder(ConfiguredYamlPreferences) },
-	func() Decoder { return NewYamlDecoder(ConfiguredYamlPreferences) },
-	func(indent int) Encoder {
-		prefs := ConfiguredYamlPreferences.Copy()
-		prefs.Indent = indent
-		prefs.ColorsEnabled = false
-		return NewYamlEncoder(prefs)
-	},
-}
-
-var JSONFormat = &Format{"json", []string{"j"}, "json",
-	func() Encoder { return NewJSONEncoder(ConfiguredJSONPreferences) },
-	func() Decoder { return NewJSONDecoder() },
-	func(indent int) Encoder {
-		prefs := ConfiguredJSONPreferences.Copy()
-		prefs.Indent = indent
-		prefs.ColorsEnabled = false
-		prefs.UnwrapScalar = false
-		return NewJSONEncoder(prefs)
-	},
-}
-
-var CSVFormat = &Format{"csv", []string{"c"}, "csv",
-	func() Encoder { return NewCsvEncoder(ConfiguredCsvPreferences) },
-	func() Decoder { return NewCSVObjectDecoder(ConfiguredCsvPreferences) },
-	nil,
-}
-
-var TSVFormat = &Format{"tsv", []string{"t"}, "tsv",
-	func() Encoder { return NewCsvEncoder(ConfiguredTsvPreferences) },
-	func() Decoder { return NewCSVObjectDecoder(ConfiguredTsvPreferences) },
-	nil,
-}
-
-var Base64Format = &Format{"base64", []string{}, "txt",
-	func() Encoder { return NewBase64Encoder() },
-	func() Decoder { return NewBase64Decoder() },
-	nil,
-}
-
-var UriFormat = &Format{"uri", []string{}, "txt",
-	func() Encoder { return NewUriEncoder() },
-	func() Decoder { return NewUriDecoder() },
-	nil,
-}
-
-var ShFormat = &Format{"", nil, "sh",
-	func() Encoder { return NewShEncoder() },
-	nil,
-	nil,
-}
-
-var TomlFormat = &Format{"toml", []string{}, "toml",
-	func() Encoder { return NewTomlEncoder() },
-	func() Decoder { return NewTomlDecoder() },
-	nil,
-}
-
-var ShellVariablesFormat = &Format{"shell", []string{"s", "sh"}, "sh",
-	func() Encoder { return NewShellVariablesEncoder() },
-	nil,
-	nil,
-}
-
-var LuaFormat = &Format{"lua", []string{"l"}, "lua",
-	func() Encoder { return NewLuaEncoder(ConfiguredLuaPreferences) },
-	func() Decoder { return NewLuaDecoder(ConfiguredLuaPreferences) },
-	nil,
-}
-
-var Formats = []*Format{
-	YamlFormat,
-	JSONFormat,
-	CSVFormat,
-	TSVFormat,
-	Base64Format,
-	UriFormat,
-	ShFormat,
-	TomlFormat,
-	ShellVariablesFormat,
-	LuaFormat,
-}
-
-func RegisterFormat(f *Format) {
-	Formats = append(Formats, f)
-}
-
-func (f *Format) MatchesName(name string) bool {
-	if f.FormalName == name {
-		return true
-	}
-	for _, n := range f.Names {
-		if n == name {
-			return true
-		}
-	}
-	return false
-}
-
-func (f *Format) GetInDocumentEncoder(indent int) Encoder {
-	if f.InDocumentEncoderFactory != nil {
-		return f.InDocumentEncoderFactory(indent)
-	}
-	return f.EncoderFactory()
-}
-
-func FormatStringFromFilename(filename string) string {
-
-	if filename != "" {
-		GetLogger().Debugf("checking file extension '%s' for auto format detection", filename)
-		nPos := strings.LastIndex(filename, ".")
-		if nPos > -1 {
-			format := filename[nPos+1:]
-			GetLogger().Debugf("detected format '%s'", format)
-			return format
-		}
-	}
-
-	GetLogger().Debugf("using default inputFormat 'yaml'")
-	return "yaml"
-}
-
-func FormatFromString(format string) (*Format, error) {
-	for _, printerFormat := range Formats {
-		if printerFormat.MatchesName(format) {
-			return printerFormat, nil
-		}
-	}
-	return nil, fmt.Errorf("unknown format '%v' please use [%v]", format, GetAvailableOutputFormatString())
-}
-
-func GetAvailableOutputFormats() []*Format {
-	var formats = []*Format{}
-	for _, printerFormat := range Formats {
-		if printerFormat.EncoderFactory != nil {
-			formats = append(formats, printerFormat)
-		}
-	}
-	return formats
-}
-
-func GetAvailableOutputFormatString() string {
-	var formats = []string{}
-	for _, printerFormat := range GetAvailableOutputFormats() {
-
-		if printerFormat.FormalName != "" {
-			formats = append(formats, printerFormat.FormalName)
-		}
-		if len(printerFormat.Names) >= 1 {
-			formats = append(formats, printerFormat.Names[0])
-		}
-	}
-	return strings.Join(formats, "|")
-}
-
-func GetAvailableInputFormats() []*Format {
-	var formats = []*Format{}
-	for _, printerFormat := range Formats {
-		if printerFormat.DecoderFactory != nil {
-			formats = append(formats, printerFormat)
-		}
-	}
-	return formats
-}
-
-func GetAvailableInputFormatString() string {
-	var formats = []string{}
-	for _, printerFormat := range GetAvailableInputFormats() {
-
-		if printerFormat.FormalName != "" {
-			formats = append(formats, printerFormat.FormalName)
-		}
-		if len(printerFormat.Names) >= 1 {
-			formats = append(formats, printerFormat.Names[0])
-		}
-	}
-	return strings.Join(formats, "|")
-}

pkg/yqlib/formatting_expressions_test.go

@@ -1,91 +0,0 @@
-package yqlib
-
-import (
-	"bufio"
-	"fmt"
-	"strings"
-	"testing"
-
-	"github.com/mikefarah/yq/v4/test"
-)
-
-var formattingExpressionScenarios = []FormatScenario{
-	{
-		description: "Using expression files and comments",
-		skipDoc:     true,
-		input:       "a:\n  b: old",
-		expression:  "#! yq\n\n# This is a yq expression that updates the map\n# for several great reasons outlined here.\n\n.a.b = \"new\" # line comment here\n| .a.c = \"frog\"\n\n# Now good things will happen.\n",
-		expected:    "a:\n  b: new\n  c: frog\n",
-	},
-	{
-		description:    "Using expression files and comments",
-		subdescription: "Note that you can execute the file directly - but make sure you make the expression file executable.",
-		input:          "a:\n  b: old",
-		expression:     "#! yq\n\n# This is a yq expression that updates the map\n# for several great reasons outlined here.\n\n.a.b = \"new\" # line comment here\n| .a.c = \"frog\"\n\n# Now good things will happen.\n",
-		expected:       "a:\n  b: new\n  c: frog\n",
-		scenarioType:   "shebang",
-	},
-	{
-		description:    "Flags in expression files",
-		subdescription: "You can specify flags on the shebang line, this only works when executing the file directly.",
-		input:          "a:\n  b: old",
-		expression:     "#! yq -oj\n\n# This is a yq expression that updates the map\n# for several great reasons outlined here.\n\n.a.b = \"new\" # line comment here\n| .a.c = \"frog\"\n\n# Now good things will happen.\n",
-		expected:       "a:\n  b: new\n  c: frog\n",
-		scenarioType:   "shebang-json",
-	},
-	{
-		description:    "Commenting out yq expressions",
-		subdescription: "Note that `c` is no longer set to 'frog'. In this example we're calling yq directly and passing the expression file into `--from-file`, this is no different from executing the expression file directly.",
-		input:          "a:\n  b: old",
-		expression:     "#! yq\n# This is a yq expression that updates the map\n# for several great reasons outlined here.\n\n.a.b = \"new\" # line comment here\n# | .a.c = \"frog\"\n\n# Now good things will happen.\n",
-		expected:       "a:\n  b: new\n",
-	},
-}
-
-func documentExpressionScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
-	s := i.(FormatScenario)
-
-	if s.skipDoc {
-		return
-	}
-	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
-
-	if s.subdescription != "" {
-		writeOrPanic(w, s.subdescription)
-		writeOrPanic(w, "\n\n")
-	}
-
-	writeOrPanic(w, "Given a sample.yaml file of:\n")
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input))
-
-	writeOrPanic(w, "And an 'update.yq' expression file of:\n")
-	writeOrPanic(w, fmt.Sprintf("```bash\n%v```\n", s.expression))
-
-	writeOrPanic(w, "then\n")
-	if strings.HasPrefix(s.scenarioType, "shebang") {
-		writeOrPanic(w, "```bash\n./update.yq sample.yaml\n```\n")
-	} else {
-		writeOrPanic(w, "```bash\nyq --from-file update.yq sample.yml\n```\n")
-	}
-	writeOrPanic(w, "will output\n")
-	encoder := NewYamlEncoder(ConfiguredYamlPreferences)
-
-	if s.scenarioType == "shebang-json" {
-		encoder = NewJSONEncoder(ConfiguredJSONPreferences)
-	}
-
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), encoder)))
-}
-
-func TestExpressionCommentScenarios(t *testing.T) {
-	for _, tt := range formattingExpressionScenarios {
-		test.AssertResultComplexWithContext(t, tt.expected,
-			mustProcessFormatScenario(tt, NewYamlDecoder(ConfiguredYamlPreferences), NewYamlEncoder(ConfiguredYamlPreferences)),
-			tt.description)
-	}
-	genericScenarios := make([]interface{}, len(formattingExpressionScenarios))
-	for i, s := range formattingExpressionScenarios {
-		genericScenarios[i] = s
-	}
-	documentScenarios(t, "usage", "formatting-expressions", genericScenarios, documentExpressionScenario)
-}

pkg/yqlib/goccy_yaml_test.go

@@ -6,7 +6,7 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
-var goccyYamlFormatScenarios = []FormatScenario{
+var goccyYamlFormatScenarios = []formatScenario{
 	{
 		description: "basic - 3",
 		skipDoc:     true,
@@ -159,8 +159,8 @@ var goccyYamlFormatScenarios = []FormatScenario{
 	// },
 }
 
-func testGoccyYamlScenario(t *testing.T, s FormatScenario) {
-	test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewGoccyYAMLDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
+func testGoccyYamlScenario(t *testing.T, s formatScenario) {
+	test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewGoccyYAMLDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
 }
 
 func TestGoccyYmlFormatScenarios(t *testing.T) {

pkg/yqlib/json.go

@@ -1,25 +0,0 @@
-package yqlib
-
-type JsonPreferences struct {
-	Indent        int
-	ColorsEnabled bool
-	UnwrapScalar  bool
-}
-
-func NewDefaultJsonPreferences() JsonPreferences {
-	return JsonPreferences{
-		Indent:        2,
-		ColorsEnabled: true,
-		UnwrapScalar:  true,
-	}
-}
-
-func (p *JsonPreferences) Copy() JsonPreferences {
-	return JsonPreferences{
-		Indent:        p.Indent,
-		ColorsEnabled: p.ColorsEnabled,
-		UnwrapScalar:  p.UnwrapScalar,
-	}
-}
-
-var ConfiguredJSONPreferences = NewDefaultJsonPreferences()

pkg/yqlib/json_test.go

@@ -79,37 +79,12 @@ const roundTripMultiLineJson = `{
 }
 `
 
-var jsonScenarios = []FormatScenario{
-	{
-		description:  "array empty",
-		skipDoc:      true,
-		input:        "[]",
-		scenarioType: "roundtrip-ndjson",
-		indent:       0,
-		expected:     "[]\n",
-	},
-	{
-		description:  "array has scalar",
-		skipDoc:      true,
-		input:        "[3]",
-		scenarioType: "roundtrip-ndjson",
-		indent:       0,
-		expected:     "[3]\n",
-	},
-	{
-		description:  "array has object",
-		skipDoc:      true,
-		input:        `[{"x": 3}]`,
-		scenarioType: "roundtrip-ndjson",
-		indent:       0,
-		expected:     "[{\"x\":3}]\n",
-	},
+var jsonScenarios = []formatScenario{
 	{
 		description:  "array null",
 		skipDoc:      true,
 		input:        "[null]",
 		scenarioType: "roundtrip-ndjson",
-		indent:       0,
 		expected:     "[null]\n",
 	},
 	{
@@ -118,7 +93,6 @@ var jsonScenarios = []FormatScenario{
 		input:        "[{}]",
 		expression:   `[.. | type]`,
 		scenarioType: "roundtrip-ndjson",
-		indent:       0,
 		expected:     "[\"!!seq\",\"!!map\"]\n",
 	},
 	{
@@ -221,15 +195,15 @@ var jsonScenarios = []FormatScenario{
 		scenarioType:   "encode",
 	},
 	{
-		description:  "Roundtrip JSON Lines / NDJSON",
-		input:        sampleNdJson,
-		expected:     expectedRoundTripSampleNdJson,
-		scenarioType: "roundtrip-ndjson",
-		indent:       0,
+		description:    "Roundtrip NDJSON",
+		subdescription: "Unfortunately the json encoder strips leading spaces of values.",
+		input:          sampleNdJson,
+		expected:       expectedRoundTripSampleNdJson,
+		scenarioType:   "roundtrip-ndjson",
 	},
 	{
 		description:    "Roundtrip multi-document JSON",
-		subdescription: "The parser can also handle multiple multi-line json documents in a single file (despite this not being in the JSON Lines / NDJSON spec). Typically you would have one entire JSON document per line, but the parser also supports multiple multi-line json documents",
+		subdescription: "The NDJSON parser can also handle multiple multi-line json documents in a single file!",
 		input:          sampleMultiLineJson,
 		expected:       roundTripMultiLineJson,
 		scenarioType:   "roundtrip-multi",
@@ -251,13 +225,13 @@ var jsonScenarios = []FormatScenario{
 		scenarioType:   "roundtrip-ndjson",
 	},
 	{
-		description:  "Decode JSON Lines / NDJSON",
+		description:  "Decode NDJSON",
 		input:        sampleNdJson,
 		expected:     expectedNdJsonYaml,
 		scenarioType: "decode-ndjson",
 	},
 	{
-		description:  "Decode JSON Lines / NDJSON, maintain key order",
+		description:  "Decode NDJSON, maintain key order",
 		skipDoc:      true,
 		input:        sampleNdJsonKey,
 		expected:     expectedJsonKeysInOrder,
@@ -307,7 +281,7 @@ var jsonScenarios = []FormatScenario{
 	},
 }
 
-func documentRoundtripNdJsonScenario(w *bufio.Writer, s FormatScenario, indent int) {
+func documentRoundtripNdJsonScenario(w *bufio.Writer, s formatScenario, indent int) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -328,14 +302,11 @@ func documentRoundtripNdJsonScenario(w *bufio.Writer, s FormatScenario, indent i
 	}
 
 	writeOrPanic(w, "will output\n")
-	prefs := ConfiguredJSONPreferences.Copy()
-	prefs.Indent = indent
-	prefs.UnwrapScalar = false
 
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(prefs))))
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(indent, false, false))))
 }
 
-func documentDecodeNdJsonScenario(w *bufio.Writer, s FormatScenario) {
+func documentDecodeNdJsonScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -357,7 +328,7 @@ func documentDecodeNdJsonScenario(w *bufio.Writer, s FormatScenario) {
 
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(ConfiguredYamlPreferences))))
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))))
 }
 
 func decodeJSON(t *testing.T, jsonString string) *CandidateNode {
@@ -385,24 +356,20 @@ func decodeJSON(t *testing.T, jsonString string) *CandidateNode {
 	return context.MatchingNodes.Front().Value.(*CandidateNode)
 }
 
-func testJSONScenario(t *testing.T, s FormatScenario) {
-	prefs := ConfiguredJSONPreferences.Copy()
-	prefs.Indent = s.indent
-	prefs.UnwrapScalar = false
+func testJSONScenario(t *testing.T, s formatScenario) {
 	switch s.scenarioType {
 	case "encode":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(prefs)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false, false)), s.description)
 	case "decode":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(prefs)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(s.indent, false, false)), s.description)
 	case "decode-ndjson":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
 	case "roundtrip-ndjson":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(prefs)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(0, false, false)), s.description)
 	case "roundtrip-multi":
-		prefs.Indent = 2
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(prefs)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(2, false, false)), s.description)
 	case "decode-error":
-		result, err := processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(prefs))
+		result, err := processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(2, false, false))
 		if err == nil {
 			t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result)
 		} else {
@@ -413,7 +380,7 @@ func testJSONScenario(t *testing.T, s FormatScenario) {
 	}
 }
 
-func documentJSONDecodeScenario(t *testing.T, w *bufio.Writer, s FormatScenario) {
+func documentJSONDecodeScenario(t *testing.T, w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -429,7 +396,7 @@ func documentJSONDecodeScenario(t *testing.T, w *bufio.Writer, s FormatScenario)
 	writeOrPanic(w, "will output\n")
 
 	var output bytes.Buffer
-	printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), true, 2, true)
+	printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
 
 	node := decodeJSON(t, s.input)
 
@@ -443,7 +410,7 @@ func documentJSONDecodeScenario(t *testing.T, w *bufio.Writer, s FormatScenario)
 }
 
 func documentJSONScenario(t *testing.T, w *bufio.Writer, i interface{}) {
-	s := i.(FormatScenario)
+	s := i.(formatScenario)
 	if s.skipDoc {
 		return
 	}
@@ -464,7 +431,7 @@ func documentJSONScenario(t *testing.T, w *bufio.Writer, i interface{}) {
 	}
 }
 
-func documentJSONEncodeScenario(w *bufio.Writer, s FormatScenario) {
+func documentJSONEncodeScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -488,11 +455,8 @@ func documentJSONEncodeScenario(w *bufio.Writer, s FormatScenario) {
 		writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=json -I=%v '%v' sample.yml\n```\n", s.indent, expression))
 	}
 	writeOrPanic(w, "will output\n")
-	prefs := ConfiguredJSONPreferences.Copy()
-	prefs.Indent = s.indent
-	prefs.UnwrapScalar = false
 
-	writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(prefs))))
+	writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false, false))))
 }
 
 func TestJSONScenarios(t *testing.T) {

pkg/yqlib/lexer.go

@@ -51,8 +51,9 @@ func (t *token) toString(detail bool) string {
 	} else if t.TokenType == traverseArrayCollect {
 		return ".["
 
+	} else {
+		return "NFI"
 	}
-	return "NFI"
 }
 
 func unwrap(value string) string {
@@ -111,11 +112,11 @@ func handleToken(tokens []*token, index int, postProcessedTokens []*token) (toke
 		//need to put a traverse array then a collect currentToken
 		// do this by adding traverse then converting currentToken to collect
 
-		log.Debug("adding self")
+		log.Debug("  adding self")
 		op := &Operation{OperationType: selfReferenceOpType, StringValue: "SELF"}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 
-		log.Debug("adding traverse array")
+		log.Debug("  adding traverse array")
 		op = &Operation{OperationType: traverseArrayOpType, StringValue: "TRAVERSE_ARRAY"}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 
@@ -135,13 +136,13 @@ func handleToken(tokens []*token, index int, postProcessedTokens []*token) (toke
 
 	if index != len(tokens)-1 && currentToken.AssignOperation != nil &&
 		tokenIsOpType(tokens[index+1], assignOpType) {
-		log.Debug("its an update assign")
+		log.Debug("  its an update assign")
 		currentToken.Operation = currentToken.AssignOperation
 		currentToken.Operation.UpdateAssign = tokens[index+1].Operation.UpdateAssign
 		skipNextToken = true
 	}
 
-	log.Debug("adding token to the fixed list")
+	log.Debug("  adding token to the fixed list")
 	postProcessedTokens = append(postProcessedTokens, currentToken)
 
 	if tokenIsOpType(currentToken, createMapOpType) {
@@ -158,7 +159,7 @@ func handleToken(tokens []*token, index int, postProcessedTokens []*token) (toke
 	if index != len(tokens)-1 &&
 		((currentToken.TokenType == openCollect && tokens[index+1].TokenType == closeCollect) ||
 			(currentToken.TokenType == openCollectObject && tokens[index+1].TokenType == closeCollectObject)) {
-		log.Debug("adding empty")
+		log.Debug("  adding empty")
 		op := &Operation{OperationType: emptyOpType, StringValue: "EMPTY"}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 	}
@@ -167,14 +168,14 @@ func handleToken(tokens []*token, index int, postProcessedTokens []*token) (toke
 
 		(tokenIsOpType(tokens[index+1], traversePathOpType) ||
 			(tokens[index+1].TokenType == traverseArrayCollect)) {
-		log.Debug("adding pipe because the next thing is traverse")
+		log.Debug("  adding pipe because the next thing is traverse")
 		op := &Operation{OperationType: shortPipeOpType, Value: "PIPE", StringValue: "."}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 	}
 	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
 		tokens[index+1].TokenType == openCollect {
 
-		log.Debug("adding traverseArray because next is opencollect")
+		log.Debug("  adding traverseArray because next is opencollect")
 		op := &Operation{OperationType: traverseArrayOpType}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 	}

pkg/yqlib/lexer_participle.go

@@ -7,7 +7,7 @@ import (
 	"github.com/alecthomas/participle/v2/lexer"
 )
 
-var participleYqRules = []*ParticipleYqRule{
+var participleYqRules = []*participleYqRule{
 	{"LINE_COMMENT", `line_?comment|lineComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true}), 0},
 	{"HEAD_COMMENT", `head_?comment|headComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{HeadComment: true}), 0},
 	{"FOOT_COMMENT", `foot_?comment|footComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{FootComment: true}), 0},
@@ -57,34 +57,45 @@ var participleYqRules = []*ParticipleYqRule{
 
 	{"ArrayToMap", "array_?to_?map", expressionOpToken(`(.[] | select(. != null) ) as $i ireduce({}; .[$i | key] = $i)`), 0},
 
-	{"YamlEncodeWithIndent", `to_?yaml\([0-9]+\)`, CreateEncodeYqActionParsingIndent(YamlFormat), 0},
+	{"YamlEncodeWithIndent", `to_?yaml\([0-9]+\)`, encodeParseIndent(YamlOutputFormat), 0},
+	{"XMLEncodeWithIndent", `to_?xml\([0-9]+\)`, encodeParseIndent(XMLOutputFormat), 0},
+	{"JSONEncodeWithIndent", `to_?json\([0-9]+\)`, encodeParseIndent(JSONOutputFormat), 0},
 
-	{"JSONEncodeWithIndent", `to_?json\([0-9]+\)`, CreateEncodeYqActionParsingIndent(JSONFormat), 0},
+	{"YamlDecode", `from_?yaml|@yamld|from_?json|@jsond`, decodeOp(YamlInputFormat), 0},
+	{"YamlEncode", `to_?yaml|@yaml`, encodeWithIndent(YamlOutputFormat, 2), 0},
 
-	{"YamlDecode", `from_?yaml|@yamld|from_?json|@jsond`, CreateDecodeOpYqAction(YamlFormat), 0},
-	{"YamlEncode", `to_?yaml|@yaml`, CreateEncodeOpYqAction(YamlFormat, 2), 0},
+	{"JSONEncode", `to_?json`, encodeWithIndent(JSONOutputFormat, 2), 0},
+	{"JSONEncodeNoIndent", `@json`, encodeWithIndent(JSONOutputFormat, 0), 0},
 
-	{"JSONEncode", `to_?json`, CreateEncodeOpYqAction(JSONFormat, 2), 0},
-	{"JSONEncodeNoIndent", `@json`, CreateEncodeOpYqAction(JSONFormat, 0), 0},
+	{"PropertiesDecode", `from_?props|@propsd`, decodeOp(PropertiesInputFormat), 0},
+	{"PropsEncode", `to_?props|@props`, encodeWithIndent(PropsOutputFormat, 2), 0},
 
-	{"CSVDecode", `from_?csv|@csvd`, CreateDecodeOpYqAction(CSVFormat), 0},
-	{"CSVEncode", `to_?csv|@csv`, CreateEncodeOpYqAction(CSVFormat, 0), 0},
+	{"XmlDecode", `from_?xml|@xmld`, decodeOp(XMLInputFormat), 0},
+	{"XMLEncode", `to_?xml`, encodeWithIndent(XMLOutputFormat, 2), 0},
+	{"XMLEncodeNoIndent", `@xml`, encodeWithIndent(XMLOutputFormat, 0), 0},
 
-	{"TSVDecode", `from_?tsv|@tsvd`, CreateDecodeOpYqAction(TSVFormat), 0},
-	{"TSVEncode", `to_?tsv|@tsv`, CreateEncodeOpYqAction(TSVFormat, 0), 0},
+	{"CSVDecode", `from_?csv|@csvd`, decodeOp(CSVObjectInputFormat), 0},
+	{"CSVEncode", `to_?csv|@csv`, encodeWithIndent(CSVOutputFormat, 0), 0},
 
-	{"Base64d", `@base64d`, CreateDecodeOpYqAction(Base64Format), 0},
-	{"Base64", `@base64`, CreateEncodeOpYqAction(Base64Format, 0), 0},
+	{"TSVDecode", `from_?tsv|@tsvd`, decodeOp(TSVObjectInputFormat), 0},
+	{"TSVEncode", `to_?tsv|@tsv`, encodeWithIndent(TSVOutputFormat, 0), 0},
 
-	{"Urid", `@urid`, CreateDecodeOpYqAction(UriFormat), 0},
-	{"Uri", `@uri`, CreateEncodeOpYqAction(UriFormat, 0), 0},
-	{"SH", `@sh`, CreateEncodeOpYqAction(ShFormat, 0), 0},
+	{"Base64d", `@base64d`, decodeOp(Base64InputFormat), 0},
+	{"Base64", `@base64`, encodeWithIndent(Base64OutputFormat, 0), 0},
 
-	{"LoadBase64", `load_?base64`, CreateLoadOpYqAction(NewBase64Decoder()), 0},
+	{"Urid", `@urid`, decodeOp(UriInputFormat), 0},
+	{"Uri", `@uri`, encodeWithIndent(UriOutputFormat, 0), 0},
+	{"SH", `@sh`, encodeWithIndent(ShOutputFormat, 0), 0},
 
-	simpleOp(`load_?str|str_?load`, loadStringOpType),
+	{"LoadXML", `load_?xml|xml_?load`, loadOp(NewXMLDecoder(ConfiguredXMLPreferences), false), 0},
 
-	{"LoadYaml", `load`, CreateLoadOpYqAction(NewYamlDecoder(LoadYamlPreferences)), 0},
+	{"LoadBase64", `load_?base64`, loadOp(NewBase64Decoder(), false), 0},
+
+	{"LoadProperties", `load_?props`, loadOp(NewPropertiesDecoder(), false), 0},
+
+	{"LoadString", `load_?str|str_?load`, loadOp(nil, true), 0},
+
+	{"LoadYaml", `load`, loadOp(NewYamlDecoder(LoadYamlPreferences), false), 0},
 
 	{"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0},
 
@@ -119,9 +130,7 @@ var participleYqRules = []*ParticipleYqRule{
 
 	simpleOp("contains", containsOpType),
 	simpleOp("split", splitStringOpType),
-
-	{"ParentWithLevel", `parent\([0-9]+\)`, parentWithLevel(), 0},
-	{"ParentWithDefaultLevel", `parent`, parentWithDefaultLevel(), 0},
+	simpleOp("parent", getParentOpType),
 
 	simpleOp("keys", keysOpType),
 	simpleOp("key", getKeyOpType),
@@ -159,7 +168,6 @@ var participleYqRules = []*ParticipleYqRule{
 	{"Uppercase", `upcase|ascii_?upcase`, opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: true}), 0},
 	{"Downcase", `downcase|ascii_?downcase`, opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: false}), 0},
 	simpleOp("trim", trimOpType),
-	simpleOp("to_?string", toStringOpType),
 
 	{"HexValue", `0[xX][0-9A-Fa-f]+`, hexValue(), 0},
 	{"FloatValueScientific", `-?[1-9](\.\d+)?[Ee][-+]?\d+`, floatValue(), 0},
@@ -213,12 +221,11 @@ var participleYqRules = []*ParticipleYqRule{
 
 	{"SubtractAssign", `\-=`, opToken(subtractAssignOpType), 0},
 	{"Subtract", `\-`, opToken(subtractOpType), 0},
-	{"Comment", `#.*`, nil, 0},
 }
 
 type yqAction func(lexer.Token) (*token, error)
 
-type ParticipleYqRule struct {
+type participleYqRule struct {
 	Name                string
 	Pattern             string
 	CreateYqToken       yqAction
@@ -229,16 +236,12 @@ type participleLexer struct {
 	lexerDefinition lexer.StringDefinition
 }
 
-func simpleOp(name string, opType *operationType) *ParticipleYqRule {
-	return &ParticipleYqRule{strings.ToUpper(string(name[1])) + name[1:], name, opToken(opType), 0}
+func simpleOp(name string, opType *operationType) *participleYqRule {
+	return &participleYqRule{strings.ToUpper(string(name[1])) + name[1:], name, opToken(opType), 0}
 }
 
-func assignableOp(name string, opType *operationType, assignOpType *operationType) *ParticipleYqRule {
-	return &ParticipleYqRule{strings.ToUpper(string(name[1])) + name[1:], name, opTokenWithPrefs(opType, assignOpType, nil), 0}
-}
-
-func RegisterRules(rules []*ParticipleYqRule) {
-	participleYqRules = append(participleYqRules, rules...)
+func assignableOp(name string, opType *operationType, assignOpType *operationType) *participleYqRule {
+	return &participleYqRule{strings.ToUpper(string(name[1])) + name[1:], name, opTokenWithPrefs(opType, assignOpType, nil), 0}
 }
 
 func newParticipleLexer() expressionTokeniser {
@@ -295,7 +298,7 @@ func opTokenWithPrefs(opType *operationType, assignOpType *operationType, prefer
 		if assignOpType != nil {
 			assign = &Operation{OperationType: assignOpType, Value: assignOpType.Type, StringValue: value, Preferences: preferences}
 		}
-		return &token{TokenType: operationToken, Operation: op, AssignOperation: assign, CheckForPostTraverse: op.OperationType.CheckForPostTraverse}, nil
+		return &token{TokenType: operationToken, Operation: op, AssignOperation: assign}, nil
 	}
 }
 
@@ -369,11 +372,7 @@ func stringValue() yqAction {
 		value = strings.ReplaceAll(value, "\\\"", "\"")
 		value = strings.ReplaceAll(value, "\\n", "\n")
 		log.Debug("replaced: %v", value)
-		return &token{TokenType: operationToken, Operation: &Operation{
-			OperationType: stringInterpolationOpType,
-			StringValue:   value,
-			Value:         value,
-		}}, nil
+		return &token{TokenType: operationToken, Operation: createValueOperation(value, value)}, nil
 	}
 }
 
@@ -496,29 +495,7 @@ func numberValue() yqAction {
 	}
 }
 
-func parentWithLevel() yqAction {
-	return func(rawToken lexer.Token) (*token, error) {
-		value := rawToken.Value
-		var level, errParsingInt = extractNumberParameter(value)
-		if errParsingInt != nil {
-			return nil, errParsingInt
-		}
-
-		prefs := parentOpPreferences{Level: level}
-		op := &Operation{OperationType: getParentOpType, Value: getParentOpType.Type, StringValue: value, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func parentWithDefaultLevel() yqAction {
-	return func(rawToken lexer.Token) (*token, error) {
-		prefs := parentOpPreferences{Level: 1}
-		op := &Operation{OperationType: getParentOpType, Value: getParentOpType.Type, StringValue: getParentOpType.Type, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func CreateEncodeYqActionParsingIndent(outputFormat *Format) yqAction {
+func encodeParseIndent(outputFormat PrinterOutputFormat) yqAction {
 	return func(rawToken lexer.Token) (*token, error) {
 		value := rawToken.Value
 		var indent, errParsingInt = extractNumberParameter(value)
@@ -532,18 +509,18 @@ func CreateEncodeYqActionParsingIndent(outputFormat *Format) yqAction {
 	}
 }
 
-func CreateEncodeOpYqAction(outputFormat *Format, indent int) yqAction {
+func encodeWithIndent(outputFormat PrinterOutputFormat, indent int) yqAction {
 	prefs := encoderPreferences{format: outputFormat, indent: indent}
 	return opTokenWithPrefs(encodeOpType, nil, prefs)
 }
 
-func CreateDecodeOpYqAction(format *Format) yqAction {
-	prefs := decoderPreferences{format: format}
+func decodeOp(inputFormat InputFormat) yqAction {
+	prefs := decoderPreferences{format: inputFormat}
 	return opTokenWithPrefs(decodeOpType, nil, prefs)
 }
 
-func CreateLoadOpYqAction(decoder Decoder) yqAction {
-	prefs := loadPrefs{decoder}
+func loadOp(decoder Decoder, loadAsString bool) yqAction {
+	prefs := loadPrefs{decoder: decoder, loadAsString: loadAsString}
 	return opTokenWithPrefs(loadOpType, nil, prefs)
 }
 
@@ -557,13 +534,13 @@ func literalToken(tt tokenType, checkForPost bool) yqAction {
 	}
 }
 
-func (p *participleLexer) getYqDefinition(rawToken lexer.Token) *ParticipleYqRule {
+func (p *participleLexer) getYqDefinition(rawToken lexer.Token) *participleYqRule {
 	for _, yqRule := range participleYqRules {
 		if yqRule.ParticipleTokenType == rawToken.Type {
 			return yqRule
 		}
 	}
-	return &ParticipleYqRule{}
+	return &participleYqRule{}
 }
 
 func (p *participleLexer) Tokenise(expression string) ([]*token, error) {

pkg/yqlib/lib.go

@@ -28,6 +28,167 @@ func GetLogger() *logging.Logger {
 	return log
 }
 
+type operationType struct {
+	Type       string
+	NumArgs    uint // number of arguments to the op
+	Precedence uint
+	Handler    operatorHandler
+}
+
+var orOpType = &operationType{Type: "OR", NumArgs: 2, Precedence: 20, Handler: orOperator}
+var andOpType = &operationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: andOperator}
+var reduceOpType = &operationType{Type: "REDUCE", NumArgs: 2, Precedence: 35, Handler: reduceOperator}
+
+var blockOpType = &operationType{Type: "BLOCK", Precedence: 10, NumArgs: 2, Handler: emptyOperator}
+
+var unionOpType = &operationType{Type: "UNION", NumArgs: 2, Precedence: 10, Handler: unionOperator}
+
+var pipeOpType = &operationType{Type: "PIPE", NumArgs: 2, Precedence: 30, Handler: pipeOperator}
+
+var assignOpType = &operationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: assignUpdateOperator}
+var addAssignOpType = &operationType{Type: "ADD_ASSIGN", NumArgs: 2, Precedence: 40, Handler: addAssignOperator}
+var subtractAssignOpType = &operationType{Type: "SUBTRACT_ASSIGN", NumArgs: 2, Precedence: 40, Handler: subtractAssignOperator}
+
+var assignAttributesOpType = &operationType{Type: "ASSIGN_ATTRIBUTES", NumArgs: 2, Precedence: 40, Handler: assignAttributesOperator}
+var assignStyleOpType = &operationType{Type: "ASSIGN_STYLE", NumArgs: 2, Precedence: 40, Handler: assignStyleOperator}
+var assignVariableOpType = &operationType{Type: "ASSIGN_VARIABLE", NumArgs: 2, Precedence: 40, Handler: useWithPipe}
+var assignTagOpType = &operationType{Type: "ASSIGN_TAG", NumArgs: 2, Precedence: 40, Handler: assignTagOperator}
+var assignCommentOpType = &operationType{Type: "ASSIGN_COMMENT", NumArgs: 2, Precedence: 40, Handler: assignCommentsOperator}
+var assignAnchorOpType = &operationType{Type: "ASSIGN_ANCHOR", NumArgs: 2, Precedence: 40, Handler: assignAnchorOperator}
+var assignAliasOpType = &operationType{Type: "ASSIGN_ALIAS", NumArgs: 2, Precedence: 40, Handler: assignAliasOperator}
+
+var multiplyOpType = &operationType{Type: "MULTIPLY", NumArgs: 2, Precedence: 42, Handler: multiplyOperator}
+var multiplyAssignOpType = &operationType{Type: "MULTIPLY_ASSIGN", NumArgs: 2, Precedence: 42, Handler: multiplyAssignOperator}
+
+var divideOpType = &operationType{Type: "DIVIDE", NumArgs: 2, Precedence: 42, Handler: divideOperator}
+
+var moduloOpType = &operationType{Type: "MODULO", NumArgs: 2, Precedence: 42, Handler: moduloOperator}
+
+var addOpType = &operationType{Type: "ADD", NumArgs: 2, Precedence: 42, Handler: addOperator}
+var subtractOpType = &operationType{Type: "SUBTRACT", NumArgs: 2, Precedence: 42, Handler: subtractOperator}
+var alternativeOpType = &operationType{Type: "ALTERNATIVE", NumArgs: 2, Precedence: 42, Handler: alternativeOperator}
+
+var equalsOpType = &operationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: equalsOperator}
+var notEqualsOpType = &operationType{Type: "NOT_EQUALS", NumArgs: 2, Precedence: 40, Handler: notEqualsOperator}
+
+var compareOpType = &operationType{Type: "COMPARE", NumArgs: 2, Precedence: 40, Handler: compareOperator}
+
+// createmap needs to be above union, as we use union to build the components of the objects
+var createMapOpType = &operationType{Type: "CREATE_MAP", NumArgs: 2, Precedence: 15, Handler: createMapOperator}
+
+var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence: 45, Handler: pipeOperator}
+
+var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: lengthOperator}
+var lineOpType = &operationType{Type: "LINE", NumArgs: 0, Precedence: 50, Handler: lineOperator}
+var columnOpType = &operationType{Type: "LINE", NumArgs: 0, Precedence: 50, Handler: columnOperator}
+
+var expressionOpType = &operationType{Type: "EXP", NumArgs: 0, Precedence: 50, Handler: expressionOperator}
+
+var collectOpType = &operationType{Type: "COLLECT", NumArgs: 1, Precedence: 50, Handler: collectOperator}
+var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 50, Handler: mapOperator}
+var filterOpType = &operationType{Type: "FILTER", NumArgs: 1, Precedence: 50, Handler: filterOperator}
+var errorOpType = &operationType{Type: "ERROR", NumArgs: 1, Precedence: 50, Handler: errorOperator}
+var pickOpType = &operationType{Type: "PICK", NumArgs: 1, Precedence: 50, Handler: pickOperator}
+var evalOpType = &operationType{Type: "EVAL", NumArgs: 1, Precedence: 50, Handler: evalOperator}
+var mapValuesOpType = &operationType{Type: "MAP_VALUES", NumArgs: 1, Precedence: 50, Handler: mapValuesOperator}
+
+var formatDateTimeOpType = &operationType{Type: "FORMAT_DATE_TIME", NumArgs: 1, Precedence: 50, Handler: formatDateTime}
+var withDtFormatOpType = &operationType{Type: "WITH_DATE_TIME_FORMAT", NumArgs: 1, Precedence: 50, Handler: withDateTimeFormat}
+var nowOpType = &operationType{Type: "NOW", NumArgs: 0, Precedence: 50, Handler: nowOp}
+var tzOpType = &operationType{Type: "TIMEZONE", NumArgs: 1, Precedence: 50, Handler: tzOp}
+var fromUnixOpType = &operationType{Type: "FROM_UNIX", NumArgs: 0, Precedence: 50, Handler: fromUnixOp}
+var toUnixOpType = &operationType{Type: "TO_UNIX", NumArgs: 0, Precedence: 50, Handler: toUnixOp}
+
+var encodeOpType = &operationType{Type: "ENCODE", NumArgs: 0, Precedence: 50, Handler: encodeOperator}
+var decodeOpType = &operationType{Type: "DECODE", NumArgs: 0, Precedence: 50, Handler: decodeOperator}
+
+var anyOpType = &operationType{Type: "ANY", NumArgs: 0, Precedence: 50, Handler: anyOperator}
+var allOpType = &operationType{Type: "ALL", NumArgs: 0, Precedence: 50, Handler: allOperator}
+var containsOpType = &operationType{Type: "CONTAINS", NumArgs: 1, Precedence: 50, Handler: containsOperator}
+var anyConditionOpType = &operationType{Type: "ANY_CONDITION", NumArgs: 1, Precedence: 50, Handler: anyOperator}
+var allConditionOpType = &operationType{Type: "ALL_CONDITION", NumArgs: 1, Precedence: 50, Handler: allOperator}
+
+var toEntriesOpType = &operationType{Type: "TO_ENTRIES", NumArgs: 0, Precedence: 50, Handler: toEntriesOperator}
+var fromEntriesOpType = &operationType{Type: "FROM_ENTRIES", NumArgs: 0, Precedence: 50, Handler: fromEntriesOperator}
+var withEntriesOpType = &operationType{Type: "WITH_ENTRIES", NumArgs: 1, Precedence: 50, Handler: withEntriesOperator}
+
+var withOpType = &operationType{Type: "WITH", NumArgs: 1, Precedence: 50, Handler: withOperator}
+
+var splitDocumentOpType = &operationType{Type: "SPLIT_DOC", NumArgs: 0, Precedence: 50, Handler: splitDocumentOperator}
+var getVariableOpType = &operationType{Type: "GET_VARIABLE", NumArgs: 0, Precedence: 55, Handler: getVariableOperator}
+var getStyleOpType = &operationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: getStyleOperator}
+var getTagOpType = &operationType{Type: "GET_TAG", NumArgs: 0, Precedence: 50, Handler: getTagOperator}
+var getKindOpType = &operationType{Type: "GET_KIND", NumArgs: 0, Precedence: 50, Handler: getKindOperator}
+
+var getKeyOpType = &operationType{Type: "GET_KEY", NumArgs: 0, Precedence: 50, Handler: getKeyOperator}
+var isKeyOpType = &operationType{Type: "IS_KEY", NumArgs: 0, Precedence: 50, Handler: isKeyOperator}
+var getParentOpType = &operationType{Type: "GET_PARENT", NumArgs: 0, Precedence: 50, Handler: getParentOperator}
+
+var getCommentOpType = &operationType{Type: "GET_COMMENT", NumArgs: 0, Precedence: 50, Handler: getCommentsOperator}
+var getAnchorOpType = &operationType{Type: "GET_ANCHOR", NumArgs: 0, Precedence: 50, Handler: getAnchorOperator}
+var getAliasOpType = &operationType{Type: "GET_ALIAS", NumArgs: 0, Precedence: 50, Handler: getAliasOperator}
+var getDocumentIndexOpType = &operationType{Type: "GET_DOCUMENT_INDEX", NumArgs: 0, Precedence: 50, Handler: getDocumentIndexOperator}
+var getFilenameOpType = &operationType{Type: "GET_FILENAME", NumArgs: 0, Precedence: 50, Handler: getFilenameOperator}
+var getFileIndexOpType = &operationType{Type: "GET_FILE_INDEX", NumArgs: 0, Precedence: 50, Handler: getFileIndexOperator}
+
+var getPathOpType = &operationType{Type: "GET_PATH", NumArgs: 0, Precedence: 50, Handler: getPathOperator}
+var setPathOpType = &operationType{Type: "SET_PATH", NumArgs: 1, Precedence: 50, Handler: setPathOperator}
+var delPathsOpType = &operationType{Type: "DEL_PATHS", NumArgs: 1, Precedence: 50, Handler: delPathsOperator}
+
+var explodeOpType = &operationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: explodeOperator}
+var sortByOpType = &operationType{Type: "SORT_BY", NumArgs: 1, Precedence: 50, Handler: sortByOperator}
+var reverseOpType = &operationType{Type: "REVERSE", NumArgs: 0, Precedence: 50, Handler: reverseOperator}
+var sortOpType = &operationType{Type: "SORT", NumArgs: 0, Precedence: 50, Handler: sortOperator}
+var shuffleOpType = &operationType{Type: "SHUFFLE", NumArgs: 0, Precedence: 50, Handler: shuffleOperator}
+
+var sortKeysOpType = &operationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Handler: sortKeysOperator}
+
+var joinStringOpType = &operationType{Type: "JOIN", NumArgs: 1, Precedence: 50, Handler: joinStringOperator}
+var subStringOpType = &operationType{Type: "SUBSTR", NumArgs: 1, Precedence: 50, Handler: substituteStringOperator}
+var matchOpType = &operationType{Type: "MATCH", NumArgs: 1, Precedence: 50, Handler: matchOperator}
+var captureOpType = &operationType{Type: "CAPTURE", NumArgs: 1, Precedence: 50, Handler: captureOperator}
+var testOpType = &operationType{Type: "TEST", NumArgs: 1, Precedence: 50, Handler: testOperator}
+var splitStringOpType = &operationType{Type: "SPLIT", NumArgs: 1, Precedence: 50, Handler: splitStringOperator}
+var changeCaseOpType = &operationType{Type: "CHANGE_CASE", NumArgs: 0, Precedence: 50, Handler: changeCaseOperator}
+var trimOpType = &operationType{Type: "TRIM", NumArgs: 0, Precedence: 50, Handler: trimSpaceOperator}
+
+var loadOpType = &operationType{Type: "LOAD", NumArgs: 1, Precedence: 50, Handler: loadYamlOperator}
+
+var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 50, Handler: keysOperator}
+
+var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator}
+var traversePathOpType = &operationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 55, Handler: traversePathOperator}
+var traverseArrayOpType = &operationType{Type: "TRAVERSE_ARRAY", NumArgs: 2, Precedence: 50, Handler: traverseArrayOperator}
+
+var selfReferenceOpType = &operationType{Type: "SELF", NumArgs: 0, Precedence: 55, Handler: selfOperator}
+var valueOpType = &operationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: valueOperator}
+var referenceOpType = &operationType{Type: "REF", NumArgs: 0, Precedence: 50, Handler: referenceOperator}
+var envOpType = &operationType{Type: "ENV", NumArgs: 0, Precedence: 50, Handler: envOperator}
+var notOpType = &operationType{Type: "NOT", NumArgs: 0, Precedence: 50, Handler: notOperator}
+var toNumberOpType = &operationType{Type: "TO_NUMBER", NumArgs: 0, Precedence: 50, Handler: toNumberOperator}
+var emptyOpType = &operationType{Type: "EMPTY", Precedence: 50, Handler: emptyOperator}
+
+var envsubstOpType = &operationType{Type: "ENVSUBST", NumArgs: 0, Precedence: 50, Handler: envsubstOperator}
+
+var recursiveDescentOpType = &operationType{Type: "RECURSIVE_DESCENT", NumArgs: 0, Precedence: 50, Handler: recursiveDescentOperator}
+
+var selectOpType = &operationType{Type: "SELECT", NumArgs: 1, Precedence: 50, Handler: selectOperator}
+var hasOpType = &operationType{Type: "HAS", NumArgs: 1, Precedence: 50, Handler: hasOperator}
+var uniqueOpType = &operationType{Type: "UNIQUE", NumArgs: 0, Precedence: 50, Handler: unique}
+var uniqueByOpType = &operationType{Type: "UNIQUE_BY", NumArgs: 1, Precedence: 50, Handler: uniqueBy}
+var groupByOpType = &operationType{Type: "GROUP_BY", NumArgs: 1, Precedence: 50, Handler: groupBy}
+var flattenOpType = &operationType{Type: "FLATTEN_BY", NumArgs: 0, Precedence: 50, Handler: flattenOp}
+var deleteChildOpType = &operationType{Type: "DELETE", NumArgs: 1, Precedence: 40, Handler: deleteChildOperator}
+
+type Operation struct {
+	OperationType *operationType
+	Value         interface{}
+	StringValue   string
+	CandidateNode *CandidateNode // used for Value Path elements
+	Preferences   interface{}
+	UpdateAssign  bool // used for assign ops, when true it means we evaluate the rhs given the lhs
+}
+
 func recurseNodeArrayEqual(lhs *CandidateNode, rhs *CandidateNode) bool {
 	if len(lhs.Content) != len(rhs.Content) {
 		return false
@@ -95,11 +256,6 @@ func parseSnippet(value string) (*CandidateNode, error) {
 	if err != nil {
 		return nil, err
 	}
-	if result.Tag == "!!str" {
-		// use the original string value, as
-		// decoding drops new lines
-		return CreateScalarNode(value, value), nil
-	}
 	result.Line = 0
 	result.Column = 0
 	return result, err
@@ -114,8 +270,8 @@ func recursiveNodeEqual(lhs *CandidateNode, rhs *CandidateNode) bool {
 		//process custom tags of scalar nodes.
 		//dont worry about matching tags of maps or arrays.
 
-		lhsTag := lhs.GuessTagFromCustomType()
-		rhsTag := rhs.GuessTagFromCustomType()
+		lhsTag := lhs.guessTagFromCustomType()
+		rhsTag := rhs.guessTagFromCustomType()
 
 		if lhsTag != rhsTag {
 			return false
@@ -161,6 +317,50 @@ func parseInt(numberString string) (int, error) {
 	return int(parsed), err
 }
 
+func headAndLineComment(node *CandidateNode) string {
+	return headComment(node) + lineComment(node)
+}
+
+func headComment(node *CandidateNode) string {
+	return strings.Replace(node.HeadComment, "#", "", 1)
+}
+
+func lineComment(node *CandidateNode) string {
+	return strings.Replace(node.LineComment, "#", "", 1)
+}
+
+func footComment(node *CandidateNode) string {
+	return strings.Replace(node.FootComment, "#", "", 1)
+}
+
+func createValueOperation(value interface{}, stringValue string) *Operation {
+	log.Debug("creating value op for string %v", stringValue)
+	var node = createScalarNode(value, stringValue)
+
+	return &Operation{
+		OperationType: valueOpType,
+		Value:         value,
+		StringValue:   stringValue,
+		CandidateNode: node,
+	}
+}
+
+// debugging purposes only
+func (p *Operation) toString() string {
+	if p == nil {
+		return "OP IS NIL"
+	}
+	if p.OperationType == traversePathOpType {
+		return fmt.Sprintf("%v", p.Value)
+	} else if p.OperationType == selfReferenceOpType {
+		return "SELF"
+	} else if p.OperationType == valueOpType {
+		return fmt.Sprintf("%v (%T)", p.Value, p.Value)
+	} else {
+		return fmt.Sprintf("%v", p.OperationType.Type)
+	}
+}
+
 // use for debugging only
 func NodesToString(collection *list.List) string {
 	if !log.IsEnabledFor(logging.DEBUG) {

pkg/yqlib/lua_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
-var luaScenarios = []FormatScenario{
+var luaScenarios = []formatScenario{
 	{
 		description: "Basic input example",
 		input: `return {
@@ -25,10 +25,10 @@ var luaScenarios = []FormatScenario{
 `,
 		expected: `country: Australia
 cities:
-  - Sydney
-  - Melbourne
-  - Brisbane
-  - Perth
+    - Sydney
+    - Melbourne
+    - Brisbane
+    - Perth
 `,
 	},
 	{
@@ -250,10 +250,10 @@ numbers:
 	},
 }
 
-func testLuaScenario(t *testing.T, s FormatScenario) {
+func testLuaScenario(t *testing.T, s formatScenario) {
 	switch s.scenarioType {
 	case "", "decode":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewYamlEncoder(4, false, ConfiguredYamlPreferences)), s.description)
 	case "encode":
 		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(ConfiguredLuaPreferences)), s.description)
 	case "roundtrip":
@@ -277,8 +277,8 @@ func testLuaScenario(t *testing.T, s FormatScenario) {
 	}
 }
 
-func documentLuaScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
-	s := i.(FormatScenario)
+func documentLuaScenario(t *testing.T, w *bufio.Writer, i interface{}) {
+	s := i.(formatScenario)
 
 	if s.skipDoc {
 		return
@@ -295,7 +295,7 @@ func documentLuaScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
 	}
 }
 
-func documentLuaDecodeScenario(w *bufio.Writer, s FormatScenario) {
+func documentLuaDecodeScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -314,10 +314,10 @@ func documentLuaDecodeScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("```bash\nyq -oy '%v' sample.lua\n```\n", expression))
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewYamlEncoder(ConfiguredYamlPreferences))))
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences))))
 }
 
-func documentLuaEncodeScenario(w *bufio.Writer, s FormatScenario) {
+func documentLuaEncodeScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -359,7 +359,7 @@ func documentLuaEncodeScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("```lua\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(prefs))))
 }
 
-func documentLuaRoundTripScenario(w *bufio.Writer, s FormatScenario) {
+func documentLuaRoundTripScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {

pkg/yqlib/matchKeyString_test.go

@@ -12,7 +12,6 @@ func TestDeepMatch(t *testing.T) {
 		ok      bool
 	}{
 		{"", "", true},
-		{"<<", "<<", true},
 		{"", "x", false},
 		{"x", "", false},
 		{"abc", "abc", true},

pkg/yqlib/operation.go

@@ -1,201 +0,0 @@
-package yqlib
-
-import "fmt"
-
-type Operation struct {
-	OperationType *operationType
-	Value         interface{}
-	StringValue   string
-	CandidateNode *CandidateNode // used for Value Path elements
-	Preferences   interface{}
-	UpdateAssign  bool // used for assign ops, when true it means we evaluate the rhs given the lhs
-}
-
-type operationType struct {
-	Type                 string
-	NumArgs              uint // number of arguments to the op
-	Precedence           uint
-	Handler              operatorHandler
-	CheckForPostTraverse bool
-	ToString             func(o *Operation) string
-}
-
-var valueToStringFunc = func(p *Operation) string {
-	return fmt.Sprintf("%v (%T)", p.Value, p.Value)
-}
-
-func createValueOperation(value interface{}, stringValue string) *Operation {
-	log.Debug("creating value op for string %v", stringValue)
-	var node = CreateScalarNode(value, stringValue)
-
-	return &Operation{
-		OperationType: valueOpType,
-		Value:         value,
-		StringValue:   stringValue,
-		CandidateNode: node,
-	}
-}
-
-var orOpType = &operationType{Type: "OR", NumArgs: 2, Precedence: 20, Handler: orOperator}
-var andOpType = &operationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: andOperator}
-var reduceOpType = &operationType{Type: "REDUCE", NumArgs: 2, Precedence: 35, Handler: reduceOperator}
-
-var blockOpType = &operationType{Type: "BLOCK", Precedence: 10, NumArgs: 2, Handler: emptyOperator}
-
-var unionOpType = &operationType{Type: "UNION", NumArgs: 2, Precedence: 10, Handler: unionOperator}
-
-var pipeOpType = &operationType{Type: "PIPE", NumArgs: 2, Precedence: 30, Handler: pipeOperator}
-
-var assignOpType = &operationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: assignUpdateOperator}
-var addAssignOpType = &operationType{Type: "ADD_ASSIGN", NumArgs: 2, Precedence: 40, Handler: addAssignOperator}
-var subtractAssignOpType = &operationType{Type: "SUBTRACT_ASSIGN", NumArgs: 2, Precedence: 40, Handler: subtractAssignOperator}
-
-var assignAttributesOpType = &operationType{Type: "ASSIGN_ATTRIBUTES", NumArgs: 2, Precedence: 40, Handler: assignAttributesOperator}
-var assignStyleOpType = &operationType{Type: "ASSIGN_STYLE", NumArgs: 2, Precedence: 40, Handler: assignStyleOperator}
-var assignVariableOpType = &operationType{Type: "ASSIGN_VARIABLE", NumArgs: 2, Precedence: 40, Handler: useWithPipe}
-var assignTagOpType = &operationType{Type: "ASSIGN_TAG", NumArgs: 2, Precedence: 40, Handler: assignTagOperator}
-var assignCommentOpType = &operationType{Type: "ASSIGN_COMMENT", NumArgs: 2, Precedence: 40, Handler: assignCommentsOperator}
-var assignAnchorOpType = &operationType{Type: "ASSIGN_ANCHOR", NumArgs: 2, Precedence: 40, Handler: assignAnchorOperator}
-var assignAliasOpType = &operationType{Type: "ASSIGN_ALIAS", NumArgs: 2, Precedence: 40, Handler: assignAliasOperator}
-
-var multiplyOpType = &operationType{Type: "MULTIPLY", NumArgs: 2, Precedence: 42, Handler: multiplyOperator}
-var multiplyAssignOpType = &operationType{Type: "MULTIPLY_ASSIGN", NumArgs: 2, Precedence: 42, Handler: multiplyAssignOperator}
-
-var divideOpType = &operationType{Type: "DIVIDE", NumArgs: 2, Precedence: 42, Handler: divideOperator}
-
-var moduloOpType = &operationType{Type: "MODULO", NumArgs: 2, Precedence: 42, Handler: moduloOperator}
-
-var addOpType = &operationType{Type: "ADD", NumArgs: 2, Precedence: 42, Handler: addOperator}
-var subtractOpType = &operationType{Type: "SUBTRACT", NumArgs: 2, Precedence: 42, Handler: subtractOperator}
-var alternativeOpType = &operationType{Type: "ALTERNATIVE", NumArgs: 2, Precedence: 42, Handler: alternativeOperator}
-
-var equalsOpType = &operationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: equalsOperator}
-var notEqualsOpType = &operationType{Type: "NOT_EQUALS", NumArgs: 2, Precedence: 40, Handler: notEqualsOperator}
-
-var compareOpType = &operationType{Type: "COMPARE", NumArgs: 2, Precedence: 40, Handler: compareOperator}
-
-// createmap needs to be above union, as we use union to build the components of the objects
-var createMapOpType = &operationType{Type: "CREATE_MAP", NumArgs: 2, Precedence: 15, Handler: createMapOperator}
-
-var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence: 45, Handler: pipeOperator}
-
-var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: lengthOperator}
-var lineOpType = &operationType{Type: "LINE", NumArgs: 0, Precedence: 50, Handler: lineOperator}
-var columnOpType = &operationType{Type: "LINE", NumArgs: 0, Precedence: 50, Handler: columnOperator}
-
-var expressionOpType = &operationType{Type: "EXP", NumArgs: 0, Precedence: 50, Handler: expressionOperator}
-
-var collectOpType = &operationType{Type: "COLLECT", NumArgs: 1, Precedence: 50, Handler: collectOperator}
-var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 50, Handler: mapOperator}
-var filterOpType = &operationType{Type: "FILTER", NumArgs: 1, Precedence: 50, Handler: filterOperator}
-var errorOpType = &operationType{Type: "ERROR", NumArgs: 1, Precedence: 50, Handler: errorOperator}
-var pickOpType = &operationType{Type: "PICK", NumArgs: 1, Precedence: 50, Handler: pickOperator}
-var evalOpType = &operationType{Type: "EVAL", NumArgs: 1, Precedence: 50, Handler: evalOperator}
-var mapValuesOpType = &operationType{Type: "MAP_VALUES", NumArgs: 1, Precedence: 50, Handler: mapValuesOperator}
-
-var formatDateTimeOpType = &operationType{Type: "FORMAT_DATE_TIME", NumArgs: 1, Precedence: 50, Handler: formatDateTime}
-var withDtFormatOpType = &operationType{Type: "WITH_DATE_TIME_FORMAT", NumArgs: 1, Precedence: 50, Handler: withDateTimeFormat}
-var nowOpType = &operationType{Type: "NOW", NumArgs: 0, Precedence: 50, Handler: nowOp}
-var tzOpType = &operationType{Type: "TIMEZONE", NumArgs: 1, Precedence: 50, Handler: tzOp}
-var fromUnixOpType = &operationType{Type: "FROM_UNIX", NumArgs: 0, Precedence: 50, Handler: fromUnixOp}
-var toUnixOpType = &operationType{Type: "TO_UNIX", NumArgs: 0, Precedence: 50, Handler: toUnixOp}
-
-var encodeOpType = &operationType{Type: "ENCODE", NumArgs: 0, Precedence: 50, Handler: encodeOperator}
-var decodeOpType = &operationType{Type: "DECODE", NumArgs: 0, Precedence: 50, Handler: decodeOperator}
-
-var anyOpType = &operationType{Type: "ANY", NumArgs: 0, Precedence: 50, Handler: anyOperator}
-var allOpType = &operationType{Type: "ALL", NumArgs: 0, Precedence: 50, Handler: allOperator}
-var containsOpType = &operationType{Type: "CONTAINS", NumArgs: 1, Precedence: 50, Handler: containsOperator}
-var anyConditionOpType = &operationType{Type: "ANY_CONDITION", NumArgs: 1, Precedence: 50, Handler: anyOperator}
-var allConditionOpType = &operationType{Type: "ALL_CONDITION", NumArgs: 1, Precedence: 50, Handler: allOperator}
-
-var toEntriesOpType = &operationType{Type: "TO_ENTRIES", NumArgs: 0, Precedence: 52, Handler: toEntriesOperator, CheckForPostTraverse: true}
-var fromEntriesOpType = &operationType{Type: "FROM_ENTRIES", NumArgs: 0, Precedence: 50, Handler: fromEntriesOperator}
-var withEntriesOpType = &operationType{Type: "WITH_ENTRIES", NumArgs: 1, Precedence: 50, Handler: withEntriesOperator}
-
-var withOpType = &operationType{Type: "WITH", NumArgs: 1, Precedence: 50, Handler: withOperator}
-
-var splitDocumentOpType = &operationType{Type: "SPLIT_DOC", NumArgs: 0, Precedence: 50, Handler: splitDocumentOperator}
-var getVariableOpType = &operationType{Type: "GET_VARIABLE", NumArgs: 0, Precedence: 55, Handler: getVariableOperator}
-var getStyleOpType = &operationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: getStyleOperator}
-var getTagOpType = &operationType{Type: "GET_TAG", NumArgs: 0, Precedence: 50, Handler: getTagOperator}
-var getKindOpType = &operationType{Type: "GET_KIND", NumArgs: 0, Precedence: 50, Handler: getKindOperator}
-
-var getKeyOpType = &operationType{Type: "GET_KEY", NumArgs: 0, Precedence: 50, Handler: getKeyOperator}
-var isKeyOpType = &operationType{Type: "IS_KEY", NumArgs: 0, Precedence: 50, Handler: isKeyOperator}
-var getParentOpType = &operationType{Type: "GET_PARENT", NumArgs: 0, Precedence: 50, Handler: getParentOperator}
-
-var getCommentOpType = &operationType{Type: "GET_COMMENT", NumArgs: 0, Precedence: 50, Handler: getCommentsOperator}
-var getAnchorOpType = &operationType{Type: "GET_ANCHOR", NumArgs: 0, Precedence: 50, Handler: getAnchorOperator}
-var getAliasOpType = &operationType{Type: "GET_ALIAS", NumArgs: 0, Precedence: 50, Handler: getAliasOperator}
-var getDocumentIndexOpType = &operationType{Type: "GET_DOCUMENT_INDEX", NumArgs: 0, Precedence: 50, Handler: getDocumentIndexOperator}
-var getFilenameOpType = &operationType{Type: "GET_FILENAME", NumArgs: 0, Precedence: 50, Handler: getFilenameOperator}
-var getFileIndexOpType = &operationType{Type: "GET_FILE_INDEX", NumArgs: 0, Precedence: 50, Handler: getFileIndexOperator}
-
-var getPathOpType = &operationType{Type: "GET_PATH", NumArgs: 0, Precedence: 50, Handler: getPathOperator}
-var setPathOpType = &operationType{Type: "SET_PATH", NumArgs: 1, Precedence: 50, Handler: setPathOperator}
-var delPathsOpType = &operationType{Type: "DEL_PATHS", NumArgs: 1, Precedence: 50, Handler: delPathsOperator}
-
-var explodeOpType = &operationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: explodeOperator}
-var sortByOpType = &operationType{Type: "SORT_BY", NumArgs: 1, Precedence: 50, Handler: sortByOperator}
-var reverseOpType = &operationType{Type: "REVERSE", NumArgs: 0, Precedence: 50, Handler: reverseOperator}
-var sortOpType = &operationType{Type: "SORT", NumArgs: 0, Precedence: 50, Handler: sortOperator}
-var shuffleOpType = &operationType{Type: "SHUFFLE", NumArgs: 0, Precedence: 50, Handler: shuffleOperator}
-
-var sortKeysOpType = &operationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Handler: sortKeysOperator}
-
-var joinStringOpType = &operationType{Type: "JOIN", NumArgs: 1, Precedence: 50, Handler: joinStringOperator}
-var subStringOpType = &operationType{Type: "SUBSTR", NumArgs: 1, Precedence: 50, Handler: substituteStringOperator}
-var matchOpType = &operationType{Type: "MATCH", NumArgs: 1, Precedence: 50, Handler: matchOperator}
-var captureOpType = &operationType{Type: "CAPTURE", NumArgs: 1, Precedence: 50, Handler: captureOperator}
-var testOpType = &operationType{Type: "TEST", NumArgs: 1, Precedence: 50, Handler: testOperator}
-var splitStringOpType = &operationType{Type: "SPLIT", NumArgs: 1, Precedence: 50, Handler: splitStringOperator}
-var changeCaseOpType = &operationType{Type: "CHANGE_CASE", NumArgs: 0, Precedence: 50, Handler: changeCaseOperator}
-var trimOpType = &operationType{Type: "TRIM", NumArgs: 0, Precedence: 50, Handler: trimSpaceOperator}
-var toStringOpType = &operationType{Type: "TO_STRING", NumArgs: 0, Precedence: 50, Handler: toStringOperator}
-var stringInterpolationOpType = &operationType{Type: "STRING_INT", NumArgs: 0, Precedence: 50, Handler: stringInterpolationOperator, ToString: valueToStringFunc}
-
-var loadOpType = &operationType{Type: "LOAD", NumArgs: 1, Precedence: 52, Handler: loadOperator}
-var loadStringOpType = &operationType{Type: "LOAD_STRING", NumArgs: 1, Precedence: 52, Handler: loadStringOperator}
-
-var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 50, Handler: keysOperator}
-
-var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator}
-
-var traversePathOpType = &operationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 55, Handler: traversePathOperator,
-	ToString: func(p *Operation) string {
-		return fmt.Sprintf("%v", p.Value)
-	}}
-
-var traverseArrayOpType = &operationType{Type: "TRAVERSE_ARRAY", NumArgs: 2, Precedence: 50, Handler: traverseArrayOperator}
-
-var selfReferenceOpType = &operationType{Type: "SELF", NumArgs: 0, Precedence: 55, Handler: selfOperator}
-var valueOpType = &operationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: valueOperator, ToString: valueToStringFunc}
-var referenceOpType = &operationType{Type: "REF", NumArgs: 0, Precedence: 50, Handler: referenceOperator}
-var envOpType = &operationType{Type: "ENV", NumArgs: 0, Precedence: 50, Handler: envOperator}
-var notOpType = &operationType{Type: "NOT", NumArgs: 0, Precedence: 50, Handler: notOperator}
-var toNumberOpType = &operationType{Type: "TO_NUMBER", NumArgs: 0, Precedence: 50, Handler: toNumberOperator}
-var emptyOpType = &operationType{Type: "EMPTY", Precedence: 50, Handler: emptyOperator}
-
-var envsubstOpType = &operationType{Type: "ENVSUBST", NumArgs: 0, Precedence: 50, Handler: envsubstOperator}
-
-var recursiveDescentOpType = &operationType{Type: "RECURSIVE_DESCENT", NumArgs: 0, Precedence: 50, Handler: recursiveDescentOperator}
-
-var selectOpType = &operationType{Type: "SELECT", NumArgs: 1, Precedence: 50, Handler: selectOperator}
-var hasOpType = &operationType{Type: "HAS", NumArgs: 1, Precedence: 50, Handler: hasOperator}
-var uniqueOpType = &operationType{Type: "UNIQUE", NumArgs: 0, Precedence: 50, Handler: unique}
-var uniqueByOpType = &operationType{Type: "UNIQUE_BY", NumArgs: 1, Precedence: 50, Handler: uniqueBy}
-var groupByOpType = &operationType{Type: "GROUP_BY", NumArgs: 1, Precedence: 50, Handler: groupBy}
-var flattenOpType = &operationType{Type: "FLATTEN_BY", NumArgs: 0, Precedence: 50, Handler: flattenOp}
-var deleteChildOpType = &operationType{Type: "DELETE", NumArgs: 1, Precedence: 40, Handler: deleteChildOperator}
-
-// debugging purposes only
-func (p *Operation) toString() string {
-	if p == nil {
-		return "OP IS NIL"
-	}
-	if p.OperationType.ToString != nil {
-		return p.OperationType.ToString(p)
-	}
-	return fmt.Sprintf("%v", p.OperationType.Type)
-}

pkg/yqlib/operator_add.go

@@ -42,7 +42,7 @@ func addOperator(d *dataTreeNavigator, context Context, expressionNode *Expressi
 	return crossFunction(d, context.ReadOnlyClone(), expressionNode, add, false)
 }
 
-func add(_ *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+func add(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	lhsNode := lhs
 
 	if lhsNode.Tag == "!!null" {
@@ -74,11 +74,11 @@ func add(_ *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Candida
 
 func addScalars(context Context, target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error {
 	lhsTag := lhs.Tag
-	rhsTag := rhs.GuessTagFromCustomType()
+	rhsTag := rhs.guessTagFromCustomType()
 	lhsIsCustom := false
 	if !strings.HasPrefix(lhsTag, "!!") {
 		// custom tag - we have to have a guess
-		lhsTag = lhs.GuessTagFromCustomType()
+		lhsTag = lhs.guessTagFromCustomType()
 		lhsIsCustom = true
 	}
 

pkg/yqlib/operator_alternative.go

@@ -1,7 +1,7 @@
 package yqlib
 
 func alternativeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	log.Debugf("alternative")
+	log.Debugf("-- alternative")
 	prefs := crossFunctionPreferences{
 		CalcWhenEmpty: true,
 		Calculation:   alternativeFunc,
@@ -19,7 +19,7 @@ func alternativeOperator(d *dataTreeNavigator, context Context, expressionNode *
 	return crossFunctionWithPrefs(d, context, expressionNode, prefs)
 }
 
-func alternativeFunc(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+func alternativeFunc(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	if lhs == nil {
 		return rhs, nil
 	}

pkg/yqlib/operator_anchors_aliases.go

@@ -48,7 +48,7 @@ func assignAliasOperator(d *dataTreeNavigator, context Context, expressionNode *
 	return context, nil
 }
 
-func getAliasOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {
+func getAliasOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("GetAlias operator!")
 	var results = list.New()
 
@@ -102,7 +102,7 @@ func assignAnchorOperator(d *dataTreeNavigator, context Context, expressionNode
 	return context, nil
 }
 
-func getAnchorOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {
+func getAnchorOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("GetAnchor operator!")
 	var results = list.New()
 
@@ -116,7 +116,7 @@ func getAnchorOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode)
 }
 
 func explodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	log.Debugf("ExplodeOperation")
+	log.Debugf("-- ExplodeOperation")
 
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)

pkg/yqlib/operator_assign.go

@@ -6,10 +6,6 @@ type assignPreferences struct {
 	ClobberCustomTags   bool
 }
 
-func PerformAssignment(context Context, path []interface{}) {
-
-}
-
 func assignUpdateFunc(prefs assignPreferences) crossFunctionCalculation {
 	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 		if !prefs.OnlyWriteNull || lhs.Tag == "!!null" {

pkg/yqlib/operator_booleans.go

@@ -35,7 +35,7 @@ func getOwner(lhs *CandidateNode, rhs *CandidateNode) *CandidateNode {
 	return owner
 }
 
-func returnRhsTruthy(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+func returnRhsTruthy(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	owner := getOwner(lhs, rhs)
 	rhsBool := isTruthyNode(rhs)
 
@@ -88,7 +88,7 @@ func allOperator(d *dataTreeNavigator, context Context, expressionNode *Expressi
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 		if candidate.Kind != SequenceNode {
-			return Context{}, fmt.Errorf("all only supports arrays, was %v", candidate.Tag)
+			return Context{}, fmt.Errorf("any only supports arrays, was %v", candidate.Tag)
 		}
 		booleanResult, err := findBoolean(false, d, context, expressionNode.RHS, candidate)
 		if err != nil {
@@ -136,8 +136,8 @@ func andOperator(d *dataTreeNavigator, context Context, expressionNode *Expressi
 	return crossFunctionWithPrefs(d, context.ReadOnlyClone(), expressionNode, prefs)
 }
 
-func notOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {
-	log.Debugf("notOperation")
+func notOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	log.Debugf("-- notOperation")
 	var results = list.New()
 
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {

pkg/yqlib/operator_collect.go

@@ -22,7 +22,7 @@ func collectTogether(d *dataTreeNavigator, context Context, expressionNode *Expr
 }
 
 func collectOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	log.Debugf("collectOperation")
+	log.Debugf("-- collectOperation")
 
 	if context.MatchingNodes.Len() == 0 {
 		log.Debugf("nothing to collect")

pkg/yqlib/operator_collect_object.go

@@ -15,14 +15,14 @@ import (
 ...
 */
 
-func collectObjectOperator(d *dataTreeNavigator, originalContext Context, _ *ExpressionNode) (Context, error) {
-	log.Debugf("collectObjectOperation")
+func collectObjectOperator(d *dataTreeNavigator, originalContext Context, expressionNode *ExpressionNode) (Context, error) {
+	log.Debugf("-- collectObjectOperation")
 
 	context := originalContext.WritableClone()
 
 	if context.MatchingNodes.Len() == 0 {
 		candidate := &CandidateNode{Kind: MappingNode, Tag: "!!map", Value: "{}"}
-		log.Debugf("collectObjectOperation - starting with empty map")
+		log.Debugf("-- collectObjectOperation - starting with empty map")
 		return context.SingleChildContext(candidate), nil
 	}
 	first := context.MatchingNodes.Front().Value.(*CandidateNode)
@@ -41,7 +41,7 @@ func collectObjectOperator(d *dataTreeNavigator, originalContext Context, _ *Exp
 			rotated[i].PushBack(candidateNode.Content[i])
 		}
 	}
-	log.Debugf("collectObjectOperation, length of rotated is %v", len(rotated))
+	log.Debugf("-- collectObjectOperation, length of rotated is %v", len(rotated))
 
 	newObject := list.New()
 	for i := 0; i < len(first.Content); i++ {
@@ -58,7 +58,7 @@ func collectObjectOperator(d *dataTreeNavigator, originalContext Context, _ *Exp
 			additionCopy.SetParent(nil)
 			additionCopy.Key = nil
 
-			log.Debugf("collectObjectOperation, adding result %v", NodeToString(additionCopy))
+			log.Debugf("-- collectObjectOperation, adding result %v", NodeToString(additionCopy))
 
 			newObject.PushBack(additionCopy)
 		}
@@ -74,7 +74,7 @@ func collect(d *dataTreeNavigator, context Context, remainingMatches *list.List)
 	}
 
 	candidate := remainingMatches.Remove(remainingMatches.Front()).(*CandidateNode)
-	log.Debugf("collectObjectOperation - collect %v", NodeToString(candidate))
+	log.Debugf("-- collectObjectOperation - collect %v", NodeToString(candidate))
 
 	splatted, err := splat(context.SingleChildContext(candidate),
 		traversePreferences{DontFollowAlias: true, IncludeMapKeys: false})
@@ -84,7 +84,7 @@ func collect(d *dataTreeNavigator, context Context, remainingMatches *list.List)
 	}
 
 	if context.MatchingNodes.Len() == 0 {
-		log.Debugf("collectObjectOperation - collect context is empty, next")
+		log.Debugf("-- collectObjectOperation - collect context is empty, next")
 		return collect(d, splatted, remainingMatches)
 	}
 
@@ -94,9 +94,9 @@ func collect(d *dataTreeNavigator, context Context, remainingMatches *list.List)
 		aggCandidate := el.Value.(*CandidateNode)
 		for splatEl := splatted.MatchingNodes.Front(); splatEl != nil; splatEl = splatEl.Next() {
 			splatCandidate := splatEl.Value.(*CandidateNode)
-			log.Debugf("collectObjectOperation; splatCandidate: %v", NodeToString(splatCandidate))
+			log.Debugf("-- collectObjectOperation; splatCandidate: %v", NodeToString(splatCandidate))
 			newCandidate := aggCandidate.Copy()
-			log.Debugf("collectObjectOperation; aggCandidate: %v", NodeToString(aggCandidate))
+			log.Debugf("-- collectObjectOperation; aggCandidate: %v", NodeToString(aggCandidate))
 
 			newCandidate, err = multiply(multiplyPreferences{AppendArrays: false})(d, context, newCandidate, splatCandidate)
 

pkg/yqlib/operator_column.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 )
 
-func columnOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {
+func columnOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("columnOperator")
 
 	var results = list.New()

pkg/yqlib/operator_comments.go

@@ -72,7 +72,7 @@ func assignCommentsOperator(d *dataTreeNavigator, context Context, expressionNod
 	return context, nil
 }
 
-func getCommentsOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	preferences := expressionNode.Operation.Preferences.(commentOpPreferences)
 	var startCommentCharacterRegExp = regexp.MustCompile(`^# `)
 	var subsequentCommentCharacterRegExp = regexp.MustCompile(`\n# `)
@@ -80,10 +80,9 @@ func getCommentsOperator(_ *dataTreeNavigator, context Context, expressionNode *
 	log.Debugf("GetComments operator!")
 	var results = list.New()
 
-	yamlPrefs := ConfiguredYamlPreferences.Copy()
+	yamlPrefs := NewDefaultYamlPreferences()
 	yamlPrefs.PrintDocSeparators = false
 	yamlPrefs.UnwrapScalar = false
-	yamlPrefs.ColorsEnabled = false
 
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
@@ -95,7 +94,7 @@ func getCommentsOperator(_ *dataTreeNavigator, context Context, expressionNode *
 			var chompRegexp = regexp.MustCompile(`\n$`)
 			var output bytes.Buffer
 			var writer = bufio.NewWriter(&output)
-			var encoder = NewYamlEncoder(yamlPrefs)
+			var encoder = NewYamlEncoder(2, false, yamlPrefs)
 			if err := encoder.PrintLeadingContent(writer, candidate.LeadingContent); err != nil {
 				return Context{}, err
 			}