wip

Bumps golang from 1.22.0 to 1.22.1.

---
updated-dependencies:
- dependency-name: golang
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/workflows/codeql.yml

@@ -42,7 +42,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -53,7 +53,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -67,4 +67,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

.github/workflows/docker-release.yml

@@ -45,7 +45,7 @@ jobs:
           docker buildx build \
             --label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \
             --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \
-            --label "org.opencontainers.image.description=yq is a portable command-line YAML processor" \
+            --label "org.opencontainers.image.description=yq is a portable command-line data file processor" \
             --label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \
             --label "org.opencontainers.image.licenses=MIT" \
             --label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \
@@ -65,7 +65,7 @@ jobs:
           docker buildx build \
             --label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \
             --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \
-            --label "org.opencontainers.image.description=yq is a portable command-line YAML processor" \
+            --label "org.opencontainers.image.description=yq is a portable command-line data file processor" \
             --label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \
             --label "org.opencontainers.image.licenses=MIT" \
             --label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \

.github/workflows/go.yml

@@ -11,7 +11,7 @@ jobs:
     steps:
 
     - name: Set up Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: '^1.20'
       id: go

.github/workflows/release.yml

@@ -2,7 +2,7 @@ name: Release YQ
 on:
   push:
     tags:
-      - 'v*'
+      - 'v4.*'
       - 'draft-*'
 
 jobs:
@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: '^1.20'
           check-latest: true

.golangci.yml

@@ -20,12 +20,15 @@ linters:
     - unparam
 linters-settings:
   depguard:
-    list-type: blacklist
-    include-go-root: true
-    packages:
-      - io/ioutil
-    packages-with-error-message:
-      - io/ioutil: "The 'io/ioutil' package is deprecated. Use corresponding 'os' or 'io' functions instead."
+    rules:
+      prevent_unmaintained_packages:
+        list-mode: lax
+        files:
+          - $all
+          - "!$test"
+        deny:
+          - pkg: io/ioutil
+            desc: "replaced by io and os packages since Go 1.16: https://tip.golang.org/doc/go1.16#ioutil"
 issues:
   exclude-rules:
   - linters:

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21.1 as builder
+FROM golang:1.22.1 as builder
 
 WORKDIR /go/src/mikefarah/yq
 

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM golang:1.21.1
+FROM golang:1.22.1
 
 COPY scripts/devtools.sh /opt/devtools.sh
 
@@ -6,5 +6,7 @@ RUN set -e -x \
     && /opt/devtools.sh
 ENV PATH=/go/bin:$PATH
 
+RUN apt-get update && apt-get install -y npm && npm install -g npx cspell@latest
+
 ENV CGO_ENABLED 0
 ENV GOPATH /go:/yq

README.md

@@ -31,6 +31,10 @@ NAME=mike yq -i '.a.b[0].c = strenv(NAME)' file.yaml
 
 Merge multiple files
 ```bash
+# merge two files
+yq -n 'load("file1.yaml") * load("file2.yaml")'
+
+# merge using globs:
 # note the use of `ea` to evaluate all the files at once
 # instead of in sequence
 yq ea '. as $item ireduce ({}; . * $item )' path/to/*.yml

acceptance_tests/basic.sh

@@ -96,6 +96,17 @@ testBasicExpressionFromFile() {
   assertEquals '{"xyz":"meow","cool":"frog"}' "$X"
 }
 
+testBasicExpressionFromFileDos() {
+  ./yq -n ".xyz = 123" > test.yml
+  echo '.xyz = "meow" | .cool = "frog"' | sed 's/$'"/`echo \\\r`/" > instructions.txt
+  
+  X=$(./yq --from-file instructions.txt test.yml -o=j -I=0)
+  assertEquals '{"xyz":"meow","cool":"frog"}' "$X"
+
+  X=$(./yq ea --from-file instructions.txt test.yml -o=j -I=0)
+  assertEquals '{"xyz":"meow","cool":"frog"}' "$X"
+}
+
 testBasicGitHubAction() {
   ./yq -n ".a = 123" > test.yml
   X=$(cat /dev/null | ./yq test.yml)
@@ -351,6 +362,12 @@ EOM
   assertEquals "$expected" "$X"
 }
 
-
+testBasicClosedStdIn() {
+  cat >test.yml <<EOL
+a: 1
+EOL
+  X=$(./yq e '.a' test.yml <&-)
+  assertEquals "1" "$X"
+}
 
 source ./scripts/shunit2

acceptance_tests/flags.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+setUp() {
+  rm test*.yml || true
+  cat >test.yml <<EOL
+# comment
+EOL
+}
+
+testStringInterpolation() {
+    X=$(./yq -n '"Mike \(3 + 4)"')
+    assertEquals "Mike 7" "$X"
+}
+
+testNoStringInterpolation() {
+    X=$(./yq --string-interpolation=f -n '"Mike \(3 + 4)"')
+    assertEquals "Mike \(3 + 4)" "$X"
+}
+
+
+source ./scripts/shunit2

acceptance_tests/inputs-format.sh

@@ -63,6 +63,61 @@ EOM
   assertEquals "$expected" "$X"
 }
 
+testInputCSVCustomSeparator() {
+  cat >test.csv <<EOL
+fruit;yumLevel
+apple;5
+banana;4
+EOL
+
+  read -r -d '' expected << EOM
+- fruit: apple
+  yumLevel: 5
+- fruit: banana
+  yumLevel: 4
+EOM
+
+  X=$(./yq -p=csv --csv-separator ";" test.csv)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -p=csv --csv-separator ";" test.csv)
+  assertEquals "$expected" "$X"
+}
+
+testInputCSVNoAuto() {
+  cat >test.csv <<EOL
+thing1
+name: cat
+EOL
+
+  read -r -d '' expected << EOM
+- thing1: 'name: cat'
+EOM
+
+  X=$(./yq --csv-auto-parse=f test.csv -oy)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --csv-auto-parse=f test.csv -oy)
+  assertEquals "$expected" "$X"
+}
+
+testInputTSVNoAuto() {
+  cat >test.tsv <<EOL
+thing1
+name: cat
+EOL
+
+  read -r -d '' expected << EOM
+- thing1: 'name: cat'
+EOM
+
+  X=$(./yq --tsv-auto-parse=f test.tsv -oy)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --tsv-auto-parse=f test.tsv -oy)
+  assertEquals "$expected" "$X"
+}
+
 testInputCSVUTF8() {
   read -r -d '' expected << EOM
 - id: 1

acceptance_tests/load-file.sh

@@ -9,7 +9,7 @@ testLoadFileNotExist() {
 testLoadFileExpNotExist() {
   result=$(./yq e -n 'load(.a)' 2>&1)
   assertEquals 1 $?
-  assertEquals "Error: Filename expression returned nil" "$result"
+  assertEquals "Error: filename expression returned nil" "$result"
 }
 
 testStrLoadFileNotExist() {
@@ -21,7 +21,7 @@ testStrLoadFileNotExist() {
 testStrLoadFileExpNotExist() {
   result=$(./yq e -n 'strload(.a)' 2>&1)
   assertEquals 1 $?
-  assertEquals "Error: Filename expression returned nil" "$result"
+  assertEquals "Error: filename expression returned nil" "$result"
 }
 
 source ./scripts/shunit2

acceptance_tests/output-format.sh

@@ -72,6 +72,15 @@ EOL
   assertEquals "\"cat\"" "$X"
 }
 
+testOutputYamlRawOnRoot() {
+  cat >test.yml <<EOL
+'a'
+EOL
+
+  X=$(./yq e -r '.' test.yml)
+  assertEquals "a" "$X"
+}
+
 testOutputJsonRaw() {
   cat >test.yml <<EOL
 a: cat
@@ -189,6 +198,27 @@ EOM
   assertEquals "$expected" "$X"
 }
 
+testOutputCSVCustomSeparator() {
+  cat >test.yml <<EOL
+- fruit: apple
+  yumLevel: 5
+- fruit: banana
+  yumLevel: 4
+EOL
+
+  read -r -d '' expected << EOM
+fruit;yumLevel
+apple;5
+banana;4
+EOM
+
+  X=$(./yq -oc --csv-separator ";" test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -o=csv --csv-separator ";" test.yml)
+  assertEquals "$expected" "$X"
+}
+
 testOutputTSV() {
   cat >test.yml <<EOL
 - fruit: apple
@@ -271,4 +301,43 @@ EOM
   assertEquals "$expected" "$X"
 }
 
-source ./scripts/shunit2
+testLuaOutputPretty() {
+  cat >test.yml <<EOL
+animals:
+  cat: meow
+EOL
+
+  read -r -d '' expected << EOM
+return {
+	["animals"] = {
+		["cat"] = "meow";
+	};
+};
+EOM
+
+  X=$(./yq e --output-format=lua test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq e --output-format=lua --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+
+}
+
+testLuaOutputSubset() {
+  cat >test.yml <<EOL
+animals:
+  cat: meow
+EOL
+
+  read -r -d '' expected << EOM
+return {
+	["cat"] = "meow";
+};
+EOM
+
+  X=$(./yq e --output-format=lua '.animals' test.yml)
+  assertEquals "$expected" "$X"
+
+}
+
+source ./scripts/shunit2

acceptance_tests/shebang.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+setUp() {
+  rm test*.yq || true
+  cat >test.yq <<EOL
+#!./yq
+.a.b
+EOL
+chmod +x test.yq
+
+rm test*.yml || true
+  cat >test.yml <<EOL
+a: {b: apple}
+EOL
+}
+
+testCanExecYqFile() {
+  read -r -d '' expected << EOM
+apple
+EOM
+   X=$(./test.yq test.yml)
+  assertEquals "$expected" "$X"
+}
+
+source ./scripts/shunit2
+

cmd/evaluate_all_command.go

@@ -12,6 +12,12 @@ func createEvaluateAllCommand() *cobra.Command {
 		Use:     "eval-all [expression] [yaml_file1]...",
 		Aliases: []string{"ea"},
 		Short:   "Loads _all_ yaml documents of _all_ yaml files and runs expression once",
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			if len(args) == 0 {
+				return nil, cobra.ShellCompDirectiveNoFileComp
+			}
+			return nil, cobra.ShellCompDirectiveDefault
+		},
 		Example: `
 # Merge f2.yml into f1.yml (in place)
 yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' f1.yml f2.yml
@@ -26,7 +32,7 @@ yq ea '. as $item ireduce ({}; . * $item )' file1.yml file2.yml ...
 ## use '-' as a filename to pipe from STDIN
 cat file2.yml | yq ea '.a.b' file1.yml - file3.yml
 `,
-		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
+		Long: `yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) 
 See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.
 
 ## Evaluate All ##
@@ -70,7 +76,7 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 		}()
 	}
 
-	format, err := yqlib.OutputFormatFromString(outputFormat)
+	format, err := yqlib.FormatFromString(outputFormat)
 	if err != nil {
 		return err
 	}

cmd/evaluate_sequence_command.go

@@ -13,6 +13,12 @@ func createEvaluateSequenceCommand() *cobra.Command {
 		Use:     "eval [expression] [yaml_file1]...",
 		Aliases: []string{"e"},
 		Short:   "(default) Apply the expression to each document in each yaml file in sequence",
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			if len(args) == 0 {
+				return nil, cobra.ShellCompDirectiveNoFileComp
+			}
+			return nil, cobra.ShellCompDirectiveDefault
+		},
 		Example: `
 # Reads field under the given path for each file
 yq e '.a.b' f1.yml f2.yml 
@@ -31,7 +37,7 @@ yq e -n '.a.b.c = "cat"'
 # Update a file in place
 yq e '.a.b = "cool"' -i file.yaml 
 `,
-		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
+		Long: `yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) 
 See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.
 
 ## Evaluate Sequence ##
@@ -84,7 +90,7 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 		}()
 	}
 
-	format, err := yqlib.OutputFormatFromString(outputFormat)
+	format, err := yqlib.FormatFromString(outputFormat)
 	if err != nil {
 		return err
 	}

cmd/root.go

@@ -1,21 +1,54 @@
 package cmd
 
 import (
+	"fmt"
 	"os"
+	"strings"
 
+	"github.com/mikefarah/yq/v4/pkg/properties"
+	"github.com/mikefarah/yq/v4/pkg/xml"
 	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"github.com/spf13/cobra"
 	logging "gopkg.in/op/go-logging.v1"
 )
 
+type runeValue rune
+
+func newRuneVar(p *rune) *runeValue {
+	return (*runeValue)(p)
+}
+
+func (r *runeValue) String() string {
+	return string(*r)
+}
+
+func (r *runeValue) Set(rawVal string) error {
+	val := strings.ReplaceAll(rawVal, "\\n", "\n")
+	val = strings.ReplaceAll(val, "\\t", "\t")
+	val = strings.ReplaceAll(val, "\\r", "\r")
+	val = strings.ReplaceAll(val, "\\f", "\f")
+	val = strings.ReplaceAll(val, "\\v", "\v")
+	if len(val) != 1 {
+		return fmt.Errorf("[%v] is not a valid character. Must be length 1 was %v", val, len(val))
+	}
+	*r = runeValue(rune(val[0]))
+	return nil
+}
+
+func (r *runeValue) Type() string {
+	return "char"
+}
+
 func New() *cobra.Command {
 	var rootCmd = &cobra.Command{
 		Use:   "yq",
-		Short: "yq is a lightweight and portable command-line YAML processor.",
-		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
+		Short: "yq is a lightweight and portable command-line data file processor.",
+		Long: `yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) 
 See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 		Example: `
-# yq defaults to 'eval' command if no command is specified. See "yq eval --help" for more examples.
+# yq tries to auto-detect the file format based off the extension, and defaults to YAML if it's unknown (or piping through STDIN)
+# Use the '-p/--input-format' flag to specify a format type.
+cat file.xml | yq -p xml
 
 # read the "stuff" node from "myfile.yml"
 yq '.stuff' < myfile.yml
@@ -24,7 +57,7 @@ yq '.stuff' < myfile.yml
 yq -i '.stuff = "foo"' myfile.yml
 
 # print contents of sample.json as idiomatic YAML
-yq -P sample.json
+yq -P -oy sample.json
 `,
 
 		RunE: func(cmd *cobra.Command, args []string) error {
@@ -37,18 +70,19 @@ yq -P sample.json
 		},
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 			cmd.SetOut(cmd.OutOrStdout())
+			level := logging.WARNING
+			stringFormat := `[%{level}] %{color}%{time:15:04:05}%{color:reset} %{message}`
 
-			var format = logging.MustStringFormatter(
-				`%{color}%{time:15:04:05} %{shortfunc} [%{level:.4s}]%{color:reset} %{message}`,
-			)
+			if verbose {
+				level = logging.DEBUG
+				stringFormat = `[%{level:5.5s}] %{color}%{time:15:04:05}%{color:bold} %{shortfile:-33s} %{shortfunc:-25s}%{color:reset} %{message}`
+			}
+
+			var format = logging.MustStringFormatter(stringFormat)
 			var backend = logging.AddModuleLevel(
 				logging.NewBackendFormatter(logging.NewLogBackend(os.Stderr, "", 0), format))
 
-			if verbose {
-				backend.SetLevel(logging.DEBUG, "")
-			} else {
-				backend.SetLevel(logging.WARNING, "")
-			}
+			backend.SetLevel(level, "")
 
 			logging.SetBackend(backend)
 			yqlib.InitExpressionParser()
@@ -65,28 +99,76 @@ yq -P sample.json
 		panic(err)
 	}
 
-	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "auto", "[auto|a|yaml|y|json|j|props|p|xml|x|tsv|t|csv|c] output format type.")
-	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "auto", "[auto|a|yaml|y|props|p|xml|x|tsv|t|csv|c|toml] parse format for input. Note that json is a subset of yaml.")
+	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "auto", fmt.Sprintf("[auto|a|%v] output format type.", yqlib.GetAvailableOutputFormatString()))
+	var outputCompletions = []string{"auto"}
+	for _, formats := range yqlib.GetAvailableOutputFormats() {
+		outputCompletions = append(outputCompletions, formats.FormalName)
+	}
 
-	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", yqlib.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes")
-	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", yqlib.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.StrictMode, "xml-strict-mode", yqlib.ConfiguredXMLPreferences.StrictMode, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.KeepNamespace, "xml-keep-namespace", yqlib.ConfiguredXMLPreferences.KeepNamespace, "enables keeping namespace after parsing attributes")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.UseRawToken, "xml-raw-token", yqlib.ConfiguredXMLPreferences.UseRawToken, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.")
-	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ProcInstPrefix, "xml-proc-inst-prefix", yqlib.ConfiguredXMLPreferences.ProcInstPrefix, "prefix for xml processing instructions (e.g. <?xml version=\"1\"?>)")
-	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.DirectiveName, "xml-directive-name", yqlib.ConfiguredXMLPreferences.DirectiveName, "name for xml directives (e.g. <!DOCTYPE thing cat>)")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipProcInst, "xml-skip-proc-inst", yqlib.ConfiguredXMLPreferences.SkipProcInst, "skip over process instructions (e.g. <?xml version=\"1\"?>)")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipDirectives, "xml-skip-directives", yqlib.ConfiguredXMLPreferences.SkipDirectives, "skip over directives (e.g. <!DOCTYPE thing cat>)")
+	if err = rootCmd.RegisterFlagCompletionFunc("output-format", cobra.FixedCompletions(outputCompletions, cobra.ShellCompDirectiveNoFileComp)); err != nil {
+		panic(err)
+	}
+	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "auto", fmt.Sprintf("[auto|a|%v] parse format for input.", yqlib.GetAvailableInputFormatString()))
+
+	var inputCompletions = []string{"auto"}
+	for _, formats := range yqlib.GetAvailableInputFormats() {
+		inputCompletions = append(inputCompletions, formats.FormalName)
+	}
+
+	if err = rootCmd.RegisterFlagCompletionFunc("input-format", cobra.FixedCompletions(inputCompletions, cobra.ShellCompDirectiveNoFileComp)); err != nil {
+		panic(err)
+	}
+
+	rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", xml.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes")
+	if err = rootCmd.RegisterFlagCompletionFunc("xml-attribute-prefix", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
+	rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.ContentName, "xml-content-name", xml.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).")
+	if err = rootCmd.RegisterFlagCompletionFunc("xml-content-name", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
+	rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.StrictMode, "xml-strict-mode", xml.ConfiguredXMLPreferences.StrictMode, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.")
+	rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.KeepNamespace, "xml-keep-namespace", xml.ConfiguredXMLPreferences.KeepNamespace, "enables keeping namespace after parsing attributes")
+	rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.UseRawToken, "xml-raw-token", xml.ConfiguredXMLPreferences.UseRawToken, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.")
+	rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.ProcInstPrefix, "xml-proc-inst-prefix", xml.ConfiguredXMLPreferences.ProcInstPrefix, "prefix for xml processing instructions (e.g. <?xml version=\"1\"?>)")
+	if err = rootCmd.RegisterFlagCompletionFunc("xml-proc-inst-prefix", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
+	rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.DirectiveName, "xml-directive-name", xml.ConfiguredXMLPreferences.DirectiveName, "name for xml directives (e.g. <!DOCTYPE thing cat>)")
+	if err = rootCmd.RegisterFlagCompletionFunc("xml-directive-name", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
+	rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.SkipProcInst, "xml-skip-proc-inst", xml.ConfiguredXMLPreferences.SkipProcInst, "skip over process instructions (e.g. <?xml version=\"1\"?>)")
+	rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.SkipDirectives, "xml-skip-directives", xml.ConfiguredXMLPreferences.SkipDirectives, "skip over directives (e.g. <!DOCTYPE thing cat>)")
+
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredCsvPreferences.AutoParse, "csv-auto-parse", yqlib.ConfiguredCsvPreferences.AutoParse, "parse CSV YAML/JSON values")
+	rootCmd.PersistentFlags().Var(newRuneVar(&yqlib.ConfiguredCsvPreferences.Separator), "csv-separator", "CSV Separator character")
+
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredTsvPreferences.AutoParse, "tsv-auto-parse", yqlib.ConfiguredTsvPreferences.AutoParse, "parse TSV YAML/JSON values")
 
 	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredLuaPreferences.DocPrefix, "lua-prefix", yqlib.ConfiguredLuaPreferences.DocPrefix, "prefix")
+	if err = rootCmd.RegisterFlagCompletionFunc("lua-prefix", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
 	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredLuaPreferences.DocSuffix, "lua-suffix", yqlib.ConfiguredLuaPreferences.DocSuffix, "suffix")
+	if err = rootCmd.RegisterFlagCompletionFunc("lua-suffix", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
 	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.UnquotedKeys, "lua-unquoted", yqlib.ConfiguredLuaPreferences.UnquotedKeys, "output unquoted string keys (e.g. {foo=\"bar\"})")
 	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.Globals, "lua-globals", yqlib.ConfiguredLuaPreferences.Globals, "output keys as top-level global variables")
 
+	rootCmd.PersistentFlags().StringVar(&properties.ConfiguredPropertiesPreferences.KeyValueSeparator, "properties-separator", properties.ConfiguredPropertiesPreferences.KeyValueSeparator, "separator to use between keys and values")
+	rootCmd.PersistentFlags().BoolVar(&properties.ConfiguredPropertiesPreferences.UseArrayBrackets, "properties-array-brackets", properties.ConfiguredPropertiesPreferences.UseArrayBrackets, "use [x] in array paths (e.g. for SpringBoot)")
+
+	rootCmd.PersistentFlags().BoolVar(&yqlib.StringInterpolationEnabled, "string-interpolation", yqlib.StringInterpolationEnabled, "Toggles strings interpolation of \\(exp)")
+
 	rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.")
 	rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)")
 
 	rootCmd.PersistentFlags().IntVarP(&indent, "indent", "I", 2, "sets indent level for output")
+	if err = rootCmd.RegisterFlagCompletionFunc("indent", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
 	rootCmd.Flags().BoolVarP(&version, "version", "V", false, "Print version information and quit")
 	rootCmd.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the file in place of first file given.")
 	rootCmd.PersistentFlags().VarP(unwrapScalarFlag, "unwrapScalar", "r", "unwrap scalar, print the value with no quotes, colors or comments. Defaults to true for yaml")
@@ -99,13 +181,28 @@ yq -P sample.json
 	rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors")
 	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
 	rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
+	if err = rootCmd.RegisterFlagCompletionFunc("front-matter", cobra.FixedCompletions([]string{"extract", "process"}, cobra.ShellCompDirectiveNoFileComp)); err != nil {
+		panic(err)
+	}
 	rootCmd.PersistentFlags().StringVarP(&forceExpression, "expression", "", "", "forcibly set the expression argument. Useful when yq argument detection thinks your expression is a file.")
+	if err = rootCmd.RegisterFlagCompletionFunc("expression", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
 	rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredYamlPreferences.LeadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.")
 
 	rootCmd.PersistentFlags().StringVarP(&splitFileExp, "split-exp", "s", "", "print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter.")
+	if err = rootCmd.RegisterFlagCompletionFunc("split-exp", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
 	rootCmd.PersistentFlags().StringVarP(&splitFileExpFile, "split-exp-file", "", "", "Use a file to specify the split-exp expression.")
+	if err = rootCmd.MarkPersistentFlagFilename("split-exp-file"); err != nil {
+		panic(err)
+	}
 
 	rootCmd.PersistentFlags().StringVarP(&expressionFile, "from-file", "", "", "Load expression from specified file.")
+	if err = rootCmd.MarkPersistentFlagFilename("from-file"); err != nil {
+		panic(err)
+	}
 
 	rootCmd.AddCommand(
 		createEvaluateSequenceCommand(),

cmd/shell-completion.go

@@ -47,7 +47,7 @@ $ yq shell-completion fish > ~/.config/fish/completions/yq.fish
 		var err error = nil
 		switch args[0] {
 		case "bash":
-			err = cmd.Root().GenBashCompletion(os.Stdout)
+			err = cmd.Root().GenBashCompletionV2(os.Stdout, true)
 		case "zsh":
 			err = cmd.Root().GenZshCompletion(os.Stdout)
 		case "fish":

cmd/utils.go

@@ -4,7 +4,10 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strings"
 
+	"github.com/mikefarah/yq/v4/pkg/properties"
+	"github.com/mikefarah/yq/v4/pkg/xml"
 	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"github.com/spf13/cobra"
 	"gopkg.in/op/go-logging.v1"
@@ -63,9 +66,9 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 	}
 	if inputFormat == "" || inputFormat == "auto" || inputFormat == "a" {
 
-		inputFormat = yqlib.FormatFromFilename(inputFilename)
+		inputFormat = yqlib.FormatStringFromFilename(inputFilename)
 
-		_, err := yqlib.InputFormatFromString(inputFormat)
+		_, err := yqlib.FormatFromString(inputFormat)
 		if err != nil {
 			// unknown file type, default to yaml
 			yqlib.GetLogger().Debug("Unknown file format extension '%v', defaulting to yaml", inputFormat)
@@ -85,14 +88,14 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 		// before this was introduced, `yq -pcsv things.csv`
 		// would produce *yaml* output.
 		//
-		outputFormat = yqlib.FormatFromFilename(inputFilename)
+		outputFormat = yqlib.FormatStringFromFilename(inputFilename)
 		if inputFilename != "-" {
 			yqlib.GetLogger().Warning("yq default output is now 'auto' (based on the filename extension). Normally yq would output '%v', but for backwards compatibility 'yaml' has been set. Please use -oy to specify yaml, or drop the -p flag.", outputFormat)
 		}
 		outputFormat = "yaml"
 	}
 
-	outputFormatType, err := yqlib.OutputFormatFromString(outputFormat)
+	outputFormatType, err := yqlib.FormatFromString(outputFormat)
 
 	if err != nil {
 		return "", nil, err
@@ -100,57 +103,32 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 	yqlib.GetLogger().Debug("Using input format %v", inputFormat)
 	yqlib.GetLogger().Debug("Using output format %v", outputFormat)
 
-	if outputFormatType == yqlib.YamlOutputFormat ||
-		outputFormatType == yqlib.PropsOutputFormat {
+	if outputFormatType == yqlib.YamlFormat ||
+		outputFormatType == properties.PropertiesFormat {
 		unwrapScalar = true
 	}
 	if unwrapScalarFlag.IsExplicitlySet() {
 		unwrapScalar = unwrapScalarFlag.IsSet()
 	}
 
-	//copy preference form global setting
-	yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar
-
-	yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators
-
 	return expression, args, nil
 }
 
 func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
-	yqlibInputFormat, err := yqlib.InputFormatFromString(inputFormat)
+	format, err := yqlib.FormatFromString(inputFormat)
 	if err != nil {
 		return nil, err
 	}
-	yqlibDecoder, err := createDecoder(yqlibInputFormat, evaluateTogether)
+	yqlib.ConfiguredYamlPreferences.EvaluateTogether = evaluateTogether
+
+	yqlibDecoder := format.DecoderFactory()
 	if yqlibDecoder == nil {
 		return nil, fmt.Errorf("no support for %s input format", inputFormat)
 	}
-	return yqlibDecoder, err
+	return yqlibDecoder, nil
 }
 
-func createDecoder(format yqlib.InputFormat, evaluateTogether bool) (yqlib.Decoder, error) {
-	switch format {
-	case yqlib.XMLInputFormat:
-		return yqlib.NewXMLDecoder(yqlib.ConfiguredXMLPreferences), nil
-	case yqlib.PropertiesInputFormat:
-		return yqlib.NewPropertiesDecoder(), nil
-	case yqlib.JsonInputFormat:
-		return yqlib.NewJSONDecoder(), nil
-	case yqlib.CSVObjectInputFormat:
-		return yqlib.NewCSVObjectDecoder(','), nil
-	case yqlib.TSVObjectInputFormat:
-		return yqlib.NewCSVObjectDecoder('\t'), nil
-	case yqlib.TomlInputFormat:
-		return yqlib.NewTomlDecoder(), nil
-	case yqlib.YamlInputFormat:
-		prefs := yqlib.ConfiguredYamlPreferences
-		prefs.EvaluateTogether = evaluateTogether
-		return yqlib.NewYamlDecoder(prefs), nil
-	}
-	return nil, fmt.Errorf("invalid decoder: %v", format)
-}
-
-func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yqlib.PrinterWriter, error) {
+func configurePrinterWriter(format *yqlib.Format, out io.Writer) (yqlib.PrinterWriter, error) {
 
 	var printerWriter yqlib.PrinterWriter
 
@@ -168,39 +146,29 @@ func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yq
 }
 
 func configureEncoder() (yqlib.Encoder, error) {
-	yqlibOutputFormat, err := yqlib.OutputFormatFromString(outputFormat)
+	yqlibOutputFormat, err := yqlib.FormatFromString(outputFormat)
 	if err != nil {
 		return nil, err
 	}
-	yqlibEncoder, err := createEncoder(yqlibOutputFormat)
-	if yqlibEncoder == nil {
+	xml.ConfiguredXMLPreferences.Indent = indent
+	yqlib.ConfiguredYamlPreferences.Indent = indent
+	yqlib.ConfiguredJSONPreferences.Indent = indent
+
+	yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar
+	properties.ConfiguredPropertiesPreferences.UnwrapScalar = unwrapScalar
+	yqlib.ConfiguredJSONPreferences.UnwrapScalar = unwrapScalar
+
+	yqlib.ConfiguredYamlPreferences.ColorsEnabled = colorsEnabled
+	yqlib.ConfiguredJSONPreferences.ColorsEnabled = colorsEnabled
+
+	yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators
+
+	encoder := yqlibOutputFormat.EncoderFactory()
+
+	if encoder == nil {
 		return nil, fmt.Errorf("no support for %s output format", outputFormat)
 	}
-	return yqlibEncoder, err
-}
-
-func createEncoder(format yqlib.PrinterOutputFormat) (yqlib.Encoder, error) {
-	switch format {
-	case yqlib.JSONOutputFormat:
-		return yqlib.NewJSONEncoder(indent, colorsEnabled, unwrapScalar), nil
-	case yqlib.PropsOutputFormat:
-		return yqlib.NewPropertiesEncoder(unwrapScalar), nil
-	case yqlib.CSVOutputFormat:
-		return yqlib.NewCsvEncoder(','), nil
-	case yqlib.TSVOutputFormat:
-		return yqlib.NewCsvEncoder('\t'), nil
-	case yqlib.YamlOutputFormat:
-		return yqlib.NewYamlEncoder(indent, colorsEnabled, yqlib.ConfiguredYamlPreferences), nil
-	case yqlib.XMLOutputFormat:
-		return yqlib.NewXMLEncoder(indent, yqlib.ConfiguredXMLPreferences), nil
-	case yqlib.TomlOutputFormat:
-		return yqlib.NewTomlEncoder(), nil
-	case yqlib.ShellVariablesOutputFormat:
-		return yqlib.NewShellVariablesEncoder(), nil
-	case yqlib.LuaOutputFormat:
-		return yqlib.NewLuaEncoder(yqlib.ConfiguredLuaPreferences), nil
-	}
-	return nil, fmt.Errorf("invalid encoder: %v", format)
+	return encoder, err
 }
 
 // this is a hack to enable backwards compatibility with githubactions (which pipe /dev/null into everything)
@@ -224,8 +192,11 @@ func maybeFile(str string) bool {
 }
 
 func processStdInArgs(args []string) []string {
-	stat, _ := os.Stdin.Stat()
-	pipingStdin := (stat.Mode() & os.ModeCharDevice) == 0
+	stat, err := os.Stdin.Stat()
+	if err != nil {
+		yqlib.GetLogger().Debugf("error getting stdin: %v", err)
+	}
+	pipingStdin := stat != nil && (stat.Mode()&os.ModeCharDevice) == 0
 
 	// if we've been given a file, don't automatically
 	// read from stdin.
@@ -249,15 +220,25 @@ func processStdInArgs(args []string) []string {
 
 func processArgs(originalArgs []string) (string, []string, error) {
 	expression := forceExpression
+	args := processStdInArgs(originalArgs)
+	maybeFirstArgIsAFile := len(args) > 0 && maybeFile(args[0])
+
+	if expressionFile == "" && maybeFirstArgIsAFile && strings.HasSuffix(args[0], ".yq") {
+		// lets check if an expression file was given
+		yqlib.GetLogger().Debug("Assuming arg %v is an expression file", args[0])
+		expressionFile = args[0]
+		args = args[1:]
+	}
+
 	if expressionFile != "" {
 		expressionBytes, err := os.ReadFile(expressionFile)
 		if err != nil {
 			return "", nil, err
 		}
-		expression = string(expressionBytes)
+		//replace \r\n (windows) with good ol' unix file endings.
+		expression = strings.ReplaceAll(string(expressionBytes), "\r\n", "\n")
 	}
 
-	args := processStdInArgs(originalArgs)
 	yqlib.GetLogger().Debugf("processed args: %v", args)
 	if expression == "" && len(args) > 0 && args[0] != "-" && !maybeFile(args[0]) {
 		yqlib.GetLogger().Debug("assuming expression is '%v'", args[0])

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "v4.35.2"
+	Version = "v4.42.1"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

debian/changelog

@@ -135,7 +135,7 @@ yq (4.9.6) focal; urgency=medium
 yq (3.3.2) focal; urgency=medium
 
   * Bug fix: existStatus bug (#459)
-  * Automatically makes a os temp directory if it does not exist (#461)
+  * Automatically makes an os temp directory if it does not exist (#461)
 
  -- Roberto Mier Escandon <rmescandon@gmail.com>  Fri, 07 Aug 2020 18:53:01 +0200
 

debian/control

@@ -16,7 +16,7 @@ XSBC-Original-Maintainer: Roberto Mier Escandón <rmescandon@gmail.com>
 Package: yq
 Architecture: any
 Depends: ${shlibs:Depends}, ${misc:Depends}
-Description: lightweight and portable command-line YAML processor
+Description: lightweight and portable command-line data file processor
  .
  The aim of the project is to be the 
  [jq](https://github.com/stedolan/jq) or sed of yaml files.

examples/data.lua

@@ -0,0 +1,9 @@
+return {
+	["country"] = "Australia"; -- this place
+	["cities"] = {
+		"Sydney",
+		"Melbourne",
+		"Brisbane",
+		"Perth",
+	};
+};

examples/data1.yaml

@@ -1 +1,11 @@
-["foobar", "foobaz", "blarp"]
+# block comments come through
+person: # neither do comments on maps
+    name: Mike Wazowski # comments on values appear
+    pets: 
+    - cat # comments on array values appear
+    - dog # comments on array values appear
+    - things:
+        - frog
+    food: [pizza] # comments on arrays do not
+emptyArray: []
+emptyMap: []

examples/environment.yq

@@ -0,0 +1,5 @@
+#! yq
+.[] |(
+    ( select(kind == "scalar") | key + "='" + . + "'"),
+    ( select(kind == "seq") | key + "=(" + (map("'" + . + "'") | join(",")) + ")")
+)

examples/mike.xml

@@ -1 +1 @@
-<cat>3</cat>
+<zoo><thing><frog>boing</frog></thing></zoo>

examples/sample.yaml

@@ -1,11 +1,2 @@
-# Some doc
-
-a: true
-b:
-  c: 2
-  d: [3, 4, 5]
-  e:
-    - name: fred
-      value: 3
-    - name: sam
-      value: 4
+ # things
+a: apple

go.mod

@@ -2,21 +2,22 @@ module github.com/mikefarah/yq/v4
 
 require (
 	github.com/a8m/envsubst v1.4.2
-	github.com/alecthomas/participle/v2 v2.1.0
-	github.com/alecthomas/repr v0.2.0
+	github.com/alecthomas/participle/v2 v2.1.1
+	github.com/alecthomas/repr v0.4.0
 	github.com/dimchansky/utfbom v1.1.1
-	github.com/elliotchance/orderedmap v1.5.0
-	github.com/fatih/color v1.15.0
+	github.com/elliotchance/orderedmap v1.5.1
+	github.com/fatih/color v1.16.0
 	github.com/goccy/go-json v0.10.2
-	github.com/goccy/go-yaml v1.11.2
+	github.com/goccy/go-yaml v1.11.3
 	github.com/jinzhu/copier v0.4.0
 	github.com/magiconair/properties v1.8.7
-	github.com/pelletier/go-toml/v2 v2.1.0
+	github.com/pelletier/go-toml/v2 v2.1.1
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
-	github.com/spf13/cobra v1.7.0
+	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
-	golang.org/x/net v0.15.0
-	golang.org/x/text v0.13.0
+	github.com/yuin/gopher-lua v1.1.1
+	golang.org/x/net v0.22.0
+	golang.org/x/text v0.14.0
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -24,9 +25,9 @@ require (
 require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.17 // indirect
-	golang.org/x/sys v0.12.0 // indirect
-	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 )
 
-go 1.20
+go 1.21

go.sum

@@ -1,45 +1,57 @@
 github.com/a8m/envsubst v1.4.2 h1:4yWIHXOLEJHQEFd4UjrWDrYeYlV7ncFWJOCBRLOZHQg=
 github.com/a8m/envsubst v1.4.2/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
 github.com/alecthomas/assert/v2 v2.3.0 h1:mAsH2wmvjsuvyBvAmCtm7zFsBlb8mIHx5ySLVdDZXL0=
-github.com/alecthomas/participle/v2 v2.1.0 h1:z7dElHRrOEEq45F2TG5cbQihMtNTv8vwldytDj7Wrz4=
-github.com/alecthomas/participle/v2 v2.1.0/go.mod h1:Y1+hAs8DHPmc3YUFzqllV+eSQ9ljPTk0ZkPMtEdAx2c=
-github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk=
-github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
-github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/alecthomas/assert/v2 v2.3.0/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ=
+github.com/alecthomas/participle/v2 v2.1.1 h1:hrjKESvSqGHzRb4yW1ciisFJ4p3MGYih6icjJvbsmV8=
+github.com/alecthomas/participle/v2 v2.1.1/go.mod h1:Y1+hAs8DHPmc3YUFzqllV+eSQ9ljPTk0ZkPMtEdAx2c=
+github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
+github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
 github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
-github.com/elliotchance/orderedmap v1.5.0 h1:1IsExUsjv5XNBD3ZdC7jkAAqLWOOKdbPTmkHx63OsBg=
-github.com/elliotchance/orderedmap v1.5.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
-github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
-github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
+github.com/elliotchance/orderedmap v1.5.1 h1:G1X4PYlljzimbdQ3RXmtIZiQ9d6aRQ3sH1nzjq5mECE=
+github.com/elliotchance/orderedmap v1.5.1/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
+github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
+github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
+github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
+github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
+github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
+github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
+github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
-github.com/goccy/go-yaml v1.11.2 h1:joq77SxuyIs9zzxEjgyLBugMQ9NEgTWxXfz2wVqwAaQ=
-github.com/goccy/go-yaml v1.11.2/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU=
+github.com/goccy/go-yaml v1.11.3 h1:B3W9IdWbvrUu2OYQGwvU1nZtvMQJPBKgBUuweJjLj6I=
+github.com/goccy/go-yaml v1.11.3/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
+github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
 github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
+github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
+github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
 github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
-github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
-github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
+github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
-github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
+github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
+github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -50,15 +62,20 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
-golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
+github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
+github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
+golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
-golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0=
-golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
+golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW8s2qTSe3wGBtvo0MbVQG/c5k8RE=

pkg/properties/decoder_properties.go

@@ -0,0 +1,117 @@
+package properties
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+
+	"github.com/magiconair/properties"
+	"github.com/mikefarah/yq/v4/pkg/yqlib"
+)
+
+type propertiesDecoder struct {
+	reader   io.Reader
+	finished bool
+	d        yqlib.DataTreeNavigator
+}
+
+func NewPropertiesDecoder() yqlib.Decoder {
+	return &propertiesDecoder{d: yqlib.NewDataTreeNavigator(), finished: false}
+}
+
+func (dec *propertiesDecoder) Init(reader io.Reader) error {
+	dec.reader = reader
+	dec.finished = false
+	return nil
+}
+
+func parsePropKey(key string) []interface{} {
+	pathStrArray := strings.Split(key, ".")
+	path := make([]interface{}, len(pathStrArray))
+	for i, pathStr := range pathStrArray {
+		num, err := strconv.ParseInt(pathStr, 10, 32)
+		if err == nil {
+			path[i] = num
+		} else {
+			path[i] = pathStr
+		}
+	}
+	return path
+}
+
+func (dec *propertiesDecoder) processComment(c string) string {
+	if c == "" {
+		return ""
+	}
+	return "# " + c
+}
+
+func (dec *propertiesDecoder) applyPropertyComments(context yqlib.Context, path []interface{}, comments []string) error {
+	rhsCandidateNode := &yqlib.CandidateNode{
+		Tag:         "!!str",
+		Value:       fmt.Sprintf("%v", path[len(path)-1]),
+		HeadComment: dec.processComment(strings.Join(comments, "\n")),
+		Kind:        yqlib.ScalarNode,
+	}
+	rhsCandidateNode.Tag = rhsCandidateNode.GuessTagFromCustomType()
+	return dec.d.DeeplyAssignKey(context, path, rhsCandidateNode)
+}
+
+func (dec *propertiesDecoder) applyProperty(context yqlib.Context, properties *properties.Properties, key string) error {
+	value, _ := properties.Get(key)
+	path := parsePropKey(key)
+
+	propertyComments := properties.GetComments(key)
+	if len(propertyComments) > 0 {
+		err := dec.applyPropertyComments(context, path, propertyComments)
+		if err != nil {
+			return nil
+		}
+	}
+
+	rhsNode := yqlib.CreateStringScalarNode(value)
+	rhsNode.Tag = rhsNode.GuessTagFromCustomType()
+
+	return dec.d.DeeplyAssign(context, path, rhsNode)
+}
+
+func (dec *propertiesDecoder) Decode() (*yqlib.CandidateNode, error) {
+	if dec.finished {
+		return nil, io.EOF
+	}
+	buf := new(bytes.Buffer)
+
+	if _, err := buf.ReadFrom(dec.reader); err != nil {
+		return nil, err
+	}
+	if buf.Len() == 0 {
+		dec.finished = true
+		return nil, io.EOF
+	}
+	properties, err := properties.LoadString(buf.String())
+	if err != nil {
+		return nil, err
+	}
+	properties.DisableExpansion = true
+
+	rootMap := &yqlib.CandidateNode{
+		Kind: yqlib.MappingNode,
+		Tag:  "!!map",
+	}
+
+	context := yqlib.Context{}
+	context = context.SingleChildContext(rootMap)
+
+	for _, key := range properties.Keys() {
+		if err := dec.applyProperty(context, properties, key); err != nil {
+			return nil, err
+		}
+
+	}
+	dec.finished = true
+
+	return rootMap, nil
+
+}

pkg/properties/encoder_properties.go

@@ -1,4 +1,4 @@
-package yqlib
+package properties
 
 import (
 	"bufio"
@@ -8,16 +8,16 @@ import (
 	"strings"
 
 	"github.com/magiconair/properties"
-	yaml "gopkg.in/yaml.v3"
+	"github.com/mikefarah/yq/v4/pkg/yqlib"
 )
 
 type propertiesEncoder struct {
-	unwrapScalar bool
+	prefs PropertiesPreferences
 }
 
-func NewPropertiesEncoder(unwrapScalar bool) Encoder {
+func NewPropertiesEncoder(prefs PropertiesPreferences) yqlib.Encoder {
 	return &propertiesEncoder{
-		unwrapScalar: unwrapScalar,
+		prefs: prefs,
 	}
 }
 
@@ -25,7 +25,7 @@ func (pe *propertiesEncoder) CanHandleAliases() bool {
 	return false
 }
 
-func (pe *propertiesEncoder) PrintDocumentSeparator(writer io.Writer) error {
+func (pe *propertiesEncoder) PrintDocumentSeparator(_ io.Writer) error {
 	return nil
 }
 
@@ -44,7 +44,7 @@ func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content strin
 			}
 
 		} else {
-			if err := writeString(writer, readline); err != nil {
+			if err := yqlib.WriteString(writer, readline); err != nil {
 				return err
 			}
 		}
@@ -52,7 +52,7 @@ func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content strin
 		if errors.Is(errReading, io.EOF) {
 			if readline != "" {
 				// the last comment we read didn't have a newline, put one in
-				if err := writeString(writer, "\n"); err != nil {
+				if err := yqlib.WriteString(writer, "\n"); err != nil {
 					return err
 				}
 			}
@@ -62,14 +62,15 @@ func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content strin
 	return nil
 }
 
-func (pe *propertiesEncoder) Encode(writer io.Writer, node *yaml.Node) error {
+func (pe *propertiesEncoder) Encode(writer io.Writer, node *yqlib.CandidateNode) error {
 
-	if node.Kind == yaml.ScalarNode {
-		return writeString(writer, node.Value+"\n")
+	if node.Kind == yqlib.ScalarNode {
+		return yqlib.WriteString(writer, node.Value+"\n")
 	}
 
-	mapKeysToStrings(node)
+	node.ConvertKeysToStrings()
 	p := properties.NewProperties()
+	p.WriteSeparator = pe.prefs.KeyValueSeparator
 	err := pe.doEncode(p, node, "", nil)
 	if err != nil {
 		return err
@@ -79,34 +80,32 @@ func (pe *propertiesEncoder) Encode(writer io.Writer, node *yaml.Node) error {
 	return err
 }
 
-func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yaml.Node, path string, keyNode *yaml.Node) error {
+func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yqlib.CandidateNode, path string, keyNode *yqlib.CandidateNode) error {
 
 	comments := ""
 	if keyNode != nil {
 		// include the key node comments if present
-		comments = headAndLineComment(keyNode)
+		comments = keyNode.CleanHeadAndLineComment()
 	}
-	comments = comments + headAndLineComment(node)
+	comments = comments + node.CleanHeadAndLineComment()
 	commentsWithSpaces := strings.ReplaceAll(comments, "\n", "\n ")
 	p.SetComments(path, strings.Split(commentsWithSpaces, "\n"))
 
 	switch node.Kind {
-	case yaml.ScalarNode:
+	case yqlib.ScalarNode:
 		var nodeValue string
-		if pe.unwrapScalar || !strings.Contains(node.Value, " ") {
+		if pe.prefs.UnwrapScalar || !strings.Contains(node.Value, " ") {
 			nodeValue = node.Value
 		} else {
 			nodeValue = fmt.Sprintf("%q", node.Value)
 		}
 		_, _, err := p.Set(path, nodeValue)
 		return err
-	case yaml.DocumentNode:
-		return pe.doEncode(p, node.Content[0], path, node)
-	case yaml.SequenceNode:
+	case yqlib.SequenceNode:
 		return pe.encodeArray(p, node.Content, path)
-	case yaml.MappingNode:
+	case yqlib.MappingNode:
 		return pe.encodeMap(p, node.Content, path)
-	case yaml.AliasNode:
+	case yqlib.AliasNode:
 		return pe.doEncode(p, node.Alias, path, nil)
 	default:
 		return fmt.Errorf("Unsupported node %v", node.Tag)
@@ -117,10 +116,17 @@ func (pe *propertiesEncoder) appendPath(path string, key interface{}) string {
 	if path == "" {
 		return fmt.Sprintf("%v", key)
 	}
+	switch key.(type) {
+	case int:
+		if pe.prefs.UseArrayBrackets {
+			return fmt.Sprintf("%v[%v]", path, key)
+		}
+
+	}
 	return fmt.Sprintf("%v.%v", path, key)
 }
 
-func (pe *propertiesEncoder) encodeArray(p *properties.Properties, kids []*yaml.Node, path string) error {
+func (pe *propertiesEncoder) encodeArray(p *properties.Properties, kids []*yqlib.CandidateNode, path string) error {
 	for index, child := range kids {
 		err := pe.doEncode(p, child, pe.appendPath(path, index), nil)
 		if err != nil {
@@ -130,7 +136,7 @@ func (pe *propertiesEncoder) encodeArray(p *properties.Properties, kids []*yaml.
 	return nil
 }
 
-func (pe *propertiesEncoder) encodeMap(p *properties.Properties, kids []*yaml.Node, path string) error {
+func (pe *propertiesEncoder) encodeMap(p *properties.Properties, kids []*yqlib.CandidateNode, path string) error {
 	for index := 0; index < len(kids); index = index + 2 {
 		key := kids[index]
 		value := kids[index+1]

pkg/properties/encoder_properties_test.go

@@ -0,0 +1,208 @@
+package properties
+
+import (
+	"bufio"
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+type keyValuePair struct {
+	key     string
+	value   string
+	comment string
+}
+
+func (kv *keyValuePair) String(unwrap bool, sep string) string {
+	builder := strings.Builder{}
+
+	if kv.comment != "" {
+		builder.WriteString(kv.comment)
+		builder.WriteString("\n")
+	}
+
+	builder.WriteString(kv.key)
+	builder.WriteString(sep)
+
+	if unwrap {
+		builder.WriteString(kv.value)
+	} else {
+		builder.WriteString("\"")
+		builder.WriteString(kv.value)
+		builder.WriteString("\"")
+	}
+
+	return builder.String()
+}
+
+type testProperties struct {
+	pairs []keyValuePair
+}
+
+func (tp *testProperties) String(unwrap bool, sep string) string {
+	kvs := []string{}
+
+	for _, kv := range tp.pairs {
+		kvs = append(kvs, kv.String(unwrap, sep))
+	}
+
+	return strings.Join(kvs, "\n")
+}
+
+func yamlToProps(sampleYaml string, unwrapScalar bool, separator string) string {
+	var output bytes.Buffer
+	writer := bufio.NewWriter(&output)
+
+	var propsEncoder = NewPropertiesEncoder(PropertiesPreferences{KeyValueSeparator: separator, UnwrapScalar: unwrapScalar})
+	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
+	if err != nil {
+		panic(err)
+	}
+	node := inputs.Front().Value.(*CandidateNode)
+	err = propsEncoder.Encode(writer, node)
+	if err != nil {
+		panic(err)
+	}
+	writer.Flush()
+
+	return strings.TrimSuffix(output.String(), "\n")
+}
+
+func doTest(t *testing.T, sampleYaml string, props testProperties, testUnwrapped, testWrapped bool) {
+	wraps := []bool{}
+	if testUnwrapped {
+		wraps = append(wraps, true)
+	}
+	if testWrapped {
+		wraps = append(wraps, false)
+	}
+
+	for _, unwrap := range wraps {
+		for _, sep := range []string{" = ", ";", "=", " "} {
+			var actualProps = yamlToProps(sampleYaml, unwrap, sep)
+			test.AssertResult(t, props.String(unwrap, sep), actualProps)
+		}
+	}
+}
+
+func TestPropertiesEncoderSimple(t *testing.T) {
+	var sampleYaml = `a: 'bob cool'`
+
+	doTest(
+		t, sampleYaml,
+		testProperties{
+			pairs: []keyValuePair{
+				{
+					key:   "a",
+					value: "bob cool",
+				},
+			},
+		},
+		true, true,
+	)
+}
+
+func TestPropertiesEncoderSimpleWithComments(t *testing.T) {
+	var sampleYaml = `a: 'bob cool' # line`
+
+	doTest(
+		t, sampleYaml,
+		testProperties{
+			pairs: []keyValuePair{
+				{
+					key:     "a",
+					value:   "bob cool",
+					comment: "# line",
+				},
+			},
+		},
+		true, true,
+	)
+}
+
+func TestPropertiesEncoderDeep(t *testing.T) {
+	var sampleYaml = `a: 
+  b: "bob cool"
+`
+
+	doTest(
+		t, sampleYaml,
+		testProperties{
+			pairs: []keyValuePair{
+				{
+					key:   "a.b",
+					value: "bob cool",
+				},
+			},
+		},
+		true, true,
+	)
+}
+
+func TestPropertiesEncoderDeepWithComments(t *testing.T) {
+	var sampleYaml = `a:  # a thing
+  b: "bob cool" # b thing
+`
+
+	doTest(
+		t, sampleYaml,
+		testProperties{
+			pairs: []keyValuePair{
+				{
+					key:     "a.b",
+					value:   "bob cool",
+					comment: "# b thing",
+				},
+			},
+		},
+		true, true,
+	)
+}
+
+func TestPropertiesEncoderArray_Unwrapped(t *testing.T) {
+	var sampleYaml = `a: 
+  b: [{c: dog}, {c: cat}]
+`
+
+	doTest(
+		t, sampleYaml,
+		testProperties{
+			pairs: []keyValuePair{
+				{
+					key:   "a.b.0.c",
+					value: "dog",
+				},
+				{
+					key:   "a.b.1.c",
+					value: "cat",
+				},
+			},
+		},
+		true, false,
+	)
+}
+
+func TestPropertiesEncoderArray_Wrapped(t *testing.T) {
+	var sampleYaml = `a: 
+  b: [{c: dog named jim}, {c: cat named jim}]
+`
+
+	doTest(
+		t, sampleYaml,
+		testProperties{
+			pairs: []keyValuePair{
+				{
+					key:   "a.b.0.c",
+					value: "dog named jim",
+				},
+				{
+					key:   "a.b.1.c",
+					value: "cat named jim",
+				},
+			},
+		},
+		false, true,
+	)
+}

pkg/properties/properties.go

@@ -0,0 +1,45 @@
+package properties
+
+import "github.com/mikefarah/yq/v4/pkg/yqlib"
+
+type PropertiesPreferences struct {
+	UnwrapScalar      bool
+	KeyValueSeparator string
+	UseArrayBrackets  bool
+}
+
+func NewDefaultPropertiesPreferences() PropertiesPreferences {
+	return PropertiesPreferences{
+		UnwrapScalar:      true,
+		KeyValueSeparator: " = ",
+		UseArrayBrackets:  false,
+	}
+}
+
+func (p *PropertiesPreferences) Copy() PropertiesPreferences {
+	return PropertiesPreferences{
+		UnwrapScalar:      p.UnwrapScalar,
+		KeyValueSeparator: p.KeyValueSeparator,
+		UseArrayBrackets:  p.UseArrayBrackets,
+	}
+}
+
+var PropertiesFormat = &yqlib.Format{"props", []string{"p", "properties"}, "properties",
+	func() yqlib.Encoder { return NewPropertiesEncoder(ConfiguredPropertiesPreferences) },
+	func() yqlib.Decoder { return NewPropertiesDecoder() },
+	nil,
+}
+
+var propertyYqRules = []*yqlib.ParticipleYqRule{
+	{"PropertiesDecode", `from_?props|@propsd`, yqlib.CreateDecodeOpYqAction(PropertiesFormat), 0},
+	{"PropsEncode", `to_?props|@props`, yqlib.CreateEncodeOpYqAction(PropertiesFormat, 2), 0},
+	{"LoadProperties", `load_?props`, yqlib.CreateLoadOpYqAction(NewPropertiesDecoder()), 0},
+}
+
+func RegisterPropertiesFormat() {
+	yqlib.RegisterFormat(PropertiesFormat)
+	yqlib.RegisterRules(propertyYqRules)
+
+}
+
+var ConfiguredPropertiesPreferences = NewDefaultPropertiesPreferences()

pkg/xml/decoder_xml.go

@@ -1,6 +1,6 @@
 //go:build !yq_noxml
 
-package yqlib
+package xml
 
 import (
 	"encoding/xml"
@@ -11,8 +11,8 @@ import (
 	"strings"
 	"unicode"
 
+	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"golang.org/x/net/html/charset"
-	yaml "gopkg.in/yaml.v3"
 )
 
 type xmlDecoder struct {
@@ -22,7 +22,7 @@ type xmlDecoder struct {
 	prefs        XmlPreferences
 }
 
-func NewXMLDecoder(prefs XmlPreferences) Decoder {
+func NewXMLDecoder(prefs XmlPreferences) yqlib.Decoder {
 	return &xmlDecoder{
 		finished: false,
 		prefs:    prefs,
@@ -36,14 +36,14 @@ func (dec *xmlDecoder) Init(reader io.Reader) error {
 	return nil
 }
 
-func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yaml.Node, error) {
-	yamlNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yqlib.CandidateNode, error) {
+	yamlNode := &yqlib.CandidateNode{Kind: yqlib.SequenceNode, Tag: "!!seq"}
 	for _, child := range nodes {
 		yamlChild, err := dec.convertToYamlNode(child)
 		if err != nil {
 			return nil, err
 		}
-		yamlNode.Content = append(yamlNode.Content, yamlChild)
+		yamlNode.AddChild(yamlChild)
 	}
 
 	return yamlNode, nil
@@ -64,36 +64,36 @@ func (dec *xmlDecoder) processComment(c string) string {
 	return replacement
 }
 
-func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
-	log.Debug("createMap: headC: %v, lineC: %v, footC: %v", n.HeadComment, n.LineComment, n.FootComment)
-	yamlNode := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
+func (dec *xmlDecoder) createMap(n *xmlNode) (*yqlib.CandidateNode, error) {
+	yqlib.GetLogger().Debug("createMap: headC: %v, lineC: %v, footC: %v", n.HeadComment, n.LineComment, n.FootComment)
+	yamlNode := &yqlib.CandidateNode{Kind: yqlib.MappingNode, Tag: "!!map"}
 
 	if len(n.Data) > 0 {
-		log.Debugf("creating content node for map: %v", dec.prefs.ContentName)
+		yqlib.GetLogger().Debugf("creating content node for map: %v", dec.prefs.ContentName)
 		label := dec.prefs.ContentName
-		labelNode := createScalarNode(label, label)
+		labelNode := yqlib.CreateScalarNode(label, label)
 		labelNode.HeadComment = dec.processComment(n.HeadComment)
 		labelNode.LineComment = dec.processComment(n.LineComment)
 		labelNode.FootComment = dec.processComment(n.FootComment)
-		yamlNode.Content = append(yamlNode.Content, labelNode, dec.createValueNodeFromData(n.Data))
+		yamlNode.AddKeyValueChild(labelNode, dec.createValueNodeFromData(n.Data))
 	}
 
 	for i, keyValuePair := range n.Children {
 		label := keyValuePair.K
 		children := keyValuePair.V
-		labelNode := createScalarNode(label, label)
-		var valueNode *yaml.Node
+		labelNode := yqlib.CreateScalarNode(label, label)
+		var valueNode *yqlib.CandidateNode
 		var err error
 
 		if i == 0 {
-			log.Debugf("head comment here")
+			yqlib.GetLogger().Debugf("head comment here")
 			labelNode.HeadComment = dec.processComment(n.HeadComment)
 
 		}
-		log.Debugf("label=%v, i=%v, keyValuePair.FootComment: %v", label, i, keyValuePair.FootComment)
+		yqlib.GetLogger().Debugf("label=%v, i=%v, keyValuePair.FootComment: %v", label, i, keyValuePair.FootComment)
 		labelNode.FootComment = dec.processComment(keyValuePair.FootComment)
 
-		log.Debug("len of children in %v is %v", label, len(children))
+		yqlib.GetLogger().Debug("len of children in %v is %v", label, len(children))
 		if len(children) > 1 {
 			valueNode, err = dec.createSequence(children)
 			if err != nil {
@@ -106,7 +106,7 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
 			if len(children[0].Children) == 0 && children[0].HeadComment != "" {
 				if len(children[0].Data) > 0 {
 
-					log.Debug("scalar comment hack, currentlabel [%v]", labelNode.HeadComment)
+					yqlib.GetLogger().Debug("scalar comment hack, currentlabel [%v]", labelNode.HeadComment)
 					labelNode.HeadComment = joinComments([]string{labelNode.HeadComment, strings.TrimSpace(children[0].HeadComment)}, "\n")
 					children[0].HeadComment = ""
 				} else {
@@ -120,39 +120,39 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
 				return nil, err
 			}
 		}
-		yamlNode.Content = append(yamlNode.Content, labelNode, valueNode)
+		yamlNode.AddKeyValueChild(labelNode, valueNode)
 	}
 
 	return yamlNode, nil
 }
 
-func (dec *xmlDecoder) createValueNodeFromData(values []string) *yaml.Node {
+func (dec *xmlDecoder) createValueNodeFromData(values []string) *yqlib.CandidateNode {
 	switch len(values) {
 	case 0:
-		return createScalarNode(nil, "")
+		return yqlib.CreateScalarNode(nil, "")
 	case 1:
-		return createScalarNode(values[0], values[0])
+		return yqlib.CreateScalarNode(values[0], values[0])
 	default:
-		content := make([]*yaml.Node, 0)
+		content := make([]*yqlib.CandidateNode, 0)
 		for _, value := range values {
-			content = append(content, createScalarNode(value, value))
+			content = append(content, yqlib.CreateScalarNode(value, value))
 		}
-		return &yaml.Node{
-			Kind:    yaml.SequenceNode,
+		return &yqlib.CandidateNode{
+			Kind:    yqlib.SequenceNode,
 			Tag:     "!!seq",
 			Content: content,
 		}
 	}
 }
 
-func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yaml.Node, error) {
+func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yqlib.CandidateNode, error) {
 	if len(n.Children) > 0 {
 		return dec.createMap(n)
 	}
 
 	scalar := dec.createValueNodeFromData(n.Data)
 
-	log.Debug("scalar (%v), headC: %v, lineC: %v, footC: %v", scalar.Tag, n.HeadComment, n.LineComment, n.FootComment)
+	yqlib.GetLogger().Debug("scalar (%v), headC: %v, lineC: %v, footC: %v", scalar.Tag, n.HeadComment, n.LineComment, n.FootComment)
 	scalar.HeadComment = dec.processComment(n.HeadComment)
 	scalar.LineComment = dec.processComment(n.LineComment)
 	if scalar.Tag == "!!seq" {
@@ -165,7 +165,7 @@ func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yaml.Node, error) {
 	return scalar, nil
 }
 
-func (dec *xmlDecoder) Decode() (*CandidateNode, error) {
+func (dec *xmlDecoder) Decode() (*yqlib.CandidateNode, error) {
 	if dec.finished {
 		return nil, io.EOF
 	}
@@ -189,12 +189,7 @@ func (dec *xmlDecoder) Decode() (*CandidateNode, error) {
 	dec.readAnything = true
 	dec.finished = true
 
-	return &CandidateNode{
-		Node: &yaml.Node{
-			Kind:    yaml.DocumentNode,
-			Content: []*yaml.Node{firstNode},
-		},
-	}, nil
+	return firstNode, nil
 }
 
 type xmlNode struct {
@@ -217,17 +212,17 @@ func (n *xmlNode) AddChild(s string, c *xmlNode) {
 	if n.Children == nil {
 		n.Children = make([]*xmlChildrenKv, 0)
 	}
-	log.Debug("looking for %s", s)
+	yqlib.GetLogger().Debug("looking for %s", s)
 	// see if we can find an existing entry to add to
 	for _, childEntry := range n.Children {
 		if childEntry.K == s {
-			log.Debug("found it, appending an entry%s", s)
+			yqlib.GetLogger().Debug("found it, appending an entry%s", s)
 			childEntry.V = append(childEntry.V, c)
-			log.Debug("yay len of children in %v is %v", s, len(childEntry.V))
+			yqlib.GetLogger().Debug("yay len of children in %v is %v", s, len(childEntry.V))
 			return
 		}
 	}
-	log.Debug("not there, making a new one %s", s)
+	yqlib.GetLogger().Debug("not there, making a new one %s", s)
 	n.Children = append(n.Children, &xmlChildrenKv{K: s, V: []*xmlNode{c}})
 }
 
@@ -273,7 +268,7 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
 
 		switch se := t.(type) {
 		case xml.StartElement:
-			log.Debug("start element %v", se.Name.Local)
+			yqlib.GetLogger().Debug("start element %v", se.Name.Local)
 			elem.state = "started"
 			// Build new a new current element and link it to its parent
 			elem = &element{
@@ -302,10 +297,14 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
 			if len(newBit) > 0 {
 				elem.n.Data = append(elem.n.Data, newBit)
 				elem.state = "chardata"
-				log.Debug("chardata [%v] for %v", elem.n.Data, elem.label)
+				yqlib.GetLogger().Debug("chardata [%v] for %v", elem.n.Data, elem.label)
 			}
 		case xml.EndElement:
-			log.Debug("end element %v", elem.label)
+			if elem == nil {
+				yqlib.GetLogger().Debug("no element, probably bad xml")
+				continue
+			}
+			yqlib.GetLogger().Debug("end element %v", elem.label)
 			elem.state = "finished"
 			// And add it to its parent list
 			if elem.parent != nil {
@@ -321,10 +320,10 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
 				applyFootComment(elem, commentStr)
 
 			} else if elem.state == "chardata" {
-				log.Debug("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
+				yqlib.GetLogger().Debug("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
 				elem.n.LineComment = joinComments([]string{elem.n.LineComment, commentStr}, " ")
 			} else {
-				log.Debug("got a head comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
+				yqlib.GetLogger().Debug("got a head comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
 				elem.n.HeadComment = joinComments([]string{elem.n.HeadComment, commentStr}, " ")
 			}
 
@@ -349,7 +348,7 @@ func applyFootComment(elem *element, commentStr string) {
 	if len(elem.n.Children) > 0 {
 		lastChildIndex := len(elem.n.Children) - 1
 		childKv := elem.n.Children[lastChildIndex]
-		log.Debug("got a foot comment, putting on last child for %v: [%v]", childKv.K, commentStr)
+		yqlib.GetLogger().Debug("got a foot comment, putting on last child for %v: [%v]", childKv.K, commentStr)
 		// if it's an array of scalars, put the foot comment on the scalar itself
 		if len(childKv.V) > 0 && len(childKv.V[0].Children) == 0 {
 			nodeToUpdate := childKv.V[len(childKv.V)-1]
@@ -358,7 +357,7 @@ func applyFootComment(elem *element, commentStr string) {
 			childKv.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, " ")
 		}
 	} else {
-		log.Debug("got a foot comment for %v: [%v]", elem.label, commentStr)
+		yqlib.GetLogger().Debug("got a foot comment for %v: [%v]", elem.label, commentStr)
 		elem.n.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, " ")
 	}
 }

pkg/xml/encoder_xml.go

@@ -1,6 +1,6 @@
 //go:build !yq_noxml
 
-package yqlib
+package xml
 
 import (
 	"encoding/xml"
@@ -9,7 +9,7 @@ import (
 	"regexp"
 	"strings"
 
-	yaml "gopkg.in/yaml.v3"
+	"github.com/mikefarah/yq/v4/pkg/yqlib"
 )
 
 type xmlEncoder struct {
@@ -19,10 +19,10 @@ type xmlEncoder struct {
 	leadingContent string
 }
 
-func NewXMLEncoder(indent int, prefs XmlPreferences) Encoder {
+func NewXMLEncoder(prefs XmlPreferences) yqlib.Encoder {
 	var indentString = ""
 
-	for index := 0; index < indent; index++ {
+	for index := 0; index < prefs.Indent; index++ {
 		indentString = indentString + " "
 	}
 	return &xmlEncoder{indentString, nil, prefs, ""}
@@ -32,28 +32,27 @@ func (e *xmlEncoder) CanHandleAliases() bool {
 	return false
 }
 
-func (e *xmlEncoder) PrintDocumentSeparator(writer io.Writer) error {
+func (e *xmlEncoder) PrintDocumentSeparator(_ io.Writer) error {
 	return nil
 }
 
-func (e *xmlEncoder) PrintLeadingContent(writer io.Writer, content string) error {
+func (e *xmlEncoder) PrintLeadingContent(_ io.Writer, content string) error {
 	e.leadingContent = content
 	return nil
 }
 
-func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
+func (e *xmlEncoder) Encode(writer io.Writer, node *yqlib.CandidateNode) error {
 	encoder := xml.NewEncoder(writer)
 	// hack so we can manually add newlines to procInst and directives
 	e.writer = writer
 	encoder.Indent("", e.indentString)
 	var newLine xml.CharData = []byte("\n")
 
-	mapNode := unwrapDoc(node)
-	if mapNode.Tag == "!!map" {
+	if node.Tag == "!!map" {
 		// make sure <?xml .. ?> processing instructions are encoded first
-		for i := 0; i < len(mapNode.Content); i += 2 {
-			key := mapNode.Content[i]
-			value := mapNode.Content[i+1]
+		for i := 0; i < len(node.Content); i += 2 {
+			key := node.Content[i]
+			value := node.Content[i+1]
 
 			if key.Value == (e.prefs.ProcInstPrefix + "xml") {
 				name := strings.Replace(key.Value, e.prefs.ProcInstPrefix, "", 1)
@@ -62,7 +61,7 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
 					return err
 				}
 				if _, err := e.writer.Write([]byte("\n")); err != nil {
-					log.Warning("Unable to write newline, skipping: %w", err)
+					yqlib.GetLogger().Warning("Unable to write newline, skipping: %w", err)
 				}
 			}
 		}
@@ -82,29 +81,12 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
 	}
 
 	switch node.Kind {
-	case yaml.MappingNode:
+	case yqlib.MappingNode:
 		err := e.encodeTopLevelMap(encoder, node)
 		if err != nil {
 			return err
 		}
-	case yaml.DocumentNode:
-		err := e.encodeComment(encoder, headAndLineComment(node))
-		if err != nil {
-			return err
-		}
-		unwrappedNode := unwrapDoc(node)
-		if unwrappedNode.Kind != yaml.MappingNode {
-			return fmt.Errorf("cannot encode %v to XML - only maps can be encoded", unwrappedNode.Tag)
-		}
-		err = e.encodeTopLevelMap(encoder, unwrappedNode)
-		if err != nil {
-			return err
-		}
-		err = e.encodeComment(encoder, footComment(node))
-		if err != nil {
-			return err
-		}
-	case yaml.ScalarNode:
+	case yqlib.ScalarNode:
 		var charData xml.CharData = []byte(node.Value)
 		err := encoder.EncodeToken(charData)
 		if err != nil {
@@ -112,15 +94,15 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
 		}
 		return encoder.Flush()
 	default:
-		return fmt.Errorf("unsupported type %v", node.Tag)
+		return fmt.Errorf("cannot encode %v to XML - only maps can be encoded", node.Tag)
 	}
 
 	return encoder.EncodeToken(newLine)
 
 }
 
-func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node) error {
-	err := e.encodeComment(encoder, headAndLineComment(node))
+func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yqlib.CandidateNode) error {
+	err := e.encodeComment(encoder, node.CleanHeadAndLineComment())
 	if err != nil {
 		return err
 	}
@@ -129,12 +111,12 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node) er
 		value := node.Content[i+1]
 
 		start := xml.StartElement{Name: xml.Name{Local: key.Value}}
-		log.Debugf("comments of key %v", key.Value)
-		err := e.encodeComment(encoder, headAndLineComment(key))
+		yqlib.GetLogger().Debugf("comments of key %v", key.Value)
+		err := e.encodeComment(encoder, key.CleanHeadAndLineComment())
 		if err != nil {
 			return err
 		}
-		if headAndLineComment(key) != "" {
+		if key.CleanHeadAndLineComment() != "" {
 			var newLine xml.CharData = []byte("\n")
 			err = encoder.EncodeToken(newLine)
 			if err != nil {
@@ -142,7 +124,7 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node) er
 			}
 		}
 
-		if key.Value == (e.prefs.ProcInstPrefix + "xml") {
+		if key.Value == (e.prefs.ProcInstPrefix + "xml") { //nolint
 			// dont double process these.
 		} else if strings.HasPrefix(key.Value, e.prefs.ProcInstPrefix) {
 			name := strings.Replace(key.Value, e.prefs.ProcInstPrefix, "", 1)
@@ -151,7 +133,7 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node) er
 				return err
 			}
 			if _, err := e.writer.Write([]byte("\n")); err != nil {
-				log.Warning("Unable to write newline, skipping: %w", err)
+				yqlib.GetLogger().Warning("Unable to write newline, skipping: %w", err)
 			}
 		} else if key.Value == e.prefs.DirectiveName {
 			var directive xml.Directive = []byte(value.Value)
@@ -159,48 +141,48 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node) er
 				return err
 			}
 			if _, err := e.writer.Write([]byte("\n")); err != nil {
-				log.Warning("Unable to write newline, skipping: %w", err)
+				yqlib.GetLogger().Warning("Unable to write newline, skipping: %w", err)
 			}
 		} else {
 
-			log.Debugf("recursing")
+			yqlib.GetLogger().Debugf("recursing")
 
 			err = e.doEncode(encoder, value, start)
 			if err != nil {
 				return err
 			}
 		}
-		err = e.encodeComment(encoder, footComment(key))
+		err = e.encodeComment(encoder, key.CleanFootComment())
 		if err != nil {
 			return err
 		}
 	}
-	return e.encodeComment(encoder, footComment(node))
+	return e.encodeComment(encoder, node.CleanFootComment())
 }
 
-func (e *xmlEncoder) encodeStart(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
+func (e *xmlEncoder) encodeStart(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
 	err := encoder.EncodeToken(start)
 	if err != nil {
 		return err
 	}
-	return e.encodeComment(encoder, headComment(node))
+	return e.encodeComment(encoder, node.CleanHeadComment())
 }
 
-func (e *xmlEncoder) encodeEnd(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
+func (e *xmlEncoder) encodeEnd(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
 	err := encoder.EncodeToken(start.End())
 	if err != nil {
 		return err
 	}
-	return e.encodeComment(encoder, footComment(node))
+	return e.encodeComment(encoder, node.CleanFootComment())
 }
 
-func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
+func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
 	switch node.Kind {
-	case yaml.MappingNode:
+	case yqlib.MappingNode:
 		return e.encodeMap(encoder, node, start)
-	case yaml.SequenceNode:
+	case yqlib.SequenceNode:
 		return e.encodeArray(encoder, node, start)
-	case yaml.ScalarNode:
+	case yqlib.ScalarNode:
 		err := e.encodeStart(encoder, node, start)
 		if err != nil {
 			return err
@@ -212,7 +194,7 @@ func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *yaml.Node, start xml.S
 			return err
 		}
 
-		if err = e.encodeComment(encoder, lineComment(node)); err != nil {
+		if err = e.encodeComment(encoder, node.CleanLineComment()); err != nil {
 			return err
 		}
 
@@ -227,13 +209,13 @@ var chompRegexp = regexp.MustCompile(`\n$`)
 
 func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) error {
 	if commentStr != "" {
-		log.Debugf("got comment [%v]", commentStr)
+		yqlib.GetLogger().Debugf("got comment [%v]", commentStr)
 		// multi line string
 		if len(commentStr) > 2 && strings.Contains(commentStr[1:len(commentStr)-1], "\n") {
 			commentStr = chompRegexp.ReplaceAllString(commentStr, "")
-			log.Debugf("chompRegexp [%v]", commentStr)
+			yqlib.GetLogger().Debugf("chompRegexp [%v]", commentStr)
 			commentStr = xmlEncodeMultilineCommentRegex.ReplaceAllString(commentStr, "$1$2")
-			log.Debugf("processed multiline [%v]", commentStr)
+			yqlib.GetLogger().Debugf("processed multiline [%v]", commentStr)
 			// if the first line is non blank, add a space
 			if commentStr[0] != '\n' && commentStr[0] != ' ' {
 				commentStr = " " + commentStr
@@ -245,9 +227,9 @@ func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) erro
 
 		if !strings.HasSuffix(commentStr, " ") && !strings.HasSuffix(commentStr, "\n") {
 			commentStr = commentStr + " "
-			log.Debugf("added suffix [%v]", commentStr)
+			yqlib.GetLogger().Debugf("added suffix [%v]", commentStr)
 		}
-		log.Debugf("encoding comment [%v]", commentStr)
+		yqlib.GetLogger().Debugf("encoding comment [%v]", commentStr)
 
 		var comment xml.Comment = []byte(commentStr)
 		err := encoder.EncodeToken(comment)
@@ -258,9 +240,9 @@ func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) erro
 	return nil
 }
 
-func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
+func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
 
-	if err := e.encodeComment(encoder, headAndLineComment(node)); err != nil {
+	if err := e.encodeComment(encoder, node.CleanHeadAndLineComment()); err != nil {
 		return err
 	}
 
@@ -270,7 +252,7 @@ func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *yaml.Node, start xm
 			return err
 		}
 	}
-	return e.encodeComment(encoder, footComment(node))
+	return e.encodeComment(encoder, node.CleanFootComment())
 }
 
 func (e *xmlEncoder) isAttribute(name string) bool {
@@ -280,8 +262,8 @@ func (e *xmlEncoder) isAttribute(name string) bool {
 		!strings.HasPrefix(name, e.prefs.ProcInstPrefix)
 }
 
-func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
-	log.Debug("its a map")
+func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
+	yqlib.GetLogger().Debug("its a map")
 
 	//first find all the attributes and put them on the start token
 	for i := 0; i < len(node.Content); i += 2 {
@@ -289,7 +271,7 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yaml.Node, start xml.
 		value := node.Content[i+1]
 
 		if e.isAttribute(key.Value) {
-			if value.Kind == yaml.ScalarNode {
+			if value.Kind == yqlib.ScalarNode {
 				attributeName := strings.Replace(key.Value, e.prefs.AttributePrefix, "", 1)
 				start.Attr = append(start.Attr, xml.Attr{Name: xml.Name{Local: attributeName}, Value: value.Value})
 			} else {
@@ -308,7 +290,7 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yaml.Node, start xml.
 		key := node.Content[i]
 		value := node.Content[i+1]
 
-		err := e.encodeComment(encoder, headAndLineComment(key))
+		err := e.encodeComment(encoder, key.CleanHeadAndLineComment())
 		if err != nil {
 			return err
 		}
@@ -325,7 +307,7 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yaml.Node, start xml.
 			}
 		} else if key.Value == e.prefs.ContentName {
 			// directly encode the contents
-			err = e.encodeComment(encoder, headAndLineComment(value))
+			err = e.encodeComment(encoder, value.CleanHeadAndLineComment())
 			if err != nil {
 				return err
 			}
@@ -334,7 +316,7 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yaml.Node, start xml.
 			if err != nil {
 				return err
 			}
-			err = e.encodeComment(encoder, footComment(value))
+			err = e.encodeComment(encoder, value.CleanFootComment())
 			if err != nil {
 				return err
 			}
@@ -345,7 +327,7 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yaml.Node, start xml.
 				return err
 			}
 		}
-		err = e.encodeComment(encoder, footComment(key))
+		err = e.encodeComment(encoder, key.CleanFootComment())
 		if err != nil {
 			return err
 		}

pkg/xml/xml.go

@@ -0,0 +1,72 @@
+package xml
+
+import "github.com/mikefarah/yq/v4/pkg/yqlib"
+
+type XmlPreferences struct {
+	Indent          int
+	AttributePrefix string
+	ContentName     string
+	StrictMode      bool
+	KeepNamespace   bool
+	UseRawToken     bool
+	ProcInstPrefix  string
+	DirectiveName   string
+	SkipProcInst    bool
+	SkipDirectives  bool
+}
+
+func NewDefaultXmlPreferences() XmlPreferences {
+	return XmlPreferences{
+		Indent:          2,
+		AttributePrefix: "+@",
+		ContentName:     "+content",
+		StrictMode:      false,
+		KeepNamespace:   true,
+		UseRawToken:     true,
+		ProcInstPrefix:  "+p_",
+		DirectiveName:   "+directive",
+		SkipProcInst:    false,
+		SkipDirectives:  false,
+	}
+}
+
+func (p *XmlPreferences) Copy() XmlPreferences {
+	return XmlPreferences{
+		Indent:          p.Indent,
+		AttributePrefix: p.AttributePrefix,
+		ContentName:     p.ContentName,
+		StrictMode:      p.StrictMode,
+		KeepNamespace:   p.KeepNamespace,
+		UseRawToken:     p.UseRawToken,
+		ProcInstPrefix:  p.ProcInstPrefix,
+		DirectiveName:   p.DirectiveName,
+		SkipProcInst:    p.SkipProcInst,
+		SkipDirectives:  p.SkipDirectives,
+	}
+}
+
+var ConfiguredXMLPreferences = NewDefaultXmlPreferences()
+
+var XMLFormat = &yqlib.Format{"xml", []string{"x"}, "xml",
+	func() yqlib.Encoder { return NewXMLEncoder(ConfiguredXMLPreferences) },
+	func() yqlib.Decoder { return NewXMLDecoder(ConfiguredXMLPreferences) },
+	func(indent int) yqlib.Encoder {
+		prefs := ConfiguredXMLPreferences.Copy()
+		prefs.Indent = indent
+		return NewXMLEncoder(prefs)
+	},
+}
+
+var xmlYqRules = []*yqlib.ParticipleYqRule{
+	{"XMLEncodeWithIndent", `to_?xml\([0-9]+\)`, yqlib.CreateEncodeYqActionParsingIndent(XMLFormat), 0},
+	{"XmlDecode", `from_?xml|@xmld`, yqlib.CreateDecodeOpYqAction(XMLFormat), 0},
+	{"XMLEncode", `to_?xml`, yqlib.CreateEncodeOpYqAction(XMLFormat, 2), 0},
+	{"XMLEncodeNoIndent", `@xml`, yqlib.CreateEncodeOpYqAction(XMLFormat, 0), 0},
+	{"LoadXML", `load_?xml|xml_?load`, yqlib.CreateLoadOpYqAction(NewXMLDecoder(ConfiguredXMLPreferences)), 0},
+}
+
+func RegisterXmlFormat() {
+	yqlib.RegisterFormat(XMLFormat)
+	yqlib.RegisterRules(xmlYqRules)
+
+}

pkg/xml/xml_test.go

@@ -1,12 +1,13 @@
 //go:build !yq_noxml
 
-package yqlib
+package xml
 
 import (
 	"bufio"
 	"fmt"
 	"testing"
 
+	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"github.com/mikefarah/yq/v4/test"
 )
 
@@ -262,7 +263,13 @@ const expectedXmlWithProcInstAndDirectives = `<?xml version="1.0"?>
 </apple>
 `
 
-var xmlScenarios = []formatScenario{
+var xmlScenarios = []yqlib.FormatScenario{
+	{
+		skipDoc:     true,
+		description: "bad xml",
+		input:       `<?xml version="1.0" encoding="UTF-8"?></Child></Root>`,
+		expected:    "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\n",
+	},
 	{
 		skipDoc:  true,
 		input:    "  <root>value<!-- comment--> </root>",
@@ -364,11 +371,10 @@ var xmlScenarios = []formatScenario{
 		expected:       "zoo:\n    animal:\n        - cat\n",
 	},
 	{
-		description:    "Parse xml: force all as an array",
-		subdescription: "Because of the way yq works, when updating everything you need to update the children before the parents. By default `..` will match parents first, so we reverse that before updating.",
-		input:          "<zoo><thing><frog>boing</frog></thing></zoo>",
-		expression:     "([..] | reverse | .[]) |= [] + .",
-		expected:       "- zoo:\n    - thing:\n        - frog:\n            - boing\n",
+		description: "Parse xml: force all as an array",
+		input:       "<zoo><thing><frog>boing</frog></thing></zoo>",
+		expression:  ".. |= [] + .",
+		expected:    "- zoo:\n    - thing:\n        - frog:\n            - boing\n",
 	},
 	{
 		description:    "Parse xml: attributes",
@@ -537,13 +543,6 @@ var xmlScenarios = []formatScenario{
 		expectedError: "cannot encode !!seq to XML - only maps can be encoded",
 		scenarioType:  "encode-error",
 	},
-	{
-		description:   "scalars cannot be encoded",
-		skipDoc:       true,
-		input:         "mike",
-		expectedError: "cannot encode !!str to XML - only maps can be encoded",
-		scenarioType:  "encode-error",
-	},
 	{
 		description:    "Encode xml: attributes with content",
 		subdescription: "Fields with the matching xml-content-name is assumed to be content.",
@@ -616,39 +615,41 @@ var xmlScenarios = []formatScenario{
 	},
 }
 
-func testXMLScenario(t *testing.T, s formatScenario) {
+func testXMLScenario(t *testing.T, s yqlib.FormatScenario) {
 	switch s.scenarioType {
 	case "", "decode":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(4, false, ConfiguredYamlPreferences)), s.description)
+		yamlPrefs := yqlib.ConfiguredYamlPreferences.Copy()
+		yamlPrefs.Indent = 4
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(yamlPrefs)), s.description)
 	case "encode":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(ConfiguredXMLPreferences)), s.description)
 	case "roundtrip":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(ConfiguredXMLPreferences)), s.description)
 	case "decode-keep-ns":
 		prefs := NewDefaultXmlPreferences()
 		prefs.KeepNamespace = true
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
 	case "decode-raw-token":
 		prefs := NewDefaultXmlPreferences()
 		prefs.UseRawToken = true
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
 	case "decode-raw-token-off":
 		prefs := NewDefaultXmlPreferences()
 		prefs.UseRawToken = false
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
 	case "roundtrip-skip-directives":
 		prefs := NewDefaultXmlPreferences()
 		prefs.SkipDirectives = true
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(prefs)), s.description)
 	case "decode-error":
-		result, err := processFormatScenario(s, NewXMLDecoder(NewDefaultXmlPreferences()), NewYamlEncoder(2, false, ConfiguredYamlPreferences))
+		result, err := processFormatScenario(s, NewXMLDecoder(NewDefaultXmlPreferences()), NewYamlEncoder(ConfiguredYamlPreferences))
 		if err == nil {
 			t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result)
 		} else {
 			test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description)
 		}
 	case "encode-error":
-		result, err := processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, NewDefaultXmlPreferences()))
+		result, err := processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(NewDefaultXmlPreferences()))
 		if err == nil {
 			t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result)
 		} else {
@@ -660,8 +661,8 @@ func testXMLScenario(t *testing.T, s formatScenario) {
 	}
 }
 
-func documentXMLScenario(t *testing.T, w *bufio.Writer, i interface{}) {
-	s := i.(formatScenario)
+func documentXMLScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
+	s := i.(FormatScenario)
 
 	if s.skipDoc {
 		return
@@ -685,7 +686,7 @@ func documentXMLScenario(t *testing.T, w *bufio.Writer, i interface{}) {
 	}
 }
 
-func documentXMLDecodeScenario(w *bufio.Writer, s formatScenario) {
+func documentXMLDecodeScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -704,10 +705,10 @@ func documentXMLDecodeScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, fmt.Sprintf("```bash\nyq -oy '%v' sample.xml\n```\n", expression))
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences))))
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(ConfiguredYamlPreferences))))
 }
 
-func documentXMLDecodeKeepNsScenario(w *bufio.Writer, s formatScenario) {
+func documentXMLDecodeKeepNsScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -723,15 +724,15 @@ func documentXMLDecodeKeepNsScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, "will output\n")
 	prefs := NewDefaultXmlPreferences()
 	prefs.KeepNamespace = false
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs))))
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(prefs))))
 
 	prefsWithout := NewDefaultXmlPreferences()
 	prefs.KeepNamespace = true
 	writeOrPanic(w, "instead of\n")
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(2, prefsWithout))))
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(prefsWithout))))
 }
 
-func documentXMLDecodeKeepNsRawTokenScenario(w *bufio.Writer, s formatScenario) {
+func documentXMLDecodeKeepNsRawTokenScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -749,16 +750,16 @@ func documentXMLDecodeKeepNsRawTokenScenario(w *bufio.Writer, s formatScenario)
 	prefs := NewDefaultXmlPreferences()
 	prefs.UseRawToken = false
 
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs))))
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(prefs))))
 
 	prefsWithout := NewDefaultXmlPreferences()
 	prefsWithout.UseRawToken = true
 
 	writeOrPanic(w, "instead of\n")
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(2, prefsWithout))))
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(prefsWithout))))
 }
 
-func documentXMLEncodeScenario(w *bufio.Writer, s formatScenario) {
+func documentXMLEncodeScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -770,13 +771,13 @@ func documentXMLEncodeScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input))
 
 	writeOrPanic(w, "then\n")
-	writeOrPanic(w, "```bash\nyq -o=xml '.' sample.yml\n```\n")
+	writeOrPanic(w, "```bash\nyq -o=xml sample.yml\n```\n")
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences))))
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(ConfiguredXMLPreferences))))
 }
 
-func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) {
+func documentXMLRoundTripScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -791,10 +792,10 @@ func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, "```bash\nyq '.' sample.xml\n```\n")
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences))))
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(ConfiguredXMLPreferences))))
 }
 
-func documentXMLSkipDirectivesScenario(w *bufio.Writer, s formatScenario) {
+func documentXMLSkipDirectivesScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -811,7 +812,7 @@ func documentXMLSkipDirectivesScenario(w *bufio.Writer, s formatScenario) {
 	prefs := NewDefaultXmlPreferences()
 	prefs.SkipDirectives = true
 
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs))))
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(prefs))))
 }
 
 func TestXMLScenarios(t *testing.T) {

pkg/yqlib/all_at_once_evaluator.go

@@ -2,8 +2,6 @@ package yqlib
 
 import (
 	"container/list"
-
-	yaml "gopkg.in/yaml.v3"
 )
 
 // A yaml expression evaluator that runs the expression once against all files/nodes in memory.
@@ -11,7 +9,7 @@ type Evaluator interface {
 	EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error
 
 	// EvaluateNodes takes an expression and one or more yaml nodes, returning a list of matching candidate nodes
-	EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error)
+	EvaluateNodes(expression string, nodes ...*CandidateNode) (*list.List, error)
 
 	// EvaluateCandidateNodes takes an expression and list of candidate nodes, returning a list of matching candidate nodes
 	EvaluateCandidateNodes(expression string, inputCandidateNodes *list.List) (*list.List, error)
@@ -26,10 +24,10 @@ func NewAllAtOnceEvaluator() Evaluator {
 	return &allAtOnceEvaluator{treeNavigator: NewDataTreeNavigator()}
 }
 
-func (e *allAtOnceEvaluator) EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error) {
+func (e *allAtOnceEvaluator) EvaluateNodes(expression string, nodes ...*CandidateNode) (*list.List, error) {
 	inputCandidates := list.New()
 	for _, node := range nodes {
-		inputCandidates.PushBack(&CandidateNode{Node: node})
+		inputCandidates.PushBack(node)
 	}
 	return e.EvaluateCandidateNodes(expression, inputCandidates)
 }
@@ -65,13 +63,7 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
 	}
 
 	if allDocuments.Len() == 0 {
-		candidateNode := &CandidateNode{
-			Document:       0,
-			Filename:       "",
-			Node:           &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
-			FileIndex:      0,
-			LeadingContent: "",
-		}
+		candidateNode := CreateScalarNode(nil, "")
 		allDocuments.PushBack(candidateNode)
 	}
 

pkg/yqlib/all_at_once_evaluator_test.go

@@ -1,6 +1,8 @@
 package yqlib
 
 import (
+	"bufio"
+	"strings"
 	"testing"
 
 	"github.com/mikefarah/yq/v4/test"
@@ -18,7 +20,7 @@ var evaluateNodesScenario = []expressionScenario{
 		document:   `a: hello`,
 		expression: `.`,
 		expected: []string{
-			"D0, P[], (doc)::a: hello\n",
+			"D0, P[], (!!map)::a: hello\n",
 		},
 	},
 	{
@@ -32,9 +34,23 @@ var evaluateNodesScenario = []expressionScenario{
 
 func TestAllAtOnceEvaluateNodes(t *testing.T) {
 	var evaluator = NewAllAtOnceEvaluator()
+	// logging.SetLevel(logging.DEBUG, "")
 	for _, tt := range evaluateNodesScenario {
-		node := test.ParseData(tt.document)
-		list, _ := evaluator.EvaluateNodes(tt.expression, &node)
+		decoder := NewYamlDecoder(ConfiguredYamlPreferences)
+		reader := bufio.NewReader(strings.NewReader(tt.document))
+		err := decoder.Init(reader)
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		candidateNode, errorReading := decoder.Decode()
+
+		if errorReading != nil {
+			t.Error(errorReading)
+			return
+		}
+
+		list, _ := evaluator.EvaluateNodes(tt.expression, candidateNode)
 		test.AssertResultComplex(t, tt.expected, resultsToString(t, list))
 	}
 }

pkg/yqlib/candidate_node.go

@@ -3,51 +3,189 @@ package yqlib
 import (
 	"container/list"
 	"fmt"
+	"strconv"
 	"strings"
-
-	"github.com/jinzhu/copier"
-	yaml "gopkg.in/yaml.v3"
 )
 
+type Kind uint32
+
+const (
+	SequenceNode Kind = 1 << iota
+	MappingNode
+	ScalarNode
+	AliasNode
+)
+
+type Style uint32
+
+const (
+	TaggedStyle Style = 1 << iota
+	DoubleQuotedStyle
+	SingleQuotedStyle
+	LiteralStyle
+	FoldedStyle
+	FlowStyle
+)
+
+func CreateStringScalarNode(stringValue string) *CandidateNode {
+	var node = &CandidateNode{Kind: ScalarNode}
+	node.Value = stringValue
+	node.Tag = "!!str"
+	return node
+}
+
+func CreateScalarNode(value interface{}, stringValue string) *CandidateNode {
+	var node = &CandidateNode{Kind: ScalarNode}
+	node.Value = stringValue
+
+	switch value.(type) {
+	case float32, float64:
+		node.Tag = "!!float"
+	case int, int64, int32:
+		node.Tag = "!!int"
+	case bool:
+		node.Tag = "!!bool"
+	case string:
+		node.Tag = "!!str"
+	case nil:
+		node.Tag = "!!null"
+	}
+	return node
+}
+
 type CandidateNode struct {
-	Node   *yaml.Node     // the actual node
+	Kind  Kind
+	Style Style
+
+	Tag     string
+	Value   string
+	Anchor  string
+	Alias   *CandidateNode
+	Content []*CandidateNode
+
+	HeadComment string
+	LineComment string
+	FootComment string
+
 	Parent *CandidateNode // parent node
-	Key    *yaml.Node     // node key, if this is a value from a map (or index in an array)
+	Key    *CandidateNode // node key, if this is a value from a map (or index in an array)
 
-	LeadingContent  string
-	TrailingContent string
+	LeadingContent string
 
-	Path      []interface{} /// the path we took to get to this node
-	Document  uint          // the document index of this node
-	Filename  string
-	FileIndex int
+	document uint // the document index of this node
+	filename string
+
+	Line   int
+	Column int
+
+	fileIndex int
 	// when performing op against all nodes given, this will treat all the nodes as one
 	// (e.g. top level cross document merge). This property does not propagate to child nodes.
 	EvaluateTogether bool
 	IsMapKey         bool
 }
 
+func (n *CandidateNode) CreateChild() *CandidateNode {
+	return &CandidateNode{
+		Parent: n,
+	}
+}
+
+func (n *CandidateNode) SetDocument(idx uint) {
+	n.document = idx
+}
+
+func (n *CandidateNode) GetDocument() uint {
+	// defer to parent
+	if n.Parent != nil {
+		return n.Parent.GetDocument()
+	}
+	return n.document
+}
+
+func (n *CandidateNode) SetFilename(name string) {
+	n.filename = name
+}
+
+func (n *CandidateNode) GetFilename() string {
+	if n.Parent != nil {
+		return n.Parent.GetFilename()
+	}
+	return n.filename
+}
+
+func (n *CandidateNode) SetFileIndex(idx int) {
+	n.fileIndex = idx
+}
+
+func (n *CandidateNode) GetFileIndex() int {
+	if n.Parent != nil {
+		return n.Parent.GetFileIndex()
+	}
+	return n.fileIndex
+}
+
 func (n *CandidateNode) GetKey() string {
 	keyPrefix := ""
 	if n.IsMapKey {
-		keyPrefix = "key-"
+		keyPrefix = fmt.Sprintf("key-%v-", n.Value)
 	}
-	return fmt.Sprintf("%v%v - %v", keyPrefix, n.Document, n.Path)
+	key := ""
+	if n.Key != nil {
+		key = n.Key.Value
+	}
+	return fmt.Sprintf("%v%v - %v", keyPrefix, n.GetDocument(), key)
 }
 
-func (n *CandidateNode) GetNiceTag() string {
-	return unwrapDoc(n.Node).Tag
+func (n *CandidateNode) getParsedKey() interface{} {
+	if n.IsMapKey {
+		return n.Value
+	}
+	if n.Key == nil {
+		return nil
+	}
+	if n.Key.Tag == "!!str" {
+		return n.Key.Value
+	}
+	index, err := parseInt(n.Key.Value)
+	if err != nil {
+		return n.Key.Value
+	}
+	return index
+
+}
+
+func (n *CandidateNode) GetPath() []interface{} {
+	key := n.getParsedKey()
+	if n.Parent != nil && key != nil {
+		return append(n.Parent.GetPath(), key)
+	}
+
+	if key != nil {
+		return []interface{}{key}
+	}
+	return make([]interface{}, 0)
 }
 
 func (n *CandidateNode) GetNicePath() string {
-	if n.Path != nil && len(n.Path) >= 0 {
-		pathStr := make([]string, len(n.Path))
-		for i, v := range n.Path {
-			pathStr[i] = fmt.Sprintf("%v", v)
+	var sb strings.Builder
+	path := n.GetPath()
+	for i, element := range path {
+		elementStr := fmt.Sprintf("%v", element)
+		switch element.(type) {
+		case int:
+			sb.WriteString("[" + elementStr + "]")
+		default:
+			if i == 0 {
+				sb.WriteString(elementStr)
+			} else if strings.ContainsRune(elementStr, '.') {
+				sb.WriteString("[" + elementStr + "]")
+			} else {
+				sb.WriteString("." + elementStr)
+			}
 		}
-		return strings.Join(pathStr, ".")
 	}
-	return ""
+	return sb.String()
 }
 
 func (n *CandidateNode) AsList() *list.List {
@@ -56,134 +194,268 @@ func (n *CandidateNode) AsList() *list.List {
 	return elMap
 }
 
-func (n *CandidateNode) CreateChildInMap(key *yaml.Node, node *yaml.Node) *CandidateNode {
-	var value interface{}
-	if key != nil {
-		value = key.Value
+func (n *CandidateNode) SetParent(parent *CandidateNode) {
+	n.Parent = parent
+}
+
+func (n *CandidateNode) AddKeyValueChild(rawKey *CandidateNode, rawValue *CandidateNode) (*CandidateNode, *CandidateNode) {
+	key := rawKey.Copy()
+	key.SetParent(n)
+	key.IsMapKey = true
+
+	value := rawValue.Copy()
+	value.SetParent(n)
+	value.IsMapKey = false // force this, incase we are creating a value from a key
+	value.Key = key
+
+	n.Content = append(n.Content, key, value)
+	return key, value
+}
+
+func (n *CandidateNode) AddChild(rawChild *CandidateNode) {
+	value := rawChild.Copy()
+	value.SetParent(n)
+	if value.Key != nil {
+		value.Key.SetParent(n)
+	} else {
+		index := len(n.Content)
+		keyNode := CreateScalarNode(index, fmt.Sprintf("%v", index))
+		keyNode.SetParent(n)
+		value.Key = keyNode
 	}
-	return &CandidateNode{
-		Node:      node,
-		Path:      n.createChildPath(value),
-		Parent:    n,
-		Key:       key,
-		Document:  n.Document,
-		Filename:  n.Filename,
-		FileIndex: n.FileIndex,
+	n.Content = append(n.Content, value)
+}
+
+func (n *CandidateNode) AddChildren(children []*CandidateNode) {
+	if n.Kind == MappingNode {
+		for i := 0; i < len(children); i += 2 {
+			key := children[i]
+			value := children[i+1]
+			n.AddKeyValueChild(key, value)
+		}
+
+	} else {
+		for _, rawChild := range children {
+			n.AddChild(rawChild)
+		}
 	}
 }
 
-func (n *CandidateNode) CreateChildInArray(index int, node *yaml.Node) *CandidateNode {
-	return &CandidateNode{
-		Node:      node,
-		Path:      n.createChildPath(index),
-		Parent:    n,
-		Key:       &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%v", index), Tag: "!!int"},
-		Document:  n.Document,
-		Filename:  n.Filename,
-		FileIndex: n.FileIndex,
+func (n *CandidateNode) GetValueRep() (interface{}, error) {
+	log.Debugf("GetValueRep for %v value: %v", n.GetNicePath(), n.Value)
+	realTag := n.GuessTagFromCustomType()
+
+	switch realTag {
+	case "!!int":
+		_, val, err := parseInt64(n.Value)
+		return val, err
+	case "!!float":
+		// need to test this
+		return strconv.ParseFloat(n.Value, 64)
+	case "!!bool":
+		return isTruthyNode(n), nil
+	case "!!null":
+		return nil, nil
 	}
+
+	return n.Value, nil
 }
 
-func (n *CandidateNode) CreateReplacement(node *yaml.Node) *CandidateNode {
-	return &CandidateNode{
-		Node:      node,
-		Path:      n.createChildPath(nil),
-		Parent:    n.Parent,
-		Key:       n.Key,
-		IsMapKey:  n.IsMapKey,
-		Document:  n.Document,
-		Filename:  n.Filename,
-		FileIndex: n.FileIndex,
+func (n *CandidateNode) GuessTagFromCustomType() string {
+	if strings.HasPrefix(n.Tag, "!!") {
+		return n.Tag
+	} else if n.Value == "" {
+		log.Debug("GuessTagFromCustomType: node has no value to guess the type with")
+		return n.Tag
 	}
+	dataBucket, errorReading := parseSnippet(n.Value)
+
+	if errorReading != nil {
+		log.Debug("GuessTagFromCustomType: could not guess underlying tag type %v", errorReading)
+		return n.Tag
+	}
+	guessedTag := dataBucket.Tag
+	log.Info("im guessing the tag %v is a %v", n.Tag, guessedTag)
+	return guessedTag
 }
 
-func (n *CandidateNode) CreateReplacementWithDocWrappers(node *yaml.Node) *CandidateNode {
-	replacement := n.CreateReplacement(node)
+func (n *CandidateNode) CreateReplacement(kind Kind, tag string, value string) *CandidateNode {
+	node := &CandidateNode{
+		Kind:  kind,
+		Tag:   tag,
+		Value: value,
+	}
+	return n.CopyAsReplacement(node)
+}
+
+func (n *CandidateNode) CopyAsReplacement(replacement *CandidateNode) *CandidateNode {
+	newCopy := replacement.Copy()
+	newCopy.Parent = n.Parent
+
+	if n.IsMapKey {
+		newCopy.Key = n
+	} else {
+		newCopy.Key = n.Key
+	}
+
+	return newCopy
+}
+
+func (n *CandidateNode) CreateReplacementWithComments(kind Kind, tag string, style Style) *CandidateNode {
+	replacement := n.CreateReplacement(kind, tag, "")
 	replacement.LeadingContent = n.LeadingContent
-	replacement.TrailingContent = n.TrailingContent
+	replacement.HeadComment = n.HeadComment
+	replacement.LineComment = n.LineComment
+	replacement.FootComment = n.FootComment
+	replacement.Style = style
 	return replacement
 }
 
-func (n *CandidateNode) createChildPath(path interface{}) []interface{} {
-	if path == nil {
-		newPath := make([]interface{}, len(n.Path))
-		copy(newPath, n.Path)
-		return newPath
-	}
-
-	//don't use append as they may actually modify the path of the original node!
-	newPath := make([]interface{}, len(n.Path)+1)
-	copy(newPath, n.Path)
-	newPath[len(n.Path)] = path
-	return newPath
+func (n *CandidateNode) Copy() *CandidateNode {
+	return n.doCopy(true)
 }
 
-func (n *CandidateNode) Copy() (*CandidateNode, error) {
-	clone := &CandidateNode{}
-	err := copier.Copy(clone, n)
-	if err != nil {
-		return nil, err
+func (n *CandidateNode) CopyWithoutContent() *CandidateNode {
+	return n.doCopy(false)
+}
+
+func (n *CandidateNode) doCopy(cloneContent bool) *CandidateNode {
+	var content []*CandidateNode
+
+	var copyKey *CandidateNode
+	if n.Key != nil {
+		copyKey = n.Key.Copy()
 	}
-	clone.Node = deepClone(n.Node)
-	return clone, nil
+
+	clone := &CandidateNode{
+		Kind:  n.Kind,
+		Style: n.Style,
+
+		Tag:    n.Tag,
+		Value:  n.Value,
+		Anchor: n.Anchor,
+
+		// ok not to clone this,
+		// as its a reference to somewhere else.
+		Alias:   n.Alias,
+		Content: content,
+
+		HeadComment: n.HeadComment,
+		LineComment: n.LineComment,
+		FootComment: n.FootComment,
+
+		Parent: n.Parent,
+		Key:    copyKey,
+
+		LeadingContent: n.LeadingContent,
+
+		document:  n.document,
+		filename:  n.filename,
+		fileIndex: n.fileIndex,
+
+		Line:   n.Line,
+		Column: n.Column,
+
+		EvaluateTogether: n.EvaluateTogether,
+		IsMapKey:         n.IsMapKey,
+	}
+
+	if cloneContent {
+		clone.AddChildren(n.Content)
+	}
+
+	return clone
 }
 
 // updates this candidate from the given candidate node
 func (n *CandidateNode) UpdateFrom(other *CandidateNode, prefs assignPreferences) {
-
+	if n == other {
+		log.Debugf("UpdateFrom, no need to update from myself.")
+		return
+	}
 	// if this is an empty map or empty array, use the style of other node.
-	if (n.Node.Kind != yaml.ScalarNode && len(n.Node.Content) == 0) ||
+	if (n.Kind != ScalarNode && len(n.Content) == 0) ||
 		// if the tag has changed (e.g. from str to bool)
-		(guessTagFromCustomType(n.Node) != guessTagFromCustomType(other.Node)) {
-		n.Node.Style = other.Node.Style
+		(n.GuessTagFromCustomType() != other.GuessTagFromCustomType()) {
+		n.Style = other.Style
 	}
 
-	n.Node.Content = deepCloneContent(other.Node.Content)
-	n.Node.Kind = other.Node.Kind
-	n.Node.Value = other.Node.Value
+	n.Content = make([]*CandidateNode, 0)
+	n.Kind = other.Kind
+	n.AddChildren(other.Content)
+
+	n.Value = other.Value
 
 	n.UpdateAttributesFrom(other, prefs)
 
 }
 
 func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignPreferences) {
-	log.Debug("UpdateAttributesFrom: n: %v other: %v", n.GetKey(), other.GetKey())
-	if n.Node.Kind != other.Node.Kind {
+	log.Debug("UpdateAttributesFrom: n: %v other: %v", NodeToString(n), NodeToString(other))
+	if n.Kind != other.Kind {
 		// clear out the contents when switching to a different type
 		// e.g. map to array
-		n.Node.Content = make([]*yaml.Node, 0)
-		n.Node.Value = ""
+		n.Content = make([]*CandidateNode, 0)
+		n.Value = ""
 	}
-	n.Node.Kind = other.Node.Kind
+	n.Kind = other.Kind
 
 	// don't clobber custom tags...
-	if prefs.ClobberCustomTags || strings.HasPrefix(n.Node.Tag, "!!") || n.Node.Tag == "" {
-		n.Node.Tag = other.Node.Tag
+	if prefs.ClobberCustomTags || strings.HasPrefix(n.Tag, "!!") || n.Tag == "" {
+		n.Tag = other.Tag
 	}
 
-	n.Node.Alias = other.Node.Alias
+	n.Alias = other.Alias
 
 	if !prefs.DontOverWriteAnchor {
-		n.Node.Anchor = other.Node.Anchor
+		n.Anchor = other.Anchor
 	}
 
 	// merge will pickup the style of the new thing
 	// when autocreating nodes
 
-	if n.Node.Style == 0 {
-		n.Node.Style = other.Node.Style
+	if n.Style == 0 {
+		n.Style = other.Style
 	}
 
-	if other.Node.FootComment != "" {
-		n.Node.FootComment = other.Node.FootComment
+	if other.FootComment != "" {
+		n.FootComment = other.FootComment
 	}
-	if other.TrailingContent != "" {
-		n.TrailingContent = other.TrailingContent
+	if other.HeadComment != "" {
+		n.HeadComment = other.HeadComment
 	}
-	if other.Node.HeadComment != "" {
-		n.Node.HeadComment = other.Node.HeadComment
+	if other.LineComment != "" {
+		n.LineComment = other.LineComment
 	}
-	if other.Node.LineComment != "" {
-		n.Node.LineComment = other.Node.LineComment
+}
+
+func (n *CandidateNode) CleanHeadAndLineComment() string {
+	return n.CleanHeadComment() + n.CleanLineComment()
+}
+
+func (n *CandidateNode) CleanHeadComment() string {
+	return strings.Replace(n.HeadComment, "#", "", 1)
+}
+
+func (n *CandidateNode) CleanLineComment() string {
+	return strings.Replace(n.LineComment, "#", "", 1)
+}
+
+func (n *CandidateNode) CleanFootComment() string {
+	return strings.Replace(n.FootComment, "#", "", 1)
+}
+
+func (n *CandidateNode) ConvertKeysToStrings() {
+
+	if n.Kind == MappingNode {
+		for index, child := range n.Content {
+			if index%2 == 0 { // its a map key
+				child.Tag = "!!str"
+			}
+		}
+	}
+
+	for _, child := range n.Content {
+		child.ConvertKeysToStrings()
 	}
 }

pkg/yqlib/candidate_node_goccy_yaml.go

@@ -0,0 +1,180 @@
+package yqlib
+
+import (
+	"fmt"
+	"strings"
+
+	yaml "github.com/goccy/go-yaml"
+	"github.com/goccy/go-yaml/ast"
+	goccyToken "github.com/goccy/go-yaml/token"
+)
+
+func (o *CandidateNode) goccyDecodeIntoChild(childNode ast.Node, cm yaml.CommentMap) (*CandidateNode, error) {
+	newChild := o.CreateChild()
+
+	err := newChild.UnmarshalGoccyYAML(childNode, cm)
+	return newChild, err
+}
+
+func (o *CandidateNode) UnmarshalGoccyYAML(node ast.Node, cm yaml.CommentMap) error {
+	log.Debugf("UnmarshalYAML %v", node)
+	log.Debugf("UnmarshalYAML %v", node.Type().String())
+	log.Debugf("UnmarshalYAML Node Value: %v", node.String())
+	log.Debugf("UnmarshalYAML Node GetComment: %v", node.GetComment())
+
+	if node.GetComment() != nil {
+		commentMapComments := cm[node.GetPath()]
+		for _, comment := range node.GetComment().Comments {
+			// need to use the comment map to find the position :/
+			log.Debugf("%v has a comment of [%v]", node.GetPath(), comment.Token.Value)
+			for _, commentMapComment := range commentMapComments {
+				commentMapValue := strings.Join(commentMapComment.Texts, "\n")
+				if commentMapValue == comment.Token.Value {
+					log.Debug("found a matching entry in comment map")
+					// we found the comment in the comment map,
+					// now we can process the position
+					switch commentMapComment.Position {
+					case yaml.CommentHeadPosition:
+						o.HeadComment = comment.String()
+						log.Debug("its a head comment %v", comment.String())
+					case yaml.CommentLinePosition:
+						o.LineComment = comment.String()
+						log.Debug("its a line comment %v", comment.String())
+					case yaml.CommentFootPosition:
+						o.FootComment = comment.String()
+						log.Debug("its a foot comment %v", comment.String())
+					}
+				}
+			}
+
+		}
+	}
+
+	o.Value = node.String()
+	switch node.Type() {
+	case ast.IntegerType:
+		o.Kind = ScalarNode
+		o.Tag = "!!int"
+	case ast.FloatType:
+		o.Kind = ScalarNode
+		o.Tag = "!!float"
+	case ast.BoolType:
+		o.Kind = ScalarNode
+		o.Tag = "!!bool"
+	case ast.NullType:
+		o.Kind = ScalarNode
+		o.Tag = "!!null"
+		o.Value = node.GetToken().Value
+	case ast.StringType:
+		o.Kind = ScalarNode
+		o.Tag = "!!str"
+		switch node.GetToken().Type {
+		case goccyToken.SingleQuoteType:
+			o.Style = SingleQuotedStyle
+		case goccyToken.DoubleQuoteType:
+			o.Style = DoubleQuotedStyle
+		}
+		o.Value = node.(*ast.StringNode).Value
+		log.Debugf("string value %v", node.(*ast.StringNode).Value)
+	case ast.LiteralType:
+		o.Kind = ScalarNode
+		o.Tag = "!!str"
+		o.Style = LiteralStyle
+		astLiteral := node.(*ast.LiteralNode)
+		log.Debugf("astLiteral.Start.Type %v", astLiteral.Start.Type)
+		if astLiteral.Start.Type == goccyToken.FoldedType {
+			log.Debugf("folded Type %v", astLiteral.Start.Type)
+			o.Style = FoldedStyle
+		}
+		log.Debug("start value: %v ", node.(*ast.LiteralNode).Start.Value)
+		log.Debug("start value: %v ", node.(*ast.LiteralNode).Start.Type)
+		// TODO: here I could put the original value with line breaks
+		// to solve the multiline > problem
+		o.Value = astLiteral.Value.Value
+	case ast.TagType:
+		if err := o.UnmarshalGoccyYAML(node.(*ast.TagNode).Value, cm); err != nil {
+			return err
+		}
+		o.Tag = node.(*ast.TagNode).Start.Value
+	case ast.MappingType:
+		log.Debugf("UnmarshalYAML -  a mapping node")
+		o.Kind = MappingNode
+		o.Tag = "!!map"
+
+		mappingNode := node.(*ast.MappingNode)
+		if mappingNode.IsFlowStyle {
+			o.Style = FlowStyle
+		}
+		for _, mappingValueNode := range mappingNode.Values {
+			err := o.goccyProcessMappingValueNode(mappingValueNode, cm)
+			if err != nil {
+				return ast.ErrInvalidAnchorName
+			}
+		}
+		if mappingNode.FootComment != nil {
+			log.Debugf("mapping node has a foot comment of: %v", mappingNode.FootComment)
+			o.FootComment = mappingNode.FootComment.String()
+		}
+	case ast.MappingValueType:
+		log.Debugf("UnmarshalYAML -  a mapping node")
+		o.Kind = MappingNode
+		o.Tag = "!!map"
+		mappingValueNode := node.(*ast.MappingValueNode)
+		err := o.goccyProcessMappingValueNode(mappingValueNode, cm)
+		if err != nil {
+			return ast.ErrInvalidAnchorName
+		}
+	case ast.SequenceType:
+		log.Debugf("UnmarshalYAML -  a sequence node")
+		o.Kind = SequenceNode
+		o.Tag = "!!seq"
+		sequenceNode := node.(*ast.SequenceNode)
+		if sequenceNode.IsFlowStyle {
+			o.Style = FlowStyle
+		}
+		astSeq := sequenceNode.Values
+		o.Content = make([]*CandidateNode, len(astSeq))
+		for i := 0; i < len(astSeq); i++ {
+			keyNode := o.CreateChild()
+			keyNode.IsMapKey = true
+			keyNode.Tag = "!!int"
+			keyNode.Kind = ScalarNode
+			keyNode.Value = fmt.Sprintf("%v", i)
+
+			valueNode, err := o.goccyDecodeIntoChild(astSeq[i], cm)
+			if err != nil {
+				return err
+			}
+
+			valueNode.Key = keyNode
+			o.Content[i] = valueNode
+		}
+
+	default:
+		log.Debugf("UnmarshalYAML -  node idea of the type!!")
+	}
+	log.Debugf("KIND: %v", o.Kind)
+	return nil
+}
+
+func (o *CandidateNode) goccyProcessMappingValueNode(mappingEntry *ast.MappingValueNode, cm yaml.CommentMap) error {
+	log.Debug("UnmarshalYAML MAP KEY entry %v", mappingEntry.Key)
+	keyNode, err := o.goccyDecodeIntoChild(mappingEntry.Key, cm)
+	if err != nil {
+		return err
+	}
+	keyNode.IsMapKey = true
+
+	log.Debug("UnmarshalYAML MAP VALUE entry %v", mappingEntry.Value)
+	valueNode, err := o.goccyDecodeIntoChild(mappingEntry.Value, cm)
+	if err != nil {
+		return err
+	}
+
+	if mappingEntry.FootComment != nil {
+		valueNode.FootComment = mappingEntry.FootComment.String()
+	}
+	o.AddKeyValueChild(keyNode, valueNode)
+
+	return nil
+}

pkg/yqlib/candidate_node_test.go

@@ -0,0 +1,162 @@
+package yqlib
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+type valueRepScenario struct {
+	input    string
+	tag      string
+	expected interface{}
+}
+
+var valueRepScenarios = []valueRepScenario{
+	{
+		input:    `"cat"`,
+		expected: `"cat"`,
+	},
+	{
+		input:    `3`,
+		expected: int64(3),
+	},
+	{
+		input:    `3.1`,
+		expected: float64(3.1),
+	},
+	{
+		input:    `true`,
+		expected: true,
+	},
+	{
+		input:    `y`,
+		tag:      "!!bool",
+		expected: true,
+	},
+	{
+		tag:      "!!null",
+		expected: nil,
+	},
+}
+
+func TestCandidateNodeGetValueRepScenarios(t *testing.T) {
+	for _, tt := range valueRepScenarios {
+		node := CandidateNode{Value: tt.input, Tag: tt.tag}
+		actual, err := node.GetValueRep()
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		test.AssertResult(t, tt.expected, actual)
+	}
+}
+
+func TestCandidateNodeChildWhenParentUpdated(t *testing.T) {
+	parent := CandidateNode{}
+	child := parent.CreateChild()
+	parent.SetDocument(1)
+	parent.SetFileIndex(2)
+	parent.SetFilename("meow")
+	test.AssertResultWithContext(t, "meow", child.GetFilename(), "filename")
+	test.AssertResultWithContext(t, 2, child.GetFileIndex(), "file index")
+	test.AssertResultWithContext(t, uint(1), child.GetDocument(), "document index")
+}
+
+type CreateScalarNodeScenario struct {
+	value       interface{}
+	stringValue string
+	expectedTag string
+}
+
+var createScalarScenarios = []CreateScalarNodeScenario{
+	{
+		value:       "mike",
+		stringValue: "mike",
+		expectedTag: "!!str",
+	},
+	{
+		value:       3,
+		stringValue: "3",
+		expectedTag: "!!int",
+	},
+	{
+		value:       3.1,
+		stringValue: "3.1",
+		expectedTag: "!!float",
+	},
+	{
+		value:       true,
+		stringValue: "true",
+		expectedTag: "!!bool",
+	},
+	{
+		value:       nil,
+		stringValue: "~",
+		expectedTag: "!!null",
+	},
+}
+
+func TestCreateScalarNodeScenarios(t *testing.T) {
+	for _, tt := range createScalarScenarios {
+		actual := CreateScalarNode(tt.value, tt.stringValue)
+		test.AssertResultWithContext(t, tt.stringValue, actual.Value, fmt.Sprintf("Value for: Value: [%v], String: %v", tt.value, tt.stringValue))
+		test.AssertResultWithContext(t, tt.expectedTag, actual.Tag, fmt.Sprintf("Value for: Value: [%v], String: %v", tt.value, tt.stringValue))
+	}
+}
+
+func TestGetKeyForMapValue(t *testing.T) {
+	key := CreateStringScalarNode("yourKey")
+	n := CandidateNode{Key: key, Value: "meow", document: 3}
+	test.AssertResult(t, "3 - yourKey", n.GetKey())
+}
+
+func TestGetKeyForMapKey(t *testing.T) {
+	key := CreateStringScalarNode("yourKey")
+	key.IsMapKey = true
+	key.document = 3
+	test.AssertResult(t, "key-yourKey-3 - ", key.GetKey())
+}
+
+func TestGetKeyForValue(t *testing.T) {
+	n := CandidateNode{Value: "meow", document: 3}
+	test.AssertResult(t, "3 - ", n.GetKey())
+}
+
+func TestGetParsedKeyForMapKey(t *testing.T) {
+	key := CreateStringScalarNode("yourKey")
+	key.IsMapKey = true
+	key.document = 3
+	test.AssertResult(t, "yourKey", key.getParsedKey())
+}
+
+func TestGetParsedKeyForLooseValue(t *testing.T) {
+	n := CandidateNode{Value: "meow", document: 3}
+	test.AssertResult(t, nil, n.getParsedKey())
+}
+
+func TestGetParsedKeyForMapValue(t *testing.T) {
+	key := CreateStringScalarNode("yourKey")
+	n := CandidateNode{Key: key, Value: "meow", document: 3}
+	test.AssertResult(t, "yourKey", n.getParsedKey())
+}
+
+func TestGetParsedKeyForArrayValue(t *testing.T) {
+	key := CreateScalarNode(4, "4")
+	n := CandidateNode{Key: key, Value: "meow", document: 3}
+	test.AssertResult(t, 4, n.getParsedKey())
+}
+
+func TestCandidateNodeAddKeyValueChild(t *testing.T) {
+	key := CandidateNode{Value: "cool", IsMapKey: true}
+	node := CandidateNode{}
+
+	// if we use a key in a new node as a value, it should no longer be marked as a key
+
+	_, keyIsValueNow := node.AddKeyValueChild(&CandidateNode{Value: "newKey"}, &key)
+
+	test.AssertResult(t, keyIsValueNow.IsMapKey, false)
+	test.AssertResult(t, key.IsMapKey, true)
+
+}

pkg/yqlib/candidate_node_yaml.go

@@ -0,0 +1,218 @@
+package yqlib
+
+import (
+	"fmt"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+func MapYamlStyle(original yaml.Style) Style {
+	switch original {
+	case yaml.TaggedStyle:
+		return TaggedStyle
+	case yaml.DoubleQuotedStyle:
+		return DoubleQuotedStyle
+	case yaml.SingleQuotedStyle:
+		return SingleQuotedStyle
+	case yaml.LiteralStyle:
+		return LiteralStyle
+	case yaml.FoldedStyle:
+		return FoldedStyle
+	case yaml.FlowStyle:
+		return FlowStyle
+	case 0:
+		return 0
+	}
+	return Style(original)
+}
+
+func MapToYamlStyle(original Style) yaml.Style {
+	switch original {
+	case TaggedStyle:
+		return yaml.TaggedStyle
+	case DoubleQuotedStyle:
+		return yaml.DoubleQuotedStyle
+	case SingleQuotedStyle:
+		return yaml.SingleQuotedStyle
+	case LiteralStyle:
+		return yaml.LiteralStyle
+	case FoldedStyle:
+		return yaml.FoldedStyle
+	case FlowStyle:
+		return yaml.FlowStyle
+	case 0:
+		return 0
+	}
+	return yaml.Style(original)
+}
+
+func (o *CandidateNode) copyFromYamlNode(node *yaml.Node, anchorMap map[string]*CandidateNode) {
+	o.Style = MapYamlStyle(node.Style)
+
+	o.Tag = node.Tag
+	o.Value = node.Value
+	o.Anchor = node.Anchor
+
+	if o.Anchor != "" {
+		anchorMap[o.Anchor] = o
+		log.Debug("set anchor %v to %v", o.Anchor, NodeToString(o))
+	}
+
+	// its a single alias
+	if node.Alias != nil && node.Alias.Anchor != "" {
+		o.Alias = anchorMap[node.Alias.Anchor]
+		log.Debug("set alias to %v", NodeToString(anchorMap[node.Alias.Anchor]))
+	}
+	o.HeadComment = node.HeadComment
+	o.LineComment = node.LineComment
+	o.FootComment = node.FootComment
+
+	o.Line = node.Line
+	o.Column = node.Column
+}
+
+func (o *CandidateNode) copyToYamlNode(node *yaml.Node) {
+	node.Style = MapToYamlStyle(o.Style)
+
+	node.Tag = o.Tag
+	node.Value = o.Value
+	node.Anchor = o.Anchor
+
+	// node.Alias = TODO - find Alias in our own structure
+	// might need to be a post process thing
+
+	node.HeadComment = o.HeadComment
+
+	node.LineComment = o.LineComment
+	node.FootComment = o.FootComment
+
+	node.Line = o.Line
+	node.Column = o.Column
+}
+
+func (o *CandidateNode) decodeIntoChild(childNode *yaml.Node, anchorMap map[string]*CandidateNode) (*CandidateNode, error) {
+	newChild := o.CreateChild()
+
+	// null yaml.Nodes to not end up calling UnmarshalYAML
+	// so we call it explicitly
+	if childNode.Tag == "!!null" {
+		newChild.Kind = ScalarNode
+		newChild.copyFromYamlNode(childNode, anchorMap)
+		return newChild, nil
+	}
+
+	err := newChild.UnmarshalYAML(childNode, anchorMap)
+	return newChild, err
+}
+
+func (o *CandidateNode) UnmarshalYAML(node *yaml.Node, anchorMap map[string]*CandidateNode) error {
+	log.Debugf("UnmarshalYAML %v", node.Tag)
+	switch node.Kind {
+	case yaml.AliasNode:
+		log.Debug("UnmarshalYAML - alias from yaml: %v", o.Tag)
+		o.Kind = AliasNode
+		o.copyFromYamlNode(node, anchorMap)
+		return nil
+	case yaml.ScalarNode:
+		log.Debugf("UnmarshalYAML -  a scalar")
+		o.Kind = ScalarNode
+		o.copyFromYamlNode(node, anchorMap)
+		return nil
+	case yaml.MappingNode:
+		log.Debugf("UnmarshalYAML -  a mapping node")
+		o.Kind = MappingNode
+		o.copyFromYamlNode(node, anchorMap)
+		o.Content = make([]*CandidateNode, len(node.Content))
+		for i := 0; i < len(node.Content); i += 2 {
+
+			keyNode, err := o.decodeIntoChild(node.Content[i], anchorMap)
+			if err != nil {
+				return err
+			}
+
+			keyNode.IsMapKey = true
+
+			valueNode, err := o.decodeIntoChild(node.Content[i+1], anchorMap)
+			if err != nil {
+				return err
+			}
+
+			valueNode.Key = keyNode
+
+			o.Content[i] = keyNode
+			o.Content[i+1] = valueNode
+		}
+		log.Debugf("UnmarshalYAML -  finished mapping node")
+		return nil
+	case yaml.SequenceNode:
+		log.Debugf("UnmarshalYAML -  a sequence: %v", len(node.Content))
+		o.Kind = SequenceNode
+
+		o.copyFromYamlNode(node, anchorMap)
+		log.Debugf("node Style: %v", node.Style)
+		log.Debugf("o Style: %v", o.Style)
+		o.Content = make([]*CandidateNode, len(node.Content))
+		for i := 0; i < len(node.Content); i++ {
+			keyNode := o.CreateChild()
+			keyNode.IsMapKey = true
+			keyNode.Tag = "!!int"
+			keyNode.Kind = ScalarNode
+			keyNode.Value = fmt.Sprintf("%v", i)
+
+			valueNode, err := o.decodeIntoChild(node.Content[i], anchorMap)
+			if err != nil {
+				return err
+			}
+
+			valueNode.Key = keyNode
+			o.Content[i] = valueNode
+		}
+		return nil
+	case 0:
+		// not sure when this happens
+		o.copyFromYamlNode(node, anchorMap)
+		log.Debugf("UnmarshalYAML -  err.. %v", NodeToString(o))
+		return nil
+	default:
+		return fmt.Errorf("orderedMap: invalid yaml node")
+	}
+}
+
+func (o *CandidateNode) MarshalYAML() (*yaml.Node, error) {
+	log.Debug("MarshalYAML to yaml: %v", o.Tag)
+	switch o.Kind {
+	case AliasNode:
+		log.Debug("MarshalYAML - alias to yaml: %v", o.Tag)
+		target := &yaml.Node{Kind: yaml.AliasNode}
+		o.copyToYamlNode(target)
+		return target, nil
+	case ScalarNode:
+		log.Debug("MarshalYAML - scalar: %v", o.Value)
+		target := &yaml.Node{Kind: yaml.ScalarNode}
+		o.copyToYamlNode(target)
+		return target, nil
+	case MappingNode, SequenceNode:
+		targetKind := yaml.MappingNode
+		if o.Kind == SequenceNode {
+			targetKind = yaml.SequenceNode
+		}
+		target := &yaml.Node{Kind: targetKind}
+		o.copyToYamlNode(target)
+		log.Debugf("original style: %v", o.Style)
+		log.Debugf("original: %v, tag: %v, style: %v, kind: %v", NodeToString(o), target.Tag, target.Style, target.Kind == yaml.SequenceNode)
+		target.Content = make([]*yaml.Node, len(o.Content))
+		for i := 0; i < len(o.Content); i++ {
+
+			child, err := o.Content[i].MarshalYAML()
+
+			if err != nil {
+				return nil, err
+			}
+			target.Content[i] = child
+		}
+		return target, nil
+	}
+	target := &yaml.Node{}
+	o.copyToYamlNode(target)
+	return target, nil
+}

pkg/yqlib/candidiate_node_json.go

@@ -0,0 +1,177 @@
+package yqlib
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+
+	"github.com/goccy/go-json"
+)
+
+func (o *CandidateNode) setScalarFromJson(value interface{}) error {
+	o.Kind = ScalarNode
+	switch rawData := value.(type) {
+	case nil:
+		o.Tag = "!!null"
+		o.Value = "null"
+	case float64, float32:
+		o.Value = fmt.Sprintf("%v", value)
+		o.Tag = "!!float"
+		// json decoder returns ints as float.
+		if value == float64(int64(rawData.(float64))) {
+			// aha it's an int disguised as a float
+			o.Tag = "!!int"
+			o.Value = fmt.Sprintf("%v", int64(value.(float64)))
+		}
+	case int, int64, int32:
+		o.Value = fmt.Sprintf("%v", value)
+		o.Tag = "!!int"
+	case bool:
+		o.Value = fmt.Sprintf("%v", value)
+		o.Tag = "!!bool"
+	case string:
+		o.Value = rawData
+		o.Tag = "!!str"
+	default:
+		return fmt.Errorf("unrecognised type :( %v", rawData)
+	}
+	return nil
+}
+
+func (o *CandidateNode) UnmarshalJSON(data []byte) error {
+	log.Debug("UnmarshalJSON")
+	switch data[0] {
+	case '{':
+		log.Debug("UnmarshalJSON -  its a map!")
+		// its a map
+		o.Kind = MappingNode
+		o.Tag = "!!map"
+
+		dec := json.NewDecoder(bytes.NewReader(data))
+		_, err := dec.Token() // open object
+		if err != nil {
+			return err
+		}
+
+		// cycle through k/v
+		var tok json.Token
+		for tok, err = dec.Token(); err == nil; tok, err = dec.Token() {
+			// we can expect two types: string or Delim. Delim automatically means
+			// that it is the closing bracket of the object, whereas string means
+			// that there is another key.
+			if _, ok := tok.(json.Delim); ok {
+				break
+			}
+
+			childKey := o.CreateChild()
+			childKey.IsMapKey = true
+			childKey.Value = tok.(string)
+			childKey.Kind = ScalarNode
+			childKey.Tag = "!!str"
+
+			childValue := o.CreateChild()
+			childValue.Key = childKey
+
+			if err := dec.Decode(childValue); err != nil {
+				return err
+			}
+			o.Content = append(o.Content, childKey, childValue)
+		}
+		// unexpected error
+		if err != nil && !errors.Is(err, io.EOF) {
+			return err
+		}
+		return nil
+	case '[':
+		o.Kind = SequenceNode
+		o.Tag = "!!seq"
+		log.Debug("UnmarshalJSON -  its an array!")
+		var children []*CandidateNode
+		if err := json.Unmarshal(data, &children); err != nil {
+			return err
+		}
+		// now we put the children into the content, and set a key value for them
+		for i, child := range children {
+
+			if child == nil {
+				// need to represent it as a null scalar
+				child = CreateScalarNode(nil, "null")
+			}
+			childKey := o.CreateChild()
+			childKey.Kind = ScalarNode
+			childKey.Tag = "!!int"
+			childKey.Value = fmt.Sprintf("%v", i)
+			childKey.IsMapKey = true
+
+			child.Parent = o
+			child.Key = childKey
+			o.Content = append(o.Content, child)
+		}
+		return nil
+	}
+	log.Debug("UnmarshalJSON -  its a scalar!")
+	// otherwise, must be a scalar
+	var scalar interface{}
+	err := json.Unmarshal(data, &scalar)
+
+	if err != nil {
+		return err
+	}
+	log.Debug("UnmarshalJSON -  scalar is %v", scalar)
+
+	return o.setScalarFromJson(scalar)
+
+}
+
+func (o *CandidateNode) MarshalJSON() ([]byte, error) {
+	log.Debugf("MarshalJSON %v", NodeToString(o))
+	buf := new(bytes.Buffer)
+	enc := json.NewEncoder(buf)
+	enc.SetIndent("", " ")
+	enc.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
+
+	switch o.Kind {
+	case AliasNode:
+		log.Debugf("MarshalJSON AliasNode")
+		err := enc.Encode(o.Alias)
+		return buf.Bytes(), err
+	case ScalarNode:
+		log.Debugf("MarshalJSON ScalarNode")
+		value, err := o.GetValueRep()
+		if err != nil {
+			return buf.Bytes(), err
+		}
+		err = enc.Encode(value)
+		return buf.Bytes(), err
+	case MappingNode:
+		log.Debugf("MarshalJSON MappingNode")
+		buf.WriteByte('{')
+		for i := 0; i < len(o.Content); i += 2 {
+			if err := enc.Encode(o.Content[i].Value); err != nil {
+				return nil, err
+			}
+			buf.WriteByte(':')
+			if err := enc.Encode(o.Content[i+1]); err != nil {
+				return nil, err
+			}
+			if i != len(o.Content)-2 {
+				buf.WriteByte(',')
+			}
+		}
+		buf.WriteByte('}')
+		return buf.Bytes(), nil
+	case SequenceNode:
+		log.Debugf("MarshalJSON SequenceNode, %v, len: %v", o.Content, len(o.Content))
+		var err error
+		if len(o.Content) == 0 {
+			buf.WriteString("[]")
+		} else {
+			err = enc.Encode(o.Content)
+		}
+		return buf.Bytes(), err
+	default:
+		err := enc.Encode(nil)
+		return buf.Bytes(), err
+	}
+}

pkg/yqlib/context.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/jinzhu/copier"
 	logging "gopkg.in/op/go-logging.v1"
 )
 
@@ -58,13 +57,19 @@ func (n *Context) SetVariable(name string, value *list.List) {
 func (n *Context) ChildContext(results *list.List) Context {
 	clone := Context{DontAutoCreate: n.DontAutoCreate, datetimeLayout: n.datetimeLayout}
 	clone.Variables = make(map[string]*list.List)
-	if len(n.Variables) > 0 {
-		err := copier.Copy(&clone.Variables, n.Variables)
-		if err != nil {
-			log.Error("Error cloning context :(")
-			panic(err)
+	for variableKey, originalValueList := range n.Variables {
+
+		variableCopyList := list.New()
+		for el := originalValueList.Front(); el != nil; el = el.Next() {
+			// note that we dont make a copy of the candidate node
+			// this is so the 'ref' operator can work correctly.
+			clonedNode := el.Value.(*CandidateNode)
+			variableCopyList.PushBack(clonedNode)
 		}
+
+		clone.Variables[variableKey] = variableCopyList
 	}
+
 	clone.MatchingNodes = results
 	return clone
 }
@@ -78,35 +83,18 @@ func (n *Context) ToString() string {
 }
 
 func (n *Context) DeepClone() Context {
-	clone := Context{}
-	err := copier.Copy(&clone, n)
-	// copier doesn't do lists properly for some reason
-	clone.MatchingNodes = list.New()
+
+	clonedContent := list.New()
 	for el := n.MatchingNodes.Front(); el != nil; el = el.Next() {
-		clonedNode, err := el.Value.(*CandidateNode).Copy()
-		if err != nil {
-			log.Error("Error cloning context :(")
-			panic(err)
-		}
-		clone.MatchingNodes.PushBack(clonedNode)
+		clonedNode := el.Value.(*CandidateNode).Copy()
+		clonedContent.PushBack(clonedNode)
 	}
 
-	if err != nil {
-		log.Error("Error cloning context :(")
-		panic(err)
-	}
-	return clone
+	return n.ChildContext(clonedContent)
 }
 
 func (n *Context) Clone() Context {
-	clone := Context{}
-	err := copier.Copy(&clone, n)
-
-	if err != nil {
-		log.Error("Error cloning context :(")
-		panic(err)
-	}
-	return clone
+	return n.ChildContext(n.MatchingNodes)
 }
 
 func (n *Context) ReadOnlyClone() Context {

pkg/yqlib/context_test.go

@@ -0,0 +1,51 @@
+package yqlib
+
+import (
+	"container/list"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+func TestChildContext(t *testing.T) {
+
+	expectedOriginal := make(map[string]*list.List)
+	expectedOriginal["dog"] = list.New()
+	expectedOriginal["dog"].PushBack(&CandidateNode{Value: "woof"})
+
+	originalVariables := make(map[string]*list.List)
+	originalVariables["dog"] = list.New()
+	originalVariables["dog"].PushBack(&CandidateNode{Value: "woof"})
+
+	original := Context{
+		DontAutoCreate: true,
+		datetimeLayout: "cat",
+		Variables:      originalVariables,
+	}
+
+	newResults := list.New()
+	newResults.PushBack(&CandidateNode{Value: "bar"})
+
+	clone := original.ChildContext(newResults)
+	test.AssertResultComplex(t, originalVariables, clone.Variables)
+
+	clone.Variables["dog"].PushBack("bark")
+	// ensure this is a separate copy
+	test.AssertResultComplex(t, 1, originalVariables["dog"].Len())
+
+}
+
+func TestChildContextNoVariables(t *testing.T) {
+
+	original := Context{
+		DontAutoCreate: true,
+		datetimeLayout: "cat",
+	}
+
+	newResults := list.New()
+	newResults.PushBack(&CandidateNode{Value: "bar"})
+
+	clone := original.ChildContext(newResults)
+	test.AssertResultComplex(t, make(map[string]*list.List), clone.Variables)
+
+}

pkg/yqlib/csv.go

@@ -0,0 +1,23 @@
+package yqlib
+
+type CsvPreferences struct {
+	Separator rune
+	AutoParse bool
+}
+
+func NewDefaultCsvPreferences() CsvPreferences {
+	return CsvPreferences{
+		Separator: ',',
+		AutoParse: true,
+	}
+}
+
+func NewDefaultTsvPreferences() CsvPreferences {
+	return CsvPreferences{
+		Separator: '\t',
+		AutoParse: true,
+	}
+}
+
+var ConfiguredCsvPreferences = NewDefaultCsvPreferences()
+var ConfiguredTsvPreferences = NewDefaultTsvPreferences()

pkg/yqlib/csv_test.go

@@ -12,6 +12,11 @@ const csvSimple = `name,numberOfCats,likesApples,height
 Gary,1,true,168.8
 Samantha's Rabbit,2,false,-188.8
 `
+
+const csvSimpleWithObject = `name,numberOfCats,likesApples,height,facts
+Gary,1,true,168.8,cool: true
+Samantha's Rabbit,2,false,-188.8,tall: indeed
+`
 const csvMissing = `name,numberOfCats,likesApples,height
 ,null,,168.8
 `
@@ -39,6 +44,31 @@ const expectedYamlFromCSV = `- name: Gary
   likesApples: false
   height: -188.8
 `
+const expectedYamlFromCSVWithObject = `- name: Gary
+  numberOfCats: 1
+  likesApples: true
+  height: 168.8
+  facts:
+    cool: true
+- name: Samantha's Rabbit
+  numberOfCats: 2
+  likesApples: false
+  height: -188.8
+  facts:
+    tall: indeed
+`
+
+const expectedYamlFromCSVNoParsing = `- name: Gary
+  numberOfCats: 1
+  likesApples: true
+  height: 168.8
+  facts: 'cool: true'
+- name: Samantha's Rabbit
+  numberOfCats: 2
+  likesApples: false
+  height: -188.8
+  facts: 'tall: indeed'
+`
 
 const expectedYamlFromCSVMissingData = `- name: Gary
   numberOfCats: 1
@@ -64,7 +94,7 @@ const tsvTestExpectedSimpleCsv = `i	like	csv
 because	excel	is	cool
 `
 
-var csvScenarios = []formatScenario{
+var csvScenarios = []FormatScenario{
 	{
 		description:  "Encode CSV simple",
 		input:        csvTestSimpleYaml,
@@ -119,12 +149,42 @@ var csvScenarios = []formatScenario{
 		expected:     csvMissing,
 		scenarioType: "roundtrip-csv",
 	},
+	{
+		description:  "decode csv key",
+		skipDoc:      true,
+		input:        csvSimple,
+		expression:   ".[0].name | key",
+		expected:     "name\n",
+		scenarioType: "decode-csv",
+	},
+	{
+		description:  "decode csv parent",
+		skipDoc:      true,
+		input:        csvSimple,
+		expression:   ".[0].name | parent | .height",
+		expected:     "168.8\n",
+		scenarioType: "decode-csv",
+	},
 	{
 		description:    "Parse CSV into an array of objects",
-		subdescription: "First row is assumed to be the header row.",
-		input:          csvSimple,
-		expected:       expectedYamlFromCSV,
-		scenarioType:   "decode-csv-object",
+		subdescription: "First row is assumed to be the header row. By default, entries with YAML/JSON formatting will be parsed!",
+		input:          csvSimpleWithObject,
+		expected:       expectedYamlFromCSVWithObject,
+		scenarioType:   "decode-csv",
+	},
+	{
+		description:  "Decode CSV line breaks",
+		skipDoc:      true,
+		input:        "heading1\n\"some data\nwith a line break\"\n",
+		expected:     "- heading1: |-\n    some data\n    with a line break\n",
+		scenarioType: "decode-csv",
+	},
+	{
+		description:    "Parse CSV into an array of objects, no auto-parsing",
+		subdescription: "First row is assumed to be the header row. Entries with YAML/JSON will be left as strings.",
+		input:          csvSimpleWithObject,
+		expected:       expectedYamlFromCSVNoParsing,
+		scenarioType:   "decode-csv-no-auto",
 	},
 	{
 		description:  "Scalar roundtrip",
@@ -150,24 +210,26 @@ var csvScenarios = []formatScenario{
 	},
 }
 
-func testCSVScenario(t *testing.T, s formatScenario) {
+func testCSVScenario(t *testing.T, s FormatScenario) {
 	switch s.scenarioType {
 	case "encode-csv":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(',')), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(ConfiguredCsvPreferences)), s.description)
 	case "encode-tsv":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder('\t')), s.description)
-	case "decode-csv-object":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(','), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(ConfiguredTsvPreferences)), s.description)
+	case "decode-csv":
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredCsvPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
+	case "decode-csv-no-auto":
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: ',', AutoParse: false}), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
 	case "decode-tsv-object":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder('\t'), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredTsvPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
 	case "roundtrip-csv":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(','), NewCsvEncoder(',')), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredCsvPreferences), NewCsvEncoder(ConfiguredCsvPreferences)), s.description)
 	default:
 		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
 	}
 }
 
-func documentCSVDecodeObjectScenario(w *bufio.Writer, s formatScenario, formatType string) {
+func documentCSVDecodeObjectScenario(w *bufio.Writer, s FormatScenario, formatType string) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -188,11 +250,36 @@ func documentCSVDecodeObjectScenario(w *bufio.Writer, s formatScenario, formatTy
 	}
 
 	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n",
-		mustProcessFormatScenario(s, NewCSVObjectDecoder(separator), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))),
+		mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: true}), NewYamlEncoder(ConfiguredYamlPreferences))),
 	)
 }
 
-func documentCSVEncodeScenario(w *bufio.Writer, s formatScenario, formatType string) {
+func documentCSVDecodeObjectNoAutoScenario(w *bufio.Writer, s FormatScenario, formatType string) {
+	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
+
+	if s.subdescription != "" {
+		writeOrPanic(w, s.subdescription)
+		writeOrPanic(w, "\n\n")
+	}
+
+	writeOrPanic(w, fmt.Sprintf("Given a sample.%v file of:\n", formatType))
+	writeOrPanic(w, fmt.Sprintf("```%v\n%v\n```\n", formatType, s.input))
+
+	writeOrPanic(w, "then\n")
+	writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=%v --csv-auto-parse=f sample.%v\n```\n", formatType, formatType))
+	writeOrPanic(w, "will output\n")
+
+	separator := ','
+	if formatType == "tsv" {
+		separator = '\t'
+	}
+
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n",
+		mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: false}), NewYamlEncoder(ConfiguredYamlPreferences))),
+	)
+}
+
+func documentCSVEncodeScenario(w *bufio.Writer, s FormatScenario, formatType string) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -218,13 +305,14 @@ func documentCSVEncodeScenario(w *bufio.Writer, s formatScenario, formatType str
 	if formatType == "tsv" {
 		separator = '\t'
 	}
-
+	csvPrefs := NewDefaultCsvPreferences()
+	csvPrefs.Separator = separator
 	writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
-		mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(separator))),
+		mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(csvPrefs))),
 	)
 }
 
-func documentCSVRoundTripScenario(w *bufio.Writer, s formatScenario, formatType string) {
+func documentCSVRoundTripScenario(w *bufio.Writer, s FormatScenario, formatType string) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -251,13 +339,16 @@ func documentCSVRoundTripScenario(w *bufio.Writer, s formatScenario, formatType
 		separator = '\t'
 	}
 
+	csvPrefs := NewDefaultCsvPreferences()
+	csvPrefs.Separator = separator
+
 	writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
-		mustProcessFormatScenario(s, NewCSVObjectDecoder(separator), NewCsvEncoder(separator))),
+		mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: true}), NewCsvEncoder(csvPrefs))),
 	)
 }
 
-func documentCSVScenario(t *testing.T, w *bufio.Writer, i interface{}) {
-	s := i.(formatScenario)
+func documentCSVScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
+	s := i.(FormatScenario)
 	if s.skipDoc {
 		return
 	}
@@ -266,8 +357,10 @@ func documentCSVScenario(t *testing.T, w *bufio.Writer, i interface{}) {
 		documentCSVEncodeScenario(w, s, "csv")
 	case "encode-tsv":
 		documentCSVEncodeScenario(w, s, "tsv")
-	case "decode-csv-object":
+	case "decode-csv":
 		documentCSVDecodeObjectScenario(w, s, "csv")
+	case "decode-csv-no-auto":
+		documentCSVDecodeObjectNoAutoScenario(w, s, "csv")
 	case "decode-tsv-object":
 		documentCSVDecodeObjectScenario(w, s, "tsv")
 	case "roundtrip-csv":

pkg/yqlib/data_tree_navigator.go

@@ -4,16 +4,16 @@ import (
 	"fmt"
 
 	logging "gopkg.in/op/go-logging.v1"
-	yaml "gopkg.in/yaml.v3"
 )
 
 type DataTreeNavigator interface {
-	// given the context and a expressionNode,
+	// given the context and an expressionNode,
 	// this will process the against the given expressionNode and return
 	// a new context of matching candidates
 	GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error)
 
-	DeeplyAssign(context Context, path []interface{}, rhsNode *yaml.Node) error
+	DeeplyAssign(context Context, path []interface{}, rhsNode *CandidateNode) error
+	DeeplyAssignKey(context Context, path []interface{}, rhsNode *CandidateNode) error
 }
 
 type dataTreeNavigator struct {
@@ -23,20 +23,34 @@ func NewDataTreeNavigator() DataTreeNavigator {
 	return &dataTreeNavigator{}
 }
 
-func (d *dataTreeNavigator) DeeplyAssign(context Context, path []interface{}, rhsNode *yaml.Node) error {
+func (d *dataTreeNavigator) DeeplyAssign(context Context, path []interface{}, rhsCandidateNode *CandidateNode) error {
+	return d.deeplyAssign(context, path, rhsCandidateNode, false)
+}
 
-	rhsCandidateNode := &CandidateNode{
-		Path: path,
-		Node: rhsNode,
-	}
+func (d *dataTreeNavigator) DeeplyAssignKey(context Context, path []interface{}, rhsCandidateNode *CandidateNode) error {
+	return d.deeplyAssign(context, path, rhsCandidateNode, true)
+}
+
+func (d *dataTreeNavigator) deeplyAssign(context Context, path []interface{}, rhsCandidateNode *CandidateNode, targetKey bool) error {
 
 	assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
 
+	if rhsCandidateNode.Kind == MappingNode {
+		log.Debug("DeeplyAssign: deeply merging object")
+		// if the rhs is a map, we need to deeply merge it in.
+		// otherwise we'll clobber any existing fields
+		assignmentOp = &Operation{OperationType: multiplyAssignOpType, Preferences: multiplyPreferences{
+			AppendArrays:  true,
+			TraversePrefs: traversePreferences{DontFollowAlias: true},
+			AssignPrefs:   assignPreferences{},
+		}}
+	}
+
 	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}
 
 	assignmentOpNode := &ExpressionNode{
 		Operation: assignmentOp,
-		LHS:       createTraversalTree(path, traversePreferences{}, false),
+		LHS:       createTraversalTree(path, traversePreferences{}, targetKey),
 		RHS:       &ExpressionNode{Operation: rhsOp},
 	}
 
@@ -55,7 +69,6 @@ func (d *dataTreeNavigator) GetMatchingNodes(context Context, expressionNode *Ex
 			log.Debug(NodeToString(el.Value.(*CandidateNode)))
 		}
 	}
-	log.Debug(">>")
 	handler := expressionNode.Operation.OperationType.Handler
 	if handler != nil {
 		return handler(d, context, expressionNode)

pkg/yqlib/decoder.go

@@ -1,63 +1,10 @@
 package yqlib
 
 import (
-	"fmt"
 	"io"
-	"strings"
-)
-
-type InputFormat uint
-
-const (
-	YamlInputFormat = 1 << iota
-	XMLInputFormat
-	PropertiesInputFormat
-	Base64InputFormat
-	JsonInputFormat
-	CSVObjectInputFormat
-	TSVObjectInputFormat
-	TomlInputFormat
-	UriInputFormat
 )
 
 type Decoder interface {
 	Init(reader io.Reader) error
 	Decode() (*CandidateNode, error)
 }
-
-func InputFormatFromString(format string) (InputFormat, error) {
-	switch format {
-	case "yaml", "yml", "y":
-		return YamlInputFormat, nil
-	case "xml", "x":
-		return XMLInputFormat, nil
-	case "properties", "props", "p":
-		return PropertiesInputFormat, nil
-	case "json", "ndjson", "j":
-		return JsonInputFormat, nil
-	case "csv", "c":
-		return CSVObjectInputFormat, nil
-	case "tsv", "t":
-		return TSVObjectInputFormat, nil
-	case "toml":
-		return TomlInputFormat, nil
-	default:
-		return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml|toml]", format)
-	}
-}
-
-func FormatFromFilename(filename string) string {
-
-	if filename != "" {
-		GetLogger().Debugf("checking file extension '%s' for auto format detection", filename)
-		nPos := strings.LastIndex(filename, ".")
-		if nPos > -1 {
-			format := filename[nPos+1:]
-			GetLogger().Debugf("detected format '%s'", format)
-			return format
-		}
-	}
-
-	GetLogger().Debugf("using default inputFormat 'yaml'")
-	return "yaml"
-}

pkg/yqlib/decoder_base64.go

@@ -5,8 +5,6 @@ import (
 	"encoding/base64"
 	"io"
 	"strings"
-
-	yaml "gopkg.in/yaml.v3"
 )
 
 type base64Padder struct {
@@ -70,11 +68,5 @@ func (dec *base64Decoder) Decode() (*CandidateNode, error) {
 		}
 	}
 	dec.readAnything = true
-	return &CandidateNode{
-		Node: &yaml.Node{
-			Kind:  yaml.ScalarNode,
-			Tag:   "!!str",
-			Value: buf.String(),
-		},
-	}, nil
+	return CreateStringScalarNode(buf.String()), nil
 }

pkg/yqlib/decoder_csv_object.go

@@ -6,44 +6,42 @@ import (
 	"io"
 
 	"github.com/dimchansky/utfbom"
-	yaml "gopkg.in/yaml.v3"
 )
 
 type csvObjectDecoder struct {
-	separator rune
-	reader    csv.Reader
-	finished  bool
+	prefs    CsvPreferences
+	reader   csv.Reader
+	finished bool
 }
 
-func NewCSVObjectDecoder(separator rune) Decoder {
-	return &csvObjectDecoder{separator: separator}
+func NewCSVObjectDecoder(prefs CsvPreferences) Decoder {
+	return &csvObjectDecoder{prefs: prefs}
 }
 
 func (dec *csvObjectDecoder) Init(reader io.Reader) error {
 	cleanReader, enc := utfbom.Skip(reader)
 	log.Debugf("Detected encoding: %s\n", enc)
 	dec.reader = *csv.NewReader(cleanReader)
-	dec.reader.Comma = dec.separator
+	dec.reader.Comma = dec.prefs.Separator
 	dec.finished = false
 	return nil
 }
 
-func (dec *csvObjectDecoder) convertToYamlNode(content string) *yaml.Node {
+func (dec *csvObjectDecoder) convertToNode(content string) *CandidateNode {
 	node, err := parseSnippet(content)
-	if err != nil {
-		return createScalarNode(content, content)
+	// if we're not auto-parsing, then we wont put in parsed objects or arrays
+	// but we still parse scalars
+	if err != nil || (!dec.prefs.AutoParse && node.Kind != ScalarNode) {
+		return CreateScalarNode(content, content)
 	}
 	return node
 }
 
-func (dec *csvObjectDecoder) createObject(headerRow []string, contentRow []string) *yaml.Node {
-	objectNode := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
+func (dec *csvObjectDecoder) createObject(headerRow []string, contentRow []string) *CandidateNode {
+	objectNode := &CandidateNode{Kind: MappingNode, Tag: "!!map"}
 
 	for i, header := range headerRow {
-		objectNode.Content = append(
-			objectNode.Content,
-			createScalarNode(header, header),
-			dec.convertToYamlNode(contentRow[i]))
+		objectNode.AddKeyValueChild(CreateScalarNode(header, header), dec.convertToNode(contentRow[i]))
 	}
 	return objectNode
 }
@@ -58,13 +56,13 @@ func (dec *csvObjectDecoder) Decode() (*CandidateNode, error) {
 		return nil, err
 	}
 
-	rootArray := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+	rootArray := &CandidateNode{Kind: SequenceNode, Tag: "!!seq"}
 
 	contentRow, err := dec.reader.Read()
 
 	for err == nil && len(contentRow) > 0 {
 		log.Debugf("Adding contentRow: %v", contentRow)
-		rootArray.Content = append(rootArray.Content, dec.createObject(headerRow, contentRow))
+		rootArray.AddChild(dec.createObject(headerRow, contentRow))
 		contentRow, err = dec.reader.Read()
 		log.Debugf("Read next contentRow: %v, %v", contentRow, err)
 	}
@@ -72,10 +70,5 @@ func (dec *csvObjectDecoder) Decode() (*CandidateNode, error) {
 		return nil, err
 	}
 
-	return &CandidateNode{
-		Node: &yaml.Node{
-			Kind:    yaml.DocumentNode,
-			Content: []*yaml.Node{rootArray},
-		},
-	}, nil
+	return rootArray, nil
 }

pkg/yqlib/decoder_goccy_yaml.go

@@ -0,0 +1,46 @@
+//go:build !yq_noyaml
+
+//
+// NOTE this is still a WIP - not yet ready.
+//
+
+package yqlib
+
+import (
+	"io"
+
+	yaml "github.com/goccy/go-yaml"
+	"github.com/goccy/go-yaml/ast"
+)
+
+type goccyYamlDecoder struct {
+	decoder yaml.Decoder
+	cm      yaml.CommentMap
+}
+
+func NewGoccyYAMLDecoder() Decoder {
+	return &goccyYamlDecoder{}
+}
+
+func (dec *goccyYamlDecoder) Init(reader io.Reader) error {
+	dec.cm = yaml.CommentMap{}
+	dec.decoder = *yaml.NewDecoder(reader, yaml.CommentToMap(dec.cm), yaml.UseOrderedMap())
+	return nil
+}
+
+func (dec *goccyYamlDecoder) Decode() (*CandidateNode, error) {
+
+	var ast ast.Node
+
+	err := dec.decoder.Decode(&ast)
+	if err != nil {
+		return nil, err
+	}
+
+	candidateNode := &CandidateNode{}
+	if err := candidateNode.UnmarshalGoccyYAML(ast, dec.cm); err != nil {
+		return nil, err
+	}
+
+	return candidateNode, nil
+}

pkg/yqlib/decoder_json.go

@@ -3,11 +3,9 @@
 package yqlib
 
 import (
-	"fmt"
 	"io"
 
 	"github.com/goccy/go-json"
-	yaml "gopkg.in/yaml.v3"
 )
 
 type jsonDecoder struct {
@@ -25,76 +23,11 @@ func (dec *jsonDecoder) Init(reader io.Reader) error {
 
 func (dec *jsonDecoder) Decode() (*CandidateNode, error) {
 
-	var dataBucket orderedMap
-	log.Debug("going to decode")
+	var dataBucket CandidateNode
 	err := dec.decoder.Decode(&dataBucket)
 	if err != nil {
 		return nil, err
 	}
-	node, err := dec.convertToYamlNode(&dataBucket)
 
-	if err != nil {
-		return nil, err
-	}
-
-	return &CandidateNode{
-		Node: &yaml.Node{
-			Kind:    yaml.DocumentNode,
-			Content: []*yaml.Node{node},
-		},
-	}, nil
-}
-
-func (dec *jsonDecoder) convertToYamlNode(data *orderedMap) (*yaml.Node, error) {
-	if data == nil {
-		return createScalarNode(nil, "null"), nil
-	}
-	if data.kv == nil {
-		switch rawData := data.altVal.(type) {
-		case nil:
-			return createScalarNode(nil, "null"), nil
-		case float64, float32:
-			// json decoder returns ints as float.'
-			intNum := int(rawData.(float64))
-
-			// if the integer representation is the same as the original
-			// then its an int.
-			if float64(intNum) == rawData.(float64) {
-				return createScalarNode(intNum, fmt.Sprintf("%v", intNum)), nil
-			}
-
-			return createScalarNode(rawData, fmt.Sprintf("%v", rawData)), nil
-		case int, int64, int32, string, bool:
-			return createScalarNode(rawData, fmt.Sprintf("%v", rawData)), nil
-		case []*orderedMap:
-			return dec.parseArray(rawData)
-		default:
-			return nil, fmt.Errorf("unrecognised type :( %v", rawData)
-		}
-	}
-
-	var yamlMap = &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
-	for i, keyValuePair := range data.kv {
-		yamlValue, err := dec.convertToYamlNode(&data.kv[i].V)
-		if err != nil {
-			return nil, err
-		}
-		yamlMap.Content = append(yamlMap.Content, createScalarNode(keyValuePair.K, keyValuePair.K), yamlValue)
-	}
-	return yamlMap, nil
-
-}
-
-func (dec *jsonDecoder) parseArray(dataArray []*orderedMap) (*yaml.Node, error) {
-
-	var yamlMap = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
-
-	for _, value := range dataArray {
-		yamlValue, err := dec.convertToYamlNode(value)
-		if err != nil {
-			return nil, err
-		}
-		yamlMap.Content = append(yamlMap.Content, yamlValue)
-	}
-	return yamlMap, nil
+	return &dataBucket, nil
 }

pkg/yqlib/decoder_lua.go

@@ -0,0 +1,165 @@
+//go:build !yq_nolua
+
+package yqlib
+
+import (
+	"fmt"
+	"io"
+	"math"
+
+	lua "github.com/yuin/gopher-lua"
+)
+
+type luaDecoder struct {
+	reader   io.Reader
+	finished bool
+	prefs    LuaPreferences
+}
+
+func NewLuaDecoder(prefs LuaPreferences) Decoder {
+	return &luaDecoder{
+		prefs: prefs,
+	}
+}
+
+func (dec *luaDecoder) Init(reader io.Reader) error {
+	dec.reader = reader
+	return nil
+}
+
+func (dec *luaDecoder) convertToYamlNode(ls *lua.LState, lv lua.LValue) *CandidateNode {
+	switch lv.Type() {
+	case lua.LTNil:
+		return &CandidateNode{
+			Kind:  ScalarNode,
+			Tag:   "!!null",
+			Value: "",
+		}
+	case lua.LTBool:
+		return &CandidateNode{
+			Kind:  ScalarNode,
+			Tag:   "!!bool",
+			Value: lv.String(),
+		}
+	case lua.LTNumber:
+		n := float64(lua.LVAsNumber(lv))
+		// various special case floats
+		if math.IsNaN(n) {
+			return &CandidateNode{
+				Kind:  ScalarNode,
+				Tag:   "!!float",
+				Value: ".nan",
+			}
+		}
+		if math.IsInf(n, 1) {
+			return &CandidateNode{
+				Kind:  ScalarNode,
+				Tag:   "!!float",
+				Value: ".inf",
+			}
+		}
+		if math.IsInf(n, -1) {
+			return &CandidateNode{
+				Kind:  ScalarNode,
+				Tag:   "!!float",
+				Value: "-.inf",
+			}
+		}
+
+		// does it look like an integer?
+		if n == float64(int(n)) {
+			return &CandidateNode{
+				Kind:  ScalarNode,
+				Tag:   "!!int",
+				Value: lv.String(),
+			}
+		}
+
+		return &CandidateNode{
+			Kind:  ScalarNode,
+			Tag:   "!!float",
+			Value: lv.String(),
+		}
+	case lua.LTString:
+		return &CandidateNode{
+			Kind:  ScalarNode,
+			Tag:   "!!str",
+			Value: lv.String(),
+		}
+	case lua.LTFunction:
+		return &CandidateNode{
+			Kind:  ScalarNode,
+			Tag:   "tag:lua.org,2006,function",
+			Value: lv.String(),
+		}
+	case lua.LTTable:
+		// Simultaneously create a sequence and a map, pick which one to return
+		// based on whether all keys were consecutive integers
+		i := 1
+		yaml_sequence := &CandidateNode{
+			Kind: SequenceNode,
+			Tag:  "!!seq",
+		}
+		yaml_map := &CandidateNode{
+			Kind: MappingNode,
+			Tag:  "!!map",
+		}
+		t := lv.(*lua.LTable)
+		k, v := ls.Next(t, lua.LNil)
+		for k != lua.LNil {
+			if ki, ok := k.(lua.LNumber); i != 0 && ok && math.Mod(float64(ki), 1) == 0 && int(ki) == i {
+				i++
+			} else {
+				i = 0
+			}
+			newKey := dec.convertToYamlNode(ls, k)
+
+			yv := dec.convertToYamlNode(ls, v)
+			yaml_map.AddKeyValueChild(newKey, yv)
+
+			if i != 0 {
+				yaml_sequence.AddChild(yv)
+			}
+			k, v = ls.Next(t, k)
+		}
+		if i != 0 {
+			return yaml_sequence
+		}
+		return yaml_map
+	default:
+		return &CandidateNode{
+			Kind:        ScalarNode,
+			LineComment: fmt.Sprintf("Unhandled Lua type: %s", lv.Type().String()),
+			Tag:         "!!null",
+			Value:       lv.String(),
+		}
+	}
+}
+
+func (dec *luaDecoder) decideTopLevelNode(ls *lua.LState) *CandidateNode {
+	if ls.GetTop() == 0 {
+		// no items were explicitly returned, encode the globals table instead
+		return dec.convertToYamlNode(ls, ls.Get(lua.GlobalsIndex))
+	}
+	return dec.convertToYamlNode(ls, ls.Get(1))
+}
+
+func (dec *luaDecoder) Decode() (*CandidateNode, error) {
+	if dec.finished {
+		return nil, io.EOF
+	}
+	ls := lua.NewState(lua.Options{SkipOpenLibs: true})
+	defer ls.Close()
+	fn, err := ls.Load(dec.reader, "@input")
+	if err != nil {
+		return nil, err
+	}
+	ls.Push(fn)
+	err = ls.PCall(0, lua.MultRet, nil)
+	if err != nil {
+		return nil, err
+	}
+	firstNode := dec.decideTopLevelNode(ls)
+	dec.finished = true
+	return firstNode, nil
+}

pkg/yqlib/decoder_properties.go

@@ -1,145 +0,0 @@
-package yqlib
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"strconv"
-	"strings"
-
-	"github.com/magiconair/properties"
-	"gopkg.in/yaml.v3"
-)
-
-type propertiesDecoder struct {
-	reader   io.Reader
-	finished bool
-	d        DataTreeNavigator
-}
-
-func NewPropertiesDecoder() Decoder {
-	return &propertiesDecoder{d: NewDataTreeNavigator(), finished: false}
-}
-
-func (dec *propertiesDecoder) Init(reader io.Reader) error {
-	dec.reader = reader
-	dec.finished = false
-	return nil
-}
-
-func parsePropKey(key string) []interface{} {
-	pathStrArray := strings.Split(key, ".")
-	path := make([]interface{}, len(pathStrArray))
-	for i, pathStr := range pathStrArray {
-		num, err := strconv.ParseInt(pathStr, 10, 32)
-		if err == nil {
-			path[i] = num
-		} else {
-			path[i] = pathStr
-		}
-	}
-	return path
-}
-
-func (dec *propertiesDecoder) processComment(c string) string {
-	if c == "" {
-		return ""
-	}
-	return "# " + c
-}
-
-func (dec *propertiesDecoder) applyPropertyComments(context Context, path []interface{}, comments []string) error {
-	assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
-
-	rhsCandidateNode := &CandidateNode{
-		Path: path,
-		Node: &yaml.Node{
-			Tag:         "!!str",
-			Value:       fmt.Sprintf("%v", path[len(path)-1]),
-			HeadComment: dec.processComment(strings.Join(comments, "\n")),
-			Kind:        yaml.ScalarNode,
-		},
-	}
-
-	rhsCandidateNode.Node.Tag = guessTagFromCustomType(rhsCandidateNode.Node)
-
-	rhsOp := &Operation{OperationType: referenceOpType, CandidateNode: rhsCandidateNode}
-
-	assignmentOpNode := &ExpressionNode{
-		Operation: assignmentOp,
-		LHS:       createTraversalTree(path, traversePreferences{}, true),
-		RHS:       &ExpressionNode{Operation: rhsOp},
-	}
-
-	_, err := dec.d.GetMatchingNodes(context, assignmentOpNode)
-	return err
-}
-
-func (dec *propertiesDecoder) applyProperty(context Context, properties *properties.Properties, key string) error {
-	value, _ := properties.Get(key)
-	path := parsePropKey(key)
-
-	propertyComments := properties.GetComments(key)
-	if len(propertyComments) > 0 {
-		err := dec.applyPropertyComments(context, path, propertyComments)
-		if err != nil {
-			return nil
-		}
-	}
-
-	rhsNode := &yaml.Node{
-		Value: value,
-		Tag:   "!!str",
-		Kind:  yaml.ScalarNode,
-	}
-
-	rhsNode.Tag = guessTagFromCustomType(rhsNode)
-
-	return dec.d.DeeplyAssign(context, path, rhsNode)
-}
-
-func (dec *propertiesDecoder) Decode() (*CandidateNode, error) {
-	if dec.finished {
-		return nil, io.EOF
-	}
-	buf := new(bytes.Buffer)
-
-	if _, err := buf.ReadFrom(dec.reader); err != nil {
-		return nil, err
-	}
-	if buf.Len() == 0 {
-		dec.finished = true
-		return nil, io.EOF
-	}
-	properties, err := properties.LoadString(buf.String())
-	if err != nil {
-		return nil, err
-	}
-	properties.DisableExpansion = true
-
-	rootMap := &CandidateNode{
-		Node: &yaml.Node{
-			Kind: yaml.MappingNode,
-			Tag:  "!!map",
-		},
-	}
-
-	context := Context{}
-	context = context.SingleChildContext(rootMap)
-
-	for _, key := range properties.Keys() {
-		if err := dec.applyProperty(context, properties, key); err != nil {
-			return nil, err
-		}
-
-	}
-	dec.finished = true
-
-	return &CandidateNode{
-		Node: &yaml.Node{
-			Kind:    yaml.DocumentNode,
-			Content: []*yaml.Node{rootMap.Node},
-		},
-	}, nil
-
-}

pkg/yqlib/decoder_test.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 )
 
-type formatScenario struct {
+type FormatScenario struct {
 	input          string
 	indent         int
 	expression     string
@@ -19,7 +19,7 @@ type formatScenario struct {
 	expectedError  string
 }
 
-func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) (string, error) {
+func processFormatScenario(s FormatScenario, decoder Decoder, encoder Encoder) (string, error) {
 	var output bytes.Buffer
 	writer := bufio.NewWriter(&output)
 
@@ -27,11 +27,14 @@ func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) (
 		decoder = NewYamlDecoder(ConfiguredYamlPreferences)
 	}
 
+	log.Debugf("reading docs")
 	inputs, err := readDocuments(strings.NewReader(s.input), "sample.yml", 0, decoder)
 	if err != nil {
 		return "", err
 	}
 
+	log.Debugf("done reading the documents")
+
 	expression := s.expression
 	if expression == "" {
 		expression = "."
@@ -45,6 +48,8 @@ func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) (
 
 	context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: inputs}, exp)
 
+	log.Debugf("Going to print: %v", NodesToString(context.MatchingNodes))
+
 	if err != nil {
 		return "", err
 	}
@@ -59,7 +64,7 @@ func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) (
 	return output.String(), nil
 }
 
-func mustProcessFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) string {
+func mustProcessFormatScenario(s FormatScenario, decoder Decoder, encoder Encoder) string {
 
 	result, err := processFormatScenario(s, decoder, encoder)
 	if err != nil {

pkg/yqlib/decoder_toml.go

@@ -11,7 +11,6 @@ import (
 	"time"
 
 	toml "github.com/pelletier/go-toml/v2/unstable"
-	yaml "gopkg.in/yaml.v3"
 )
 
 type tomlDecoder struct {
@@ -37,10 +36,9 @@ func (dec *tomlDecoder) Init(reader io.Reader) error {
 	}
 	dec.parser.Reset(buf.Bytes())
 	dec.rootMap = &CandidateNode{
-		Node: &yaml.Node{
-			Kind: yaml.MappingNode,
-			Tag:  "!!map",
-		}}
+		Kind: MappingNode,
+		Tag:  "!!map",
+	}
 	return nil
 }
 
@@ -58,12 +56,13 @@ func (dec *tomlDecoder) getFullPath(tomlNode *toml.Node) []interface{} {
 func (dec *tomlDecoder) processKeyValueIntoMap(rootMap *CandidateNode, tomlNode *toml.Node) error {
 	value := tomlNode.Value()
 	path := dec.getFullPath(value.Next())
-	log.Debug("!!!processKeyValueIntoMap: %v", path)
+	log.Debug("processKeyValueIntoMap: %v", path)
 
 	valueNode, err := dec.decodeNode(value)
 	if err != nil {
 		return err
 	}
+
 	context := Context{}
 	context = context.SingleChildContext(rootMap)
 
@@ -71,14 +70,14 @@ func (dec *tomlDecoder) processKeyValueIntoMap(rootMap *CandidateNode, tomlNode
 }
 
 func (dec *tomlDecoder) decodeKeyValuesIntoMap(rootMap *CandidateNode, tomlNode *toml.Node) (bool, error) {
-	log.Debug("!! DECODE_KV_INTO_MAP -- processing first (current) entry")
+	log.Debug("decodeKeyValuesIntoMap -- processing first (current) entry")
 	if err := dec.processKeyValueIntoMap(rootMap, tomlNode); err != nil {
 		return false, err
 	}
 
 	for dec.parser.NextExpression() {
 		nextItem := dec.parser.Expression()
-		log.Debug("!! DECODE_KV_INTO_MAP -- next exp, its a %v", nextItem.Kind)
+		log.Debug("decodeKeyValuesIntoMap -- next exp, its a %v", nextItem.Kind)
 
 		if nextItem.Kind == toml.KeyValue {
 			if err := dec.processKeyValueIntoMap(rootMap, nextItem); err != nil {
@@ -86,18 +85,17 @@ func (dec *tomlDecoder) decodeKeyValuesIntoMap(rootMap *CandidateNode, tomlNode
 			}
 		} else {
 			// run out of key values
-			log.Debug("! DECODE_KV_INTO_MAP - ok we are done in decodeKeyValuesIntoMap, gota a %v", nextItem.Kind)
-			log.Debug("! DECODE_KV_INTO_MAP - processAgainstCurrentExp = true!")
+			log.Debug("done in decodeKeyValuesIntoMap, gota a %v", nextItem.Kind)
 			return true, nil
 		}
 	}
-	log.Debug("! DECODE_KV_INTO_MAP - no more things to read in")
+	log.Debug("no more things to read in")
 	return false, nil
 }
 
-func (dec *tomlDecoder) createInlineTableMap(tomlNode *toml.Node) (*yaml.Node, error) {
-	content := make([]*yaml.Node, 0)
-	log.Debug("!! createInlineTableMap")
+func (dec *tomlDecoder) createInlineTableMap(tomlNode *toml.Node) (*CandidateNode, error) {
+	content := make([]*CandidateNode, 0)
+	log.Debug("createInlineTableMap")
 
 	iterator := tomlNode.Children()
 	for iterator.Next() {
@@ -107,28 +105,26 @@ func (dec *tomlDecoder) createInlineTableMap(tomlNode *toml.Node) (*yaml.Node, e
 		}
 
 		keyValues := &CandidateNode{
-			Node: &yaml.Node{
-				Kind: yaml.MappingNode,
-				Tag:  "!!map",
-			},
+			Kind: MappingNode,
+			Tag:  "!!map",
 		}
 
 		if err := dec.processKeyValueIntoMap(keyValues, child); err != nil {
 			return nil, err
 		}
 
-		content = append(content, keyValues.Node.Content...)
+		content = append(content, keyValues.Content...)
 	}
 
-	return &yaml.Node{
-		Kind:    yaml.MappingNode,
+	return &CandidateNode{
+		Kind:    MappingNode,
 		Tag:     "!!map",
 		Content: content,
 	}, nil
 }
 
-func (dec *tomlDecoder) createArray(tomlNode *toml.Node) (*yaml.Node, error) {
-	content := make([]*yaml.Node, 0)
+func (dec *tomlDecoder) createArray(tomlNode *toml.Node) (*CandidateNode, error) {
+	content := make([]*CandidateNode, 0)
 	iterator := tomlNode.Children()
 	for iterator.Next() {
 		child := iterator.Node()
@@ -139,43 +135,43 @@ func (dec *tomlDecoder) createArray(tomlNode *toml.Node) (*yaml.Node, error) {
 		content = append(content, yamlNode)
 	}
 
-	return &yaml.Node{
-		Kind:    yaml.SequenceNode,
+	return &CandidateNode{
+		Kind:    SequenceNode,
 		Tag:     "!!seq",
 		Content: content,
 	}, nil
 
 }
 
-func (dec *tomlDecoder) createStringScalar(tomlNode *toml.Node) (*yaml.Node, error) {
+func (dec *tomlDecoder) createStringScalar(tomlNode *toml.Node) (*CandidateNode, error) {
 	content := string(tomlNode.Data)
-	return createScalarNode(content, content), nil
+	return CreateScalarNode(content, content), nil
 }
 
-func (dec *tomlDecoder) createBoolScalar(tomlNode *toml.Node) (*yaml.Node, error) {
+func (dec *tomlDecoder) createBoolScalar(tomlNode *toml.Node) (*CandidateNode, error) {
 	content := string(tomlNode.Data)
-	return createScalarNode(content == "true", content), nil
+	return CreateScalarNode(content == "true", content), nil
 }
 
-func (dec *tomlDecoder) createIntegerScalar(tomlNode *toml.Node) (*yaml.Node, error) {
+func (dec *tomlDecoder) createIntegerScalar(tomlNode *toml.Node) (*CandidateNode, error) {
 	content := string(tomlNode.Data)
 	_, num, err := parseInt64(content)
-	return createScalarNode(num, content), err
+	return CreateScalarNode(num, content), err
 }
 
-func (dec *tomlDecoder) createDateTimeScalar(tomlNode *toml.Node) (*yaml.Node, error) {
+func (dec *tomlDecoder) createDateTimeScalar(tomlNode *toml.Node) (*CandidateNode, error) {
 	content := string(tomlNode.Data)
 	val, err := parseDateTime(time.RFC3339, content)
-	return createScalarNode(val, content), err
+	return CreateScalarNode(val, content), err
 }
 
-func (dec *tomlDecoder) createFloatScalar(tomlNode *toml.Node) (*yaml.Node, error) {
+func (dec *tomlDecoder) createFloatScalar(tomlNode *toml.Node) (*CandidateNode, error) {
 	content := string(tomlNode.Data)
 	num, err := strconv.ParseFloat(content, 64)
-	return createScalarNode(num, content), err
+	return CreateScalarNode(num, content), err
 }
 
-func (dec *tomlDecoder) decodeNode(tomlNode *toml.Node) (*yaml.Node, error) {
+func (dec *tomlDecoder) decodeNode(tomlNode *toml.Node) (*CandidateNode, error) {
 	switch tomlNode.Kind {
 	case toml.Key, toml.String:
 		return dec.createStringScalar(tomlNode)
@@ -241,81 +237,75 @@ func (dec *tomlDecoder) Decode() (*CandidateNode, error) {
 	// must have finished
 	dec.finished = true
 
-	if len(dec.rootMap.Node.Content) == 0 {
+	if len(dec.rootMap.Content) == 0 {
 		return nil, io.EOF
 	}
 
-	return &CandidateNode{
-		Node: &yaml.Node{
-			Kind:    yaml.DocumentNode,
-			Content: []*yaml.Node{dec.rootMap.Node},
-		},
-	}, deferredError
+	return dec.rootMap, deferredError
 
 }
 
 func (dec *tomlDecoder) processTopLevelNode(currentNode *toml.Node) (bool, error) {
 	var runAgainstCurrentExp bool
 	var err error
-	log.Debug("!!!!!!!!!!!!Going to process %v state is current %v", currentNode.Kind, NodeToString(dec.rootMap))
+	log.Debug("processTopLevelNode: Going to process %v state is current %v", currentNode.Kind, NodeToString(dec.rootMap))
 	if currentNode.Kind == toml.Table {
-		runAgainstCurrentExp, err = dec.processTable((currentNode))
+		runAgainstCurrentExp, err = dec.processTable(currentNode)
 	} else if currentNode.Kind == toml.ArrayTable {
-		runAgainstCurrentExp, err = dec.processArrayTable((currentNode))
+		runAgainstCurrentExp, err = dec.processArrayTable(currentNode)
 	} else {
 		runAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(dec.rootMap, currentNode)
 	}
 
-	log.Debug("!!!!!!!!!!!!DONE Processing state is now %v", NodeToString(dec.rootMap))
+	log.Debug("processTopLevelNode: DONE Processing state is now %v", NodeToString(dec.rootMap))
 	return runAgainstCurrentExp, err
 }
 
 func (dec *tomlDecoder) processTable(currentNode *toml.Node) (bool, error) {
-	log.Debug("!!! processing table")
+	log.Debug("Enter processTable")
 	fullPath := dec.getFullPath(currentNode.Child())
-	log.Debug("!!!fullpath: %v", fullPath)
-
-	hasValue := dec.parser.NextExpression()
-	if !hasValue {
-		return false, fmt.Errorf("error retrieving table %v value: %w", fullPath, dec.parser.Error())
-	}
+	log.Debug("fullpath: %v", fullPath)
 
 	tableNodeValue := &CandidateNode{
-		Node: &yaml.Node{
-			Kind: yaml.MappingNode,
-			Tag:  "!!map",
-		},
+		Kind:    MappingNode,
+		Tag:     "!!map",
+		Content: make([]*CandidateNode, 0),
 	}
 
-	tableValue := dec.parser.Expression()
-	if tableValue.Kind != toml.KeyValue {
-		log.Debug("got an empty table, returning")
-		return true, nil
+	var tableValue *toml.Node
+	runAgainstCurrentExp := false
+	var err error
+	hasValue := dec.parser.NextExpression()
+	// check to see if there is any table data
+	if hasValue {
+		tableValue = dec.parser.Expression()
+		// next expression is not table data, so we are done
+		if tableValue.Kind != toml.KeyValue {
+			log.Debug("got an empty table, returning")
+			return true, nil
+		}
+
+		runAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue)
+		if err != nil && !errors.Is(io.EOF, err) {
+			return false, err
+		}
 	}
 
-	runAgainstCurrentExp, err := dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue)
-	log.Debugf("table node err: %w", err)
-	if err != nil && !errors.Is(io.EOF, err) {
-		return false, err
-	}
 	c := Context{}
-
 	c = c.SingleChildContext(dec.rootMap)
-	err = dec.d.DeeplyAssign(c, fullPath, tableNodeValue.Node)
+	err = dec.d.DeeplyAssign(c, fullPath, tableNodeValue)
 	if err != nil {
 		return false, err
 	}
 	return runAgainstCurrentExp, nil
 }
 
-func (dec *tomlDecoder) arrayAppend(context Context, path []interface{}, rhsNode *yaml.Node) error {
+func (dec *tomlDecoder) arrayAppend(context Context, path []interface{}, rhsNode *CandidateNode) error {
+	log.Debug("arrayAppend to path: %v,%v", path, NodeToString(rhsNode))
 	rhsCandidateNode := &CandidateNode{
-		Path: path,
-		Node: &yaml.Node{
-			Kind:    yaml.SequenceNode,
-			Tag:     "!!seq",
-			Content: []*yaml.Node{rhsNode},
-		},
+		Kind:    SequenceNode,
+		Tag:     "!!seq",
+		Content: []*CandidateNode{rhsNode},
 	}
 
 	assignmentOp := &Operation{OperationType: addAssignOpType}
@@ -333,9 +323,9 @@ func (dec *tomlDecoder) arrayAppend(context Context, path []interface{}, rhsNode
 }
 
 func (dec *tomlDecoder) processArrayTable(currentNode *toml.Node) (bool, error) {
-	log.Debug("!!! processing table")
+	log.Debug("Entering processArrayTable")
 	fullPath := dec.getFullPath(currentNode.Child())
-	log.Debug("!!!fullpath: %v", fullPath)
+	log.Debug("Fullpath: %v", fullPath)
 
 	// need to use the array append exp to add another entry to
 	// this array: fullpath += [ thing ]
@@ -346,10 +336,8 @@ func (dec *tomlDecoder) processArrayTable(currentNode *toml.Node) (bool, error)
 	}
 
 	tableNodeValue := &CandidateNode{
-		Node: &yaml.Node{
-			Kind: yaml.MappingNode,
-			Tag:  "!!map",
-		},
+		Kind: MappingNode,
+		Tag:  "!!map",
 	}
 
 	tableValue := dec.parser.Expression()
@@ -363,7 +351,7 @@ func (dec *tomlDecoder) processArrayTable(currentNode *toml.Node) (bool, error)
 	c = c.SingleChildContext(dec.rootMap)
 
 	// += function
-	err = dec.arrayAppend(c, fullPath, tableNodeValue.Node)
+	err = dec.arrayAppend(c, fullPath, tableNodeValue)
 
 	return runAgainstCurrentExp, err
 }

pkg/yqlib/decoder_uri.go

@@ -4,8 +4,6 @@ import (
 	"bytes"
 	"io"
 	"net/url"
-
-	yaml "gopkg.in/yaml.v3"
 )
 
 type uriDecoder struct {
@@ -50,11 +48,5 @@ func (dec *uriDecoder) Decode() (*CandidateNode, error) {
 		return nil, err
 	}
 	dec.readAnything = true
-	return &CandidateNode{
-		Node: &yaml.Node{
-			Kind:  yaml.ScalarNode,
-			Tag:   "!!str",
-			Value: newValue,
-		},
-	}, nil
+	return CreateStringScalarNode(newValue), nil
 }

pkg/yqlib/decoder_yaml.go

@@ -20,8 +20,12 @@ type yamlDecoder struct {
 	leadingContent string
 	bufferRead     bytes.Buffer
 
-	readAnything bool
-	firstFile    bool
+	// anchor map persists over multiple documents for convenience.
+	anchorMap map[string]*CandidateNode
+
+	readAnything  bool
+	firstFile     bool
+	documentIndex uint
 }
 
 func NewYamlDecoder(prefs YamlPreferences) Decoder {
@@ -33,7 +37,7 @@ func (dec *yamlDecoder) processReadStream(reader *bufio.Reader) (io.Reader, stri
 	var yamlDirectiveLineRegEx = regexp.MustCompile(`^\s*%YA`)
 	var sb strings.Builder
 	for {
-		peekBytes, err := reader.Peek(3)
+		peekBytes, err := reader.Peek(4)
 		if errors.Is(err, io.EOF) {
 			// EOF are handled else where..
 			return reader, sb.String(), nil
@@ -47,7 +51,15 @@ func (dec *yamlDecoder) processReadStream(reader *bufio.Reader) (io.Reader, stri
 			} else if err != nil {
 				return reader, sb.String(), err
 			}
-		} else if string(peekBytes) == "---" {
+		} else if string(peekBytes) == "--- " {
+			_, err := reader.ReadString(' ')
+			sb.WriteString("$yqDocSeparator$\n")
+			if errors.Is(err, io.EOF) {
+				return reader, sb.String(), nil
+			} else if err != nil {
+				return reader, sb.String(), err
+			}
+		} else if string(peekBytes) == "---\n" {
 			_, err := reader.ReadString('\n')
 			sb.WriteString("$yqDocSeparator$\n")
 			if errors.Is(err, io.EOF) {
@@ -93,12 +105,15 @@ func (dec *yamlDecoder) Init(reader io.Reader) error {
 	dec.readAnything = false
 	dec.decoder = *yaml.NewDecoder(readerToUse)
 	dec.firstFile = false
+	dec.documentIndex = 0
+	dec.anchorMap = make(map[string]*CandidateNode)
 	return nil
 }
 
 func (dec *yamlDecoder) Decode() (*CandidateNode, error) {
-	var dataBucket yaml.Node
-	err := dec.decoder.Decode(&dataBucket)
+	var yamlNode yaml.Node
+	err := dec.decoder.Decode(&yamlNode)
+
 	if errors.Is(err, io.EOF) && dec.leadingContent != "" && !dec.readAnything {
 		// force returning an empty node with a comment.
 		dec.readAnything = true
@@ -116,28 +131,27 @@ func (dec *yamlDecoder) Decode() (*CandidateNode, error) {
 		return nil, err
 	}
 
-	candidateNode := &CandidateNode{
-		Node: &dataBucket,
+	candidateNode := CandidateNode{document: dec.documentIndex}
+	// don't bother with the DocumentNode
+	err = candidateNode.UnmarshalYAML(yamlNode.Content[0], dec.anchorMap)
+	if err != nil {
+		return nil, err
 	}
 
+	candidateNode.HeadComment = yamlNode.HeadComment + candidateNode.HeadComment
+	candidateNode.FootComment = yamlNode.FootComment + candidateNode.FootComment
+
 	if dec.leadingContent != "" {
 		candidateNode.LeadingContent = dec.leadingContent
 		dec.leadingContent = ""
 	}
 	dec.readAnything = true
-	// move document comments into candidate node
-	// otherwise unwrap drops them.
-	candidateNode.TrailingContent = dataBucket.FootComment
-	dataBucket.FootComment = ""
-	return candidateNode, nil
+	dec.documentIndex++
+	return &candidateNode, nil
 }
 
 func (dec *yamlDecoder) blankNodeWithComment() *CandidateNode {
-	return &CandidateNode{
-		Document:       0,
-		Filename:       "",
-		Node:           &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
-		FileIndex:      0,
-		LeadingContent: dec.leadingContent,
-	}
+	node := CreateScalarNode(nil, "")
+	node.LeadingContent = dec.leadingContent
+	return node
 }

pkg/yqlib/doc/operators/boolean-operators.md

@@ -4,7 +4,7 @@ The `or` and `and` operators take two parameters and return a boolean result.
 
 `not` flips a boolean from true to false, or vice versa. 
 
-`any` will return `true` if there are any `true` values in a array sequence, and `all` will return true if _all_ elements in an array are true.
+`any` will return `true` if there are any `true` values in an array sequence, and `all` will return true if _all_ elements in an array are true.
 
 `any_c(condition)` and `all_c(condition)` are like `any` and `all` but they take a condition expression that is used against each element to determine if it's `true`. Note: in `jq` you can simply pass a condition to `any` or `all` and it simply works - `yq` isn't that clever..yet
 

pkg/yqlib/doc/operators/column.md

@@ -2,6 +2,8 @@
 
 Returns the column of the matching node. Starts from 1, 0 indicates there was no column data.
 
+Column is the number of characters that precede that node on the line it starts.
+
 ## Returns column of _value_ node
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/operators/comment-operators.md

@@ -187,7 +187,6 @@ yq '. head_comment="single"' sample.yml
 will output
 ```yaml
 # single
-
 a: cat
 ```
 

pkg/yqlib/doc/operators/create-collect-into-object.md

@@ -66,6 +66,7 @@ will output
 ```yaml
 Mike: cat
 Mike: dog
+---
 Rosey: monkey
 Rosey: sheep
 ```

pkg/yqlib/doc/operators/document-index.md

@@ -85,6 +85,7 @@ will output
 ```yaml
 match: cat
 doc: 0
+---
 match: frog
 doc: 1
 ```

pkg/yqlib/doc/operators/file-operators.md

@@ -54,7 +54,6 @@ yq eval-all 'file_index' sample.yml another.yml
 will output
 ```yaml
 0
----
 1
 ```
 

pkg/yqlib/doc/operators/headers/Main.md

@@ -1,5 +1,5 @@
 # NAME
-  *yq* is a portable command-line YAML processor
+  *yq* is a portable command-line data file processor
 
 # SYNOPSIS 
 
@@ -11,7 +11,7 @@ eval-all/ea - Loads all yaml documents of all yaml files and runs expression onc
 
 # DESCRIPTION
 
-a lightweight and portable command-line YAML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.
+a lightweight and portable command-line data file processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml, json, xml, csv, properties and TOML files. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.
 
 This documentation is also available at https://mikefarah.gitbook.io/yq/
 # QUICK GUIDE 

pkg/yqlib/doc/operators/headers/boolean-operators.md

@@ -4,7 +4,7 @@ The `or` and `and` operators take two parameters and return a boolean result.
 
 `not` flips a boolean from true to false, or vice versa. 
 
-`any` will return `true` if there are any `true` values in a array sequence, and `all` will return true if _all_ elements in an array are true.
+`any` will return `true` if there are any `true` values in an array sequence, and `all` will return true if _all_ elements in an array are true.
 
 `any_c(condition)` and `all_c(condition)` are like `any` and `all` but they take a condition expression that is used against each element to determine if it's `true`. Note: in `jq` you can simply pass a condition to `any` or `all` and it simply works - `yq` isn't that clever..yet
 

pkg/yqlib/doc/operators/headers/column.md

@@ -1,3 +1,5 @@
 # Column
 
 Returns the column of the matching node. Starts from 1, 0 indicates there was no column data.
+
+Column is the number of characters that precede that node on the line it starts.

pkg/yqlib/doc/operators/headers/multiply-merge.md

@@ -3,7 +3,9 @@
 Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently numbers, arrays and objects are supported.
 
 ## Objects and arrays - merging
-Objects are merged deeply matching on matching keys. By default, array values override and are not deeply merged.
+Objects are merged _deeply_ matching on matching keys. By default, array values override and are not deeply merged.
+
+You can use the add operator `+`, to shallow merge objects, see more info [here](https://mikefarah.gitbook.io/yq/operators/add).
 
 Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.
 
@@ -16,6 +18,7 @@ You can control how objects are merged by using one or more of the following fla
 - `n` only merge _new_ fields
 - `c` clobber custom tags
 
+To perform a shallow merge only, use the add operator `+`, see more info [here](https://mikefarah.gitbook.io/yq/operators/add).
 
 ### Merge two files together
 This uses the load operator to merge file2 into file1.

pkg/yqlib/doc/operators/headers/style.md

@@ -1,3 +1,5 @@
 # Style
 
-The style operator can be used to get or set the style of nodes (e.g. string style, yaml style)
+The style operator can be used to get or set the style of nodes (e.g. string style, yaml style).
+Use this to control the formatting of the document in yaml.
+

pkg/yqlib/doc/operators/headers/to_number.md

@@ -0,0 +1,2 @@
+# To Number
+Parses the input as a number. yq will try to parse values as an int first, failing that it will try float. Values that already ints or floats will be left alone.

pkg/yqlib/doc/operators/kind.md

@@ -0,0 +1,77 @@
+
+## Get kind
+Given a sample.yml file of:
+```yaml
+a: cat
+b: 5
+c: 3.2
+e: true
+f: []
+g: {}
+h: null
+```
+then
+```bash
+yq '.. | kind' sample.yml
+```
+will output
+```yaml
+map
+scalar
+scalar
+scalar
+scalar
+seq
+map
+scalar
+```
+
+## Get kind, ignores custom tags
+Unlike tag, kind is not affected by custom tags.
+
+Given a sample.yml file of:
+```yaml
+a: !!thing cat
+b: !!foo {}
+c: !!bar []
+```
+then
+```bash
+yq '.. | kind' sample.yml
+```
+will output
+```yaml
+map
+scalar
+map
+seq
+```
+
+## Add comments only to scalars
+An example of how you can use kind
+
+Given a sample.yml file of:
+```yaml
+a:
+  b: 5
+  c: 3.2
+e: true
+f: []
+g: {}
+h: null
+```
+then
+```bash
+yq '(.. | select(kind == "scalar")) line_comment = "this is a scalar"' sample.yml
+```
+will output
+```yaml
+a:
+  b: 5 # this is a scalar
+  c: 3.2 # this is a scalar
+e: true # this is a scalar
+f: []
+g: {}
+h: null # this is a scalar
+```
+

pkg/yqlib/doc/operators/multiply-merge.md

@@ -3,7 +3,9 @@
 Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently numbers, arrays and objects are supported.
 
 ## Objects and arrays - merging
-Objects are merged deeply matching on matching keys. By default, array values override and are not deeply merged.
+Objects are merged _deeply_ matching on matching keys. By default, array values override and are not deeply merged.
+
+You can use the add operator `+`, to shallow merge objects, see more info [here](https://mikefarah.gitbook.io/yq/operators/add).
 
 Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.
 
@@ -16,6 +18,7 @@ You can control how objects are merged by using one or more of the following fla
 - `n` only merge _new_ fields
 - `c` clobber custom tags
 
+To perform a shallow merge only, use the add operator `+`, see more info [here](https://mikefarah.gitbook.io/yq/operators/add).
 
 ### Merge two files together
 This uses the load operator to merge file2 into file1.
@@ -519,7 +522,7 @@ will output
 some: thing
 ```
 
-## Merging an null with an array
+## Merging a null with an array
 Running
 ```bash
 yq --null-input 'null * ["some"]'

pkg/yqlib/doc/operators/parent.md

@@ -37,6 +37,43 @@ fruit: banana
 name: sam
 ```
 
+## N-th parent
+You can optionally supply the number of levels to go up for the parent, the default being 1.
+
+Given a sample.yml file of:
+```yaml
+a:
+  b:
+    c: cat
+```
+then
+```bash
+yq '.a.b.c | parent(2)' sample.yml
+```
+will output
+```yaml
+b:
+  c: cat
+```
+
+## N-th parent - another level
+Given a sample.yml file of:
+```yaml
+a:
+  b:
+    c: cat
+```
+then
+```bash
+yq '.a.b.c | parent(3)' sample.yml
+```
+will output
+```yaml
+a:
+  b:
+    c: cat
+```
+
 ## No parent
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/operators/string-operators.md

@@ -56,6 +56,40 @@ IFS= read -rd '' output < <(cat my_file)
 output=$output ./yq '.data.values = strenv(output)' first.yml
 ```
 
+## Interpolation
+Given a sample.yml file of:
+```yaml
+value: things
+another: stuff
+```
+then
+```bash
+yq '.message = "I like \(.value) and \(.another)"' sample.yml
+```
+will output
+```yaml
+value: things
+another: stuff
+message: I like things and stuff
+```
+
+## Interpolation - not a string
+Given a sample.yml file of:
+```yaml
+value:
+  an: apple
+```
+then
+```bash
+yq '.message = "I like \(.value)"' sample.yml
+```
+will output
+```yaml
+value:
+  an: apple
+message: 'I like an: apple'
+```
+
 ## To up (upper) case
 Works with unicode characters
 
@@ -383,3 +417,32 @@ will output
 - word
 ```
 
+## To string
+Note that you may want to force `yq` to leave scalar values wrapped by passing in `--unwrapScalar=false` or `-r=f`
+
+Given a sample.yml file of:
+```yaml
+- 1
+- true
+- null
+- ~
+- cat
+- an: object
+- - array
+  - 2
+```
+then
+```bash
+yq '.[] |= to_string' sample.yml
+```
+will output
+```yaml
+- "1"
+- "true"
+- "null"
+- "~"
+- cat
+- "an: object"
+- "- array\n- 2"
+```
+

pkg/yqlib/doc/operators/style.md

@@ -1,6 +1,8 @@
 # Style
 
-The style operator can be used to get or set the style of nodes (e.g. string style, yaml style)
+The style operator can be used to get or set the style of nodes (e.g. string style, yaml style).
+Use this to control the formatting of the document in yaml.
+
 
 ## Update and set style of a particular node (simple)
 Given a sample.yml file of:
@@ -45,6 +47,12 @@ a: cat
 b: 5
 c: 3.2
 e: true
+f:
+  - 1
+  - 2
+  - 3
+g:
+  something: cool
 ```
 then
 ```bash
@@ -57,6 +65,12 @@ a: !!str cat
 b: !!int 5
 c: !!float 3.2
 e: !!bool true
+f: !!seq
+  - !!int 1
+  - !!int 2
+  - !!int 3
+g: !!map
+  something: !!str cool
 ```
 
 ## Set double quote style
@@ -66,6 +80,12 @@ a: cat
 b: 5
 c: 3.2
 e: true
+f:
+  - 1
+  - 2
+  - 3
+g:
+  something: cool
 ```
 then
 ```bash
@@ -77,6 +97,12 @@ a: "cat"
 b: "5"
 c: "3.2"
 e: "true"
+f:
+  - "1"
+  - "2"
+  - "3"
+g:
+  something: "cool"
 ```
 
 ## Set double quote style on map keys too
@@ -86,6 +112,12 @@ a: cat
 b: 5
 c: 3.2
 e: true
+f:
+  - 1
+  - 2
+  - 3
+g:
+  something: cool
 ```
 then
 ```bash
@@ -97,6 +129,12 @@ will output
 "b": "5"
 "c": "3.2"
 "e": "true"
+"f":
+  - "1"
+  - "2"
+  - "3"
+"g":
+  "something": "cool"
 ```
 
 ## Set single quote style
@@ -106,6 +144,12 @@ a: cat
 b: 5
 c: 3.2
 e: true
+f:
+  - 1
+  - 2
+  - 3
+g:
+  something: cool
 ```
 then
 ```bash
@@ -117,6 +161,12 @@ a: 'cat'
 b: '5'
 c: '3.2'
 e: 'true'
+f:
+  - '1'
+  - '2'
+  - '3'
+g:
+  something: 'cool'
 ```
 
 ## Set literal quote style
@@ -126,6 +176,12 @@ a: cat
 b: 5
 c: 3.2
 e: true
+f:
+  - 1
+  - 2
+  - 3
+g:
+  something: cool
 ```
 then
 ```bash
@@ -141,6 +197,16 @@ c: |-
   3.2
 e: |-
   true
+f:
+  - |-
+    1
+  - |-
+    2
+  - |-
+    3
+g:
+  something: |-
+    cool
 ```
 
 ## Set folded quote style
@@ -150,6 +216,12 @@ a: cat
 b: 5
 c: 3.2
 e: true
+f:
+  - 1
+  - 2
+  - 3
+g:
+  something: cool
 ```
 then
 ```bash
@@ -165,6 +237,16 @@ c: >-
   3.2
 e: >-
   true
+f:
+  - >-
+    1
+  - >-
+    2
+  - >-
+    3
+g:
+  something: >-
+    cool
 ```
 
 ## Set flow quote style
@@ -174,6 +256,12 @@ a: cat
 b: 5
 c: 3.2
 e: true
+f:
+  - 1
+  - 2
+  - 3
+g:
+  something: cool
 ```
 then
 ```bash
@@ -181,7 +269,7 @@ yq '.. style="flow"' sample.yml
 ```
 will output
 ```yaml
-{a: cat, b: 5, c: 3.2, e: true}
+{a: cat, b: 5, c: 3.2, e: true, f: [1, 2, 3], g: {something: cool}}
 ```
 
 ## Reset style - or pretty print
@@ -189,10 +277,7 @@ Set empty (default) quote style, note the usage of `...` to match keys too. Note
 
 Given a sample.yml file of:
 ```yaml
-a: cat
-"b": 5
-'c': 3.2
-"e": true
+{a: cat, "b": 5, 'c': 3.2, "e": true,  f: [1,2,3], "g": { something: "cool"} }
 ```
 then
 ```bash
@@ -204,6 +289,12 @@ a: cat
 b: 5
 c: 3.2
 e: true
+f:
+  - 1
+  - 2
+  - 3
+g:
+  something: cool
 ```
 
 ## Set style relatively with assign-update

pkg/yqlib/doc/operators/to_number.md

@@ -0,0 +1,49 @@
+# To Number
+Parses the input as a number. yq will try to parse values as an int first, failing that it will try float. Values that already ints or floats will be left alone.
+
+## Converts strings to numbers
+Given a sample.yml file of:
+```yaml
+- "3"
+- "3.1"
+- "-1e3"
+```
+then
+```bash
+yq '.[] | to_number' sample.yml
+```
+will output
+```yaml
+3
+3.1
+-1e3
+```
+
+## Doesn't change numbers
+Given a sample.yml file of:
+```yaml
+- 3
+- 3.1
+- -1e3
+```
+then
+```bash
+yq '.[] | to_number' sample.yml
+```
+will output
+```yaml
+3
+3.1
+-1e3
+```
+
+## Cannot convert null
+Running
+```bash
+yq --null-input '.a.b | to_number'
+```
+will output
+```bash
+Error: cannot convert node value [null] at path a.b of tag !!null to number
+```
+

pkg/yqlib/doc/usage/convert.md

@@ -14,7 +14,7 @@ Given a sample.json file of:
 ```
 then
 ```bash
-yq -P '.' sample.json
+yq -p=json sample.json
 ```
 will output
 ```yaml
@@ -30,7 +30,7 @@ Given a sample.json file of:
 ```
 then
 ```bash
-yq -P '.' sample.json
+yq -p=json sample.json
 ```
 will output
 ```yaml
@@ -125,9 +125,7 @@ will output
 {"whatever":"cat"}
 ```
 
-## Roundtrip NDJSON
-Unfortunately the json encoder strips leading spaces of values.
-
+## Roundtrip JSON Lines / NDJSON
 Given a sample.json file of:
 ```json
 {"this": "is a multidoc json file"}
@@ -147,7 +145,7 @@ will output
 ```
 
 ## Roundtrip multi-document JSON
-The NDJSON parser can also handle multiple multi-line json documents in a single file!
+The parser can also handle multiple multi-line json documents in a single file (despite this not being in the JSON Lines / NDJSON spec). Typically you would have one entire JSON document per line, but the parser also supports multiple multi-line json documents
 
 Given a sample.json file of:
 ```json
@@ -229,7 +227,7 @@ will output
 {"a number":4}
 ```
 
-## Decode NDJSON
+## Decode JSON Lines / NDJSON
 Given a sample.json file of:
 ```json
 {"this": "is a multidoc json file"}
@@ -246,7 +244,7 @@ will output
 this: is a multidoc json file
 ---
 each:
-    - line is a valid json document
+  - line is a valid json document
 ---
 a number: 4
 ```

pkg/yqlib/doc/usage/csv-tsv.md

@@ -136,13 +136,13 @@ Samantha's Rabbit,,-188.8
 ```
 
 ## Parse CSV into an array of objects
-First row is assumed to be the header row.
+First row is assumed to be the header row. By default, entries with YAML/JSON formatting will be parsed!
 
 Given a sample.csv file of:
 ```csv
-name,numberOfCats,likesApples,height
-Gary,1,true,168.8
-Samantha's Rabbit,2,false,-188.8
+name,numberOfCats,likesApples,height,facts
+Gary,1,true,168.8,cool: true
+Samantha's Rabbit,2,false,-188.8,tall: indeed
 
 ```
 then
@@ -155,10 +155,42 @@ will output
   numberOfCats: 1
   likesApples: true
   height: 168.8
+  facts:
+    cool: true
 - name: Samantha's Rabbit
   numberOfCats: 2
   likesApples: false
   height: -188.8
+  facts:
+    tall: indeed
+```
+
+## Parse CSV into an array of objects, no auto-parsing
+First row is assumed to be the header row. Entries with YAML/JSON will be left as strings.
+
+Given a sample.csv file of:
+```csv
+name,numberOfCats,likesApples,height,facts
+Gary,1,true,168.8,cool: true
+Samantha's Rabbit,2,false,-188.8,tall: indeed
+
+```
+then
+```bash
+yq -p=csv --csv-auto-parse=f sample.csv
+```
+will output
+```yaml
+- name: Gary
+  numberOfCats: 1
+  likesApples: true
+  height: 168.8
+  facts: 'cool: true'
+- name: Samantha's Rabbit
+  numberOfCats: 2
+  likesApples: false
+  height: -188.8
+  facts: 'tall: indeed'
 ```
 
 ## Parse TSV into an array of objects

pkg/yqlib/doc/usage/formatting-expressions.md

@@ -0,0 +1,100 @@
+# Formatting Expressions
+
+`From version v4.41+`
+
+You can put expressions into `.yq` files, use whitespace and comments to break up complex expressions and explain what's going on.
+
+## Using expression files and comments
+Note that you can execute the file directly - but make sure you make the expression file executable.
+
+Given a sample.yaml file of:
+```yaml
+a:
+  b: old
+```
+And an 'update.yq' expression file of:
+```bash
+#! yq
+
+# This is a yq expression that updates the map
+# for several great reasons outlined here.
+
+.a.b = "new" # line comment here
+| .a.c = "frog"
+
+# Now good things will happen.
+```
+then
+```bash
+./update.yq sample.yaml
+```
+will output
+```yaml
+a:
+  b: new
+  c: frog
+```
+
+## Flags in expression files
+You can specify flags on the shebang line, this only works when executing the file directly.
+
+Given a sample.yaml file of:
+```yaml
+a:
+  b: old
+```
+And an 'update.yq' expression file of:
+```bash
+#! yq -oj
+
+# This is a yq expression that updates the map
+# for several great reasons outlined here.
+
+.a.b = "new" # line comment here
+| .a.c = "frog"
+
+# Now good things will happen.
+```
+then
+```bash
+./update.yq sample.yaml
+```
+will output
+```yaml
+{
+  "a": {
+    "b": "new",
+    "c": "frog"
+  }
+}
+```
+
+## Commenting out yq expressions
+Note that `c` is no longer set to 'frog'. In this example we're calling yq directly and passing the expression file into `--from-file`, this is no different from executing the expression file directly.
+
+Given a sample.yaml file of:
+```yaml
+a:
+  b: old
+```
+And an 'update.yq' expression file of:
+```bash
+#! yq
+# This is a yq expression that updates the map
+# for several great reasons outlined here.
+
+.a.b = "new" # line comment here
+# | .a.c = "frog"
+
+# Now good things will happen.
+```
+then
+```bash
+yq --from-file update.yq sample.yml
+```
+will output
+```yaml
+a:
+  b: new
+```
+

pkg/yqlib/doc/usage/headers/formatting-expressions.md

@@ -0,0 +1,5 @@
+# Formatting Expressions
+
+`From version v4.41+`
+
+You can put expressions into `.yq` files, use whitespace and comments to break up complex expressions and explain what's going on.

pkg/yqlib/doc/usage/lua.md

@@ -1,5 +1,33 @@
 
-## Basic example
+## Basic input example
+Given a sample.lua file of:
+```lua
+return {
+	["country"] = "Australia"; -- this place
+	["cities"] = {
+		"Sydney",
+		"Melbourne",
+		"Brisbane",
+		"Perth",
+	};
+};
+
+```
+then
+```bash
+yq -oy '.' sample.lua
+```
+will output
+```yaml
+country: Australia
+cities:
+  - Sydney
+  - Melbourne
+  - Brisbane
+  - Perth
+```
+
+## Basic output example
 Given a sample.yml file of:
 ```yaml
 ---
@@ -101,6 +129,8 @@ numbers:
   - octal: 0o30
   - float: 123.45
   - infinity: .inf
+    plus_infinity: +.inf
+    minus_infinity: -.inf
   - not: .nan
 
 ```
@@ -134,6 +164,8 @@ return {
 		},
 		{
 			["infinity"] = (1/0);
+			["plus_infinity"] = (1/0);
+			["minus_infinity"] = (-1/0);
 		},
 		{
 			["not"] = (0/0);

pkg/yqlib/doc/usage/properties.md

@@ -14,6 +14,8 @@ person: # neither do comments on maps
     name: Mike Wazowski # comments on values appear
     pets: 
     - cat # comments on array values appear
+    - nested:
+        - list entry
     food: [pizza] # comments on arrays do not
 emptyArray: []
 emptyMap: []
@@ -31,9 +33,76 @@ person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = cat
+person.pets.1.nested.0 = list entry
 person.food.0 = pizza
 ```
 
+## Encode properties with array brackets
+Declare the --properties-array-brackets flag to give array paths in brackets (e.g. SpringBoot).
+
+Given a sample.yml file of:
+```yaml
+# block comments come through
+person: # neither do comments on maps
+    name: Mike Wazowski # comments on values appear
+    pets: 
+    - cat # comments on array values appear
+    - nested:
+        - list entry
+    food: [pizza] # comments on arrays do not
+emptyArray: []
+emptyMap: []
+
+```
+then
+```bash
+yq -o=props --properties-array-brackets sample.yml
+```
+will output
+```properties
+# block comments come through
+# comments on values appear
+person.name = Mike Wazowski
+
+# comments on array values appear
+person.pets[0] = cat
+person.pets[1].nested[0] = list entry
+person.food[0] = pizza
+```
+
+## Encode properties - custom separator
+Use the --properties-customer-separator flag to specify your own key/value separator.
+
+Given a sample.yml file of:
+```yaml
+# block comments come through
+person: # neither do comments on maps
+    name: Mike Wazowski # comments on values appear
+    pets: 
+    - cat # comments on array values appear
+    - nested:
+        - list entry
+    food: [pizza] # comments on arrays do not
+emptyArray: []
+emptyMap: []
+
+```
+then
+```bash
+yq -o=props --properties-customer-separator=" :@ " sample.yml
+```
+will output
+```properties
+# block comments come through
+# comments on values appear
+person.name :@ Mike Wazowski
+
+# comments on array values appear
+person.pets.0 :@ cat
+person.pets.1.nested.0 :@ list entry
+person.food.0 :@ pizza
+```
+
 ## Encode properties: scalar encapsulation
 Note that string values with blank characters in them are encapsulated with double quotes
 
@@ -44,6 +113,8 @@ person: # neither do comments on maps
     name: Mike Wazowski # comments on values appear
     pets: 
     - cat # comments on array values appear
+    - nested:
+        - list entry
     food: [pizza] # comments on arrays do not
 emptyArray: []
 emptyMap: []
@@ -61,6 +132,7 @@ person.name = "Mike Wazowski"
 
 # comments on array values appear
 person.pets.0 = cat
+person.pets.1.nested.0 = "list entry"
 person.food.0 = pizza
 ```
 
@@ -72,6 +144,8 @@ person: # neither do comments on maps
     name: Mike Wazowski # comments on values appear
     pets: 
     - cat # comments on array values appear
+    - nested:
+        - list entry
     food: [pizza] # comments on arrays do not
 emptyArray: []
 emptyMap: []
@@ -85,6 +159,7 @@ will output
 ```properties
 person.name = Mike Wazowski
 person.pets.0 = cat
+person.pets.1.nested.0 = list entry
 person.food.0 = pizza
 ```
 
@@ -98,6 +173,8 @@ person: # neither do comments on maps
     name: Mike Wazowski # comments on values appear
     pets: 
     - cat # comments on array values appear
+    - nested:
+        - list entry
     food: [pizza] # comments on arrays do not
 emptyArray: []
 emptyMap: []
@@ -115,6 +192,7 @@ person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = cat
+person.pets.1.nested.0 = list entry
 person.food.0 = pizza
 emptyArray = 
 emptyMap = 
@@ -129,6 +207,7 @@ person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = cat
+person.pets.1.nested.0 = list entry
 person.food.0 = pizza
 
 ```
@@ -139,14 +218,16 @@ yq -p=props sample.properties
 will output
 ```yaml
 person:
-    # block comments come through
-    # comments on values appear
-    name: Mike Wazowski
-    pets:
-        # comments on array values appear
-        - cat
-    food:
-        - pizza
+  # block comments come through
+  # comments on values appear
+  name: Mike Wazowski
+  pets:
+    # comments on array values appear
+    - cat
+    - nested:
+        - list entry
+  food:
+    - pizza
 ```
 
 ## Decode properties - array should be a map
@@ -163,7 +244,7 @@ yq -p=props '.things |= array_to_map' sample.properties
 will output
 ```yaml
 things:
-    10: mike
+  10: mike
 ```
 
 ## Roundtrip
@@ -175,6 +256,7 @@ person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = cat
+person.pets.1.nested.0 = list entry
 person.food.0 = pizza
 
 ```
@@ -190,6 +272,7 @@ person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = dog
+person.pets.1.nested.0 = list entry
 person.food.0 = pizza
 ```
 

pkg/yqlib/doc/usage/recipes.md

@@ -59,6 +59,44 @@ will output
 - The expression `. + 1` increments the numBuckets counter.
 - See the [assign](https://mikefarah.gitbook.io/yq/operators/assign-update) and [add](https://mikefarah.gitbook.io/yq/operators/add) operators for more information.
 
+## Deeply prune a tree
+Say we are only interested in child1 and child2, and want to filter everything else out.
+
+Given a sample.yml file of:
+```yaml
+parentA:
+  - bob
+parentB:
+  child1: i am child1
+  child3: hiya
+parentC:
+  childX: cool
+  child2: me child2
+```
+then
+```bash
+yq '(
+  .. | # recurse through all the nodes
+  select(has("child1") or has("child2")) | # match parents that have either child1 or child2
+  (.child1, .child2) | # select those children
+  select(.) # filter out nulls
+) as $i ireduce({};  # using that set of nodes, create a new result map
+  setpath($i | path; $i) # and put in each node, using its original path
+)' sample.yml
+```
+will output
+```yaml
+parentB:
+  child1: i am child1
+parentC:
+  child2: me child2
+```
+
+### Explanation:
+- Find all the matching child1 and child2 nodes
+- Using ireduce, create a new map using just those nodes
+- Set each node into the new map using its original path
+
 ## Multiple or complex updates to items in an array
 We have an array and we want to _update_ the elements with a particular name in reference to its type.
 
@@ -117,7 +155,7 @@ myArray:
 - So, we use `|=` to update `.myArray`. This is the same as doing `.myArray = (.myArray | sort_by(.numBuckets))`
 
 ## Filter, flatten, sort and unique
-Lets
+Lets find the unique set of names from the document.
 
 Given a sample.yml file of:
 ```yaml
@@ -153,3 +191,74 @@ will output
 - Next we pipe `|` that through `sort` and then `unique` to get a sorted, unique list of the names!
 - See the [flatten](https://mikefarah.gitbook.io/yq/operators/flatten), [sort](https://mikefarah.gitbook.io/yq/operators/sort) and [unique](https://mikefarah.gitbook.io/yq/operators/unique) for more information and examples.
 
+## Export as environment variables (script), or any custom format
+Given a yaml document, lets output a script that will configure environment variables with that data. This same approach can be used for exporting into custom formats.
+
+Given a sample.yml file of:
+```yaml
+var0: string0
+var1: string1
+fruit:
+  - apple
+  - banana
+  - peach
+```
+then
+```bash
+yq '.[] |(
+	( select(kind == "scalar") | key + "='\''" + . + "'\''"),
+	( select(kind == "seq") | key + "=(" + (map("'\''" + . + "'\''") | join(",")) + ")")
+)' sample.yml
+```
+will output
+```yaml
+var0='string0'
+var1='string1'
+fruit=('apple','banana','peach')
+```
+
+### Explanation:
+- `.[]` matches all top level elements
+- We need a string expression for each of the different types that will produce the bash syntax, we'll use the union operator, to join them together
+- Scalars, we just need the key and quoted value: `( select(kind == "scalar") | key + "='" + . + "'")`
+- Sequences (or arrays) are trickier, we need to quote each value and `join` them with `,`: `map("'" + . + "'") | join(",")`
+
+## Custom format with nested data
+Like the previous example, but lets handle nested data structures. In this custom example, we're going to join the property paths with _. The important thing to keep in mind is that our expression is not recursive (despite the data structure being so). Instead we match _all_ elements on the tree and operate on them.
+
+Given a sample.yml file of:
+```yaml
+simple: string0
+simpleArray:
+  - apple
+  - banana
+  - peach
+deep:
+  property: value
+  array:
+    - cat
+```
+then
+```bash
+yq '.. |(
+	( select(kind == "scalar" and parent | kind != "seq") | (path | join("_")) + "='\''" + . + "'\''"),
+	( select(kind == "seq") | (path | join("_")) + "=(" + (map("'\''" + . + "'\''") | join(",")) + ")")
+)' sample.yml
+```
+will output
+```yaml
+simple='string0'
+deep_property='value'
+simpleArray=('apple','banana','peach')
+deep_array=('cat')
+```
+
+### Explanation:
+- You'll need to understand how the previous example works to understand this extension.
+- `..` matches _all_ elements, instead of `.[]` from the previous example that just matches top level elements.
+- Like before, we need a string expression for each of the different types that will produce the bash syntax, we'll use the union operator, to join them together
+- This time, however, our expression matches every node in the data structure.
+- We only want to print scalars that are not in arrays (because we handle the separately), so well add `and parent | kind != "seq"` to the select operator expression for scalars
+- We don't just want the key any more, we want the full path. So instead of `key` we have `path | join("_")`
+- The expression for sequences follows the same logic
+

pkg/yqlib/doc/usage/toml.md

@@ -104,3 +104,19 @@ owner:
       suburb: nice
 ```
 
+## Parse: Empty Table
+Given a sample.toml file of:
+```toml
+
+[dependencies]
+
+```
+then
+```bash
+yq -oy '.' sample.toml
+```
+will output
+```yaml
+dependencies: {}
+```
+

pkg/yqlib/doc/usage/xml.md

@@ -129,15 +129,13 @@ zoo:
 ```
 
 ## Parse xml: force all as an array
-Because of the way yq works, when updating everything you need to update the children before the parents. By default `..` will match parents first, so we reverse that before updating.
-
 Given a sample.xml file of:
 ```xml
 <zoo><thing><frog>boing</frog></thing></zoo>
 ```
 then
 ```bash
-yq -oy '([..] | reverse | .[]) |= [] + .' sample.xml
+yq -oy '.. |= [] + .' sample.xml
 ```
 will output
 ```yaml
@@ -372,7 +370,7 @@ cat: purrs
 ```
 then
 ```bash
-yq -o=xml '.' sample.yml
+yq -o=xml sample.yml
 ```
 will output
 ```xml
@@ -389,7 +387,7 @@ pets:
 ```
 then
 ```bash
-yq -o=xml '.' sample.yml
+yq -o=xml sample.yml
 ```
 will output
 ```xml
@@ -411,7 +409,7 @@ cat:
 ```
 then
 ```bash
-yq -o=xml '.' sample.yml
+yq -o=xml sample.yml
 ```
 will output
 ```xml
@@ -432,7 +430,7 @@ cat:
 ```
 then
 ```bash
-yq -o=xml '.' sample.yml
+yq -o=xml sample.yml
 ```
 will output
 ```xml
@@ -459,7 +457,7 @@ cat: # inline_cat
 ```
 then
 ```bash
-yq -o=xml '.' sample.yml
+yq -o=xml sample.yml
 ```
 will output
 ```xml
@@ -489,7 +487,7 @@ apple:
 ```
 then
 ```bash
-yq -o=xml '.' sample.yml
+yq -o=xml sample.yml
 ```
 will output
 ```xml

pkg/yqlib/encoder.go

@@ -2,28 +2,11 @@ package yqlib
 
 import (
 	"io"
-
-	yaml "gopkg.in/yaml.v3"
 )
 
 type Encoder interface {
-	Encode(writer io.Writer, node *yaml.Node) error
+	Encode(writer io.Writer, node *CandidateNode) error
 	PrintDocumentSeparator(writer io.Writer) error
 	PrintLeadingContent(writer io.Writer, content string) error
 	CanHandleAliases() bool
 }
-
-func mapKeysToStrings(node *yaml.Node) {
-
-	if node.Kind == yaml.MappingNode {
-		for index, child := range node.Content {
-			if index%2 == 0 { // its a map key
-				child.Tag = "!!str"
-			}
-		}
-	}
-
-	for _, child := range node.Content {
-		mapKeysToStrings(child)
-	}
-}

pkg/yqlib/encoder_base64.go

@@ -4,8 +4,6 @@ import (
 	"encoding/base64"
 	"fmt"
 	"io"
-
-	yaml "gopkg.in/yaml.v3"
 )
 
 type base64Encoder struct {
@@ -20,19 +18,18 @@ func (e *base64Encoder) CanHandleAliases() bool {
 	return false
 }
 
-func (e *base64Encoder) PrintDocumentSeparator(writer io.Writer) error {
+func (e *base64Encoder) PrintDocumentSeparator(_ io.Writer) error {
 	return nil
 }
 
-func (e *base64Encoder) PrintLeadingContent(writer io.Writer, content string) error {
+func (e *base64Encoder) PrintLeadingContent(_ io.Writer, _ string) error {
 	return nil
 }
 
-func (e *base64Encoder) Encode(writer io.Writer, originalNode *yaml.Node) error {
-	node := unwrapDoc(originalNode)
-	if guessTagFromCustomType(node) != "!!str" {
-		return fmt.Errorf("cannot encode %v as base64, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string", node.Tag)
+func (e *base64Encoder) Encode(writer io.Writer, node *CandidateNode) error {
+	if node.GuessTagFromCustomType() != "!!str" {
+		return fmt.Errorf("cannot encode %v as base64, can only operate on strings", node.Tag)
 	}
-	_, err := writer.Write([]byte(e.encoding.EncodeToString([]byte(originalNode.Value))))
+	_, err := writer.Write([]byte(e.encoding.EncodeToString([]byte(node.Value))))
 	return err
 }

pkg/yqlib/encoder_csv.go

@@ -4,36 +4,34 @@ import (
 	"encoding/csv"
 	"fmt"
 	"io"
-
-	yaml "gopkg.in/yaml.v3"
 )
 
 type csvEncoder struct {
 	separator rune
 }
 
-func NewCsvEncoder(separator rune) Encoder {
-	return &csvEncoder{separator: separator}
+func NewCsvEncoder(prefs CsvPreferences) Encoder {
+	return &csvEncoder{separator: prefs.Separator}
 }
 
 func (e *csvEncoder) CanHandleAliases() bool {
 	return false
 }
 
-func (e *csvEncoder) PrintDocumentSeparator(writer io.Writer) error {
+func (e *csvEncoder) PrintDocumentSeparator(_ io.Writer) error {
 	return nil
 }
 
-func (e *csvEncoder) PrintLeadingContent(writer io.Writer, content string) error {
+func (e *csvEncoder) PrintLeadingContent(_ io.Writer, _ string) error {
 	return nil
 }
 
-func (e *csvEncoder) encodeRow(csvWriter *csv.Writer, contents []*yaml.Node) error {
+func (e *csvEncoder) encodeRow(csvWriter *csv.Writer, contents []*CandidateNode) error {
 	stringValues := make([]string, len(contents))
 
 	for i, child := range contents {
 
-		if child.Kind != yaml.ScalarNode {
+		if child.Kind != ScalarNode {
 			return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag)
 		}
 		stringValues[i] = child.Value
@@ -41,10 +39,10 @@ func (e *csvEncoder) encodeRow(csvWriter *csv.Writer, contents []*yaml.Node) err
 	return csvWriter.Write(stringValues)
 }
 
-func (e *csvEncoder) encodeArrays(csvWriter *csv.Writer, content []*yaml.Node) error {
+func (e *csvEncoder) encodeArrays(csvWriter *csv.Writer, content []*CandidateNode) error {
 	for i, child := range content {
 
-		if child.Kind != yaml.SequenceNode {
+		if child.Kind != SequenceNode {
 			return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag)
 		}
 		err := e.encodeRow(csvWriter, child.Content)
@@ -55,19 +53,19 @@ func (e *csvEncoder) encodeArrays(csvWriter *csv.Writer, content []*yaml.Node) e
 	return nil
 }
 
-func (e *csvEncoder) extractHeader(child *yaml.Node) ([]*yaml.Node, error) {
-	if child.Kind != yaml.MappingNode {
+func (e *csvEncoder) extractHeader(child *CandidateNode) ([]*CandidateNode, error) {
+	if child.Kind != MappingNode {
 		return nil, fmt.Errorf("csv object encoding only works for arrays of flat objects (string key => string/numbers/boolean value), child[0] is a %v", child.Tag)
 	}
 	mapKeys := getMapKeys(child)
 	return mapKeys.Content, nil
 }
 
-func (e *csvEncoder) createChildRow(child *yaml.Node, headers []*yaml.Node) []*yaml.Node {
-	childRow := make([]*yaml.Node, 0)
+func (e *csvEncoder) createChildRow(child *CandidateNode, headers []*CandidateNode) []*CandidateNode {
+	childRow := make([]*CandidateNode, 0)
 	for _, header := range headers {
 		keyIndex := findKeyInMap(child, header)
-		value := createScalarNode(nil, "")
+		value := CreateScalarNode(nil, "")
 		if keyIndex != -1 {
 			value = child.Content[keyIndex+1]
 		}
@@ -77,7 +75,7 @@ func (e *csvEncoder) createChildRow(child *yaml.Node, headers []*yaml.Node) []*y
 
 }
 
-func (e *csvEncoder) encodeObjects(csvWriter *csv.Writer, content []*yaml.Node) error {
+func (e *csvEncoder) encodeObjects(csvWriter *csv.Writer, content []*CandidateNode) error {
 	headers, err := e.extractHeader(content[0])
 	if err != nil {
 		return nil
@@ -89,7 +87,7 @@ func (e *csvEncoder) encodeObjects(csvWriter *csv.Writer, content []*yaml.Node)
 	}
 
 	for i, child := range content {
-		if child.Kind != yaml.MappingNode {
+		if child.Kind != MappingNode {
 			return fmt.Errorf("csv object encoding only works for arrays of flat objects (string key => string/numbers/boolean value), child[%v] is a %v", i, child.Tag)
 		}
 		row := e.createChildRow(child, headers)
@@ -102,26 +100,25 @@ func (e *csvEncoder) encodeObjects(csvWriter *csv.Writer, content []*yaml.Node)
 	return nil
 }
 
-func (e *csvEncoder) Encode(writer io.Writer, originalNode *yaml.Node) error {
-	if originalNode.Kind == yaml.ScalarNode {
-		return writeString(writer, originalNode.Value+"\n")
+func (e *csvEncoder) Encode(writer io.Writer, node *CandidateNode) error {
+	if node.Kind == ScalarNode {
+		return WriteString(writer, node.Value+"\n")
 	}
 
 	csvWriter := csv.NewWriter(writer)
 	csvWriter.Comma = e.separator
 
 	// node must be a sequence
-	node := unwrapDoc(originalNode)
-	if node.Kind != yaml.SequenceNode {
+	if node.Kind != SequenceNode {
 		return fmt.Errorf("csv encoding only works for arrays, got: %v", node.Tag)
 	} else if len(node.Content) == 0 {
 		return nil
 	}
-	if node.Content[0].Kind == yaml.ScalarNode {
+	if node.Content[0].Kind == ScalarNode {
 		return e.encodeRow(csvWriter, node.Content)
 	}
 
-	if node.Content[0].Kind == yaml.MappingNode {
+	if node.Content[0].Kind == MappingNode {
 		return e.encodeObjects(csvWriter, node.Content)
 	}
 

pkg/yqlib/encoder_json.go

@@ -7,46 +7,46 @@ import (
 	"io"
 
 	"github.com/goccy/go-json"
-	yaml "gopkg.in/yaml.v3"
 )
 
 type jsonEncoder struct {
+	prefs        JsonPreferences
 	indentString string
-	colorise     bool
-	UnwrapScalar bool
 }
 
-func NewJSONEncoder(indent int, colorise bool, unwrapScalar bool) Encoder {
+func NewJSONEncoder(prefs JsonPreferences) Encoder {
 	var indentString = ""
 
-	for index := 0; index < indent; index++ {
+	for index := 0; index < prefs.Indent; index++ {
 		indentString = indentString + " "
 	}
 
-	return &jsonEncoder{indentString, colorise, unwrapScalar}
+	return &jsonEncoder{prefs, indentString}
 }
 
 func (je *jsonEncoder) CanHandleAliases() bool {
 	return false
 }
 
-func (je *jsonEncoder) PrintDocumentSeparator(writer io.Writer) error {
+func (je *jsonEncoder) PrintDocumentSeparator(_ io.Writer) error {
 	return nil
 }
 
-func (je *jsonEncoder) PrintLeadingContent(writer io.Writer, content string) error {
+func (je *jsonEncoder) PrintLeadingContent(_ io.Writer, _ string) error {
 	return nil
 }
 
-func (je *jsonEncoder) Encode(writer io.Writer, node *yaml.Node) error {
+func (je *jsonEncoder) Encode(writer io.Writer, node *CandidateNode) error {
+	log.Debugf("I need to encode %v", NodeToString(node))
+	log.Debugf("kids %v", len(node.Content))
 
-	if node.Kind == yaml.ScalarNode && je.UnwrapScalar {
-		return writeString(writer, node.Value+"\n")
+	if node.Kind == ScalarNode && je.prefs.UnwrapScalar {
+		return WriteString(writer, node.Value+"\n")
 	}
 
 	destination := writer
 	tempBuffer := bytes.NewBuffer(nil)
-	if je.colorise {
+	if je.prefs.ColorsEnabled {
 		destination = tempBuffer
 	}
 
@@ -54,18 +54,11 @@ func (je *jsonEncoder) Encode(writer io.Writer, node *yaml.Node) error {
 	encoder.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
 	encoder.SetIndent("", je.indentString)
 
-	var dataBucket orderedMap
-	// firstly, convert all map keys to strings
-	mapKeysToStrings(node)
-	errorDecoding := node.Decode(&dataBucket)
-	if errorDecoding != nil {
-		return errorDecoding
-	}
-	err := encoder.Encode(dataBucket)
+	err := encoder.Encode(node)
 	if err != nil {
 		return err
 	}
-	if je.colorise {
+	if je.prefs.ColorsEnabled {
 		return colorizeAndPrint(tempBuffer.Bytes(), writer)
 	}
 	return nil

pkg/yqlib/encoder_lua.go

@@ -1,11 +1,11 @@
+//go:build !yq_nolua
+
 package yqlib
 
 import (
 	"fmt"
 	"io"
 	"strings"
-
-	yaml "gopkg.in/yaml.v3"
 )
 
 type luaEncoder struct {
@@ -72,52 +72,52 @@ func NewLuaEncoder(prefs LuaPreferences) Encoder {
 	return &luaEncoder{unescape.Replace(prefs.DocPrefix), unescape.Replace(prefs.DocSuffix), 0, "\t", prefs.UnquotedKeys, prefs.Globals, escape}
 }
 
-func (le *luaEncoder) PrintDocumentSeparator(writer io.Writer) error {
+func (le *luaEncoder) PrintDocumentSeparator(_ io.Writer) error {
 	return nil
 }
 
-func (le *luaEncoder) PrintLeadingContent(writer io.Writer, content string) error {
+func (le *luaEncoder) PrintLeadingContent(_ io.Writer, _ string) error {
 	return nil
 }
 
-func (le *luaEncoder) encodeString(writer io.Writer, node *yaml.Node) error {
+func (le *luaEncoder) encodeString(writer io.Writer, node *CandidateNode) error {
 	quote := "\""
 	switch node.Style {
-	case yaml.LiteralStyle, yaml.FoldedStyle, yaml.FlowStyle:
+	case LiteralStyle, FoldedStyle, FlowStyle:
 		for i := 0; i < 10; i++ {
 			if !strings.Contains(node.Value, "]"+strings.Repeat("=", i)+"]") {
-				err := writeString(writer, "["+strings.Repeat("=", i)+"[\n")
+				err := WriteString(writer, "["+strings.Repeat("=", i)+"[\n")
 				if err != nil {
 					return err
 				}
-				err = writeString(writer, node.Value)
+				err = WriteString(writer, node.Value)
 				if err != nil {
 					return err
 				}
-				return writeString(writer, "]"+strings.Repeat("=", i)+"]")
+				return WriteString(writer, "]"+strings.Repeat("=", i)+"]")
 			}
 		}
-	case yaml.SingleQuotedStyle:
+	case SingleQuotedStyle:
 		quote = "'"
 
 		// fallthrough to regular ol' string
 	}
-	return writeString(writer, quote+le.escape.Replace(node.Value)+quote)
+	return WriteString(writer, quote+le.escape.Replace(node.Value)+quote)
 }
 
 func (le *luaEncoder) writeIndent(writer io.Writer) error {
 	if le.indentStr == "" {
 		return nil
 	}
-	err := writeString(writer, "\n")
+	err := WriteString(writer, "\n")
 	if err != nil {
 		return err
 	}
-	return writeString(writer, strings.Repeat(le.indentStr, le.indent))
+	return WriteString(writer, strings.Repeat(le.indentStr, le.indent))
 }
 
-func (le *luaEncoder) encodeArray(writer io.Writer, node *yaml.Node) error {
-	err := writeString(writer, "{")
+func (le *luaEncoder) encodeArray(writer io.Writer, node *CandidateNode) error {
+	err := WriteString(writer, "{")
 	if err != nil {
 		return err
 	}
@@ -127,17 +127,17 @@ func (le *luaEncoder) encodeArray(writer io.Writer, node *yaml.Node) error {
 		if err != nil {
 			return err
 		}
-		err := le.Encode(writer, child)
+		err := le.encodeAny(writer, child)
 		if err != nil {
 			return err
 		}
-		err = writeString(writer, ",")
+		err = WriteString(writer, ",")
 		if err != nil {
 			return err
 		}
 		if child.LineComment != "" {
 			sansPrefix, _ := strings.CutPrefix(child.LineComment, "#")
-			err = writeString(writer, " --"+sansPrefix)
+			err = WriteString(writer, " --"+sansPrefix)
 			if err != nil {
 				return err
 			}
@@ -150,7 +150,7 @@ func (le *luaEncoder) encodeArray(writer io.Writer, node *yaml.Node) error {
 			return err
 		}
 	}
-	return writeString(writer, "}")
+	return WriteString(writer, "}")
 }
 
 func needsQuoting(s string) bool {
@@ -179,9 +179,9 @@ func needsQuoting(s string) bool {
 	return false
 }
 
-func (le *luaEncoder) encodeMap(writer io.Writer, node *yaml.Node, global bool) error {
+func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bool) error {
 	if !global {
-		err := writeString(writer, "{")
+		err := WriteString(writer, "{")
 		if err != nil {
 			return err
 		}
@@ -190,11 +190,11 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *yaml.Node, global bool)
 	for i, child := range node.Content {
 		if (i % 2) == 1 {
 			// value
-			err := le.Encode(writer, child)
+			err := le.encodeAny(writer, child)
 			if err != nil {
 				return err
 			}
-			err = writeString(writer, ";")
+			err = WriteString(writer, ";")
 			if err != nil {
 				return err
 			}
@@ -207,19 +207,19 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *yaml.Node, global bool)
 				}
 			}
 			if (le.unquoted || global) && child.Tag == "!!str" && !needsQuoting(child.Value) {
-				err := writeString(writer, child.Value+" = ")
+				err := WriteString(writer, child.Value+" = ")
 				if err != nil {
 					return err
 				}
 			} else {
 				if global {
 					// This only works in Lua 5.2+
-					err := writeString(writer, "_ENV")
+					err := WriteString(writer, "_ENV")
 					if err != nil {
 						return err
 					}
 				}
-				err := writeString(writer, "[")
+				err := WriteString(writer, "[")
 				if err != nil {
 					return err
 				}
@@ -227,7 +227,7 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *yaml.Node, global bool)
 				if err != nil {
 					return err
 				}
-				err = writeString(writer, "] = ")
+				err = WriteString(writer, "] = ")
 				if err != nil {
 					return err
 				}
@@ -235,7 +235,7 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *yaml.Node, global bool)
 		}
 		if child.LineComment != "" {
 			sansPrefix, _ := strings.CutPrefix(child.LineComment, "#")
-			err := writeString(writer, strings.Repeat(" ", i%2)+"--"+sansPrefix)
+			err := WriteString(writer, strings.Repeat(" ", i%2)+"--"+sansPrefix)
 			if err != nil {
 				return err
 			}
@@ -249,7 +249,7 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *yaml.Node, global bool)
 		}
 	}
 	if global {
-		return writeString(writer, "\n")
+		return WriteString(writer, "\n")
 	}
 	le.indent--
 	if len(node.Content) != 0 {
@@ -258,71 +258,73 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *yaml.Node, global bool)
 			return err
 		}
 	}
-	return writeString(writer, "}")
+	return WriteString(writer, "}")
 }
 
-func (le *luaEncoder) encodeAny(writer io.Writer, node *yaml.Node) error {
+func (le *luaEncoder) encodeAny(writer io.Writer, node *CandidateNode) error {
 	switch node.Kind {
-	case yaml.SequenceNode:
+	case SequenceNode:
 		return le.encodeArray(writer, node)
-	case yaml.MappingNode:
+	case MappingNode:
 		return le.encodeMap(writer, node, false)
-	case yaml.ScalarNode:
+	case ScalarNode:
 		switch node.Tag {
 		case "!!str":
 			return le.encodeString(writer, node)
 		case "!!null":
 			// TODO reject invalid use as a table key
-			return writeString(writer, "nil")
+			return WriteString(writer, "nil")
 		case "!!bool":
 			// Yaml 1.2 has case variation e.g. True, FALSE etc but Lua only has
 			// lower case
-			return writeString(writer, strings.ToLower(node.Value))
+			return WriteString(writer, strings.ToLower(node.Value))
 		case "!!int":
 			if strings.HasPrefix(node.Value, "0o") {
-				var octalValue int
-				err := node.Decode(&octalValue)
+				_, octalValue, err := parseInt64(node.Value)
 				if err != nil {
 					return err
 				}
-				return writeString(writer, fmt.Sprintf("%d", octalValue))
+				return WriteString(writer, fmt.Sprintf("%d", octalValue))
 			}
-			return writeString(writer, strings.ToLower(node.Value))
+			return WriteString(writer, strings.ToLower(node.Value))
 		case "!!float":
 			switch strings.ToLower(node.Value) {
-			case ".inf":
-				return writeString(writer, "(1/0)")
+			case ".inf", "+.inf":
+				return WriteString(writer, "(1/0)")
 			case "-.inf":
-				return writeString(writer, "(-1/0)")
+				return WriteString(writer, "(-1/0)")
 			case ".nan":
-				return writeString(writer, "(0/0)")
+				return WriteString(writer, "(0/0)")
 			default:
-				return writeString(writer, node.Value)
+				return WriteString(writer, node.Value)
 			}
 		default:
-			return fmt.Errorf("Lua encoder NYI -- %s", node.ShortTag())
+			return fmt.Errorf("Lua encoder NYI -- %s", node.Tag)
 		}
-	case yaml.DocumentNode:
-		if le.globals {
-			if node.Content[0].Kind != yaml.MappingNode {
-				return fmt.Errorf("--lua-global requires a top level MappingNode")
-			}
-			return le.encodeMap(writer, node.Content[0], true)
-		}
-		err := writeString(writer, le.docPrefix)
-		if err != nil {
-			return err
-		}
-		err = le.encodeAny(writer, node.Content[0])
-		if err != nil {
-			return err
-		}
-		return writeString(writer, le.docSuffix)
 	default:
-		return fmt.Errorf("Lua encoder NYI -- %s", node.ShortTag())
+		return fmt.Errorf("Lua encoder NYI -- %s", node.Tag)
 	}
 }
 
-func (le *luaEncoder) Encode(writer io.Writer, node *yaml.Node) error {
-	return le.encodeAny(writer, node)
+func (le *luaEncoder) encodeTopLevel(writer io.Writer, node *CandidateNode) error {
+	err := WriteString(writer, le.docPrefix)
+	if err != nil {
+		return err
+	}
+	err = le.encodeAny(writer, node)
+	if err != nil {
+		return err
+	}
+	return WriteString(writer, le.docSuffix)
+}
+
+func (le *luaEncoder) Encode(writer io.Writer, node *CandidateNode) error {
+
+	if le.globals {
+		if node.Kind != MappingNode {
+			return fmt.Errorf("--lua-global requires a top level MappingNode")
+		}
+		return le.encodeMap(writer, node, true)
+	}
+	return le.encodeTopLevel(writer, node)
 }

pkg/yqlib/encoder_properties_test.go

@@ -1,127 +0,0 @@
-package yqlib
-
-import (
-	"bufio"
-	"bytes"
-	"strings"
-	"testing"
-
-	"github.com/mikefarah/yq/v4/test"
-)
-
-func yamlToProps(sampleYaml string, unwrapScalar bool) string {
-	var output bytes.Buffer
-	writer := bufio.NewWriter(&output)
-
-	var propsEncoder = NewPropertiesEncoder(unwrapScalar)
-	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
-	if err != nil {
-		panic(err)
-	}
-	node := inputs.Front().Value.(*CandidateNode).Node
-	err = propsEncoder.Encode(writer, node)
-	if err != nil {
-		panic(err)
-	}
-	writer.Flush()
-
-	return strings.TrimSuffix(output.String(), "\n")
-}
-
-func TestPropertiesEncoderSimple_Unwrapped(t *testing.T) {
-	var sampleYaml = `a: 'bob cool'`
-
-	var expectedProps = `a = bob cool`
-	var actualProps = yamlToProps(sampleYaml, true)
-	test.AssertResult(t, expectedProps, actualProps)
-}
-
-func TestPropertiesEncoderSimple_Wrapped(t *testing.T) {
-	var sampleYaml = `a: 'bob cool'`
-
-	var expectedProps = `a = "bob cool"`
-	var actualProps = yamlToProps(sampleYaml, false)
-	test.AssertResult(t, expectedProps, actualProps)
-}
-
-func TestPropertiesEncoderSimpleWithComments_Unwrapped(t *testing.T) {
-	var sampleYaml = `a: 'bob cool' # line`
-
-	var expectedProps = `# line
-a = bob cool`
-	var actualProps = yamlToProps(sampleYaml, true)
-	test.AssertResult(t, expectedProps, actualProps)
-}
-
-func TestPropertiesEncoderSimpleWithComments_Wrapped(t *testing.T) {
-	var sampleYaml = `a: 'bob cool' # line`
-
-	var expectedProps = `# line
-a = "bob cool"`
-	var actualProps = yamlToProps(sampleYaml, false)
-	test.AssertResult(t, expectedProps, actualProps)
-}
-
-func TestPropertiesEncoderDeep_Unwrapped(t *testing.T) {
-	var sampleYaml = `a: 
-  b: "bob cool"
-`
-
-	var expectedProps = `a.b = bob cool`
-	var actualProps = yamlToProps(sampleYaml, true)
-	test.AssertResult(t, expectedProps, actualProps)
-}
-
-func TestPropertiesEncoderDeep_Wrapped(t *testing.T) {
-	var sampleYaml = `a: 
-  b: "bob cool"
-`
-
-	var expectedProps = `a.b = "bob cool"`
-	var actualProps = yamlToProps(sampleYaml, false)
-	test.AssertResult(t, expectedProps, actualProps)
-}
-
-func TestPropertiesEncoderDeepWithComments_Unwrapped(t *testing.T) {
-	var sampleYaml = `a:  # a thing
-  b: "bob cool" # b thing
-`
-
-	var expectedProps = `# b thing
-a.b = bob cool`
-	var actualProps = yamlToProps(sampleYaml, true)
-	test.AssertResult(t, expectedProps, actualProps)
-}
-
-func TestPropertiesEncoderDeepWithComments_Wrapped(t *testing.T) {
-	var sampleYaml = `a:  # a thing
-  b: "bob cool" # b thing
-`
-
-	var expectedProps = `# b thing
-a.b = "bob cool"`
-	var actualProps = yamlToProps(sampleYaml, false)
-	test.AssertResult(t, expectedProps, actualProps)
-}
-
-func TestPropertiesEncoderArray_Unwrapped(t *testing.T) {
-	var sampleYaml = `a: 
-  b: [{c: dog}, {c: cat}]
-`
-
-	var expectedProps = `a.b.0.c = dog
-a.b.1.c = cat`
-	var actualProps = yamlToProps(sampleYaml, true)
-	test.AssertResult(t, expectedProps, actualProps)
-}
-
-func TestPropertiesEncoderArray_Wrapped(t *testing.T) {
-	var sampleYaml = `a: 
-  b: [{c: dog named jim}, {c: cat named jam}]
-`
-
-	var expectedProps = `a.b.0.c = "dog named jim"
-a.b.1.c = "cat named jam"`
-	var actualProps = yamlToProps(sampleYaml, false)
-	test.AssertResult(t, expectedProps, actualProps)
-}

pkg/yqlib/encoder_sh.go

@@ -5,8 +5,6 @@ import (
 	"io"
 	"regexp"
 	"strings"
-
-	yaml "gopkg.in/yaml.v3"
 )
 
 var unsafeChars = regexp.MustCompile(`[^\w@%+=:,./-]`)
@@ -23,21 +21,20 @@ func (e *shEncoder) CanHandleAliases() bool {
 	return false
 }
 
-func (e *shEncoder) PrintDocumentSeparator(writer io.Writer) error {
+func (e *shEncoder) PrintDocumentSeparator(_ io.Writer) error {
 	return nil
 }
 
-func (e *shEncoder) PrintLeadingContent(writer io.Writer, content string) error {
+func (e *shEncoder) PrintLeadingContent(_ io.Writer, _ string) error {
 	return nil
 }
 
-func (e *shEncoder) Encode(writer io.Writer, originalNode *yaml.Node) error {
-	node := unwrapDoc(originalNode)
-	if guessTagFromCustomType(node) != "!!str" {
+func (e *shEncoder) Encode(writer io.Writer, node *CandidateNode) error {
+	if node.GuessTagFromCustomType() != "!!str" {
 		return fmt.Errorf("cannot encode %v as URI, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string", node.Tag)
 	}
 
-	return writeString(writer, e.encode(originalNode.Value))
+	return WriteString(writer, e.encode(node.Value))
 }
 
 // put any (shell-unsafe) characters into a single-quoted block, close the block lazily

pkg/yqlib/encoder_shellvariables.go

@@ -7,7 +7,6 @@ import (
 	"unicode/utf8"
 
 	"golang.org/x/text/unicode/norm"
-	yaml "gopkg.in/yaml.v3"
 )
 
 type shellVariablesEncoder struct {
@@ -29,9 +28,9 @@ func (pe *shellVariablesEncoder) PrintLeadingContent(_ io.Writer, _ string) erro
 	return nil
 }
 
-func (pe *shellVariablesEncoder) Encode(writer io.Writer, node *yaml.Node) error {
+func (pe *shellVariablesEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 
-	mapKeysToStrings(node)
+	node.ConvertKeysToStrings()
 	err := pe.doEncode(&writer, node, "")
 	if err != nil {
 		return err
@@ -40,12 +39,12 @@ func (pe *shellVariablesEncoder) Encode(writer io.Writer, node *yaml.Node) error
 	return err
 }
 
-func (pe *shellVariablesEncoder) doEncode(w *io.Writer, node *yaml.Node, path string) error {
+func (pe *shellVariablesEncoder) doEncode(w *io.Writer, node *CandidateNode, path string) error {
 
 	// Note this drops all comments.
 
 	switch node.Kind {
-	case yaml.ScalarNode:
+	case ScalarNode:
 		nonemptyPath := path
 		if path == "" {
 			// We can't assign an empty variable "=somevalue" because that would error out if sourced in a shell,
@@ -55,9 +54,7 @@ func (pe *shellVariablesEncoder) doEncode(w *io.Writer, node *yaml.Node, path st
 		}
 		_, err := io.WriteString(*w, nonemptyPath+"="+quoteValue(node.Value)+"\n")
 		return err
-	case yaml.DocumentNode:
-		return pe.doEncode(w, node.Content[0], path)
-	case yaml.SequenceNode:
+	case SequenceNode:
 		for index, child := range node.Content {
 			err := pe.doEncode(w, child, appendPath(path, index))
 			if err != nil {
@@ -65,7 +62,7 @@ func (pe *shellVariablesEncoder) doEncode(w *io.Writer, node *yaml.Node, path st
 			}
 		}
 		return nil
-	case yaml.MappingNode:
+	case MappingNode:
 		for index := 0; index < len(node.Content); index = index + 2 {
 			key := node.Content[index]
 			value := node.Content[index+1]
@@ -75,7 +72,7 @@ func (pe *shellVariablesEncoder) doEncode(w *io.Writer, node *yaml.Node, path st
 			}
 		}
 		return nil
-	case yaml.AliasNode:
+	case AliasNode:
 		return pe.doEncode(w, node.Alias, path)
 	default:
 		return fmt.Errorf("Unsupported node %v", node.Tag)

pkg/yqlib/encoder_shellvariables_test.go

@@ -18,7 +18,7 @@ func assertEncodesTo(t *testing.T, yaml string, shellvars string) {
 	if err != nil {
 		panic(err)
 	}
-	node := inputs.Front().Value.(*CandidateNode).Node
+	node := inputs.Front().Value.(*CandidateNode)
 	err = encoder.Encode(writer, node)
 	if err != nil {
 		panic(err)

pkg/yqlib/encoder_test.go

@@ -11,16 +11,23 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
-func yamlToJSON(sampleYaml string, indent int) string {
+func yamlToJSON(t *testing.T, sampleYaml string, indent int) string {
+	t.Helper()
 	var output bytes.Buffer
 	writer := bufio.NewWriter(&output)
 
-	var jsonEncoder = NewJSONEncoder(indent, false, false)
+	prefs := ConfiguredJSONPreferences.Copy()
+	prefs.Indent = indent
+	prefs.UnwrapScalar = false
+	var jsonEncoder = NewJSONEncoder(prefs)
 	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {
 		panic(err)
 	}
-	node := inputs.Front().Value.(*CandidateNode).Node
+	node := inputs.Front().Value.(*CandidateNode)
+	log.Debugf("%v", NodeToString(node))
+	// log.Debugf("Content[0] %v", NodeToString(node.Content[0]))
+
 	err = jsonEncoder.Encode(writer, node)
 	if err != nil {
 		panic(err)
@@ -46,31 +53,31 @@ banana:
     }
   ]
 }`
-	var actualJSON = yamlToJSON(sampleYaml, 2)
+	var actualJSON = yamlToJSON(t, sampleYaml, 2)
 	test.AssertResult(t, expectedJSON, actualJSON)
 }
 
 func TestJsonNullInArray(t *testing.T) {
 	var sampleYaml = `[null]`
-	var actualJSON = yamlToJSON(sampleYaml, 0)
+	var actualJSON = yamlToJSON(t, sampleYaml, 0)
 	test.AssertResult(t, sampleYaml, actualJSON)
 }
 
 func TestJsonNull(t *testing.T) {
 	var sampleYaml = `null`
-	var actualJSON = yamlToJSON(sampleYaml, 0)
+	var actualJSON = yamlToJSON(t, sampleYaml, 0)
 	test.AssertResult(t, sampleYaml, actualJSON)
 }
 
 func TestJsonNullInObject(t *testing.T) {
 	var sampleYaml = `{x: null}`
-	var actualJSON = yamlToJSON(sampleYaml, 0)
+	var actualJSON = yamlToJSON(t, sampleYaml, 0)
 	test.AssertResult(t, `{"x":null}`, actualJSON)
 }
 
 func TestJsonEncoderDoesNotEscapeHTMLChars(t *testing.T) {
 	var sampleYaml = `build: "( ./lint && ./format && ./compile ) < src.code"`
 	var expectedJSON = `{"build":"( ./lint && ./format && ./compile ) < src.code"}`
-	var actualJSON = yamlToJSON(sampleYaml, 0)
+	var actualJSON = yamlToJSON(t, sampleYaml, 0)
 	test.AssertResult(t, expectedJSON, actualJSON)
 }

pkg/yqlib/encoder_toml.go

@@ -3,8 +3,6 @@ package yqlib
 import (
 	"fmt"
 	"io"
-
-	yaml "gopkg.in/yaml.v3"
 )
 
 type tomlEncoder struct {
@@ -14,18 +12,18 @@ func NewTomlEncoder() Encoder {
 	return &tomlEncoder{}
 }
 
-func (te *tomlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
-	if node.Kind == yaml.ScalarNode {
-		return writeString(writer, node.Value+"\n")
+func (te *tomlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
+	if node.Kind == ScalarNode {
+		return WriteString(writer, node.Value+"\n")
 	}
 	return fmt.Errorf("only scalars (e.g. strings, numbers, booleans) are supported for TOML output at the moment. Please use yaml output format (-oy) until the encoder has been fully implemented")
 }
 
-func (te *tomlEncoder) PrintDocumentSeparator(writer io.Writer) error {
+func (te *tomlEncoder) PrintDocumentSeparator(_ io.Writer) error {
 	return nil
 }
 
-func (te *tomlEncoder) PrintLeadingContent(writer io.Writer, content string) error {
+func (te *tomlEncoder) PrintLeadingContent(_ io.Writer, _ string) error {
 	return nil
 }
 

pkg/yqlib/encoder_uri.go

@@ -4,8 +4,6 @@ import (
 	"fmt"
 	"io"
 	"net/url"
-
-	yaml "gopkg.in/yaml.v3"
 )
 
 type uriEncoder struct {
@@ -19,19 +17,18 @@ func (e *uriEncoder) CanHandleAliases() bool {
 	return false
 }
 
-func (e *uriEncoder) PrintDocumentSeparator(writer io.Writer) error {
+func (e *uriEncoder) PrintDocumentSeparator(_ io.Writer) error {
 	return nil
 }
 
-func (e *uriEncoder) PrintLeadingContent(writer io.Writer, content string) error {
+func (e *uriEncoder) PrintLeadingContent(_ io.Writer, _ string) error {
 	return nil
 }
 
-func (e *uriEncoder) Encode(writer io.Writer, originalNode *yaml.Node) error {
-	node := unwrapDoc(originalNode)
-	if guessTagFromCustomType(node) != "!!str" {
+func (e *uriEncoder) Encode(writer io.Writer, node *CandidateNode) error {
+	if node.GuessTagFromCustomType() != "!!str" {
 		return fmt.Errorf("cannot encode %v as URI, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string", node.Tag)
 	}
-	_, err := writer.Write([]byte(url.QueryEscape(originalNode.Value)))
+	_, err := writer.Write([]byte(url.QueryEscape(node.Value)))
 	return err
 }