wip

Bumps golang from 1.22.0 to 1.22.1.

---
updated-dependencies:
- dependency-name: golang
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.22.0 as builder
+FROM golang:1.22.1 as builder
 
 WORKDIR /go/src/mikefarah/yq
 

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM golang:1.22.0
+FROM golang:1.22.1
 
 COPY scripts/devtools.sh /opt/devtools.sh
 

acceptance_tests/flags.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+setUp() {
+  rm test*.yml || true
+  cat >test.yml <<EOL
+# comment
+EOL
+}
+
+testStringInterpolation() {
+    X=$(./yq -n '"Mike \(3 + 4)"')
+    assertEquals "Mike 7" "$X"
+}
+
+testNoStringInterpolation() {
+    X=$(./yq --string-interpolation=f -n '"Mike \(3 + 4)"')
+    assertEquals "Mike \(3 + 4)" "$X"
+}
+
+
+source ./scripts/shunit2

acceptance_tests/inputs-format.sh

@@ -63,6 +63,27 @@ EOM
   assertEquals "$expected" "$X"
 }
 
+testInputCSVCustomSeparator() {
+  cat >test.csv <<EOL
+fruit;yumLevel
+apple;5
+banana;4
+EOL
+
+  read -r -d '' expected << EOM
+- fruit: apple
+  yumLevel: 5
+- fruit: banana
+  yumLevel: 4
+EOM
+
+  X=$(./yq -p=csv --csv-separator ";" test.csv)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -p=csv --csv-separator ";" test.csv)
+  assertEquals "$expected" "$X"
+}
+
 testInputCSVNoAuto() {
   cat >test.csv <<EOL
 thing1

acceptance_tests/output-format.sh

@@ -198,6 +198,27 @@ EOM
   assertEquals "$expected" "$X"
 }
 
+testOutputCSVCustomSeparator() {
+  cat >test.yml <<EOL
+- fruit: apple
+  yumLevel: 5
+- fruit: banana
+  yumLevel: 4
+EOL
+
+  read -r -d '' expected << EOM
+fruit;yumLevel
+apple;5
+banana;4
+EOM
+
+  X=$(./yq -oc --csv-separator ";" test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -o=csv --csv-separator ";" test.yml)
+  assertEquals "$expected" "$X"
+}
+
 testOutputTSV() {
   cat >test.yml <<EOL
 - fruit: apple

acceptance_tests/shebang.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+setUp() {
+  rm test*.yq || true
+  cat >test.yq <<EOL
+#!./yq
+.a.b
+EOL
+chmod +x test.yq
+
+rm test*.yml || true
+  cat >test.yml <<EOL
+a: {b: apple}
+EOL
+}
+
+testCanExecYqFile() {
+  read -r -d '' expected << EOM
+apple
+EOM
+   X=$(./test.yq test.yml)
+  assertEquals "$expected" "$X"
+}
+
+source ./scripts/shunit2
+

cmd/evaluate_all_command.go

@@ -12,6 +12,12 @@ func createEvaluateAllCommand() *cobra.Command {
 		Use:     "eval-all [expression] [yaml_file1]...",
 		Aliases: []string{"ea"},
 		Short:   "Loads _all_ yaml documents of _all_ yaml files and runs expression once",
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			if len(args) == 0 {
+				return nil, cobra.ShellCompDirectiveNoFileComp
+			}
+			return nil, cobra.ShellCompDirectiveDefault
+		},
 		Example: `
 # Merge f2.yml into f1.yml (in place)
 yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' f1.yml f2.yml
@@ -70,7 +76,7 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 		}()
 	}
 
-	format, err := yqlib.OutputFormatFromString(outputFormat)
+	format, err := yqlib.FormatFromString(outputFormat)
 	if err != nil {
 		return err
 	}

cmd/evaluate_sequence_command.go

@@ -13,6 +13,12 @@ func createEvaluateSequenceCommand() *cobra.Command {
 		Use:     "eval [expression] [yaml_file1]...",
 		Aliases: []string{"e"},
 		Short:   "(default) Apply the expression to each document in each yaml file in sequence",
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			if len(args) == 0 {
+				return nil, cobra.ShellCompDirectiveNoFileComp
+			}
+			return nil, cobra.ShellCompDirectiveDefault
+		},
 		Example: `
 # Reads field under the given path for each file
 yq e '.a.b' f1.yml f2.yml 
@@ -84,7 +90,7 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 		}()
 	}
 
-	format, err := yqlib.OutputFormatFromString(outputFormat)
+	format, err := yqlib.FormatFromString(outputFormat)
 	if err != nil {
 		return err
 	}

cmd/root.go

@@ -1,13 +1,44 @@
 package cmd
 
 import (
+	"fmt"
 	"os"
+	"strings"
 
+	"github.com/mikefarah/yq/v4/pkg/properties"
+	"github.com/mikefarah/yq/v4/pkg/xml"
 	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"github.com/spf13/cobra"
 	logging "gopkg.in/op/go-logging.v1"
 )
 
+type runeValue rune
+
+func newRuneVar(p *rune) *runeValue {
+	return (*runeValue)(p)
+}
+
+func (r *runeValue) String() string {
+	return string(*r)
+}
+
+func (r *runeValue) Set(rawVal string) error {
+	val := strings.ReplaceAll(rawVal, "\\n", "\n")
+	val = strings.ReplaceAll(val, "\\t", "\t")
+	val = strings.ReplaceAll(val, "\\r", "\r")
+	val = strings.ReplaceAll(val, "\\f", "\f")
+	val = strings.ReplaceAll(val, "\\v", "\v")
+	if len(val) != 1 {
+		return fmt.Errorf("[%v] is not a valid character. Must be length 1 was %v", val, len(val))
+	}
+	*r = runeValue(rune(val[0]))
+	return nil
+}
+
+func (r *runeValue) Type() string {
+	return "char"
+}
+
 func New() *cobra.Command {
 	var rootCmd = &cobra.Command{
 		Use:   "yq",
@@ -68,31 +99,76 @@ yq -P -oy sample.json
 		panic(err)
 	}
 
-	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "auto", "[auto|a|yaml|y|json|j|props|p|xml|x|tsv|t|csv|c] output format type.")
-	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "auto", "[auto|a|yaml|y|props|p|xml|x|tsv|t|csv|c|toml] parse format for input. Note that json is a subset of yaml.")
+	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "auto", fmt.Sprintf("[auto|a|%v] output format type.", yqlib.GetAvailableOutputFormatString()))
+	var outputCompletions = []string{"auto"}
+	for _, formats := range yqlib.GetAvailableOutputFormats() {
+		outputCompletions = append(outputCompletions, formats.FormalName)
+	}
 
-	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", yqlib.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes")
-	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", yqlib.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.StrictMode, "xml-strict-mode", yqlib.ConfiguredXMLPreferences.StrictMode, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.KeepNamespace, "xml-keep-namespace", yqlib.ConfiguredXMLPreferences.KeepNamespace, "enables keeping namespace after parsing attributes")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.UseRawToken, "xml-raw-token", yqlib.ConfiguredXMLPreferences.UseRawToken, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.")
-	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ProcInstPrefix, "xml-proc-inst-prefix", yqlib.ConfiguredXMLPreferences.ProcInstPrefix, "prefix for xml processing instructions (e.g. <?xml version=\"1\"?>)")
-	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.DirectiveName, "xml-directive-name", yqlib.ConfiguredXMLPreferences.DirectiveName, "name for xml directives (e.g. <!DOCTYPE thing cat>)")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipProcInst, "xml-skip-proc-inst", yqlib.ConfiguredXMLPreferences.SkipProcInst, "skip over process instructions (e.g. <?xml version=\"1\"?>)")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipDirectives, "xml-skip-directives", yqlib.ConfiguredXMLPreferences.SkipDirectives, "skip over directives (e.g. <!DOCTYPE thing cat>)")
+	if err = rootCmd.RegisterFlagCompletionFunc("output-format", cobra.FixedCompletions(outputCompletions, cobra.ShellCompDirectiveNoFileComp)); err != nil {
+		panic(err)
+	}
+	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "auto", fmt.Sprintf("[auto|a|%v] parse format for input.", yqlib.GetAvailableInputFormatString()))
+
+	var inputCompletions = []string{"auto"}
+	for _, formats := range yqlib.GetAvailableInputFormats() {
+		inputCompletions = append(inputCompletions, formats.FormalName)
+	}
+
+	if err = rootCmd.RegisterFlagCompletionFunc("input-format", cobra.FixedCompletions(inputCompletions, cobra.ShellCompDirectiveNoFileComp)); err != nil {
+		panic(err)
+	}
+
+	rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", xml.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes")
+	if err = rootCmd.RegisterFlagCompletionFunc("xml-attribute-prefix", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
+	rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.ContentName, "xml-content-name", xml.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).")
+	if err = rootCmd.RegisterFlagCompletionFunc("xml-content-name", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
+	rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.StrictMode, "xml-strict-mode", xml.ConfiguredXMLPreferences.StrictMode, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.")
+	rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.KeepNamespace, "xml-keep-namespace", xml.ConfiguredXMLPreferences.KeepNamespace, "enables keeping namespace after parsing attributes")
+	rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.UseRawToken, "xml-raw-token", xml.ConfiguredXMLPreferences.UseRawToken, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.")
+	rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.ProcInstPrefix, "xml-proc-inst-prefix", xml.ConfiguredXMLPreferences.ProcInstPrefix, "prefix for xml processing instructions (e.g. <?xml version=\"1\"?>)")
+	if err = rootCmd.RegisterFlagCompletionFunc("xml-proc-inst-prefix", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
+	rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.DirectiveName, "xml-directive-name", xml.ConfiguredXMLPreferences.DirectiveName, "name for xml directives (e.g. <!DOCTYPE thing cat>)")
+	if err = rootCmd.RegisterFlagCompletionFunc("xml-directive-name", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
+	rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.SkipProcInst, "xml-skip-proc-inst", xml.ConfiguredXMLPreferences.SkipProcInst, "skip over process instructions (e.g. <?xml version=\"1\"?>)")
+	rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.SkipDirectives, "xml-skip-directives", xml.ConfiguredXMLPreferences.SkipDirectives, "skip over directives (e.g. <!DOCTYPE thing cat>)")
 
 	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredCsvPreferences.AutoParse, "csv-auto-parse", yqlib.ConfiguredCsvPreferences.AutoParse, "parse CSV YAML/JSON values")
+	rootCmd.PersistentFlags().Var(newRuneVar(&yqlib.ConfiguredCsvPreferences.Separator), "csv-separator", "CSV Separator character")
+
 	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredTsvPreferences.AutoParse, "tsv-auto-parse", yqlib.ConfiguredTsvPreferences.AutoParse, "parse TSV YAML/JSON values")
 
 	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredLuaPreferences.DocPrefix, "lua-prefix", yqlib.ConfiguredLuaPreferences.DocPrefix, "prefix")
+	if err = rootCmd.RegisterFlagCompletionFunc("lua-prefix", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
 	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredLuaPreferences.DocSuffix, "lua-suffix", yqlib.ConfiguredLuaPreferences.DocSuffix, "suffix")
+	if err = rootCmd.RegisterFlagCompletionFunc("lua-suffix", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
 	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.UnquotedKeys, "lua-unquoted", yqlib.ConfiguredLuaPreferences.UnquotedKeys, "output unquoted string keys (e.g. {foo=\"bar\"})")
 	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.Globals, "lua-globals", yqlib.ConfiguredLuaPreferences.Globals, "output keys as top-level global variables")
 
+	rootCmd.PersistentFlags().StringVar(&properties.ConfiguredPropertiesPreferences.KeyValueSeparator, "properties-separator", properties.ConfiguredPropertiesPreferences.KeyValueSeparator, "separator to use between keys and values")
+	rootCmd.PersistentFlags().BoolVar(&properties.ConfiguredPropertiesPreferences.UseArrayBrackets, "properties-array-brackets", properties.ConfiguredPropertiesPreferences.UseArrayBrackets, "use [x] in array paths (e.g. for SpringBoot)")
+
+	rootCmd.PersistentFlags().BoolVar(&yqlib.StringInterpolationEnabled, "string-interpolation", yqlib.StringInterpolationEnabled, "Toggles strings interpolation of \\(exp)")
+
 	rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.")
 	rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)")
 
 	rootCmd.PersistentFlags().IntVarP(&indent, "indent", "I", 2, "sets indent level for output")
+	if err = rootCmd.RegisterFlagCompletionFunc("indent", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
 	rootCmd.Flags().BoolVarP(&version, "version", "V", false, "Print version information and quit")
 	rootCmd.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the file in place of first file given.")
 	rootCmd.PersistentFlags().VarP(unwrapScalarFlag, "unwrapScalar", "r", "unwrap scalar, print the value with no quotes, colors or comments. Defaults to true for yaml")
@@ -105,13 +181,28 @@ yq -P -oy sample.json
 	rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors")
 	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
 	rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
+	if err = rootCmd.RegisterFlagCompletionFunc("front-matter", cobra.FixedCompletions([]string{"extract", "process"}, cobra.ShellCompDirectiveNoFileComp)); err != nil {
+		panic(err)
+	}
 	rootCmd.PersistentFlags().StringVarP(&forceExpression, "expression", "", "", "forcibly set the expression argument. Useful when yq argument detection thinks your expression is a file.")
+	if err = rootCmd.RegisterFlagCompletionFunc("expression", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
 	rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredYamlPreferences.LeadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.")
 
 	rootCmd.PersistentFlags().StringVarP(&splitFileExp, "split-exp", "s", "", "print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter.")
+	if err = rootCmd.RegisterFlagCompletionFunc("split-exp", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
 	rootCmd.PersistentFlags().StringVarP(&splitFileExpFile, "split-exp-file", "", "", "Use a file to specify the split-exp expression.")
+	if err = rootCmd.MarkPersistentFlagFilename("split-exp-file"); err != nil {
+		panic(err)
+	}
 
 	rootCmd.PersistentFlags().StringVarP(&expressionFile, "from-file", "", "", "Load expression from specified file.")
+	if err = rootCmd.MarkPersistentFlagFilename("from-file"); err != nil {
+		panic(err)
+	}
 
 	rootCmd.AddCommand(
 		createEvaluateSequenceCommand(),

cmd/shell-completion.go

@@ -47,7 +47,7 @@ $ yq shell-completion fish > ~/.config/fish/completions/yq.fish
 		var err error = nil
 		switch args[0] {
 		case "bash":
-			err = cmd.Root().GenBashCompletion(os.Stdout)
+			err = cmd.Root().GenBashCompletionV2(os.Stdout, true)
 		case "zsh":
 			err = cmd.Root().GenZshCompletion(os.Stdout)
 		case "fish":

cmd/utils.go

@@ -6,6 +6,8 @@ import (
 	"os"
 	"strings"
 
+	"github.com/mikefarah/yq/v4/pkg/properties"
+	"github.com/mikefarah/yq/v4/pkg/xml"
 	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"github.com/spf13/cobra"
 	"gopkg.in/op/go-logging.v1"
@@ -64,9 +66,9 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 	}
 	if inputFormat == "" || inputFormat == "auto" || inputFormat == "a" {
 
-		inputFormat = yqlib.FormatFromFilename(inputFilename)
+		inputFormat = yqlib.FormatStringFromFilename(inputFilename)
 
-		_, err := yqlib.InputFormatFromString(inputFormat)
+		_, err := yqlib.FormatFromString(inputFormat)
 		if err != nil {
 			// unknown file type, default to yaml
 			yqlib.GetLogger().Debug("Unknown file format extension '%v', defaulting to yaml", inputFormat)
@@ -86,14 +88,14 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 		// before this was introduced, `yq -pcsv things.csv`
 		// would produce *yaml* output.
 		//
-		outputFormat = yqlib.FormatFromFilename(inputFilename)
+		outputFormat = yqlib.FormatStringFromFilename(inputFilename)
 		if inputFilename != "-" {
 			yqlib.GetLogger().Warning("yq default output is now 'auto' (based on the filename extension). Normally yq would output '%v', but for backwards compatibility 'yaml' has been set. Please use -oy to specify yaml, or drop the -p flag.", outputFormat)
 		}
 		outputFormat = "yaml"
 	}
 
-	outputFormatType, err := yqlib.OutputFormatFromString(outputFormat)
+	outputFormatType, err := yqlib.FormatFromString(outputFormat)
 
 	if err != nil {
 		return "", nil, err
@@ -101,59 +103,32 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 	yqlib.GetLogger().Debug("Using input format %v", inputFormat)
 	yqlib.GetLogger().Debug("Using output format %v", outputFormat)
 
-	if outputFormatType == yqlib.YamlOutputFormat ||
-		outputFormatType == yqlib.PropsOutputFormat {
+	if outputFormatType == yqlib.YamlFormat ||
+		outputFormatType == properties.PropertiesFormat {
 		unwrapScalar = true
 	}
 	if unwrapScalarFlag.IsExplicitlySet() {
 		unwrapScalar = unwrapScalarFlag.IsSet()
 	}
 
-	//copy preference form global setting
-	yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar
-
-	yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators
-
 	return expression, args, nil
 }
 
 func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
-	yqlibInputFormat, err := yqlib.InputFormatFromString(inputFormat)
+	format, err := yqlib.FormatFromString(inputFormat)
 	if err != nil {
 		return nil, err
 	}
-	yqlibDecoder, err := createDecoder(yqlibInputFormat, evaluateTogether)
+	yqlib.ConfiguredYamlPreferences.EvaluateTogether = evaluateTogether
+
+	yqlibDecoder := format.DecoderFactory()
 	if yqlibDecoder == nil {
 		return nil, fmt.Errorf("no support for %s input format", inputFormat)
 	}
-	return yqlibDecoder, err
+	return yqlibDecoder, nil
 }
 
-func createDecoder(format yqlib.InputFormat, evaluateTogether bool) (yqlib.Decoder, error) {
-	switch format {
-	case yqlib.LuaInputFormat:
-		return yqlib.NewLuaDecoder(yqlib.ConfiguredLuaPreferences), nil
-	case yqlib.XMLInputFormat:
-		return yqlib.NewXMLDecoder(yqlib.ConfiguredXMLPreferences), nil
-	case yqlib.PropertiesInputFormat:
-		return yqlib.NewPropertiesDecoder(), nil
-	case yqlib.JsonInputFormat:
-		return yqlib.NewJSONDecoder(), nil
-	case yqlib.CSVObjectInputFormat:
-		return yqlib.NewCSVObjectDecoder(yqlib.ConfiguredCsvPreferences), nil
-	case yqlib.TSVObjectInputFormat:
-		return yqlib.NewCSVObjectDecoder(yqlib.ConfiguredTsvPreferences), nil
-	case yqlib.TomlInputFormat:
-		return yqlib.NewTomlDecoder(), nil
-	case yqlib.YamlInputFormat:
-		prefs := yqlib.ConfiguredYamlPreferences
-		prefs.EvaluateTogether = evaluateTogether
-		return yqlib.NewYamlDecoder(prefs), nil
-	}
-	return nil, fmt.Errorf("invalid decoder: %v", format)
-}
-
-func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yqlib.PrinterWriter, error) {
+func configurePrinterWriter(format *yqlib.Format, out io.Writer) (yqlib.PrinterWriter, error) {
 
 	var printerWriter yqlib.PrinterWriter
 
@@ -171,39 +146,29 @@ func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yq
 }
 
 func configureEncoder() (yqlib.Encoder, error) {
-	yqlibOutputFormat, err := yqlib.OutputFormatFromString(outputFormat)
+	yqlibOutputFormat, err := yqlib.FormatFromString(outputFormat)
 	if err != nil {
 		return nil, err
 	}
-	yqlibEncoder, err := createEncoder(yqlibOutputFormat)
-	if yqlibEncoder == nil {
+	xml.ConfiguredXMLPreferences.Indent = indent
+	yqlib.ConfiguredYamlPreferences.Indent = indent
+	yqlib.ConfiguredJSONPreferences.Indent = indent
+
+	yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar
+	properties.ConfiguredPropertiesPreferences.UnwrapScalar = unwrapScalar
+	yqlib.ConfiguredJSONPreferences.UnwrapScalar = unwrapScalar
+
+	yqlib.ConfiguredYamlPreferences.ColorsEnabled = colorsEnabled
+	yqlib.ConfiguredJSONPreferences.ColorsEnabled = colorsEnabled
+
+	yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators
+
+	encoder := yqlibOutputFormat.EncoderFactory()
+
+	if encoder == nil {
 		return nil, fmt.Errorf("no support for %s output format", outputFormat)
 	}
-	return yqlibEncoder, err
-}
-
-func createEncoder(format yqlib.PrinterOutputFormat) (yqlib.Encoder, error) {
-	switch format {
-	case yqlib.JSONOutputFormat:
-		return yqlib.NewJSONEncoder(indent, colorsEnabled, unwrapScalar), nil
-	case yqlib.PropsOutputFormat:
-		return yqlib.NewPropertiesEncoder(unwrapScalar), nil
-	case yqlib.CSVOutputFormat:
-		return yqlib.NewCsvEncoder(','), nil
-	case yqlib.TSVOutputFormat:
-		return yqlib.NewCsvEncoder('\t'), nil
-	case yqlib.YamlOutputFormat:
-		return yqlib.NewYamlEncoder(indent, colorsEnabled, yqlib.ConfiguredYamlPreferences), nil
-	case yqlib.XMLOutputFormat:
-		return yqlib.NewXMLEncoder(indent, yqlib.ConfiguredXMLPreferences), nil
-	case yqlib.TomlOutputFormat:
-		return yqlib.NewTomlEncoder(), nil
-	case yqlib.ShellVariablesOutputFormat:
-		return yqlib.NewShellVariablesEncoder(), nil
-	case yqlib.LuaOutputFormat:
-		return yqlib.NewLuaEncoder(yqlib.ConfiguredLuaPreferences), nil
-	}
-	return nil, fmt.Errorf("invalid encoder: %v", format)
+	return encoder, err
 }
 
 // this is a hack to enable backwards compatibility with githubactions (which pipe /dev/null into everything)
@@ -255,6 +220,16 @@ func processStdInArgs(args []string) []string {
 
 func processArgs(originalArgs []string) (string, []string, error) {
 	expression := forceExpression
+	args := processStdInArgs(originalArgs)
+	maybeFirstArgIsAFile := len(args) > 0 && maybeFile(args[0])
+
+	if expressionFile == "" && maybeFirstArgIsAFile && strings.HasSuffix(args[0], ".yq") {
+		// lets check if an expression file was given
+		yqlib.GetLogger().Debug("Assuming arg %v is an expression file", args[0])
+		expressionFile = args[0]
+		args = args[1:]
+	}
+
 	if expressionFile != "" {
 		expressionBytes, err := os.ReadFile(expressionFile)
 		if err != nil {
@@ -264,7 +239,6 @@ func processArgs(originalArgs []string) (string, []string, error) {
 		expression = strings.ReplaceAll(string(expressionBytes), "\r\n", "\n")
 	}
 
-	args := processStdInArgs(originalArgs)
 	yqlib.GetLogger().Debugf("processed args: %v", args)
 	if expression == "" && len(args) > 0 && args[0] != "-" && !maybeFile(args[0]) {
 		yqlib.GetLogger().Debug("assuming expression is '%v'", args[0])

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "v4.41.1"
+	Version = "v4.42.1"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

examples/data1.yaml

@@ -1,2 +1,11 @@
-a: #things
-  meow
+# block comments come through
+person: # neither do comments on maps
+    name: Mike Wazowski # comments on values appear
+    pets: 
+    - cat # comments on array values appear
+    - dog # comments on array values appear
+    - things:
+        - frog
+    food: [pizza] # comments on arrays do not
+emptyArray: []
+emptyMap: []

examples/environment.yq

@@ -1,3 +1,4 @@
+#! yq
 .[] |(
     ( select(kind == "scalar") | key + "='" + . + "'"),
     ( select(kind == "seq") | key + "=(" + (map("'" + . + "'") | join(",")) + ")")

go.mod

@@ -3,7 +3,7 @@ module github.com/mikefarah/yq/v4
 require (
 	github.com/a8m/envsubst v1.4.2
 	github.com/alecthomas/participle/v2 v2.1.1
-	github.com/alecthomas/repr v0.3.0
+	github.com/alecthomas/repr v0.4.0
 	github.com/dimchansky/utfbom v1.1.1
 	github.com/elliotchance/orderedmap v1.5.1
 	github.com/fatih/color v1.16.0
@@ -16,7 +16,7 @@ require (
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
 	github.com/yuin/gopher-lua v1.1.1
-	golang.org/x/net v0.21.0
+	golang.org/x/net v0.22.0
 	golang.org/x/text v0.14.0
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.1
@@ -26,7 +26,7 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 )
 

go.sum

@@ -4,8 +4,8 @@ github.com/alecthomas/assert/v2 v2.3.0 h1:mAsH2wmvjsuvyBvAmCtm7zFsBlb8mIHx5ySLVd
 github.com/alecthomas/assert/v2 v2.3.0/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ=
 github.com/alecthomas/participle/v2 v2.1.1 h1:hrjKESvSqGHzRb4yW1ciisFJ4p3MGYih6icjJvbsmV8=
 github.com/alecthomas/participle/v2 v2.1.1/go.mod h1:Y1+hAs8DHPmc3YUFzqllV+eSQ9ljPTk0ZkPMtEdAx2c=
-github.com/alecthomas/repr v0.3.0 h1:NeYzUPfjjlqHY4KtzgKJiWd6sVq2eNUPTi34PiFGjY8=
-github.com/alecthomas/repr v0.3.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
+github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -64,14 +64,14 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
 github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
-golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
+golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=

pkg/properties/decoder_properties.go

@@ -1,4 +1,4 @@
-package yqlib
+package properties
 
 import (
 	"bytes"
@@ -8,16 +8,17 @@ import (
 	"strings"
 
 	"github.com/magiconair/properties"
+	"github.com/mikefarah/yq/v4/pkg/yqlib"
 )
 
 type propertiesDecoder struct {
 	reader   io.Reader
 	finished bool
-	d        DataTreeNavigator
+	d        yqlib.DataTreeNavigator
 }
 
-func NewPropertiesDecoder() Decoder {
-	return &propertiesDecoder{d: NewDataTreeNavigator(), finished: false}
+func NewPropertiesDecoder() yqlib.Decoder {
+	return &propertiesDecoder{d: yqlib.NewDataTreeNavigator(), finished: false}
 }
 
 func (dec *propertiesDecoder) Init(reader io.Reader) error {
@@ -47,31 +48,18 @@ func (dec *propertiesDecoder) processComment(c string) string {
 	return "# " + c
 }
 
-func (dec *propertiesDecoder) applyPropertyComments(context Context, path []interface{}, comments []string) error {
-	assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
-
-	rhsCandidateNode := &CandidateNode{
+func (dec *propertiesDecoder) applyPropertyComments(context yqlib.Context, path []interface{}, comments []string) error {
+	rhsCandidateNode := &yqlib.CandidateNode{
 		Tag:         "!!str",
 		Value:       fmt.Sprintf("%v", path[len(path)-1]),
 		HeadComment: dec.processComment(strings.Join(comments, "\n")),
-		Kind:        ScalarNode,
+		Kind:        yqlib.ScalarNode,
 	}
-
-	rhsCandidateNode.Tag = rhsCandidateNode.guessTagFromCustomType()
-
-	rhsOp := &Operation{OperationType: referenceOpType, CandidateNode: rhsCandidateNode}
-
-	assignmentOpNode := &ExpressionNode{
-		Operation: assignmentOp,
-		LHS:       createTraversalTree(path, traversePreferences{}, true),
-		RHS:       &ExpressionNode{Operation: rhsOp},
-	}
-
-	_, err := dec.d.GetMatchingNodes(context, assignmentOpNode)
-	return err
+	rhsCandidateNode.Tag = rhsCandidateNode.GuessTagFromCustomType()
+	return dec.d.DeeplyAssignKey(context, path, rhsCandidateNode)
 }
 
-func (dec *propertiesDecoder) applyProperty(context Context, properties *properties.Properties, key string) error {
+func (dec *propertiesDecoder) applyProperty(context yqlib.Context, properties *properties.Properties, key string) error {
 	value, _ := properties.Get(key)
 	path := parsePropKey(key)
 
@@ -83,13 +71,13 @@ func (dec *propertiesDecoder) applyProperty(context Context, properties *propert
 		}
 	}
 
-	rhsNode := createStringScalarNode(value)
-	rhsNode.Tag = rhsNode.guessTagFromCustomType()
+	rhsNode := yqlib.CreateStringScalarNode(value)
+	rhsNode.Tag = rhsNode.GuessTagFromCustomType()
 
 	return dec.d.DeeplyAssign(context, path, rhsNode)
 }
 
-func (dec *propertiesDecoder) Decode() (*CandidateNode, error) {
+func (dec *propertiesDecoder) Decode() (*yqlib.CandidateNode, error) {
 	if dec.finished {
 		return nil, io.EOF
 	}
@@ -108,12 +96,12 @@ func (dec *propertiesDecoder) Decode() (*CandidateNode, error) {
 	}
 	properties.DisableExpansion = true
 
-	rootMap := &CandidateNode{
-		Kind: MappingNode,
+	rootMap := &yqlib.CandidateNode{
+		Kind: yqlib.MappingNode,
 		Tag:  "!!map",
 	}
 
-	context := Context{}
+	context := yqlib.Context{}
 	context = context.SingleChildContext(rootMap)
 
 	for _, key := range properties.Keys() {

pkg/properties/encoder_properties.go

@@ -1,4 +1,4 @@
-package yqlib
+package properties
 
 import (
 	"bufio"
@@ -8,15 +8,16 @@ import (
 	"strings"
 
 	"github.com/magiconair/properties"
+	"github.com/mikefarah/yq/v4/pkg/yqlib"
 )
 
 type propertiesEncoder struct {
-	unwrapScalar bool
+	prefs PropertiesPreferences
 }
 
-func NewPropertiesEncoder(unwrapScalar bool) Encoder {
+func NewPropertiesEncoder(prefs PropertiesPreferences) yqlib.Encoder {
 	return &propertiesEncoder{
-		unwrapScalar: unwrapScalar,
+		prefs: prefs,
 	}
 }
 
@@ -43,7 +44,7 @@ func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content strin
 			}
 
 		} else {
-			if err := writeString(writer, readline); err != nil {
+			if err := yqlib.WriteString(writer, readline); err != nil {
 				return err
 			}
 		}
@@ -51,7 +52,7 @@ func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content strin
 		if errors.Is(errReading, io.EOF) {
 			if readline != "" {
 				// the last comment we read didn't have a newline, put one in
-				if err := writeString(writer, "\n"); err != nil {
+				if err := yqlib.WriteString(writer, "\n"); err != nil {
 					return err
 				}
 			}
@@ -61,14 +62,15 @@ func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content strin
 	return nil
 }
 
-func (pe *propertiesEncoder) Encode(writer io.Writer, node *CandidateNode) error {
+func (pe *propertiesEncoder) Encode(writer io.Writer, node *yqlib.CandidateNode) error {
 
-	if node.Kind == ScalarNode {
-		return writeString(writer, node.Value+"\n")
+	if node.Kind == yqlib.ScalarNode {
+		return yqlib.WriteString(writer, node.Value+"\n")
 	}
 
-	mapKeysToStrings(node)
+	node.ConvertKeysToStrings()
 	p := properties.NewProperties()
+	p.WriteSeparator = pe.prefs.KeyValueSeparator
 	err := pe.doEncode(p, node, "", nil)
 	if err != nil {
 		return err
@@ -78,32 +80,32 @@ func (pe *propertiesEncoder) Encode(writer io.Writer, node *CandidateNode) error
 	return err
 }
 
-func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *CandidateNode, path string, keyNode *CandidateNode) error {
+func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yqlib.CandidateNode, path string, keyNode *yqlib.CandidateNode) error {
 
 	comments := ""
 	if keyNode != nil {
 		// include the key node comments if present
-		comments = headAndLineComment(keyNode)
+		comments = keyNode.CleanHeadAndLineComment()
 	}
-	comments = comments + headAndLineComment(node)
+	comments = comments + node.CleanHeadAndLineComment()
 	commentsWithSpaces := strings.ReplaceAll(comments, "\n", "\n ")
 	p.SetComments(path, strings.Split(commentsWithSpaces, "\n"))
 
 	switch node.Kind {
-	case ScalarNode:
+	case yqlib.ScalarNode:
 		var nodeValue string
-		if pe.unwrapScalar || !strings.Contains(node.Value, " ") {
+		if pe.prefs.UnwrapScalar || !strings.Contains(node.Value, " ") {
 			nodeValue = node.Value
 		} else {
 			nodeValue = fmt.Sprintf("%q", node.Value)
 		}
 		_, _, err := p.Set(path, nodeValue)
 		return err
-	case SequenceNode:
+	case yqlib.SequenceNode:
 		return pe.encodeArray(p, node.Content, path)
-	case MappingNode:
+	case yqlib.MappingNode:
 		return pe.encodeMap(p, node.Content, path)
-	case AliasNode:
+	case yqlib.AliasNode:
 		return pe.doEncode(p, node.Alias, path, nil)
 	default:
 		return fmt.Errorf("Unsupported node %v", node.Tag)
@@ -114,10 +116,17 @@ func (pe *propertiesEncoder) appendPath(path string, key interface{}) string {
 	if path == "" {
 		return fmt.Sprintf("%v", key)
 	}
+	switch key.(type) {
+	case int:
+		if pe.prefs.UseArrayBrackets {
+			return fmt.Sprintf("%v[%v]", path, key)
+		}
+
+	}
 	return fmt.Sprintf("%v.%v", path, key)
 }
 
-func (pe *propertiesEncoder) encodeArray(p *properties.Properties, kids []*CandidateNode, path string) error {
+func (pe *propertiesEncoder) encodeArray(p *properties.Properties, kids []*yqlib.CandidateNode, path string) error {
 	for index, child := range kids {
 		err := pe.doEncode(p, child, pe.appendPath(path, index), nil)
 		if err != nil {
@@ -127,7 +136,7 @@ func (pe *propertiesEncoder) encodeArray(p *properties.Properties, kids []*Candi
 	return nil
 }
 
-func (pe *propertiesEncoder) encodeMap(p *properties.Properties, kids []*CandidateNode, path string) error {
+func (pe *propertiesEncoder) encodeMap(p *properties.Properties, kids []*yqlib.CandidateNode, path string) error {
 	for index := 0; index < len(kids); index = index + 2 {
 		key := kids[index]
 		value := kids[index+1]

pkg/properties/encoder_properties_test.go

@@ -0,0 +1,208 @@
+package properties
+
+import (
+	"bufio"
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+type keyValuePair struct {
+	key     string
+	value   string
+	comment string
+}
+
+func (kv *keyValuePair) String(unwrap bool, sep string) string {
+	builder := strings.Builder{}
+
+	if kv.comment != "" {
+		builder.WriteString(kv.comment)
+		builder.WriteString("\n")
+	}
+
+	builder.WriteString(kv.key)
+	builder.WriteString(sep)
+
+	if unwrap {
+		builder.WriteString(kv.value)
+	} else {
+		builder.WriteString("\"")
+		builder.WriteString(kv.value)
+		builder.WriteString("\"")
+	}
+
+	return builder.String()
+}
+
+type testProperties struct {
+	pairs []keyValuePair
+}
+
+func (tp *testProperties) String(unwrap bool, sep string) string {
+	kvs := []string{}
+
+	for _, kv := range tp.pairs {
+		kvs = append(kvs, kv.String(unwrap, sep))
+	}
+
+	return strings.Join(kvs, "\n")
+}
+
+func yamlToProps(sampleYaml string, unwrapScalar bool, separator string) string {
+	var output bytes.Buffer
+	writer := bufio.NewWriter(&output)
+
+	var propsEncoder = NewPropertiesEncoder(PropertiesPreferences{KeyValueSeparator: separator, UnwrapScalar: unwrapScalar})
+	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
+	if err != nil {
+		panic(err)
+	}
+	node := inputs.Front().Value.(*CandidateNode)
+	err = propsEncoder.Encode(writer, node)
+	if err != nil {
+		panic(err)
+	}
+	writer.Flush()
+
+	return strings.TrimSuffix(output.String(), "\n")
+}
+
+func doTest(t *testing.T, sampleYaml string, props testProperties, testUnwrapped, testWrapped bool) {
+	wraps := []bool{}
+	if testUnwrapped {
+		wraps = append(wraps, true)
+	}
+	if testWrapped {
+		wraps = append(wraps, false)
+	}
+
+	for _, unwrap := range wraps {
+		for _, sep := range []string{" = ", ";", "=", " "} {
+			var actualProps = yamlToProps(sampleYaml, unwrap, sep)
+			test.AssertResult(t, props.String(unwrap, sep), actualProps)
+		}
+	}
+}
+
+func TestPropertiesEncoderSimple(t *testing.T) {
+	var sampleYaml = `a: 'bob cool'`
+
+	doTest(
+		t, sampleYaml,
+		testProperties{
+			pairs: []keyValuePair{
+				{
+					key:   "a",
+					value: "bob cool",
+				},
+			},
+		},
+		true, true,
+	)
+}
+
+func TestPropertiesEncoderSimpleWithComments(t *testing.T) {
+	var sampleYaml = `a: 'bob cool' # line`
+
+	doTest(
+		t, sampleYaml,
+		testProperties{
+			pairs: []keyValuePair{
+				{
+					key:     "a",
+					value:   "bob cool",
+					comment: "# line",
+				},
+			},
+		},
+		true, true,
+	)
+}
+
+func TestPropertiesEncoderDeep(t *testing.T) {
+	var sampleYaml = `a: 
+  b: "bob cool"
+`
+
+	doTest(
+		t, sampleYaml,
+		testProperties{
+			pairs: []keyValuePair{
+				{
+					key:   "a.b",
+					value: "bob cool",
+				},
+			},
+		},
+		true, true,
+	)
+}
+
+func TestPropertiesEncoderDeepWithComments(t *testing.T) {
+	var sampleYaml = `a:  # a thing
+  b: "bob cool" # b thing
+`
+
+	doTest(
+		t, sampleYaml,
+		testProperties{
+			pairs: []keyValuePair{
+				{
+					key:     "a.b",
+					value:   "bob cool",
+					comment: "# b thing",
+				},
+			},
+		},
+		true, true,
+	)
+}
+
+func TestPropertiesEncoderArray_Unwrapped(t *testing.T) {
+	var sampleYaml = `a: 
+  b: [{c: dog}, {c: cat}]
+`
+
+	doTest(
+		t, sampleYaml,
+		testProperties{
+			pairs: []keyValuePair{
+				{
+					key:   "a.b.0.c",
+					value: "dog",
+				},
+				{
+					key:   "a.b.1.c",
+					value: "cat",
+				},
+			},
+		},
+		true, false,
+	)
+}
+
+func TestPropertiesEncoderArray_Wrapped(t *testing.T) {
+	var sampleYaml = `a: 
+  b: [{c: dog named jim}, {c: cat named jim}]
+`
+
+	doTest(
+		t, sampleYaml,
+		testProperties{
+			pairs: []keyValuePair{
+				{
+					key:   "a.b.0.c",
+					value: "dog named jim",
+				},
+				{
+					key:   "a.b.1.c",
+					value: "cat named jim",
+				},
+			},
+		},
+		false, true,
+	)
+}

pkg/properties/properties.go

@@ -0,0 +1,45 @@
+package properties
+
+import "github.com/mikefarah/yq/v4/pkg/yqlib"
+
+type PropertiesPreferences struct {
+	UnwrapScalar      bool
+	KeyValueSeparator string
+	UseArrayBrackets  bool
+}
+
+func NewDefaultPropertiesPreferences() PropertiesPreferences {
+	return PropertiesPreferences{
+		UnwrapScalar:      true,
+		KeyValueSeparator: " = ",
+		UseArrayBrackets:  false,
+	}
+}
+
+func (p *PropertiesPreferences) Copy() PropertiesPreferences {
+	return PropertiesPreferences{
+		UnwrapScalar:      p.UnwrapScalar,
+		KeyValueSeparator: p.KeyValueSeparator,
+		UseArrayBrackets:  p.UseArrayBrackets,
+	}
+}
+
+var PropertiesFormat = &yqlib.Format{"props", []string{"p", "properties"}, "properties",
+	func() yqlib.Encoder { return NewPropertiesEncoder(ConfiguredPropertiesPreferences) },
+	func() yqlib.Decoder { return NewPropertiesDecoder() },
+	nil,
+}
+
+var propertyYqRules = []*yqlib.ParticipleYqRule{
+	{"PropertiesDecode", `from_?props|@propsd`, yqlib.CreateDecodeOpYqAction(PropertiesFormat), 0},
+	{"PropsEncode", `to_?props|@props`, yqlib.CreateEncodeOpYqAction(PropertiesFormat, 2), 0},
+	{"LoadProperties", `load_?props`, yqlib.CreateLoadOpYqAction(NewPropertiesDecoder()), 0},
+}
+
+func RegisterPropertiesFormat() {
+	yqlib.RegisterFormat(PropertiesFormat)
+	yqlib.RegisterRules(propertyYqRules)
+
+}
+
+var ConfiguredPropertiesPreferences = NewDefaultPropertiesPreferences()

pkg/xml/decoder_xml.go

@@ -1,6 +1,6 @@
 //go:build !yq_noxml
 
-package yqlib
+package xml
 
 import (
 	"encoding/xml"
@@ -11,6 +11,7 @@ import (
 	"strings"
 	"unicode"
 
+	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"golang.org/x/net/html/charset"
 )
 
@@ -21,7 +22,7 @@ type xmlDecoder struct {
 	prefs        XmlPreferences
 }
 
-func NewXMLDecoder(prefs XmlPreferences) Decoder {
+func NewXMLDecoder(prefs XmlPreferences) yqlib.Decoder {
 	return &xmlDecoder{
 		finished: false,
 		prefs:    prefs,
@@ -35,8 +36,8 @@ func (dec *xmlDecoder) Init(reader io.Reader) error {
 	return nil
 }
 
-func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*CandidateNode, error) {
-	yamlNode := &CandidateNode{Kind: SequenceNode, Tag: "!!seq"}
+func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yqlib.CandidateNode, error) {
+	yamlNode := &yqlib.CandidateNode{Kind: yqlib.SequenceNode, Tag: "!!seq"}
 	for _, child := range nodes {
 		yamlChild, err := dec.convertToYamlNode(child)
 		if err != nil {
@@ -63,14 +64,14 @@ func (dec *xmlDecoder) processComment(c string) string {
 	return replacement
 }
 
-func (dec *xmlDecoder) createMap(n *xmlNode) (*CandidateNode, error) {
-	log.Debug("createMap: headC: %v, lineC: %v, footC: %v", n.HeadComment, n.LineComment, n.FootComment)
-	yamlNode := &CandidateNode{Kind: MappingNode, Tag: "!!map"}
+func (dec *xmlDecoder) createMap(n *xmlNode) (*yqlib.CandidateNode, error) {
+	yqlib.GetLogger().Debug("createMap: headC: %v, lineC: %v, footC: %v", n.HeadComment, n.LineComment, n.FootComment)
+	yamlNode := &yqlib.CandidateNode{Kind: yqlib.MappingNode, Tag: "!!map"}
 
 	if len(n.Data) > 0 {
-		log.Debugf("creating content node for map: %v", dec.prefs.ContentName)
+		yqlib.GetLogger().Debugf("creating content node for map: %v", dec.prefs.ContentName)
 		label := dec.prefs.ContentName
-		labelNode := createScalarNode(label, label)
+		labelNode := yqlib.CreateScalarNode(label, label)
 		labelNode.HeadComment = dec.processComment(n.HeadComment)
 		labelNode.LineComment = dec.processComment(n.LineComment)
 		labelNode.FootComment = dec.processComment(n.FootComment)
@@ -80,19 +81,19 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*CandidateNode, error) {
 	for i, keyValuePair := range n.Children {
 		label := keyValuePair.K
 		children := keyValuePair.V
-		labelNode := createScalarNode(label, label)
-		var valueNode *CandidateNode
+		labelNode := yqlib.CreateScalarNode(label, label)
+		var valueNode *yqlib.CandidateNode
 		var err error
 
 		if i == 0 {
-			log.Debugf("head comment here")
+			yqlib.GetLogger().Debugf("head comment here")
 			labelNode.HeadComment = dec.processComment(n.HeadComment)
 
 		}
-		log.Debugf("label=%v, i=%v, keyValuePair.FootComment: %v", label, i, keyValuePair.FootComment)
+		yqlib.GetLogger().Debugf("label=%v, i=%v, keyValuePair.FootComment: %v", label, i, keyValuePair.FootComment)
 		labelNode.FootComment = dec.processComment(keyValuePair.FootComment)
 
-		log.Debug("len of children in %v is %v", label, len(children))
+		yqlib.GetLogger().Debug("len of children in %v is %v", label, len(children))
 		if len(children) > 1 {
 			valueNode, err = dec.createSequence(children)
 			if err != nil {
@@ -105,7 +106,7 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*CandidateNode, error) {
 			if len(children[0].Children) == 0 && children[0].HeadComment != "" {
 				if len(children[0].Data) > 0 {
 
-					log.Debug("scalar comment hack, currentlabel [%v]", labelNode.HeadComment)
+					yqlib.GetLogger().Debug("scalar comment hack, currentlabel [%v]", labelNode.HeadComment)
 					labelNode.HeadComment = joinComments([]string{labelNode.HeadComment, strings.TrimSpace(children[0].HeadComment)}, "\n")
 					children[0].HeadComment = ""
 				} else {
@@ -125,33 +126,33 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*CandidateNode, error) {
 	return yamlNode, nil
 }
 
-func (dec *xmlDecoder) createValueNodeFromData(values []string) *CandidateNode {
+func (dec *xmlDecoder) createValueNodeFromData(values []string) *yqlib.CandidateNode {
 	switch len(values) {
 	case 0:
-		return createScalarNode(nil, "")
+		return yqlib.CreateScalarNode(nil, "")
 	case 1:
-		return createScalarNode(values[0], values[0])
+		return yqlib.CreateScalarNode(values[0], values[0])
 	default:
-		content := make([]*CandidateNode, 0)
+		content := make([]*yqlib.CandidateNode, 0)
 		for _, value := range values {
-			content = append(content, createScalarNode(value, value))
+			content = append(content, yqlib.CreateScalarNode(value, value))
 		}
-		return &CandidateNode{
-			Kind:    SequenceNode,
+		return &yqlib.CandidateNode{
+			Kind:    yqlib.SequenceNode,
 			Tag:     "!!seq",
 			Content: content,
 		}
 	}
 }
 
-func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*CandidateNode, error) {
+func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yqlib.CandidateNode, error) {
 	if len(n.Children) > 0 {
 		return dec.createMap(n)
 	}
 
 	scalar := dec.createValueNodeFromData(n.Data)
 
-	log.Debug("scalar (%v), headC: %v, lineC: %v, footC: %v", scalar.Tag, n.HeadComment, n.LineComment, n.FootComment)
+	yqlib.GetLogger().Debug("scalar (%v), headC: %v, lineC: %v, footC: %v", scalar.Tag, n.HeadComment, n.LineComment, n.FootComment)
 	scalar.HeadComment = dec.processComment(n.HeadComment)
 	scalar.LineComment = dec.processComment(n.LineComment)
 	if scalar.Tag == "!!seq" {
@@ -164,7 +165,7 @@ func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*CandidateNode, error) {
 	return scalar, nil
 }
 
-func (dec *xmlDecoder) Decode() (*CandidateNode, error) {
+func (dec *xmlDecoder) Decode() (*yqlib.CandidateNode, error) {
 	if dec.finished {
 		return nil, io.EOF
 	}
@@ -211,17 +212,17 @@ func (n *xmlNode) AddChild(s string, c *xmlNode) {
 	if n.Children == nil {
 		n.Children = make([]*xmlChildrenKv, 0)
 	}
-	log.Debug("looking for %s", s)
+	yqlib.GetLogger().Debug("looking for %s", s)
 	// see if we can find an existing entry to add to
 	for _, childEntry := range n.Children {
 		if childEntry.K == s {
-			log.Debug("found it, appending an entry%s", s)
+			yqlib.GetLogger().Debug("found it, appending an entry%s", s)
 			childEntry.V = append(childEntry.V, c)
-			log.Debug("yay len of children in %v is %v", s, len(childEntry.V))
+			yqlib.GetLogger().Debug("yay len of children in %v is %v", s, len(childEntry.V))
 			return
 		}
 	}
-	log.Debug("not there, making a new one %s", s)
+	yqlib.GetLogger().Debug("not there, making a new one %s", s)
 	n.Children = append(n.Children, &xmlChildrenKv{K: s, V: []*xmlNode{c}})
 }
 
@@ -267,7 +268,7 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
 
 		switch se := t.(type) {
 		case xml.StartElement:
-			log.Debug("start element %v", se.Name.Local)
+			yqlib.GetLogger().Debug("start element %v", se.Name.Local)
 			elem.state = "started"
 			// Build new a new current element and link it to its parent
 			elem = &element{
@@ -296,14 +297,14 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
 			if len(newBit) > 0 {
 				elem.n.Data = append(elem.n.Data, newBit)
 				elem.state = "chardata"
-				log.Debug("chardata [%v] for %v", elem.n.Data, elem.label)
+				yqlib.GetLogger().Debug("chardata [%v] for %v", elem.n.Data, elem.label)
 			}
 		case xml.EndElement:
 			if elem == nil {
-				log.Debug("no element, probably bad xml")
+				yqlib.GetLogger().Debug("no element, probably bad xml")
 				continue
 			}
-			log.Debug("end element %v", elem.label)
+			yqlib.GetLogger().Debug("end element %v", elem.label)
 			elem.state = "finished"
 			// And add it to its parent list
 			if elem.parent != nil {
@@ -319,10 +320,10 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
 				applyFootComment(elem, commentStr)
 
 			} else if elem.state == "chardata" {
-				log.Debug("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
+				yqlib.GetLogger().Debug("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
 				elem.n.LineComment = joinComments([]string{elem.n.LineComment, commentStr}, " ")
 			} else {
-				log.Debug("got a head comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
+				yqlib.GetLogger().Debug("got a head comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
 				elem.n.HeadComment = joinComments([]string{elem.n.HeadComment, commentStr}, " ")
 			}
 
@@ -347,7 +348,7 @@ func applyFootComment(elem *element, commentStr string) {
 	if len(elem.n.Children) > 0 {
 		lastChildIndex := len(elem.n.Children) - 1
 		childKv := elem.n.Children[lastChildIndex]
-		log.Debug("got a foot comment, putting on last child for %v: [%v]", childKv.K, commentStr)
+		yqlib.GetLogger().Debug("got a foot comment, putting on last child for %v: [%v]", childKv.K, commentStr)
 		// if it's an array of scalars, put the foot comment on the scalar itself
 		if len(childKv.V) > 0 && len(childKv.V[0].Children) == 0 {
 			nodeToUpdate := childKv.V[len(childKv.V)-1]
@@ -356,7 +357,7 @@ func applyFootComment(elem *element, commentStr string) {
 			childKv.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, " ")
 		}
 	} else {
-		log.Debug("got a foot comment for %v: [%v]", elem.label, commentStr)
+		yqlib.GetLogger().Debug("got a foot comment for %v: [%v]", elem.label, commentStr)
 		elem.n.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, " ")
 	}
 }

pkg/xml/encoder_xml.go

@@ -1,6 +1,6 @@
 //go:build !yq_noxml
 
-package yqlib
+package xml
 
 import (
 	"encoding/xml"
@@ -8,6 +8,8 @@ import (
 	"io"
 	"regexp"
 	"strings"
+
+	"github.com/mikefarah/yq/v4/pkg/yqlib"
 )
 
 type xmlEncoder struct {
@@ -17,10 +19,10 @@ type xmlEncoder struct {
 	leadingContent string
 }
 
-func NewXMLEncoder(indent int, prefs XmlPreferences) Encoder {
+func NewXMLEncoder(prefs XmlPreferences) yqlib.Encoder {
 	var indentString = ""
 
-	for index := 0; index < indent; index++ {
+	for index := 0; index < prefs.Indent; index++ {
 		indentString = indentString + " "
 	}
 	return &xmlEncoder{indentString, nil, prefs, ""}
@@ -39,7 +41,7 @@ func (e *xmlEncoder) PrintLeadingContent(_ io.Writer, content string) error {
 	return nil
 }
 
-func (e *xmlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
+func (e *xmlEncoder) Encode(writer io.Writer, node *yqlib.CandidateNode) error {
 	encoder := xml.NewEncoder(writer)
 	// hack so we can manually add newlines to procInst and directives
 	e.writer = writer
@@ -59,7 +61,7 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 					return err
 				}
 				if _, err := e.writer.Write([]byte("\n")); err != nil {
-					log.Warning("Unable to write newline, skipping: %w", err)
+					yqlib.GetLogger().Warning("Unable to write newline, skipping: %w", err)
 				}
 			}
 		}
@@ -79,12 +81,12 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 	}
 
 	switch node.Kind {
-	case MappingNode:
+	case yqlib.MappingNode:
 		err := e.encodeTopLevelMap(encoder, node)
 		if err != nil {
 			return err
 		}
-	case ScalarNode:
+	case yqlib.ScalarNode:
 		var charData xml.CharData = []byte(node.Value)
 		err := encoder.EncodeToken(charData)
 		if err != nil {
@@ -99,8 +101,8 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 
 }
 
-func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *CandidateNode) error {
-	err := e.encodeComment(encoder, headAndLineComment(node))
+func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yqlib.CandidateNode) error {
+	err := e.encodeComment(encoder, node.CleanHeadAndLineComment())
 	if err != nil {
 		return err
 	}
@@ -109,12 +111,12 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *CandidateNode
 		value := node.Content[i+1]
 
 		start := xml.StartElement{Name: xml.Name{Local: key.Value}}
-		log.Debugf("comments of key %v", key.Value)
-		err := e.encodeComment(encoder, headAndLineComment(key))
+		yqlib.GetLogger().Debugf("comments of key %v", key.Value)
+		err := e.encodeComment(encoder, key.CleanHeadAndLineComment())
 		if err != nil {
 			return err
 		}
-		if headAndLineComment(key) != "" {
+		if key.CleanHeadAndLineComment() != "" {
 			var newLine xml.CharData = []byte("\n")
 			err = encoder.EncodeToken(newLine)
 			if err != nil {
@@ -131,7 +133,7 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *CandidateNode
 				return err
 			}
 			if _, err := e.writer.Write([]byte("\n")); err != nil {
-				log.Warning("Unable to write newline, skipping: %w", err)
+				yqlib.GetLogger().Warning("Unable to write newline, skipping: %w", err)
 			}
 		} else if key.Value == e.prefs.DirectiveName {
 			var directive xml.Directive = []byte(value.Value)
@@ -139,48 +141,48 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *CandidateNode
 				return err
 			}
 			if _, err := e.writer.Write([]byte("\n")); err != nil {
-				log.Warning("Unable to write newline, skipping: %w", err)
+				yqlib.GetLogger().Warning("Unable to write newline, skipping: %w", err)
 			}
 		} else {
 
-			log.Debugf("recursing")
+			yqlib.GetLogger().Debugf("recursing")
 
 			err = e.doEncode(encoder, value, start)
 			if err != nil {
 				return err
 			}
 		}
-		err = e.encodeComment(encoder, footComment(key))
+		err = e.encodeComment(encoder, key.CleanFootComment())
 		if err != nil {
 			return err
 		}
 	}
-	return e.encodeComment(encoder, footComment(node))
+	return e.encodeComment(encoder, node.CleanFootComment())
 }
 
-func (e *xmlEncoder) encodeStart(encoder *xml.Encoder, node *CandidateNode, start xml.StartElement) error {
+func (e *xmlEncoder) encodeStart(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
 	err := encoder.EncodeToken(start)
 	if err != nil {
 		return err
 	}
-	return e.encodeComment(encoder, headComment(node))
+	return e.encodeComment(encoder, node.CleanHeadComment())
 }
 
-func (e *xmlEncoder) encodeEnd(encoder *xml.Encoder, node *CandidateNode, start xml.StartElement) error {
+func (e *xmlEncoder) encodeEnd(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
 	err := encoder.EncodeToken(start.End())
 	if err != nil {
 		return err
 	}
-	return e.encodeComment(encoder, footComment(node))
+	return e.encodeComment(encoder, node.CleanFootComment())
 }
 
-func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *CandidateNode, start xml.StartElement) error {
+func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
 	switch node.Kind {
-	case MappingNode:
+	case yqlib.MappingNode:
 		return e.encodeMap(encoder, node, start)
-	case SequenceNode:
+	case yqlib.SequenceNode:
 		return e.encodeArray(encoder, node, start)
-	case ScalarNode:
+	case yqlib.ScalarNode:
 		err := e.encodeStart(encoder, node, start)
 		if err != nil {
 			return err
@@ -192,7 +194,7 @@ func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *CandidateNode, start x
 			return err
 		}
 
-		if err = e.encodeComment(encoder, lineComment(node)); err != nil {
+		if err = e.encodeComment(encoder, node.CleanLineComment()); err != nil {
 			return err
 		}
 
@@ -207,13 +209,13 @@ var chompRegexp = regexp.MustCompile(`\n$`)
 
 func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) error {
 	if commentStr != "" {
-		log.Debugf("got comment [%v]", commentStr)
+		yqlib.GetLogger().Debugf("got comment [%v]", commentStr)
 		// multi line string
 		if len(commentStr) > 2 && strings.Contains(commentStr[1:len(commentStr)-1], "\n") {
 			commentStr = chompRegexp.ReplaceAllString(commentStr, "")
-			log.Debugf("chompRegexp [%v]", commentStr)
+			yqlib.GetLogger().Debugf("chompRegexp [%v]", commentStr)
 			commentStr = xmlEncodeMultilineCommentRegex.ReplaceAllString(commentStr, "$1$2")
-			log.Debugf("processed multiline [%v]", commentStr)
+			yqlib.GetLogger().Debugf("processed multiline [%v]", commentStr)
 			// if the first line is non blank, add a space
 			if commentStr[0] != '\n' && commentStr[0] != ' ' {
 				commentStr = " " + commentStr
@@ -225,9 +227,9 @@ func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) erro
 
 		if !strings.HasSuffix(commentStr, " ") && !strings.HasSuffix(commentStr, "\n") {
 			commentStr = commentStr + " "
-			log.Debugf("added suffix [%v]", commentStr)
+			yqlib.GetLogger().Debugf("added suffix [%v]", commentStr)
 		}
-		log.Debugf("encoding comment [%v]", commentStr)
+		yqlib.GetLogger().Debugf("encoding comment [%v]", commentStr)
 
 		var comment xml.Comment = []byte(commentStr)
 		err := encoder.EncodeToken(comment)
@@ -238,9 +240,9 @@ func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) erro
 	return nil
 }
 
-func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *CandidateNode, start xml.StartElement) error {
+func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
 
-	if err := e.encodeComment(encoder, headAndLineComment(node)); err != nil {
+	if err := e.encodeComment(encoder, node.CleanHeadAndLineComment()); err != nil {
 		return err
 	}
 
@@ -250,7 +252,7 @@ func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *CandidateNode, star
 			return err
 		}
 	}
-	return e.encodeComment(encoder, footComment(node))
+	return e.encodeComment(encoder, node.CleanFootComment())
 }
 
 func (e *xmlEncoder) isAttribute(name string) bool {
@@ -260,8 +262,8 @@ func (e *xmlEncoder) isAttribute(name string) bool {
 		!strings.HasPrefix(name, e.prefs.ProcInstPrefix)
 }
 
-func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *CandidateNode, start xml.StartElement) error {
-	log.Debug("its a map")
+func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
+	yqlib.GetLogger().Debug("its a map")
 
 	//first find all the attributes and put them on the start token
 	for i := 0; i < len(node.Content); i += 2 {
@@ -269,7 +271,7 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *CandidateNode, start
 		value := node.Content[i+1]
 
 		if e.isAttribute(key.Value) {
-			if value.Kind == ScalarNode {
+			if value.Kind == yqlib.ScalarNode {
 				attributeName := strings.Replace(key.Value, e.prefs.AttributePrefix, "", 1)
 				start.Attr = append(start.Attr, xml.Attr{Name: xml.Name{Local: attributeName}, Value: value.Value})
 			} else {
@@ -288,7 +290,7 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *CandidateNode, start
 		key := node.Content[i]
 		value := node.Content[i+1]
 
-		err := e.encodeComment(encoder, headAndLineComment(key))
+		err := e.encodeComment(encoder, key.CleanHeadAndLineComment())
 		if err != nil {
 			return err
 		}
@@ -305,7 +307,7 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *CandidateNode, start
 			}
 		} else if key.Value == e.prefs.ContentName {
 			// directly encode the contents
-			err = e.encodeComment(encoder, headAndLineComment(value))
+			err = e.encodeComment(encoder, value.CleanHeadAndLineComment())
 			if err != nil {
 				return err
 			}
@@ -314,7 +316,7 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *CandidateNode, start
 			if err != nil {
 				return err
 			}
-			err = e.encodeComment(encoder, footComment(value))
+			err = e.encodeComment(encoder, value.CleanFootComment())
 			if err != nil {
 				return err
 			}
@@ -325,7 +327,7 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *CandidateNode, start
 				return err
 			}
 		}
-		err = e.encodeComment(encoder, footComment(key))
+		err = e.encodeComment(encoder, key.CleanFootComment())
 		if err != nil {
 			return err
 		}

pkg/xml/xml.go

@@ -0,0 +1,72 @@
+package xml
+
+import "github.com/mikefarah/yq/v4/pkg/yqlib"
+
+type XmlPreferences struct {
+	Indent          int
+	AttributePrefix string
+	ContentName     string
+	StrictMode      bool
+	KeepNamespace   bool
+	UseRawToken     bool
+	ProcInstPrefix  string
+	DirectiveName   string
+	SkipProcInst    bool
+	SkipDirectives  bool
+}
+
+func NewDefaultXmlPreferences() XmlPreferences {
+	return XmlPreferences{
+		Indent:          2,
+		AttributePrefix: "+@",
+		ContentName:     "+content",
+		StrictMode:      false,
+		KeepNamespace:   true,
+		UseRawToken:     true,
+		ProcInstPrefix:  "+p_",
+		DirectiveName:   "+directive",
+		SkipProcInst:    false,
+		SkipDirectives:  false,
+	}
+}
+
+func (p *XmlPreferences) Copy() XmlPreferences {
+	return XmlPreferences{
+		Indent:          p.Indent,
+		AttributePrefix: p.AttributePrefix,
+		ContentName:     p.ContentName,
+		StrictMode:      p.StrictMode,
+		KeepNamespace:   p.KeepNamespace,
+		UseRawToken:     p.UseRawToken,
+		ProcInstPrefix:  p.ProcInstPrefix,
+		DirectiveName:   p.DirectiveName,
+		SkipProcInst:    p.SkipProcInst,
+		SkipDirectives:  p.SkipDirectives,
+	}
+}
+
+var ConfiguredXMLPreferences = NewDefaultXmlPreferences()
+
+var XMLFormat = &yqlib.Format{"xml", []string{"x"}, "xml",
+	func() yqlib.Encoder { return NewXMLEncoder(ConfiguredXMLPreferences) },
+	func() yqlib.Decoder { return NewXMLDecoder(ConfiguredXMLPreferences) },
+	func(indent int) yqlib.Encoder {
+		prefs := ConfiguredXMLPreferences.Copy()
+		prefs.Indent = indent
+		return NewXMLEncoder(prefs)
+	},
+}
+
+var xmlYqRules = []*yqlib.ParticipleYqRule{
+	{"XMLEncodeWithIndent", `to_?xml\([0-9]+\)`, yqlib.CreateEncodeYqActionParsingIndent(XMLFormat), 0},
+	{"XmlDecode", `from_?xml|@xmld`, yqlib.CreateDecodeOpYqAction(XMLFormat), 0},
+	{"XMLEncode", `to_?xml`, yqlib.CreateEncodeOpYqAction(XMLFormat, 2), 0},
+	{"XMLEncodeNoIndent", `@xml`, yqlib.CreateEncodeOpYqAction(XMLFormat, 0), 0},
+	{"LoadXML", `load_?xml|xml_?load`, yqlib.CreateLoadOpYqAction(NewXMLDecoder(ConfiguredXMLPreferences)), 0},
+}
+
+func RegisterXmlFormat() {
+	yqlib.RegisterFormat(XMLFormat)
+	yqlib.RegisterRules(xmlYqRules)
+
+}

pkg/xml/xml_test.go

@@ -1,12 +1,13 @@
 //go:build !yq_noxml
 
-package yqlib
+package xml
 
 import (
 	"bufio"
 	"fmt"
 	"testing"
 
+	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"github.com/mikefarah/yq/v4/test"
 )
 
@@ -262,7 +263,7 @@ const expectedXmlWithProcInstAndDirectives = `<?xml version="1.0"?>
 </apple>
 `
 
-var xmlScenarios = []formatScenario{
+var xmlScenarios = []yqlib.FormatScenario{
 	{
 		skipDoc:     true,
 		description: "bad xml",
@@ -614,39 +615,41 @@ var xmlScenarios = []formatScenario{
 	},
 }
 
-func testXMLScenario(t *testing.T, s formatScenario) {
+func testXMLScenario(t *testing.T, s yqlib.FormatScenario) {
 	switch s.scenarioType {
 	case "", "decode":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(4, false, ConfiguredYamlPreferences)), s.description)
+		yamlPrefs := yqlib.ConfiguredYamlPreferences.Copy()
+		yamlPrefs.Indent = 4
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(yamlPrefs)), s.description)
 	case "encode":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(ConfiguredXMLPreferences)), s.description)
 	case "roundtrip":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(ConfiguredXMLPreferences)), s.description)
 	case "decode-keep-ns":
 		prefs := NewDefaultXmlPreferences()
 		prefs.KeepNamespace = true
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
 	case "decode-raw-token":
 		prefs := NewDefaultXmlPreferences()
 		prefs.UseRawToken = true
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
 	case "decode-raw-token-off":
 		prefs := NewDefaultXmlPreferences()
 		prefs.UseRawToken = false
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
 	case "roundtrip-skip-directives":
 		prefs := NewDefaultXmlPreferences()
 		prefs.SkipDirectives = true
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(prefs)), s.description)
 	case "decode-error":
-		result, err := processFormatScenario(s, NewXMLDecoder(NewDefaultXmlPreferences()), NewYamlEncoder(2, false, ConfiguredYamlPreferences))
+		result, err := processFormatScenario(s, NewXMLDecoder(NewDefaultXmlPreferences()), NewYamlEncoder(ConfiguredYamlPreferences))
 		if err == nil {
 			t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result)
 		} else {
 			test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description)
 		}
 	case "encode-error":
-		result, err := processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, NewDefaultXmlPreferences()))
+		result, err := processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(NewDefaultXmlPreferences()))
 		if err == nil {
 			t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result)
 		} else {
@@ -659,7 +662,7 @@ func testXMLScenario(t *testing.T, s formatScenario) {
 }
 
 func documentXMLScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
-	s := i.(formatScenario)
+	s := i.(FormatScenario)
 
 	if s.skipDoc {
 		return
@@ -683,7 +686,7 @@ func documentXMLScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
 	}
 }
 
-func documentXMLDecodeScenario(w *bufio.Writer, s formatScenario) {
+func documentXMLDecodeScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -702,10 +705,10 @@ func documentXMLDecodeScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, fmt.Sprintf("```bash\nyq -oy '%v' sample.xml\n```\n", expression))
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences))))
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(ConfiguredYamlPreferences))))
 }
 
-func documentXMLDecodeKeepNsScenario(w *bufio.Writer, s formatScenario) {
+func documentXMLDecodeKeepNsScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -721,15 +724,15 @@ func documentXMLDecodeKeepNsScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, "will output\n")
 	prefs := NewDefaultXmlPreferences()
 	prefs.KeepNamespace = false
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs))))
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(prefs))))
 
 	prefsWithout := NewDefaultXmlPreferences()
 	prefs.KeepNamespace = true
 	writeOrPanic(w, "instead of\n")
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(2, prefsWithout))))
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(prefsWithout))))
 }
 
-func documentXMLDecodeKeepNsRawTokenScenario(w *bufio.Writer, s formatScenario) {
+func documentXMLDecodeKeepNsRawTokenScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -747,16 +750,16 @@ func documentXMLDecodeKeepNsRawTokenScenario(w *bufio.Writer, s formatScenario)
 	prefs := NewDefaultXmlPreferences()
 	prefs.UseRawToken = false
 
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs))))
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(prefs))))
 
 	prefsWithout := NewDefaultXmlPreferences()
 	prefsWithout.UseRawToken = true
 
 	writeOrPanic(w, "instead of\n")
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(2, prefsWithout))))
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(prefsWithout))))
 }
 
-func documentXMLEncodeScenario(w *bufio.Writer, s formatScenario) {
+func documentXMLEncodeScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -771,10 +774,10 @@ func documentXMLEncodeScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, "```bash\nyq -o=xml sample.yml\n```\n")
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences))))
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(ConfiguredXMLPreferences))))
 }
 
-func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) {
+func documentXMLRoundTripScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -789,10 +792,10 @@ func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, "```bash\nyq '.' sample.xml\n```\n")
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences))))
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(ConfiguredXMLPreferences))))
 }
 
-func documentXMLSkipDirectivesScenario(w *bufio.Writer, s formatScenario) {
+func documentXMLSkipDirectivesScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -809,7 +812,7 @@ func documentXMLSkipDirectivesScenario(w *bufio.Writer, s formatScenario) {
 	prefs := NewDefaultXmlPreferences()
 	prefs.SkipDirectives = true
 
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs))))
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(prefs))))
 }
 
 func TestXMLScenarios(t *testing.T) {

pkg/yqlib/all_at_once_evaluator.go

@@ -63,7 +63,7 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
 	}
 
 	if allDocuments.Len() == 0 {
-		candidateNode := createScalarNode(nil, "")
+		candidateNode := CreateScalarNode(nil, "")
 		allDocuments.PushBack(candidateNode)
 	}
 

pkg/yqlib/all_at_once_evaluator_test.go

@@ -36,7 +36,7 @@ func TestAllAtOnceEvaluateNodes(t *testing.T) {
 	var evaluator = NewAllAtOnceEvaluator()
 	// logging.SetLevel(logging.DEBUG, "")
 	for _, tt := range evaluateNodesScenario {
-		decoder := NewYamlDecoder(NewDefaultYamlPreferences())
+		decoder := NewYamlDecoder(ConfiguredYamlPreferences)
 		reader := bufio.NewReader(strings.NewReader(tt.document))
 		err := decoder.Init(reader)
 		if err != nil {

pkg/yqlib/candidate_node.go

@@ -27,14 +27,14 @@ const (
 	FlowStyle
 )
 
-func createStringScalarNode(stringValue string) *CandidateNode {
+func CreateStringScalarNode(stringValue string) *CandidateNode {
 	var node = &CandidateNode{Kind: ScalarNode}
 	node.Value = stringValue
 	node.Tag = "!!str"
 	return node
 }
 
-func createScalarNode(value interface{}, stringValue string) *CandidateNode {
+func CreateScalarNode(value interface{}, stringValue string) *CandidateNode {
 	var node = &CandidateNode{Kind: ScalarNode}
 	node.Value = stringValue
 
@@ -219,7 +219,7 @@ func (n *CandidateNode) AddChild(rawChild *CandidateNode) {
 		value.Key.SetParent(n)
 	} else {
 		index := len(n.Content)
-		keyNode := createScalarNode(index, fmt.Sprintf("%v", index))
+		keyNode := CreateScalarNode(index, fmt.Sprintf("%v", index))
 		keyNode.SetParent(n)
 		value.Key = keyNode
 	}
@@ -243,7 +243,7 @@ func (n *CandidateNode) AddChildren(children []*CandidateNode) {
 
 func (n *CandidateNode) GetValueRep() (interface{}, error) {
 	log.Debugf("GetValueRep for %v value: %v", n.GetNicePath(), n.Value)
-	realTag := n.guessTagFromCustomType()
+	realTag := n.GuessTagFromCustomType()
 
 	switch realTag {
 	case "!!int":
@@ -261,17 +261,17 @@ func (n *CandidateNode) GetValueRep() (interface{}, error) {
 	return n.Value, nil
 }
 
-func (n *CandidateNode) guessTagFromCustomType() string {
+func (n *CandidateNode) GuessTagFromCustomType() string {
 	if strings.HasPrefix(n.Tag, "!!") {
 		return n.Tag
 	} else if n.Value == "" {
-		log.Debug("guessTagFromCustomType: node has no value to guess the type with")
+		log.Debug("GuessTagFromCustomType: node has no value to guess the type with")
 		return n.Tag
 	}
 	dataBucket, errorReading := parseSnippet(n.Value)
 
 	if errorReading != nil {
-		log.Debug("guessTagFromCustomType: could not guess underlying tag type %v", errorReading)
+		log.Debug("GuessTagFromCustomType: could not guess underlying tag type %v", errorReading)
 		return n.Tag
 	}
 	guessedTag := dataBucket.Tag
@@ -376,7 +376,7 @@ func (n *CandidateNode) UpdateFrom(other *CandidateNode, prefs assignPreferences
 	// if this is an empty map or empty array, use the style of other node.
 	if (n.Kind != ScalarNode && len(n.Content) == 0) ||
 		// if the tag has changed (e.g. from str to bool)
-		(n.guessTagFromCustomType() != other.guessTagFromCustomType()) {
+		(n.GuessTagFromCustomType() != other.GuessTagFromCustomType()) {
 		n.Style = other.Style
 	}
 
@@ -428,3 +428,34 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignP
 		n.LineComment = other.LineComment
 	}
 }
+
+func (n *CandidateNode) CleanHeadAndLineComment() string {
+	return n.CleanHeadComment() + n.CleanLineComment()
+}
+
+func (n *CandidateNode) CleanHeadComment() string {
+	return strings.Replace(n.HeadComment, "#", "", 1)
+}
+
+func (n *CandidateNode) CleanLineComment() string {
+	return strings.Replace(n.LineComment, "#", "", 1)
+}
+
+func (n *CandidateNode) CleanFootComment() string {
+	return strings.Replace(n.FootComment, "#", "", 1)
+}
+
+func (n *CandidateNode) ConvertKeysToStrings() {
+
+	if n.Kind == MappingNode {
+		for index, child := range n.Content {
+			if index%2 == 0 { // its a map key
+				child.Tag = "!!str"
+			}
+		}
+	}
+
+	for _, child := range n.Content {
+		child.ConvertKeysToStrings()
+	}
+}

pkg/yqlib/candidate_node_test.go

@@ -64,13 +64,13 @@ func TestCandidateNodeChildWhenParentUpdated(t *testing.T) {
 	test.AssertResultWithContext(t, uint(1), child.GetDocument(), "document index")
 }
 
-type createScalarNodeScenario struct {
+type CreateScalarNodeScenario struct {
 	value       interface{}
 	stringValue string
 	expectedTag string
 }
 
-var createScalarScenarios = []createScalarNodeScenario{
+var createScalarScenarios = []CreateScalarNodeScenario{
 	{
 		value:       "mike",
 		stringValue: "mike",
@@ -100,20 +100,20 @@ var createScalarScenarios = []createScalarNodeScenario{
 
 func TestCreateScalarNodeScenarios(t *testing.T) {
 	for _, tt := range createScalarScenarios {
-		actual := createScalarNode(tt.value, tt.stringValue)
+		actual := CreateScalarNode(tt.value, tt.stringValue)
 		test.AssertResultWithContext(t, tt.stringValue, actual.Value, fmt.Sprintf("Value for: Value: [%v], String: %v", tt.value, tt.stringValue))
 		test.AssertResultWithContext(t, tt.expectedTag, actual.Tag, fmt.Sprintf("Value for: Value: [%v], String: %v", tt.value, tt.stringValue))
 	}
 }
 
 func TestGetKeyForMapValue(t *testing.T) {
-	key := createStringScalarNode("yourKey")
+	key := CreateStringScalarNode("yourKey")
 	n := CandidateNode{Key: key, Value: "meow", document: 3}
 	test.AssertResult(t, "3 - yourKey", n.GetKey())
 }
 
 func TestGetKeyForMapKey(t *testing.T) {
-	key := createStringScalarNode("yourKey")
+	key := CreateStringScalarNode("yourKey")
 	key.IsMapKey = true
 	key.document = 3
 	test.AssertResult(t, "key-yourKey-3 - ", key.GetKey())
@@ -125,7 +125,7 @@ func TestGetKeyForValue(t *testing.T) {
 }
 
 func TestGetParsedKeyForMapKey(t *testing.T) {
-	key := createStringScalarNode("yourKey")
+	key := CreateStringScalarNode("yourKey")
 	key.IsMapKey = true
 	key.document = 3
 	test.AssertResult(t, "yourKey", key.getParsedKey())
@@ -137,13 +137,13 @@ func TestGetParsedKeyForLooseValue(t *testing.T) {
 }
 
 func TestGetParsedKeyForMapValue(t *testing.T) {
-	key := createStringScalarNode("yourKey")
+	key := CreateStringScalarNode("yourKey")
 	n := CandidateNode{Key: key, Value: "meow", document: 3}
 	test.AssertResult(t, "yourKey", n.getParsedKey())
 }
 
 func TestGetParsedKeyForArrayValue(t *testing.T) {
-	key := createScalarNode(4, "4")
+	key := CreateScalarNode(4, "4")
 	n := CandidateNode{Key: key, Value: "meow", document: 3}
 	test.AssertResult(t, 4, n.getParsedKey())
 }

pkg/yqlib/candidiate_node_json.go

@@ -96,7 +96,7 @@ func (o *CandidateNode) UnmarshalJSON(data []byte) error {
 
 			if child == nil {
 				// need to represent it as a null scalar
-				child = createScalarNode(nil, "null")
+				child = CreateScalarNode(nil, "null")
 			}
 			childKey := o.CreateChild()
 			childKey.Kind = ScalarNode

pkg/yqlib/csv_test.go

@@ -94,7 +94,7 @@ const tsvTestExpectedSimpleCsv = `i	like	csv
 because	excel	is	cool
 `
 
-var csvScenarios = []formatScenario{
+var csvScenarios = []FormatScenario{
 	{
 		description:  "Encode CSV simple",
 		input:        csvTestSimpleYaml,
@@ -172,6 +172,13 @@ var csvScenarios = []formatScenario{
 		expected:       expectedYamlFromCSVWithObject,
 		scenarioType:   "decode-csv",
 	},
+	{
+		description:  "Decode CSV line breaks",
+		skipDoc:      true,
+		input:        "heading1\n\"some data\nwith a line break\"\n",
+		expected:     "- heading1: |-\n    some data\n    with a line break\n",
+		scenarioType: "decode-csv",
+	},
 	{
 		description:    "Parse CSV into an array of objects, no auto-parsing",
 		subdescription: "First row is assumed to be the header row. Entries with YAML/JSON will be left as strings.",
@@ -203,26 +210,26 @@ var csvScenarios = []formatScenario{
 	},
 }
 
-func testCSVScenario(t *testing.T, s formatScenario) {
+func testCSVScenario(t *testing.T, s FormatScenario) {
 	switch s.scenarioType {
 	case "encode-csv":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(',')), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(ConfiguredCsvPreferences)), s.description)
 	case "encode-tsv":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder('\t')), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(ConfiguredTsvPreferences)), s.description)
 	case "decode-csv":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredCsvPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredCsvPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
 	case "decode-csv-no-auto":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: ',', AutoParse: false}), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: ',', AutoParse: false}), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
 	case "decode-tsv-object":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredTsvPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredTsvPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
 	case "roundtrip-csv":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredCsvPreferences), NewCsvEncoder(',')), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredCsvPreferences), NewCsvEncoder(ConfiguredCsvPreferences)), s.description)
 	default:
 		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
 	}
 }
 
-func documentCSVDecodeObjectScenario(w *bufio.Writer, s formatScenario, formatType string) {
+func documentCSVDecodeObjectScenario(w *bufio.Writer, s FormatScenario, formatType string) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -243,11 +250,11 @@ func documentCSVDecodeObjectScenario(w *bufio.Writer, s formatScenario, formatTy
 	}
 
 	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n",
-		mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: true}), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))),
+		mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: true}), NewYamlEncoder(ConfiguredYamlPreferences))),
 	)
 }
 
-func documentCSVDecodeObjectNoAutoScenario(w *bufio.Writer, s formatScenario, formatType string) {
+func documentCSVDecodeObjectNoAutoScenario(w *bufio.Writer, s FormatScenario, formatType string) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -268,11 +275,11 @@ func documentCSVDecodeObjectNoAutoScenario(w *bufio.Writer, s formatScenario, fo
 	}
 
 	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n",
-		mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: false}), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))),
+		mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: false}), NewYamlEncoder(ConfiguredYamlPreferences))),
 	)
 }
 
-func documentCSVEncodeScenario(w *bufio.Writer, s formatScenario, formatType string) {
+func documentCSVEncodeScenario(w *bufio.Writer, s FormatScenario, formatType string) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -298,13 +305,14 @@ func documentCSVEncodeScenario(w *bufio.Writer, s formatScenario, formatType str
 	if formatType == "tsv" {
 		separator = '\t'
 	}
-
+	csvPrefs := NewDefaultCsvPreferences()
+	csvPrefs.Separator = separator
 	writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
-		mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(separator))),
+		mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(csvPrefs))),
 	)
 }
 
-func documentCSVRoundTripScenario(w *bufio.Writer, s formatScenario, formatType string) {
+func documentCSVRoundTripScenario(w *bufio.Writer, s FormatScenario, formatType string) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -331,13 +339,16 @@ func documentCSVRoundTripScenario(w *bufio.Writer, s formatScenario, formatType
 		separator = '\t'
 	}
 
+	csvPrefs := NewDefaultCsvPreferences()
+	csvPrefs.Separator = separator
+
 	writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
-		mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: true}), NewCsvEncoder(separator))),
+		mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: true}), NewCsvEncoder(csvPrefs))),
 	)
 }
 
 func documentCSVScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
-	s := i.(formatScenario)
+	s := i.(FormatScenario)
 	if s.skipDoc {
 		return
 	}

pkg/yqlib/data_tree_navigator.go

@@ -13,6 +13,7 @@ type DataTreeNavigator interface {
 	GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error)
 
 	DeeplyAssign(context Context, path []interface{}, rhsNode *CandidateNode) error
+	DeeplyAssignKey(context Context, path []interface{}, rhsNode *CandidateNode) error
 }
 
 type dataTreeNavigator struct {
@@ -23,6 +24,14 @@ func NewDataTreeNavigator() DataTreeNavigator {
 }
 
 func (d *dataTreeNavigator) DeeplyAssign(context Context, path []interface{}, rhsCandidateNode *CandidateNode) error {
+	return d.deeplyAssign(context, path, rhsCandidateNode, false)
+}
+
+func (d *dataTreeNavigator) DeeplyAssignKey(context Context, path []interface{}, rhsCandidateNode *CandidateNode) error {
+	return d.deeplyAssign(context, path, rhsCandidateNode, true)
+}
+
+func (d *dataTreeNavigator) deeplyAssign(context Context, path []interface{}, rhsCandidateNode *CandidateNode, targetKey bool) error {
 
 	assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
 
@@ -41,7 +50,7 @@ func (d *dataTreeNavigator) DeeplyAssign(context Context, path []interface{}, rh
 
 	assignmentOpNode := &ExpressionNode{
 		Operation: assignmentOp,
-		LHS:       createTraversalTree(path, traversePreferences{}, false),
+		LHS:       createTraversalTree(path, traversePreferences{}, targetKey),
 		RHS:       &ExpressionNode{Operation: rhsOp},
 	}
 

pkg/yqlib/decoder.go

@@ -1,66 +1,10 @@
 package yqlib
 
 import (
-	"fmt"
 	"io"
-	"strings"
-)
-
-type InputFormat uint
-
-const (
-	YamlInputFormat = 1 << iota
-	XMLInputFormat
-	PropertiesInputFormat
-	Base64InputFormat
-	JsonInputFormat
-	CSVObjectInputFormat
-	TSVObjectInputFormat
-	TomlInputFormat
-	UriInputFormat
-	LuaInputFormat
 )
 
 type Decoder interface {
 	Init(reader io.Reader) error
 	Decode() (*CandidateNode, error)
 }
-
-func InputFormatFromString(format string) (InputFormat, error) {
-	switch format {
-	case "yaml", "yml", "y":
-		return YamlInputFormat, nil
-	case "xml", "x":
-		return XMLInputFormat, nil
-	case "properties", "props", "p":
-		return PropertiesInputFormat, nil
-	case "json", "ndjson", "j":
-		return JsonInputFormat, nil
-	case "csv", "c":
-		return CSVObjectInputFormat, nil
-	case "tsv", "t":
-		return TSVObjectInputFormat, nil
-	case "toml":
-		return TomlInputFormat, nil
-	case "lua", "l":
-		return LuaInputFormat, nil
-	default:
-		return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml|toml]", format)
-	}
-}
-
-func FormatFromFilename(filename string) string {
-
-	if filename != "" {
-		GetLogger().Debugf("checking file extension '%s' for auto format detection", filename)
-		nPos := strings.LastIndex(filename, ".")
-		if nPos > -1 {
-			format := filename[nPos+1:]
-			GetLogger().Debugf("detected format '%s'", format)
-			return format
-		}
-	}
-
-	GetLogger().Debugf("using default inputFormat 'yaml'")
-	return "yaml"
-}

pkg/yqlib/decoder_base64.go

@@ -68,5 +68,5 @@ func (dec *base64Decoder) Decode() (*CandidateNode, error) {
 		}
 	}
 	dec.readAnything = true
-	return createStringScalarNode(buf.String()), nil
+	return CreateStringScalarNode(buf.String()), nil
 }

pkg/yqlib/decoder_csv_object.go

@@ -32,7 +32,7 @@ func (dec *csvObjectDecoder) convertToNode(content string) *CandidateNode {
 	// if we're not auto-parsing, then we wont put in parsed objects or arrays
 	// but we still parse scalars
 	if err != nil || (!dec.prefs.AutoParse && node.Kind != ScalarNode) {
-		return createScalarNode(content, content)
+		return CreateScalarNode(content, content)
 	}
 	return node
 }
@@ -41,7 +41,7 @@ func (dec *csvObjectDecoder) createObject(headerRow []string, contentRow []strin
 	objectNode := &CandidateNode{Kind: MappingNode, Tag: "!!map"}
 
 	for i, header := range headerRow {
-		objectNode.AddKeyValueChild(createScalarNode(header, header), dec.convertToNode(contentRow[i]))
+		objectNode.AddKeyValueChild(CreateScalarNode(header, header), dec.convertToNode(contentRow[i]))
 	}
 	return objectNode
 }

pkg/yqlib/decoder_goccy_yaml.go

@@ -1,5 +1,9 @@
 //go:build !yq_noyaml
 
+//
+// NOTE this is still a WIP - not yet ready.
+//
+
 package yqlib
 
 import (

pkg/yqlib/decoder_test.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 )
 
-type formatScenario struct {
+type FormatScenario struct {
 	input          string
 	indent         int
 	expression     string
@@ -19,7 +19,7 @@ type formatScenario struct {
 	expectedError  string
 }
 
-func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) (string, error) {
+func processFormatScenario(s FormatScenario, decoder Decoder, encoder Encoder) (string, error) {
 	var output bytes.Buffer
 	writer := bufio.NewWriter(&output)
 
@@ -64,7 +64,7 @@ func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) (
 	return output.String(), nil
 }
 
-func mustProcessFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) string {
+func mustProcessFormatScenario(s FormatScenario, decoder Decoder, encoder Encoder) string {
 
 	result, err := processFormatScenario(s, decoder, encoder)
 	if err != nil {

pkg/yqlib/decoder_toml.go

@@ -145,30 +145,30 @@ func (dec *tomlDecoder) createArray(tomlNode *toml.Node) (*CandidateNode, error)
 
 func (dec *tomlDecoder) createStringScalar(tomlNode *toml.Node) (*CandidateNode, error) {
 	content := string(tomlNode.Data)
-	return createScalarNode(content, content), nil
+	return CreateScalarNode(content, content), nil
 }
 
 func (dec *tomlDecoder) createBoolScalar(tomlNode *toml.Node) (*CandidateNode, error) {
 	content := string(tomlNode.Data)
-	return createScalarNode(content == "true", content), nil
+	return CreateScalarNode(content == "true", content), nil
 }
 
 func (dec *tomlDecoder) createIntegerScalar(tomlNode *toml.Node) (*CandidateNode, error) {
 	content := string(tomlNode.Data)
 	_, num, err := parseInt64(content)
-	return createScalarNode(num, content), err
+	return CreateScalarNode(num, content), err
 }
 
 func (dec *tomlDecoder) createDateTimeScalar(tomlNode *toml.Node) (*CandidateNode, error) {
 	content := string(tomlNode.Data)
 	val, err := parseDateTime(time.RFC3339, content)
-	return createScalarNode(val, content), err
+	return CreateScalarNode(val, content), err
 }
 
 func (dec *tomlDecoder) createFloatScalar(tomlNode *toml.Node) (*CandidateNode, error) {
 	content := string(tomlNode.Data)
 	num, err := strconv.ParseFloat(content, 64)
-	return createScalarNode(num, content), err
+	return CreateScalarNode(num, content), err
 }
 
 func (dec *tomlDecoder) decodeNode(tomlNode *toml.Node) (*CandidateNode, error) {

pkg/yqlib/decoder_uri.go

@@ -48,5 +48,5 @@ func (dec *uriDecoder) Decode() (*CandidateNode, error) {
 		return nil, err
 	}
 	dec.readAnything = true
-	return createStringScalarNode(newValue), nil
+	return CreateStringScalarNode(newValue), nil
 }

pkg/yqlib/decoder_yaml.go

@@ -151,7 +151,7 @@ func (dec *yamlDecoder) Decode() (*CandidateNode, error) {
 }
 
 func (dec *yamlDecoder) blankNodeWithComment() *CandidateNode {
-	node := createScalarNode(nil, "")
+	node := CreateScalarNode(nil, "")
 	node.LeadingContent = dec.leadingContent
 	return node
 }

pkg/yqlib/doc/operators/column.md

@@ -2,6 +2,8 @@
 
 Returns the column of the matching node. Starts from 1, 0 indicates there was no column data.
 
+Column is the number of characters that precede that node on the line it starts.
+
 ## Returns column of _value_ node
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/operators/headers/column.md

@@ -1,3 +1,5 @@
 # Column
 
 Returns the column of the matching node. Starts from 1, 0 indicates there was no column data.
+
+Column is the number of characters that precede that node on the line it starts.

pkg/yqlib/doc/operators/parent.md

@@ -37,6 +37,43 @@ fruit: banana
 name: sam
 ```
 
+## N-th parent
+You can optionally supply the number of levels to go up for the parent, the default being 1.
+
+Given a sample.yml file of:
+```yaml
+a:
+  b:
+    c: cat
+```
+then
+```bash
+yq '.a.b.c | parent(2)' sample.yml
+```
+will output
+```yaml
+b:
+  c: cat
+```
+
+## N-th parent - another level
+Given a sample.yml file of:
+```yaml
+a:
+  b:
+    c: cat
+```
+then
+```bash
+yq '.a.b.c | parent(3)' sample.yml
+```
+will output
+```yaml
+a:
+  b:
+    c: cat
+```
+
 ## No parent
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/operators/string-operators.md

@@ -56,6 +56,40 @@ IFS= read -rd '' output < <(cat my_file)
 output=$output ./yq '.data.values = strenv(output)' first.yml
 ```
 
+## Interpolation
+Given a sample.yml file of:
+```yaml
+value: things
+another: stuff
+```
+then
+```bash
+yq '.message = "I like \(.value) and \(.another)"' sample.yml
+```
+will output
+```yaml
+value: things
+another: stuff
+message: I like things and stuff
+```
+
+## Interpolation - not a string
+Given a sample.yml file of:
+```yaml
+value:
+  an: apple
+```
+then
+```bash
+yq '.message = "I like \(.value)"' sample.yml
+```
+will output
+```yaml
+value:
+  an: apple
+message: 'I like an: apple'
+```
+
 ## To up (upper) case
 Works with unicode characters
 
@@ -383,3 +417,32 @@ will output
 - word
 ```
 
+## To string
+Note that you may want to force `yq` to leave scalar values wrapped by passing in `--unwrapScalar=false` or `-r=f`
+
+Given a sample.yml file of:
+```yaml
+- 1
+- true
+- null
+- ~
+- cat
+- an: object
+- - array
+  - 2
+```
+then
+```bash
+yq '.[] |= to_string' sample.yml
+```
+will output
+```yaml
+- "1"
+- "true"
+- "null"
+- "~"
+- cat
+- "an: object"
+- "- array\n- 2"
+```
+

pkg/yqlib/doc/usage/convert.md

@@ -36,10 +36,10 @@ will output
 ```yaml
 a: Easy! as one two three
 b:
-    c: 2
-    d:
-        - 3
-        - 4
+  c: 2
+  d:
+    - 3
+    - 4
 ```
 
 ## Encode json: simple
@@ -244,7 +244,7 @@ will output
 this: is a multidoc json file
 ---
 each:
-    - line is a valid json document
+  - line is a valid json document
 ---
 a number: 4
 ```

pkg/yqlib/doc/usage/formatting-expressions.md

@@ -5,6 +5,8 @@
 You can put expressions into `.yq` files, use whitespace and comments to break up complex expressions and explain what's going on.
 
 ## Using expression files and comments
+Note that you can execute the file directly - but make sure you make the expression file executable.
+
 Given a sample.yaml file of:
 ```yaml
 a:
@@ -12,6 +14,8 @@ a:
 ```
 And an 'update.yq' expression file of:
 ```bash
+#! yq
+
 # This is a yq expression that updates the map
 # for several great reasons outlined here.
 
@@ -22,7 +26,7 @@ And an 'update.yq' expression file of:
 ```
 then
 ```bash
-yq --from-file update.yq sample.yml
+./update.yq sample.yaml
 ```
 will output
 ```yaml
@@ -31,8 +35,8 @@ a:
   c: frog
 ```
 
-## Commenting out yq expressions
-Note that `c` is no longer set to 'frog'.
+## Flags in expression files
+You can specify flags on the shebang line, this only works when executing the file directly.
 
 Given a sample.yaml file of:
 ```yaml
@@ -41,6 +45,41 @@ a:
 ```
 And an 'update.yq' expression file of:
 ```bash
+#! yq -oj
+
+# This is a yq expression that updates the map
+# for several great reasons outlined here.
+
+.a.b = "new" # line comment here
+| .a.c = "frog"
+
+# Now good things will happen.
+```
+then
+```bash
+./update.yq sample.yaml
+```
+will output
+```yaml
+{
+  "a": {
+    "b": "new",
+    "c": "frog"
+  }
+}
+```
+
+## Commenting out yq expressions
+Note that `c` is no longer set to 'frog'. In this example we're calling yq directly and passing the expression file into `--from-file`, this is no different from executing the expression file directly.
+
+Given a sample.yaml file of:
+```yaml
+a:
+  b: old
+```
+And an 'update.yq' expression file of:
+```bash
+#! yq
 # This is a yq expression that updates the map
 # for several great reasons outlined here.
 

pkg/yqlib/doc/usage/properties.md

@@ -14,6 +14,8 @@ person: # neither do comments on maps
     name: Mike Wazowski # comments on values appear
     pets: 
     - cat # comments on array values appear
+    - nested:
+        - list entry
     food: [pizza] # comments on arrays do not
 emptyArray: []
 emptyMap: []
@@ -31,9 +33,76 @@ person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = cat
+person.pets.1.nested.0 = list entry
 person.food.0 = pizza
 ```
 
+## Encode properties with array brackets
+Declare the --properties-array-brackets flag to give array paths in brackets (e.g. SpringBoot).
+
+Given a sample.yml file of:
+```yaml
+# block comments come through
+person: # neither do comments on maps
+    name: Mike Wazowski # comments on values appear
+    pets: 
+    - cat # comments on array values appear
+    - nested:
+        - list entry
+    food: [pizza] # comments on arrays do not
+emptyArray: []
+emptyMap: []
+
+```
+then
+```bash
+yq -o=props --properties-array-brackets sample.yml
+```
+will output
+```properties
+# block comments come through
+# comments on values appear
+person.name = Mike Wazowski
+
+# comments on array values appear
+person.pets[0] = cat
+person.pets[1].nested[0] = list entry
+person.food[0] = pizza
+```
+
+## Encode properties - custom separator
+Use the --properties-customer-separator flag to specify your own key/value separator.
+
+Given a sample.yml file of:
+```yaml
+# block comments come through
+person: # neither do comments on maps
+    name: Mike Wazowski # comments on values appear
+    pets: 
+    - cat # comments on array values appear
+    - nested:
+        - list entry
+    food: [pizza] # comments on arrays do not
+emptyArray: []
+emptyMap: []
+
+```
+then
+```bash
+yq -o=props --properties-customer-separator=" :@ " sample.yml
+```
+will output
+```properties
+# block comments come through
+# comments on values appear
+person.name :@ Mike Wazowski
+
+# comments on array values appear
+person.pets.0 :@ cat
+person.pets.1.nested.0 :@ list entry
+person.food.0 :@ pizza
+```
+
 ## Encode properties: scalar encapsulation
 Note that string values with blank characters in them are encapsulated with double quotes
 
@@ -44,6 +113,8 @@ person: # neither do comments on maps
     name: Mike Wazowski # comments on values appear
     pets: 
     - cat # comments on array values appear
+    - nested:
+        - list entry
     food: [pizza] # comments on arrays do not
 emptyArray: []
 emptyMap: []
@@ -61,6 +132,7 @@ person.name = "Mike Wazowski"
 
 # comments on array values appear
 person.pets.0 = cat
+person.pets.1.nested.0 = "list entry"
 person.food.0 = pizza
 ```
 
@@ -72,6 +144,8 @@ person: # neither do comments on maps
     name: Mike Wazowski # comments on values appear
     pets: 
     - cat # comments on array values appear
+    - nested:
+        - list entry
     food: [pizza] # comments on arrays do not
 emptyArray: []
 emptyMap: []
@@ -85,6 +159,7 @@ will output
 ```properties
 person.name = Mike Wazowski
 person.pets.0 = cat
+person.pets.1.nested.0 = list entry
 person.food.0 = pizza
 ```
 
@@ -98,6 +173,8 @@ person: # neither do comments on maps
     name: Mike Wazowski # comments on values appear
     pets: 
     - cat # comments on array values appear
+    - nested:
+        - list entry
     food: [pizza] # comments on arrays do not
 emptyArray: []
 emptyMap: []
@@ -115,6 +192,7 @@ person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = cat
+person.pets.1.nested.0 = list entry
 person.food.0 = pizza
 emptyArray = 
 emptyMap = 
@@ -129,6 +207,7 @@ person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = cat
+person.pets.1.nested.0 = list entry
 person.food.0 = pizza
 
 ```
@@ -139,14 +218,16 @@ yq -p=props sample.properties
 will output
 ```yaml
 person:
-    # block comments come through
-    # comments on values appear
-    name: Mike Wazowski
-    pets:
-        # comments on array values appear
-        - cat
-    food:
-        - pizza
+  # block comments come through
+  # comments on values appear
+  name: Mike Wazowski
+  pets:
+    # comments on array values appear
+    - cat
+    - nested:
+        - list entry
+  food:
+    - pizza
 ```
 
 ## Decode properties - array should be a map
@@ -163,7 +244,7 @@ yq -p=props '.things |= array_to_map' sample.properties
 will output
 ```yaml
 things:
-    10: mike
+  10: mike
 ```
 
 ## Roundtrip
@@ -175,6 +256,7 @@ person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = cat
+person.pets.1.nested.0 = list entry
 person.food.0 = pizza
 
 ```
@@ -190,6 +272,7 @@ person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = dog
+person.pets.1.nested.0 = list entry
 person.food.0 = pizza
 ```
 

pkg/yqlib/doc/usage/recipes.md

@@ -59,6 +59,44 @@ will output
 - The expression `. + 1` increments the numBuckets counter.
 - See the [assign](https://mikefarah.gitbook.io/yq/operators/assign-update) and [add](https://mikefarah.gitbook.io/yq/operators/add) operators for more information.
 
+## Deeply prune a tree
+Say we are only interested in child1 and child2, and want to filter everything else out.
+
+Given a sample.yml file of:
+```yaml
+parentA:
+  - bob
+parentB:
+  child1: i am child1
+  child3: hiya
+parentC:
+  childX: cool
+  child2: me child2
+```
+then
+```bash
+yq '(
+  .. | # recurse through all the nodes
+  select(has("child1") or has("child2")) | # match parents that have either child1 or child2
+  (.child1, .child2) | # select those children
+  select(.) # filter out nulls
+) as $i ireduce({};  # using that set of nodes, create a new result map
+  setpath($i | path; $i) # and put in each node, using its original path
+)' sample.yml
+```
+will output
+```yaml
+parentB:
+  child1: i am child1
+parentC:
+  child2: me child2
+```
+
+### Explanation:
+- Find all the matching child1 and child2 nodes
+- Using ireduce, create a new map using just those nodes
+- Set each node into the new map using its original path
+
 ## Multiple or complex updates to items in an array
 We have an array and we want to _update_ the elements with a particular name in reference to its type.
 

pkg/yqlib/encoder.go

@@ -10,18 +10,3 @@ type Encoder interface {
 	PrintLeadingContent(writer io.Writer, content string) error
 	CanHandleAliases() bool
 }
-
-func mapKeysToStrings(node *CandidateNode) {
-
-	if node.Kind == MappingNode {
-		for index, child := range node.Content {
-			if index%2 == 0 { // its a map key
-				child.Tag = "!!str"
-			}
-		}
-	}
-
-	for _, child := range node.Content {
-		mapKeysToStrings(child)
-	}
-}

pkg/yqlib/encoder_base64.go

@@ -27,8 +27,8 @@ func (e *base64Encoder) PrintLeadingContent(_ io.Writer, _ string) error {
 }
 
 func (e *base64Encoder) Encode(writer io.Writer, node *CandidateNode) error {
-	if node.guessTagFromCustomType() != "!!str" {
-		return fmt.Errorf("cannot encode %v as base64, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string", node.Tag)
+	if node.GuessTagFromCustomType() != "!!str" {
+		return fmt.Errorf("cannot encode %v as base64, can only operate on strings", node.Tag)
 	}
 	_, err := writer.Write([]byte(e.encoding.EncodeToString([]byte(node.Value))))
 	return err

pkg/yqlib/encoder_csv.go

@@ -10,8 +10,8 @@ type csvEncoder struct {
 	separator rune
 }
 
-func NewCsvEncoder(separator rune) Encoder {
-	return &csvEncoder{separator: separator}
+func NewCsvEncoder(prefs CsvPreferences) Encoder {
+	return &csvEncoder{separator: prefs.Separator}
 }
 
 func (e *csvEncoder) CanHandleAliases() bool {
@@ -65,7 +65,7 @@ func (e *csvEncoder) createChildRow(child *CandidateNode, headers []*CandidateNo
 	childRow := make([]*CandidateNode, 0)
 	for _, header := range headers {
 		keyIndex := findKeyInMap(child, header)
-		value := createScalarNode(nil, "")
+		value := CreateScalarNode(nil, "")
 		if keyIndex != -1 {
 			value = child.Content[keyIndex+1]
 		}
@@ -102,7 +102,7 @@ func (e *csvEncoder) encodeObjects(csvWriter *csv.Writer, content []*CandidateNo
 
 func (e *csvEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 	if node.Kind == ScalarNode {
-		return writeString(writer, node.Value+"\n")
+		return WriteString(writer, node.Value+"\n")
 	}
 
 	csvWriter := csv.NewWriter(writer)

pkg/yqlib/encoder_json.go

@@ -10,19 +10,18 @@ import (
 )
 
 type jsonEncoder struct {
+	prefs        JsonPreferences
 	indentString string
-	colorise     bool
-	UnwrapScalar bool
 }
 
-func NewJSONEncoder(indent int, colorise bool, unwrapScalar bool) Encoder {
+func NewJSONEncoder(prefs JsonPreferences) Encoder {
 	var indentString = ""
 
-	for index := 0; index < indent; index++ {
+	for index := 0; index < prefs.Indent; index++ {
 		indentString = indentString + " "
 	}
 
-	return &jsonEncoder{indentString, colorise, unwrapScalar}
+	return &jsonEncoder{prefs, indentString}
 }
 
 func (je *jsonEncoder) CanHandleAliases() bool {
@@ -41,13 +40,13 @@ func (je *jsonEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 	log.Debugf("I need to encode %v", NodeToString(node))
 	log.Debugf("kids %v", len(node.Content))
 
-	if node.Kind == ScalarNode && je.UnwrapScalar {
-		return writeString(writer, node.Value+"\n")
+	if node.Kind == ScalarNode && je.prefs.UnwrapScalar {
+		return WriteString(writer, node.Value+"\n")
 	}
 
 	destination := writer
 	tempBuffer := bytes.NewBuffer(nil)
-	if je.colorise {
+	if je.prefs.ColorsEnabled {
 		destination = tempBuffer
 	}
 
@@ -59,7 +58,7 @@ func (je *jsonEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 	if err != nil {
 		return err
 	}
-	if je.colorise {
+	if je.prefs.ColorsEnabled {
 		return colorizeAndPrint(tempBuffer.Bytes(), writer)
 	}
 	return nil

pkg/yqlib/encoder_lua.go

@@ -86,15 +86,15 @@ func (le *luaEncoder) encodeString(writer io.Writer, node *CandidateNode) error
 	case LiteralStyle, FoldedStyle, FlowStyle:
 		for i := 0; i < 10; i++ {
 			if !strings.Contains(node.Value, "]"+strings.Repeat("=", i)+"]") {
-				err := writeString(writer, "["+strings.Repeat("=", i)+"[\n")
+				err := WriteString(writer, "["+strings.Repeat("=", i)+"[\n")
 				if err != nil {
 					return err
 				}
-				err = writeString(writer, node.Value)
+				err = WriteString(writer, node.Value)
 				if err != nil {
 					return err
 				}
-				return writeString(writer, "]"+strings.Repeat("=", i)+"]")
+				return WriteString(writer, "]"+strings.Repeat("=", i)+"]")
 			}
 		}
 	case SingleQuotedStyle:
@@ -102,22 +102,22 @@ func (le *luaEncoder) encodeString(writer io.Writer, node *CandidateNode) error
 
 		// fallthrough to regular ol' string
 	}
-	return writeString(writer, quote+le.escape.Replace(node.Value)+quote)
+	return WriteString(writer, quote+le.escape.Replace(node.Value)+quote)
 }
 
 func (le *luaEncoder) writeIndent(writer io.Writer) error {
 	if le.indentStr == "" {
 		return nil
 	}
-	err := writeString(writer, "\n")
+	err := WriteString(writer, "\n")
 	if err != nil {
 		return err
 	}
-	return writeString(writer, strings.Repeat(le.indentStr, le.indent))
+	return WriteString(writer, strings.Repeat(le.indentStr, le.indent))
 }
 
 func (le *luaEncoder) encodeArray(writer io.Writer, node *CandidateNode) error {
-	err := writeString(writer, "{")
+	err := WriteString(writer, "{")
 	if err != nil {
 		return err
 	}
@@ -131,13 +131,13 @@ func (le *luaEncoder) encodeArray(writer io.Writer, node *CandidateNode) error {
 		if err != nil {
 			return err
 		}
-		err = writeString(writer, ",")
+		err = WriteString(writer, ",")
 		if err != nil {
 			return err
 		}
 		if child.LineComment != "" {
 			sansPrefix, _ := strings.CutPrefix(child.LineComment, "#")
-			err = writeString(writer, " --"+sansPrefix)
+			err = WriteString(writer, " --"+sansPrefix)
 			if err != nil {
 				return err
 			}
@@ -150,7 +150,7 @@ func (le *luaEncoder) encodeArray(writer io.Writer, node *CandidateNode) error {
 			return err
 		}
 	}
-	return writeString(writer, "}")
+	return WriteString(writer, "}")
 }
 
 func needsQuoting(s string) bool {
@@ -181,7 +181,7 @@ func needsQuoting(s string) bool {
 
 func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bool) error {
 	if !global {
-		err := writeString(writer, "{")
+		err := WriteString(writer, "{")
 		if err != nil {
 			return err
 		}
@@ -194,7 +194,7 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bo
 			if err != nil {
 				return err
 			}
-			err = writeString(writer, ";")
+			err = WriteString(writer, ";")
 			if err != nil {
 				return err
 			}
@@ -207,19 +207,19 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bo
 				}
 			}
 			if (le.unquoted || global) && child.Tag == "!!str" && !needsQuoting(child.Value) {
-				err := writeString(writer, child.Value+" = ")
+				err := WriteString(writer, child.Value+" = ")
 				if err != nil {
 					return err
 				}
 			} else {
 				if global {
 					// This only works in Lua 5.2+
-					err := writeString(writer, "_ENV")
+					err := WriteString(writer, "_ENV")
 					if err != nil {
 						return err
 					}
 				}
-				err := writeString(writer, "[")
+				err := WriteString(writer, "[")
 				if err != nil {
 					return err
 				}
@@ -227,7 +227,7 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bo
 				if err != nil {
 					return err
 				}
-				err = writeString(writer, "] = ")
+				err = WriteString(writer, "] = ")
 				if err != nil {
 					return err
 				}
@@ -235,7 +235,7 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bo
 		}
 		if child.LineComment != "" {
 			sansPrefix, _ := strings.CutPrefix(child.LineComment, "#")
-			err := writeString(writer, strings.Repeat(" ", i%2)+"--"+sansPrefix)
+			err := WriteString(writer, strings.Repeat(" ", i%2)+"--"+sansPrefix)
 			if err != nil {
 				return err
 			}
@@ -249,7 +249,7 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bo
 		}
 	}
 	if global {
-		return writeString(writer, "\n")
+		return WriteString(writer, "\n")
 	}
 	le.indent--
 	if len(node.Content) != 0 {
@@ -258,7 +258,7 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bo
 			return err
 		}
 	}
-	return writeString(writer, "}")
+	return WriteString(writer, "}")
 }
 
 func (le *luaEncoder) encodeAny(writer io.Writer, node *CandidateNode) error {
@@ -273,30 +273,30 @@ func (le *luaEncoder) encodeAny(writer io.Writer, node *CandidateNode) error {
 			return le.encodeString(writer, node)
 		case "!!null":
 			// TODO reject invalid use as a table key
-			return writeString(writer, "nil")
+			return WriteString(writer, "nil")
 		case "!!bool":
 			// Yaml 1.2 has case variation e.g. True, FALSE etc but Lua only has
 			// lower case
-			return writeString(writer, strings.ToLower(node.Value))
+			return WriteString(writer, strings.ToLower(node.Value))
 		case "!!int":
 			if strings.HasPrefix(node.Value, "0o") {
 				_, octalValue, err := parseInt64(node.Value)
 				if err != nil {
 					return err
 				}
-				return writeString(writer, fmt.Sprintf("%d", octalValue))
+				return WriteString(writer, fmt.Sprintf("%d", octalValue))
 			}
-			return writeString(writer, strings.ToLower(node.Value))
+			return WriteString(writer, strings.ToLower(node.Value))
 		case "!!float":
 			switch strings.ToLower(node.Value) {
 			case ".inf", "+.inf":
-				return writeString(writer, "(1/0)")
+				return WriteString(writer, "(1/0)")
 			case "-.inf":
-				return writeString(writer, "(-1/0)")
+				return WriteString(writer, "(-1/0)")
 			case ".nan":
-				return writeString(writer, "(0/0)")
+				return WriteString(writer, "(0/0)")
 			default:
-				return writeString(writer, node.Value)
+				return WriteString(writer, node.Value)
 			}
 		default:
 			return fmt.Errorf("Lua encoder NYI -- %s", node.Tag)
@@ -307,7 +307,7 @@ func (le *luaEncoder) encodeAny(writer io.Writer, node *CandidateNode) error {
 }
 
 func (le *luaEncoder) encodeTopLevel(writer io.Writer, node *CandidateNode) error {
-	err := writeString(writer, le.docPrefix)
+	err := WriteString(writer, le.docPrefix)
 	if err != nil {
 		return err
 	}
@@ -315,7 +315,7 @@ func (le *luaEncoder) encodeTopLevel(writer io.Writer, node *CandidateNode) erro
 	if err != nil {
 		return err
 	}
-	return writeString(writer, le.docSuffix)
+	return WriteString(writer, le.docSuffix)
 }
 
 func (le *luaEncoder) Encode(writer io.Writer, node *CandidateNode) error {

pkg/yqlib/encoder_properties_test.go

@@ -1,127 +0,0 @@
-package yqlib
-
-import (
-	"bufio"
-	"bytes"
-	"strings"
-	"testing"
-
-	"github.com/mikefarah/yq/v4/test"
-)
-
-func yamlToProps(sampleYaml string, unwrapScalar bool) string {
-	var output bytes.Buffer
-	writer := bufio.NewWriter(&output)
-
-	var propsEncoder = NewPropertiesEncoder(unwrapScalar)
-	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
-	if err != nil {
-		panic(err)
-	}
-	node := inputs.Front().Value.(*CandidateNode)
-	err = propsEncoder.Encode(writer, node)
-	if err != nil {
-		panic(err)
-	}
-	writer.Flush()
-
-	return strings.TrimSuffix(output.String(), "\n")
-}
-
-func TestPropertiesEncoderSimple_Unwrapped(t *testing.T) {
-	var sampleYaml = `a: 'bob cool'`
-
-	var expectedProps = `a = bob cool`
-	var actualProps = yamlToProps(sampleYaml, true)
-	test.AssertResult(t, expectedProps, actualProps)
-}
-
-func TestPropertiesEncoderSimple_Wrapped(t *testing.T) {
-	var sampleYaml = `a: 'bob cool'`
-
-	var expectedProps = `a = "bob cool"`
-	var actualProps = yamlToProps(sampleYaml, false)
-	test.AssertResult(t, expectedProps, actualProps)
-}
-
-func TestPropertiesEncoderSimpleWithComments_Unwrapped(t *testing.T) {
-	var sampleYaml = `a: 'bob cool' # line`
-
-	var expectedProps = `# line
-a = bob cool`
-	var actualProps = yamlToProps(sampleYaml, true)
-	test.AssertResult(t, expectedProps, actualProps)
-}
-
-func TestPropertiesEncoderSimpleWithComments_Wrapped(t *testing.T) {
-	var sampleYaml = `a: 'bob cool' # line`
-
-	var expectedProps = `# line
-a = "bob cool"`
-	var actualProps = yamlToProps(sampleYaml, false)
-	test.AssertResult(t, expectedProps, actualProps)
-}
-
-func TestPropertiesEncoderDeep_Unwrapped(t *testing.T) {
-	var sampleYaml = `a: 
-  b: "bob cool"
-`
-
-	var expectedProps = `a.b = bob cool`
-	var actualProps = yamlToProps(sampleYaml, true)
-	test.AssertResult(t, expectedProps, actualProps)
-}
-
-func TestPropertiesEncoderDeep_Wrapped(t *testing.T) {
-	var sampleYaml = `a: 
-  b: "bob cool"
-`
-
-	var expectedProps = `a.b = "bob cool"`
-	var actualProps = yamlToProps(sampleYaml, false)
-	test.AssertResult(t, expectedProps, actualProps)
-}
-
-func TestPropertiesEncoderDeepWithComments_Unwrapped(t *testing.T) {
-	var sampleYaml = `a:  # a thing
-  b: "bob cool" # b thing
-`
-
-	var expectedProps = `# b thing
-a.b = bob cool`
-	var actualProps = yamlToProps(sampleYaml, true)
-	test.AssertResult(t, expectedProps, actualProps)
-}
-
-func TestPropertiesEncoderDeepWithComments_Wrapped(t *testing.T) {
-	var sampleYaml = `a:  # a thing
-  b: "bob cool" # b thing
-`
-
-	var expectedProps = `# b thing
-a.b = "bob cool"`
-	var actualProps = yamlToProps(sampleYaml, false)
-	test.AssertResult(t, expectedProps, actualProps)
-}
-
-func TestPropertiesEncoderArray_Unwrapped(t *testing.T) {
-	var sampleYaml = `a: 
-  b: [{c: dog}, {c: cat}]
-`
-
-	var expectedProps = `a.b.0.c = dog
-a.b.1.c = cat`
-	var actualProps = yamlToProps(sampleYaml, true)
-	test.AssertResult(t, expectedProps, actualProps)
-}
-
-func TestPropertiesEncoderArray_Wrapped(t *testing.T) {
-	var sampleYaml = `a: 
-  b: [{c: dog named jim}, {c: cat named jam}]
-`
-
-	var expectedProps = `a.b.0.c = "dog named jim"
-a.b.1.c = "cat named jam"`
-	var actualProps = yamlToProps(sampleYaml, false)
-	test.AssertResult(t, expectedProps, actualProps)
-}

pkg/yqlib/encoder_sh.go

@@ -30,11 +30,11 @@ func (e *shEncoder) PrintLeadingContent(_ io.Writer, _ string) error {
 }
 
 func (e *shEncoder) Encode(writer io.Writer, node *CandidateNode) error {
-	if node.guessTagFromCustomType() != "!!str" {
+	if node.GuessTagFromCustomType() != "!!str" {
 		return fmt.Errorf("cannot encode %v as URI, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string", node.Tag)
 	}
 
-	return writeString(writer, e.encode(node.Value))
+	return WriteString(writer, e.encode(node.Value))
 }
 
 // put any (shell-unsafe) characters into a single-quoted block, close the block lazily

pkg/yqlib/encoder_shellvariables.go

@@ -30,7 +30,7 @@ func (pe *shellVariablesEncoder) PrintLeadingContent(_ io.Writer, _ string) erro
 
 func (pe *shellVariablesEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 
-	mapKeysToStrings(node)
+	node.ConvertKeysToStrings()
 	err := pe.doEncode(&writer, node, "")
 	if err != nil {
 		return err

pkg/yqlib/encoder_test.go

@@ -16,7 +16,10 @@ func yamlToJSON(t *testing.T, sampleYaml string, indent int) string {
 	var output bytes.Buffer
 	writer := bufio.NewWriter(&output)
 
-	var jsonEncoder = NewJSONEncoder(indent, false, false)
+	prefs := ConfiguredJSONPreferences.Copy()
+	prefs.Indent = indent
+	prefs.UnwrapScalar = false
+	var jsonEncoder = NewJSONEncoder(prefs)
 	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {
 		panic(err)

pkg/yqlib/encoder_toml.go

@@ -14,7 +14,7 @@ func NewTomlEncoder() Encoder {
 
 func (te *tomlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 	if node.Kind == ScalarNode {
-		return writeString(writer, node.Value+"\n")
+		return WriteString(writer, node.Value+"\n")
 	}
 	return fmt.Errorf("only scalars (e.g. strings, numbers, booleans) are supported for TOML output at the moment. Please use yaml output format (-oy) until the encoder has been fully implemented")
 }

pkg/yqlib/encoder_uri.go

@@ -26,7 +26,7 @@ func (e *uriEncoder) PrintLeadingContent(_ io.Writer, _ string) error {
 }
 
 func (e *uriEncoder) Encode(writer io.Writer, node *CandidateNode) error {
-	if node.guessTagFromCustomType() != "!!str" {
+	if node.GuessTagFromCustomType() != "!!str" {
 		return fmt.Errorf("cannot encode %v as URI, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string", node.Tag)
 	}
 	_, err := writer.Write([]byte(url.QueryEscape(node.Value)))

pkg/yqlib/encoder_yaml.go

@@ -12,16 +12,11 @@ import (
 )
 
 type yamlEncoder struct {
-	indent   int
-	colorise bool
-	prefs    YamlPreferences
+	prefs YamlPreferences
 }
 
-func NewYamlEncoder(indent int, colorise bool, prefs YamlPreferences) Encoder {
-	if indent < 0 {
-		indent = 0
-	}
-	return &yamlEncoder{indent, colorise, prefs}
+func NewYamlEncoder(prefs YamlPreferences) Encoder {
+	return &yamlEncoder{prefs}
 }
 
 func (ye *yamlEncoder) CanHandleAliases() bool {
@@ -31,7 +26,7 @@ func (ye *yamlEncoder) CanHandleAliases() bool {
 func (ye *yamlEncoder) PrintDocumentSeparator(writer io.Writer) error {
 	if ye.prefs.PrintDocSeparators {
 		log.Debug("writing doc sep")
-		if err := writeString(writer, "---\n"); err != nil {
+		if err := WriteString(writer, "---\n"); err != nil {
 			return err
 		}
 	}
@@ -59,7 +54,7 @@ func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) err
 			if len(readline) > 0 && readline != "\n" && readline[0] != '%' && !commentLineRegEx.MatchString(readline) {
 				readline = "# " + readline
 			}
-			if err := writeString(writer, readline); err != nil {
+			if err := WriteString(writer, readline); err != nil {
 				return err
 			}
 		}
@@ -67,7 +62,7 @@ func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) err
 		if errors.Is(errReading, io.EOF) {
 			if readline != "" {
 				// the last comment we read didn't have a newline, put one in
-				if err := writeString(writer, "\n"); err != nil {
+				if err := WriteString(writer, "\n"); err != nil {
 					return err
 				}
 			}
@@ -85,18 +80,18 @@ func (ye *yamlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 		if node.LeadingContent == "" || valueToPrint != "" {
 			valueToPrint = valueToPrint + "\n"
 		}
-		return writeString(writer, valueToPrint)
+		return WriteString(writer, valueToPrint)
 	}
 
 	destination := writer
 	tempBuffer := bytes.NewBuffer(nil)
-	if ye.colorise {
+	if ye.prefs.ColorsEnabled {
 		destination = tempBuffer
 	}
 
 	var encoder = yaml.NewEncoder(destination)
 
-	encoder.SetIndent(ye.indent)
+	encoder.SetIndent(ye.prefs.Indent)
 
 	target, err := node.MarshalYAML()
 
@@ -115,7 +110,7 @@ func (ye *yamlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 		return err
 	}
 
-	if ye.colorise {
+	if ye.prefs.ColorsEnabled {
 		return colorizeAndPrint(tempBuffer.Bytes(), writer)
 	}
 	return nil

pkg/yqlib/format.go

@@ -0,0 +1,205 @@
+package yqlib
+
+import (
+	"fmt"
+	"strings"
+)
+
+type EncoderFactoryFunction func() Encoder
+type InDocumentEncoderFactoryFunction func(indent int) Encoder
+type DecoderFactoryFunction func() Decoder
+
+type Format struct {
+	FormalName       string
+	Names            []string
+	DefaultExtension string
+	EncoderFactory   EncoderFactoryFunction
+	DecoderFactory   DecoderFactoryFunction
+
+	/**
+	* Like the Encoder Factory, but for encoding content within the document itself.
+	* Should turn off colors and other settings to ensure the content comes out right.
+	* If this function is not configured, it will default to the EncoderFactory.
+	**/
+	InDocumentEncoderFactory InDocumentEncoderFactoryFunction
+}
+
+var YamlFormat = &Format{"yaml", []string{"y", "yml"}, "yml",
+	func() Encoder { return NewYamlEncoder(ConfiguredYamlPreferences) },
+	func() Decoder { return NewYamlDecoder(ConfiguredYamlPreferences) },
+	func(indent int) Encoder {
+		prefs := ConfiguredYamlPreferences.Copy()
+		prefs.Indent = indent
+		prefs.ColorsEnabled = false
+		return NewYamlEncoder(prefs)
+	},
+}
+
+var JSONFormat = &Format{"json", []string{"j"}, "json",
+	func() Encoder { return NewJSONEncoder(ConfiguredJSONPreferences) },
+	func() Decoder { return NewJSONDecoder() },
+	func(indent int) Encoder {
+		prefs := ConfiguredJSONPreferences.Copy()
+		prefs.Indent = indent
+		prefs.ColorsEnabled = false
+		prefs.UnwrapScalar = false
+		return NewJSONEncoder(prefs)
+	},
+}
+
+var CSVFormat = &Format{"csv", []string{"c"}, "csv",
+	func() Encoder { return NewCsvEncoder(ConfiguredCsvPreferences) },
+	func() Decoder { return NewCSVObjectDecoder(ConfiguredCsvPreferences) },
+	nil,
+}
+
+var TSVFormat = &Format{"tsv", []string{"t"}, "tsv",
+	func() Encoder { return NewCsvEncoder(ConfiguredTsvPreferences) },
+	func() Decoder { return NewCSVObjectDecoder(ConfiguredTsvPreferences) },
+	nil,
+}
+
+var Base64Format = &Format{"base64", []string{}, "txt",
+	func() Encoder { return NewBase64Encoder() },
+	func() Decoder { return NewBase64Decoder() },
+	nil,
+}
+
+var UriFormat = &Format{"uri", []string{}, "txt",
+	func() Encoder { return NewUriEncoder() },
+	func() Decoder { return NewUriDecoder() },
+	nil,
+}
+
+var ShFormat = &Format{"", nil, "sh",
+	func() Encoder { return NewShEncoder() },
+	nil,
+	nil,
+}
+
+var TomlFormat = &Format{"toml", []string{}, "toml",
+	func() Encoder { return NewTomlEncoder() },
+	func() Decoder { return NewTomlDecoder() },
+	nil,
+}
+
+var ShellVariablesFormat = &Format{"shell", []string{"s", "sh"}, "sh",
+	func() Encoder { return NewShellVariablesEncoder() },
+	nil,
+	nil,
+}
+
+var LuaFormat = &Format{"lua", []string{"l"}, "lua",
+	func() Encoder { return NewLuaEncoder(ConfiguredLuaPreferences) },
+	func() Decoder { return NewLuaDecoder(ConfiguredLuaPreferences) },
+	nil,
+}
+
+var Formats = []*Format{
+	YamlFormat,
+	JSONFormat,
+	CSVFormat,
+	TSVFormat,
+	Base64Format,
+	UriFormat,
+	ShFormat,
+	TomlFormat,
+	ShellVariablesFormat,
+	LuaFormat,
+}
+
+func RegisterFormat(f *Format) {
+	Formats = append(Formats, f)
+}
+
+func (f *Format) MatchesName(name string) bool {
+	if f.FormalName == name {
+		return true
+	}
+	for _, n := range f.Names {
+		if n == name {
+			return true
+		}
+	}
+	return false
+}
+
+func (f *Format) GetInDocumentEncoder(indent int) Encoder {
+	if f.InDocumentEncoderFactory != nil {
+		return f.InDocumentEncoderFactory(indent)
+	}
+	return f.EncoderFactory()
+}
+
+func FormatStringFromFilename(filename string) string {
+
+	if filename != "" {
+		GetLogger().Debugf("checking file extension '%s' for auto format detection", filename)
+		nPos := strings.LastIndex(filename, ".")
+		if nPos > -1 {
+			format := filename[nPos+1:]
+			GetLogger().Debugf("detected format '%s'", format)
+			return format
+		}
+	}
+
+	GetLogger().Debugf("using default inputFormat 'yaml'")
+	return "yaml"
+}
+
+func FormatFromString(format string) (*Format, error) {
+	for _, printerFormat := range Formats {
+		if printerFormat.MatchesName(format) {
+			return printerFormat, nil
+		}
+	}
+	return nil, fmt.Errorf("unknown format '%v' please use [%v]", format, GetAvailableOutputFormatString())
+}
+
+func GetAvailableOutputFormats() []*Format {
+	var formats = []*Format{}
+	for _, printerFormat := range Formats {
+		if printerFormat.EncoderFactory != nil {
+			formats = append(formats, printerFormat)
+		}
+	}
+	return formats
+}
+
+func GetAvailableOutputFormatString() string {
+	var formats = []string{}
+	for _, printerFormat := range GetAvailableOutputFormats() {
+
+		if printerFormat.FormalName != "" {
+			formats = append(formats, printerFormat.FormalName)
+		}
+		if len(printerFormat.Names) >= 1 {
+			formats = append(formats, printerFormat.Names[0])
+		}
+	}
+	return strings.Join(formats, "|")
+}
+
+func GetAvailableInputFormats() []*Format {
+	var formats = []*Format{}
+	for _, printerFormat := range Formats {
+		if printerFormat.DecoderFactory != nil {
+			formats = append(formats, printerFormat)
+		}
+	}
+	return formats
+}
+
+func GetAvailableInputFormatString() string {
+	var formats = []string{}
+	for _, printerFormat := range GetAvailableInputFormats() {
+
+		if printerFormat.FormalName != "" {
+			formats = append(formats, printerFormat.FormalName)
+		}
+		if len(printerFormat.Names) >= 1 {
+			formats = append(formats, printerFormat.Names[0])
+		}
+	}
+	return strings.Join(formats, "|")
+}

pkg/yqlib/formatting_expressions_test.go

@@ -3,29 +3,47 @@ package yqlib
 import (
 	"bufio"
 	"fmt"
+	"strings"
 	"testing"
 
 	"github.com/mikefarah/yq/v4/test"
 )
 
-var formattingExpressionScenarios = []formatScenario{
+var formattingExpressionScenarios = []FormatScenario{
 	{
 		description: "Using expression files and comments",
+		skipDoc:     true,
 		input:       "a:\n  b: old",
-		expression:  "\n# This is a yq expression that updates the map\n# for several great reasons outlined here.\n\n.a.b = \"new\" # line comment here\n| .a.c = \"frog\"\n\n# Now good things will happen.\n",
+		expression:  "#! yq\n\n# This is a yq expression that updates the map\n# for several great reasons outlined here.\n\n.a.b = \"new\" # line comment here\n| .a.c = \"frog\"\n\n# Now good things will happen.\n",
 		expected:    "a:\n  b: new\n  c: frog\n",
 	},
 	{
-		description:    "Commenting out yq expressions",
-		subdescription: "Note that `c` is no longer set to 'frog'.",
+		description:    "Using expression files and comments",
+		subdescription: "Note that you can execute the file directly - but make sure you make the expression file executable.",
 		input:          "a:\n  b: old",
-		expression:     "\n# This is a yq expression that updates the map\n# for several great reasons outlined here.\n\n.a.b = \"new\" # line comment here\n# | .a.c = \"frog\"\n\n# Now good things will happen.\n",
+		expression:     "#! yq\n\n# This is a yq expression that updates the map\n# for several great reasons outlined here.\n\n.a.b = \"new\" # line comment here\n| .a.c = \"frog\"\n\n# Now good things will happen.\n",
+		expected:       "a:\n  b: new\n  c: frog\n",
+		scenarioType:   "shebang",
+	},
+	{
+		description:    "Flags in expression files",
+		subdescription: "You can specify flags on the shebang line, this only works when executing the file directly.",
+		input:          "a:\n  b: old",
+		expression:     "#! yq -oj\n\n# This is a yq expression that updates the map\n# for several great reasons outlined here.\n\n.a.b = \"new\" # line comment here\n| .a.c = \"frog\"\n\n# Now good things will happen.\n",
+		expected:       "a:\n  b: new\n  c: frog\n",
+		scenarioType:   "shebang-json",
+	},
+	{
+		description:    "Commenting out yq expressions",
+		subdescription: "Note that `c` is no longer set to 'frog'. In this example we're calling yq directly and passing the expression file into `--from-file`, this is no different from executing the expression file directly.",
+		input:          "a:\n  b: old",
+		expression:     "#! yq\n# This is a yq expression that updates the map\n# for several great reasons outlined here.\n\n.a.b = \"new\" # line comment here\n# | .a.c = \"frog\"\n\n# Now good things will happen.\n",
 		expected:       "a:\n  b: new\n",
 	},
 }
 
 func documentExpressionScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
-	s := i.(formatScenario)
+	s := i.(FormatScenario)
 
 	if s.skipDoc {
 		return
@@ -41,19 +59,28 @@ func documentExpressionScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
 	writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input))
 
 	writeOrPanic(w, "And an 'update.yq' expression file of:\n")
-	writeOrPanic(w, fmt.Sprintf("```bash%v```\n", s.expression))
+	writeOrPanic(w, fmt.Sprintf("```bash\n%v```\n", s.expression))
 
 	writeOrPanic(w, "then\n")
-	writeOrPanic(w, "```bash\nyq --from-file update.yq sample.yml\n```\n")
+	if strings.HasPrefix(s.scenarioType, "shebang") {
+		writeOrPanic(w, "```bash\n./update.yq sample.yaml\n```\n")
+	} else {
+		writeOrPanic(w, "```bash\nyq --from-file update.yq sample.yml\n```\n")
+	}
 	writeOrPanic(w, "will output\n")
+	encoder := NewYamlEncoder(ConfiguredYamlPreferences)
 
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences))))
+	if s.scenarioType == "shebang-json" {
+		encoder = NewJSONEncoder(ConfiguredJSONPreferences)
+	}
+
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), encoder)))
 }
 
 func TestExpressionCommentScenarios(t *testing.T) {
 	for _, tt := range formattingExpressionScenarios {
 		test.AssertResultComplexWithContext(t, tt.expected,
-			mustProcessFormatScenario(tt, NewYamlDecoder(ConfiguredYamlPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences)),
+			mustProcessFormatScenario(tt, NewYamlDecoder(ConfiguredYamlPreferences), NewYamlEncoder(ConfiguredYamlPreferences)),
 			tt.description)
 	}
 	genericScenarios := make([]interface{}, len(formattingExpressionScenarios))

pkg/yqlib/goccy_yaml_test.go

@@ -6,7 +6,7 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
-var goccyYamlFormatScenarios = []formatScenario{
+var goccyYamlFormatScenarios = []FormatScenario{
 	{
 		description: "basic - 3",
 		skipDoc:     true,
@@ -159,8 +159,8 @@ var goccyYamlFormatScenarios = []formatScenario{
 	// },
 }
 
-func testGoccyYamlScenario(t *testing.T, s formatScenario) {
-	test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewGoccyYAMLDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
+func testGoccyYamlScenario(t *testing.T, s FormatScenario) {
+	test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewGoccyYAMLDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
 }
 
 func TestGoccyYmlFormatScenarios(t *testing.T) {

pkg/yqlib/json.go

@@ -0,0 +1,25 @@
+package yqlib
+
+type JsonPreferences struct {
+	Indent        int
+	ColorsEnabled bool
+	UnwrapScalar  bool
+}
+
+func NewDefaultJsonPreferences() JsonPreferences {
+	return JsonPreferences{
+		Indent:        2,
+		ColorsEnabled: true,
+		UnwrapScalar:  true,
+	}
+}
+
+func (p *JsonPreferences) Copy() JsonPreferences {
+	return JsonPreferences{
+		Indent:        p.Indent,
+		ColorsEnabled: p.ColorsEnabled,
+		UnwrapScalar:  p.UnwrapScalar,
+	}
+}
+
+var ConfiguredJSONPreferences = NewDefaultJsonPreferences()

pkg/yqlib/json_test.go

@@ -79,12 +79,13 @@ const roundTripMultiLineJson = `{
 }
 `
 
-var jsonScenarios = []formatScenario{
+var jsonScenarios = []FormatScenario{
 	{
 		description:  "array empty",
 		skipDoc:      true,
 		input:        "[]",
 		scenarioType: "roundtrip-ndjson",
+		indent:       0,
 		expected:     "[]\n",
 	},
 	{
@@ -92,6 +93,7 @@ var jsonScenarios = []formatScenario{
 		skipDoc:      true,
 		input:        "[3]",
 		scenarioType: "roundtrip-ndjson",
+		indent:       0,
 		expected:     "[3]\n",
 	},
 	{
@@ -99,6 +101,7 @@ var jsonScenarios = []formatScenario{
 		skipDoc:      true,
 		input:        `[{"x": 3}]`,
 		scenarioType: "roundtrip-ndjson",
+		indent:       0,
 		expected:     "[{\"x\":3}]\n",
 	},
 	{
@@ -106,6 +109,7 @@ var jsonScenarios = []formatScenario{
 		skipDoc:      true,
 		input:        "[null]",
 		scenarioType: "roundtrip-ndjson",
+		indent:       0,
 		expected:     "[null]\n",
 	},
 	{
@@ -114,6 +118,7 @@ var jsonScenarios = []formatScenario{
 		input:        "[{}]",
 		expression:   `[.. | type]`,
 		scenarioType: "roundtrip-ndjson",
+		indent:       0,
 		expected:     "[\"!!seq\",\"!!map\"]\n",
 	},
 	{
@@ -220,6 +225,7 @@ var jsonScenarios = []formatScenario{
 		input:        sampleNdJson,
 		expected:     expectedRoundTripSampleNdJson,
 		scenarioType: "roundtrip-ndjson",
+		indent:       0,
 	},
 	{
 		description:    "Roundtrip multi-document JSON",
@@ -301,7 +307,7 @@ var jsonScenarios = []formatScenario{
 	},
 }
 
-func documentRoundtripNdJsonScenario(w *bufio.Writer, s formatScenario, indent int) {
+func documentRoundtripNdJsonScenario(w *bufio.Writer, s FormatScenario, indent int) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -322,11 +328,14 @@ func documentRoundtripNdJsonScenario(w *bufio.Writer, s formatScenario, indent i
 	}
 
 	writeOrPanic(w, "will output\n")
+	prefs := ConfiguredJSONPreferences.Copy()
+	prefs.Indent = indent
+	prefs.UnwrapScalar = false
 
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(indent, false, false))))
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(prefs))))
 }
 
-func documentDecodeNdJsonScenario(w *bufio.Writer, s formatScenario) {
+func documentDecodeNdJsonScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -348,7 +357,7 @@ func documentDecodeNdJsonScenario(w *bufio.Writer, s formatScenario) {
 
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))))
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(ConfiguredYamlPreferences))))
 }
 
 func decodeJSON(t *testing.T, jsonString string) *CandidateNode {
@@ -376,20 +385,24 @@ func decodeJSON(t *testing.T, jsonString string) *CandidateNode {
 	return context.MatchingNodes.Front().Value.(*CandidateNode)
 }
 
-func testJSONScenario(t *testing.T, s formatScenario) {
+func testJSONScenario(t *testing.T, s FormatScenario) {
+	prefs := ConfiguredJSONPreferences.Copy()
+	prefs.Indent = s.indent
+	prefs.UnwrapScalar = false
 	switch s.scenarioType {
 	case "encode":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false, false)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(prefs)), s.description)
 	case "decode":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(s.indent, false, false)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(prefs)), s.description)
 	case "decode-ndjson":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
 	case "roundtrip-ndjson":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(0, false, false)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(prefs)), s.description)
 	case "roundtrip-multi":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(2, false, false)), s.description)
+		prefs.Indent = 2
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(prefs)), s.description)
 	case "decode-error":
-		result, err := processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(2, false, false))
+		result, err := processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(prefs))
 		if err == nil {
 			t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result)
 		} else {
@@ -400,7 +413,7 @@ func testJSONScenario(t *testing.T, s formatScenario) {
 	}
 }
 
-func documentJSONDecodeScenario(t *testing.T, w *bufio.Writer, s formatScenario) {
+func documentJSONDecodeScenario(t *testing.T, w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -416,7 +429,7 @@ func documentJSONDecodeScenario(t *testing.T, w *bufio.Writer, s formatScenario)
 	writeOrPanic(w, "will output\n")
 
 	var output bytes.Buffer
-	printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
+	printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), true, 2, true)
 
 	node := decodeJSON(t, s.input)
 
@@ -430,7 +443,7 @@ func documentJSONDecodeScenario(t *testing.T, w *bufio.Writer, s formatScenario)
 }
 
 func documentJSONScenario(t *testing.T, w *bufio.Writer, i interface{}) {
-	s := i.(formatScenario)
+	s := i.(FormatScenario)
 	if s.skipDoc {
 		return
 	}
@@ -451,7 +464,7 @@ func documentJSONScenario(t *testing.T, w *bufio.Writer, i interface{}) {
 	}
 }
 
-func documentJSONEncodeScenario(w *bufio.Writer, s formatScenario) {
+func documentJSONEncodeScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -475,8 +488,11 @@ func documentJSONEncodeScenario(w *bufio.Writer, s formatScenario) {
 		writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=json -I=%v '%v' sample.yml\n```\n", s.indent, expression))
 	}
 	writeOrPanic(w, "will output\n")
+	prefs := ConfiguredJSONPreferences.Copy()
+	prefs.Indent = s.indent
+	prefs.UnwrapScalar = false
 
-	writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false, false))))
+	writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(prefs))))
 }
 
 func TestJSONScenarios(t *testing.T) {

pkg/yqlib/lexer_participle.go

@@ -7,7 +7,7 @@ import (
 	"github.com/alecthomas/participle/v2/lexer"
 )
 
-var participleYqRules = []*participleYqRule{
+var participleYqRules = []*ParticipleYqRule{
 	{"LINE_COMMENT", `line_?comment|lineComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true}), 0},
 	{"HEAD_COMMENT", `head_?comment|headComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{HeadComment: true}), 0},
 	{"FOOT_COMMENT", `foot_?comment|footComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{FootComment: true}), 0},
@@ -57,45 +57,34 @@ var participleYqRules = []*participleYqRule{
 
 	{"ArrayToMap", "array_?to_?map", expressionOpToken(`(.[] | select(. != null) ) as $i ireduce({}; .[$i | key] = $i)`), 0},
 
-	{"YamlEncodeWithIndent", `to_?yaml\([0-9]+\)`, encodeParseIndent(YamlOutputFormat), 0},
-	{"XMLEncodeWithIndent", `to_?xml\([0-9]+\)`, encodeParseIndent(XMLOutputFormat), 0},
-	{"JSONEncodeWithIndent", `to_?json\([0-9]+\)`, encodeParseIndent(JSONOutputFormat), 0},
+	{"YamlEncodeWithIndent", `to_?yaml\([0-9]+\)`, CreateEncodeYqActionParsingIndent(YamlFormat), 0},
 
-	{"YamlDecode", `from_?yaml|@yamld|from_?json|@jsond`, decodeOp(YamlInputFormat), 0},
-	{"YamlEncode", `to_?yaml|@yaml`, encodeWithIndent(YamlOutputFormat, 2), 0},
+	{"JSONEncodeWithIndent", `to_?json\([0-9]+\)`, CreateEncodeYqActionParsingIndent(JSONFormat), 0},
 
-	{"JSONEncode", `to_?json`, encodeWithIndent(JSONOutputFormat, 2), 0},
-	{"JSONEncodeNoIndent", `@json`, encodeWithIndent(JSONOutputFormat, 0), 0},
+	{"YamlDecode", `from_?yaml|@yamld|from_?json|@jsond`, CreateDecodeOpYqAction(YamlFormat), 0},
+	{"YamlEncode", `to_?yaml|@yaml`, CreateEncodeOpYqAction(YamlFormat, 2), 0},
 
-	{"PropertiesDecode", `from_?props|@propsd`, decodeOp(PropertiesInputFormat), 0},
-	{"PropsEncode", `to_?props|@props`, encodeWithIndent(PropsOutputFormat, 2), 0},
+	{"JSONEncode", `to_?json`, CreateEncodeOpYqAction(JSONFormat, 2), 0},
+	{"JSONEncodeNoIndent", `@json`, CreateEncodeOpYqAction(JSONFormat, 0), 0},
 
-	{"XmlDecode", `from_?xml|@xmld`, decodeOp(XMLInputFormat), 0},
-	{"XMLEncode", `to_?xml`, encodeWithIndent(XMLOutputFormat, 2), 0},
-	{"XMLEncodeNoIndent", `@xml`, encodeWithIndent(XMLOutputFormat, 0), 0},
+	{"CSVDecode", `from_?csv|@csvd`, CreateDecodeOpYqAction(CSVFormat), 0},
+	{"CSVEncode", `to_?csv|@csv`, CreateEncodeOpYqAction(CSVFormat, 0), 0},
 
-	{"CSVDecode", `from_?csv|@csvd`, decodeOp(CSVObjectInputFormat), 0},
-	{"CSVEncode", `to_?csv|@csv`, encodeWithIndent(CSVOutputFormat, 0), 0},
+	{"TSVDecode", `from_?tsv|@tsvd`, CreateDecodeOpYqAction(TSVFormat), 0},
+	{"TSVEncode", `to_?tsv|@tsv`, CreateEncodeOpYqAction(TSVFormat, 0), 0},
 
-	{"TSVDecode", `from_?tsv|@tsvd`, decodeOp(TSVObjectInputFormat), 0},
-	{"TSVEncode", `to_?tsv|@tsv`, encodeWithIndent(TSVOutputFormat, 0), 0},
+	{"Base64d", `@base64d`, CreateDecodeOpYqAction(Base64Format), 0},
+	{"Base64", `@base64`, CreateEncodeOpYqAction(Base64Format, 0), 0},
 
-	{"Base64d", `@base64d`, decodeOp(Base64InputFormat), 0},
-	{"Base64", `@base64`, encodeWithIndent(Base64OutputFormat, 0), 0},
+	{"Urid", `@urid`, CreateDecodeOpYqAction(UriFormat), 0},
+	{"Uri", `@uri`, CreateEncodeOpYqAction(UriFormat, 0), 0},
+	{"SH", `@sh`, CreateEncodeOpYqAction(ShFormat, 0), 0},
 
-	{"Urid", `@urid`, decodeOp(UriInputFormat), 0},
-	{"Uri", `@uri`, encodeWithIndent(UriOutputFormat, 0), 0},
-	{"SH", `@sh`, encodeWithIndent(ShOutputFormat, 0), 0},
+	{"LoadBase64", `load_?base64`, CreateLoadOpYqAction(NewBase64Decoder()), 0},
 
-	{"LoadXML", `load_?xml|xml_?load`, loadOp(NewXMLDecoder(ConfiguredXMLPreferences), false), 0},
+	simpleOp(`load_?str|str_?load`, loadStringOpType),
 
-	{"LoadBase64", `load_?base64`, loadOp(NewBase64Decoder(), false), 0},
-
-	{"LoadProperties", `load_?props`, loadOp(NewPropertiesDecoder(), false), 0},
-
-	{"LoadString", `load_?str|str_?load`, loadOp(nil, true), 0},
-
-	{"LoadYaml", `load`, loadOp(NewYamlDecoder(LoadYamlPreferences), false), 0},
+	{"LoadYaml", `load`, CreateLoadOpYqAction(NewYamlDecoder(LoadYamlPreferences)), 0},
 
 	{"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0},
 
@@ -130,7 +119,9 @@ var participleYqRules = []*participleYqRule{
 
 	simpleOp("contains", containsOpType),
 	simpleOp("split", splitStringOpType),
-	simpleOp("parent", getParentOpType),
+
+	{"ParentWithLevel", `parent\([0-9]+\)`, parentWithLevel(), 0},
+	{"ParentWithDefaultLevel", `parent`, parentWithDefaultLevel(), 0},
 
 	simpleOp("keys", keysOpType),
 	simpleOp("key", getKeyOpType),
@@ -168,6 +159,7 @@ var participleYqRules = []*participleYqRule{
 	{"Uppercase", `upcase|ascii_?upcase`, opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: true}), 0},
 	{"Downcase", `downcase|ascii_?downcase`, opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: false}), 0},
 	simpleOp("trim", trimOpType),
+	simpleOp("to_?string", toStringOpType),
 
 	{"HexValue", `0[xX][0-9A-Fa-f]+`, hexValue(), 0},
 	{"FloatValueScientific", `-?[1-9](\.\d+)?[Ee][-+]?\d+`, floatValue(), 0},
@@ -226,7 +218,7 @@ var participleYqRules = []*participleYqRule{
 
 type yqAction func(lexer.Token) (*token, error)
 
-type participleYqRule struct {
+type ParticipleYqRule struct {
 	Name                string
 	Pattern             string
 	CreateYqToken       yqAction
@@ -237,12 +229,16 @@ type participleLexer struct {
 	lexerDefinition lexer.StringDefinition
 }
 
-func simpleOp(name string, opType *operationType) *participleYqRule {
-	return &participleYqRule{strings.ToUpper(string(name[1])) + name[1:], name, opToken(opType), 0}
+func simpleOp(name string, opType *operationType) *ParticipleYqRule {
+	return &ParticipleYqRule{strings.ToUpper(string(name[1])) + name[1:], name, opToken(opType), 0}
 }
 
-func assignableOp(name string, opType *operationType, assignOpType *operationType) *participleYqRule {
-	return &participleYqRule{strings.ToUpper(string(name[1])) + name[1:], name, opTokenWithPrefs(opType, assignOpType, nil), 0}
+func assignableOp(name string, opType *operationType, assignOpType *operationType) *ParticipleYqRule {
+	return &ParticipleYqRule{strings.ToUpper(string(name[1])) + name[1:], name, opTokenWithPrefs(opType, assignOpType, nil), 0}
+}
+
+func RegisterRules(rules []*ParticipleYqRule) {
+	participleYqRules = append(participleYqRules, rules...)
 }
 
 func newParticipleLexer() expressionTokeniser {
@@ -373,7 +369,11 @@ func stringValue() yqAction {
 		value = strings.ReplaceAll(value, "\\\"", "\"")
 		value = strings.ReplaceAll(value, "\\n", "\n")
 		log.Debug("replaced: %v", value)
-		return &token{TokenType: operationToken, Operation: createValueOperation(value, value)}, nil
+		return &token{TokenType: operationToken, Operation: &Operation{
+			OperationType: stringInterpolationOpType,
+			StringValue:   value,
+			Value:         value,
+		}}, nil
 	}
 }
 
@@ -496,7 +496,29 @@ func numberValue() yqAction {
 	}
 }
 
-func encodeParseIndent(outputFormat PrinterOutputFormat) yqAction {
+func parentWithLevel() yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		value := rawToken.Value
+		var level, errParsingInt = extractNumberParameter(value)
+		if errParsingInt != nil {
+			return nil, errParsingInt
+		}
+
+		prefs := parentOpPreferences{Level: level}
+		op := &Operation{OperationType: getParentOpType, Value: getParentOpType.Type, StringValue: value, Preferences: prefs}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+}
+
+func parentWithDefaultLevel() yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		prefs := parentOpPreferences{Level: 1}
+		op := &Operation{OperationType: getParentOpType, Value: getParentOpType.Type, StringValue: getParentOpType.Type, Preferences: prefs}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+}
+
+func CreateEncodeYqActionParsingIndent(outputFormat *Format) yqAction {
 	return func(rawToken lexer.Token) (*token, error) {
 		value := rawToken.Value
 		var indent, errParsingInt = extractNumberParameter(value)
@@ -510,18 +532,18 @@ func encodeParseIndent(outputFormat PrinterOutputFormat) yqAction {
 	}
 }
 
-func encodeWithIndent(outputFormat PrinterOutputFormat, indent int) yqAction {
+func CreateEncodeOpYqAction(outputFormat *Format, indent int) yqAction {
 	prefs := encoderPreferences{format: outputFormat, indent: indent}
 	return opTokenWithPrefs(encodeOpType, nil, prefs)
 }
 
-func decodeOp(inputFormat InputFormat) yqAction {
-	prefs := decoderPreferences{format: inputFormat}
+func CreateDecodeOpYqAction(format *Format) yqAction {
+	prefs := decoderPreferences{format: format}
 	return opTokenWithPrefs(decodeOpType, nil, prefs)
 }
 
-func loadOp(decoder Decoder, loadAsString bool) yqAction {
-	prefs := loadPrefs{decoder: decoder, loadAsString: loadAsString}
+func CreateLoadOpYqAction(decoder Decoder) yqAction {
+	prefs := loadPrefs{decoder}
 	return opTokenWithPrefs(loadOpType, nil, prefs)
 }
 
@@ -535,13 +557,13 @@ func literalToken(tt tokenType, checkForPost bool) yqAction {
 	}
 }
 
-func (p *participleLexer) getYqDefinition(rawToken lexer.Token) *participleYqRule {
+func (p *participleLexer) getYqDefinition(rawToken lexer.Token) *ParticipleYqRule {
 	for _, yqRule := range participleYqRules {
 		if yqRule.ParticipleTokenType == rawToken.Type {
 			return yqRule
 		}
 	}
-	return &participleYqRule{}
+	return &ParticipleYqRule{}
 }
 
 func (p *participleLexer) Tokenise(expression string) ([]*token, error) {

pkg/yqlib/lib.go

@@ -28,168 +28,6 @@ func GetLogger() *logging.Logger {
 	return log
 }
 
-type operationType struct {
-	Type                 string
-	NumArgs              uint // number of arguments to the op
-	Precedence           uint
-	Handler              operatorHandler
-	CheckForPostTraverse bool
-}
-
-var orOpType = &operationType{Type: "OR", NumArgs: 2, Precedence: 20, Handler: orOperator}
-var andOpType = &operationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: andOperator}
-var reduceOpType = &operationType{Type: "REDUCE", NumArgs: 2, Precedence: 35, Handler: reduceOperator}
-
-var blockOpType = &operationType{Type: "BLOCK", Precedence: 10, NumArgs: 2, Handler: emptyOperator}
-
-var unionOpType = &operationType{Type: "UNION", NumArgs: 2, Precedence: 10, Handler: unionOperator}
-
-var pipeOpType = &operationType{Type: "PIPE", NumArgs: 2, Precedence: 30, Handler: pipeOperator}
-
-var assignOpType = &operationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: assignUpdateOperator}
-var addAssignOpType = &operationType{Type: "ADD_ASSIGN", NumArgs: 2, Precedence: 40, Handler: addAssignOperator}
-var subtractAssignOpType = &operationType{Type: "SUBTRACT_ASSIGN", NumArgs: 2, Precedence: 40, Handler: subtractAssignOperator}
-
-var assignAttributesOpType = &operationType{Type: "ASSIGN_ATTRIBUTES", NumArgs: 2, Precedence: 40, Handler: assignAttributesOperator}
-var assignStyleOpType = &operationType{Type: "ASSIGN_STYLE", NumArgs: 2, Precedence: 40, Handler: assignStyleOperator}
-var assignVariableOpType = &operationType{Type: "ASSIGN_VARIABLE", NumArgs: 2, Precedence: 40, Handler: useWithPipe}
-var assignTagOpType = &operationType{Type: "ASSIGN_TAG", NumArgs: 2, Precedence: 40, Handler: assignTagOperator}
-var assignCommentOpType = &operationType{Type: "ASSIGN_COMMENT", NumArgs: 2, Precedence: 40, Handler: assignCommentsOperator}
-var assignAnchorOpType = &operationType{Type: "ASSIGN_ANCHOR", NumArgs: 2, Precedence: 40, Handler: assignAnchorOperator}
-var assignAliasOpType = &operationType{Type: "ASSIGN_ALIAS", NumArgs: 2, Precedence: 40, Handler: assignAliasOperator}
-
-var multiplyOpType = &operationType{Type: "MULTIPLY", NumArgs: 2, Precedence: 42, Handler: multiplyOperator}
-var multiplyAssignOpType = &operationType{Type: "MULTIPLY_ASSIGN", NumArgs: 2, Precedence: 42, Handler: multiplyAssignOperator}
-
-var divideOpType = &operationType{Type: "DIVIDE", NumArgs: 2, Precedence: 42, Handler: divideOperator}
-
-var moduloOpType = &operationType{Type: "MODULO", NumArgs: 2, Precedence: 42, Handler: moduloOperator}
-
-var addOpType = &operationType{Type: "ADD", NumArgs: 2, Precedence: 42, Handler: addOperator}
-var subtractOpType = &operationType{Type: "SUBTRACT", NumArgs: 2, Precedence: 42, Handler: subtractOperator}
-var alternativeOpType = &operationType{Type: "ALTERNATIVE", NumArgs: 2, Precedence: 42, Handler: alternativeOperator}
-
-var equalsOpType = &operationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: equalsOperator}
-var notEqualsOpType = &operationType{Type: "NOT_EQUALS", NumArgs: 2, Precedence: 40, Handler: notEqualsOperator}
-
-var compareOpType = &operationType{Type: "COMPARE", NumArgs: 2, Precedence: 40, Handler: compareOperator}
-
-// createmap needs to be above union, as we use union to build the components of the objects
-var createMapOpType = &operationType{Type: "CREATE_MAP", NumArgs: 2, Precedence: 15, Handler: createMapOperator}
-
-var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence: 45, Handler: pipeOperator}
-
-var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: lengthOperator}
-var lineOpType = &operationType{Type: "LINE", NumArgs: 0, Precedence: 50, Handler: lineOperator}
-var columnOpType = &operationType{Type: "LINE", NumArgs: 0, Precedence: 50, Handler: columnOperator}
-
-var expressionOpType = &operationType{Type: "EXP", NumArgs: 0, Precedence: 50, Handler: expressionOperator}
-
-var collectOpType = &operationType{Type: "COLLECT", NumArgs: 1, Precedence: 50, Handler: collectOperator}
-var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 50, Handler: mapOperator}
-var filterOpType = &operationType{Type: "FILTER", NumArgs: 1, Precedence: 50, Handler: filterOperator}
-var errorOpType = &operationType{Type: "ERROR", NumArgs: 1, Precedence: 50, Handler: errorOperator}
-var pickOpType = &operationType{Type: "PICK", NumArgs: 1, Precedence: 50, Handler: pickOperator}
-var evalOpType = &operationType{Type: "EVAL", NumArgs: 1, Precedence: 50, Handler: evalOperator}
-var mapValuesOpType = &operationType{Type: "MAP_VALUES", NumArgs: 1, Precedence: 50, Handler: mapValuesOperator}
-
-var formatDateTimeOpType = &operationType{Type: "FORMAT_DATE_TIME", NumArgs: 1, Precedence: 50, Handler: formatDateTime}
-var withDtFormatOpType = &operationType{Type: "WITH_DATE_TIME_FORMAT", NumArgs: 1, Precedence: 50, Handler: withDateTimeFormat}
-var nowOpType = &operationType{Type: "NOW", NumArgs: 0, Precedence: 50, Handler: nowOp}
-var tzOpType = &operationType{Type: "TIMEZONE", NumArgs: 1, Precedence: 50, Handler: tzOp}
-var fromUnixOpType = &operationType{Type: "FROM_UNIX", NumArgs: 0, Precedence: 50, Handler: fromUnixOp}
-var toUnixOpType = &operationType{Type: "TO_UNIX", NumArgs: 0, Precedence: 50, Handler: toUnixOp}
-
-var encodeOpType = &operationType{Type: "ENCODE", NumArgs: 0, Precedence: 50, Handler: encodeOperator}
-var decodeOpType = &operationType{Type: "DECODE", NumArgs: 0, Precedence: 50, Handler: decodeOperator}
-
-var anyOpType = &operationType{Type: "ANY", NumArgs: 0, Precedence: 50, Handler: anyOperator}
-var allOpType = &operationType{Type: "ALL", NumArgs: 0, Precedence: 50, Handler: allOperator}
-var containsOpType = &operationType{Type: "CONTAINS", NumArgs: 1, Precedence: 50, Handler: containsOperator}
-var anyConditionOpType = &operationType{Type: "ANY_CONDITION", NumArgs: 1, Precedence: 50, Handler: anyOperator}
-var allConditionOpType = &operationType{Type: "ALL_CONDITION", NumArgs: 1, Precedence: 50, Handler: allOperator}
-
-var toEntriesOpType = &operationType{Type: "TO_ENTRIES", NumArgs: 0, Precedence: 52, Handler: toEntriesOperator, CheckForPostTraverse: true}
-var fromEntriesOpType = &operationType{Type: "FROM_ENTRIES", NumArgs: 0, Precedence: 50, Handler: fromEntriesOperator}
-var withEntriesOpType = &operationType{Type: "WITH_ENTRIES", NumArgs: 1, Precedence: 50, Handler: withEntriesOperator}
-
-var withOpType = &operationType{Type: "WITH", NumArgs: 1, Precedence: 50, Handler: withOperator}
-
-var splitDocumentOpType = &operationType{Type: "SPLIT_DOC", NumArgs: 0, Precedence: 50, Handler: splitDocumentOperator}
-var getVariableOpType = &operationType{Type: "GET_VARIABLE", NumArgs: 0, Precedence: 55, Handler: getVariableOperator}
-var getStyleOpType = &operationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: getStyleOperator}
-var getTagOpType = &operationType{Type: "GET_TAG", NumArgs: 0, Precedence: 50, Handler: getTagOperator}
-var getKindOpType = &operationType{Type: "GET_KIND", NumArgs: 0, Precedence: 50, Handler: getKindOperator}
-
-var getKeyOpType = &operationType{Type: "GET_KEY", NumArgs: 0, Precedence: 50, Handler: getKeyOperator}
-var isKeyOpType = &operationType{Type: "IS_KEY", NumArgs: 0, Precedence: 50, Handler: isKeyOperator}
-var getParentOpType = &operationType{Type: "GET_PARENT", NumArgs: 0, Precedence: 50, Handler: getParentOperator}
-
-var getCommentOpType = &operationType{Type: "GET_COMMENT", NumArgs: 0, Precedence: 50, Handler: getCommentsOperator}
-var getAnchorOpType = &operationType{Type: "GET_ANCHOR", NumArgs: 0, Precedence: 50, Handler: getAnchorOperator}
-var getAliasOpType = &operationType{Type: "GET_ALIAS", NumArgs: 0, Precedence: 50, Handler: getAliasOperator}
-var getDocumentIndexOpType = &operationType{Type: "GET_DOCUMENT_INDEX", NumArgs: 0, Precedence: 50, Handler: getDocumentIndexOperator}
-var getFilenameOpType = &operationType{Type: "GET_FILENAME", NumArgs: 0, Precedence: 50, Handler: getFilenameOperator}
-var getFileIndexOpType = &operationType{Type: "GET_FILE_INDEX", NumArgs: 0, Precedence: 50, Handler: getFileIndexOperator}
-
-var getPathOpType = &operationType{Type: "GET_PATH", NumArgs: 0, Precedence: 50, Handler: getPathOperator}
-var setPathOpType = &operationType{Type: "SET_PATH", NumArgs: 1, Precedence: 50, Handler: setPathOperator}
-var delPathsOpType = &operationType{Type: "DEL_PATHS", NumArgs: 1, Precedence: 50, Handler: delPathsOperator}
-
-var explodeOpType = &operationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: explodeOperator}
-var sortByOpType = &operationType{Type: "SORT_BY", NumArgs: 1, Precedence: 50, Handler: sortByOperator}
-var reverseOpType = &operationType{Type: "REVERSE", NumArgs: 0, Precedence: 50, Handler: reverseOperator}
-var sortOpType = &operationType{Type: "SORT", NumArgs: 0, Precedence: 50, Handler: sortOperator}
-var shuffleOpType = &operationType{Type: "SHUFFLE", NumArgs: 0, Precedence: 50, Handler: shuffleOperator}
-
-var sortKeysOpType = &operationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Handler: sortKeysOperator}
-
-var joinStringOpType = &operationType{Type: "JOIN", NumArgs: 1, Precedence: 50, Handler: joinStringOperator}
-var subStringOpType = &operationType{Type: "SUBSTR", NumArgs: 1, Precedence: 50, Handler: substituteStringOperator}
-var matchOpType = &operationType{Type: "MATCH", NumArgs: 1, Precedence: 50, Handler: matchOperator}
-var captureOpType = &operationType{Type: "CAPTURE", NumArgs: 1, Precedence: 50, Handler: captureOperator}
-var testOpType = &operationType{Type: "TEST", NumArgs: 1, Precedence: 50, Handler: testOperator}
-var splitStringOpType = &operationType{Type: "SPLIT", NumArgs: 1, Precedence: 50, Handler: splitStringOperator}
-var changeCaseOpType = &operationType{Type: "CHANGE_CASE", NumArgs: 0, Precedence: 50, Handler: changeCaseOperator}
-var trimOpType = &operationType{Type: "TRIM", NumArgs: 0, Precedence: 50, Handler: trimSpaceOperator}
-
-var loadOpType = &operationType{Type: "LOAD", NumArgs: 1, Precedence: 52, Handler: loadYamlOperator}
-
-var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 50, Handler: keysOperator}
-
-var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator}
-var traversePathOpType = &operationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 55, Handler: traversePathOperator}
-var traverseArrayOpType = &operationType{Type: "TRAVERSE_ARRAY", NumArgs: 2, Precedence: 50, Handler: traverseArrayOperator}
-
-var selfReferenceOpType = &operationType{Type: "SELF", NumArgs: 0, Precedence: 55, Handler: selfOperator}
-var valueOpType = &operationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: valueOperator}
-var referenceOpType = &operationType{Type: "REF", NumArgs: 0, Precedence: 50, Handler: referenceOperator}
-var envOpType = &operationType{Type: "ENV", NumArgs: 0, Precedence: 50, Handler: envOperator}
-var notOpType = &operationType{Type: "NOT", NumArgs: 0, Precedence: 50, Handler: notOperator}
-var toNumberOpType = &operationType{Type: "TO_NUMBER", NumArgs: 0, Precedence: 50, Handler: toNumberOperator}
-var emptyOpType = &operationType{Type: "EMPTY", Precedence: 50, Handler: emptyOperator}
-
-var envsubstOpType = &operationType{Type: "ENVSUBST", NumArgs: 0, Precedence: 50, Handler: envsubstOperator}
-
-var recursiveDescentOpType = &operationType{Type: "RECURSIVE_DESCENT", NumArgs: 0, Precedence: 50, Handler: recursiveDescentOperator}
-
-var selectOpType = &operationType{Type: "SELECT", NumArgs: 1, Precedence: 50, Handler: selectOperator}
-var hasOpType = &operationType{Type: "HAS", NumArgs: 1, Precedence: 50, Handler: hasOperator}
-var uniqueOpType = &operationType{Type: "UNIQUE", NumArgs: 0, Precedence: 50, Handler: unique}
-var uniqueByOpType = &operationType{Type: "UNIQUE_BY", NumArgs: 1, Precedence: 50, Handler: uniqueBy}
-var groupByOpType = &operationType{Type: "GROUP_BY", NumArgs: 1, Precedence: 50, Handler: groupBy}
-var flattenOpType = &operationType{Type: "FLATTEN_BY", NumArgs: 0, Precedence: 50, Handler: flattenOp}
-var deleteChildOpType = &operationType{Type: "DELETE", NumArgs: 1, Precedence: 40, Handler: deleteChildOperator}
-
-type Operation struct {
-	OperationType *operationType
-	Value         interface{}
-	StringValue   string
-	CandidateNode *CandidateNode // used for Value Path elements
-	Preferences   interface{}
-	UpdateAssign  bool // used for assign ops, when true it means we evaluate the rhs given the lhs
-}
-
 func recurseNodeArrayEqual(lhs *CandidateNode, rhs *CandidateNode) bool {
 	if len(lhs.Content) != len(rhs.Content) {
 		return false
@@ -257,6 +95,11 @@ func parseSnippet(value string) (*CandidateNode, error) {
 	if err != nil {
 		return nil, err
 	}
+	if result.Tag == "!!str" {
+		// use the original string value, as
+		// decoding drops new lines
+		return CreateScalarNode(value, value), nil
+	}
 	result.Line = 0
 	result.Column = 0
 	return result, err
@@ -271,8 +114,8 @@ func recursiveNodeEqual(lhs *CandidateNode, rhs *CandidateNode) bool {
 		//process custom tags of scalar nodes.
 		//dont worry about matching tags of maps or arrays.
 
-		lhsTag := lhs.guessTagFromCustomType()
-		rhsTag := rhs.guessTagFromCustomType()
+		lhsTag := lhs.GuessTagFromCustomType()
+		rhsTag := rhs.GuessTagFromCustomType()
 
 		if lhsTag != rhsTag {
 			return false
@@ -318,49 +161,6 @@ func parseInt(numberString string) (int, error) {
 	return int(parsed), err
 }
 
-func headAndLineComment(node *CandidateNode) string {
-	return headComment(node) + lineComment(node)
-}
-
-func headComment(node *CandidateNode) string {
-	return strings.Replace(node.HeadComment, "#", "", 1)
-}
-
-func lineComment(node *CandidateNode) string {
-	return strings.Replace(node.LineComment, "#", "", 1)
-}
-
-func footComment(node *CandidateNode) string {
-	return strings.Replace(node.FootComment, "#", "", 1)
-}
-
-func createValueOperation(value interface{}, stringValue string) *Operation {
-	log.Debug("creating value op for string %v", stringValue)
-	var node = createScalarNode(value, stringValue)
-
-	return &Operation{
-		OperationType: valueOpType,
-		Value:         value,
-		StringValue:   stringValue,
-		CandidateNode: node,
-	}
-}
-
-// debugging purposes only
-func (p *Operation) toString() string {
-	if p == nil {
-		return "OP IS NIL"
-	}
-	if p.OperationType == traversePathOpType {
-		return fmt.Sprintf("%v", p.Value)
-	} else if p.OperationType == selfReferenceOpType {
-		return "SELF"
-	} else if p.OperationType == valueOpType {
-		return fmt.Sprintf("%v (%T)", p.Value, p.Value)
-	}
-	return fmt.Sprintf("%v", p.OperationType.Type)
-}
-
 // use for debugging only
 func NodesToString(collection *list.List) string {
 	if !log.IsEnabledFor(logging.DEBUG) {

pkg/yqlib/lua_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
-var luaScenarios = []formatScenario{
+var luaScenarios = []FormatScenario{
 	{
 		description: "Basic input example",
 		input: `return {
@@ -25,10 +25,10 @@ var luaScenarios = []formatScenario{
 `,
 		expected: `country: Australia
 cities:
-    - Sydney
-    - Melbourne
-    - Brisbane
-    - Perth
+  - Sydney
+  - Melbourne
+  - Brisbane
+  - Perth
 `,
 	},
 	{
@@ -250,10 +250,10 @@ numbers:
 	},
 }
 
-func testLuaScenario(t *testing.T, s formatScenario) {
+func testLuaScenario(t *testing.T, s FormatScenario) {
 	switch s.scenarioType {
 	case "", "decode":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewYamlEncoder(4, false, ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
 	case "encode":
 		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(ConfiguredLuaPreferences)), s.description)
 	case "roundtrip":
@@ -278,7 +278,7 @@ func testLuaScenario(t *testing.T, s formatScenario) {
 }
 
 func documentLuaScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
-	s := i.(formatScenario)
+	s := i.(FormatScenario)
 
 	if s.skipDoc {
 		return
@@ -295,7 +295,7 @@ func documentLuaScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
 	}
 }
 
-func documentLuaDecodeScenario(w *bufio.Writer, s formatScenario) {
+func documentLuaDecodeScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -314,10 +314,10 @@ func documentLuaDecodeScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, fmt.Sprintf("```bash\nyq -oy '%v' sample.lua\n```\n", expression))
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences))))
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewYamlEncoder(ConfiguredYamlPreferences))))
 }
 
-func documentLuaEncodeScenario(w *bufio.Writer, s formatScenario) {
+func documentLuaEncodeScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -359,7 +359,7 @@ func documentLuaEncodeScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, fmt.Sprintf("```lua\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(prefs))))
 }
 
-func documentLuaRoundTripScenario(w *bufio.Writer, s formatScenario) {
+func documentLuaRoundTripScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {

pkg/yqlib/operation.go

@@ -0,0 +1,201 @@
+package yqlib
+
+import "fmt"
+
+type Operation struct {
+	OperationType *operationType
+	Value         interface{}
+	StringValue   string
+	CandidateNode *CandidateNode // used for Value Path elements
+	Preferences   interface{}
+	UpdateAssign  bool // used for assign ops, when true it means we evaluate the rhs given the lhs
+}
+
+type operationType struct {
+	Type                 string
+	NumArgs              uint // number of arguments to the op
+	Precedence           uint
+	Handler              operatorHandler
+	CheckForPostTraverse bool
+	ToString             func(o *Operation) string
+}
+
+var valueToStringFunc = func(p *Operation) string {
+	return fmt.Sprintf("%v (%T)", p.Value, p.Value)
+}
+
+func createValueOperation(value interface{}, stringValue string) *Operation {
+	log.Debug("creating value op for string %v", stringValue)
+	var node = CreateScalarNode(value, stringValue)
+
+	return &Operation{
+		OperationType: valueOpType,
+		Value:         value,
+		StringValue:   stringValue,
+		CandidateNode: node,
+	}
+}
+
+var orOpType = &operationType{Type: "OR", NumArgs: 2, Precedence: 20, Handler: orOperator}
+var andOpType = &operationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: andOperator}
+var reduceOpType = &operationType{Type: "REDUCE", NumArgs: 2, Precedence: 35, Handler: reduceOperator}
+
+var blockOpType = &operationType{Type: "BLOCK", Precedence: 10, NumArgs: 2, Handler: emptyOperator}
+
+var unionOpType = &operationType{Type: "UNION", NumArgs: 2, Precedence: 10, Handler: unionOperator}
+
+var pipeOpType = &operationType{Type: "PIPE", NumArgs: 2, Precedence: 30, Handler: pipeOperator}
+
+var assignOpType = &operationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: assignUpdateOperator}
+var addAssignOpType = &operationType{Type: "ADD_ASSIGN", NumArgs: 2, Precedence: 40, Handler: addAssignOperator}
+var subtractAssignOpType = &operationType{Type: "SUBTRACT_ASSIGN", NumArgs: 2, Precedence: 40, Handler: subtractAssignOperator}
+
+var assignAttributesOpType = &operationType{Type: "ASSIGN_ATTRIBUTES", NumArgs: 2, Precedence: 40, Handler: assignAttributesOperator}
+var assignStyleOpType = &operationType{Type: "ASSIGN_STYLE", NumArgs: 2, Precedence: 40, Handler: assignStyleOperator}
+var assignVariableOpType = &operationType{Type: "ASSIGN_VARIABLE", NumArgs: 2, Precedence: 40, Handler: useWithPipe}
+var assignTagOpType = &operationType{Type: "ASSIGN_TAG", NumArgs: 2, Precedence: 40, Handler: assignTagOperator}
+var assignCommentOpType = &operationType{Type: "ASSIGN_COMMENT", NumArgs: 2, Precedence: 40, Handler: assignCommentsOperator}
+var assignAnchorOpType = &operationType{Type: "ASSIGN_ANCHOR", NumArgs: 2, Precedence: 40, Handler: assignAnchorOperator}
+var assignAliasOpType = &operationType{Type: "ASSIGN_ALIAS", NumArgs: 2, Precedence: 40, Handler: assignAliasOperator}
+
+var multiplyOpType = &operationType{Type: "MULTIPLY", NumArgs: 2, Precedence: 42, Handler: multiplyOperator}
+var multiplyAssignOpType = &operationType{Type: "MULTIPLY_ASSIGN", NumArgs: 2, Precedence: 42, Handler: multiplyAssignOperator}
+
+var divideOpType = &operationType{Type: "DIVIDE", NumArgs: 2, Precedence: 42, Handler: divideOperator}
+
+var moduloOpType = &operationType{Type: "MODULO", NumArgs: 2, Precedence: 42, Handler: moduloOperator}
+
+var addOpType = &operationType{Type: "ADD", NumArgs: 2, Precedence: 42, Handler: addOperator}
+var subtractOpType = &operationType{Type: "SUBTRACT", NumArgs: 2, Precedence: 42, Handler: subtractOperator}
+var alternativeOpType = &operationType{Type: "ALTERNATIVE", NumArgs: 2, Precedence: 42, Handler: alternativeOperator}
+
+var equalsOpType = &operationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: equalsOperator}
+var notEqualsOpType = &operationType{Type: "NOT_EQUALS", NumArgs: 2, Precedence: 40, Handler: notEqualsOperator}
+
+var compareOpType = &operationType{Type: "COMPARE", NumArgs: 2, Precedence: 40, Handler: compareOperator}
+
+// createmap needs to be above union, as we use union to build the components of the objects
+var createMapOpType = &operationType{Type: "CREATE_MAP", NumArgs: 2, Precedence: 15, Handler: createMapOperator}
+
+var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence: 45, Handler: pipeOperator}
+
+var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: lengthOperator}
+var lineOpType = &operationType{Type: "LINE", NumArgs: 0, Precedence: 50, Handler: lineOperator}
+var columnOpType = &operationType{Type: "LINE", NumArgs: 0, Precedence: 50, Handler: columnOperator}
+
+var expressionOpType = &operationType{Type: "EXP", NumArgs: 0, Precedence: 50, Handler: expressionOperator}
+
+var collectOpType = &operationType{Type: "COLLECT", NumArgs: 1, Precedence: 50, Handler: collectOperator}
+var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 50, Handler: mapOperator}
+var filterOpType = &operationType{Type: "FILTER", NumArgs: 1, Precedence: 50, Handler: filterOperator}
+var errorOpType = &operationType{Type: "ERROR", NumArgs: 1, Precedence: 50, Handler: errorOperator}
+var pickOpType = &operationType{Type: "PICK", NumArgs: 1, Precedence: 50, Handler: pickOperator}
+var evalOpType = &operationType{Type: "EVAL", NumArgs: 1, Precedence: 50, Handler: evalOperator}
+var mapValuesOpType = &operationType{Type: "MAP_VALUES", NumArgs: 1, Precedence: 50, Handler: mapValuesOperator}
+
+var formatDateTimeOpType = &operationType{Type: "FORMAT_DATE_TIME", NumArgs: 1, Precedence: 50, Handler: formatDateTime}
+var withDtFormatOpType = &operationType{Type: "WITH_DATE_TIME_FORMAT", NumArgs: 1, Precedence: 50, Handler: withDateTimeFormat}
+var nowOpType = &operationType{Type: "NOW", NumArgs: 0, Precedence: 50, Handler: nowOp}
+var tzOpType = &operationType{Type: "TIMEZONE", NumArgs: 1, Precedence: 50, Handler: tzOp}
+var fromUnixOpType = &operationType{Type: "FROM_UNIX", NumArgs: 0, Precedence: 50, Handler: fromUnixOp}
+var toUnixOpType = &operationType{Type: "TO_UNIX", NumArgs: 0, Precedence: 50, Handler: toUnixOp}
+
+var encodeOpType = &operationType{Type: "ENCODE", NumArgs: 0, Precedence: 50, Handler: encodeOperator}
+var decodeOpType = &operationType{Type: "DECODE", NumArgs: 0, Precedence: 50, Handler: decodeOperator}
+
+var anyOpType = &operationType{Type: "ANY", NumArgs: 0, Precedence: 50, Handler: anyOperator}
+var allOpType = &operationType{Type: "ALL", NumArgs: 0, Precedence: 50, Handler: allOperator}
+var containsOpType = &operationType{Type: "CONTAINS", NumArgs: 1, Precedence: 50, Handler: containsOperator}
+var anyConditionOpType = &operationType{Type: "ANY_CONDITION", NumArgs: 1, Precedence: 50, Handler: anyOperator}
+var allConditionOpType = &operationType{Type: "ALL_CONDITION", NumArgs: 1, Precedence: 50, Handler: allOperator}
+
+var toEntriesOpType = &operationType{Type: "TO_ENTRIES", NumArgs: 0, Precedence: 52, Handler: toEntriesOperator, CheckForPostTraverse: true}
+var fromEntriesOpType = &operationType{Type: "FROM_ENTRIES", NumArgs: 0, Precedence: 50, Handler: fromEntriesOperator}
+var withEntriesOpType = &operationType{Type: "WITH_ENTRIES", NumArgs: 1, Precedence: 50, Handler: withEntriesOperator}
+
+var withOpType = &operationType{Type: "WITH", NumArgs: 1, Precedence: 50, Handler: withOperator}
+
+var splitDocumentOpType = &operationType{Type: "SPLIT_DOC", NumArgs: 0, Precedence: 50, Handler: splitDocumentOperator}
+var getVariableOpType = &operationType{Type: "GET_VARIABLE", NumArgs: 0, Precedence: 55, Handler: getVariableOperator}
+var getStyleOpType = &operationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: getStyleOperator}
+var getTagOpType = &operationType{Type: "GET_TAG", NumArgs: 0, Precedence: 50, Handler: getTagOperator}
+var getKindOpType = &operationType{Type: "GET_KIND", NumArgs: 0, Precedence: 50, Handler: getKindOperator}
+
+var getKeyOpType = &operationType{Type: "GET_KEY", NumArgs: 0, Precedence: 50, Handler: getKeyOperator}
+var isKeyOpType = &operationType{Type: "IS_KEY", NumArgs: 0, Precedence: 50, Handler: isKeyOperator}
+var getParentOpType = &operationType{Type: "GET_PARENT", NumArgs: 0, Precedence: 50, Handler: getParentOperator}
+
+var getCommentOpType = &operationType{Type: "GET_COMMENT", NumArgs: 0, Precedence: 50, Handler: getCommentsOperator}
+var getAnchorOpType = &operationType{Type: "GET_ANCHOR", NumArgs: 0, Precedence: 50, Handler: getAnchorOperator}
+var getAliasOpType = &operationType{Type: "GET_ALIAS", NumArgs: 0, Precedence: 50, Handler: getAliasOperator}
+var getDocumentIndexOpType = &operationType{Type: "GET_DOCUMENT_INDEX", NumArgs: 0, Precedence: 50, Handler: getDocumentIndexOperator}
+var getFilenameOpType = &operationType{Type: "GET_FILENAME", NumArgs: 0, Precedence: 50, Handler: getFilenameOperator}
+var getFileIndexOpType = &operationType{Type: "GET_FILE_INDEX", NumArgs: 0, Precedence: 50, Handler: getFileIndexOperator}
+
+var getPathOpType = &operationType{Type: "GET_PATH", NumArgs: 0, Precedence: 50, Handler: getPathOperator}
+var setPathOpType = &operationType{Type: "SET_PATH", NumArgs: 1, Precedence: 50, Handler: setPathOperator}
+var delPathsOpType = &operationType{Type: "DEL_PATHS", NumArgs: 1, Precedence: 50, Handler: delPathsOperator}
+
+var explodeOpType = &operationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: explodeOperator}
+var sortByOpType = &operationType{Type: "SORT_BY", NumArgs: 1, Precedence: 50, Handler: sortByOperator}
+var reverseOpType = &operationType{Type: "REVERSE", NumArgs: 0, Precedence: 50, Handler: reverseOperator}
+var sortOpType = &operationType{Type: "SORT", NumArgs: 0, Precedence: 50, Handler: sortOperator}
+var shuffleOpType = &operationType{Type: "SHUFFLE", NumArgs: 0, Precedence: 50, Handler: shuffleOperator}
+
+var sortKeysOpType = &operationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Handler: sortKeysOperator}
+
+var joinStringOpType = &operationType{Type: "JOIN", NumArgs: 1, Precedence: 50, Handler: joinStringOperator}
+var subStringOpType = &operationType{Type: "SUBSTR", NumArgs: 1, Precedence: 50, Handler: substituteStringOperator}
+var matchOpType = &operationType{Type: "MATCH", NumArgs: 1, Precedence: 50, Handler: matchOperator}
+var captureOpType = &operationType{Type: "CAPTURE", NumArgs: 1, Precedence: 50, Handler: captureOperator}
+var testOpType = &operationType{Type: "TEST", NumArgs: 1, Precedence: 50, Handler: testOperator}
+var splitStringOpType = &operationType{Type: "SPLIT", NumArgs: 1, Precedence: 50, Handler: splitStringOperator}
+var changeCaseOpType = &operationType{Type: "CHANGE_CASE", NumArgs: 0, Precedence: 50, Handler: changeCaseOperator}
+var trimOpType = &operationType{Type: "TRIM", NumArgs: 0, Precedence: 50, Handler: trimSpaceOperator}
+var toStringOpType = &operationType{Type: "TO_STRING", NumArgs: 0, Precedence: 50, Handler: toStringOperator}
+var stringInterpolationOpType = &operationType{Type: "STRING_INT", NumArgs: 0, Precedence: 50, Handler: stringInterpolationOperator, ToString: valueToStringFunc}
+
+var loadOpType = &operationType{Type: "LOAD", NumArgs: 1, Precedence: 52, Handler: loadOperator}
+var loadStringOpType = &operationType{Type: "LOAD_STRING", NumArgs: 1, Precedence: 52, Handler: loadStringOperator}
+
+var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 50, Handler: keysOperator}
+
+var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator}
+
+var traversePathOpType = &operationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 55, Handler: traversePathOperator,
+	ToString: func(p *Operation) string {
+		return fmt.Sprintf("%v", p.Value)
+	}}
+
+var traverseArrayOpType = &operationType{Type: "TRAVERSE_ARRAY", NumArgs: 2, Precedence: 50, Handler: traverseArrayOperator}
+
+var selfReferenceOpType = &operationType{Type: "SELF", NumArgs: 0, Precedence: 55, Handler: selfOperator}
+var valueOpType = &operationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: valueOperator, ToString: valueToStringFunc}
+var referenceOpType = &operationType{Type: "REF", NumArgs: 0, Precedence: 50, Handler: referenceOperator}
+var envOpType = &operationType{Type: "ENV", NumArgs: 0, Precedence: 50, Handler: envOperator}
+var notOpType = &operationType{Type: "NOT", NumArgs: 0, Precedence: 50, Handler: notOperator}
+var toNumberOpType = &operationType{Type: "TO_NUMBER", NumArgs: 0, Precedence: 50, Handler: toNumberOperator}
+var emptyOpType = &operationType{Type: "EMPTY", Precedence: 50, Handler: emptyOperator}
+
+var envsubstOpType = &operationType{Type: "ENVSUBST", NumArgs: 0, Precedence: 50, Handler: envsubstOperator}
+
+var recursiveDescentOpType = &operationType{Type: "RECURSIVE_DESCENT", NumArgs: 0, Precedence: 50, Handler: recursiveDescentOperator}
+
+var selectOpType = &operationType{Type: "SELECT", NumArgs: 1, Precedence: 50, Handler: selectOperator}
+var hasOpType = &operationType{Type: "HAS", NumArgs: 1, Precedence: 50, Handler: hasOperator}
+var uniqueOpType = &operationType{Type: "UNIQUE", NumArgs: 0, Precedence: 50, Handler: unique}
+var uniqueByOpType = &operationType{Type: "UNIQUE_BY", NumArgs: 1, Precedence: 50, Handler: uniqueBy}
+var groupByOpType = &operationType{Type: "GROUP_BY", NumArgs: 1, Precedence: 50, Handler: groupBy}
+var flattenOpType = &operationType{Type: "FLATTEN_BY", NumArgs: 0, Precedence: 50, Handler: flattenOp}
+var deleteChildOpType = &operationType{Type: "DELETE", NumArgs: 1, Precedence: 40, Handler: deleteChildOperator}
+
+// debugging purposes only
+func (p *Operation) toString() string {
+	if p == nil {
+		return "OP IS NIL"
+	}
+	if p.OperationType.ToString != nil {
+		return p.OperationType.ToString(p)
+	}
+	return fmt.Sprintf("%v", p.OperationType.Type)
+}

pkg/yqlib/operator_add.go

@@ -74,11 +74,11 @@ func add(_ *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Candida
 
 func addScalars(context Context, target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error {
 	lhsTag := lhs.Tag
-	rhsTag := rhs.guessTagFromCustomType()
+	rhsTag := rhs.GuessTagFromCustomType()
 	lhsIsCustom := false
 	if !strings.HasPrefix(lhsTag, "!!") {
 		// custom tag - we have to have a guess
-		lhsTag = lhs.guessTagFromCustomType()
+		lhsTag = lhs.GuessTagFromCustomType()
 		lhsIsCustom = true
 	}
 

pkg/yqlib/operator_assign.go

@@ -6,6 +6,10 @@ type assignPreferences struct {
 	ClobberCustomTags   bool
 }
 
+func PerformAssignment(context Context, path []interface{}) {
+
+}
+
 func assignUpdateFunc(prefs assignPreferences) crossFunctionCalculation {
 	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 		if !prefs.OnlyWriteNull || lhs.Tag == "!!null" {

pkg/yqlib/operator_comments.go

@@ -80,9 +80,10 @@ func getCommentsOperator(_ *dataTreeNavigator, context Context, expressionNode *
 	log.Debugf("GetComments operator!")
 	var results = list.New()
 
-	yamlPrefs := NewDefaultYamlPreferences()
+	yamlPrefs := ConfiguredYamlPreferences.Copy()
 	yamlPrefs.PrintDocSeparators = false
 	yamlPrefs.UnwrapScalar = false
+	yamlPrefs.ColorsEnabled = false
 
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
@@ -94,7 +95,7 @@ func getCommentsOperator(_ *dataTreeNavigator, context Context, expressionNode *
 			var chompRegexp = regexp.MustCompile(`\n$`)
 			var output bytes.Buffer
 			var writer = bufio.NewWriter(&output)
-			var encoder = NewYamlEncoder(2, false, yamlPrefs)
+			var encoder = NewYamlEncoder(yamlPrefs)
 			if err := encoder.PrintLeadingContent(writer, candidate.LeadingContent); err != nil {
 				return Context{}, err
 			}

pkg/yqlib/operator_compare.go

@@ -69,8 +69,8 @@ func compareDateTime(layout string, prefs compareTypePref, lhs *CandidateNode, r
 }
 
 func compareScalars(context Context, prefs compareTypePref, lhs *CandidateNode, rhs *CandidateNode) (bool, error) {
-	lhsTag := lhs.guessTagFromCustomType()
-	rhsTag := rhs.guessTagFromCustomType()
+	lhsTag := lhs.GuessTagFromCustomType()
+	rhsTag := rhs.GuessTagFromCustomType()
 
 	isDateTime := lhs.Tag == "!!timestamp"
 	// if the lhs is a string, it might be a timestamp in a custom format.

pkg/yqlib/operator_datetime.go

@@ -141,7 +141,7 @@ func fromUnixOp(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Conte
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 
-		actualTag := candidate.guessTagFromCustomType()
+		actualTag := candidate.GuessTagFromCustomType()
 
 		if actualTag != "!!int" && actualTag != "!!float" {
 			return Context{}, fmt.Errorf("from_unix only works on numbers, found %v instead", candidate.Tag)

pkg/yqlib/operator_divide.go

@@ -32,11 +32,11 @@ func divide(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateN
 
 func divideScalars(target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error {
 	lhsTag := lhs.Tag
-	rhsTag := rhs.guessTagFromCustomType()
+	rhsTag := rhs.GuessTagFromCustomType()
 	lhsIsCustom := false
 	if !strings.HasPrefix(lhsTag, "!!") {
 		// custom tag - we have to have a guess
-		lhsTag = lhs.guessTagFromCustomType()
+		lhsTag = lhs.GuessTagFromCustomType()
 		lhsIsCustom = true
 	}
 

pkg/yqlib/operator_encoder_decoder.go

@@ -9,35 +9,11 @@ import (
 	"strings"
 )
 
-func configureEncoder(format PrinterOutputFormat, indent int) Encoder {
-	switch format {
-	case JSONOutputFormat:
-		return NewJSONEncoder(indent, false, false)
-	case PropsOutputFormat:
-		return NewPropertiesEncoder(true)
-	case CSVOutputFormat:
-		return NewCsvEncoder(',')
-	case TSVOutputFormat:
-		return NewCsvEncoder('\t')
-	case YamlOutputFormat:
-		return NewYamlEncoder(indent, false, ConfiguredYamlPreferences)
-	case XMLOutputFormat:
-		return NewXMLEncoder(indent, ConfiguredXMLPreferences)
-	case Base64OutputFormat:
-		return NewBase64Encoder()
-	case UriOutputFormat:
-		return NewUriEncoder()
-	case ShOutputFormat:
-		return NewShEncoder()
-	}
-	panic("invalid encoder")
-}
-
 func encodeToString(candidate *CandidateNode, prefs encoderPreferences) (string, error) {
 	var output bytes.Buffer
 	log.Debug("printing with indent: %v", prefs.indent)
 
-	encoder := configureEncoder(prefs.format, prefs.indent)
+	encoder := prefs.format.GetInDocumentEncoder(prefs.indent)
 	if encoder == nil {
 		return "", errors.New("no support for output format")
 	}
@@ -48,11 +24,12 @@ func encodeToString(candidate *CandidateNode, prefs encoderPreferences) (string,
 }
 
 type encoderPreferences struct {
-	format PrinterOutputFormat
+	format *Format
 	indent int
 }
 
 /* encodes object as yaml string */
+var chomper = regexp.MustCompile("\n+$")
 
 func encodeOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	preferences := expressionNode.Operation.Preferences.(encoderPreferences)
@@ -60,7 +37,6 @@ func encodeOperator(_ *dataTreeNavigator, context Context, expressionNode *Expre
 
 	hasOnlyOneNewLine := regexp.MustCompile("[^\n].*\n$")
 	endWithNewLine := regexp.MustCompile(".*\n$")
-	chomper := regexp.MustCompile("\n+$")
 
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
@@ -83,9 +59,9 @@ func encodeOperator(_ *dataTreeNavigator, context Context, expressionNode *Expre
 		}
 
 		// dont print a newline when printing json on a single line.
-		if (preferences.format == JSONOutputFormat && preferences.indent == 0) ||
-			preferences.format == CSVOutputFormat ||
-			preferences.format == TSVOutputFormat {
+		if (preferences.format == JSONFormat && preferences.indent == 0) ||
+			preferences.format == CSVFormat ||
+			preferences.format == TSVFormat {
 			stringValue = chomper.ReplaceAllString(stringValue, "")
 		}
 
@@ -95,30 +71,7 @@ func encodeOperator(_ *dataTreeNavigator, context Context, expressionNode *Expre
 }
 
 type decoderPreferences struct {
-	format InputFormat
-}
-
-func createDecoder(format InputFormat) Decoder {
-	var decoder Decoder
-	switch format {
-	case JsonInputFormat:
-		decoder = NewJSONDecoder()
-	case YamlInputFormat:
-		decoder = NewYamlDecoder(ConfiguredYamlPreferences)
-	case XMLInputFormat:
-		decoder = NewXMLDecoder(ConfiguredXMLPreferences)
-	case Base64InputFormat:
-		decoder = NewBase64Decoder()
-	case PropertiesInputFormat:
-		decoder = NewPropertiesDecoder()
-	case CSVObjectInputFormat:
-		decoder = NewCSVObjectDecoder(ConfiguredCsvPreferences)
-	case TSVObjectInputFormat:
-		decoder = NewCSVObjectDecoder(ConfiguredTsvPreferences)
-	case UriInputFormat:
-		decoder = NewUriDecoder()
-	}
-	return decoder
+	format *Format
 }
 
 /* takes a string and decodes it back into an object */
@@ -126,7 +79,7 @@ func decodeOperator(_ *dataTreeNavigator, context Context, expressionNode *Expre
 
 	preferences := expressionNode.Operation.Preferences.(decoderPreferences)
 
-	decoder := createDecoder(preferences.format)
+	decoder := preferences.format.DecoderFactory()
 	if decoder == nil {
 		return Context{}, errors.New("no support for input format")
 	}

pkg/yqlib/operator_entries.go

@@ -64,7 +64,7 @@ func toEntriesOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode)
 func parseEntry(candidateNode *CandidateNode, position int) (*CandidateNode, *CandidateNode, error) {
 	prefs := traversePreferences{DontAutoCreate: true}
 
-	keyResults, err := traverseMap(Context{}, candidateNode, createStringScalarNode("key"), prefs, false)
+	keyResults, err := traverseMap(Context{}, candidateNode, CreateStringScalarNode("key"), prefs, false)
 
 	if err != nil {
 		return nil, nil, err
@@ -72,7 +72,7 @@ func parseEntry(candidateNode *CandidateNode, position int) (*CandidateNode, *Ca
 		return nil, nil, fmt.Errorf("expected to find one 'key' entry but found %v in position %v", keyResults.Len(), position)
 	}
 
-	valueResults, err := traverseMap(Context{}, candidateNode, createStringScalarNode("value"), prefs, false)
+	valueResults, err := traverseMap(Context{}, candidateNode, CreateStringScalarNode("value"), prefs, false)
 
 	if err != nil {
 		return nil, nil, err

pkg/yqlib/operator_load.go

@@ -15,8 +15,7 @@ var LoadYamlPreferences = YamlPreferences{
 }
 
 type loadPrefs struct {
-	loadAsString bool
-	decoder      Decoder
+	decoder Decoder
 }
 
 func loadString(filename string) (*CandidateNode, error) {
@@ -31,7 +30,7 @@ func loadString(filename string) (*CandidateNode, error) {
 	return &CandidateNode{Kind: ScalarNode, Tag: "!!str", Value: string(filebytes)}, nil
 }
 
-func loadYaml(filename string, decoder Decoder) (*CandidateNode, error) {
+func loadWithDecoder(filename string, decoder Decoder) (*CandidateNode, error) {
 	if decoder == nil {
 		return nil, fmt.Errorf("could not load %s", filename)
 	}
@@ -62,8 +61,39 @@ func loadYaml(filename string, decoder Decoder) (*CandidateNode, error) {
 	return sequenceNode, nil
 }
 
-func loadYamlOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	log.Debugf("loadYamlOperator")
+func loadStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	log.Debugf("loadString")
+
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+
+		rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS)
+		if err != nil {
+			return Context{}, err
+		}
+		if rhs.MatchingNodes.Front() == nil {
+			return Context{}, fmt.Errorf("filename expression returned nil")
+		}
+		nameCandidateNode := rhs.MatchingNodes.Front().Value.(*CandidateNode)
+
+		filename := nameCandidateNode.Value
+
+		contentsCandidate, err := loadString(filename)
+		if err != nil {
+			return Context{}, fmt.Errorf("Failed to load %v: %w", filename, err)
+		}
+
+		results.PushBack(contentsCandidate)
+
+	}
+
+	return context.ChildContext(results), nil
+}
+
+func loadOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	log.Debugf("loadOperator")
 
 	loadPrefs := expressionNode.Operation.Preferences.(loadPrefs)
 
@@ -86,13 +116,7 @@ func loadYamlOperator(d *dataTreeNavigator, context Context, expressionNode *Exp
 
 		filename := nameCandidateNode.Value
 
-		var contentsCandidate *CandidateNode
-
-		if loadPrefs.loadAsString {
-			contentsCandidate, err = loadString(filename)
-		} else {
-			contentsCandidate, err = loadYaml(filename, loadPrefs.decoder)
-		}
+		contentsCandidate, err := loadWithDecoder(filename, loadPrefs.decoder)
 		if err != nil {
 			return Context{}, fmt.Errorf("Failed to load %v: %w", filename, err)
 		}

pkg/yqlib/operator_modulo.go

@@ -33,11 +33,11 @@ func modulo(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateN
 
 func moduloScalars(target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error {
 	lhsTag := lhs.Tag
-	rhsTag := rhs.guessTagFromCustomType()
+	rhsTag := rhs.GuessTagFromCustomType()
 	lhsIsCustom := false
 	if !strings.HasPrefix(lhsTag, "!!") {
 		// custom tag - we have to have a guess
-		lhsTag = lhs.guessTagFromCustomType()
+		lhsTag = lhs.GuessTagFromCustomType()
 		lhsIsCustom = true
 	}
 

pkg/yqlib/operator_multiply.go

@@ -79,11 +79,11 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
 
 func multiplyScalars(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	lhsTag := lhs.Tag
-	rhsTag := rhs.guessTagFromCustomType()
+	rhsTag := rhs.GuessTagFromCustomType()
 	lhsIsCustom := false
 	if !strings.HasPrefix(lhsTag, "!!") {
 		// custom tag - we have to have a guess
-		lhsTag = lhs.guessTagFromCustomType()
+		lhsTag = lhs.GuessTagFromCustomType()
 		lhsIsCustom = true
 	}
 

pkg/yqlib/operator_parent.go

@@ -2,15 +2,30 @@ package yqlib
 
 import "container/list"
 
-func getParentOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {
+type parentOpPreferences struct {
+	Level int
+}
+
+func getParentOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("getParentOperator")
 
 	var results = list.New()
 
+	prefs := expressionNode.Operation.Preferences.(parentOpPreferences)
+
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
-		if candidate.Parent != nil {
-			results.PushBack(candidate.Parent)
+		currentLevel := 0
+		for currentLevel < prefs.Level && candidate != nil {
+			log.Debugf("currentLevel: %v, desired: %v", currentLevel, prefs.Level)
+			log.Debugf("candidate: %v", NodeToString(candidate))
+			candidate = candidate.Parent
+			currentLevel++
+		}
+
+		log.Debugf("found candidate: %v", NodeToString(candidate))
+		if candidate != nil {
+			results.PushBack(candidate)
 		}
 	}
 

pkg/yqlib/operator_parent_test.go

@@ -21,6 +21,23 @@ var parentOperatorScenarios = []expressionScenario{
 			"D0, P[b], (!!map)::{fruit: banana, name: sam}\n",
 		},
 	},
+	{
+		description:    "N-th parent",
+		subdescription: "You can optionally supply the number of levels to go up for the parent, the default being 1.",
+		document:       "a:\n  b:\n    c: cat\n",
+		expression:     `.a.b.c | parent(2)`,
+		expected: []string{
+			"D0, P[a], (!!map)::b:\n    c: cat\n",
+		},
+	},
+	{
+		description: "N-th parent - another level",
+		document:    "a:\n  b:\n    c: cat\n",
+		expression:  `.a.b.c | parent(3)`,
+		expected: []string{
+			"D0, P[], (!!map)::a:\n    b:\n        c: cat\n",
+		},
+	},
 	{
 		description: "No parent",
 		document:    `{}`,

pkg/yqlib/operator_sort.go

@@ -92,12 +92,12 @@ func (a sortableNodeArray) compare(lhs *CandidateNode, rhs *CandidateNode, dateT
 
 	if !strings.HasPrefix(lhsTag, "!!") {
 		// custom tag - we have to have a guess
-		lhsTag = lhs.guessTagFromCustomType()
+		lhsTag = lhs.GuessTagFromCustomType()
 	}
 
 	if !strings.HasPrefix(rhsTag, "!!") {
 		// custom tag - we have to have a guess
-		rhsTag = rhs.guessTagFromCustomType()
+		rhsTag = rhs.GuessTagFromCustomType()
 	}
 
 	isDateTime := lhsTag == "!!timestamp" && rhsTag == "!!timestamp"

pkg/yqlib/operator_strings.go

@@ -7,16 +7,130 @@ import (
 	"strings"
 )
 
+var StringInterpolationEnabled = true
+
 type changeCasePrefs struct {
 	ToUpperCase bool
 }
 
+func encodeToYamlString(node *CandidateNode) (string, error) {
+	encoderPrefs := encoderPreferences{
+		format: YamlFormat,
+		indent: ConfiguredYamlPreferences.Indent,
+	}
+	result, err := encodeToString(node, encoderPrefs)
+
+	if err != nil {
+		return "", err
+	}
+	return chomper.ReplaceAllString(result, ""), nil
+}
+
+func evaluate(d *dataTreeNavigator, context Context, expStr string) (string, error) {
+	exp, err := ExpressionParser.ParseExpression(expStr)
+	if err != nil {
+		return "", err
+	}
+	result, err := d.GetMatchingNodes(context, exp)
+	if err != nil {
+		return "", err
+	}
+	if result.MatchingNodes.Len() == 0 {
+		return "", nil
+	}
+	node := result.MatchingNodes.Front().Value.(*CandidateNode)
+	if node.Kind != ScalarNode {
+		return encodeToYamlString(node)
+	}
+	return node.Value, nil
+}
+
+func interpolate(d *dataTreeNavigator, context Context, str string) (string, error) {
+	var sb strings.Builder
+	var expSb strings.Builder
+	inExpression := false
+	nestedBracketsCounter := 0
+	runes := []rune(str)
+	for i := 0; i < len(runes); i++ {
+		char := runes[i]
+		if !inExpression {
+			if char == '\\' && i != len(runes)-1 && runes[i+1] == '(' {
+				inExpression = true
+				i = i + 1 // skip over the next open bracket
+				continue
+			}
+			sb.WriteRune(char)
+		} else { // we are in an expression
+			if char == ')' && nestedBracketsCounter == 0 {
+				// finished the expression!
+				log.Debugf("Expression is :%v", expSb.String())
+				value, err := evaluate(d, context, expSb.String())
+				if err != nil {
+					return "", err
+				}
+
+				inExpression = false
+				expSb = strings.Builder{} // reset this
+
+				sb.WriteString(value)
+				continue
+			} else if char == '(' {
+				nestedBracketsCounter++
+			} else if char == '\\' && i != len(runes)-1 && runes[i+1] == ')' {
+				// close brackets is escaped, skip over it
+				expSb.WriteRune(char)
+				expSb.WriteRune(runes[i+1])
+				i = i + 1
+				continue
+			} else if char == ')' {
+				nestedBracketsCounter--
+			}
+			expSb.WriteRune(char)
+		}
+
+	}
+	if inExpression {
+		return "", fmt.Errorf("unclosed interpolation string \\(")
+	}
+	return sb.String(), nil
+}
+
+func stringInterpolationOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	if !StringInterpolationEnabled {
+		return context.SingleChildContext(
+			CreateScalarNode(expressionNode.Operation.StringValue, expressionNode.Operation.StringValue),
+		), nil
+	}
+	if context.MatchingNodes.Len() == 0 {
+		value, err := interpolate(d, context, expressionNode.Operation.StringValue)
+		if err != nil {
+			return Context{}, err
+		}
+		node := CreateScalarNode(value, value)
+		return context.SingleChildContext(node), nil
+	}
+
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		value, err := interpolate(d, context.SingleChildContext(candidate), expressionNode.Operation.StringValue)
+		if err != nil {
+			return Context{}, err
+		}
+		node := CreateScalarNode(value, value)
+		results.PushBack(node)
+	}
+
+	return context.ChildContext(results), nil
+}
+
 func trimSpaceOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {
 	results := list.New()
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		node := el.Value.(*CandidateNode)
 
-		if node.guessTagFromCustomType() != "!!str" {
+		if node.GuessTagFromCustomType() != "!!str" {
 			return Context{}, fmt.Errorf("cannot trim %v, can only operate on strings. ", node.Tag)
 		}
 
@@ -29,6 +143,31 @@ func trimSpaceOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode)
 	return context.ChildContext(results), nil
 }
 
+func toStringOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {
+	results := list.New()
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		node := el.Value.(*CandidateNode)
+		var newStringNode *CandidateNode
+		if node.Tag == "!!str" {
+			newStringNode = node.CreateReplacement(ScalarNode, "!!str", node.Value)
+		} else if node.Kind == ScalarNode {
+			newStringNode = node.CreateReplacement(ScalarNode, "!!str", node.Value)
+			newStringNode.Style = DoubleQuotedStyle
+		} else {
+			result, err := encodeToYamlString(node)
+			if err != nil {
+				return Context{}, err
+			}
+			newStringNode = node.CreateReplacement(ScalarNode, "!!str", result)
+			newStringNode.Style = DoubleQuotedStyle
+		}
+		newStringNode.Tag = "!!str"
+		results.PushBack(newStringNode)
+	}
+
+	return context.ChildContext(results), nil
+}
+
 func changeCaseOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	results := list.New()
 	prefs := expressionNode.Operation.Preferences.(changeCasePrefs)
@@ -36,7 +175,7 @@ func changeCaseOperator(_ *dataTreeNavigator, context Context, expressionNode *E
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		node := el.Value.(*CandidateNode)
 
-		if node.guessTagFromCustomType() != "!!str" {
+		if node.GuessTagFromCustomType() != "!!str" {
 			return Context{}, fmt.Errorf("cannot change case with %v, can only operate on strings. ", node.Tag)
 		}
 
@@ -107,7 +246,7 @@ func substituteStringOperator(d *dataTreeNavigator, context Context, expressionN
 
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		node := el.Value.(*CandidateNode)
-		if node.guessTagFromCustomType() != "!!str" {
+		if node.GuessTagFromCustomType() != "!!str" {
 			return Context{}, fmt.Errorf("cannot substitute with %v, can only substitute strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag)
 		}
 
@@ -122,29 +261,29 @@ func substituteStringOperator(d *dataTreeNavigator, context Context, expressionN
 func addMatch(original []*CandidateNode, match string, offset int, name string) []*CandidateNode {
 
 	newContent := append(original,
-		createScalarNode("string", "string"))
+		CreateScalarNode("string", "string"))
 
 	if offset < 0 {
 		// offset of -1 means there was no match, force a null value like jq
 		newContent = append(newContent,
-			createScalarNode(nil, "null"),
+			CreateScalarNode(nil, "null"),
 		)
 	} else {
 		newContent = append(newContent,
-			createScalarNode(match, match),
+			CreateScalarNode(match, match),
 		)
 	}
 
 	newContent = append(newContent,
-		createScalarNode("offset", "offset"),
-		createScalarNode(offset, fmt.Sprintf("%v", offset)),
-		createScalarNode("length", "length"),
-		createScalarNode(len(match), fmt.Sprintf("%v", len(match))))
+		CreateScalarNode("offset", "offset"),
+		CreateScalarNode(offset, fmt.Sprintf("%v", offset)),
+		CreateScalarNode("length", "length"),
+		CreateScalarNode(len(match), fmt.Sprintf("%v", len(match))))
 
 	if name != "" {
 		newContent = append(newContent,
-			createScalarNode("name", "name"),
-			createScalarNode(name, name),
+			CreateScalarNode("name", "name"),
+			CreateScalarNode(name, name),
 		)
 	}
 	return newContent
@@ -191,7 +330,7 @@ func match(matchPrefs matchPreferences, regEx *regexp.Regexp, candidate *Candida
 
 		node := candidate.CreateReplacement(MappingNode, "!!map", "")
 		node.AddChildren(addMatch(node.Content, match, allIndices[i][0], ""))
-		node.AddKeyValueChild(createScalarNode("captures", "captures"), capturesListNode)
+		node.AddKeyValueChild(CreateScalarNode("captures", "captures"), capturesListNode)
 		results.PushBack(node)
 
 	}
@@ -214,15 +353,15 @@ func capture(matchPrefs matchPreferences, regEx *regexp.Regexp, candidate *Candi
 		_, submatches := matches[0], matches[1:]
 		for j, submatch := range submatches {
 
-			keyNode := createScalarNode(subNames[j+1], subNames[j+1])
+			keyNode := CreateScalarNode(subNames[j+1], subNames[j+1])
 			var valueNode *CandidateNode
 
 			offset := allIndices[i][2+j*2]
 			// offset of -1 means there was no match, force a null value like jq
 			if offset < 0 {
-				valueNode = createScalarNode(nil, "null")
+				valueNode = CreateScalarNode(nil, "null")
 			} else {
-				valueNode = createScalarNode(submatch, submatch)
+				valueNode = CreateScalarNode(submatch, submatch)
 			}
 			capturesNode.AddKeyValueChild(keyNode, valueNode)
 		}
@@ -286,7 +425,7 @@ func matchOperator(d *dataTreeNavigator, context Context, expressionNode *Expres
 
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		node := el.Value.(*CandidateNode)
-		if node.guessTagFromCustomType() != "!!str" {
+		if node.GuessTagFromCustomType() != "!!str" {
 			return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag)
 		}
 
@@ -306,7 +445,7 @@ func captureOperator(d *dataTreeNavigator, context Context, expressionNode *Expr
 
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		node := el.Value.(*CandidateNode)
-		if node.guessTagFromCustomType() != "!!str" {
+		if node.GuessTagFromCustomType() != "!!str" {
 			return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag)
 		}
 		capture(matchPrefs, regEx, node, node.Value, results)
@@ -326,7 +465,7 @@ func testOperator(d *dataTreeNavigator, context Context, expressionNode *Express
 
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		node := el.Value.(*CandidateNode)
-		if node.guessTagFromCustomType() != "!!str" {
+		if node.GuessTagFromCustomType() != "!!str" {
 			return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag)
 		}
 		matches := regEx.FindStringSubmatch(node.Value)
@@ -396,7 +535,7 @@ func splitStringOperator(d *dataTreeNavigator, context Context, expressionNode *
 			continue
 		}
 
-		if node.guessTagFromCustomType() != "!!str" {
+		if node.GuessTagFromCustomType() != "!!str" {
 			return Context{}, fmt.Errorf("cannot split %v, can only split strings", node.Tag)
 		}
 		kind, tag, content := split(node.Value, splitStr)

pkg/yqlib/operator_strings_test.go

@@ -5,6 +5,64 @@ import (
 )
 
 var stringsOperatorScenarios = []expressionScenario{
+	{
+		description: "Interpolation",
+		document:    "value: things\nanother: stuff",
+		expression:  `.message = "I like \(.value) and \(.another)"`,
+		expected: []string{
+			"D0, P[], (!!map)::value: things\nanother: stuff\nmessage: I like things and stuff\n",
+		},
+	},
+	{
+		description: "Interpolation - not a string",
+		document:    `value: {an: apple}`,
+		expression:  `.message = "I like \(.value)"`,
+		expected: []string{
+			"D0, P[], (!!map)::value: {an: apple}\nmessage: 'I like {an: apple}'\n",
+		},
+	},
+	{
+		skipDoc:     true,
+		description: "Interpolation - just escape",
+		expression:  `"\\"`,
+		expected: []string{
+			"D0, P[], (!!str)::\\\\\n",
+		},
+	},
+	{
+		skipDoc:     true,
+		description: "Interpolation - nested",
+		document:    `value: things`,
+		expression:  `"Hi \( (.value) )"`,
+		expected: []string{
+			"D0, P[], (!!str)::Hi things\n",
+		},
+	},
+	{
+		skipDoc:     true,
+		description: "Interpolation - don't",
+		document:    `value: things`,
+		expression:  `"Hi (.value)"`,
+		expected: []string{
+			"D0, P[], (!!str)::Hi (.value)\n",
+		},
+	},
+	{
+		skipDoc:     true,
+		description: "Interpolation - random close bracket",
+		document:    `value: things`,
+		expression:  `"Hi )"`,
+		expected: []string{
+			"D0, P[], (!!str)::Hi )\n",
+		},
+	},
+	{
+		skipDoc:       true,
+		description:   "Interpolation - unclosed interpolation string",
+		document:      `value: things`,
+		expression:    `"Hi \("`,
+		expectedError: "unclosed interpolation string \\(",
+	},
 	{
 		description:    "To up (upper) case",
 		subdescription: "Works with unicode characters",
@@ -267,6 +325,15 @@ var stringsOperatorScenarios = []expressionScenario{
 		expression: `split("; ")`,
 		expected:   []string{},
 	},
+	{
+		description:    "To string",
+		subdescription: "Note that you may want to force `yq` to leave scalar values wrapped by passing in `--unwrapScalar=false` or `-r=f`",
+		document:       `[1, true, null, ~, cat, {an: object}, [array, 2]]`,
+		expression:     ".[] |= to_string",
+		expected: []string{
+			"D0, P[], (!!seq)::[\"1\", \"true\", \"null\", \"~\", cat, \"{an: object}\", \"[array, 2]\"]\n",
+		},
+	},
 }
 
 func TestStringsOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_subtract.go

@@ -77,13 +77,13 @@ func subtractScalars(context Context, target *CandidateNode, lhs *CandidateNode,
 	lhsIsCustom := false
 	if !strings.HasPrefix(lhsTag, "!!") {
 		// custom tag - we have to have a guess
-		lhsTag = lhs.guessTagFromCustomType()
+		lhsTag = lhs.GuessTagFromCustomType()
 		lhsIsCustom = true
 	}
 
 	if !strings.HasPrefix(rhsTag, "!!") {
 		// custom tag - we have to have a guess
-		rhsTag = rhs.guessTagFromCustomType()
+		rhsTag = rhs.GuessTagFromCustomType()
 	}
 
 	isDateTime := lhsTag == "!!timestamp"

pkg/yqlib/operator_traverse_path.go

@@ -54,7 +54,7 @@ func traverse(context Context, matchingNode *CandidateNode, operation *Operation
 	switch matchingNode.Kind {
 	case MappingNode:
 		log.Debug("its a map with %v entries", len(matchingNode.Content)/2)
-		return traverseMap(context, matchingNode, createStringScalarNode(operation.StringValue), operation.Preferences.(traversePreferences), false)
+		return traverseMap(context, matchingNode, CreateStringScalarNode(operation.StringValue), operation.Preferences.(traversePreferences), false)
 
 	case SequenceNode:
 		log.Debug("its a sequence of %v things!", len(matchingNode.Content))
@@ -149,7 +149,7 @@ func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesT
 
 func traverseMapWithIndices(context Context, candidate *CandidateNode, indices []*CandidateNode, prefs traversePreferences) (*list.List, error) {
 	if len(indices) == 0 {
-		return traverseMap(context, candidate, createStringScalarNode(""), prefs, true)
+		return traverseMap(context, candidate, CreateStringScalarNode(""), prefs, true)
 	}
 
 	var matchingNodeMap = list.New()
@@ -196,7 +196,7 @@ func traverseArrayWithIndices(node *CandidateNode, indices []*CandidateNode, pre
 				node.Style = 0
 			}
 
-			valueNode := createScalarNode(nil, "null")
+			valueNode := CreateScalarNode(nil, "null")
 			node.AddChild(valueNode)
 			contentLength = len(node.Content)
 		}

pkg/yqlib/operators_test.go

@@ -32,7 +32,9 @@ type expressionScenario struct {
 }
 
 func TestMain(m *testing.M) {
-	logging.SetLevel(logging.DEBUG, "")
+	logging.SetLevel(logging.ERROR, "")
+	ConfiguredYamlPreferences.ColorsEnabled = false
+	ConfiguredJSONPreferences.ColorsEnabled = false
 	Now = func() time.Time {
 		return time.Date(2021, time.May, 19, 1, 2, 3, 4, time.UTC)
 	}
@@ -40,11 +42,12 @@ func TestMain(m *testing.M) {
 	os.Exit(code)
 }
 
-func NewSimpleYamlPrinter(writer io.Writer, _ PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
-	prefs := NewDefaultYamlPreferences()
+func NewSimpleYamlPrinter(writer io.Writer, unwrapScalar bool, indent int, printDocSeparators bool) Printer {
+	prefs := ConfiguredYamlPreferences.Copy()
 	prefs.PrintDocSeparators = printDocSeparators
 	prefs.UnwrapScalar = unwrapScalar
-	return NewPrinter(NewYamlEncoder(indent, colorsEnabled, prefs), NewSinglePrinterWriter(writer))
+	prefs.Indent = indent
+	return NewPrinter(NewYamlEncoder(prefs), NewSinglePrinterWriter(writer))
 }
 
 func readDocument(content string, fakefilename string, fakeFileIndex int) (*list.List, error) {
@@ -107,14 +110,14 @@ func testScenario(t *testing.T, s *expressionScenario) {
 
 	if s.requiresFormat != "" {
 		format := s.requiresFormat
-		inputFormat, err := InputFormatFromString(format)
+		inputFormat, err := FormatFromString(format)
 		if err != nil {
 			t.Error(err)
 		}
-		if decoder := createDecoder(inputFormat); decoder == nil {
+		if decoder := inputFormat.DecoderFactory(); decoder == nil {
 			t.Skipf("no support for %s input format", format)
 		}
-		outputFormat, err := OutputFormatFromString(format)
+		outputFormat, err := FormatFromString(format)
 		if err != nil {
 			t.Error(err)
 		}
@@ -132,7 +135,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
 func resultToString(t *testing.T, n *CandidateNode) string {
 	var valueBuffer bytes.Buffer
 	log.Debugf("printing result %v", NodeToString(n))
-	printer := NewSimpleYamlPrinter(bufio.NewWriter(&valueBuffer), YamlOutputFormat, true, false, 4, true)
+	printer := NewSimpleYamlPrinter(bufio.NewWriter(&valueBuffer), true, 4, true)
 
 	err := printer.PrintResults(n.AsList())
 	if err != nil {
@@ -182,7 +185,7 @@ func copySnippet(source string, out *os.File) error {
 
 func formatYaml(yaml string, filename string) string {
 	var output bytes.Buffer
-	printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
+	printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), true, 2, true)
 
 	node, err := getExpressionParser().ParseExpression(".. style= \"\"")
 	if err != nil {
@@ -331,7 +334,7 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) {
 func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formattedDoc string, formattedDoc2 string) {
 	var output bytes.Buffer
 	var err error
-	printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
+	printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), true, 2, true)
 
 	node, err := getExpressionParser().ParseExpression(s.expression)
 	if err != nil {

pkg/yqlib/printer.go

@@ -17,48 +17,6 @@ type Printer interface {
 	SetNulSepOutput(nulSepOutput bool)
 }
 
-type PrinterOutputFormat uint32
-
-const (
-	YamlOutputFormat = 1 << iota
-	JSONOutputFormat
-	PropsOutputFormat
-	CSVOutputFormat
-	TSVOutputFormat
-	XMLOutputFormat
-	Base64OutputFormat
-	UriOutputFormat
-	ShOutputFormat
-	TomlOutputFormat
-	ShellVariablesOutputFormat
-	LuaOutputFormat
-)
-
-func OutputFormatFromString(format string) (PrinterOutputFormat, error) {
-	switch format {
-	case "yaml", "y", "yml":
-		return YamlOutputFormat, nil
-	case "json", "j":
-		return JSONOutputFormat, nil
-	case "props", "p", "properties":
-		return PropsOutputFormat, nil
-	case "csv", "c":
-		return CSVOutputFormat, nil
-	case "tsv", "t":
-		return TSVOutputFormat, nil
-	case "xml", "x":
-		return XMLOutputFormat, nil
-	case "toml":
-		return TomlOutputFormat, nil
-	case "shell", "s", "sh":
-		return ShellVariablesOutputFormat, nil
-	case "lua", "l":
-		return LuaOutputFormat, nil
-	default:
-		return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml|toml|shell|lua]", format)
-	}
-}
-
 type resultsPrinter struct {
 	encoder           Encoder
 	printerWriter     PrinterWriter

pkg/yqlib/printer_test.go

@@ -36,7 +36,7 @@ func nodeToList(candidate *CandidateNode) *list.List {
 func TestPrinterMultipleDocsInSequenceOnly(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewSimpleYamlPrinter(writer, true, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {
@@ -74,7 +74,7 @@ func TestPrinterMultipleDocsInSequenceOnly(t *testing.T) {
 func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewSimpleYamlPrinter(writer, true, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {
@@ -116,7 +116,7 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
 func TestPrinterMultipleFilesInSequence(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewSimpleYamlPrinter(writer, true, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {
@@ -163,7 +163,7 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) {
 func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewSimpleYamlPrinter(writer, true, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {
@@ -213,7 +213,7 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
 func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewSimpleYamlPrinter(writer, true, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {
@@ -232,7 +232,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewSimpleYamlPrinter(writer, true, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {
@@ -261,7 +261,7 @@ a: coconut
 func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewSimpleYamlPrinter(writer, true, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {
@@ -287,7 +287,7 @@ a: coconut
 func TestPrinterScalarWithLeadingCont(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewSimpleYamlPrinter(writer, true, 2, true)
 
 	node, err := getExpressionParser().ParseExpression(".a")
 	if err != nil {
@@ -314,7 +314,9 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	// note printDocSeparators is true, it should still not print document separators
 	// when outputting JSON.
-	encoder := NewJSONEncoder(0, false, false)
+	prefs := ConfiguredJSONPreferences.Copy()
+	prefs.Indent = 0
+	encoder := NewJSONEncoder(prefs)
 	if encoder == nil {
 		t.Skipf("no support for %s output format", "json")
 	}
@@ -344,7 +346,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
 func TestPrinterNulSeparator(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, false)
+	printer := NewSimpleYamlPrinter(writer, true, 2, false)
 	printer.SetNulSepOutput(true)
 	node, err := getExpressionParser().ParseExpression(".a")
 	if err != nil {
@@ -366,7 +368,9 @@ func TestPrinterNulSeparatorWithJson(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	// note printDocSeparators is true, it should still not print document separators
 	// when outputting JSON.
-	encoder := NewJSONEncoder(0, false, false)
+	prefs := ConfiguredJSONPreferences.Copy()
+	prefs.Indent = 0
+	encoder := NewJSONEncoder(prefs)
 	if encoder == nil {
 		t.Skipf("no support for %s output format", "json")
 	}
@@ -394,7 +398,7 @@ func TestPrinterNulSeparatorWithJson(t *testing.T) {
 func TestPrinterRootUnwrap(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, false)
+	printer := NewSimpleYamlPrinter(writer, true, 2, false)
 	node, err := getExpressionParser().ParseExpression(".")
 	if err != nil {
 		panic(err)

pkg/yqlib/printer_writer.go

@@ -33,15 +33,8 @@ type multiPrintWriter struct {
 	index          int
 }
 
-func NewMultiPrinterWriter(expression *ExpressionNode, format PrinterOutputFormat) PrinterWriter {
-	extension := "yml"
-
-	switch format {
-	case JSONOutputFormat:
-		extension = "json"
-	case PropsOutputFormat:
-		extension = "properties"
-	}
+func NewMultiPrinterWriter(expression *ExpressionNode, format *Format) PrinterWriter {
+	extension := format.DefaultExtension
 
 	return &multiPrintWriter{
 		nameExpression: expression,

pkg/yqlib/properties_test.go

@@ -55,6 +55,8 @@ person: # neither do comments on maps
     name: Mike Wazowski # comments on values appear
     pets: 
     - cat # comments on array values appear
+    - nested:
+        - list entry
     food: [pizza] # comments on arrays do not
 emptyArray: []
 emptyMap: []
@@ -66,15 +68,37 @@ person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = cat
+person.pets.1.nested.0 = list entry
 person.food.0 = pizza
 `
 
+const expectedPropertiesUnwrappedArrayBrackets = `# block comments come through
+# comments on values appear
+person.name = Mike Wazowski
+
+# comments on array values appear
+person.pets[0] = cat
+person.pets[1].nested[0] = list entry
+person.food[0] = pizza
+`
+
+const expectedPropertiesUnwrappedCustomSeparator = `# block comments come through
+# comments on values appear
+person.name :@ Mike Wazowski
+
+# comments on array values appear
+person.pets.0 :@ cat
+person.pets.1.nested.0 :@ list entry
+person.food.0 :@ pizza
+`
+
 const expectedPropertiesWrapped = `# block comments come through
 # comments on values appear
 person.name = "Mike Wazowski"
 
 # comments on array values appear
 person.pets.0 = cat
+person.pets.1.nested.0 = "list entry"
 person.food.0 = pizza
 `
 
@@ -84,6 +108,7 @@ person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = dog
+person.pets.1.nested.0 = list entry
 person.food.0 = pizza
 `
 
@@ -94,6 +119,8 @@ const expectedDecodedYaml = `person:
   pets:
     # comments on array values appear
     - cat
+    - nested:
+        - list entry
   food:
     - pizza
 `
@@ -104,12 +131,15 @@ name: Mike Wazowski
 pets:
   # comments on array values appear
   - cat
+  - nested:
+      - list entry
 food:
   - pizza
 `
 
 const expectedPropertiesNoComments = `person.name = Mike Wazowski
 person.pets.0 = cat
+person.pets.1.nested.0 = list entry
 person.food.0 = pizza
 `
 
@@ -119,18 +149,33 @@ person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = cat
+person.pets.1.nested.0 = list entry
 person.food.0 = pizza
 emptyArray = 
 emptyMap = 
 `
 
-var propertyScenarios = []formatScenario{
+var propertyScenarios = []FormatScenario{
 	{
 		description:    "Encode properties",
 		subdescription: "Note that empty arrays and maps are not encoded by default.",
 		input:          samplePropertiesYaml,
 		expected:       expectedPropertiesUnwrapped,
 	},
+	{
+		description:    "Encode properties with array brackets",
+		subdescription: "Declare the --properties-array-brackets flag to give array paths in brackets (e.g. SpringBoot).",
+		input:          samplePropertiesYaml,
+		expected:       expectedPropertiesUnwrappedArrayBrackets,
+		scenarioType:   "encode-array-brackets",
+	},
+	{
+		description:    "Encode properties - custom separator",
+		subdescription: "Use the --properties-customer-separator flag to specify your own key/value separator.",
+		input:          samplePropertiesYaml,
+		expected:       expectedPropertiesUnwrappedCustomSeparator,
+		scenarioType:   "encode-custom-separator",
+	},
 	{
 		description:    "Encode properties: scalar encapsulation",
 		subdescription: "Note that string values with blank characters in them are encapsulated with double quotes",
@@ -250,7 +295,7 @@ var propertyScenarios = []formatScenario{
 	},
 }
 
-func documentUnwrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) {
+func documentUnwrappedEncodePropertyScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -264,18 +309,29 @@ func documentUnwrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario)
 	writeOrPanic(w, "then\n")
 
 	expression := s.expression
+	prefs := NewDefaultPropertiesPreferences()
+	useArrayBracketsFlag := ""
+	useCustomSeparatorFlag := ""
+	if s.scenarioType == "encode-array-brackets" {
+		useArrayBracketsFlag = " --properties-array-brackets"
+		prefs.UseArrayBrackets = true
+	} else if s.scenarioType == "encode-custom-separator" {
+		prefs.KeyValueSeparator = " :@ "
+		useCustomSeparatorFlag = ` --properties-customer-separator=" :@ "`
+	}
 
 	if expression != "" {
-		writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=props '%v' sample.yml\n```\n", expression))
+		writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=props%v%v '%v' sample.yml\n```\n", useArrayBracketsFlag, useCustomSeparatorFlag, expression))
 	} else {
-		writeOrPanic(w, "```bash\nyq -o=props sample.yml\n```\n")
+		writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=props%v%v sample.yml\n```\n", useArrayBracketsFlag, useCustomSeparatorFlag))
 	}
 	writeOrPanic(w, "will output\n")
+	prefs.UnwrapScalar = true
 
-	writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true))))
+	writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(prefs))))
 }
 
-func documentWrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) {
+func documentWrappedEncodePropertyScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -296,11 +352,12 @@ func documentWrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) {
 		writeOrPanic(w, "```bash\nyq -o=props --unwrapScalar=false sample.yml\n```\n")
 	}
 	writeOrPanic(w, "will output\n")
-
-	writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false))))
+	prefs := ConfiguredPropertiesPreferences.Copy()
+	prefs.UnwrapScalar = false
+	writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(prefs))))
 }
 
-func documentDecodePropertyScenario(w *bufio.Writer, s formatScenario) {
+func documentDecodePropertyScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -322,10 +379,10 @@ func documentDecodePropertyScenario(w *bufio.Writer, s formatScenario) {
 
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))))
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(ConfiguredYamlPreferences))))
 }
 
-func documentRoundTripPropertyScenario(w *bufio.Writer, s formatScenario) {
+func documentRoundTripPropertyScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -347,16 +404,16 @@ func documentRoundTripPropertyScenario(w *bufio.Writer, s formatScenario) {
 
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", mustProcessFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(true))))
+	writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", mustProcessFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(ConfiguredPropertiesPreferences))))
 }
 
 func documentPropertyScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
-	s := i.(formatScenario)
+	s := i.(FormatScenario)
 	if s.skipDoc {
 		return
 	}
 	switch s.scenarioType {
-	case "":
+	case "", "encode-array-brackets", "encode-custom-separator":
 		documentUnwrappedEncodePropertyScenario(w, s)
 	case "decode":
 		documentDecodePropertyScenario(w, s)
@@ -374,13 +431,24 @@ func TestPropertyScenarios(t *testing.T) {
 	for _, s := range propertyScenarios {
 		switch s.scenarioType {
 		case "":
-			test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true)), s.description)
+			test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(ConfiguredPropertiesPreferences)), s.description)
 		case "decode":
-			test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
+			test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
 		case "encode-wrapped":
-			test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false)), s.description)
+			prefs := ConfiguredPropertiesPreferences.Copy()
+			prefs.UnwrapScalar = false
+			test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(prefs)), s.description)
+		case "encode-array-brackets":
+			prefs := ConfiguredPropertiesPreferences.Copy()
+			prefs.KeyValueSeparator = " = "
+			prefs.UseArrayBrackets = true
+			test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(prefs)), s.description)
+		case "encode-custom-separator":
+			prefs := ConfiguredPropertiesPreferences.Copy()
+			prefs.KeyValueSeparator = " :@ "
+			test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(prefs)), s.description)
 		case "roundtrip":
-			test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(true)), s.description)
+			test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(ConfiguredPropertiesPreferences)), s.description)
 
 		default:
 			panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))

pkg/yqlib/recipes_test.go

@@ -14,6 +14,15 @@ var nestedBashEnvScript = `.. |(
 	( select(kind == "seq") | (path | join("_")) + "=(" + (map("'" + . + "'") | join(",")) + ")")
 )`
 
+var deepPruneExpression = `(
+  .. | # recurse through all the nodes
+  select(has("child1") or has("child2")) | # match parents that have either child1 or child2
+  (.child1, .child2) | # select those children
+  select(.) # filter out nulls
+) as $i ireduce({};  # using that set of nodes, create a new result map
+  setpath($i | path; $i) # and put in each node, using its original path
+)`
+
 var recipes = []expressionScenario{
 	{
 		description:    "Find items in an array",
@@ -46,6 +55,20 @@ var recipes = []expressionScenario{
 			"D0, P[], (!!seq)::[{name: Foo, numBuckets: 1}, {name: Bar, numBuckets: 0}]\n",
 		},
 	},
+	{
+		description:    "Deeply prune a tree",
+		subdescription: "Say we are only interested in child1 and child2, and want to filter everything else out.",
+		document:       `{parentA: [bob],parentB: {child1: i am child1, child3: hiya},parentC: {childX: "cool",child2: me child2}}`,
+		expression:     deepPruneExpression,
+		explanation: []string{
+			"Find all the matching child1 and child2 nodes",
+			"Using ireduce, create a new map using just those nodes",
+			"Set each node into the new map using its original path",
+		},
+		expected: []string{
+			"D0, P[], (!!map)::parentB:\n    child1: i am child1\nparentC:\n    child2: me child2\n",
+		},
+	},
 	{
 		description:    "Multiple or complex updates to items in an array",
 		subdescription: "We have an array and we want to _update_ the elements with a particular name in reference to its type.",

pkg/yqlib/shellvariables_test.go

@@ -8,7 +8,7 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
-var shellVariablesScenarios = []formatScenario{
+var shellVariablesScenarios = []FormatScenario{
 	{
 		description:    "Encode shell variables",
 		subdescription: "Note that comments are dropped and values will be enclosed in single quotes as needed.",
@@ -69,7 +69,7 @@ func TestShellVariableScenarios(t *testing.T) {
 }
 
 func documentShellVariableScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
-	s := i.(formatScenario)
+	s := i.(FormatScenario)
 	if s.skipDoc {
 		return
 	}

pkg/yqlib/stream_evaluator.go

@@ -31,7 +31,7 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error
 	if err != nil {
 		return err
 	}
-	candidateNode := createScalarNode(nil, "")
+	candidateNode := CreateScalarNode(nil, "")
 	inputList := list.New()
 	inputList.PushBack(candidateNode)
 

pkg/yqlib/string_evaluator.go

@@ -4,19 +4,16 @@ import (
 	"bufio"
 	"bytes"
 	"container/list"
-	"errors"
-	"fmt"
-	"io"
 	"strings"
 )
 
 type StringEvaluator interface {
 	Evaluate(expression string, input string, encoder Encoder, decoder Decoder) (string, error)
+	EvaluateAll(expression string, input string, encoder Encoder, decoder Decoder) (string, error)
 }
 
 type stringEvaluator struct {
 	treeNavigator DataTreeNavigator
-	fileIndex     int
 }
 
 func NewStringEvaluator() StringEvaluator {
@@ -25,6 +22,29 @@ func NewStringEvaluator() StringEvaluator {
 	}
 }
 
+func (s *stringEvaluator) EvaluateAll(expression string, input string, encoder Encoder, decoder Decoder) (string, error) {
+	reader := bufio.NewReader(strings.NewReader(input))
+	var documents *list.List
+	var results *list.List
+	var err error
+
+	if documents, err = ReadDocuments(reader, decoder); err != nil {
+		return "", err
+	}
+
+	evaluator := NewAllAtOnceEvaluator()
+	if results, err = evaluator.EvaluateCandidateNodes(expression, documents); err != nil {
+		return "", err
+	}
+
+	out := new(bytes.Buffer)
+	printer := NewPrinter(encoder, NewSinglePrinterWriter(out))
+	if err := printer.PrintResults(results); err != nil {
+		return "", err
+	}
+	return out.String(), nil
+}
+
 func (s *stringEvaluator) Evaluate(expression string, input string, encoder Encoder, decoder Decoder) (string, error) {
 
 	// Use bytes.Buffer for output of string
@@ -38,36 +58,9 @@ func (s *stringEvaluator) Evaluate(expression string, input string, encoder Enco
 	}
 
 	reader := bufio.NewReader(strings.NewReader(input))
-
-	var currentIndex uint
-	err = decoder.Init(reader)
-	if err != nil {
+	evaluator := NewStreamEvaluator()
+	if _, err := evaluator.Evaluate("", reader, node, printer, decoder); err != nil {
 		return "", err
 	}
-	for {
-		candidateNode, errorReading := decoder.Decode()
-
-		if errors.Is(errorReading, io.EOF) {
-			s.fileIndex = s.fileIndex + 1
-			return out.String(), nil
-		} else if errorReading != nil {
-			return "", fmt.Errorf("bad input '%v': %w", input, errorReading)
-		}
-		candidateNode.document = currentIndex
-		candidateNode.fileIndex = s.fileIndex
-
-		inputList := list.New()
-		inputList.PushBack(candidateNode)
-
-		result, errorParsing := s.treeNavigator.GetMatchingNodes(Context{MatchingNodes: inputList}, node)
-		if errorParsing != nil {
-			return "", errorParsing
-		}
-		err = printer.PrintResults(result.MatchingNodes)
-
-		if err != nil {
-			return "", err
-		}
-		currentIndex = currentIndex + 1
-	}
+	return out.String(), nil
 }

pkg/yqlib/string_evaluator_test.go

@@ -6,6 +6,20 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
+func TestStringEvaluator_MultipleDocumentMerge(t *testing.T) {
+	yamlString := "a: Hello\n---\na: Goodbye\n"
+	expected_output := "a: Goodbye\n"
+
+	encoder := NewYamlEncoder(ConfiguredYamlPreferences)
+	decoder := NewYamlDecoder(ConfiguredYamlPreferences)
+	result, err := NewStringEvaluator().EvaluateAll("select(di==0) * select(di==1)", yamlString, encoder, decoder)
+	if err != nil {
+		t.Error(err)
+	} else {
+		test.AssertResult(t, expected_output, result)
+	}
+}
+
 func TestStringEvaluator_Evaluate_Nominal(t *testing.T) {
 	expected_output := `` +
 		`yq` + "\n" +
@@ -18,13 +32,13 @@ func TestStringEvaluator_Evaluate_Nominal(t *testing.T) {
 		`---` + "\n" +
 		` - name: jq` + "\n" +
 		`   description: Command-line JSON processor` + "\n"
-	encoder := NewYamlEncoder(2, true, ConfiguredYamlPreferences)
+	encoder := NewYamlEncoder(ConfiguredYamlPreferences)
 	decoder := NewYamlDecoder(ConfiguredYamlPreferences)
 
 	result, err := NewStringEvaluator().Evaluate(expression, input, encoder, decoder)
 	if err != nil {
 		t.Error(err)
+	} else {
+		test.AssertResult(t, expected_output, result)
 	}
-
-	test.AssertResult(t, expected_output, result)
 }

pkg/yqlib/toml_test.go

@@ -79,7 +79,7 @@ var expectedSampleWithHeader = `servers:
     ip: 10.0.0.1
 `
 
-var tomlScenarios = []formatScenario{
+var tomlScenarios = []FormatScenario{
 	{
 		skipDoc:      true,
 		description:  "blank",
@@ -240,12 +240,12 @@ var tomlScenarios = []formatScenario{
 	},
 }
 
-func testTomlScenario(t *testing.T, s formatScenario) {
+func testTomlScenario(t *testing.T, s FormatScenario) {
 	switch s.scenarioType {
 	case "", "decode":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewTomlDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewTomlDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
 	case "decode-error":
-		result, err := processFormatScenario(s, NewTomlDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences))
+		result, err := processFormatScenario(s, NewTomlDecoder(), NewYamlEncoder(ConfiguredYamlPreferences))
 		if err == nil {
 			t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result)
 		} else {
@@ -256,7 +256,7 @@ func testTomlScenario(t *testing.T, s formatScenario) {
 	}
 }
 
-func documentTomlDecodeScenario(w *bufio.Writer, s formatScenario) {
+func documentTomlDecodeScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -275,10 +275,10 @@ func documentTomlDecodeScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, fmt.Sprintf("```bash\nyq -oy '%v' sample.toml\n```\n", expression))
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewTomlDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences))))
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewTomlDecoder(), NewYamlEncoder(ConfiguredYamlPreferences))))
 }
 
-func documentTomlRoundtripScenario(w *bufio.Writer, s formatScenario) {
+func documentTomlRoundtripScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -301,7 +301,7 @@ func documentTomlRoundtripScenario(w *bufio.Writer, s formatScenario) {
 }
 
 func documentTomlScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
-	s := i.(formatScenario)
+	s := i.(FormatScenario)
 
 	if s.skipDoc {
 		return

pkg/yqlib/utils.go

@@ -26,11 +26,15 @@ func readStream(filename string) (io.Reader, error) {
 
 }
 
-func writeString(writer io.Writer, txt string) error {
+func WriteString(writer io.Writer, txt string) error {
 	_, errorWriting := writer.Write([]byte(txt))
 	return errorWriting
 }
 
+func ReadDocuments(reader io.Reader, decoder Decoder) (*list.List, error) {
+	return readDocuments(reader, "", 0, decoder)
+}
+
 func readDocuments(reader io.Reader, filename string, fileIndex int, decoder Decoder) (*list.List, error) {
 	err := decoder.Init(reader)
 	if err != nil {

pkg/yqlib/xml.go

@@ -1,29 +0,0 @@
-package yqlib
-
-type XmlPreferences struct {
-	AttributePrefix string
-	ContentName     string
-	StrictMode      bool
-	KeepNamespace   bool
-	UseRawToken     bool
-	ProcInstPrefix  string
-	DirectiveName   string
-	SkipProcInst    bool
-	SkipDirectives  bool
-}
-
-func NewDefaultXmlPreferences() XmlPreferences {
-	return XmlPreferences{
-		AttributePrefix: "+@",
-		ContentName:     "+content",
-		StrictMode:      false,
-		KeepNamespace:   true,
-		UseRawToken:     true,
-		ProcInstPrefix:  "+p_",
-		DirectiveName:   "+directive",
-		SkipProcInst:    false,
-		SkipDirectives:  false,
-	}
-}
-
-var ConfiguredXMLPreferences = NewDefaultXmlPreferences()

pkg/yqlib/yaml.go

@@ -1,6 +1,8 @@
 package yqlib
 
 type YamlPreferences struct {
+	Indent                      int
+	ColorsEnabled               bool
 	LeadingContentPreProcessing bool
 	PrintDocSeparators          bool
 	UnwrapScalar                bool
@@ -9,6 +11,8 @@ type YamlPreferences struct {
 
 func NewDefaultYamlPreferences() YamlPreferences {
 	return YamlPreferences{
+		Indent:                      2,
+		ColorsEnabled:               false,
 		LeadingContentPreProcessing: true,
 		PrintDocSeparators:          true,
 		UnwrapScalar:                true,
@@ -16,4 +20,15 @@ func NewDefaultYamlPreferences() YamlPreferences {
 	}
 }
 
+func (p *YamlPreferences) Copy() YamlPreferences {
+	return YamlPreferences{
+		Indent:                      p.Indent,
+		ColorsEnabled:               p.ColorsEnabled,
+		LeadingContentPreProcessing: p.LeadingContentPreProcessing,
+		PrintDocSeparators:          p.PrintDocSeparators,
+		UnwrapScalar:                p.UnwrapScalar,
+		EvaluateTogether:            p.EvaluateTogether,
+	}
+}
+
 var ConfiguredYamlPreferences = NewDefaultYamlPreferences()