Merge branch 'master' into xml-proc-inst

README.md

@@ -59,7 +59,7 @@ Take a look at the discussions for [common questions](https://github.com/mikefar
 ### [Download the latest binary](https://github.com/mikefarah/yq/releases/latest)
 
 ### wget
-Use wget to download, gzipped pre-compiled binaries:
+Use wget to download the pre-compiled binaries:
 
 #### Compressed via tar.gz
 ```bash
@@ -76,13 +76,6 @@ wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY} -O /
 
 For instance, VERSION=v4.2.0 and BINARY=yq_linux_amd64
 
-Use VERSION=latest to always get the latest version:
-
-```bash
-wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq &&\
-    chmod +x /usr/bin/yq
-```
-
 ### MacOS / Linux via Homebrew:
 Using [Homebrew](https://brew.sh/)
 ```

acceptance_tests/empty.sh

@@ -9,7 +9,7 @@ EOL
 
 testEmptyEval() {
   X=$(./yq e test.yml)
-  expected="# comment"
+  expected=$(cat test.yml)
   assertEquals 0 $?
   assertEquals "$expected" "$X"
 }

cmd/constant.go

@@ -1,5 +1,6 @@
 package cmd
 
+var leadingContentPreProcessing = true
 var unwrapScalar = true
 
 var writeInplace = false

cmd/evaluate_all_command.go

@@ -75,7 +75,7 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 		return err
 	}
 
-	decoder, err := configureDecoder(true)
+	decoder, err := configureDecoder()
 	if err != nil {
 		return err
 	}
@@ -109,13 +109,13 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 	switch len(args) {
 	case 0:
 		if nullInput {
-			err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(expression), printer)
+			err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(expression), printer, "")
 		} else {
 			cmd.Println(cmd.UsageString())
 			return nil
 		}
 	default:
-		err = allAtOnceEvaluator.EvaluateFiles(processExpression(expression), args, printer, decoder)
+		err = allAtOnceEvaluator.EvaluateFiles(processExpression(expression), args, printer, leadingContentPreProcessing, decoder)
 	}
 
 	completedSuccessfully = err == nil

cmd/evalute_sequence_command.go

@@ -97,7 +97,7 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 
 	printer := yqlib.NewPrinter(encoder, printerWriter)
 
-	decoder, err := configureDecoder(false)
+	decoder, err := configureDecoder()
 	if err != nil {
 		return err
 	}
@@ -123,13 +123,13 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 	switch len(args) {
 	case 0:
 		if nullInput {
-			err = streamEvaluator.EvaluateNew(processExpression(expression), printer)
+			err = streamEvaluator.EvaluateNew(processExpression(expression), printer, "")
 		} else {
 			cmd.Println(cmd.UsageString())
 			return nil
 		}
 	default:
-		err = streamEvaluator.EvaluateFiles(processExpression(expression), args, printer, decoder)
+		err = streamEvaluator.EvaluateFiles(processExpression(expression), args, printer, leadingContentPreProcessing, decoder)
 	}
 	completedSuccessfully = err == nil
 

cmd/root.go

@@ -54,16 +54,11 @@ yq -P sample.json
 			yqlib.InitExpressionParser()
 			if (inputFormat == "x" || inputFormat == "xml") &&
 				outputFormat != "x" && outputFormat != "xml" &&
-				yqlib.ConfiguredXMLPreferences.AttributePrefix == "+" {
+				yqlib.XMLPreferences.AttributePrefix == "+" {
 				yqlib.GetLogger().Warning("The default xml-attribute-prefix will change in the v4.30 to `+@` to avoid " +
 					"naming conflicts with the default content name, directive name and proc inst prefix. If you need to keep " +
 					"`+` please set that value explicityly with --xml-attribute-prefix.")
 			}
-
-			//copy preference form global setting
-			yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar
-
-			yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators
 		},
 	}
 
@@ -78,15 +73,15 @@ yq -P sample.json
 	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "yaml", "[yaml|y|json|j|props|p|xml|x] output format type.")
 	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "yaml", "[yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.")
 
-	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", "+", "prefix for xml attributes")
-	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", "+content", "name for xml content (if no attribute name is present).")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.StrictMode, "xml-strict-mode", false, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.KeepNamespace, "xml-keep-namespace", true, "enables keeping namespace after parsing attributes")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.UseRawToken, "xml-raw-token", true, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.")
-	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ProcInstPrefix, "xml-proc-inst-prefix", "+p_", "prefix for xml processing instructions (e.g. <?xml version=\"1\"?>)")
-	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.DirectiveName, "xml-directive-name", "+directive", "name for xml directives (e.g. <!DOCTYPE thing cat>)")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipProcInst, "xml-skip-proc-inst", false, "skip over process instructions (e.g. <?xml version=\"1\"?>)")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipDirectives, "xml-skip-directives", false, "skip over directives (e.g. <!DOCTYPE thing cat>)")
+	rootCmd.PersistentFlags().StringVar(&yqlib.XMLPreferences.AttributePrefix, "xml-attribute-prefix", "+", "prefix for xml attributes")
+	rootCmd.PersistentFlags().StringVar(&yqlib.XMLPreferences.ContentName, "xml-content-name", "+content", "name for xml content (if no attribute name is present).")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.XMLPreferences.StrictMode, "xml-strict-mode", false, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.XMLPreferences.KeepNamespace, "xml-keep-namespace", true, "enables keeping namespace after parsing attributes")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.XMLPreferences.UseRawToken, "xml-raw-token", true, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.")
+	rootCmd.PersistentFlags().StringVar(&yqlib.XMLPreferences.ProcInstPrefix, "xml-proc-inst-prefix", "+p_", "prefix for xml processing instructions (e.g. <?xml version=\"1\"?>)")
+	rootCmd.PersistentFlags().StringVar(&yqlib.XMLPreferences.DirectiveName, "xml-directive-name", "+directive", "name for xml directives (e.g. <!DOCTYPE thing cat>)")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.XMLPreferences.SkipProcInst, "xml-skip-proc-inst", false, "skip over process instructions (e.g. <?xml version=\"1\"?>)")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.XMLPreferences.SkipDirectives, "xml-skip-directives", false, "skip over directives (e.g. <!DOCTYPE thing cat>)")
 
 	rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.")
 	rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)")
@@ -102,7 +97,7 @@ yq -P sample.json
 	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
 	rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
 	rootCmd.PersistentFlags().StringVarP(&forceExpression, "expression", "", "", "forcibly set the expression argument. Useful when yq argument detection thinks your expression is a file.")
-	rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredYamlPreferences.LeadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.")
+	rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.")
 
 	rootCmd.PersistentFlags().StringVarP(&splitFileExp, "split-exp", "s", "", "print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter.")
 	rootCmd.PersistentFlags().StringVarP(&splitFileExpFile, "split-exp-file", "", "", "Use a file to specify the split-exp expression.")

cmd/utils.go

@@ -56,14 +56,14 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 	return expression, args, nil
 }
 
-func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
+func configureDecoder() (yqlib.Decoder, error) {
 	yqlibInputFormat, err := yqlib.InputFormatFromString(inputFormat)
 	if err != nil {
 		return nil, err
 	}
 	switch yqlibInputFormat {
 	case yqlib.XMLInputFormat:
-		return yqlib.NewXMLDecoder(yqlib.ConfiguredXMLPreferences), nil
+		return yqlib.NewXMLDecoder(yqlib.XMLPreferences), nil
 	case yqlib.PropertiesInputFormat:
 		return yqlib.NewPropertiesDecoder(), nil
 	case yqlib.JsonInputFormat:
@@ -73,9 +73,8 @@ func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
 	case yqlib.TSVObjectInputFormat:
 		return yqlib.NewCSVObjectDecoder('\t'), nil
 	}
-	prefs := yqlib.ConfiguredYamlPreferences
-	prefs.EvaluateTogether = evaluateTogether
-	return yqlib.NewYamlDecoder(prefs), nil
+
+	return yqlib.NewYamlDecoder(), nil
 }
 
 func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yqlib.PrinterWriter, error) {
@@ -106,9 +105,9 @@ func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder {
 	case yqlib.TSVOutputFormat:
 		return yqlib.NewCsvEncoder('\t')
 	case yqlib.YamlOutputFormat:
-		return yqlib.NewYamlEncoder(indent, colorsEnabled, yqlib.ConfiguredYamlPreferences)
+		return yqlib.NewYamlEncoder(indent, colorsEnabled, !noDocSeparators, unwrapScalar)
 	case yqlib.XMLOutputFormat:
-		return yqlib.NewXMLEncoder(indent, yqlib.ConfiguredXMLPreferences)
+		return yqlib.NewXMLEncoder(indent, yqlib.XMLPreferences)
 	}
 	panic("invalid encoder")
 }

examples/array.yaml

@@ -1,2 +1,11 @@
-- [cat, dog, frog, cow]
-- [apple, banana, grape, mango]
+---
+- become: true
+  gather_facts: false
+  hosts: lalaland
+  name: "Apply smth"
+  roles:
+    - lala
+    - land
+  serial: 1
+- become: false
+  gather_facts: true

go.mod

@@ -8,11 +8,11 @@ require (
 	github.com/elliotchance/orderedmap v1.5.0
 	github.com/fatih/color v1.13.0
 	github.com/goccy/go-json v0.9.11
-	github.com/goccy/go-yaml v1.9.6
+	github.com/goccy/go-yaml v1.9.5
 	github.com/jinzhu/copier v0.3.5
 	github.com/magiconair/properties v1.8.6
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
-	github.com/spf13/cobra v1.6.1
+	github.com/spf13/cobra v1.6.0
 	golang.org/x/net v0.0.0-20220708220712-1185a9018129
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.1

go.sum

@@ -21,8 +21,8 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk=
 github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
-github.com/goccy/go-yaml v1.9.6 h1:KhAu1zf9JXnm3vbG49aDE0E5uEBUsM4uwD31/58ZWyI=
-github.com/goccy/go-yaml v1.9.6/go.mod h1:JubOolP3gh0HpiBc4BLRD4YmjEjHAmIIB2aaXKkTfoE=
+github.com/goccy/go-yaml v1.9.5 h1:Eh/+3uk9kLxG4koCX6lRMAPS1OaMSAi+FJcya0INdB0=
+github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
 github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
@@ -43,8 +43,8 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
-github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
+github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI=
+github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -62,7 +62,6 @@ golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e h1:NHvCuwuS43lGnYhten69ZWqi2QOj/CiDNcKbVqwVoew=
 golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

pkg/yqlib/all_at_once_evaluator.go

@@ -8,7 +8,7 @@ import (
 
 // A yaml expression evaluator that runs the expression once against all files/nodes in memory.
 type Evaluator interface {
-	EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error
+	EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error
 
 	// EvaluateNodes takes an expression and one or more yaml nodes, returning a list of matching candidate nodes
 	EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error)
@@ -46,16 +46,21 @@ func (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCand
 	return context.MatchingNodes, nil
 }
 
-func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error {
+func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error {
 	fileIndex := 0
+	firstFileLeadingContent := ""
 
 	var allDocuments = list.New()
 	for _, filename := range filenames {
-		reader, err := readStream(filename)
+		reader, leadingContent, err := readStream(filename, fileIndex == 0 && leadingContentPreProcessing)
 		if err != nil {
 			return err
 		}
 
+		if fileIndex == 0 {
+			firstFileLeadingContent = leadingContent
+		}
+
 		fileDocuments, err := readDocuments(reader, filename, fileIndex, decoder)
 		if err != nil {
 			return err
@@ -70,9 +75,11 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
 			Filename:       "",
 			Node:           &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
 			FileIndex:      0,
-			LeadingContent: "",
+			LeadingContent: firstFileLeadingContent,
 		}
 		allDocuments.PushBack(candidateNode)
+	} else {
+		allDocuments.Front().Value.(*CandidateNode).LeadingContent = firstFileLeadingContent
 	}
 
 	matches, err := e.EvaluateCandidateNodes(expression, allDocuments)

pkg/yqlib/csv_test.go

@@ -136,13 +136,13 @@ var csvScenarios = []formatScenario{
 func testCSVScenario(t *testing.T, s formatScenario) {
 	switch s.scenarioType {
 	case "encode-csv":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(',')), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder(',')), s.description)
 	case "encode-tsv":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder('\t')), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder('\t')), s.description)
 	case "decode-csv-object":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewYamlEncoder(2, false, true, true)), s.description)
 	case "decode-tsv-object":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder('\t'), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder('\t'), NewYamlEncoder(2, false, true, true)), s.description)
 	case "roundtrip-csv":
 		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewCsvEncoder(',')), s.description)
 	default:
@@ -171,7 +171,7 @@ func documentCSVDecodeObjectScenario(w *bufio.Writer, s formatScenario, formatTy
 	}
 
 	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n",
-		processFormatScenario(s, NewCSVObjectDecoder(separator), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))),
+		processFormatScenario(s, NewCSVObjectDecoder(separator), NewYamlEncoder(s.indent, false, true, true))),
 	)
 }
 
@@ -203,7 +203,7 @@ func documentCSVEncodeScenario(w *bufio.Writer, s formatScenario, formatType str
 	}
 
 	writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
-		processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(separator))),
+		processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder(separator))),
 	)
 }
 

pkg/yqlib/decoder.go

@@ -3,6 +3,8 @@ package yqlib
 import (
 	"fmt"
 	"io"
+
+	yaml "gopkg.in/yaml.v3"
 )
 
 type InputFormat uint
@@ -18,8 +20,8 @@ const (
 )
 
 type Decoder interface {
-	Init(reader io.Reader) error
-	Decode() (*CandidateNode, error)
+	Init(reader io.Reader)
+	Decode(node *yaml.Node) error
 }
 
 func InputFormatFromString(format string) (InputFormat, error) {

pkg/yqlib/decoder_base64.go

@@ -19,22 +19,21 @@ func NewBase64Decoder() Decoder {
 	return &base64Decoder{finished: false, encoding: *base64.StdEncoding}
 }
 
-func (dec *base64Decoder) Init(reader io.Reader) error {
+func (dec *base64Decoder) Init(reader io.Reader) {
 	dec.reader = reader
 	dec.readAnything = false
 	dec.finished = false
-	return nil
 }
 
-func (dec *base64Decoder) Decode() (*CandidateNode, error) {
+func (dec *base64Decoder) Decode(rootYamlNode *yaml.Node) error {
 	if dec.finished {
-		return nil, io.EOF
+		return io.EOF
 	}
 	base64Reader := base64.NewDecoder(&dec.encoding, dec.reader)
 	buf := new(bytes.Buffer)
 
 	if _, err := buf.ReadFrom(base64Reader); err != nil {
-		return nil, err
+		return err
 	}
 	if buf.Len() == 0 {
 		dec.finished = true
@@ -43,15 +42,12 @@ func (dec *base64Decoder) Decode() (*CandidateNode, error) {
 		// otherwise if we've already read some bytes, and now we get
 		// an empty string, then we are done.
 		if dec.readAnything {
-			return nil, io.EOF
+			return io.EOF
 		}
 	}
 	dec.readAnything = true
-	return &CandidateNode{
-		Node: &yaml.Node{
-			Kind:  yaml.ScalarNode,
-			Tag:   "!!str",
-			Value: buf.String(),
-		},
-	}, nil
+	rootYamlNode.Kind = yaml.ScalarNode
+	rootYamlNode.Tag = "!!str"
+	rootYamlNode.Value = buf.String()
+	return nil
 }

pkg/yqlib/decoder_csv_object.go

@@ -19,13 +19,12 @@ func NewCSVObjectDecoder(separator rune) Decoder {
 	return &csvObjectDecoder{separator: separator}
 }
 
-func (dec *csvObjectDecoder) Init(reader io.Reader) error {
+func (dec *csvObjectDecoder) Init(reader io.Reader) {
 	cleanReader, enc := utfbom.Skip(reader)
 	log.Debugf("Detected encoding: %s\n", enc)
 	dec.reader = *csv.NewReader(cleanReader)
 	dec.reader.Comma = dec.separator
 	dec.finished = false
-	return nil
 }
 
 func (dec *csvObjectDecoder) convertToYamlNode(content string) *yaml.Node {
@@ -48,14 +47,14 @@ func (dec *csvObjectDecoder) createObject(headerRow []string, contentRow []strin
 	return objectNode
 }
 
-func (dec *csvObjectDecoder) Decode() (*CandidateNode, error) {
+func (dec *csvObjectDecoder) Decode(rootYamlNode *yaml.Node) error {
 	if dec.finished {
-		return nil, io.EOF
+		return io.EOF
 	}
 	headerRow, err := dec.reader.Read()
 	log.Debugf(": headerRow%v", headerRow)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
 	rootArray := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
@@ -69,13 +68,13 @@ func (dec *csvObjectDecoder) Decode() (*CandidateNode, error) {
 		log.Debugf("Read next contentRow: %v, %v", contentRow, err)
 	}
 	if !errors.Is(err, io.EOF) {
-		return nil, err
+		return err
 	}
 
-	return &CandidateNode{
-		Node: &yaml.Node{
-			Kind:    yaml.DocumentNode,
-			Content: []*yaml.Node{rootArray},
-		},
-	}, nil
+	log.Debugf("finished, contentRow%v", contentRow)
+	log.Debugf("err: %v", err)
+
+	rootYamlNode.Kind = yaml.DocumentNode
+	rootYamlNode.Content = []*yaml.Node{rootArray}
+	return nil
 }

pkg/yqlib/decoder_json.go

@@ -16,31 +16,26 @@ func NewJSONDecoder() Decoder {
 	return &jsonDecoder{}
 }
 
-func (dec *jsonDecoder) Init(reader io.Reader) error {
+func (dec *jsonDecoder) Init(reader io.Reader) {
 	dec.decoder = *json.NewDecoder(reader)
-	return nil
 }
 
-func (dec *jsonDecoder) Decode() (*CandidateNode, error) {
+func (dec *jsonDecoder) Decode(rootYamlNode *yaml.Node) error {
 
 	var dataBucket orderedMap
 	log.Debug("going to decode")
 	err := dec.decoder.Decode(&dataBucket)
 	if err != nil {
-		return nil, err
+		return err
 	}
 	node, err := dec.convertToYamlNode(&dataBucket)
 
 	if err != nil {
-		return nil, err
+		return err
 	}
-
-	return &CandidateNode{
-		Node: &yaml.Node{
-			Kind:    yaml.DocumentNode,
-			Content: []*yaml.Node{node},
-		},
-	}, nil
+	rootYamlNode.Kind = yaml.DocumentNode
+	rootYamlNode.Content = []*yaml.Node{node}
+	return nil
 }
 
 func (dec *jsonDecoder) convertToYamlNode(data *orderedMap) (*yaml.Node, error) {

pkg/yqlib/decoder_properties.go

@@ -2,7 +2,6 @@ package yqlib
 
 import (
 	"bytes"
-	"fmt"
 	"io"
 	"strconv"
 	"strings"
@@ -21,10 +20,9 @@ func NewPropertiesDecoder() Decoder {
 	return &propertiesDecoder{d: NewDataTreeNavigator(), finished: false}
 }
 
-func (dec *propertiesDecoder) Init(reader io.Reader) error {
+func (dec *propertiesDecoder) Init(reader io.Reader) {
 	dec.reader = reader
 	dec.finished = false
-	return nil
 }
 
 func parsePropKey(key string) []interface{} {
@@ -48,49 +46,15 @@ func (dec *propertiesDecoder) processComment(c string) string {
 	return "# " + c
 }
 
-func (dec *propertiesDecoder) applyPropertyComments(context Context, path []interface{}, comments []string) error {
-	assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
-
-	rhsCandidateNode := &CandidateNode{
-		Path: path,
-		Node: &yaml.Node{
-			Tag:         "!!str",
-			Value:       fmt.Sprintf("%v", path[len(path)-1]),
-			HeadComment: dec.processComment(strings.Join(comments, "\n")),
-			Kind:        yaml.ScalarNode,
-		},
-	}
-
-	rhsCandidateNode.Node.Tag = guessTagFromCustomType(rhsCandidateNode.Node)
-
-	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}
-
-	assignmentOpNode := &ExpressionNode{
-		Operation: assignmentOp,
-		LHS:       createTraversalTree(path, traversePreferences{}, true),
-		RHS:       &ExpressionNode{Operation: rhsOp},
-	}
-
-	_, err := dec.d.GetMatchingNodes(context, assignmentOpNode)
-	return err
-}
-
-func (dec *propertiesDecoder) applyProperty(context Context, properties *properties.Properties, key string) error {
+func (dec *propertiesDecoder) applyProperty(properties *properties.Properties, context Context, key string) error {
 	value, _ := properties.Get(key)
 	path := parsePropKey(key)
 
-	propertyComments := properties.GetComments(key)
-	if len(propertyComments) > 0 {
-		err := dec.applyPropertyComments(context, path, propertyComments)
-		if err != nil {
-			return nil
-		}
-	}
-
 	rhsNode := &yaml.Node{
-		Value: value,
-		Tag:   "!!str",
-		Kind:  yaml.ScalarNode,
+		Value:       value,
+		Tag:         "!!str",
+		Kind:        yaml.ScalarNode,
+		LineComment: dec.processComment(properties.GetComment(key)),
 	}
 
 	rhsNode.Tag = guessTagFromCustomType(rhsNode)
@@ -114,22 +78,22 @@ func (dec *propertiesDecoder) applyProperty(context Context, properties *propert
 	return err
 }
 
-func (dec *propertiesDecoder) Decode() (*CandidateNode, error) {
+func (dec *propertiesDecoder) Decode(rootYamlNode *yaml.Node) error {
 	if dec.finished {
-		return nil, io.EOF
+		return io.EOF
 	}
 	buf := new(bytes.Buffer)
 
 	if _, err := buf.ReadFrom(dec.reader); err != nil {
-		return nil, err
+		return err
 	}
 	if buf.Len() == 0 {
 		dec.finished = true
-		return nil, io.EOF
+		return io.EOF
 	}
 	properties, err := properties.LoadString(buf.String())
 	if err != nil {
-		return nil, err
+		return err
 	}
 	properties.DisableExpansion = true
 
@@ -144,18 +108,15 @@ func (dec *propertiesDecoder) Decode() (*CandidateNode, error) {
 	context = context.SingleChildContext(rootMap)
 
 	for _, key := range properties.Keys() {
-		if err := dec.applyProperty(context, properties, key); err != nil {
-			return nil, err
+		if err := dec.applyProperty(properties, context, key); err != nil {
+			return err
 		}
 
 	}
-	dec.finished = true
 
-	return &CandidateNode{
-		Node: &yaml.Node{
-			Kind:    yaml.DocumentNode,
-			Content: []*yaml.Node{rootMap.Node},
-		},
-	}, nil
+	rootYamlNode.Kind = yaml.DocumentNode
+	rootYamlNode.Content = []*yaml.Node{rootMap.Node}
+	dec.finished = true
+	return nil
 
 }

pkg/yqlib/decoder_test.go

@@ -23,7 +23,7 @@ func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) s
 	writer := bufio.NewWriter(&output)
 
 	if decoder == nil {
-		decoder = NewYamlDecoder(ConfiguredYamlPreferences)
+		decoder = NewYamlDecoder()
 	}
 
 	inputs, err := readDocuments(strings.NewReader(s.input), "sample.yml", 0, decoder)

pkg/yqlib/decoder_xml.go

@@ -15,21 +15,20 @@ type xmlDecoder struct {
 	reader       io.Reader
 	readAnything bool
 	finished     bool
-	prefs        XmlPreferences
+	prefs        xmlPreferences
 }
 
-func NewXMLDecoder(prefs XmlPreferences) Decoder {
+func NewXMLDecoder(prefs xmlPreferences) Decoder {
 	return &xmlDecoder{
 		finished: false,
 		prefs:    prefs,
 	}
 }
 
-func (dec *xmlDecoder) Init(reader io.Reader) error {
+func (dec *xmlDecoder) Init(reader io.Reader) {
 	dec.reader = reader
 	dec.readAnything = false
 	dec.finished = false
-	return nil
 }
 
 func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yaml.Node, error) {
@@ -119,36 +118,32 @@ func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yaml.Node, error) {
 	return scalar, nil
 }
 
-func (dec *xmlDecoder) Decode() (*CandidateNode, error) {
+func (dec *xmlDecoder) Decode(rootYamlNode *yaml.Node) error {
 	if dec.finished {
-		return nil, io.EOF
+		return io.EOF
 	}
 	root := &xmlNode{}
 	// cant use xj - it doesn't keep map order.
 	err := dec.decodeXML(root)
 
 	if err != nil {
-		return nil, err
+		return err
 	}
 	firstNode, err := dec.convertToYamlNode(root)
 
 	if err != nil {
-		return nil, err
+		return err
 	} else if firstNode.Tag == "!!null" {
 		dec.finished = true
 		if dec.readAnything {
-			return nil, io.EOF
+			return io.EOF
 		}
 	}
 	dec.readAnything = true
+	rootYamlNode.Kind = yaml.DocumentNode
+	rootYamlNode.Content = []*yaml.Node{firstNode}
 	dec.finished = true
-
-	return &CandidateNode{
-		Node: &yaml.Node{
-			Kind:    yaml.DocumentNode,
-			Content: []*yaml.Node{firstNode},
-		},
-	}, nil
+	return nil
 }
 
 type xmlNode struct {

pkg/yqlib/decoder_yaml.go

@@ -1,114 +1,23 @@
 package yqlib
 
 import (
-	"bufio"
-	"errors"
 	"io"
-	"regexp"
-	"strings"
 
 	yaml "gopkg.in/yaml.v3"
 )
 
 type yamlDecoder struct {
 	decoder yaml.Decoder
-	// work around of various parsing issues by yaml.v3 with document headers
-	prefs          YamlPreferences
-	leadingContent string
-	readAnything   bool
-	firstFile      bool
 }
 
-func NewYamlDecoder(prefs YamlPreferences) Decoder {
-	return &yamlDecoder{prefs: prefs, firstFile: true}
+func NewYamlDecoder() Decoder {
+	return &yamlDecoder{}
 }
 
-func (dec *yamlDecoder) processReadStream(reader *bufio.Reader) (io.Reader, string, error) {
-	var commentLineRegEx = regexp.MustCompile(`^\s*#`)
-	var sb strings.Builder
-	for {
-		peekBytes, err := reader.Peek(3)
-		if errors.Is(err, io.EOF) {
-			// EOF are handled else where..
-			return reader, sb.String(), nil
-		} else if err != nil {
-			return reader, sb.String(), err
-		} else if string(peekBytes) == "---" {
-			_, err := reader.ReadString('\n')
-			sb.WriteString("$yqDocSeperator$\n")
-			if errors.Is(err, io.EOF) {
-				return reader, sb.String(), nil
-			} else if err != nil {
-				return reader, sb.String(), err
-			}
-		} else if commentLineRegEx.MatchString(string(peekBytes)) {
-			line, err := reader.ReadString('\n')
-			sb.WriteString(line)
-			if errors.Is(err, io.EOF) {
-				return reader, sb.String(), nil
-			} else if err != nil {
-				return reader, sb.String(), err
-			}
-		} else {
-			return reader, sb.String(), nil
-		}
-	}
+func (dec *yamlDecoder) Init(reader io.Reader) {
+	dec.decoder = *yaml.NewDecoder(reader)
 }
 
-func (dec *yamlDecoder) Init(reader io.Reader) error {
-	readerToUse := reader
-	leadingContent := ""
-	var err error
-	// if we 'evaluating together' - we only process the leading content
-	// of the first file - this ensures comments from subsequent files are
-	// merged together correctly.
-	if dec.prefs.LeadingContentPreProcessing && (!dec.prefs.EvaluateTogether || dec.firstFile) {
-		readerToUse, leadingContent, err = dec.processReadStream(bufio.NewReader(reader))
-		if err != nil {
-			return err
-		}
-	}
-	dec.leadingContent = leadingContent
-	dec.readAnything = false
-	dec.decoder = *yaml.NewDecoder(readerToUse)
-	dec.firstFile = false
-	return nil
-}
-
-func (dec *yamlDecoder) Decode() (*CandidateNode, error) {
-	var dataBucket yaml.Node
-
-	err := dec.decoder.Decode(&dataBucket)
-	if errors.Is(err, io.EOF) && dec.leadingContent != "" && !dec.readAnything {
-		// force returning an empty node with a comment.
-		dec.readAnything = true
-		return dec.blankNodeWithComment(), nil
-
-	} else if err != nil {
-		return nil, err
-	}
-
-	candidateNode := &CandidateNode{
-		Node: &dataBucket,
-	}
-
-	if dec.leadingContent != "" {
-		candidateNode.LeadingContent = dec.leadingContent
-		dec.leadingContent = ""
-	}
-	// move document comments into candidate node
-	// otherwise unwrap drops them.
-	candidateNode.TrailingContent = dataBucket.FootComment
-	dataBucket.FootComment = ""
-	return candidateNode, nil
-}
-
-func (dec *yamlDecoder) blankNodeWithComment() *CandidateNode {
-	return &CandidateNode{
-		Document:       0,
-		Filename:       "",
-		Node:           &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
-		FileIndex:      0,
-		LeadingContent: dec.leadingContent,
-	}
+func (dec *yamlDecoder) Decode(rootYamlNode *yaml.Node) error {
+	return dec.decoder.Decode(rootYamlNode)
 }

pkg/yqlib/doc/operators/comment-operators.md

@@ -246,7 +246,6 @@ Note the use of `...` to ensure key nodes are included.
 
 Given a sample.yml file of:
 ```yaml
-# hi
 a: cat # comment
 # great
 b: # key comment
@@ -264,7 +263,6 @@ b:
 ## Get line comment
 Given a sample.yml file of:
 ```yaml
-# welcome!
 a: cat # meow
 # have a great day
 ```

pkg/yqlib/doc/operators/slice-array.md

@@ -1,77 +0,0 @@
-
-## Slicing arrays
-Given a sample.yml file of:
-```yaml
-- cat
-- dog
-- frog
-- cow
-```
-then
-```bash
-yq '.[1:3]' sample.yml
-```
-will output
-```yaml
-- dog
-- frog
-```
-
-## Slicing arrays - without the first number
-Starts from the start of the array
-
-Given a sample.yml file of:
-```yaml
-- cat
-- dog
-- frog
-- cow
-```
-then
-```bash
-yq '.[:2]' sample.yml
-```
-will output
-```yaml
-- cat
-- dog
-```
-
-## Slicing arrays - without the second number
-Finishes at the end of the array
-
-Given a sample.yml file of:
-```yaml
-- cat
-- dog
-- frog
-- cow
-```
-then
-```bash
-yq '.[2:]' sample.yml
-```
-will output
-```yaml
-- frog
-- cow
-```
-
-## Slicing arrays - use negative numbers to count backwards from the end
-Given a sample.yml file of:
-```yaml
-- cat
-- dog
-- frog
-- cow
-```
-then
-```bash
-yq '.[1:-1]' sample.yml
-```
-will output
-```yaml
-- dog
-- frog
-```
-

pkg/yqlib/doc/usage/properties.md

@@ -9,7 +9,7 @@ Note that empty arrays and maps are not encoded by default.
 
 Given a sample.yml file of:
 ```yaml
-# block comments come through
+# block comments don't come through
 person: # neither do comments on maps
     name: Mike Wazowski # comments on values appear
     pets: 
@@ -25,7 +25,6 @@ yq -o=props sample.yml
 ```
 will output
 ```properties
-# block comments come through
 # comments on values appear
 person.name = Mike Wazowski
 
@@ -39,7 +38,7 @@ Note that string values with blank characters in them are encapsulated with doub
 
 Given a sample.yml file of:
 ```yaml
-# block comments come through
+# block comments don't come through
 person: # neither do comments on maps
     name: Mike Wazowski # comments on values appear
     pets: 
@@ -55,7 +54,6 @@ yq -o=props --unwrapScalar=false sample.yml
 ```
 will output
 ```properties
-# block comments come through
 # comments on values appear
 person.name = "Mike Wazowski"
 
@@ -67,7 +65,7 @@ person.food.0 = pizza
 ## Encode properties: no comments
 Given a sample.yml file of:
 ```yaml
-# block comments come through
+# block comments don't come through
 person: # neither do comments on maps
     name: Mike Wazowski # comments on values appear
     pets: 
@@ -93,7 +91,7 @@ Use a yq expression to set the empty maps and sequences to your desired value.
 
 Given a sample.yml file of:
 ```yaml
-# block comments come through
+# block comments don't come through
 person: # neither do comments on maps
     name: Mike Wazowski # comments on values appear
     pets: 
@@ -109,7 +107,6 @@ yq -o=props '(.. | select( (tag == "!!map" or tag =="!!seq") and length == 0)) =
 ```
 will output
 ```properties
-# block comments come through
 # comments on values appear
 person.name = Mike Wazowski
 
@@ -123,7 +120,6 @@ emptyMap =
 ## Decode properties
 Given a sample.properties file of:
 ```properties
-# block comments come through
 # comments on values appear
 person.name = Mike Wazowski
 
@@ -139,12 +135,9 @@ yq -p=props sample.properties
 will output
 ```yaml
 person:
-    # block comments come through
-    # comments on values appear
-    name: Mike Wazowski
+    name: Mike Wazowski # comments on values appear
     pets:
-        # comments on array values appear
-        - cat
+        - cat # comments on array values appear
     food:
         - pizza
 ```
@@ -152,7 +145,6 @@ person:
 ## Roundtrip
 Given a sample.properties file of:
 ```properties
-# block comments come through
 # comments on values appear
 person.name = Mike Wazowski
 
@@ -167,7 +159,6 @@ yq -p=props -o=props '.person.pets.0 = "dog"' sample.properties
 ```
 will output
 ```properties
-# block comments come through
 # comments on values appear
 person.name = Mike Wazowski
 

pkg/yqlib/doc/usage/xml.md

@@ -386,7 +386,6 @@ A best attempt is made to copy comments to xml.
 
 Given a sample.yml file of:
 ```yaml
-# header comment
 # above_cat
 cat: # inline_cat
   # above_array
@@ -403,10 +402,7 @@ yq -o=xml '.' sample.yml
 ```
 will output
 ```xml
-<!--
- header comment
- above_cat
- --><!-- inline_cat --><cat><!-- above_array inline_array -->
+<!-- above_cat inline_cat --><cat><!-- above_array inline_array -->
   <array>val1<!-- inline_val1 --></array>
   <array><!-- above_val2 -->val2<!-- inline_val2 --></array>
 </cat><!-- below_cat -->

pkg/yqlib/encoder_properties.go

@@ -65,7 +65,7 @@ func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content strin
 func (pe *propertiesEncoder) Encode(writer io.Writer, node *yaml.Node) error {
 	mapKeysToStrings(node)
 	p := properties.NewProperties()
-	err := pe.doEncode(p, node, "", nil)
+	err := pe.doEncode(p, node, "")
 	if err != nil {
 		return err
 	}
@@ -74,17 +74,8 @@ func (pe *propertiesEncoder) Encode(writer io.Writer, node *yaml.Node) error {
 	return err
 }
 
-func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yaml.Node, path string, keyNode *yaml.Node) error {
-
-	comments := ""
-	if keyNode != nil {
-		// include the key node comments if present
-		comments = headAndLineComment(keyNode)
-	}
-	comments = comments + headAndLineComment(node)
-	commentsWithSpaces := strings.ReplaceAll(comments, "\n", "\n ")
-	p.SetComments(path, strings.Split(commentsWithSpaces, "\n"))
-
+func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yaml.Node, path string) error {
+	p.SetComment(path, headAndLineComment(node))
 	switch node.Kind {
 	case yaml.ScalarNode:
 		var nodeValue string
@@ -96,13 +87,13 @@ func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yaml.Node,
 		_, _, err := p.Set(path, nodeValue)
 		return err
 	case yaml.DocumentNode:
-		return pe.doEncode(p, node.Content[0], path, node)
+		return pe.doEncode(p, node.Content[0], path)
 	case yaml.SequenceNode:
 		return pe.encodeArray(p, node.Content, path)
 	case yaml.MappingNode:
 		return pe.encodeMap(p, node.Content, path)
 	case yaml.AliasNode:
-		return pe.doEncode(p, node.Alias, path, nil)
+		return pe.doEncode(p, node.Alias, path)
 	default:
 		return fmt.Errorf("Unsupported node %v", node.Tag)
 	}
@@ -117,7 +108,7 @@ func (pe *propertiesEncoder) appendPath(path string, key interface{}) string {
 
 func (pe *propertiesEncoder) encodeArray(p *properties.Properties, kids []*yaml.Node, path string) error {
 	for index, child := range kids {
-		err := pe.doEncode(p, child, pe.appendPath(path, index), nil)
+		err := pe.doEncode(p, child, pe.appendPath(path, index))
 		if err != nil {
 			return err
 		}
@@ -129,7 +120,7 @@ func (pe *propertiesEncoder) encodeMap(p *properties.Properties, kids []*yaml.No
 	for index := 0; index < len(kids); index = index + 2 {
 		key := kids[index]
 		value := kids[index+1]
-		err := pe.doEncode(p, value, pe.appendPath(path, key.Value), key)
+		err := pe.doEncode(p, value, pe.appendPath(path, key.Value))
 		if err != nil {
 			return err
 		}

pkg/yqlib/encoder_properties_test.go

@@ -14,7 +14,7 @@ func yamlToProps(sampleYaml string, unwrapScalar bool) string {
 	writer := bufio.NewWriter(&output)
 
 	var propsEncoder = NewPropertiesEncoder(unwrapScalar)
-	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
+	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
 		panic(err)
 	}

pkg/yqlib/encoder_test.go

@@ -14,7 +14,7 @@ func yamlToJSON(sampleYaml string, indent int) string {
 	writer := bufio.NewWriter(&output)
 
 	var jsonEncoder = NewJSONEncoder(indent, false)
-	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
+	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
 		panic(err)
 	}

pkg/yqlib/encoder_xml.go

@@ -4,28 +4,24 @@ import (
 	"encoding/xml"
 	"fmt"
 	"io"
-	"regexp"
 	"strings"
 
 	yaml "gopkg.in/yaml.v3"
 )
 
 type xmlEncoder struct {
-	indentString   string
-	writer         io.Writer
-	prefs          XmlPreferences
-	leadingContent string
+	indentString string
+	writer       io.Writer
+	prefs        xmlPreferences
 }
 
-var commentPrefix = regexp.MustCompile(`(^|\n)\s*#`)
-
-func NewXMLEncoder(indent int, prefs XmlPreferences) Encoder {
+func NewXMLEncoder(indent int, prefs xmlPreferences) Encoder {
 	var indentString = ""
 
 	for index := 0; index < indent; index++ {
 		indentString = indentString + " "
 	}
-	return &xmlEncoder{indentString, nil, prefs, ""}
+	return &xmlEncoder{indentString, nil, prefs}
 }
 
 func (e *xmlEncoder) CanHandleAliases() bool {
@@ -37,7 +33,6 @@ func (e *xmlEncoder) PrintDocumentSeparator(writer io.Writer) error {
 }
 
 func (e *xmlEncoder) PrintLeadingContent(writer io.Writer, content string) error {
-	e.leadingContent = commentPrefix.ReplaceAllString(content, "\n")
 	return nil
 }
 
@@ -47,13 +42,6 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
 	e.writer = writer
 	encoder.Indent("", e.indentString)
 
-	if e.leadingContent != "" {
-		err := e.encodeComment(encoder, e.leadingContent)
-		if err != nil {
-			return err
-		}
-	}
-
 	switch node.Kind {
 	case yaml.MappingNode:
 		err := e.encodeTopLevelMap(encoder, node)

pkg/yqlib/encoder_yaml.go

@@ -11,16 +11,17 @@ import (
 )
 
 type yamlEncoder struct {
-	indent   int
-	colorise bool
-	prefs    YamlPreferences
+	indent             int
+	colorise           bool
+	printDocSeparators bool
+	unwrapScalar       bool
 }
 
-func NewYamlEncoder(indent int, colorise bool, prefs YamlPreferences) Encoder {
+func NewYamlEncoder(indent int, colorise bool, printDocSeparators bool, unwrapScalar bool) Encoder {
 	if indent < 0 {
 		indent = 0
 	}
-	return &yamlEncoder{indent, colorise, prefs}
+	return &yamlEncoder{indent, colorise, printDocSeparators, unwrapScalar}
 }
 
 func (ye *yamlEncoder) CanHandleAliases() bool {
@@ -28,7 +29,7 @@ func (ye *yamlEncoder) CanHandleAliases() bool {
 }
 
 func (ye *yamlEncoder) PrintDocumentSeparator(writer io.Writer) error {
-	if ye.prefs.PrintDocSeparators {
+	if ye.printDocSeparators {
 		log.Debug("-- writing doc sep")
 		if err := writeString(writer, "---\n"); err != nil {
 			return err
@@ -75,7 +76,7 @@ func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) err
 
 func (ye *yamlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
 
-	if node.Kind == yaml.ScalarNode && ye.prefs.UnwrapScalar {
+	if node.Kind == yaml.ScalarNode && ye.unwrapScalar {
 		return writeString(writer, node.Value+"\n")
 	}
 

pkg/yqlib/json_test.go

@@ -262,11 +262,11 @@ func documentDecodeNdJsonScenario(w *bufio.Writer, s formatScenario) {
 
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))))
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(s.indent, false, true, true))))
 }
 
 func decodeJSON(t *testing.T, jsonString string) *CandidateNode {
-	docs, err := readDocument(jsonString, "sample.json", 0)
+	docs, err := readDocumentWithLeadingContent(jsonString, "sample.json", 0)
 
 	if err != nil {
 		t.Error(err)
@@ -293,12 +293,12 @@ func decodeJSON(t *testing.T, jsonString string) *CandidateNode {
 func testJSONScenario(t *testing.T, s formatScenario) {
 	switch s.scenarioType {
 	case "encode", "decode":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewJSONEncoder(s.indent, false)), s.description)
 	case "":
 		var actual = resultToString(t, decodeJSON(t, s.input))
 		test.AssertResultWithContext(t, s.expected, actual, s.description)
 	case "decode-ndjson":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(2, false, true, true)), s.description)
 	case "roundtrip-ndjson":
 		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(0, false)), s.description)
 	case "roundtrip-multi":
@@ -385,7 +385,7 @@ func documentJSONEncodeScenario(w *bufio.Writer, s formatScenario) {
 	}
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false))))
+	writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewJSONEncoder(s.indent, false))))
 }
 
 func TestJSONScenarios(t *testing.T) {

pkg/yqlib/lexer.go

@@ -3,6 +3,7 @@ package yqlib
 import (
 	"fmt"
 	"regexp"
+	"strconv"
 )
 
 type expressionTokeniser interface {
@@ -63,11 +64,11 @@ func unwrap(value string) string {
 func extractNumberParameter(value string) (int, error) {
 	parameterParser := regexp.MustCompile(`.*\(([0-9]+)\)`)
 	matches := parameterParser.FindStringSubmatch(value)
-	var indent, errParsingInt = parseInt(matches[1])
+	var indent, errParsingInt = strconv.ParseInt(matches[1], 10, 32)
 	if errParsingInt != nil {
 		return 0, errParsingInt
 	}
-	return indent, nil
+	return int(indent), nil
 }
 
 func hasOptionParameter(value string, option string) bool {

pkg/yqlib/lexer_participle.go

@@ -1,7 +1,6 @@
 package yqlib
 
 import (
-	"regexp"
 	"strconv"
 	"strings"
 
@@ -13,10 +12,6 @@ var participleYqRules = []*participleYqRule{
 	{"HEAD_COMMENT", `head_?comment|headComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{HeadComment: true}), 0},
 	{"FOOT_COMMENT", `foot_?comment|footComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{FootComment: true}), 0},
 
-	{"SliceArray", `\.\[-?[0-9]+:-?[0-9]+\]`, sliceArrayTwoNumbers(), 0},
-	{"SliceArraySecond", `\.\[\:-?[0-9]+\]`, sliceArraySecondNumberOnly(), 0},
-	{"SliceArrayFirst", `\.\[-?[0-9]+\:\]`, sliceArrayFirstNumberOnly(), 0},
-
 	{"OpenBracket", `\(`, literalToken(openBracket, false), 0},
 	{"CloseBracket", `\)`, literalToken(closeBracket, true), 0},
 	{"OpenTraverseArrayCollect", `\.\[`, literalToken(traverseArrayCollect, false), 0},
@@ -81,7 +76,7 @@ var participleYqRules = []*participleYqRule{
 	{"Base64d", `@base64d`, decodeOp(Base64InputFormat), 0},
 	{"Base64", `@base64`, encodeWithIndent(Base64OutputFormat, 0), 0},
 
-	{"LoadXML", `load_?xml|xml_?load`, loadOp(NewXMLDecoder(ConfiguredXMLPreferences), false), 0},
+	{"LoadXML", `load_?xml|xml_?load`, loadOp(NewXMLDecoder(XMLPreferences), false), 0},
 
 	{"LoadBase64", `load_?base64`, loadOp(NewBase64Decoder(), false), 0},
 
@@ -89,7 +84,7 @@ var participleYqRules = []*participleYqRule{
 
 	{"LoadString", `load_?str|str_?load`, loadOp(nil, true), 0},
 
-	{"LoadYaml", `load`, loadOp(NewYamlDecoder(ConfiguredYamlPreferences), false), 0},
+	{"LoadYaml", `load`, loadOp(NewYamlDecoder(), false), 0},
 
 	{"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0},
 
@@ -305,84 +300,6 @@ func flattenWithDepth() yqAction {
 	}
 }
 
-func sliceArrayTwoNumbers() yqAction {
-	return func(rawToken lexer.Token) (*token, error) {
-		value := rawToken.Value
-		sliceArrayNumbers := regexp.MustCompile(`\.\[(-?[0-9]+)\:(-?[0-9]+)\]`)
-		matches := sliceArrayNumbers.FindStringSubmatch(value)
-		log.Debug("sliceArrayTwoNumbers value: %v", value)
-		log.Debug("Matches: %v", matches)
-
-		firstNumber, err := parseInt(matches[1])
-		if err != nil {
-			return nil, err
-		}
-		secondNumber, err := parseInt(matches[2])
-		if err != nil {
-			return nil, err
-		}
-
-		prefs := sliceArrayPreferences{
-			firstNumber:         firstNumber,
-			secondNumber:        secondNumber,
-			secondNumberDefined: true,
-		}
-		log.Debug("%v", prefs)
-
-		op := &Operation{OperationType: sliceArrayOpType, Value: sliceArrayOpType.Type, StringValue: value, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func sliceArraySecondNumberOnly() yqAction {
-	return func(rawToken lexer.Token) (*token, error) {
-		value := rawToken.Value
-		sliceArrayNumbers := regexp.MustCompile(`\.\[\:(-?[0-9]+)\]`)
-		matches := sliceArrayNumbers.FindStringSubmatch(value)
-		log.Debug("sliceArraySecondNumberOnly value: %v", value)
-		log.Debug("Matches: %v", matches)
-
-		secondNumber, err := parseInt(matches[1])
-		if err != nil {
-			return nil, err
-		}
-
-		prefs := sliceArrayPreferences{
-			firstNumber:         0,
-			secondNumber:        secondNumber,
-			secondNumberDefined: true,
-		}
-		log.Debug("%v", prefs)
-
-		op := &Operation{OperationType: sliceArrayOpType, Value: sliceArrayOpType.Type, StringValue: value, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func sliceArrayFirstNumberOnly() yqAction {
-	return func(rawToken lexer.Token) (*token, error) {
-		value := rawToken.Value
-		sliceArrayNumbers := regexp.MustCompile(`\.\[(-?[0-9]+)\:\]`)
-		matches := sliceArrayNumbers.FindStringSubmatch(value)
-		log.Debug("sliceArrayFirstNumberOnly value: %v", value)
-		log.Debug("Matches: %v", matches)
-
-		firstNumber, err := parseInt(matches[1])
-		if err != nil {
-			return nil, err
-		}
-
-		prefs := sliceArrayPreferences{
-			firstNumber:         firstNumber,
-			secondNumberDefined: false,
-		}
-		log.Debug("%v", prefs)
-
-		op := &Operation{OperationType: sliceArrayOpType, Value: sliceArrayOpType.Type, StringValue: value, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
 func assignAllCommentsOp(updateAssign bool) yqAction {
 	return func(rawToken lexer.Token) (*token, error) {
 		log.Debug("assignAllCommentsOp %v", rawToken.Value)

pkg/yqlib/lexer_participle_test.go

@@ -14,62 +14,6 @@ type participleLexerScenario struct {
 }
 
 var participleLexerScenarios = []participleLexerScenario{
-	{
-		expression: ".[1:3]",
-		tokens: []*token{
-			{
-				TokenType: operationToken,
-				Operation: &Operation{
-					OperationType: sliceArrayOpType,
-					Value:         "SLICE",
-					StringValue:   ".[1:3]",
-					Preferences:   sliceArrayPreferences{firstNumber: 1, secondNumber: 3, secondNumberDefined: true},
-				},
-			},
-		},
-	},
-	{
-		expression: ".[:3]",
-		tokens: []*token{
-			{
-				TokenType: operationToken,
-				Operation: &Operation{
-					OperationType: sliceArrayOpType,
-					Value:         "SLICE",
-					StringValue:   ".[:3]",
-					Preferences:   sliceArrayPreferences{firstNumber: 0, secondNumber: 3, secondNumberDefined: true},
-				},
-			},
-		},
-	},
-	{
-		expression: ".[1:]",
-		tokens: []*token{
-			{
-				TokenType: operationToken,
-				Operation: &Operation{
-					OperationType: sliceArrayOpType,
-					Value:         "SLICE",
-					StringValue:   ".[1:]",
-					Preferences:   sliceArrayPreferences{firstNumber: 1, secondNumber: 0, secondNumberDefined: false},
-				},
-			},
-		},
-	},
-	{
-		expression: ".[-100:-54]",
-		tokens: []*token{
-			{
-				TokenType: operationToken,
-				Operation: &Operation{
-					OperationType: sliceArrayOpType,
-					Value:         "SLICE",
-					StringValue:   ".[-100:-54]",
-					Preferences:   sliceArrayPreferences{firstNumber: -100, secondNumber: -54, secondNumberDefined: true},
-				},
-			},
-		},
-	},
 	{
 		expression: ".a",
 		tokens: []*token{

pkg/yqlib/lib.go

@@ -21,6 +21,34 @@ func InitExpressionParser() {
 	}
 }
 
+type xmlPreferences struct {
+	AttributePrefix string
+	ContentName     string
+	StrictMode      bool
+	KeepNamespace   bool
+	UseRawToken     bool
+	ProcInstPrefix  string
+	DirectiveName   string
+	SkipProcInst    bool
+	SkipDirectives  bool
+}
+
+func NewDefaultXmlPreferences() xmlPreferences {
+	return xmlPreferences{
+		AttributePrefix: "+",
+		ContentName:     "+content",
+		StrictMode:      false,
+		KeepNamespace:   true,
+		UseRawToken:     false,
+		ProcInstPrefix:  "+p_",
+		DirectiveName:   "+directive",
+		SkipProcInst:    false,
+		SkipDirectives:  false,
+	}
+}
+
+var XMLPreferences = NewDefaultXmlPreferences()
+
 var log = logging.MustGetLogger("yq-lib")
 
 var PrettyPrintExp = `(... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not))  ) style=""`
@@ -81,7 +109,6 @@ var lineOpType = &operationType{Type: "LINE", NumArgs: 0, Precedence: 50, Handle
 var columnOpType = &operationType{Type: "LINE", NumArgs: 0, Precedence: 50, Handler: columnOperator}
 
 var collectOpType = &operationType{Type: "COLLECT", NumArgs: 1, Precedence: 50, Handler: collectOperator}
-var sliceArrayOpType = &operationType{Type: "SLICE", NumArgs: 0, Precedence: 50, Handler: sliceArrayOperator}
 var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 50, Handler: mapOperator}
 var errorOpType = &operationType{Type: "ERROR", NumArgs: 1, Precedence: 50, Handler: errorOperator}
 var pickOpType = &operationType{Type: "PICK", NumArgs: 1, Precedence: 50, Handler: pickOperator}
@@ -249,16 +276,14 @@ func guessTagFromCustomType(node *yaml.Node) string {
 }
 
 func parseSnippet(value string) (*yaml.Node, error) {
-	decoder := NewYamlDecoder(ConfiguredYamlPreferences)
-	err := decoder.Init(strings.NewReader(value))
-	if err != nil {
-		return nil, err
-	}
-	parsedNode, err := decoder.Decode()
-	if len(parsedNode.Node.Content) == 0 {
+	decoder := NewYamlDecoder()
+	decoder.Init(strings.NewReader(value))
+	var dataBucket yaml.Node
+	err := decoder.Decode(&dataBucket)
+	if len(dataBucket.Content) == 0 {
 		return nil, fmt.Errorf("bad data")
 	}
-	return unwrapDoc(parsedNode.Node), err
+	return dataBucket.Content[0], err
 }
 
 func recursiveNodeEqual(lhs *yaml.Node, rhs *yaml.Node) bool {
@@ -353,8 +378,8 @@ func parseInt(numberString string) (int, error) {
 
 	if err != nil {
 		return 0, err
-	} else if parsed > math.MaxInt || parsed < math.MinInt {
-		return 0, fmt.Errorf("%v is not within [%v, %v]", parsed, math.MinInt, math.MaxInt)
+	} else if parsed > math.MaxInt {
+		return 0, fmt.Errorf("%v is too big (larger than %v)", parsed, math.MaxInt)
 	}
 
 	return int(parsed), err

pkg/yqlib/operator_collect.go

@@ -56,7 +56,7 @@ func collectOperator(d *dataTreeNavigator, context Context, expressionNode *Expr
 		collectedNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
 		collectCandidate := candidate.CreateReplacement(collectedNode)
 
-		collectExpResults, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.RHS)
+		collectExpResults, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS)
 		if err != nil {
 			return Context{}, err
 		}

pkg/yqlib/operator_collect_test.go

@@ -20,14 +20,6 @@ var collectOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::- 1\n- 2\n- 3\n",
 		},
 	},
-	{
-		skipDoc:     true,
-		description: "update in collect",
-		expression:  `[.a = "cat"]`,
-		expected: []string{
-			"D0, P[], (!!seq)::- a: cat\n",
-		},
-	},
 	{
 		description: "Collect empty",
 		document:    ``,

pkg/yqlib/operator_comments.go

@@ -83,10 +83,6 @@ func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *
 	log.Debugf("GetComments operator!")
 	var results = list.New()
 
-	yamlPrefs := NewDefaultYamlPreferences()
-	yamlPrefs.PrintDocSeparators = false
-	yamlPrefs.UnwrapScalar = false
-
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 		comment := ""
@@ -96,7 +92,7 @@ func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *
 			var chompRegexp = regexp.MustCompile(`\n$`)
 			var output bytes.Buffer
 			var writer = bufio.NewWriter(&output)
-			var encoder = NewYamlEncoder(2, false, yamlPrefs)
+			var encoder = NewYamlEncoder(2, false, false, false)
 			if err := encoder.PrintLeadingContent(writer, candidate.LeadingContent); err != nil {
 				return Context{}, err
 			}

pkg/yqlib/operator_datetime.go

@@ -4,6 +4,7 @@ import (
 	"container/list"
 	"errors"
 	"fmt"
+	"strings"
 	"time"
 
 	"gopkg.in/yaml.v3"
@@ -53,6 +54,7 @@ func nowOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode
 func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	format, err := getStringParamter("format", d, context, expressionNode.RHS)
 	layout := context.GetDateTimeLayout()
+	decoder := NewYamlDecoder()
 
 	if err != nil {
 		return Context{}, err
@@ -67,15 +69,19 @@ func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *Expre
 			return Context{}, fmt.Errorf("could not parse datetime of [%v]: %w", candidate.GetNicePath(), err)
 		}
 		formattedTimeStr := parsedTime.Format(format)
-
-		node, errorReading := parseSnippet(formattedTimeStr)
+		decoder.Init(strings.NewReader(formattedTimeStr))
+		var dataBucket yaml.Node
+		errorReading := decoder.Decode(&dataBucket)
+		var node *yaml.Node
 		if errorReading != nil {
-			log.Debugf("could not parse %v - lets just leave it as a string: %w", formattedTimeStr, errorReading)
+			log.Debugf("could not parse %v - lets just leave it as a string", formattedTimeStr)
 			node = &yaml.Node{
 				Kind:  yaml.ScalarNode,
 				Tag:   "!!str",
 				Value: formattedTimeStr,
 			}
+		} else {
+			node = unwrapDoc(&dataBucket)
 		}
 
 		results.PushBack(candidate.CreateReplacement(node))

pkg/yqlib/operator_encoder_decoder.go

@@ -21,9 +21,9 @@ func configureEncoder(format PrinterOutputFormat, indent int) Encoder {
 	case TSVOutputFormat:
 		return NewCsvEncoder('\t')
 	case YamlOutputFormat:
-		return NewYamlEncoder(indent, false, ConfiguredYamlPreferences)
+		return NewYamlEncoder(indent, false, true, true)
 	case XMLOutputFormat:
-		return NewXMLEncoder(indent, ConfiguredXMLPreferences)
+		return NewXMLEncoder(indent, XMLPreferences)
 	case Base64OutputFormat:
 		return NewBase64Encoder()
 	}
@@ -102,9 +102,9 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 	var decoder Decoder
 	switch preferences.format {
 	case YamlInputFormat:
-		decoder = NewYamlDecoder(ConfiguredYamlPreferences)
+		decoder = NewYamlDecoder()
 	case XMLInputFormat:
-		decoder = NewXMLDecoder(ConfiguredXMLPreferences)
+		decoder = NewXMLDecoder(XMLPreferences)
 	case Base64InputFormat:
 		decoder = NewBase64Decoder()
 	case PropertiesInputFormat:
@@ -121,19 +121,17 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 
 		context.SetVariable("decoded: "+candidate.GetKey(), candidate.AsList())
 
+		var dataBucket yaml.Node
 		log.Debugf("got: [%v]", candidate.Node.Value)
 
-		err := decoder.Init(strings.NewReader(unwrapDoc(candidate.Node).Value))
-		if err != nil {
-			return Context{}, err
-		}
+		decoder.Init(strings.NewReader(unwrapDoc(candidate.Node).Value))
 
-		decodedNode, errorReading := decoder.Decode()
+		errorReading := decoder.Decode(&dataBucket)
 		if errorReading != nil {
 			return Context{}, errorReading
 		}
 		//first node is a doc
-		node := unwrapDoc(decodedNode.Node)
+		node := unwrapDoc(&dataBucket)
 
 		results.PushBack(candidate.CreateReplacement(node))
 	}

pkg/yqlib/operator_load.go

@@ -48,7 +48,7 @@ func loadYaml(filename string, decoder Decoder) (*CandidateNode, error) {
 	} else {
 		sequenceNode := &CandidateNode{Node: &yaml.Node{Kind: yaml.SequenceNode}}
 		for doc := documents.Front(); doc != nil; doc = doc.Next() {
-			sequenceNode.Node.Content = append(sequenceNode.Node.Content, unwrapDoc(doc.Value.(*CandidateNode).Node))
+			sequenceNode.Node.Content = append(sequenceNode.Node.Content, doc.Value.(*CandidateNode).Node)
 		}
 		return sequenceNode, nil
 	}

pkg/yqlib/operator_load_test.go

@@ -5,30 +5,6 @@ import (
 )
 
 var loadScenarios = []expressionScenario{
-	{
-		skipDoc:     true,
-		description: "Load empty file with a comment",
-		expression:  `load("../../examples/empty.yaml")`,
-		expected: []string{
-			"D0, P[], (doc)::# comment\n\n",
-		},
-	},
-	{
-		skipDoc:     true,
-		description: "Load empty file with no comment",
-		expression:  `load("../../examples/empty-no-comment.yaml")`,
-		expected: []string{
-			"D0, P[], (!!null)::\n",
-		},
-	},
-	{
-		skipDoc:     true,
-		description: "Load multiple documents",
-		expression:  `load("../../examples/multiple_docs_small.yaml")`,
-		expected: []string{
-			"D0, P[], ()::- a: Easy! as one two three\n- another:\n    document: here\n- - 1\n  - 2\n",
-		},
-	},
 	{
 		description: "Simple example",
 		document:    `{myFile: "../../examples/thing.yml"}`,

pkg/yqlib/operator_slice.go

@@ -1,63 +0,0 @@
-package yqlib
-
-import (
-	"container/list"
-
-	yaml "gopkg.in/yaml.v3"
-)
-
-type sliceArrayPreferences struct {
-	firstNumber         int
-	secondNumber        int
-	secondNumberDefined bool
-}
-
-func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-
-	lhs, err := d.GetMatchingNodes(context, expressionNode.LHS)
-	if err != nil {
-		return Context{}, err
-	}
-	prefs := expressionNode.Operation.Preferences.(sliceArrayPreferences)
-	firstNumber := prefs.firstNumber
-	secondNumber := prefs.secondNumber
-
-	results := list.New()
-
-	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
-		lhsNode := el.Value.(*CandidateNode)
-		original := unwrapDoc(lhsNode.Node)
-
-		relativeFirstNumber := firstNumber
-		if relativeFirstNumber < 0 {
-			relativeFirstNumber = len(original.Content) + firstNumber
-		}
-
-		relativeSecondNumber := len(original.Content)
-		if prefs.secondNumberDefined {
-			relativeSecondNumber = secondNumber
-			if relativeSecondNumber < 0 {
-				relativeSecondNumber = len(original.Content) + secondNumber
-			}
-		}
-
-		log.Debug("calculateIndicesToTraverse: slice from %v to %v", relativeFirstNumber, relativeSecondNumber)
-
-		var newResults []*yaml.Node
-		for i := relativeFirstNumber; i < relativeSecondNumber; i++ {
-			newResults = append(newResults, original.Content[i])
-		}
-
-		slicedArrayNode := &yaml.Node{
-			Kind:    yaml.SequenceNode,
-			Tag:     original.Tag,
-			Content: newResults,
-		}
-		results.PushBack(lhsNode.CreateReplacement(slicedArrayNode))
-
-	}
-
-	// result is now the context that has the nodes we need to put back into a sequence.
-	//what about multiple arrays in the context? I think we need to create an array for each one
-	return context.ChildContext(results), nil
-}

pkg/yqlib/operator_slice_test.go

@@ -1,81 +0,0 @@
-package yqlib
-
-import "testing"
-
-var sliceArrayScenarios = []expressionScenario{
-	{
-		description: "Slicing arrays",
-		document:    `[cat, dog, frog, cow]`,
-		expression:  `.[1:3]`,
-		expected: []string{
-			"D0, P[], (!!seq)::- dog\n- frog\n",
-		},
-	},
-	{
-		description:    "Slicing arrays - without the first number",
-		subdescription: "Starts from the start of the array",
-		document:       `[cat, dog, frog, cow]`,
-		expression:     `.[:2]`,
-		expected: []string{
-			"D0, P[], (!!seq)::- cat\n- dog\n",
-		},
-	},
-	{
-		description:    "Slicing arrays - without the second number",
-		subdescription: "Finishes at the end of the array",
-		document:       `[cat, dog, frog, cow]`,
-		expression:     `.[2:]`,
-		expected: []string{
-			"D0, P[], (!!seq)::- frog\n- cow\n",
-		},
-	},
-	{
-		description: "Slicing arrays - use negative numbers to count backwards from the end",
-		document:    `[cat, dog, frog, cow]`,
-		expression:  `.[1:-1]`,
-		expected: []string{
-			"D0, P[], (!!seq)::- dog\n- frog\n",
-		},
-	},
-	{
-		skipDoc:    true,
-		document:   `[[cat, dog, frog, cow], [apple, banana, grape, mango]]`,
-		expression: `.[] | .[1:3]`,
-		expected: []string{
-			"D0, P[0], (!!seq)::- dog\n- frog\n",
-			"D0, P[1], (!!seq)::- banana\n- grape\n",
-		},
-	},
-	{
-		skipDoc:    true,
-		document:   `[[cat, dog, frog, cow], [apple, banana, grape, mango]]`,
-		expression: `.[] | .[-2:-1]`,
-		expected: []string{
-			"D0, P[0], (!!seq)::- frog\n",
-			"D0, P[1], (!!seq)::- grape\n",
-		},
-	},
-	{
-		skipDoc:    true,
-		document:   `[cat1, cat2, cat3, cat4, cat5, cat6, cat7, cat8, cat9, cat10, cat11]`,
-		expression: `.[10:11]`,
-		expected: []string{
-			"D0, P[], (!!seq)::- cat11\n",
-		},
-	},
-	{
-		skipDoc:    true,
-		document:   `[cat1, cat2, cat3, cat4, cat5, cat6, cat7, cat8, cat9, cat10, cat11]`,
-		expression: `.[-11:-10]`,
-		expected: []string{
-			"D0, P[], (!!seq)::- cat1\n",
-		},
-	},
-}
-
-func TestSliceOperatorScenarios(t *testing.T) {
-	for _, tt := range sliceArrayScenarios {
-		testScenario(t, &tt)
-	}
-	documentOperatorScenarios(t, "slice-array", sliceArrayScenarios)
-}

pkg/yqlib/operator_traverse_path.go

@@ -76,6 +76,7 @@ func traverse(context Context, matchingNode *CandidateNode, operation *Operation
 }
 
 func traverseArrayOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
 	//lhs may update the variable context, we should pass that into the RHS
 	// BUT we still return the original context back (see jq)
 	// https://stedolan.github.io/jq/manual/#Variable/SymbolicBindingOperator:...as$identifier|...

pkg/yqlib/operators_test.go

@@ -31,7 +31,7 @@ type expressionScenario struct {
 }
 
 func TestMain(m *testing.M) {
-	logging.SetLevel(logging.DEBUG, "")
+	logging.SetLevel(logging.ERROR, "")
 	Now = func() time.Time {
 		return time.Date(2021, time.May, 19, 1, 2, 3, 4, time.UTC)
 	}
@@ -40,16 +40,21 @@ func TestMain(m *testing.M) {
 }
 
 func NewSimpleYamlPrinter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
-	prefs := NewDefaultYamlPreferences()
-	prefs.PrintDocSeparators = printDocSeparators
-	prefs.UnwrapScalar = unwrapScalar
-	return NewPrinter(NewYamlEncoder(indent, colorsEnabled, prefs), NewSinglePrinterWriter(writer))
+	return NewPrinter(NewYamlEncoder(indent, colorsEnabled, printDocSeparators, unwrapScalar), NewSinglePrinterWriter(writer))
 }
 
-func readDocument(content string, fakefilename string, fakeFileIndex int) (*list.List, error) {
-	reader := bufio.NewReader(strings.NewReader(content))
+func readDocumentWithLeadingContent(content string, fakefilename string, fakeFileIndex int) (*list.List, error) {
+	reader, firstFileLeadingContent, err := processReadStream(bufio.NewReader(strings.NewReader(content)))
+	if err != nil {
+		return nil, err
+	}
 
-	return readDocuments(reader, fakefilename, fakeFileIndex, NewYamlDecoder(ConfiguredYamlPreferences))
+	inputs, err := readDocuments(reader, fakefilename, fakeFileIndex, NewYamlDecoder())
+	if err != nil {
+		return nil, err
+	}
+	inputs.Front().Value.(*CandidateNode).LeadingContent = firstFileLeadingContent
+	return inputs, nil
 }
 
 func testScenario(t *testing.T, s *expressionScenario) {
@@ -62,7 +67,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
 	inputs := list.New()
 
 	if s.document != "" {
-		inputs, err = readDocument(s.document, "sample.yml", 0)
+		inputs, err = readDocumentWithLeadingContent(s.document, "sample.yml", 0)
 
 		if err != nil {
 			t.Error(err, s.document, s.expression)
@@ -70,7 +75,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
 		}
 
 		if s.document2 != "" {
-			moreInputs, err := readDocument(s.document2, "another.yml", 1)
+			moreInputs, err := readDocumentWithLeadingContent(s.document2, "another.yml", 1)
 			if err != nil {
 				t.Error(err, s.document2, s.expression)
 				return
@@ -171,7 +176,7 @@ func formatYaml(yaml string, filename string) string {
 		panic(err)
 	}
 	streamEvaluator := NewStreamEvaluator()
-	_, err = streamEvaluator.Evaluate(filename, strings.NewReader(yaml), node, printer, NewYamlDecoder(ConfiguredYamlPreferences))
+	_, err = streamEvaluator.Evaluate(filename, strings.NewReader(yaml), node, printer, "", NewYamlDecoder())
 	if err != nil {
 		panic(err)
 	}
@@ -317,13 +322,13 @@ func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formatt
 
 	if s.document != "" {
 
-		inputs, err = readDocument(formattedDoc, "sample.yml", 0)
+		inputs, err = readDocumentWithLeadingContent(formattedDoc, "sample.yml", 0)
 		if err != nil {
 			t.Error(err, s.document, s.expression)
 			return
 		}
 		if s.document2 != "" {
-			moreInputs, err := readDocument(formattedDoc2, "another.yml", 1)
+			moreInputs, err := readDocumentWithLeadingContent(formattedDoc2, "another.yml", 1)
 			if err != nil {
 				t.Error(err, s.document, s.expression)
 				return

pkg/yqlib/printer.go

@@ -111,7 +111,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 
 		mappedDoc := el.Value.(*CandidateNode)
 		log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document)
-		log.Debug("%v", NodeToString(mappedDoc))
+
 		writer, errorWriting := p.printerWriter.GetWriter(mappedDoc)
 		if errorWriting != nil {
 			return errorWriting

pkg/yqlib/printer_test.go

@@ -38,7 +38,7 @@ func TestPrinterMultipleDocsInSequenceOnly(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
-	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
 		panic(err)
 	}
@@ -76,7 +76,7 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
-	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
 		panic(err)
 	}
@@ -118,7 +118,7 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
-	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
 		panic(err)
 	}
@@ -165,7 +165,7 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
-	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
 		panic(err)
 	}
@@ -215,7 +215,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
-	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
 		panic(err)
 	}
@@ -234,7 +234,7 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
-	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
 		panic(err)
 	}
@@ -263,7 +263,7 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
-	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
 		panic(err)
 	}
@@ -294,7 +294,7 @@ func TestPrinterScalarWithLeadingCont(t *testing.T) {
 		panic(err)
 	}
 	streamEvaluator := NewStreamEvaluator()
-	_, err = streamEvaluator.Evaluate("sample", strings.NewReader(multiDocSample), node, printer, NewYamlDecoder(ConfiguredYamlPreferences))
+	_, err = streamEvaluator.Evaluate("sample", strings.NewReader(multiDocSample), node, printer, "# blah\n", NewYamlDecoder())
 	if err != nil {
 		panic(err)
 	}
@@ -316,7 +316,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
 	// when outputing JSON.
 	printer := NewPrinter(NewJSONEncoder(0, false), NewSinglePrinterWriter(writer))
 
-	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
 		panic(err)
 	}

pkg/yqlib/properties_test.go

@@ -8,49 +8,7 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
-const propertiesWithCommentsOnMap = `this.thing = hi hi
-# important notes
-# about this value
-this.value = cool
-`
-
-const expectedPropertiesWithCommentsOnMapProps = `this.thing = hi hi
-
-# important notes
-# about this value
-this.value = cool
-`
-
-const expectedPropertiesWithCommentsOnMapYaml = `this:
-  thing: hi hi
-  # important notes
-  # about this value
-  value: cool
-`
-
-const propertiesWithCommentInArray = `
-this.array.0 = cat
-# important notes
-# about dogs
-this.array.1 = dog
-`
-
-const expectedPropertiesWithCommentInArrayProps = `this.array.0 = cat
-
-# important notes
-# about dogs
-this.array.1 = dog
-`
-
-const expectedPropertiesWithCommentInArrayYaml = `this:
-  array:
-    - cat
-    # important notes
-    # about dogs
-    - dog
-`
-
-const samplePropertiesYaml = `# block comments come through
+const samplePropertiesYaml = `# block comments don't come through
 person: # neither do comments on maps
     name: Mike Wazowski # comments on values appear
     pets: 
@@ -60,8 +18,7 @@ emptyArray: []
 emptyMap: []
 `
 
-const expectedPropertiesUnwrapped = `# block comments come through
-# comments on values appear
+const expectedPropertiesUnwrapped = `# comments on values appear
 person.name = Mike Wazowski
 
 # comments on array values appear
@@ -69,8 +26,7 @@ person.pets.0 = cat
 person.food.0 = pizza
 `
 
-const expectedPropertiesWrapped = `# block comments come through
-# comments on values appear
+const expectedPropertiesWrapped = `# comments on values appear
 person.name = "Mike Wazowski"
 
 # comments on array values appear
@@ -78,8 +34,7 @@ person.pets.0 = cat
 person.food.0 = pizza
 `
 
-const expectedUpdatedProperties = `# block comments come through
-# comments on values appear
+const expectedUpdatedProperties = `# comments on values appear
 person.name = Mike Wazowski
 
 # comments on array values appear
@@ -88,12 +43,9 @@ person.food.0 = pizza
 `
 
 const expectedDecodedYaml = `person:
-  # block comments come through
-  # comments on values appear
-  name: Mike Wazowski
+  name: Mike Wazowski # comments on values appear
   pets:
-    # comments on array values appear
-    - cat
+    - cat # comments on array values appear
   food:
     - pizza
 `
@@ -103,8 +55,7 @@ person.pets.0 = cat
 person.food.0 = pizza
 `
 
-const expectedPropertiesWithEmptyMapsAndArrays = `# block comments come through
-# comments on values appear
+const expectedPropertiesWithEmptyMapsAndArrays = `# comments on values appear
 person.name = Mike Wazowski
 
 # comments on array values appear
@@ -161,34 +112,6 @@ var propertyScenarios = []formatScenario{
 		expected:     expectedUpdatedProperties,
 		scenarioType: "roundtrip",
 	},
-	{
-		skipDoc:      true,
-		description:  "comments on arrays roundtrip",
-		input:        propertiesWithCommentInArray,
-		expected:     expectedPropertiesWithCommentInArrayProps,
-		scenarioType: "roundtrip",
-	},
-	{
-		skipDoc:      true,
-		description:  "comments on arrays decode",
-		input:        propertiesWithCommentInArray,
-		expected:     expectedPropertiesWithCommentInArrayYaml,
-		scenarioType: "decode",
-	},
-	{
-		skipDoc:      true,
-		description:  "comments on map roundtrip",
-		input:        propertiesWithCommentsOnMap,
-		expected:     expectedPropertiesWithCommentsOnMapProps,
-		scenarioType: "roundtrip",
-	},
-	{
-		skipDoc:      true,
-		description:  "comments on map decode",
-		input:        propertiesWithCommentsOnMap,
-		expected:     expectedPropertiesWithCommentsOnMapYaml,
-		scenarioType: "decode",
-	},
 	{
 		description:  "Empty doc",
 		skipDoc:      true,
@@ -220,7 +143,7 @@ func documentUnwrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario)
 	}
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true))))
+	writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(true))))
 }
 
 func documentWrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) {
@@ -245,7 +168,7 @@ func documentWrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) {
 	}
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false))))
+	writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(false))))
 }
 
 func documentDecodePropertyScenario(w *bufio.Writer, s formatScenario) {
@@ -270,7 +193,7 @@ func documentDecodePropertyScenario(w *bufio.Writer, s formatScenario) {
 
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))))
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(s.indent, false, true, true))))
 }
 
 func documentRoundTripPropertyScenario(w *bufio.Writer, s formatScenario) {
@@ -322,11 +245,11 @@ func TestPropertyScenarios(t *testing.T) {
 	for _, s := range propertyScenarios {
 		switch s.scenarioType {
 		case "":
-			test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true)), s.description)
+			test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(true)), s.description)
 		case "decode":
-			test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
+			test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(2, false, true, true)), s.description)
 		case "encode-wrapped":
-			test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false)), s.description)
+			test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(false)), s.description)
 		case "roundtrip":
 			test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(true)), s.description)
 

pkg/yqlib/stream_evaluator.go

@@ -14,9 +14,9 @@ import (
 // Uses less memory than loading all documents and running the expression once, but this cannot process
 // cross document expressions.
 type StreamEvaluator interface {
-	Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, decoder Decoder) (uint, error)
-	EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error
-	EvaluateNew(expression string, printer Printer) error
+	Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, leadingContent string, decoder Decoder) (uint, error)
+	EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error
+	EvaluateNew(expression string, printer Printer, leadingContent string) error
 }
 
 type streamEvaluator struct {
@@ -28,16 +28,17 @@ func NewStreamEvaluator() StreamEvaluator {
 	return &streamEvaluator{treeNavigator: NewDataTreeNavigator()}
 }
 
-func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error {
+func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadingContent string) error {
 	node, err := ExpressionParser.ParseExpression(expression)
 	if err != nil {
 		return err
 	}
 	candidateNode := &CandidateNode{
-		Document:  0,
-		Filename:  "",
-		Node:      &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
-		FileIndex: 0,
+		Document:       0,
+		Filename:       "",
+		Node:           &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
+		FileIndex:      0,
+		LeadingContent: leadingContent,
 	}
 	inputList := list.New()
 	inputList.PushBack(candidateNode)
@@ -49,20 +50,27 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error
 	return printer.PrintResults(result.MatchingNodes)
 }
 
-func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error {
+func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error {
 	var totalProcessDocs uint
 	node, err := ExpressionParser.ParseExpression(expression)
 	if err != nil {
 		return err
 	}
 
-	for _, filename := range filenames {
-		reader, err := readStream(filename)
+	var firstFileLeadingContent string
+
+	for index, filename := range filenames {
+		reader, leadingContent, err := readStream(filename, leadingContentPreProcessing)
+		log.Debug("leadingContent: %v", leadingContent)
+
+		if index == 0 {
+			firstFileLeadingContent = leadingContent
+		}
 
 		if err != nil {
 			return err
 		}
-		processedDocs, err := s.Evaluate(filename, reader, node, printer, decoder)
+		processedDocs, err := s.Evaluate(filename, reader, node, printer, leadingContent, decoder)
 		if err != nil {
 			return err
 		}
@@ -75,22 +83,19 @@ func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, p
 	}
 
 	if totalProcessDocs == 0 {
-		// problem is I've already slurped the leading content sadface
-		return s.EvaluateNew(expression, printer)
+		return s.EvaluateNew(expression, printer, firstFileLeadingContent)
 	}
 
 	return nil
 }
 
-func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, decoder Decoder) (uint, error) {
+func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, leadingContent string, decoder Decoder) (uint, error) {
 
 	var currentIndex uint
-	err := decoder.Init(reader)
-	if err != nil {
-		return 0, err
-	}
+	decoder.Init(reader)
 	for {
-		candidateNode, errorReading := decoder.Decode()
+		var dataBucket yaml.Node
+		errorReading := decoder.Decode(&dataBucket)
 
 		if errors.Is(errorReading, io.EOF) {
 			s.fileIndex = s.fileIndex + 1
@@ -98,10 +103,21 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr
 		} else if errorReading != nil {
 			return currentIndex, fmt.Errorf("bad file '%v': %w", filename, errorReading)
 		}
-		candidateNode.Document = currentIndex
-		candidateNode.Filename = filename
-		candidateNode.FileIndex = s.fileIndex
 
+		candidateNode := &CandidateNode{
+			Document:  currentIndex,
+			Filename:  filename,
+			Node:      &dataBucket,
+			FileIndex: s.fileIndex,
+		}
+		// move document comments into candidate node
+		// otherwise unwrap drops them.
+		candidateNode.TrailingContent = dataBucket.FootComment
+		dataBucket.FootComment = ""
+
+		if currentIndex == 0 {
+			candidateNode.LeadingContent = leadingContent
+		}
 		inputList := list.New()
 		inputList.PushBack(candidateNode)
 

pkg/yqlib/string_evaluator.go

@@ -1,17 +1,17 @@
 package yqlib
 
 import (
-	"bufio"
 	"bytes"
 	"container/list"
 	"errors"
 	"fmt"
 	"io"
-	"strings"
+
+	yaml "gopkg.in/yaml.v3"
 )
 
 type StringEvaluator interface {
-	Evaluate(expression string, input string, encoder Encoder, decoder Decoder) (string, error)
+	Evaluate(expression string, input string, encoder Encoder, leadingContentPreProcessing bool, decoder Decoder) (string, error)
 }
 
 type stringEvaluator struct {
@@ -25,7 +25,7 @@ func NewStringEvaluator() StringEvaluator {
 	}
 }
 
-func (s *stringEvaluator) Evaluate(expression string, input string, encoder Encoder, decoder Decoder) (string, error) {
+func (s *stringEvaluator) Evaluate(expression string, input string, encoder Encoder, leadingContentPreProcessing bool, decoder Decoder) (string, error) {
 
 	// Use bytes.Buffer for output of string
 	out := new(bytes.Buffer)
@@ -37,15 +37,16 @@ func (s *stringEvaluator) Evaluate(expression string, input string, encoder Enco
 		return "", err
 	}
 
-	reader := bufio.NewReader(strings.NewReader(input))
-
-	var currentIndex uint
-	err = decoder.Init(reader)
+	reader, leadingContent, err := readString(input, leadingContentPreProcessing)
 	if err != nil {
 		return "", err
 	}
+
+	var currentIndex uint
+	decoder.Init(reader)
 	for {
-		candidateNode, errorReading := decoder.Decode()
+		var dataBucket yaml.Node
+		errorReading := decoder.Decode(&dataBucket)
 
 		if errors.Is(errorReading, io.EOF) {
 			s.fileIndex = s.fileIndex + 1
@@ -53,9 +54,20 @@ func (s *stringEvaluator) Evaluate(expression string, input string, encoder Enco
 		} else if errorReading != nil {
 			return "", fmt.Errorf("bad input '%v': %w", input, errorReading)
 		}
-		candidateNode.Document = currentIndex
-		candidateNode.FileIndex = s.fileIndex
 
+		candidateNode := &CandidateNode{
+			Document:  currentIndex,
+			Node:      &dataBucket,
+			FileIndex: s.fileIndex,
+		}
+		// move document comments into candidate node
+		// otherwise unwrap drops them.
+		candidateNode.TrailingContent = dataBucket.FootComment
+		dataBucket.FootComment = ""
+
+		if currentIndex == 0 {
+			candidateNode.LeadingContent = leadingContent
+		}
 		inputList := list.New()
 		inputList.PushBack(candidateNode)
 

pkg/yqlib/string_evaluator_test.go

@@ -18,10 +18,10 @@ func TestStringEvaluator_Evaluate_Nominal(t *testing.T) {
 		`---` + "\n" +
 		` - name: jq` + "\n" +
 		`   description: Command-line JSON processor` + "\n"
-	encoder := NewYamlEncoder(2, true, ConfiguredYamlPreferences)
-	decoder := NewYamlDecoder(ConfiguredYamlPreferences)
+	encoder := NewYamlEncoder(2, true, true, true)
+	decoder := NewYamlDecoder()
 
-	result, err := NewStringEvaluator().Evaluate(expression, input, encoder, decoder)
+	result, err := NewStringEvaluator().Evaluate(expression, input, encoder, true, decoder)
 	if err != nil {
 		t.Error(err)
 	}

pkg/yqlib/utils.go

@@ -7,9 +7,13 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"regexp"
+	"strings"
+
+	yaml "gopkg.in/yaml.v3"
 )
 
-func readStream(filename string) (io.Reader, error) {
+func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, string, error) {
 	var reader *bufio.Reader
 	if filename == "-" {
 		reader = bufio.NewReader(os.Stdin)
@@ -18,12 +22,23 @@ func readStream(filename string) (io.Reader, error) {
 		// and ensuring that it's not possible to give a path to a file outside thar directory.
 		file, err := os.Open(filename) // #nosec
 		if err != nil {
-			return nil, err
+			return nil, "", err
 		}
 		reader = bufio.NewReader(file)
 	}
-	return reader, nil
 
+	if !leadingContentPreProcessing {
+		return reader, "", nil
+	}
+	return processReadStream(reader)
+}
+
+func readString(input string, leadingContentPreProcessing bool) (io.Reader, string, error) {
+	reader := bufio.NewReader(strings.NewReader(input))
+	if !leadingContentPreProcessing {
+		return reader, "", nil
+	}
+	return processReadStream(reader)
 }
 
 func writeString(writer io.Writer, txt string) error {
@@ -31,16 +46,46 @@ func writeString(writer io.Writer, txt string) error {
 	return errorWriting
 }
 
-func readDocuments(reader io.Reader, filename string, fileIndex int, decoder Decoder) (*list.List, error) {
-	err := decoder.Init(reader)
-	if err != nil {
-		return nil, err
+func processReadStream(reader *bufio.Reader) (io.Reader, string, error) {
+	var commentLineRegEx = regexp.MustCompile(`^\s*#`)
+	var sb strings.Builder
+	for {
+		peekBytes, err := reader.Peek(3)
+		if errors.Is(err, io.EOF) {
+			// EOF are handled else where..
+			return reader, sb.String(), nil
+		} else if err != nil {
+			return reader, sb.String(), err
+		} else if string(peekBytes) == "---" {
+			_, err := reader.ReadString('\n')
+			sb.WriteString("$yqDocSeperator$\n")
+			if errors.Is(err, io.EOF) {
+				return reader, sb.String(), nil
+			} else if err != nil {
+				return reader, sb.String(), err
+			}
+		} else if commentLineRegEx.MatchString(string(peekBytes)) {
+			line, err := reader.ReadString('\n')
+			sb.WriteString(line)
+			if errors.Is(err, io.EOF) {
+				return reader, sb.String(), nil
+			} else if err != nil {
+				return reader, sb.String(), err
+			}
+		} else {
+			return reader, sb.String(), nil
+		}
 	}
+}
+
+func readDocuments(reader io.Reader, filename string, fileIndex int, decoder Decoder) (*list.List, error) {
+	decoder.Init(reader)
 	inputList := list.New()
 	var currentIndex uint
 
 	for {
-		candidateNode, errorReading := decoder.Decode()
+		var dataBucket yaml.Node
+		errorReading := decoder.Decode(&dataBucket)
 
 		if errors.Is(errorReading, io.EOF) {
 			switch reader := reader.(type) {
@@ -51,10 +96,18 @@ func readDocuments(reader io.Reader, filename string, fileIndex int, decoder Dec
 		} else if errorReading != nil {
 			return nil, fmt.Errorf("bad file '%v': %w", filename, errorReading)
 		}
-		candidateNode.Document = currentIndex
-		candidateNode.Filename = filename
-		candidateNode.FileIndex = fileIndex
-		candidateNode.EvaluateTogether = true
+		candidateNode := &CandidateNode{
+			Document:         currentIndex,
+			Filename:         filename,
+			Node:             &dataBucket,
+			FileIndex:        fileIndex,
+			EvaluateTogether: true,
+		}
+
+		//move document comments into candidate node
+		// otherwise unwrap drops them.
+		candidateNode.TrailingContent = dataBucket.FootComment
+		dataBucket.FootComment = ""
 
 		inputList.PushBack(candidateNode)
 

pkg/yqlib/xml.go

@@ -1,29 +0,0 @@
-package yqlib
-
-type XmlPreferences struct {
-	AttributePrefix string
-	ContentName     string
-	StrictMode      bool
-	KeepNamespace   bool
-	UseRawToken     bool
-	ProcInstPrefix  string
-	DirectiveName   string
-	SkipProcInst    bool
-	SkipDirectives  bool
-}
-
-func NewDefaultXmlPreferences() XmlPreferences {
-	return XmlPreferences{
-		AttributePrefix: "+",
-		ContentName:     "+content",
-		StrictMode:      false,
-		KeepNamespace:   true,
-		UseRawToken:     false,
-		ProcInstPrefix:  "+p_",
-		DirectiveName:   "+directive",
-		SkipProcInst:    false,
-		SkipDirectives:  false,
-	}
-}
-
-var ConfiguredXMLPreferences = NewDefaultXmlPreferences()

pkg/yqlib/xml_test.go

@@ -137,8 +137,7 @@ in d before -->
 </cat><!-- after cat -->
 `
 
-const yamlWithComments = `# header comment
-# above_cat
+const yamlWithComments = `# above_cat
 cat: # inline_cat
   # above_array
   array: # inline_array
@@ -148,10 +147,7 @@ cat: # inline_cat
 # below_cat
 `
 
-const expectedXMLWithComments = `<!--
- header comment
- above_cat
- --><!-- inline_cat --><cat><!-- above_array inline_array -->
+const expectedXMLWithComments = `<!-- above_cat inline_cat --><cat><!-- above_array inline_array -->
   <array>val1<!-- inline_val1 --></array>
   <array><!-- above_val2 -->val2<!-- inline_val2 --></array>
 </cat><!-- below_cat -->
@@ -418,19 +414,19 @@ var xmlScenarios = []formatScenario{
 func testXMLScenario(t *testing.T, s formatScenario) {
 	switch s.scenarioType {
 	case "", "decode":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(4, false, ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(XMLPreferences), NewYamlEncoder(4, false, true, true)), s.description)
 	case "encode":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewXMLEncoder(2, XMLPreferences)), s.description)
 	case "roundtrip":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(XMLPreferences), NewXMLEncoder(2, XMLPreferences)), s.description)
 	case "decode-keep-ns":
 		prefs := NewDefaultXmlPreferences()
 		prefs.KeepNamespace = true
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, true, true)), s.description)
 	case "decode-raw-token":
 		prefs := NewDefaultXmlPreferences()
 		prefs.UseRawToken = true
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, true, true)), s.description)
 	case "roundtrip-skip-directives":
 		prefs := NewDefaultXmlPreferences()
 		prefs.SkipDirectives = true
@@ -484,7 +480,7 @@ func documentXMLDecodeScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=xml '%v' sample.xml\n```\n", expression))
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences))))
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(XMLPreferences), NewYamlEncoder(2, false, true, true))))
 }
 
 func documentXMLDecodeKeepNsScenario(w *bufio.Writer, s formatScenario) {
@@ -553,7 +549,7 @@ func documentXMLEncodeScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, "```bash\nyq -o=xml '.' sample.yml\n```\n")
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences))))
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewXMLEncoder(2, XMLPreferences))))
 }
 
 func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) {
@@ -571,7 +567,7 @@ func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, "```bash\nyq -p=xml -o=xml '.' sample.xml\n```\n")
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences))))
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(XMLPreferences), NewXMLEncoder(2, XMLPreferences))))
 }
 
 func documentXMLSkipDirectrivesScenario(w *bufio.Writer, s formatScenario) {

pkg/yqlib/yaml.go

@@ -1,19 +0,0 @@
-package yqlib
-
-type YamlPreferences struct {
-	LeadingContentPreProcessing bool
-	PrintDocSeparators          bool
-	UnwrapScalar                bool
-	EvaluateTogether            bool
-}
-
-func NewDefaultYamlPreferences() YamlPreferences {
-	return YamlPreferences{
-		LeadingContentPreProcessing: true,
-		PrintDocSeparators:          true,
-		UnwrapScalar:                true,
-		EvaluateTogether:            false,
-	}
-}
-
-var ConfiguredYamlPreferences = NewDefaultYamlPreferences()

release_notes.txt

@@ -1,8 +1,3 @@
-4.29.1:
-  - Fixed Square brackets removing update #1342
-  - XML decoder/encoder now parses directives and proc instructions (#1344). Please use the new skip flags [documented here](https://mikefarah.gitbook.io/yq/usage/xml) to ignore them.
-  - Bumped Go compiler version
-
 4.28.2:
   - Fixed Github Actions issues (thanks @mattphelps-8451)
   - Fixed bug - can now delete documents #1377

test/utils.go

@@ -23,29 +23,17 @@ func ParseData(rawData string) yaml.Node {
 	return parsedData
 }
 
-func printDifference(t *testing.T, expectedValue interface{}, actualValue interface{}) {
-	opts := []write.Option{write.TerminalColor()}
-	var differenceBuffer bytes.Buffer
-	expectedString := fmt.Sprintf("%v", expectedValue)
-	actualString := fmt.Sprintf("%v", actualValue)
-	if err := diff.Text("expected", "actual", expectedString, actualString, bufio.NewWriter(&differenceBuffer), opts...); err != nil {
-		t.Error(err)
-	} else {
-		t.Error(differenceBuffer.String())
-	}
-}
-
 func AssertResult(t *testing.T, expectedValue interface{}, actualValue interface{}) {
 	t.Helper()
 	if expectedValue != actualValue {
-		printDifference(t, expectedValue, actualValue)
+		t.Error("Expected <", expectedValue, "> but got <", actualValue, ">", fmt.Sprintf("%T", actualValue))
 	}
 }
 
 func AssertResultComplex(t *testing.T, expectedValue interface{}, actualValue interface{}) {
 	t.Helper()
 	if !reflect.DeepEqual(expectedValue, actualValue) {
-		printDifference(t, expectedValue, actualValue)
+		t.Error("\nExpected <", expectedValue, ">\nbut got  <", actualValue, ">", fmt.Sprintf("%T", actualValue))
 	}
 }
 
@@ -53,7 +41,7 @@ func AssertResultComplexWithContext(t *testing.T, expectedValue interface{}, act
 	t.Helper()
 	if !reflect.DeepEqual(expectedValue, actualValue) {
 		t.Error(context)
-		printDifference(t, expectedValue, actualValue)
+		t.Error("\nExpected <", expectedValue, ">\nbut got  <", actualValue, ">", fmt.Sprintf("%T", actualValue))
 	}
 }