Slice array

Refactored decoder responsibilities (#1402 )
- improved comment handling - yaml decoder now responsible for leading content work around
2026-08-25 00:44:43 +08:00 · 2022-10-28 20:33:50 +11:00 · 2022-10-28 14:16:46 +11:00 · 2022-10-27 15:11:06 +11:00 · 2022-10-27 15:10:56 +11:00 · 2022-10-26 13:12:20 +11:00
58 changed files with 1006 additions and 406 deletions
@@ -59,7 +59,7 @@ Take a look at the discussions for [common questions](https://github.com/mikefar
 ### [Download the latest binary](https://github.com/mikefarah/yq/releases/latest)

 ### wget
-Use wget to download the pre-compiled binaries:
+Use wget to download, gzipped pre-compiled binaries:

 #### Compressed via tar.gz
 ```bash
@@ -76,6 +76,13 @@ wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY} -O /

 For instance, VERSION=v4.2.0 and BINARY=yq_linux_amd64

+Use VERSION=latest to always get the latest version:
+
+```bash
+wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq &&\
+    chmod +x /usr/bin/yq
+```
+
 ### MacOS / Linux via Homebrew:
 Using [Homebrew](https://brew.sh/)
 ```
@@ -9,7 +9,7 @@ EOL

 testEmptyEval() {
  X=$(./yq e test.yml)
-  expected=$(cat test.yml)
+  expected="# comment"
  assertEquals 0 $?
  assertEquals "$expected" "$X"
 }
@@ -1,6 +1,5 @@
 package cmd

-var leadingContentPreProcessing = true
 var unwrapScalar = true

 var writeInplace = false
@@ -75,7 +75,7 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 		return err
 	}

-	decoder, err := configureDecoder()
+	decoder, err := configureDecoder(true)
 	if err != nil {
 		return err
 	}
@@ -109,13 +109,13 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 	switch len(args) {
 	case 0:
 		if nullInput {
-			err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(expression), printer, "")
+			err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(expression), printer)
 		} else {
 			cmd.Println(cmd.UsageString())
 			return nil
 		}
 	default:
-		err = allAtOnceEvaluator.EvaluateFiles(processExpression(expression), args, printer, leadingContentPreProcessing, decoder)
+		err = allAtOnceEvaluator.EvaluateFiles(processExpression(expression), args, printer, decoder)
 	}

 	completedSuccessfully = err == nil
@@ -97,7 +97,7 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {

 	printer := yqlib.NewPrinter(encoder, printerWriter)

-	decoder, err := configureDecoder()
+	decoder, err := configureDecoder(false)
 	if err != nil {
 		return err
 	}
@@ -123,13 +123,13 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 	switch len(args) {
 	case 0:
 		if nullInput {
-			err = streamEvaluator.EvaluateNew(processExpression(expression), printer, "")
+			err = streamEvaluator.EvaluateNew(processExpression(expression), printer)
 		} else {
 			cmd.Println(cmd.UsageString())
 			return nil
 		}
 	default:
-		err = streamEvaluator.EvaluateFiles(processExpression(expression), args, printer, leadingContentPreProcessing, decoder)
+		err = streamEvaluator.EvaluateFiles(processExpression(expression), args, printer, decoder)
 	}
 	completedSuccessfully = err == nil

@@ -54,11 +54,16 @@ yq -P sample.json
 			yqlib.InitExpressionParser()
 			if (inputFormat == "x" || inputFormat == "xml") &&
 				outputFormat != "x" && outputFormat != "xml" &&
-				yqlib.XMLPreferences.AttributePrefix == "+" {
+				yqlib.ConfiguredXMLPreferences.AttributePrefix == "+" {
 				yqlib.GetLogger().Warning("The default xml-attribute-prefix will change in the v4.30 to `+@` to avoid " +
 					"naming conflicts with the default content name, directive name and proc inst prefix. If you need to keep " +
 					"`+` please set that value explicityly with --xml-attribute-prefix.")
 			}
+
+			//copy preference form global setting
+			yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar
+
+			yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators
 		},
 	}

@@ -73,15 +78,15 @@ yq -P sample.json
 	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "yaml", "[yaml|y|json|j|props|p|xml|x] output format type.")
 	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "yaml", "[yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.")

-	rootCmd.PersistentFlags().StringVar(&yqlib.XMLPreferences.AttributePrefix, "xml-attribute-prefix", "+", "prefix for xml attributes")
-	rootCmd.PersistentFlags().StringVar(&yqlib.XMLPreferences.ContentName, "xml-content-name", "+content", "name for xml content (if no attribute name is present).")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.XMLPreferences.StrictMode, "xml-strict-mode", false, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.XMLPreferences.KeepNamespace, "xml-keep-namespace", true, "enables keeping namespace after parsing attributes")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.XMLPreferences.UseRawToken, "xml-raw-token", true, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.")
-	rootCmd.PersistentFlags().StringVar(&yqlib.XMLPreferences.ProcInstPrefix, "xml-proc-inst-prefix", "+p_", "prefix for xml processing instructions (e.g. <?xml version=\"1\"?>)")
-	rootCmd.PersistentFlags().StringVar(&yqlib.XMLPreferences.DirectiveName, "xml-directive-name", "+directive", "name for xml directives (e.g. <!DOCTYPE thing cat>)")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.XMLPreferences.SkipProcInst, "xml-skip-proc-inst", false, "skip over process instructions (e.g. <?xml version=\"1\"?>)")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.XMLPreferences.SkipDirectives, "xml-skip-directives", false, "skip over directives (e.g. <!DOCTYPE thing cat>)")
+	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", "+", "prefix for xml attributes")
+	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", "+content", "name for xml content (if no attribute name is present).")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.StrictMode, "xml-strict-mode", false, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.KeepNamespace, "xml-keep-namespace", true, "enables keeping namespace after parsing attributes")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.UseRawToken, "xml-raw-token", true, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.")
+	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ProcInstPrefix, "xml-proc-inst-prefix", "+p_", "prefix for xml processing instructions (e.g. <?xml version=\"1\"?>)")
+	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.DirectiveName, "xml-directive-name", "+directive", "name for xml directives (e.g. <!DOCTYPE thing cat>)")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipProcInst, "xml-skip-proc-inst", false, "skip over process instructions (e.g. <?xml version=\"1\"?>)")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipDirectives, "xml-skip-directives", false, "skip over directives (e.g. <!DOCTYPE thing cat>)")

 	rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.")
 	rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)")
@@ -97,7 +102,7 @@ yq -P sample.json
 	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
 	rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
 	rootCmd.PersistentFlags().StringVarP(&forceExpression, "expression", "", "", "forcibly set the expression argument. Useful when yq argument detection thinks your expression is a file.")
-	rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.")
+	rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredYamlPreferences.LeadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.")

 	rootCmd.PersistentFlags().StringVarP(&splitFileExp, "split-exp", "s", "", "print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter.")
 	rootCmd.PersistentFlags().StringVarP(&splitFileExpFile, "split-exp-file", "", "", "Use a file to specify the split-exp expression.")
@@ -56,14 +56,14 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 	return expression, args, nil
 }

-func configureDecoder() (yqlib.Decoder, error) {
+func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
 	yqlibInputFormat, err := yqlib.InputFormatFromString(inputFormat)
 	if err != nil {
 		return nil, err
 	}
 	switch yqlibInputFormat {
 	case yqlib.XMLInputFormat:
-		return yqlib.NewXMLDecoder(yqlib.XMLPreferences), nil
+		return yqlib.NewXMLDecoder(yqlib.ConfiguredXMLPreferences), nil
 	case yqlib.PropertiesInputFormat:
 		return yqlib.NewPropertiesDecoder(), nil
 	case yqlib.JsonInputFormat:
@@ -73,8 +73,9 @@ func configureDecoder() (yqlib.Decoder, error) {
 	case yqlib.TSVObjectInputFormat:
 		return yqlib.NewCSVObjectDecoder('\t'), nil
 	}
-
-	return yqlib.NewYamlDecoder(), nil
+	prefs := yqlib.ConfiguredYamlPreferences
+	prefs.EvaluateTogether = evaluateTogether
+	return yqlib.NewYamlDecoder(prefs), nil
 }

 func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yqlib.PrinterWriter, error) {
@@ -105,9 +106,9 @@ func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder {
 	case yqlib.TSVOutputFormat:
 		return yqlib.NewCsvEncoder('\t')
 	case yqlib.YamlOutputFormat:
-		return yqlib.NewYamlEncoder(indent, colorsEnabled, !noDocSeparators, unwrapScalar)
+		return yqlib.NewYamlEncoder(indent, colorsEnabled, yqlib.ConfiguredYamlPreferences)
 	case yqlib.XMLOutputFormat:
-		return yqlib.NewXMLEncoder(indent, yqlib.XMLPreferences)
+		return yqlib.NewXMLEncoder(indent, yqlib.ConfiguredXMLPreferences)
 	}
 	panic("invalid encoder")
 }
@@ -1,11 +1,2 @@
---
- become: true
-  gather_facts: false
-  hosts: lalaland
-  name: "Apply smth"
-  roles:
-    - lala
-    - land
-  serial: 1
- become: false
-  gather_facts: true
+- [cat, dog, frog, cow]
+- [apple, banana, grape, mango]
@@ -8,11 +8,11 @@ require (
 	github.com/elliotchance/orderedmap v1.5.0
 	github.com/fatih/color v1.13.0
 	github.com/goccy/go-json v0.9.11
-	github.com/goccy/go-yaml v1.9.5
+	github.com/goccy/go-yaml v1.9.6
 	github.com/jinzhu/copier v0.3.5
 	github.com/magiconair/properties v1.8.6
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
-	github.com/spf13/cobra v1.6.0
+	github.com/spf13/cobra v1.6.1
 	golang.org/x/net v0.0.0-20220708220712-1185a9018129
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.1
@@ -21,8 +21,8 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk=
 github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
-github.com/goccy/go-yaml v1.9.5 h1:Eh/+3uk9kLxG4koCX6lRMAPS1OaMSAi+FJcya0INdB0=
-github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
+github.com/goccy/go-yaml v1.9.6 h1:KhAu1zf9JXnm3vbG49aDE0E5uEBUsM4uwD31/58ZWyI=
+github.com/goccy/go-yaml v1.9.6/go.mod h1:JubOolP3gh0HpiBc4BLRD4YmjEjHAmIIB2aaXKkTfoE=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
 github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
@@ -43,8 +43,8 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI=
-github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
+github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
+github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -62,6 +62,7 @@ golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e h1:NHvCuwuS43lGnYhten69ZWqi2QOj/CiDNcKbVqwVoew=
 golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -8,7 +8,7 @@ import (

 // A yaml expression evaluator that runs the expression once against all files/nodes in memory.
 type Evaluator interface {
-	EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error
+	EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error

 	// EvaluateNodes takes an expression and one or more yaml nodes, returning a list of matching candidate nodes
 	EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error)
@@ -46,21 +46,16 @@ func (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCand
 	return context.MatchingNodes, nil
 }

-func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error {
+func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error {
 	fileIndex := 0
-	firstFileLeadingContent := ""

 	var allDocuments = list.New()
 	for _, filename := range filenames {
-		reader, leadingContent, err := readStream(filename, fileIndex == 0 && leadingContentPreProcessing)
+		reader, err := readStream(filename)
 		if err != nil {
 			return err
 		}

-		if fileIndex == 0 {
-			firstFileLeadingContent = leadingContent
-		}
-
 		fileDocuments, err := readDocuments(reader, filename, fileIndex, decoder)
 		if err != nil {
 			return err
@@ -75,11 +70,9 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
 			Filename:       "",
 			Node:           &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
 			FileIndex:      0,
-			LeadingContent: firstFileLeadingContent,
+			LeadingContent: "",
 		}
 		allDocuments.PushBack(candidateNode)
-	} else {
-		allDocuments.Front().Value.(*CandidateNode).LeadingContent = firstFileLeadingContent
 	}

 	matches, err := e.EvaluateCandidateNodes(expression, allDocuments)
@@ -136,13 +136,13 @@ var csvScenarios = []formatScenario{
 func testCSVScenario(t *testing.T, s formatScenario) {
 	switch s.scenarioType {
 	case "encode-csv":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder(',')), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(',')), s.description)
 	case "encode-tsv":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder('\t')), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder('\t')), s.description)
 	case "decode-csv-object":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewYamlEncoder(2, false, true, true)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
 	case "decode-tsv-object":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder('\t'), NewYamlEncoder(2, false, true, true)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder('\t'), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
 	case "roundtrip-csv":
 		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewCsvEncoder(',')), s.description)
 	default:
@@ -171,7 +171,7 @@ func documentCSVDecodeObjectScenario(w *bufio.Writer, s formatScenario, formatTy
 	}

 	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n",
-		processFormatScenario(s, NewCSVObjectDecoder(separator), NewYamlEncoder(s.indent, false, true, true))),
+		processFormatScenario(s, NewCSVObjectDecoder(separator), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))),
 	)
 }

@@ -203,7 +203,7 @@ func documentCSVEncodeScenario(w *bufio.Writer, s formatScenario, formatType str
 	}

 	writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
-		processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder(separator))),
+		processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(separator))),
 	)
 }

@@ -3,8 +3,6 @@ package yqlib
 import (
 	"fmt"
 	"io"
-
-	yaml "gopkg.in/yaml.v3"
 )

 type InputFormat uint
@@ -20,8 +18,8 @@ const (
 )

 type Decoder interface {
-	Init(reader io.Reader)
-	Decode(node *yaml.Node) error
+	Init(reader io.Reader) error
+	Decode() (*CandidateNode, error)
 }

 func InputFormatFromString(format string) (InputFormat, error) {
@@ -19,21 +19,22 @@ func NewBase64Decoder() Decoder {
 	return &base64Decoder{finished: false, encoding: *base64.StdEncoding}
 }

-func (dec *base64Decoder) Init(reader io.Reader) {
+func (dec *base64Decoder) Init(reader io.Reader) error {
 	dec.reader = reader
 	dec.readAnything = false
 	dec.finished = false
+	return nil
 }

-func (dec *base64Decoder) Decode(rootYamlNode *yaml.Node) error {
+func (dec *base64Decoder) Decode() (*CandidateNode, error) {
 	if dec.finished {
-		return io.EOF
+		return nil, io.EOF
 	}
 	base64Reader := base64.NewDecoder(&dec.encoding, dec.reader)
 	buf := new(bytes.Buffer)

 	if _, err := buf.ReadFrom(base64Reader); err != nil {
-		return err
+		return nil, err
 	}
 	if buf.Len() == 0 {
 		dec.finished = true
@@ -42,12 +43,15 @@ func (dec *base64Decoder) Decode(rootYamlNode *yaml.Node) error {
 		// otherwise if we've already read some bytes, and now we get
 		// an empty string, then we are done.
 		if dec.readAnything {
-			return io.EOF
+			return nil, io.EOF
 		}
 	}
 	dec.readAnything = true
-	rootYamlNode.Kind = yaml.ScalarNode
-	rootYamlNode.Tag = "!!str"
-	rootYamlNode.Value = buf.String()
-	return nil
+	return &CandidateNode{
+		Node: &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "!!str",
+			Value: buf.String(),
+		},
+	}, nil
 }
@@ -19,12 +19,13 @@ func NewCSVObjectDecoder(separator rune) Decoder {
 	return &csvObjectDecoder{separator: separator}
 }

-func (dec *csvObjectDecoder) Init(reader io.Reader) {
+func (dec *csvObjectDecoder) Init(reader io.Reader) error {
 	cleanReader, enc := utfbom.Skip(reader)
 	log.Debugf("Detected encoding: %s\n", enc)
 	dec.reader = *csv.NewReader(cleanReader)
 	dec.reader.Comma = dec.separator
 	dec.finished = false
+	return nil
 }

 func (dec *csvObjectDecoder) convertToYamlNode(content string) *yaml.Node {
@@ -47,14 +48,14 @@ func (dec *csvObjectDecoder) createObject(headerRow []string, contentRow []strin
 	return objectNode
 }

-func (dec *csvObjectDecoder) Decode(rootYamlNode *yaml.Node) error {
+func (dec *csvObjectDecoder) Decode() (*CandidateNode, error) {
 	if dec.finished {
-		return io.EOF
+		return nil, io.EOF
 	}
 	headerRow, err := dec.reader.Read()
 	log.Debugf(": headerRow%v", headerRow)
 	if err != nil {
-		return err
+		return nil, err
 	}

 	rootArray := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
@@ -68,13 +69,13 @@ func (dec *csvObjectDecoder) Decode(rootYamlNode *yaml.Node) error {
 		log.Debugf("Read next contentRow: %v, %v", contentRow, err)
 	}
 	if !errors.Is(err, io.EOF) {
-		return err
+		return nil, err
 	}

-	log.Debugf("finished, contentRow%v", contentRow)
-	log.Debugf("err: %v", err)
-
-	rootYamlNode.Kind = yaml.DocumentNode
-	rootYamlNode.Content = []*yaml.Node{rootArray}
-	return nil
+	return &CandidateNode{
+		Node: &yaml.Node{
+			Kind:    yaml.DocumentNode,
+			Content: []*yaml.Node{rootArray},
+		},
+	}, nil
 }
@@ -16,26 +16,31 @@ func NewJSONDecoder() Decoder {
 	return &jsonDecoder{}
 }

-func (dec *jsonDecoder) Init(reader io.Reader) {
+func (dec *jsonDecoder) Init(reader io.Reader) error {
 	dec.decoder = *json.NewDecoder(reader)
+	return nil
 }

-func (dec *jsonDecoder) Decode(rootYamlNode *yaml.Node) error {
+func (dec *jsonDecoder) Decode() (*CandidateNode, error) {

 	var dataBucket orderedMap
 	log.Debug("going to decode")
 	err := dec.decoder.Decode(&dataBucket)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	node, err := dec.convertToYamlNode(&dataBucket)

 	if err != nil {
-		return err
+		return nil, err
 	}
-	rootYamlNode.Kind = yaml.DocumentNode
-	rootYamlNode.Content = []*yaml.Node{node}
-	return nil
+
+	return &CandidateNode{
+		Node: &yaml.Node{
+			Kind:    yaml.DocumentNode,
+			Content: []*yaml.Node{node},
+		},
+	}, nil
 }

 func (dec *jsonDecoder) convertToYamlNode(data *orderedMap) (*yaml.Node, error) {
@@ -2,6 +2,7 @@ package yqlib

 import (
 	"bytes"
+	"fmt"
 	"io"
 	"strconv"
 	"strings"
@@ -20,9 +21,10 @@ func NewPropertiesDecoder() Decoder {
 	return &propertiesDecoder{d: NewDataTreeNavigator(), finished: false}
 }

-func (dec *propertiesDecoder) Init(reader io.Reader) {
+func (dec *propertiesDecoder) Init(reader io.Reader) error {
 	dec.reader = reader
 	dec.finished = false
+	return nil
 }

 func parsePropKey(key string) []interface{} {
@@ -46,15 +48,49 @@ func (dec *propertiesDecoder) processComment(c string) string {
 	return "# " + c
 }

-func (dec *propertiesDecoder) applyProperty(properties *properties.Properties, context Context, key string) error {
+func (dec *propertiesDecoder) applyPropertyComments(context Context, path []interface{}, comments []string) error {
+	assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
+
+	rhsCandidateNode := &CandidateNode{
+		Path: path,
+		Node: &yaml.Node{
+			Tag:         "!!str",
+			Value:       fmt.Sprintf("%v", path[len(path)-1]),
+			HeadComment: dec.processComment(strings.Join(comments, "\n")),
+			Kind:        yaml.ScalarNode,
+		},
+	}
+
+	rhsCandidateNode.Node.Tag = guessTagFromCustomType(rhsCandidateNode.Node)
+
+	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}
+
+	assignmentOpNode := &ExpressionNode{
+		Operation: assignmentOp,
+		LHS:       createTraversalTree(path, traversePreferences{}, true),
+		RHS:       &ExpressionNode{Operation: rhsOp},
+	}
+
+	_, err := dec.d.GetMatchingNodes(context, assignmentOpNode)
+	return err
+}
+
+func (dec *propertiesDecoder) applyProperty(context Context, properties *properties.Properties, key string) error {
 	value, _ := properties.Get(key)
 	path := parsePropKey(key)

+	propertyComments := properties.GetComments(key)
+	if len(propertyComments) > 0 {
+		err := dec.applyPropertyComments(context, path, propertyComments)
+		if err != nil {
+			return nil
+		}
+	}
+
 	rhsNode := &yaml.Node{
-		Value:       value,
-		Tag:         "!!str",
-		Kind:        yaml.ScalarNode,
-		LineComment: dec.processComment(properties.GetComment(key)),
+		Value: value,
+		Tag:   "!!str",
+		Kind:  yaml.ScalarNode,
 	}

 	rhsNode.Tag = guessTagFromCustomType(rhsNode)
@@ -78,22 +114,22 @@ func (dec *propertiesDecoder) applyProperty(properties *properties.Properties, c
 	return err
 }

-func (dec *propertiesDecoder) Decode(rootYamlNode *yaml.Node) error {
+func (dec *propertiesDecoder) Decode() (*CandidateNode, error) {
 	if dec.finished {
-		return io.EOF
+		return nil, io.EOF
 	}
 	buf := new(bytes.Buffer)

 	if _, err := buf.ReadFrom(dec.reader); err != nil {
-		return err
+		return nil, err
 	}
 	if buf.Len() == 0 {
 		dec.finished = true
-		return io.EOF
+		return nil, io.EOF
 	}
 	properties, err := properties.LoadString(buf.String())
 	if err != nil {
-		return err
+		return nil, err
 	}
 	properties.DisableExpansion = true

@@ -108,15 +144,18 @@ func (dec *propertiesDecoder) Decode(rootYamlNode *yaml.Node) error {
 	context = context.SingleChildContext(rootMap)

 	for _, key := range properties.Keys() {
-		if err := dec.applyProperty(properties, context, key); err != nil {
-			return err
+		if err := dec.applyProperty(context, properties, key); err != nil {
+			return nil, err
 		}

 	}
-
-	rootYamlNode.Kind = yaml.DocumentNode
-	rootYamlNode.Content = []*yaml.Node{rootMap.Node}
 	dec.finished = true
-	return nil
+
+	return &CandidateNode{
+		Node: &yaml.Node{
+			Kind:    yaml.DocumentNode,
+			Content: []*yaml.Node{rootMap.Node},
+		},
+	}, nil

 }
@@ -23,7 +23,7 @@ func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) s
 	writer := bufio.NewWriter(&output)

 	if decoder == nil {
-		decoder = NewYamlDecoder()
+		decoder = NewYamlDecoder(ConfiguredYamlPreferences)
 	}

 	inputs, err := readDocuments(strings.NewReader(s.input), "sample.yml", 0, decoder)
@@ -15,20 +15,21 @@ type xmlDecoder struct {
 	reader       io.Reader
 	readAnything bool
 	finished     bool
-	prefs        xmlPreferences
+	prefs        XmlPreferences
 }

-func NewXMLDecoder(prefs xmlPreferences) Decoder {
+func NewXMLDecoder(prefs XmlPreferences) Decoder {
 	return &xmlDecoder{
 		finished: false,
 		prefs:    prefs,
 	}
 }

-func (dec *xmlDecoder) Init(reader io.Reader) {
+func (dec *xmlDecoder) Init(reader io.Reader) error {
 	dec.reader = reader
 	dec.readAnything = false
 	dec.finished = false
+	return nil
 }

 func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yaml.Node, error) {
@@ -118,32 +119,36 @@ func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yaml.Node, error) {
 	return scalar, nil
 }

-func (dec *xmlDecoder) Decode(rootYamlNode *yaml.Node) error {
+func (dec *xmlDecoder) Decode() (*CandidateNode, error) {
 	if dec.finished {
-		return io.EOF
+		return nil, io.EOF
 	}
 	root := &xmlNode{}
 	// cant use xj - it doesn't keep map order.
 	err := dec.decodeXML(root)

 	if err != nil {
-		return err
+		return nil, err
 	}
 	firstNode, err := dec.convertToYamlNode(root)

 	if err != nil {
-		return err
+		return nil, err
 	} else if firstNode.Tag == "!!null" {
 		dec.finished = true
 		if dec.readAnything {
-			return io.EOF
+			return nil, io.EOF
 		}
 	}
 	dec.readAnything = true
-	rootYamlNode.Kind = yaml.DocumentNode
-	rootYamlNode.Content = []*yaml.Node{firstNode}
 	dec.finished = true
-	return nil
+
+	return &CandidateNode{
+		Node: &yaml.Node{
+			Kind:    yaml.DocumentNode,
+			Content: []*yaml.Node{firstNode},
+		},
+	}, nil
 }

 type xmlNode struct {
@@ -1,23 +1,114 @@
 package yqlib

 import (
+	"bufio"
+	"errors"
 	"io"
+	"regexp"
+	"strings"

 	yaml "gopkg.in/yaml.v3"
 )

 type yamlDecoder struct {
 	decoder yaml.Decoder
+	// work around of various parsing issues by yaml.v3 with document headers
+	prefs          YamlPreferences
+	leadingContent string
+	readAnything   bool
+	firstFile      bool
 }

-func NewYamlDecoder() Decoder {
-	return &yamlDecoder{}
+func NewYamlDecoder(prefs YamlPreferences) Decoder {
+	return &yamlDecoder{prefs: prefs, firstFile: true}
 }

-func (dec *yamlDecoder) Init(reader io.Reader) {
-	dec.decoder = *yaml.NewDecoder(reader)
+func (dec *yamlDecoder) processReadStream(reader *bufio.Reader) (io.Reader, string, error) {
+	var commentLineRegEx = regexp.MustCompile(`^\s*#`)
+	var sb strings.Builder
+	for {
+		peekBytes, err := reader.Peek(3)
+		if errors.Is(err, io.EOF) {
+			// EOF are handled else where..
+			return reader, sb.String(), nil
+		} else if err != nil {
+			return reader, sb.String(), err
+		} else if string(peekBytes) == "---" {
+			_, err := reader.ReadString('\n')
+			sb.WriteString("$yqDocSeperator$\n")
+			if errors.Is(err, io.EOF) {
+				return reader, sb.String(), nil
+			} else if err != nil {
+				return reader, sb.String(), err
+			}
+		} else if commentLineRegEx.MatchString(string(peekBytes)) {
+			line, err := reader.ReadString('\n')
+			sb.WriteString(line)
+			if errors.Is(err, io.EOF) {
+				return reader, sb.String(), nil
+			} else if err != nil {
+				return reader, sb.String(), err
+			}
+		} else {
+			return reader, sb.String(), nil
+		}
+	}
 }

-func (dec *yamlDecoder) Decode(rootYamlNode *yaml.Node) error {
-	return dec.decoder.Decode(rootYamlNode)
+func (dec *yamlDecoder) Init(reader io.Reader) error {
+	readerToUse := reader
+	leadingContent := ""
+	var err error
+	// if we 'evaluating together' - we only process the leading content
+	// of the first file - this ensures comments from subsequent files are
+	// merged together correctly.
+	if dec.prefs.LeadingContentPreProcessing && (!dec.prefs.EvaluateTogether || dec.firstFile) {
+		readerToUse, leadingContent, err = dec.processReadStream(bufio.NewReader(reader))
+		if err != nil {
+			return err
+		}
+	}
+	dec.leadingContent = leadingContent
+	dec.readAnything = false
+	dec.decoder = *yaml.NewDecoder(readerToUse)
+	dec.firstFile = false
+	return nil
+}
+
+func (dec *yamlDecoder) Decode() (*CandidateNode, error) {
+	var dataBucket yaml.Node
+
+	err := dec.decoder.Decode(&dataBucket)
+	if errors.Is(err, io.EOF) && dec.leadingContent != "" && !dec.readAnything {
+		// force returning an empty node with a comment.
+		dec.readAnything = true
+		return dec.blankNodeWithComment(), nil
+
+	} else if err != nil {
+		return nil, err
+	}
+
+	candidateNode := &CandidateNode{
+		Node: &dataBucket,
+	}
+
+	if dec.leadingContent != "" {
+		candidateNode.LeadingContent = dec.leadingContent
+		dec.leadingContent = ""
+	}
+	// move document comments into candidate node
+	// otherwise unwrap drops them.
+	candidateNode.TrailingContent = dataBucket.FootComment
+	dataBucket.FootComment = ""
+	return candidateNode, nil
+}
+
+func (dec *yamlDecoder) blankNodeWithComment() *CandidateNode {
+	return &CandidateNode{
+		Document:       0,
+		Filename:       "",
+		Node:           &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
+		FileIndex:      0,
+		LeadingContent: dec.leadingContent,
+	}
 }
@@ -246,6 +246,7 @@ Note the use of `...` to ensure key nodes are included.

 Given a sample.yml file of:
 ```yaml
+# hi
 a: cat # comment
 # great
 b: # key comment
@@ -263,6 +264,7 @@ b:
 ## Get line comment
 Given a sample.yml file of:
 ```yaml
+# welcome!
 a: cat # meow
 # have a great day
 ```
@@ -0,0 +1,77 @@
+
+## Slicing arrays
+Given a sample.yml file of:
+```yaml
+- cat
+- dog
+- frog
+- cow
+```
+then
+```bash
+yq '.[1:3]' sample.yml
+```
+will output
+```yaml
+- dog
+- frog
+```
+
+## Slicing arrays - without the first number
+Starts from the start of the array
+
+Given a sample.yml file of:
+```yaml
+- cat
+- dog
+- frog
+- cow
+```
+then
+```bash
+yq '.[:2]' sample.yml
+```
+will output
+```yaml
+- cat
+- dog
+```
+
+## Slicing arrays - without the second number
+Finishes at the end of the array
+
+Given a sample.yml file of:
+```yaml
+- cat
+- dog
+- frog
+- cow
+```
+then
+```bash
+yq '.[2:]' sample.yml
+```
+will output
+```yaml
+- frog
+- cow
+```
+
+## Slicing arrays - use negative numbers to count backwards from the end
+Given a sample.yml file of:
+```yaml
+- cat
+- dog
+- frog
+- cow
+```
+then
+```bash
+yq '.[1:-1]' sample.yml
+```
+will output
+```yaml
+- dog
+- frog
+```
+
@@ -9,7 +9,7 @@ Note that empty arrays and maps are not encoded by default.

 Given a sample.yml file of:
 ```yaml
-# block comments don't come through
+# block comments come through
 person: # neither do comments on maps
    name: Mike Wazowski # comments on values appear
    pets: 
@@ -25,6 +25,7 @@ yq -o=props sample.yml
 ```
 will output
 ```properties
+# block comments come through
 # comments on values appear
 person.name = Mike Wazowski

@@ -38,7 +39,7 @@ Note that string values with blank characters in them are encapsulated with doub

 Given a sample.yml file of:
 ```yaml
-# block comments don't come through
+# block comments come through
 person: # neither do comments on maps
    name: Mike Wazowski # comments on values appear
    pets: 
@@ -54,6 +55,7 @@ yq -o=props --unwrapScalar=false sample.yml
 ```
 will output
 ```properties
+# block comments come through
 # comments on values appear
 person.name = "Mike Wazowski"

@@ -65,7 +67,7 @@ person.food.0 = pizza
 ## Encode properties: no comments
 Given a sample.yml file of:
 ```yaml
-# block comments don't come through
+# block comments come through
 person: # neither do comments on maps
    name: Mike Wazowski # comments on values appear
    pets: 
@@ -91,7 +93,7 @@ Use a yq expression to set the empty maps and sequences to your desired value.

 Given a sample.yml file of:
 ```yaml
-# block comments don't come through
+# block comments come through
 person: # neither do comments on maps
    name: Mike Wazowski # comments on values appear
    pets: 
@@ -107,6 +109,7 @@ yq -o=props '(.. | select( (tag == "!!map" or tag =="!!seq") and length == 0)) =
 ```
 will output
 ```properties
+# block comments come through
 # comments on values appear
 person.name = Mike Wazowski

@@ -120,6 +123,7 @@ emptyMap =
 ## Decode properties
 Given a sample.properties file of:
 ```properties
+# block comments come through
 # comments on values appear
 person.name = Mike Wazowski

@@ -135,9 +139,12 @@ yq -p=props sample.properties
 will output
 ```yaml
 person:
-    name: Mike Wazowski # comments on values appear
+    # block comments come through
+    # comments on values appear
+    name: Mike Wazowski
    pets:
-        - cat # comments on array values appear
+        # comments on array values appear
+        - cat
    food:
        - pizza
 ```
@@ -145,6 +152,7 @@ person:
 ## Roundtrip
 Given a sample.properties file of:
 ```properties
+# block comments come through
 # comments on values appear
 person.name = Mike Wazowski

@@ -159,6 +167,7 @@ yq -p=props -o=props '.person.pets.0 = "dog"' sample.properties
 ```
 will output
 ```properties
+# block comments come through
 # comments on values appear
 person.name = Mike Wazowski

@@ -386,6 +386,7 @@ A best attempt is made to copy comments to xml.

 Given a sample.yml file of:
 ```yaml
+# header comment
 # above_cat
 cat: # inline_cat
  # above_array
@@ -402,7 +403,10 @@ yq -o=xml '.' sample.yml
 ```
 will output
 ```xml
-<!-- above_cat inline_cat --><cat><!-- above_array inline_array -->
+<!--
+ header comment
+ above_cat
+ --><!-- inline_cat --><cat><!-- above_array inline_array -->
  <array>val1<!-- inline_val1 --></array>
  <array><!-- above_val2 -->val2<!-- inline_val2 --></array>
 </cat><!-- below_cat -->
@@ -65,7 +65,7 @@ func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content strin
 func (pe *propertiesEncoder) Encode(writer io.Writer, node *yaml.Node) error {
 	mapKeysToStrings(node)
 	p := properties.NewProperties()
-	err := pe.doEncode(p, node, "")
+	err := pe.doEncode(p, node, "", nil)
 	if err != nil {
 		return err
 	}
@@ -74,8 +74,17 @@ func (pe *propertiesEncoder) Encode(writer io.Writer, node *yaml.Node) error {
 	return err
 }

-func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yaml.Node, path string) error {
-	p.SetComment(path, headAndLineComment(node))
+func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yaml.Node, path string, keyNode *yaml.Node) error {
+
+	comments := ""
+	if keyNode != nil {
+		// include the key node comments if present
+		comments = headAndLineComment(keyNode)
+	}
+	comments = comments + headAndLineComment(node)
+	commentsWithSpaces := strings.ReplaceAll(comments, "\n", "\n ")
+	p.SetComments(path, strings.Split(commentsWithSpaces, "\n"))
+
 	switch node.Kind {
 	case yaml.ScalarNode:
 		var nodeValue string
@@ -87,13 +96,13 @@ func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yaml.Node,
 		_, _, err := p.Set(path, nodeValue)
 		return err
 	case yaml.DocumentNode:
-		return pe.doEncode(p, node.Content[0], path)
+		return pe.doEncode(p, node.Content[0], path, node)
 	case yaml.SequenceNode:
 		return pe.encodeArray(p, node.Content, path)
 	case yaml.MappingNode:
 		return pe.encodeMap(p, node.Content, path)
 	case yaml.AliasNode:
-		return pe.doEncode(p, node.Alias, path)
+		return pe.doEncode(p, node.Alias, path, nil)
 	default:
 		return fmt.Errorf("Unsupported node %v", node.Tag)
 	}
@@ -108,7 +117,7 @@ func (pe *propertiesEncoder) appendPath(path string, key interface{}) string {

 func (pe *propertiesEncoder) encodeArray(p *properties.Properties, kids []*yaml.Node, path string) error {
 	for index, child := range kids {
-		err := pe.doEncode(p, child, pe.appendPath(path, index))
+		err := pe.doEncode(p, child, pe.appendPath(path, index), nil)
 		if err != nil {
 			return err
 		}
@@ -120,7 +129,7 @@ func (pe *propertiesEncoder) encodeMap(p *properties.Properties, kids []*yaml.No
 	for index := 0; index < len(kids); index = index + 2 {
 		key := kids[index]
 		value := kids[index+1]
-		err := pe.doEncode(p, value, pe.appendPath(path, key.Value))
+		err := pe.doEncode(p, value, pe.appendPath(path, key.Value), key)
 		if err != nil {
 			return err
 		}
@@ -14,7 +14,7 @@ func yamlToProps(sampleYaml string, unwrapScalar bool) string {
 	writer := bufio.NewWriter(&output)

 	var propsEncoder = NewPropertiesEncoder(unwrapScalar)
-	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
+	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {
 		panic(err)
 	}
@@ -14,7 +14,7 @@ func yamlToJSON(sampleYaml string, indent int) string {
 	writer := bufio.NewWriter(&output)

 	var jsonEncoder = NewJSONEncoder(indent, false)
-	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
+	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {
 		panic(err)
 	}
@@ -4,24 +4,28 @@ import (
 	"encoding/xml"
 	"fmt"
 	"io"
+	"regexp"
 	"strings"

 	yaml "gopkg.in/yaml.v3"
 )

 type xmlEncoder struct {
-	indentString string
-	writer       io.Writer
-	prefs        xmlPreferences
+	indentString   string
+	writer         io.Writer
+	prefs          XmlPreferences
+	leadingContent string
 }

-func NewXMLEncoder(indent int, prefs xmlPreferences) Encoder {
+var commentPrefix = regexp.MustCompile(`(^|\n)\s*#`)
+
+func NewXMLEncoder(indent int, prefs XmlPreferences) Encoder {
 	var indentString = ""

 	for index := 0; index < indent; index++ {
 		indentString = indentString + " "
 	}
-	return &xmlEncoder{indentString, nil, prefs}
+	return &xmlEncoder{indentString, nil, prefs, ""}
 }

 func (e *xmlEncoder) CanHandleAliases() bool {
@@ -33,6 +37,7 @@ func (e *xmlEncoder) PrintDocumentSeparator(writer io.Writer) error {
 }

 func (e *xmlEncoder) PrintLeadingContent(writer io.Writer, content string) error {
+	e.leadingContent = commentPrefix.ReplaceAllString(content, "\n")
 	return nil
 }

@@ -42,6 +47,13 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
 	e.writer = writer
 	encoder.Indent("", e.indentString)

+	if e.leadingContent != "" {
+		err := e.encodeComment(encoder, e.leadingContent)
+		if err != nil {
+			return err
+		}
+	}
+
 	switch node.Kind {
 	case yaml.MappingNode:
 		err := e.encodeTopLevelMap(encoder, node)
@@ -11,17 +11,16 @@ import (
 )

 type yamlEncoder struct {
-	indent             int
-	colorise           bool
-	printDocSeparators bool
-	unwrapScalar       bool
+	indent   int
+	colorise bool
+	prefs    YamlPreferences
 }

-func NewYamlEncoder(indent int, colorise bool, printDocSeparators bool, unwrapScalar bool) Encoder {
+func NewYamlEncoder(indent int, colorise bool, prefs YamlPreferences) Encoder {
 	if indent < 0 {
 		indent = 0
 	}
-	return &yamlEncoder{indent, colorise, printDocSeparators, unwrapScalar}
+	return &yamlEncoder{indent, colorise, prefs}
 }

 func (ye *yamlEncoder) CanHandleAliases() bool {
@@ -29,7 +28,7 @@ func (ye *yamlEncoder) CanHandleAliases() bool {
 }

 func (ye *yamlEncoder) PrintDocumentSeparator(writer io.Writer) error {
-	if ye.printDocSeparators {
+	if ye.prefs.PrintDocSeparators {
 		log.Debug("-- writing doc sep")
 		if err := writeString(writer, "---\n"); err != nil {
 			return err
@@ -76,7 +75,7 @@ func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) err

 func (ye *yamlEncoder) Encode(writer io.Writer, node *yaml.Node) error {

-	if node.Kind == yaml.ScalarNode && ye.unwrapScalar {
+	if node.Kind == yaml.ScalarNode && ye.prefs.UnwrapScalar {
 		return writeString(writer, node.Value+"\n")
 	}

@@ -262,11 +262,11 @@ func documentDecodeNdJsonScenario(w *bufio.Writer, s formatScenario) {

 	writeOrPanic(w, "will output\n")

-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(s.indent, false, true, true))))
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))))
 }

 func decodeJSON(t *testing.T, jsonString string) *CandidateNode {
-	docs, err := readDocumentWithLeadingContent(jsonString, "sample.json", 0)
+	docs, err := readDocument(jsonString, "sample.json", 0)

 	if err != nil {
 		t.Error(err)
@@ -293,12 +293,12 @@ func decodeJSON(t *testing.T, jsonString string) *CandidateNode {
 func testJSONScenario(t *testing.T, s formatScenario) {
 	switch s.scenarioType {
 	case "encode", "decode":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewJSONEncoder(s.indent, false)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false)), s.description)
 	case "":
 		var actual = resultToString(t, decodeJSON(t, s.input))
 		test.AssertResultWithContext(t, s.expected, actual, s.description)
 	case "decode-ndjson":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(2, false, true, true)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
 	case "roundtrip-ndjson":
 		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(0, false)), s.description)
 	case "roundtrip-multi":
@@ -385,7 +385,7 @@ func documentJSONEncodeScenario(w *bufio.Writer, s formatScenario) {
 	}
 	writeOrPanic(w, "will output\n")

-	writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewJSONEncoder(s.indent, false))))
+	writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false))))
 }

 func TestJSONScenarios(t *testing.T) {
@@ -3,7 +3,6 @@ package yqlib
 import (
 	"fmt"
 	"regexp"
-	"strconv"
 )

 type expressionTokeniser interface {
@@ -64,11 +63,11 @@ func unwrap(value string) string {
 func extractNumberParameter(value string) (int, error) {
 	parameterParser := regexp.MustCompile(`.*\(([0-9]+)\)`)
 	matches := parameterParser.FindStringSubmatch(value)
-	var indent, errParsingInt = strconv.ParseInt(matches[1], 10, 32)
+	var indent, errParsingInt = parseInt(matches[1])
 	if errParsingInt != nil {
 		return 0, errParsingInt
 	}
-	return int(indent), nil
+	return indent, nil
 }

 func hasOptionParameter(value string, option string) bool {
@@ -1,6 +1,7 @@
 package yqlib

 import (
+	"regexp"
 	"strconv"
 	"strings"

@@ -12,6 +13,10 @@ var participleYqRules = []*participleYqRule{
 	{"HEAD_COMMENT", `head_?comment|headComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{HeadComment: true}), 0},
 	{"FOOT_COMMENT", `foot_?comment|footComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{FootComment: true}), 0},

+	{"SliceArray", `\.\[-?[0-9]+:-?[0-9]+\]`, sliceArrayTwoNumbers(), 0},
+	{"SliceArraySecond", `\.\[\:-?[0-9]+\]`, sliceArraySecondNumberOnly(), 0},
+	{"SliceArrayFirst", `\.\[-?[0-9]+\:\]`, sliceArrayFirstNumberOnly(), 0},
+
 	{"OpenBracket", `\(`, literalToken(openBracket, false), 0},
 	{"CloseBracket", `\)`, literalToken(closeBracket, true), 0},
 	{"OpenTraverseArrayCollect", `\.\[`, literalToken(traverseArrayCollect, false), 0},
@@ -76,7 +81,7 @@ var participleYqRules = []*participleYqRule{
 	{"Base64d", `@base64d`, decodeOp(Base64InputFormat), 0},
 	{"Base64", `@base64`, encodeWithIndent(Base64OutputFormat, 0), 0},

-	{"LoadXML", `load_?xml|xml_?load`, loadOp(NewXMLDecoder(XMLPreferences), false), 0},
+	{"LoadXML", `load_?xml|xml_?load`, loadOp(NewXMLDecoder(ConfiguredXMLPreferences), false), 0},

 	{"LoadBase64", `load_?base64`, loadOp(NewBase64Decoder(), false), 0},

@@ -84,7 +89,7 @@ var participleYqRules = []*participleYqRule{

 	{"LoadString", `load_?str|str_?load`, loadOp(nil, true), 0},

-	{"LoadYaml", `load`, loadOp(NewYamlDecoder(), false), 0},
+	{"LoadYaml", `load`, loadOp(NewYamlDecoder(ConfiguredYamlPreferences), false), 0},

 	{"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0},

@@ -300,6 +305,84 @@ func flattenWithDepth() yqAction {
 	}
 }

+func sliceArrayTwoNumbers() yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		value := rawToken.Value
+		sliceArrayNumbers := regexp.MustCompile(`\.\[(-?[0-9]+)\:(-?[0-9]+)\]`)
+		matches := sliceArrayNumbers.FindStringSubmatch(value)
+		log.Debug("sliceArrayTwoNumbers value: %v", value)
+		log.Debug("Matches: %v", matches)
+
+		firstNumber, err := parseInt(matches[1])
+		if err != nil {
+			return nil, err
+		}
+		secondNumber, err := parseInt(matches[2])
+		if err != nil {
+			return nil, err
+		}
+
+		prefs := sliceArrayPreferences{
+			firstNumber:         firstNumber,
+			secondNumber:        secondNumber,
+			secondNumberDefined: true,
+		}
+		log.Debug("%v", prefs)
+
+		op := &Operation{OperationType: sliceArrayOpType, Value: sliceArrayOpType.Type, StringValue: value, Preferences: prefs}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+}
+
+func sliceArraySecondNumberOnly() yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		value := rawToken.Value
+		sliceArrayNumbers := regexp.MustCompile(`\.\[\:(-?[0-9]+)\]`)
+		matches := sliceArrayNumbers.FindStringSubmatch(value)
+		log.Debug("sliceArraySecondNumberOnly value: %v", value)
+		log.Debug("Matches: %v", matches)
+
+		secondNumber, err := parseInt(matches[1])
+		if err != nil {
+			return nil, err
+		}
+
+		prefs := sliceArrayPreferences{
+			firstNumber:         0,
+			secondNumber:        secondNumber,
+			secondNumberDefined: true,
+		}
+		log.Debug("%v", prefs)
+
+		op := &Operation{OperationType: sliceArrayOpType, Value: sliceArrayOpType.Type, StringValue: value, Preferences: prefs}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+}
+
+func sliceArrayFirstNumberOnly() yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		value := rawToken.Value
+		sliceArrayNumbers := regexp.MustCompile(`\.\[(-?[0-9]+)\:\]`)
+		matches := sliceArrayNumbers.FindStringSubmatch(value)
+		log.Debug("sliceArrayFirstNumberOnly value: %v", value)
+		log.Debug("Matches: %v", matches)
+
+		firstNumber, err := parseInt(matches[1])
+		if err != nil {
+			return nil, err
+		}
+
+		prefs := sliceArrayPreferences{
+			firstNumber:         firstNumber,
+			secondNumberDefined: false,
+		}
+		log.Debug("%v", prefs)
+
+		op := &Operation{OperationType: sliceArrayOpType, Value: sliceArrayOpType.Type, StringValue: value, Preferences: prefs}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+}
+
 func assignAllCommentsOp(updateAssign bool) yqAction {
 	return func(rawToken lexer.Token) (*token, error) {
 		log.Debug("assignAllCommentsOp %v", rawToken.Value)
@@ -14,6 +14,62 @@ type participleLexerScenario struct {
 }

 var participleLexerScenarios = []participleLexerScenario{
+	{
+		expression: ".[1:3]",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: sliceArrayOpType,
+					Value:         "SLICE",
+					StringValue:   ".[1:3]",
+					Preferences:   sliceArrayPreferences{firstNumber: 1, secondNumber: 3, secondNumberDefined: true},
+				},
+			},
+		},
+	},
+	{
+		expression: ".[:3]",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: sliceArrayOpType,
+					Value:         "SLICE",
+					StringValue:   ".[:3]",
+					Preferences:   sliceArrayPreferences{firstNumber: 0, secondNumber: 3, secondNumberDefined: true},
+				},
+			},
+		},
+	},
+	{
+		expression: ".[1:]",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: sliceArrayOpType,
+					Value:         "SLICE",
+					StringValue:   ".[1:]",
+					Preferences:   sliceArrayPreferences{firstNumber: 1, secondNumber: 0, secondNumberDefined: false},
+				},
+			},
+		},
+	},
+	{
+		expression: ".[-100:-54]",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: sliceArrayOpType,
+					Value:         "SLICE",
+					StringValue:   ".[-100:-54]",
+					Preferences:   sliceArrayPreferences{firstNumber: -100, secondNumber: -54, secondNumberDefined: true},
+				},
+			},
+		},
+	},
 	{
 		expression: ".a",
 		tokens: []*token{
@@ -21,34 +21,6 @@ func InitExpressionParser() {
 	}
 }

-type xmlPreferences struct {
-	AttributePrefix string
-	ContentName     string
-	StrictMode      bool
-	KeepNamespace   bool
-	UseRawToken     bool
-	ProcInstPrefix  string
-	DirectiveName   string
-	SkipProcInst    bool
-	SkipDirectives  bool
-}
-
-func NewDefaultXmlPreferences() xmlPreferences {
-	return xmlPreferences{
-		AttributePrefix: "+",
-		ContentName:     "+content",
-		StrictMode:      false,
-		KeepNamespace:   true,
-		UseRawToken:     false,
-		ProcInstPrefix:  "+p_",
-		DirectiveName:   "+directive",
-		SkipProcInst:    false,
-		SkipDirectives:  false,
-	}
-}
-
-var XMLPreferences = NewDefaultXmlPreferences()
-
 var log = logging.MustGetLogger("yq-lib")

 var PrettyPrintExp = `(... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not))  ) style=""`
@@ -109,6 +81,7 @@ var lineOpType = &operationType{Type: "LINE", NumArgs: 0, Precedence: 50, Handle
 var columnOpType = &operationType{Type: "LINE", NumArgs: 0, Precedence: 50, Handler: columnOperator}

 var collectOpType = &operationType{Type: "COLLECT", NumArgs: 1, Precedence: 50, Handler: collectOperator}
+var sliceArrayOpType = &operationType{Type: "SLICE", NumArgs: 0, Precedence: 50, Handler: sliceArrayOperator}
 var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 50, Handler: mapOperator}
 var errorOpType = &operationType{Type: "ERROR", NumArgs: 1, Precedence: 50, Handler: errorOperator}
 var pickOpType = &operationType{Type: "PICK", NumArgs: 1, Precedence: 50, Handler: pickOperator}
@@ -276,14 +249,16 @@ func guessTagFromCustomType(node *yaml.Node) string {
 }

 func parseSnippet(value string) (*yaml.Node, error) {
-	decoder := NewYamlDecoder()
-	decoder.Init(strings.NewReader(value))
-	var dataBucket yaml.Node
-	err := decoder.Decode(&dataBucket)
-	if len(dataBucket.Content) == 0 {
+	decoder := NewYamlDecoder(ConfiguredYamlPreferences)
+	err := decoder.Init(strings.NewReader(value))
+	if err != nil {
+		return nil, err
+	}
+	parsedNode, err := decoder.Decode()
+	if len(parsedNode.Node.Content) == 0 {
 		return nil, fmt.Errorf("bad data")
 	}
-	return dataBucket.Content[0], err
+	return unwrapDoc(parsedNode.Node), err
 }

 func recursiveNodeEqual(lhs *yaml.Node, rhs *yaml.Node) bool {
@@ -378,8 +353,8 @@ func parseInt(numberString string) (int, error) {

 	if err != nil {
 		return 0, err
-	} else if parsed > math.MaxInt {
-		return 0, fmt.Errorf("%v is too big (larger than %v)", parsed, math.MaxInt)
+	} else if parsed > math.MaxInt || parsed < math.MinInt {
+		return 0, fmt.Errorf("%v is not within [%v, %v]", parsed, math.MinInt, math.MaxInt)
 	}

 	return int(parsed), err
@@ -56,7 +56,7 @@ func collectOperator(d *dataTreeNavigator, context Context, expressionNode *Expr
 		collectedNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
 		collectCandidate := candidate.CreateReplacement(collectedNode)

-		collectExpResults, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS)
+		collectExpResults, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.RHS)
 		if err != nil {
 			return Context{}, err
 		}
@@ -20,6 +20,14 @@ var collectOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::- 1\n- 2\n- 3\n",
 		},
 	},
+	{
+		skipDoc:     true,
+		description: "update in collect",
+		expression:  `[.a = "cat"]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- a: cat\n",
+		},
+	},
 	{
 		description: "Collect empty",
 		document:    ``,
@@ -83,6 +83,10 @@ func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *
 	log.Debugf("GetComments operator!")
 	var results = list.New()

+	yamlPrefs := NewDefaultYamlPreferences()
+	yamlPrefs.PrintDocSeparators = false
+	yamlPrefs.UnwrapScalar = false
+
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 		comment := ""
@@ -92,7 +96,7 @@ func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *
 			var chompRegexp = regexp.MustCompile(`\n$`)
 			var output bytes.Buffer
 			var writer = bufio.NewWriter(&output)
-			var encoder = NewYamlEncoder(2, false, false, false)
+			var encoder = NewYamlEncoder(2, false, yamlPrefs)
 			if err := encoder.PrintLeadingContent(writer, candidate.LeadingContent); err != nil {
 				return Context{}, err
 			}
@@ -4,7 +4,6 @@ import (
 	"container/list"
 	"errors"
 	"fmt"
-	"strings"
 	"time"

 	"gopkg.in/yaml.v3"
@@ -54,7 +53,6 @@ func nowOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode
 func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	format, err := getStringParamter("format", d, context, expressionNode.RHS)
 	layout := context.GetDateTimeLayout()
-	decoder := NewYamlDecoder()

 	if err != nil {
 		return Context{}, err
@@ -69,19 +67,15 @@ func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *Expre
 			return Context{}, fmt.Errorf("could not parse datetime of [%v]: %w", candidate.GetNicePath(), err)
 		}
 		formattedTimeStr := parsedTime.Format(format)
-		decoder.Init(strings.NewReader(formattedTimeStr))
-		var dataBucket yaml.Node
-		errorReading := decoder.Decode(&dataBucket)
-		var node *yaml.Node
+
+		node, errorReading := parseSnippet(formattedTimeStr)
 		if errorReading != nil {
-			log.Debugf("could not parse %v - lets just leave it as a string", formattedTimeStr)
+			log.Debugf("could not parse %v - lets just leave it as a string: %w", formattedTimeStr, errorReading)
 			node = &yaml.Node{
 				Kind:  yaml.ScalarNode,
 				Tag:   "!!str",
 				Value: formattedTimeStr,
 			}
-		} else {
-			node = unwrapDoc(&dataBucket)
 		}

 		results.PushBack(candidate.CreateReplacement(node))
@@ -21,9 +21,9 @@ func configureEncoder(format PrinterOutputFormat, indent int) Encoder {
 	case TSVOutputFormat:
 		return NewCsvEncoder('\t')
 	case YamlOutputFormat:
-		return NewYamlEncoder(indent, false, true, true)
+		return NewYamlEncoder(indent, false, ConfiguredYamlPreferences)
 	case XMLOutputFormat:
-		return NewXMLEncoder(indent, XMLPreferences)
+		return NewXMLEncoder(indent, ConfiguredXMLPreferences)
 	case Base64OutputFormat:
 		return NewBase64Encoder()
 	}
@@ -102,9 +102,9 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 	var decoder Decoder
 	switch preferences.format {
 	case YamlInputFormat:
-		decoder = NewYamlDecoder()
+		decoder = NewYamlDecoder(ConfiguredYamlPreferences)
 	case XMLInputFormat:
-		decoder = NewXMLDecoder(XMLPreferences)
+		decoder = NewXMLDecoder(ConfiguredXMLPreferences)
 	case Base64InputFormat:
 		decoder = NewBase64Decoder()
 	case PropertiesInputFormat:
@@ -121,17 +121,19 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre

 		context.SetVariable("decoded: "+candidate.GetKey(), candidate.AsList())

-		var dataBucket yaml.Node
 		log.Debugf("got: [%v]", candidate.Node.Value)

-		decoder.Init(strings.NewReader(unwrapDoc(candidate.Node).Value))
+		err := decoder.Init(strings.NewReader(unwrapDoc(candidate.Node).Value))
+		if err != nil {
+			return Context{}, err
+		}

-		errorReading := decoder.Decode(&dataBucket)
+		decodedNode, errorReading := decoder.Decode()
 		if errorReading != nil {
 			return Context{}, errorReading
 		}
 		//first node is a doc
-		node := unwrapDoc(&dataBucket)
+		node := unwrapDoc(decodedNode.Node)

 		results.PushBack(candidate.CreateReplacement(node))
 	}
@@ -48,7 +48,7 @@ func loadYaml(filename string, decoder Decoder) (*CandidateNode, error) {
 	} else {
 		sequenceNode := &CandidateNode{Node: &yaml.Node{Kind: yaml.SequenceNode}}
 		for doc := documents.Front(); doc != nil; doc = doc.Next() {
-			sequenceNode.Node.Content = append(sequenceNode.Node.Content, doc.Value.(*CandidateNode).Node)
+			sequenceNode.Node.Content = append(sequenceNode.Node.Content, unwrapDoc(doc.Value.(*CandidateNode).Node))
 		}
 		return sequenceNode, nil
 	}
@@ -5,6 +5,30 @@ import (
 )

 var loadScenarios = []expressionScenario{
+	{
+		skipDoc:     true,
+		description: "Load empty file with a comment",
+		expression:  `load("../../examples/empty.yaml")`,
+		expected: []string{
+			"D0, P[], (doc)::# comment\n\n",
+		},
+	},
+	{
+		skipDoc:     true,
+		description: "Load empty file with no comment",
+		expression:  `load("../../examples/empty-no-comment.yaml")`,
+		expected: []string{
+			"D0, P[], (!!null)::\n",
+		},
+	},
+	{
+		skipDoc:     true,
+		description: "Load multiple documents",
+		expression:  `load("../../examples/multiple_docs_small.yaml")`,
+		expected: []string{
+			"D0, P[], ()::- a: Easy! as one two three\n- another:\n    document: here\n- - 1\n  - 2\n",
+		},
+	},
 	{
 		description: "Simple example",
 		document:    `{myFile: "../../examples/thing.yml"}`,
@@ -0,0 +1,63 @@
+package yqlib
+
+import (
+	"container/list"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+type sliceArrayPreferences struct {
+	firstNumber         int
+	secondNumber        int
+	secondNumberDefined bool
+}
+
+func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	lhs, err := d.GetMatchingNodes(context, expressionNode.LHS)
+	if err != nil {
+		return Context{}, err
+	}
+	prefs := expressionNode.Operation.Preferences.(sliceArrayPreferences)
+	firstNumber := prefs.firstNumber
+	secondNumber := prefs.secondNumber
+
+	results := list.New()
+
+	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
+		lhsNode := el.Value.(*CandidateNode)
+		original := unwrapDoc(lhsNode.Node)
+
+		relativeFirstNumber := firstNumber
+		if relativeFirstNumber < 0 {
+			relativeFirstNumber = len(original.Content) + firstNumber
+		}
+
+		relativeSecondNumber := len(original.Content)
+		if prefs.secondNumberDefined {
+			relativeSecondNumber = secondNumber
+			if relativeSecondNumber < 0 {
+				relativeSecondNumber = len(original.Content) + secondNumber
+			}
+		}
+
+		log.Debug("calculateIndicesToTraverse: slice from %v to %v", relativeFirstNumber, relativeSecondNumber)
+
+		var newResults []*yaml.Node
+		for i := relativeFirstNumber; i < relativeSecondNumber; i++ {
+			newResults = append(newResults, original.Content[i])
+		}
+
+		slicedArrayNode := &yaml.Node{
+			Kind:    yaml.SequenceNode,
+			Tag:     original.Tag,
+			Content: newResults,
+		}
+		results.PushBack(lhsNode.CreateReplacement(slicedArrayNode))
+
+	}
+
+	// result is now the context that has the nodes we need to put back into a sequence.
+	//what about multiple arrays in the context? I think we need to create an array for each one
+	return context.ChildContext(results), nil
+}
@@ -0,0 +1,81 @@
+package yqlib
+
+import "testing"
+
+var sliceArrayScenarios = []expressionScenario{
+	{
+		description: "Slicing arrays",
+		document:    `[cat, dog, frog, cow]`,
+		expression:  `.[1:3]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- dog\n- frog\n",
+		},
+	},
+	{
+		description:    "Slicing arrays - without the first number",
+		subdescription: "Starts from the start of the array",
+		document:       `[cat, dog, frog, cow]`,
+		expression:     `.[:2]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- cat\n- dog\n",
+		},
+	},
+	{
+		description:    "Slicing arrays - without the second number",
+		subdescription: "Finishes at the end of the array",
+		document:       `[cat, dog, frog, cow]`,
+		expression:     `.[2:]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- frog\n- cow\n",
+		},
+	},
+	{
+		description: "Slicing arrays - use negative numbers to count backwards from the end",
+		document:    `[cat, dog, frog, cow]`,
+		expression:  `.[1:-1]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- dog\n- frog\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[[cat, dog, frog, cow], [apple, banana, grape, mango]]`,
+		expression: `.[] | .[1:3]`,
+		expected: []string{
+			"D0, P[0], (!!seq)::- dog\n- frog\n",
+			"D0, P[1], (!!seq)::- banana\n- grape\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[[cat, dog, frog, cow], [apple, banana, grape, mango]]`,
+		expression: `.[] | .[-2:-1]`,
+		expected: []string{
+			"D0, P[0], (!!seq)::- frog\n",
+			"D0, P[1], (!!seq)::- grape\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[cat1, cat2, cat3, cat4, cat5, cat6, cat7, cat8, cat9, cat10, cat11]`,
+		expression: `.[10:11]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- cat11\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[cat1, cat2, cat3, cat4, cat5, cat6, cat7, cat8, cat9, cat10, cat11]`,
+		expression: `.[-11:-10]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- cat1\n",
+		},
+	},
+}
+
+func TestSliceOperatorScenarios(t *testing.T) {
+	for _, tt := range sliceArrayScenarios {
+		testScenario(t, &tt)
+	}
+	documentOperatorScenarios(t, "slice-array", sliceArrayScenarios)
+}
@@ -76,7 +76,6 @@ func traverse(context Context, matchingNode *CandidateNode, operation *Operation
 }

 func traverseArrayOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-
 	//lhs may update the variable context, we should pass that into the RHS
 	// BUT we still return the original context back (see jq)
 	// https://stedolan.github.io/jq/manual/#Variable/SymbolicBindingOperator:...as$identifier|...
@@ -31,7 +31,7 @@ type expressionScenario struct {
 }

 func TestMain(m *testing.M) {
-	logging.SetLevel(logging.ERROR, "")
+	logging.SetLevel(logging.DEBUG, "")
 	Now = func() time.Time {
 		return time.Date(2021, time.May, 19, 1, 2, 3, 4, time.UTC)
 	}
@@ -40,21 +40,16 @@ func TestMain(m *testing.M) {
 }

 func NewSimpleYamlPrinter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
-	return NewPrinter(NewYamlEncoder(indent, colorsEnabled, printDocSeparators, unwrapScalar), NewSinglePrinterWriter(writer))
+	prefs := NewDefaultYamlPreferences()
+	prefs.PrintDocSeparators = printDocSeparators
+	prefs.UnwrapScalar = unwrapScalar
+	return NewPrinter(NewYamlEncoder(indent, colorsEnabled, prefs), NewSinglePrinterWriter(writer))
 }

-func readDocumentWithLeadingContent(content string, fakefilename string, fakeFileIndex int) (*list.List, error) {
-	reader, firstFileLeadingContent, err := processReadStream(bufio.NewReader(strings.NewReader(content)))
-	if err != nil {
-		return nil, err
-	}
+func readDocument(content string, fakefilename string, fakeFileIndex int) (*list.List, error) {
+	reader := bufio.NewReader(strings.NewReader(content))

-	inputs, err := readDocuments(reader, fakefilename, fakeFileIndex, NewYamlDecoder())
-	if err != nil {
-		return nil, err
-	}
-	inputs.Front().Value.(*CandidateNode).LeadingContent = firstFileLeadingContent
-	return inputs, nil
+	return readDocuments(reader, fakefilename, fakeFileIndex, NewYamlDecoder(ConfiguredYamlPreferences))
 }

 func testScenario(t *testing.T, s *expressionScenario) {
@@ -67,7 +62,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
 	inputs := list.New()

 	if s.document != "" {
-		inputs, err = readDocumentWithLeadingContent(s.document, "sample.yml", 0)
+		inputs, err = readDocument(s.document, "sample.yml", 0)

 		if err != nil {
 			t.Error(err, s.document, s.expression)
@@ -75,7 +70,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
 		}

 		if s.document2 != "" {
-			moreInputs, err := readDocumentWithLeadingContent(s.document2, "another.yml", 1)
+			moreInputs, err := readDocument(s.document2, "another.yml", 1)
 			if err != nil {
 				t.Error(err, s.document2, s.expression)
 				return
@@ -176,7 +171,7 @@ func formatYaml(yaml string, filename string) string {
 		panic(err)
 	}
 	streamEvaluator := NewStreamEvaluator()
-	_, err = streamEvaluator.Evaluate(filename, strings.NewReader(yaml), node, printer, "", NewYamlDecoder())
+	_, err = streamEvaluator.Evaluate(filename, strings.NewReader(yaml), node, printer, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {
 		panic(err)
 	}
@@ -322,13 +317,13 @@ func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formatt

 	if s.document != "" {

-		inputs, err = readDocumentWithLeadingContent(formattedDoc, "sample.yml", 0)
+		inputs, err = readDocument(formattedDoc, "sample.yml", 0)
 		if err != nil {
 			t.Error(err, s.document, s.expression)
 			return
 		}
 		if s.document2 != "" {
-			moreInputs, err := readDocumentWithLeadingContent(formattedDoc2, "another.yml", 1)
+			moreInputs, err := readDocument(formattedDoc2, "another.yml", 1)
 			if err != nil {
 				t.Error(err, s.document, s.expression)
 				return
@@ -111,7 +111,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {

 		mappedDoc := el.Value.(*CandidateNode)
 		log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document)
-
+		log.Debug("%v", NodeToString(mappedDoc))
 		writer, errorWriting := p.printerWriter.GetWriter(mappedDoc)
 		if errorWriting != nil {
 			return errorWriting
@@ -38,7 +38,7 @@ func TestPrinterMultipleDocsInSequenceOnly(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)

-	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {
 		panic(err)
 	}
@@ -76,7 +76,7 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)

-	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {
 		panic(err)
 	}
@@ -118,7 +118,7 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)

-	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {
 		panic(err)
 	}
@@ -165,7 +165,7 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)

-	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {
 		panic(err)
 	}
@@ -215,7 +215,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)

-	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {
 		panic(err)
 	}
@@ -234,7 +234,7 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)

-	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {
 		panic(err)
 	}
@@ -263,7 +263,7 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)

-	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {
 		panic(err)
 	}
@@ -294,7 +294,7 @@ func TestPrinterScalarWithLeadingCont(t *testing.T) {
 		panic(err)
 	}
 	streamEvaluator := NewStreamEvaluator()
-	_, err = streamEvaluator.Evaluate("sample", strings.NewReader(multiDocSample), node, printer, "# blah\n", NewYamlDecoder())
+	_, err = streamEvaluator.Evaluate("sample", strings.NewReader(multiDocSample), node, printer, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {
 		panic(err)
 	}
@@ -316,7 +316,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
 	// when outputing JSON.
 	printer := NewPrinter(NewJSONEncoder(0, false), NewSinglePrinterWriter(writer))

-	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {
 		panic(err)
 	}
@@ -8,7 +8,49 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )

-const samplePropertiesYaml = `# block comments don't come through
+const propertiesWithCommentsOnMap = `this.thing = hi hi
+# important notes
+# about this value
+this.value = cool
+`
+
+const expectedPropertiesWithCommentsOnMapProps = `this.thing = hi hi
+
+# important notes
+# about this value
+this.value = cool
+`
+
+const expectedPropertiesWithCommentsOnMapYaml = `this:
+  thing: hi hi
+  # important notes
+  # about this value
+  value: cool
+`
+
+const propertiesWithCommentInArray = `
+this.array.0 = cat
+# important notes
+# about dogs
+this.array.1 = dog
+`
+
+const expectedPropertiesWithCommentInArrayProps = `this.array.0 = cat
+
+# important notes
+# about dogs
+this.array.1 = dog
+`
+
+const expectedPropertiesWithCommentInArrayYaml = `this:
+  array:
+    - cat
+    # important notes
+    # about dogs
+    - dog
+`
+
+const samplePropertiesYaml = `# block comments come through
 person: # neither do comments on maps
    name: Mike Wazowski # comments on values appear
    pets: 
@@ -18,7 +60,8 @@ emptyArray: []
 emptyMap: []
 `

-const expectedPropertiesUnwrapped = `# comments on values appear
+const expectedPropertiesUnwrapped = `# block comments come through
+# comments on values appear
 person.name = Mike Wazowski

 # comments on array values appear
@@ -26,7 +69,8 @@ person.pets.0 = cat
 person.food.0 = pizza
 `

-const expectedPropertiesWrapped = `# comments on values appear
+const expectedPropertiesWrapped = `# block comments come through
+# comments on values appear
 person.name = "Mike Wazowski"

 # comments on array values appear
@@ -34,7 +78,8 @@ person.pets.0 = cat
 person.food.0 = pizza
 `

-const expectedUpdatedProperties = `# comments on values appear
+const expectedUpdatedProperties = `# block comments come through
+# comments on values appear
 person.name = Mike Wazowski

 # comments on array values appear
@@ -43,9 +88,12 @@ person.food.0 = pizza
 `

 const expectedDecodedYaml = `person:
-  name: Mike Wazowski # comments on values appear
+  # block comments come through
+  # comments on values appear
+  name: Mike Wazowski
  pets:
-    - cat # comments on array values appear
+    # comments on array values appear
+    - cat
  food:
    - pizza
 `
@@ -55,7 +103,8 @@ person.pets.0 = cat
 person.food.0 = pizza
 `

-const expectedPropertiesWithEmptyMapsAndArrays = `# comments on values appear
+const expectedPropertiesWithEmptyMapsAndArrays = `# block comments come through
+# comments on values appear
 person.name = Mike Wazowski

 # comments on array values appear
@@ -112,6 +161,34 @@ var propertyScenarios = []formatScenario{
 		expected:     expectedUpdatedProperties,
 		scenarioType: "roundtrip",
 	},
+	{
+		skipDoc:      true,
+		description:  "comments on arrays roundtrip",
+		input:        propertiesWithCommentInArray,
+		expected:     expectedPropertiesWithCommentInArrayProps,
+		scenarioType: "roundtrip",
+	},
+	{
+		skipDoc:      true,
+		description:  "comments on arrays decode",
+		input:        propertiesWithCommentInArray,
+		expected:     expectedPropertiesWithCommentInArrayYaml,
+		scenarioType: "decode",
+	},
+	{
+		skipDoc:      true,
+		description:  "comments on map roundtrip",
+		input:        propertiesWithCommentsOnMap,
+		expected:     expectedPropertiesWithCommentsOnMapProps,
+		scenarioType: "roundtrip",
+	},
+	{
+		skipDoc:      true,
+		description:  "comments on map decode",
+		input:        propertiesWithCommentsOnMap,
+		expected:     expectedPropertiesWithCommentsOnMapYaml,
+		scenarioType: "decode",
+	},
 	{
 		description:  "Empty doc",
 		skipDoc:      true,
@@ -143,7 +220,7 @@ func documentUnwrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario)
 	}
 	writeOrPanic(w, "will output\n")

-	writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(true))))
+	writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true))))
 }

 func documentWrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) {
@@ -168,7 +245,7 @@ func documentWrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) {
 	}
 	writeOrPanic(w, "will output\n")

-	writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(false))))
+	writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false))))
 }

 func documentDecodePropertyScenario(w *bufio.Writer, s formatScenario) {
@@ -193,7 +270,7 @@ func documentDecodePropertyScenario(w *bufio.Writer, s formatScenario) {

 	writeOrPanic(w, "will output\n")

-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(s.indent, false, true, true))))
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))))
 }

 func documentRoundTripPropertyScenario(w *bufio.Writer, s formatScenario) {
@@ -245,11 +322,11 @@ func TestPropertyScenarios(t *testing.T) {
 	for _, s := range propertyScenarios {
 		switch s.scenarioType {
 		case "":
-			test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(true)), s.description)
+			test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true)), s.description)
 		case "decode":
-			test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(2, false, true, true)), s.description)
+			test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
 		case "encode-wrapped":
-			test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(false)), s.description)
+			test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false)), s.description)
 		case "roundtrip":
 			test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(true)), s.description)

@@ -14,9 +14,9 @@ import (
 // Uses less memory than loading all documents and running the expression once, but this cannot process
 // cross document expressions.
 type StreamEvaluator interface {
-	Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, leadingContent string, decoder Decoder) (uint, error)
-	EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error
-	EvaluateNew(expression string, printer Printer, leadingContent string) error
+	Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, decoder Decoder) (uint, error)
+	EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error
+	EvaluateNew(expression string, printer Printer) error
 }

 type streamEvaluator struct {
@@ -28,17 +28,16 @@ func NewStreamEvaluator() StreamEvaluator {
 	return &streamEvaluator{treeNavigator: NewDataTreeNavigator()}
 }

-func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadingContent string) error {
+func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error {
 	node, err := ExpressionParser.ParseExpression(expression)
 	if err != nil {
 		return err
 	}
 	candidateNode := &CandidateNode{
-		Document:       0,
-		Filename:       "",
-		Node:           &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
-		FileIndex:      0,
-		LeadingContent: leadingContent,
+		Document:  0,
+		Filename:  "",
+		Node:      &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
+		FileIndex: 0,
 	}
 	inputList := list.New()
 	inputList.PushBack(candidateNode)
@@ -50,27 +49,20 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadin
 	return printer.PrintResults(result.MatchingNodes)
 }

-func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error {
+func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error {
 	var totalProcessDocs uint
 	node, err := ExpressionParser.ParseExpression(expression)
 	if err != nil {
 		return err
 	}

-	var firstFileLeadingContent string
-
-	for index, filename := range filenames {
-		reader, leadingContent, err := readStream(filename, leadingContentPreProcessing)
-		log.Debug("leadingContent: %v", leadingContent)
-
-		if index == 0 {
-			firstFileLeadingContent = leadingContent
-		}
+	for _, filename := range filenames {
+		reader, err := readStream(filename)

 		if err != nil {
 			return err
 		}
-		processedDocs, err := s.Evaluate(filename, reader, node, printer, leadingContent, decoder)
+		processedDocs, err := s.Evaluate(filename, reader, node, printer, decoder)
 		if err != nil {
 			return err
 		}
@@ -83,19 +75,22 @@ func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, p
 	}

 	if totalProcessDocs == 0 {
-		return s.EvaluateNew(expression, printer, firstFileLeadingContent)
+		// problem is I've already slurped the leading content sadface
+		return s.EvaluateNew(expression, printer)
 	}

 	return nil
 }

-func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, leadingContent string, decoder Decoder) (uint, error) {
+func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, decoder Decoder) (uint, error) {

 	var currentIndex uint
-	decoder.Init(reader)
+	err := decoder.Init(reader)
+	if err != nil {
+		return 0, err
+	}
 	for {
-		var dataBucket yaml.Node
-		errorReading := decoder.Decode(&dataBucket)
+		candidateNode, errorReading := decoder.Decode()

 		if errors.Is(errorReading, io.EOF) {
 			s.fileIndex = s.fileIndex + 1
@@ -103,21 +98,10 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr
 		} else if errorReading != nil {
 			return currentIndex, fmt.Errorf("bad file '%v': %w", filename, errorReading)
 		}
+		candidateNode.Document = currentIndex
+		candidateNode.Filename = filename
+		candidateNode.FileIndex = s.fileIndex

-		candidateNode := &CandidateNode{
-			Document:  currentIndex,
-			Filename:  filename,
-			Node:      &dataBucket,
-			FileIndex: s.fileIndex,
-		}
-		// move document comments into candidate node
-		// otherwise unwrap drops them.
-		candidateNode.TrailingContent = dataBucket.FootComment
-		dataBucket.FootComment = ""
-
-		if currentIndex == 0 {
-			candidateNode.LeadingContent = leadingContent
-		}
 		inputList := list.New()
 		inputList.PushBack(candidateNode)

@@ -1,17 +1,17 @@
 package yqlib

 import (
+	"bufio"
 	"bytes"
 	"container/list"
 	"errors"
 	"fmt"
 	"io"
-
-	yaml "gopkg.in/yaml.v3"
+	"strings"
 )

 type StringEvaluator interface {
-	Evaluate(expression string, input string, encoder Encoder, leadingContentPreProcessing bool, decoder Decoder) (string, error)
+	Evaluate(expression string, input string, encoder Encoder, decoder Decoder) (string, error)
 }

 type stringEvaluator struct {
@@ -25,7 +25,7 @@ func NewStringEvaluator() StringEvaluator {
 	}
 }

-func (s *stringEvaluator) Evaluate(expression string, input string, encoder Encoder, leadingContentPreProcessing bool, decoder Decoder) (string, error) {
+func (s *stringEvaluator) Evaluate(expression string, input string, encoder Encoder, decoder Decoder) (string, error) {

 	// Use bytes.Buffer for output of string
 	out := new(bytes.Buffer)
@@ -37,16 +37,15 @@ func (s *stringEvaluator) Evaluate(expression string, input string, encoder Enco
 		return "", err
 	}

-	reader, leadingContent, err := readString(input, leadingContentPreProcessing)
+	reader := bufio.NewReader(strings.NewReader(input))
+
+	var currentIndex uint
+	err = decoder.Init(reader)
 	if err != nil {
 		return "", err
 	}
-
-	var currentIndex uint
-	decoder.Init(reader)
 	for {
-		var dataBucket yaml.Node
-		errorReading := decoder.Decode(&dataBucket)
+		candidateNode, errorReading := decoder.Decode()

 		if errors.Is(errorReading, io.EOF) {
 			s.fileIndex = s.fileIndex + 1
@@ -54,20 +53,9 @@ func (s *stringEvaluator) Evaluate(expression string, input string, encoder Enco
 		} else if errorReading != nil {
 			return "", fmt.Errorf("bad input '%v': %w", input, errorReading)
 		}
+		candidateNode.Document = currentIndex
+		candidateNode.FileIndex = s.fileIndex

-		candidateNode := &CandidateNode{
-			Document:  currentIndex,
-			Node:      &dataBucket,
-			FileIndex: s.fileIndex,
-		}
-		// move document comments into candidate node
-		// otherwise unwrap drops them.
-		candidateNode.TrailingContent = dataBucket.FootComment
-		dataBucket.FootComment = ""
-
-		if currentIndex == 0 {
-			candidateNode.LeadingContent = leadingContent
-		}
 		inputList := list.New()
 		inputList.PushBack(candidateNode)

@@ -18,10 +18,10 @@ func TestStringEvaluator_Evaluate_Nominal(t *testing.T) {
 		`---` + "\n" +
 		` - name: jq` + "\n" +
 		`   description: Command-line JSON processor` + "\n"
-	encoder := NewYamlEncoder(2, true, true, true)
-	decoder := NewYamlDecoder()
+	encoder := NewYamlEncoder(2, true, ConfiguredYamlPreferences)
+	decoder := NewYamlDecoder(ConfiguredYamlPreferences)

-	result, err := NewStringEvaluator().Evaluate(expression, input, encoder, true, decoder)
+	result, err := NewStringEvaluator().Evaluate(expression, input, encoder, decoder)
 	if err != nil {
 		t.Error(err)
 	}
@@ -7,13 +7,9 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"regexp"
-	"strings"
-
-	yaml "gopkg.in/yaml.v3"
 )

-func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, string, error) {
+func readStream(filename string) (io.Reader, error) {
 	var reader *bufio.Reader
 	if filename == "-" {
 		reader = bufio.NewReader(os.Stdin)
@@ -22,23 +18,12 @@ func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, s
 		// and ensuring that it's not possible to give a path to a file outside thar directory.
 		file, err := os.Open(filename) // #nosec
 		if err != nil {
-			return nil, "", err
+			return nil, err
 		}
 		reader = bufio.NewReader(file)
 	}
+	return reader, nil

-	if !leadingContentPreProcessing {
-		return reader, "", nil
-	}
-	return processReadStream(reader)
-}
-
-func readString(input string, leadingContentPreProcessing bool) (io.Reader, string, error) {
-	reader := bufio.NewReader(strings.NewReader(input))
-	if !leadingContentPreProcessing {
-		return reader, "", nil
-	}
-	return processReadStream(reader)
 }

 func writeString(writer io.Writer, txt string) error {
@@ -46,46 +31,16 @@ func writeString(writer io.Writer, txt string) error {
 	return errorWriting
 }

-func processReadStream(reader *bufio.Reader) (io.Reader, string, error) {
-	var commentLineRegEx = regexp.MustCompile(`^\s*#`)
-	var sb strings.Builder
-	for {
-		peekBytes, err := reader.Peek(3)
-		if errors.Is(err, io.EOF) {
-			// EOF are handled else where..
-			return reader, sb.String(), nil
-		} else if err != nil {
-			return reader, sb.String(), err
-		} else if string(peekBytes) == "---" {
-			_, err := reader.ReadString('\n')
-			sb.WriteString("$yqDocSeperator$\n")
-			if errors.Is(err, io.EOF) {
-				return reader, sb.String(), nil
-			} else if err != nil {
-				return reader, sb.String(), err
-			}
-		} else if commentLineRegEx.MatchString(string(peekBytes)) {
-			line, err := reader.ReadString('\n')
-			sb.WriteString(line)
-			if errors.Is(err, io.EOF) {
-				return reader, sb.String(), nil
-			} else if err != nil {
-				return reader, sb.String(), err
-			}
-		} else {
-			return reader, sb.String(), nil
-		}
-	}
-}
-
 func readDocuments(reader io.Reader, filename string, fileIndex int, decoder Decoder) (*list.List, error) {
-	decoder.Init(reader)
+	err := decoder.Init(reader)
+	if err != nil {
+		return nil, err
+	}
 	inputList := list.New()
 	var currentIndex uint

 	for {
-		var dataBucket yaml.Node
-		errorReading := decoder.Decode(&dataBucket)
+		candidateNode, errorReading := decoder.Decode()

 		if errors.Is(errorReading, io.EOF) {
 			switch reader := reader.(type) {
@@ -96,18 +51,10 @@ func readDocuments(reader io.Reader, filename string, fileIndex int, decoder Dec
 		} else if errorReading != nil {
 			return nil, fmt.Errorf("bad file '%v': %w", filename, errorReading)
 		}
-		candidateNode := &CandidateNode{
-			Document:         currentIndex,
-			Filename:         filename,
-			Node:             &dataBucket,
-			FileIndex:        fileIndex,
-			EvaluateTogether: true,
-		}
-
-		//move document comments into candidate node
-		// otherwise unwrap drops them.
-		candidateNode.TrailingContent = dataBucket.FootComment
-		dataBucket.FootComment = ""
+		candidateNode.Document = currentIndex
+		candidateNode.Filename = filename
+		candidateNode.FileIndex = fileIndex
+		candidateNode.EvaluateTogether = true

 		inputList.PushBack(candidateNode)

@@ -0,0 +1,29 @@
+package yqlib
+
+type XmlPreferences struct {
+	AttributePrefix string
+	ContentName     string
+	StrictMode      bool
+	KeepNamespace   bool
+	UseRawToken     bool
+	ProcInstPrefix  string
+	DirectiveName   string
+	SkipProcInst    bool
+	SkipDirectives  bool
+}
+
+func NewDefaultXmlPreferences() XmlPreferences {
+	return XmlPreferences{
+		AttributePrefix: "+",
+		ContentName:     "+content",
+		StrictMode:      false,
+		KeepNamespace:   true,
+		UseRawToken:     false,
+		ProcInstPrefix:  "+p_",
+		DirectiveName:   "+directive",
+		SkipProcInst:    false,
+		SkipDirectives:  false,
+	}
+}
+
+var ConfiguredXMLPreferences = NewDefaultXmlPreferences()
@@ -137,7 +137,8 @@ in d before -->
 </cat><!-- after cat -->
 `

-const yamlWithComments = `# above_cat
+const yamlWithComments = `# header comment
+# above_cat
 cat: # inline_cat
  # above_array
  array: # inline_array
@@ -147,7 +148,10 @@ cat: # inline_cat
 # below_cat
 `

-const expectedXMLWithComments = `<!-- above_cat inline_cat --><cat><!-- above_array inline_array -->
+const expectedXMLWithComments = `<!--
+ header comment
+ above_cat
+ --><!-- inline_cat --><cat><!-- above_array inline_array -->
  <array>val1<!-- inline_val1 --></array>
  <array><!-- above_val2 -->val2<!-- inline_val2 --></array>
 </cat><!-- below_cat -->
@@ -414,19 +418,19 @@ var xmlScenarios = []formatScenario{
 func testXMLScenario(t *testing.T, s formatScenario) {
 	switch s.scenarioType {
 	case "", "decode":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(XMLPreferences), NewYamlEncoder(4, false, true, true)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(4, false, ConfiguredYamlPreferences)), s.description)
 	case "encode":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewXMLEncoder(2, XMLPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description)
 	case "roundtrip":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(XMLPreferences), NewXMLEncoder(2, XMLPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description)
 	case "decode-keep-ns":
 		prefs := NewDefaultXmlPreferences()
 		prefs.KeepNamespace = true
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, true, true)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
 	case "decode-raw-token":
 		prefs := NewDefaultXmlPreferences()
 		prefs.UseRawToken = true
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, true, true)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
 	case "roundtrip-skip-directives":
 		prefs := NewDefaultXmlPreferences()
 		prefs.SkipDirectives = true
@@ -480,7 +484,7 @@ func documentXMLDecodeScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=xml '%v' sample.xml\n```\n", expression))
 	writeOrPanic(w, "will output\n")

-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(XMLPreferences), NewYamlEncoder(2, false, true, true))))
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences))))
 }

 func documentXMLDecodeKeepNsScenario(w *bufio.Writer, s formatScenario) {
@@ -549,7 +553,7 @@ func documentXMLEncodeScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, "```bash\nyq -o=xml '.' sample.yml\n```\n")
 	writeOrPanic(w, "will output\n")

-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewXMLEncoder(2, XMLPreferences))))
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences))))
 }

 func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) {
@@ -567,7 +571,7 @@ func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, "```bash\nyq -p=xml -o=xml '.' sample.xml\n```\n")
 	writeOrPanic(w, "will output\n")

-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(XMLPreferences), NewXMLEncoder(2, XMLPreferences))))
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences))))
 }

 func documentXMLSkipDirectrivesScenario(w *bufio.Writer, s formatScenario) {
@@ -0,0 +1,19 @@
+package yqlib
+
+type YamlPreferences struct {
+	LeadingContentPreProcessing bool
+	PrintDocSeparators          bool
+	UnwrapScalar                bool
+	EvaluateTogether            bool
+}
+
+func NewDefaultYamlPreferences() YamlPreferences {
+	return YamlPreferences{
+		LeadingContentPreProcessing: true,
+		PrintDocSeparators:          true,
+		UnwrapScalar:                true,
+		EvaluateTogether:            false,
+	}
+}
+
+var ConfiguredYamlPreferences = NewDefaultYamlPreferences()
@@ -1,3 +1,8 @@
+4.29.1:
+  - Fixed Square brackets removing update #1342
+  - XML decoder/encoder now parses directives and proc instructions (#1344). Please use the new skip flags [documented here](https://mikefarah.gitbook.io/yq/usage/xml) to ignore them.
+  - Bumped Go compiler version
+
 4.28.2:
  - Fixed Github Actions issues (thanks @mattphelps-8451)
  - Fixed bug - can now delete documents #1377
@@ -23,17 +23,29 @@ func ParseData(rawData string) yaml.Node {
 	return parsedData
 }

+func printDifference(t *testing.T, expectedValue interface{}, actualValue interface{}) {
+	opts := []write.Option{write.TerminalColor()}
+	var differenceBuffer bytes.Buffer
+	expectedString := fmt.Sprintf("%v", expectedValue)
+	actualString := fmt.Sprintf("%v", actualValue)
+	if err := diff.Text("expected", "actual", expectedString, actualString, bufio.NewWriter(&differenceBuffer), opts...); err != nil {
+		t.Error(err)
+	} else {
+		t.Error(differenceBuffer.String())
+	}
+}
+
 func AssertResult(t *testing.T, expectedValue interface{}, actualValue interface{}) {
 	t.Helper()
 	if expectedValue != actualValue {
-		t.Error("Expected <", expectedValue, "> but got <", actualValue, ">", fmt.Sprintf("%T", actualValue))
+		printDifference(t, expectedValue, actualValue)
 	}
 }

 func AssertResultComplex(t *testing.T, expectedValue interface{}, actualValue interface{}) {
 	t.Helper()
 	if !reflect.DeepEqual(expectedValue, actualValue) {
-		t.Error("\nExpected <", expectedValue, ">\nbut got  <", actualValue, ">", fmt.Sprintf("%T", actualValue))
+		printDifference(t, expectedValue, actualValue)
 	}
 }

@@ -41,7 +53,7 @@ func AssertResultComplexWithContext(t *testing.T, expectedValue interface{}, act
 	t.Helper()
 	if !reflect.DeepEqual(expectedValue, actualValue) {
 		t.Error(context)
-		t.Error("\nExpected <", expectedValue, ">\nbut got  <", actualValue, ">", fmt.Sprintf("%T", actualValue))
+		printDifference(t, expectedValue, actualValue)
 	}
 }
Author	SHA1	Message	Date
Mike Farah	48d6ad58e7	Slice array	2022-10-28 20:33:50 +11:00
Mike FarahandGitHub	880397d549	Refactored decoder responsibilities (#1402 ) - improved comment handling - yaml decoder now responsible for leading content work around	2022-10-28 14:16:46 +11:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	46c32f4c79	Bump github.com/spf13/cobra from 1.6.0 to 1.6.1 (#1399 ) Bumps [github.com/spf13/cobra](https://github.com/spf13/cobra) from 1.6.0 to 1.6.1. - [Release notes](https://github.com/spf13/cobra/releases) - [Commits](https://github.com/spf13/cobra/compare/v1.6.0...v1.6.1) --- updated-dependencies: - dependency-name: github.com/spf13/cobra dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2022-10-27 15:11:06 +11:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	b25114c1fc	Bump github.com/goccy/go-yaml from 1.9.5 to 1.9.6 (#1400 ) Bumps [github.com/goccy/go-yaml](https://github.com/goccy/go-yaml) from 1.9.5 to 1.9.6. - [Release notes](https://github.com/goccy/go-yaml/releases) - [Changelog](https://github.com/goccy/go-yaml/blob/master/CHANGELOG.md) - [Commits](https://github.com/goccy/go-yaml/compare/v1.9.5...v1.9.6) --- updated-dependencies: - dependency-name: github.com/goccy/go-yaml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2022-10-27 15:10:56 +11:00
Mike Farah	44552c151b	Improving readme	2022-10-26 13:12:20 +11:00
Sho CeandGitHub	108cfeb4bb	README: how to easily download the latest release (#1397 )	2022-10-25 20:43:24 +11:00
Mike Farah	24bbffd71e	xml prefs should be part of API	2022-10-25 14:27:16 +11:00
Mike Farah	c62e18f9b2	Fixed load operator bug	2022-10-25 13:30:38 +11:00
Mike Farah	c1640fb10d	Removing old version notice	2022-10-25 12:47:13 +11:00
Mike Farah	0c33690596	Updating release notes	2022-10-25 09:36:00 +11:00
Mike Farah	6bf3defe85	Fixing updates in square brackets #1342	2022-10-24 17:55:19 +11:00
Mike FarahandGitHub	6d6b693fb3	Added XML processing instructions and directive support (#1396 )	2022-10-24 10:09:42 +11:00