Fixing from review

Added CSV roundtrip
Adding CSV decoder
2026-08-24 16:39:58 +08:00 · 2022-08-01 10:16:59 +10:00 · 2022-08-01 10:05:26 +10:00 · 2022-07-31 17:35:36 +10:00 · 2022-07-31 17:31:27 +10:00 · 2022-07-29 10:28:06 +10:00
57 changed files with 2979 additions and 899 deletions
@@ -54,3 +54,6 @@ yq.1
 # debian pkg
 _build
 debian/files
+
+# intellij
+/.idea
@@ -1,3 +1,5 @@
+# Development
+
 1. Install (golang)[https://golang.org/]
 1. Run  `scripts/devtools.sh` to install the required devtools
 2. Run `make [local] vendor` to install the vendor dependencies
@@ -6,3 +8,33 @@
 4. Make the code changes.
 5. `make [local] test` to lint code and run tests
 6. Profit! ok no profit, but raise a PR and get kudos :)
+
+
+# Documentation
+
+The documentation is a bit of a mixed bag (sorry in advanced, I do plan on simplifying it...) - with some parts automatically generated and stiched together and some statically defined.
+
+Documentation is written in markdown, and is published in the 'gitbook' branch.
+
+The various operator documentation (e.g. 'strings') are generated from the 'master' branch, and have a statically defined header (e.g. `pkg/yqlib/doc/operators/headers/add.md`) and the bulk of the docs are generated from the unit tests e.g. `pkg/yqlib/operator_add_test.go`.
+
+The pipeline will run the tests and automatically concatenate the files together, and put them under 
+`pkg/qylib/doc/add.md`. These files are checked in the master branch (and are copied to the gitbook branch as part of the release process).
+
+## How to contribute
+
+The first step is to find if what you want is automatically generated or not - start by looking in the master branch. 
+
+### Updating dynamic documentation from master
+- Search for the documentation you want to update. If you find matches in a `*_test.go` file - update that, as that will automatically update the matching `*.md` file 
+- Assuming you are updating a `*_test.go` file, once updated, run the test to regenerated the docs. E.g. for the 'Add' test generated docs, from the pkg/yqlib folder run:
+`go test -run TestAddOperatorScenarios` which will run that test defined in the `operator_add_test.go` file.
+- Ensure the tests still pass, and check the generated documentation have your update.
+- Note: If the documentation is only in a `headers/*.md` file, then just update that directly
+- Raise a PR to merge the changes into master!
+
+### Updating static documentation from the gitbook branch
+If you haven't found what you want to update in the master branch, then check the gitbook branch directly as there are a few pages in there that are not in master.
+
+- Update the `*.md` files
+- Raise a PR to merge the changes into gitbook.
@@ -1,4 +1,4 @@
-FROM golang:1.18.3 as builder
+FROM golang:1.18.4 as builder

 WORKDIR /go/src/mikefarah/yq

@@ -1,4 +1,4 @@
-FROM golang:1.18.3
+FROM golang:1.18.4

 COPY scripts/devtools.sh /opt/devtools.sh

@@ -331,4 +331,26 @@ EOM
  assertEquals "$expected" "$X"
 }

+testBasicMultiplyAssignMultiDoc() {
+      cat >test.yml <<EOL
+a: 1
+---
+b: 2
+EOL
+
+read -r -d '' expected << EOM
+a: 1
+c: 3
+---
+b: 2
+c: 3
+EOM
+
+
+  X=$(./yq '. *= {"c":3}' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+
+
 source ./scripts/shunit2
@@ -3,6 +3,8 @@
 setUp() {
  rm test*.yml 2>/dev/null || true
  rm test*.properties 2>/dev/null || true
+  rm test*.csv 2>/dev/null || true
+  rm test*.tsv 2>/dev/null || true
  rm test*.xml 2>/dev/null || true
 }

@@ -40,6 +42,51 @@ EOM
  assertEquals "$expected" "$X"
 }

+testInputCSV() {
+  cat >test.csv <<EOL
+fruit,yumLevel
+apple,5
+banana,4
+EOL
+
+  read -r -d '' expected << EOM
+- fruit: apple
+  yumLevel: 5
+- fruit: banana
+  yumLevel: 4
+EOM
+
+  X=$(./yq e -p=csv test.csv)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -p=csv test.csv)
+  assertEquals "$expected" "$X"
+}
+
+testInputTSV() {
+  cat >test.tsv <<EOL
+fruit	yumLevel
+apple	5
+banana	4
+EOL
+
+  read -r -d '' expected << EOM
+- fruit: apple
+  yumLevel: 5
+- fruit: banana
+  yumLevel: 4
+EOM
+
+  X=$(./yq e -p=t test.tsv)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -p=t test.tsv)
+  assertEquals "$expected" "$X"
+}
+
+
+
+
 testInputXml() {
  cat >test.yml <<EOL
 <cat legs="4">BiBi</cat>
@@ -102,6 +102,48 @@ EOM
  assertEquals "$expected" "$X"
 }

+testOutputCSV() {
+  cat >test.yml <<EOL
+- fruit: apple
+  yumLevel: 5
+- fruit: banana
+  yumLevel: 4
+EOL
+
+  read -r -d '' expected << EOM
+fruit,yumLevel
+apple,5
+banana,4
+EOM
+
+  X=$(./yq -o=c test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -o=csv test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testOutputTSV() {
+  cat >test.yml <<EOL
+- fruit: apple
+  yumLevel: 5
+- fruit: banana
+  yumLevel: 4
+EOL
+
+  read -r -d '' expected << EOM
+fruit	yumLevel
+apple	5
+banana	4
+EOM
+
+  X=$(./yq -o=t test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -o=tsv test.yml)
+  assertEquals "$expected" "$X"
+}
+
 testOutputXml() {
  cat >test.yml <<EOL
 a: {b: {c: ["cat"]}}
@@ -66,6 +66,12 @@ func configureDecoder() (yqlib.Decoder, error) {
 		return yqlib.NewXMLDecoder(xmlAttributePrefix, xmlContentName, xmlStrictMode, xmlKeepNamespace, xmlUseRawToken), nil
 	case yqlib.PropertiesInputFormat:
 		return yqlib.NewPropertiesDecoder(), nil
+	case yqlib.JsonInputFormat:
+		return yqlib.NewJSONDecoder(), nil
+	case yqlib.CSVObjectInputFormat:
+		return yqlib.NewCSVObjectDecoder(','), nil
+	case yqlib.TSVObjectInputFormat:
+		return yqlib.NewCSVObjectDecoder('\t'), nil
 	}

 	return yqlib.NewYamlDecoder(), nil
@@ -91,9 +97,9 @@ func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yq
 func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder {
 	switch format {
 	case yqlib.JSONOutputFormat:
-		return yqlib.NewJONEncoder(indent, colorsEnabled)
+		return yqlib.NewJSONEncoder(indent, colorsEnabled)
 	case yqlib.PropsOutputFormat:
-		return yqlib.NewPropertiesEncoder()
+		return yqlib.NewPropertiesEncoder(unwrapScalar)
 	case yqlib.CSVOutputFormat:
 		return yqlib.NewCsvEncoder(',')
 	case yqlib.TSVOutputFormat:
@@ -11,7 +11,7 @@ var (
 	GitDescribe string

 	// Version is main version number that is being run at the moment.
-	Version = "4.25.3"
+	Version = "4.26.1"

 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release
@@ -0,0 +1,3 @@
+name,numberOfCats,likesApples,height
+Gary,1,true,168.8
+Samantha's Rabbit,2,false,-188.8
@@ -2,15 +2,17 @@ module github.com/mikefarah/yq/v4

 require (
 	github.com/a8m/envsubst v1.3.0
+	github.com/alecthomas/participle/v2 v2.0.0-beta.4
+	github.com/alecthomas/repr v0.1.0
 	github.com/elliotchance/orderedmap v1.4.0
 	github.com/fatih/color v1.13.0
+	github.com/goccy/go-json v0.9.10
 	github.com/goccy/go-yaml v1.9.5
 	github.com/jinzhu/copier v0.3.5
 	github.com/magiconair/properties v1.8.6
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
 	github.com/spf13/cobra v1.5.0
-	github.com/timtadh/lexmachine v0.2.2
-	golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
+	golang.org/x/net v0.0.0-20220708220712-1185a9018129
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -20,10 +22,9 @@ require (
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/timtadh/data-structures v0.5.3 // indirect
-	golang.org/x/sys v0.0.0-20211205182925-97ca703d548d // indirect
+	golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e // indirect
 	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 )

 go 1.18
@@ -1,5 +1,10 @@
 github.com/a8m/envsubst v1.3.0 h1:GmXKmVssap0YtlU3E230W98RWtWCyIZzjtf1apWWyAg=
 github.com/a8m/envsubst v1.3.0/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
+github.com/alecthomas/assert/v2 v2.0.3 h1:WKqJODfOiQG0nEJKFKzDIG3E29CN2/4zR9XGJzKIkbg=
+github.com/alecthomas/participle/v2 v2.0.0-beta.4 h1:ublfGBm+x+p2j7KotHhrUMbKtejT7M0Gv1Mt1u3absw=
+github.com/alecthomas/participle/v2 v2.0.0-beta.4/go.mod h1:RC764t6n4L8D8ITAJv0qdokritYSNR3wV5cVwmIEaMM=
+github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE=
+github.com/alecthomas/repr v0.1.0/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -12,8 +17,11 @@ github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvSc
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
+github.com/goccy/go-json v0.9.10 h1:hCeNmprSNLB8B8vQKWl6DpuH0t60oEs+TAk9a7CScKc=
+github.com/goccy/go-json v0.9.10/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/goccy/go-yaml v1.9.5 h1:Eh/+3uk9kLxG4koCX6lRMAPS1OaMSAi+FJcya0INdB0=
 github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
+github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg=
@@ -41,30 +49,27 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/timtadh/data-structures v0.5.3 h1:F2tEjoG9qWIyUjbvXVgJqEOGJPMIiYn7U5W5mE+i/vQ=
-github.com/timtadh/data-structures v0.5.3/go.mod h1:9R4XODhJ8JdWFEI8P/HJKqxuJctfBQw6fDibMQny2oU=
-github.com/timtadh/lexmachine v0.2.2 h1:g55RnjdYazm5wnKv59pwFcBJHOyvTPfDEoz21s4PHmY=
-github.com/timtadh/lexmachine v0.2.2/go.mod h1:GBJvD5OAfRn/gnp92zb9KTgHLB7akKyxmVivoYCcjQI=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c=
-golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220708220712-1185a9018129 h1:vucSRfWwTsoXro7P+3Cjlr6flUMtzCwzlvkxEQtHHB0=
+golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211205182925-97ca703d548d h1:FjkYO/PPp4Wi0EAUOVLxePm7qVW4r4ctbWpURyuOD0E=
-golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e h1:NHvCuwuS43lGnYhten69ZWqi2QOj/CiDNcKbVqwVoew=
+golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0=
+golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW8s2qTSe3wGBtvo0MbVQG/c5k8RE=
@@ -0,0 +1,273 @@
+package yqlib
+
+import (
+	"bufio"
+	"fmt"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+const csvSimple = `name,numberOfCats,likesApples,height
+Gary,1,true,168.8
+Samantha's Rabbit,2,false,-188.8
+`
+
+const expectedUpdatedSimpleCsv = `name,numberOfCats,likesApples,height
+Gary,3,true,168.8
+Samantha's Rabbit,2,false,-188.8
+`
+
+const csvSimpleShort = `Name,Number of Cats
+Gary,1
+Samantha's Rabbit,2
+`
+
+const tsvSimple = `name	numberOfCats	likesApples	height
+Gary	1	true	168.8
+Samantha's Rabbit	2	false	-188.8
+`
+
+const expectedYamlFromCSV = `- name: Gary
+  numberOfCats: 1
+  likesApples: true
+  height: 168.8
+- name: Samantha's Rabbit
+  numberOfCats: 2
+  likesApples: false
+  height: -188.8
+`
+
+const expectedYamlFromCSVMissingData = `- name: Gary
+  numberOfCats: 1
+  height: 168.8
+- name: Samantha's Rabbit
+  height: -188.8
+  likesApples: false
+`
+
+const csvSimpleMissingData = `name,numberOfCats,height
+Gary,1,168.8
+Samantha's Rabbit,,-188.8
+`
+
+const csvTestSimpleYaml = `- [i, like, csv]
+- [because, excel, is, cool]`
+
+const expectedSimpleCsv = `i,like,csv
+because,excel,is,cool
+`
+
+const tsvTestExpectedSimpleCsv = `i	like	csv
+because	excel	is	cool
+`
+
+var csvScenarios = []formatScenario{
+	{
+		description:  "Encode CSV simple",
+		input:        csvTestSimpleYaml,
+		expected:     expectedSimpleCsv,
+		scenarioType: "encode-csv",
+	},
+	{
+		description:  "Encode TSV simple",
+		input:        csvTestSimpleYaml,
+		expected:     tsvTestExpectedSimpleCsv,
+		scenarioType: "encode-tsv",
+	},
+	{
+		description:  "Encode Empty",
+		skipDoc:      true,
+		input:        `[]`,
+		expected:     "",
+		scenarioType: "encode-csv",
+	},
+	{
+		description:  "Comma in value",
+		skipDoc:      true,
+		input:        `["comma, in, value", things]`,
+		expected:     "\"comma, in, value\",things\n",
+		scenarioType: "encode-csv",
+	},
+	{
+		description:  "Encode array of objects to csv",
+		input:        expectedYamlFromCSV,
+		expected:     csvSimple,
+		scenarioType: "encode-csv",
+	},
+	{
+		description:    "Encode array of objects to custom csv format",
+		subdescription: "Add the header row manually, then the we convert each object into an array of values - resulting in an array of arrays. Pick the columns and call the header whatever you like.",
+		input:          expectedYamlFromCSV,
+		expected:       csvSimpleShort,
+		expression:     `[["Name", "Number of Cats"]] +  [.[] | [.name, .numberOfCats ]]`,
+		scenarioType:   "encode-csv",
+	},
+	{
+		description:    "Encode array of objects to csv - missing fields behaviour",
+		subdescription: "First entry is used to determine the headers, and it is missing 'likesApples', so it is not included in the csv. Second entry does not have 'numberOfCats' so that is blank",
+		input:          expectedYamlFromCSVMissingData,
+		expected:       csvSimpleMissingData,
+		scenarioType:   "encode-csv",
+	},
+	{
+		description:    "Parse CSV into an array of objects",
+		subdescription: "First row is assumed to be the header row.",
+		input:          csvSimple,
+		expected:       expectedYamlFromCSV,
+		scenarioType:   "decode-csv-object",
+	},
+	{
+		description:    "Parse TSV into an array of objects",
+		subdescription: "First row is assumed to be the header row.",
+		input:          tsvSimple,
+		expected:       expectedYamlFromCSV,
+		scenarioType:   "decode-tsv-object",
+	},
+	{
+		description:  "Round trip",
+		input:        csvSimple,
+		expected:     expectedUpdatedSimpleCsv,
+		expression:   `(.[] | select(.name == "Gary") | .numberOfCats) = 3`,
+		scenarioType: "roundtrip-csv",
+	},
+}
+
+func testCSVScenario(t *testing.T, s formatScenario) {
+	switch s.scenarioType {
+	case "encode-csv":
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder(',')), s.description)
+	case "encode-tsv":
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder('\t')), s.description)
+	case "decode-csv-object":
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewYamlEncoder(2, false, true, true)), s.description)
+	case "decode-tsv-object":
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder('\t'), NewYamlEncoder(2, false, true, true)), s.description)
+	case "roundtrip-csv":
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewCsvEncoder(',')), s.description)
+	default:
+		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
+	}
+}
+
+func documentCSVDecodeObjectScenario(t *testing.T, w *bufio.Writer, s formatScenario, formatType string) {
+	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
+
+	if s.subdescription != "" {
+		writeOrPanic(w, s.subdescription)
+		writeOrPanic(w, "\n\n")
+	}
+
+	writeOrPanic(w, fmt.Sprintf("Given a sample.%v file of:\n", formatType))
+	writeOrPanic(w, fmt.Sprintf("```%v\n%v\n```\n", formatType, s.input))
+
+	writeOrPanic(w, "then\n")
+	writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=%v sample.%v\n```\n", formatType, formatType))
+	writeOrPanic(w, "will output\n")
+
+	separator := ','
+	if formatType == "tsv" {
+		separator = '\t'
+	}
+
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n",
+		processFormatScenario(s, NewCSVObjectDecoder(separator), NewYamlEncoder(s.indent, false, true, true))),
+	)
+}
+
+func documentCSVEncodeScenario(w *bufio.Writer, s formatScenario, formatType string) {
+	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
+
+	if s.subdescription != "" {
+		writeOrPanic(w, s.subdescription)
+		writeOrPanic(w, "\n\n")
+	}
+
+	writeOrPanic(w, "Given a sample.yml file of:\n")
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input))
+
+	writeOrPanic(w, "then\n")
+
+	expression := s.expression
+
+	if expression != "" {
+		writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=%v '%v' sample.yml\n```\n", formatType, expression))
+	} else {
+		writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=%v sample.yml\n```\n", formatType))
+	}
+	writeOrPanic(w, "will output\n")
+
+	separator := ','
+	if formatType == "tsv" {
+		separator = '\t'
+	}
+
+	writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
+		processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder(separator))),
+	)
+}
+
+func documentCSVRoundTripScenario(w *bufio.Writer, s formatScenario, formatType string) {
+	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
+
+	if s.subdescription != "" {
+		writeOrPanic(w, s.subdescription)
+		writeOrPanic(w, "\n\n")
+	}
+
+	writeOrPanic(w, fmt.Sprintf("Given a sample.%v file of:\n", formatType))
+	writeOrPanic(w, fmt.Sprintf("```%v\n%v\n```\n", formatType, s.input))
+
+	writeOrPanic(w, "then\n")
+
+	expression := s.expression
+
+	if expression != "" {
+		writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=%v -o=%v '%v' sample.%v\n```\n", formatType, formatType, expression, formatType))
+	} else {
+		writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=%v -o=%v sample.%v\n```\n", formatType, formatType, formatType))
+	}
+	writeOrPanic(w, "will output\n")
+
+	separator := ','
+	if formatType == "tsv" {
+		separator = '\t'
+	}
+
+	writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
+		processFormatScenario(s, NewCSVObjectDecoder(separator), NewCsvEncoder(separator))),
+	)
+}
+
+func documentCSVScenario(t *testing.T, w *bufio.Writer, i interface{}) {
+	s := i.(formatScenario)
+	if s.skipDoc {
+		return
+	}
+	switch s.scenarioType {
+	case "encode-csv":
+		documentCSVEncodeScenario(w, s, "csv")
+	case "encode-tsv":
+		documentCSVEncodeScenario(w, s, "tsv")
+	case "decode-csv-object":
+		documentCSVDecodeObjectScenario(t, w, s, "csv")
+	case "decode-tsv-object":
+		documentCSVDecodeObjectScenario(t, w, s, "tsv")
+	case "roundtrip-csv":
+		documentCSVRoundTripScenario(w, s, "csv")
+
+	default:
+		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
+	}
+}
+
+func TestCSVScenarios(t *testing.T) {
+	for _, tt := range csvScenarios {
+		testCSVScenario(t, tt)
+	}
+	genericScenarios := make([]interface{}, len(csvScenarios))
+	for i, s := range csvScenarios {
+		genericScenarios[i] = s
+	}
+	documentScenarios(t, "usage", "csv-tsv", genericScenarios, documentCSVScenario)
+}
@@ -14,6 +14,9 @@ const (
 	XMLInputFormat
 	PropertiesInputFormat
 	Base64InputFormat
+	JsonInputFormat
+	CSVObjectInputFormat
+	TSVObjectInputFormat
 )

 type Decoder interface {
@@ -29,6 +32,12 @@ func InputFormatFromString(format string) (InputFormat, error) {
 		return XMLInputFormat, nil
 	case "props", "p":
 		return PropertiesInputFormat, nil
+	case "json", "ndjson", "j":
+		return JsonInputFormat, nil
+	case "csv", "c":
+		return CSVObjectInputFormat, nil
+	case "tsv", "t":
+		return TSVObjectInputFormat, nil
 	default:
 		return 0, fmt.Errorf("unknown format '%v' please use [yaml|xml|props]", format)
 	}
@@ -0,0 +1,77 @@
+package yqlib
+
+import (
+	"encoding/csv"
+	"errors"
+	"io"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+type csvObjectDecoder struct {
+	separator rune
+	reader    csv.Reader
+	finished  bool
+}
+
+func NewCSVObjectDecoder(separator rune) Decoder {
+	return &csvObjectDecoder{separator: separator}
+}
+
+func (dec *csvObjectDecoder) Init(reader io.Reader) {
+	dec.reader = *csv.NewReader(reader)
+	dec.reader.Comma = dec.separator
+	dec.finished = false
+}
+
+func (dec *csvObjectDecoder) convertToYamlNode(content string) *yaml.Node {
+	node, err := parseSnippet(content)
+	if err != nil {
+		return createScalarNode(content, content)
+	}
+	return node
+}
+
+func (dec *csvObjectDecoder) createObject(headerRow []string, contentRow []string) *yaml.Node {
+	objectNode := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
+
+	for i, header := range headerRow {
+		objectNode.Content = append(
+			objectNode.Content,
+			createScalarNode(header, header),
+			dec.convertToYamlNode(contentRow[i]))
+	}
+	return objectNode
+}
+
+func (dec *csvObjectDecoder) Decode(rootYamlNode *yaml.Node) error {
+	if dec.finished {
+		return io.EOF
+	}
+	headerRow, err := dec.reader.Read()
+	log.Debugf(": headerRow%v", headerRow)
+	if err != nil {
+		return err
+	}
+
+	rootArray := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+
+	contentRow, err := dec.reader.Read()
+
+	for err == nil && len(contentRow) > 0 {
+		log.Debugf("Adding contentRow: %v", contentRow)
+		rootArray.Content = append(rootArray.Content, dec.createObject(headerRow, contentRow))
+		contentRow, err = dec.reader.Read()
+		log.Debugf("Read next contentRow: %v, %v", contentRow, err)
+	}
+	if !errors.Is(err, io.EOF) {
+		return err
+	}
+
+	log.Debugf("finished, contentRow%v", contentRow)
+	log.Debugf("err: %v", err)
+
+	rootYamlNode.Kind = yaml.DocumentNode
+	rootYamlNode.Content = []*yaml.Node{rootArray}
+	return nil
+}
@@ -0,0 +1,85 @@
+package yqlib
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/goccy/go-json"
+	yaml "gopkg.in/yaml.v3"
+)
+
+type jsonDecoder struct {
+	decoder json.Decoder
+}
+
+func NewJSONDecoder() Decoder {
+	return &jsonDecoder{}
+}
+
+func (dec *jsonDecoder) Init(reader io.Reader) {
+	dec.decoder = *json.NewDecoder(reader)
+}
+
+func (dec *jsonDecoder) Decode(rootYamlNode *yaml.Node) error {
+
+	var dataBucket interface{}
+	log.Debug("going to decode")
+	err := dec.decoder.Decode(&dataBucket)
+	if err != nil {
+		return err
+	}
+	node, err := dec.convertToYamlNode(dataBucket)
+
+	if err != nil {
+		return err
+	}
+	rootYamlNode.Kind = yaml.DocumentNode
+	rootYamlNode.Content = []*yaml.Node{node}
+	return nil
+}
+
+func (dec *jsonDecoder) convertToYamlNode(data interface{}) (*yaml.Node, error) {
+	switch data := data.(type) {
+	case nil:
+		return createScalarNode(nil, "null"), nil
+	case float64, float32:
+		// json decoder returns ints as float.
+		return parseSnippet(fmt.Sprintf("%v", data))
+	case int, int64, int32, string, bool:
+		return createScalarNode(data, fmt.Sprintf("%v", data)), nil
+	case map[string]interface{}:
+		return dec.parseMap(data)
+	case []interface{}:
+		return dec.parseArray(data)
+	default:
+		return nil, fmt.Errorf("unrecognised type :(")
+	}
+}
+
+func (dec *jsonDecoder) parseMap(dataMap map[string]interface{}) (*yaml.Node, error) {
+
+	var yamlMap = &yaml.Node{Kind: yaml.MappingNode}
+
+	for key, value := range dataMap {
+		yamlValue, err := dec.convertToYamlNode(value)
+		if err != nil {
+			return nil, err
+		}
+		yamlMap.Content = append(yamlMap.Content, createScalarNode(key, fmt.Sprintf("%v", key)), yamlValue)
+	}
+	return yamlMap, nil
+}
+
+func (dec *jsonDecoder) parseArray(dataArray []interface{}) (*yaml.Node, error) {
+
+	var yamlMap = &yaml.Node{Kind: yaml.SequenceNode}
+
+	for _, value := range dataArray {
+		yamlValue, err := dec.convertToYamlNode(value)
+		if err != nil {
+			return nil, err
+		}
+		yamlMap.Content = append(yamlMap.Content, yamlValue)
+	}
+	return yamlMap, nil
+}
@@ -43,7 +43,7 @@ func (dec *xmlDecoder) Init(reader io.Reader) {
 }

 func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yaml.Node, error) {
-	yamlNode := &yaml.Node{Kind: yaml.SequenceNode}
+	yamlNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
 	for _, child := range nodes {
 		yamlChild, err := dec.convertToYamlNode(child)
 		if err != nil {
@@ -64,7 +64,7 @@ func (dec *xmlDecoder) processComment(c string) string {

 func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
 	log.Debug("createMap: headC: %v, footC: %v", n.HeadComment, n.FootComment)
-	yamlNode := &yaml.Node{Kind: yaml.MappingNode}
+	yamlNode := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}

 	if len(n.Data) > 0 {
 		label := dec.contentName
@@ -11,14 +11,14 @@ These operators are useful to process yaml documents that have stringified embed
 | --- | -- | --|
 | Yaml | from_yaml | to_yaml(i)/@yaml |
 | JSON | from_json | to_json(i)/@json |
-| Properties | from_props  | to_props/@props |
-| CSV |  | to_csv/@csv |
-| TSV |  | to_tsv/@tsv |
+| Properties | from_props/@propsd  | to_props/@props |
+| CSV | from_csv/@csvd | to_csv/@csv |
+| TSV | from_tsv/@tsvd | to_tsv/@tsv |
 | XML | from_xml | to_xml(i)/@xml |
 | Base64 | @base64d | @base64 |


-CSV and TSV format both accept either a single array or scalars (representing a single row), or an array of array of scalars (representing multiple rows). 
+See CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) for accepted formats.

 XML uses the `--xml-attribute-prefix` and `xml-content-name` flags to identify attributes and content fields.

@@ -132,7 +132,7 @@ a: |-
 ```
 then
 ```bash
-yq '.a |= from_props' sample.yml
+yq '.a |= @propsd' sample.yml
 ```
 will output
 ```yaml
@@ -141,6 +141,42 @@ a:
  dogs: cool as well
 ```

+## Decode csv encoded string
+Given a sample.yml file of:
+```yaml
+a: |-
+  cats,dogs
+  great,cool as well
+```
+then
+```bash
+yq '.a |= @csvd' sample.yml
+```
+will output
+```yaml
+a:
+  - cats: great
+    dogs: cool as well
+```
+
+## Decode tsv encoded string
+Given a sample.yml file of:
+```yaml
+a: |-
+  cats	dogs
+  great	cool as well
+```
+then
+```bash
+yq '.a |= @tsvd' sample.yml
+```
+will output
+```yaml
+a:
+  - cats: great
+    dogs: cool as well
+```
+
 ## Encode value as yaml string
 Indent defaults to 2

@@ -0,0 +1,56 @@
+# Error
+
+Use this operation to short-circuit expressions. Useful for validation.
+
+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
+## Validate a particular value
+Given a sample.yml file of:
+```yaml
+a: hello
+```
+then
+```bash
+yq 'select(.a == "howdy") or error(".a [" + .a + "] is not howdy!")' sample.yml
+```
+will output
+```bash
+Error: .a [hello] is not howdy!
+```
+
+## Validate the environment variable is a number - invalid
+Running
+```bash
+numberOfCats="please" yq --null-input 'env(numberOfCats) | select(tag == "!!int") or error("numberOfCats is not a number :(")'
+```
+will output
+```bash
+Error: numberOfCats is not a number :(
+```
+
+## Validate the environment variable is a number - valid
+`with` can be a convenient way of encapsulating validation.
+
+Given a sample.yml file of:
+```yaml
+name: Bob
+favouriteAnimal: cat
+```
+then
+```bash
+numberOfCats="3" yq '
+	with(env(numberOfCats); select(tag == "!!int") or error("numberOfCats is not a number :(")) | 
+	.numPets = env(numberOfCats)
+' sample.yml
+```
+will output
+```yaml
+name: Bob
+favouriteAnimal: cat
+numPets: 3
+```
+
@@ -11,14 +11,14 @@ These operators are useful to process yaml documents that have stringified embed
 | --- | -- | --|
 | Yaml | from_yaml | to_yaml(i)/@yaml |
 | JSON | from_json | to_json(i)/@json |
-| Properties | from_props  | to_props/@props |
-| CSV |  | to_csv/@csv |
-| TSV |  | to_tsv/@tsv |
+| Properties | from_props/@propsd  | to_props/@props |
+| CSV | from_csv/@csvd | to_csv/@csv |
+| TSV | from_tsv/@tsvd | to_tsv/@tsv |
 | XML | from_xml | to_xml(i)/@xml |
 | Base64 | @base64d | @base64 |


-CSV and TSV format both accept either a single array or scalars (representing a single row), or an array of array of scalars (representing multiple rows). 
+See CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) for accepted formats.

 XML uses the `--xml-attribute-prefix` and `xml-content-name` flags to identify attributes and content fields.

@@ -0,0 +1,3 @@
+# Error
+
+Use this operation to short-circuit expressions. Useful for validation.
@@ -1,6 +1,8 @@
 # JSON

-Encode and decode to and from JSON. Note that YAML is a _superset_ of JSON - so `yq` can read any json file without doing anything special.
+Encode and decode to and from JSON. Note that, unless you have multiple JSON documents in a single file, YAML is a _superset_ of JSON - so `yq` can read any json file without doing anything special.
+
+If you do have mulitple JSON documents in a single file (e.g. NDJSON) then you will need to explicity use the json parser `-p=json`.

 This means you don't need to 'convert' a JSON file to YAML - however if you want idiomatic YAML styling, then you can use the `-P/--prettyPrint` flag, see examples below.

@@ -130,3 +132,129 @@ will output
 {"whatever":"cat"}
 ```

+## Roundtrip NDJSON
+Unfortunately the json encoder strips leading spaces of values.
+
+Given a sample.json file of:
+```json
+{"this": "is a multidoc json file"}
+{"each": ["line is a valid json document"]}
+{"a number": 4}
+
+```
+then
+```bash
+yq -p=json -o=json -I=0 sample.json
+```
+will output
+```yaml
+{"this":"is a multidoc json file"}
+{"each":["line is a valid json document"]}
+{"a number":4}
+```
+
+## Roundtrip multi-document JSON
+The NDJSON parser can also handle multiple multi-line json documents in a single file!
+
+Given a sample.json file of:
+```json
+{
+	"this": "is a multidoc json file"
+}
+{
+	"it": [
+		"has",
+		"consecutive",
+		"json documents"
+	]
+}
+{
+	"a number": 4
+}
+
+```
+then
+```bash
+yq -p=json -o=json -I=2 sample.json
+```
+will output
+```yaml
+{
+  "this": "is a multidoc json file"
+}
+{
+  "it": [
+    "has",
+    "consecutive",
+    "json documents"
+  ]
+}
+{
+  "a number": 4
+}
+```
+
+## Update a specific document in a multi-document json
+Documents are indexed by the `documentIndex` or `di` operator.
+
+Given a sample.json file of:
+```json
+{"this": "is a multidoc json file"}
+{"each": ["line is a valid json document"]}
+{"a number": 4}
+
+```
+then
+```bash
+yq -p=json -o=json -I=0 '(select(di == 1) | .each ) += "cool"' sample.json
+```
+will output
+```yaml
+{"this":"is a multidoc json file"}
+{"each":["line is a valid json document","cool"]}
+{"a number":4}
+```
+
+## Find and update a specific document in a multi-document json
+Use expressions as you normally would.
+
+Given a sample.json file of:
+```json
+{"this": "is a multidoc json file"}
+{"each": ["line is a valid json document"]}
+{"a number": 4}
+
+```
+then
+```bash
+yq -p=json -o=json -I=0 '(select(has("each")) | .each ) += "cool"' sample.json
+```
+will output
+```yaml
+{"this":"is a multidoc json file"}
+{"each":["line is a valid json document","cool"]}
+{"a number":4}
+```
+
+## Decode NDJSON
+Given a sample.json file of:
+```json
+{"this": "is a multidoc json file"}
+{"each": ["line is a valid json document"]}
+{"a number": 4}
+
+```
+then
+```bash
+yq -p=json sample.json
+```
+will output
+```yaml
+this: is a multidoc json file
+---
+each:
+    - line is a valid json document
+---
+a number: 4
+```
+
@@ -0,0 +1,214 @@
+# CSV
+Encode/Decode/Roundtrip CSV and TSV files.
+
+## Encode 
+Currently supports arrays of homogenous flat objects, that is: no nesting and it assumes the _first_ object has all the keys required:
+
+```yaml
+- name: Bobo
+  type: dog
+- name: Fifi
+  type: cat
+```
+
+As well as arrays of arrays of scalars (strings/numbers/booleans):
+
+```yaml
+- [Bobo, dog]
+- [Fifi, cat]
+```
+
+## Decode
+Decode assumes the first CSV/TSV row is the header row, and all rows beneath are the entries.
+The data will be coded into an array of objects, using the header rows as keys.
+
+```csv
+name,type
+Bobo,dog
+Fifi,cat
+```
+
+
+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
+## Encode CSV simple
+Given a sample.yml file of:
+```yaml
+- [i, like, csv]
+- [because, excel, is, cool]
+```
+then
+```bash
+yq -o=csv sample.yml
+```
+will output
+```csv
+i,like,csv
+because,excel,is,cool
+```
+
+## Encode TSV simple
+Given a sample.yml file of:
+```yaml
+- [i, like, csv]
+- [because, excel, is, cool]
+```
+then
+```bash
+yq -o=tsv sample.yml
+```
+will output
+```tsv
+i	like	csv
+because	excel	is	cool
+```
+
+## Encode array of objects to csv
+Given a sample.yml file of:
+```yaml
+- name: Gary
+  numberOfCats: 1
+  likesApples: true
+  height: 168.8
+- name: Samantha's Rabbit
+  numberOfCats: 2
+  likesApples: false
+  height: -188.8
+
+```
+then
+```bash
+yq -o=csv sample.yml
+```
+will output
+```csv
+name,numberOfCats,likesApples,height
+Gary,1,true,168.8
+Samantha's Rabbit,2,false,-188.8
+```
+
+## Encode array of objects to custom csv format
+Add the header row manually, then the we convert each object into an array of values - resulting in an array of arrays. Pick the columns and call the header whatever you like.
+
+Given a sample.yml file of:
+```yaml
+- name: Gary
+  numberOfCats: 1
+  likesApples: true
+  height: 168.8
+- name: Samantha's Rabbit
+  numberOfCats: 2
+  likesApples: false
+  height: -188.8
+
+```
+then
+```bash
+yq -o=csv '[["Name", "Number of Cats"]] +  [.[] | [.name, .numberOfCats ]]' sample.yml
+```
+will output
+```csv
+Name,Number of Cats
+Gary,1
+Samantha's Rabbit,2
+```
+
+## Encode array of objects to csv - missing fields behaviour
+First entry is used to determine the headers, and it is missing 'likesApples', so it is not included in the csv. Second entry does not have 'numberOfCats' so that is blank
+
+Given a sample.yml file of:
+```yaml
+- name: Gary
+  numberOfCats: 1
+  height: 168.8
+- name: Samantha's Rabbit
+  height: -188.8
+  likesApples: false
+
+```
+then
+```bash
+yq -o=csv sample.yml
+```
+will output
+```csv
+name,numberOfCats,height
+Gary,1,168.8
+Samantha's Rabbit,,-188.8
+```
+
+## Parse CSV into an array of objects
+First row is assumed to be the header row.
+
+Given a sample.csv file of:
+```csv
+name,numberOfCats,likesApples,height
+Gary,1,true,168.8
+Samantha's Rabbit,2,false,-188.8
+
+```
+then
+```bash
+yq -p=csv sample.csv
+```
+will output
+```yaml
+- name: Gary
+  numberOfCats: 1
+  likesApples: true
+  height: 168.8
+- name: Samantha's Rabbit
+  numberOfCats: 2
+  likesApples: false
+  height: -188.8
+```
+
+## Parse TSV into an array of objects
+First row is assumed to be the header row.
+
+Given a sample.tsv file of:
+```tsv
+name	numberOfCats	likesApples	height
+Gary	1	true	168.8
+Samantha's Rabbit	2	false	-188.8
+
+```
+then
+```bash
+yq -p=tsv sample.tsv
+```
+will output
+```yaml
+- name: Gary
+  numberOfCats: 1
+  likesApples: true
+  height: 168.8
+- name: Samantha's Rabbit
+  numberOfCats: 2
+  likesApples: false
+  height: -188.8
+```
+
+## Round trip
+Given a sample.csv file of:
+```csv
+name,numberOfCats,likesApples,height
+Gary,1,true,168.8
+Samantha's Rabbit,2,false,-188.8
+
+```
+then
+```bash
+yq -p=csv -o=csv '(.[] | select(.name == "Gary") | .numberOfCats) = 3' sample.csv
+```
+will output
+```csv
+name,numberOfCats,likesApples,height
+Gary,3,true,168.8
+Samantha's Rabbit,2,false,-188.8
+```
+
@@ -1,5 +1,7 @@
 # JSON

-Encode and decode to and from JSON. Note that YAML is a _superset_ of JSON - so `yq` can read any json file without doing anything special.
+Encode and decode to and from JSON. Note that, unless you have multiple JSON documents in a single file, YAML is a _superset_ of JSON - so `yq` can read any json file without doing anything special.
+
+If you do have mulitple JSON documents in a single file (e.g. NDJSON) then you will need to explicity use the json parser `-p=json`.

 This means you don't need to 'convert' a JSON file to YAML - however if you want idiomatic YAML styling, then you can use the `-P/--prettyPrint` flag, see examples below.
@@ -0,0 +1,30 @@
+# CSV
+Encode/Decode/Roundtrip CSV and TSV files.
+
+## Encode 
+Currently supports arrays of homogenous flat objects, that is: no nesting and it assumes the _first_ object has all the keys required:
+
+```yaml
+- name: Bobo
+  type: dog
+- name: Fifi
+  type: cat
+```
+
+As well as arrays of arrays of scalars (strings/numbers/booleans):
+
+```yaml
+- [Bobo, dog]
+- [Fifi, cat]
+```
+
+## Decode
+Decode assumes the first CSV/TSV row is the header row, and all rows beneath are the entries.
+The data will be coded into an array of objects, using the header rows as keys.
+
+```csv
+name,type
+Bobo,dog
+Fifi,cat
+```
+
@@ -1,5 +1,5 @@
 # Properties

-Encode to a property file (decode not yet supported). Line comments on value nodes will be copied across.
+Encode/Decode/Roundtrip to/from a property file. Line comments on value nodes will be copied across.

 By default, empty maps and arrays are not encoded - see below for an example on how to encode a value for these.
@@ -1,6 +1,6 @@
 # Properties

-Encode to a property file (decode not yet supported). Line comments on value nodes will be copied across.
+Encode/Decode/Roundtrip to/from a property file. Line comments on value nodes will be copied across.

 By default, empty maps and arrays are not encoded - see below for an example on how to encode a value for these.

@@ -17,7 +17,7 @@ Given a sample.yml file of:
 ```yaml
 # block comments don't come through
 person: # neither do comments on maps
-    name: Mike # comments on values appear
+    name: Mike Wazowski # comments on values appear
    pets: 
    - cat # comments on array values appear
    food: [pizza] # comments on arrays do not
@@ -32,7 +32,36 @@ yq -o=props sample.yml
 will output
 ```properties
 # comments on values appear
-person.name = Mike
+person.name = Mike Wazowski
+
+# comments on array values appear
+person.pets.0 = cat
+person.food.0 = pizza
+```
+
+## Encode properties: scalar encapsulation
+Note that string values with blank characters in them are encapsulated with double quotes
+
+Given a sample.yml file of:
+```yaml
+# block comments don't come through
+person: # neither do comments on maps
+    name: Mike Wazowski # comments on values appear
+    pets: 
+    - cat # comments on array values appear
+    food: [pizza] # comments on arrays do not
+emptyArray: []
+emptyMap: []
+
+```
+then
+```bash
+yq -o=props --unwrapScalar=false sample.yml
+```
+will output
+```properties
+# comments on values appear
+person.name = "Mike Wazowski"

 # comments on array values appear
 person.pets.0 = cat
@@ -44,7 +73,7 @@ Given a sample.yml file of:
 ```yaml
 # block comments don't come through
 person: # neither do comments on maps
-    name: Mike # comments on values appear
+    name: Mike Wazowski # comments on values appear
    pets: 
    - cat # comments on array values appear
    food: [pizza] # comments on arrays do not
@@ -58,7 +87,7 @@ yq -o=props '... comments = ""' sample.yml
 ```
 will output
 ```properties
-person.name = Mike
+person.name = Mike Wazowski
 person.pets.0 = cat
 person.food.0 = pizza
 ```
@@ -70,7 +99,7 @@ Given a sample.yml file of:
 ```yaml
 # block comments don't come through
 person: # neither do comments on maps
-    name: Mike # comments on values appear
+    name: Mike Wazowski # comments on values appear
    pets: 
    - cat # comments on array values appear
    food: [pizza] # comments on arrays do not
@@ -85,7 +114,7 @@ yq -o=props '(.. | select( (tag == "!!map" or tag =="!!seq") and length == 0)) =
 will output
 ```properties
 # comments on values appear
-person.name = Mike
+person.name = Mike Wazowski

 # comments on array values appear
 person.pets.0 = cat
@@ -98,7 +127,7 @@ emptyMap =
 Given a sample.properties file of:
 ```properties
 # comments on values appear
-person.name = Mike
+person.name = Mike Wazowski

 # comments on array values appear
 person.pets.0 = cat
@@ -112,7 +141,7 @@ yq -p=props sample.properties
 will output
 ```yaml
 person:
-    name: Mike # comments on values appear
+    name: Mike Wazowski # comments on values appear
    pets:
        - cat # comments on array values appear
    food:
@@ -123,7 +152,7 @@ person:
 Given a sample.properties file of:
 ```properties
 # comments on values appear
-person.name = Mike
+person.name = Mike Wazowski

 # comments on array values appear
 person.pets.0 = cat
@@ -137,7 +166,7 @@ yq -p=props -o=props '.person.pets.0 = "dog"' sample.properties
 will output
 ```properties
 # comments on values appear
-person.name = Mike
+person.name = Mike Wazowski

 # comments on array values appear
 person.pets.0 = dog
@@ -13,7 +13,7 @@ type csvEncoder struct {
 }

 func NewCsvEncoder(separator rune) Encoder {
-	return &csvEncoder{separator}
+	return &csvEncoder{separator: separator}
 }

 func (e *csvEncoder) CanHandleAliases() bool {
@@ -41,6 +41,67 @@ func (e *csvEncoder) encodeRow(csvWriter *csv.Writer, contents []*yaml.Node) err
 	return csvWriter.Write(stringValues)
 }

+func (e *csvEncoder) encodeArrays(csvWriter *csv.Writer, content []*yaml.Node) error {
+	for i, child := range content {
+
+		if child.Kind != yaml.SequenceNode {
+			return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag)
+		}
+		err := e.encodeRow(csvWriter, child.Content)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (e *csvEncoder) extractHeader(child *yaml.Node) ([]*yaml.Node, error) {
+	if child.Kind != yaml.MappingNode {
+		return nil, fmt.Errorf("csv object encoding only works for arrays of flat objects (string key => string/numbers/boolean value), child[0] is a %v", child.Tag)
+	}
+	mapKeys := getMapKeys(child)
+	return mapKeys.Content, nil
+}
+
+func (e *csvEncoder) createChildRow(child *yaml.Node, headers []*yaml.Node) []*yaml.Node {
+	childRow := make([]*yaml.Node, 0)
+	for _, header := range headers {
+		keyIndex := findKeyInMap(child, header)
+		value := createScalarNode(nil, "")
+		if keyIndex != -1 {
+			value = child.Content[keyIndex+1]
+		}
+		childRow = append(childRow, value)
+	}
+	return childRow
+
+}
+
+func (e *csvEncoder) encodeObjects(csvWriter *csv.Writer, content []*yaml.Node) error {
+	headers, err := e.extractHeader(content[0])
+	if err != nil {
+		return nil
+	}
+
+	err = e.encodeRow(csvWriter, headers)
+	if err != nil {
+		return nil
+	}
+
+	for i, child := range content {
+		if child.Kind != yaml.MappingNode {
+			return fmt.Errorf("csv object encoding only works for arrays of flat objects (string key => string/numbers/boolean value), child[%v] is a %v", i, child.Tag)
+		}
+		row := e.createChildRow(child, headers)
+		err = e.encodeRow(csvWriter, row)
+		if err != nil {
+			return err
+		}
+
+	}
+	return nil
+}
+
 func (e *csvEncoder) Encode(writer io.Writer, originalNode *yaml.Node) error {
 	csvWriter := csv.NewWriter(writer)
 	csvWriter.Comma = e.separator
@@ -56,15 +117,10 @@ func (e *csvEncoder) Encode(writer io.Writer, originalNode *yaml.Node) error {
 		return e.encodeRow(csvWriter, node.Content)
 	}

-	for i, child := range node.Content {
-
-		if child.Kind != yaml.SequenceNode {
-			return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag)
-		}
-		err := e.encodeRow(csvWriter, child.Content)
-		if err != nil {
-			return err
-		}
+	if node.Content[0].Kind == yaml.MappingNode {
+		return e.encodeObjects(csvWriter, node.Content)
 	}
-	return nil
+
+	return e.encodeArrays(csvWriter, node.Content)
+
 }
@@ -1,60 +0,0 @@
-package yqlib
-
-import (
-	"bufio"
-	"bytes"
-	"strings"
-	"testing"
-
-	"github.com/mikefarah/yq/v4/test"
-)
-
-func yamlToCsv(sampleYaml string, separator rune) string {
-	var output bytes.Buffer
-	writer := bufio.NewWriter(&output)
-
-	var jsonEncoder = NewCsvEncoder(separator)
-	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
-	if err != nil {
-		panic(err)
-	}
-	node := inputs.Front().Value.(*CandidateNode).Node
-	err = jsonEncoder.Encode(writer, node)
-	if err != nil {
-		panic(err)
-	}
-	writer.Flush()
-
-	return strings.TrimSuffix(output.String(), "\n")
-}
-
-var sampleYaml = `["apple", apple2, "comma, in, value", "new
-line", 3, 3.40, true, "tab	here"]`
-
-var sampleYamlArray = "[" + sampleYaml + ", [bob, cat, meow, puss]]"
-
-func TestCsvEncoderEmptyArray(t *testing.T) {
-	var actualCsv = yamlToCsv(`[]`, ',')
-	test.AssertResult(t, "", actualCsv)
-}
-
-func TestCsvEncoder(t *testing.T) {
-	var expectedCsv = `apple,apple2,"comma, in, value",new line,3,3.40,true,tab	here`
-
-	var actualCsv = yamlToCsv(sampleYaml, ',')
-	test.AssertResult(t, expectedCsv, actualCsv)
-}
-
-func TestCsvEncoderArrayOfArrays(t *testing.T) {
-	var actualCsv = yamlToCsv(sampleYamlArray, ',')
-	var expectedCsv = "apple,apple2,\"comma, in, value\",new line,3,3.40,true,tab	here\nbob,cat,meow,puss"
-	test.AssertResult(t, expectedCsv, actualCsv)
-}
-
-func TestTsvEncoder(t *testing.T) {
-
-	var expectedCsv = `apple	apple2	comma, in, value	new line	3	3.40	true	"tab	here"`
-
-	var actualCsv = yamlToCsv(sampleYaml, '\t')
-	test.AssertResult(t, expectedCsv, actualCsv)
-}
@@ -2,9 +2,9 @@ package yqlib

 import (
 	"bytes"
-	"encoding/json"
 	"io"

+	"github.com/goccy/go-json"
 	yaml "gopkg.in/yaml.v3"
 )

@@ -28,7 +28,7 @@ func mapKeysToStrings(node *yaml.Node) {
 	}
 }

-func NewJONEncoder(indent int, colorise bool) Encoder {
+func NewJSONEncoder(indent int, colorise bool) Encoder {
 	var indentString = ""

 	for index := 0; index < indent; index++ {
@@ -12,10 +12,13 @@ import (
 )

 type propertiesEncoder struct {
+	unwrapScalar bool
 }

-func NewPropertiesEncoder() Encoder {
-	return &propertiesEncoder{}
+func NewPropertiesEncoder(unwrapScalar bool) Encoder {
+	return &propertiesEncoder{
+		unwrapScalar: unwrapScalar,
+	}
 }

 func (pe *propertiesEncoder) CanHandleAliases() bool {
@@ -75,7 +78,13 @@ func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yaml.Node,
 	p.SetComment(path, headAndLineComment(node))
 	switch node.Kind {
 	case yaml.ScalarNode:
-		_, _, err := p.Set(path, node.Value)
+		var nodeValue string
+		if pe.unwrapScalar || !strings.Contains(node.Value, " ") {
+			nodeValue = node.Value
+		} else {
+			nodeValue = fmt.Sprintf("%q", node.Value)
+		}
+		_, _, err := p.Set(path, nodeValue)
 		return err
 	case yaml.DocumentNode:
 		return pe.doEncode(p, node.Content[0], path)
@@ -9,11 +9,11 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )

-func yamlToProps(sampleYaml string) string {
+func yamlToProps(sampleYaml string, unwrapScalar bool) string {
 	var output bytes.Buffer
 	writer := bufio.NewWriter(&output)

-	var propsEncoder = NewPropertiesEncoder()
+	var propsEncoder = NewPropertiesEncoder(unwrapScalar)
 	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
 		panic(err)
@@ -28,51 +28,100 @@ func yamlToProps(sampleYaml string) string {
 	return strings.TrimSuffix(output.String(), "\n")
 }

-func TestPropertiesEncoderSimple(t *testing.T) {
+func TestPropertiesEncoderSimple_Unwrapped(t *testing.T) {
 	var sampleYaml = `a: 'bob cool'`

 	var expectedProps = `a = bob cool`
-	var actualProps = yamlToProps(sampleYaml)
+	var actualProps = yamlToProps(sampleYaml, true)
 	test.AssertResult(t, expectedProps, actualProps)
 }

-func TestPropertiesEncoderSimpleWithComments(t *testing.T) {
+func TestPropertiesEncoderSimple_Wrapped(t *testing.T) {
+	var sampleYaml = `a: 'bob cool'`
+
+	var expectedProps = `a = "bob cool"`
+	var actualProps = yamlToProps(sampleYaml, false)
+	test.AssertResult(t, expectedProps, actualProps)
+}
+
+func TestPropertiesEncoderSimpleWithComments_Unwrapped(t *testing.T) {
 	var sampleYaml = `a: 'bob cool' # line`

 	var expectedProps = `# line
 a = bob cool`
-	var actualProps = yamlToProps(sampleYaml)
+	var actualProps = yamlToProps(sampleYaml, true)
 	test.AssertResult(t, expectedProps, actualProps)
 }

-func TestPropertiesEncoderDeep(t *testing.T) {
+func TestPropertiesEncoderSimpleWithComments_Wrapped(t *testing.T) {
+	var sampleYaml = `a: 'bob cool' # line`
+
+	var expectedProps = `# line
+a = "bob cool"`
+	var actualProps = yamlToProps(sampleYaml, false)
+	test.AssertResult(t, expectedProps, actualProps)
+}
+
+func TestPropertiesEncoderDeep_Unwrapped(t *testing.T) {
 	var sampleYaml = `a: 
  b: "bob cool"
 `

 	var expectedProps = `a.b = bob cool`
-	var actualProps = yamlToProps(sampleYaml)
+	var actualProps = yamlToProps(sampleYaml, true)
 	test.AssertResult(t, expectedProps, actualProps)
 }

-func TestPropertiesEncoderDeepWithComments(t *testing.T) {
+func TestPropertiesEncoderDeep_Wrapped(t *testing.T) {
+	var sampleYaml = `a: 
+  b: "bob cool"
+`
+
+	var expectedProps = `a.b = "bob cool"`
+	var actualProps = yamlToProps(sampleYaml, false)
+	test.AssertResult(t, expectedProps, actualProps)
+}
+
+func TestPropertiesEncoderDeepWithComments_Unwrapped(t *testing.T) {
 	var sampleYaml = `a:  # a thing
  b: "bob cool" # b thing
 `

 	var expectedProps = `# b thing
 a.b = bob cool`
-	var actualProps = yamlToProps(sampleYaml)
+	var actualProps = yamlToProps(sampleYaml, true)
 	test.AssertResult(t, expectedProps, actualProps)
 }

-func TestPropertiesEncoderArray(t *testing.T) {
+func TestPropertiesEncoderDeepWithComments_Wrapped(t *testing.T) {
+	var sampleYaml = `a:  # a thing
+  b: "bob cool" # b thing
+`
+
+	var expectedProps = `# b thing
+a.b = "bob cool"`
+	var actualProps = yamlToProps(sampleYaml, false)
+	test.AssertResult(t, expectedProps, actualProps)
+}
+
+func TestPropertiesEncoderArray_Unwrapped(t *testing.T) {
 	var sampleYaml = `a: 
  b: [{c: dog}, {c: cat}]
 `

 	var expectedProps = `a.b.0.c = dog
 a.b.1.c = cat`
-	var actualProps = yamlToProps(sampleYaml)
+	var actualProps = yamlToProps(sampleYaml, true)
+	test.AssertResult(t, expectedProps, actualProps)
+}
+
+func TestPropertiesEncoderArray_Wrapped(t *testing.T) {
+	var sampleYaml = `a: 
+  b: [{c: dog named jim}, {c: cat named jam}]
+`
+
+	var expectedProps = `a.b.0.c = "dog named jim"
+a.b.1.c = "cat named jam"`
+	var actualProps = yamlToProps(sampleYaml, false)
 	test.AssertResult(t, expectedProps, actualProps)
 }
@@ -13,7 +13,7 @@ func yamlToJSON(sampleYaml string, indent int) string {
 	var output bytes.Buffer
 	writer := bufio.NewWriter(&output)

-	var jsonEncoder = NewJONEncoder(indent, false)
+	var jsonEncoder = NewJSONEncoder(indent, false)
 	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
 		panic(err)
@@ -21,7 +21,7 @@ type expressionParserImpl struct {
 }

 func newExpressionParser() ExpressionParserInterface {
-	return &expressionParserImpl{newExpressionTokeniser(), newExpressionPostFixer()}
+	return &expressionParserImpl{newParticipleLexer(), newExpressionPostFixer()}
 }

 func (p *expressionParserImpl) ParseExpression(expression string) (*ExpressionNode, error) {
@@ -76,7 +76,7 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
 			// on the close op - move it to the traverse array op
 			// allows for .["cat"]?
 			prefs := traversePreferences{}
-			closeTokenMatch := string(currentToken.Match.Bytes)
+			closeTokenMatch := currentToken.Match
 			if closeTokenMatch[len(closeTokenMatch)-1:] == "?" {
 				prefs.OptionalTraverse = true
 			}
@@ -312,7 +312,7 @@ var pathTests = []struct {
 	},
 }

-var tokeniser = newExpressionTokeniser()
+var tokeniser = newParticipleLexer()
 var postFixer = newExpressionPostFixer()

 func TestPathParsing(t *testing.T) {
@@ -1,689 +0,0 @@
-package yqlib
-
-import (
-	"fmt"
-	"regexp"
-	"strconv"
-	"strings"
-
-	lex "github.com/timtadh/lexmachine"
-	"github.com/timtadh/lexmachine/machines"
-)
-
-func skip(*lex.Scanner, *machines.Match) (interface{}, error) {
-	return nil, nil
-}
-
-type tokenType uint32
-
-const (
-	operationToken = 1 << iota
-	openBracket
-	closeBracket
-	openCollect
-	closeCollect
-	openCollectObject
-	closeCollectObject
-	traverseArrayCollect
-)
-
-type token struct {
-	TokenType            tokenType
-	Operation            *Operation
-	AssignOperation      *Operation      // e.g. tag (GetTag) op becomes AssignTag if '=' follows it
-	CheckForPostTraverse bool            // e.g. [1]cat should really be [1].cat
-	Match                *machines.Match // match that created this token
-
-}
-
-func (t *token) toString(detail bool) string {
-	if t.TokenType == operationToken {
-		if detail {
-			return fmt.Sprintf("%v (%v)", t.Operation.toString(), t.Operation.OperationType.Precedence)
-		}
-		return t.Operation.toString()
-	} else if t.TokenType == openBracket {
-		return "("
-	} else if t.TokenType == closeBracket {
-		return ")"
-	} else if t.TokenType == openCollect {
-		return "["
-	} else if t.TokenType == closeCollect {
-		return "]"
-	} else if t.TokenType == openCollectObject {
-		return "{"
-	} else if t.TokenType == closeCollectObject {
-		return "}"
-	} else if t.TokenType == traverseArrayCollect {
-		return ".["
-
-	} else {
-		return "NFI"
-	}
-}
-
-func pathToken(wrapped bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-		prefs := traversePreferences{}
-
-		if value[len(value)-1:] == "?" {
-			prefs.OptionalTraverse = true
-			value = value[:len(value)-1]
-		}
-
-		value = value[1:]
-		if wrapped {
-			value = unwrap(value)
-		}
-		log.Debug("PathToken %v", value)
-		op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil
-	}
-}
-
-func opToken(op *operationType) lex.Action {
-	return opTokenWithPrefs(op, nil, nil)
-}
-
-func opAssignableToken(opType *operationType, assignOpType *operationType) lex.Action {
-	return opTokenWithPrefs(opType, assignOpType, nil)
-}
-
-func assignOpToken(updateAssign bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		log.Debug("assignOpToken %v", string(m.Bytes))
-		value := string(m.Bytes)
-		prefs := assignPreferences{DontOverWriteAnchor: true}
-		op := &Operation{OperationType: assignOpType, Value: assignOpType.Type, StringValue: value, UpdateAssign: updateAssign, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func multiplyWithPrefs(op *operationType) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		prefs := multiplyPreferences{}
-		options := string(m.Bytes)
-		if strings.Contains(options, "+") {
-			prefs.AppendArrays = true
-		}
-		if strings.Contains(options, "?") {
-			prefs.TraversePrefs = traversePreferences{DontAutoCreate: true}
-		}
-		if strings.Contains(options, "n") {
-			prefs.AssignPrefs = assignPreferences{OnlyWriteNull: true}
-		}
-		if strings.Contains(options, "d") {
-			prefs.DeepMergeArrays = true
-		}
-		prefs.TraversePrefs.DontFollowAlias = true
-		op := &Operation{OperationType: op, Value: multiplyOpType.Type, StringValue: options, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func opTokenWithPrefs(op *operationType, assignOpType *operationType, preferences interface{}) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		log.Debug("opTokenWithPrefs %v", string(m.Bytes))
-		value := string(m.Bytes)
-		op := &Operation{OperationType: op, Value: op.Type, StringValue: value, Preferences: preferences}
-		var assign *Operation
-		if assignOpType != nil {
-			assign = &Operation{OperationType: assignOpType, Value: assignOpType.Type, StringValue: value, Preferences: preferences}
-		}
-		return &token{TokenType: operationToken, Operation: op, AssignOperation: assign}, nil
-	}
-}
-
-func hasOptionParameter(value string, option string) bool {
-	parameterParser := regexp.MustCompile(`.*\([^\)]*\)`)
-	matches := parameterParser.FindStringSubmatch(value)
-	if len(matches) == 0 {
-		return false
-	}
-	parameterString := matches[0]
-	optionParser := regexp.MustCompile(fmt.Sprintf("\\b%v\\b", option))
-	return len(optionParser.FindStringSubmatch(parameterString)) > 0
-}
-
-func extractNumberParameter(value string) (int, error) {
-	parameterParser := regexp.MustCompile(`.*\(([0-9]+)\)`)
-	matches := parameterParser.FindStringSubmatch(value)
-	var indent, errParsingInt = strconv.ParseInt(matches[1], 10, 32)
-	if errParsingInt != nil {
-		return 0, errParsingInt
-	}
-	return int(indent), nil
-}
-
-func envSubstWithOptions() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-		noEmpty := hasOptionParameter(value, "ne")
-		noUnset := hasOptionParameter(value, "nu")
-		failFast := hasOptionParameter(value, "ff")
-		envsubstOpType.Type = "ENVSUBST"
-		prefs := envOpPreferences{NoUnset: noUnset, NoEmpty: noEmpty, FailFast: failFast}
-		if noEmpty {
-			envsubstOpType.Type = envsubstOpType.Type + "_NO_EMPTY"
-		}
-		if noUnset {
-			envsubstOpType.Type = envsubstOpType.Type + "_NO_UNSET"
-		}
-
-		op := &Operation{OperationType: envsubstOpType, Value: envsubstOpType.Type, StringValue: value, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func flattenWithDepth() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-		var depth, errParsingInt = extractNumberParameter(value)
-		if errParsingInt != nil {
-			return nil, errParsingInt
-		}
-
-		prefs := flattenPreferences{depth: depth}
-		op := &Operation{OperationType: flattenOpType, Value: flattenOpType.Type, StringValue: value, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func encodeWithIndent(outputFormat PrinterOutputFormat) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-		var indent, errParsingInt = extractNumberParameter(value)
-		if errParsingInt != nil {
-			return nil, errParsingInt
-		}
-
-		prefs := encoderPreferences{format: outputFormat, indent: indent}
-		op := &Operation{OperationType: encodeOpType, Value: encodeOpType.Type, StringValue: value, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func assignAllCommentsOp(updateAssign bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		log.Debug("assignAllCommentsOp %v", string(m.Bytes))
-		value := string(m.Bytes)
-		op := &Operation{
-			OperationType: assignCommentOpType,
-			Value:         assignCommentOpType.Type,
-			StringValue:   value,
-			UpdateAssign:  updateAssign,
-			Preferences:   commentOpPreferences{LineComment: true, HeadComment: true, FootComment: true},
-		}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func literalToken(pType tokenType, checkForPost bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		return &token{TokenType: pType, CheckForPostTraverse: checkForPost, Match: m}, nil
-	}
-}
-
-func unwrap(value string) string {
-	return value[1 : len(value)-1]
-}
-
-func numberValue() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		var numberString = string(m.Bytes)
-		var number, errParsingInt = strconv.ParseInt(numberString, 10, 64)
-		if errParsingInt != nil {
-			return nil, errParsingInt
-		}
-
-		return &token{TokenType: operationToken, Operation: createValueOperation(number, numberString)}, nil
-	}
-}
-
-func hexValue() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		var originalString = string(m.Bytes)
-		var numberString = originalString[2:]
-		log.Debugf("numberString: %v", numberString)
-		var number, errParsingInt = strconv.ParseInt(numberString, 16, 64)
-		if errParsingInt != nil {
-			return nil, errParsingInt
-		}
-
-		return &token{TokenType: operationToken, Operation: createValueOperation(number, originalString)}, nil
-	}
-}
-
-func floatValue() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		var numberString = string(m.Bytes)
-		var number, errParsingInt = strconv.ParseFloat(numberString, 64)
-		if errParsingInt != nil {
-			return nil, errParsingInt
-		}
-		return &token{TokenType: operationToken, Operation: createValueOperation(number, numberString)}, nil
-	}
-}
-
-func booleanValue(val bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		return &token{TokenType: operationToken, Operation: createValueOperation(val, string(m.Bytes))}, nil
-	}
-}
-
-func stringValue(wrapped bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-		if wrapped {
-			value = unwrap(value)
-		}
-		value = strings.ReplaceAll(value, "\\\"", "\"")
-		return &token{TokenType: operationToken, Operation: createValueOperation(value, value)}, nil
-	}
-}
-
-func getVariableOpToken() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-
-		value = value[1:]
-
-		getVarOperation := createValueOperation(value, value)
-		getVarOperation.OperationType = getVariableOpType
-
-		return &token{TokenType: operationToken, Operation: getVarOperation, CheckForPostTraverse: true}, nil
-	}
-}
-
-func envOp(strenv bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-		preferences := envOpPreferences{}
-
-		if strenv {
-			// strenv( )
-			value = value[7 : len(value)-1]
-			preferences.StringValue = true
-		} else {
-			//env( )
-			value = value[4 : len(value)-1]
-		}
-
-		envOperation := createValueOperation(value, value)
-		envOperation.OperationType = envOpType
-		envOperation.Preferences = preferences
-
-		return &token{TokenType: operationToken, Operation: envOperation}, nil
-	}
-}
-
-func nullValue() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		return &token{TokenType: operationToken, Operation: createValueOperation(nil, string(m.Bytes))}, nil
-	}
-}
-
-func selfToken() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		op := &Operation{OperationType: selfReferenceOpType}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func initLexer() (*lex.Lexer, error) {
-	lexer := lex.NewLexer()
-	lexer.Add([]byte(`\(`), literalToken(openBracket, false))
-	lexer.Add([]byte(`\)`), literalToken(closeBracket, true))
-
-	lexer.Add([]byte(`\.\[`), literalToken(traverseArrayCollect, false))
-	lexer.Add([]byte(`\.\.`), opTokenWithPrefs(recursiveDescentOpType, nil, recursiveDescentPreferences{RecurseArray: true,
-		TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: false}}))
-
-	lexer.Add([]byte(`\.\.\.`), opTokenWithPrefs(recursiveDescentOpType, nil, recursiveDescentPreferences{RecurseArray: true,
-		TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: true}}))
-
-	lexer.Add([]byte(`,`), opToken(unionOpType))
-	lexer.Add([]byte(`:\s*`), opToken(createMapOpType))
-	lexer.Add([]byte(`length`), opToken(lengthOpType))
-	lexer.Add([]byte(`line`), opToken(lineOpType))
-	lexer.Add([]byte(`column`), opToken(columnOpType))
-
-	lexer.Add([]byte(`eval`), opToken(evalOpType))
-
-	lexer.Add([]byte(`map`), opToken(mapOpType))
-	lexer.Add([]byte(`map_values`), opToken(mapValuesOpType))
-	lexer.Add([]byte(`pick`), opToken(pickOpType))
-
-	lexer.Add([]byte(`flatten\([0-9]+\)`), flattenWithDepth())
-	lexer.Add([]byte(`flatten`), opTokenWithPrefs(flattenOpType, nil, flattenPreferences{depth: -1}))
-
-	lexer.Add([]byte(`format_datetime`), opToken(formatDateTimeOpType))
-	lexer.Add([]byte(`now`), opToken(nowOpType))
-	lexer.Add([]byte(`tz`), opToken(tzOpType))
-	lexer.Add([]byte(`with_dtf`), opToken(withDtFormatOpType))
-
-	lexer.Add([]byte(`toyaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat))
-	lexer.Add([]byte(`to_yaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat))
-
-	lexer.Add([]byte(`toxml\([0-9]+\)`), encodeWithIndent(XMLOutputFormat))
-	lexer.Add([]byte(`to_xml\([0-9]+\)`), encodeWithIndent(XMLOutputFormat))
-
-	lexer.Add([]byte(`tojson\([0-9]+\)`), encodeWithIndent(JSONOutputFormat))
-	lexer.Add([]byte(`to_json\([0-9]+\)`), encodeWithIndent(JSONOutputFormat))
-
-	lexer.Add([]byte(`toyaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2}))
-	lexer.Add([]byte(`to_yaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2}))
-	// 0 indent doesn't work with yaml.
-	lexer.Add([]byte(`@yaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2}))
-
-	lexer.Add([]byte(`tojson`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JSONOutputFormat, indent: 2}))
-	lexer.Add([]byte(`to_json`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JSONOutputFormat, indent: 2}))
-	lexer.Add([]byte(`@json`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JSONOutputFormat, indent: 0}))
-
-	lexer.Add([]byte(`toprops`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2}))
-	lexer.Add([]byte(`to_props`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2}))
-	lexer.Add([]byte(`@props`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2}))
-
-	lexer.Add([]byte(`tocsv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: CSVOutputFormat}))
-	lexer.Add([]byte(`to_csv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: CSVOutputFormat}))
-	lexer.Add([]byte(`@csv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: CSVOutputFormat}))
-
-	lexer.Add([]byte(`totsv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: TSVOutputFormat}))
-	lexer.Add([]byte(`to_tsv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: TSVOutputFormat}))
-	lexer.Add([]byte(`@tsv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: TSVOutputFormat}))
-
-	lexer.Add([]byte(`toxml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: XMLOutputFormat}))
-	lexer.Add([]byte(`to_xml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: XMLOutputFormat, indent: 2}))
-	lexer.Add([]byte(`@xml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: XMLOutputFormat, indent: 0}))
-
-	lexer.Add([]byte(`@base64`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: Base64OutputFormat}))
-	lexer.Add([]byte(`@base64d`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: Base64InputFormat}))
-
-	lexer.Add([]byte(`fromyaml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat}))
-	lexer.Add([]byte(`fromjson`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat}))
-	lexer.Add([]byte(`fromxml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: XMLInputFormat}))
-	lexer.Add([]byte(`fromprops`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: PropertiesInputFormat}))
-
-	lexer.Add([]byte(`from_yaml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat}))
-	lexer.Add([]byte(`from_json`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat}))
-	lexer.Add([]byte(`from_xml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: XMLInputFormat}))
-	lexer.Add([]byte(`from_props`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: PropertiesInputFormat}))
-
-	lexer.Add([]byte(`@yamld`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat}))
-	lexer.Add([]byte(`@jsond`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat}))
-	lexer.Add([]byte(`@xmld`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: XMLInputFormat}))
-	lexer.Add([]byte(`@propsd`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: PropertiesInputFormat}))
-
-	lexer.Add([]byte(`sortKeys`), opToken(sortKeysOpType))
-	lexer.Add([]byte(`sort_keys`), opToken(sortKeysOpType))
-
-	lexer.Add([]byte(`load`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewYamlDecoder()}))
-
-	lexer.Add([]byte(`xmlload`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName, XMLPreferences.StrictMode, XMLPreferences.KeepNamespace, XMLPreferences.UseRawToken)}))
-	lexer.Add([]byte(`load_xml`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName, XMLPreferences.StrictMode, XMLPreferences.KeepNamespace, XMLPreferences.UseRawToken)}))
-	lexer.Add([]byte(`loadxml`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName, XMLPreferences.StrictMode, XMLPreferences.KeepNamespace, XMLPreferences.UseRawToken)}))
-
-	lexer.Add([]byte(`load_base64`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewBase64Decoder()}))
-
-	lexer.Add([]byte(`load_props`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewPropertiesDecoder()}))
-	lexer.Add([]byte(`loadprops`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewPropertiesDecoder()}))
-
-	lexer.Add([]byte(`strload`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: true}))
-	lexer.Add([]byte(`load_str`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: true}))
-	lexer.Add([]byte(`loadstr`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: true}))
-
-	lexer.Add([]byte(`select`), opToken(selectOpType))
-	lexer.Add([]byte(`has`), opToken(hasOpType))
-	lexer.Add([]byte(`unique`), opToken(uniqueOpType))
-	lexer.Add([]byte(`unique_by`), opToken(uniqueByOpType))
-	lexer.Add([]byte(`group_by`), opToken(groupByOpType))
-	lexer.Add([]byte(`explode`), opToken(explodeOpType))
-	lexer.Add([]byte(`or`), opToken(orOpType))
-	lexer.Add([]byte(`and`), opToken(andOpType))
-	lexer.Add([]byte(`not`), opToken(notOpType))
-	lexer.Add([]byte(`ireduce`), opToken(reduceOpType))
-	lexer.Add([]byte(`;`), opToken(blockOpType))
-	lexer.Add([]byte(`\/\/`), opToken(alternativeOpType))
-
-	lexer.Add([]byte(`documentIndex`), opToken(getDocumentIndexOpType))
-	lexer.Add([]byte(`document_index`), opToken(getDocumentIndexOpType))
-
-	lexer.Add([]byte(`di`), opToken(getDocumentIndexOpType))
-
-	lexer.Add([]byte(`splitDoc`), opToken(splitDocumentOpType))
-	lexer.Add([]byte(`split_doc`), opToken(splitDocumentOpType))
-
-	lexer.Add([]byte(`join`), opToken(joinStringOpType))
-	lexer.Add([]byte(`sub`), opToken(subStringOpType))
-	lexer.Add([]byte(`match`), opToken(matchOpType))
-	lexer.Add([]byte(`capture`), opToken(captureOpType))
-	lexer.Add([]byte(`test`), opToken(testOpType))
-
-	lexer.Add([]byte(`upcase`), opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: true}))
-	lexer.Add([]byte(`ascii_upcase`), opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: true}))
-
-	lexer.Add([]byte(`downcase`), opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: false}))
-	lexer.Add([]byte(`ascii_downcase`), opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: false}))
-
-	lexer.Add([]byte(`sort`), opToken(sortOpType))
-	lexer.Add([]byte(`sort_by`), opToken(sortByOpType))
-	lexer.Add([]byte(`reverse`), opToken(reverseOpType))
-
-	lexer.Add([]byte(`any`), opToken(anyOpType))
-	lexer.Add([]byte(`any_c`), opToken(anyConditionOpType))
-	lexer.Add([]byte(`all`), opToken(allOpType))
-	lexer.Add([]byte(`all_c`), opToken(allConditionOpType))
-	lexer.Add([]byte(`contains`), opToken(containsOpType))
-
-	lexer.Add([]byte(`split`), opToken(splitStringOpType))
-
-	lexer.Add([]byte(`parent`), opToken(getParentOpType))
-	lexer.Add([]byte(`key`), opToken(getKeyOpType))
-	lexer.Add([]byte(`keys`), opToken(keysOpType))
-
-	lexer.Add([]byte(`style`), opAssignableToken(getStyleOpType, assignStyleOpType))
-
-	lexer.Add([]byte(`tag|type`), opAssignableToken(getTagOpType, assignTagOpType))
-
-	lexer.Add([]byte(`anchor`), opAssignableToken(getAnchorOpType, assignAnchorOpType))
-	lexer.Add([]byte(`alias`), opAssignableToken(getAliasOptype, assignAliasOpType))
-	lexer.Add([]byte(`filename`), opToken(getFilenameOpType))
-
-	lexer.Add([]byte(`fileIndex`), opToken(getFileIndexOpType))
-	lexer.Add([]byte(`file_index`), opToken(getFileIndexOpType))
-
-	lexer.Add([]byte(`fi`), opToken(getFileIndexOpType))
-	lexer.Add([]byte(`path`), opToken(getPathOpType))
-	lexer.Add([]byte(`to_entries`), opToken(toEntriesOpType))
-	lexer.Add([]byte(`from_entries`), opToken(fromEntriesOpType))
-	lexer.Add([]byte(`with_entries`), opToken(withEntriesOpType))
-
-	lexer.Add([]byte(`with`), opToken(withOpType))
-
-	lexer.Add([]byte(`lineComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true}))
-	lexer.Add([]byte(`line_comment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true}))
-
-	lexer.Add([]byte(`headComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{HeadComment: true}))
-	lexer.Add([]byte(`head_comment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{HeadComment: true}))
-
-	lexer.Add([]byte(`footComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{FootComment: true}))
-	lexer.Add([]byte(`foot_comment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{FootComment: true}))
-
-	lexer.Add([]byte(`comments\s*=`), assignAllCommentsOp(false))
-	lexer.Add([]byte(`comments\s*\|=`), assignAllCommentsOp(true))
-
-	lexer.Add([]byte(`collect`), opToken(collectOpType))
-
-	lexer.Add([]byte(`\s*==\s*`), opToken(equalsOpType))
-	lexer.Add([]byte(`\s*!=\s*`), opToken(notEqualsOpType))
-
-	lexer.Add([]byte(`\s*>=\s*`), opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: true, Greater: true}))
-	lexer.Add([]byte(`\s*>\s*`), opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: false, Greater: true}))
-
-	lexer.Add([]byte(`\s*<=\s*`), opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: true, Greater: false}))
-	lexer.Add([]byte(`\s*<\s*`), opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: false, Greater: false}))
-
-	lexer.Add([]byte(`\s*=\s*`), assignOpToken(false))
-
-	lexer.Add([]byte(`del`), opToken(deleteChildOpType))
-
-	lexer.Add([]byte(`\s*\|=\s*`), assignOpToken(true))
-
-	lexer.Add([]byte("( |\t|\n|\r)+"), skip)
-
-	lexer.Add([]byte(`\."[^ "]+"\??`), pathToken(true))
-	lexer.Add([]byte(`\.[^ ;\}\{\:\[\],\|\.\[\(\)=\n]+\??`), pathToken(false))
-	lexer.Add([]byte(`\.`), selfToken())
-
-	lexer.Add([]byte(`\|`), opToken(pipeOpType))
-
-	lexer.Add([]byte(`0[xX][0-9A-Fa-f]+`), hexValue())
-	lexer.Add([]byte(`-?\d+(\.\d+)`), floatValue())
-	lexer.Add([]byte(`-?[1-9](\.\d+)?[Ee][-+]?\d+`), floatValue())
-	lexer.Add([]byte(`-?\d+`), numberValue())
-
-	lexer.Add([]byte(`[Tt][Rr][Uu][Ee]`), booleanValue(true))
-	lexer.Add([]byte(`[Ff][Aa][Ll][Ss][Ee]`), booleanValue(false))
-
-	lexer.Add([]byte(`[Nn][Uu][Ll][Ll]`), nullValue())
-	lexer.Add([]byte(`~`), nullValue())
-
-	lexer.Add([]byte(`"([^"\\]*(\\.[^"\\]*)*)"`), stringValue(true))
-	lexer.Add([]byte(`strenv\([^\)]+\)`), envOp(true))
-	lexer.Add([]byte(`env\([^\)]+\)`), envOp(false))
-
-	lexer.Add([]byte(`envsubst\((ne|nu|ff| |,)+\)`), envSubstWithOptions())
-	lexer.Add([]byte(`envsubst`), opToken(envsubstOpType))
-
-	lexer.Add([]byte(`\[`), literalToken(openCollect, false))
-	lexer.Add([]byte(`\]\??`), literalToken(closeCollect, true))
-	lexer.Add([]byte(`\{`), literalToken(openCollectObject, false))
-	lexer.Add([]byte(`\}`), literalToken(closeCollectObject, true))
-	lexer.Add([]byte(`\*=[\+|\?dn]*`), multiplyWithPrefs(multiplyAssignOpType))
-	lexer.Add([]byte(`\*[\+|\?dn]*`), multiplyWithPrefs(multiplyOpType))
-
-	lexer.Add([]byte(`\+`), opToken(addOpType))
-	lexer.Add([]byte(`\+=`), opToken(addAssignOpType))
-
-	lexer.Add([]byte(`\-`), opToken(subtractOpType))
-	lexer.Add([]byte(`\-=`), opToken(subtractAssignOpType))
-	lexer.Add([]byte(`\$[a-zA-Z_-0-9]+`), getVariableOpToken())
-	lexer.Add([]byte(`as`), opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{}))
-	lexer.Add([]byte(`ref`), opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{IsReference: true}))
-
-	err := lexer.CompileNFA()
-	if err != nil {
-		return nil, err
-	}
-	return lexer, nil
-}
-
-type expressionTokeniser interface {
-	Tokenise(expression string) ([]*token, error)
-}
-
-type expressionTokeniserImpl struct {
-	lexer *lex.Lexer
-}
-
-func newExpressionTokeniser() expressionTokeniser {
-	var lexer, err = initLexer()
-	if err != nil {
-		panic(err)
-	}
-	return &expressionTokeniserImpl{lexer}
-}
-
-func (p *expressionTokeniserImpl) Tokenise(expression string) ([]*token, error) {
-	scanner, err := p.lexer.Scanner([]byte(expression))
-
-	if err != nil {
-		return nil, fmt.Errorf("parsing expression: %w", err)
-	}
-	var tokens []*token
-	for tok, err, eof := scanner.Next(); !eof; tok, err, eof = scanner.Next() {
-
-		if tok != nil {
-			currentToken := tok.(*token)
-			log.Debugf("Tokenising %v", currentToken.toString(true))
-			tokens = append(tokens, currentToken)
-		}
-		if err != nil {
-			return nil, fmt.Errorf("parsing expression: %w", err)
-		}
-	}
-	var postProcessedTokens = make([]*token, 0)
-
-	skipNextToken := false
-
-	for index := range tokens {
-		if skipNextToken {
-			skipNextToken = false
-		} else {
-			postProcessedTokens, skipNextToken = p.handleToken(tokens, index, postProcessedTokens)
-		}
-	}
-
-	return postProcessedTokens, nil
-}
-
-func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postProcessedTokens []*token) (tokensAccum []*token, skipNextToken bool) {
-	skipNextToken = false
-	currentToken := tokens[index]
-
-	log.Debug("processing %v", currentToken.toString(true))
-
-	if currentToken.TokenType == traverseArrayCollect {
-		// `.[exp]`` works by creating a traversal array of [self, exp] and piping that into the traverse array operator
-		//need to put a traverse array then a collect currentToken
-		// do this by adding traverse then converting currentToken to collect
-
-		log.Debug("  adding self")
-		op := &Operation{OperationType: selfReferenceOpType, StringValue: "SELF"}
-		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
-
-		log.Debug("  adding traverse array")
-		op = &Operation{OperationType: traverseArrayOpType, StringValue: "TRAVERSE_ARRAY"}
-		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
-
-		currentToken = &token{TokenType: openCollect}
-
-	}
-
-	if index != len(tokens)-1 && currentToken.AssignOperation != nil &&
-		tokens[index+1].TokenType == operationToken &&
-		tokens[index+1].Operation.OperationType == assignOpType {
-		log.Debug("  its an update assign")
-		currentToken.Operation = currentToken.AssignOperation
-		currentToken.Operation.UpdateAssign = tokens[index+1].Operation.UpdateAssign
-		skipNextToken = true
-	}
-
-	log.Debug("  adding token to the fixed list")
-	postProcessedTokens = append(postProcessedTokens, currentToken)
-
-	if index != len(tokens)-1 &&
-		((currentToken.TokenType == openCollect && tokens[index+1].TokenType == closeCollect) ||
-			(currentToken.TokenType == openCollectObject && tokens[index+1].TokenType == closeCollectObject)) {
-		log.Debug("  adding empty")
-		op := &Operation{OperationType: emptyOpType, StringValue: "EMPTY"}
-		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
-	}
-
-	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
-		((tokens[index+1].TokenType == operationToken && (tokens[index+1].Operation.OperationType == traversePathOpType)) ||
-			(tokens[index+1].TokenType == traverseArrayCollect)) {
-		log.Debug("  adding pipe because the next thing is traverse")
-		op := &Operation{OperationType: shortPipeOpType, Value: "PIPE", StringValue: "."}
-		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
-	}
-	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
-		tokens[index+1].TokenType == openCollect {
-
-		log.Debug("  adding traverArray because next is opencollect")
-		op := &Operation{OperationType: traverseArrayOpType}
-		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
-	}
-	return postProcessedTokens, skipNextToken
-}
@@ -9,7 +9,7 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )

-var complexExpectYaml = `D0, P[], (!!map)::a: Easy! as one two three
+const complexExpectYaml = `D0, P[], (!!map)::a: Easy! as one two three
 b:
    c: 2
    d:
@@ -17,6 +17,59 @@ b:
        - 4
 `

+const sampleNdJson = `{"this": "is a multidoc json file"}
+{"each": ["line is a valid json document"]}
+{"a number": 4}
+`
+
+const expectedNdJsonYaml = `this: is a multidoc json file
+---
+each:
+  - line is a valid json document
+---
+a number: 4
+`
+
+const expectedRoundTripSampleNdJson = `{"this":"is a multidoc json file"}
+{"each":["line is a valid json document"]}
+{"a number":4}
+`
+
+const expectedUpdatedMultilineJson = `{"this":"is a multidoc json file"}
+{"each":["line is a valid json document","cool"]}
+{"a number":4}
+`
+
+const sampleMultiLineJson = `{
+	"this": "is a multidoc json file"
+}
+{
+	"it": [
+		"has",
+		"consecutive",
+		"json documents"
+	]
+}
+{
+	"a number": 4
+}
+`
+
+const roundTripMultiLineJson = `{
+  "this": "is a multidoc json file"
+}
+{
+  "it": [
+    "has",
+    "consecutive",
+    "json documents"
+  ]
+}
+{
+  "a number": 4
+}
+`
+
 var jsonScenarios = []formatScenario{
 	{
 		description:    "Parse json: simple",
@@ -68,6 +121,120 @@ var jsonScenarios = []formatScenario{
 		expected:       "{\"stuff\":\"cool\"}\n{\"whatever\":\"cat\"}\n",
 		scenarioType:   "encode",
 	},
+	{
+		description:    "Roundtrip NDJSON",
+		subdescription: "Unfortunately the json encoder strips leading spaces of values.",
+		input:          sampleNdJson,
+		expected:       expectedRoundTripSampleNdJson,
+		scenarioType:   "roundtrip-ndjson",
+	},
+	{
+		description:    "Roundtrip multi-document JSON",
+		subdescription: "The NDJSON parser can also handle multiple multi-line json documents in a single file!",
+		input:          sampleMultiLineJson,
+		expected:       roundTripMultiLineJson,
+		scenarioType:   "roundtrip-multi",
+	},
+	{
+		description:    "Update a specific document in a multi-document json",
+		subdescription: "Documents are indexed by the `documentIndex` or `di` operator.",
+		input:          sampleNdJson,
+		expected:       expectedUpdatedMultilineJson,
+		expression:     `(select(di == 1) | .each ) += "cool"`,
+		scenarioType:   "roundtrip-ndjson",
+	},
+	{
+		description:    "Find and update a specific document in a multi-document json",
+		subdescription: "Use expressions as you normally would.",
+		input:          sampleNdJson,
+		expected:       expectedUpdatedMultilineJson,
+		expression:     `(select(has("each")) | .each ) += "cool"`,
+		scenarioType:   "roundtrip-ndjson",
+	},
+	{
+		description:  "Decode NDJSON",
+		input:        sampleNdJson,
+		expected:     expectedNdJsonYaml,
+		scenarioType: "decode-ndjson",
+	},
+	{
+		description:  "numbers",
+		skipDoc:      true,
+		input:        "[3, 3.0, 3.1, -1]",
+		expected:     "- 3\n- 3\n- 3.1\n- -1\n",
+		scenarioType: "decode-ndjson",
+	},
+	{
+		description:  "strings",
+		skipDoc:      true,
+		input:        `["", "cat"]`,
+		expected:     "- \"\"\n- cat\n",
+		scenarioType: "decode-ndjson",
+	},
+	{
+		description:  "null",
+		skipDoc:      true,
+		input:        `null`,
+		expected:     "null\n",
+		scenarioType: "decode-ndjson",
+	},
+	{
+		description:  "booleans",
+		skipDoc:      true,
+		input:        `[true, false]`,
+		expected:     "- true\n- false\n",
+		scenarioType: "decode-ndjson",
+	},
+}
+
+func documentRoundtripNdJsonScenario(w *bufio.Writer, s formatScenario, indent int) {
+	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
+
+	if s.subdescription != "" {
+		writeOrPanic(w, s.subdescription)
+		writeOrPanic(w, "\n\n")
+	}
+
+	writeOrPanic(w, "Given a sample.json file of:\n")
+	writeOrPanic(w, fmt.Sprintf("```json\n%v\n```\n", s.input))
+
+	writeOrPanic(w, "then\n")
+
+	expression := s.expression
+	if expression != "" {
+		writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=json -o=json -I=%v '%v' sample.json\n```\n", indent, expression))
+	} else {
+		writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=json -o=json -I=%v sample.json\n```\n", indent))
+	}
+
+	writeOrPanic(w, "will output\n")
+
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(indent, false))))
+}
+
+func documentDecodeNdJsonScenario(w *bufio.Writer, s formatScenario) {
+	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
+
+	if s.subdescription != "" {
+		writeOrPanic(w, s.subdescription)
+		writeOrPanic(w, "\n\n")
+	}
+
+	writeOrPanic(w, "Given a sample.json file of:\n")
+	writeOrPanic(w, fmt.Sprintf("```json\n%v\n```\n", s.input))
+
+	writeOrPanic(w, "then\n")
+
+	expression := s.expression
+	if expression != "" {
+		writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=json '%v' sample.json\n```\n", expression))
+	} else {
+		writeOrPanic(w, "```bash\nyq -p=json sample.json\n```\n")
+	}
+
+	writeOrPanic(w, "will output\n")
+
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(s.indent, false, true, true))))
 }

 func decodeJSON(t *testing.T, jsonString string) *CandidateNode {
@@ -96,11 +263,21 @@ func decodeJSON(t *testing.T, jsonString string) *CandidateNode {
 }

 func testJSONScenario(t *testing.T, s formatScenario) {
-	if s.scenarioType == "encode" || s.scenarioType == "roundtrip" {
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewJONEncoder(s.indent, false)), s.description)
-	} else {
+	switch s.scenarioType {
+	case "encode", "decode":
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewJSONEncoder(s.indent, false)), s.description)
+	case "":
 		var actual = resultToString(t, decodeJSON(t, s.input))
 		test.AssertResultWithContext(t, s.expected, actual, s.description)
+	case "decode-ndjson":
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(2, false, true, true)), s.description)
+	case "roundtrip-ndjson":
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(0, false)), s.description)
+	case "roundtrip-multi":
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(2, false)), s.description)
+
+	default:
+		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
 	}
 }

@@ -135,14 +312,23 @@ func documentJSONDecodeScenario(t *testing.T, w *bufio.Writer, s formatScenario)

 func documentJSONScenario(t *testing.T, w *bufio.Writer, i interface{}) {
 	s := i.(formatScenario)
-
 	if s.skipDoc {
 		return
 	}
-	if s.scenarioType == "encode" {
-		documentJSONEncodeScenario(w, s)
-	} else {
+	switch s.scenarioType {
+	case "":
 		documentJSONDecodeScenario(t, w, s)
+	case "encode":
+		documentJSONEncodeScenario(w, s)
+	case "decode-ndjson":
+		documentDecodeNdJsonScenario(w, s)
+	case "roundtrip-ndjson":
+		documentRoundtripNdJsonScenario(w, s, 0)
+	case "roundtrip-multi":
+		documentRoundtripNdJsonScenario(w, s, 2)
+
+	default:
+		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
 	}
 }

@@ -171,7 +357,7 @@ func documentJSONEncodeScenario(w *bufio.Writer, s formatScenario) {
 	}
 	writeOrPanic(w, "will output\n")

-	writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewJONEncoder(s.indent, false))))
+	writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewJSONEncoder(s.indent, false))))
 }

 func TestJSONScenarios(t *testing.T) {
@@ -0,0 +1,159 @@
+package yqlib
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+)
+
+type expressionTokeniser interface {
+	Tokenise(expression string) ([]*token, error)
+}
+
+type tokenType uint32
+
+const (
+	operationToken = 1 << iota
+	openBracket
+	closeBracket
+	openCollect
+	closeCollect
+	openCollectObject
+	closeCollectObject
+	traverseArrayCollect
+)
+
+type token struct {
+	TokenType            tokenType
+	Operation            *Operation
+	AssignOperation      *Operation // e.g. tag (GetTag) op becomes AssignTag if '=' follows it
+	CheckForPostTraverse bool       // e.g. [1]cat should really be [1].cat
+	Match                string
+}
+
+func (t *token) toString(detail bool) string {
+	if t.TokenType == operationToken {
+		if detail {
+			return fmt.Sprintf("%v (%v)", t.Operation.toString(), t.Operation.OperationType.Precedence)
+		}
+		return t.Operation.toString()
+	} else if t.TokenType == openBracket {
+		return "("
+	} else if t.TokenType == closeBracket {
+		return ")"
+	} else if t.TokenType == openCollect {
+		return "["
+	} else if t.TokenType == closeCollect {
+		return "]"
+	} else if t.TokenType == openCollectObject {
+		return "{"
+	} else if t.TokenType == closeCollectObject {
+		return "}"
+	} else if t.TokenType == traverseArrayCollect {
+		return ".["
+
+	} else {
+		return "NFI"
+	}
+}
+
+func unwrap(value string) string {
+	return value[1 : len(value)-1]
+}
+
+func extractNumberParameter(value string) (int, error) {
+	parameterParser := regexp.MustCompile(`.*\(([0-9]+)\)`)
+	matches := parameterParser.FindStringSubmatch(value)
+	var indent, errParsingInt = strconv.ParseInt(matches[1], 10, 32)
+	if errParsingInt != nil {
+		return 0, errParsingInt
+	}
+	return int(indent), nil
+}
+
+func hasOptionParameter(value string, option string) bool {
+	parameterParser := regexp.MustCompile(`.*\([^\)]*\)`)
+	matches := parameterParser.FindStringSubmatch(value)
+	if len(matches) == 0 {
+		return false
+	}
+	parameterString := matches[0]
+	optionParser := regexp.MustCompile(fmt.Sprintf("\\b%v\\b", option))
+	return len(optionParser.FindStringSubmatch(parameterString)) > 0
+}
+
+func postProcessTokens(tokens []*token) []*token {
+	var postProcessedTokens = make([]*token, 0)
+
+	skipNextToken := false
+
+	for index := range tokens {
+		if skipNextToken {
+			skipNextToken = false
+		} else {
+			postProcessedTokens, skipNextToken = handleToken(tokens, index, postProcessedTokens)
+		}
+	}
+
+	return postProcessedTokens
+}
+
+func handleToken(tokens []*token, index int, postProcessedTokens []*token) (tokensAccum []*token, skipNextToken bool) {
+	skipNextToken = false
+	currentToken := tokens[index]
+
+	log.Debug("processing %v", currentToken.toString(true))
+
+	if currentToken.TokenType == traverseArrayCollect {
+		// `.[exp]`` works by creating a traversal array of [self, exp] and piping that into the traverse array operator
+		//need to put a traverse array then a collect currentToken
+		// do this by adding traverse then converting currentToken to collect
+
+		log.Debug("  adding self")
+		op := &Operation{OperationType: selfReferenceOpType, StringValue: "SELF"}
+		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
+
+		log.Debug("  adding traverse array")
+		op = &Operation{OperationType: traverseArrayOpType, StringValue: "TRAVERSE_ARRAY"}
+		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
+
+		currentToken = &token{TokenType: openCollect}
+
+	}
+
+	if index != len(tokens)-1 && currentToken.AssignOperation != nil &&
+		tokens[index+1].TokenType == operationToken &&
+		tokens[index+1].Operation.OperationType == assignOpType {
+		log.Debug("  its an update assign")
+		currentToken.Operation = currentToken.AssignOperation
+		currentToken.Operation.UpdateAssign = tokens[index+1].Operation.UpdateAssign
+		skipNextToken = true
+	}
+
+	log.Debug("  adding token to the fixed list")
+	postProcessedTokens = append(postProcessedTokens, currentToken)
+
+	if index != len(tokens)-1 &&
+		((currentToken.TokenType == openCollect && tokens[index+1].TokenType == closeCollect) ||
+			(currentToken.TokenType == openCollectObject && tokens[index+1].TokenType == closeCollectObject)) {
+		log.Debug("  adding empty")
+		op := &Operation{OperationType: emptyOpType, StringValue: "EMPTY"}
+		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
+	}
+
+	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
+		((tokens[index+1].TokenType == operationToken && (tokens[index+1].Operation.OperationType == traversePathOpType)) ||
+			(tokens[index+1].TokenType == traverseArrayCollect)) {
+		log.Debug("  adding pipe because the next thing is traverse")
+		op := &Operation{OperationType: shortPipeOpType, Value: "PIPE", StringValue: "."}
+		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
+	}
+	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
+		tokens[index+1].TokenType == openCollect {
+
+		log.Debug("  adding traverArray because next is opencollect")
+		op := &Operation{OperationType: traverseArrayOpType}
+		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
+	}
+	return postProcessedTokens, skipNextToken
+}
@@ -0,0 +1,533 @@
+package yqlib
+
+import (
+	"strconv"
+	"strings"
+
+	"github.com/alecthomas/participle/v2/lexer"
+)
+
+var participleYqRules = []*participleYqRule{
+	{"LINE_COMMENT", `line_?comment|lineComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true}), 0},
+	{"HEAD_COMMENT", `head_?comment|headComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{HeadComment: true}), 0},
+	{"FOOT_COMMENT", `foot_?comment|footComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{FootComment: true}), 0},
+
+	{"OpenBracket", `\(`, literalToken(openBracket, false), 0},
+	{"CloseBracket", `\)`, literalToken(closeBracket, true), 0},
+	{"OpenTraverseArrayCollect", `\.\[`, literalToken(traverseArrayCollect, false), 0},
+
+	{"OpenCollect", `\[`, literalToken(openCollect, false), 0},
+	{"CloseCollect", `\]\??`, literalToken(closeCollect, true), 0},
+
+	{"OpenCollectObject", `\{`, literalToken(openCollectObject, false), 0},
+	{"CloseCollectObject", `\}`, literalToken(closeCollectObject, true), 0},
+
+	{"RecursiveDecentIncludingKeys", `\.\.\.`, recursiveDecentOpToken(true), 0},
+	{"RecursiveDecent", `\.\.`, recursiveDecentOpToken(false), 0},
+
+	{"GetVariable", `\$[a-zA-Z_\-0-9]+`, getVariableOpToken(), 0},
+	{"AsignAsVariable", `as`, opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{}), 0},
+	{"AsignRefVariable", `ref`, opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{IsReference: true}), 0},
+
+	{"CreateMap", `:\s*`, opToken(createMapOpType), 0},
+	simpleOp("length", lengthOpType),
+	simpleOp("line", lineOpType),
+	simpleOp("column", columnOpType),
+	simpleOp("eval", evalOpType),
+
+	{"MapValues", `map_?values`, opToken(mapValuesOpType), 0},
+	simpleOp("map", mapOpType),
+	simpleOp("pick", pickOpType),
+
+	{"FlattenWithDepth", `flatten\([0-9]+\)`, flattenWithDepth(), 0},
+	{"Flatten", `flatten`, opTokenWithPrefs(flattenOpType, nil, flattenPreferences{depth: -1}), 0},
+
+	simpleOp("format_datetime", formatDateTimeOpType),
+	simpleOp("now", nowOpType),
+	simpleOp("tz", tzOpType),
+	simpleOp("with_dtf", withDtFormatOpType),
+	simpleOp("error", errorOpType),
+	simpleOp("sortKeys", sortKeysOpType),
+	simpleOp("sort_?keys", sortKeysOpType),
+
+	{"YamlEncodeWithIndent", `to_?yaml\([0-9]+\)`, encodeParseIndent(YamlOutputFormat), 0},
+	{"XMLEncodeWithIndent", `to_?xml\([0-9]+\)`, encodeParseIndent(XMLOutputFormat), 0},
+	{"JSONEncodeWithIndent", `to_?json\([0-9]+\)`, encodeParseIndent(JSONOutputFormat), 0},
+
+	{"YamlDecode", `from_?yaml|@yamld|from_?json|@jsond`, decodeOp(YamlInputFormat), 0},
+	{"YamlEncode", `to_?yaml|@yaml`, encodeWithIndent(YamlOutputFormat, 2), 0},
+
+	{"JSONEncode", `to_?json`, encodeWithIndent(JSONOutputFormat, 2), 0},
+	{"JSONEncodeNoIndent", `@json`, encodeWithIndent(JSONOutputFormat, 0), 0},
+
+	{"PropertiesDecode", `from_?props|@propsd`, decodeOp(PropertiesInputFormat), 0},
+	{"PropsEncode", `to_?props|@props`, encodeWithIndent(PropsOutputFormat, 2), 0},
+
+	{"XmlDecode", `from_?xml|@xmld`, decodeOp(XMLInputFormat), 0},
+	{"XMLEncode", `to_?xml`, encodeWithIndent(XMLOutputFormat, 2), 0},
+	{"XMLEncodeNoIndent", `@xml`, encodeWithIndent(XMLOutputFormat, 0), 0},
+
+	{"CSVDecode", `from_?csv|@csvd`, decodeOp(CSVObjectInputFormat), 0},
+	{"CSVEncode", `to_?csv|@csv`, encodeWithIndent(CSVOutputFormat, 0), 0},
+
+	{"TSVDecode", `from_?tsv|@tsvd`, decodeOp(TSVObjectInputFormat), 0},
+	{"TSVEncode", `to_?tsv|@tsv`, encodeWithIndent(TSVOutputFormat, 0), 0},
+
+	{"Base64d", `@base64d`, decodeOp(Base64InputFormat), 0},
+	{"Base64", `@base64`, encodeWithIndent(Base64OutputFormat, 0), 0},
+
+	{"LoadXML", `load_?xml|xml_?load`, loadOp(NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName, XMLPreferences.StrictMode, XMLPreferences.KeepNamespace, XMLPreferences.UseRawToken), false), 0},
+
+	{"LoadBase64", `load_?base64`, loadOp(NewBase64Decoder(), false), 0},
+
+	{"LoadProperties", `load_?props`, loadOp(NewPropertiesDecoder(), false), 0},
+
+	{"LoadString", `load_?str|str_?load`, loadOp(nil, true), 0},
+
+	{"LoadYaml", `load`, loadOp(NewYamlDecoder(), false), 0},
+
+	{"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0},
+
+	simpleOp("select", selectOpType),
+	simpleOp("has", hasOpType),
+	simpleOp("unique_?by", uniqueByOpType),
+	simpleOp("unique", uniqueOpType),
+
+	simpleOp("group_?by", groupByOpType),
+	simpleOp("explode", explodeOpType),
+	simpleOp("or", orOpType),
+	simpleOp("and", andOpType),
+	simpleOp("not", notOpType),
+	simpleOp("ireduce", reduceOpType),
+
+	simpleOp("join", joinStringOpType),
+	simpleOp("sub", subStringOpType),
+	simpleOp("match", matchOpType),
+	simpleOp("capture", captureOpType),
+	simpleOp("test", testOpType),
+
+	simpleOp("sort_?by", sortByOpType),
+	simpleOp("sort", sortOpType),
+
+	simpleOp("reverse", reverseOpType),
+
+	simpleOp("any_c", anyConditionOpType),
+	simpleOp("any", anyOpType),
+
+	simpleOp("all_c", allConditionOpType),
+	simpleOp("all", allOpType),
+
+	simpleOp("contains", containsOpType),
+	simpleOp("split", splitStringOpType),
+	simpleOp("parent", getParentOpType),
+
+	simpleOp("keys", keysOpType),
+	simpleOp("key", getKeyOpType),
+
+	simpleOp("file_?name|fileName", getFilenameOpType),
+	simpleOp("file_?index|fileIndex|fi", getFileIndexOpType),
+	simpleOp("path", getPathOpType),
+
+	simpleOp("to_?entries|toEntries", toEntriesOpType),
+	simpleOp("from_?entries|fromEntries", fromEntriesOpType),
+	simpleOp("with_?entries|withEntries", withEntriesOpType),
+
+	simpleOp("with", withOpType),
+
+	simpleOp("collect", collectOpType),
+	simpleOp("del", deleteChildOpType),
+
+	assignableOp("style", getStyleOpType, assignStyleOpType),
+	assignableOp("tag|type", getTagOpType, assignTagOpType),
+	assignableOp("anchor", getAnchorOpType, assignAnchorOpType),
+	assignableOp("alias", getAliasOpType, assignAliasOpType),
+
+	{"ALL_COMMENTS", `comments\s*=`, assignAllCommentsOp(false), 0},
+	{"ALL_COMMENTS_ASSIGN_RELATIVE", `comments\s*\|=`, assignAllCommentsOp(true), 0},
+
+	{"Block", `;`, opToken(blockOpType), 0},
+	{"Alternative", `\/\/`, opToken(alternativeOpType), 0},
+
+	{"DocumentIndex", `documentIndex|document_?index|di`, opToken(getDocumentIndexOpType), 0},
+
+	{"Uppercase", `upcase|ascii_?upcase`, opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: true}), 0},
+	{"Downcase", `downcase|ascii_?downcase`, opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: false}), 0},
+
+	{"HexValue", `0[xX][0-9A-Fa-f]+`, hexValue(), 0},
+	{"FloatValueScientific", `-?[1-9](\.\d+)?[Ee][-+]?\d+`, floatValue(), 0},
+	{"FloatValue", `-?\d+(\.\d+)`, floatValue(), 0},
+
+	{"NumberValue", `-?\d+`, numberValue(), 0},
+
+	{"TrueBooleanValue", `[Tt][Rr][Uu][Ee]`, booleanValue(true), 0},
+	{"FalseBooleanValue", `[Ff][Aa][Ll][Ss][Ee]`, booleanValue(false), 0},
+
+	{"NullValue", `[Nn][Uu][Ll][Ll]|~`, nullValue(), 0},
+
+	{"QuotedStringValue", `"([^"\\]*(\\.[^"\\]*)*)"`, stringValue(), 0},
+
+	{"StrEnvOp", `strenv\([^\)]+\)`, envOp(true), 0},
+	{"EnvOp", `env\([^\)]+\)`, envOp(false), 0},
+
+	{"EnvSubstWithOptions", `envsubst\((ne|nu|ff| |,)+\)`, envSubstWithOptions(), 0},
+	simpleOp("envsubst", envsubstOpType),
+
+	{"Equals", `\s*==\s*`, opToken(equalsOpType), 0},
+	{"NotEquals", `\s*!=\s*`, opToken(notEqualsOpType), 0},
+
+	{"GreaterThanEquals", `\s*>=\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: true, Greater: true}), 0},
+	{"LessThanEquals", `\s*<=\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: true, Greater: false}), 0},
+
+	{"GreaterThan", `\s*>\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: false, Greater: true}), 0},
+	{"LessThan", `\s*<\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: false, Greater: false}), 0},
+
+	{"AssignRelative", `\|=`, assignOpToken(true), 0},
+	{"Assign", `=`, assignOpToken(false), 0},
+
+	{`whitespace`, `[ \t\n]+`, nil, 0},
+
+	{"WrappedPathElement", `\."[^ "]+"\??`, pathToken(true), 0},
+	{"PathElement", `\.[^ ;\}\{\:\[\],\|\.\[\(\)=\n]+\??`, pathToken(false), 0},
+	{"Pipe", `\|`, opToken(pipeOpType), 0},
+	{"Self", `\.`, opToken(selfReferenceOpType), 0},
+
+	{"Union", `,`, opToken(unionOpType), 0},
+
+	{"MultiplyAssign", `\*=[\+|\?dn]*`, multiplyWithPrefs(multiplyAssignOpType), 0},
+	{"Multiply", `\*[\+|\?dn]*`, multiplyWithPrefs(multiplyOpType), 0},
+
+	{"AddAssign", `\+=`, opToken(addAssignOpType), 0},
+	{"Add", `\+`, opToken(addOpType), 0},
+
+	{"SubtractAssign", `\-=`, opToken(subtractAssignOpType), 0},
+	{"Subtract", `\-`, opToken(subtractOpType), 0},
+}
+
+type yqAction func(lexer.Token) (*token, error)
+
+type participleYqRule struct {
+	Name                string
+	Pattern             string
+	CreateYqToken       yqAction
+	ParticipleTokenType lexer.TokenType
+}
+
+type participleLexer struct {
+	lexerDefinition lexer.StringDefinition
+}
+
+func simpleOp(name string, opType *operationType) *participleYqRule {
+	return &participleYqRule{strings.ToUpper(string(name[1])) + name[1:], name, opToken(opType), 0}
+}
+
+func assignableOp(name string, opType *operationType, assignOpType *operationType) *participleYqRule {
+	return &participleYqRule{strings.ToUpper(string(name[1])) + name[1:], name, opTokenWithPrefs(opType, assignOpType, nil), 0}
+}
+
+func newParticipleLexer() expressionTokeniser {
+	simpleRules := make([]lexer.SimpleRule, len(participleYqRules))
+	for i, yqRule := range participleYqRules {
+		simpleRules[i] = lexer.SimpleRule{Name: yqRule.Name, Pattern: yqRule.Pattern}
+	}
+	lexerDefinition := lexer.MustSimple(simpleRules)
+	symbols := lexerDefinition.Symbols()
+
+	for _, yqRule := range participleYqRules {
+		yqRule.ParticipleTokenType = symbols[yqRule.Name]
+	}
+
+	return &participleLexer{lexerDefinition}
+}
+
+func pathToken(wrapped bool) yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		value := rawToken.Value
+		prefs := traversePreferences{}
+
+		if value[len(value)-1:] == "?" {
+			prefs.OptionalTraverse = true
+			value = value[:len(value)-1]
+		}
+
+		value = value[1:]
+		if wrapped {
+			value = unwrap(value)
+		}
+		log.Debug("PathToken %v", value)
+		op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: prefs}
+		return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil
+	}
+}
+
+func recursiveDecentOpToken(includeMapKeys bool) yqAction {
+	prefs := recursiveDescentPreferences{
+		RecurseArray: true,
+		TraversePreferences: traversePreferences{
+			DontFollowAlias: true,
+			IncludeMapKeys:  includeMapKeys,
+		},
+	}
+	return opTokenWithPrefs(recursiveDescentOpType, nil, prefs)
+}
+
+func opTokenWithPrefs(opType *operationType, assignOpType *operationType, preferences interface{}) yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		value := rawToken.Value
+		op := &Operation{OperationType: opType, Value: opType.Type, StringValue: value, Preferences: preferences}
+		var assign *Operation
+		if assignOpType != nil {
+			assign = &Operation{OperationType: assignOpType, Value: assignOpType.Type, StringValue: value, Preferences: preferences}
+		}
+		return &token{TokenType: operationToken, Operation: op, AssignOperation: assign}, nil
+	}
+}
+
+func flattenWithDepth() yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		value := rawToken.Value
+		var depth, errParsingInt = extractNumberParameter(value)
+		if errParsingInt != nil {
+			return nil, errParsingInt
+		}
+
+		prefs := flattenPreferences{depth: depth}
+		op := &Operation{OperationType: flattenOpType, Value: flattenOpType.Type, StringValue: value, Preferences: prefs}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+}
+
+func assignAllCommentsOp(updateAssign bool) yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		log.Debug("assignAllCommentsOp %v", rawToken.Value)
+		value := rawToken.Value
+		op := &Operation{
+			OperationType: assignCommentOpType,
+			Value:         assignCommentOpType.Type,
+			StringValue:   value,
+			UpdateAssign:  updateAssign,
+			Preferences:   commentOpPreferences{LineComment: true, HeadComment: true, FootComment: true},
+		}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+}
+
+func assignOpToken(updateAssign bool) yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		log.Debug("assignOpToken %v", rawToken.Value)
+		value := rawToken.Value
+		prefs := assignPreferences{DontOverWriteAnchor: true}
+		op := &Operation{OperationType: assignOpType, Value: assignOpType.Type, StringValue: value, UpdateAssign: updateAssign, Preferences: prefs}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+}
+
+func booleanValue(val bool) yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		return &token{TokenType: operationToken, Operation: createValueOperation(val, rawToken.Value)}, nil
+	}
+}
+
+func nullValue() yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		return &token{TokenType: operationToken, Operation: createValueOperation(nil, rawToken.Value)}, nil
+	}
+}
+
+func stringValue() yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		value := unwrap(rawToken.Value)
+		value = strings.ReplaceAll(value, "\\\"", "\"")
+		return &token{TokenType: operationToken, Operation: createValueOperation(value, value)}, nil
+	}
+}
+
+func envOp(strenv bool) yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		value := rawToken.Value
+		preferences := envOpPreferences{}
+
+		if strenv {
+			// strenv( )
+			value = value[7 : len(value)-1]
+			preferences.StringValue = true
+		} else {
+			//env( )
+			value = value[4 : len(value)-1]
+		}
+
+		envOperation := createValueOperation(value, value)
+		envOperation.OperationType = envOpType
+		envOperation.Preferences = preferences
+
+		return &token{TokenType: operationToken, Operation: envOperation}, nil
+	}
+}
+
+func envSubstWithOptions() yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		value := rawToken.Value
+		noEmpty := hasOptionParameter(value, "ne")
+		noUnset := hasOptionParameter(value, "nu")
+		failFast := hasOptionParameter(value, "ff")
+		envsubstOpType.Type = "ENVSUBST"
+		prefs := envOpPreferences{NoUnset: noUnset, NoEmpty: noEmpty, FailFast: failFast}
+		if noEmpty {
+			envsubstOpType.Type = envsubstOpType.Type + "_NO_EMPTY"
+		}
+		if noUnset {
+			envsubstOpType.Type = envsubstOpType.Type + "_NO_UNSET"
+		}
+
+		op := &Operation{OperationType: envsubstOpType, Value: envsubstOpType.Type, StringValue: value, Preferences: prefs}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+}
+
+func multiplyWithPrefs(op *operationType) yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		prefs := multiplyPreferences{}
+		options := rawToken.Value
+		if strings.Contains(options, "+") {
+			prefs.AppendArrays = true
+		}
+		if strings.Contains(options, "?") {
+			prefs.TraversePrefs = traversePreferences{DontAutoCreate: true}
+		}
+		if strings.Contains(options, "n") {
+			prefs.AssignPrefs = assignPreferences{OnlyWriteNull: true}
+		}
+		if strings.Contains(options, "d") {
+			prefs.DeepMergeArrays = true
+		}
+		prefs.TraversePrefs.DontFollowAlias = true
+		op := &Operation{OperationType: op, Value: multiplyOpType.Type, StringValue: options, Preferences: prefs}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+
+}
+
+func getVariableOpToken() yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		value := rawToken.Value
+
+		value = value[1:]
+
+		getVarOperation := createValueOperation(value, value)
+		getVarOperation.OperationType = getVariableOpType
+
+		return &token{TokenType: operationToken, Operation: getVarOperation, CheckForPostTraverse: true}, nil
+	}
+}
+
+func hexValue() yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		var originalString = rawToken.Value
+		var numberString = originalString[2:]
+		log.Debugf("numberString: %v", numberString)
+		var number, errParsingInt = strconv.ParseInt(numberString, 16, 64)
+		if errParsingInt != nil {
+			return nil, errParsingInt
+		}
+
+		return &token{TokenType: operationToken, Operation: createValueOperation(number, originalString)}, nil
+	}
+}
+
+func floatValue() yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		var numberString = rawToken.Value
+		var number, errParsingInt = strconv.ParseFloat(numberString, 64)
+		if errParsingInt != nil {
+			return nil, errParsingInt
+		}
+		return &token{TokenType: operationToken, Operation: createValueOperation(number, numberString)}, nil
+	}
+}
+
+func numberValue() yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		var numberString = rawToken.Value
+		var number, errParsingInt = strconv.ParseInt(numberString, 10, 64)
+		if errParsingInt != nil {
+			return nil, errParsingInt
+		}
+
+		return &token{TokenType: operationToken, Operation: createValueOperation(number, numberString)}, nil
+	}
+}
+
+func encodeParseIndent(outputFormat PrinterOutputFormat) yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		value := rawToken.Value
+		var indent, errParsingInt = extractNumberParameter(value)
+		if errParsingInt != nil {
+			return nil, errParsingInt
+		}
+
+		prefs := encoderPreferences{format: outputFormat, indent: indent}
+		op := &Operation{OperationType: encodeOpType, Value: encodeOpType.Type, StringValue: value, Preferences: prefs}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+}
+
+func encodeWithIndent(outputFormat PrinterOutputFormat, indent int) yqAction {
+	prefs := encoderPreferences{format: outputFormat, indent: indent}
+	return opTokenWithPrefs(encodeOpType, nil, prefs)
+}
+
+func decodeOp(inputFormat InputFormat) yqAction {
+	prefs := decoderPreferences{format: inputFormat}
+	return opTokenWithPrefs(decodeOpType, nil, prefs)
+}
+
+func loadOp(decoder Decoder, loadAsString bool) yqAction {
+	prefs := loadPrefs{decoder: decoder, loadAsString: loadAsString}
+	return opTokenWithPrefs(loadOpType, nil, prefs)
+}
+
+func opToken(op *operationType) yqAction {
+	return opTokenWithPrefs(op, nil, nil)
+}
+
+func literalToken(tt tokenType, checkForPost bool) yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		return &token{TokenType: tt, CheckForPostTraverse: checkForPost, Match: rawToken.Value}, nil
+	}
+}
+
+func (p *participleLexer) getYqDefinition(rawToken lexer.Token) *participleYqRule {
+	for _, yqRule := range participleYqRules {
+		if yqRule.ParticipleTokenType == rawToken.Type {
+			return yqRule
+		}
+	}
+	return &participleYqRule{}
+}
+
+func (p *participleLexer) Tokenise(expression string) ([]*token, error) {
+	myLexer, err := p.lexerDefinition.LexString("", expression)
+	if err != nil {
+		return nil, err
+	}
+	tokens := make([]*token, 0)
+
+	for {
+		rawToken, e := myLexer.Next()
+		if e != nil {
+			return nil, e
+		} else if rawToken.Type == lexer.EOF {
+			return postProcessTokens(tokens), nil
+		}
+
+		definition := p.getYqDefinition(rawToken)
+		if definition.CreateYqToken != nil {
+			token, e := definition.CreateYqToken(rawToken)
+			if e != nil {
+				return nil, e
+			}
+			tokens = append(tokens, token)
+		}
+
+	}
+
+}
@@ -0,0 +1,537 @@
+package yqlib
+
+import (
+	"testing"
+
+	"github.com/alecthomas/repr"
+	"github.com/mikefarah/yq/v4/test"
+	"gopkg.in/yaml.v3"
+)
+
+type participleLexerScenario struct {
+	expression string
+	tokens     []*token
+}
+
+var participleLexerScenarios = []participleLexerScenario{
+	{
+		expression: ".a",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "a",
+					StringValue:   "a",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+		},
+	},
+	{
+		expression: ".a.b",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "a",
+					StringValue:   "a",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: shortPipeOpType,
+					Value:         "PIPE",
+					StringValue:   ".",
+					Preferences:   nil,
+				},
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "b",
+					StringValue:   "b",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+		},
+	},
+	{
+		expression: ".a.b?",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "a",
+					StringValue:   "a",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: shortPipeOpType,
+					Value:         "PIPE",
+					StringValue:   ".",
+					Preferences:   nil,
+				},
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "b",
+					StringValue:   "b",
+					Preferences: traversePreferences{
+						OptionalTraverse: true,
+					},
+				},
+				CheckForPostTraverse: true,
+			},
+		},
+	},
+	{
+		expression: `.a."b?"`,
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "a",
+					StringValue:   "a",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: shortPipeOpType,
+					Value:         "PIPE",
+					StringValue:   ".",
+					Preferences:   nil,
+				},
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "b?",
+					StringValue:   "b?",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+		},
+	},
+	{
+		expression: `   .a  ."b?"`,
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "a",
+					StringValue:   "a",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: shortPipeOpType,
+					Value:         "PIPE",
+					StringValue:   ".",
+					Preferences:   nil,
+				},
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "b?",
+					StringValue:   "b?",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+		},
+	},
+	{
+		expression: `.a | .b`,
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "a",
+					StringValue:   "a",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: pipeOpType,
+					Value:         "PIPE",
+					StringValue:   "|",
+					Preferences:   nil,
+				},
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "b",
+					StringValue:   "b",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+		},
+	},
+	{
+		expression: "(.a)",
+		tokens: []*token{
+			{
+				TokenType: openBracket,
+				Match:     "(",
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "a",
+					StringValue:   "a",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+			{
+				TokenType:            closeBracket,
+				Match:                ")",
+				CheckForPostTraverse: true,
+			},
+		},
+	},
+	{
+		expression: "..",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: recursiveDescentOpType,
+					Value:         "RECURSIVE_DESCENT",
+					StringValue:   "..",
+					Preferences: recursiveDescentPreferences{
+						RecurseArray: true,
+						TraversePreferences: traversePreferences{
+							DontFollowAlias: true,
+							IncludeMapKeys:  false,
+						},
+					},
+				},
+			},
+		},
+	},
+	{
+		expression: "...",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: recursiveDescentOpType,
+					Value:         "RECURSIVE_DESCENT",
+					StringValue:   "...",
+					Preferences: recursiveDescentPreferences{
+						RecurseArray: true,
+						TraversePreferences: traversePreferences{
+							DontFollowAlias: true,
+							IncludeMapKeys:  true,
+						},
+					},
+				},
+			},
+		},
+	},
+	{
+		expression: ".a,.b",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "a",
+					StringValue:   "a",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: unionOpType,
+					Value:         "UNION",
+					StringValue:   ",",
+					Preferences:   nil,
+				},
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "b",
+					StringValue:   "b",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+		},
+	},
+	{
+		expression: "map_values",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: mapValuesOpType,
+					Value:         "MAP_VALUES",
+					StringValue:   "map_values",
+					Preferences:   nil,
+				},
+			},
+		},
+	},
+	{
+		expression: "mapvalues",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: mapValuesOpType,
+					Value:         "MAP_VALUES",
+					StringValue:   "mapvalues",
+					Preferences:   nil,
+				},
+			},
+		},
+	},
+	{
+		expression: "flatten(3)",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: flattenOpType,
+					Value:         "FLATTEN_BY",
+					StringValue:   "flatten(3)",
+					Preferences:   flattenPreferences{depth: 3},
+				},
+			},
+		},
+	},
+	{
+		expression: "flatten",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: flattenOpType,
+					Value:         "FLATTEN_BY",
+					StringValue:   "flatten",
+					Preferences:   flattenPreferences{depth: -1},
+				},
+			},
+		},
+	},
+	{
+		expression: "length",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: lengthOpType,
+					Value:         "LENGTH",
+					StringValue:   "length",
+					Preferences:   nil,
+				},
+			},
+		},
+	},
+	{
+		expression: "format_datetime",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: formatDateTimeOpType,
+					Value:         "FORMAT_DATE_TIME",
+					StringValue:   "format_datetime",
+					Preferences:   nil,
+				},
+			},
+		},
+	},
+	{
+		expression: "to_yaml(3)",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: encodeOpType,
+					Value:         "ENCODE",
+					StringValue:   "to_yaml(3)",
+					Preferences: encoderPreferences{
+						format: YamlOutputFormat,
+						indent: 3,
+					},
+				},
+			},
+		},
+	},
+	{
+		expression: "tojson(2)",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: encodeOpType,
+					Value:         "ENCODE",
+					StringValue:   "tojson(2)",
+					Preferences: encoderPreferences{
+						format: JSONOutputFormat,
+						indent: 2,
+					},
+				},
+			},
+		},
+	},
+	{
+		expression: "@yaml",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: encodeOpType,
+					Value:         "ENCODE",
+					StringValue:   "@yaml",
+					Preferences: encoderPreferences{
+						format: YamlOutputFormat,
+						indent: 2,
+					},
+				},
+			},
+		},
+	},
+	{
+		expression: "to_props",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: encodeOpType,
+					Value:         "ENCODE",
+					StringValue:   "to_props",
+					Preferences: encoderPreferences{
+						format: PropsOutputFormat,
+						indent: 2,
+					},
+				},
+			},
+		},
+	},
+	{
+		expression: "@base64d",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: decodeOpType,
+					Value:         "DECODE",
+					StringValue:   "@base64d",
+					Preferences: decoderPreferences{
+						format: Base64InputFormat,
+					},
+				},
+			},
+		},
+	},
+	{
+		expression: "@base64",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: encodeOpType,
+					Value:         "ENCODE",
+					StringValue:   "@base64",
+					Preferences: encoderPreferences{
+						format: Base64OutputFormat,
+					},
+				},
+			},
+		},
+	},
+	{
+		expression: "@yamld",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: decodeOpType,
+					Value:         "DECODE",
+					StringValue:   "@yamld",
+					Preferences: decoderPreferences{
+						format: YamlInputFormat,
+					},
+				},
+			},
+		},
+	},
+	{
+		expression: `"string with a \""`,
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: valueOpType,
+					Value:         `string with a "`,
+					StringValue:   `string with a "`,
+					Preferences:   nil,
+					CandidateNode: &CandidateNode{
+						Node: &yaml.Node{
+							Kind:  yaml.ScalarNode,
+							Tag:   "!!str",
+							Value: `string with a "`,
+						},
+					},
+				},
+			},
+		},
+	},
+}
+
+func TestParticipleLexer(t *testing.T) {
+	lexer := newParticipleLexer()
+
+	for _, scenario := range participleLexerScenarios {
+		actual, err := lexer.Tokenise(scenario.expression)
+		if err != nil {
+			t.Error(err)
+		} else {
+			test.AssertResultWithContext(t, repr.String(scenario.tokens, repr.Indent(" ")), repr.String(actual, repr.Indent(" ")), scenario.expression)
+		}
+
+	}
+}
@@ -91,6 +91,7 @@ var columnOpType = &operationType{Type: "LINE", NumArgs: 0, Precedence: 50, Hand

 var collectOpType = &operationType{Type: "COLLECT", NumArgs: 1, Precedence: 50, Handler: collectOperator}
 var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 50, Handler: mapOperator}
+var errorOpType = &operationType{Type: "ERROR", NumArgs: 1, Precedence: 50, Handler: errorOperator}
 var pickOpType = &operationType{Type: "PICK", NumArgs: 1, Precedence: 50, Handler: pickOperator}
 var evalOpType = &operationType{Type: "EVAL", NumArgs: 1, Precedence: 50, Handler: evalOperator}
 var mapValuesOpType = &operationType{Type: "MAP_VALUES", NumArgs: 1, Precedence: 50, Handler: mapValuesOperator}
@@ -125,7 +126,7 @@ var getParentOpType = &operationType{Type: "GET_PARENT", NumArgs: 0, Precedence:

 var getCommentOpType = &operationType{Type: "GET_COMMENT", NumArgs: 0, Precedence: 50, Handler: getCommentsOperator}
 var getAnchorOpType = &operationType{Type: "GET_ANCHOR", NumArgs: 0, Precedence: 50, Handler: getAnchorOperator}
-var getAliasOptype = &operationType{Type: "GET_ALIAS", NumArgs: 0, Precedence: 50, Handler: getAliasOperator}
+var getAliasOpType = &operationType{Type: "GET_ALIAS", NumArgs: 0, Precedence: 50, Handler: getAliasOperator}
 var getDocumentIndexOpType = &operationType{Type: "GET_DOCUMENT_INDEX", NumArgs: 0, Precedence: 50, Handler: getDocumentIndexOperator}
 var getFilenameOpType = &operationType{Type: "GET_FILENAME", NumArgs: 0, Precedence: 50, Handler: getFilenameOperator}
 var getFileIndexOpType = &operationType{Type: "GET_FILE_INDEX", NumArgs: 0, Precedence: 50, Handler: getFileIndexOperator}
@@ -204,10 +205,10 @@ func findInArray(array *yaml.Node, item *yaml.Node) int {
 	return -1
 }

-func findKeyInMap(array *yaml.Node, item *yaml.Node) int {
+func findKeyInMap(dataMap *yaml.Node, item *yaml.Node) int {

-	for index := 0; index < len(array.Content); index = index + 2 {
-		if recursiveNodeEqual(array.Content[index], item) {
+	for index := 0; index < len(dataMap.Content); index = index + 2 {
+		if recursiveNodeEqual(dataMap.Content[index], item) {
 			return index
 		}
 	}
@@ -239,20 +240,28 @@ func guessTagFromCustomType(node *yaml.Node) string {
 		log.Warning("node has no value to guess the type with")
 		return node.Tag
 	}
+	dataBucket, errorReading := parseSnippet(node.Value)

-	decoder := NewYamlDecoder()
-	decoder.Init(strings.NewReader(node.Value))
-	var dataBucket yaml.Node
-	errorReading := decoder.Decode(&dataBucket)
 	if errorReading != nil {
 		log.Warning("could not guess underlying tag type %v", errorReading)
 		return node.Tag
 	}
-	guessedTag := unwrapDoc(&dataBucket).Tag
+	guessedTag := unwrapDoc(dataBucket).Tag
 	log.Info("im guessing the tag %v is a %v", node.Tag, guessedTag)
 	return guessedTag
 }

+func parseSnippet(value string) (*yaml.Node, error) {
+	decoder := NewYamlDecoder()
+	decoder.Init(strings.NewReader(value))
+	var dataBucket yaml.Node
+	err := decoder.Decode(&dataBucket)
+	if len(dataBucket.Content) == 0 {
+		return nil, fmt.Errorf("bad data")
+	}
+	return dataBucket.Content[0], err
+}
+
 func recursiveNodeEqual(lhs *yaml.Node, rhs *yaml.Node) bool {
 	if lhs.Kind != rhs.Kind {
 		return false
@@ -56,7 +56,9 @@ func add(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Candida
 		return lhs.CreateReplacement(rhs.Node), nil
 	}

-	target := lhs.CreateReplacement(&yaml.Node{})
+	target := lhs.CreateReplacement(&yaml.Node{
+		Anchor: lhs.Node.Anchor,
+	})

 	switch lhsNode.Kind {
 	case yaml.MappingNode:
@@ -341,6 +341,15 @@ var addOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: !horse [a, b]\nb: !goat [b]\n",
 		},
 	},
+	{
+		skipDoc:     true,
+		description: "Keep anchors",
+		document:    "a: &horse [1]",
+		expression:  `.a += 2`,
+		expected: []string{
+			"D0, P[], (doc)::a: &horse [1, 2]\n",
+		},
+	},
 }

 func TestAddOperatorScenarios(t *testing.T) {
@@ -22,8 +22,10 @@ func assignUpdateOperator(d *dataTreeNavigator, context Context, expressionNode
 	}

 	prefs := assignPreferences{}
-	if expressionNode.Operation.Preferences != nil {
-		prefs = expressionNode.Operation.Preferences.(assignPreferences)
+	// they way *= (multipleAssign) is handled, we set the multiplePrefs
+	// on the node, not assignPrefs. Long story.
+	if p, ok := expressionNode.Operation.Preferences.(assignPreferences); ok {
+		prefs = p
 	}

 	if !expressionNode.Operation.UpdateAssign {
@@ -13,9 +13,9 @@ import (
 func configureEncoder(format PrinterOutputFormat, indent int) Encoder {
 	switch format {
 	case JSONOutputFormat:
-		return NewJONEncoder(indent, false)
+		return NewJSONEncoder(indent, false)
 	case PropsOutputFormat:
-		return NewPropertiesEncoder()
+		return NewPropertiesEncoder(true)
 	case CSVOutputFormat:
 		return NewCsvEncoder(',')
 	case TSVOutputFormat:
@@ -114,6 +114,10 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 		decoder = NewBase64Decoder()
 	case PropertiesInputFormat:
 		decoder = NewPropertiesDecoder()
+	case CSVObjectInputFormat:
+		decoder = NewCSVObjectDecoder(',')
+	case TSVObjectInputFormat:
+		decoder = NewCSVObjectDecoder('\t')
 	}

 	var results = list.New()
@@ -66,11 +66,27 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 	{
 		description: "Decode props encoded string",
 		document:    `a: "cats=great\ndogs=cool as well"`,
-		expression:  `.a |= from_props`,
+		expression:  `.a |= @propsd`,
 		expected: []string{
 			"D0, P[], (doc)::a:\n    cats: great\n    dogs: cool as well\n",
 		},
 	},
+	{
+		description: "Decode csv encoded string",
+		document:    `a: "cats,dogs\ngreat,cool as well"`,
+		expression:  `.a |= @csvd`,
+		expected: []string{
+			"D0, P[], (doc)::a:\n    - cats: great\n      dogs: cool as well\n",
+		},
+	},
+	{
+		description: "Decode tsv encoded string",
+		document: `a: "cats	dogs\ngreat	cool as well"`,
+		expression: `.a |= @tsvd`,
+		expected: []string{
+			"D0, P[], (doc)::a:\n    - cats: great\n      dogs: cool as well\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   "a:\n  cool:\n    bob: dylan",
@@ -0,0 +1,20 @@
+package yqlib
+
+import (
+	"fmt"
+)
+
+func errorOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	log.Debugf("-- errorOperation")
+
+	rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS)
+	if err != nil {
+		return Context{}, err
+	}
+	errorMessage := "aborted"
+	if rhs.MatchingNodes.Len() > 0 {
+		errorMessage = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node.Value
+	}
+	return Context{}, fmt.Errorf(errorMessage)
+}
@@ -0,0 +1,40 @@
+package yqlib
+
+import "testing"
+
+const validationExpression = `
+	with(env(numberOfCats); select(tag == "!!int") or error("numberOfCats is not a number :(")) | 
+	.numPets = env(numberOfCats)
+`
+
+var errorOperatorScenarios = []expressionScenario{
+	{
+		description:   "Validate a particular value",
+		document:      `a: hello`,
+		expression:    `select(.a == "howdy") or error(".a [" + .a + "] is not howdy!")`,
+		expectedError: ".a [hello] is not howdy!",
+	},
+	{
+		description:          "Validate the environment variable is a number - invalid",
+		environmentVariables: map[string]string{"numberOfCats": "please"},
+		expression:           `env(numberOfCats) | select(tag == "!!int") or error("numberOfCats is not a number :(")`,
+		expectedError:        "numberOfCats is not a number :(",
+	},
+	{
+		description:          "Validate the environment variable is a number - valid",
+		subdescription:       "`with` can be a convenient way of encapsulating validation.",
+		environmentVariables: map[string]string{"numberOfCats": "3"},
+		document:             "name: Bob\nfavouriteAnimal: cat\n",
+		expression:           validationExpression,
+		expected: []string{
+			"D0, P[], (doc)::name: Bob\nfavouriteAnimal: cat\nnumPets: 3\n",
+		},
+	},
+}
+
+func TestErrorOperatorScenarios(t *testing.T) {
+	for _, tt := range errorOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentOperatorScenarios(t, "error", errorOperatorScenarios)
+}
@@ -27,7 +27,6 @@ func createMultiplyOp(prefs interface{}) func(lhs *ExpressionNode, rhs *Expressi

 func multiplyAssignOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	var multiplyPrefs = expressionNode.Operation.Preferences
-	expressionNode.Operation.Preferences = nil

 	return compoundAssignFunction(d, context, expressionNode, createMultiplyOp(multiplyPrefs))
 }
@@ -67,9 +67,9 @@ func pickOperator(d *dataTreeNavigator, context Context, expressionNode *Express
 		node := unwrapDoc(candidate.Node)

 		var replacement *yaml.Node
-		if node.Tag == "!!map" {
+		if node.Kind == yaml.MappingNode {
 			replacement = pickMap(node, indicesToPick)
-		} else if node.Tag == "!!seq" {
+		} else if node.Kind == yaml.SequenceNode {
 			replacement, err = pickSequence(node, indicesToPick)
 			if err != nil {
 				return Context{}, err
@@ -14,6 +14,15 @@ var pickOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::myMap: {hamster: squeek, cat: meow}\n",
 		},
 	},
+	{
+		description: "Pick keys from map",
+		skipDoc:     true,
+		document:    "!things myMap: {cat: meow, dog: bark, thing: hamster, hamster: squeek}\n",
+		expression:  `.myMap |= pick(["hamster", "cat", "goat"])`,
+		expected: []string{
+			"D0, P[], (doc)::!things myMap: {hamster: squeek, cat: meow}\n",
+		},
+	},
 	{
 		description: "Pick keys from map with comments",
 		skipDoc:     true,
@@ -314,7 +314,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	// note printDocSeparators is true, it should still not print document separators
 	// when outputing JSON.
-	printer := NewPrinter(NewJONEncoder(0, false), NewSinglePrinterWriter(writer))
+	printer := NewPrinter(NewJSONEncoder(0, false), NewSinglePrinterWriter(writer))

 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
@@ -8,9 +8,9 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )

-var samplePropertiesYaml = `# block comments don't come through
+const samplePropertiesYaml = `# block comments don't come through
 person: # neither do comments on maps
-    name: Mike # comments on values appear
+    name: Mike Wazowski # comments on values appear
    pets: 
    - cat # comments on array values appear
    food: [pizza] # comments on arrays do not
@@ -18,37 +18,45 @@ emptyArray: []
 emptyMap: []
 `

-var expectedProperties = `# comments on values appear
-person.name = Mike
+const expectedPropertiesUnwrapped = `# comments on values appear
+person.name = Mike Wazowski

 # comments on array values appear
 person.pets.0 = cat
 person.food.0 = pizza
 `

-var expectedUpdatedProperties = `# comments on values appear
-person.name = Mike
+const expectedPropertiesWrapped = `# comments on values appear
+person.name = "Mike Wazowski"
+
+# comments on array values appear
+person.pets.0 = cat
+person.food.0 = pizza
+`
+
+const expectedUpdatedProperties = `# comments on values appear
+person.name = Mike Wazowski

 # comments on array values appear
 person.pets.0 = dog
 person.food.0 = pizza
 `

-var expectedDecodedYaml = `person:
-  name: Mike # comments on values appear
+const expectedDecodedYaml = `person:
+  name: Mike Wazowski # comments on values appear
  pets:
    - cat # comments on array values appear
  food:
    - pizza
 `

-var expectedPropertiesNoComments = `person.name = Mike
+const expectedPropertiesNoComments = `person.name = Mike Wazowski
 person.pets.0 = cat
 person.food.0 = pizza
 `

-var expectedPropertiesWithEmptyMapsAndArrays = `# comments on values appear
-person.name = Mike
+const expectedPropertiesWithEmptyMapsAndArrays = `# comments on values appear
+person.name = Mike Wazowski

 # comments on array values appear
 person.pets.0 = cat
@@ -62,7 +70,14 @@ var propertyScenarios = []formatScenario{
 		description:    "Encode properties",
 		subdescription: "Note that empty arrays and maps are not encoded by default.",
 		input:          samplePropertiesYaml,
-		expected:       expectedProperties,
+		expected:       expectedPropertiesUnwrapped,
+	},
+	{
+		description:    "Encode properties: scalar encapsulation",
+		subdescription: "Note that string values with blank characters in them are encapsulated with double quotes",
+		input:          samplePropertiesYaml,
+		expected:       expectedPropertiesWrapped,
+		scenarioType:   "encode-wrapped",
 	},
 	{
 		description: "Encode properties: no comments",
@@ -79,7 +94,7 @@ var propertyScenarios = []formatScenario{
 	},
 	{
 		description:  "Decode properties",
-		input:        expectedProperties,
+		input:        expectedPropertiesUnwrapped,
 		expected:     expectedDecodedYaml,
 		scenarioType: "decode",
 	},
@@ -92,7 +107,7 @@ var propertyScenarios = []formatScenario{
 	},
 	{
 		description:  "Roundtrip",
-		input:        expectedProperties,
+		input:        expectedPropertiesUnwrapped,
 		expression:   `.person.pets.0 = "dog"`,
 		expected:     expectedUpdatedProperties,
 		scenarioType: "roundtrip",
@@ -106,7 +121,7 @@ var propertyScenarios = []formatScenario{
 	},
 }

-func documentEncodePropertyScenario(w *bufio.Writer, s formatScenario) {
+func documentUnwrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))

 	if s.subdescription != "" {
@@ -128,7 +143,32 @@ func documentEncodePropertyScenario(w *bufio.Writer, s formatScenario) {
 	}
 	writeOrPanic(w, "will output\n")

-	writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder())))
+	writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(true))))
+}
+
+func documentWrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) {
+	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
+
+	if s.subdescription != "" {
+		writeOrPanic(w, s.subdescription)
+		writeOrPanic(w, "\n\n")
+	}
+
+	writeOrPanic(w, "Given a sample.yml file of:\n")
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input))
+
+	writeOrPanic(w, "then\n")
+
+	expression := s.expression
+
+	if expression != "" {
+		writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=props --unwrapScalar=false '%v' sample.yml\n```\n", expression))
+	} else {
+		writeOrPanic(w, "```bash\nyq -o=props --unwrapScalar=false sample.yml\n```\n")
+	}
+	writeOrPanic(w, "will output\n")
+
+	writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(false))))
 }

 func documentDecodePropertyScenario(w *bufio.Writer, s formatScenario) {
@@ -178,7 +218,7 @@ func documentRoundTripPropertyScenario(w *bufio.Writer, s formatScenario) {

 	writeOrPanic(w, "will output\n")

-	writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder())))
+	writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(true))))
 }

 func documentPropertyScenario(t *testing.T, w *bufio.Writer, i interface{}) {
@@ -186,24 +226,35 @@ func documentPropertyScenario(t *testing.T, w *bufio.Writer, i interface{}) {
 	if s.skipDoc {
 		return
 	}
-	if s.scenarioType == "decode" {
+	switch s.scenarioType {
+	case "":
+		documentUnwrappedEncodePropertyScenario(w, s)
+	case "decode":
 		documentDecodePropertyScenario(w, s)
-	} else if s.scenarioType == "roundtrip" {
+	case "encode-wrapped":
+		documentWrappedEncodePropertyScenario(w, s)
+	case "roundtrip":
 		documentRoundTripPropertyScenario(w, s)
-	} else {
-		documentEncodePropertyScenario(w, s)
-	}

+	default:
+		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
+	}
 }

 func TestPropertyScenarios(t *testing.T) {
 	for _, s := range propertyScenarios {
-		if s.scenarioType == "decode" {
+		switch s.scenarioType {
+		case "":
+			test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(true)), s.description)
+		case "decode":
 			test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(2, false, true, true)), s.description)
-		} else if s.scenarioType == "roundtrip" {
-			test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder()), s.description)
-		} else {
-			test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder()), s.description)
+		case "encode-wrapped":
+			test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(false)), s.description)
+		case "roundtrip":
+			test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(true)), s.description)
+
+		default:
+			panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
 		}
 	}
 	genericScenarios := make([]interface{}, len(propertyScenarios))
@@ -8,7 +8,7 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )

-var inputXMLWithComments = `
+const inputXMLWithComments = `
 <!-- before cat -->
 <cat>
 	<!-- in cat before -->
@@ -26,7 +26,7 @@ for x --></x>
 </cat>
 <!-- after cat -->
 `
-var inputXMLWithCommentsWithSubChild = `
+const inputXMLWithCommentsWithSubChild = `
 <!-- before cat -->
 <cat>
 	<!-- in cat before -->
@@ -45,7 +45,7 @@ for x --></x>
 <!-- after cat -->
 `

-var expectedDecodeYamlWithSubChild = `# before cat
+const expectedDecodeYamlWithSubChild = `# before cat
 cat:
    # in cat before
    x: "3" # multi
@@ -65,7 +65,7 @@ cat:
 # after cat
 `

-var inputXMLWithCommentsWithArray = `
+const inputXMLWithCommentsWithArray = `
 <!-- before cat -->
 <cat>
 	<!-- in cat before -->
@@ -85,7 +85,7 @@ for x --></x>
 <!-- after cat -->
 `

-var expectedDecodeYamlWithArray = `# before cat
+const expectedDecodeYamlWithArray = `# before cat
 cat:
    # in cat before
    x: "3" # multi
@@ -109,7 +109,7 @@ cat:
 # after cat
 `

-var expectedDecodeYamlWithComments = `# before cat
+const expectedDecodeYamlWithComments = `# before cat
 cat:
    # in cat before
    x: "3" # multi
@@ -126,7 +126,7 @@ cat:
 # after cat
 `

-var expectedRoundtripXMLWithComments = `<!-- before cat --><cat><!-- in cat before -->
+const expectedRoundtripXMLWithComments = `<!-- before cat --><cat><!-- in cat before -->
  <x>3<!-- multi
 line comment 
 for x --></x><!-- before y -->
@@ -137,7 +137,7 @@ in d before -->
 </cat><!-- after cat -->
 `

-var yamlWithComments = `# above_cat
+const yamlWithComments = `# above_cat
 cat: # inline_cat
  # above_array
  array: # inline_array
@@ -147,31 +147,31 @@ cat: # inline_cat
 # below_cat
 `

-var expectedXMLWithComments = `<!-- above_cat inline_cat --><cat><!-- above_array inline_array -->
+const expectedXMLWithComments = `<!-- above_cat inline_cat --><cat><!-- above_array inline_array -->
  <array>val1<!-- inline_val1 --></array>
  <array><!-- above_val2 -->val2<!-- inline_val2 --></array>
 </cat><!-- below_cat -->
 `

-var inputXMLWithNamespacedAttr = `
+const inputXMLWithNamespacedAttr = `
 <?xml version="1.0"?>
 <map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
 </map>
 `

-var expectedYAMLWithNamespacedAttr = `map:
+const expectedYAMLWithNamespacedAttr = `map:
  +xmlns: some-namespace
  +xmlns:xsi: some-instance
  +some-instance:schemaLocation: some-url
 `

-var expectedYAMLWithRawNamespacedAttr = `map:
+const expectedYAMLWithRawNamespacedAttr = `map:
  +xmlns: some-namespace
  +xmlns:xsi: some-instance
  +xsi:schemaLocation: some-url
 `

-var xmlWithCustomDtd = `
+const xmlWithCustomDtd = `
 <?xml version="1.0"?>
 <!DOCTYPE root [
 <!ENTITY writer "Blah.">
@@ -181,7 +181,7 @@ var xmlWithCustomDtd = `
    <item>&writer;&copyright;</item>
 </root>`

-var expectedDtd = `root:
+const expectedDtd = `root:
    item: '&writer;&copyright;'
 `

@@ -292,6 +292,22 @@ var xmlScenarios = []formatScenario{
 		expected:     "<cat>purrs</cat>\n",
 		scenarioType: "encode",
 	},
+	{
+		description:  "includes map tags",
+		skipDoc:      true,
+		input:        "<cat>purrs</cat>\n",
+		expression:   `tag`,
+		expected:     "!!map\n",
+		scenarioType: "decode",
+	},
+	{
+		description:  "includes array tags",
+		skipDoc:      true,
+		input:        "<cat>purrs</cat><cat>purrs</cat>\n",
+		expression:   `.cat | tag`,
+		expected:     "!!seq\n",
+		scenarioType: "decode",
+	},
 	{
 		description:  "Encode xml: array",
 		input:        "pets:\n  cat:\n    - purrs\n    - meows",
@@ -336,6 +352,8 @@ var xmlScenarios = []formatScenario{

 func testXMLScenario(t *testing.T, s formatScenario) {
 	switch s.scenarioType {
+	case "", "decode":
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false), NewYamlEncoder(4, false, true, true)), s.description)
 	case "encode":
 		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewXMLEncoder(2, "+", "+content")), s.description)
 	case "roundtrip":
@@ -344,8 +362,9 @@ func testXMLScenario(t *testing.T, s formatScenario) {
 		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, true, false), NewYamlEncoder(2, false, true, true)), s.description)
 	case "decode-raw-token":
 		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, true, true), NewYamlEncoder(2, false, true, true)), s.description)
+
 	default:
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false), NewYamlEncoder(4, false, true, true)), s.description)
+		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
 	}
 }

@@ -356,6 +375,8 @@ func documentXMLScenario(t *testing.T, w *bufio.Writer, i interface{}) {
 		return
 	}
 	switch s.scenarioType {
+	case "", "decode":
+		documentXMLDecodeScenario(w, s)
 	case "encode":
 		documentXMLEncodeScenario(w, s)
 	case "roundtrip":
@@ -364,10 +385,10 @@ func documentXMLScenario(t *testing.T, w *bufio.Writer, i interface{}) {
 		documentXMLDecodeKeepNsScenario(w, s)
 	case "decode-raw-token":
 		documentXMLDecodeKeepNsRawTokenScenario(w, s)
-	default:
-		documentXMLDecodeScenario(w, s)
-	}

+	default:
+		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
+	}
 }

 func documentXMLDecodeScenario(w *bufio.Writer, s formatScenario) {
@@ -1,3 +1,18 @@
+4.27.1:
+  - Added 'json' decoder for support for multiple JSON documents in a single file (e.g. NDJSON)
+  - Fixed XML decoding issue (#1284)
+
+4.26.1:
+  - Switched to new expression parser (#1264)
+  - Don't clobber anchor when adding nodes (#1269)
+  - New error operator for custom validation (#1259)
+  - Added support for --wrapScalar=false in properties encoder (#1241) Thanks @dcarbone
+  - Fix error on multiple assign (#1257) Thanks @care0717
+  - Bumped dependency versions
+
+4.25.4:
+  - Fixed panic when using multiply assign on multiple documents #1256 Thanks @care0717
+
 4.25.3:
  - xml decoder now maintains namespaces by default. Use new flags to disable if required. Thanks @rndmit
  - Length and other similar operators no longer return comments (#1231)
@@ -1,5 +1,5 @@
 name: yq
-version: '4.25.3'
+version: '4.26.1'
 summary: A lightweight and portable command-line YAML processor
 description: |
  The aim of the project is to be the jq or sed of yaml files.
Author	SHA1	Message	Date
Mike Farah	d8f8661567	Fixing from review	2022-08-01 10:16:59 +10:00
Mike Farah	9628aaf8c5	Added CSV roundtrip	2022-08-01 10:05:26 +10:00
Mike Farah	cb0bae128f	Adding CSV decoder	2022-07-31 17:35:36 +10:00
Mike Farah	183a42c249	WIP: adding CSV decoder	2022-07-31 17:31:27 +10:00
Mike Farah	4508bc2dc2	Updating release notes for next release	2022-07-29 10:28:06 +10:00
Mike Farah	8c3a8c085e	Fixes xml decode missing tags #1284	2022-07-29 10:26:50 +10:00
Mike FarahandGitHub	b9a1ef89fe	Added NDJSON decoder (#1281 )	2022-07-27 12:26:22 +10:00
Mike Farah	c2c48bbe1d	Bumping version	2022-07-15 10:27:18 +10:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	97f89c0edb	Bump golang from 1.18.3 to 1.18.4 (#1273 ) Bumps golang from 1.18.3 to 1.18.4. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2022-07-14 15:38:26 +10:00
Mike FarahandGitHub	03ba38eaff	Using new lexer by alecthomas! (#1270 )	2022-07-13 14:45:28 +10:00
Mike Farah	f393cf37c2	Dont clobber anchor when adding nodes - fixes #1269	2022-07-13 11:12:15 +10:00
Mike Farah	f3a8c4b2ec	Added contributing guide on documentation	2022-07-07 11:53:23 +10:00
Mike Farah	86c2b03630	Added error operator for custom validation #1259	2022-06-25 12:46:24 +10:00
Daniel CarboneandGitHub	98b411f82e	adding support for --wrapScalar=false in properties encoder (#1241 ) * adding support for --wrapScalar=false in properties encoder * altering encoder and decoder tests somewhat * adding .idea * Revert "altering encoder and decoder tests somewhat" This reverts commit `e3655130e2`. * adding test scenario for encoding with wrapped scalars	2022-06-25 12:22:03 +10:00
Mike Farah	06d2aaad80	Adding acceptance test for #1257 , fixed panic	2022-06-25 12:15:28 +10:00
HayamaandGitHub	9b47a29864	hotfix: fix panic multiply assign (#1257 )	2022-06-25 12:08:52 +10:00