Added XML decoder

.gitignore

@@ -48,8 +48,4 @@ test*.yml
 
 # man page
 man.md
-yq.1
-
-# debian pkg
-_build
-debian/files
+yq.1

.golangci.yml

@@ -18,14 +18,6 @@ linters:
     - revive
     - unconvert
     - unparam
-linters-settings:
-  depguard:
-    list-type: blacklist
-    include-go-root: true
-    packages:
-      - io/ioutil
-    packages-with-error-message:
-      - io/ioutil: "The 'io/ioutil' package is deprecated. Use corresponding 'os' or 'io' functions instead."
 issues:
   exclude-rules:
   - linters:

README.md

@@ -104,9 +104,6 @@ rm /etc/myfile.tmp
 docker run --rm -v "${PWD}":/workdir mikefarah/yq <command> [flags] [expression ]FILE...
 ```
 
-Note that you can run `yq` in docker without network access and other privileges if you desire,
-namely `--security-opt=no-new-privileges --cap-drop all --network none`.
-
 ```bash
 podman run --rm -v "${PWD}":/workdir mikefarah/yq <command> [flags] [expression ]FILE...
 ```
@@ -198,12 +195,6 @@ webi yq
 See [webi](https://webinstall.dev/)
 Supported by @adithyasunil26 (https://github.com/webinstall/webi-installers/tree/master/yq)
 
-### Arch Linux
-
-```
-pacman -S go-yq
-```
-
 ### Windows:
 [![Chocolatey](https://img.shields.io/chocolatey/v/yq.svg)](https://chocolatey.org/packages/yq)
 [![Chocolatey](https://img.shields.io/chocolatey/dt/yq.svg)](https://chocolatey.org/packages/yq)

acceptance_tests/inputs-format.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-
-setUp() {
-  rm test*.yml || true
-}
-
-testInputXml() {
-  cat >test.yml <<EOL
-<cat legs="4">BiBi</cat>
-EOL
-
-  read -r -d '' expected << EOM
-cat:
-  +content: BiBi
-  +legs: "4"
-EOM
-
-  X=$(./yq e -p=xml test.yml)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea -p=xml test.yml)
-  assertEquals "$expected" "$X"
-}
-
-source ./scripts/shunit2

acceptance_tests/output-format.sh

@@ -102,65 +102,4 @@ EOM
   assertEquals "$expected" "$X"
 }
 
-testOutputXml() {
-  cat >test.yml <<EOL
-a: {b: {c: ["cat"]}}
-EOL
-
-  read -r -d '' expected << EOM
-<a>
-  <b>
-    <c>cat</c>
-  </b>
-</a>
-EOM
-
-  X=$(./yq e --output-format=xml test.yml)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea --output-format=xml test.yml)
-  assertEquals "$expected" "$X"
-}
-
-testOutputXmlShort() {
-  cat >test.yml <<EOL
-a: {b: {c: ["cat"]}}
-EOL
-
-  read -r -d '' expected << EOM
-<a>
-  <b>
-    <c>cat</c>
-  </b>
-</a>
-EOM
-
-  X=$(./yq e --output-format=x test.yml)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea --output-format=x test.yml)
-  assertEquals "$expected" "$X"
-}
-
-testOutputXmComplex() {
-  cat >test.yml <<EOL
-a: {b: {c: ["cat", "dog"], +f: meow}}
-EOL
-
-  read -r -d '' expected << EOM
-<a>
-  <b f="meow">
-    <c>cat</c>
-    <c>dog</c>
-  </b>
-</a>
-EOM
-
-  X=$(./yq e --output-format=x test.yml)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea --output-format=x test.yml)
-  assertEquals "$expected" "$X"
-}
-
 source ./scripts/shunit2

cmd/evaluate_all_command.go

@@ -39,7 +39,7 @@ Note that it consumes more memory than eval.
 	}
 	return cmdEvalAll
 }
-func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
+func evaluateAll(cmd *cobra.Command, args []string) error {
 	// 0 args, read std in
 	// 1 arg, null input, process expression
 	// 1 arg, read file in sequence
@@ -67,11 +67,7 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 		}
 		// need to indirectly call the function so  that completedSuccessfully is
 		// passed when we finish execution as opposed to now
-		defer func() {
-			if cmdError == nil {
-				cmdError = writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully)
-			}
-		}()
+		defer func() { writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully) }()
 	}
 
 	format, err := yqlib.OutputFormatFromString(outputFormat)
@@ -85,9 +81,8 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 	}
 
 	printerWriter := configurePrinterWriter(format, out)
-	encoder := configureEncoder(format)
 
-	printer := yqlib.NewPrinter(encoder, printerWriter)
+	printer := yqlib.NewPrinter(printerWriter, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
 
 	if frontMatter != "" {
 		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])

cmd/evalute_sequence_command.go

@@ -44,16 +44,16 @@ expression and prints the result in sequence.`,
 }
 
 func processExpression(expression string) string {
-
+	var prettyPrintExp = `(... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not))  ) style=""`
 	if prettyPrint && expression == "" {
-		return yqlib.PrettyPrintExp
+		return prettyPrintExp
 	} else if prettyPrint {
-		return fmt.Sprintf("%v | %v", expression, yqlib.PrettyPrintExp)
+		return fmt.Sprintf("%v | %v", expression, prettyPrintExp)
 	}
 	return expression
 }
 
-func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
+func evaluateSequence(cmd *cobra.Command, args []string) error {
 	// 0 args, read std in
 	// 1 arg, null input, process expression
 	// 1 arg, read file in sequence
@@ -80,11 +80,7 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 		}
 		// need to indirectly call the function so  that completedSuccessfully is
 		// passed when we finish execution as opposed to now
-		defer func() {
-			if cmdError == nil {
-				cmdError = writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully)
-			}
-		}()
+		defer func() { writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully) }()
 	}
 
 	format, err := yqlib.OutputFormatFromString(outputFormat)
@@ -93,9 +89,8 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 	}
 
 	printerWriter := configurePrinterWriter(format, out)
-	encoder := configureEncoder(format)
 
-	printer := yqlib.NewPrinter(encoder, printerWriter)
+	printer := yqlib.NewPrinter(printerWriter, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
 
 	decoder, err := configureDecoder()
 	if err != nil {

cmd/root.go

@@ -3,7 +3,6 @@ package cmd
 import (
 	"os"
 
-	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"github.com/spf13/cobra"
 	logging "gopkg.in/op/go-logging.v1"
 )
@@ -38,8 +37,6 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 			}
 
 			logging.SetBackend(backend)
-			yqlib.XmlPreferences.AttributePrefix = xmlAttributePrefix
-			yqlib.XmlPreferences.ContentName = xmlContentName
 		},
 	}
 
@@ -51,18 +48,18 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 		panic(err)
 	}
 
-	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "yaml", "[yaml|y|json|j|props|p|xml|x] output format type.")
-	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "yaml", "[yaml|y|xml|x] parse format for input. Note that json is a subset of yaml.")
+	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "yaml", "[yaml|y|json|j|props|p] output format type.")
+	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "yaml", "[yaml|y|xml|x] input format type.")
 
 	rootCmd.PersistentFlags().StringVar(&xmlAttributePrefix, "xml-attribute-prefix", "+", "prefix for xml attributes")
 	rootCmd.PersistentFlags().StringVar(&xmlContentName, "xml-content-name", "+content", "name for xml content (if no attribute name is present).")
 
-	rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.")
+	rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating yaml docs from scratch.")
 	rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)")
 
 	rootCmd.PersistentFlags().IntVarP(&indent, "indent", "I", 2, "sets indent level for output")
 	rootCmd.Flags().BoolVarP(&version, "version", "V", false, "Print version information and quit")
-	rootCmd.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the file inplace of first file given.")
+	rootCmd.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace of first yaml file given.")
 	rootCmd.PersistentFlags().BoolVarP(&unwrapScalar, "unwrapScalar", "", true, "unwrap scalar, print the value with no quotes, colors or comments")
 	rootCmd.PersistentFlags().BoolVarP(&prettyPrint, "prettyPrint", "P", false, "pretty print, shorthand for '... style = \"\"'")
 	rootCmd.PersistentFlags().BoolVarP(&exitStatus, "exit-status", "e", false, "set exit status if there are no matches or null or false is returned")
@@ -70,7 +67,7 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 	rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors")
 	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
 	rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
-	rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.")
+	rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression. This is a workaround for go-yaml to persist header content properly.")
 
 	rootCmd.PersistentFlags().StringVarP(&splitFileExp, "split-exp", "s", "", "print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter.")
 

cmd/utils.go

@@ -73,21 +73,3 @@ func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) yql
 	}
 	return printerWriter
 }
-
-func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder {
-	switch format {
-	case yqlib.JsonOutputFormat:
-		return yqlib.NewJsonEncoder(indent)
-	case yqlib.PropsOutputFormat:
-		return yqlib.NewPropertiesEncoder()
-	case yqlib.CsvOutputFormat:
-		return yqlib.NewCsvEncoder(',')
-	case yqlib.TsvOutputFormat:
-		return yqlib.NewCsvEncoder('\t')
-	case yqlib.YamlOutputFormat:
-		return yqlib.NewYamlEncoder(indent, colorsEnabled, !noDocSeparators, unwrapScalar)
-	case yqlib.XmlOutputFormat:
-		return yqlib.NewXmlEncoder(indent, xmlAttributePrefix, xmlContentName)
-	}
-	panic("invalid encoder")
-}

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.17.2"
+	Version = "4.16.2"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

debian/changelog

@@ -1,112 +1,3 @@
-yq (4.16.2) focal; urgency=medium
-
-  * Fixed with semicolon space issue
-  * Updating with documentation
-  * Added STDIN example to the top
-  * minor readme cleanup
-  * Help text tweak
-  * Fixed docker timeout - simplify docker builds
-  * New release with docker build fixes
-  * Updating to go 1.17 to fix CVE #944
-  * Fix a typo in root.go
-  * Skip the tests if the nocheck Debian build option is specified
-  * Fixed select bug (#958)
-  * Sped up explode operator
-  * Slight performance improvement to context.ChildContext
-  * Speed up multiply
-  * Update README with recently added / changed options
-  * Make deepMatch report in linear time
-  * Removed leadingContentPreProcessing flag - header preprocessing is stable
-  * Revert "Removed leadingContentPreProcessing flag - header preprocessing is stable"
-  * Keep flag, it is needed in corner cases
-  * Updated Readme
-  * Man page
-  * Fixed expression parsing bug #970
-  * Bumping go-lang, docker versions
-  * Added test release flow
-  * Updated github action release to generate man page
-  * Bumping version
-  * Removing no longer needed github action
-  * Added decoder op
-  * Fixed newline handling when decoding/encoding
-  * Fixed newline handling in encoder/decoder
-  * Can specify indent in encode ops
-  * Added group_by operator
-  * Added flatten operator
-  * Fixed flatten error message
-  * Improving docs
-  * Split printer
-  * Refactored command logic
-  * Fix JSON encoding removing null #985
-  * Fixed acceptance tests
-  * gitbook
-  * Update document generation script
-  * Updating README
-  * Updating release instructions
-  * github action no longer uses data1.yml
-  * Create dependabot.yml
-  * Bump actions/create-release from 1.0.0 to 1.1.4
-  * Bump actions/setup-go from 1 to 2.1.4
-  * Bump github.com/goccy/go-yaml from 1.8.9 to 1.9.4
-  * Bump github.com/jinzhu/copier from 0.2.8 to 0.3.2
-  * Bump github.com/fatih/color from 1.10.0 to 1.13.0
-  * Bump github.com/spf13/cobra from 1.1.3 to 1.2.1
-  * Update dependabot.yml
-  * Update go.yml
-  * add build check to PRs
-  * Include secure as part of build process
-  * Fixing bad label in github action
-  * fixed printer test
-  * remove leading content indicator
-  * Fixed header preprocessing!
-  * lint : define golangci configuration file
-  * Update check.sh
-  * Load file acceptance test
-  * Minor improvement on handling front matter
-  * Improved load doc
-  * feature: detect MANPATh and install there
-  * Update install-man-page.sh
-  * simplify prod stage, move version label to action
-  * add labels, quote some values
-  * enable errorlint linter
-  * Added errorlint to devtools
-  * Added key operator
-  * Added more tests
-  * Fixing comments
-  * Attempt to fix golint problem
-  * Include version query for tools
-  * Clean up errored file?
-  * enable misspell linter
-  * updated readme
-  * update Golangci version to v1.43.0
-  * gci linter
-  * Better merge array by key example
-  * Added credit for merge by array example
-  * Better formatting of merge arrays example
-  * Better merge example
-  * Add accessor for the yq logger instance (#1013)
-  * Fixed collect op when working with multiple nodes
-  * Added map, map_values
-  * Add support for Podman as well as Docker (#1026)
-  * Bump github.com/jinzhu/copier from 0.3.2 to 0.3.4 (#1027)
-  * Added csv, tsv output formats
-  * Added encoder tests
-  * Cleanup test
-  * Fixed docker permission issue #1014
-  * Recording release notes for next release
-  * Assignment op no longer clobbers anchor (#1029)
-  * Added sort_by operator
-  * Improved error message
-  * Improved tips and tricks
-  * Report while filename failed to parse #1030
-  * Added script for extracting checksums
-  * Improved extract-checksum.sh
-  * Bump github.com/spf13/cobra from 1.2.1 to 1.3.0 (#1039)
-  * enable more linters (#1043)
-  * Bump golang compiler #1037
-
- -- Roberto Mier Escandon <rmescandon@gmail.com>  Tue, 21 Dec 2021 09:41:44 +0000
-
 yq (4.13.0) focal; urgency=medium
 
   * New `with` operator for making multiple changes to a given path

debian/control

@@ -3,8 +3,7 @@ Section: devel
 Priority: optional
 Maintainer: Roberto Mier Escandón <rmescandon@gmail.com>
 Build-Depends: debhelper (>=10),
-               golang-1.17-go,
-               pandoc,
+               golang-1.15,
                rsync
 Standards-Version: 4.1.4
 Homepage: https://github.com/mikefarah/yq.git

debian/rules

@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 #
-# Copyright (C) 2018-2021 Roberto Mier Escandón <rmescandon@gmail.com>
+# Copyright (C) 2018 Roberto Mier Escandón <rmescandon@gmail.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
@@ -17,21 +17,20 @@
 PROJECT := yq
 OWNER := mikefarah
 REPO := github.com
+GOVERSION := 1.15
 
 export DH_OPTIONS
 export DH_GOPKG := ${REPO}/${OWNER}/${PROJECT}
-export GOROOT := /usr/local/go
+export GOROOT := /usr/lib/go-${GOVERSION}
 export GOPATH := ${CURDIR}/_build
 export GOBIN := ${GOPATH}/bin
 export PATH := ${GOROOT}/bin:${GOBIN}:${PATH}
 export GOCACHE := /tmp/gocache
 export GOFLAGS := -mod=vendor
-export GO111MODULE := on
 
 SRCDIR := ${GOPATH}/src/${DH_GOPKG}
 DESTDIR := ${CURDIR}/debian/${PROJECT}
 BINDIR := /usr/bin
-MANDIR := /usr/share/man/man1/
 ASSETSDIR := /usr/share/${PROJECT}
 
 %:
@@ -43,17 +42,7 @@ override_dh_auto_build:
 	# copy project to local srcdir to build from there
 	rsync -avz --progress --exclude=_build --exclude=debian --exclude=tmp. --exclude=go.mod --exclude=docs . $(SRCDIR)
 	# build go code
-	( \
-		cd ${SRCDIR} && \
-		go install -buildmode=pie ./... \
-	)
-
-	# build man page
-	( \
-		cd ${SRCDIR} && \
-		./scripts/generate-man-page-md.sh && \
-		./scripts/generate-man-page.sh \
-	)
+	(cd ${SRCDIR} && go install -buildmode=pie ./...)
 	
 override_dh_auto_test:
 ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS)))
@@ -65,10 +54,6 @@ override_dh_auto_install:
 	cp -f ${SRCDIR}/LICENSE ${DESTDIR}/${ASSETSDIR}
 	chmod a+x ${DESTDIR}/${BINDIR}/yq
 
-	# man
-	mkdir -p "${DESTDIR}"/"${MANDIR}"
-	cp "${SRCDIR}"/yq.1 "${DESTDIR}"/"${MANDIR}" \
-
 override_dh_auto_clean:
 	dh_clean
 	rm -rf ${CURDIR}/_build

examples/data1.yaml

@@ -1 +1,15 @@
-cat: purrs
+strings:
+  - a: banana
+  - a: cat
+  - a: apple
+
+numbers:
+  - a: 12
+  - a: 13
+  - a: 120
+
+
+obj:
+  - a: {cat: "adog"}
+  - a: {cat: "doga"}
+  - a: apple

examples/mike.xml

@@ -1,14 +1,4 @@
-<!-- before cat -->
-<cat>
-	<!-- in cat before -->
-	<x>3<!-- multi
-line comment
-for x --></x>
-	<y>
-		<!-- in y before -->
-		<d><!-- in d before -->4<!-- in d after --></d>
-		<!-- in y after -->
-	</y>
-	<!-- in_cat_after -->
-</cat>
-<!-- after cat -->
+  <?xml version="1.0" encoding="UTF-8"?>
+  <cat>3f</cat>
+  <dog>meow:as</dog>
+  <dog3>true</dog3>

go.mod

@@ -3,10 +3,9 @@ module github.com/mikefarah/yq/v4
 require (
 	github.com/elliotchance/orderedmap v1.4.0
 	github.com/fatih/color v1.13.0
-	github.com/goccy/go-yaml v1.9.5
-	github.com/jinzhu/copier v0.3.5
+	github.com/goccy/go-yaml v1.9.4
+	github.com/jinzhu/copier v0.3.4
 	github.com/magiconair/properties v1.8.5
-	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
 	github.com/spf13/cobra v1.3.0
 	github.com/timtadh/lexmachine v0.2.2
 	golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d

go.sum

@@ -123,8 +123,8 @@ github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTM
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/goccy/go-yaml v1.9.5 h1:Eh/+3uk9kLxG4koCX6lRMAPS1OaMSAi+FJcya0INdB0=
-github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
+github.com/goccy/go-yaml v1.9.4 h1:S0GCYjwHKVI6IHqio7QWNKNThUl6NLzFd/g8Z65Axw8=
+github.com/goccy/go-yaml v1.9.4/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
@@ -235,8 +235,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg=
-github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
+github.com/jinzhu/copier v0.3.4 h1:mfU6jI9PtCeUjkjQ322dlff9ELjGDu975C2p/nrubVI=
+github.com/jinzhu/copier v0.3.4/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -292,8 +292,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
-github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
-github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=

pkg/yqlib/decode_xml_test.go

@@ -0,0 +1,110 @@
+package yqlib
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+	yaml "gopkg.in/yaml.v3"
+)
+
+func decodeXml(t *testing.T, xml string) *CandidateNode {
+	decoder := NewXmlDecoder("+", "+content")
+
+	decoder.Init(strings.NewReader(xml))
+
+	node := &yaml.Node{}
+	err := decoder.Decode(node)
+	if err != nil {
+		t.Error(err, "fail to decode", xml)
+	}
+	return &CandidateNode{Node: node}
+}
+
+type xmlScenario struct {
+	inputXml       string
+	expected       string
+	description    string
+	subdescription string
+	skipDoc        bool
+}
+
+var xmlScenarios = []xmlScenario{
+	{
+		description: "Parse xml: simple",
+		inputXml:    "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat>meow</cat>",
+		expected:    "D0, P[], (doc)::cat: meow\n",
+	},
+	{
+		description:    "Parse xml: array",
+		subdescription: "Consecutive nodes with identical xml names are assumed to be arrays.",
+		inputXml:       "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<animal>1</animal>\n<animal>2</animal>",
+		expected:       "D0, P[], (doc)::animal:\n    - \"1\"\n    - \"2\"\n",
+	},
+	{
+		description:    "Parse xml: attributes",
+		subdescription: "Attributes are converted to fields, with the attribute prefix.",
+		inputXml:       "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">\n  <legs>7</legs>\n</cat>",
+		expected:       "D0, P[], (doc)::cat:\n    +legs: \"4\"\n    legs: \"7\"\n",
+	},
+	{
+		description:    "Parse xml: attributes with content",
+		subdescription: "Content is added as a field, using the content name",
+		inputXml:       "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">meow</cat>",
+		expected:       "D0, P[], (doc)::cat:\n    +content: meow\n    +legs: \"4\"\n",
+	},
+}
+
+func testXmlScenario(t *testing.T, s *xmlScenario) {
+	var actual = resultToString(t, decodeXml(t, s.inputXml))
+	test.AssertResult(t, s.expected, actual)
+}
+
+func documentXmlScenario(t *testing.T, w *bufio.Writer, i interface{}) {
+	s := i.(xmlScenario)
+
+	if s.skipDoc {
+		return
+	}
+	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
+
+	if s.subdescription != "" {
+		writeOrPanic(w, s.subdescription)
+		writeOrPanic(w, "\n\n")
+	}
+
+	writeOrPanic(w, "Given a sample.xml file of:\n")
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.inputXml))
+
+	writeOrPanic(w, "then\n")
+	writeOrPanic(w, "```bash\nyq e sample.xml\n```\n")
+	writeOrPanic(w, "will output\n")
+
+	var output bytes.Buffer
+	printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
+
+	node := decodeXml(t, s.inputXml)
+
+	err := printer.PrintResults(node.AsList())
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", output.String()))
+
+}
+
+func TestXmlScenarios(t *testing.T) {
+	for _, tt := range xmlScenarios {
+		testXmlScenario(t, &tt)
+	}
+	genericScenarios := make([]interface{}, len(xmlScenarios))
+	for i, s := range xmlScenarios {
+		genericScenarios[i] = s
+	}
+	documentScenarios(t, "usage", "xml", genericScenarios, documentXmlScenario)
+}

pkg/yqlib/decoder_xml.go

@@ -4,7 +4,6 @@ import (
 	"encoding/xml"
 	"fmt"
 	"io"
-	"strings"
 	"unicode"
 
 	"golang.org/x/net/html/charset"
@@ -32,15 +31,15 @@ func InputFormatFromString(format string) (InputFormat, error) {
 type xmlDecoder struct {
 	reader          io.Reader
 	attributePrefix string
-	contentName     string
+	contentPrefix   string
 	finished        bool
 }
 
-func NewXmlDecoder(attributePrefix string, contentName string) Decoder {
-	if contentName == "" {
-		contentName = "content"
+func NewXmlDecoder(attributePrefix string, contentPrefix string) Decoder {
+	if contentPrefix == "" {
+		contentPrefix = "content"
 	}
-	return &xmlDecoder{attributePrefix: attributePrefix, contentName: contentName, finished: false}
+	return &xmlDecoder{attributePrefix: attributePrefix, contentPrefix: contentPrefix, finished: false}
 }
 
 func (dec *xmlDecoder) Init(reader io.Reader) {
@@ -61,41 +60,20 @@ func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yaml.Node, error) {
 	return yamlNode, nil
 }
 
-func (dec *xmlDecoder) processComment(c string) string {
-	if c == "" {
-		return ""
-	}
-	return "#" + strings.TrimRight(c, " ")
-}
-
 func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
-	log.Debug("createMap: headC: %v, footC: %v", n.HeadComment, n.FootComment)
-	yamlNode := &yaml.Node{Kind: yaml.MappingNode}
+	yamlNode := &yaml.Node{Kind: yaml.MappingNode, HeadComment: n.Comment}
 
 	if len(n.Data) > 0 {
-		label := dec.contentName
-		labelNode := createScalarNode(label, label)
-		labelNode.HeadComment = dec.processComment(n.HeadComment)
-		labelNode.FootComment = dec.processComment(n.FootComment)
-		yamlNode.Content = append(yamlNode.Content, labelNode, createScalarNode(n.Data, n.Data))
+		label := dec.contentPrefix
+		yamlNode.Content = append(yamlNode.Content, createScalarNode(label, label), createScalarNode(n.Data, n.Data))
 	}
 
-	for i, keyValuePair := range n.Children {
+	for _, keyValuePair := range n.Children {
 		label := keyValuePair.K
 		children := keyValuePair.V
 		labelNode := createScalarNode(label, label)
 		var valueNode *yaml.Node
 		var err error
-
-		if i == 0 {
-			labelNode.HeadComment = dec.processComment(n.HeadComment)
-
-		}
-
-		// if i == len(n.Children)-1 {
-		labelNode.FootComment = dec.processComment(keyValuePair.FootComment)
-		// }
-
 		log.Debug("len of children in %v is %v", label, len(children))
 		if len(children) > 1 {
 			valueNode, err = dec.createSequence(children)
@@ -103,13 +81,6 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
 				return nil, err
 			}
 		} else {
-			// comment hack for maps of scalars
-			// if the value is a scalar, the head comment of the scalar needs to go on the key?
-			// add tests for <z/> as well as multiple <ds> of inputXmlWithComments > yaml
-			if len(children[0].Children) == 0 && children[0].HeadComment != "" {
-				labelNode.HeadComment = labelNode.HeadComment + "\n" + strings.TrimSpace(children[0].HeadComment)
-				children[0].HeadComment = ""
-			}
 			valueNode, err = dec.convertToYamlNode(children[0])
 			if err != nil {
 				return nil, err
@@ -126,11 +97,7 @@ func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yaml.Node, error) {
 		return dec.createMap(n)
 	}
 	scalar := createScalarNode(n.Data, n.Data)
-	log.Debug("scalar headC: %v, footC: %v", n.HeadComment, n.FootComment)
-	scalar.HeadComment = dec.processComment(n.HeadComment)
-	scalar.LineComment = dec.processComment(n.LineComment)
-	scalar.FootComment = dec.processComment(n.FootComment)
-
+	scalar.HeadComment = n.Comment
 	return scalar, nil
 }
 
@@ -157,17 +124,14 @@ func (dec *xmlDecoder) Decode(rootYamlNode *yaml.Node) error {
 }
 
 type xmlNode struct {
-	Children    []*xmlChildrenKv
-	HeadComment string
-	FootComment string
-	LineComment string
-	Data        string
+	Children []*xmlChildrenKv
+	Comment  string
+	Data     string
 }
 
 type xmlChildrenKv struct {
-	K           string
-	V           []*xmlNode
-	FootComment string
+	K string
+	V []*xmlNode
 }
 
 // AddChild appends a node to the list of children
@@ -194,7 +158,6 @@ type element struct {
 	parent *element
 	n      *xmlNode
 	label  string
-	state  string
 }
 
 // this code is heavily based on https://github.com/basgys/goxml2json
@@ -220,8 +183,6 @@ func (dec *xmlDecoder) decodeXml(root *xmlNode) error {
 
 		switch se := t.(type) {
 		case xml.StartElement:
-			log.Debug("start element %v", se.Name.Local)
-			elem.state = "started"
 			// Build new a new current element and link it to its parent
 			elem = &element{
 				parent: elem,
@@ -236,13 +197,7 @@ func (dec *xmlDecoder) decodeXml(root *xmlNode) error {
 		case xml.CharData:
 			// Extract XML data (if any)
 			elem.n.Data = trimNonGraphic(string(se))
-			if elem.n.Data != "" {
-				elem.state = "chardata"
-				log.Debug("chardata [%v] for %v", elem.n.Data, elem.label)
-			}
 		case xml.EndElement:
-			log.Debug("end element %v", elem.label)
-			elem.state = "finished"
 			// And add it to its parent list
 			if elem.parent != nil {
 				elem.parent.n.AddChild(elem.label, elem.n)
@@ -251,49 +206,13 @@ func (dec *xmlDecoder) decodeXml(root *xmlNode) error {
 			// Then change the current element to its parent
 			elem = elem.parent
 		case xml.Comment:
-
-			commentStr := string(xml.CharData(se))
-			if elem.state == "started" {
-				applyFootComment(elem, commentStr)
-
-			} else if elem.state == "chardata" {
-				log.Debug("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
-				elem.n.LineComment = joinFilter([]string{elem.n.LineComment, commentStr})
-			} else {
-				log.Debug("got a head comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
-				elem.n.HeadComment = joinFilter([]string{elem.n.HeadComment, commentStr})
-			}
-
+			elem.n.Comment = trimNonGraphic(string(xml.CharData(se)))
 		}
 	}
 
 	return nil
 }
 
-func applyFootComment(elem *element, commentStr string) {
-
-	// first lets try to put the comment on the last child
-	if len(elem.n.Children) > 0 {
-		lastChildIndex := len(elem.n.Children) - 1
-		childKv := elem.n.Children[lastChildIndex]
-		log.Debug("got a foot comment for %v: [%v]", childKv.K, commentStr)
-		childKv.FootComment = joinFilter([]string{elem.n.FootComment, commentStr})
-	} else {
-		log.Debug("got a foot comment for %v: [%v]", elem.label, commentStr)
-		elem.n.FootComment = joinFilter([]string{elem.n.FootComment, commentStr})
-	}
-}
-
-func joinFilter(rawStrings []string) string {
-	stringsToJoin := make([]string, 0)
-	for _, str := range rawStrings {
-		if str != "" {
-			stringsToJoin = append(stringsToJoin, str)
-		}
-	}
-	return strings.Join(stringsToJoin, " ")
-}
-
 // trimNonGraphic returns a slice of the string s, with all leading and trailing
 // non graphic characters and spaces removed.
 //

pkg/yqlib/doc/operators/encode-decode.md

@@ -14,13 +14,10 @@ These operators are useful to process yaml documents that have stringified embed
 | Properties |  | to_props/@props |
 | CSV |  | to_csv/@csv |
 | TSV |  | to_tsv/@tsv |
-| XML | from_xml | to_xml(i)/@xml |
 
 
 CSV and TSV format both accept either a single array or scalars (representing a single row), or an array of array of scalars (representing multiple rows). 
 
-XML uses the `--xml-attribute-prefix` and `xml-content-name` flags to identify attributes and content fields.
-
 
 ## Encode value as json string
 Given a sample.yml file of:
@@ -274,64 +271,6 @@ cat	thing1,thing2	true	3.40
 dog	thing3	false	12
 ```
 
-## Encode value as xml string
-Given a sample.yml file of:
-```yaml
-a:
-  cool:
-    foo: bar
-    +id: hi
-```
-then
-```bash
-yq eval '.a | to_xml' sample.yml
-```
-will output
-```yaml
-<cool id="hi">
-  <foo>bar</foo>
-</cool>
-
-```
-
-## Encode value as xml string on a single line
-Given a sample.yml file of:
-```yaml
-a:
-  cool:
-    foo: bar
-    +id: hi
-```
-then
-```bash
-yq eval '.a | @xml' sample.yml
-```
-will output
-```yaml
-<cool id="hi"><foo>bar</foo></cool>
-
-```
-
-## Encode value as xml string with custom indentation
-Given a sample.yml file of:
-```yaml
-a:
-  cool:
-    foo: bar
-    +id: hi
-```
-then
-```bash
-yq eval '{"cat": .a | to_xml(1)}' sample.yml
-```
-will output
-```yaml
-cat: |
-  <cool id="hi">
-   <foo>bar</foo>
-  </cool>
-```
-
 ## Decode a xml encoded string
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/operators/headers/encode-decode.md

@@ -14,10 +14,8 @@ These operators are useful to process yaml documents that have stringified embed
 | Properties |  | to_props/@props |
 | CSV |  | to_csv/@csv |
 | TSV |  | to_tsv/@tsv |
-| XML | from_xml | to_xml(i)/@xml |
+| XML | from_xml |  |
 
 
 CSV and TSV format both accept either a single array or scalars (representing a single row), or an array of array of scalars (representing multiple rows). 
 
-XML uses the `--xml-attribute-prefix` and `xml-content-name` flags to identify attributes and content fields.
-

pkg/yqlib/doc/operators/headers/multiply-merge.md

@@ -10,10 +10,9 @@ Note that when merging objects, this operator returns the merged object (not the
 ### Merge Flags
 You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`.  See examples below
 
-- `+` append arrays
-- `d` deeply merge arrays
-- `?` only merge _existing_ fields
-- `n` only merge _new_ fields
+- `+` to append arrays
+- `?` to only merge existing fields
+- `d` to deeply merge arrays
 
 ### Merging files
 Note the use of `eval-all` to ensure all documents are loaded into memory.

pkg/yqlib/doc/operators/multiply-merge.md

@@ -10,10 +10,9 @@ Note that when merging objects, this operator returns the merged object (not the
 ### Merge Flags
 You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`.  See examples below
 
-- `+` append arrays
-- `d` deeply merge arrays
-- `?` only merge _existing_ fields
-- `n` only merge _new_ fields
+- `+` to append arrays
+- `?` to only merge existing fields
+- `d` to deeply merge arrays
 
 ### Merging files
 Note the use of `eval-all` to ensure all documents are loaded into memory.
@@ -149,27 +148,6 @@ thing: two
 cat: frog
 ```
 
-## Merge, only new fields
-Given a sample.yml file of:
-```yaml
-a:
-  thing: one
-  cat: frog
-b:
-  missing: two
-  thing: two
-```
-then
-```bash
-yq eval '.a *n .b' sample.yml
-```
-will output
-```yaml
-thing: one
-cat: frog
-missing: two
-```
-
 ## Merge, appending arrays
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/operators/select.md

@@ -2,7 +2,7 @@
 
 Select is used to filter arrays and maps by a boolean expression.
 
-## Select elements from array using wildcard prefix
+## Select elements from array
 Given a sample.yml file of:
 ```yaml
 - cat
@@ -19,99 +19,7 @@ cat
 goat
 ```
 
-## Select elements from array using wildcard suffix
-Given a sample.yml file of:
-```yaml
-- go-kart
-- goat
-- dog
-```
-then
-```bash
-yq eval '.[] | select(. == "go*")' sample.yml
-```
-will output
-```yaml
-go-kart
-goat
-```
-
-## Select elements from array using wildcard prefix and suffix
-Given a sample.yml file of:
-```yaml
-- ago
-- go
-- meow
-- going
-```
-then
-```bash
-yq eval '.[] | select(. == "*go*")' sample.yml
-```
-will output
-```yaml
-ago
-go
-going
-```
-
-## Select elements from array with regular expression
-See more regular expression examples under the `string` operator docs.
-
-Given a sample.yml file of:
-```yaml
-- this_0
-- not_this
-- nor_0_this
-- thisTo_4
-```
-then
-```bash
-yq eval '.[] | select(test("[a-zA-Z]+_[0-9]$"))' sample.yml
-```
-will output
-```yaml
-this_0
-thisTo_4
-```
-
-## Select items from a map
-Given a sample.yml file of:
-```yaml
-things: cat
-bob: goat
-horse: dog
-```
-then
-```bash
-yq eval '.[] | select(. == "cat" or test("og$"))' sample.yml
-```
-will output
-```yaml
-cat
-dog
-```
-
-## Use select and with_entries to filter map keys
-Given a sample.yml file of:
-```yaml
-name: bob
-legs: 2
-game: poker
-```
-then
-```bash
-yq eval 'with_entries(select(.key | test("ame$")))' sample.yml
-```
-will output
-```yaml
-name: bob
-game: poker
-```
-
-## Select multiple items in a map and update
-Note the brackets around the entire LHS.
-
+## Select and update matching values in map
 Given a sample.yml file of:
 ```yaml
 a:
@@ -121,7 +29,7 @@ a:
 ```
 then
 ```bash
-yq eval '(.a.[] | select(. == "cat" or . == "goat")) |= "rabbit"' sample.yml
+yq eval '(.a.[] | select(. == "*at")) |= "rabbit"' sample.yml
 ```
 will output
 ```yaml

pkg/yqlib/doc/usage/convert.md

@@ -1,126 +0,0 @@
-# JSON
-
-Encode and decode to and from JSON. Note that YAML is a _superset_ of JSON - so `yq` can read any json file without doing anything special.
-
-This means you don't need to 'convert' a JSON file to YAML - however if you want idiomatic YAML styling, then you can use the `-P/--prettyPrint` flag, see examples below.
-
-## Parse json: simple
-JSON is a subset of yaml, so all you need to do is prettify the output
-
-Given a sample.json file of:
-```json
-{"cat": "meow"}
-```
-then
-```bash
-yq e -P '.' sample.json
-```
-will output
-```yaml
-cat: meow
-```
-
-## Parse json: complex
-JSON is a subset of yaml, so all you need to do is prettify the output
-
-Given a sample.json file of:
-```json
-{"a":"Easy! as one two three","b":{"c":2,"d":[3,4]}}
-```
-then
-```bash
-yq e -P '.' sample.json
-```
-will output
-```yaml
-a: Easy! as one two three
-b:
-  c: 2
-  d:
-    - 3
-    - 4
-```
-
-## Encode json: simple
-Given a sample.yml file of:
-```yaml
-cat: meow
-```
-then
-```bash
-yq e -o=json '.' sample.yml
-```
-will output
-```json
-{
-  "cat": "meow"
-}
-```
-
-## Encode json: simple - in one line
-Given a sample.yml file of:
-```yaml
-cat: meow # this is a comment, and it will be dropped.
-```
-then
-```bash
-yq e -o=json -I=0 '.' sample.yml
-```
-will output
-```json
-{"cat":"meow"}
-```
-
-## Encode json: comments
-Given a sample.yml file of:
-```yaml
-cat: meow # this is a comment, and it will be dropped.
-```
-then
-```bash
-yq e -o=json '.' sample.yml
-```
-will output
-```json
-{
-  "cat": "meow"
-}
-```
-
-## Encode json: anchors
-Anchors are dereferenced
-
-Given a sample.yml file of:
-```yaml
-cat: &ref meow
-anotherCat: *ref
-```
-then
-```bash
-yq e -o=json '.' sample.yml
-```
-will output
-```json
-{
-  "cat": "meow",
-  "anotherCat": "meow"
-}
-```
-
-## Encode json: multiple results
-Each matching node is converted into a json doc. This is best used with 0 indent (json document per line)
-
-Given a sample.yml file of:
-```yaml
-things: [{stuff: cool}, {whatever: cat}]
-```
-then
-```bash
-yq e -o=json -I=0 '.things[]' sample.yml
-```
-will output
-```json
-{"stuff":"cool"}
-{"whatever":"cat"}
-```
-

pkg/yqlib/doc/usage/headers/convert.md

@@ -1,5 +0,0 @@
-# JSON
-
-Encode and decode to and from JSON. Note that YAML is a _superset_ of JSON - so `yq` can read any json file without doing anything special.
-
-This means you don't need to 'convert' a JSON file to YAML - however if you want idiomatic YAML styling, then you can use the `-P/--prettyPrint` flag, see examples below.

pkg/yqlib/doc/usage/headers/xml.md

@@ -1,8 +1,8 @@
 # XML
 
-Encode and decode to and from XML. Whitespace is not conserved for round trips - but the order of the fields are.
+At the moment, `yq` only supports decoding `xml` (into one of the other supported output formats).
 
-Consecutive xml nodes with the same name are assumed to be arrays.
+As yaml does not have the concept of attributes, these are converted to regular fields with a prefix to prevent clobbering. Consecutive xml nodes with the same name are assumed to be arrays.
 
 All values in XML are assumed to be strings - but you can use `from_yaml` to parse them into their correct types:
 
@@ -10,10 +10,3 @@ All values in XML are assumed to be strings - but you can use `from_yaml` to par
 ```
 yq e -p=xml '.myNumberField |= from_yaml' my.xml
 ```
-
-
-```xml
-<cat name="tiger">meow</cat>
-```
-
-The content of the node will be set as a field in the map with the key "+content". Use the `--xml-content-name` flag to change this.

pkg/yqlib/doc/usage/xml.md

@@ -1,8 +1,8 @@
 # XML
 
-Encode and decode to and from XML. Whitespace is not conserved for round trips - but the order of the fields are.
+At the moment, `yq` only supports decoding `xml` (into one of the other supported output formats).
 
-Consecutive xml nodes with the same name are assumed to be arrays.
+As yaml does not have the concept of attributes, these are converted to regular fields with a prefix to prevent clobbering. Consecutive xml nodes with the same name are assumed to be arrays.
 
 All values in XML are assumed to be strings - but you can use `from_yaml` to parse them into their correct types:
 
@@ -11,13 +11,6 @@ All values in XML are assumed to be strings - but you can use `from_yaml` to par
 yq e -p=xml '.myNumberField |= from_yaml' my.xml
 ```
 
-
-```xml
-<cat name="tiger">meow</cat>
-```
-
-The content of the node will be set as a field in the map with the key "+content". Use the `--xml-content-name` flag to change this.
-
 ## Parse xml: simple
 Given a sample.xml file of:
 ```xml
@@ -26,7 +19,7 @@ Given a sample.xml file of:
 ```
 then
 ```bash
-yq e -p=xml '.' sample.xml
+yq e sample.xml
 ```
 will output
 ```yaml
@@ -44,7 +37,7 @@ Given a sample.xml file of:
 ```
 then
 ```bash
-yq e -p=xml '.' sample.xml
+yq e sample.xml
 ```
 will output
 ```yaml
@@ -54,7 +47,7 @@ animal:
 ```
 
 ## Parse xml: attributes
-Attributes are converted to fields, with the default attribute prefix '+'. Use '--xml-attribute-prefix` to set your own.
+Attributes are converted to fields, with the attribute prefix.
 
 Given a sample.xml file of:
 ```xml
@@ -65,7 +58,7 @@ Given a sample.xml file of:
 ```
 then
 ```bash
-yq e -p=xml '.' sample.xml
+yq e sample.xml
 ```
 will output
 ```yaml
@@ -75,7 +68,7 @@ cat:
 ```
 
 ## Parse xml: attributes with content
-Content is added as a field, using the default content name of '+content'. Use `--xml-content-name` to set your own.
+Content is added as a field, using the content name
 
 Given a sample.xml file of:
 ```xml
@@ -84,7 +77,7 @@ Given a sample.xml file of:
 ```
 then
 ```bash
-yq e -p=xml '.' sample.xml
+yq e sample.xml
 ```
 will output
 ```yaml
@@ -93,192 +86,3 @@ cat:
   +legs: "4"
 ```
 
-## Parse xml: with comments
-A best attempt is made to preserve comments.
-
-Given a sample.xml file of:
-```xml
-
-<!-- before cat -->
-<cat>
-	<!-- in cat before -->
-	<x>3<!-- multi
-line comment 
-for x --></x>
-	<!-- before y -->
-	<y>
-		<!-- in y before -->
-		<d><!-- in d before -->z<!-- in d after --></d>
-		
-		<!-- in y after -->
-	</y>
-	<!-- in_cat_after -->
-</cat>
-<!-- after cat -->
-
-```
-then
-```bash
-yq e -p=xml '.' sample.xml
-```
-will output
-```yaml
-# before cat
-cat:
-  # in cat before
-  x: "3" # multi
-  # line comment 
-  # for x
-  # before y
-
-  y:
-    # in y before
-    # in d before
-    d: z # in d after
-    # in y after
-  # in_cat_after
-# after cat
-```
-
-## Encode xml: simple
-Given a sample.yml file of:
-```yaml
-cat: purrs
-```
-then
-```bash
-yq e -o=xml '.' sample.yml
-```
-will output
-```xml
-<cat>purrs</cat>
-```
-
-## Encode xml: array
-Given a sample.yml file of:
-```yaml
-pets:
-  cat:
-    - purrs
-    - meows
-```
-then
-```bash
-yq e -o=xml '.' sample.yml
-```
-will output
-```xml
-<pets>
-  <cat>purrs</cat>
-  <cat>meows</cat>
-</pets>
-```
-
-## Encode xml: attributes
-Fields with the matching xml-attribute-prefix are assumed to be attributes.
-
-Given a sample.yml file of:
-```yaml
-cat:
-  +name: tiger
-  meows: true
-
-```
-then
-```bash
-yq e -o=xml '.' sample.yml
-```
-will output
-```xml
-<cat name="tiger">
-  <meows>true</meows>
-</cat>
-```
-
-## Encode xml: attributes with content
-Fields with the matching xml-content-name is assumed to be content.
-
-Given a sample.yml file of:
-```yaml
-cat:
-  +name: tiger
-  +content: cool
-
-```
-then
-```bash
-yq e -o=xml '.' sample.yml
-```
-will output
-```xml
-<cat name="tiger">cool</cat>
-```
-
-## Encode xml: comments
-A best attempt is made to copy comments to xml.
-
-Given a sample.yml file of:
-```yaml
-# above_cat
-cat: # inline_cat
-  # above_array
-  array: # inline_array
-    - val1 # inline_val1
-    # above_val2
-    - val2 # inline_val2
-# below_cat
-
-```
-then
-```bash
-yq e -o=xml '.' sample.yml
-```
-will output
-```xml
-<!-- above_cat inline_cat --><cat><!-- above_array inline_array -->
-  <array>val1<!-- inline_val1 --></array>
-  <array><!-- above_val2 -->val2<!-- inline_val2 --></array>
-</cat><!-- below_cat -->
-```
-
-## Round trip: with comments
-A best effort is made, but comment positions and white space are not preserved perfectly.
-
-Given a sample.xml file of:
-```xml
-
-<!-- before cat -->
-<cat>
-	<!-- in cat before -->
-	<x>3<!-- multi
-line comment 
-for x --></x>
-	<!-- before y -->
-	<y>
-		<!-- in y before -->
-		<d><!-- in d before -->z<!-- in d after --></d>
-		
-		<!-- in y after -->
-	</y>
-	<!-- in_cat_after -->
-</cat>
-<!-- after cat -->
-
-```
-then
-```bash
-yq e -p=xml -o=xml '.' sample.xml
-```
-will output
-```xml
-<!-- before cat --><cat><!-- in cat before -->
-  <x>3<!-- multi
-line comment 
-for x --></x><!-- before y -->
-  <y><!-- in y before
-in d before -->
-    <d>z<!-- in d after --></d><!-- in y after -->
-  </y><!-- in_cat_after -->
-</cat><!-- after cat -->
-```
-

pkg/yqlib/encoder.go

@@ -11,10 +11,92 @@ import (
 )
 
 type Encoder interface {
-	Encode(writer io.Writer, node *yaml.Node) error
-	PrintDocumentSeparator(writer io.Writer) error
-	PrintLeadingContent(writer io.Writer, content string) error
-	CanHandleAliases() bool
+	Encode(node *yaml.Node) error
+}
+
+type yamlEncoder struct {
+	destination io.Writer
+	indent      int
+	colorise    bool
+	firstDoc    bool
+}
+
+func NewYamlEncoder(destination io.Writer, indent int, colorise bool) Encoder {
+	if indent < 0 {
+		indent = 0
+	}
+	return &yamlEncoder{destination, indent, colorise, true}
+}
+
+func (ye *yamlEncoder) Encode(node *yaml.Node) error {
+
+	destination := ye.destination
+	tempBuffer := bytes.NewBuffer(nil)
+	if ye.colorise {
+		destination = tempBuffer
+	}
+
+	var encoder = yaml.NewEncoder(destination)
+
+	encoder.SetIndent(ye.indent)
+	// TODO: work out if the first doc had a separator or not.
+	if ye.firstDoc {
+		ye.firstDoc = false
+	} else if _, err := destination.Write([]byte("---\n")); err != nil {
+		return err
+	}
+
+	if err := encoder.Encode(node); err != nil {
+		return err
+	}
+
+	if ye.colorise {
+		return colorizeAndPrint(tempBuffer.Bytes(), ye.destination)
+	}
+	return nil
+}
+
+type jsonEncoder struct {
+	encoder *json.Encoder
+}
+
+func mapKeysToStrings(node *yaml.Node) {
+
+	if node.Kind == yaml.MappingNode {
+		for index, child := range node.Content {
+			if index%2 == 0 { // its a map key
+				child.Tag = "!!str"
+			}
+		}
+	}
+
+	for _, child := range node.Content {
+		mapKeysToStrings(child)
+	}
+}
+
+func NewJsonEncoder(destination io.Writer, indent int) Encoder {
+	var encoder = json.NewEncoder(destination)
+	encoder.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
+
+	var indentString = ""
+
+	for index := 0; index < indent; index++ {
+		indentString = indentString + " "
+	}
+	encoder.SetIndent("", indentString)
+	return &jsonEncoder{encoder}
+}
+
+func (je *jsonEncoder) Encode(node *yaml.Node) error {
+	var dataBucket orderedMap
+	// firstly, convert all map keys to strings
+	mapKeysToStrings(node)
+	errorDecoding := node.Decode(&dataBucket)
+	if errorDecoding != nil {
+		return errorDecoding
+	}
+	return je.encoder.Encode(dataBucket)
 }
 
 // orderedMap allows to marshal and unmarshal JSON and YAML values keeping the

pkg/yqlib/encoder_csv.go

@@ -9,26 +9,16 @@ import (
 )
 
 type csvEncoder struct {
-	separator rune
+	destination csv.Writer
 }
 
-func NewCsvEncoder(separator rune) Encoder {
-	return &csvEncoder{separator}
+func NewCsvEncoder(destination io.Writer, separator rune) Encoder {
+	csvWriter := *csv.NewWriter(destination)
+	csvWriter.Comma = separator
+	return &csvEncoder{csvWriter}
 }
 
-func (e *csvEncoder) CanHandleAliases() bool {
-	return false
-}
-
-func (e *csvEncoder) PrintDocumentSeparator(writer io.Writer) error {
-	return nil
-}
-
-func (e *csvEncoder) PrintLeadingContent(writer io.Writer, content string) error {
-	return nil
-}
-
-func (e *csvEncoder) encodeRow(csvWriter *csv.Writer, contents []*yaml.Node) error {
+func (e *csvEncoder) encodeRow(contents []*yaml.Node) error {
 	stringValues := make([]string, len(contents))
 
 	for i, child := range contents {
@@ -38,13 +28,10 @@ func (e *csvEncoder) encodeRow(csvWriter *csv.Writer, contents []*yaml.Node) err
 		}
 		stringValues[i] = child.Value
 	}
-	return csvWriter.Write(stringValues)
+	return e.destination.Write(stringValues)
 }
 
-func (e *csvEncoder) Encode(writer io.Writer, originalNode *yaml.Node) error {
-	csvWriter := csv.NewWriter(writer)
-	csvWriter.Comma = e.separator
-
+func (e *csvEncoder) Encode(originalNode *yaml.Node) error {
 	// node must be a sequence
 	node := unwrapDoc(originalNode)
 	if node.Kind != yaml.SequenceNode {
@@ -53,7 +40,7 @@ func (e *csvEncoder) Encode(writer io.Writer, originalNode *yaml.Node) error {
 		return nil
 	}
 	if node.Content[0].Kind == yaml.ScalarNode {
-		return e.encodeRow(csvWriter, node.Content)
+		return e.encodeRow(node.Content)
 	}
 
 	for i, child := range node.Content {
@@ -61,7 +48,7 @@ func (e *csvEncoder) Encode(writer io.Writer, originalNode *yaml.Node) error {
 		if child.Kind != yaml.SequenceNode {
 			return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag)
 		}
-		err := e.encodeRow(csvWriter, child.Content)
+		err := e.encodeRow(child.Content)
 		if err != nil {
 			return err
 		}

pkg/yqlib/encoder_csv_test.go

@@ -13,13 +13,13 @@ func yamlToCsv(sampleYaml string, separator rune) string {
 	var output bytes.Buffer
 	writer := bufio.NewWriter(&output)
 
-	var jsonEncoder = NewCsvEncoder(separator)
+	var jsonEncoder = NewCsvEncoder(writer, separator)
 	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
 		panic(err)
 	}
 	node := inputs.Front().Value.(*CandidateNode).Node
-	err = jsonEncoder.Encode(writer, node)
+	err = jsonEncoder.Encode(node)
 	if err != nil {
 		panic(err)
 	}

pkg/yqlib/encoder_json.go

@@ -1,64 +0,0 @@
-package yqlib
-
-import (
-	"encoding/json"
-	"io"
-
-	yaml "gopkg.in/yaml.v3"
-)
-
-type jsonEncoder struct {
-	indentString string
-}
-
-func mapKeysToStrings(node *yaml.Node) {
-
-	if node.Kind == yaml.MappingNode {
-		for index, child := range node.Content {
-			if index%2 == 0 { // its a map key
-				child.Tag = "!!str"
-			}
-		}
-	}
-
-	for _, child := range node.Content {
-		mapKeysToStrings(child)
-	}
-}
-
-func NewJsonEncoder(indent int) Encoder {
-	var indentString = ""
-
-	for index := 0; index < indent; index++ {
-		indentString = indentString + " "
-	}
-
-	return &jsonEncoder{indentString}
-}
-
-func (je *jsonEncoder) CanHandleAliases() bool {
-	return false
-}
-
-func (je *jsonEncoder) PrintDocumentSeparator(writer io.Writer) error {
-	return nil
-}
-
-func (je *jsonEncoder) PrintLeadingContent(writer io.Writer, content string) error {
-	return nil
-}
-
-func (je *jsonEncoder) Encode(writer io.Writer, node *yaml.Node) error {
-	var encoder = json.NewEncoder(writer)
-	encoder.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
-	encoder.SetIndent("", je.indentString)
-
-	var dataBucket orderedMap
-	// firstly, convert all map keys to strings
-	mapKeysToStrings(node)
-	errorDecoding := node.Decode(&dataBucket)
-	if errorDecoding != nil {
-		return errorDecoding
-	}
-	return encoder.Encode(dataBucket)
-}

pkg/yqlib/encoder_properties.go

@@ -1,8 +1,6 @@
 package yqlib
 
 import (
-	"bufio"
-	"errors"
 	"fmt"
 	"io"
 	"strings"
@@ -12,54 +10,14 @@ import (
 )
 
 type propertiesEncoder struct {
+	destination io.Writer
 }
 
-func NewPropertiesEncoder() Encoder {
-	return &propertiesEncoder{}
+func NewPropertiesEncoder(destination io.Writer) Encoder {
+	return &propertiesEncoder{destination}
 }
 
-func (pe *propertiesEncoder) CanHandleAliases() bool {
-	return false
-}
-
-func (pe *propertiesEncoder) PrintDocumentSeparator(writer io.Writer) error {
-	return nil
-}
-
-func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content string) error {
-	reader := bufio.NewReader(strings.NewReader(content))
-	for {
-
-		readline, errReading := reader.ReadString('\n')
-		if errReading != nil && !errors.Is(errReading, io.EOF) {
-			return errReading
-		}
-		if strings.Contains(readline, "$yqDocSeperator$") {
-
-			if err := pe.PrintDocumentSeparator(writer); err != nil {
-				return err
-			}
-
-		} else {
-			if err := writeString(writer, readline); err != nil {
-				return err
-			}
-		}
-
-		if errors.Is(errReading, io.EOF) {
-			if readline != "" {
-				// the last comment we read didn't have a new line, put one in
-				if err := writeString(writer, "\n"); err != nil {
-					return err
-				}
-			}
-			break
-		}
-	}
-	return nil
-}
-
-func (pe *propertiesEncoder) Encode(writer io.Writer, node *yaml.Node) error {
+func (pe *propertiesEncoder) Encode(node *yaml.Node) error {
 	mapKeysToStrings(node)
 	p := properties.NewProperties()
 	err := pe.doEncode(p, node, "")
@@ -67,12 +25,14 @@ func (pe *propertiesEncoder) Encode(writer io.Writer, node *yaml.Node) error {
 		return err
 	}
 
-	_, err = p.WriteComment(writer, "#", properties.UTF8)
+	_, err = p.WriteComment(pe.destination, "#", properties.UTF8)
 	return err
 }
 
 func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yaml.Node, path string) error {
-	p.SetComment(path, headAndLineComment(node))
+	p.SetComment(path,
+		strings.Replace(node.HeadComment, "#", "", 1)+
+			strings.Replace(node.LineComment, "#", "", 1))
 	switch node.Kind {
 	case yaml.ScalarNode:
 		_, _, err := p.Set(path, node.Value)

pkg/yqlib/encoder_properties_test.go

@@ -13,13 +13,13 @@ func yamlToProps(sampleYaml string) string {
 	var output bytes.Buffer
 	writer := bufio.NewWriter(&output)
 
-	var propsEncoder = NewPropertiesEncoder()
+	var propsEncoder = NewPropertiesEncoder(writer)
 	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
 		panic(err)
 	}
 	node := inputs.Front().Value.(*CandidateNode).Node
-	err = propsEncoder.Encode(writer, node)
+	err = propsEncoder.Encode(node)
 	if err != nil {
 		panic(err)
 	}

pkg/yqlib/encoder_test.go

@@ -13,13 +13,13 @@ func yamlToJson(sampleYaml string, indent int) string {
 	var output bytes.Buffer
 	writer := bufio.NewWriter(&output)
 
-	var jsonEncoder = NewJsonEncoder(indent)
+	var jsonEncoder = NewJsonEncoder(writer, indent)
 	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
 		panic(err)
 	}
 	node := inputs.Front().Value.(*CandidateNode).Node
-	err = jsonEncoder.Encode(writer, node)
+	err = jsonEncoder.Encode(node)
 	if err != nil {
 		panic(err)
 	}

pkg/yqlib/encoder_xml.go

@@ -1,245 +0,0 @@
-package yqlib
-
-import (
-	"encoding/xml"
-	"fmt"
-	"io"
-	"strings"
-
-	yaml "gopkg.in/yaml.v3"
-)
-
-var XmlPreferences = xmlPreferences{AttributePrefix: "+", ContentName: "+content"}
-
-type xmlEncoder struct {
-	attributePrefix string
-	contentName     string
-	indentString    string
-}
-
-func NewXmlEncoder(indent int, attributePrefix string, contentName string) Encoder {
-	var indentString = ""
-
-	for index := 0; index < indent; index++ {
-		indentString = indentString + " "
-	}
-	return &xmlEncoder{attributePrefix, contentName, indentString}
-}
-
-func (e *xmlEncoder) CanHandleAliases() bool {
-	return false
-}
-
-func (e *xmlEncoder) PrintDocumentSeparator(writer io.Writer) error {
-	return nil
-}
-
-func (e *xmlEncoder) PrintLeadingContent(writer io.Writer, content string) error {
-	return nil
-}
-
-func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
-	encoder := xml.NewEncoder(writer)
-	encoder.Indent("", e.indentString)
-
-	switch node.Kind {
-	case yaml.MappingNode:
-		err := e.encodeTopLevelMap(encoder, node)
-		if err != nil {
-			return err
-		}
-	case yaml.DocumentNode:
-		err := e.encodeComment(encoder, headAndLineComment(node))
-		if err != nil {
-			return err
-		}
-		err = e.encodeTopLevelMap(encoder, unwrapDoc(node))
-		if err != nil {
-			return err
-		}
-		err = e.encodeComment(encoder, footComment(node))
-		if err != nil {
-			return err
-		}
-	case yaml.ScalarNode:
-		var charData xml.CharData = []byte(node.Value)
-		err := encoder.EncodeToken(charData)
-		if err != nil {
-			return err
-		}
-		return encoder.Flush()
-	default:
-		return fmt.Errorf("unsupported type %v", node.Tag)
-	}
-	var charData xml.CharData = []byte("\n")
-	return encoder.EncodeToken(charData)
-
-}
-
-func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node) error {
-	err := e.encodeComment(encoder, headAndLineComment(node))
-	if err != nil {
-		return err
-	}
-	for i := 0; i < len(node.Content); i += 2 {
-		key := node.Content[i]
-		value := node.Content[i+1]
-
-		start := xml.StartElement{Name: xml.Name{Local: key.Value}}
-		log.Debugf("comments of key %v", key.Value)
-		err := e.encodeComment(encoder, headAndLineComment(key))
-		if err != nil {
-			return err
-		}
-
-		log.Debugf("recursing")
-
-		err = e.doEncode(encoder, value, start)
-		if err != nil {
-			return err
-		}
-		err = e.encodeComment(encoder, footComment(key))
-		if err != nil {
-			return err
-		}
-	}
-	return e.encodeComment(encoder, footComment(node))
-}
-
-func (e *xmlEncoder) encodeStart(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
-	err := encoder.EncodeToken(start)
-	if err != nil {
-		return err
-	}
-	return e.encodeComment(encoder, headComment(node))
-}
-
-func (e *xmlEncoder) encodeEnd(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
-	err := encoder.EncodeToken(start.End())
-	if err != nil {
-		return err
-	}
-	return e.encodeComment(encoder, footComment(node))
-}
-
-func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
-	switch node.Kind {
-	case yaml.MappingNode:
-		return e.encodeMap(encoder, node, start)
-	case yaml.SequenceNode:
-		return e.encodeArray(encoder, node, start)
-	case yaml.ScalarNode:
-		err := e.encodeStart(encoder, node, start)
-		if err != nil {
-			return err
-		}
-
-		var charData xml.CharData = []byte(node.Value)
-		err = encoder.EncodeToken(charData)
-		if err != nil {
-			return err
-		}
-
-		if err = e.encodeComment(encoder, lineComment(node)); err != nil {
-			return err
-		}
-
-		return e.encodeEnd(encoder, node, start)
-	}
-	return fmt.Errorf("unsupported type %v", node.Tag)
-}
-
-func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) error {
-	if commentStr != "" {
-		log.Debugf("encoding comment %v", commentStr)
-		if !strings.HasSuffix(commentStr, " ") {
-			commentStr = commentStr + " "
-		}
-
-		var comment xml.Comment = []byte(commentStr)
-		err := encoder.EncodeToken(comment)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
-
-	if err := e.encodeComment(encoder, headAndLineComment(node)); err != nil {
-		return err
-	}
-
-	for i := 0; i < len(node.Content); i++ {
-		value := node.Content[i]
-		if err := e.doEncode(encoder, value, start.Copy()); err != nil {
-			return err
-		}
-	}
-	return e.encodeComment(encoder, footComment(node))
-}
-
-func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
-	log.Debug("its a map")
-
-	//first find all the attributes and put them on the start token
-	for i := 0; i < len(node.Content); i += 2 {
-		key := node.Content[i]
-		value := node.Content[i+1]
-
-		if strings.HasPrefix(key.Value, e.attributePrefix) && key.Value != e.contentName {
-			if value.Kind == yaml.ScalarNode {
-				attributeName := strings.Replace(key.Value, e.attributePrefix, "", 1)
-				start.Attr = append(start.Attr, xml.Attr{Name: xml.Name{Local: attributeName}, Value: value.Value})
-			} else {
-				return fmt.Errorf("cannot use %v as attribute, only scalars are supported", value.Tag)
-			}
-		}
-	}
-
-	err := e.encodeStart(encoder, node, start)
-	if err != nil {
-		return err
-	}
-
-	//now we encode non attribute tokens
-	for i := 0; i < len(node.Content); i += 2 {
-		key := node.Content[i]
-		value := node.Content[i+1]
-
-		err := e.encodeComment(encoder, headAndLineComment(key))
-		if err != nil {
-			return err
-		}
-
-		if !strings.HasPrefix(key.Value, e.attributePrefix) && key.Value != e.contentName {
-			start := xml.StartElement{Name: xml.Name{Local: key.Value}}
-			err := e.doEncode(encoder, value, start)
-			if err != nil {
-				return err
-			}
-		} else if key.Value == e.contentName {
-			// directly encode the contents
-			err = e.encodeComment(encoder, headAndLineComment(value))
-			if err != nil {
-				return err
-			}
-			var charData xml.CharData = []byte(value.Value)
-			err = encoder.EncodeToken(charData)
-			if err != nil {
-				return err
-			}
-			err = e.encodeComment(encoder, footComment(value))
-			if err != nil {
-				return err
-			}
-		}
-		err = e.encodeComment(encoder, footComment(key))
-		if err != nil {
-			return err
-		}
-	}
-
-	return e.encodeEnd(encoder, node, start)
-}

pkg/yqlib/encoder_yaml.go

@@ -1,101 +0,0 @@
-package yqlib
-
-import (
-	"bufio"
-	"bytes"
-	"errors"
-	"io"
-	"strings"
-
-	yaml "gopkg.in/yaml.v3"
-)
-
-type yamlEncoder struct {
-	indent             int
-	colorise           bool
-	printDocSeparators bool
-	unwrapScalar       bool
-}
-
-func NewYamlEncoder(indent int, colorise bool, printDocSeparators bool, unwrapScalar bool) Encoder {
-	if indent < 0 {
-		indent = 0
-	}
-	return &yamlEncoder{indent, colorise, printDocSeparators, unwrapScalar}
-}
-
-func (ye *yamlEncoder) CanHandleAliases() bool {
-	return true
-}
-
-func (ye *yamlEncoder) PrintDocumentSeparator(writer io.Writer) error {
-	if ye.printDocSeparators {
-		log.Debug("-- writing doc sep")
-		if err := writeString(writer, "---\n"); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) error {
-	// log.Debug("headcommentwas [%v]", content)
-	reader := bufio.NewReader(strings.NewReader(content))
-
-	for {
-
-		readline, errReading := reader.ReadString('\n')
-		if errReading != nil && !errors.Is(errReading, io.EOF) {
-			return errReading
-		}
-		if strings.Contains(readline, "$yqDocSeperator$") {
-
-			if err := ye.PrintDocumentSeparator(writer); err != nil {
-				return err
-			}
-
-		} else {
-			if err := writeString(writer, readline); err != nil {
-				return err
-			}
-		}
-
-		if errors.Is(errReading, io.EOF) {
-			if readline != "" {
-				// the last comment we read didn't have a new line, put one in
-				if err := writeString(writer, "\n"); err != nil {
-					return err
-				}
-			}
-			break
-		}
-	}
-
-	return nil
-}
-
-func (ye *yamlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
-
-	if node.Kind == yaml.ScalarNode && ye.unwrapScalar {
-		return writeString(writer, node.Value+"\n")
-	}
-
-	destination := writer
-	tempBuffer := bytes.NewBuffer(nil)
-	if ye.colorise {
-		destination = tempBuffer
-	}
-
-	var encoder = yaml.NewEncoder(destination)
-
-	encoder.SetIndent(ye.indent)
-
-	if err := encoder.Encode(node); err != nil {
-		return err
-	}
-
-	if ye.colorise {
-		return colorizeAndPrint(tempBuffer.Bytes(), writer)
-	}
-	return nil
-}

pkg/yqlib/expression_tokeniser.go

@@ -110,9 +110,6 @@ func multiplyWithPrefs() lex.Action {
 		if strings.Contains(options, "?") {
 			prefs.TraversePrefs = traversePreferences{DontAutoCreate: true}
 		}
-		if strings.Contains(options, "n") {
-			prefs.AssignPrefs = assignPreferences{OnlyWriteNull: true}
-		}
 		if strings.Contains(options, "d") {
 			prefs.DeepMergeArrays = true
 		}
@@ -325,9 +322,6 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`toyaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat))
 	lexer.Add([]byte(`to_yaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat))
 
-	lexer.Add([]byte(`toxml\([0-9]+\)`), encodeWithIndent(XmlOutputFormat))
-	lexer.Add([]byte(`to_xml\([0-9]+\)`), encodeWithIndent(XmlOutputFormat))
-
 	lexer.Add([]byte(`tojson\([0-9]+\)`), encodeWithIndent(JsonOutputFormat))
 	lexer.Add([]byte(`to_json\([0-9]+\)`), encodeWithIndent(JsonOutputFormat))
 
@@ -352,10 +346,6 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`to_tsv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: TsvOutputFormat}))
 	lexer.Add([]byte(`@tsv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: TsvOutputFormat}))
 
-	lexer.Add([]byte(`toxml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: XmlOutputFormat}))
-	lexer.Add([]byte(`to_xml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: XmlOutputFormat, indent: 2}))
-	lexer.Add([]byte(`@xml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: XmlOutputFormat, indent: 0}))
-
 	lexer.Add([]byte(`fromyaml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat}))
 	lexer.Add([]byte(`fromjson`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat}))
 	lexer.Add([]byte(`fromxml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: XmlInputFormat}))
@@ -369,9 +359,9 @@ func initLexer() (*lex.Lexer, error) {
 
 	lexer.Add([]byte(`load`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewYamlDecoder()}))
 
-	lexer.Add([]byte(`xmlload`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXmlDecoder(XmlPreferences.AttributePrefix, XmlPreferences.ContentName)}))
-	lexer.Add([]byte(`load_xml`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXmlDecoder(XmlPreferences.AttributePrefix, XmlPreferences.ContentName)}))
-	lexer.Add([]byte(`loadxml`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXmlDecoder(XmlPreferences.AttributePrefix, XmlPreferences.ContentName)}))
+	lexer.Add([]byte(`xmlload`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXmlDecoder("+", "+content")}))
+	lexer.Add([]byte(`load_xml`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXmlDecoder("+", "+content")}))
+	lexer.Add([]byte(`loadxml`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXmlDecoder("+", "+content")}))
 
 	lexer.Add([]byte(`strload`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: true}))
 	lexer.Add([]byte(`load_str`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: true}))
@@ -476,7 +466,7 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`\]\??`), literalToken(closeCollect, true))
 	lexer.Add([]byte(`\{`), literalToken(openCollectObject, false))
 	lexer.Add([]byte(`\}`), literalToken(closeCollectObject, true))
-	lexer.Add([]byte(`\*[\+|\?dn]*`), multiplyWithPrefs())
+	lexer.Add([]byte(`\*[\+|\?d]*`), multiplyWithPrefs())
 	lexer.Add([]byte(`\+`), opToken(addOpType))
 	lexer.Add([]byte(`\+=`), opToken(addAssignOpType))
 	lexer.Add([]byte(`\-`), opToken(subtractOpType))

pkg/yqlib/file_utils.go

@@ -1,27 +1,30 @@
 package yqlib
 
 import (
-	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 )
 
-func tryRenameFile(from string, to string) error {
+func safelyRenameFile(from string, to string) {
 	if renameError := os.Rename(from, to); renameError != nil {
 		log.Debugf("Error renaming from %v to %v, attempting to copy contents", from, to)
 		log.Debug(renameError.Error())
-		log.Debug("going to try copying instead")
 		// can't do this rename when running in docker to a file targeted in a mounted volume,
 		// so gracefully degrade to copying the entire contents.
 		if copyError := copyFileContents(from, to); copyError != nil {
-			return fmt.Errorf("failed copying from %v to %v: %w", from, to, copyError)
+			log.Errorf("Failed copying from %v to %v", from, to)
+			log.Error(copyError.Error())
+		} else {
+			removeErr := os.Remove(from)
+			if removeErr != nil {
+				log.Errorf("failed removing original file: %s", from)
+			}
 		}
-		tryRemoveTempFile(from)
 	}
-	return nil
 }
 
-func tryRemoveTempFile(filename string) {
+func tryRemoveFile(filename string) {
 	log.Debug("Removing temp file: %v", filename)
 	removeErr := os.Remove(filename)
 	if removeErr != nil {
@@ -76,7 +79,7 @@ func createTempFile() (*os.File, error) {
 		return nil, err
 	}
 
-	file, err := os.CreateTemp("", "temp")
+	file, err := ioutil.TempFile("", "temp")
 	if err != nil {
 		return nil, err
 	}

pkg/yqlib/front_matter.go

@@ -33,7 +33,7 @@ func (f *frontMatterHandlerImpl) GetContentReader() io.Reader {
 }
 
 func (f *frontMatterHandlerImpl) CleanUp() {
-	tryRemoveTempFile(f.yamlFrontMatterFilename)
+	tryRemoveFile(f.yamlFrontMatterFilename)
 }
 
 // Splits the given file by yaml front matter

pkg/yqlib/front_matter_test.go

@@ -1,8 +1,7 @@
 package yqlib
 
 import (
-	"io"
-	"os"
+	"io/ioutil"
 	"testing"
 
 	"github.com/mikefarah/yq/v4/test"
@@ -25,7 +24,7 @@ func createTestFile(content string) string {
 }
 
 func readFile(filename string) string {
-	bytes, err := os.ReadFile(filename)
+	bytes, err := ioutil.ReadFile(filename)
 	if err != nil {
 		panic(err)
 	}
@@ -61,13 +60,13 @@ yaml: doc
 
 	test.AssertResult(t, expectedYamlFm, yamlFm)
 
-	contentBytes, err := io.ReadAll(fmHandler.GetContentReader())
+	contentBytes, err := ioutil.ReadAll(fmHandler.GetContentReader())
 	if err != nil {
 		panic(err)
 	}
 	test.AssertResult(t, expectedContent, string(contentBytes))
 
-	tryRemoveTempFile(file)
+	tryRemoveFile(file)
 	fmHandler.CleanUp()
 }
 
@@ -98,13 +97,13 @@ yaml: doc
 
 	test.AssertResult(t, expectedYamlFm, yamlFm)
 
-	contentBytes, err := io.ReadAll(fmHandler.GetContentReader())
+	contentBytes, err := ioutil.ReadAll(fmHandler.GetContentReader())
 	if err != nil {
 		panic(err)
 	}
 	test.AssertResult(t, expectedContent, string(contentBytes))
 
-	tryRemoveTempFile(file)
+	tryRemoveFile(file)
 	fmHandler.CleanUp()
 }
 
@@ -132,12 +131,12 @@ yaml: doc
 
 	test.AssertResult(t, expectedYamlFm, yamlFm)
 
-	contentBytes, err := io.ReadAll(fmHandler.GetContentReader())
+	contentBytes, err := ioutil.ReadAll(fmHandler.GetContentReader())
 	if err != nil {
 		panic(err)
 	}
 	test.AssertResult(t, expectedContent, string(contentBytes))
 
-	tryRemoveTempFile(file)
+	tryRemoveFile(file)
 	fmHandler.CleanUp()
 }

pkg/yqlib/json_test.go

@@ -1,229 +0,0 @@
-package yqlib
-
-import (
-	"bufio"
-	"bytes"
-	"fmt"
-	"strings"
-	"testing"
-
-	"github.com/mikefarah/yq/v4/test"
-)
-
-var complexExpectYaml = `D0, P[], (!!map)::a: Easy! as one two three
-b:
-    c: 2
-    d:
-        - 3
-        - 4
-`
-
-var jsonScenarios = []formatScenario{
-	{
-		description:    "Parse json: simple",
-		subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output",
-		input:          `{"cat": "meow"}`,
-		expected:       "D0, P[], (!!map)::cat: meow\n",
-	},
-	{
-		description:    "Parse json: complex",
-		subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output",
-		input:          `{"a":"Easy! as one two three","b":{"c":2,"d":[3,4]}}`,
-		expected:       complexExpectYaml,
-	},
-	{
-		description:  "Encode json: simple",
-		input:        `cat: meow`,
-		indent:       2,
-		expected:     "{\n  \"cat\": \"meow\"\n}\n",
-		scenarioType: "encode",
-	},
-	{
-		description:  "Encode json: simple - in one line",
-		input:        `cat: meow # this is a comment, and it will be dropped.`,
-		indent:       0,
-		expected:     "{\"cat\":\"meow\"}\n",
-		scenarioType: "encode",
-	},
-	{
-		description:  "Encode json: comments",
-		input:        `cat: meow # this is a comment, and it will be dropped.`,
-		indent:       2,
-		expected:     "{\n  \"cat\": \"meow\"\n}\n",
-		scenarioType: "encode",
-	},
-	{
-		description:    "Encode json: anchors",
-		subdescription: "Anchors are dereferenced",
-		input:          "cat: &ref meow\nanotherCat: *ref",
-		indent:         2,
-		expected:       "{\n  \"cat\": \"meow\",\n  \"anotherCat\": \"meow\"\n}\n",
-		scenarioType:   "encode",
-	},
-	{
-		description:    "Encode json: multiple results",
-		subdescription: "Each matching node is converted into a json doc. This is best used with 0 indent (json document per line)",
-		input:          `things: [{stuff: cool}, {whatever: cat}]`,
-		expression:     `.things[]`,
-		indent:         0,
-		expected:       "{\"stuff\":\"cool\"}\n{\"whatever\":\"cat\"}\n",
-		scenarioType:   "encode",
-	},
-}
-
-func decodeJson(t *testing.T, jsonString string) *CandidateNode {
-	docs, err := readDocumentWithLeadingContent(jsonString, "sample.json", 0)
-
-	if err != nil {
-		t.Error(err)
-		return nil
-	}
-
-	exp, err := NewExpressionParser().ParseExpression(PrettyPrintExp)
-
-	if err != nil {
-		t.Error(err)
-		return nil
-	}
-
-	context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: docs}, exp)
-
-	if err != nil {
-		t.Error(err)
-		return nil
-	}
-
-	return context.MatchingNodes.Front().Value.(*CandidateNode)
-}
-
-func testJsonScenario(t *testing.T, s formatScenario) {
-	if s.scenarioType == "encode" || s.scenarioType == "roundtrip" {
-		test.AssertResultWithContext(t, s.expected, processJsonScenario(s), s.description)
-	} else {
-		var actual = resultToString(t, decodeJson(t, s.input))
-		test.AssertResultWithContext(t, s.expected, actual, s.description)
-	}
-}
-
-func processJsonScenario(s formatScenario) string {
-
-	var output bytes.Buffer
-	writer := bufio.NewWriter(&output)
-
-	var encoder = NewJsonEncoder(s.indent)
-
-	var decoder = NewYamlDecoder()
-
-	inputs, err := readDocuments(strings.NewReader(s.input), "sample.yml", 0, decoder)
-	if err != nil {
-		panic(err)
-	}
-
-	expression := s.expression
-	if expression == "" {
-		expression = "."
-	}
-
-	exp, err := NewExpressionParser().ParseExpression(expression)
-
-	if err != nil {
-		panic(err)
-	}
-
-	context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: inputs}, exp)
-
-	if err != nil {
-		panic(err)
-	}
-
-	printer := NewPrinter(encoder, NewSinglePrinterWriter(writer))
-	err = printer.PrintResults(context.MatchingNodes)
-	if err != nil {
-		panic(err)
-	}
-	writer.Flush()
-
-	return output.String()
-
-}
-
-func documentJsonDecodeScenario(t *testing.T, w *bufio.Writer, s formatScenario) {
-	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
-
-	if s.subdescription != "" {
-		writeOrPanic(w, s.subdescription)
-		writeOrPanic(w, "\n\n")
-	}
-
-	writeOrPanic(w, "Given a sample.json file of:\n")
-	writeOrPanic(w, fmt.Sprintf("```json\n%v\n```\n", s.input))
-
-	writeOrPanic(w, "then\n")
-	writeOrPanic(w, "```bash\nyq e -P '.' sample.json\n```\n")
-	writeOrPanic(w, "will output\n")
-
-	var output bytes.Buffer
-	printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
-
-	node := decodeJson(t, s.input)
-
-	err := printer.PrintResults(node.AsList())
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", output.String()))
-}
-
-func documentJsonScenario(t *testing.T, w *bufio.Writer, i interface{}) {
-	s := i.(formatScenario)
-
-	if s.skipDoc {
-		return
-	}
-	if s.scenarioType == "encode" {
-		documentJsonEncodeScenario(w, s)
-	} else {
-		documentJsonDecodeScenario(t, w, s)
-	}
-}
-
-func documentJsonEncodeScenario(w *bufio.Writer, s formatScenario) {
-	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
-
-	if s.subdescription != "" {
-		writeOrPanic(w, s.subdescription)
-		writeOrPanic(w, "\n\n")
-	}
-
-	writeOrPanic(w, "Given a sample.yml file of:\n")
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input))
-
-	writeOrPanic(w, "then\n")
-
-	expression := s.expression
-	if expression == "" {
-		expression = "."
-	}
-
-	if s.indent == 2 {
-		writeOrPanic(w, fmt.Sprintf("```bash\nyq e -o=json '%v' sample.yml\n```\n", expression))
-	} else {
-		writeOrPanic(w, fmt.Sprintf("```bash\nyq e -o=json -I=%v '%v' sample.yml\n```\n", s.indent, expression))
-	}
-	writeOrPanic(w, "will output\n")
-
-	writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", processJsonScenario(s)))
-}
-
-func TestJsonScenarios(t *testing.T) {
-	for _, tt := range jsonScenarios {
-		testJsonScenario(t, tt)
-	}
-	genericScenarios := make([]interface{}, len(jsonScenarios))
-	for i, s := range jsonScenarios {
-		genericScenarios[i] = s
-	}
-	documentScenarios(t, "usage", "convert", genericScenarios, documentJsonScenario)
-}

pkg/yqlib/lib.go

@@ -13,15 +13,8 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
-type xmlPreferences struct {
-	AttributePrefix string
-	ContentName     string
-}
-
 var log = logging.MustGetLogger("yq-lib")
 
-var PrettyPrintExp = `(... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not))  ) style=""`
-
 // GetLogger returns the yq logger instance.
 func GetLogger() *logging.Logger {
 	return log
@@ -236,22 +229,6 @@ func createScalarNode(value interface{}, stringValue string) *yaml.Node {
 	return node
 }
 
-func headAndLineComment(node *yaml.Node) string {
-	return headComment(node) + lineComment(node)
-}
-
-func headComment(node *yaml.Node) string {
-	return strings.Replace(node.HeadComment, "#", "", 1)
-}
-
-func lineComment(node *yaml.Node) string {
-	return strings.Replace(node.LineComment, "#", "", 1)
-}
-
-func footComment(node *yaml.Node) string {
-	return strings.Replace(node.FootComment, "#", "", 1)
-}
-
 func createValueOperation(value interface{}, stringValue string) *Operation {
 	var node *yaml.Node = createScalarNode(value, stringValue)
 

pkg/yqlib/operator_assign.go

@@ -2,15 +2,13 @@ package yqlib
 
 type assignPreferences struct {
 	DontOverWriteAnchor bool
-	OnlyWriteNull       bool
 }
 
 func assignUpdateFunc(prefs assignPreferences) crossFunctionCalculation {
 	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 		rhs.Node = unwrapDoc(rhs.Node)
-		if !prefs.OnlyWriteNull || lhs.Node.Tag == "!!null" {
-			lhs.UpdateFrom(rhs, prefs)
-		}
+
+		lhs.UpdateFrom(rhs, prefs)
 		return lhs, nil
 	}
 }
@@ -78,9 +76,7 @@ func assignAttributesOperator(d *dataTreeNavigator, context Context, expressionN
 			if expressionNode.Operation.Preferences != nil {
 				prefs = expressionNode.Operation.Preferences.(assignPreferences)
 			}
-			if !prefs.OnlyWriteNull || candidate.Node.Tag == "!!null" {
-				candidate.UpdateAttributesFrom(first.Value.(*CandidateNode), prefs)
-			}
+			candidate.UpdateAttributesFrom(first.Value.(*CandidateNode), prefs)
 		}
 	}
 	return context, nil

pkg/yqlib/operator_comments.go

@@ -86,8 +86,7 @@ func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *
 			var chompRegexp = regexp.MustCompile(`\n$`)
 			var output bytes.Buffer
 			var writer = bufio.NewWriter(&output)
-			var encoder = NewYamlEncoder(2, false, false, false)
-			if err := encoder.PrintLeadingContent(writer, candidate.LeadingContent); err != nil {
+			if err := processLeadingContent(candidate, writer, false, YamlOutputFormat); err != nil {
 				return Context{}, err
 			}
 			if err := writer.Flush(); err != nil {

pkg/yqlib/operator_encoder_decoder.go

@@ -10,31 +10,11 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
-func configureEncoder(format PrinterOutputFormat, indent int) Encoder {
-	switch format {
-	case JsonOutputFormat:
-		return NewJsonEncoder(indent)
-	case PropsOutputFormat:
-		return NewPropertiesEncoder()
-	case CsvOutputFormat:
-		return NewCsvEncoder(',')
-	case TsvOutputFormat:
-		return NewCsvEncoder('\t')
-	case YamlOutputFormat:
-		return NewYamlEncoder(indent, false, true, true)
-	case XmlOutputFormat:
-		return NewXmlEncoder(indent, XmlPreferences.AttributePrefix, XmlPreferences.ContentName)
-	}
-	panic("invalid encoder")
-}
-
-func encodeToString(candidate *CandidateNode, prefs encoderPreferences) (string, error) {
+func yamlToString(candidate *CandidateNode, prefs encoderPreferences) (string, error) {
 	var output bytes.Buffer
 	log.Debug("printing with indent: %v", prefs.indent)
 
-	encoder := configureEncoder(prefs.format, prefs.indent)
-
-	printer := NewPrinter(encoder, NewSinglePrinterWriter(bufio.NewWriter(&output)))
+	printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), prefs.format, true, false, prefs.indent, true)
 	err := printer.PrintResults(candidate.AsList())
 	return output.String(), err
 }
@@ -56,7 +36,7 @@ func encodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
-		stringValue, err := encodeToString(candidate, preferences)
+		stringValue, err := yamlToString(candidate, preferences)
 
 		if err != nil {
 			return Context{}, err
@@ -102,7 +82,7 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 	case YamlInputFormat:
 		decoder = NewYamlDecoder()
 	case XmlInputFormat:
-		decoder = NewXmlDecoder(XmlPreferences.AttributePrefix, XmlPreferences.ContentName)
+		decoder = NewXmlDecoder("+a", "+content")
 	}
 
 	var results = list.New()

pkg/yqlib/operator_encoder_decoder_test.go

@@ -168,30 +168,6 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: \"foo:\\n  a: frog\"\n",
 		},
 	},
-	{
-		description: "Encode value as xml string",
-		document:    `{a: {cool: {foo: "bar", +id: hi}}}`,
-		expression:  `.a | to_xml`,
-		expected: []string{
-			"D0, P[a], (!!str)::<cool id=\"hi\">\n  <foo>bar</foo>\n</cool>\n\n",
-		},
-	},
-	{
-		description: "Encode value as xml string on a single line",
-		document:    `{a: {cool: {foo: "bar", +id: hi}}}`,
-		expression:  `.a | @xml`,
-		expected: []string{
-			"D0, P[a], (!!str)::<cool id=\"hi\"><foo>bar</foo></cool>\n\n",
-		},
-	},
-	{
-		description: "Encode value as xml string with custom indentation",
-		document:    `{a: {cool: {foo: "bar", +id: hi}}}`,
-		expression:  `{"cat": .a | to_xml(1)}`,
-		expected: []string{
-			"D0, P[], (!!map)::cat: |\n    <cool id=\"hi\">\n     <foo>bar</foo>\n    </cool>\n",
-		},
-	},
 	{
 		description: "Decode a xml encoded string",
 		document:    `a: "<foo>bar</foo>"`,

pkg/yqlib/operator_load.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"container/list"
 	"fmt"
+	"io/ioutil"
 	"os"
 
 	"gopkg.in/yaml.v3"
@@ -18,7 +19,7 @@ func loadString(filename string) (*CandidateNode, error) {
 	// ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory,
 	// and ensuring that it's not possible to give a path to a file outside that directory.
 
-	filebytes, err := os.ReadFile(filename) // #nosec
+	filebytes, err := ioutil.ReadFile(filename) // #nosec
 	if err != nil {
 		return nil, err
 	}

pkg/yqlib/operator_multiply.go

@@ -13,7 +13,6 @@ type multiplyPreferences struct {
 	AppendArrays    bool
 	DeepMergeArrays bool
 	TraversePrefs   traversePreferences
-	AssignPrefs     assignPreferences
 }
 
 func multiplyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
@@ -141,7 +140,7 @@ func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom
 	lhsPath := rhs.Path[pathIndexToStartFrom:]
 	log.Debugf("merge - lhsPath %v", lhsPath)
 
-	assignmentOp := &Operation{OperationType: assignAttributesOpType, Preferences: preferences.AssignPrefs}
+	assignmentOp := &Operation{OperationType: assignAttributesOpType}
 	if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode {
 		assignmentOp.OperationType = addAssignOpType
 		log.Debugf("merge - assignmentOp.OperationType = addAssignOpType")
@@ -158,11 +157,7 @@ func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom
 	}
 	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs}
 
-	assignmentOpNode := &ExpressionNode{
-		Operation: assignmentOp,
-		Lhs:       createTraversalTree(lhsPath, preferences.TraversePrefs, rhs.IsMapKey),
-		Rhs:       &ExpressionNode{Operation: rhsOp},
-	}
+	assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath, preferences.TraversePrefs, rhs.IsMapKey), Rhs: &ExpressionNode{Operation: rhsOp}}
 
 	_, err := d.GetMatchingNodes(context.SingleChildContext(lhs), assignmentOpNode)
 

pkg/yqlib/operator_multiply_test.go

@@ -344,14 +344,6 @@ var multiplyOperatorScenarios = []expressionScenario{
 			"D0, P[a], (!!map)::{thing: two, cat: frog}\n",
 		},
 	},
-	{
-		description: "Merge, only new fields",
-		document:    `{a: {thing: one, cat: frog}, b: {missing: two, thing: two}}`,
-		expression:  `.a *n .b`,
-		expected: []string{
-			"D0, P[a], (!!map)::{thing: one, cat: frog, missing: two}\n",
-		},
-	},
 	{
 		skipDoc:    true,
 		document:   `{a: [{thing: one}], b: [{missing: two, thing: two}]}`,
@@ -360,14 +352,6 @@ var multiplyOperatorScenarios = []expressionScenario{
 			"D0, P[a], (!!seq)::[{thing: two}]\n",
 		},
 	},
-	{
-		skipDoc:    true,
-		document:   `{a: [{thing: one}], b: [{missing: two, thing: two}]}`,
-		expression: `.a *nd .b`,
-		expected: []string{
-			"D0, P[a], (!!seq)::[{thing: one, missing: two}]\n",
-		},
-	},
 	{
 		skipDoc:    true,
 		document:   `{a: {array: [1]}, b: {}}`,
@@ -392,16 +376,6 @@ var multiplyOperatorScenarios = []expressionScenario{
 			"D0, P[a], (!!map)::{thing: [1, 2, 3, 4]}\n",
 		},
 	},
-	{
-		description:    "Merge, only new fields, appending arrays",
-		subdescription: "Append (+) with (n) has no effect.",
-		skipDoc:        true,
-		document:       `{a: {thing: [1,2]}, b: {thing: [3,4], another: [1]}}`,
-		expression:     `.a *n+ .b`,
-		expected: []string{
-			"D0, P[a], (!!map)::{thing: [1, 2], another: [1]}\n",
-		},
-	},
 	{
 		description:    "Merge, deeply merging arrays",
 		subdescription: "Merging arrays deeply means arrays are merge like objects, with indexes as their key. In this case, we merge the first item in the array, and do nothing with the second.",

pkg/yqlib/operator_select_test.go

@@ -28,7 +28,7 @@ var selectOperatorScenarios = []expressionScenario{
 		expected:   []string{},
 	},
 	{
-		description: "Select elements from array using wildcard prefix",
+		description: "Select elements from array",
 		document:    `[cat,goat,dog]`,
 		expression:  `.[] | select(. == "*at")`,
 		expected: []string{
@@ -36,35 +36,6 @@ var selectOperatorScenarios = []expressionScenario{
 			"D0, P[1], (!!str)::goat\n",
 		},
 	},
-	{
-		description: "Select elements from array using wildcard suffix",
-		document:    `[go-kart,goat,dog]`,
-		expression:  `.[] | select(. == "go*")`,
-		expected: []string{
-			"D0, P[0], (!!str)::go-kart\n",
-			"D0, P[1], (!!str)::goat\n",
-		},
-	},
-	{
-		description: "Select elements from array using wildcard prefix and suffix",
-		document:    `[ago, go, meow, going]`,
-		expression:  `.[] | select(. == "*go*")`,
-		expected: []string{
-			"D0, P[0], (!!str)::ago\n",
-			"D0, P[1], (!!str)::go\n",
-			"D0, P[3], (!!str)::going\n",
-		},
-	},
-	{
-		description:    "Select elements from array with regular expression",
-		subdescription: "See more regular expression examples under the `string` operator docs.",
-		document:       `[this_0, not_this, nor_0_this, thisTo_4]`,
-		expression:     `.[] | select(test("[a-zA-Z]+_[0-9]$"))`,
-		expected: []string{
-			"D0, P[0], (!!str)::this_0\n",
-			"D0, P[3], (!!str)::thisTo_4\n",
-		},
-	},
 	{
 		skipDoc:    true,
 		document:   "a: hello",
@@ -98,27 +69,9 @@ var selectOperatorScenarios = []expressionScenario{
 			"D0, P[a 1], (!!str)::goat\n"},
 	},
 	{
-		description: "Select items from a map",
-		document:    `{ things: cat, bob: goat, horse: dog }`,
-		expression:  `.[] | select(. == "cat" or test("og$"))`,
-		expected: []string{
-			"D0, P[things], (!!str)::cat\n",
-			"D0, P[horse], (!!str)::dog\n",
-		},
-	},
-	{
-		description: "Use select and with_entries to filter map keys",
-		document:    `{name: bob, legs: 2, game: poker}`,
-		expression:  `with_entries(select(.key | test("ame$")))`,
-		expected: []string{
-			"D0, P[], (!!map)::name: bob\ngame: poker\n",
-		},
-	},
-	{
-		description:    "Select multiple items in a map and update",
-		subdescription: "Note the brackets around the entire LHS.",
-		document:       `a: { things: cat, bob: goat, horse: dog }`,
-		expression:     `(.a.[] | select(. == "cat" or . == "goat")) |= "rabbit"`,
+		description: "Select and update matching values in map",
+		document:    `a: { things: cat, bob: goat, horse: dog }`,
+		expression:  `(.a.[] | select(. == "*at")) |= "rabbit"`,
 		expected: []string{
 			"D0, P[], (doc)::a: {things: rabbit, bob: rabbit, horse: dog}\n",
 		},

pkg/yqlib/operator_union.go

@@ -1,7 +1,5 @@
 package yqlib
 
-import "container/list"
-
 func unionOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debug("unionOperator")
 	log.Debug("context: %v", NodesToString(context.MatchingNodes))
@@ -20,25 +18,19 @@ func unionOperator(d *dataTreeNavigator, context Context, expressionNode *Expres
 	log.Debug("lhs: %v", lhs.ToString())
 	log.Debug("rhs: %v", rhs.ToString())
 
-	results := lhs.ChildContext(list.New())
-	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
-		node := el.Value.(*CandidateNode)
-		results.MatchingNodes.PushBack(node)
-	}
-
 	// this can happen when both expressions modify the context
 	// instead of creating their own.
 	/// (.foo = "bar"), (.thing = "cat")
+
 	if rhs.MatchingNodes != lhs.MatchingNodes {
 
 		for el := rhs.MatchingNodes.Front(); el != nil; el = el.Next() {
 			node := el.Value.(*CandidateNode)
 			log.Debug("processing %v", NodeToString(node))
 
-			results.MatchingNodes.PushBack(node)
+			lhs.MatchingNodes.PushBack(node)
 		}
 	}
-	log.Debug("and lets print it out")
-	log.Debug("all together: %v", results.ToString())
-	return results, nil
+	log.Debug("all together: %v", lhs.ToString())
+	return lhs, nil
 }

pkg/yqlib/operator_union_test.go

@@ -13,14 +13,6 @@ var unionOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{}\n",
 		},
 	},
-	{
-		skipDoc:     true,
-		description: "clone test",
-		expression:  `"abc" as $a | [$a, "cat"]`,
-		expected: []string{
-			"D0, P[], (!!seq)::- abc\n- cat\n",
-		},
-	},
 	{
 		skipDoc:    true,
 		expression: `(.foo = "bar"), (.toe = "jam")`,

pkg/yqlib/operators.go

@@ -18,7 +18,7 @@ func compoundAssignFunction(d *dataTreeNavigator, context Context, expressionNod
 		return Context{}, err
 	}
 
-	assignmentOp := &Operation{OperationType: assignOpType, Preferences: expressionNode.Operation.Preferences}
+	assignmentOp := &Operation{OperationType: assignOpType}
 	valueOp := &Operation{OperationType: valueOpType}
 
 	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {

pkg/yqlib/operators_test.go

@@ -26,10 +26,6 @@ type expressionScenario struct {
 	dontFormatInputForDoc bool // dont format input doc for documentation generation
 }
 
-func NewSimpleYamlPrinter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
-	return NewPrinter(NewYamlEncoder(indent, colorsEnabled, printDocSeparators, unwrapScalar), NewSinglePrinterWriter(writer))
-}
-
 func readDocumentWithLeadingContent(content string, fakefilename string, fakeFileIndex int) (*list.List, error) {
 	reader, firstFileLeadingContent, err := processReadStream(bufio.NewReader(strings.NewReader(content)))
 	if err != nil {
@@ -96,7 +92,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
 
 func resultToString(t *testing.T, n *CandidateNode) string {
 	var valueBuffer bytes.Buffer
-	printer := NewSimpleYamlPrinter(bufio.NewWriter(&valueBuffer), YamlOutputFormat, true, false, 4, true)
+	printer := NewPrinterWithSingleWriter(bufio.NewWriter(&valueBuffer), YamlOutputFormat, true, false, 4, true)
 
 	err := printer.PrintResults(n.AsList())
 	if err != nil {
@@ -149,7 +145,7 @@ func copyFromHeader(folder string, title string, out *os.File) error {
 
 func formatYaml(yaml string, filename string) string {
 	var output bytes.Buffer
-	printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
 
 	node, err := NewExpressionParser().ParseExpression(".. style= \"\"")
 	if err != nil {
@@ -272,7 +268,7 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) {
 func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formattedDoc string, formattedDoc2 string) {
 	var output bytes.Buffer
 	var err error
-	printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
 
 	node, err := NewExpressionParser().ParseExpression(s.expression)
 	if err != nil {

pkg/yqlib/printer.go

@@ -25,7 +25,6 @@ const (
 	PropsOutputFormat
 	CsvOutputFormat
 	TsvOutputFormat
-	XmlOutputFormat
 )
 
 func OutputFormatFromString(format string) (PrinterOutputFormat, error) {
@@ -40,30 +39,40 @@ func OutputFormatFromString(format string) (PrinterOutputFormat, error) {
 		return CsvOutputFormat, nil
 	case "tsv", "t":
 		return TsvOutputFormat, nil
-	case "xml", "x":
-		return XmlOutputFormat, nil
 	default:
-		return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml]", format)
+		return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv]", format)
 	}
 }
 
 type resultsPrinter struct {
-	encoder           Encoder
-	printerWriter     PrinterWriter
-	firstTimePrinting bool
-	previousDocIndex  uint
-	previousFileIndex int
-	printedMatches    bool
-	treeNavigator     DataTreeNavigator
-	appendixReader    io.Reader
+	outputFormat       PrinterOutputFormat
+	unwrapScalar       bool
+	colorsEnabled      bool
+	indent             int
+	printDocSeparators bool
+	printerWriter      PrinterWriter
+	firstTimePrinting  bool
+	previousDocIndex   uint
+	previousFileIndex  int
+	printedMatches     bool
+	treeNavigator      DataTreeNavigator
+	appendixReader     io.Reader
 }
 
-func NewPrinter(encoder Encoder, printerWriter PrinterWriter) Printer {
+func NewPrinterWithSingleWriter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
+	return NewPrinter(NewSinglePrinterWriter(writer), outputFormat, unwrapScalar, colorsEnabled, indent, printDocSeparators)
+}
+
+func NewPrinter(printerWriter PrinterWriter, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
 	return &resultsPrinter{
-		encoder:           encoder,
-		printerWriter:     printerWriter,
-		firstTimePrinting: true,
-		treeNavigator:     NewDataTreeNavigator(),
+		printerWriter:      printerWriter,
+		outputFormat:       outputFormat,
+		unwrapScalar:       unwrapScalar,
+		colorsEnabled:      colorsEnabled,
+		indent:             indent,
+		printDocSeparators: outputFormat == YamlOutputFormat && printDocSeparators,
+		firstTimePrinting:  true,
+		treeNavigator:      NewDataTreeNavigator(),
 	}
 }
 
@@ -78,7 +87,26 @@ func (p *resultsPrinter) PrintedAnything() bool {
 func (p *resultsPrinter) printNode(node *yaml.Node, writer io.Writer) error {
 	p.printedMatches = p.printedMatches || (node.Tag != "!!null" &&
 		(node.Tag != "!!bool" || node.Value != "false"))
-	return p.encoder.Encode(writer, node)
+
+	var encoder Encoder
+	if node.Kind == yaml.ScalarNode && p.unwrapScalar && p.outputFormat == YamlOutputFormat {
+		return writeString(writer, node.Value+"\n")
+	}
+
+	switch p.outputFormat {
+	case JsonOutputFormat:
+		encoder = NewJsonEncoder(writer, p.indent)
+	case PropsOutputFormat:
+		encoder = NewPropertiesEncoder(writer)
+	case CsvOutputFormat:
+		encoder = NewCsvEncoder(writer, ',')
+	case TsvOutputFormat:
+		encoder = NewCsvEncoder(writer, '\t')
+	case YamlOutputFormat:
+		encoder = NewYamlEncoder(writer, p.indent, p.colorsEnabled)
+	}
+
+	return encoder.Encode(node)
 }
 
 func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
@@ -89,7 +117,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 		return nil
 	}
 
-	if !p.encoder.CanHandleAliases() {
+	if p.outputFormat != YamlOutputFormat {
 		explodeOp := Operation{OperationType: explodeOpType}
 		explodeNode := ExpressionNode{Operation: &explodeOp}
 		context, err := p.treeNavigator.GetMatchingNodes(Context{MatchingNodes: matchingNodes}, &explodeNode)
@@ -109,7 +137,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 	for el := matchingNodes.Front(); el != nil; el = el.Next() {
 
 		mappedDoc := el.Value.(*CandidateNode)
-		log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document)
+		log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v, printDocSeparators: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document, p.printDocSeparators)
 
 		writer, errorWriting := p.printerWriter.GetWriter(mappedDoc)
 		if errorWriting != nil {
@@ -119,13 +147,14 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 		commentsStartWithSepExp := regexp.MustCompile(`^\$yqDocSeperator\$`)
 		commentStartsWithSeparator := commentsStartWithSepExp.MatchString(mappedDoc.LeadingContent)
 
-		if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && !commentStartsWithSeparator {
-			if err := p.encoder.PrintDocumentSeparator(writer); err != nil {
+		if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators && !commentStartsWithSeparator {
+			log.Debug("-- writing doc sep")
+			if err := writeString(writer, "---\n"); err != nil {
 				return err
 			}
 		}
 
-		if err := p.encoder.PrintLeadingContent(writer, mappedDoc.LeadingContent); err != nil {
+		if err := processLeadingContent(mappedDoc, writer, p.printDocSeparators, p.outputFormat); err != nil {
 			return err
 		}
 
@@ -137,11 +166,9 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 		if err := writer.Flush(); err != nil {
 			return err
 		}
-		log.Debugf("done printing results")
 	}
 
-	// what happens if I remove output format check?
-	if p.appendixReader != nil {
+	if p.appendixReader != nil && p.outputFormat == YamlOutputFormat {
 		writer, err := p.printerWriter.GetWriter(nil)
 		if err != nil {
 			return err

pkg/yqlib/printer_test.go

@@ -33,10 +33,10 @@ func nodeToList(candidate *CandidateNode) *list.List {
 	return elMap
 }
 
-func TestPrinterMultipleDocsInSequenceOnly(t *testing.T) {
+func TestPrinterMultipleDocsInSequence(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
@@ -74,7 +74,7 @@ func TestPrinterMultipleDocsInSequenceOnly(t *testing.T) {
 func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
@@ -116,7 +116,7 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
 func TestPrinterMultipleFilesInSequence(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
@@ -163,7 +163,7 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) {
 func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
@@ -213,7 +213,7 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
 func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
@@ -232,7 +232,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
@@ -261,7 +261,7 @@ a: coconut
 func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
@@ -287,7 +287,7 @@ a: coconut
 func TestPrinterScalarWithLeadingCont(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	node, err := NewExpressionParser().ParseExpression(".a")
 	if err != nil {
@@ -314,7 +314,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	// note printDocSeparators is true, it should still not print document separators
 	// when outputing JSON.
-	printer := NewPrinter(NewJsonEncoder(0), NewSinglePrinterWriter(writer))
+	printer := NewPrinterWithSingleWriter(writer, JsonOutputFormat, true, false, 0, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {

pkg/yqlib/utils.go

@@ -38,6 +38,43 @@ func writeString(writer io.Writer, txt string) error {
 	return errorWriting
 }
 
+func processLeadingContent(mappedDoc *CandidateNode, writer io.Writer, printDocSeparators bool, outputFormat PrinterOutputFormat) error {
+	log.Debug("headcommentwas %v", mappedDoc.LeadingContent)
+	log.Debug("finished headcomment")
+	reader := bufio.NewReader(strings.NewReader(mappedDoc.LeadingContent))
+
+	for {
+
+		readline, errReading := reader.ReadString('\n')
+		if errReading != nil && !errors.Is(errReading, io.EOF) {
+			return errReading
+		}
+		if strings.Contains(readline, "$yqDocSeperator$") {
+			if printDocSeparators {
+				if err := writeString(writer, "---\n"); err != nil {
+					return err
+				}
+			}
+		} else if outputFormat == YamlOutputFormat {
+			if err := writeString(writer, readline); err != nil {
+				return err
+			}
+		}
+
+		if errors.Is(errReading, io.EOF) {
+			if readline != "" {
+				// the last comment we read didn't have a new line, put one in
+				if err := writeString(writer, "\n"); err != nil {
+					return err
+				}
+			}
+			break
+		}
+	}
+
+	return nil
+}
+
 func processReadStream(reader *bufio.Reader) (io.Reader, string, error) {
 	var commentLineRegEx = regexp.MustCompile(`^\s*#`)
 	var sb strings.Builder

pkg/yqlib/write_in_place_handler.go

@@ -6,7 +6,7 @@ import (
 
 type writeInPlaceHandler interface {
 	CreateTempFile() (*os.File, error)
-	FinishWriteInPlace(evaluatedSuccessfully bool) error
+	FinishWriteInPlace(evaluatedSuccessfully bool)
 }
 
 type writeInPlaceHandlerImpl struct {
@@ -39,14 +39,13 @@ func (w *writeInPlaceHandlerImpl) CreateTempFile() (*os.File, error) {
 	return file, err
 }
 
-func (w *writeInPlaceHandlerImpl) FinishWriteInPlace(evaluatedSuccessfully bool) error {
+func (w *writeInPlaceHandlerImpl) FinishWriteInPlace(evaluatedSuccessfully bool) {
 	log.Debug("Going to write-inplace, evaluatedSuccessfully=%v, target=%v", evaluatedSuccessfully, w.inputFilename)
 	safelyCloseFile(w.tempFile)
 	if evaluatedSuccessfully {
 		log.Debug("Moving temp file to target")
-		return tryRenameFile(w.tempFile.Name(), w.inputFilename)
+		safelyRenameFile(w.tempFile.Name(), w.inputFilename)
+	} else {
+		tryRemoveFile(w.tempFile.Name())
 	}
-	tryRemoveTempFile(w.tempFile.Name())
-
-	return nil
 }

pkg/yqlib/xml_test.go

@@ -1,400 +0,0 @@
-package yqlib
-
-import (
-	"bufio"
-	"bytes"
-	"fmt"
-	"strings"
-	"testing"
-
-	"github.com/mikefarah/yq/v4/test"
-	yaml "gopkg.in/yaml.v3"
-)
-
-func decodeXml(t *testing.T, xml string) *CandidateNode {
-	decoder := NewXmlDecoder("+", "+content")
-
-	decoder.Init(strings.NewReader(xml))
-
-	node := &yaml.Node{}
-	err := decoder.Decode(node)
-	if err != nil {
-		t.Error(err, "fail to decode", xml)
-	}
-	return &CandidateNode{Node: node}
-}
-
-func processXmlScenario(s formatScenario) string {
-	var output bytes.Buffer
-	writer := bufio.NewWriter(&output)
-
-	var encoder = NewXmlEncoder(2, "+", "+content")
-
-	var decoder = NewYamlDecoder()
-	if s.scenarioType == "roundtrip" {
-		decoder = NewXmlDecoder("+", "+content")
-	}
-
-	inputs, err := readDocuments(strings.NewReader(s.input), "sample.yml", 0, decoder)
-	if err != nil {
-		panic(err)
-	}
-	node := inputs.Front().Value.(*CandidateNode).Node
-	err = encoder.Encode(writer, node)
-	if err != nil {
-		panic(err)
-	}
-	writer.Flush()
-
-	return output.String()
-}
-
-type formatScenario struct {
-	input          string
-	indent         int
-	expression     string
-	expected       string
-	description    string
-	subdescription string
-	skipDoc        bool
-	scenarioType   string
-}
-
-var inputXmlWithComments = `
-<!-- before cat -->
-<cat>
-	<!-- in cat before -->
-	<x>3<!-- multi
-line comment 
-for x --></x>
-	<!-- before y -->
-	<y>
-		<!-- in y before -->
-		<d><!-- in d before -->z<!-- in d after --></d>
-		
-		<!-- in y after -->
-	</y>
-	<!-- in_cat_after -->
-</cat>
-<!-- after cat -->
-`
-var inputXmlWithCommentsWithSubChild = `
-<!-- before cat -->
-<cat>
-	<!-- in cat before -->
-	<x>3<!-- multi
-line comment 
-for x --></x>
-	<!-- before y -->
-	<y>
-		<!-- in y before -->
-		<d><!-- in d before --><z sweet="cool"/><!-- in d after --></d>
-		
-		<!-- in y after -->
-	</y>
-	<!-- in_cat_after -->
-</cat>
-<!-- after cat -->
-`
-
-var expectedDecodeYamlWithSubChild = `D0, P[], (doc)::# before cat
-cat:
-    # in cat before
-    x: "3" # multi
-    # line comment 
-    # for x
-    # before y
-
-    y:
-        # in y before
-        d:
-            # in d before
-            z:
-                +sweet: cool
-            # in d after
-        # in y after
-    # in_cat_after
-# after cat
-`
-
-var inputXmlWithCommentsWithArray = `
-<!-- before cat -->
-<cat>
-	<!-- in cat before -->
-	<x>3<!-- multi
-line comment 
-for x --></x>
-	<!-- before y -->
-	<y>
-		<!-- in y before -->
-		<d><!-- in d before --><z sweet="cool"/><!-- in d after --></d>
-        <d><!-- in d2 before --><z sweet="cool2"/><!-- in d2 after --></d>
-		
-		<!-- in y after -->
-	</y>
-	<!-- in_cat_after -->
-</cat>
-<!-- after cat -->
-`
-
-var expectedDecodeYamlWithArray = `D0, P[], (doc)::# before cat
-cat:
-    # in cat before
-    x: "3" # multi
-    # line comment 
-    # for x
-    # before y
-
-    y:
-        # in y before
-        d:
-            - # in d before
-              z:
-                +sweet: cool
-              # in d after
-            - # in d2 before
-              z:
-                +sweet: cool2
-              # in d2 after
-        # in y after
-    # in_cat_after
-# after cat
-`
-
-var expectedDecodeYamlWithComments = `D0, P[], (doc)::# before cat
-cat:
-    # in cat before
-    x: "3" # multi
-    # line comment 
-    # for x
-    # before y
-
-    y:
-        # in y before
-        # in d before
-        d: z # in d after
-        # in y after
-    # in_cat_after
-# after cat
-`
-
-var expectedRoundtripXmlWithComments = `<!-- before cat --><cat><!-- in cat before -->
-  <x>3<!-- multi
-line comment 
-for x --></x><!-- before y -->
-  <y><!-- in y before
-in d before -->
-    <d>z<!-- in d after --></d><!-- in y after -->
-  </y><!-- in_cat_after -->
-</cat><!-- after cat -->
-`
-
-var yamlWithComments = `# above_cat
-cat: # inline_cat
-  # above_array
-  array: # inline_array
-    - val1 # inline_val1
-    # above_val2
-    - val2 # inline_val2
-# below_cat
-`
-
-var expectedXmlWithComments = `<!-- above_cat inline_cat --><cat><!-- above_array inline_array -->
-  <array>val1<!-- inline_val1 --></array>
-  <array><!-- above_val2 -->val2<!-- inline_val2 --></array>
-</cat><!-- below_cat -->
-`
-
-var xmlScenarios = []formatScenario{
-	{
-		description: "Parse xml: simple",
-		input:       "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat>meow</cat>",
-		expected:    "D0, P[], (doc)::cat: meow\n",
-	},
-	{
-		description:    "Parse xml: array",
-		subdescription: "Consecutive nodes with identical xml names are assumed to be arrays.",
-		input:          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<animal>1</animal>\n<animal>2</animal>",
-		expected:       "D0, P[], (doc)::animal:\n    - \"1\"\n    - \"2\"\n",
-	},
-	{
-		description:    "Parse xml: attributes",
-		subdescription: "Attributes are converted to fields, with the default attribute prefix '+'. Use '--xml-attribute-prefix` to set your own.",
-		input:          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">\n  <legs>7</legs>\n</cat>",
-		expected:       "D0, P[], (doc)::cat:\n    +legs: \"4\"\n    legs: \"7\"\n",
-	},
-	{
-		description:    "Parse xml: attributes with content",
-		subdescription: "Content is added as a field, using the default content name of '+content'. Use `--xml-content-name` to set your own.",
-		input:          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">meow</cat>",
-		expected:       "D0, P[], (doc)::cat:\n    +content: meow\n    +legs: \"4\"\n",
-	},
-	{
-		description:    "Parse xml: with comments",
-		subdescription: "A best attempt is made to preserve comments.",
-		input:          inputXmlWithComments,
-		expected:       expectedDecodeYamlWithComments,
-		scenarioType:   "decode",
-	},
-	{
-		description:  "Parse xml: with comments subchild",
-		skipDoc:      true,
-		input:        inputXmlWithCommentsWithSubChild,
-		expected:     expectedDecodeYamlWithSubChild,
-		scenarioType: "decode",
-	},
-	{
-		description:  "Parse xml: with comments array",
-		skipDoc:      true,
-		input:        inputXmlWithCommentsWithArray,
-		expected:     expectedDecodeYamlWithArray,
-		scenarioType: "decode",
-	},
-	{
-		description:  "Encode xml: simple",
-		input:        "cat: purrs",
-		expected:     "<cat>purrs</cat>\n",
-		scenarioType: "encode",
-	},
-	{
-		description:  "Encode xml: array",
-		input:        "pets:\n  cat:\n    - purrs\n    - meows",
-		expected:     "<pets>\n  <cat>purrs</cat>\n  <cat>meows</cat>\n</pets>\n",
-		scenarioType: "encode",
-	},
-	{
-		description:    "Encode xml: attributes",
-		subdescription: "Fields with the matching xml-attribute-prefix are assumed to be attributes.",
-		input:          "cat:\n  +name: tiger\n  meows: true\n",
-		expected:       "<cat name=\"tiger\">\n  <meows>true</meows>\n</cat>\n",
-		scenarioType:   "encode",
-	},
-	{
-		skipDoc:      true,
-		input:        "cat:\n  ++name: tiger\n  meows: true\n",
-		expected:     "<cat +name=\"tiger\">\n  <meows>true</meows>\n</cat>\n",
-		scenarioType: "encode",
-	},
-	{
-		description:    "Encode xml: attributes with content",
-		subdescription: "Fields with the matching xml-content-name is assumed to be content.",
-		input:          "cat:\n  +name: tiger\n  +content: cool\n",
-		expected:       "<cat name=\"tiger\">cool</cat>\n",
-		scenarioType:   "encode",
-	},
-	{
-		description:    "Encode xml: comments",
-		subdescription: "A best attempt is made to copy comments to xml.",
-		input:          yamlWithComments,
-		expected:       expectedXmlWithComments,
-		scenarioType:   "encode",
-	},
-	{
-		description:    "Round trip: with comments",
-		subdescription: "A best effort is made, but comment positions and white space are not preserved perfectly.",
-		input:          inputXmlWithComments,
-		expected:       expectedRoundtripXmlWithComments,
-		scenarioType:   "roundtrip",
-	},
-}
-
-func testXmlScenario(t *testing.T, s formatScenario) {
-	if s.scenarioType == "encode" || s.scenarioType == "roundtrip" {
-		test.AssertResultWithContext(t, s.expected, processXmlScenario(s), s.description)
-	} else {
-		var actual = resultToString(t, decodeXml(t, s.input))
-		test.AssertResultWithContext(t, s.expected, actual, s.description)
-	}
-}
-
-func documentXmlScenario(t *testing.T, w *bufio.Writer, i interface{}) {
-	s := i.(formatScenario)
-
-	if s.skipDoc {
-		return
-	}
-	if s.scenarioType == "encode" {
-		documentXmlEncodeScenario(w, s)
-	} else if s.scenarioType == "roundtrip" {
-		documentXmlRoundTripScenario(w, s)
-	} else {
-		documentXmlDecodeScenario(t, w, s)
-	}
-
-}
-
-func documentXmlDecodeScenario(t *testing.T, w *bufio.Writer, s formatScenario) {
-	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
-
-	if s.subdescription != "" {
-		writeOrPanic(w, s.subdescription)
-		writeOrPanic(w, "\n\n")
-	}
-
-	writeOrPanic(w, "Given a sample.xml file of:\n")
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input))
-
-	writeOrPanic(w, "then\n")
-	writeOrPanic(w, "```bash\nyq e -p=xml '.' sample.xml\n```\n")
-	writeOrPanic(w, "will output\n")
-
-	var output bytes.Buffer
-	printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
-
-	node := decodeXml(t, s.input)
-
-	err := printer.PrintResults(node.AsList())
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", output.String()))
-}
-
-func documentXmlEncodeScenario(w *bufio.Writer, s formatScenario) {
-	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
-
-	if s.subdescription != "" {
-		writeOrPanic(w, s.subdescription)
-		writeOrPanic(w, "\n\n")
-	}
-
-	writeOrPanic(w, "Given a sample.yml file of:\n")
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input))
-
-	writeOrPanic(w, "then\n")
-	writeOrPanic(w, "```bash\nyq e -o=xml '.' sample.yml\n```\n")
-	writeOrPanic(w, "will output\n")
-
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processXmlScenario(s)))
-}
-
-func documentXmlRoundTripScenario(w *bufio.Writer, s formatScenario) {
-	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
-
-	if s.subdescription != "" {
-		writeOrPanic(w, s.subdescription)
-		writeOrPanic(w, "\n\n")
-	}
-
-	writeOrPanic(w, "Given a sample.xml file of:\n")
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input))
-
-	writeOrPanic(w, "then\n")
-	writeOrPanic(w, "```bash\nyq e -p=xml -o=xml '.' sample.xml\n```\n")
-	writeOrPanic(w, "will output\n")
-
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processXmlScenario(s)))
-}
-
-func TestXmlScenarios(t *testing.T) {
-	for _, tt := range xmlScenarios {
-		testXmlScenario(t, tt)
-	}
-	genericScenarios := make([]interface{}, len(xmlScenarios))
-	for i, s := range xmlScenarios {
-		genericScenarios[i] = s
-	}
-	documentScenarios(t, "usage", "xml", genericScenarios, documentXmlScenario)
-}

release_instructions.txt

@@ -1,11 +1,8 @@
 - update release notes
+- check dependabot
 - increment version in version.go
 - increment version in snapcraft.yaml
 - make sure local build passes
-- check manpage 
-  - ./scripts/generate-man-page-md.sh
-  - ./scripts/generate-man-page.sh
-  - man ./yq.1
 - run ./scripts/copy-docs.sh (and commit the changed in the yq-book branch)
 - commit version update changes
 - tag git with same version number
@@ -36,15 +33,7 @@
   - build and push latest and new version tag
   - docker build .  -t mikefarah/yq:latest -t mikefarah/yq:3 -t mikefarah/yq:3.X
 
-- debian package (with release script)
-  - execute the script `./scripts/release-deb.sh` *on the machine having the private
-      gpg key to sign the generated sources* providing the version to release, the
-      ppa where deploying the files and the passphrase of the private key if needed:
-      ```
-      ./scripts/release-deb.sh -o output -p -s --passphrase PASSPHRASE VERSION
-      ```
-
-- debian package (manually)
+- debian package
   - ensure you get all vendor dependencies before packaging
 	```go mod vendor```
   - execute 

release_notes.txt

@@ -1,13 +1,3 @@
-4.17.2:
-  - Fixed manpath issue (thanks @mr-pmillz)
-  
-4.17.1:
-  - Added XML support (#491)
-  - New merge flag (n) to only merge new fields (#1038)
-  - Fixed exit status bug for permission denied error (#1062)
-  - Fixed using multiple variables with union (,) operator (#1048)
-  - Bumped some versions of dependencies
-
 4.16.2:
   - Bumped go-lang compiler to fix CVE-2021-44717 (#1037)
   - Dependency version bumps via dependabot

scripts/generate-man-page-md.sh

@@ -3,14 +3,10 @@ set -e
 
 # note that this reqires pandoc to be installed.
 
-cat ./pkg/yqlib/doc/operators/headers/Main.md > man.md
+cat ./pkg/yqlib/doc/headers/Main.md > man.md
 printf "\n# HOW IT WORKS\n" >> man.md
 tail -n +2 how-it-works.md >> man.md
 
-for f in ./pkg/yqlib/doc/operators/*.md; do
-  cat "$f" >> man.md
-done
-
-for f in ./pkg/yqlib/doc/usage/*.md; do
+for f in ./pkg/yqlib/doc/*.md; do 
   cat "$f" >> man.md
 done

scripts/release-deb.sh

@@ -1,264 +0,0 @@
-#!/bin/bash -eux
-#
-# Copyright (C) 2021 Roberto Mier Escandón <rmescandon@gmail.com>
-#
-# This script creates a .deb package file with yq valid for ubuntu 20.04 by default
-# You can pass 
-
-DOCKER_IMAGE_NAME=yq-deb-builder
-DOCKER_IMAGE_TAG=$(git describe --always --tags)
-OUTPUT=
-GOVERSION="1.17.4"
-KEYID=
-MAINTAINER=
-DO_PUBLISH=
-PPA="rmescandon/yq"
-VERSION=
-DISTRIBUTION=
-DO_SIGN=
-PASSPHRASE=
-
-show_help() {
-  echo "  usage: $(basename "$0") VERSION [options...]"
-  echo ""
-  echo "  positional arguments"
-  echo "    VERSION"
-  echo ""
-  echo "  optional arguments:"
-  echo "    -h, --help		              Shows this help"
-  echo "    -d, --distribution DISTRO   The distribution to use for the changelog generation. If not provided, last changelog entry"
-  echo "                                  distribution is considered"
-  echo "    --goversion VERSION         The version of golang to use. Default to $GOVERSION"
-  echo "    -k, --sign-key KEYID        Sign the package sources with the provided gpg key id (long format). When not provided this"
-  echo "                                  paramater, the generated sources are not signed"
-  echo "    -s, --sign                  Sign the package sources with a gpg key of the maintainer"
-  echo "    -m, --maintainer WHO        The maintainer used as author of the changelog. git.name and git.email (see git config) is"
-  echo "                                  the considered format"
-  echo "    -o DIR, --output DIR        The path where leaving the generated debian package. Default to a temporary folder if not set"
-  echo "    -p                          The resultant file is being published to ppa"
-  echo "    --ppa PPA                   Push resultant files to indicated ppa. This option should be given along with a signing key."
-  echo "                                  Otherwise, the server could reject the package building. Default is set to 'rmescandon/yq'"
-  echo "    --passphrase PASSPHRASE     Passphrase to decrypt the signage key"
-  exit 1
-}
-# read input args
-while [ $# -ne 0 ]; do
-  case $1 in
-    -h|--help)
-      show_help
-      ;;
-    -d|--distribution)
-      shift
-      DISTRIBUTION="$1"
-      ;;
-    --goversion)
-      shift
-      GOVERSION="$1"
-      ;;
-    -k|--sign-key)
-      shift
-      DO_SIGN='y'
-      KEYID="$1"
-      ;;
-    -s|--sign)
-      DO_SIGN='y'
-      ;;
-    -m|--maintainer)
-      shift
-      MAINTAINER="$1"
-      ;;
-    -o|--output)
-      shift
-      OUTPUT="$1"
-      ;;
-    -p)
-      DO_PUBLISH="y"
-      ;;
-    --ppa)
-      shift
-      DO_PUBLISH="y"
-      PPA="$1"
-      ;;
-    --passphrase)
-      shift
-      PASSPHRASE="$1"
-      ;;
-    *)
-      if [ -z "$VERSION" ]; then
-        VERSION="$1"
-      else
-        show_help
-      fi
-  esac
-  shift
-done
-
-[ -n "$VERSION" ] || (echo "error - you have to provide a version" && show_help)
-
-if [ -n "$OUTPUT" ]; then
-  OUTPUT="$(realpath "$OUTPUT")"
-  mkdir -p "$OUTPUT"
-else
-  # Temporary folder where leaving the built deb package in case that output folder is not provided
-  OUTPUT="$(mktemp -d)"
-fi 
-
-# Define the folders with the source project and the build artifacts and files
-srcdir="$(realpath "$(dirname "$0")"/..)"
-blddir="$(cd "${srcdir}" && mkdir -p build && cd build && echo "$(pwd)")"
-# clean on exit
-cleanup() {
-  rm -f "${blddir}/build.sh" || true
-  rm -f "${blddir}/Dockerfile" || true
-  rm -f "${blddir}/dput.cf" || true
-  docker rmi "${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" -f > /dev/null 2>&1 || true
-}
-trap cleanup EXIT INT
-
-# configure the dput config in case publishing is requested
-lp_id="$(echo "$PPA" | cut -d'/' -f1)"
-ppa_id="$(echo "$PPA" | cut -d'/' -f2)"
-cat << EOF > ${blddir}/dput.cf
-[ppa]
-fqdn                    = ppa.launchpad.net
-method                  = ftp
-incoming                = ~${lp_id}/ubuntu/${ppa_id}
-login                   = anonymous
-EOF
-
-# create the main script
-cat << EOF > ${blddir}/build.sh
-#!/bin/bash 
-set -e -o pipefail
-
-PATH=$PATH:/usr/local/go/bin
-export GPG_TTY=$(tty)
-
-go mod vendor
-
-### bump debian/changelog
-
-# maintainer
-export DEBEMAIL="$MAINTAINER"
-if [ -z "$MAINTAINER" ]; then
-  export DEBEMAIL="\$(dpkg-parsechangelog -S Maintainer)"
-fi
-
-# prepend a 'v' char to complete the tag name from where calculating the changelog
-SINCE="v\$(dpkg-parsechangelog -S Version)"
-
-# distribution
-DISTRIBUTION="$DISTRIBUTION"
-if [ -z "$DISTRIBUTION" ]; then
-  DISTRIBUTION="\$(dpkg-parsechangelog -S Distribution)"
-fi
-
-# generate changelog
-gbp dch --ignore-branch --no-multimaint -N "$VERSION" -s "\$SINCE" -D "\$DISTRIBUTION"
-
-# using -d to prevent failing when searching for golang dep on control file
-params=("-d" "-S")
-
-# add the -sa option for signing along with the key to use when provided key id
-if [ -n "$DO_SIGN" ]; then
-  params+=("-sa")
-
-  # read from gpg the key id associated with the maintainer if not provided explicitly
-  if [ -z "$KEYID" ]; then
-    KEYID="\$(gpg --list-keys "\$(dpkg-parsechangelog -S Maintainer)" | head -2 | tail -1 | xargs)"
-  else
-    KEYID="$KEYID"
-  fi
-  
-  params+=("--sign-key="\$KEYID"")
-
-  if [ -n "$PASSPHRASE" ]; then
-    gpg-agent --verbose --daemon --options /home/yq/.gnupg/gpg-agent.conf --log-file /tmp/gpg-agent.log --allow-preset-passphrase --default-cache-ttl=31536000
-    KEYGRIP="\$(gpg --with-keygrip -k "\$KEYID" | grep 'Keygrip = ' | cut -d'=' -f2 | head -1 | xargs)"
-    /usr/lib/gnupg/gpg-preset-passphrase --preset --passphrase "$PASSPHRASE" "\$KEYGRIP"
-  fi
-
-else
-  params+=("-us" "-uc")
-fi
-
-debuild  \${params[@]}
-mv ../yq_* /home/yq/output
-
-echo ""
-echo -e "\tfind resulting package at: "$OUTPUT""
-
-# publish to ppa whether given
-if [ -n "$DO_PUBLISH" ]; then
-  dput -c /etc/dput.cf ppa /home/yq/output/yq_*.changes
-fi
-EOF
-chmod +x "${blddir}"/build.sh
-
-# build the docker image with all dependencies
-cat << EOF > ${blddir}/Dockerfile
-FROM bitnami/minideb:bullseye as base
-ENV LANG C.UTF-8
-ENV LC_ALL C.UTF-8
-ENV DEBIAN_FRONTEND noninteractive
-ENV GO111MODULE on
-ENV GOMODCACHE /home/yq/go
-
-RUN set -e \
-  && sed -i -- 's/# deb-src/deb-src/g' /etc/apt/sources.list \
-  && apt-get -qq update
-
-# install golang on its $GOVERSION
-FROM base as golang
-RUN apt-get -qq -y --no-install-recommends install \
-    ca-certificates \
-    wget
-RUN wget "https://golang.org/dl/go${GOVERSION}.linux-amd64.tar.gz" -4
-RUN tar -C /usr/local -xvf "go${GOVERSION}.linux-amd64.tar.gz"
-
-FROM base
-RUN apt-get -qq -y --no-install-recommends install \
-    build-essential \
-    debhelper \
-    devscripts \
-    dput \
-    fakeroot \
-    git-buildpackage \
-    gpg-agent \
-    libdistro-info-perl \
-    pandoc \
-    rsync \
-    sensible-utils && \
-  apt-get clean && \
-  rm -rf /tmp/* /var/tmp/*
-
-COPY --from=golang /usr/local/go /usr/local/go
-
-# build debian package as yq user
-RUN useradd -ms /bin/bash yq && \
-  mkdir /home/yq/src && chown -R yq: /home/yq/src && \
-  mkdir /home/yq/output && chown -R yq: /home/yq/output
-
-ADD ./build/dput.cf /etc/dput.cf
-ADD ./build/build.sh /usr/bin/build.sh
-RUN chmod +x /usr/bin/build.sh && chown -R yq: /usr/bin/build.sh
-
-USER yq
-
-WORKDIR /home/yq/src
-VOLUME ["/home/yq/src"]
-
-# dir where output packages are finally left
-VOLUME ["/home/yq/output"]
-
-CMD ["/usr/bin/build.sh"]
-EOF
-
-DOCKER_BUILDKIT=1 docker build --pull -f "${blddir}"/Dockerfile -t "${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" .
-
-docker run --rm -i \
-  -v "${srcdir}":/home/yq/src:delegated \
-  -v "${OUTPUT}":/home/yq/output \
-  -v "${HOME}"/.gnupg:/home/yq/.gnupg:delegated \
-  -u "$(id -u)" \
-  "${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.17.2'
+version: '4.16.2'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.
@@ -16,7 +16,7 @@ apps:
 parts:
   yq:
     plugin: go
-    go-channel: 1.17/stable
+    go-channel: 1.15/stable
     source: .
     source-type: git
     go-importpath: github.com/mikefarah/yq

test/utils.go

@@ -1,7 +1,6 @@
 package test
 
 import (
-	"bufio"
 	"bytes"
 	"fmt"
 	"os"
@@ -9,8 +8,6 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/pkg/diff"
-	"github.com/pkg/diff/write"
 	"github.com/spf13/cobra"
 	yaml "gopkg.in/yaml.v3"
 )
@@ -66,14 +63,8 @@ func AssertResultComplexWithContext(t *testing.T, expectedValue interface{}, act
 
 func AssertResultWithContext(t *testing.T, expectedValue interface{}, actualValue interface{}, context interface{}) {
 	t.Helper()
-	opts := []write.Option{write.TerminalColor()}
 	if expectedValue != actualValue {
 		t.Error(context)
-		var differenceBuffer bytes.Buffer
-		if err := diff.Text("expected", "actual", expectedValue, actualValue, bufio.NewWriter(&differenceBuffer), opts...); err != nil {
-			t.Error(err)
-		} else {
-			t.Error(differenceBuffer.String())
-		}
+		t.Error(": expected <", expectedValue, "> but got <", actualValue, ">")
 	}
 }