Bumping version

* Refactor ordered_map into separate files

Separate json and xml, from the regular yaml.

Makes it possible to compile, without those...

* Refactor encoder and decoder creation

Use more consistent parameters vs globals

Return errors instead of calling panic()

* Allow build without json and xml support

cmd/evaluate_all_command.go

@@ -84,7 +84,10 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 	if err != nil {
 		return err
 	}
-	encoder := configureEncoder(format)
+	encoder, err := configureEncoder()
+	if err != nil {
+		return err
+	}
 
 	printer := yqlib.NewPrinter(encoder, printerWriter)
 

cmd/evalute_sequence_command.go

@@ -93,7 +93,10 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 	if err != nil {
 		return err
 	}
-	encoder := configureEncoder(format)
+	encoder, err := configureEncoder()
+	if err != nil {
+		return err
+	}
 
 	printer := yqlib.NewPrinter(encoder, printerWriter)
 

cmd/utils.go

@@ -61,7 +61,15 @@ func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
 	if err != nil {
 		return nil, err
 	}
-	switch yqlibInputFormat {
+	yqlibDecoder, err := createDecoder(yqlibInputFormat, evaluateTogether)
+	if yqlibDecoder == nil {
+		return nil, fmt.Errorf("no support for %s input format", inputFormat)
+	}
+	return yqlibDecoder, err
+}
+
+func createDecoder(format yqlib.InputFormat, evaluateTogether bool) (yqlib.Decoder, error) {
+	switch format {
 	case yqlib.XMLInputFormat:
 		return yqlib.NewXMLDecoder(yqlib.ConfiguredXMLPreferences), nil
 	case yqlib.PropertiesInputFormat:
@@ -72,10 +80,12 @@ func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
 		return yqlib.NewCSVObjectDecoder(','), nil
 	case yqlib.TSVObjectInputFormat:
 		return yqlib.NewCSVObjectDecoder('\t'), nil
+	case yqlib.YamlInputFormat:
+		prefs := yqlib.ConfiguredYamlPreferences
+		prefs.EvaluateTogether = evaluateTogether
+		return yqlib.NewYamlDecoder(prefs), nil
 	}
-	prefs := yqlib.ConfiguredYamlPreferences
-	prefs.EvaluateTogether = evaluateTogether
-	return yqlib.NewYamlDecoder(prefs), nil
+	return nil, fmt.Errorf("invalid decoder: %v", format)
 }
 
 func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yqlib.PrinterWriter, error) {
@@ -95,22 +105,34 @@ func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yq
 	return printerWriter, nil
 }
 
-func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder {
+func configureEncoder() (yqlib.Encoder, error) {
+	yqlibOutputFormat, err := yqlib.OutputFormatFromString(outputFormat)
+	if err != nil {
+		return nil, err
+	}
+	yqlibEncoder, err := createEncoder(yqlibOutputFormat)
+	if yqlibEncoder == nil {
+		return nil, fmt.Errorf("no support for %s output format", outputFormat)
+	}
+	return yqlibEncoder, err
+}
+
+func createEncoder(format yqlib.PrinterOutputFormat) (yqlib.Encoder, error) {
 	switch format {
 	case yqlib.JSONOutputFormat:
-		return yqlib.NewJSONEncoder(indent, colorsEnabled, unwrapScalar)
+		return yqlib.NewJSONEncoder(indent, colorsEnabled, unwrapScalar), nil
 	case yqlib.PropsOutputFormat:
-		return yqlib.NewPropertiesEncoder(unwrapScalar)
+		return yqlib.NewPropertiesEncoder(unwrapScalar), nil
 	case yqlib.CSVOutputFormat:
-		return yqlib.NewCsvEncoder(',')
+		return yqlib.NewCsvEncoder(','), nil
 	case yqlib.TSVOutputFormat:
-		return yqlib.NewCsvEncoder('\t')
+		return yqlib.NewCsvEncoder('\t'), nil
 	case yqlib.YamlOutputFormat:
-		return yqlib.NewYamlEncoder(indent, colorsEnabled, yqlib.ConfiguredYamlPreferences)
+		return yqlib.NewYamlEncoder(indent, colorsEnabled, yqlib.ConfiguredYamlPreferences), nil
 	case yqlib.XMLOutputFormat:
-		return yqlib.NewXMLEncoder(indent, yqlib.ConfiguredXMLPreferences)
+		return yqlib.NewXMLEncoder(indent, yqlib.ConfiguredXMLPreferences), nil
 	}
-	panic("invalid encoder")
+	return nil, fmt.Errorf("invalid encoder: %v", format)
 }
 
 // this is a hack to enable backwards compatibility with githubactions (which pipe /dev/null into everything)

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "v4.31.1"
+	Version = "v4.31.2"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

go.mod

@@ -1,7 +1,7 @@
 module github.com/mikefarah/yq/v4
 
 require (
-	github.com/a8m/envsubst v1.4.1
+	github.com/a8m/envsubst v1.4.2
 	github.com/alecthomas/participle/v2 v2.0.0-beta.5
 	github.com/alecthomas/repr v0.2.0
 	github.com/dimchansky/utfbom v1.1.1
@@ -14,7 +14,7 @@ require (
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/pflag v1.0.5
-	golang.org/x/net v0.1.1-0.20221104162952-702349b0e862
+	golang.org/x/net v0.7.0
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -23,8 +23,8 @@ require (
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
-	golang.org/x/sys v0.3.0 // indirect
-	golang.org/x/text v0.4.0 // indirect
+	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/text v0.7.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 )
 

go.sum

@@ -1,5 +1,5 @@
-github.com/a8m/envsubst v1.4.1 h1:RShc2OKbQpIIp5qR2glVsBOJCImTVHFrDkJvmGh0Qi0=
-github.com/a8m/envsubst v1.4.1/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
+github.com/a8m/envsubst v1.4.2 h1:4yWIHXOLEJHQEFd4UjrWDrYeYlV7ncFWJOCBRLOZHQg=
+github.com/a8m/envsubst v1.4.2/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
 github.com/alecthomas/assert/v2 v2.0.3 h1:WKqJODfOiQG0nEJKFKzDIG3E29CN2/4zR9XGJzKIkbg=
 github.com/alecthomas/participle/v2 v2.0.0-beta.5 h1:y6dsSYVb1G5eK6mgmy+BgI3Mw35a3WghArZ/Hbebrjo=
 github.com/alecthomas/participle/v2 v2.0.0-beta.5/go.mod h1:RC764t6n4L8D8ITAJv0qdokritYSNR3wV5cVwmIEaMM=
@@ -54,20 +54,20 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.1.1-0.20221104162952-702349b0e862 h1:KrLJ+iz8J6j6VVr/OCfULAcK+xozUmWE43fKpMR4MlI=
-golang.org/x/net v0.1.1-0.20221104162952-702349b0e862/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
+golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
-golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
-golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0=

pkg/yqlib/decoder_json.go

@@ -1,3 +1,5 @@
+//go:build !yq_nojson
+
 package yqlib
 
 import (

pkg/yqlib/decoder_xml.go

@@ -1,3 +1,5 @@
+//go:build !yq_noxml
+
 package yqlib
 
 import (
@@ -5,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"regexp"
 	"strings"
 	"unicode"
 
@@ -46,11 +49,19 @@ func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yaml.Node, error) {
 	return yamlNode, nil
 }
 
+var decoderCommentPrefix = regexp.MustCompile(`(^|\n)([[:alpha:]])`)
+
 func (dec *xmlDecoder) processComment(c string) string {
 	if c == "" {
 		return ""
 	}
-	return "#" + strings.TrimRight(c, " ")
+	//need to replace "cat " with "# cat"
+	// "\ncat\n" with "\n cat\n"
+	// ensure non-empty comments starting with newline have a space in front
+
+	replacement := decoderCommentPrefix.ReplaceAllString(c, "$1 $2")
+	replacement = "#" + strings.ReplaceAll(strings.TrimRight(replacement, " "), "\n", "\n#")
+	return replacement
 }
 
 func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
@@ -75,6 +86,7 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
 		var err error
 
 		if i == 0 {
+			log.Debugf("head comment here")
 			labelNode.HeadComment = dec.processComment(n.HeadComment)
 
 		}

pkg/yqlib/doc/usage/xml.md

@@ -407,8 +407,10 @@ A best attempt is made to copy comments to xml.
 
 Given a sample.yml file of:
 ```yaml
+#
 # header comment
 # above_cat
+#
 cat: # inline_cat
   # above_array
   array: # inline_array
@@ -425,9 +427,11 @@ yq -o=xml '.' sample.yml
 will output
 ```xml
 <!--
- header comment
- above_cat
- --><!-- inline_cat --><cat><!-- above_array inline_array -->
+header comment
+above_cat
+-->
+<!-- inline_cat -->
+<cat><!-- above_array inline_array -->
   <array>val1<!-- inline_val1 --></array>
   <array><!-- above_val2 -->val2<!-- inline_val2 --></array>
 </cat><!-- below_cat -->
@@ -489,7 +493,8 @@ yq -p=xml -o=xml '.' sample.xml
 ```
 will output
 ```xml
-<!-- before cat --><cat><!-- in cat before -->
+<!-- before cat -->
+<cat><!-- in cat before -->
   <x>3<!-- multi
 line comment 
 for x --></x><!-- before y -->

pkg/yqlib/encoder.go

@@ -1,10 +1,6 @@
 package yqlib
 
 import (
-	"bytes"
-	"encoding/json"
-	"errors"
-	"fmt"
 	"io"
 
 	yaml "gopkg.in/yaml.v3"
@@ -17,162 +13,17 @@ type Encoder interface {
 	CanHandleAliases() bool
 }
 
-// orderedMap allows to marshal and unmarshal JSON and YAML values keeping the
-// order of keys and values in a map or an object.
-type orderedMap struct {
-	// if this is an object, kv != nil. If this is not an object, kv == nil.
-	kv     []orderedMapKV
-	altVal interface{}
-}
+func mapKeysToStrings(node *yaml.Node) {
 
-type orderedMapKV struct {
-	K string
-	V orderedMap
-}
-
-func (o *orderedMap) UnmarshalJSON(data []byte) error {
-	switch data[0] {
-	case '{':
-		// initialise so that even if the object is empty it is not nil
-		o.kv = []orderedMapKV{}
-
-		// create decoder
-		dec := json.NewDecoder(bytes.NewReader(data))
-		_, err := dec.Token() // open object
-		if err != nil {
-			return err
-		}
-
-		// cycle through k/v
-		var tok json.Token
-		for tok, err = dec.Token(); err == nil; tok, err = dec.Token() {
-			// we can expect two types: string or Delim. Delim automatically means
-			// that it is the closing bracket of the object, whereas string means
-			// that there is another key.
-			if _, ok := tok.(json.Delim); ok {
-				break
+	if node.Kind == yaml.MappingNode {
+		for index, child := range node.Content {
+			if index%2 == 0 { // its a map key
+				child.Tag = "!!str"
 			}
-			kv := orderedMapKV{
-				K: tok.(string),
-			}
-			if err := dec.Decode(&kv.V); err != nil {
-				return err
-			}
-			o.kv = append(o.kv, kv)
 		}
-		// unexpected error
-		if err != nil && !errors.Is(err, io.EOF) {
-			return err
-		}
-		return nil
-	case '[':
-		var res []*orderedMap
-		if err := json.Unmarshal(data, &res); err != nil {
-			return err
-		}
-		o.altVal = res
-		o.kv = nil
-		return nil
 	}
 
-	return json.Unmarshal(data, &o.altVal)
-}
-
-func (o orderedMap) MarshalJSON() ([]byte, error) {
-	buf := new(bytes.Buffer)
-	enc := json.NewEncoder(buf)
-	enc.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
-	if o.kv == nil {
-		if err := enc.Encode(o.altVal); err != nil {
-			return nil, err
-		}
-		return buf.Bytes(), nil
-	}
-	buf.WriteByte('{')
-	for idx, el := range o.kv {
-		if err := enc.Encode(el.K); err != nil {
-			return nil, err
-		}
-		buf.WriteByte(':')
-		if err := enc.Encode(el.V); err != nil {
-			return nil, err
-		}
-		if idx != len(o.kv)-1 {
-			buf.WriteByte(',')
-		}
-	}
-	buf.WriteByte('}')
-	return buf.Bytes(), nil
-}
-
-func (o *orderedMap) UnmarshalYAML(node *yaml.Node) error {
-	switch node.Kind {
-	case yaml.DocumentNode:
-		if len(node.Content) == 0 {
-			return nil
-		}
-		return o.UnmarshalYAML(node.Content[0])
-	case yaml.AliasNode:
-		return o.UnmarshalYAML(node.Alias)
-	case yaml.ScalarNode:
-		return node.Decode(&o.altVal)
-	case yaml.MappingNode:
-		// set kv to non-nil
-		o.kv = []orderedMapKV{}
-		for i := 0; i < len(node.Content); i += 2 {
-			var key string
-			var val orderedMap
-			if err := node.Content[i].Decode(&key); err != nil {
-				return err
-			}
-			if err := node.Content[i+1].Decode(&val); err != nil {
-				return err
-			}
-			o.kv = append(o.kv, orderedMapKV{
-				K: key,
-				V: val,
-			})
-		}
-		return nil
-	case yaml.SequenceNode:
-		// note that this has to be a pointer, so that nulls can be represented.
-		var res []*orderedMap
-		if err := node.Decode(&res); err != nil {
-			return err
-		}
-		o.altVal = res
-		o.kv = nil
-		return nil
-	case 0:
-		// null
-		o.kv = nil
-		o.altVal = nil
-		return nil
-	default:
-		return fmt.Errorf("orderedMap: invalid yaml node")
+	for _, child := range node.Content {
+		mapKeysToStrings(child)
 	}
 }
-
-func (o *orderedMap) MarshalYAML() (interface{}, error) {
-	// fast path: kv is nil, use altVal
-	if o.kv == nil {
-		return o.altVal, nil
-	}
-	content := make([]*yaml.Node, 0, len(o.kv)*2)
-	for _, val := range o.kv {
-		n := new(yaml.Node)
-		if err := n.Encode(val.V); err != nil {
-			return nil, err
-		}
-		content = append(content, &yaml.Node{
-			Kind:  yaml.ScalarNode,
-			Tag:   "!!str",
-			Value: val.K,
-		}, n)
-	}
-	return &yaml.Node{
-		Kind:    yaml.MappingNode,
-		Tag:     "!!map",
-		Content: content,
-	}, nil
-}

pkg/yqlib/encoder_json.go

@@ -1,3 +1,5 @@
+//go:build !yq_nojson
+
 package yqlib
 
 import (
@@ -14,21 +16,6 @@ type jsonEncoder struct {
 	UnwrapScalar bool
 }
 
-func mapKeysToStrings(node *yaml.Node) {
-
-	if node.Kind == yaml.MappingNode {
-		for index, child := range node.Content {
-			if index%2 == 0 { // its a map key
-				child.Tag = "!!str"
-			}
-		}
-	}
-
-	for _, child := range node.Content {
-		mapKeysToStrings(child)
-	}
-}
-
 func NewJSONEncoder(indent int, colorise bool, unwrapScalar bool) Encoder {
 	var indentString = ""
 

pkg/yqlib/encoder_test.go

@@ -1,3 +1,5 @@
+//go:build !yq_nojson
+
 package yqlib
 
 import (

pkg/yqlib/encoder_xml.go

@@ -1,3 +1,5 @@
+//go:build !yq_noxml
+
 package yqlib
 
 import (
@@ -17,8 +19,6 @@ type xmlEncoder struct {
 	leadingContent string
 }
 
-var commentPrefix = regexp.MustCompile(`(^|\n)\s*#`)
-
 func NewXMLEncoder(indent int, prefs XmlPreferences) Encoder {
 	var indentString = ""
 
@@ -37,7 +37,7 @@ func (e *xmlEncoder) PrintDocumentSeparator(writer io.Writer) error {
 }
 
 func (e *xmlEncoder) PrintLeadingContent(writer io.Writer, content string) error {
-	e.leadingContent = commentPrefix.ReplaceAllString(content, "\n")
+	e.leadingContent = content
 	return nil
 }
 
@@ -46,12 +46,39 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
 	// hack so we can manually add newlines to procInst and directives
 	e.writer = writer
 	encoder.Indent("", e.indentString)
+	var newLine xml.CharData = []byte("\n")
+
+	mapNode := unwrapDoc(node)
+	if mapNode.Tag == "!!map" {
+		// make sure <?xml .. ?> processing instructions are encoded first
+		for i := 0; i < len(mapNode.Content); i += 2 {
+			key := mapNode.Content[i]
+			value := mapNode.Content[i+1]
+
+			if key.Value == (e.prefs.ProcInstPrefix + "xml") {
+				name := strings.Replace(key.Value, e.prefs.ProcInstPrefix, "", 1)
+				procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)}
+				if err := encoder.EncodeToken(procInst); err != nil {
+					return err
+				}
+				if _, err := e.writer.Write([]byte("\n")); err != nil {
+					log.Warning("Unable to write newline, skipping: %w", err)
+				}
+			}
+		}
+	}
 
 	if e.leadingContent != "" {
+
+		// remove first and last newlines if present
 		err := e.encodeComment(encoder, e.leadingContent)
 		if err != nil {
 			return err
 		}
+		err = encoder.EncodeToken(newLine)
+		if err != nil {
+			return err
+		}
 	}
 
 	switch node.Kind {
@@ -87,29 +114,12 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
 	default:
 		return fmt.Errorf("unsupported type %v", node.Tag)
 	}
-	var charData xml.CharData = []byte("\n")
-	return encoder.EncodeToken(charData)
+
+	return encoder.EncodeToken(newLine)
 
 }
 
 func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node) error {
-	// make sure <?xml .. ?> processing instructions are encoded first
-	for i := 0; i < len(node.Content); i += 2 {
-		key := node.Content[i]
-		value := node.Content[i+1]
-
-		if key.Value == (e.prefs.ProcInstPrefix + "xml") {
-			name := strings.Replace(key.Value, e.prefs.ProcInstPrefix, "", 1)
-			procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)}
-			if err := encoder.EncodeToken(procInst); err != nil {
-				return err
-			}
-			if _, err := e.writer.Write([]byte("\n")); err != nil {
-				log.Warning("Unable to write newline, skipping: %w", err)
-			}
-		}
-	}
-
 	err := e.encodeComment(encoder, headAndLineComment(node))
 	if err != nil {
 		return err
@@ -124,6 +134,13 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node) er
 		if err != nil {
 			return err
 		}
+		if headAndLineComment(key) != "" {
+			var newLine xml.CharData = []byte("\n")
+			err = encoder.EncodeToken(newLine)
+			if err != nil {
+				return err
+			}
+		}
 
 		if key.Value == (e.prefs.ProcInstPrefix + "xml") {
 			// dont double process these.
@@ -204,13 +221,34 @@ func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *yaml.Node, start xml.S
 	return fmt.Errorf("unsupported type %v", node.Tag)
 }
 
+var xmlEncodeMultilineCommentRegex = regexp.MustCompile(`(^|\n) *# ?(.*)`)
+var xmlEncodeSingleLineCommentRegex = regexp.MustCompile(`^\s*#(.*)\n?`)
+var chompRegexp = regexp.MustCompile(`\n$`)
+
 func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) error {
 	if commentStr != "" {
-		log.Debugf("encoding comment %v", commentStr)
-		if !strings.HasSuffix(commentStr, " ") {
-			commentStr = commentStr + " "
+		log.Debugf("got comment [%v]", commentStr)
+		// multi line string
+		if len(commentStr) > 2 && strings.Contains(commentStr[1:len(commentStr)-1], "\n") {
+			commentStr = chompRegexp.ReplaceAllString(commentStr, "")
+			log.Debugf("chompRegexp [%v]", commentStr)
+			commentStr = xmlEncodeMultilineCommentRegex.ReplaceAllString(commentStr, "$1$2")
+			log.Debugf("processed multine [%v]", commentStr)
+			// if the first line is non blank, add a space
+			if commentStr[0] != '\n' && commentStr[0] != ' ' {
+				commentStr = " " + commentStr
+			}
+
+		} else {
+			commentStr = xmlEncodeSingleLineCommentRegex.ReplaceAllString(commentStr, "$1")
 		}
 
+		if !strings.HasSuffix(commentStr, " ") && !strings.HasSuffix(commentStr, "\n") {
+			commentStr = commentStr + " "
+			log.Debugf("added suffix [%v]", commentStr)
+		}
+		log.Debugf("encoding comment [%v]", commentStr)
+
 		var comment xml.Comment = []byte(commentStr)
 		err := encoder.EncodeToken(comment)
 		if err != nil {

pkg/yqlib/json_test.go

@@ -1,3 +1,5 @@
+//go:build !yq_nojson
+
 package yqlib
 
 import (

pkg/yqlib/lib.go

@@ -53,7 +53,7 @@ var subtractAssignOpType = &operationType{Type: "SUBTRACT_ASSIGN", NumArgs: 2, P
 
 var assignAttributesOpType = &operationType{Type: "ASSIGN_ATTRIBUTES", NumArgs: 2, Precedence: 40, Handler: assignAttributesOperator}
 var assignStyleOpType = &operationType{Type: "ASSIGN_STYLE", NumArgs: 2, Precedence: 40, Handler: assignStyleOperator}
-var assignVariableOpType = &operationType{Type: "ASSIGN_VARIABLE", NumArgs: 2, Precedence: 40, Handler: assignVariableOperator}
+var assignVariableOpType = &operationType{Type: "ASSIGN_VARIABLE", NumArgs: 2, Precedence: 40, Handler: useWithPipe}
 var assignTagOpType = &operationType{Type: "ASSIGN_TAG", NumArgs: 2, Precedence: 40, Handler: assignTagOperator}
 var assignCommentOpType = &operationType{Type: "ASSIGN_COMMENT", NumArgs: 2, Precedence: 40, Handler: assignCommentsOperator}
 var assignAnchorOpType = &operationType{Type: "ASSIGN_ANCHOR", NumArgs: 2, Precedence: 40, Handler: assignAnchorOperator}
@@ -338,7 +338,7 @@ func deepCloneWithOptions(node *yaml.Node, cloneContent bool) *yaml.Node {
 		Tag:         node.Tag,
 		Value:       node.Value,
 		Anchor:      node.Anchor,
-		Alias:       deepClone(node.Alias),
+		Alias:       node.Alias,
 		HeadComment: node.HeadComment,
 		LineComment: node.LineComment,
 		FootComment: node.FootComment,

pkg/yqlib/no_json.go

@@ -0,0 +1,11 @@
+//go:build yq_nojson
+
+package yqlib
+
+func NewJSONDecoder() Decoder {
+	return nil
+}
+
+func NewJSONEncoder(indent int, colorise bool, unwrapScalar bool) Encoder {
+	return nil
+}

pkg/yqlib/no_xml.go

@@ -0,0 +1,11 @@
+//go:build yq_noxml
+
+package yqlib
+
+func NewXMLDecoder(prefs XmlPreferences) Decoder {
+	return nil
+}
+
+func NewXMLEncoder(indent int, prefs XmlPreferences) Encoder {
+	return nil
+}

pkg/yqlib/operator_add_test.go

@@ -34,7 +34,7 @@ var addOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `{}`,
-		expression: "(.a + .b) as $x",
+		expression: "(.a + .b) as $x | .",
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},

pkg/yqlib/operator_alternative_test.go

@@ -16,7 +16,7 @@ var alternativeOperatorScenarios = []expressionScenario{
 	},
 	{
 		skipDoc:    true,
-		expression: `(.b // "hello") as $x`,
+		expression: `(.b // "hello") as $x | .`,
 		document:   `a: bridge`,
 		expected: []string{
 			"D0, P[], (doc)::a: bridge\n",

pkg/yqlib/operator_assign_test.go

@@ -4,6 +4,11 @@ import (
 	"testing"
 )
 
+var mergeAnchorAssign = `a: &a
+  x: OriginalValue
+b:
+  <<: *a`
+
 var assignOperatorScenarios = []expressionScenario{
 	{
 		description: "Create yaml file",
@@ -20,6 +25,14 @@ var assignOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: null\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   mergeAnchorAssign,
+		expression: `.c = .b | .a.x = "ModifiedValue" | explode(.)`,
+		expected: []string{
+			"D0, P[], (doc)::a:\n    x: ModifiedValue\nb:\n    x: ModifiedValue\nc:\n    x: ModifiedValue\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   "{}",

pkg/yqlib/operator_booleans_test.go

@@ -102,7 +102,7 @@ var booleanOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `[{pet: cat}]`,
-		expression: `any_c(.name == "harry") as $c`,
+		expression: `any_c(.name == "harry") as $c | .`,
 		expected: []string{
 			"D0, P[], (doc)::[{pet: cat}]\n",
 		},
@@ -110,9 +110,17 @@ var booleanOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `[{pet: cat}]`,
-		expression: `all_c(.name == "harry") as $c`,
+		expression: `any_c(.name == "harry") as $c | $c`,
 		expected: []string{
-			"D0, P[], (doc)::[{pet: cat}]\n",
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{pet: cat}]`,
+		expression: `all_c(.name == "harry") as $c | $c`,
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
 		},
 	},
 	{
@@ -185,7 +193,7 @@ var booleanOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `{}`,
-		expression: `(.a.b or .c) as $x`,
+		expression: `(.a.b or .c) as $x | .`,
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},
@@ -193,7 +201,7 @@ var booleanOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `{}`,
-		expression: `(.a.b and .c) as $x`,
+		expression: `(.a.b and .c) as $x | .`,
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},

pkg/yqlib/operator_encoder_decoder.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"bytes"
 	"container/list"
+	"errors"
 	"regexp"
 	"strings"
 
@@ -39,6 +40,9 @@ func encodeToString(candidate *CandidateNode, prefs encoderPreferences) (string,
 	log.Debug("printing with indent: %v", prefs.indent)
 
 	encoder := configureEncoder(prefs.format, prefs.indent)
+	if encoder == nil {
+		return "", errors.New("no support for output format")
+	}
 
 	printer := NewPrinter(encoder, NewSinglePrinterWriter(bufio.NewWriter(&output)))
 	err := printer.PrintResults(candidate.AsList())
@@ -98,13 +102,11 @@ type decoderPreferences struct {
 	format InputFormat
 }
 
-/* takes a string and decodes it back into an object */
-func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-
-	preferences := expressionNode.Operation.Preferences.(decoderPreferences)
-
+func createDecoder(format InputFormat) Decoder {
 	var decoder Decoder
-	switch preferences.format {
+	switch format {
+	case JsonInputFormat:
+		decoder = NewJSONDecoder()
 	case YamlInputFormat:
 		decoder = NewYamlDecoder(ConfiguredYamlPreferences)
 	case XMLInputFormat:
@@ -120,6 +122,18 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 	case UriInputFormat:
 		decoder = NewUriDecoder()
 	}
+	return decoder
+}
+
+/* takes a string and decodes it back into an object */
+func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	preferences := expressionNode.Operation.Preferences.(decoderPreferences)
+
+	decoder := createDecoder(preferences.format)
+	if decoder == nil {
+		return Context{}, errors.New("no support for input format")
+	}
 
 	var results = list.New()
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {

pkg/yqlib/operator_encoder_decoder_test.go

@@ -8,15 +8,17 @@ var prefix = "D0, P[], (doc)::a:\n    cool:\n        bob: dylan\n"
 
 var encoderDecoderOperatorScenarios = []expressionScenario{
 	{
-		description: "Encode value as json string",
-		document:    `{a: {cool: "thing"}}`,
-		expression:  `.b = (.a | to_json)`,
+		requiresFormat: "json",
+		description:    "Encode value as json string",
+		document:       `{a: {cool: "thing"}}`,
+		expression:     `.b = (.a | to_json)`,
 		expected: []string{
 			`D0, P[], (doc)::{a: {cool: "thing"}, b: "{\n  \"cool\": \"thing\"\n}\n"}
 `,
 		},
 	},
 	{
+		requiresFormat: "json",
 		description:    "Encode value as json string, on one line",
 		subdescription: "Pass in a 0 indent to print json on a single line.",
 		document:       `{a: {cool: "thing"}}`,
@@ -27,6 +29,7 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
+		requiresFormat: "json",
 		description:    "Encode value as json string, on one line shorthand",
 		subdescription: "Pass in a 0 indent to print json on a single line.",
 		document:       `{a: {cool: "thing"}}`,
@@ -37,6 +40,7 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
+		requiresFormat: "json",
 		description:    "Decode a json encoded string",
 		subdescription: "Keep in mind JSON is a subset of YAML. If you want idiomatic yaml, pipe through the style operator to clear out the JSON styling.",
 		document:       `a: '{"cool":"thing"}'`,
@@ -193,33 +197,37 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description: "Encode value as xml string",
-		document:    `{a: {cool: {foo: "bar", +@id: hi}}}`,
-		expression:  `.a | to_xml`,
+		requiresFormat: "xml",
+		description:    "Encode value as xml string",
+		document:       `{a: {cool: {foo: "bar", +@id: hi}}}`,
+		expression:     `.a | to_xml`,
 		expected: []string{
 			"D0, P[a], (!!str)::<cool id=\"hi\">\n  <foo>bar</foo>\n</cool>\n\n",
 		},
 	},
 	{
-		description: "Encode value as xml string on a single line",
-		document:    `{a: {cool: {foo: "bar", +@id: hi}}}`,
-		expression:  `.a | @xml`,
+		requiresFormat: "xml",
+		description:    "Encode value as xml string on a single line",
+		document:       `{a: {cool: {foo: "bar", +@id: hi}}}`,
+		expression:     `.a | @xml`,
 		expected: []string{
 			"D0, P[a], (!!str)::<cool id=\"hi\"><foo>bar</foo></cool>\n\n",
 		},
 	},
 	{
-		description: "Encode value as xml string with custom indentation",
-		document:    `{a: {cool: {foo: "bar", +@id: hi}}}`,
-		expression:  `{"cat": .a | to_xml(1)}`,
+		requiresFormat: "xml",
+		description:    "Encode value as xml string with custom indentation",
+		document:       `{a: {cool: {foo: "bar", +@id: hi}}}`,
+		expression:     `{"cat": .a | to_xml(1)}`,
 		expected: []string{
 			"D0, P[], (!!map)::cat: |\n    <cool id=\"hi\">\n     <foo>bar</foo>\n    </cool>\n",
 		},
 	},
 	{
-		description: "Decode a xml encoded string",
-		document:    `a: "<foo>bar</foo>"`,
-		expression:  `.b = (.a | from_xml)`,
+		requiresFormat: "xml",
+		description:    "Decode a xml encoded string",
+		document:       `a: "<foo>bar</foo>"`,
+		expression:     `.b = (.a | from_xml)`,
 		expected: []string{
 			"D0, P[], (doc)::a: \"<foo>bar</foo>\"\nb:\n    foo: bar\n",
 		},
@@ -303,9 +311,10 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description: "empty xml decode",
-		skipDoc:     true,
-		expression:  `"" | @xmld`,
+		requiresFormat: "xml",
+		description:    "empty xml decode",
+		skipDoc:        true,
+		expression:     `"" | @xmld`,
 		expected: []string{
 			"D0, P[], (!!null)::\n",
 		},

pkg/yqlib/operator_equals_test.go

@@ -47,7 +47,7 @@ var equalsOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   "{}",
-		expression: "(.a == .b) as $x",
+		expression: "(.a == .b) as $x | .",
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},
@@ -63,7 +63,7 @@ var equalsOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   "{}",
-		expression: "(.a != .b) as $x",
+		expression: "(.a != .b) as $x | .",
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},

pkg/yqlib/operator_has_test.go

@@ -16,7 +16,7 @@ var hasOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `a: hello`,
-		expression: `has(.b) as $c`,
+		expression: `has(.b) as $c | .`,
 		expected: []string{
 			"D0, P[], (doc)::a: hello\n",
 		},

pkg/yqlib/operator_load.go

@@ -34,6 +34,9 @@ func loadString(filename string) (*CandidateNode, error) {
 }
 
 func loadYaml(filename string, decoder Decoder) (*CandidateNode, error) {
+	if decoder == nil {
+		return nil, fmt.Errorf("could not load %s", filename)
+	}
 
 	file, err := os.Open(filename) // #nosec
 	if err != nil {

pkg/yqlib/operator_load_test.go

@@ -74,9 +74,10 @@ var loadScenarios = []expressionScenario{
 		},
 	},
 	{
-		description: "Load from XML",
-		document:    "cool: things",
-		expression:  `.more_stuff = load_xml("../../examples/small.xml")`,
+		requiresFormat: "xml",
+		description:    "Load from XML",
+		document:       "cool: things",
+		expression:     `.more_stuff = load_xml("../../examples/small.xml")`,
 		expected: []string{
 			"D0, P[], (doc)::cool: things\nmore_stuff:\n    this: is some xml\n",
 		},

pkg/yqlib/operator_pipe.go

@@ -2,9 +2,9 @@ package yqlib
 
 func pipeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 
-	//lhs may update the variable context, we should pass that into the RHS
-	// BUT we still return the original context back (see jq)
-	// https://stedolan.github.io/jq/manual/#Variable/SymbolicBindingOperator:...as$identifier|...
+	if expressionNode.LHS.Operation.OperationType == assignVariableOpType {
+		return variableLoop(d, context, expressionNode)
+	}
 
 	lhs, err := d.GetMatchingNodes(context, expressionNode.LHS)
 	if err != nil {

pkg/yqlib/operator_subtract_test.go

@@ -8,7 +8,7 @@ var subtractOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `{}`,
-		expression: "(.a - .b) as $x",
+		expression: "(.a - .b) as $x | .",
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},

pkg/yqlib/operator_traverse_path_test.go

@@ -151,7 +151,7 @@ var traversePathOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `c: dog`,
-		expression: `.[.a.b] as $x`,
+		expression: `.[.a.b] as $x | .`,
 		expected: []string{
 			"D0, P[], (doc)::c: dog\n",
 		},

pkg/yqlib/operator_union_test.go

@@ -8,7 +8,7 @@ var unionOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   "{}",
-		expression: `(.a, .b.c) as $x`,
+		expression: `(.a, .b.c) as $x | .`,
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},

pkg/yqlib/operator_variables.go

@@ -19,24 +19,81 @@ type assignVarPreferences struct {
 	IsReference bool
 }
 
-func assignVariableOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	lhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.LHS)
+func useWithPipe(d *dataTreeNavigator, context Context, originalExp *ExpressionNode) (Context, error) {
+	return Context{}, fmt.Errorf("must use variable with a pipe, e.g. `exp as $x | ...`")
+}
+
+// variables are like loops in jq
+// https://stedolan.github.io/jq/manual/#Variable
+func variableLoop(d *dataTreeNavigator, context Context, originalExp *ExpressionNode) (Context, error) {
+	log.Debug("variable loop!")
+	results := list.New()
+	var evaluateAllTogether = true
+	for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() {
+		evaluateAllTogether = evaluateAllTogether && matchEl.Value.(*CandidateNode).EvaluateTogether
+		if !evaluateAllTogether {
+			break
+		}
+	}
+	if evaluateAllTogether {
+		return variableLoopSingleChild(d, context, originalExp)
+	}
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		result, err := variableLoopSingleChild(d, context.SingleChildContext(el.Value.(*CandidateNode)), originalExp)
+		if err != nil {
+			return Context{}, err
+		}
+		results.PushBackList(result.MatchingNodes)
+	}
+	return context.ChildContext(results), nil
+}
+
+func variableLoopSingleChild(d *dataTreeNavigator, context Context, originalExp *ExpressionNode) (Context, error) {
+
+	variableExp := originalExp.LHS
+	lhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), variableExp.LHS)
 	if err != nil {
 		return Context{}, err
 	}
-	if expressionNode.RHS.Operation.OperationType.Type != "GET_VARIABLE" {
+	if variableExp.RHS.Operation.OperationType.Type != "GET_VARIABLE" {
 		return Context{}, fmt.Errorf("RHS of 'as' operator must be a variable name e.g. $foo")
 	}
-	variableName := expressionNode.RHS.Operation.StringValue
+	variableName := variableExp.RHS.Operation.StringValue
 
-	prefs := expressionNode.Operation.Preferences.(assignVarPreferences)
+	prefs := variableExp.Operation.Preferences.(assignVarPreferences)
 
-	var variableValue *list.List
-	if prefs.IsReference {
-		variableValue = lhs.MatchingNodes
-	} else {
-		variableValue = lhs.DeepClone().MatchingNodes
+	results := list.New()
+
+	// now we loop over lhs, set variable to each result and calculate originalExp.Rhs
+	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
+		log.Debug("PROCESSING VARIABLE: ", NodeToString(el.Value.(*CandidateNode)))
+		var variableValue = list.New()
+		if prefs.IsReference {
+			variableValue.PushBack(el.Value)
+		} else {
+			candidateCopy, err := el.Value.(*CandidateNode).Copy()
+			if err != nil {
+				return Context{}, err
+			}
+			variableValue.PushBack(candidateCopy)
+		}
+		newContext := context.ChildContext(context.MatchingNodes)
+		newContext.SetVariable(variableName, variableValue)
+
+		rhs, err := d.GetMatchingNodes(newContext, originalExp.RHS)
+		log.Debug("PROCESSING VARIABLE DONE, got back: ", rhs.MatchingNodes.Len())
+		if err != nil {
+			return Context{}, err
+		}
+		results.PushBackList(rhs.MatchingNodes)
 	}
-	context.SetVariable(variableName, variableValue)
-	return context, nil
+
+	// if there is no LHS - then I guess we just calculate originalExp.Rhs
+	if lhs.MatchingNodes.Len() == 0 {
+		return d.GetMatchingNodes(context, originalExp.RHS)
+	}
+
+	return context.ChildContext(results), nil
+
 }

pkg/yqlib/operator_variables_test.go

@@ -8,7 +8,7 @@ var variableOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `{}`,
-		expression: `.a.b as $foo`,
+		expression: `.a.b as $foo | .`,
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},
@@ -16,7 +16,7 @@ var variableOperatorScenarios = []expressionScenario{
 	{
 		document:      "a: [cat]",
 		skipDoc:       true,
-		expression:    "(.[] | {.name: .}) as $item",
+		expression:    "(.[] | {.name: .}) as $item | .",
 		expectedError: `cannot index array with 'name' (strconv.ParseInt: parsing "name": invalid syntax)`,
 	},
 	{
@@ -36,6 +36,22 @@ var variableOperatorScenarios = []expressionScenario{
 			"D0, P[1], (!!str)::dog\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `[1, 2]`,
+		expression: `.[] | . as $f | select($f == 2)`,
+		expected: []string{
+			"D0, P[1], (!!int)::2\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[1, 2]`,
+		expression: `[.[] | . as $f | $f + 1]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- 2\n- 3\n",
+		},
+	},
 	{
 		description:    "Using variables as a lookup",
 		subdescription: "Example taken from [jq](https://stedolan.github.io/jq/manual/#Variable/SymbolicBindingOperator:...as$identifier|...)",

pkg/yqlib/operators_test.go

@@ -28,6 +28,7 @@ type expressionScenario struct {
 	skipDoc               bool
 	expectedError         string
 	dontFormatInputForDoc bool // dont format input doc for documentation generation
+	requiresFormat        string
 }
 
 func TestMain(m *testing.M) {
@@ -103,6 +104,23 @@ func testScenario(t *testing.T, s *expressionScenario) {
 		return
 	}
 
+	if s.requiresFormat != "" {
+		format := s.requiresFormat
+		inputFormat, err := InputFormatFromString(format)
+		if err != nil {
+			t.Error(err)
+		}
+		if decoder := createDecoder(inputFormat); decoder == nil {
+			t.Skipf("no support for %s input format", format)
+		}
+		outputFormat, err := OutputFormatFromString(format)
+		if err != nil {
+			t.Error(err)
+		}
+		if encoder := configureEncoder(outputFormat, 4); encoder == nil {
+			t.Skipf("no support for %s output format", format)
+		}
+	}
 	if err != nil {
 		t.Error(fmt.Errorf("%w: %v: %v", err, s.description, s.expression))
 		return

pkg/yqlib/ordered_map.go

@@ -0,0 +1,14 @@
+package yqlib
+
+// orderedMap allows to marshal and unmarshal JSON and YAML values keeping the
+// order of keys and values in a map or an object.
+type orderedMap struct {
+	// if this is an object, kv != nil. If this is not an object, kv == nil.
+	kv     []orderedMapKV
+	altVal interface{}
+}
+
+type orderedMapKV struct {
+	K string
+	V orderedMap
+}

pkg/yqlib/ordered_map_json.go

@@ -0,0 +1,83 @@
+package yqlib
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"io"
+)
+
+func (o *orderedMap) UnmarshalJSON(data []byte) error {
+	switch data[0] {
+	case '{':
+		// initialise so that even if the object is empty it is not nil
+		o.kv = []orderedMapKV{}
+
+		// create decoder
+		dec := json.NewDecoder(bytes.NewReader(data))
+		_, err := dec.Token() // open object
+		if err != nil {
+			return err
+		}
+
+		// cycle through k/v
+		var tok json.Token
+		for tok, err = dec.Token(); err == nil; tok, err = dec.Token() {
+			// we can expect two types: string or Delim. Delim automatically means
+			// that it is the closing bracket of the object, whereas string means
+			// that there is another key.
+			if _, ok := tok.(json.Delim); ok {
+				break
+			}
+			kv := orderedMapKV{
+				K: tok.(string),
+			}
+			if err := dec.Decode(&kv.V); err != nil {
+				return err
+			}
+			o.kv = append(o.kv, kv)
+		}
+		// unexpected error
+		if err != nil && !errors.Is(err, io.EOF) {
+			return err
+		}
+		return nil
+	case '[':
+		var res []*orderedMap
+		if err := json.Unmarshal(data, &res); err != nil {
+			return err
+		}
+		o.altVal = res
+		o.kv = nil
+		return nil
+	}
+
+	return json.Unmarshal(data, &o.altVal)
+}
+
+func (o orderedMap) MarshalJSON() ([]byte, error) {
+	buf := new(bytes.Buffer)
+	enc := json.NewEncoder(buf)
+	enc.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
+	if o.kv == nil {
+		if err := enc.Encode(o.altVal); err != nil {
+			return nil, err
+		}
+		return buf.Bytes(), nil
+	}
+	buf.WriteByte('{')
+	for idx, el := range o.kv {
+		if err := enc.Encode(el.K); err != nil {
+			return nil, err
+		}
+		buf.WriteByte(':')
+		if err := enc.Encode(el.V); err != nil {
+			return nil, err
+		}
+		if idx != len(o.kv)-1 {
+			buf.WriteByte(',')
+		}
+	}
+	buf.WriteByte('}')
+	return buf.Bytes(), nil
+}

pkg/yqlib/ordered_map_yaml.go

@@ -0,0 +1,79 @@
+package yqlib
+
+import (
+	"fmt"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+func (o *orderedMap) UnmarshalYAML(node *yaml.Node) error {
+	switch node.Kind {
+	case yaml.DocumentNode:
+		if len(node.Content) == 0 {
+			return nil
+		}
+		return o.UnmarshalYAML(node.Content[0])
+	case yaml.AliasNode:
+		return o.UnmarshalYAML(node.Alias)
+	case yaml.ScalarNode:
+		return node.Decode(&o.altVal)
+	case yaml.MappingNode:
+		// set kv to non-nil
+		o.kv = []orderedMapKV{}
+		for i := 0; i < len(node.Content); i += 2 {
+			var key string
+			var val orderedMap
+			if err := node.Content[i].Decode(&key); err != nil {
+				return err
+			}
+			if err := node.Content[i+1].Decode(&val); err != nil {
+				return err
+			}
+			o.kv = append(o.kv, orderedMapKV{
+				K: key,
+				V: val,
+			})
+		}
+		return nil
+	case yaml.SequenceNode:
+		// note that this has to be a pointer, so that nulls can be represented.
+		var res []*orderedMap
+		if err := node.Decode(&res); err != nil {
+			return err
+		}
+		o.altVal = res
+		o.kv = nil
+		return nil
+	case 0:
+		// null
+		o.kv = nil
+		o.altVal = nil
+		return nil
+	default:
+		return fmt.Errorf("orderedMap: invalid yaml node")
+	}
+}
+
+func (o *orderedMap) MarshalYAML() (interface{}, error) {
+	// fast path: kv is nil, use altVal
+	if o.kv == nil {
+		return o.altVal, nil
+	}
+	content := make([]*yaml.Node, 0, len(o.kv)*2)
+	for _, val := range o.kv {
+		n := new(yaml.Node)
+		if err := n.Encode(val.V); err != nil {
+			return nil, err
+		}
+		content = append(content, &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "!!str",
+			Value: val.K,
+		}, n)
+	}
+	return &yaml.Node{
+		Kind:    yaml.MappingNode,
+		Tag:     "!!map",
+		Content: content,
+	}, nil
+}

pkg/yqlib/printer_test.go

@@ -314,7 +314,11 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	// note printDocSeparators is true, it should still not print document separators
 	// when outputing JSON.
-	printer := NewPrinter(NewJSONEncoder(0, false, false), NewSinglePrinterWriter(writer))
+	encoder := NewJSONEncoder(0, false, false)
+	if encoder == nil {
+		t.Skipf("no support for %s output format", "json")
+	}
+	printer := NewPrinter(encoder, NewSinglePrinterWriter(writer))
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {

pkg/yqlib/xml_test.go

@@ -1,3 +1,5 @@
+//go:build !yq_noxml
+
 package yqlib
 
 import (
@@ -8,6 +10,29 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
+const yamlInputWithProcInstAndHeadComment = `# cats
++p_xml: version="1.0"
+this: is some xml`
+
+const expectedXmlProcInstAndHeadComment = `<?xml version="1.0"?>
+<!-- cats -->
+<this>is some xml</this>
+`
+
+const xmlProcInstAndHeadCommentBlock = `<?xml version="1.0"?>
+<!--
+cats
+-->
+<this>is some xml</this>
+`
+
+const expectedYamlProcInstAndHeadCommentBlock = `#
+# cats
+#
++p_xml: version="1.0"
+this: is some xml
+`
+
 const inputXMLWithComments = `
 <!-- before cat -->
 <cat>
@@ -126,7 +151,8 @@ cat:
 # after cat
 `
 
-const expectedRoundtripXMLWithComments = `<!-- before cat --><cat><!-- in cat before -->
+const expectedRoundtripXMLWithComments = `<!-- before cat -->
+<cat><!-- in cat before -->
   <x>3<!-- multi
 line comment 
 for x --></x><!-- before y -->
@@ -137,8 +163,10 @@ in d before -->
 </cat><!-- after cat -->
 `
 
-const yamlWithComments = `# header comment
+const yamlWithComments = `#
+# header comment
 # above_cat
+#
 cat: # inline_cat
   # above_array
   array: # inline_array
@@ -149,9 +177,11 @@ cat: # inline_cat
 `
 
 const expectedXMLWithComments = `<!--
- header comment
- above_cat
- --><!-- inline_cat --><cat><!-- above_array inline_array -->
+header comment
+above_cat
+-->
+<!-- inline_cat -->
+<cat><!-- above_array inline_array -->
   <array>val1<!-- inline_val1 --></array>
   <array><!-- above_val2 -->val2<!-- inline_val2 --></array>
 </cat><!-- below_cat -->
@@ -264,6 +294,41 @@ var xmlScenarios = []formatScenario{
 		input:    "<root><cats><cat>quick</cat><cat>soft</cat><!-- kitty_comment--><cat>squishy</cat></cats></root>",
 		expected: "root:\n    cats:\n        cat:\n            - quick\n            - soft\n            # kitty_comment\n\n            - squishy\n",
 	},
+	{
+		description:  "ProcInst with head comment",
+		skipDoc:      true,
+		input:        yamlInputWithProcInstAndHeadComment,
+		expected:     expectedXmlProcInstAndHeadComment,
+		scenarioType: "encode",
+	},
+	{
+		description:  "ProcInst with head comment round trip",
+		skipDoc:      true,
+		input:        expectedXmlProcInstAndHeadComment,
+		expected:     expectedXmlProcInstAndHeadComment,
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "ProcInst with block head comment to yaml",
+		skipDoc:      true,
+		input:        xmlProcInstAndHeadCommentBlock,
+		expected:     expectedYamlProcInstAndHeadCommentBlock,
+		scenarioType: "decode",
+	},
+	{
+		description:  "ProcInst with block head comment from yaml",
+		skipDoc:      true,
+		input:        expectedYamlProcInstAndHeadCommentBlock,
+		expected:     xmlProcInstAndHeadCommentBlock,
+		scenarioType: "encode",
+	},
+	{
+		description:  "ProcInst with head comment round trip block",
+		skipDoc:      true,
+		input:        xmlProcInstAndHeadCommentBlock,
+		expected:     xmlProcInstAndHeadCommentBlock,
+		scenarioType: "roundtrip",
+	},
 	{
 		description:    "Parse xml: simple",
 		subdescription: "Notice how all the values are strings, see the next example on how you can fix that.",
@@ -464,6 +529,41 @@ var xmlScenarios = []formatScenario{
 		expected:       "<cat name=\"tiger\">cool</cat>\n",
 		scenarioType:   "encode",
 	},
+	{
+		description:  "round trip multiline 1",
+		skipDoc:      true,
+		input:        "<x><!-- cats --></x>\n",
+		expected:     "<x><!-- cats --></x>\n",
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "round trip multiline 2",
+		skipDoc:      true,
+		input:        "<x><!--\n cats\n --></x>\n",
+		expected:     "<x><!--\ncats\n--></x>\n",
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "round trip multiline 3",
+		skipDoc:      true,
+		input:        "<x><!--\n\tcats\n --></x>\n",
+		expected:     "<x><!--\n\tcats\n--></x>\n",
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "round trip multiline 4",
+		skipDoc:      true,
+		input:        "<x><!--\n\tcats\n\tdogs\n--></x>\n",
+		expected:     "<x><!--\n\tcats\n\tdogs\n--></x>\n",
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "round trip multiline 5",
+		skipDoc:      true, // pity spaces aren't kept atm.
+		input:        "<x><!--\ncats\ndogs\n--></x>\n",
+		expected:     "<x><!--\ncats\ndogs\n--></x>\n",
+		scenarioType: "roundtrip",
+	},
 	{
 		description:    "Encode xml: comments",
 		subdescription: "A best attempt is made to copy comments to xml.",

release_notes.txt

@@ -1,3 +1,9 @@
+4.31.2 (draft):
+  - Fixed variable handling #1458, #1566
+  - Fixed merged anchor reference problem #1482
+  - Allow build without json and xml support (#1556) Thanks @afbjorklund
+  - Bumped dependencies
+
 4.31.1:
   - Added shuffle command #1503
   - Added ability to sort by multiple fields #1541

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: 'v4.31.1'
+version: 'v4.31.2'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.