Fixed variable operator to work like jq

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.20.2 as builder
+FROM golang:1.20.1 as builder
 
 WORKDIR /go/src/mikefarah/yq
 

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM golang:1.20.2
+FROM golang:1.20.1
 
 COPY scripts/devtools.sh /opt/devtools.sh
 

acceptance_tests/inputs-format-auto.sh

@@ -1,211 +0,0 @@
-#!/bin/bash
-
-setUp() {
-  rm test*.yml 2>/dev/null || true
-  rm test*.json 2>/dev/null || true
-  rm test*.properties 2>/dev/null || true
-  rm test*.csv 2>/dev/null || true
-  rm test*.tsv 2>/dev/null || true
-  rm test*.xml 2>/dev/null || true
-}
-
-testInputJson() {
-  cat >test.json <<EOL
-{ "mike" : { "things": "cool" } }
-EOL
-
-  read -r -d '' expected << EOM
-{
-  "mike": {
-    "things": "cool"
-  }
-}
-EOM
-
-  X=$(./yq test.json)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea test.json)
-  assertEquals "$expected" "$X"
-}
-
-testInputJsonOutputYaml() {
-  cat >test.json <<EOL
-{ "mike" : { "things": "cool" } }
-EOL
-
-  read -r -d '' expected << EOM
-mike:
-  things: cool
-EOM
-
-  X=$(./yq test.json -oy)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea test.json -oy)
-  assertEquals "$expected" "$X"
-}
-
-testInputProperties() {
-  cat >test.properties <<EOL
-mike.things = hello
-EOL
-
-  read -r -d '' expected << EOM
-mike.things = hello
-EOM
-
-  X=$(./yq e test.properties)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq test.properties)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea test.properties)
-  assertEquals "$expected" "$X"
-}
-
-testInputPropertiesGitHubAction() {
-  cat >test.properties <<EOL
-mike.things = hello
-EOL
-
-  read -r -d '' expected << EOM
-mike.things = hello
-EOM
-
-  X=$(cat /dev/null | ./yq e test.properties)
-  assertEquals "$expected" "$X"
-
-  X=$(cat /dev/null | ./yq ea test.properties)
-  assertEquals "$expected" "$X"
-}
-
-testInputCSV() {
-  cat >test.csv <<EOL
-fruit,yumLevel
-apple,5
-banana,4
-EOL
-
-  read -r -d '' expected << EOM
-fruit,yumLevel
-apple,5
-banana,4
-EOM
-
-  X=$(./yq e test.csv)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea test.csv)
-  assertEquals "$expected" "$X"
-}
-
-testInputCSVUTF8() {
-  read -r -d '' expected << EOM
-id,first,last
-1,john,smith
-1,jane,smith
-EOM
-
-  X=$(./yq utf8.csv)
-  assertEquals "$expected" "$X"
-}
-
-testInputTSV() {
-  cat >test.tsv <<EOL
-fruit	yumLevel
-apple	5
-banana	4
-EOL
-
-  read -r -d '' expected << EOM
-fruit	yumLevel
-apple	5
-banana	4
-EOM
-
-  X=$(./yq e test.tsv)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea test.tsv)
-  assertEquals "$expected" "$X"
-}
-
-
-
-
-testInputXml() {
-  cat >test.xml <<EOL
-<cat legs="4">BiBi</cat>
-EOL
-
-  read -r -d '' expected << EOM
-<cat legs="4">BiBi</cat>
-EOM
-
-  X=$(./yq e test.xml)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea test.xml)
-  assertEquals "$expected" "$X"
-}
-
-testInputXmlNamespaces() {
-  cat >test.xml <<EOL
-<?xml version="1.0"?>
-<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
-</map>
-EOL
-
-  read -r -d '' expected << EOM
-<?xml version="1.0"?>
-<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url"></map>
-EOM
-
-  X=$(./yq e test.xml)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea test.xml)
-  assertEquals "$expected" "$X"
-}
-
-
-
-testInputXmlStrict() {
-  cat >test.xml <<EOL
-<?xml version="1.0"?>
-<!DOCTYPE root [
-<!ENTITY writer "Catherine.">
-<!ENTITY copyright "(r) Great">
-]>
-<root>
-    <item>&writer;&copyright;</item>
-</root>
-EOL
-
-  X=$(./yq --xml-strict-mode test.xml  2>&1)
-  assertEquals 1 $?
-  assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X"
-
-  X=$(./yq ea --xml-strict-mode test.xml  2>&1)
-  assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X"
-}
-
-testInputXmlGithubAction() {
-  cat >test.xml <<EOL
-<cat legs="4">BiBi</cat>
-EOL
-
-  read -r -d '' expected << EOM
-<cat legs="4">BiBi</cat>
-EOM
-
-  X=$(cat /dev/null | ./yq e test.xml)
-  assertEquals "$expected" "$X"
-
-  X=$(cat /dev/null | ./yq ea test.xml)
-  assertEquals "$expected" "$X"
-}
-
-source ./scripts/shunit2

acceptance_tests/inputs-format.sh

@@ -120,7 +120,7 @@ EOM
 }
 
 testInputXmlNamespaces() {
-  cat >test.xml <<EOL
+  cat >test.yml <<EOL
 <?xml version="1.0"?>
 <map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
 </map>
@@ -134,10 +134,10 @@ map:
   +@xsi:schemaLocation: some-url
 EOM
 
-  X=$(./yq e -p=xml test.xml)
+  X=$(./yq e -p=xml test.yml)
   assertEquals "$expected" "$X"
 
-  X=$(./yq ea -p=xml test.xml)
+  X=$(./yq ea -p=xml test.yml)
   assertEquals "$expected" "$X"
 }
 

cmd/constant.go

@@ -6,10 +6,8 @@ var unwrapScalar = false
 
 var writeInplace = false
 var outputToJSON = false
-
-var outputFormat = ""
-
-var inputFormat = ""
+var outputFormat = "yaml"
+var inputFormat = "yaml"
 
 var exitStatus = false
 var forceColor = false

cmd/evaluate_all_command.go

@@ -84,10 +84,7 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 	if err != nil {
 		return err
 	}
-	encoder, err := configureEncoder()
-	if err != nil {
-		return err
-	}
+	encoder := configureEncoder(format)
 
 	printer := yqlib.NewPrinter(encoder, printerWriter)
 

cmd/evalute_sequence_command.go

@@ -93,10 +93,7 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 	if err != nil {
 		return err
 	}
-	encoder, err := configureEncoder()
-	if err != nil {
-		return err
-	}
+	encoder := configureEncoder(format)
 
 	printer := yqlib.NewPrinter(encoder, printerWriter)
 

cmd/root.go

@@ -53,6 +53,25 @@ yq -P sample.json
 			logging.SetBackend(backend)
 			yqlib.InitExpressionParser()
 
+			outputFormatType, err := yqlib.OutputFormatFromString(outputFormat)
+
+			if err != nil {
+				return err
+			}
+
+			if outputFormatType == yqlib.YamlOutputFormat ||
+				outputFormatType == yqlib.PropsOutputFormat {
+				unwrapScalar = true
+			}
+			if unwrapScalarFlag.IsExplicitySet() {
+				unwrapScalar = unwrapScalarFlag.IsSet()
+			}
+
+			//copy preference form global setting
+			yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar
+
+			yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators
+
 			return nil
 		},
 	}
@@ -65,8 +84,8 @@ yq -P sample.json
 		panic(err)
 	}
 
-	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "auto", "[auto|a|yaml|y|json|j|props|p|xml|x] output format type.")
-	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "auto", "[auto|a|yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.")
+	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "yaml", "[yaml|y|json|j|props|p|xml|x] output format type.")
+	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "yaml", "[yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.")
 
 	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", yqlib.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes")
 	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", yqlib.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).")

cmd/utils.go

@@ -53,48 +53,6 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 		return "", nil, fmt.Errorf("cannot pass files in when using null-input flag")
 	}
 
-	inputFilename := ""
-	if len(args) > 0 {
-		inputFilename = args[0]
-	}
-	if inputFormat == "" || inputFormat == "auto" || inputFormat == "a" {
-
-		inputFormat = yqlib.FormatFromFilename(inputFilename)
-		if outputFormat == "" || outputFormat == "auto" || outputFormat == "a" {
-			outputFormat = yqlib.FormatFromFilename(inputFilename)
-		}
-	} else if outputFormat == "" || outputFormat == "auto" || outputFormat == "a" {
-		// backwards compatibility -
-		// before this was introduced, `yq -pcsv things.csv`
-		// would produce *yaml* output.
-		//
-		outputFormat = yqlib.FormatFromFilename(inputFilename)
-		if inputFilename != "-" {
-			yqlib.GetLogger().Warning("yq default output is now 'auto' (based on the filename extension). Normally yq would output '%v', but for backwards compatibility 'yaml' has been set. Please use -oy to specify yaml, or drop the -p flag.", outputFormat)
-		}
-		outputFormat = "yaml"
-	}
-
-	outputFormatType, err := yqlib.OutputFormatFromString(outputFormat)
-
-	if err != nil {
-		return "", nil, err
-	}
-	yqlib.GetLogger().Debug("Using outputformat %v", outputFormat)
-
-	if outputFormatType == yqlib.YamlOutputFormat ||
-		outputFormatType == yqlib.PropsOutputFormat {
-		unwrapScalar = true
-	}
-	if unwrapScalarFlag.IsExplicitySet() {
-		unwrapScalar = unwrapScalarFlag.IsSet()
-	}
-
-	//copy preference form global setting
-	yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar
-
-	yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators
-
 	return expression, args, nil
 }
 
@@ -103,15 +61,7 @@ func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
 	if err != nil {
 		return nil, err
 	}
-	yqlibDecoder, err := createDecoder(yqlibInputFormat, evaluateTogether)
-	if yqlibDecoder == nil {
-		return nil, fmt.Errorf("no support for %s input format", inputFormat)
-	}
-	return yqlibDecoder, err
-}
-
-func createDecoder(format yqlib.InputFormat, evaluateTogether bool) (yqlib.Decoder, error) {
-	switch format {
+	switch yqlibInputFormat {
 	case yqlib.XMLInputFormat:
 		return yqlib.NewXMLDecoder(yqlib.ConfiguredXMLPreferences), nil
 	case yqlib.PropertiesInputFormat:
@@ -122,12 +72,10 @@ func createDecoder(format yqlib.InputFormat, evaluateTogether bool) (yqlib.Decod
 		return yqlib.NewCSVObjectDecoder(','), nil
 	case yqlib.TSVObjectInputFormat:
 		return yqlib.NewCSVObjectDecoder('\t'), nil
-	case yqlib.YamlInputFormat:
-		prefs := yqlib.ConfiguredYamlPreferences
-		prefs.EvaluateTogether = evaluateTogether
-		return yqlib.NewYamlDecoder(prefs), nil
 	}
-	return nil, fmt.Errorf("invalid decoder: %v", format)
+	prefs := yqlib.ConfiguredYamlPreferences
+	prefs.EvaluateTogether = evaluateTogether
+	return yqlib.NewYamlDecoder(prefs), nil
 }
 
 func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yqlib.PrinterWriter, error) {
@@ -147,34 +95,22 @@ func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yq
 	return printerWriter, nil
 }
 
-func configureEncoder() (yqlib.Encoder, error) {
-	yqlibOutputFormat, err := yqlib.OutputFormatFromString(outputFormat)
-	if err != nil {
-		return nil, err
-	}
-	yqlibEncoder, err := createEncoder(yqlibOutputFormat)
-	if yqlibEncoder == nil {
-		return nil, fmt.Errorf("no support for %s output format", outputFormat)
-	}
-	return yqlibEncoder, err
-}
-
-func createEncoder(format yqlib.PrinterOutputFormat) (yqlib.Encoder, error) {
+func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder {
 	switch format {
 	case yqlib.JSONOutputFormat:
-		return yqlib.NewJSONEncoder(indent, colorsEnabled, unwrapScalar), nil
+		return yqlib.NewJSONEncoder(indent, colorsEnabled, unwrapScalar)
 	case yqlib.PropsOutputFormat:
-		return yqlib.NewPropertiesEncoder(unwrapScalar), nil
+		return yqlib.NewPropertiesEncoder(unwrapScalar)
 	case yqlib.CSVOutputFormat:
-		return yqlib.NewCsvEncoder(','), nil
+		return yqlib.NewCsvEncoder(',')
 	case yqlib.TSVOutputFormat:
-		return yqlib.NewCsvEncoder('\t'), nil
+		return yqlib.NewCsvEncoder('\t')
 	case yqlib.YamlOutputFormat:
-		return yqlib.NewYamlEncoder(indent, colorsEnabled, yqlib.ConfiguredYamlPreferences), nil
+		return yqlib.NewYamlEncoder(indent, colorsEnabled, yqlib.ConfiguredYamlPreferences)
 	case yqlib.XMLOutputFormat:
-		return yqlib.NewXMLEncoder(indent, yqlib.ConfiguredXMLPreferences), nil
+		return yqlib.NewXMLEncoder(indent, yqlib.ConfiguredXMLPreferences)
 	}
-	return nil, fmt.Errorf("invalid encoder: %v", format)
+	panic("invalid encoder")
 }
 
 // this is a hack to enable backwards compatibility with githubactions (which pipe /dev/null into everything)

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "v4.31.2"
+	Version = "v4.31.1"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

examples/mike.xml

@@ -1 +1,5 @@
-<cat>3</cat>
+date: "2022-11-25"
+raw:
+  id: 1
+  XML: text_outside_1<Tag1 />text_outside_2<Tag2Kling Klong</Tag2>text_outside_3
+  time: 09:19:00

go.mod

@@ -1,20 +1,20 @@
 module github.com/mikefarah/yq/v4
 
 require (
-	github.com/a8m/envsubst v1.4.2
+	github.com/a8m/envsubst v1.4.1
 	github.com/alecthomas/participle/v2 v2.0.0-beta.5
 	github.com/alecthomas/repr v0.2.0
 	github.com/dimchansky/utfbom v1.1.1
 	github.com/elliotchance/orderedmap v1.5.0
 	github.com/fatih/color v1.14.1
 	github.com/goccy/go-json v0.10.0
-	github.com/goccy/go-yaml v1.10.0
+	github.com/goccy/go-yaml v1.9.8
 	github.com/jinzhu/copier v0.3.5
 	github.com/magiconair/properties v1.8.7
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/pflag v1.0.5
-	golang.org/x/net v0.8.0
+	golang.org/x/net v0.1.1-0.20221104162952-702349b0e862
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -23,8 +23,8 @@ require (
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
-	golang.org/x/sys v0.6.0 // indirect
-	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/sys v0.3.0 // indirect
+	golang.org/x/text v0.4.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 )
 

go.sum

@@ -1,5 +1,5 @@
-github.com/a8m/envsubst v1.4.2 h1:4yWIHXOLEJHQEFd4UjrWDrYeYlV7ncFWJOCBRLOZHQg=
-github.com/a8m/envsubst v1.4.2/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
+github.com/a8m/envsubst v1.4.1 h1:RShc2OKbQpIIp5qR2glVsBOJCImTVHFrDkJvmGh0Qi0=
+github.com/a8m/envsubst v1.4.1/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
 github.com/alecthomas/assert/v2 v2.0.3 h1:WKqJODfOiQG0nEJKFKzDIG3E29CN2/4zR9XGJzKIkbg=
 github.com/alecthomas/participle/v2 v2.0.0-beta.5 h1:y6dsSYVb1G5eK6mgmy+BgI3Mw35a3WghArZ/Hbebrjo=
 github.com/alecthomas/participle/v2 v2.0.0-beta.5/go.mod h1:RC764t6n4L8D8ITAJv0qdokritYSNR3wV5cVwmIEaMM=
@@ -21,9 +21,8 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA=
 github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
-github.com/goccy/go-yaml v1.10.0 h1:rBi+5HGuznOxx0JZ+60LDY85gc0dyIJCIMvsMJTKSKQ=
-github.com/goccy/go-yaml v1.10.0/go.mod h1:h/18Lr6oSQ3mvmqFoWmQ47KChOgpfHpTyIHl3yVmpiY=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/goccy/go-yaml v1.9.8 h1:5gMyLUeU1/6zl+WFfR1hN7D2kf+1/eRGa7DFtToiBvQ=
+github.com/goccy/go-yaml v1.9.8/go.mod h1:JubOolP3gh0HpiBc4BLRD4YmjEjHAmIIB2aaXKkTfoE=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
 github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
@@ -55,20 +54,20 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
-golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+golang.org/x/net v0.1.1-0.20221104162952-702349b0e862 h1:KrLJ+iz8J6j6VVr/OCfULAcK+xozUmWE43fKpMR4MlI=
+golang.org/x/net v0.1.1-0.20221104162952-702349b0e862/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
+golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
-golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0=

pkg/yqlib/decoder.go

@@ -3,7 +3,6 @@ package yqlib
 import (
 	"fmt"
 	"io"
-	"strings"
 )
 
 type InputFormat uint
@@ -26,11 +25,11 @@ type Decoder interface {
 
 func InputFormatFromString(format string) (InputFormat, error) {
 	switch format {
-	case "yaml", "yml", "y":
+	case "yaml", "y":
 		return YamlInputFormat, nil
 	case "xml", "x":
 		return XMLInputFormat, nil
-	case "properties", "props", "p":
+	case "props", "p":
 		return PropertiesInputFormat, nil
 	case "json", "ndjson", "j":
 		return JsonInputFormat, nil
@@ -39,22 +38,6 @@ func InputFormatFromString(format string) (InputFormat, error) {
 	case "tsv", "t":
 		return TSVObjectInputFormat, nil
 	default:
-		return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml]", format)
+		return 0, fmt.Errorf("unknown format '%v' please use [yaml|xml|props]", format)
 	}
 }
-
-func FormatFromFilename(filename string) string {
-
-	if filename != "" {
-		GetLogger().Debugf("checking file extension '%s' for auto format detection", filename)
-		nPos := strings.LastIndex(filename, ".")
-		if nPos > -1 {
-			format := filename[nPos+1:]
-			GetLogger().Debugf("detected format '%s'", format)
-			return format
-		}
-	}
-
-	GetLogger().Debugf("using default inputFormat 'yaml'")
-	return "yaml"
-}

pkg/yqlib/decoder_json.go

@@ -1,5 +1,3 @@
-//go:build !yq_nojson
-
 package yqlib
 
 import (

pkg/yqlib/decoder_xml.go

@@ -1,5 +1,3 @@
-//go:build !yq_noxml
-
 package yqlib
 
 import (
@@ -7,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"regexp"
 	"strings"
 	"unicode"
 
@@ -49,19 +46,11 @@ func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yaml.Node, error) {
 	return yamlNode, nil
 }
 
-var decoderCommentPrefix = regexp.MustCompile(`(^|\n)([[:alpha:]])`)
-
 func (dec *xmlDecoder) processComment(c string) string {
 	if c == "" {
 		return ""
 	}
-	//need to replace "cat " with "# cat"
-	// "\ncat\n" with "\n cat\n"
-	// ensure non-empty comments starting with newline have a space in front
-
-	replacement := decoderCommentPrefix.ReplaceAllString(c, "$1 $2")
-	replacement = "#" + strings.ReplaceAll(strings.TrimRight(replacement, " "), "\n", "\n#")
-	return replacement
+	return "#" + strings.TrimRight(c, " ")
 }
 
 func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
@@ -86,7 +75,6 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
 		var err error
 
 		if i == 0 {
-			log.Debugf("head comment here")
 			labelNode.HeadComment = dec.processComment(n.HeadComment)
 
 		}

pkg/yqlib/doc/operators/filter.md

@@ -1,18 +0,0 @@
-
-## Filter array
-Given a sample.yml file of:
-```yaml
-- 1
-- 2
-- 3
-```
-then
-```bash
-yq 'filter(. < 3)' sample.yml
-```
-will output
-```yaml
-- 1
-- 2
-```
-

pkg/yqlib/doc/usage/xml.md

@@ -407,10 +407,8 @@ A best attempt is made to copy comments to xml.
 
 Given a sample.yml file of:
 ```yaml
-#
 # header comment
 # above_cat
-#
 cat: # inline_cat
   # above_array
   array: # inline_array
@@ -427,11 +425,9 @@ yq -o=xml '.' sample.yml
 will output
 ```xml
 <!--
-header comment
-above_cat
--->
-<!-- inline_cat -->
-<cat><!-- above_array inline_array -->
+ header comment
+ above_cat
+ --><!-- inline_cat --><cat><!-- above_array inline_array -->
   <array>val1<!-- inline_val1 --></array>
   <array><!-- above_val2 -->val2<!-- inline_val2 --></array>
 </cat><!-- below_cat -->
@@ -493,8 +489,7 @@ yq -p=xml -o=xml '.' sample.xml
 ```
 will output
 ```xml
-<!-- before cat -->
-<cat><!-- in cat before -->
+<!-- before cat --><cat><!-- in cat before -->
   <x>3<!-- multi
 line comment 
 for x --></x><!-- before y -->

pkg/yqlib/encoder.go

@@ -1,6 +1,10 @@
 package yqlib
 
 import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
 	"io"
 
 	yaml "gopkg.in/yaml.v3"
@@ -13,17 +17,162 @@ type Encoder interface {
 	CanHandleAliases() bool
 }
 
-func mapKeysToStrings(node *yaml.Node) {
+// orderedMap allows to marshal and unmarshal JSON and YAML values keeping the
+// order of keys and values in a map or an object.
+type orderedMap struct {
+	// if this is an object, kv != nil. If this is not an object, kv == nil.
+	kv     []orderedMapKV
+	altVal interface{}
+}
 
-	if node.Kind == yaml.MappingNode {
-		for index, child := range node.Content {
-			if index%2 == 0 { // its a map key
-				child.Tag = "!!str"
+type orderedMapKV struct {
+	K string
+	V orderedMap
+}
+
+func (o *orderedMap) UnmarshalJSON(data []byte) error {
+	switch data[0] {
+	case '{':
+		// initialise so that even if the object is empty it is not nil
+		o.kv = []orderedMapKV{}
+
+		// create decoder
+		dec := json.NewDecoder(bytes.NewReader(data))
+		_, err := dec.Token() // open object
+		if err != nil {
+			return err
+		}
+
+		// cycle through k/v
+		var tok json.Token
+		for tok, err = dec.Token(); err == nil; tok, err = dec.Token() {
+			// we can expect two types: string or Delim. Delim automatically means
+			// that it is the closing bracket of the object, whereas string means
+			// that there is another key.
+			if _, ok := tok.(json.Delim); ok {
+				break
 			}
+			kv := orderedMapKV{
+				K: tok.(string),
+			}
+			if err := dec.Decode(&kv.V); err != nil {
+				return err
+			}
+			o.kv = append(o.kv, kv)
+		}
+		// unexpected error
+		if err != nil && !errors.Is(err, io.EOF) {
+			return err
+		}
+		return nil
+	case '[':
+		var res []*orderedMap
+		if err := json.Unmarshal(data, &res); err != nil {
+			return err
+		}
+		o.altVal = res
+		o.kv = nil
+		return nil
+	}
+
+	return json.Unmarshal(data, &o.altVal)
+}
+
+func (o orderedMap) MarshalJSON() ([]byte, error) {
+	buf := new(bytes.Buffer)
+	enc := json.NewEncoder(buf)
+	enc.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
+	if o.kv == nil {
+		if err := enc.Encode(o.altVal); err != nil {
+			return nil, err
+		}
+		return buf.Bytes(), nil
+	}
+	buf.WriteByte('{')
+	for idx, el := range o.kv {
+		if err := enc.Encode(el.K); err != nil {
+			return nil, err
+		}
+		buf.WriteByte(':')
+		if err := enc.Encode(el.V); err != nil {
+			return nil, err
+		}
+		if idx != len(o.kv)-1 {
+			buf.WriteByte(',')
 		}
 	}
+	buf.WriteByte('}')
+	return buf.Bytes(), nil
+}
 
-	for _, child := range node.Content {
-		mapKeysToStrings(child)
+func (o *orderedMap) UnmarshalYAML(node *yaml.Node) error {
+	switch node.Kind {
+	case yaml.DocumentNode:
+		if len(node.Content) == 0 {
+			return nil
+		}
+		return o.UnmarshalYAML(node.Content[0])
+	case yaml.AliasNode:
+		return o.UnmarshalYAML(node.Alias)
+	case yaml.ScalarNode:
+		return node.Decode(&o.altVal)
+	case yaml.MappingNode:
+		// set kv to non-nil
+		o.kv = []orderedMapKV{}
+		for i := 0; i < len(node.Content); i += 2 {
+			var key string
+			var val orderedMap
+			if err := node.Content[i].Decode(&key); err != nil {
+				return err
+			}
+			if err := node.Content[i+1].Decode(&val); err != nil {
+				return err
+			}
+			o.kv = append(o.kv, orderedMapKV{
+				K: key,
+				V: val,
+			})
+		}
+		return nil
+	case yaml.SequenceNode:
+		// note that this has to be a pointer, so that nulls can be represented.
+		var res []*orderedMap
+		if err := node.Decode(&res); err != nil {
+			return err
+		}
+		o.altVal = res
+		o.kv = nil
+		return nil
+	case 0:
+		// null
+		o.kv = nil
+		o.altVal = nil
+		return nil
+	default:
+		return fmt.Errorf("orderedMap: invalid yaml node")
 	}
 }
+
+func (o *orderedMap) MarshalYAML() (interface{}, error) {
+	// fast path: kv is nil, use altVal
+	if o.kv == nil {
+		return o.altVal, nil
+	}
+	content := make([]*yaml.Node, 0, len(o.kv)*2)
+	for _, val := range o.kv {
+		n := new(yaml.Node)
+		if err := n.Encode(val.V); err != nil {
+			return nil, err
+		}
+		content = append(content, &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "!!str",
+			Value: val.K,
+		}, n)
+	}
+	return &yaml.Node{
+		Kind:    yaml.MappingNode,
+		Tag:     "!!map",
+		Content: content,
+	}, nil
+}

pkg/yqlib/encoder_json.go

@@ -1,5 +1,3 @@
-//go:build !yq_nojson
-
 package yqlib
 
 import (
@@ -16,6 +14,21 @@ type jsonEncoder struct {
 	UnwrapScalar bool
 }
 
+func mapKeysToStrings(node *yaml.Node) {
+
+	if node.Kind == yaml.MappingNode {
+		for index, child := range node.Content {
+			if index%2 == 0 { // its a map key
+				child.Tag = "!!str"
+			}
+		}
+	}
+
+	for _, child := range node.Content {
+		mapKeysToStrings(child)
+	}
+}
+
 func NewJSONEncoder(indent int, colorise bool, unwrapScalar bool) Encoder {
 	var indentString = ""
 

pkg/yqlib/encoder_test.go

@@ -1,5 +1,3 @@
-//go:build !yq_nojson
-
 package yqlib
 
 import (

pkg/yqlib/encoder_xml.go

@@ -1,5 +1,3 @@
-//go:build !yq_noxml
-
 package yqlib
 
 import (
@@ -19,6 +17,8 @@ type xmlEncoder struct {
 	leadingContent string
 }
 
+var commentPrefix = regexp.MustCompile(`(^|\n)\s*#`)
+
 func NewXMLEncoder(indent int, prefs XmlPreferences) Encoder {
 	var indentString = ""
 
@@ -37,7 +37,7 @@ func (e *xmlEncoder) PrintDocumentSeparator(writer io.Writer) error {
 }
 
 func (e *xmlEncoder) PrintLeadingContent(writer io.Writer, content string) error {
-	e.leadingContent = content
+	e.leadingContent = commentPrefix.ReplaceAllString(content, "\n")
 	return nil
 }
 
@@ -46,39 +46,12 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
 	// hack so we can manually add newlines to procInst and directives
 	e.writer = writer
 	encoder.Indent("", e.indentString)
-	var newLine xml.CharData = []byte("\n")
-
-	mapNode := unwrapDoc(node)
-	if mapNode.Tag == "!!map" {
-		// make sure <?xml .. ?> processing instructions are encoded first
-		for i := 0; i < len(mapNode.Content); i += 2 {
-			key := mapNode.Content[i]
-			value := mapNode.Content[i+1]
-
-			if key.Value == (e.prefs.ProcInstPrefix + "xml") {
-				name := strings.Replace(key.Value, e.prefs.ProcInstPrefix, "", 1)
-				procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)}
-				if err := encoder.EncodeToken(procInst); err != nil {
-					return err
-				}
-				if _, err := e.writer.Write([]byte("\n")); err != nil {
-					log.Warning("Unable to write newline, skipping: %w", err)
-				}
-			}
-		}
-	}
 
 	if e.leadingContent != "" {
-
-		// remove first and last newlines if present
 		err := e.encodeComment(encoder, e.leadingContent)
 		if err != nil {
 			return err
 		}
-		err = encoder.EncodeToken(newLine)
-		if err != nil {
-			return err
-		}
 	}
 
 	switch node.Kind {
@@ -114,12 +87,29 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
 	default:
 		return fmt.Errorf("unsupported type %v", node.Tag)
 	}
-
-	return encoder.EncodeToken(newLine)
+	var charData xml.CharData = []byte("\n")
+	return encoder.EncodeToken(charData)
 
 }
 
 func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node) error {
+	// make sure <?xml .. ?> processing instructions are encoded first
+	for i := 0; i < len(node.Content); i += 2 {
+		key := node.Content[i]
+		value := node.Content[i+1]
+
+		if key.Value == (e.prefs.ProcInstPrefix + "xml") {
+			name := strings.Replace(key.Value, e.prefs.ProcInstPrefix, "", 1)
+			procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)}
+			if err := encoder.EncodeToken(procInst); err != nil {
+				return err
+			}
+			if _, err := e.writer.Write([]byte("\n")); err != nil {
+				log.Warning("Unable to write newline, skipping: %w", err)
+			}
+		}
+	}
+
 	err := e.encodeComment(encoder, headAndLineComment(node))
 	if err != nil {
 		return err
@@ -134,13 +124,6 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node) er
 		if err != nil {
 			return err
 		}
-		if headAndLineComment(key) != "" {
-			var newLine xml.CharData = []byte("\n")
-			err = encoder.EncodeToken(newLine)
-			if err != nil {
-				return err
-			}
-		}
 
 		if key.Value == (e.prefs.ProcInstPrefix + "xml") {
 			// dont double process these.
@@ -221,33 +204,12 @@ func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *yaml.Node, start xml.S
 	return fmt.Errorf("unsupported type %v", node.Tag)
 }
 
-var xmlEncodeMultilineCommentRegex = regexp.MustCompile(`(^|\n) *# ?(.*)`)
-var xmlEncodeSingleLineCommentRegex = regexp.MustCompile(`^\s*#(.*)\n?`)
-var chompRegexp = regexp.MustCompile(`\n$`)
-
 func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) error {
 	if commentStr != "" {
-		log.Debugf("got comment [%v]", commentStr)
-		// multi line string
-		if len(commentStr) > 2 && strings.Contains(commentStr[1:len(commentStr)-1], "\n") {
-			commentStr = chompRegexp.ReplaceAllString(commentStr, "")
-			log.Debugf("chompRegexp [%v]", commentStr)
-			commentStr = xmlEncodeMultilineCommentRegex.ReplaceAllString(commentStr, "$1$2")
-			log.Debugf("processed multine [%v]", commentStr)
-			// if the first line is non blank, add a space
-			if commentStr[0] != '\n' && commentStr[0] != ' ' {
-				commentStr = " " + commentStr
-			}
-
-		} else {
-			commentStr = xmlEncodeSingleLineCommentRegex.ReplaceAllString(commentStr, "$1")
-		}
-
-		if !strings.HasSuffix(commentStr, " ") && !strings.HasSuffix(commentStr, "\n") {
+		log.Debugf("encoding comment %v", commentStr)
+		if !strings.HasSuffix(commentStr, " ") {
 			commentStr = commentStr + " "
-			log.Debugf("added suffix [%v]", commentStr)
 		}
-		log.Debugf("encoding comment [%v]", commentStr)
 
 		var comment xml.Comment = []byte(commentStr)
 		err := encoder.EncodeToken(comment)

pkg/yqlib/json_test.go

@@ -1,5 +1,3 @@
-//go:build !yq_nojson
-
 package yqlib
 
 import (

pkg/yqlib/lexer_participle.go

@@ -37,7 +37,6 @@ var participleYqRules = []*participleYqRule{
 
 	{"MapValues", `map_?values`, opToken(mapValuesOpType), 0},
 	simpleOp("map", mapOpType),
-	simpleOp("filter", filterOpType),
 	simpleOp("pick", pickOpType),
 
 	{"FlattenWithDepth", `flatten\([0-9]+\)`, flattenWithDepth(), 0},

pkg/yqlib/lib.go

@@ -84,7 +84,6 @@ var expressionOpType = &operationType{Type: "EXP", NumArgs: 0, Precedence: 50, H
 
 var collectOpType = &operationType{Type: "COLLECT", NumArgs: 1, Precedence: 50, Handler: collectOperator}
 var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 50, Handler: mapOperator}
-var filterOpType = &operationType{Type: "FILTER", NumArgs: 1, Precedence: 50, Handler: filterOperator}
 var errorOpType = &operationType{Type: "ERROR", NumArgs: 1, Precedence: 50, Handler: errorOperator}
 var pickOpType = &operationType{Type: "PICK", NumArgs: 1, Precedence: 50, Handler: pickOperator}
 var evalOpType = &operationType{Type: "EVAL", NumArgs: 1, Precedence: 50, Handler: evalOperator}
@@ -339,7 +338,7 @@ func deepCloneWithOptions(node *yaml.Node, cloneContent bool) *yaml.Node {
 		Tag:         node.Tag,
 		Value:       node.Value,
 		Anchor:      node.Anchor,
-		Alias:       node.Alias,
+		Alias:       deepClone(node.Alias),
 		HeadComment: node.HeadComment,
 		LineComment: node.LineComment,
 		FootComment: node.FootComment,

pkg/yqlib/no_json.go

@@ -1,11 +0,0 @@
-//go:build yq_nojson
-
-package yqlib
-
-func NewJSONDecoder() Decoder {
-	return nil
-}
-
-func NewJSONEncoder(indent int, colorise bool, unwrapScalar bool) Encoder {
-	return nil
-}

pkg/yqlib/no_xml.go

@@ -1,11 +0,0 @@
-//go:build yq_noxml
-
-package yqlib
-
-func NewXMLDecoder(prefs XmlPreferences) Decoder {
-	return nil
-}
-
-func NewXMLEncoder(indent int, prefs XmlPreferences) Encoder {
-	return nil
-}

pkg/yqlib/operator_assign_test.go

@@ -4,11 +4,6 @@ import (
 	"testing"
 )
 
-var mergeAnchorAssign = `a: &a
-  x: OriginalValue
-b:
-  <<: *a`
-
 var assignOperatorScenarios = []expressionScenario{
 	{
 		description: "Create yaml file",
@@ -25,14 +20,6 @@ var assignOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: null\n",
 		},
 	},
-	{
-		skipDoc:    true,
-		document:   mergeAnchorAssign,
-		expression: `.c = .b | .a.x = "ModifiedValue" | explode(.)`,
-		expected: []string{
-			"D0, P[], (doc)::a:\n    x: ModifiedValue\nb:\n    x: ModifiedValue\nc:\n    x: ModifiedValue\n",
-		},
-	},
 	{
 		skipDoc:    true,
 		document:   "{}",

pkg/yqlib/operator_encoder_decoder.go

@@ -4,7 +4,6 @@ import (
 	"bufio"
 	"bytes"
 	"container/list"
-	"errors"
 	"regexp"
 	"strings"
 
@@ -40,9 +39,6 @@ func encodeToString(candidate *CandidateNode, prefs encoderPreferences) (string,
 	log.Debug("printing with indent: %v", prefs.indent)
 
 	encoder := configureEncoder(prefs.format, prefs.indent)
-	if encoder == nil {
-		return "", errors.New("no support for output format")
-	}
 
 	printer := NewPrinter(encoder, NewSinglePrinterWriter(bufio.NewWriter(&output)))
 	err := printer.PrintResults(candidate.AsList())
@@ -102,11 +98,13 @@ type decoderPreferences struct {
 	format InputFormat
 }
 
-func createDecoder(format InputFormat) Decoder {
+/* takes a string and decodes it back into an object */
+func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	preferences := expressionNode.Operation.Preferences.(decoderPreferences)
+
 	var decoder Decoder
-	switch format {
-	case JsonInputFormat:
-		decoder = NewJSONDecoder()
+	switch preferences.format {
 	case YamlInputFormat:
 		decoder = NewYamlDecoder(ConfiguredYamlPreferences)
 	case XMLInputFormat:
@@ -122,18 +120,6 @@ func createDecoder(format InputFormat) Decoder {
 	case UriInputFormat:
 		decoder = NewUriDecoder()
 	}
-	return decoder
-}
-
-/* takes a string and decodes it back into an object */
-func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-
-	preferences := expressionNode.Operation.Preferences.(decoderPreferences)
-
-	decoder := createDecoder(preferences.format)
-	if decoder == nil {
-		return Context{}, errors.New("no support for input format")
-	}
 
 	var results = list.New()
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {

pkg/yqlib/operator_encoder_decoder_test.go

@@ -8,17 +8,15 @@ var prefix = "D0, P[], (doc)::a:\n    cool:\n        bob: dylan\n"
 
 var encoderDecoderOperatorScenarios = []expressionScenario{
 	{
-		requiresFormat: "json",
-		description:    "Encode value as json string",
-		document:       `{a: {cool: "thing"}}`,
-		expression:     `.b = (.a | to_json)`,
+		description: "Encode value as json string",
+		document:    `{a: {cool: "thing"}}`,
+		expression:  `.b = (.a | to_json)`,
 		expected: []string{
 			`D0, P[], (doc)::{a: {cool: "thing"}, b: "{\n  \"cool\": \"thing\"\n}\n"}
 `,
 		},
 	},
 	{
-		requiresFormat: "json",
 		description:    "Encode value as json string, on one line",
 		subdescription: "Pass in a 0 indent to print json on a single line.",
 		document:       `{a: {cool: "thing"}}`,
@@ -29,7 +27,6 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		requiresFormat: "json",
 		description:    "Encode value as json string, on one line shorthand",
 		subdescription: "Pass in a 0 indent to print json on a single line.",
 		document:       `{a: {cool: "thing"}}`,
@@ -40,7 +37,6 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		requiresFormat: "json",
 		description:    "Decode a json encoded string",
 		subdescription: "Keep in mind JSON is a subset of YAML. If you want idiomatic yaml, pipe through the style operator to clear out the JSON styling.",
 		document:       `a: '{"cool":"thing"}'`,
@@ -197,37 +193,33 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		requiresFormat: "xml",
-		description:    "Encode value as xml string",
-		document:       `{a: {cool: {foo: "bar", +@id: hi}}}`,
-		expression:     `.a | to_xml`,
+		description: "Encode value as xml string",
+		document:    `{a: {cool: {foo: "bar", +@id: hi}}}`,
+		expression:  `.a | to_xml`,
 		expected: []string{
 			"D0, P[a], (!!str)::<cool id=\"hi\">\n  <foo>bar</foo>\n</cool>\n\n",
 		},
 	},
 	{
-		requiresFormat: "xml",
-		description:    "Encode value as xml string on a single line",
-		document:       `{a: {cool: {foo: "bar", +@id: hi}}}`,
-		expression:     `.a | @xml`,
+		description: "Encode value as xml string on a single line",
+		document:    `{a: {cool: {foo: "bar", +@id: hi}}}`,
+		expression:  `.a | @xml`,
 		expected: []string{
 			"D0, P[a], (!!str)::<cool id=\"hi\"><foo>bar</foo></cool>\n\n",
 		},
 	},
 	{
-		requiresFormat: "xml",
-		description:    "Encode value as xml string with custom indentation",
-		document:       `{a: {cool: {foo: "bar", +@id: hi}}}`,
-		expression:     `{"cat": .a | to_xml(1)}`,
+		description: "Encode value as xml string with custom indentation",
+		document:    `{a: {cool: {foo: "bar", +@id: hi}}}`,
+		expression:  `{"cat": .a | to_xml(1)}`,
 		expected: []string{
 			"D0, P[], (!!map)::cat: |\n    <cool id=\"hi\">\n     <foo>bar</foo>\n    </cool>\n",
 		},
 	},
 	{
-		requiresFormat: "xml",
-		description:    "Decode a xml encoded string",
-		document:       `a: "<foo>bar</foo>"`,
-		expression:     `.b = (.a | from_xml)`,
+		description: "Decode a xml encoded string",
+		document:    `a: "<foo>bar</foo>"`,
+		expression:  `.b = (.a | from_xml)`,
 		expected: []string{
 			"D0, P[], (doc)::a: \"<foo>bar</foo>\"\nb:\n    foo: bar\n",
 		},
@@ -311,10 +303,9 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		requiresFormat: "xml",
-		description:    "empty xml decode",
-		skipDoc:        true,
-		expression:     `"" | @xmld`,
+		description: "empty xml decode",
+		skipDoc:     true,
+		expression:  `"" | @xmld`,
 		expected: []string{
 			"D0, P[], (!!null)::\n",
 		},

pkg/yqlib/operator_filter.go

@@ -1,32 +0,0 @@
-package yqlib
-
-import (
-	"container/list"
-)
-
-func filterOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	log.Debugf("-- filterOperation")
-	var results = list.New()
-
-	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
-		candidate := el.Value.(*CandidateNode)
-		children := context.SingleChildContext(candidate)
-		splatted, err := splat(children, traversePreferences{})
-		if err != nil {
-			return Context{}, err
-		}
-		filtered, err := selectOperator(d, splatted, expressionNode)
-		if err != nil {
-			return Context{}, err
-		}
-
-		selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
-		collected, err := collectTogether(d, filtered, selfExpression)
-		if err != nil {
-			return Context{}, err
-		}
-		collected.Node.Style = unwrapDoc(candidate.Node).Style
-		results.PushBack(collected)
-	}
-	return context.ChildContext(results), nil
-}

pkg/yqlib/operator_filter_test.go

@@ -1,49 +0,0 @@
-package yqlib
-
-import (
-	"testing"
-)
-
-var filterOperatorScenarios = []expressionScenario{
-	{
-		description: "Filter array",
-		document:    `[1,2,3]`,
-		expression:  `filter(. < 3)`,
-		expected: []string{
-			"D0, P[], (!!seq)::[1, 2]\n",
-		},
-	},
-	{
-		skipDoc:    true,
-		document:   `[1,2,3]`,
-		expression: `filter(. > 1)`,
-		expected: []string{
-			"D0, P[], (!!seq)::[2, 3]\n",
-		},
-	},
-	{
-		skipDoc:     true,
-		description: "Filter array to empty",
-		document:    `[1,2,3]`,
-		expression:  `filter(. > 4)`,
-		expected: []string{
-			"D0, P[], (!!seq)::[]\n",
-		},
-	},
-	{
-		skipDoc:     true,
-		description: "Filter empty array",
-		document:    `[]`,
-		expression:  `filter(. > 1)`,
-		expected: []string{
-			"D0, P[], (!!seq)::[]\n",
-		},
-	},
-}
-
-func TestFilterOperatorScenarios(t *testing.T) {
-	for _, tt := range filterOperatorScenarios {
-		testScenario(t, &tt)
-	}
-	documentOperatorScenarios(t, "filter", filterOperatorScenarios)
-}

pkg/yqlib/operator_load.go

@@ -34,9 +34,6 @@ func loadString(filename string) (*CandidateNode, error) {
 }
 
 func loadYaml(filename string, decoder Decoder) (*CandidateNode, error) {
-	if decoder == nil {
-		return nil, fmt.Errorf("could not load %s", filename)
-	}
 
 	file, err := os.Open(filename) // #nosec
 	if err != nil {

pkg/yqlib/operator_load_test.go

@@ -74,10 +74,9 @@ var loadScenarios = []expressionScenario{
 		},
 	},
 	{
-		requiresFormat: "xml",
-		description:    "Load from XML",
-		document:       "cool: things",
-		expression:     `.more_stuff = load_xml("../../examples/small.xml")`,
+		description: "Load from XML",
+		document:    "cool: things",
+		expression:  `.more_stuff = load_xml("../../examples/small.xml")`,
 		expected: []string{
 			"D0, P[], (doc)::cool: things\nmore_stuff:\n    this: is some xml\n",
 		},

pkg/yqlib/operators_test.go

@@ -28,7 +28,6 @@ type expressionScenario struct {
 	skipDoc               bool
 	expectedError         string
 	dontFormatInputForDoc bool // dont format input doc for documentation generation
-	requiresFormat        string
 }
 
 func TestMain(m *testing.M) {
@@ -104,23 +103,6 @@ func testScenario(t *testing.T, s *expressionScenario) {
 		return
 	}
 
-	if s.requiresFormat != "" {
-		format := s.requiresFormat
-		inputFormat, err := InputFormatFromString(format)
-		if err != nil {
-			t.Error(err)
-		}
-		if decoder := createDecoder(inputFormat); decoder == nil {
-			t.Skipf("no support for %s input format", format)
-		}
-		outputFormat, err := OutputFormatFromString(format)
-		if err != nil {
-			t.Error(err)
-		}
-		if encoder := configureEncoder(outputFormat, 4); encoder == nil {
-			t.Skipf("no support for %s output format", format)
-		}
-	}
 	if err != nil {
 		t.Error(fmt.Errorf("%w: %v: %v", err, s.description, s.expression))
 		return

pkg/yqlib/ordered_map.go

@@ -1,14 +0,0 @@
-package yqlib
-
-// orderedMap allows to marshal and unmarshal JSON and YAML values keeping the
-// order of keys and values in a map or an object.
-type orderedMap struct {
-	// if this is an object, kv != nil. If this is not an object, kv == nil.
-	kv     []orderedMapKV
-	altVal interface{}
-}
-
-type orderedMapKV struct {
-	K string
-	V orderedMap
-}

pkg/yqlib/ordered_map_json.go

@@ -1,83 +0,0 @@
-package yqlib
-
-import (
-	"bytes"
-	"encoding/json"
-	"errors"
-	"io"
-)
-
-func (o *orderedMap) UnmarshalJSON(data []byte) error {
-	switch data[0] {
-	case '{':
-		// initialise so that even if the object is empty it is not nil
-		o.kv = []orderedMapKV{}
-
-		// create decoder
-		dec := json.NewDecoder(bytes.NewReader(data))
-		_, err := dec.Token() // open object
-		if err != nil {
-			return err
-		}
-
-		// cycle through k/v
-		var tok json.Token
-		for tok, err = dec.Token(); err == nil; tok, err = dec.Token() {
-			// we can expect two types: string or Delim. Delim automatically means
-			// that it is the closing bracket of the object, whereas string means
-			// that there is another key.
-			if _, ok := tok.(json.Delim); ok {
-				break
-			}
-			kv := orderedMapKV{
-				K: tok.(string),
-			}
-			if err := dec.Decode(&kv.V); err != nil {
-				return err
-			}
-			o.kv = append(o.kv, kv)
-		}
-		// unexpected error
-		if err != nil && !errors.Is(err, io.EOF) {
-			return err
-		}
-		return nil
-	case '[':
-		var res []*orderedMap
-		if err := json.Unmarshal(data, &res); err != nil {
-			return err
-		}
-		o.altVal = res
-		o.kv = nil
-		return nil
-	}
-
-	return json.Unmarshal(data, &o.altVal)
-}
-
-func (o orderedMap) MarshalJSON() ([]byte, error) {
-	buf := new(bytes.Buffer)
-	enc := json.NewEncoder(buf)
-	enc.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
-	if o.kv == nil {
-		if err := enc.Encode(o.altVal); err != nil {
-			return nil, err
-		}
-		return buf.Bytes(), nil
-	}
-	buf.WriteByte('{')
-	for idx, el := range o.kv {
-		if err := enc.Encode(el.K); err != nil {
-			return nil, err
-		}
-		buf.WriteByte(':')
-		if err := enc.Encode(el.V); err != nil {
-			return nil, err
-		}
-		if idx != len(o.kv)-1 {
-			buf.WriteByte(',')
-		}
-	}
-	buf.WriteByte('}')
-	return buf.Bytes(), nil
-}

pkg/yqlib/ordered_map_yaml.go

@@ -1,79 +0,0 @@
-package yqlib
-
-import (
-	"fmt"
-
-	yaml "gopkg.in/yaml.v3"
-)
-
-func (o *orderedMap) UnmarshalYAML(node *yaml.Node) error {
-	switch node.Kind {
-	case yaml.DocumentNode:
-		if len(node.Content) == 0 {
-			return nil
-		}
-		return o.UnmarshalYAML(node.Content[0])
-	case yaml.AliasNode:
-		return o.UnmarshalYAML(node.Alias)
-	case yaml.ScalarNode:
-		return node.Decode(&o.altVal)
-	case yaml.MappingNode:
-		// set kv to non-nil
-		o.kv = []orderedMapKV{}
-		for i := 0; i < len(node.Content); i += 2 {
-			var key string
-			var val orderedMap
-			if err := node.Content[i].Decode(&key); err != nil {
-				return err
-			}
-			if err := node.Content[i+1].Decode(&val); err != nil {
-				return err
-			}
-			o.kv = append(o.kv, orderedMapKV{
-				K: key,
-				V: val,
-			})
-		}
-		return nil
-	case yaml.SequenceNode:
-		// note that this has to be a pointer, so that nulls can be represented.
-		var res []*orderedMap
-		if err := node.Decode(&res); err != nil {
-			return err
-		}
-		o.altVal = res
-		o.kv = nil
-		return nil
-	case 0:
-		// null
-		o.kv = nil
-		o.altVal = nil
-		return nil
-	default:
-		return fmt.Errorf("orderedMap: invalid yaml node")
-	}
-}
-
-func (o *orderedMap) MarshalYAML() (interface{}, error) {
-	// fast path: kv is nil, use altVal
-	if o.kv == nil {
-		return o.altVal, nil
-	}
-	content := make([]*yaml.Node, 0, len(o.kv)*2)
-	for _, val := range o.kv {
-		n := new(yaml.Node)
-		if err := n.Encode(val.V); err != nil {
-			return nil, err
-		}
-		content = append(content, &yaml.Node{
-			Kind:  yaml.ScalarNode,
-			Tag:   "!!str",
-			Value: val.K,
-		}, n)
-	}
-	return &yaml.Node{
-		Kind:    yaml.MappingNode,
-		Tag:     "!!map",
-		Content: content,
-	}, nil
-}

pkg/yqlib/printer.go

@@ -33,11 +33,11 @@ const (
 
 func OutputFormatFromString(format string) (PrinterOutputFormat, error) {
 	switch format {
-	case "yaml", "y", "yml":
+	case "yaml", "y":
 		return YamlOutputFormat, nil
 	case "json", "j":
 		return JSONOutputFormat, nil
-	case "props", "p", "properties":
+	case "props", "p":
 		return PropsOutputFormat, nil
 	case "csv", "c":
 		return CSVOutputFormat, nil

pkg/yqlib/printer_test.go

@@ -314,11 +314,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	// note printDocSeparators is true, it should still not print document separators
 	// when outputing JSON.
-	encoder := NewJSONEncoder(0, false, false)
-	if encoder == nil {
-		t.Skipf("no support for %s output format", "json")
-	}
-	printer := NewPrinter(encoder, NewSinglePrinterWriter(writer))
+	printer := NewPrinter(NewJSONEncoder(0, false, false), NewSinglePrinterWriter(writer))
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {

pkg/yqlib/xml_test.go

@@ -1,5 +1,3 @@
-//go:build !yq_noxml
-
 package yqlib
 
 import (
@@ -10,29 +8,6 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
-const yamlInputWithProcInstAndHeadComment = `# cats
-+p_xml: version="1.0"
-this: is some xml`
-
-const expectedXmlProcInstAndHeadComment = `<?xml version="1.0"?>
-<!-- cats -->
-<this>is some xml</this>
-`
-
-const xmlProcInstAndHeadCommentBlock = `<?xml version="1.0"?>
-<!--
-cats
--->
-<this>is some xml</this>
-`
-
-const expectedYamlProcInstAndHeadCommentBlock = `#
-# cats
-#
-+p_xml: version="1.0"
-this: is some xml
-`
-
 const inputXMLWithComments = `
 <!-- before cat -->
 <cat>
@@ -151,8 +126,7 @@ cat:
 # after cat
 `
 
-const expectedRoundtripXMLWithComments = `<!-- before cat -->
-<cat><!-- in cat before -->
+const expectedRoundtripXMLWithComments = `<!-- before cat --><cat><!-- in cat before -->
   <x>3<!-- multi
 line comment 
 for x --></x><!-- before y -->
@@ -163,10 +137,8 @@ in d before -->
 </cat><!-- after cat -->
 `
 
-const yamlWithComments = `#
-# header comment
+const yamlWithComments = `# header comment
 # above_cat
-#
 cat: # inline_cat
   # above_array
   array: # inline_array
@@ -177,11 +149,9 @@ cat: # inline_cat
 `
 
 const expectedXMLWithComments = `<!--
-header comment
-above_cat
--->
-<!-- inline_cat -->
-<cat><!-- above_array inline_array -->
+ header comment
+ above_cat
+ --><!-- inline_cat --><cat><!-- above_array inline_array -->
   <array>val1<!-- inline_val1 --></array>
   <array><!-- above_val2 -->val2<!-- inline_val2 --></array>
 </cat><!-- below_cat -->
@@ -294,41 +264,6 @@ var xmlScenarios = []formatScenario{
 		input:    "<root><cats><cat>quick</cat><cat>soft</cat><!-- kitty_comment--><cat>squishy</cat></cats></root>",
 		expected: "root:\n    cats:\n        cat:\n            - quick\n            - soft\n            # kitty_comment\n\n            - squishy\n",
 	},
-	{
-		description:  "ProcInst with head comment",
-		skipDoc:      true,
-		input:        yamlInputWithProcInstAndHeadComment,
-		expected:     expectedXmlProcInstAndHeadComment,
-		scenarioType: "encode",
-	},
-	{
-		description:  "ProcInst with head comment round trip",
-		skipDoc:      true,
-		input:        expectedXmlProcInstAndHeadComment,
-		expected:     expectedXmlProcInstAndHeadComment,
-		scenarioType: "roundtrip",
-	},
-	{
-		description:  "ProcInst with block head comment to yaml",
-		skipDoc:      true,
-		input:        xmlProcInstAndHeadCommentBlock,
-		expected:     expectedYamlProcInstAndHeadCommentBlock,
-		scenarioType: "decode",
-	},
-	{
-		description:  "ProcInst with block head comment from yaml",
-		skipDoc:      true,
-		input:        expectedYamlProcInstAndHeadCommentBlock,
-		expected:     xmlProcInstAndHeadCommentBlock,
-		scenarioType: "encode",
-	},
-	{
-		description:  "ProcInst with head comment round trip block",
-		skipDoc:      true,
-		input:        xmlProcInstAndHeadCommentBlock,
-		expected:     xmlProcInstAndHeadCommentBlock,
-		scenarioType: "roundtrip",
-	},
 	{
 		description:    "Parse xml: simple",
 		subdescription: "Notice how all the values are strings, see the next example on how you can fix that.",
@@ -529,41 +464,6 @@ var xmlScenarios = []formatScenario{
 		expected:       "<cat name=\"tiger\">cool</cat>\n",
 		scenarioType:   "encode",
 	},
-	{
-		description:  "round trip multiline 1",
-		skipDoc:      true,
-		input:        "<x><!-- cats --></x>\n",
-		expected:     "<x><!-- cats --></x>\n",
-		scenarioType: "roundtrip",
-	},
-	{
-		description:  "round trip multiline 2",
-		skipDoc:      true,
-		input:        "<x><!--\n cats\n --></x>\n",
-		expected:     "<x><!--\ncats\n--></x>\n",
-		scenarioType: "roundtrip",
-	},
-	{
-		description:  "round trip multiline 3",
-		skipDoc:      true,
-		input:        "<x><!--\n\tcats\n --></x>\n",
-		expected:     "<x><!--\n\tcats\n--></x>\n",
-		scenarioType: "roundtrip",
-	},
-	{
-		description:  "round trip multiline 4",
-		skipDoc:      true,
-		input:        "<x><!--\n\tcats\n\tdogs\n--></x>\n",
-		expected:     "<x><!--\n\tcats\n\tdogs\n--></x>\n",
-		scenarioType: "roundtrip",
-	},
-	{
-		description:  "round trip multiline 5",
-		skipDoc:      true, // pity spaces aren't kept atm.
-		input:        "<x><!--\ncats\ndogs\n--></x>\n",
-		expected:     "<x><!--\ncats\ndogs\n--></x>\n",
-		scenarioType: "roundtrip",
-	},
 	{
 		description:    "Encode xml: comments",
 		subdescription: "A best attempt is made to copy comments to xml.",

release_notes.txt

@@ -1,10 +1,3 @@
-4.31.2:
-  - Fixed variable handling #1458, #1566
-  - Fixed merged anchor reference problem #1482
-  - Fixed xml encoding of ProcInst #1563, improved XML comment handling
-  - Allow build without json and xml support (#1556) Thanks @afbjorklund
-  - Bumped dependencies
-
 4.31.1:
   - Added shuffle command #1503
   - Added ability to sort by multiple fields #1541

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: 'v4.31.2'
+version: 'v4.31.1'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.