Auto output format (#1599)

* Use file extension to auto detect output format!

* Use file extension to auto detect output format!

* formatting

acceptance_tests/inputs-format-auto.sh

@@ -15,8 +15,11 @@ testInputJson() {
 EOL
 
   read -r -d '' expected << EOM
-mike:
+{
-  things: cool
+  "mike": {
+    "things": "cool"
+  }
+}
 EOM
 
   X=$(./yq test.json)
@@ -26,19 +29,38 @@ EOM
   assertEquals "$expected" "$X"
 }
 
+testInputJsonOutputYaml() {
+  cat >test.json <<EOL
+{ "mike" : { "things": "cool" } }
+EOL
+
+  read -r -d '' expected << EOM
+mike:
+  things: cool
+EOM
+
+  X=$(./yq test.json -oy)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea test.json -oy)
+  assertEquals "$expected" "$X"
+}
+
 testInputProperties() {
   cat >test.properties <<EOL
 mike.things = hello
 EOL
 
   read -r -d '' expected << EOM
-mike:
+mike.things = hello
-  things: hello
 EOM
 
   X=$(./yq e test.properties)
   assertEquals "$expected" "$X"
 
+  X=$(./yq test.properties)
+  assertEquals "$expected" "$X"
+
   X=$(./yq ea test.properties)
   assertEquals "$expected" "$X"
 }
@@ -49,8 +71,7 @@ mike.things = hello
 EOL
 
   read -r -d '' expected << EOM
-mike:
+mike.things = hello
-  things: hello
 EOM
 
   X=$(cat /dev/null | ./yq e test.properties)
@@ -68,10 +89,9 @@ banana,4
 EOL
 
   read -r -d '' expected << EOM
-- fruit: apple
+fruit,yumLevel
-  yumLevel: 5
+apple,5
-- fruit: banana
+banana,4
-  yumLevel: 4
 EOM
 
   X=$(./yq e test.csv)
@@ -83,12 +103,9 @@ EOM
 
 testInputCSVUTF8() {
   read -r -d '' expected << EOM
-- id: 1
+id,first,last
-  first: john
+1,john,smith
-  last: smith
+1,jane,smith
-- id: 1
-  first: jane
-  last: smith
 EOM
 
   X=$(./yq utf8.csv)
@@ -103,10 +120,9 @@ banana	4
 EOL
 
   read -r -d '' expected << EOM
-- fruit: apple
+fruit	yumLevel
-  yumLevel: 5
+apple	5
-- fruit: banana
+banana	4
-  yumLevel: 4
 EOM
 
   X=$(./yq e test.tsv)
@@ -125,9 +141,7 @@ testInputXml() {
 EOL
 
   read -r -d '' expected << EOM
-cat:
+<cat legs="4">BiBi</cat>
-  +content: BiBi
-  +@legs: "4"
 EOM
 
   X=$(./yq e test.xml)
@@ -145,11 +159,8 @@ testInputXmlNamespaces() {
 EOL
 
   read -r -d '' expected << EOM
-+p_xml: version="1.0"
+<?xml version="1.0"?>
-map:
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url"></map>
-  +@xmlns: some-namespace
-  +@xmlns:xsi: some-instance
-  +@xsi:schemaLocation: some-url
 EOM
 
   X=$(./yq e test.xml)
@@ -159,25 +170,6 @@ EOM
   assertEquals "$expected" "$X"
 }
 
-testInputXmlRoundtrip() {
-  cat >test.xml <<EOL
-<?xml version="1.0"?>
-<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
-<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">Meow</map>
-EOL
-
-  read -r -d '' expected << EOM
-<?xml version="1.0"?>
-<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
-<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">Meow</map>
-EOM
-
-  X=$(./yq -o=xml test.xml)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea -o=xml test.xml)
-  assertEquals "$expected" "$X"
-}
 
 
 testInputXmlStrict() {
@@ -192,11 +184,11 @@ testInputXmlStrict() {
 </root>
 EOL
 
-  X=$(./yq --xml-strict-mode test.xml -o=xml 2>&1)
+  X=$(./yq --xml-strict-mode test.xml  2>&1)
   assertEquals 1 $?
   assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X"
 
-  X=$(./yq ea --xml-strict-mode test.xml -o=xml 2>&1)
+  X=$(./yq ea --xml-strict-mode test.xml  2>&1)
   assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X"
 }
 
@@ -206,9 +198,7 @@ testInputXmlGithubAction() {
 EOL
 
   read -r -d '' expected << EOM
-cat:
+<cat legs="4">BiBi</cat>
-  +content: BiBi
-  +@legs: "4"
 EOM
 
   X=$(cat /dev/null | ./yq e test.xml)

acceptance_tests/inputs-format.sh

@@ -120,7 +120,7 @@ EOM
 }
 
 testInputXmlNamespaces() {
-  cat >test.yml <<EOL
+  cat >test.xml <<EOL
 <?xml version="1.0"?>
 <map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
 </map>
@@ -134,10 +134,10 @@ map:
   +@xsi:schemaLocation: some-url
 EOM
 
-  X=$(./yq e -p=xml test.yml)
+  X=$(./yq e -p=xml test.xml)
   assertEquals "$expected" "$X"
 
-  X=$(./yq ea -p=xml test.yml)
+  X=$(./yq ea -p=xml test.xml)
   assertEquals "$expected" "$X"
 }
 

cmd/constant.go

@@ -6,8 +6,9 @@ var unwrapScalar = false
 
 var writeInplace = false
 var outputToJSON = false
-var outputFormat = "yaml"
+
-var inputFormatDefault = "yaml"
+var outputFormat = ""
+
 var inputFormat = ""
 
 var exitStatus = false

cmd/evaluate_all_command.go

@@ -75,12 +75,7 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 		return err
 	}
 
-	inputFilename := ""
+	decoder, err := configureDecoder(true)
-	if len(args) > 0 {
-		inputFilename = args[0]
-	}
-
-	decoder, err := configureDecoder(true, inputFilename)
 	if err != nil {
 		return err
 	}

cmd/evalute_sequence_command.go

@@ -100,12 +100,7 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 
 	printer := yqlib.NewPrinter(encoder, printerWriter)
 
-	inputFilename := ""
+	decoder, err := configureDecoder(false)
-	if len(args) > 0 {
-		inputFilename = args[0]
-	}
-
-	decoder, err := configureDecoder(false, inputFilename)
 	if err != nil {
 		return err
 	}

cmd/root.go

@@ -53,25 +53,6 @@ yq -P sample.json
 			logging.SetBackend(backend)
 			yqlib.InitExpressionParser()
 
-			outputFormatType, err := yqlib.OutputFormatFromString(outputFormat)
-
-			if err != nil {
-				return err
-			}
-
-			if outputFormatType == yqlib.YamlOutputFormat ||
-				outputFormatType == yqlib.PropsOutputFormat {
-				unwrapScalar = true
-			}
-			if unwrapScalarFlag.IsExplicitySet() {
-				unwrapScalar = unwrapScalarFlag.IsSet()
-			}
-
-			//copy preference form global setting
-			yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar
-
-			yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators
-
 			return nil
 		},
 	}
@@ -84,8 +65,8 @@ yq -P sample.json
 		panic(err)
 	}
 
-	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "yaml", "[yaml|y|json|j|props|p|xml|x] output format type.")
+	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "auto", "[auto|a|yaml|y|json|j|props|p|xml|x] output format type.")
-	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "", "[yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.")
+	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "auto", "[auto|a|yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.")
 
 	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", yqlib.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes")
 	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", yqlib.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).")

cmd/utils.go

@@ -53,15 +53,52 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 		return "", nil, fmt.Errorf("cannot pass files in when using null-input flag")
 	}
 
+	inputFilename := ""
+	if len(args) > 0 {
+		inputFilename = args[0]
+	}
+	if inputFormat == "" || inputFormat == "auto" || inputFormat == "a" {
+
+		inputFormat = yqlib.FormatFromFilename(inputFilename)
+		if outputFormat == "" || outputFormat == "auto" || outputFormat == "a" {
+			outputFormat = yqlib.FormatFromFilename(inputFilename)
+		}
+	} else if outputFormat == "" || outputFormat == "auto" || outputFormat == "a" {
+		// backwards compatibility -
+		// before this was introduced, `yq -pcsv things.csv`
+		// would produce *yaml* output.
+		//
+		outputFormat = yqlib.FormatFromFilename(inputFilename)
+		if inputFilename != "-" {
+			yqlib.GetLogger().Warning("yq default output is now 'auto' (based on the filename extension). Normally yq would output '%v', but for backwards compatibility 'yaml' has been set. Please use -oy to specify yaml, or drop the -p flag.", outputFormat)
+		}
+		outputFormat = "yaml"
+	}
+
+	outputFormatType, err := yqlib.OutputFormatFromString(outputFormat)
+
+	if err != nil {
+		return "", nil, err
+	}
+	yqlib.GetLogger().Debug("Using outputformat %v", outputFormat)
+
+	if outputFormatType == yqlib.YamlOutputFormat ||
+		outputFormatType == yqlib.PropsOutputFormat {
+		unwrapScalar = true
+	}
+	if unwrapScalarFlag.IsExplicitySet() {
+		unwrapScalar = unwrapScalarFlag.IsSet()
+	}
+
+	//copy preference form global setting
+	yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar
+
+	yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators
+
 	return expression, args, nil
 }
 
-func configureDecoder(evaluateTogether bool, inputFilename string) (yqlib.Decoder, error) {
+func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
-	if inputFormat == "" {
-		inputFormat = yqlib.InputFormatFromFilename(inputFilename, inputFormatDefault)
-	} else {
-		yqlib.GetLogger().Debugf("user specified inputFormat '%s'", inputFormat)
-	}
 	yqlibInputFormat, err := yqlib.InputFormatFromString(inputFormat)
 	if err != nil {
 		return nil, err

examples/mike.xml

@@ -1,5 +1 @@
-date: "2022-11-25"
+<cat>3</cat>
-raw:
-  id: 1
-  XML: text_outside_1<Tag1 />text_outside_2<Tag2Kling Klong</Tag2>text_outside_3
-  time: 09:19:00

pkg/yqlib/decoder.go

@@ -39,21 +39,22 @@ func InputFormatFromString(format string) (InputFormat, error) {
 	case "tsv", "t":
 		return TSVObjectInputFormat, nil
 	default:
-		return 0, fmt.Errorf("unknown format '%v' please use [yaml|xml|props]", format)
+		return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml]", format)
 	}
 }
 
-func InputFormatFromFilename(filename string, defaultFormat string) string {
+func FormatFromFilename(filename string) string {
+
 	if filename != "" {
-		GetLogger().Debugf("checking filename '%s' for inputFormat", filename)
+		GetLogger().Debugf("checking file extension '%s' for auto format detection", filename)
 		nPos := strings.LastIndex(filename, ".")
 		if nPos > -1 {
-			inputFormat := filename[nPos+1:]
+			format := filename[nPos+1:]
-			GetLogger().Debugf("detected inputFormat '%s'", inputFormat)
+			GetLogger().Debugf("detected format '%s'", format)
-			return inputFormat
+			return format
 		}
 	}
 
-	GetLogger().Debugf("using default inputFormat '%s'", defaultFormat)
+	GetLogger().Debugf("using default inputFormat 'yaml'")
-	return defaultFormat
+	return "yaml"
 }

pkg/yqlib/operator_filter.go

@@ -30,4 +30,3 @@ func filterOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 	}
 	return context.ChildContext(results), nil
 }
-

pkg/yqlib/operator_filter_test.go

@@ -7,22 +7,22 @@ import (
 var filterOperatorScenarios = []expressionScenario{
 	{
 		description: "Filter array",
-		document:   `[1,2,3]`,
+		document:    `[1,2,3]`,
-		expression: `filter(. < 3)`,
+		expression:  `filter(. < 3)`,
 		expected: []string{
 			"D0, P[], (!!seq)::[1, 2]\n",
 		},
 	},
 	{
-		skipDoc: true,
+		skipDoc:    true,
-		document:    `[1,2,3]`,
+		document:   `[1,2,3]`,
-		expression:  `filter(. > 1)`,
+		expression: `filter(. > 1)`,
 		expected: []string{
 			"D0, P[], (!!seq)::[2, 3]\n",
 		},
 	},
 	{
-		skipDoc: true,
+		skipDoc:     true,
 		description: "Filter array to empty",
 		document:    `[1,2,3]`,
 		expression:  `filter(. > 4)`,
@@ -31,7 +31,7 @@ var filterOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		skipDoc: true,
+		skipDoc:     true,
 		description: "Filter empty array",
 		document:    `[]`,
 		expression:  `filter(. > 1)`,

pkg/yqlib/printer.go

@@ -33,11 +33,11 @@ const (
 
 func OutputFormatFromString(format string) (PrinterOutputFormat, error) {
 	switch format {
-	case "yaml", "y":
+	case "yaml", "y", "yml":
 		return YamlOutputFormat, nil
 	case "json", "j":
 		return JSONOutputFormat, nil
-	case "props", "p":
+	case "props", "p", "properties":
 		return PropsOutputFormat, nil
 	case "csv", "c":
 		return CSVOutputFormat, nil