From d30941b57515969a7e9f6373b63a18c2c32a0a66 Mon Sep 17 00:00:00 2001 From: ryenus Date: Thu, 9 Mar 2023 07:17:19 +0800 Subject: [PATCH] Detect input format based on file name extension (#1582) * detect inputFormat from filename * refactor and extract func InputFormatFromFilename * detect inputFormat only when file is provided * add test for automatic input format detection --- acceptance_tests/inputs-format-auto.sh | 221 +++++++++++++++++++++++++ cmd/constant.go | 3 +- cmd/evaluate_all_command.go | 7 +- cmd/evalute_sequence_command.go | 7 +- cmd/root.go | 2 +- cmd/utils.go | 7 +- pkg/yqlib/decoder.go | 20 ++- 7 files changed, 260 insertions(+), 7 deletions(-) create mode 100755 acceptance_tests/inputs-format-auto.sh diff --git a/acceptance_tests/inputs-format-auto.sh b/acceptance_tests/inputs-format-auto.sh new file mode 100755 index 00000000..4732eb81 --- /dev/null +++ b/acceptance_tests/inputs-format-auto.sh @@ -0,0 +1,221 @@ +#!/bin/bash + +setUp() { + rm test*.yml 2>/dev/null || true + rm test*.json 2>/dev/null || true + rm test*.properties 2>/dev/null || true + rm test*.csv 2>/dev/null || true + rm test*.tsv 2>/dev/null || true + rm test*.xml 2>/dev/null || true +} + +testInputJson() { + cat >test.json <test.properties <test.properties <test.csv <test.tsv <test.xml <BiBi +EOL + + read -r -d '' expected << EOM +cat: + +content: BiBi + +@legs: "4" +EOM + + X=$(./yq e test.xml) + assertEquals "$expected" "$X" + + X=$(./yq ea test.xml) + assertEquals "$expected" "$X" +} + +testInputXmlNamespaces() { + cat >test.xml < + + +EOL + + read -r -d '' expected << EOM ++p_xml: version="1.0" +map: + +@xmlns: some-namespace + +@xmlns:xsi: some-instance + +@xsi:schemaLocation: some-url +EOM + + X=$(./yq e test.xml) + assertEquals "$expected" "$X" + + X=$(./yq ea test.xml) + assertEquals "$expected" "$X" +} + +testInputXmlRoundtrip() { + cat >test.xml < + +Meow +EOL + + read -r -d '' expected << EOM + + +Meow +EOM + + X=$(./yq -o=xml test.xml) + assertEquals "$expected" "$X" + + X=$(./yq ea -o=xml test.xml) + assertEquals "$expected" "$X" +} + + +testInputXmlStrict() { + cat >test.xml < + + +]> + + &writer;©right; + +EOL + + X=$(./yq --xml-strict-mode test.xml -o=xml 2>&1) + assertEquals 1 $? + assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X" + + X=$(./yq ea --xml-strict-mode test.xml -o=xml 2>&1) + assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X" +} + +testInputXmlGithubAction() { + cat >test.xml <BiBi +EOL + + read -r -d '' expected << EOM +cat: + +content: BiBi + +@legs: "4" +EOM + + X=$(cat /dev/null | ./yq e test.xml) + assertEquals "$expected" "$X" + + X=$(cat /dev/null | ./yq ea test.xml) + assertEquals "$expected" "$X" +} + +source ./scripts/shunit2 diff --git a/cmd/constant.go b/cmd/constant.go index 573a72fb..8b94dc2c 100644 --- a/cmd/constant.go +++ b/cmd/constant.go @@ -7,7 +7,8 @@ var unwrapScalar = false var writeInplace = false var outputToJSON = false var outputFormat = "yaml" -var inputFormat = "yaml" +var inputFormatDefault = "yaml" +var inputFormat = "" var exitStatus = false var forceColor = false diff --git a/cmd/evaluate_all_command.go b/cmd/evaluate_all_command.go index fff64fa5..b56a9cc4 100644 --- a/cmd/evaluate_all_command.go +++ b/cmd/evaluate_all_command.go @@ -75,7 +75,12 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) { return err } - decoder, err := configureDecoder(true) + inputFilename := "" + if len(args) > 0 { + inputFilename = args[0] + } + + decoder, err := configureDecoder(true, inputFilename) if err != nil { return err } diff --git a/cmd/evalute_sequence_command.go b/cmd/evalute_sequence_command.go index dab803c3..bc69698f 100644 --- a/cmd/evalute_sequence_command.go +++ b/cmd/evalute_sequence_command.go @@ -100,7 +100,12 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) { printer := yqlib.NewPrinter(encoder, printerWriter) - decoder, err := configureDecoder(false) + inputFilename := "" + if len(args) > 0 { + inputFilename = args[0] + } + + decoder, err := configureDecoder(false, inputFilename) if err != nil { return err } diff --git a/cmd/root.go b/cmd/root.go index 97d9beaf..76782ccb 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -85,7 +85,7 @@ yq -P sample.json } rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "yaml", "[yaml|y|json|j|props|p|xml|x] output format type.") - rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "yaml", "[yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.") + rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "", "[yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.") rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", yqlib.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes") rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", yqlib.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).") diff --git a/cmd/utils.go b/cmd/utils.go index 671dd86b..cc20a827 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -56,7 +56,12 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) { return expression, args, nil } -func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) { +func configureDecoder(evaluateTogether bool, inputFilename string) (yqlib.Decoder, error) { + if inputFormat == "" { + inputFormat = yqlib.InputFormatFromFilename(inputFilename, inputFormatDefault) + } else { + yqlib.GetLogger().Debugf("user specified inputFormat '%s'", inputFormat) + } yqlibInputFormat, err := yqlib.InputFormatFromString(inputFormat) if err != nil { return nil, err diff --git a/pkg/yqlib/decoder.go b/pkg/yqlib/decoder.go index ff7ab2db..c06a5a72 100644 --- a/pkg/yqlib/decoder.go +++ b/pkg/yqlib/decoder.go @@ -3,6 +3,7 @@ package yqlib import ( "fmt" "io" + "strings" ) type InputFormat uint @@ -25,11 +26,11 @@ type Decoder interface { func InputFormatFromString(format string) (InputFormat, error) { switch format { - case "yaml", "y": + case "yaml", "yml", "y": return YamlInputFormat, nil case "xml", "x": return XMLInputFormat, nil - case "props", "p": + case "properties", "props", "p": return PropertiesInputFormat, nil case "json", "ndjson", "j": return JsonInputFormat, nil @@ -41,3 +42,18 @@ func InputFormatFromString(format string) (InputFormat, error) { return 0, fmt.Errorf("unknown format '%v' please use [yaml|xml|props]", format) } } + +func InputFormatFromFilename(filename string, defaultFormat string) string { + if filename != "" { + GetLogger().Debugf("checking filename '%s' for inputFormat", filename) + nPos := strings.LastIndex(filename, ".") + if nPos > -1 { + inputFormat := filename[nPos+1:] + GetLogger().Debugf("detected inputFormat '%s'", inputFormat) + return inputFormat + } + } + + GetLogger().Debugf("using default inputFormat '%s'", defaultFormat) + return defaultFormat +}