Detect input format based on file name extension (#1582)

* detect inputFormat from filename

* refactor and extract func InputFormatFromFilename

* detect inputFormat only when file is provided

* add test for automatic input format detection
This commit is contained in:
ryenus 2023-03-09 07:17:19 +08:00 committed by GitHub
parent fed96f67ea
commit d30941b575
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 260 additions and 7 deletions

View File

@ -0,0 +1,221 @@
#!/bin/bash
setUp() {
rm test*.yml 2>/dev/null || true
rm test*.json 2>/dev/null || true
rm test*.properties 2>/dev/null || true
rm test*.csv 2>/dev/null || true
rm test*.tsv 2>/dev/null || true
rm test*.xml 2>/dev/null || true
}
testInputJson() {
cat >test.json <<EOL
{ "mike" : { "things": "cool" } }
EOL
read -r -d '' expected << EOM
mike:
things: cool
EOM
X=$(./yq test.json)
assertEquals "$expected" "$X"
X=$(./yq ea test.json)
assertEquals "$expected" "$X"
}
testInputProperties() {
cat >test.properties <<EOL
mike.things = hello
EOL
read -r -d '' expected << EOM
mike:
things: hello
EOM
X=$(./yq e test.properties)
assertEquals "$expected" "$X"
X=$(./yq ea test.properties)
assertEquals "$expected" "$X"
}
testInputPropertiesGitHubAction() {
cat >test.properties <<EOL
mike.things = hello
EOL
read -r -d '' expected << EOM
mike:
things: hello
EOM
X=$(cat /dev/null | ./yq e test.properties)
assertEquals "$expected" "$X"
X=$(cat /dev/null | ./yq ea test.properties)
assertEquals "$expected" "$X"
}
testInputCSV() {
cat >test.csv <<EOL
fruit,yumLevel
apple,5
banana,4
EOL
read -r -d '' expected << EOM
- fruit: apple
yumLevel: 5
- fruit: banana
yumLevel: 4
EOM
X=$(./yq e test.csv)
assertEquals "$expected" "$X"
X=$(./yq ea test.csv)
assertEquals "$expected" "$X"
}
testInputCSVUTF8() {
read -r -d '' expected << EOM
- id: 1
first: john
last: smith
- id: 1
first: jane
last: smith
EOM
X=$(./yq utf8.csv)
assertEquals "$expected" "$X"
}
testInputTSV() {
cat >test.tsv <<EOL
fruit yumLevel
apple 5
banana 4
EOL
read -r -d '' expected << EOM
- fruit: apple
yumLevel: 5
- fruit: banana
yumLevel: 4
EOM
X=$(./yq e test.tsv)
assertEquals "$expected" "$X"
X=$(./yq ea test.tsv)
assertEquals "$expected" "$X"
}
testInputXml() {
cat >test.xml <<EOL
<cat legs="4">BiBi</cat>
EOL
read -r -d '' expected << EOM
cat:
+content: BiBi
+@legs: "4"
EOM
X=$(./yq e test.xml)
assertEquals "$expected" "$X"
X=$(./yq ea test.xml)
assertEquals "$expected" "$X"
}
testInputXmlNamespaces() {
cat >test.xml <<EOL
<?xml version="1.0"?>
<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
</map>
EOL
read -r -d '' expected << EOM
+p_xml: version="1.0"
map:
+@xmlns: some-namespace
+@xmlns:xsi: some-instance
+@xsi:schemaLocation: some-url
EOM
X=$(./yq e test.xml)
assertEquals "$expected" "$X"
X=$(./yq ea test.xml)
assertEquals "$expected" "$X"
}
testInputXmlRoundtrip() {
cat >test.xml <<EOL
<?xml version="1.0"?>
<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">Meow</map>
EOL
read -r -d '' expected << EOM
<?xml version="1.0"?>
<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">Meow</map>
EOM
X=$(./yq -o=xml test.xml)
assertEquals "$expected" "$X"
X=$(./yq ea -o=xml test.xml)
assertEquals "$expected" "$X"
}
testInputXmlStrict() {
cat >test.xml <<EOL
<?xml version="1.0"?>
<!DOCTYPE root [
<!ENTITY writer "Catherine.">
<!ENTITY copyright "(r) Great">
]>
<root>
<item>&writer;&copyright;</item>
</root>
EOL
X=$(./yq --xml-strict-mode test.xml -o=xml 2>&1)
assertEquals 1 $?
assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X"
X=$(./yq ea --xml-strict-mode test.xml -o=xml 2>&1)
assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X"
}
testInputXmlGithubAction() {
cat >test.xml <<EOL
<cat legs="4">BiBi</cat>
EOL
read -r -d '' expected << EOM
cat:
+content: BiBi
+@legs: "4"
EOM
X=$(cat /dev/null | ./yq e test.xml)
assertEquals "$expected" "$X"
X=$(cat /dev/null | ./yq ea test.xml)
assertEquals "$expected" "$X"
}
source ./scripts/shunit2

View File

@ -7,7 +7,8 @@ var unwrapScalar = false
var writeInplace = false var writeInplace = false
var outputToJSON = false var outputToJSON = false
var outputFormat = "yaml" var outputFormat = "yaml"
var inputFormat = "yaml" var inputFormatDefault = "yaml"
var inputFormat = ""
var exitStatus = false var exitStatus = false
var forceColor = false var forceColor = false

View File

@ -75,7 +75,12 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
return err return err
} }
decoder, err := configureDecoder(true) inputFilename := ""
if len(args) > 0 {
inputFilename = args[0]
}
decoder, err := configureDecoder(true, inputFilename)
if err != nil { if err != nil {
return err return err
} }

View File

@ -100,7 +100,12 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
printer := yqlib.NewPrinter(encoder, printerWriter) printer := yqlib.NewPrinter(encoder, printerWriter)
decoder, err := configureDecoder(false) inputFilename := ""
if len(args) > 0 {
inputFilename = args[0]
}
decoder, err := configureDecoder(false, inputFilename)
if err != nil { if err != nil {
return err return err
} }

View File

@ -85,7 +85,7 @@ yq -P sample.json
} }
rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "yaml", "[yaml|y|json|j|props|p|xml|x] output format type.") rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "yaml", "[yaml|y|json|j|props|p|xml|x] output format type.")
rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "yaml", "[yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.") rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "", "[yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.")
rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", yqlib.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes") rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", yqlib.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes")
rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", yqlib.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).") rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", yqlib.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).")

View File

@ -56,7 +56,12 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
return expression, args, nil return expression, args, nil
} }
func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) { func configureDecoder(evaluateTogether bool, inputFilename string) (yqlib.Decoder, error) {
if inputFormat == "" {
inputFormat = yqlib.InputFormatFromFilename(inputFilename, inputFormatDefault)
} else {
yqlib.GetLogger().Debugf("user specified inputFormat '%s'", inputFormat)
}
yqlibInputFormat, err := yqlib.InputFormatFromString(inputFormat) yqlibInputFormat, err := yqlib.InputFormatFromString(inputFormat)
if err != nil { if err != nil {
return nil, err return nil, err

View File

@ -3,6 +3,7 @@ package yqlib
import ( import (
"fmt" "fmt"
"io" "io"
"strings"
) )
type InputFormat uint type InputFormat uint
@ -25,11 +26,11 @@ type Decoder interface {
func InputFormatFromString(format string) (InputFormat, error) { func InputFormatFromString(format string) (InputFormat, error) {
switch format { switch format {
case "yaml", "y": case "yaml", "yml", "y":
return YamlInputFormat, nil return YamlInputFormat, nil
case "xml", "x": case "xml", "x":
return XMLInputFormat, nil return XMLInputFormat, nil
case "props", "p": case "properties", "props", "p":
return PropertiesInputFormat, nil return PropertiesInputFormat, nil
case "json", "ndjson", "j": case "json", "ndjson", "j":
return JsonInputFormat, nil return JsonInputFormat, nil
@ -41,3 +42,18 @@ func InputFormatFromString(format string) (InputFormat, error) {
return 0, fmt.Errorf("unknown format '%v' please use [yaml|xml|props]", format) return 0, fmt.Errorf("unknown format '%v' please use [yaml|xml|props]", format)
} }
} }
func InputFormatFromFilename(filename string, defaultFormat string) string {
if filename != "" {
GetLogger().Debugf("checking filename '%s' for inputFormat", filename)
nPos := strings.LastIndex(filename, ".")
if nPos > -1 {
inputFormat := filename[nPos+1:]
GetLogger().Debugf("detected inputFormat '%s'", inputFormat)
return inputFormat
}
}
GetLogger().Debugf("using default inputFormat '%s'", defaultFormat)
return defaultFormat
}