suyiiyii
/
yq
mirror of https://github.com/mikefarah/yq.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Auto output format (#1599) 

* Use file extension to auto detect output format!

* Use file extension to auto detect output format!

* formatting
This commit is contained in:
Mike Farah 2023-03-15 13:22:58 +11:00 committed by GitHub
parent 2c14c98408
commit 08a6cb65fe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 114 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
acceptance_tests/inputs-format-auto.sh
View File

@ -15,8 +15,11 @@ testInputJson() {
EOL
read -r -d '' expected << EOM
mike:
things: cool
{
"mike": {
"things": "cool"
}
}
EOM
X=$(./yq test.json)
@ -26,19 +29,38 @@ EOM
assertEquals "$expected" "$X"
}
testInputJsonOutputYaml() {
cat >test.json <<EOL
{ "mike" : { "things": "cool" } }
EOL
read -r -d '' expected << EOM
mike:
things: cool
EOM
X=$(./yq test.json -oy)
assertEquals "$expected" "$X"
X=$(./yq ea test.json -oy)
assertEquals "$expected" "$X"
}
testInputProperties() {
cat >test.properties <<EOL
mike.things = hello
EOL
read -r -d '' expected << EOM
mike:
things: hello
mike.things = hello
EOM
X=$(./yq e test.properties)
assertEquals "$expected" "$X"
X=$(./yq test.properties)
assertEquals "$expected" "$X"
X=$(./yq ea test.properties)
assertEquals "$expected" "$X"
}
@ -49,8 +71,7 @@ mike.things = hello
EOL
read -r -d '' expected << EOM
mike:
things: hello
mike.things = hello
EOM
X=$(cat /dev/null | ./yq e test.properties)
@ -68,10 +89,9 @@ banana,4
EOL
read -r -d '' expected << EOM
- fruit: apple
yumLevel: 5
- fruit: banana
yumLevel: 4
fruit,yumLevel
apple,5
banana,4
EOM
X=$(./yq e test.csv)
@ -83,12 +103,9 @@ EOM
testInputCSVUTF8() {
read -r -d '' expected << EOM
- id: 1
first: john
last: smith
- id: 1
first: jane
last: smith
id,first,last
1,john,smith
1,jane,smith
EOM
X=$(./yq utf8.csv)
@ -103,10 +120,9 @@ banana 4
EOL
read -r -d '' expected << EOM
- fruit: apple
yumLevel: 5
- fruit: banana
yumLevel: 4
fruit yumLevel
apple 5
banana 4
EOM
X=$(./yq e test.tsv)
@ -125,9 +141,7 @@ testInputXml() {
EOL
read -r -d '' expected << EOM
cat:
+content: BiBi
+@legs: "4"
<cat legs="4">BiBi</cat>
EOM
X=$(./yq e test.xml)
@ -145,11 +159,8 @@ testInputXmlNamespaces() {
EOL
read -r -d '' expected << EOM
+p_xml: version="1.0"
map:
+@xmlns: some-namespace
+@xmlns:xsi: some-instance
+@xsi:schemaLocation: some-url
<?xml version="1.0"?>
<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url"></map>
EOM
X=$(./yq e test.xml)
@ -159,25 +170,6 @@ EOM
assertEquals "$expected" "$X"
}
testInputXmlRoundtrip() {
cat >test.xml <<EOL
<?xml version="1.0"?>
<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">Meow</map>
EOL
read -r -d '' expected << EOM
<?xml version="1.0"?>
<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">Meow</map>
EOM
X=$(./yq -o=xml test.xml)
assertEquals "$expected" "$X"
X=$(./yq ea -o=xml test.xml)
assertEquals "$expected" "$X"
}
testInputXmlStrict() {
@ -192,11 +184,11 @@ testInputXmlStrict() {
</root>
EOL
X=$(./yq --xml-strict-mode test.xml -o=xml 2>&1)
X=$(./yq --xml-strict-mode test.xml 2>&1)
assertEquals 1 $?
assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X"
X=$(./yq ea --xml-strict-mode test.xml -o=xml 2>&1)
X=$(./yq ea --xml-strict-mode test.xml 2>&1)
assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X"
}
@ -206,9 +198,7 @@ testInputXmlGithubAction() {
EOL
read -r -d '' expected << EOM
cat:
+content: BiBi
+@legs: "4"
<cat legs="4">BiBi</cat>
EOM
X=$(cat /dev/null | ./yq e test.xml)

6
acceptance_tests/inputs-format.sh
View File

@ -120,7 +120,7 @@ EOM
}
testInputXmlNamespaces() {
cat >test.yml <<EOL
cat >test.xml <<EOL
<?xml version="1.0"?>
<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
</map>
@ -134,10 +134,10 @@ map:
+@xsi:schemaLocation: some-url
EOM
X=$(./yq e -p=xml test.yml)
X=$(./yq e -p=xml test.xml)
assertEquals "$expected" "$X"
X=$(./yq ea -p=xml test.yml)
X=$(./yq ea -p=xml test.xml)
assertEquals "$expected" "$X"
}

5
cmd/constant.go
View File

@ -6,8 +6,9 @@ var unwrapScalar = false
var writeInplace = false
var outputToJSON = false
var outputFormat = "yaml"
var inputFormatDefault = "yaml"
var outputFormat = ""
var inputFormat = ""
var exitStatus = false

7
cmd/evaluate_all_command.go
View File

@ -75,12 +75,7 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
return err
}
inputFilename := ""
if len(args) > 0 {
inputFilename = args[0]
}
decoder, err := configureDecoder(true, inputFilename)
decoder, err := configureDecoder(true)
if err != nil {
return err
}

7
cmd/evalute_sequence_command.go
View File

@ -100,12 +100,7 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
printer := yqlib.NewPrinter(encoder, printerWriter)
inputFilename := ""
if len(args) > 0 {
inputFilename = args[0]
}
decoder, err := configureDecoder(false, inputFilename)
decoder, err := configureDecoder(false)
if err != nil {
return err
}

23
cmd/root.go
View File

@ -53,25 +53,6 @@ yq -P sample.json
logging.SetBackend(backend)
yqlib.InitExpressionParser()
outputFormatType, err := yqlib.OutputFormatFromString(outputFormat)
if err != nil {
return err
}
if outputFormatType == yqlib.YamlOutputFormat ||
outputFormatType == yqlib.PropsOutputFormat {
unwrapScalar = true
}
if unwrapScalarFlag.IsExplicitySet() {
unwrapScalar = unwrapScalarFlag.IsSet()
}
//copy preference form global setting
yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar
yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators
return nil
},
}
@ -84,8 +65,8 @@ yq -P sample.json
panic(err)
}
rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "yaml", "[yaml|y|json|j|props|p|xml|x] output format type.")
rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "", "[yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.")
rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "auto", "[auto|a|yaml|y|json|j|props|p|xml|x] output format type.")
rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "auto", "[auto|a|yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.")
rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", yqlib.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes")
rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", yqlib.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).")

49
cmd/utils.go
View File

@ -53,15 +53,52 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
return "", nil, fmt.Errorf("cannot pass files in when using null-input flag")
}
inputFilename := ""
if len(args) > 0 {
inputFilename = args[0]
}
if inputFormat == "" || inputFormat == "auto" || inputFormat == "a" {
inputFormat = yqlib.FormatFromFilename(inputFilename)
if outputFormat == "" || outputFormat == "auto" || outputFormat == "a" {
outputFormat = yqlib.FormatFromFilename(inputFilename)
}
} else if outputFormat == "" || outputFormat == "auto" || outputFormat == "a" {
// backwards compatibility -
// before this was introduced, `yq -pcsv things.csv`
// would produce *yaml* output.
//
outputFormat = yqlib.FormatFromFilename(inputFilename)
if inputFilename != "-" {
yqlib.GetLogger().Warning("yq default output is now 'auto' (based on the filename extension). Normally yq would output '%v', but for backwards compatibility 'yaml' has been set. Please use -oy to specify yaml, or drop the -p flag.", outputFormat)
}
outputFormat = "yaml"
}
outputFormatType, err := yqlib.OutputFormatFromString(outputFormat)
if err != nil {
return "", nil, err
}
yqlib.GetLogger().Debug("Using outputformat %v", outputFormat)
if outputFormatType == yqlib.YamlOutputFormat ||
outputFormatType == yqlib.PropsOutputFormat {
unwrapScalar = true
}
if unwrapScalarFlag.IsExplicitySet() {
unwrapScalar = unwrapScalarFlag.IsSet()
}
//copy preference form global setting
yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar
yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators
return expression, args, nil
}
func configureDecoder(evaluateTogether bool, inputFilename string) (yqlib.Decoder, error) {
if inputFormat == "" {
inputFormat = yqlib.InputFormatFromFilename(inputFilename, inputFormatDefault)
} else {
yqlib.GetLogger().Debugf("user specified inputFormat '%s'", inputFormat)
}
func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
yqlibInputFormat, err := yqlib.InputFormatFromString(inputFormat)
if err != nil {
return nil, err

6
examples/mike.xml
View File

@ -1,5 +1 @@
date: "2022-11-25"
raw:
id: 1
XML: text_outside_1<Tag1 />text_outside_2<Tag2Kling Klong</Tag2>text_outside_3
time: 09:19:00
<cat>3</cat>

17
pkg/yqlib/decoder.go
View File

@ -39,21 +39,22 @@ func InputFormatFromString(format string) (InputFormat, error) {
case "tsv", "t":
return TSVObjectInputFormat, nil
default:
return 0, fmt.Errorf("unknown format '%v' please use [yaml|xml|props]", format)
return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml]", format)
}
}
func InputFormatFromFilename(filename string, defaultFormat string) string {
func FormatFromFilename(filename string) string {
if filename != "" {
GetLogger().Debugf("checking filename '%s' for inputFormat", filename)
GetLogger().Debugf("checking file extension '%s' for auto format detection", filename)
nPos := strings.LastIndex(filename, ".")
if nPos > -1 {
inputFormat := filename[nPos+1:]
GetLogger().Debugf("detected inputFormat '%s'", inputFormat)
return inputFormat
format := filename[nPos+1:]
GetLogger().Debugf("detected format '%s'", format)
return format
}
}
GetLogger().Debugf("using default inputFormat '%s'", defaultFormat)
return defaultFormat
GetLogger().Debugf("using default inputFormat 'yaml'")
return "yaml"
}

1
pkg/yqlib/operator_filter.go
View File

@ -30,4 +30,3 @@ func filterOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
}
return context.ChildContext(results), nil
}

14
pkg/yqlib/operator_filter_test.go
View File

@ -7,22 +7,22 @@ import (
var filterOperatorScenarios = []expressionScenario{
{
description: "Filter array",
document: `[1,2,3]`,
expression: `filter(. < 3)`,
document: `[1,2,3]`,
expression: `filter(. < 3)`,
expected: []string{
"D0, P[], (!!seq)::[1, 2]\n",
},
},
{
skipDoc: true,
document: `[1,2,3]`,
expression: `filter(. > 1)`,
skipDoc: true,
document: `[1,2,3]`,
expression: `filter(. > 1)`,
expected: []string{
"D0, P[], (!!seq)::[2, 3]\n",
},
},
{
skipDoc: true,
skipDoc: true,
description: "Filter array to empty",
document: `[1,2,3]`,
expression: `filter(. > 4)`,
@ -31,7 +31,7 @@ var filterOperatorScenarios = []expressionScenario{
},
},
{
skipDoc: true,
skipDoc: true,
description: "Filter empty array",
document: `[]`,
expression: `filter(. > 1)`,

4
pkg/yqlib/printer.go
View File

@ -33,11 +33,11 @@ const (
func OutputFormatFromString(format string) (PrinterOutputFormat, error) {
switch format {
case "yaml", "y":
case "yaml", "y", "yml":
return YamlOutputFormat, nil
case "json", "j":
return JSONOutputFormat, nil
case "props", "p":
case "props", "p", "properties":
return PropsOutputFormat, nil
case "csv", "c":
return CSVOutputFormat, nil