mirror of
https://github.com/mikefarah/yq.git
synced 2026-08-24 16:39:58 +08:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
48d6ad58e7 |
+1
-1
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.19.3 as builder
|
||||
FROM golang:1.19.2 as builder
|
||||
|
||||
WORKDIR /go/src/mikefarah/yq
|
||||
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.19.3
|
||||
FROM golang:1.19.2
|
||||
|
||||
COPY scripts/devtools.sh /opt/devtools.sh
|
||||
|
||||
|
||||
@@ -61,9 +61,6 @@ Take a look at the discussions for [common questions](https://github.com/mikefar
|
||||
### wget
|
||||
Use wget to download, gzipped pre-compiled binaries:
|
||||
|
||||
|
||||
For instance, VERSION=v4.2.0 and BINARY=yq_linux_amd64
|
||||
|
||||
#### Compressed via tar.gz
|
||||
```bash
|
||||
wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY}.tar.gz -O - |\
|
||||
@@ -77,7 +74,9 @@ wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY} -O /
|
||||
chmod +x /usr/bin/yq
|
||||
```
|
||||
|
||||
#### Latest version
|
||||
For instance, VERSION=v4.2.0 and BINARY=yq_linux_amd64
|
||||
|
||||
Use VERSION=latest to always get the latest version:
|
||||
|
||||
```bash
|
||||
wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq &&\
|
||||
|
||||
@@ -48,55 +48,6 @@ EOM
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testOutputYamlRawDefault() {
|
||||
cat >test.yml <<EOL
|
||||
a: "cat"
|
||||
EOL
|
||||
|
||||
X=$(./yq e '.a' test.yml)
|
||||
assertEquals "cat" "$X"
|
||||
|
||||
X=$(./yq ea '.a' test.yml)
|
||||
assertEquals "cat" "$X"
|
||||
}
|
||||
|
||||
testOutputYamlRawOff() {
|
||||
cat >test.yml <<EOL
|
||||
a: "cat"
|
||||
EOL
|
||||
|
||||
X=$(./yq e -r=false '.a' test.yml)
|
||||
assertEquals "\"cat\"" "$X"
|
||||
|
||||
X=$(./yq ea -r=false '.a' test.yml)
|
||||
assertEquals "\"cat\"" "$X"
|
||||
}
|
||||
|
||||
testOutputJsonRaw() {
|
||||
cat >test.yml <<EOL
|
||||
a: cat
|
||||
EOL
|
||||
|
||||
X=$(./yq e -r --output-format=json '.a' test.yml)
|
||||
assertEquals "cat" "$X"
|
||||
|
||||
X=$(./yq ea -r --output-format=json '.a' test.yml)
|
||||
assertEquals "cat" "$X"
|
||||
}
|
||||
|
||||
testOutputJsonDefault() {
|
||||
cat >test.yml <<EOL
|
||||
a: cat
|
||||
EOL
|
||||
|
||||
X=$(./yq e --output-format=json '.a' test.yml)
|
||||
assertEquals "\"cat\"" "$X"
|
||||
|
||||
X=$(./yq ea --output-format=json '.a' test.yml)
|
||||
assertEquals "\"cat\"" "$X"
|
||||
}
|
||||
|
||||
|
||||
testOutputJsonShort() {
|
||||
cat >test.yml <<EOL
|
||||
a: {b: ["cat"]}
|
||||
@@ -121,11 +72,11 @@ EOM
|
||||
|
||||
testOutputProperties() {
|
||||
cat >test.yml <<EOL
|
||||
a: {b: {c: ["cat cat"]}}
|
||||
a: {b: {c: ["cat"]}}
|
||||
EOL
|
||||
|
||||
read -r -d '' expected << EOM
|
||||
a.b.c.0 = cat cat
|
||||
a.b.c.0 = cat
|
||||
EOM
|
||||
|
||||
X=$(./yq e --output-format=props test.yml)
|
||||
@@ -135,30 +86,13 @@ EOM
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testOutputPropertiesDontUnwrap() {
|
||||
cat >test.yml <<EOL
|
||||
a: {b: {c: ["cat cat"]}}
|
||||
EOL
|
||||
|
||||
read -r -d '' expected << EOM
|
||||
a.b.c.0 = "cat cat"
|
||||
EOM
|
||||
|
||||
X=$(./yq e -r=false --output-format=props test.yml)
|
||||
assertEquals "$expected" "$X"
|
||||
|
||||
X=$(./yq ea -r=false --output-format=props test.yml)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
|
||||
testOutputPropertiesShort() {
|
||||
cat >test.yml <<EOL
|
||||
a: {b: {c: ["cat cat"]}}
|
||||
a: {b: {c: ["cat"]}}
|
||||
EOL
|
||||
|
||||
read -r -d '' expected << EOM
|
||||
a.b.c.0 = cat cat
|
||||
a.b.c.0 = cat
|
||||
EOM
|
||||
|
||||
X=$(./yq e -o=p test.yml)
|
||||
|
||||
+1
-3
@@ -1,8 +1,6 @@
|
||||
package cmd
|
||||
|
||||
var unwrapScalarFlag = newUnwrapFlag()
|
||||
|
||||
var unwrapScalar = false
|
||||
var unwrapScalar = true
|
||||
|
||||
var writeInplace = false
|
||||
var outputToJSON = false
|
||||
|
||||
+6
-33
@@ -35,7 +35,7 @@ yq -P sample.json
|
||||
return evaluateSequence(cmd, args)
|
||||
|
||||
},
|
||||
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
PersistentPreRun: func(cmd *cobra.Command, args []string) {
|
||||
cmd.SetOut(cmd.OutOrStdout())
|
||||
|
||||
var format = logging.MustStringFormatter(
|
||||
@@ -52,43 +52,18 @@ yq -P sample.json
|
||||
|
||||
logging.SetBackend(backend)
|
||||
yqlib.InitExpressionParser()
|
||||
|
||||
outputFormatType, err := yqlib.OutputFormatFromString(outputFormat)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
inputFormatType, err := yqlib.InputFormatFromString(inputFormat)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if (inputFormatType == yqlib.XMLInputFormat &&
|
||||
outputFormatType != yqlib.XMLOutputFormat ||
|
||||
inputFormatType != yqlib.XMLInputFormat &&
|
||||
outputFormatType == yqlib.XMLOutputFormat) &&
|
||||
yqlib.ConfiguredXMLPreferences.AttributePrefix == "+@" {
|
||||
yqlib.GetLogger().Warning("The default xml-attribute-prefix has changed in the v4.30 to `+@` to avoid " +
|
||||
if (inputFormat == "x" || inputFormat == "xml") &&
|
||||
outputFormat != "x" && outputFormat != "xml" &&
|
||||
yqlib.ConfiguredXMLPreferences.AttributePrefix == "+" {
|
||||
yqlib.GetLogger().Warning("The default xml-attribute-prefix will change in the v4.30 to `+@` to avoid " +
|
||||
"naming conflicts with the default content name, directive name and proc inst prefix. If you need to keep " +
|
||||
"`+` please set that value explicityly with --xml-attribute-prefix.")
|
||||
}
|
||||
|
||||
if outputFormatType == yqlib.YamlOutputFormat ||
|
||||
outputFormatType == yqlib.PropsOutputFormat {
|
||||
unwrapScalar = true
|
||||
}
|
||||
if unwrapScalarFlag.IsExplicitySet() {
|
||||
unwrapScalar = unwrapScalarFlag.IsSet()
|
||||
}
|
||||
|
||||
//copy preference form global setting
|
||||
yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar
|
||||
|
||||
yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
@@ -119,9 +94,7 @@ yq -P sample.json
|
||||
rootCmd.PersistentFlags().IntVarP(&indent, "indent", "I", 2, "sets indent level for output")
|
||||
rootCmd.Flags().BoolVarP(&version, "version", "V", false, "Print version information and quit")
|
||||
rootCmd.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the file inplace of first file given.")
|
||||
rootCmd.PersistentFlags().VarP(unwrapScalarFlag, "unwrapScalar", "r", "unwrap scalar, print the value with no quotes, colors or comments. Defaults to true for yaml")
|
||||
rootCmd.PersistentFlags().Lookup("unwrapScalar").NoOptDefVal = "true"
|
||||
|
||||
rootCmd.PersistentFlags().BoolVarP(&unwrapScalar, "unwrapScalar", "r", true, "unwrap scalar, print the value with no quotes, colors or comments")
|
||||
rootCmd.PersistentFlags().BoolVarP(&prettyPrint, "prettyPrint", "P", false, "pretty print, shorthand for '... style = \"\"'")
|
||||
rootCmd.PersistentFlags().BoolVarP(&exitStatus, "exit-status", "e", false, "set exit status if there are no matches or null or false is returned")
|
||||
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
type boolFlag interface {
|
||||
pflag.Value
|
||||
IsExplicitySet() bool
|
||||
IsSet() bool
|
||||
}
|
||||
|
||||
type unwrapScalarFlagStrc struct {
|
||||
explicitySet bool
|
||||
value bool
|
||||
}
|
||||
|
||||
func newUnwrapFlag() boolFlag {
|
||||
return &unwrapScalarFlagStrc{value: true}
|
||||
}
|
||||
|
||||
func (f *unwrapScalarFlagStrc) IsExplicitySet() bool {
|
||||
return f.explicitySet
|
||||
}
|
||||
|
||||
func (f *unwrapScalarFlagStrc) IsSet() bool {
|
||||
return f.value
|
||||
}
|
||||
|
||||
func (f *unwrapScalarFlagStrc) String() string {
|
||||
return strconv.FormatBool(f.value)
|
||||
}
|
||||
|
||||
func (f *unwrapScalarFlagStrc) Set(value string) error {
|
||||
|
||||
v, err := strconv.ParseBool(value)
|
||||
f.value = v
|
||||
f.explicitySet = true
|
||||
return err
|
||||
}
|
||||
|
||||
func (*unwrapScalarFlagStrc) Type() string {
|
||||
return "bool"
|
||||
}
|
||||
+1
-1
@@ -98,7 +98,7 @@ func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yq
|
||||
func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder {
|
||||
switch format {
|
||||
case yqlib.JSONOutputFormat:
|
||||
return yqlib.NewJSONEncoder(indent, colorsEnabled, unwrapScalar)
|
||||
return yqlib.NewJSONEncoder(indent, colorsEnabled)
|
||||
case yqlib.PropsOutputFormat:
|
||||
return yqlib.NewPropertiesEncoder(unwrapScalar)
|
||||
case yqlib.CSVOutputFormat:
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@ var (
|
||||
GitDescribe string
|
||||
|
||||
// Version is main version number that is being run at the moment.
|
||||
Version = "4.30.1"
|
||||
Version = "4.28.2"
|
||||
|
||||
// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
|
||||
// then it means that it is a final release. Otherwise, this is a pre-release
|
||||
|
||||
+5
-1
@@ -1 +1,5 @@
|
||||
["foobar", "foobaz", "blarp"]
|
||||
volumes:
|
||||
beehive-conf:
|
||||
driver: local
|
||||
services:
|
||||
- beehive
|
||||
@@ -1,3 +1,3 @@
|
||||
name,numberOfCats,likesApples,height
|
||||
,1,true,168.8
|
||||
Gary,1,true,168.8
|
||||
Samantha's Rabbit,2,false,-188.8
|
||||
|
@@ -13,7 +13,6 @@ require (
|
||||
github.com/magiconair/properties v1.8.6
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
|
||||
github.com/spf13/cobra v1.6.1
|
||||
github.com/spf13/pflag v1.0.5
|
||||
golang.org/x/net v0.0.0-20220708220712-1185a9018129
|
||||
gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
@@ -23,6 +22,7 @@ require (
|
||||
github.com/inconshreveable/mousetrap v1.0.1 // indirect
|
||||
github.com/mattn/go-colorable v0.1.12 // indirect
|
||||
github.com/mattn/go-isatty v0.0.14 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
|
||||
|
||||
@@ -54,12 +54,12 @@ func compare(prefs compareTypePref) func(d *dataTreeNavigator, context Context,
|
||||
}
|
||||
|
||||
func compareDateTime(layout string, prefs compareTypePref, lhs *yaml.Node, rhs *yaml.Node) (bool, error) {
|
||||
lhsTime, err := parseDateTime(layout, lhs.Value)
|
||||
lhsTime, err := time.Parse(layout, lhs.Value)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
rhsTime, err := parseDateTime(layout, rhs.Value)
|
||||
rhsTime, err := time.Parse(layout, rhs.Value)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -81,7 +81,7 @@ func compareScalars(context Context, prefs compareTypePref, lhs *yaml.Node, rhs
|
||||
isDateTime := lhs.Tag == "!!timestamp"
|
||||
// if the lhs is a string, it might be a timestamp in a custom format.
|
||||
if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 {
|
||||
_, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value)
|
||||
_, err := time.Parse(context.GetDateTimeLayout(), lhs.Value)
|
||||
isDateTime = err == nil
|
||||
}
|
||||
if isDateTime {
|
||||
|
||||
+9
-18
@@ -12,9 +12,7 @@ const csvSimple = `name,numberOfCats,likesApples,height
|
||||
Gary,1,true,168.8
|
||||
Samantha's Rabbit,2,false,-188.8
|
||||
`
|
||||
const csvMissing = `name,numberOfCats,likesApples,height
|
||||
,null,,168.8
|
||||
`
|
||||
|
||||
const expectedUpdatedSimpleCsv = `name,numberOfCats,likesApples,height
|
||||
Gary,3,true,168.8
|
||||
Samantha's Rabbit,2,false,-188.8
|
||||
@@ -112,13 +110,6 @@ var csvScenarios = []formatScenario{
|
||||
expected: csvSimpleMissingData,
|
||||
scenarioType: "encode-csv",
|
||||
},
|
||||
{
|
||||
description: "decode csv missing",
|
||||
skipDoc: true,
|
||||
input: csvMissing,
|
||||
expected: csvMissing,
|
||||
scenarioType: "roundtrip-csv",
|
||||
},
|
||||
{
|
||||
description: "Parse CSV into an array of objects",
|
||||
subdescription: "First row is assumed to be the header row.",
|
||||
@@ -145,15 +136,15 @@ var csvScenarios = []formatScenario{
|
||||
func testCSVScenario(t *testing.T, s formatScenario) {
|
||||
switch s.scenarioType {
|
||||
case "encode-csv":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(',')), s.description)
|
||||
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(',')), s.description)
|
||||
case "encode-tsv":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder('\t')), s.description)
|
||||
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder('\t')), s.description)
|
||||
case "decode-csv-object":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(','), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
|
||||
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
|
||||
case "decode-tsv-object":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder('\t'), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
|
||||
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder('\t'), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
|
||||
case "roundtrip-csv":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(','), NewCsvEncoder(',')), s.description)
|
||||
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewCsvEncoder(',')), s.description)
|
||||
default:
|
||||
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
|
||||
}
|
||||
@@ -180,7 +171,7 @@ func documentCSVDecodeObjectScenario(w *bufio.Writer, s formatScenario, formatTy
|
||||
}
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n",
|
||||
mustProcessFormatScenario(s, NewCSVObjectDecoder(separator), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))),
|
||||
processFormatScenario(s, NewCSVObjectDecoder(separator), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -212,7 +203,7 @@ func documentCSVEncodeScenario(w *bufio.Writer, s formatScenario, formatType str
|
||||
}
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
|
||||
mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(separator))),
|
||||
processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(separator))),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -244,7 +235,7 @@ func documentCSVRoundTripScenario(w *bufio.Writer, s formatScenario, formatType
|
||||
}
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
|
||||
mustProcessFormatScenario(s, NewCSVObjectDecoder(separator), NewCsvEncoder(separator))),
|
||||
processFormatScenario(s, NewCSVObjectDecoder(separator), NewCsvEncoder(separator))),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -15,10 +15,10 @@ type formatScenario struct {
|
||||
subdescription string
|
||||
skipDoc bool
|
||||
scenarioType string
|
||||
expectedError string
|
||||
}
|
||||
|
||||
func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) (string, error) {
|
||||
func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) string {
|
||||
|
||||
var output bytes.Buffer
|
||||
writer := bufio.NewWriter(&output)
|
||||
|
||||
@@ -28,7 +28,7 @@ func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) (
|
||||
|
||||
inputs, err := readDocuments(strings.NewReader(s.input), "sample.yml", 0, decoder)
|
||||
if err != nil {
|
||||
return "", err
|
||||
panic(err)
|
||||
}
|
||||
|
||||
expression := s.expression
|
||||
@@ -39,31 +39,22 @@ func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) (
|
||||
exp, err := getExpressionParser().ParseExpression(expression)
|
||||
|
||||
if err != nil {
|
||||
return "", err
|
||||
panic(err)
|
||||
}
|
||||
|
||||
context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: inputs}, exp)
|
||||
|
||||
if err != nil {
|
||||
return "", err
|
||||
panic(err)
|
||||
}
|
||||
|
||||
printer := NewPrinter(encoder, NewSinglePrinterWriter(writer))
|
||||
err = printer.PrintResults(context.MatchingNodes)
|
||||
if err != nil {
|
||||
return "", err
|
||||
panic(err)
|
||||
}
|
||||
writer.Flush()
|
||||
|
||||
return output.String(), nil
|
||||
}
|
||||
|
||||
func mustProcessFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) string {
|
||||
|
||||
result, err := processFormatScenario(s, decoder, encoder)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return result
|
||||
return output.String()
|
||||
|
||||
}
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
# Array to Map
|
||||
|
||||
Use this operator to convert an array to..a map. Skips over null values.
|
||||
|
||||
Behind the scenes, this is implemented using reduce:
|
||||
|
||||
```
|
||||
(.[] | select(. != null) ) as $i ireduce({}; .[$i | key] = $i)
|
||||
```
|
||||
|
||||
## Simple example
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
cool:
|
||||
- null
|
||||
- null
|
||||
- hello
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.cool |= array_to_map' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
cool:
|
||||
2: hello
|
||||
```
|
||||
|
||||
@@ -1,14 +1,6 @@
|
||||
# Contains
|
||||
|
||||
This returns `true` if the context contains the passed in parameter, and false otherwise. For arrays, this will return true if the passed in array is contained within the array. For strings, it will return true if the string is a substring.
|
||||
|
||||
{% hint style="warning" %}
|
||||
|
||||
_Note_ that, just like jq, when checking if an array of strings `contains` another, this will use `contains` and _not_ equals to check each string. This means an array like `contains(["cat"])` will return true for an array `["cats"]`.
|
||||
|
||||
See the "Array has a subset array" example below on how to check for a subset.
|
||||
|
||||
{% endhint %}
|
||||
This returns `true` if the context contains the passed in parameter, and false otherwise.
|
||||
|
||||
## Array contains array
|
||||
Array is equal or subset of
|
||||
@@ -28,24 +20,6 @@ will output
|
||||
true
|
||||
```
|
||||
|
||||
## Array has a subset array
|
||||
Subtract the superset array from the subset, if there's anything left, it's not a subset
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
- foobar
|
||||
- foobaz
|
||||
- blarp
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '["baz", "bar"] - . | length == 0' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
false
|
||||
```
|
||||
|
||||
## Object included in array
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
|
||||
@@ -60,7 +60,7 @@ a: 2001-12-15
|
||||
## Format: get the day of the week
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
a: 2001-12-15
|
||||
a: 2001-12-15T02:59:43.1Z
|
||||
```
|
||||
then
|
||||
```bash
|
||||
|
||||
@@ -337,7 +337,7 @@ Given a sample.yml file of:
|
||||
a:
|
||||
cool:
|
||||
foo: bar
|
||||
+@id: hi
|
||||
+id: hi
|
||||
```
|
||||
then
|
||||
```bash
|
||||
@@ -357,7 +357,7 @@ Given a sample.yml file of:
|
||||
a:
|
||||
cool:
|
||||
foo: bar
|
||||
+@id: hi
|
||||
+id: hi
|
||||
```
|
||||
then
|
||||
```bash
|
||||
@@ -375,7 +375,7 @@ Given a sample.yml file of:
|
||||
a:
|
||||
cool:
|
||||
foo: bar
|
||||
+@id: hi
|
||||
+id: hi
|
||||
```
|
||||
then
|
||||
```bash
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
# Array to Map
|
||||
|
||||
Use this operator to convert an array to..a map. Skips over null values.
|
||||
|
||||
Behind the scenes, this is implemented using reduce:
|
||||
|
||||
```
|
||||
(.[] | select(. != null) ) as $i ireduce({}; .[$i | key] = $i)
|
||||
```
|
||||
@@ -1,11 +1,3 @@
|
||||
# Contains
|
||||
|
||||
This returns `true` if the context contains the passed in parameter, and false otherwise. For arrays, this will return true if the passed in array is contained within the array. For strings, it will return true if the string is a substring.
|
||||
|
||||
{% hint style="warning" %}
|
||||
|
||||
_Note_ that, just like jq, when checking if an array of strings `contains` another, this will use `contains` and _not_ equals to check each string. This means an array like `contains(["cat"])` will return true for an array `["cats"]`.
|
||||
|
||||
See the "Array has a subset array" example below on how to check for a subset.
|
||||
|
||||
{% endhint %}
|
||||
This returns `true` if the context contains the passed in parameter, and false otherwise.
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
# Slice Array
|
||||
|
||||
The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array.
|
||||
|
||||
You may leave out the first or second number, which will will refer to the start or end of the array respectively.
|
||||
@@ -122,32 +122,6 @@ a:
|
||||
b: things
|
||||
```
|
||||
|
||||
## Set path to prune deep paths
|
||||
Like pick but recursive. This uses `ireduce` to deeply set the selected paths into an empty object,
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
parentA: bob
|
||||
parentB:
|
||||
child1: i am child1
|
||||
child2: i am child2
|
||||
parentC:
|
||||
child1: me child1
|
||||
child2: me child2
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '(.parentB.child2, .parentC.child1) as $i
|
||||
ireduce({}; setpath($i | path; $i))' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
parentB:
|
||||
child2: i am child2
|
||||
parentC:
|
||||
child1: me child1
|
||||
```
|
||||
|
||||
## Set array path
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
|
||||
@@ -1,8 +1,3 @@
|
||||
# Slice Array
|
||||
|
||||
The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array.
|
||||
|
||||
You may leave out the first or second number, which will will refer to the start or end of the array respectively.
|
||||
|
||||
## Slicing arrays
|
||||
Given a sample.yml file of:
|
||||
@@ -80,26 +75,3 @@ will output
|
||||
- frog
|
||||
```
|
||||
|
||||
## Inserting into the middle of an array
|
||||
using an expression to find the index
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
- cat
|
||||
- dog
|
||||
- frog
|
||||
- cow
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '(.[] | select(. == "dog") | key + 1) as $pos | .[0:($pos)] + ["rabbit"] + .[$pos:]' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
- cat
|
||||
- dog
|
||||
- rabbit
|
||||
- frog
|
||||
- cow
|
||||
```
|
||||
|
||||
|
||||
@@ -135,24 +135,6 @@ will output
|
||||
- a: 100
|
||||
```
|
||||
|
||||
## Sort by custom date field
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
- a: 12-Jun-2011
|
||||
- a: 23-Dec-2010
|
||||
- a: 10-Aug-2011
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq 'with_dtf("02-Jan-2006"; sort_by(.a))' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
- a: 23-Dec-2010
|
||||
- a: 12-Jun-2011
|
||||
- a: 10-Aug-2011
|
||||
```
|
||||
|
||||
## Sort, nulls come first
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
|
||||
@@ -149,23 +149,6 @@ person:
|
||||
- pizza
|
||||
```
|
||||
|
||||
## Decode properties - array should be a map
|
||||
If you have a numeric map key in your property files, use array_to_map to convert them to maps.
|
||||
|
||||
Given a sample.properties file of:
|
||||
```properties
|
||||
things.10 = mike
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq -p=props '.things |= array_to_map' sample.properties
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
things:
|
||||
10: mike
|
||||
```
|
||||
|
||||
## Roundtrip
|
||||
Given a sample.properties file of:
|
||||
```properties
|
||||
|
||||
@@ -128,7 +128,7 @@ will output
|
||||
```yaml
|
||||
+p_xml: version="1.0" encoding="UTF-8"
|
||||
cat:
|
||||
+@legs: "4"
|
||||
+legs: "4"
|
||||
legs: "7"
|
||||
```
|
||||
|
||||
@@ -149,7 +149,7 @@ will output
|
||||
+p_xml: version="1.0" encoding="UTF-8"
|
||||
cat:
|
||||
+content: meow
|
||||
+@legs: "4"
|
||||
+legs: "4"
|
||||
```
|
||||
|
||||
## Parse xml: custom dtd
|
||||
@@ -347,7 +347,7 @@ Fields with the matching xml-attribute-prefix are assumed to be attributes.
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
cat:
|
||||
+@name: tiger
|
||||
+name: tiger
|
||||
meows: true
|
||||
|
||||
```
|
||||
@@ -368,7 +368,7 @@ Fields with the matching xml-content-name is assumed to be content.
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
cat:
|
||||
+@name: tiger
|
||||
+name: tiger
|
||||
+content: cool
|
||||
|
||||
```
|
||||
|
||||
@@ -45,7 +45,7 @@ func (o *orderedMap) UnmarshalJSON(data []byte) error {
|
||||
|
||||
// cycle through k/v
|
||||
var tok json.Token
|
||||
for tok, err = dec.Token(); err == nil; tok, err = dec.Token() {
|
||||
for tok, err = dec.Token(); !errors.Is(err, io.EOF); tok, err = dec.Token() {
|
||||
// we can expect two types: string or Delim. Delim automatically means
|
||||
// that it is the closing bracket of the object, whereas string means
|
||||
// that there is another key.
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
type jsonEncoder struct {
|
||||
indentString string
|
||||
colorise bool
|
||||
UnwrapScalar bool
|
||||
}
|
||||
|
||||
func mapKeysToStrings(node *yaml.Node) {
|
||||
@@ -29,14 +28,14 @@ func mapKeysToStrings(node *yaml.Node) {
|
||||
}
|
||||
}
|
||||
|
||||
func NewJSONEncoder(indent int, colorise bool, unwrapScalar bool) Encoder {
|
||||
func NewJSONEncoder(indent int, colorise bool) Encoder {
|
||||
var indentString = ""
|
||||
|
||||
for index := 0; index < indent; index++ {
|
||||
indentString = indentString + " "
|
||||
}
|
||||
|
||||
return &jsonEncoder{indentString, colorise, unwrapScalar}
|
||||
return &jsonEncoder{indentString, colorise}
|
||||
}
|
||||
|
||||
func (je *jsonEncoder) CanHandleAliases() bool {
|
||||
@@ -53,10 +52,6 @@ func (je *jsonEncoder) PrintLeadingContent(writer io.Writer, content string) err
|
||||
|
||||
func (je *jsonEncoder) Encode(writer io.Writer, node *yaml.Node) error {
|
||||
|
||||
if node.Kind == yaml.ScalarNode && je.UnwrapScalar {
|
||||
return writeString(writer, node.Value+"\n")
|
||||
}
|
||||
|
||||
destination := writer
|
||||
tempBuffer := bytes.NewBuffer(nil)
|
||||
if je.colorise {
|
||||
|
||||
@@ -13,7 +13,7 @@ func yamlToJSON(sampleYaml string, indent int) string {
|
||||
var output bytes.Buffer
|
||||
writer := bufio.NewWriter(&output)
|
||||
|
||||
var jsonEncoder = NewJSONEncoder(indent, false, false)
|
||||
var jsonEncoder = NewJSONEncoder(indent, false)
|
||||
inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
|
||||
@@ -65,11 +65,7 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
unwrappedNode := unwrapDoc(node)
|
||||
if unwrappedNode.Kind != yaml.MappingNode {
|
||||
return fmt.Errorf("cannot encode %v to XML - only maps can be encoded", unwrappedNode.Tag)
|
||||
}
|
||||
err = e.encodeTopLevelMap(encoder, unwrappedNode)
|
||||
err = e.encodeTopLevelMap(encoder, unwrapDoc(node))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+8
-21
@@ -84,13 +84,6 @@ var jsonScenarios = []formatScenario{
|
||||
input: `{"cat": "meow"}`,
|
||||
expected: "D0, P[], (!!map)::cat: meow\n",
|
||||
},
|
||||
{
|
||||
description: "bad json",
|
||||
skipDoc: true,
|
||||
input: `{"a": 1 "b": 2}`,
|
||||
expectedError: `bad file 'sample.yml': invalid character '"' after object key:value pair`,
|
||||
scenarioType: "decode-error",
|
||||
},
|
||||
{
|
||||
description: "Parse json: complex",
|
||||
subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output",
|
||||
@@ -244,7 +237,7 @@ func documentRoundtripNdJsonScenario(w *bufio.Writer, s formatScenario, indent i
|
||||
|
||||
writeOrPanic(w, "will output\n")
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(indent, false, false))))
|
||||
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(indent, false))))
|
||||
}
|
||||
|
||||
func documentDecodeNdJsonScenario(w *bufio.Writer, s formatScenario) {
|
||||
@@ -269,7 +262,7 @@ func documentDecodeNdJsonScenario(w *bufio.Writer, s formatScenario) {
|
||||
|
||||
writeOrPanic(w, "will output\n")
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))))
|
||||
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))))
|
||||
}
|
||||
|
||||
func decodeJSON(t *testing.T, jsonString string) *CandidateNode {
|
||||
@@ -300,23 +293,17 @@ func decodeJSON(t *testing.T, jsonString string) *CandidateNode {
|
||||
func testJSONScenario(t *testing.T, s formatScenario) {
|
||||
switch s.scenarioType {
|
||||
case "encode", "decode":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false, false)), s.description)
|
||||
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false)), s.description)
|
||||
case "":
|
||||
var actual = resultToString(t, decodeJSON(t, s.input))
|
||||
test.AssertResultWithContext(t, s.expected, actual, s.description)
|
||||
case "decode-ndjson":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
|
||||
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
|
||||
case "roundtrip-ndjson":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(0, false, false)), s.description)
|
||||
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(0, false)), s.description)
|
||||
case "roundtrip-multi":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(2, false, false)), s.description)
|
||||
case "decode-error":
|
||||
result, err := processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(2, false, false))
|
||||
if err == nil {
|
||||
t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result)
|
||||
} else {
|
||||
test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description)
|
||||
}
|
||||
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(2, false)), s.description)
|
||||
|
||||
default:
|
||||
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
|
||||
}
|
||||
@@ -398,7 +385,7 @@ func documentJSONEncodeScenario(w *bufio.Writer, s formatScenario) {
|
||||
}
|
||||
writeOrPanic(w, "will output\n")
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false, false))))
|
||||
writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false))))
|
||||
}
|
||||
|
||||
func TestJSONScenarios(t *testing.T) {
|
||||
|
||||
+3
-28
@@ -97,10 +97,6 @@ func postProcessTokens(tokens []*token) []*token {
|
||||
return postProcessedTokens
|
||||
}
|
||||
|
||||
func tokenIsOpType(token *token, opType *operationType) bool {
|
||||
return token.TokenType == operationToken && token.Operation.OperationType == opType
|
||||
}
|
||||
|
||||
func handleToken(tokens []*token, index int, postProcessedTokens []*token) (tokensAccum []*token, skipNextToken bool) {
|
||||
skipNextToken = false
|
||||
currentToken := tokens[index]
|
||||
@@ -124,18 +120,9 @@ func handleToken(tokens []*token, index int, postProcessedTokens []*token) (toke
|
||||
|
||||
}
|
||||
|
||||
if tokenIsOpType(currentToken, createMapOpType) {
|
||||
log.Debugf("tokenIsOpType: createMapOpType")
|
||||
// check the previous token is '[', means we are slice, but dont have a first number
|
||||
if tokens[index-1].TokenType == traverseArrayCollect {
|
||||
log.Debugf("previous token is : traverseArrayOpType")
|
||||
// need to put the number 0 before this token, as that is implied
|
||||
postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: createValueOperation(0, "0")})
|
||||
}
|
||||
}
|
||||
|
||||
if index != len(tokens)-1 && currentToken.AssignOperation != nil &&
|
||||
tokenIsOpType(tokens[index+1], assignOpType) {
|
||||
tokens[index+1].TokenType == operationToken &&
|
||||
tokens[index+1].Operation.OperationType == assignOpType {
|
||||
log.Debug(" its an update assign")
|
||||
currentToken.Operation = currentToken.AssignOperation
|
||||
currentToken.Operation.UpdateAssign = tokens[index+1].Operation.UpdateAssign
|
||||
@@ -145,17 +132,6 @@ func handleToken(tokens []*token, index int, postProcessedTokens []*token) (toke
|
||||
log.Debug(" adding token to the fixed list")
|
||||
postProcessedTokens = append(postProcessedTokens, currentToken)
|
||||
|
||||
if tokenIsOpType(currentToken, createMapOpType) {
|
||||
log.Debugf("tokenIsOpType: createMapOpType")
|
||||
// check the next token is ']', means we are slice, but dont have a second number
|
||||
if index != len(tokens)-1 && tokens[index+1].TokenType == closeCollect {
|
||||
log.Debugf("next token is : closeCollect")
|
||||
// need to put the number 0 before this token, as that is implied
|
||||
lengthOp := &Operation{OperationType: lengthOpType}
|
||||
postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: lengthOp})
|
||||
}
|
||||
}
|
||||
|
||||
if index != len(tokens)-1 &&
|
||||
((currentToken.TokenType == openCollect && tokens[index+1].TokenType == closeCollect) ||
|
||||
(currentToken.TokenType == openCollectObject && tokens[index+1].TokenType == closeCollectObject)) {
|
||||
@@ -165,8 +141,7 @@ func handleToken(tokens []*token, index int, postProcessedTokens []*token) (toke
|
||||
}
|
||||
|
||||
if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
|
||||
|
||||
(tokenIsOpType(tokens[index+1], traversePathOpType) ||
|
||||
((tokens[index+1].TokenType == operationToken && (tokens[index+1].Operation.OperationType == traversePathOpType)) ||
|
||||
(tokens[index+1].TokenType == traverseArrayCollect)) {
|
||||
log.Debug(" adding pipe because the next thing is traverse")
|
||||
op := &Operation{OperationType: shortPipeOpType, Value: "PIPE", StringValue: "."}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
@@ -12,6 +13,10 @@ var participleYqRules = []*participleYqRule{
|
||||
{"HEAD_COMMENT", `head_?comment|headComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{HeadComment: true}), 0},
|
||||
{"FOOT_COMMENT", `foot_?comment|footComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{FootComment: true}), 0},
|
||||
|
||||
{"SliceArray", `\.\[-?[0-9]+:-?[0-9]+\]`, sliceArrayTwoNumbers(), 0},
|
||||
{"SliceArraySecond", `\.\[\:-?[0-9]+\]`, sliceArraySecondNumberOnly(), 0},
|
||||
{"SliceArrayFirst", `\.\[-?[0-9]+\:\]`, sliceArrayFirstNumberOnly(), 0},
|
||||
|
||||
{"OpenBracket", `\(`, literalToken(openBracket, false), 0},
|
||||
{"CloseBracket", `\)`, literalToken(closeBracket, true), 0},
|
||||
{"OpenTraverseArrayCollect", `\.\[`, literalToken(traverseArrayCollect, false), 0},
|
||||
@@ -50,8 +55,6 @@ var participleYqRules = []*participleYqRule{
|
||||
simpleOp("sortKeys", sortKeysOpType),
|
||||
simpleOp("sort_?keys", sortKeysOpType),
|
||||
|
||||
{"ArrayToMap", "array_?to_?map", expressionOpToken(`(.[] | select(. != null) ) as $i ireduce({}; .[$i | key] = $i)`), 0},
|
||||
|
||||
{"YamlEncodeWithIndent", `to_?yaml\([0-9]+\)`, encodeParseIndent(YamlOutputFormat), 0},
|
||||
{"XMLEncodeWithIndent", `to_?xml\([0-9]+\)`, encodeParseIndent(XMLOutputFormat), 0},
|
||||
{"JSONEncodeWithIndent", `to_?json\([0-9]+\)`, encodeParseIndent(JSONOutputFormat), 0},
|
||||
@@ -288,14 +291,6 @@ func opTokenWithPrefs(opType *operationType, assignOpType *operationType, prefer
|
||||
}
|
||||
}
|
||||
|
||||
func expressionOpToken(expression string) yqAction {
|
||||
return func(rawToken lexer.Token) (*token, error) {
|
||||
prefs := expressionOpPreferences{expression: expression}
|
||||
expressionOp := &Operation{OperationType: expressionOpType, Preferences: prefs}
|
||||
return &token{TokenType: operationToken, Operation: expressionOp}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func flattenWithDepth() yqAction {
|
||||
return func(rawToken lexer.Token) (*token, error) {
|
||||
value := rawToken.Value
|
||||
@@ -310,6 +305,84 @@ func flattenWithDepth() yqAction {
|
||||
}
|
||||
}
|
||||
|
||||
func sliceArrayTwoNumbers() yqAction {
|
||||
return func(rawToken lexer.Token) (*token, error) {
|
||||
value := rawToken.Value
|
||||
sliceArrayNumbers := regexp.MustCompile(`\.\[(-?[0-9]+)\:(-?[0-9]+)\]`)
|
||||
matches := sliceArrayNumbers.FindStringSubmatch(value)
|
||||
log.Debug("sliceArrayTwoNumbers value: %v", value)
|
||||
log.Debug("Matches: %v", matches)
|
||||
|
||||
firstNumber, err := parseInt(matches[1])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
secondNumber, err := parseInt(matches[2])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
prefs := sliceArrayPreferences{
|
||||
firstNumber: firstNumber,
|
||||
secondNumber: secondNumber,
|
||||
secondNumberDefined: true,
|
||||
}
|
||||
log.Debug("%v", prefs)
|
||||
|
||||
op := &Operation{OperationType: sliceArrayOpType, Value: sliceArrayOpType.Type, StringValue: value, Preferences: prefs}
|
||||
return &token{TokenType: operationToken, Operation: op}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func sliceArraySecondNumberOnly() yqAction {
|
||||
return func(rawToken lexer.Token) (*token, error) {
|
||||
value := rawToken.Value
|
||||
sliceArrayNumbers := regexp.MustCompile(`\.\[\:(-?[0-9]+)\]`)
|
||||
matches := sliceArrayNumbers.FindStringSubmatch(value)
|
||||
log.Debug("sliceArraySecondNumberOnly value: %v", value)
|
||||
log.Debug("Matches: %v", matches)
|
||||
|
||||
secondNumber, err := parseInt(matches[1])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
prefs := sliceArrayPreferences{
|
||||
firstNumber: 0,
|
||||
secondNumber: secondNumber,
|
||||
secondNumberDefined: true,
|
||||
}
|
||||
log.Debug("%v", prefs)
|
||||
|
||||
op := &Operation{OperationType: sliceArrayOpType, Value: sliceArrayOpType.Type, StringValue: value, Preferences: prefs}
|
||||
return &token{TokenType: operationToken, Operation: op}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func sliceArrayFirstNumberOnly() yqAction {
|
||||
return func(rawToken lexer.Token) (*token, error) {
|
||||
value := rawToken.Value
|
||||
sliceArrayNumbers := regexp.MustCompile(`\.\[(-?[0-9]+)\:\]`)
|
||||
matches := sliceArrayNumbers.FindStringSubmatch(value)
|
||||
log.Debug("sliceArrayFirstNumberOnly value: %v", value)
|
||||
log.Debug("Matches: %v", matches)
|
||||
|
||||
firstNumber, err := parseInt(matches[1])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
prefs := sliceArrayPreferences{
|
||||
firstNumber: firstNumber,
|
||||
secondNumberDefined: false,
|
||||
}
|
||||
log.Debug("%v", prefs)
|
||||
|
||||
op := &Operation{OperationType: sliceArrayOpType, Value: sliceArrayOpType.Type, StringValue: value, Preferences: prefs}
|
||||
return &token{TokenType: operationToken, Operation: op}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func assignAllCommentsOp(updateAssign bool) yqAction {
|
||||
return func(rawToken lexer.Token) (*token, error) {
|
||||
log.Debug("assignAllCommentsOp %v", rawToken.Value)
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
|
||||
"github.com/alecthomas/repr"
|
||||
"github.com/mikefarah/yq/v4/test"
|
||||
yaml "gopkg.in/yaml.v3"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type participleLexerScenario struct {
|
||||
@@ -14,125 +14,60 @@ type participleLexerScenario struct {
|
||||
}
|
||||
|
||||
var participleLexerScenarios = []participleLexerScenario{
|
||||
{
|
||||
expression: ".[1:3]",
|
||||
tokens: []*token{
|
||||
{
|
||||
TokenType: operationToken,
|
||||
Operation: &Operation{
|
||||
OperationType: sliceArrayOpType,
|
||||
Value: "SLICE",
|
||||
StringValue: ".[1:3]",
|
||||
Preferences: sliceArrayPreferences{firstNumber: 1, secondNumber: 3, secondNumberDefined: true},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
expression: ".[:3]",
|
||||
tokens: []*token{
|
||||
{
|
||||
TokenType: operationToken,
|
||||
Operation: &Operation{
|
||||
OperationType: selfReferenceOpType,
|
||||
StringValue: "SELF",
|
||||
OperationType: sliceArrayOpType,
|
||||
Value: "SLICE",
|
||||
StringValue: ".[:3]",
|
||||
Preferences: sliceArrayPreferences{firstNumber: 0, secondNumber: 3, secondNumberDefined: true},
|
||||
},
|
||||
},
|
||||
{
|
||||
TokenType: operationToken,
|
||||
Operation: &Operation{
|
||||
OperationType: traverseArrayOpType,
|
||||
StringValue: "TRAVERSE_ARRAY",
|
||||
},
|
||||
},
|
||||
{
|
||||
TokenType: openCollect,
|
||||
},
|
||||
{
|
||||
TokenType: operationToken,
|
||||
Operation: &Operation{
|
||||
OperationType: valueOpType,
|
||||
Value: 0,
|
||||
StringValue: "0",
|
||||
CandidateNode: &CandidateNode{
|
||||
Node: &yaml.Node{
|
||||
Kind: yaml.ScalarNode,
|
||||
Tag: "!!int",
|
||||
Value: "0",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
TokenType: operationToken,
|
||||
Operation: &Operation{
|
||||
OperationType: createMapOpType,
|
||||
Value: "CREATE_MAP",
|
||||
StringValue: ":",
|
||||
},
|
||||
},
|
||||
{
|
||||
TokenType: operationToken,
|
||||
Operation: &Operation{
|
||||
OperationType: valueOpType,
|
||||
Value: 3,
|
||||
StringValue: "3",
|
||||
CandidateNode: &CandidateNode{
|
||||
Node: &yaml.Node{
|
||||
Kind: yaml.Kind(8),
|
||||
Tag: "!!int",
|
||||
Value: "3",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
TokenType: closeCollect,
|
||||
CheckForPostTraverse: true,
|
||||
Match: "]",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
expression: ".[-2:]",
|
||||
expression: ".[1:]",
|
||||
tokens: []*token{
|
||||
{
|
||||
TokenType: operationToken,
|
||||
Operation: &Operation{
|
||||
OperationType: selfReferenceOpType,
|
||||
StringValue: "SELF",
|
||||
OperationType: sliceArrayOpType,
|
||||
Value: "SLICE",
|
||||
StringValue: ".[1:]",
|
||||
Preferences: sliceArrayPreferences{firstNumber: 1, secondNumber: 0, secondNumberDefined: false},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
expression: ".[-100:-54]",
|
||||
tokens: []*token{
|
||||
{
|
||||
TokenType: operationToken,
|
||||
Operation: &Operation{
|
||||
OperationType: traverseArrayOpType,
|
||||
StringValue: "TRAVERSE_ARRAY",
|
||||
OperationType: sliceArrayOpType,
|
||||
Value: "SLICE",
|
||||
StringValue: ".[-100:-54]",
|
||||
Preferences: sliceArrayPreferences{firstNumber: -100, secondNumber: -54, secondNumberDefined: true},
|
||||
},
|
||||
},
|
||||
{
|
||||
TokenType: openCollect,
|
||||
},
|
||||
{
|
||||
TokenType: operationToken,
|
||||
Operation: &Operation{
|
||||
OperationType: valueOpType,
|
||||
Value: -2,
|
||||
StringValue: "-2",
|
||||
CandidateNode: &CandidateNode{
|
||||
Node: &yaml.Node{
|
||||
Kind: yaml.ScalarNode,
|
||||
Tag: "!!int",
|
||||
Value: "-2",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
TokenType: operationToken,
|
||||
Operation: &Operation{
|
||||
OperationType: createMapOpType,
|
||||
Value: "CREATE_MAP",
|
||||
StringValue: ":",
|
||||
},
|
||||
},
|
||||
{
|
||||
TokenType: operationToken,
|
||||
Operation: &Operation{
|
||||
OperationType: lengthOpType,
|
||||
},
|
||||
},
|
||||
{
|
||||
TokenType: closeCollect,
|
||||
CheckForPostTraverse: true,
|
||||
Match: "]",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
+1
-11
@@ -80,9 +80,8 @@ var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Ha
|
||||
var lineOpType = &operationType{Type: "LINE", NumArgs: 0, Precedence: 50, Handler: lineOperator}
|
||||
var columnOpType = &operationType{Type: "LINE", NumArgs: 0, Precedence: 50, Handler: columnOperator}
|
||||
|
||||
var expressionOpType = &operationType{Type: "EXP", NumArgs: 0, Precedence: 50, Handler: expressionOperator}
|
||||
|
||||
var collectOpType = &operationType{Type: "COLLECT", NumArgs: 1, Precedence: 50, Handler: collectOperator}
|
||||
var sliceArrayOpType = &operationType{Type: "SLICE", NumArgs: 0, Precedence: 50, Handler: sliceArrayOperator}
|
||||
var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 50, Handler: mapOperator}
|
||||
var errorOpType = &operationType{Type: "ERROR", NumArgs: 1, Precedence: 50, Handler: errorOperator}
|
||||
var pickOpType = &operationType{Type: "PICK", NumArgs: 1, Precedence: 50, Handler: pickOperator}
|
||||
@@ -250,21 +249,12 @@ func guessTagFromCustomType(node *yaml.Node) string {
|
||||
}
|
||||
|
||||
func parseSnippet(value string) (*yaml.Node, error) {
|
||||
if value == "" {
|
||||
return &yaml.Node{
|
||||
Kind: yaml.ScalarNode,
|
||||
Tag: "!!null",
|
||||
}, nil
|
||||
}
|
||||
decoder := NewYamlDecoder(ConfiguredYamlPreferences)
|
||||
err := decoder.Init(strings.NewReader(value))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
parsedNode, err := decoder.Decode()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(parsedNode.Node.Content) == 0 {
|
||||
return nil, fmt.Errorf("bad data")
|
||||
}
|
||||
|
||||
+1
-95
@@ -1,11 +1,6 @@
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/mikefarah/yq/v4/test"
|
||||
yaml "gopkg.in/yaml.v3"
|
||||
)
|
||||
import "testing"
|
||||
|
||||
func TestGetLogger(t *testing.T) {
|
||||
l := GetLogger()
|
||||
@@ -13,92 +8,3 @@ func TestGetLogger(t *testing.T) {
|
||||
t.Fatal("GetLogger should return the yq logger instance, not a copy")
|
||||
}
|
||||
}
|
||||
|
||||
type parseSnippetScenario struct {
|
||||
snippet string
|
||||
expected *yaml.Node
|
||||
expectedError string
|
||||
}
|
||||
|
||||
var parseSnippetScenarios = []parseSnippetScenario{
|
||||
{
|
||||
snippet: ":",
|
||||
expectedError: "yaml: did not find expected key",
|
||||
},
|
||||
{
|
||||
snippet: "",
|
||||
expected: &yaml.Node{
|
||||
Kind: yaml.ScalarNode,
|
||||
Tag: "!!null",
|
||||
},
|
||||
},
|
||||
{
|
||||
snippet: "null",
|
||||
expected: &yaml.Node{
|
||||
Kind: yaml.ScalarNode,
|
||||
Tag: "!!null",
|
||||
Value: "null",
|
||||
Line: 1,
|
||||
Column: 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
snippet: "3",
|
||||
expected: &yaml.Node{
|
||||
Kind: yaml.ScalarNode,
|
||||
Tag: "!!int",
|
||||
Value: "3",
|
||||
Line: 1,
|
||||
Column: 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
snippet: "cat",
|
||||
expected: &yaml.Node{
|
||||
Kind: yaml.ScalarNode,
|
||||
Tag: "!!str",
|
||||
Value: "cat",
|
||||
Line: 1,
|
||||
Column: 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
snippet: "3.1",
|
||||
expected: &yaml.Node{
|
||||
Kind: yaml.ScalarNode,
|
||||
Tag: "!!float",
|
||||
Value: "3.1",
|
||||
Line: 1,
|
||||
Column: 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
snippet: "true",
|
||||
expected: &yaml.Node{
|
||||
Kind: yaml.ScalarNode,
|
||||
Tag: "!!bool",
|
||||
Value: "true",
|
||||
Line: 1,
|
||||
Column: 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func TestParseSnippet(t *testing.T) {
|
||||
for _, tt := range parseSnippetScenarios {
|
||||
actual, err := parseSnippet(tt.snippet)
|
||||
if tt.expectedError != "" {
|
||||
if err == nil {
|
||||
t.Errorf("Expected error '%v' but it worked!", tt.expectedError)
|
||||
} else {
|
||||
test.AssertResultComplexWithContext(t, tt.expectedError, err.Error(), tt.snippet)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Error(tt.snippet)
|
||||
t.Error(err)
|
||||
}
|
||||
test.AssertResultComplexWithContext(t, tt.expected, actual, tt.snippet)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -98,7 +98,7 @@ func addScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs *yam
|
||||
|
||||
// if the lhs is a string, it might be a timestamp in a custom format.
|
||||
if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 {
|
||||
_, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value)
|
||||
_, err := time.Parse(context.GetDateTimeLayout(), lhs.Value)
|
||||
isDateTime = err == nil
|
||||
}
|
||||
|
||||
@@ -152,7 +152,7 @@ func addDateTimes(layout string, target *CandidateNode, lhs *yaml.Node, rhs *yam
|
||||
return fmt.Errorf("unable to parse duration [%v]: %w", rhs.Value, err)
|
||||
}
|
||||
|
||||
currentTime, err := parseDateTime(layout, lhs.Value)
|
||||
currentTime, err := time.Parse(layout, lhs.Value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -252,15 +252,6 @@ var addOperatorScenarios = []expressionScenario{
|
||||
"D0, P[], (doc)::a: 2021-01-01T03:10:00Z\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Date addition -date only",
|
||||
skipDoc: true,
|
||||
document: `a: 2021-01-01`,
|
||||
expression: `.a += "24h"`,
|
||||
expected: []string{
|
||||
"D0, P[], (doc)::a: 2021-01-02T00:00:00Z\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Date addition - custom format",
|
||||
subdescription: "You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",
|
||||
|
||||
@@ -2,7 +2,6 @@ package yqlib
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"fmt"
|
||||
|
||||
yaml "gopkg.in/yaml.v3"
|
||||
)
|
||||
@@ -245,9 +244,6 @@ func applyAlias(node *yaml.Node, alias *yaml.Node, aliasIndex int, newContent Co
|
||||
if alias == nil {
|
||||
return nil
|
||||
}
|
||||
if alias.Kind != yaml.MappingNode {
|
||||
return fmt.Errorf("merge anchor only supports maps, got %v instead", alias.Tag)
|
||||
}
|
||||
for index := 0; index < len(alias.Content); index = index + 2 {
|
||||
keyNode := alias.Content[index]
|
||||
log.Debugf("applying alias key %v", keyNode.Value)
|
||||
|
||||
@@ -36,13 +36,6 @@ thingTwo:
|
||||
`
|
||||
|
||||
var anchorOperatorScenarios = []expressionScenario{
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "merge anchor not map",
|
||||
document: "a: &a\n - 0\nc:\n <<: [*a]\n",
|
||||
expectedError: "merge anchor only supports maps, got !!seq instead",
|
||||
expression: "explode(.)",
|
||||
},
|
||||
{
|
||||
description: "Merge one map",
|
||||
subdescription: "see https://yaml.org/type/merge.html",
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
var arrayToMapScenarios = []expressionScenario{
|
||||
{
|
||||
description: "Simple example",
|
||||
document: `cool: [null, null, hello]`,
|
||||
expression: `.cool |= array_to_map`,
|
||||
expected: []string{
|
||||
"D0, P[], (doc)::cool:\n 2: hello\n",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func TestArrayToMapScenarios(t *testing.T) {
|
||||
for _, tt := range arrayToMapScenarios {
|
||||
testScenario(t, &tt)
|
||||
}
|
||||
documentOperatorScenarios(t, "array-to-map", arrayToMapScenarios)
|
||||
}
|
||||
@@ -33,15 +33,6 @@ var containsOperatorScenarios = []expressionScenario{
|
||||
"D0, P[], (!!bool)::true\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Array has a subset array",
|
||||
subdescription: "Subtract the superset array from the subset, if there's anything left, it's not a subset",
|
||||
document: `["foobar", "foobaz", "blarp"]`,
|
||||
expression: `["baz", "bar"] - . | length == 0`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!bool)::false\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
expression: `["dog", "cat", "giraffe"] | contains(["camel"])`,
|
||||
|
||||
@@ -50,17 +50,6 @@ func nowOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode
|
||||
|
||||
}
|
||||
|
||||
func parseDateTime(layout string, datestring string) (time.Time, error) {
|
||||
|
||||
parsedTime, err := time.Parse(layout, datestring)
|
||||
if err != nil && layout == time.RFC3339 {
|
||||
// try parsing the date time with only the date
|
||||
return time.Parse("2006-01-02", datestring)
|
||||
}
|
||||
return parsedTime, err
|
||||
|
||||
}
|
||||
|
||||
func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
|
||||
format, err := getStringParamter("format", d, context, expressionNode.RHS)
|
||||
layout := context.GetDateTimeLayout()
|
||||
@@ -73,7 +62,7 @@ func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *Expre
|
||||
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
|
||||
candidate := el.Value.(*CandidateNode)
|
||||
|
||||
parsedTime, err := parseDateTime(layout, candidate.Node.Value)
|
||||
parsedTime, err := time.Parse(layout, candidate.Node.Value)
|
||||
if err != nil {
|
||||
return Context{}, fmt.Errorf("could not parse datetime of [%v]: %w", candidate.GetNicePath(), err)
|
||||
}
|
||||
@@ -112,7 +101,7 @@ func tzOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode)
|
||||
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
|
||||
candidate := el.Value.(*CandidateNode)
|
||||
|
||||
parsedTime, err := parseDateTime(layout, candidate.Node.Value)
|
||||
parsedTime, err := time.Parse(layout, candidate.Node.Value)
|
||||
if err != nil {
|
||||
return Context{}, fmt.Errorf("could not parse datetime of [%v] using layout [%v]: %w", candidate.GetNicePath(), layout, err)
|
||||
}
|
||||
|
||||
@@ -25,7 +25,7 @@ var dateTimeOperatorScenarios = []expressionScenario{
|
||||
},
|
||||
{
|
||||
description: "Format: get the day of the week",
|
||||
document: `a: 2001-12-15`,
|
||||
document: `a: 2001-12-15T02:59:43.1Z`,
|
||||
expression: `.a | format_datetime("Monday")`,
|
||||
expected: []string{
|
||||
"D0, P[a], (!!str)::Saturday\n",
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
func configureEncoder(format PrinterOutputFormat, indent int) Encoder {
|
||||
switch format {
|
||||
case JSONOutputFormat:
|
||||
return NewJSONEncoder(indent, false, false)
|
||||
return NewJSONEncoder(indent, false)
|
||||
case PropsOutputFormat:
|
||||
return NewPropertiesEncoder(true)
|
||||
case CSVOutputFormat:
|
||||
|
||||
@@ -194,7 +194,7 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
|
||||
},
|
||||
{
|
||||
description: "Encode value as xml string",
|
||||
document: `{a: {cool: {foo: "bar", +@id: hi}}}`,
|
||||
document: `{a: {cool: {foo: "bar", +id: hi}}}`,
|
||||
expression: `.a | to_xml`,
|
||||
expected: []string{
|
||||
"D0, P[a], (!!str)::<cool id=\"hi\">\n <foo>bar</foo>\n</cool>\n\n",
|
||||
@@ -202,7 +202,7 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
|
||||
},
|
||||
{
|
||||
description: "Encode value as xml string on a single line",
|
||||
document: `{a: {cool: {foo: "bar", +@id: hi}}}`,
|
||||
document: `{a: {cool: {foo: "bar", +id: hi}}}`,
|
||||
expression: `.a | @xml`,
|
||||
expected: []string{
|
||||
"D0, P[a], (!!str)::<cool id=\"hi\"><foo>bar</foo></cool>\n\n",
|
||||
@@ -210,7 +210,7 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
|
||||
},
|
||||
{
|
||||
description: "Encode value as xml string with custom indentation",
|
||||
document: `{a: {cool: {foo: "bar", +@id: hi}}}`,
|
||||
document: `{a: {cool: {foo: "bar", +id: hi}}}`,
|
||||
expression: `{"cat": .a | to_xml(1)}`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::cat: |\n <cool id=\"hi\">\n <foo>bar</foo>\n </cool>\n",
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
package yqlib
|
||||
|
||||
type expressionOpPreferences struct {
|
||||
expression string
|
||||
}
|
||||
|
||||
func expressionOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
|
||||
|
||||
prefs := expressionNode.Operation.Preferences.(expressionOpPreferences)
|
||||
expNode, err := ExpressionParser.ParseExpression(prefs.expression)
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
|
||||
return d.GetMatchingNodes(context, expNode)
|
||||
}
|
||||
@@ -4,16 +4,6 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
var documentToPrune = `
|
||||
parentA: bob
|
||||
parentB:
|
||||
child1: i am child1
|
||||
child2: i am child2
|
||||
parentC:
|
||||
child1: me child1
|
||||
child2: me child2
|
||||
`
|
||||
|
||||
var pathOperatorScenarios = []expressionScenario{
|
||||
{
|
||||
description: "Map path",
|
||||
@@ -88,15 +78,6 @@ var pathOperatorScenarios = []expressionScenario{
|
||||
"D0, P[], ()::a:\n b: things\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Set path to prune deep paths",
|
||||
subdescription: "Like pick but recursive. This uses `ireduce` to deeply set the selected paths into an empty object,",
|
||||
document: documentToPrune,
|
||||
expression: "(.parentB.child2, .parentC.child1) as $i\n ireduce({}; setpath($i | path; $i))",
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::parentB:\n child2: i am child2\nparentC:\n child1: me child1\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Set array path",
|
||||
document: `a: [cat, frog]`,
|
||||
|
||||
+18
-27
@@ -2,52 +2,43 @@ package yqlib
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"fmt"
|
||||
|
||||
yaml "gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func getSliceNumber(d *dataTreeNavigator, context Context, node *CandidateNode, expressionNode *ExpressionNode) (int, error) {
|
||||
result, err := d.GetMatchingNodes(context.SingleChildContext(node), expressionNode)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if result.MatchingNodes.Len() != 1 {
|
||||
return 0, fmt.Errorf("expected to find 1 number, got %v instead", result.MatchingNodes.Len())
|
||||
}
|
||||
return parseInt(result.MatchingNodes.Front().Value.(*CandidateNode).Node.Value)
|
||||
type sliceArrayPreferences struct {
|
||||
firstNumber int
|
||||
secondNumber int
|
||||
secondNumberDefined bool
|
||||
}
|
||||
|
||||
func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
|
||||
|
||||
log.Debug("slice array operator!")
|
||||
log.Debug("lhs: %v", expressionNode.LHS.Operation.toString())
|
||||
log.Debug("rhs: %v", expressionNode.RHS.Operation.toString())
|
||||
lhs, err := d.GetMatchingNodes(context, expressionNode.LHS)
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
prefs := expressionNode.Operation.Preferences.(sliceArrayPreferences)
|
||||
firstNumber := prefs.firstNumber
|
||||
secondNumber := prefs.secondNumber
|
||||
|
||||
results := list.New()
|
||||
|
||||
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
|
||||
for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
|
||||
lhsNode := el.Value.(*CandidateNode)
|
||||
original := unwrapDoc(lhsNode.Node)
|
||||
|
||||
firstNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.LHS)
|
||||
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
relativeFirstNumber := firstNumber
|
||||
if relativeFirstNumber < 0 {
|
||||
relativeFirstNumber = len(original.Content) + firstNumber
|
||||
}
|
||||
|
||||
secondNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.RHS)
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
|
||||
relativeSecondNumber := secondNumber
|
||||
if relativeSecondNumber < 0 {
|
||||
relativeSecondNumber = len(original.Content) + secondNumber
|
||||
relativeSecondNumber := len(original.Content)
|
||||
if prefs.secondNumberDefined {
|
||||
relativeSecondNumber = secondNumber
|
||||
if relativeSecondNumber < 0 {
|
||||
relativeSecondNumber = len(original.Content) + secondNumber
|
||||
}
|
||||
}
|
||||
|
||||
log.Debug("calculateIndicesToTraverse: slice from %v to %v", relativeFirstNumber, relativeSecondNumber)
|
||||
|
||||
@@ -37,15 +37,6 @@ var sliceArrayScenarios = []expressionScenario{
|
||||
"D0, P[], (!!seq)::- dog\n- frog\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Inserting into the middle of an array",
|
||||
subdescription: "using an expression to find the index",
|
||||
document: `[cat, dog, frog, cow]`,
|
||||
expression: `(.[] | select(. == "dog") | key + 1) as $pos | .[0:($pos)] + ["rabbit"] + .[$pos:]`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!seq)::- cat\n- dog\n- rabbit\n- frog\n- cow\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
document: `[[cat, dog, frog, cow], [apple, banana, grape, mango]]`,
|
||||
|
||||
+13
-47
@@ -6,7 +6,6 @@ import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
yaml "gopkg.in/yaml.v3"
|
||||
)
|
||||
@@ -49,7 +48,7 @@ func sortByOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
|
||||
|
||||
log.Debug("going to compare %v by %v", NodeToString(candidate.CreateReplacement(originalNode)), NodeToString(candidate.CreateReplacement(nodeToCompare)))
|
||||
|
||||
sortableArray[i] = sortableNode{Node: originalNode, NodeToCompare: nodeToCompare, dateTimeLayout: context.GetDateTimeLayout()}
|
||||
sortableArray[i] = sortableNode{Node: originalNode, NodeToCompare: nodeToCompare}
|
||||
|
||||
if nodeToCompare.Kind != yaml.ScalarNode {
|
||||
return Context{}, fmt.Errorf("sort only works for scalars, got %v", nodeToCompare.Tag)
|
||||
@@ -71,9 +70,8 @@ func sortByOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
|
||||
}
|
||||
|
||||
type sortableNode struct {
|
||||
Node *yaml.Node
|
||||
NodeToCompare *yaml.Node
|
||||
dateTimeLayout string
|
||||
Node *yaml.Node
|
||||
NodeToCompare *yaml.Node
|
||||
}
|
||||
|
||||
type sortableNodeArray []sortableNode
|
||||
@@ -85,37 +83,15 @@ func (a sortableNodeArray) Less(i, j int) bool {
|
||||
lhs := a[i].NodeToCompare
|
||||
rhs := a[j].NodeToCompare
|
||||
|
||||
lhsTag := lhs.Tag
|
||||
rhsTag := rhs.Tag
|
||||
|
||||
if !strings.HasPrefix(lhsTag, "!!") {
|
||||
// custom tag - we have to have a guess
|
||||
lhsTag = guessTagFromCustomType(lhs)
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(rhsTag, "!!") {
|
||||
// custom tag - we have to have a guess
|
||||
rhsTag = guessTagFromCustomType(rhs)
|
||||
}
|
||||
|
||||
isDateTime := lhsTag == "!!timestamp" && rhsTag == "!!timestamp"
|
||||
layout := a[i].dateTimeLayout
|
||||
// if the lhs is a string, it might be a timestamp in a custom format.
|
||||
if lhsTag == "!!str" && layout != time.RFC3339 {
|
||||
_, errLhs := parseDateTime(layout, lhs.Value)
|
||||
_, errRhs := parseDateTime(layout, rhs.Value)
|
||||
isDateTime = errLhs == nil && errRhs == nil
|
||||
}
|
||||
|
||||
if lhsTag == "!!null" && rhsTag != "!!null" {
|
||||
if lhs.Tag == "!!null" && rhs.Tag != "!!null" {
|
||||
return true
|
||||
} else if lhsTag != "!!null" && rhsTag == "!!null" {
|
||||
} else if lhs.Tag != "!!null" && rhs.Tag == "!!null" {
|
||||
return false
|
||||
} else if lhsTag == "!!bool" && rhsTag != "!!bool" {
|
||||
} else if lhs.Tag == "!!bool" && rhs.Tag != "!!bool" {
|
||||
return true
|
||||
} else if lhsTag != "!!bool" && rhsTag == "!!bool" {
|
||||
} else if lhs.Tag != "!!bool" && rhs.Tag == "!!bool" {
|
||||
return false
|
||||
} else if lhsTag == "!!bool" && rhsTag == "!!bool" {
|
||||
} else if lhs.Tag == "!!bool" && rhs.Tag == "!!bool" {
|
||||
lhsTruthy, err := isTruthyNode(lhs)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("could not parse %v as boolean: %w", lhs.Value, err))
|
||||
@@ -127,19 +103,9 @@ func (a sortableNodeArray) Less(i, j int) bool {
|
||||
}
|
||||
|
||||
return !lhsTruthy && rhsTruthy
|
||||
} else if isDateTime {
|
||||
lhsTime, err := parseDateTime(layout, lhs.Value)
|
||||
if err != nil {
|
||||
log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %w", lhs.Value, layout, err)
|
||||
return strings.Compare(lhs.Value, rhs.Value) < 0
|
||||
}
|
||||
rhsTime, err := parseDateTime(layout, rhs.Value)
|
||||
if err != nil {
|
||||
log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %w", rhs.Value, layout, err)
|
||||
return strings.Compare(lhs.Value, rhs.Value) < 0
|
||||
}
|
||||
return lhsTime.Before(rhsTime)
|
||||
} else if lhsTag == "!!int" && rhsTag == "!!int" {
|
||||
} else if lhs.Tag != rhs.Tag || lhs.Tag == "!!str" {
|
||||
return strings.Compare(lhs.Value, rhs.Value) < 0
|
||||
} else if lhs.Tag == "!!int" && rhs.Tag == "!!int" {
|
||||
_, lhsNum, err := parseInt64(lhs.Value)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
@@ -149,7 +115,7 @@ func (a sortableNodeArray) Less(i, j int) bool {
|
||||
panic(err)
|
||||
}
|
||||
return lhsNum < rhsNum
|
||||
} else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") {
|
||||
} else if (lhs.Tag == "!!int" || lhs.Tag == "!!float") && (rhs.Tag == "!!int" || rhs.Tag == "!!float") {
|
||||
lhsNum, err := strconv.ParseFloat(lhs.Value, 64)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
@@ -161,5 +127,5 @@ func (a sortableNodeArray) Less(i, j int) bool {
|
||||
return lhsNum < rhsNum
|
||||
}
|
||||
|
||||
return strings.Compare(lhs.Value, rhs.Value) < 0
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -54,14 +54,6 @@ var sortByOperatorScenarios = []expressionScenario{
|
||||
"D0, P[], (!!seq)::[{a: 1}, {a: 10}, {a: 100}]\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Sort by custom date field",
|
||||
document: `[{a: "12-Jun-2011"},{a: "23-Dec-2010"},{a: "10-Aug-2011"}]`,
|
||||
expression: `with_dtf("02-Jan-2006"; sort_by(.a))`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!seq)::[{a: \"23-Dec-2010\"}, {a: \"12-Jun-2011\"}, {a: \"10-Aug-2011\"}]\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
document: "[{a: 1.1},{a: 1.001},{a: 1.01}]",
|
||||
|
||||
@@ -92,10 +92,10 @@ func subtractScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs
|
||||
rhsTag = guessTagFromCustomType(rhs)
|
||||
}
|
||||
|
||||
isDateTime := lhsTag == "!!timestamp"
|
||||
isDateTime := lhs.Tag == "!!timestamp"
|
||||
// if the lhs is a string, it might be a timestamp in a custom format.
|
||||
if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 {
|
||||
_, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value)
|
||||
_, err := time.Parse(context.GetDateTimeLayout(), lhs.Value)
|
||||
isDateTime = err == nil
|
||||
}
|
||||
|
||||
@@ -151,7 +151,7 @@ func subtractDateTime(layout string, target *CandidateNode, lhs *yaml.Node, rhs
|
||||
return fmt.Errorf("unable to parse duration [%v]: %w", rhs.Value, err)
|
||||
}
|
||||
|
||||
currentTime, err := parseDateTime(layout, lhs.Value)
|
||||
currentTime, err := time.Parse(layout, lhs.Value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -102,15 +102,6 @@ var subtractOperatorScenarios = []expressionScenario{
|
||||
"D0, P[], (doc)::a: 2021-01-01T00:00:00Z\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Date subtraction - only date",
|
||||
skipDoc: true,
|
||||
document: `a: 2021-01-01`,
|
||||
expression: `.a -= "24h"`,
|
||||
expected: []string{
|
||||
"D0, P[], (doc)::a: 2020-12-31T00:00:00Z\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Date subtraction - custom format",
|
||||
subdescription: "Use with_dtf to specify your datetime format. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",
|
||||
|
||||
@@ -80,10 +80,6 @@ func traverseArrayOperator(d *dataTreeNavigator, context Context, expressionNode
|
||||
// BUT we still return the original context back (see jq)
|
||||
// https://stedolan.github.io/jq/manual/#Variable/SymbolicBindingOperator:...as$identifier|...
|
||||
|
||||
if expressionNode.RHS != nil && expressionNode.RHS.RHS != nil && expressionNode.RHS.RHS.Operation.OperationType == createMapOpType {
|
||||
return sliceArrayOperator(d, context, expressionNode.RHS.RHS)
|
||||
}
|
||||
|
||||
lhs, err := d.GetMatchingNodes(context, expressionNode.LHS)
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
|
||||
@@ -31,7 +31,7 @@ type expressionScenario struct {
|
||||
}
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
logging.SetLevel(logging.ERROR, "")
|
||||
logging.SetLevel(logging.DEBUG, "")
|
||||
Now = func() time.Time {
|
||||
return time.Date(2021, time.May, 19, 1, 2, 3, 4, time.UTC)
|
||||
}
|
||||
|
||||
@@ -314,7 +314,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
|
||||
var writer = bufio.NewWriter(&output)
|
||||
// note printDocSeparators is true, it should still not print document separators
|
||||
// when outputing JSON.
|
||||
printer := NewPrinter(NewJSONEncoder(0, false, false), NewSinglePrinterWriter(writer))
|
||||
printer := NewPrinter(NewJSONEncoder(0, false), NewSinglePrinterWriter(writer))
|
||||
|
||||
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
|
||||
if err != nil {
|
||||
|
||||
@@ -147,14 +147,6 @@ var propertyScenarios = []formatScenario{
|
||||
expected: expectedDecodedYaml,
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "Decode properties - array should be a map",
|
||||
subdescription: "If you have a numeric map key in your property files, use array_to_map to convert them to maps.",
|
||||
input: `things.10 = mike`,
|
||||
expression: `.things |= array_to_map`,
|
||||
expected: "things:\n 10: mike\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "does not expand automatically",
|
||||
skipDoc: true,
|
||||
@@ -228,7 +220,7 @@ func documentUnwrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario)
|
||||
}
|
||||
writeOrPanic(w, "will output\n")
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true))))
|
||||
writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true))))
|
||||
}
|
||||
|
||||
func documentWrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) {
|
||||
@@ -253,7 +245,7 @@ func documentWrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) {
|
||||
}
|
||||
writeOrPanic(w, "will output\n")
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false))))
|
||||
writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false))))
|
||||
}
|
||||
|
||||
func documentDecodePropertyScenario(w *bufio.Writer, s formatScenario) {
|
||||
@@ -278,7 +270,7 @@ func documentDecodePropertyScenario(w *bufio.Writer, s formatScenario) {
|
||||
|
||||
writeOrPanic(w, "will output\n")
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))))
|
||||
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))))
|
||||
}
|
||||
|
||||
func documentRoundTripPropertyScenario(w *bufio.Writer, s formatScenario) {
|
||||
@@ -303,7 +295,7 @@ func documentRoundTripPropertyScenario(w *bufio.Writer, s formatScenario) {
|
||||
|
||||
writeOrPanic(w, "will output\n")
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", mustProcessFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(true))))
|
||||
writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(true))))
|
||||
}
|
||||
|
||||
func documentPropertyScenario(t *testing.T, w *bufio.Writer, i interface{}) {
|
||||
@@ -330,13 +322,13 @@ func TestPropertyScenarios(t *testing.T) {
|
||||
for _, s := range propertyScenarios {
|
||||
switch s.scenarioType {
|
||||
case "":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true)), s.description)
|
||||
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true)), s.description)
|
||||
case "decode":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
|
||||
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
|
||||
case "encode-wrapped":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false)), s.description)
|
||||
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false)), s.description)
|
||||
case "roundtrip":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(true)), s.description)
|
||||
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(true)), s.description)
|
||||
|
||||
default:
|
||||
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ type XmlPreferences struct {
|
||||
|
||||
func NewDefaultXmlPreferences() XmlPreferences {
|
||||
return XmlPreferences{
|
||||
AttributePrefix: "+@",
|
||||
AttributePrefix: "+",
|
||||
ContentName: "+content",
|
||||
StrictMode: false,
|
||||
KeepNamespace: true,
|
||||
|
||||
+28
-57
@@ -58,7 +58,7 @@ cat:
|
||||
d:
|
||||
# in d before
|
||||
z:
|
||||
+@sweet: cool
|
||||
+sweet: cool
|
||||
# in d after
|
||||
# in y after
|
||||
# in_cat_after
|
||||
@@ -98,11 +98,11 @@ cat:
|
||||
d:
|
||||
- # in d before
|
||||
z:
|
||||
+@sweet: cool
|
||||
+sweet: cool
|
||||
# in d after
|
||||
- # in d2 before
|
||||
z:
|
||||
+@sweet: cool2
|
||||
+sweet: cool2
|
||||
# in d2 after
|
||||
# in y after
|
||||
# in_cat_after
|
||||
@@ -165,16 +165,16 @@ const inputXMLWithNamespacedAttr = `
|
||||
|
||||
const expectedYAMLWithNamespacedAttr = `+p_xml: version="1.0"
|
||||
map:
|
||||
+@xmlns: some-namespace
|
||||
+@xmlns:xsi: some-instance
|
||||
+@some-instance:schemaLocation: some-url
|
||||
+xmlns: some-namespace
|
||||
+xmlns:xsi: some-instance
|
||||
+some-instance:schemaLocation: some-url
|
||||
`
|
||||
|
||||
const expectedYAMLWithRawNamespacedAttr = `+p_xml: version="1.0"
|
||||
map:
|
||||
+@xmlns: some-namespace
|
||||
+@xmlns:xsi: some-instance
|
||||
+@xsi:schemaLocation: some-url
|
||||
+xmlns: some-namespace
|
||||
+xmlns:xsi: some-instance
|
||||
+xsi:schemaLocation: some-url
|
||||
`
|
||||
|
||||
const xmlWithCustomDtd = `
|
||||
@@ -251,13 +251,13 @@ var xmlScenarios = []formatScenario{
|
||||
description: "Parse xml: attributes",
|
||||
subdescription: "Attributes are converted to fields, with the default attribute prefix '+'. Use '--xml-attribute-prefix` to set your own.",
|
||||
input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">\n <legs>7</legs>\n</cat>",
|
||||
expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n +@legs: \"4\"\n legs: \"7\"\n",
|
||||
expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n +legs: \"4\"\n legs: \"7\"\n",
|
||||
},
|
||||
{
|
||||
description: "Parse xml: attributes with content",
|
||||
subdescription: "Content is added as a field, using the default content name of `+content`. Use `--xml-content-name` to set your own.",
|
||||
input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">meow</cat>",
|
||||
expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n +content: meow\n +@legs: \"4\"\n",
|
||||
expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n +content: meow\n +legs: \"4\"\n",
|
||||
},
|
||||
{
|
||||
description: "Parse xml: custom dtd",
|
||||
@@ -367,42 +367,21 @@ var xmlScenarios = []formatScenario{
|
||||
{
|
||||
description: "Encode xml: attributes",
|
||||
subdescription: "Fields with the matching xml-attribute-prefix are assumed to be attributes.",
|
||||
input: "cat:\n +@name: tiger\n meows: true\n",
|
||||
input: "cat:\n +name: tiger\n meows: true\n",
|
||||
expected: "<cat name=\"tiger\">\n <meows>true</meows>\n</cat>\n",
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
description: "double prefix",
|
||||
skipDoc: true,
|
||||
input: "cat:\n +@+@name: tiger\n meows: true\n",
|
||||
input: "cat:\n ++@name: tiger\n meows: true\n",
|
||||
expected: "<cat +@name=\"tiger\">\n <meows>true</meows>\n</cat>\n",
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
description: "arrays cannot be encoded",
|
||||
skipDoc: true,
|
||||
input: "[cat, dog, fish]",
|
||||
expectedError: "cannot encode !!seq to XML - only maps can be encoded",
|
||||
scenarioType: "encode-error",
|
||||
},
|
||||
{
|
||||
description: "arrays cannot be encoded - 2",
|
||||
skipDoc: true,
|
||||
input: "[cat, dog]",
|
||||
expectedError: "cannot encode !!seq to XML - only maps can be encoded",
|
||||
scenarioType: "encode-error",
|
||||
},
|
||||
{
|
||||
description: "scalars cannot be encoded",
|
||||
skipDoc: true,
|
||||
input: "mike",
|
||||
expectedError: "cannot encode !!str to XML - only maps can be encoded",
|
||||
scenarioType: "encode-error",
|
||||
},
|
||||
{
|
||||
description: "Encode xml: attributes with content",
|
||||
subdescription: "Fields with the matching xml-content-name is assumed to be content.",
|
||||
input: "cat:\n +@name: tiger\n +content: cool\n",
|
||||
input: "cat:\n +name: tiger\n +content: cool\n",
|
||||
expected: "<cat name=\"tiger\">cool</cat>\n",
|
||||
scenarioType: "encode",
|
||||
},
|
||||
@@ -439,31 +418,23 @@ var xmlScenarios = []formatScenario{
|
||||
func testXMLScenario(t *testing.T, s formatScenario) {
|
||||
switch s.scenarioType {
|
||||
case "", "decode":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(4, false, ConfiguredYamlPreferences)), s.description)
|
||||
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(4, false, ConfiguredYamlPreferences)), s.description)
|
||||
case "encode":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description)
|
||||
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description)
|
||||
case "roundtrip":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description)
|
||||
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description)
|
||||
case "decode-keep-ns":
|
||||
prefs := NewDefaultXmlPreferences()
|
||||
prefs.KeepNamespace = true
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
|
||||
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
|
||||
case "decode-raw-token":
|
||||
prefs := NewDefaultXmlPreferences()
|
||||
prefs.UseRawToken = true
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
|
||||
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
|
||||
case "roundtrip-skip-directives":
|
||||
prefs := NewDefaultXmlPreferences()
|
||||
prefs.SkipDirectives = true
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs)), s.description)
|
||||
case "encode-error":
|
||||
result, err := processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, NewDefaultXmlPreferences()))
|
||||
if err == nil {
|
||||
t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result)
|
||||
} else {
|
||||
test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description)
|
||||
}
|
||||
|
||||
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs)), s.description)
|
||||
default:
|
||||
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
|
||||
}
|
||||
@@ -513,7 +484,7 @@ func documentXMLDecodeScenario(w *bufio.Writer, s formatScenario) {
|
||||
writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=xml '%v' sample.xml\n```\n", expression))
|
||||
writeOrPanic(w, "will output\n")
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences))))
|
||||
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences))))
|
||||
}
|
||||
|
||||
func documentXMLDecodeKeepNsScenario(w *bufio.Writer, s formatScenario) {
|
||||
@@ -532,12 +503,12 @@ func documentXMLDecodeKeepNsScenario(w *bufio.Writer, s formatScenario) {
|
||||
writeOrPanic(w, "will output\n")
|
||||
prefs := NewDefaultXmlPreferences()
|
||||
prefs.KeepNamespace = true
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs))))
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs))))
|
||||
|
||||
prefsWithout := NewDefaultXmlPreferences()
|
||||
prefs.KeepNamespace = false
|
||||
writeOrPanic(w, "instead of\n")
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(2, prefsWithout))))
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(2, prefsWithout))))
|
||||
}
|
||||
|
||||
func documentXMLDecodeKeepNsRawTokenScenario(w *bufio.Writer, s formatScenario) {
|
||||
@@ -558,13 +529,13 @@ func documentXMLDecodeKeepNsRawTokenScenario(w *bufio.Writer, s formatScenario)
|
||||
prefs := NewDefaultXmlPreferences()
|
||||
prefs.KeepNamespace = true
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs))))
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs))))
|
||||
|
||||
prefsWithout := NewDefaultXmlPreferences()
|
||||
prefsWithout.KeepNamespace = false
|
||||
|
||||
writeOrPanic(w, "instead of\n")
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(2, prefsWithout))))
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(2, prefsWithout))))
|
||||
}
|
||||
|
||||
func documentXMLEncodeScenario(w *bufio.Writer, s formatScenario) {
|
||||
@@ -582,7 +553,7 @@ func documentXMLEncodeScenario(w *bufio.Writer, s formatScenario) {
|
||||
writeOrPanic(w, "```bash\nyq -o=xml '.' sample.yml\n```\n")
|
||||
writeOrPanic(w, "will output\n")
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences))))
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences))))
|
||||
}
|
||||
|
||||
func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) {
|
||||
@@ -600,7 +571,7 @@ func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) {
|
||||
writeOrPanic(w, "```bash\nyq -p=xml -o=xml '.' sample.xml\n```\n")
|
||||
writeOrPanic(w, "will output\n")
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences))))
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences))))
|
||||
}
|
||||
|
||||
func documentXMLSkipDirectrivesScenario(w *bufio.Writer, s formatScenario) {
|
||||
@@ -620,7 +591,7 @@ func documentXMLSkipDirectrivesScenario(w *bufio.Writer, s formatScenario) {
|
||||
prefs := NewDefaultXmlPreferences()
|
||||
prefs.SkipDirectives = true
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs))))
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs))))
|
||||
}
|
||||
|
||||
func TestXMLScenarios(t *testing.T) {
|
||||
|
||||
@@ -1,28 +1,7 @@
|
||||
4.30.1:
|
||||
- XML users note: the default attribute prefix has change to `+@` to avoid naming conflicts!
|
||||
- Can use expressions in slice #1419
|
||||
- Fixed unhandled exception when decoding CSV thanks @washanhanzi
|
||||
- Added array_to_map operator for #1415
|
||||
- Fixed sorting by date #1412
|
||||
- Added check to ensure only maps can be encoded to XML #1408
|
||||
- Check merge alias is a map #1425
|
||||
- Explicity setting unwrap flag works for json output #437, #1409
|
||||
- Bumped go version
|
||||
|
||||
|
||||
|
||||
4.29.2:
|
||||
- Fixed null pointer exception when parsing CSV with empty field #1404
|
||||
|
||||
4.29.1:
|
||||
- Fixed Square brackets removing update #1342
|
||||
- Added slice array operator (.[10:15]) #44
|
||||
- XML decoder/encoder now parses directives and proc instructions (#1344). Please use the new skip flags [documented here](https://mikefarah.gitbook.io/yq/usage/xml) to ignore them.
|
||||
- XML users note that the default attribute prefix will change to `+@` in the 4.30 release to avoid naming conflicts!
|
||||
- Improved comment handling of decoders (breaking change for yqlib users sorry)
|
||||
- Fixed load operator bug when loading yaml file with multiple documents
|
||||
- Bumped Go compiler version
|
||||
- Bumped dependencies
|
||||
|
||||
4.28.2:
|
||||
- Fixed Github Actions issues (thanks @mattphelps-8451)
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
name: yq
|
||||
version: '4.30.1'
|
||||
version: '4.28.2'
|
||||
summary: A lightweight and portable command-line YAML processor
|
||||
description: |
|
||||
The aim of the project is to be the jq or sed of yaml files.
|
||||
|
||||
Reference in New Issue
Block a user