Compare commits

..
1 Commits
Author SHA1 Message Date
Mike Farah 48d6ad58e7 Slice array 2022-10-28 20:33:50 +11:00
63 changed files with 271 additions and 964 deletions
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.19.3 as builder
FROM golang:1.19.2 as builder
WORKDIR /go/src/mikefarah/yq
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.19.3
FROM golang:1.19.2
COPY scripts/devtools.sh /opt/devtools.sh
+3 -4
View File
@@ -61,9 +61,6 @@ Take a look at the discussions for [common questions](https://github.com/mikefar
### wget
Use wget to download, gzipped pre-compiled binaries:
For instance, VERSION=v4.2.0 and BINARY=yq_linux_amd64
#### Compressed via tar.gz
```bash
wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY}.tar.gz -O - |\
@@ -77,7 +74,9 @@ wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY} -O /
chmod +x /usr/bin/yq
```
#### Latest version
For instance, VERSION=v4.2.0 and BINARY=yq_linux_amd64
Use VERSION=latest to always get the latest version:
```bash
wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq &&\
+4 -70
View File
@@ -48,55 +48,6 @@ EOM
assertEquals "$expected" "$X"
}
testOutputYamlRawDefault() {
cat >test.yml <<EOL
a: "cat"
EOL
X=$(./yq e '.a' test.yml)
assertEquals "cat" "$X"
X=$(./yq ea '.a' test.yml)
assertEquals "cat" "$X"
}
testOutputYamlRawOff() {
cat >test.yml <<EOL
a: "cat"
EOL
X=$(./yq e -r=false '.a' test.yml)
assertEquals "\"cat\"" "$X"
X=$(./yq ea -r=false '.a' test.yml)
assertEquals "\"cat\"" "$X"
}
testOutputJsonRaw() {
cat >test.yml <<EOL
a: cat
EOL
X=$(./yq e -r --output-format=json '.a' test.yml)
assertEquals "cat" "$X"
X=$(./yq ea -r --output-format=json '.a' test.yml)
assertEquals "cat" "$X"
}
testOutputJsonDefault() {
cat >test.yml <<EOL
a: cat
EOL
X=$(./yq e --output-format=json '.a' test.yml)
assertEquals "\"cat\"" "$X"
X=$(./yq ea --output-format=json '.a' test.yml)
assertEquals "\"cat\"" "$X"
}
testOutputJsonShort() {
cat >test.yml <<EOL
a: {b: ["cat"]}
@@ -121,11 +72,11 @@ EOM
testOutputProperties() {
cat >test.yml <<EOL
a: {b: {c: ["cat cat"]}}
a: {b: {c: ["cat"]}}
EOL
read -r -d '' expected << EOM
a.b.c.0 = cat cat
a.b.c.0 = cat
EOM
X=$(./yq e --output-format=props test.yml)
@@ -135,30 +86,13 @@ EOM
assertEquals "$expected" "$X"
}
testOutputPropertiesDontUnwrap() {
cat >test.yml <<EOL
a: {b: {c: ["cat cat"]}}
EOL
read -r -d '' expected << EOM
a.b.c.0 = "cat cat"
EOM
X=$(./yq e -r=false --output-format=props test.yml)
assertEquals "$expected" "$X"
X=$(./yq ea -r=false --output-format=props test.yml)
assertEquals "$expected" "$X"
}
testOutputPropertiesShort() {
cat >test.yml <<EOL
a: {b: {c: ["cat cat"]}}
a: {b: {c: ["cat"]}}
EOL
read -r -d '' expected << EOM
a.b.c.0 = cat cat
a.b.c.0 = cat
EOM
X=$(./yq e -o=p test.yml)
+1 -3
View File
@@ -1,8 +1,6 @@
package cmd
var unwrapScalarFlag = newUnwrapFlag()
var unwrapScalar = false
var unwrapScalar = true
var writeInplace = false
var outputToJSON = false
+6 -33
View File
@@ -35,7 +35,7 @@ yq -P sample.json
return evaluateSequence(cmd, args)
},
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
PersistentPreRun: func(cmd *cobra.Command, args []string) {
cmd.SetOut(cmd.OutOrStdout())
var format = logging.MustStringFormatter(
@@ -52,43 +52,18 @@ yq -P sample.json
logging.SetBackend(backend)
yqlib.InitExpressionParser()
outputFormatType, err := yqlib.OutputFormatFromString(outputFormat)
if err != nil {
return err
}
inputFormatType, err := yqlib.InputFormatFromString(inputFormat)
if err != nil {
return err
}
if (inputFormatType == yqlib.XMLInputFormat &&
outputFormatType != yqlib.XMLOutputFormat ||
inputFormatType != yqlib.XMLInputFormat &&
outputFormatType == yqlib.XMLOutputFormat) &&
yqlib.ConfiguredXMLPreferences.AttributePrefix == "+@" {
yqlib.GetLogger().Warning("The default xml-attribute-prefix has changed in the v4.30 to `+@` to avoid " +
if (inputFormat == "x" || inputFormat == "xml") &&
outputFormat != "x" && outputFormat != "xml" &&
yqlib.ConfiguredXMLPreferences.AttributePrefix == "+" {
yqlib.GetLogger().Warning("The default xml-attribute-prefix will change in the v4.30 to `+@` to avoid " +
"naming conflicts with the default content name, directive name and proc inst prefix. If you need to keep " +
"`+` please set that value explicityly with --xml-attribute-prefix.")
}
if outputFormatType == yqlib.YamlOutputFormat ||
outputFormatType == yqlib.PropsOutputFormat {
unwrapScalar = true
}
if unwrapScalarFlag.IsExplicitySet() {
unwrapScalar = unwrapScalarFlag.IsSet()
}
//copy preference form global setting
yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar
yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators
return nil
},
}
@@ -119,9 +94,7 @@ yq -P sample.json
rootCmd.PersistentFlags().IntVarP(&indent, "indent", "I", 2, "sets indent level for output")
rootCmd.Flags().BoolVarP(&version, "version", "V", false, "Print version information and quit")
rootCmd.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the file inplace of first file given.")
rootCmd.PersistentFlags().VarP(unwrapScalarFlag, "unwrapScalar", "r", "unwrap scalar, print the value with no quotes, colors or comments. Defaults to true for yaml")
rootCmd.PersistentFlags().Lookup("unwrapScalar").NoOptDefVal = "true"
rootCmd.PersistentFlags().BoolVarP(&unwrapScalar, "unwrapScalar", "r", true, "unwrap scalar, print the value with no quotes, colors or comments")
rootCmd.PersistentFlags().BoolVarP(&prettyPrint, "prettyPrint", "P", false, "pretty print, shorthand for '... style = \"\"'")
rootCmd.PersistentFlags().BoolVarP(&exitStatus, "exit-status", "e", false, "set exit status if there are no matches or null or false is returned")
-46
View File
@@ -1,46 +0,0 @@
package cmd
import (
"strconv"
"github.com/spf13/pflag"
)
type boolFlag interface {
pflag.Value
IsExplicitySet() bool
IsSet() bool
}
type unwrapScalarFlagStrc struct {
explicitySet bool
value bool
}
func newUnwrapFlag() boolFlag {
return &unwrapScalarFlagStrc{value: true}
}
func (f *unwrapScalarFlagStrc) IsExplicitySet() bool {
return f.explicitySet
}
func (f *unwrapScalarFlagStrc) IsSet() bool {
return f.value
}
func (f *unwrapScalarFlagStrc) String() string {
return strconv.FormatBool(f.value)
}
func (f *unwrapScalarFlagStrc) Set(value string) error {
v, err := strconv.ParseBool(value)
f.value = v
f.explicitySet = true
return err
}
func (*unwrapScalarFlagStrc) Type() string {
return "bool"
}
+1 -1
View File
@@ -98,7 +98,7 @@ func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yq
func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder {
switch format {
case yqlib.JSONOutputFormat:
return yqlib.NewJSONEncoder(indent, colorsEnabled, unwrapScalar)
return yqlib.NewJSONEncoder(indent, colorsEnabled)
case yqlib.PropsOutputFormat:
return yqlib.NewPropertiesEncoder(unwrapScalar)
case yqlib.CSVOutputFormat:
+1 -1
View File
@@ -11,7 +11,7 @@ var (
GitDescribe string
// Version is main version number that is being run at the moment.
Version = "4.30.1"
Version = "4.28.2"
// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
+5 -1
View File
@@ -1 +1,5 @@
["foobar", "foobaz", "blarp"]
volumes:
beehive-conf:
driver: local
services:
- beehive
+1 -1
View File
@@ -1,3 +1,3 @@
name,numberOfCats,likesApples,height
,1,true,168.8
Gary,1,true,168.8
Samantha's Rabbit,2,false,-188.8
1 name numberOfCats likesApples height
2 Gary 1 true 168.8
3 Samantha's Rabbit 2 false -188.8
+1 -1
View File
@@ -13,7 +13,6 @@ require (
github.com/magiconair/properties v1.8.6
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
github.com/spf13/cobra v1.6.1
github.com/spf13/pflag v1.0.5
golang.org/x/net v0.0.0-20220708220712-1185a9018129
gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
gopkg.in/yaml.v3 v3.0.1
@@ -23,6 +22,7 @@ require (
github.com/inconshreveable/mousetrap v1.0.1 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/spf13/pflag v1.0.5 // indirect
golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
+3 -3
View File
@@ -54,12 +54,12 @@ func compare(prefs compareTypePref) func(d *dataTreeNavigator, context Context,
}
func compareDateTime(layout string, prefs compareTypePref, lhs *yaml.Node, rhs *yaml.Node) (bool, error) {
lhsTime, err := parseDateTime(layout, lhs.Value)
lhsTime, err := time.Parse(layout, lhs.Value)
if err != nil {
return false, err
}
rhsTime, err := parseDateTime(layout, rhs.Value)
rhsTime, err := time.Parse(layout, rhs.Value)
if err != nil {
return false, err
}
@@ -81,7 +81,7 @@ func compareScalars(context Context, prefs compareTypePref, lhs *yaml.Node, rhs
isDateTime := lhs.Tag == "!!timestamp"
// if the lhs is a string, it might be a timestamp in a custom format.
if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 {
_, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value)
_, err := time.Parse(context.GetDateTimeLayout(), lhs.Value)
isDateTime = err == nil
}
if isDateTime {
+9 -18
View File
@@ -12,9 +12,7 @@ const csvSimple = `name,numberOfCats,likesApples,height
Gary,1,true,168.8
Samantha's Rabbit,2,false,-188.8
`
const csvMissing = `name,numberOfCats,likesApples,height
,null,,168.8
`
const expectedUpdatedSimpleCsv = `name,numberOfCats,likesApples,height
Gary,3,true,168.8
Samantha's Rabbit,2,false,-188.8
@@ -112,13 +110,6 @@ var csvScenarios = []formatScenario{
expected: csvSimpleMissingData,
scenarioType: "encode-csv",
},
{
description: "decode csv missing",
skipDoc: true,
input: csvMissing,
expected: csvMissing,
scenarioType: "roundtrip-csv",
},
{
description: "Parse CSV into an array of objects",
subdescription: "First row is assumed to be the header row.",
@@ -145,15 +136,15 @@ var csvScenarios = []formatScenario{
func testCSVScenario(t *testing.T, s formatScenario) {
switch s.scenarioType {
case "encode-csv":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(',')), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(',')), s.description)
case "encode-tsv":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder('\t')), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder('\t')), s.description)
case "decode-csv-object":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(','), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
case "decode-tsv-object":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder('\t'), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder('\t'), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
case "roundtrip-csv":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(','), NewCsvEncoder(',')), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewCsvEncoder(',')), s.description)
default:
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
}
@@ -180,7 +171,7 @@ func documentCSVDecodeObjectScenario(w *bufio.Writer, s formatScenario, formatTy
}
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n",
mustProcessFormatScenario(s, NewCSVObjectDecoder(separator), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))),
processFormatScenario(s, NewCSVObjectDecoder(separator), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))),
)
}
@@ -212,7 +203,7 @@ func documentCSVEncodeScenario(w *bufio.Writer, s formatScenario, formatType str
}
writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(separator))),
processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(separator))),
)
}
@@ -244,7 +235,7 @@ func documentCSVRoundTripScenario(w *bufio.Writer, s formatScenario, formatType
}
writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
mustProcessFormatScenario(s, NewCSVObjectDecoder(separator), NewCsvEncoder(separator))),
processFormatScenario(s, NewCSVObjectDecoder(separator), NewCsvEncoder(separator))),
)
}
+7 -16
View File
@@ -15,10 +15,10 @@ type formatScenario struct {
subdescription string
skipDoc bool
scenarioType string
expectedError string
}
func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) (string, error) {
func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) string {
var output bytes.Buffer
writer := bufio.NewWriter(&output)
@@ -28,7 +28,7 @@ func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) (
inputs, err := readDocuments(strings.NewReader(s.input), "sample.yml", 0, decoder)
if err != nil {
return "", err
panic(err)
}
expression := s.expression
@@ -39,31 +39,22 @@ func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) (
exp, err := getExpressionParser().ParseExpression(expression)
if err != nil {
return "", err
panic(err)
}
context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: inputs}, exp)
if err != nil {
return "", err
panic(err)
}
printer := NewPrinter(encoder, NewSinglePrinterWriter(writer))
err = printer.PrintResults(context.MatchingNodes)
if err != nil {
return "", err
panic(err)
}
writer.Flush()
return output.String(), nil
}
func mustProcessFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) string {
result, err := processFormatScenario(s, decoder, encoder)
if err != nil {
panic(err)
}
return result
return output.String()
}
-28
View File
@@ -1,28 +0,0 @@
# Array to Map
Use this operator to convert an array to..a map. Skips over null values.
Behind the scenes, this is implemented using reduce:
```
(.[] | select(. != null) ) as $i ireduce({}; .[$i | key] = $i)
```
## Simple example
Given a sample.yml file of:
```yaml
cool:
- null
- null
- hello
```
then
```bash
yq '.cool |= array_to_map' sample.yml
```
will output
```yaml
cool:
2: hello
```
+1 -27
View File
@@ -1,14 +1,6 @@
# Contains
This returns `true` if the context contains the passed in parameter, and false otherwise. For arrays, this will return true if the passed in array is contained within the array. For strings, it will return true if the string is a substring.
{% hint style="warning" %}
_Note_ that, just like jq, when checking if an array of strings `contains` another, this will use `contains` and _not_ equals to check each string. This means an array like `contains(["cat"])` will return true for an array `["cats"]`.
See the "Array has a subset array" example below on how to check for a subset.
{% endhint %}
This returns `true` if the context contains the passed in parameter, and false otherwise.
## Array contains array
Array is equal or subset of
@@ -28,24 +20,6 @@ will output
true
```
## Array has a subset array
Subtract the superset array from the subset, if there's anything left, it's not a subset
Given a sample.yml file of:
```yaml
- foobar
- foobaz
- blarp
```
then
```bash
yq '["baz", "bar"] - . | length == 0' sample.yml
```
will output
```yaml
false
```
## Object included in array
Given a sample.yml file of:
```yaml
+1 -1
View File
@@ -60,7 +60,7 @@ a: 2001-12-15
## Format: get the day of the week
Given a sample.yml file of:
```yaml
a: 2001-12-15
a: 2001-12-15T02:59:43.1Z
```
then
```bash
+3 -3
View File
@@ -337,7 +337,7 @@ Given a sample.yml file of:
a:
cool:
foo: bar
+@id: hi
+id: hi
```
then
```bash
@@ -357,7 +357,7 @@ Given a sample.yml file of:
a:
cool:
foo: bar
+@id: hi
+id: hi
```
then
```bash
@@ -375,7 +375,7 @@ Given a sample.yml file of:
a:
cool:
foo: bar
+@id: hi
+id: hi
```
then
```bash
@@ -1,9 +0,0 @@
# Array to Map
Use this operator to convert an array to..a map. Skips over null values.
Behind the scenes, this is implemented using reduce:
```
(.[] | select(. != null) ) as $i ireduce({}; .[$i | key] = $i)
```
+1 -9
View File
@@ -1,11 +1,3 @@
# Contains
This returns `true` if the context contains the passed in parameter, and false otherwise. For arrays, this will return true if the passed in array is contained within the array. For strings, it will return true if the string is a substring.
{% hint style="warning" %}
_Note_ that, just like jq, when checking if an array of strings `contains` another, this will use `contains` and _not_ equals to check each string. This means an array like `contains(["cat"])` will return true for an array `["cats"]`.
See the "Array has a subset array" example below on how to check for a subset.
{% endhint %}
This returns `true` if the context contains the passed in parameter, and false otherwise.
@@ -1,5 +0,0 @@
# Slice Array
The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array.
You may leave out the first or second number, which will will refer to the start or end of the array respectively.
-26
View File
@@ -122,32 +122,6 @@ a:
b: things
```
## Set path to prune deep paths
Like pick but recursive. This uses `ireduce` to deeply set the selected paths into an empty object,
Given a sample.yml file of:
```yaml
parentA: bob
parentB:
child1: i am child1
child2: i am child2
parentC:
child1: me child1
child2: me child2
```
then
```bash
yq '(.parentB.child2, .parentC.child1) as $i
ireduce({}; setpath($i | path; $i))' sample.yml
```
will output
```yaml
parentB:
child2: i am child2
parentC:
child1: me child1
```
## Set array path
Given a sample.yml file of:
```yaml
-28
View File
@@ -1,8 +1,3 @@
# Slice Array
The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array.
You may leave out the first or second number, which will will refer to the start or end of the array respectively.
## Slicing arrays
Given a sample.yml file of:
@@ -80,26 +75,3 @@ will output
- frog
```
## Inserting into the middle of an array
using an expression to find the index
Given a sample.yml file of:
```yaml
- cat
- dog
- frog
- cow
```
then
```bash
yq '(.[] | select(. == "dog") | key + 1) as $pos | .[0:($pos)] + ["rabbit"] + .[$pos:]' sample.yml
```
will output
```yaml
- cat
- dog
- rabbit
- frog
- cow
```
-18
View File
@@ -135,24 +135,6 @@ will output
- a: 100
```
## Sort by custom date field
Given a sample.yml file of:
```yaml
- a: 12-Jun-2011
- a: 23-Dec-2010
- a: 10-Aug-2011
```
then
```bash
yq 'with_dtf("02-Jan-2006"; sort_by(.a))' sample.yml
```
will output
```yaml
- a: 23-Dec-2010
- a: 12-Jun-2011
- a: 10-Aug-2011
```
## Sort, nulls come first
Given a sample.yml file of:
```yaml
-17
View File
@@ -149,23 +149,6 @@ person:
- pizza
```
## Decode properties - array should be a map
If you have a numeric map key in your property files, use array_to_map to convert them to maps.
Given a sample.properties file of:
```properties
things.10 = mike
```
then
```bash
yq -p=props '.things |= array_to_map' sample.properties
```
will output
```yaml
things:
10: mike
```
## Roundtrip
Given a sample.properties file of:
```properties
+4 -4
View File
@@ -128,7 +128,7 @@ will output
```yaml
+p_xml: version="1.0" encoding="UTF-8"
cat:
+@legs: "4"
+legs: "4"
legs: "7"
```
@@ -149,7 +149,7 @@ will output
+p_xml: version="1.0" encoding="UTF-8"
cat:
+content: meow
+@legs: "4"
+legs: "4"
```
## Parse xml: custom dtd
@@ -347,7 +347,7 @@ Fields with the matching xml-attribute-prefix are assumed to be attributes.
Given a sample.yml file of:
```yaml
cat:
+@name: tiger
+name: tiger
meows: true
```
@@ -368,7 +368,7 @@ Fields with the matching xml-content-name is assumed to be content.
Given a sample.yml file of:
```yaml
cat:
+@name: tiger
+name: tiger
+content: cool
```
+1 -1
View File
@@ -45,7 +45,7 @@ func (o *orderedMap) UnmarshalJSON(data []byte) error {
// cycle through k/v
var tok json.Token
for tok, err = dec.Token(); err == nil; tok, err = dec.Token() {
for tok, err = dec.Token(); !errors.Is(err, io.EOF); tok, err = dec.Token() {
// we can expect two types: string or Delim. Delim automatically means
// that it is the closing bracket of the object, whereas string means
// that there is another key.
+2 -7
View File
@@ -11,7 +11,6 @@ import (
type jsonEncoder struct {
indentString string
colorise bool
UnwrapScalar bool
}
func mapKeysToStrings(node *yaml.Node) {
@@ -29,14 +28,14 @@ func mapKeysToStrings(node *yaml.Node) {
}
}
func NewJSONEncoder(indent int, colorise bool, unwrapScalar bool) Encoder {
func NewJSONEncoder(indent int, colorise bool) Encoder {
var indentString = ""
for index := 0; index < indent; index++ {
indentString = indentString + " "
}
return &jsonEncoder{indentString, colorise, unwrapScalar}
return &jsonEncoder{indentString, colorise}
}
func (je *jsonEncoder) CanHandleAliases() bool {
@@ -53,10 +52,6 @@ func (je *jsonEncoder) PrintLeadingContent(writer io.Writer, content string) err
func (je *jsonEncoder) Encode(writer io.Writer, node *yaml.Node) error {
if node.Kind == yaml.ScalarNode && je.UnwrapScalar {
return writeString(writer, node.Value+"\n")
}
destination := writer
tempBuffer := bytes.NewBuffer(nil)
if je.colorise {
+1 -1
View File
@@ -13,7 +13,7 @@ func yamlToJSON(sampleYaml string, indent int) string {
var output bytes.Buffer
writer := bufio.NewWriter(&output)
var jsonEncoder = NewJSONEncoder(indent, false, false)
var jsonEncoder = NewJSONEncoder(indent, false)
inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
if err != nil {
panic(err)
+1 -5
View File
@@ -65,11 +65,7 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
if err != nil {
return err
}
unwrappedNode := unwrapDoc(node)
if unwrappedNode.Kind != yaml.MappingNode {
return fmt.Errorf("cannot encode %v to XML - only maps can be encoded", unwrappedNode.Tag)
}
err = e.encodeTopLevelMap(encoder, unwrappedNode)
err = e.encodeTopLevelMap(encoder, unwrapDoc(node))
if err != nil {
return err
}
+8 -21
View File
@@ -84,13 +84,6 @@ var jsonScenarios = []formatScenario{
input: `{"cat": "meow"}`,
expected: "D0, P[], (!!map)::cat: meow\n",
},
{
description: "bad json",
skipDoc: true,
input: `{"a": 1 "b": 2}`,
expectedError: `bad file 'sample.yml': invalid character '"' after object key:value pair`,
scenarioType: "decode-error",
},
{
description: "Parse json: complex",
subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output",
@@ -244,7 +237,7 @@ func documentRoundtripNdJsonScenario(w *bufio.Writer, s formatScenario, indent i
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(indent, false, false))))
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(indent, false))))
}
func documentDecodeNdJsonScenario(w *bufio.Writer, s formatScenario) {
@@ -269,7 +262,7 @@ func documentDecodeNdJsonScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))))
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))))
}
func decodeJSON(t *testing.T, jsonString string) *CandidateNode {
@@ -300,23 +293,17 @@ func decodeJSON(t *testing.T, jsonString string) *CandidateNode {
func testJSONScenario(t *testing.T, s formatScenario) {
switch s.scenarioType {
case "encode", "decode":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false, false)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false)), s.description)
case "":
var actual = resultToString(t, decodeJSON(t, s.input))
test.AssertResultWithContext(t, s.expected, actual, s.description)
case "decode-ndjson":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
case "roundtrip-ndjson":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(0, false, false)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(0, false)), s.description)
case "roundtrip-multi":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(2, false, false)), s.description)
case "decode-error":
result, err := processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(2, false, false))
if err == nil {
t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result)
} else {
test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description)
}
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(2, false)), s.description)
default:
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
}
@@ -398,7 +385,7 @@ func documentJSONEncodeScenario(w *bufio.Writer, s formatScenario) {
}
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false, false))))
writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false))))
}
func TestJSONScenarios(t *testing.T) {
+3 -28
View File
@@ -97,10 +97,6 @@ func postProcessTokens(tokens []*token) []*token {
return postProcessedTokens
}
func tokenIsOpType(token *token, opType *operationType) bool {
return token.TokenType == operationToken && token.Operation.OperationType == opType
}
func handleToken(tokens []*token, index int, postProcessedTokens []*token) (tokensAccum []*token, skipNextToken bool) {
skipNextToken = false
currentToken := tokens[index]
@@ -124,18 +120,9 @@ func handleToken(tokens []*token, index int, postProcessedTokens []*token) (toke
}
if tokenIsOpType(currentToken, createMapOpType) {
log.Debugf("tokenIsOpType: createMapOpType")
// check the previous token is '[', means we are slice, but dont have a first number
if tokens[index-1].TokenType == traverseArrayCollect {
log.Debugf("previous token is : traverseArrayOpType")
// need to put the number 0 before this token, as that is implied
postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: createValueOperation(0, "0")})
}
}
if index != len(tokens)-1 && currentToken.AssignOperation != nil &&
tokenIsOpType(tokens[index+1], assignOpType) {
tokens[index+1].TokenType == operationToken &&
tokens[index+1].Operation.OperationType == assignOpType {
log.Debug(" its an update assign")
currentToken.Operation = currentToken.AssignOperation
currentToken.Operation.UpdateAssign = tokens[index+1].Operation.UpdateAssign
@@ -145,17 +132,6 @@ func handleToken(tokens []*token, index int, postProcessedTokens []*token) (toke
log.Debug(" adding token to the fixed list")
postProcessedTokens = append(postProcessedTokens, currentToken)
if tokenIsOpType(currentToken, createMapOpType) {
log.Debugf("tokenIsOpType: createMapOpType")
// check the next token is ']', means we are slice, but dont have a second number
if index != len(tokens)-1 && tokens[index+1].TokenType == closeCollect {
log.Debugf("next token is : closeCollect")
// need to put the number 0 before this token, as that is implied
lengthOp := &Operation{OperationType: lengthOpType}
postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: lengthOp})
}
}
if index != len(tokens)-1 &&
((currentToken.TokenType == openCollect && tokens[index+1].TokenType == closeCollect) ||
(currentToken.TokenType == openCollectObject && tokens[index+1].TokenType == closeCollectObject)) {
@@ -165,8 +141,7 @@ func handleToken(tokens []*token, index int, postProcessedTokens []*token) (toke
}
if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
(tokenIsOpType(tokens[index+1], traversePathOpType) ||
((tokens[index+1].TokenType == operationToken && (tokens[index+1].Operation.OperationType == traversePathOpType)) ||
(tokens[index+1].TokenType == traverseArrayCollect)) {
log.Debug(" adding pipe because the next thing is traverse")
op := &Operation{OperationType: shortPipeOpType, Value: "PIPE", StringValue: "."}
+83 -10
View File
@@ -1,6 +1,7 @@
package yqlib
import (
"regexp"
"strconv"
"strings"
@@ -12,6 +13,10 @@ var participleYqRules = []*participleYqRule{
{"HEAD_COMMENT", `head_?comment|headComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{HeadComment: true}), 0},
{"FOOT_COMMENT", `foot_?comment|footComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{FootComment: true}), 0},
{"SliceArray", `\.\[-?[0-9]+:-?[0-9]+\]`, sliceArrayTwoNumbers(), 0},
{"SliceArraySecond", `\.\[\:-?[0-9]+\]`, sliceArraySecondNumberOnly(), 0},
{"SliceArrayFirst", `\.\[-?[0-9]+\:\]`, sliceArrayFirstNumberOnly(), 0},
{"OpenBracket", `\(`, literalToken(openBracket, false), 0},
{"CloseBracket", `\)`, literalToken(closeBracket, true), 0},
{"OpenTraverseArrayCollect", `\.\[`, literalToken(traverseArrayCollect, false), 0},
@@ -50,8 +55,6 @@ var participleYqRules = []*participleYqRule{
simpleOp("sortKeys", sortKeysOpType),
simpleOp("sort_?keys", sortKeysOpType),
{"ArrayToMap", "array_?to_?map", expressionOpToken(`(.[] | select(. != null) ) as $i ireduce({}; .[$i | key] = $i)`), 0},
{"YamlEncodeWithIndent", `to_?yaml\([0-9]+\)`, encodeParseIndent(YamlOutputFormat), 0},
{"XMLEncodeWithIndent", `to_?xml\([0-9]+\)`, encodeParseIndent(XMLOutputFormat), 0},
{"JSONEncodeWithIndent", `to_?json\([0-9]+\)`, encodeParseIndent(JSONOutputFormat), 0},
@@ -288,14 +291,6 @@ func opTokenWithPrefs(opType *operationType, assignOpType *operationType, prefer
}
}
func expressionOpToken(expression string) yqAction {
return func(rawToken lexer.Token) (*token, error) {
prefs := expressionOpPreferences{expression: expression}
expressionOp := &Operation{OperationType: expressionOpType, Preferences: prefs}
return &token{TokenType: operationToken, Operation: expressionOp}, nil
}
}
func flattenWithDepth() yqAction {
return func(rawToken lexer.Token) (*token, error) {
value := rawToken.Value
@@ -310,6 +305,84 @@ func flattenWithDepth() yqAction {
}
}
func sliceArrayTwoNumbers() yqAction {
return func(rawToken lexer.Token) (*token, error) {
value := rawToken.Value
sliceArrayNumbers := regexp.MustCompile(`\.\[(-?[0-9]+)\:(-?[0-9]+)\]`)
matches := sliceArrayNumbers.FindStringSubmatch(value)
log.Debug("sliceArrayTwoNumbers value: %v", value)
log.Debug("Matches: %v", matches)
firstNumber, err := parseInt(matches[1])
if err != nil {
return nil, err
}
secondNumber, err := parseInt(matches[2])
if err != nil {
return nil, err
}
prefs := sliceArrayPreferences{
firstNumber: firstNumber,
secondNumber: secondNumber,
secondNumberDefined: true,
}
log.Debug("%v", prefs)
op := &Operation{OperationType: sliceArrayOpType, Value: sliceArrayOpType.Type, StringValue: value, Preferences: prefs}
return &token{TokenType: operationToken, Operation: op}, nil
}
}
func sliceArraySecondNumberOnly() yqAction {
return func(rawToken lexer.Token) (*token, error) {
value := rawToken.Value
sliceArrayNumbers := regexp.MustCompile(`\.\[\:(-?[0-9]+)\]`)
matches := sliceArrayNumbers.FindStringSubmatch(value)
log.Debug("sliceArraySecondNumberOnly value: %v", value)
log.Debug("Matches: %v", matches)
secondNumber, err := parseInt(matches[1])
if err != nil {
return nil, err
}
prefs := sliceArrayPreferences{
firstNumber: 0,
secondNumber: secondNumber,
secondNumberDefined: true,
}
log.Debug("%v", prefs)
op := &Operation{OperationType: sliceArrayOpType, Value: sliceArrayOpType.Type, StringValue: value, Preferences: prefs}
return &token{TokenType: operationToken, Operation: op}, nil
}
}
func sliceArrayFirstNumberOnly() yqAction {
return func(rawToken lexer.Token) (*token, error) {
value := rawToken.Value
sliceArrayNumbers := regexp.MustCompile(`\.\[(-?[0-9]+)\:\]`)
matches := sliceArrayNumbers.FindStringSubmatch(value)
log.Debug("sliceArrayFirstNumberOnly value: %v", value)
log.Debug("Matches: %v", matches)
firstNumber, err := parseInt(matches[1])
if err != nil {
return nil, err
}
prefs := sliceArrayPreferences{
firstNumber: firstNumber,
secondNumberDefined: false,
}
log.Debug("%v", prefs)
op := &Operation{OperationType: sliceArrayOpType, Value: sliceArrayOpType.Type, StringValue: value, Preferences: prefs}
return &token{TokenType: operationToken, Operation: op}, nil
}
}
func assignAllCommentsOp(updateAssign bool) yqAction {
return func(rawToken lexer.Token) (*token, error) {
log.Debug("assignAllCommentsOp %v", rawToken.Value)
+33 -98
View File
@@ -5,7 +5,7 @@ import (
"github.com/alecthomas/repr"
"github.com/mikefarah/yq/v4/test"
yaml "gopkg.in/yaml.v3"
"gopkg.in/yaml.v3"
)
type participleLexerScenario struct {
@@ -14,125 +14,60 @@ type participleLexerScenario struct {
}
var participleLexerScenarios = []participleLexerScenario{
{
expression: ".[1:3]",
tokens: []*token{
{
TokenType: operationToken,
Operation: &Operation{
OperationType: sliceArrayOpType,
Value: "SLICE",
StringValue: ".[1:3]",
Preferences: sliceArrayPreferences{firstNumber: 1, secondNumber: 3, secondNumberDefined: true},
},
},
},
},
{
expression: ".[:3]",
tokens: []*token{
{
TokenType: operationToken,
Operation: &Operation{
OperationType: selfReferenceOpType,
StringValue: "SELF",
OperationType: sliceArrayOpType,
Value: "SLICE",
StringValue: ".[:3]",
Preferences: sliceArrayPreferences{firstNumber: 0, secondNumber: 3, secondNumberDefined: true},
},
},
{
TokenType: operationToken,
Operation: &Operation{
OperationType: traverseArrayOpType,
StringValue: "TRAVERSE_ARRAY",
},
},
{
TokenType: openCollect,
},
{
TokenType: operationToken,
Operation: &Operation{
OperationType: valueOpType,
Value: 0,
StringValue: "0",
CandidateNode: &CandidateNode{
Node: &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!int",
Value: "0",
},
},
},
},
{
TokenType: operationToken,
Operation: &Operation{
OperationType: createMapOpType,
Value: "CREATE_MAP",
StringValue: ":",
},
},
{
TokenType: operationToken,
Operation: &Operation{
OperationType: valueOpType,
Value: 3,
StringValue: "3",
CandidateNode: &CandidateNode{
Node: &yaml.Node{
Kind: yaml.Kind(8),
Tag: "!!int",
Value: "3",
},
},
},
},
{
TokenType: closeCollect,
CheckForPostTraverse: true,
Match: "]",
},
},
},
{
expression: ".[-2:]",
expression: ".[1:]",
tokens: []*token{
{
TokenType: operationToken,
Operation: &Operation{
OperationType: selfReferenceOpType,
StringValue: "SELF",
OperationType: sliceArrayOpType,
Value: "SLICE",
StringValue: ".[1:]",
Preferences: sliceArrayPreferences{firstNumber: 1, secondNumber: 0, secondNumberDefined: false},
},
},
},
},
{
expression: ".[-100:-54]",
tokens: []*token{
{
TokenType: operationToken,
Operation: &Operation{
OperationType: traverseArrayOpType,
StringValue: "TRAVERSE_ARRAY",
OperationType: sliceArrayOpType,
Value: "SLICE",
StringValue: ".[-100:-54]",
Preferences: sliceArrayPreferences{firstNumber: -100, secondNumber: -54, secondNumberDefined: true},
},
},
{
TokenType: openCollect,
},
{
TokenType: operationToken,
Operation: &Operation{
OperationType: valueOpType,
Value: -2,
StringValue: "-2",
CandidateNode: &CandidateNode{
Node: &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!int",
Value: "-2",
},
},
},
},
{
TokenType: operationToken,
Operation: &Operation{
OperationType: createMapOpType,
Value: "CREATE_MAP",
StringValue: ":",
},
},
{
TokenType: operationToken,
Operation: &Operation{
OperationType: lengthOpType,
},
},
{
TokenType: closeCollect,
CheckForPostTraverse: true,
Match: "]",
},
},
},
{
+1 -11
View File
@@ -80,9 +80,8 @@ var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Ha
var lineOpType = &operationType{Type: "LINE", NumArgs: 0, Precedence: 50, Handler: lineOperator}
var columnOpType = &operationType{Type: "LINE", NumArgs: 0, Precedence: 50, Handler: columnOperator}
var expressionOpType = &operationType{Type: "EXP", NumArgs: 0, Precedence: 50, Handler: expressionOperator}
var collectOpType = &operationType{Type: "COLLECT", NumArgs: 1, Precedence: 50, Handler: collectOperator}
var sliceArrayOpType = &operationType{Type: "SLICE", NumArgs: 0, Precedence: 50, Handler: sliceArrayOperator}
var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 50, Handler: mapOperator}
var errorOpType = &operationType{Type: "ERROR", NumArgs: 1, Precedence: 50, Handler: errorOperator}
var pickOpType = &operationType{Type: "PICK", NumArgs: 1, Precedence: 50, Handler: pickOperator}
@@ -250,21 +249,12 @@ func guessTagFromCustomType(node *yaml.Node) string {
}
func parseSnippet(value string) (*yaml.Node, error) {
if value == "" {
return &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!null",
}, nil
}
decoder := NewYamlDecoder(ConfiguredYamlPreferences)
err := decoder.Init(strings.NewReader(value))
if err != nil {
return nil, err
}
parsedNode, err := decoder.Decode()
if err != nil {
return nil, err
}
if len(parsedNode.Node.Content) == 0 {
return nil, fmt.Errorf("bad data")
}
+1 -95
View File
@@ -1,11 +1,6 @@
package yqlib
import (
"testing"
"github.com/mikefarah/yq/v4/test"
yaml "gopkg.in/yaml.v3"
)
import "testing"
func TestGetLogger(t *testing.T) {
l := GetLogger()
@@ -13,92 +8,3 @@ func TestGetLogger(t *testing.T) {
t.Fatal("GetLogger should return the yq logger instance, not a copy")
}
}
type parseSnippetScenario struct {
snippet string
expected *yaml.Node
expectedError string
}
var parseSnippetScenarios = []parseSnippetScenario{
{
snippet: ":",
expectedError: "yaml: did not find expected key",
},
{
snippet: "",
expected: &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!null",
},
},
{
snippet: "null",
expected: &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!null",
Value: "null",
Line: 1,
Column: 1,
},
},
{
snippet: "3",
expected: &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!int",
Value: "3",
Line: 1,
Column: 1,
},
},
{
snippet: "cat",
expected: &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!str",
Value: "cat",
Line: 1,
Column: 1,
},
},
{
snippet: "3.1",
expected: &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!float",
Value: "3.1",
Line: 1,
Column: 1,
},
},
{
snippet: "true",
expected: &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!bool",
Value: "true",
Line: 1,
Column: 1,
},
},
}
func TestParseSnippet(t *testing.T) {
for _, tt := range parseSnippetScenarios {
actual, err := parseSnippet(tt.snippet)
if tt.expectedError != "" {
if err == nil {
t.Errorf("Expected error '%v' but it worked!", tt.expectedError)
} else {
test.AssertResultComplexWithContext(t, tt.expectedError, err.Error(), tt.snippet)
}
return
}
if err != nil {
t.Error(tt.snippet)
t.Error(err)
}
test.AssertResultComplexWithContext(t, tt.expected, actual, tt.snippet)
}
}
+2 -2
View File
@@ -98,7 +98,7 @@ func addScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs *yam
// if the lhs is a string, it might be a timestamp in a custom format.
if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 {
_, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value)
_, err := time.Parse(context.GetDateTimeLayout(), lhs.Value)
isDateTime = err == nil
}
@@ -152,7 +152,7 @@ func addDateTimes(layout string, target *CandidateNode, lhs *yaml.Node, rhs *yam
return fmt.Errorf("unable to parse duration [%v]: %w", rhs.Value, err)
}
currentTime, err := parseDateTime(layout, lhs.Value)
currentTime, err := time.Parse(layout, lhs.Value)
if err != nil {
return err
}
-9
View File
@@ -252,15 +252,6 @@ var addOperatorScenarios = []expressionScenario{
"D0, P[], (doc)::a: 2021-01-01T03:10:00Z\n",
},
},
{
description: "Date addition -date only",
skipDoc: true,
document: `a: 2021-01-01`,
expression: `.a += "24h"`,
expected: []string{
"D0, P[], (doc)::a: 2021-01-02T00:00:00Z\n",
},
},
{
description: "Date addition - custom format",
subdescription: "You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",
-4
View File
@@ -2,7 +2,6 @@ package yqlib
import (
"container/list"
"fmt"
yaml "gopkg.in/yaml.v3"
)
@@ -245,9 +244,6 @@ func applyAlias(node *yaml.Node, alias *yaml.Node, aliasIndex int, newContent Co
if alias == nil {
return nil
}
if alias.Kind != yaml.MappingNode {
return fmt.Errorf("merge anchor only supports maps, got %v instead", alias.Tag)
}
for index := 0; index < len(alias.Content); index = index + 2 {
keyNode := alias.Content[index]
log.Debugf("applying alias key %v", keyNode.Value)
@@ -36,13 +36,6 @@ thingTwo:
`
var anchorOperatorScenarios = []expressionScenario{
{
skipDoc: true,
description: "merge anchor not map",
document: "a: &a\n - 0\nc:\n <<: [*a]\n",
expectedError: "merge anchor only supports maps, got !!seq instead",
expression: "explode(.)",
},
{
description: "Merge one map",
subdescription: "see https://yaml.org/type/merge.html",
-23
View File
@@ -1,23 +0,0 @@
package yqlib
import (
"testing"
)
var arrayToMapScenarios = []expressionScenario{
{
description: "Simple example",
document: `cool: [null, null, hello]`,
expression: `.cool |= array_to_map`,
expected: []string{
"D0, P[], (doc)::cool:\n 2: hello\n",
},
},
}
func TestArrayToMapScenarios(t *testing.T) {
for _, tt := range arrayToMapScenarios {
testScenario(t, &tt)
}
documentOperatorScenarios(t, "array-to-map", arrayToMapScenarios)
}
-9
View File
@@ -33,15 +33,6 @@ var containsOperatorScenarios = []expressionScenario{
"D0, P[], (!!bool)::true\n",
},
},
{
description: "Array has a subset array",
subdescription: "Subtract the superset array from the subset, if there's anything left, it's not a subset",
document: `["foobar", "foobaz", "blarp"]`,
expression: `["baz", "bar"] - . | length == 0`,
expected: []string{
"D0, P[], (!!bool)::false\n",
},
},
{
skipDoc: true,
expression: `["dog", "cat", "giraffe"] | contains(["camel"])`,
+2 -13
View File
@@ -50,17 +50,6 @@ func nowOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode
}
func parseDateTime(layout string, datestring string) (time.Time, error) {
parsedTime, err := time.Parse(layout, datestring)
if err != nil && layout == time.RFC3339 {
// try parsing the date time with only the date
return time.Parse("2006-01-02", datestring)
}
return parsedTime, err
}
func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
format, err := getStringParamter("format", d, context, expressionNode.RHS)
layout := context.GetDateTimeLayout()
@@ -73,7 +62,7 @@ func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *Expre
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
parsedTime, err := parseDateTime(layout, candidate.Node.Value)
parsedTime, err := time.Parse(layout, candidate.Node.Value)
if err != nil {
return Context{}, fmt.Errorf("could not parse datetime of [%v]: %w", candidate.GetNicePath(), err)
}
@@ -112,7 +101,7 @@ func tzOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode)
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
parsedTime, err := parseDateTime(layout, candidate.Node.Value)
parsedTime, err := time.Parse(layout, candidate.Node.Value)
if err != nil {
return Context{}, fmt.Errorf("could not parse datetime of [%v] using layout [%v]: %w", candidate.GetNicePath(), layout, err)
}
+1 -1
View File
@@ -25,7 +25,7 @@ var dateTimeOperatorScenarios = []expressionScenario{
},
{
description: "Format: get the day of the week",
document: `a: 2001-12-15`,
document: `a: 2001-12-15T02:59:43.1Z`,
expression: `.a | format_datetime("Monday")`,
expected: []string{
"D0, P[a], (!!str)::Saturday\n",
+1 -1
View File
@@ -13,7 +13,7 @@ import (
func configureEncoder(format PrinterOutputFormat, indent int) Encoder {
switch format {
case JSONOutputFormat:
return NewJSONEncoder(indent, false, false)
return NewJSONEncoder(indent, false)
case PropsOutputFormat:
return NewPropertiesEncoder(true)
case CSVOutputFormat:
+3 -3
View File
@@ -194,7 +194,7 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
},
{
description: "Encode value as xml string",
document: `{a: {cool: {foo: "bar", +@id: hi}}}`,
document: `{a: {cool: {foo: "bar", +id: hi}}}`,
expression: `.a | to_xml`,
expected: []string{
"D0, P[a], (!!str)::<cool id=\"hi\">\n <foo>bar</foo>\n</cool>\n\n",
@@ -202,7 +202,7 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
},
{
description: "Encode value as xml string on a single line",
document: `{a: {cool: {foo: "bar", +@id: hi}}}`,
document: `{a: {cool: {foo: "bar", +id: hi}}}`,
expression: `.a | @xml`,
expected: []string{
"D0, P[a], (!!str)::<cool id=\"hi\"><foo>bar</foo></cool>\n\n",
@@ -210,7 +210,7 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
},
{
description: "Encode value as xml string with custom indentation",
document: `{a: {cool: {foo: "bar", +@id: hi}}}`,
document: `{a: {cool: {foo: "bar", +id: hi}}}`,
expression: `{"cat": .a | to_xml(1)}`,
expected: []string{
"D0, P[], (!!map)::cat: |\n <cool id=\"hi\">\n <foo>bar</foo>\n </cool>\n",
-16
View File
@@ -1,16 +0,0 @@
package yqlib
type expressionOpPreferences struct {
expression string
}
func expressionOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
prefs := expressionNode.Operation.Preferences.(expressionOpPreferences)
expNode, err := ExpressionParser.ParseExpression(prefs.expression)
if err != nil {
return Context{}, err
}
return d.GetMatchingNodes(context, expNode)
}
-19
View File
@@ -4,16 +4,6 @@ import (
"testing"
)
var documentToPrune = `
parentA: bob
parentB:
child1: i am child1
child2: i am child2
parentC:
child1: me child1
child2: me child2
`
var pathOperatorScenarios = []expressionScenario{
{
description: "Map path",
@@ -88,15 +78,6 @@ var pathOperatorScenarios = []expressionScenario{
"D0, P[], ()::a:\n b: things\n",
},
},
{
description: "Set path to prune deep paths",
subdescription: "Like pick but recursive. This uses `ireduce` to deeply set the selected paths into an empty object,",
document: documentToPrune,
expression: "(.parentB.child2, .parentC.child1) as $i\n ireduce({}; setpath($i | path; $i))",
expected: []string{
"D0, P[], (!!map)::parentB:\n child2: i am child2\nparentC:\n child1: me child1\n",
},
},
{
description: "Set array path",
document: `a: [cat, frog]`,
+18 -27
View File
@@ -2,52 +2,43 @@ package yqlib
import (
"container/list"
"fmt"
yaml "gopkg.in/yaml.v3"
)
func getSliceNumber(d *dataTreeNavigator, context Context, node *CandidateNode, expressionNode *ExpressionNode) (int, error) {
result, err := d.GetMatchingNodes(context.SingleChildContext(node), expressionNode)
if err != nil {
return 0, err
}
if result.MatchingNodes.Len() != 1 {
return 0, fmt.Errorf("expected to find 1 number, got %v instead", result.MatchingNodes.Len())
}
return parseInt(result.MatchingNodes.Front().Value.(*CandidateNode).Node.Value)
type sliceArrayPreferences struct {
firstNumber int
secondNumber int
secondNumberDefined bool
}
func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debug("slice array operator!")
log.Debug("lhs: %v", expressionNode.LHS.Operation.toString())
log.Debug("rhs: %v", expressionNode.RHS.Operation.toString())
lhs, err := d.GetMatchingNodes(context, expressionNode.LHS)
if err != nil {
return Context{}, err
}
prefs := expressionNode.Operation.Preferences.(sliceArrayPreferences)
firstNumber := prefs.firstNumber
secondNumber := prefs.secondNumber
results := list.New()
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
lhsNode := el.Value.(*CandidateNode)
original := unwrapDoc(lhsNode.Node)
firstNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.LHS)
if err != nil {
return Context{}, err
}
relativeFirstNumber := firstNumber
if relativeFirstNumber < 0 {
relativeFirstNumber = len(original.Content) + firstNumber
}
secondNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.RHS)
if err != nil {
return Context{}, err
}
relativeSecondNumber := secondNumber
if relativeSecondNumber < 0 {
relativeSecondNumber = len(original.Content) + secondNumber
relativeSecondNumber := len(original.Content)
if prefs.secondNumberDefined {
relativeSecondNumber = secondNumber
if relativeSecondNumber < 0 {
relativeSecondNumber = len(original.Content) + secondNumber
}
}
log.Debug("calculateIndicesToTraverse: slice from %v to %v", relativeFirstNumber, relativeSecondNumber)
-9
View File
@@ -37,15 +37,6 @@ var sliceArrayScenarios = []expressionScenario{
"D0, P[], (!!seq)::- dog\n- frog\n",
},
},
{
description: "Inserting into the middle of an array",
subdescription: "using an expression to find the index",
document: `[cat, dog, frog, cow]`,
expression: `(.[] | select(. == "dog") | key + 1) as $pos | .[0:($pos)] + ["rabbit"] + .[$pos:]`,
expected: []string{
"D0, P[], (!!seq)::- cat\n- dog\n- rabbit\n- frog\n- cow\n",
},
},
{
skipDoc: true,
document: `[[cat, dog, frog, cow], [apple, banana, grape, mango]]`,
+13 -47
View File
@@ -6,7 +6,6 @@ import (
"sort"
"strconv"
"strings"
"time"
yaml "gopkg.in/yaml.v3"
)
@@ -49,7 +48,7 @@ func sortByOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
log.Debug("going to compare %v by %v", NodeToString(candidate.CreateReplacement(originalNode)), NodeToString(candidate.CreateReplacement(nodeToCompare)))
sortableArray[i] = sortableNode{Node: originalNode, NodeToCompare: nodeToCompare, dateTimeLayout: context.GetDateTimeLayout()}
sortableArray[i] = sortableNode{Node: originalNode, NodeToCompare: nodeToCompare}
if nodeToCompare.Kind != yaml.ScalarNode {
return Context{}, fmt.Errorf("sort only works for scalars, got %v", nodeToCompare.Tag)
@@ -71,9 +70,8 @@ func sortByOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
}
type sortableNode struct {
Node *yaml.Node
NodeToCompare *yaml.Node
dateTimeLayout string
Node *yaml.Node
NodeToCompare *yaml.Node
}
type sortableNodeArray []sortableNode
@@ -85,37 +83,15 @@ func (a sortableNodeArray) Less(i, j int) bool {
lhs := a[i].NodeToCompare
rhs := a[j].NodeToCompare
lhsTag := lhs.Tag
rhsTag := rhs.Tag
if !strings.HasPrefix(lhsTag, "!!") {
// custom tag - we have to have a guess
lhsTag = guessTagFromCustomType(lhs)
}
if !strings.HasPrefix(rhsTag, "!!") {
// custom tag - we have to have a guess
rhsTag = guessTagFromCustomType(rhs)
}
isDateTime := lhsTag == "!!timestamp" && rhsTag == "!!timestamp"
layout := a[i].dateTimeLayout
// if the lhs is a string, it might be a timestamp in a custom format.
if lhsTag == "!!str" && layout != time.RFC3339 {
_, errLhs := parseDateTime(layout, lhs.Value)
_, errRhs := parseDateTime(layout, rhs.Value)
isDateTime = errLhs == nil && errRhs == nil
}
if lhsTag == "!!null" && rhsTag != "!!null" {
if lhs.Tag == "!!null" && rhs.Tag != "!!null" {
return true
} else if lhsTag != "!!null" && rhsTag == "!!null" {
} else if lhs.Tag != "!!null" && rhs.Tag == "!!null" {
return false
} else if lhsTag == "!!bool" && rhsTag != "!!bool" {
} else if lhs.Tag == "!!bool" && rhs.Tag != "!!bool" {
return true
} else if lhsTag != "!!bool" && rhsTag == "!!bool" {
} else if lhs.Tag != "!!bool" && rhs.Tag == "!!bool" {
return false
} else if lhsTag == "!!bool" && rhsTag == "!!bool" {
} else if lhs.Tag == "!!bool" && rhs.Tag == "!!bool" {
lhsTruthy, err := isTruthyNode(lhs)
if err != nil {
panic(fmt.Errorf("could not parse %v as boolean: %w", lhs.Value, err))
@@ -127,19 +103,9 @@ func (a sortableNodeArray) Less(i, j int) bool {
}
return !lhsTruthy && rhsTruthy
} else if isDateTime {
lhsTime, err := parseDateTime(layout, lhs.Value)
if err != nil {
log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %w", lhs.Value, layout, err)
return strings.Compare(lhs.Value, rhs.Value) < 0
}
rhsTime, err := parseDateTime(layout, rhs.Value)
if err != nil {
log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %w", rhs.Value, layout, err)
return strings.Compare(lhs.Value, rhs.Value) < 0
}
return lhsTime.Before(rhsTime)
} else if lhsTag == "!!int" && rhsTag == "!!int" {
} else if lhs.Tag != rhs.Tag || lhs.Tag == "!!str" {
return strings.Compare(lhs.Value, rhs.Value) < 0
} else if lhs.Tag == "!!int" && rhs.Tag == "!!int" {
_, lhsNum, err := parseInt64(lhs.Value)
if err != nil {
panic(err)
@@ -149,7 +115,7 @@ func (a sortableNodeArray) Less(i, j int) bool {
panic(err)
}
return lhsNum < rhsNum
} else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") {
} else if (lhs.Tag == "!!int" || lhs.Tag == "!!float") && (rhs.Tag == "!!int" || rhs.Tag == "!!float") {
lhsNum, err := strconv.ParseFloat(lhs.Value, 64)
if err != nil {
panic(err)
@@ -161,5 +127,5 @@ func (a sortableNodeArray) Less(i, j int) bool {
return lhsNum < rhsNum
}
return strings.Compare(lhs.Value, rhs.Value) < 0
return true
}
-8
View File
@@ -54,14 +54,6 @@ var sortByOperatorScenarios = []expressionScenario{
"D0, P[], (!!seq)::[{a: 1}, {a: 10}, {a: 100}]\n",
},
},
{
description: "Sort by custom date field",
document: `[{a: "12-Jun-2011"},{a: "23-Dec-2010"},{a: "10-Aug-2011"}]`,
expression: `with_dtf("02-Jan-2006"; sort_by(.a))`,
expected: []string{
"D0, P[], (!!seq)::[{a: \"23-Dec-2010\"}, {a: \"12-Jun-2011\"}, {a: \"10-Aug-2011\"}]\n",
},
},
{
skipDoc: true,
document: "[{a: 1.1},{a: 1.001},{a: 1.01}]",
+3 -3
View File
@@ -92,10 +92,10 @@ func subtractScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs
rhsTag = guessTagFromCustomType(rhs)
}
isDateTime := lhsTag == "!!timestamp"
isDateTime := lhs.Tag == "!!timestamp"
// if the lhs is a string, it might be a timestamp in a custom format.
if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 {
_, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value)
_, err := time.Parse(context.GetDateTimeLayout(), lhs.Value)
isDateTime = err == nil
}
@@ -151,7 +151,7 @@ func subtractDateTime(layout string, target *CandidateNode, lhs *yaml.Node, rhs
return fmt.Errorf("unable to parse duration [%v]: %w", rhs.Value, err)
}
currentTime, err := parseDateTime(layout, lhs.Value)
currentTime, err := time.Parse(layout, lhs.Value)
if err != nil {
return err
}
-9
View File
@@ -102,15 +102,6 @@ var subtractOperatorScenarios = []expressionScenario{
"D0, P[], (doc)::a: 2021-01-01T00:00:00Z\n",
},
},
{
description: "Date subtraction - only date",
skipDoc: true,
document: `a: 2021-01-01`,
expression: `.a -= "24h"`,
expected: []string{
"D0, P[], (doc)::a: 2020-12-31T00:00:00Z\n",
},
},
{
description: "Date subtraction - custom format",
subdescription: "Use with_dtf to specify your datetime format. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",
-4
View File
@@ -80,10 +80,6 @@ func traverseArrayOperator(d *dataTreeNavigator, context Context, expressionNode
// BUT we still return the original context back (see jq)
// https://stedolan.github.io/jq/manual/#Variable/SymbolicBindingOperator:...as$identifier|...
if expressionNode.RHS != nil && expressionNode.RHS.RHS != nil && expressionNode.RHS.RHS.Operation.OperationType == createMapOpType {
return sliceArrayOperator(d, context, expressionNode.RHS.RHS)
}
lhs, err := d.GetMatchingNodes(context, expressionNode.LHS)
if err != nil {
return Context{}, err
+1 -1
View File
@@ -31,7 +31,7 @@ type expressionScenario struct {
}
func TestMain(m *testing.M) {
logging.SetLevel(logging.ERROR, "")
logging.SetLevel(logging.DEBUG, "")
Now = func() time.Time {
return time.Date(2021, time.May, 19, 1, 2, 3, 4, time.UTC)
}
+1 -1
View File
@@ -314,7 +314,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
var writer = bufio.NewWriter(&output)
// note printDocSeparators is true, it should still not print document separators
// when outputing JSON.
printer := NewPrinter(NewJSONEncoder(0, false, false), NewSinglePrinterWriter(writer))
printer := NewPrinter(NewJSONEncoder(0, false), NewSinglePrinterWriter(writer))
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
if err != nil {
+8 -16
View File
@@ -147,14 +147,6 @@ var propertyScenarios = []formatScenario{
expected: expectedDecodedYaml,
scenarioType: "decode",
},
{
description: "Decode properties - array should be a map",
subdescription: "If you have a numeric map key in your property files, use array_to_map to convert them to maps.",
input: `things.10 = mike`,
expression: `.things |= array_to_map`,
expected: "things:\n 10: mike\n",
scenarioType: "decode",
},
{
description: "does not expand automatically",
skipDoc: true,
@@ -228,7 +220,7 @@ func documentUnwrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario)
}
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true))))
writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true))))
}
func documentWrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) {
@@ -253,7 +245,7 @@ func documentWrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) {
}
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false))))
writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false))))
}
func documentDecodePropertyScenario(w *bufio.Writer, s formatScenario) {
@@ -278,7 +270,7 @@ func documentDecodePropertyScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))))
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))))
}
func documentRoundTripPropertyScenario(w *bufio.Writer, s formatScenario) {
@@ -303,7 +295,7 @@ func documentRoundTripPropertyScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", mustProcessFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(true))))
writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(true))))
}
func documentPropertyScenario(t *testing.T, w *bufio.Writer, i interface{}) {
@@ -330,13 +322,13 @@ func TestPropertyScenarios(t *testing.T) {
for _, s := range propertyScenarios {
switch s.scenarioType {
case "":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true)), s.description)
case "decode":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
case "encode-wrapped":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false)), s.description)
case "roundtrip":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(true)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(true)), s.description)
default:
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
+1 -1
View File
@@ -14,7 +14,7 @@ type XmlPreferences struct {
func NewDefaultXmlPreferences() XmlPreferences {
return XmlPreferences{
AttributePrefix: "+@",
AttributePrefix: "+",
ContentName: "+content",
StrictMode: false,
KeepNamespace: true,
+28 -57
View File
@@ -58,7 +58,7 @@ cat:
d:
# in d before
z:
+@sweet: cool
+sweet: cool
# in d after
# in y after
# in_cat_after
@@ -98,11 +98,11 @@ cat:
d:
- # in d before
z:
+@sweet: cool
+sweet: cool
# in d after
- # in d2 before
z:
+@sweet: cool2
+sweet: cool2
# in d2 after
# in y after
# in_cat_after
@@ -165,16 +165,16 @@ const inputXMLWithNamespacedAttr = `
const expectedYAMLWithNamespacedAttr = `+p_xml: version="1.0"
map:
+@xmlns: some-namespace
+@xmlns:xsi: some-instance
+@some-instance:schemaLocation: some-url
+xmlns: some-namespace
+xmlns:xsi: some-instance
+some-instance:schemaLocation: some-url
`
const expectedYAMLWithRawNamespacedAttr = `+p_xml: version="1.0"
map:
+@xmlns: some-namespace
+@xmlns:xsi: some-instance
+@xsi:schemaLocation: some-url
+xmlns: some-namespace
+xmlns:xsi: some-instance
+xsi:schemaLocation: some-url
`
const xmlWithCustomDtd = `
@@ -251,13 +251,13 @@ var xmlScenarios = []formatScenario{
description: "Parse xml: attributes",
subdescription: "Attributes are converted to fields, with the default attribute prefix '+'. Use '--xml-attribute-prefix` to set your own.",
input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">\n <legs>7</legs>\n</cat>",
expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n +@legs: \"4\"\n legs: \"7\"\n",
expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n +legs: \"4\"\n legs: \"7\"\n",
},
{
description: "Parse xml: attributes with content",
subdescription: "Content is added as a field, using the default content name of `+content`. Use `--xml-content-name` to set your own.",
input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">meow</cat>",
expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n +content: meow\n +@legs: \"4\"\n",
expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n +content: meow\n +legs: \"4\"\n",
},
{
description: "Parse xml: custom dtd",
@@ -367,42 +367,21 @@ var xmlScenarios = []formatScenario{
{
description: "Encode xml: attributes",
subdescription: "Fields with the matching xml-attribute-prefix are assumed to be attributes.",
input: "cat:\n +@name: tiger\n meows: true\n",
input: "cat:\n +name: tiger\n meows: true\n",
expected: "<cat name=\"tiger\">\n <meows>true</meows>\n</cat>\n",
scenarioType: "encode",
},
{
description: "double prefix",
skipDoc: true,
input: "cat:\n +@+@name: tiger\n meows: true\n",
input: "cat:\n ++@name: tiger\n meows: true\n",
expected: "<cat +@name=\"tiger\">\n <meows>true</meows>\n</cat>\n",
scenarioType: "encode",
},
{
description: "arrays cannot be encoded",
skipDoc: true,
input: "[cat, dog, fish]",
expectedError: "cannot encode !!seq to XML - only maps can be encoded",
scenarioType: "encode-error",
},
{
description: "arrays cannot be encoded - 2",
skipDoc: true,
input: "[cat, dog]",
expectedError: "cannot encode !!seq to XML - only maps can be encoded",
scenarioType: "encode-error",
},
{
description: "scalars cannot be encoded",
skipDoc: true,
input: "mike",
expectedError: "cannot encode !!str to XML - only maps can be encoded",
scenarioType: "encode-error",
},
{
description: "Encode xml: attributes with content",
subdescription: "Fields with the matching xml-content-name is assumed to be content.",
input: "cat:\n +@name: tiger\n +content: cool\n",
input: "cat:\n +name: tiger\n +content: cool\n",
expected: "<cat name=\"tiger\">cool</cat>\n",
scenarioType: "encode",
},
@@ -439,31 +418,23 @@ var xmlScenarios = []formatScenario{
func testXMLScenario(t *testing.T, s formatScenario) {
switch s.scenarioType {
case "", "decode":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(4, false, ConfiguredYamlPreferences)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(4, false, ConfiguredYamlPreferences)), s.description)
case "encode":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description)
case "roundtrip":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description)
case "decode-keep-ns":
prefs := NewDefaultXmlPreferences()
prefs.KeepNamespace = true
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
case "decode-raw-token":
prefs := NewDefaultXmlPreferences()
prefs.UseRawToken = true
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
case "roundtrip-skip-directives":
prefs := NewDefaultXmlPreferences()
prefs.SkipDirectives = true
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs)), s.description)
case "encode-error":
result, err := processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, NewDefaultXmlPreferences()))
if err == nil {
t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result)
} else {
test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description)
}
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs)), s.description)
default:
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
}
@@ -513,7 +484,7 @@ func documentXMLDecodeScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=xml '%v' sample.xml\n```\n", expression))
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences))))
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences))))
}
func documentXMLDecodeKeepNsScenario(w *bufio.Writer, s formatScenario) {
@@ -532,12 +503,12 @@ func documentXMLDecodeKeepNsScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, "will output\n")
prefs := NewDefaultXmlPreferences()
prefs.KeepNamespace = true
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs))))
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs))))
prefsWithout := NewDefaultXmlPreferences()
prefs.KeepNamespace = false
writeOrPanic(w, "instead of\n")
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(2, prefsWithout))))
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(2, prefsWithout))))
}
func documentXMLDecodeKeepNsRawTokenScenario(w *bufio.Writer, s formatScenario) {
@@ -558,13 +529,13 @@ func documentXMLDecodeKeepNsRawTokenScenario(w *bufio.Writer, s formatScenario)
prefs := NewDefaultXmlPreferences()
prefs.KeepNamespace = true
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs))))
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs))))
prefsWithout := NewDefaultXmlPreferences()
prefsWithout.KeepNamespace = false
writeOrPanic(w, "instead of\n")
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(2, prefsWithout))))
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(2, prefsWithout))))
}
func documentXMLEncodeScenario(w *bufio.Writer, s formatScenario) {
@@ -582,7 +553,7 @@ func documentXMLEncodeScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, "```bash\nyq -o=xml '.' sample.yml\n```\n")
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences))))
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences))))
}
func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) {
@@ -600,7 +571,7 @@ func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, "```bash\nyq -p=xml -o=xml '.' sample.xml\n```\n")
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences))))
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences))))
}
func documentXMLSkipDirectrivesScenario(w *bufio.Writer, s formatScenario) {
@@ -620,7 +591,7 @@ func documentXMLSkipDirectrivesScenario(w *bufio.Writer, s formatScenario) {
prefs := NewDefaultXmlPreferences()
prefs.SkipDirectives = true
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs))))
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs))))
}
func TestXMLScenarios(t *testing.T) {
-21
View File
@@ -1,28 +1,7 @@
4.30.1:
- XML users note: the default attribute prefix has change to `+@` to avoid naming conflicts!
- Can use expressions in slice #1419
- Fixed unhandled exception when decoding CSV thanks @washanhanzi
- Added array_to_map operator for #1415
- Fixed sorting by date #1412
- Added check to ensure only maps can be encoded to XML #1408
- Check merge alias is a map #1425
- Explicity setting unwrap flag works for json output #437, #1409
- Bumped go version
4.29.2:
- Fixed null pointer exception when parsing CSV with empty field #1404
4.29.1:
- Fixed Square brackets removing update #1342
- Added slice array operator (.[10:15]) #44
- XML decoder/encoder now parses directives and proc instructions (#1344). Please use the new skip flags [documented here](https://mikefarah.gitbook.io/yq/usage/xml) to ignore them.
- XML users note that the default attribute prefix will change to `+@` in the 4.30 release to avoid naming conflicts!
- Improved comment handling of decoders (breaking change for yqlib users sorry)
- Fixed load operator bug when loading yaml file with multiple documents
- Bumped Go compiler version
- Bumped dependencies
4.28.2:
- Fixed Github Actions issues (thanks @mattphelps-8451)
+1 -1
View File
@@ -1,5 +1,5 @@
name: yq
version: '4.30.1'
version: '4.28.2'
summary: A lightweight and portable command-line YAML processor
description: |
The aim of the project is to be the jq or sed of yaml files.