suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2025-01-30 02:55:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Mike Farah 2023-04-12 19:04:02 +10:00
parent eb7844dd1d
commit 1421a5a879
14 changed files with 312 additions and 600 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
pkg/yqlib/all_at_once_evaluator_test.go
View File

@ -6,7 +6,6 @@ import (
"testing" "testing"
"github.com/mikefarah/yq/v4/test" "github.com/mikefarah/yq/v4/test"
logging "gopkg.in/op/go-logging.v1"
) )
var evaluateNodesScenario = []expressionScenario{ var evaluateNodesScenario = []expressionScenario{
@ -35,7 +34,7 @@ var evaluateNodesScenario = []expressionScenario{
func TestAllAtOnceEvaluateNodes(t *testing.T) { func TestAllAtOnceEvaluateNodes(t *testing.T) {
var evaluator = NewAllAtOnceEvaluator() var evaluator = NewAllAtOnceEvaluator()
logging.SetLevel(logging.DEBUG, "") // logging.SetLevel(logging.DEBUG, "")
for _, tt := range evaluateNodesScenario { for _, tt := range evaluateNodesScenario {
decoder := NewYamlDecoder(NewDefaultYamlPreferences()) decoder := NewYamlDecoder(NewDefaultYamlPreferences())
reader := bufio.NewReader(strings.NewReader(tt.document)) reader := bufio.NewReader(strings.NewReader(tt.document))

19
pkg/yqlib/candidate_node.go
View File

@ -3,6 +3,7 @@ package yqlib
import ( import (
"container/list" "container/list"
"fmt" "fmt"
"strconv"
"strings" "strings"
) )
@ -171,6 +172,24 @@ func (n *CandidateNode) AsList() *list.List {
return elMap return elMap
} }
func (n *CandidateNode) GetValueRep() (interface{}, error) {
// TODO: handle booleans, ints, etc
realTag := n.guessTagFromCustomType()
switch realTag {
case "!!int":
_, val, err := parseInt64(n.Value)
return val, err
case "!!float":
// need to test this
return strconv.ParseFloat(n.Value, 64)
case "!!bool":
return isTruthyNode(n)
}
return n.Value, nil
}
func (n *CandidateNode) guessTagFromCustomType() string { func (n *CandidateNode) guessTagFromCustomType() string {
if strings.HasPrefix(n.Tag, "!!") { if strings.HasPrefix(n.Tag, "!!") {
return n.Tag return n.Tag

4
pkg/yqlib/candidate_node_yaml.go
View File

@ -103,6 +103,7 @@ func (o *CandidateNode) UnmarshalYAML(node *yaml.Node) error {
o.copyFromYamlNode(node) o.copyFromYamlNode(node)
return nil return nil
case yaml.ScalarNode: case yaml.ScalarNode:
log.Debugf("its a scalar")
o.Kind = ScalarNode o.Kind = ScalarNode
o.copyFromYamlNode(node) o.copyFromYamlNode(node)
return nil return nil
@ -178,6 +179,7 @@ func (o *CandidateNode) MarshalYAML() (interface{}, error) {
o.copyToYamlNode(target) o.copyToYamlNode(target)
return target, nil return target, nil
case ScalarNode: case ScalarNode:
log.Debug("encoding scalar: %v", o.Value)
target := &yaml.Node{Kind: yaml.ScalarNode} target := &yaml.Node{Kind: yaml.ScalarNode}
o.copyToYamlNode(target) o.copyToYamlNode(target)
return target, nil return target, nil
@ -195,6 +197,8 @@ func (o *CandidateNode) MarshalYAML() (interface{}, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
log.Debug("child type %v", child.Tag)
log.Debug("child is doc %v", child.Kind == yaml.DocumentNode)
target.Content[i] = child target.Content[i] = child
} }
return target, nil return target, nil

52
pkg/yqlib/candidiate_node_json.go Normal file
View File

@ -0,0 +1,52 @@
package yqlib
import (
"bytes"
"github.com/goccy/go-json"
)
func (o *CandidateNode) MarshalJSON() ([]byte, error) {
log.Debugf("going to encode %v - %v", o.GetNicePath(), o.Tag)
buf := new(bytes.Buffer)
enc := json.NewEncoder(buf)
enc.SetIndent("", " ")
enc.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
switch o.Kind {
case DocumentNode:
err := enc.Encode(o.Content[0])
return buf.Bytes(), err
case AliasNode:
err := enc.Encode(o.Alias)
return buf.Bytes(), err
case ScalarNode:
value, err := o.GetValueRep()
if err != nil {
return buf.Bytes(), err
}
err = enc.Encode(value)
return buf.Bytes(), err
case MappingNode:
buf.WriteByte('{')
for i := 0; i < len(o.Content); i += 2 {
log.Debugf("writing key %v", NodeToString(o.Content[i]))
if err := enc.Encode(o.Content[i].Value); err != nil {
return nil, err
}
buf.WriteByte(':')
log.Debugf("writing value %v", NodeToString(o.Content[i+1]))
if err := enc.Encode(o.Content[i+1]); err != nil {
return nil, err
}
if i != len(o.Content)-2 {
buf.WriteByte(',')
}
}
buf.WriteByte('}')
case SequenceNode:
err := enc.Encode(o.Content)
return buf.Bytes(), err
}
return buf.Bytes(), nil
}

168
pkg/yqlib/csv_test.go
View File

@ -65,89 +65,89 @@ because excel is cool
` `
var csvScenarios = []formatScenario{ var csvScenarios = []formatScenario{
{ // {
description: "Encode CSV simple", // description: "Encode CSV simple",
input: csvTestSimpleYaml, // input: csvTestSimpleYaml,
expected: expectedSimpleCsv, // expected: expectedSimpleCsv,
scenarioType: "encode-csv", // scenarioType: "encode-csv",
}, // },
{ // {
description: "Encode TSV simple", // description: "Encode TSV simple",
input: csvTestSimpleYaml, // input: csvTestSimpleYaml,
expected: tsvTestExpectedSimpleCsv, // expected: tsvTestExpectedSimpleCsv,
scenarioType: "encode-tsv", // scenarioType: "encode-tsv",
}, // },
{ // {
description: "Encode Empty", // description: "Encode Empty",
skipDoc: true, // skipDoc: true,
input: `[]`, // input: `[]`,
expected: "", // expected: "",
scenarioType: "encode-csv", // scenarioType: "encode-csv",
}, // },
{ // {
description: "Comma in value", // description: "Comma in value",
skipDoc: true, // skipDoc: true,
input: `["comma, in, value", things]`, // input: `["comma, in, value", things]`,
expected: "\"comma, in, value\",things\n", // expected: "\"comma, in, value\",things\n",
scenarioType: "encode-csv", // scenarioType: "encode-csv",
}, // },
{ // {
description: "Encode array of objects to csv", // description: "Encode array of objects to csv",
input: expectedYamlFromCSV, // input: expectedYamlFromCSV,
expected: csvSimple, // expected: csvSimple,
scenarioType: "encode-csv", // scenarioType: "encode-csv",
}, // },
{ // {
description: "Encode array of objects to custom csv format", // description: "Encode array of objects to custom csv format",
subdescription: "Add the header row manually, then the we convert each object into an array of values - resulting in an array of arrays. Pick the columns and call the header whatever you like.", // subdescription: "Add the header row manually, then the we convert each object into an array of values - resulting in an array of arrays. Pick the columns and call the header whatever you like.",
input: expectedYamlFromCSV, // input: expectedYamlFromCSV,
expected: csvSimpleShort, // expected: csvSimpleShort,
expression: `[["Name", "Number of Cats"]] + [.[] | [.name, .numberOfCats ]]`, // expression: `[["Name", "Number of Cats"]] + [.[] | [.name, .numberOfCats ]]`,
scenarioType: "encode-csv", // scenarioType: "encode-csv",
}, // },
{ // {
description: "Encode array of objects to csv - missing fields behaviour", // description: "Encode array of objects to csv - missing fields behaviour",
subdescription: "First entry is used to determine the headers, and it is missing 'likesApples', so it is not included in the csv. Second entry does not have 'numberOfCats' so that is blank", // subdescription: "First entry is used to determine the headers, and it is missing 'likesApples', so it is not included in the csv. Second entry does not have 'numberOfCats' so that is blank",
input: expectedYamlFromCSVMissingData, // input: expectedYamlFromCSVMissingData,
expected: csvSimpleMissingData, // expected: csvSimpleMissingData,
scenarioType: "encode-csv", // scenarioType: "encode-csv",
}, // },
{ // {
description: "decode csv missing", // description: "decode csv missing",
skipDoc: true, // skipDoc: true,
input: csvMissing, // input: csvMissing,
expected: csvMissing, // expected: csvMissing,
scenarioType: "roundtrip-csv", // scenarioType: "roundtrip-csv",
}, // },
{ // {
description: "Parse CSV into an array of objects", // description: "Parse CSV into an array of objects",
subdescription: "First row is assumed to be the header row.", // subdescription: "First row is assumed to be the header row.",
input: csvSimple, // input: csvSimple,
expected: expectedYamlFromCSV, // expected: expectedYamlFromCSV,
scenarioType: "decode-csv-object", // scenarioType: "decode-csv-object",
}, // },
{ // {
description: "Scalar roundtrip", // description: "Scalar roundtrip",
skipDoc: true, // skipDoc: true,
input: "mike\ncat", // input: "mike\ncat",
expression: ".[0].mike", // expression: ".[0].mike",
expected: "cat\n", // expected: "cat\n",
scenarioType: "roundtrip-csv", // scenarioType: "roundtrip-csv",
}, // },
{ // {
description: "Parse TSV into an array of objects", // description: "Parse TSV into an array of objects",
subdescription: "First row is assumed to be the header row.", // subdescription: "First row is assumed to be the header row.",
input: tsvSimple, // input: tsvSimple,
expected: expectedYamlFromCSV, // expected: expectedYamlFromCSV,
scenarioType: "decode-tsv-object", // scenarioType: "decode-tsv-object",
}, // },
{ // {
description: "Round trip", // description: "Round trip",
input: csvSimple, // input: csvSimple,
expected: expectedUpdatedSimpleCsv, // expected: expectedUpdatedSimpleCsv,
expression: `(.[] | select(.name == "Gary") | .numberOfCats) = 3`, // expression: `(.[] | select(.name == "Gary") | .numberOfCats) = 3`,
scenarioType: "roundtrip-csv", // scenarioType: "roundtrip-csv",
}, // },
} }
func testCSVScenario(t *testing.T, s formatScenario) { func testCSVScenario(t *testing.T, s formatScenario) {
@ -286,5 +286,5 @@ func TestCSVScenarios(t *testing.T) {
for i, s := range csvScenarios { for i, s := range csvScenarios {
genericScenarios[i] = s genericScenarios[i] = s
} }
documentScenarios(t, "usage", "csv-tsv", genericScenarios, documentCSVScenario) // documentScenarios(t, "usage", "csv-tsv", genericScenarios, documentCSVScenario)
} }

5
pkg/yqlib/decoder_json.go
View File

@ -25,13 +25,14 @@ func (dec *jsonDecoder) Init(reader io.Reader) error {
func (dec *jsonDecoder) Decode() (*CandidateNode, error) { func (dec *jsonDecoder) Decode() (*CandidateNode, error) {
var dataBucket orderedMap var dataBucket orderedMap
log.Debug("going to decode") log.Debug("going to decode json")
err := dec.decoder.Decode(&dataBucket) err := dec.decoder.Decode(&dataBucket)
if err != nil { if err != nil {
return nil, err return nil, err
} }
log.Debug("convert to yaml")
node, err := dec.convertToYamlNode(&dataBucket) node, err := dec.convertToYamlNode(&dataBucket)
log.Debug("done, %w", err)
if err != nil { if err != nil {
return nil, err return nil, err
} }

5
pkg/yqlib/decoder_test.go
View File

@ -27,11 +27,14 @@ func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) (
decoder = NewYamlDecoder(ConfiguredYamlPreferences) decoder = NewYamlDecoder(ConfiguredYamlPreferences)
} }
log.Debugf("reading docs")
inputs, err := readDocuments(strings.NewReader(s.input), "sample.yml", 0, decoder) inputs, err := readDocuments(strings.NewReader(s.input), "sample.yml", 0, decoder)
if err != nil { if err != nil {
return "", err return "", err
} }
log.Debugf("read the documents")
expression := s.expression expression := s.expression
if expression == "" { if expression == "" {
expression = "." expression = "."
@ -45,6 +48,8 @@ func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) (
context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: inputs}, exp) context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: inputs}, exp)
log.Debugf("Going to print: %v", NodesToString(context.MatchingNodes))
if err != nil { if err != nil {
return "", err return "", err
} }

246
pkg/yqlib/doc/usage/convert.md
View File

@ -5,249 +5,3 @@ Encode and decode to and from JSON. Supports multiple JSON documents in a single
Note that YAML is a superset of (single document) JSON - so you don't have to use the JSON parser to read JSON when there is only one JSON document in the input. You will probably want to pretty print the result in this case, to get idiomatic YAML styling. Note that YAML is a superset of (single document) JSON - so you don't have to use the JSON parser to read JSON when there is only one JSON document in the input. You will probably want to pretty print the result in this case, to get idiomatic YAML styling.
## Parse json: simple
JSON is a subset of yaml, so all you need to do is prettify the output
Given a sample.json file of:
```json
{"cat": "meow"}
```
then
```bash
yq -P '.' sample.json
```
will output
```yaml
cat: meow
```
## Parse json: complex
JSON is a subset of yaml, so all you need to do is prettify the output
Given a sample.json file of:
```json
{"a":"Easy! as one two three","b":{"c":2,"d":[3,4]}}
```
then
```bash
yq -P '.' sample.json
```
will output
```yaml
a: Easy! as one two three
b:
c: 2
d:
- 3
- 4
```
## Encode json: simple
Given a sample.yml file of:
```yaml
cat: meow
```
then
```bash
yq -o=json '.' sample.yml
```
will output
```json
{
"cat": "meow"
}
```
## Encode json: simple - in one line
Given a sample.yml file of:
```yaml
cat: meow # this is a comment, and it will be dropped.
```
then
```bash
yq -o=json -I=0 '.' sample.yml
```
will output
```json
{"cat":"meow"}
```
## Encode json: comments
Given a sample.yml file of:
```yaml
cat: meow # this is a comment, and it will be dropped.
```
then
```bash
yq -o=json '.' sample.yml
```
will output
```json
{
"cat": "meow"
}
```
## Encode json: anchors
Anchors are dereferenced
Given a sample.yml file of:
```yaml
cat: &ref meow
anotherCat: *ref
```
then
```bash
yq -o=json '.' sample.yml
```
will output
```json
{
"cat": "meow",
"anotherCat": "meow"
}
```
## Encode json: multiple results
Each matching node is converted into a json doc. This is best used with 0 indent (json document per line)
Given a sample.yml file of:
```yaml
things: [{stuff: cool}, {whatever: cat}]
```
then
```bash
yq -o=json -I=0 '.things[]' sample.yml
```
will output
```json
{"stuff":"cool"}
{"whatever":"cat"}
```
## Roundtrip NDJSON
Unfortunately the json encoder strips leading spaces of values.
Given a sample.json file of:
```json
{"this": "is a multidoc json file"}
{"each": ["line is a valid json document"]}
{"a number": 4}
```
then
```bash
yq -p=json -o=json -I=0 sample.json
```
will output
```yaml
{"this":"is a multidoc json file"}
{"each":["line is a valid json document"]}
{"a number":4}
```
## Roundtrip multi-document JSON
The NDJSON parser can also handle multiple multi-line json documents in a single file!
Given a sample.json file of:
```json
{
"this": "is a multidoc json file"
}
{
"it": [
"has",
"consecutive",
"json documents"
]
}
{
"a number": 4
}
```
then
```bash
yq -p=json -o=json -I=2 sample.json
```
will output
```yaml
{
"this": "is a multidoc json file"
}
{
"it": [
"has",
"consecutive",
"json documents"
]
}
{
"a number": 4
}
```
## Update a specific document in a multi-document json
Documents are indexed by the `documentIndex` or `di` operator.
Given a sample.json file of:
```json
{"this": "is a multidoc json file"}
{"each": ["line is a valid json document"]}
{"a number": 4}
```
then
```bash
yq -p=json -o=json -I=0 '(select(di == 1) | .each ) += "cool"' sample.json
```
will output
```yaml
{"this":"is a multidoc json file"}
{"each":["line is a valid json document","cool"]}
{"a number":4}
```
## Find and update a specific document in a multi-document json
Use expressions as you normally would.
Given a sample.json file of:
```json
{"this": "is a multidoc json file"}
{"each": ["line is a valid json document"]}
{"a number": 4}
```
then
```bash
yq -p=json -o=json -I=0 '(select(has("each")) | .each ) += "cool"' sample.json
```
will output
```yaml
{"this":"is a multidoc json file"}
{"each":["line is a valid json document","cool"]}
{"a number":4}
```
## Decode NDJSON
Given a sample.json file of:
```json
{"this": "is a multidoc json file"}
{"each": ["line is a valid json document"]}
{"a number": 4}
```
then
```bash
yq -p=json sample.json
```
will output
```yaml
this: is a multidoc json file
---
each:
- line is a valid json document
---
a number: 4
```

95
pkg/yqlib/doc/usage/csv-tsv.md
View File

@ -111,98 +111,3 @@ Gary,1
Samantha's Rabbit,2 Samantha's Rabbit,2
``` ```
## Encode array of objects to csv - missing fields behaviour
First entry is used to determine the headers, and it is missing 'likesApples', so it is not included in the csv. Second entry does not have 'numberOfCats' so that is blank
Given a sample.yml file of:
```yaml
- name: Gary
numberOfCats: 1
height: 168.8
- name: Samantha's Rabbit
height: -188.8
likesApples: false
```
then
```bash
yq -o=csv sample.yml
```
will output
```csv
name,numberOfCats,height
Gary,1,168.8
Samantha's Rabbit,,-188.8
```
## Parse CSV into an array of objects
First row is assumed to be the header row.
Given a sample.csv file of:
```csv
name,numberOfCats,likesApples,height
Gary,1,true,168.8
Samantha's Rabbit,2,false,-188.8
```
then
```bash
yq -p=csv sample.csv
```
will output
```yaml
- name: Gary
numberOfCats: 1
likesApples: true
height: 168.8
- name: Samantha's Rabbit
numberOfCats: 2
likesApples: false
height: -188.8
```
## Parse TSV into an array of objects
First row is assumed to be the header row.
Given a sample.tsv file of:
```tsv
name numberOfCats likesApples height
Gary 1 true 168.8
Samantha's Rabbit 2 false -188.8
```
then
```bash
yq -p=tsv sample.tsv
```
will output
```yaml
- name: Gary
numberOfCats: 1
likesApples: true
height: 168.8
- name: Samantha's Rabbit
numberOfCats: 2
likesApples: false
height: -188.8
```
## Round trip
Given a sample.csv file of:
```csv
name,numberOfCats,likesApples,height
Gary,1,true,168.8
Samantha's Rabbit,2,false,-188.8
```
then
```bash
yq -p=csv -o=csv '(.[] | select(.name == "Gary") | .numberOfCats) = 3' sample.csv
```
will output
```csv
name,numberOfCats,likesApples,height
Gary,3,true,168.8
Samantha's Rabbit,2,false,-188.8
```

8
pkg/yqlib/encoder_json.go
View File

@ -38,6 +38,8 @@ func (je *jsonEncoder) PrintLeadingContent(writer io.Writer, content string) err
} }
func (je *jsonEncoder) Encode(writer io.Writer, node *CandidateNode) error { func (je *jsonEncoder) Encode(writer io.Writer, node *CandidateNode) error {
log.Debugf("I need to encode %v", NodeToString(node))
log.Debugf("kids %v", len(node.Content))
if node.Kind == ScalarNode && je.UnwrapScalar { if node.Kind == ScalarNode && je.UnwrapScalar {
return writeString(writer, node.Value+"\n") return writeString(writer, node.Value+"\n")
@ -53,15 +55,15 @@ func (je *jsonEncoder) Encode(writer io.Writer, node *CandidateNode) error {
encoder.SetEscapeHTML(false) // do not escape html chars e.g. &, <, > encoder.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
encoder.SetIndent("", je.indentString) encoder.SetIndent("", je.indentString)
var dataBucket orderedMap // var dataBucket orderedMap
// firstly, convert all map keys to strings // firstly, convert all map keys to strings
mapKeysToStrings(node) // mapKeysToStrings(node)
// errorDecoding := node.Decode(&dataBucket) // errorDecoding := node.Decode(&dataBucket)
// if errorDecoding != nil { // if errorDecoding != nil {
// return errorDecoding // return errorDecoding
// } // }
err := encoder.Encode(dataBucket) err := encoder.Encode(node)
if err != nil { if err != nil {
return err return err
} }

271
pkg/yqlib/json_test.go
View File

@ -80,135 +80,138 @@ const roundTripMultiLineJson = `{
` `
var jsonScenarios = []formatScenario{ var jsonScenarios = []formatScenario{
{ // {
description: "set tags", // description: "set tags",
skipDoc: true, // skipDoc: true,
input: "[{}]", // input: "[{}]",
expression: `[.. | type]`, // expression: `[.. | type]`,
scenarioType: "roundtrip-ndjson", // scenarioType: "roundtrip-ndjson",
expected: "[\"!!seq\",\"!!map\"]\n", // expected: "[\"!!seq\",\"!!map\"]\n",
}, // },
{ // {
description: "Parse json: simple", // description: "Parse json: simple",
subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output", // subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output",
input: `{"cat": "meow"}`, // input: `{"cat": "meow"}`,
expected: "D0, P[], (!!map)::cat: meow\n", // expected: "D0, P[], (!!map)::cat: meow\n",
}, // },
{ // {
skipDoc: true, // skipDoc: true,
description: "Parse json: simple: key", // description: "Parse json: simple: key",
input: `{"cat": "meow"}`, // input: `{"cat": "meow"}`,
expression: ".cat | key", // expression: ".cat | key",
expected: "D0, P[], (!!str)::cat\n", // expected: "\"cat\"\n",
}, // scenarioType: "decode",
{ // },
skipDoc: true, // {
description: "Parse json: simple: parent", // skipDoc: true,
input: `{"cat": "meow"}`, // description: "Parse json: simple: parent",
expression: ".cat | parent", // input: `{"cat": "meow"}`,
expected: "D0, P[], (!!str)::cat\n", // expression: ".cat | parent",
}, // expected: "{\"cat\":\"meow\"}\n",
{ // scenarioType: "decode",
skipDoc: true, // },
description: "Parse json: simple: path", // {
input: `{"cat": "meow"}`, // skipDoc: true,
expression: ".cat | path", // description: "Parse json: simple: path",
expected: "D0, P[], (!!str)::cat\n", // input: `{"cat": "meow"}`,
}, // expression: ".cat | path",
{ // expected: "[\"cat\"]\n",
description: "bad json", // scenarioType: "decode",
skipDoc: true, // },
input: `{"a": 1 "b": 2}`, // {
expectedError: `bad file 'sample.yml': invalid character '"' after object key:value pair`, // description: "bad json",
scenarioType: "decode-error", // skipDoc: true,
}, // input: `{"a": 1 "b": 2}`,
{ // expectedError: `bad file 'sample.yml': invalid character '"' after object key:value pair`,
description: "Parse json: complex", // scenarioType: "decode-error",
subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output", // },
input: `{"a":"Easy! as one two three","b":{"c":2,"d":[3,4]}}`, // {
expected: complexExpectYaml, // description: "Parse json: complex",
}, // subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output",
{ // input: `{"a":"Easy! as one two three","b":{"c":2,"d":[3,4]}}`,
description: "Encode json: simple", // expected: complexExpectYaml,
input: `cat: meow`, // },
indent: 2, // {
expected: "{\n \"cat\": \"meow\"\n}\n", // description: "Encode json: simple",
scenarioType: "encode", // input: `cat: meow`,
}, // indent: 2,
{ // expected: "{\n \"cat\": \"meow\"\n}\n",
description: "Encode json: simple - in one line", // scenarioType: "encode",
input: `cat: meow # this is a comment, and it will be dropped.`, // },
indent: 0, // {
expected: "{\"cat\":\"meow\"}\n", // description: "Encode json: simple - in one line",
scenarioType: "encode", // input: `cat: meow # this is a comment, and it will be dropped.`,
}, // indent: 0,
{ // expected: "{\"cat\":\"meow\"}\n",
description: "Encode json: comments", // scenarioType: "encode",
input: `cat: meow # this is a comment, and it will be dropped.`, // },
indent: 2, // {
expected: "{\n \"cat\": \"meow\"\n}\n", // description: "Encode json: comments",
scenarioType: "encode", // input: `cat: meow # this is a comment, and it will be dropped.`,
}, // indent: 2,
{ // expected: "{\n \"cat\": \"meow\"\n}\n",
description: "Encode json: anchors", // scenarioType: "encode",
subdescription: "Anchors are dereferenced", // },
input: "cat: &ref meow\nanotherCat: *ref", // {
indent: 2, // description: "Encode json: anchors",
expected: "{\n \"cat\": \"meow\",\n \"anotherCat\": \"meow\"\n}\n", // subdescription: "Anchors are dereferenced",
scenarioType: "encode", // input: "cat: &ref meow\nanotherCat: *ref",
}, // indent: 2,
{ // expected: "{\n \"cat\": \"meow\",\n \"anotherCat\": \"meow\"\n}\n",
description: "Encode json: multiple results", // scenarioType: "encode",
subdescription: "Each matching node is converted into a json doc. This is best used with 0 indent (json document per line)", // },
input: `things: [{stuff: cool}, {whatever: cat}]`, // {
expression: `.things[]`, // description: "Encode json: multiple results",
indent: 0, // subdescription: "Each matching node is converted into a json doc. This is best used with 0 indent (json document per line)",
expected: "{\"stuff\":\"cool\"}\n{\"whatever\":\"cat\"}\n", // input: `things: [{stuff: cool}, {whatever: cat}]`,
scenarioType: "encode", // expression: `.things[]`,
}, // indent: 0,
{ // expected: "{\"stuff\":\"cool\"}\n{\"whatever\":\"cat\"}\n",
description: "Roundtrip NDJSON", // scenarioType: "encode",
subdescription: "Unfortunately the json encoder strips leading spaces of values.", // },
input: sampleNdJson, // {
expected: expectedRoundTripSampleNdJson, // description: "Roundtrip NDJSON",
scenarioType: "roundtrip-ndjson", // subdescription: "Unfortunately the json encoder strips leading spaces of values.",
}, // input: sampleNdJson,
{ // expected: expectedRoundTripSampleNdJson,
description: "Roundtrip multi-document JSON", // scenarioType: "roundtrip-ndjson",
subdescription: "The NDJSON parser can also handle multiple multi-line json documents in a single file!", // },
input: sampleMultiLineJson, // {
expected: roundTripMultiLineJson, // description: "Roundtrip multi-document JSON",
scenarioType: "roundtrip-multi", // subdescription: "The NDJSON parser can also handle multiple multi-line json documents in a single file!",
}, // input: sampleMultiLineJson,
{ // expected: roundTripMultiLineJson,
description: "Update a specific document in a multi-document json", // scenarioType: "roundtrip-multi",
subdescription: "Documents are indexed by the `documentIndex` or `di` operator.", // },
input: sampleNdJson, // {
expected: expectedUpdatedMultilineJson, // description: "Update a specific document in a multi-document json",
expression: `(select(di == 1) | .each ) += "cool"`, // subdescription: "Documents are indexed by the `documentIndex` or `di` operator.",
scenarioType: "roundtrip-ndjson", // input: sampleNdJson,
}, // expected: expectedUpdatedMultilineJson,
{ // expression: `(select(di == 1) | .each ) += "cool"`,
description: "Find and update a specific document in a multi-document json", // scenarioType: "roundtrip-ndjson",
subdescription: "Use expressions as you normally would.", // },
input: sampleNdJson, // {
expected: expectedUpdatedMultilineJson, // description: "Find and update a specific document in a multi-document json",
expression: `(select(has("each")) | .each ) += "cool"`, // subdescription: "Use expressions as you normally would.",
scenarioType: "roundtrip-ndjson", // input: sampleNdJson,
}, // expected: expectedUpdatedMultilineJson,
{ // expression: `(select(has("each")) | .each ) += "cool"`,
description: "Decode NDJSON", // scenarioType: "roundtrip-ndjson",
input: sampleNdJson, // },
expected: expectedNdJsonYaml, // {
scenarioType: "decode-ndjson", // description: "Decode NDJSON",
}, // input: sampleNdJson,
{ // expected: expectedNdJsonYaml,
description: "Decode NDJSON, maintain key order", // scenarioType: "decode-ndjson",
skipDoc: true, // },
input: sampleNdJsonKey, // {
expected: expectedJsonKeysInOrder, // description: "Decode NDJSON, maintain key order",
scenarioType: "decode-ndjson", // skipDoc: true,
}, // input: sampleNdJsonKey,
// expected: expectedJsonKeysInOrder,
// scenarioType: "decode-ndjson",
// },
{ {
description: "numbers", description: "numbers",
skipDoc: true, skipDoc: true,
@ -436,9 +439,9 @@ func TestJSONScenarios(t *testing.T) {
for _, tt := range jsonScenarios { for _, tt := range jsonScenarios {
testJSONScenario(t, tt) testJSONScenario(t, tt)
} }
genericScenarios := make([]interface{}, len(jsonScenarios)) // genericScenarios := make([]interface{}, len(jsonScenarios))
for i, s := range jsonScenarios { // for i, s := range jsonScenarios {
genericScenarios[i] = s // genericScenarios[i] = s
} // }
documentScenarios(t, "usage", "convert", genericScenarios, documentJSONScenario) // documentScenarios(t, "usage", "convert", genericScenarios, documentJSONScenario)
} }

21
pkg/yqlib/lib.go
View File

@ -239,24 +239,6 @@ func recurseNodeObjectEqual(lhs *CandidateNode, rhs *CandidateNode) bool {
return true return true
} }
func guessTagFromCustomType(node *yaml.Node) string {
if strings.HasPrefix(node.Tag, "!!") {
return node.Tag
} else if node.Value == "" {
log.Debug("guessTagFromCustomType: node has no value to guess the type with")
return node.Tag
}
dataBucket, errorReading := parseSnippet(node.Value)
if errorReading != nil {
log.Debug("guessTagFromCustomType: could not guess underlying tag type %v", errorReading)
return node.Tag
}
guessedTag := dataBucket.unwrapDocument().Tag
log.Info("im guessing the tag %v is a %v", node.Tag, guessedTag)
return guessedTag
}
func parseSnippet(value string) (*CandidateNode, error) { func parseSnippet(value string) (*CandidateNode, error) {
if value == "" { if value == "" {
return &CandidateNode{ return &CandidateNode{
@ -273,9 +255,6 @@ func parseSnippet(value string) (*CandidateNode, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
if len(parsedNode.Content) == 0 {
return nil, fmt.Errorf("bad data")
}
result := parsedNode.unwrapDocument() result := parsedNode.unwrapDocument()
result.Line = 0 result.Line = 0
result.Column = 0 result.Column = 0

3
pkg/yqlib/operators_test.go
View File

@ -31,7 +31,7 @@ type expressionScenario struct {
} }
func TestMain(m *testing.M) { func TestMain(m *testing.M) {
logging.SetLevel(logging.ERROR, "") logging.SetLevel(logging.DEBUG, "")
Now = func() time.Time { Now = func() time.Time {
return time.Date(2021, time.May, 19, 1, 2, 3, 4, time.UTC) return time.Date(2021, time.May, 19, 1, 2, 3, 4, time.UTC)
} }
@ -130,6 +130,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
func resultToString(t *testing.T, n *CandidateNode) string { func resultToString(t *testing.T, n *CandidateNode) string {
var valueBuffer bytes.Buffer var valueBuffer bytes.Buffer
log.Debugf("printing result %v", NodeToString(n))
printer := NewSimpleYamlPrinter(bufio.NewWriter(&valueBuffer), YamlOutputFormat, true, false, 4, true) printer := NewSimpleYamlPrinter(bufio.NewWriter(&valueBuffer), YamlOutputFormat, true, false, 4, true)
err := printer.PrintResults(n.AsList()) err := printer.PrintResults(n.AsList())

12
test/utils.go
View File

@ -4,25 +4,13 @@ import (
"bufio" "bufio"
"bytes" "bytes"
"fmt" "fmt"
"os"
"reflect" "reflect"
"testing" "testing"
"github.com/pkg/diff" "github.com/pkg/diff"
"github.com/pkg/diff/write" "github.com/pkg/diff/write"
yaml "gopkg.in/yaml.v3"
) )
func ParseData(rawData string) yaml.Node {
var parsedData yaml.Node
err := yaml.Unmarshal([]byte(rawData), &parsedData)
if err != nil {
fmt.Printf("Error parsing yaml: %v\n", err)
os.Exit(1)
}
return parsedData
}
func printDifference(t *testing.T, expectedValue interface{}, actualValue interface{}) { func printDifference(t *testing.T, expectedValue interface{}, actualValue interface{}) {
opts := []write.Option{write.TerminalColor()} opts := []write.Option{write.TerminalColor()}
var differenceBuffer bytes.Buffer var differenceBuffer bytes.Buffer