suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2025-01-28 17:35:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Mike Farah 2023-04-12 19:04:02 +10:00
parent eb7844dd1d
commit 1421a5a879
14 changed files with 312 additions and 600 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
pkg/yqlib/all_at_once_evaluator_test.go
View File

@ -6,7 +6,6 @@ import (
"testing"
"github.com/mikefarah/yq/v4/test"
logging "gopkg.in/op/go-logging.v1"
)
var evaluateNodesScenario = []expressionScenario{
@ -35,7 +34,7 @@ var evaluateNodesScenario = []expressionScenario{
func TestAllAtOnceEvaluateNodes(t *testing.T) {
var evaluator = NewAllAtOnceEvaluator()
logging.SetLevel(logging.DEBUG, "")
// logging.SetLevel(logging.DEBUG, "")
for _, tt := range evaluateNodesScenario {
decoder := NewYamlDecoder(NewDefaultYamlPreferences())
reader := bufio.NewReader(strings.NewReader(tt.document))

19
pkg/yqlib/candidate_node.go
View File

@ -3,6 +3,7 @@ package yqlib
import (
"container/list"
"fmt"
"strconv"
"strings"
)
@ -171,6 +172,24 @@ func (n *CandidateNode) AsList() *list.List {
return elMap
}
func (n *CandidateNode) GetValueRep() (interface{}, error) {
// TODO: handle booleans, ints, etc
realTag := n.guessTagFromCustomType()
switch realTag {
case "!!int":
_, val, err := parseInt64(n.Value)
return val, err
case "!!float":
// need to test this
return strconv.ParseFloat(n.Value, 64)
case "!!bool":
return isTruthyNode(n)
}
return n.Value, nil
}
func (n *CandidateNode) guessTagFromCustomType() string {
if strings.HasPrefix(n.Tag, "!!") {
return n.Tag

4
pkg/yqlib/candidate_node_yaml.go
View File

@ -103,6 +103,7 @@ func (o *CandidateNode) UnmarshalYAML(node *yaml.Node) error {
o.copyFromYamlNode(node)
return nil
case yaml.ScalarNode:
log.Debugf("its a scalar")
o.Kind = ScalarNode
o.copyFromYamlNode(node)
return nil
@ -178,6 +179,7 @@ func (o *CandidateNode) MarshalYAML() (interface{}, error) {
o.copyToYamlNode(target)
return target, nil
case ScalarNode:
log.Debug("encoding scalar: %v", o.Value)
target := &yaml.Node{Kind: yaml.ScalarNode}
o.copyToYamlNode(target)
return target, nil
@ -195,6 +197,8 @@ func (o *CandidateNode) MarshalYAML() (interface{}, error) {
if err != nil {
return nil, err
}
log.Debug("child type %v", child.Tag)
log.Debug("child is doc %v", child.Kind == yaml.DocumentNode)
target.Content[i] = child
}
return target, nil

52
pkg/yqlib/candidiate_node_json.go Normal file
View File

@ -0,0 +1,52 @@
package yqlib
import (
"bytes"
"github.com/goccy/go-json"
)
func (o *CandidateNode) MarshalJSON() ([]byte, error) {
log.Debugf("going to encode %v - %v", o.GetNicePath(), o.Tag)
buf := new(bytes.Buffer)
enc := json.NewEncoder(buf)
enc.SetIndent("", " ")
enc.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
switch o.Kind {
case DocumentNode:
err := enc.Encode(o.Content[0])
return buf.Bytes(), err
case AliasNode:
err := enc.Encode(o.Alias)
return buf.Bytes(), err
case ScalarNode:
value, err := o.GetValueRep()
if err != nil {
return buf.Bytes(), err
}
err = enc.Encode(value)
return buf.Bytes(), err
case MappingNode:
buf.WriteByte('{')
for i := 0; i < len(o.Content); i += 2 {
log.Debugf("writing key %v", NodeToString(o.Content[i]))
if err := enc.Encode(o.Content[i].Value); err != nil {
return nil, err
}
buf.WriteByte(':')
log.Debugf("writing value %v", NodeToString(o.Content[i+1]))
if err := enc.Encode(o.Content[i+1]); err != nil {
return nil, err
}
if i != len(o.Content)-2 {
buf.WriteByte(',')
}
}
buf.WriteByte('}')
case SequenceNode:
err := enc.Encode(o.Content)
return buf.Bytes(), err
}
return buf.Bytes(), nil
}

168
pkg/yqlib/csv_test.go
View File

@ -65,89 +65,89 @@ because excel is cool
`
var csvScenarios = []formatScenario{
{
description: "Encode CSV simple",
input: csvTestSimpleYaml,
expected: expectedSimpleCsv,
scenarioType: "encode-csv",
},
{
description: "Encode TSV simple",
input: csvTestSimpleYaml,
expected: tsvTestExpectedSimpleCsv,
scenarioType: "encode-tsv",
},
{
description: "Encode Empty",
skipDoc: true,
input: `[]`,
expected: "",
scenarioType: "encode-csv",
},
{
description: "Comma in value",
skipDoc: true,
input: `["comma, in, value", things]`,
expected: "\"comma, in, value\",things\n",
scenarioType: "encode-csv",
},
{
description: "Encode array of objects to csv",
input: expectedYamlFromCSV,
expected: csvSimple,
scenarioType: "encode-csv",
},
{
description: "Encode array of objects to custom csv format",
subdescription: "Add the header row manually, then the we convert each object into an array of values - resulting in an array of arrays. Pick the columns and call the header whatever you like.",
input: expectedYamlFromCSV,
expected: csvSimpleShort,
expression: `[["Name", "Number of Cats"]] + [.[] | [.name, .numberOfCats ]]`,
scenarioType: "encode-csv",
},
{
description: "Encode array of objects to csv - missing fields behaviour",
subdescription: "First entry is used to determine the headers, and it is missing 'likesApples', so it is not included in the csv. Second entry does not have 'numberOfCats' so that is blank",
input: expectedYamlFromCSVMissingData,
expected: csvSimpleMissingData,
scenarioType: "encode-csv",
},
{
description: "decode csv missing",
skipDoc: true,
input: csvMissing,
expected: csvMissing,
scenarioType: "roundtrip-csv",
},
{
description: "Parse CSV into an array of objects",
subdescription: "First row is assumed to be the header row.",
input: csvSimple,
expected: expectedYamlFromCSV,
scenarioType: "decode-csv-object",
},
{
description: "Scalar roundtrip",
skipDoc: true,
input: "mike\ncat",
expression: ".[0].mike",
expected: "cat\n",
scenarioType: "roundtrip-csv",
},
{
description: "Parse TSV into an array of objects",
subdescription: "First row is assumed to be the header row.",
input: tsvSimple,
expected: expectedYamlFromCSV,
scenarioType: "decode-tsv-object",
},
{
description: "Round trip",
input: csvSimple,
expected: expectedUpdatedSimpleCsv,
expression: `(.[] | select(.name == "Gary") | .numberOfCats) = 3`,
scenarioType: "roundtrip-csv",
},
// {
// description: "Encode CSV simple",
// input: csvTestSimpleYaml,
// expected: expectedSimpleCsv,
// scenarioType: "encode-csv",
// },
// {
// description: "Encode TSV simple",
// input: csvTestSimpleYaml,
// expected: tsvTestExpectedSimpleCsv,
// scenarioType: "encode-tsv",
// },
// {
// description: "Encode Empty",
// skipDoc: true,
// input: `[]`,
// expected: "",
// scenarioType: "encode-csv",
// },
// {
// description: "Comma in value",
// skipDoc: true,
// input: `["comma, in, value", things]`,
// expected: "\"comma, in, value\",things\n",
// scenarioType: "encode-csv",
// },
// {
// description: "Encode array of objects to csv",
// input: expectedYamlFromCSV,
// expected: csvSimple,
// scenarioType: "encode-csv",
// },
// {
// description: "Encode array of objects to custom csv format",
// subdescription: "Add the header row manually, then the we convert each object into an array of values - resulting in an array of arrays. Pick the columns and call the header whatever you like.",
// input: expectedYamlFromCSV,
// expected: csvSimpleShort,
// expression: `[["Name", "Number of Cats"]] + [.[] | [.name, .numberOfCats ]]`,
// scenarioType: "encode-csv",
// },
// {
// description: "Encode array of objects to csv - missing fields behaviour",
// subdescription: "First entry is used to determine the headers, and it is missing 'likesApples', so it is not included in the csv. Second entry does not have 'numberOfCats' so that is blank",
// input: expectedYamlFromCSVMissingData,
// expected: csvSimpleMissingData,
// scenarioType: "encode-csv",
// },
// {
// description: "decode csv missing",
// skipDoc: true,
// input: csvMissing,
// expected: csvMissing,
// scenarioType: "roundtrip-csv",
// },
// {
// description: "Parse CSV into an array of objects",
// subdescription: "First row is assumed to be the header row.",
// input: csvSimple,
// expected: expectedYamlFromCSV,
// scenarioType: "decode-csv-object",
// },
// {
// description: "Scalar roundtrip",
// skipDoc: true,
// input: "mike\ncat",
// expression: ".[0].mike",
// expected: "cat\n",
// scenarioType: "roundtrip-csv",
// },
// {
// description: "Parse TSV into an array of objects",
// subdescription: "First row is assumed to be the header row.",
// input: tsvSimple,
// expected: expectedYamlFromCSV,
// scenarioType: "decode-tsv-object",
// },
// {
// description: "Round trip",
// input: csvSimple,
// expected: expectedUpdatedSimpleCsv,
// expression: `(.[] | select(.name == "Gary") | .numberOfCats) = 3`,
// scenarioType: "roundtrip-csv",
// },
}
func testCSVScenario(t *testing.T, s formatScenario) {
@ -286,5 +286,5 @@ func TestCSVScenarios(t *testing.T) {
for i, s := range csvScenarios {
genericScenarios[i] = s
}
documentScenarios(t, "usage", "csv-tsv", genericScenarios, documentCSVScenario)
// documentScenarios(t, "usage", "csv-tsv", genericScenarios, documentCSVScenario)
}

5
pkg/yqlib/decoder_json.go
View File

@ -25,13 +25,14 @@ func (dec *jsonDecoder) Init(reader io.Reader) error {
func (dec *jsonDecoder) Decode() (*CandidateNode, error) {
var dataBucket orderedMap
log.Debug("going to decode")
log.Debug("going to decode json")
err := dec.decoder.Decode(&dataBucket)
if err != nil {
return nil, err
}
log.Debug("convert to yaml")
node, err := dec.convertToYamlNode(&dataBucket)
log.Debug("done, %w", err)
if err != nil {
return nil, err
}

5
pkg/yqlib/decoder_test.go
View File

@ -27,11 +27,14 @@ func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) (
decoder = NewYamlDecoder(ConfiguredYamlPreferences)
}
log.Debugf("reading docs")
inputs, err := readDocuments(strings.NewReader(s.input), "sample.yml", 0, decoder)
if err != nil {
return "", err
}
log.Debugf("read the documents")
expression := s.expression
if expression == "" {
expression = "."
@ -45,6 +48,8 @@ func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) (
context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: inputs}, exp)
log.Debugf("Going to print: %v", NodesToString(context.MatchingNodes))
if err != nil {
return "", err
}

246
pkg/yqlib/doc/usage/convert.md
View File

@ -5,249 +5,3 @@ Encode and decode to and from JSON. Supports multiple JSON documents in a single
Note that YAML is a superset of (single document) JSON - so you don't have to use the JSON parser to read JSON when there is only one JSON document in the input. You will probably want to pretty print the result in this case, to get idiomatic YAML styling.
## Parse json: simple
JSON is a subset of yaml, so all you need to do is prettify the output
Given a sample.json file of:
```json
{"cat": "meow"}
```
then
```bash
yq -P '.' sample.json
```
will output
```yaml
cat: meow
```
## Parse json: complex
JSON is a subset of yaml, so all you need to do is prettify the output
Given a sample.json file of:
```json
{"a":"Easy! as one two three","b":{"c":2,"d":[3,4]}}
```
then
```bash
yq -P '.' sample.json
```
will output
```yaml
a: Easy! as one two three
b:
c: 2
d:
- 3
- 4
```
## Encode json: simple
Given a sample.yml file of:
```yaml
cat: meow
```
then
```bash
yq -o=json '.' sample.yml
```
will output
```json
{
"cat": "meow"
}
```
## Encode json: simple - in one line
Given a sample.yml file of:
```yaml
cat: meow # this is a comment, and it will be dropped.
```
then
```bash
yq -o=json -I=0 '.' sample.yml
```
will output
```json
{"cat":"meow"}
```
## Encode json: comments
Given a sample.yml file of:
```yaml
cat: meow # this is a comment, and it will be dropped.
```
then
```bash
yq -o=json '.' sample.yml
```
will output
```json
{
"cat": "meow"
}
```
## Encode json: anchors
Anchors are dereferenced
Given a sample.yml file of:
```yaml
cat: &ref meow
anotherCat: *ref
```
then
```bash
yq -o=json '.' sample.yml
```
will output
```json
{
"cat": "meow",
"anotherCat": "meow"
}
```
## Encode json: multiple results
Each matching node is converted into a json doc. This is best used with 0 indent (json document per line)
Given a sample.yml file of:
```yaml
things: [{stuff: cool}, {whatever: cat}]
```
then
```bash
yq -o=json -I=0 '.things[]' sample.yml
```
will output
```json
{"stuff":"cool"}
{"whatever":"cat"}
```
## Roundtrip NDJSON
Unfortunately the json encoder strips leading spaces of values.
Given a sample.json file of:
```json
{"this": "is a multidoc json file"}
{"each": ["line is a valid json document"]}
{"a number": 4}
```
then
```bash
yq -p=json -o=json -I=0 sample.json
```
will output
```yaml
{"this":"is a multidoc json file"}
{"each":["line is a valid json document"]}
{"a number":4}
```
## Roundtrip multi-document JSON
The NDJSON parser can also handle multiple multi-line json documents in a single file!
Given a sample.json file of:
```json
{
"this": "is a multidoc json file"
}
{
"it": [
"has",
"consecutive",
"json documents"
]
}
{
"a number": 4
}
```
then
```bash
yq -p=json -o=json -I=2 sample.json
```
will output
```yaml
{
"this": "is a multidoc json file"
}
{
"it": [
"has",
"consecutive",
"json documents"
]
}
{
"a number": 4
}
```
## Update a specific document in a multi-document json
Documents are indexed by the `documentIndex` or `di` operator.
Given a sample.json file of:
```json
{"this": "is a multidoc json file"}
{"each": ["line is a valid json document"]}
{"a number": 4}
```
then
```bash
yq -p=json -o=json -I=0 '(select(di == 1) | .each ) += "cool"' sample.json
```
will output
```yaml
{"this":"is a multidoc json file"}
{"each":["line is a valid json document","cool"]}
{"a number":4}
```
## Find and update a specific document in a multi-document json
Use expressions as you normally would.
Given a sample.json file of:
```json
{"this": "is a multidoc json file"}
{"each": ["line is a valid json document"]}
{"a number": 4}
```
then
```bash
yq -p=json -o=json -I=0 '(select(has("each")) | .each ) += "cool"' sample.json
```
will output
```yaml
{"this":"is a multidoc json file"}
{"each":["line is a valid json document","cool"]}
{"a number":4}
```
## Decode NDJSON
Given a sample.json file of:
```json
{"this": "is a multidoc json file"}
{"each": ["line is a valid json document"]}
{"a number": 4}
```
then
```bash
yq -p=json sample.json
```
will output
```yaml
this: is a multidoc json file
---
each:
- line is a valid json document
---
a number: 4
```

95
pkg/yqlib/doc/usage/csv-tsv.md
View File

@ -111,98 +111,3 @@ Gary,1
Samantha's Rabbit,2
```
## Encode array of objects to csv - missing fields behaviour
First entry is used to determine the headers, and it is missing 'likesApples', so it is not included in the csv. Second entry does not have 'numberOfCats' so that is blank
Given a sample.yml file of:
```yaml
- name: Gary
numberOfCats: 1
height: 168.8
- name: Samantha's Rabbit
height: -188.8
likesApples: false
```
then
```bash
yq -o=csv sample.yml
```
will output
```csv
name,numberOfCats,height
Gary,1,168.8
Samantha's Rabbit,,-188.8
```
## Parse CSV into an array of objects
First row is assumed to be the header row.
Given a sample.csv file of:
```csv
name,numberOfCats,likesApples,height
Gary,1,true,168.8
Samantha's Rabbit,2,false,-188.8
```
then
```bash
yq -p=csv sample.csv
```
will output
```yaml
- name: Gary
numberOfCats: 1
likesApples: true
height: 168.8
- name: Samantha's Rabbit
numberOfCats: 2
likesApples: false
height: -188.8
```
## Parse TSV into an array of objects
First row is assumed to be the header row.
Given a sample.tsv file of:
```tsv
name numberOfCats likesApples height
Gary 1 true 168.8
Samantha's Rabbit 2 false -188.8
```
then
```bash
yq -p=tsv sample.tsv
```
will output
```yaml
- name: Gary
numberOfCats: 1
likesApples: true
height: 168.8
- name: Samantha's Rabbit
numberOfCats: 2
likesApples: false
height: -188.8
```
## Round trip
Given a sample.csv file of:
```csv
name,numberOfCats,likesApples,height
Gary,1,true,168.8
Samantha's Rabbit,2,false,-188.8
```
then
```bash
yq -p=csv -o=csv '(.[] | select(.name == "Gary") | .numberOfCats) = 3' sample.csv
```
will output
```csv
name,numberOfCats,likesApples,height
Gary,3,true,168.8
Samantha's Rabbit,2,false,-188.8
```

8
pkg/yqlib/encoder_json.go
View File

@ -38,6 +38,8 @@ func (je *jsonEncoder) PrintLeadingContent(writer io.Writer, content string) err
}
func (je *jsonEncoder) Encode(writer io.Writer, node *CandidateNode) error {
log.Debugf("I need to encode %v", NodeToString(node))
log.Debugf("kids %v", len(node.Content))
if node.Kind == ScalarNode && je.UnwrapScalar {
return writeString(writer, node.Value+"\n")
@ -53,15 +55,15 @@ func (je *jsonEncoder) Encode(writer io.Writer, node *CandidateNode) error {
encoder.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
encoder.SetIndent("", je.indentString)
var dataBucket orderedMap
// var dataBucket orderedMap
// firstly, convert all map keys to strings
mapKeysToStrings(node)
// mapKeysToStrings(node)
// errorDecoding := node.Decode(&dataBucket)
// if errorDecoding != nil {
// return errorDecoding
// }
err := encoder.Encode(dataBucket)
err := encoder.Encode(node)
if err != nil {
return err
}

271
pkg/yqlib/json_test.go
View File

@ -80,135 +80,138 @@ const roundTripMultiLineJson = `{
`
var jsonScenarios = []formatScenario{
{
description: "set tags",
skipDoc: true,
input: "[{}]",
expression: `[.. | type]`,
scenarioType: "roundtrip-ndjson",
expected: "[\"!!seq\",\"!!map\"]\n",
},
{
description: "Parse json: simple",
subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output",
input: `{"cat": "meow"}`,
expected: "D0, P[], (!!map)::cat: meow\n",
},
{
skipDoc: true,
description: "Parse json: simple: key",
input: `{"cat": "meow"}`,
expression: ".cat | key",
expected: "D0, P[], (!!str)::cat\n",
},
{
skipDoc: true,
description: "Parse json: simple: parent",
input: `{"cat": "meow"}`,
expression: ".cat | parent",
expected: "D0, P[], (!!str)::cat\n",
},
{
skipDoc: true,
description: "Parse json: simple: path",
input: `{"cat": "meow"}`,
expression: ".cat | path",
expected: "D0, P[], (!!str)::cat\n",
},
{
description: "bad json",
skipDoc: true,
input: `{"a": 1 "b": 2}`,
expectedError: `bad file 'sample.yml': invalid character '"' after object key:value pair`,
scenarioType: "decode-error",
},
{
description: "Parse json: complex",
subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output",
input: `{"a":"Easy! as one two three","b":{"c":2,"d":[3,4]}}`,
expected: complexExpectYaml,
},
{
description: "Encode json: simple",
input: `cat: meow`,
indent: 2,
expected: "{\n \"cat\": \"meow\"\n}\n",
scenarioType: "encode",
},
{
description: "Encode json: simple - in one line",
input: `cat: meow # this is a comment, and it will be dropped.`,
indent: 0,
expected: "{\"cat\":\"meow\"}\n",
scenarioType: "encode",
},
{
description: "Encode json: comments",
input: `cat: meow # this is a comment, and it will be dropped.`,
indent: 2,
expected: "{\n \"cat\": \"meow\"\n}\n",
scenarioType: "encode",
},
{
description: "Encode json: anchors",
subdescription: "Anchors are dereferenced",
input: "cat: &ref meow\nanotherCat: *ref",
indent: 2,
expected: "{\n \"cat\": \"meow\",\n \"anotherCat\": \"meow\"\n}\n",
scenarioType: "encode",
},
{
description: "Encode json: multiple results",
subdescription: "Each matching node is converted into a json doc. This is best used with 0 indent (json document per line)",
input: `things: [{stuff: cool}, {whatever: cat}]`,
expression: `.things[]`,
indent: 0,
expected: "{\"stuff\":\"cool\"}\n{\"whatever\":\"cat\"}\n",
scenarioType: "encode",
},
{
description: "Roundtrip NDJSON",
subdescription: "Unfortunately the json encoder strips leading spaces of values.",
input: sampleNdJson,
expected: expectedRoundTripSampleNdJson,
scenarioType: "roundtrip-ndjson",
},
{
description: "Roundtrip multi-document JSON",
subdescription: "The NDJSON parser can also handle multiple multi-line json documents in a single file!",
input: sampleMultiLineJson,
expected: roundTripMultiLineJson,
scenarioType: "roundtrip-multi",
},
{
description: "Update a specific document in a multi-document json",
subdescription: "Documents are indexed by the `documentIndex` or `di` operator.",
input: sampleNdJson,
expected: expectedUpdatedMultilineJson,
expression: `(select(di == 1) | .each ) += "cool"`,
scenarioType: "roundtrip-ndjson",
},
{
description: "Find and update a specific document in a multi-document json",
subdescription: "Use expressions as you normally would.",
input: sampleNdJson,
expected: expectedUpdatedMultilineJson,
expression: `(select(has("each")) | .each ) += "cool"`,
scenarioType: "roundtrip-ndjson",
},
{
description: "Decode NDJSON",
input: sampleNdJson,
expected: expectedNdJsonYaml,
scenarioType: "decode-ndjson",
},
{
description: "Decode NDJSON, maintain key order",
skipDoc: true,
input: sampleNdJsonKey,
expected: expectedJsonKeysInOrder,
scenarioType: "decode-ndjson",
},
// {
// description: "set tags",
// skipDoc: true,
// input: "[{}]",
// expression: `[.. | type]`,
// scenarioType: "roundtrip-ndjson",
// expected: "[\"!!seq\",\"!!map\"]\n",
// },
// {
// description: "Parse json: simple",
// subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output",
// input: `{"cat": "meow"}`,
// expected: "D0, P[], (!!map)::cat: meow\n",
// },
// {
// skipDoc: true,
// description: "Parse json: simple: key",
// input: `{"cat": "meow"}`,
// expression: ".cat | key",
// expected: "\"cat\"\n",
// scenarioType: "decode",
// },
// {
// skipDoc: true,
// description: "Parse json: simple: parent",
// input: `{"cat": "meow"}`,
// expression: ".cat | parent",
// expected: "{\"cat\":\"meow\"}\n",
// scenarioType: "decode",
// },
// {
// skipDoc: true,
// description: "Parse json: simple: path",
// input: `{"cat": "meow"}`,
// expression: ".cat | path",
// expected: "[\"cat\"]\n",
// scenarioType: "decode",
// },
// {
// description: "bad json",
// skipDoc: true,
// input: `{"a": 1 "b": 2}`,
// expectedError: `bad file 'sample.yml': invalid character '"' after object key:value pair`,
// scenarioType: "decode-error",
// },
// {
// description: "Parse json: complex",
// subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output",
// input: `{"a":"Easy! as one two three","b":{"c":2,"d":[3,4]}}`,
// expected: complexExpectYaml,
// },
// {
// description: "Encode json: simple",
// input: `cat: meow`,
// indent: 2,
// expected: "{\n \"cat\": \"meow\"\n}\n",
// scenarioType: "encode",
// },
// {
// description: "Encode json: simple - in one line",
// input: `cat: meow # this is a comment, and it will be dropped.`,
// indent: 0,
// expected: "{\"cat\":\"meow\"}\n",
// scenarioType: "encode",
// },
// {
// description: "Encode json: comments",
// input: `cat: meow # this is a comment, and it will be dropped.`,
// indent: 2,
// expected: "{\n \"cat\": \"meow\"\n}\n",
// scenarioType: "encode",
// },
// {
// description: "Encode json: anchors",
// subdescription: "Anchors are dereferenced",
// input: "cat: &ref meow\nanotherCat: *ref",
// indent: 2,
// expected: "{\n \"cat\": \"meow\",\n \"anotherCat\": \"meow\"\n}\n",
// scenarioType: "encode",
// },
// {
// description: "Encode json: multiple results",
// subdescription: "Each matching node is converted into a json doc. This is best used with 0 indent (json document per line)",
// input: `things: [{stuff: cool}, {whatever: cat}]`,
// expression: `.things[]`,
// indent: 0,
// expected: "{\"stuff\":\"cool\"}\n{\"whatever\":\"cat\"}\n",
// scenarioType: "encode",
// },
// {
// description: "Roundtrip NDJSON",
// subdescription: "Unfortunately the json encoder strips leading spaces of values.",
// input: sampleNdJson,
// expected: expectedRoundTripSampleNdJson,
// scenarioType: "roundtrip-ndjson",
// },
// {
// description: "Roundtrip multi-document JSON",
// subdescription: "The NDJSON parser can also handle multiple multi-line json documents in a single file!",
// input: sampleMultiLineJson,
// expected: roundTripMultiLineJson,
// scenarioType: "roundtrip-multi",
// },
// {
// description: "Update a specific document in a multi-document json",
// subdescription: "Documents are indexed by the `documentIndex` or `di` operator.",
// input: sampleNdJson,
// expected: expectedUpdatedMultilineJson,
// expression: `(select(di == 1) | .each ) += "cool"`,
// scenarioType: "roundtrip-ndjson",
// },
// {
// description: "Find and update a specific document in a multi-document json",
// subdescription: "Use expressions as you normally would.",
// input: sampleNdJson,
// expected: expectedUpdatedMultilineJson,
// expression: `(select(has("each")) | .each ) += "cool"`,
// scenarioType: "roundtrip-ndjson",
// },
// {
// description: "Decode NDJSON",
// input: sampleNdJson,
// expected: expectedNdJsonYaml,
// scenarioType: "decode-ndjson",
// },
// {
// description: "Decode NDJSON, maintain key order",
// skipDoc: true,
// input: sampleNdJsonKey,
// expected: expectedJsonKeysInOrder,
// scenarioType: "decode-ndjson",
// },
{
description: "numbers",
skipDoc: true,
@ -436,9 +439,9 @@ func TestJSONScenarios(t *testing.T) {
for _, tt := range jsonScenarios {
testJSONScenario(t, tt)
}
genericScenarios := make([]interface{}, len(jsonScenarios))
for i, s := range jsonScenarios {
genericScenarios[i] = s
}
documentScenarios(t, "usage", "convert", genericScenarios, documentJSONScenario)
// genericScenarios := make([]interface{}, len(jsonScenarios))
// for i, s := range jsonScenarios {
// genericScenarios[i] = s
// }
// documentScenarios(t, "usage", "convert", genericScenarios, documentJSONScenario)
}

21
pkg/yqlib/lib.go
View File

@ -239,24 +239,6 @@ func recurseNodeObjectEqual(lhs *CandidateNode, rhs *CandidateNode) bool {
return true
}
func guessTagFromCustomType(node *yaml.Node) string {
if strings.HasPrefix(node.Tag, "!!") {
return node.Tag
} else if node.Value == "" {
log.Debug("guessTagFromCustomType: node has no value to guess the type with")
return node.Tag
}
dataBucket, errorReading := parseSnippet(node.Value)
if errorReading != nil {
log.Debug("guessTagFromCustomType: could not guess underlying tag type %v", errorReading)
return node.Tag
}
guessedTag := dataBucket.unwrapDocument().Tag
log.Info("im guessing the tag %v is a %v", node.Tag, guessedTag)
return guessedTag
}
func parseSnippet(value string) (*CandidateNode, error) {
if value == "" {
return &CandidateNode{
@ -273,9 +255,6 @@ func parseSnippet(value string) (*CandidateNode, error) {
if err != nil {
return nil, err
}
if len(parsedNode.Content) == 0 {
return nil, fmt.Errorf("bad data")
}
result := parsedNode.unwrapDocument()
result.Line = 0
result.Column = 0

3
pkg/yqlib/operators_test.go
View File

@ -31,7 +31,7 @@ type expressionScenario struct {
}
func TestMain(m *testing.M) {
logging.SetLevel(logging.ERROR, "")
logging.SetLevel(logging.DEBUG, "")
Now = func() time.Time {
return time.Date(2021, time.May, 19, 1, 2, 3, 4, time.UTC)
}
@ -130,6 +130,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
func resultToString(t *testing.T, n *CandidateNode) string {
var valueBuffer bytes.Buffer
log.Debugf("printing result %v", NodeToString(n))
printer := NewSimpleYamlPrinter(bufio.NewWriter(&valueBuffer), YamlOutputFormat, true, false, 4, true)
err := printer.PrintResults(n.AsList())

12
test/utils.go
View File

@ -4,25 +4,13 @@ import (
"bufio"
"bytes"
"fmt"
"os"
"reflect"
"testing"
"github.com/pkg/diff"
"github.com/pkg/diff/write"
yaml "gopkg.in/yaml.v3"
)
func ParseData(rawData string) yaml.Node {
var parsedData yaml.Node
err := yaml.Unmarshal([]byte(rawData), &parsedData)
if err != nil {
fmt.Printf("Error parsing yaml: %v\n", err)
os.Exit(1)
}
return parsedData
}
func printDifference(t *testing.T, expectedValue interface{}, actualValue interface{}) {
opts := []write.Option{write.TerminalColor()}
var differenceBuffer bytes.Buffer