suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2025-01-30 19:25:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixed json encode/decode

Browse Source
This commit is contained in:
Mike Farah 2023-05-04 19:22:33 +10:00
parent e3ccd05a00
commit 9ba85a0a46
3 changed files with 428 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
pkg/yqlib/candidiate_node_json.go
View File

@ -84,6 +84,8 @@ func (o *CandidateNode) UnmarshalJSON(data []byte) error {
}
return nil
case '[':
o.Kind = SequenceNode
o.Tag = "!!seq"
log.Debug("UnmarshalJSON - its an array!")
var children []*CandidateNode
if err := json.Unmarshal(data, &children); err != nil {
@ -102,7 +104,7 @@ func (o *CandidateNode) UnmarshalJSON(data []byte) error {
child.FileIndex = o.FileIndex
child.Filename = o.Filename
child.Key = childKey
o.Content[i] = child
o.Content = append(o.Content, child)
}
return nil
}
@ -143,7 +145,6 @@ func (o *CandidateNode) MarshalJSON() ([]byte, error) {
return buf.Bytes(), err
}
err = enc.Encode(value)
log.Debugf("cool I handled that")
return buf.Bytes(), err
case MappingNode:
log.Debugf("MarshalJSON MappingNode")

246
pkg/yqlib/doc/usage/convert.md
View File

@ -5,3 +5,249 @@ Encode and decode to and from JSON. Supports multiple JSON documents in a single
Note that YAML is a superset of (single document) JSON - so you don't have to use the JSON parser to read JSON when there is only one JSON document in the input. You will probably want to pretty print the result in this case, to get idiomatic YAML styling.
## Parse json: simple
JSON is a subset of yaml, so all you need to do is prettify the output
Given a sample.json file of:
```json
{"cat": "meow"}
```
then
```bash
yq -p=json sample.json
```
will output
```yaml
cat: meow
```
## Parse json: complex
JSON is a subset of yaml, so all you need to do is prettify the output
Given a sample.json file of:
```json
{"a":"Easy! as one two three","b":{"c":2,"d":[3,4]}}
```
then
```bash
yq -p=json sample.json
```
will output
```yaml
a: Easy! as one two three
b:
c: 2
d:
- 3
- 4
```
## Encode json: simple
Given a sample.yml file of:
```yaml
cat: meow
```
then
```bash
yq -o=json '.' sample.yml
```
will output
```json
{
"cat": "meow"
}
```
## Encode json: simple - in one line
Given a sample.yml file of:
```yaml
cat: meow # this is a comment, and it will be dropped.
```
then
```bash
yq -o=json -I=0 '.' sample.yml
```
will output
```json
{"cat":"meow"}
```
## Encode json: comments
Given a sample.yml file of:
```yaml
cat: meow # this is a comment, and it will be dropped.
```
then
```bash
yq -o=json '.' sample.yml
```
will output
```json
{
"cat": "meow"
}
```
## Encode json: anchors
Anchors are dereferenced
Given a sample.yml file of:
```yaml
cat: &ref meow
anotherCat: *ref
```
then
```bash
yq -o=json '.' sample.yml
```
will output
```json
{
"cat": "meow",
"anotherCat": "meow"
}
```
## Encode json: multiple results
Each matching node is converted into a json doc. This is best used with 0 indent (json document per line)
Given a sample.yml file of:
```yaml
things: [{stuff: cool}, {whatever: cat}]
```
then
```bash
yq -o=json -I=0 '.things[]' sample.yml
```
will output
```json
{"stuff":"cool"}
{"whatever":"cat"}
```
## Roundtrip NDJSON
Unfortunately the json encoder strips leading spaces of values.
Given a sample.json file of:
```json
{"this": "is a multidoc json file"}
{"each": ["line is a valid json document"]}
{"a number": 4}
```
then
```bash
yq -p=json -o=json -I=0 sample.json
```
will output
```yaml
{"this":"is a multidoc json file"}
{"each":["line is a valid json document"]}
{"a number":4}
```
## Roundtrip multi-document JSON
The NDJSON parser can also handle multiple multi-line json documents in a single file!
Given a sample.json file of:
```json
{
"this": "is a multidoc json file"
}
{
"it": [
"has",
"consecutive",
"json documents"
]
}
{
"a number": 4
}
```
then
```bash
yq -p=json -o=json -I=2 sample.json
```
will output
```yaml
{
"this": "is a multidoc json file"
}
{
"it": [
"has",
"consecutive",
"json documents"
]
}
{
"a number": 4
}
```
## Update a specific document in a multi-document json
Documents are indexed by the `documentIndex` or `di` operator.
Given a sample.json file of:
```json
{"this": "is a multidoc json file"}
{"each": ["line is a valid json document"]}
{"a number": 4}
```
then
```bash
yq -p=json -o=json -I=0 '(select(di == 1) | .each ) += "cool"' sample.json
```
will output
```yaml
{"this":"is a multidoc json file"}
{"each":["line is a valid json document","cool"]}
{"a number":4}
```
## Find and update a specific document in a multi-document json
Use expressions as you normally would.
Given a sample.json file of:
```json
{"this": "is a multidoc json file"}
{"each": ["line is a valid json document"]}
{"a number": 4}
```
then
```bash
yq -p=json -o=json -I=0 '(select(has("each")) | .each ) += "cool"' sample.json
```
will output
```yaml
{"this":"is a multidoc json file"}
{"each":["line is a valid json document","cool"]}
{"a number":4}
```
## Decode NDJSON
Given a sample.json file of:
```json
{"this": "is a multidoc json file"}
{"each": ["line is a valid json document"]}
{"a number": 4}
```
then
```bash
yq -p=json sample.json
```
will output
```yaml
this: is a multidoc json file
---
each:
- line is a valid json document
---
a number: 4
```

351
pkg/yqlib/json_test.go
View File

@ -11,7 +11,7 @@ import (
"github.com/mikefarah/yq/v4/test"
)
const complexExpectYaml = `D0, P[], (!!map)::a: Easy! as one two three
const complexExpectYaml = `a: Easy! as one two three
b:
c: 2
d:
@ -80,145 +80,147 @@ const roundTripMultiLineJson = `{
`
var jsonScenarios = []formatScenario{
// {
// description: "set tags",
// skipDoc: true,
// input: "[{}]",
// expression: `[.. | type]`,
// scenarioType: "roundtrip-ndjson",
// expected: "[\"!!seq\",\"!!map\"]\n",
// },
// {
// description: "Parse json: simple",
// subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output",
// input: `{"cat": "meow"}`,
// expected: "D0, P[], (!!map)::cat: meow\n",
// },
// {
// skipDoc: true,
// description: "Parse json: simple: key",
// input: `{"cat": "meow"}`,
// expression: ".cat | key",
// expected: "\"cat\"\n",
// scenarioType: "decode",
// },
// {
// skipDoc: true,
// description: "Parse json: simple: parent",
// input: `{"cat": "meow"}`,
// expression: ".cat | parent",
// expected: "{\"cat\":\"meow\"}\n",
// scenarioType: "decode",
// },
// {
// skipDoc: true,
// description: "Parse json: simple: path",
// input: `{"cat": "meow"}`,
// expression: ".cat | path",
// expected: "[\"cat\"]\n",
// scenarioType: "decode",
// },
// {
// description: "bad json",
// skipDoc: true,
// input: `{"a": 1 "b": 2}`,
// expectedError: `bad file 'sample.yml': invalid character '"' after object key:value pair`,
// scenarioType: "decode-error",
// },
// {
// description: "Parse json: complex",
// subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output",
// input: `{"a":"Easy! as one two three","b":{"c":2,"d":[3,4]}}`,
// expected: complexExpectYaml,
// },
// {
// description: "Encode json: simple",
// input: `cat: meow`,
// indent: 2,
// expected: "{\n \"cat\": \"meow\"\n}\n",
// scenarioType: "encode",
// },
// {
// description: "Encode json: simple - in one line",
// input: `cat: meow # this is a comment, and it will be dropped.`,
// indent: 0,
// expected: "{\"cat\":\"meow\"}\n",
// scenarioType: "encode",
// },
// {
// description: "Encode json: comments",
// input: `cat: meow # this is a comment, and it will be dropped.`,
// indent: 2,
// expected: "{\n \"cat\": \"meow\"\n}\n",
// scenarioType: "encode",
// },
// {
// description: "Encode json: anchors",
// subdescription: "Anchors are dereferenced",
// input: "cat: &ref meow\nanotherCat: *ref",
// indent: 2,
// expected: "{\n \"cat\": \"meow\",\n \"anotherCat\": \"meow\"\n}\n",
// scenarioType: "encode",
// },
// {
// description: "Encode json: multiple results",
// subdescription: "Each matching node is converted into a json doc. This is best used with 0 indent (json document per line)",
// input: `things: [{stuff: cool}, {whatever: cat}]`,
// expression: `.things[]`,
// indent: 0,
// expected: "{\"stuff\":\"cool\"}\n{\"whatever\":\"cat\"}\n",
// scenarioType: "encode",
// },
// {
// description: "Roundtrip NDJSON",
// subdescription: "Unfortunately the json encoder strips leading spaces of values.",
// input: sampleNdJson,
// expected: expectedRoundTripSampleNdJson,
// scenarioType: "roundtrip-ndjson",
// },
// {
// description: "Roundtrip multi-document JSON",
// subdescription: "The NDJSON parser can also handle multiple multi-line json documents in a single file!",
// input: sampleMultiLineJson,
// expected: roundTripMultiLineJson,
// scenarioType: "roundtrip-multi",
// },
// {
// description: "Update a specific document in a multi-document json",
// subdescription: "Documents are indexed by the `documentIndex` or `di` operator.",
// input: sampleNdJson,
// expected: expectedUpdatedMultilineJson,
// expression: `(select(di == 1) | .each ) += "cool"`,
// scenarioType: "roundtrip-ndjson",
// },
// {
// description: "Find and update a specific document in a multi-document json",
// subdescription: "Use expressions as you normally would.",
// input: sampleNdJson,
// expected: expectedUpdatedMultilineJson,
// expression: `(select(has("each")) | .each ) += "cool"`,
// scenarioType: "roundtrip-ndjson",
// },
// {
// description: "Decode NDJSON",
// input: sampleNdJson,
// expected: expectedNdJsonYaml,
// scenarioType: "decode-ndjson",
// },
// {
// description: "Decode NDJSON, maintain key order",
// skipDoc: true,
// input: sampleNdJsonKey,
// expected: expectedJsonKeysInOrder,
// scenarioType: "decode-ndjson",
// },
// {
// description: "numbers",
// skipDoc: true,
// input: "[3, 3.0, 3.1, -1]",
// expected: "- 3\n- 3\n- 3.1\n- -1\n",
// scenarioType: "decode-ndjson",
// },
{
description: "set tags",
skipDoc: true,
input: "[{}]",
expression: `[.. | type]`,
scenarioType: "roundtrip-ndjson",
expected: "[\"!!seq\",\"!!map\"]\n",
},
{
description: "Parse json: simple",
subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output",
input: `{"cat": "meow"}`,
scenarioType: "decode-ndjson",
expected: "cat: meow\n",
},
{
skipDoc: true,
description: "Parse json: simple: key",
input: `{"cat": "meow"}`,
expression: ".cat | key",
expected: "\"cat\"\n",
scenarioType: "decode",
},
{
skipDoc: true,
description: "Parse json: simple: parent",
input: `{"cat": "meow"}`,
expression: ".cat | parent",
expected: "{\"cat\":\"meow\"}\n",
scenarioType: "decode",
},
{
skipDoc: true,
description: "Parse json: simple: path",
input: `{"cat": "meow"}`,
expression: ".cat | path",
expected: "[\"cat\"]\n",
scenarioType: "decode",
},
{
description: "bad json",
skipDoc: true,
input: `{"a": 1 b": 2}`,
expectedError: `bad file 'sample.yml': json: string of object unexpected end of JSON input`,
scenarioType: "decode-error",
},
{
description: "Parse json: complex",
subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output",
input: `{"a":"Easy! as one two three","b":{"c":2,"d":[3,4]}}`,
expected: complexExpectYaml,
scenarioType: "decode-ndjson",
},
{
description: "Encode json: simple",
input: `cat: meow`,
indent: 2,
expected: "{\n \"cat\": \"meow\"\n}\n",
scenarioType: "encode",
},
{
description: "Encode json: simple - in one line",
input: `cat: meow # this is a comment, and it will be dropped.`,
indent: 0,
expected: "{\"cat\":\"meow\"}\n",
scenarioType: "encode",
},
{
description: "Encode json: comments",
input: `cat: meow # this is a comment, and it will be dropped.`,
indent: 2,
expected: "{\n \"cat\": \"meow\"\n}\n",
scenarioType: "encode",
},
{
description: "Encode json: anchors",
subdescription: "Anchors are dereferenced",
input: "cat: &ref meow\nanotherCat: *ref",
indent: 2,
expected: "{\n \"cat\": \"meow\",\n \"anotherCat\": \"meow\"\n}\n",
scenarioType: "encode",
},
{
description: "Encode json: multiple results",
subdescription: "Each matching node is converted into a json doc. This is best used with 0 indent (json document per line)",
input: `things: [{stuff: cool}, {whatever: cat}]`,
expression: `.things[]`,
indent: 0,
expected: "{\"stuff\":\"cool\"}\n{\"whatever\":\"cat\"}\n",
scenarioType: "encode",
},
{
description: "Roundtrip NDJSON",
subdescription: "Unfortunately the json encoder strips leading spaces of values.",
input: sampleNdJson,
expected: expectedRoundTripSampleNdJson,
scenarioType: "roundtrip-ndjson",
},
{
description: "Roundtrip multi-document JSON",
subdescription: "The NDJSON parser can also handle multiple multi-line json documents in a single file!",
input: sampleMultiLineJson,
expected: roundTripMultiLineJson,
scenarioType: "roundtrip-multi",
},
{
description: "Update a specific document in a multi-document json",
subdescription: "Documents are indexed by the `documentIndex` or `di` operator.",
input: sampleNdJson,
expected: expectedUpdatedMultilineJson,
expression: `(select(di == 1) | .each ) += "cool"`,
scenarioType: "roundtrip-ndjson",
},
{
description: "Find and update a specific document in a multi-document json",
subdescription: "Use expressions as you normally would.",
input: sampleNdJson,
expected: expectedUpdatedMultilineJson,
expression: `(select(has("each")) | .each ) += "cool"`,
scenarioType: "roundtrip-ndjson",
},
{
description: "Decode NDJSON",
input: sampleNdJson,
expected: expectedNdJsonYaml,
scenarioType: "decode-ndjson",
},
{
description: "Decode NDJSON, maintain key order",
skipDoc: true,
input: sampleNdJsonKey,
expected: expectedJsonKeysInOrder,
scenarioType: "decode-ndjson",
},
{
description: "numbers",
skipDoc: true,
input: "[3, 3.0, 3.1, -1]",
expected: "- 3\n- 3\n- 3.1\n- -1\n",
scenarioType: "decode-ndjson",
},
{
description: "number single",
skipDoc: true,
@ -226,34 +228,34 @@ var jsonScenarios = []formatScenario{
expected: "3\n",
scenarioType: "decode-ndjson",
},
// {
// description: "empty string",
// skipDoc: true,
// input: `""`,
// expected: "\"\"\n",
// scenarioType: "decode-ndjson",
// },
// {
// description: "strings",
// skipDoc: true,
// input: `["", "cat"]`,
// expected: "- \"\"\n- cat\n",
// scenarioType: "decode-ndjson",
// },
// {
// description: "null",
// skipDoc: true,
// input: `null`,
// expected: "null\n",
// scenarioType: "decode-ndjson",
// },
// {
// description: "booleans",
// skipDoc: true,
// input: `[true, false]`,
// expected: "- true\n- false\n",
// scenarioType: "decode-ndjson",
// },
{
description: "empty string",
skipDoc: true,
input: `""`,
expected: "\"\"\n",
scenarioType: "decode-ndjson",
},
{
description: "strings",
skipDoc: true,
input: `["", "cat"]`,
expected: "- \"\"\n- cat\n",
scenarioType: "decode-ndjson",
},
{
description: "null",
skipDoc: true,
input: `null`,
expected: "null\n",
scenarioType: "decode-ndjson",
},
{
description: "booleans",
skipDoc: true,
input: `[true, false]`,
expected: "- true\n- false\n",
scenarioType: "decode-ndjson",
},
}
func documentRoundtripNdJsonScenario(w *bufio.Writer, s formatScenario, indent int) {
@ -335,9 +337,6 @@ func testJSONScenario(t *testing.T, s formatScenario) {
switch s.scenarioType {
case "encode", "decode":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false, false)), s.description)
case "":
var actual = resultToString(t, decodeJSON(t, s.input))
test.AssertResultWithContext(t, s.expected, actual, s.description)
case "decode-ndjson":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
case "roundtrip-ndjson":
@ -439,9 +438,9 @@ func TestJSONScenarios(t *testing.T) {
for _, tt := range jsonScenarios {
testJSONScenario(t, tt)
}
// genericScenarios := make([]interface{}, len(jsonScenarios))
// for i, s := range jsonScenarios {
// genericScenarios[i] = s
// }
// documentScenarios(t, "usage", "convert", genericScenarios, documentJSONScenario)
genericScenarios := make([]interface{}, len(jsonScenarios))
for i, s := range jsonScenarios {
genericScenarios[i] = s
}
documentScenarios(t, "usage", "convert", genericScenarios, documentJSONScenario)
}