From 9ba85a0a46980f314263a45cc5cec0f04375d9a8 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Thu, 4 May 2023 19:22:33 +1000 Subject: [PATCH] fixed json encode/decode --- pkg/yqlib/candidiate_node_json.go | 5 +- pkg/yqlib/doc/usage/convert.md | 246 ++++++++++++++++++++ pkg/yqlib/json_test.go | 359 +++++++++++++++--------------- 3 files changed, 428 insertions(+), 182 deletions(-) diff --git a/pkg/yqlib/candidiate_node_json.go b/pkg/yqlib/candidiate_node_json.go index 99119511..81902207 100644 --- a/pkg/yqlib/candidiate_node_json.go +++ b/pkg/yqlib/candidiate_node_json.go @@ -84,6 +84,8 @@ func (o *CandidateNode) UnmarshalJSON(data []byte) error { } return nil case '[': + o.Kind = SequenceNode + o.Tag = "!!seq" log.Debug("UnmarshalJSON - its an array!") var children []*CandidateNode if err := json.Unmarshal(data, &children); err != nil { @@ -102,7 +104,7 @@ func (o *CandidateNode) UnmarshalJSON(data []byte) error { child.FileIndex = o.FileIndex child.Filename = o.Filename child.Key = childKey - o.Content[i] = child + o.Content = append(o.Content, child) } return nil } @@ -143,7 +145,6 @@ func (o *CandidateNode) MarshalJSON() ([]byte, error) { return buf.Bytes(), err } err = enc.Encode(value) - log.Debugf("cool I handled that") return buf.Bytes(), err case MappingNode: log.Debugf("MarshalJSON MappingNode") diff --git a/pkg/yqlib/doc/usage/convert.md b/pkg/yqlib/doc/usage/convert.md index abdce47c..5060f8da 100644 --- a/pkg/yqlib/doc/usage/convert.md +++ b/pkg/yqlib/doc/usage/convert.md @@ -5,3 +5,249 @@ Encode and decode to and from JSON. Supports multiple JSON documents in a single Note that YAML is a superset of (single document) JSON - so you don't have to use the JSON parser to read JSON when there is only one JSON document in the input. You will probably want to pretty print the result in this case, to get idiomatic YAML styling. +## Parse json: simple +JSON is a subset of yaml, so all you need to do is prettify the output + +Given a sample.json file of: +```json +{"cat": "meow"} +``` +then +```bash +yq -p=json sample.json +``` +will output +```yaml +cat: meow +``` + +## Parse json: complex +JSON is a subset of yaml, so all you need to do is prettify the output + +Given a sample.json file of: +```json +{"a":"Easy! as one two three","b":{"c":2,"d":[3,4]}} +``` +then +```bash +yq -p=json sample.json +``` +will output +```yaml +a: Easy! as one two three +b: + c: 2 + d: + - 3 + - 4 +``` + +## Encode json: simple +Given a sample.yml file of: +```yaml +cat: meow +``` +then +```bash +yq -o=json '.' sample.yml +``` +will output +```json +{ + "cat": "meow" +} +``` + +## Encode json: simple - in one line +Given a sample.yml file of: +```yaml +cat: meow # this is a comment, and it will be dropped. +``` +then +```bash +yq -o=json -I=0 '.' sample.yml +``` +will output +```json +{"cat":"meow"} +``` + +## Encode json: comments +Given a sample.yml file of: +```yaml +cat: meow # this is a comment, and it will be dropped. +``` +then +```bash +yq -o=json '.' sample.yml +``` +will output +```json +{ + "cat": "meow" +} +``` + +## Encode json: anchors +Anchors are dereferenced + +Given a sample.yml file of: +```yaml +cat: &ref meow +anotherCat: *ref +``` +then +```bash +yq -o=json '.' sample.yml +``` +will output +```json +{ + "cat": "meow", + "anotherCat": "meow" +} +``` + +## Encode json: multiple results +Each matching node is converted into a json doc. This is best used with 0 indent (json document per line) + +Given a sample.yml file of: +```yaml +things: [{stuff: cool}, {whatever: cat}] +``` +then +```bash +yq -o=json -I=0 '.things[]' sample.yml +``` +will output +```json +{"stuff":"cool"} +{"whatever":"cat"} +``` + +## Roundtrip NDJSON +Unfortunately the json encoder strips leading spaces of values. + +Given a sample.json file of: +```json +{"this": "is a multidoc json file"} +{"each": ["line is a valid json document"]} +{"a number": 4} + +``` +then +```bash +yq -p=json -o=json -I=0 sample.json +``` +will output +```yaml +{"this":"is a multidoc json file"} +{"each":["line is a valid json document"]} +{"a number":4} +``` + +## Roundtrip multi-document JSON +The NDJSON parser can also handle multiple multi-line json documents in a single file! + +Given a sample.json file of: +```json +{ + "this": "is a multidoc json file" +} +{ + "it": [ + "has", + "consecutive", + "json documents" + ] +} +{ + "a number": 4 +} + +``` +then +```bash +yq -p=json -o=json -I=2 sample.json +``` +will output +```yaml +{ + "this": "is a multidoc json file" +} +{ + "it": [ + "has", + "consecutive", + "json documents" + ] +} +{ + "a number": 4 +} +``` + +## Update a specific document in a multi-document json +Documents are indexed by the `documentIndex` or `di` operator. + +Given a sample.json file of: +```json +{"this": "is a multidoc json file"} +{"each": ["line is a valid json document"]} +{"a number": 4} + +``` +then +```bash +yq -p=json -o=json -I=0 '(select(di == 1) | .each ) += "cool"' sample.json +``` +will output +```yaml +{"this":"is a multidoc json file"} +{"each":["line is a valid json document","cool"]} +{"a number":4} +``` + +## Find and update a specific document in a multi-document json +Use expressions as you normally would. + +Given a sample.json file of: +```json +{"this": "is a multidoc json file"} +{"each": ["line is a valid json document"]} +{"a number": 4} + +``` +then +```bash +yq -p=json -o=json -I=0 '(select(has("each")) | .each ) += "cool"' sample.json +``` +will output +```yaml +{"this":"is a multidoc json file"} +{"each":["line is a valid json document","cool"]} +{"a number":4} +``` + +## Decode NDJSON +Given a sample.json file of: +```json +{"this": "is a multidoc json file"} +{"each": ["line is a valid json document"]} +{"a number": 4} + +``` +then +```bash +yq -p=json sample.json +``` +will output +```yaml +this: is a multidoc json file +--- +each: + - line is a valid json document +--- +a number: 4 +``` + diff --git a/pkg/yqlib/json_test.go b/pkg/yqlib/json_test.go index 24dd6ed4..5502b000 100644 --- a/pkg/yqlib/json_test.go +++ b/pkg/yqlib/json_test.go @@ -11,12 +11,12 @@ import ( "github.com/mikefarah/yq/v4/test" ) -const complexExpectYaml = `D0, P[], (!!map)::a: Easy! as one two three +const complexExpectYaml = `a: Easy! as one two three b: - c: 2 - d: - - 3 - - 4 + c: 2 + d: + - 3 + - 4 ` const sampleNdJson = `{"this": "is a multidoc json file"} @@ -80,145 +80,147 @@ const roundTripMultiLineJson = `{ ` var jsonScenarios = []formatScenario{ - // { - // description: "set tags", - // skipDoc: true, - // input: "[{}]", - // expression: `[.. | type]`, - // scenarioType: "roundtrip-ndjson", - // expected: "[\"!!seq\",\"!!map\"]\n", - // }, - // { - // description: "Parse json: simple", - // subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output", - // input: `{"cat": "meow"}`, - // expected: "D0, P[], (!!map)::cat: meow\n", - // }, - // { - // skipDoc: true, - // description: "Parse json: simple: key", - // input: `{"cat": "meow"}`, - // expression: ".cat | key", - // expected: "\"cat\"\n", - // scenarioType: "decode", - // }, - // { - // skipDoc: true, - // description: "Parse json: simple: parent", - // input: `{"cat": "meow"}`, - // expression: ".cat | parent", - // expected: "{\"cat\":\"meow\"}\n", - // scenarioType: "decode", - // }, - // { - // skipDoc: true, - // description: "Parse json: simple: path", - // input: `{"cat": "meow"}`, - // expression: ".cat | path", - // expected: "[\"cat\"]\n", - // scenarioType: "decode", - // }, - // { - // description: "bad json", - // skipDoc: true, - // input: `{"a": 1 "b": 2}`, - // expectedError: `bad file 'sample.yml': invalid character '"' after object key:value pair`, - // scenarioType: "decode-error", - // }, - // { - // description: "Parse json: complex", - // subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output", - // input: `{"a":"Easy! as one two three","b":{"c":2,"d":[3,4]}}`, - // expected: complexExpectYaml, - // }, - // { - // description: "Encode json: simple", - // input: `cat: meow`, - // indent: 2, - // expected: "{\n \"cat\": \"meow\"\n}\n", - // scenarioType: "encode", - // }, - // { - // description: "Encode json: simple - in one line", - // input: `cat: meow # this is a comment, and it will be dropped.`, - // indent: 0, - // expected: "{\"cat\":\"meow\"}\n", - // scenarioType: "encode", - // }, - // { - // description: "Encode json: comments", - // input: `cat: meow # this is a comment, and it will be dropped.`, - // indent: 2, - // expected: "{\n \"cat\": \"meow\"\n}\n", - // scenarioType: "encode", - // }, - // { - // description: "Encode json: anchors", - // subdescription: "Anchors are dereferenced", - // input: "cat: &ref meow\nanotherCat: *ref", - // indent: 2, - // expected: "{\n \"cat\": \"meow\",\n \"anotherCat\": \"meow\"\n}\n", - // scenarioType: "encode", - // }, - // { - // description: "Encode json: multiple results", - // subdescription: "Each matching node is converted into a json doc. This is best used with 0 indent (json document per line)", - // input: `things: [{stuff: cool}, {whatever: cat}]`, - // expression: `.things[]`, - // indent: 0, - // expected: "{\"stuff\":\"cool\"}\n{\"whatever\":\"cat\"}\n", - // scenarioType: "encode", - // }, - // { - // description: "Roundtrip NDJSON", - // subdescription: "Unfortunately the json encoder strips leading spaces of values.", - // input: sampleNdJson, - // expected: expectedRoundTripSampleNdJson, - // scenarioType: "roundtrip-ndjson", - // }, - // { - // description: "Roundtrip multi-document JSON", - // subdescription: "The NDJSON parser can also handle multiple multi-line json documents in a single file!", - // input: sampleMultiLineJson, - // expected: roundTripMultiLineJson, - // scenarioType: "roundtrip-multi", - // }, - // { - // description: "Update a specific document in a multi-document json", - // subdescription: "Documents are indexed by the `documentIndex` or `di` operator.", - // input: sampleNdJson, - // expected: expectedUpdatedMultilineJson, - // expression: `(select(di == 1) | .each ) += "cool"`, - // scenarioType: "roundtrip-ndjson", - // }, - // { - // description: "Find and update a specific document in a multi-document json", - // subdescription: "Use expressions as you normally would.", - // input: sampleNdJson, - // expected: expectedUpdatedMultilineJson, - // expression: `(select(has("each")) | .each ) += "cool"`, - // scenarioType: "roundtrip-ndjson", - // }, - // { - // description: "Decode NDJSON", - // input: sampleNdJson, - // expected: expectedNdJsonYaml, - // scenarioType: "decode-ndjson", - // }, - // { - // description: "Decode NDJSON, maintain key order", - // skipDoc: true, - // input: sampleNdJsonKey, - // expected: expectedJsonKeysInOrder, - // scenarioType: "decode-ndjson", - // }, - // { - // description: "numbers", - // skipDoc: true, - // input: "[3, 3.0, 3.1, -1]", - // expected: "- 3\n- 3\n- 3.1\n- -1\n", - // scenarioType: "decode-ndjson", - // }, + { + description: "set tags", + skipDoc: true, + input: "[{}]", + expression: `[.. | type]`, + scenarioType: "roundtrip-ndjson", + expected: "[\"!!seq\",\"!!map\"]\n", + }, + { + description: "Parse json: simple", + subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output", + input: `{"cat": "meow"}`, + scenarioType: "decode-ndjson", + expected: "cat: meow\n", + }, + { + skipDoc: true, + description: "Parse json: simple: key", + input: `{"cat": "meow"}`, + expression: ".cat | key", + expected: "\"cat\"\n", + scenarioType: "decode", + }, + { + skipDoc: true, + description: "Parse json: simple: parent", + input: `{"cat": "meow"}`, + expression: ".cat | parent", + expected: "{\"cat\":\"meow\"}\n", + scenarioType: "decode", + }, + { + skipDoc: true, + description: "Parse json: simple: path", + input: `{"cat": "meow"}`, + expression: ".cat | path", + expected: "[\"cat\"]\n", + scenarioType: "decode", + }, + { + description: "bad json", + skipDoc: true, + input: `{"a": 1 b": 2}`, + expectedError: `bad file 'sample.yml': json: string of object unexpected end of JSON input`, + scenarioType: "decode-error", + }, + { + description: "Parse json: complex", + subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output", + input: `{"a":"Easy! as one two three","b":{"c":2,"d":[3,4]}}`, + expected: complexExpectYaml, + scenarioType: "decode-ndjson", + }, + { + description: "Encode json: simple", + input: `cat: meow`, + indent: 2, + expected: "{\n \"cat\": \"meow\"\n}\n", + scenarioType: "encode", + }, + { + description: "Encode json: simple - in one line", + input: `cat: meow # this is a comment, and it will be dropped.`, + indent: 0, + expected: "{\"cat\":\"meow\"}\n", + scenarioType: "encode", + }, + { + description: "Encode json: comments", + input: `cat: meow # this is a comment, and it will be dropped.`, + indent: 2, + expected: "{\n \"cat\": \"meow\"\n}\n", + scenarioType: "encode", + }, + { + description: "Encode json: anchors", + subdescription: "Anchors are dereferenced", + input: "cat: &ref meow\nanotherCat: *ref", + indent: 2, + expected: "{\n \"cat\": \"meow\",\n \"anotherCat\": \"meow\"\n}\n", + scenarioType: "encode", + }, + { + description: "Encode json: multiple results", + subdescription: "Each matching node is converted into a json doc. This is best used with 0 indent (json document per line)", + input: `things: [{stuff: cool}, {whatever: cat}]`, + expression: `.things[]`, + indent: 0, + expected: "{\"stuff\":\"cool\"}\n{\"whatever\":\"cat\"}\n", + scenarioType: "encode", + }, + { + description: "Roundtrip NDJSON", + subdescription: "Unfortunately the json encoder strips leading spaces of values.", + input: sampleNdJson, + expected: expectedRoundTripSampleNdJson, + scenarioType: "roundtrip-ndjson", + }, + { + description: "Roundtrip multi-document JSON", + subdescription: "The NDJSON parser can also handle multiple multi-line json documents in a single file!", + input: sampleMultiLineJson, + expected: roundTripMultiLineJson, + scenarioType: "roundtrip-multi", + }, + { + description: "Update a specific document in a multi-document json", + subdescription: "Documents are indexed by the `documentIndex` or `di` operator.", + input: sampleNdJson, + expected: expectedUpdatedMultilineJson, + expression: `(select(di == 1) | .each ) += "cool"`, + scenarioType: "roundtrip-ndjson", + }, + { + description: "Find and update a specific document in a multi-document json", + subdescription: "Use expressions as you normally would.", + input: sampleNdJson, + expected: expectedUpdatedMultilineJson, + expression: `(select(has("each")) | .each ) += "cool"`, + scenarioType: "roundtrip-ndjson", + }, + { + description: "Decode NDJSON", + input: sampleNdJson, + expected: expectedNdJsonYaml, + scenarioType: "decode-ndjson", + }, + { + description: "Decode NDJSON, maintain key order", + skipDoc: true, + input: sampleNdJsonKey, + expected: expectedJsonKeysInOrder, + scenarioType: "decode-ndjson", + }, + { + description: "numbers", + skipDoc: true, + input: "[3, 3.0, 3.1, -1]", + expected: "- 3\n- 3\n- 3.1\n- -1\n", + scenarioType: "decode-ndjson", + }, { description: "number single", skipDoc: true, @@ -226,34 +228,34 @@ var jsonScenarios = []formatScenario{ expected: "3\n", scenarioType: "decode-ndjson", }, - // { - // description: "empty string", - // skipDoc: true, - // input: `""`, - // expected: "\"\"\n", - // scenarioType: "decode-ndjson", - // }, - // { - // description: "strings", - // skipDoc: true, - // input: `["", "cat"]`, - // expected: "- \"\"\n- cat\n", - // scenarioType: "decode-ndjson", - // }, - // { - // description: "null", - // skipDoc: true, - // input: `null`, - // expected: "null\n", - // scenarioType: "decode-ndjson", - // }, - // { - // description: "booleans", - // skipDoc: true, - // input: `[true, false]`, - // expected: "- true\n- false\n", - // scenarioType: "decode-ndjson", - // }, + { + description: "empty string", + skipDoc: true, + input: `""`, + expected: "\"\"\n", + scenarioType: "decode-ndjson", + }, + { + description: "strings", + skipDoc: true, + input: `["", "cat"]`, + expected: "- \"\"\n- cat\n", + scenarioType: "decode-ndjson", + }, + { + description: "null", + skipDoc: true, + input: `null`, + expected: "null\n", + scenarioType: "decode-ndjson", + }, + { + description: "booleans", + skipDoc: true, + input: `[true, false]`, + expected: "- true\n- false\n", + scenarioType: "decode-ndjson", + }, } func documentRoundtripNdJsonScenario(w *bufio.Writer, s formatScenario, indent int) { @@ -335,9 +337,6 @@ func testJSONScenario(t *testing.T, s formatScenario) { switch s.scenarioType { case "encode", "decode": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false, false)), s.description) - case "": - var actual = resultToString(t, decodeJSON(t, s.input)) - test.AssertResultWithContext(t, s.expected, actual, s.description) case "decode-ndjson": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description) case "roundtrip-ndjson": @@ -439,9 +438,9 @@ func TestJSONScenarios(t *testing.T) { for _, tt := range jsonScenarios { testJSONScenario(t, tt) } - // genericScenarios := make([]interface{}, len(jsonScenarios)) - // for i, s := range jsonScenarios { - // genericScenarios[i] = s - // } - // documentScenarios(t, "usage", "convert", genericScenarios, documentJSONScenario) + genericScenarios := make([]interface{}, len(jsonScenarios)) + for i, s := range jsonScenarios { + genericScenarios[i] = s + } + documentScenarios(t, "usage", "convert", genericScenarios, documentJSONScenario) }