diff --git a/pkg/yqlib/doc/encode-decode.md b/pkg/yqlib/doc/encode-decode.md index 384b5559..f4640496 100644 --- a/pkg/yqlib/doc/encode-decode.md +++ b/pkg/yqlib/doc/encode-decode.md @@ -6,6 +6,111 @@ Note that you can optionally pass an indent value to the encode functions (see b These operators are useful to process yaml documents that have stringified embeded yaml/json/props in them. + +| Format | Decode (from string) | Encode (to string) | +| --- | -- | --| +| Yaml | from_yaml | to_yaml(i)/@yaml | +| JSON | from_json | to_json(i)/@json | +| Properties | | to_props/@props | +| CSV | | to_csv/@csv | +| TSV | | to_tsv/@tsv | + + +CSV and TSV format both accept either a single array or scalars (representing a single row), or an array of array of scalars (representing multiple rows). + + +## Encode value as json string +Given a sample.yml file of: +```yaml +a: + cool: thing +``` +then +```bash +yq eval '.b = (.a | to_json)' sample.yml +``` +will output +```yaml +a: + cool: thing +b: | + { + "cool": "thing" + } +``` + +## Encode value as json string, on one line +Pass in a 0 indent to print json on a single line. + +Given a sample.yml file of: +```yaml +a: + cool: thing +``` +then +```bash +yq eval '.b = (.a | to_json(0))' sample.yml +``` +will output +```yaml +a: + cool: thing +b: '{"cool":"thing"}' +``` + +## Encode value as json string, on one line shorthand +Pass in a 0 indent to print json on a single line. + +Given a sample.yml file of: +```yaml +a: + cool: thing +``` +then +```bash +yq eval '.b = (.a | @json)' sample.yml +``` +will output +```yaml +a: + cool: thing +b: '{"cool":"thing"}' +``` + +## Decode a json encoded string +Keep in mind JSON is a subset of YAML. If you want idiomatic yaml, pipe through the style operator to clear out the JSON styling. + +Given a sample.yml file of: +```yaml +a: '{"cool":"thing"}' +``` +then +```bash +yq eval '.a | from_json | ... style=""' sample.yml +``` +will output +```yaml +cool: thing +``` + +## Encode value as props string +Given a sample.yml file of: +```yaml +a: + cool: thing +``` +then +```bash +yq eval '.b = (.a | @props)' sample.yml +``` +will output +```yaml +a: + cool: thing +b: | + cool = thing +``` + ## Encode value as yaml string Indent defaults to 2 @@ -72,63 +177,6 @@ b: | cool: thing ``` -## Encode value as json string -Given a sample.yml file of: -```yaml -a: - cool: thing -``` -then -```bash -yq eval '.b = (.a | to_json)' sample.yml -``` -will output -```yaml -a: - cool: thing -b: | - { - "cool": "thing" - } -``` - -## Encode value as json string, on one line -Pass in a 0 indent to print json on a single line. - -Given a sample.yml file of: -```yaml -a: - cool: thing -``` -then -```bash -yq eval '.b = (.a | to_json(0))' sample.yml -``` -will output -```yaml -a: - cool: thing -b: '{"cool":"thing"}' -``` - -## Encode value as props string -Given a sample.yml file of: -```yaml -a: - cool: thing -``` -then -```bash -yq eval '.b = (.a | to_props)' sample.yml -``` -will output -```yaml -a: - cool: thing -b: | - cool = thing -``` - ## Decode a yaml encoded string Given a sample.yml file of: ```yaml @@ -178,3 +226,68 @@ will output a: 'foo: cat' ``` +## Encode array of scalars as csv string +Scalars are strings, numbers and booleans. + +Given a sample.yml file of: +```yaml +- cat +- thing1,thing2 +- true +- 3.40 +``` +then +```bash +yq eval '@csv' sample.yml +``` +will output +```yaml +cat,"thing1,thing2",true,3.40 +``` + +## Encode array of arrays as csv string +Given a sample.yml file of: +```yaml +- - cat + - thing1,thing2 + - true + - 3.40 +- - dog + - thing3 + - false + - 12 +``` +then +```bash +yq eval '@csv' sample.yml +``` +will output +```yaml +cat,"thing1,thing2",true,3.40 +dog,thing3,false,12 +``` + +## Encode array of array scalars as tsv string +Scalars are strings, numbers and booleans. + +Given a sample.yml file of: +```yaml +- - cat + - thing1,thing2 + - true + - 3.40 +- - dog + - thing3 + - false + - 12 +``` +then +```bash +yq eval '@tsv' sample.yml +``` +will output +```yaml +cat thing1,thing2 true 3.40 +dog thing3 false 12 +``` + diff --git a/pkg/yqlib/doc/headers/encode-decode.md b/pkg/yqlib/doc/headers/encode-decode.md index 177dce21..92ff9f5a 100644 --- a/pkg/yqlib/doc/headers/encode-decode.md +++ b/pkg/yqlib/doc/headers/encode-decode.md @@ -5,3 +5,16 @@ Encode operators will take the piped in object structure and encode it as a stri Note that you can optionally pass an indent value to the encode functions (see below). These operators are useful to process yaml documents that have stringified embeded yaml/json/props in them. + + +| Format | Decode (from string) | Encode (to string) | +| --- | -- | --| +| Yaml | from_yaml | to_yaml(i)/@yaml | +| JSON | from_json | to_json(i)/@json | +| Properties | | to_props/@props | +| CSV | | to_csv/@csv | +| TSV | | to_tsv/@tsv | + + +CSV and TSV format both accept either a single array or scalars (representing a single row), or an array of array of scalars (representing multiple rows). + diff --git a/pkg/yqlib/encoder_csv.go b/pkg/yqlib/encoder_csv.go index bc5a4260..654e05ac 100644 --- a/pkg/yqlib/encoder_csv.go +++ b/pkg/yqlib/encoder_csv.go @@ -18,16 +18,10 @@ func NewCsvEncoder(destination io.Writer, separator rune) Encoder { return &csvEncoder{csvWriter} } -func (e *csvEncoder) Encode(originalNode *yaml.Node) error { - // node must be a sequence - node := unwrapDoc(originalNode) - if node.Kind != yaml.SequenceNode { - return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), got: %v", node.Tag) - } +func (e *csvEncoder) encodeRow(contents []*yaml.Node) error { + stringValues := make([]string, len(contents)) - stringValues := make([]string, len(node.Content)) - - for i, child := range node.Content { + for i, child := range contents { if child.Kind != yaml.ScalarNode { return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag) @@ -36,3 +30,28 @@ func (e *csvEncoder) Encode(originalNode *yaml.Node) error { } return e.destination.Write(stringValues) } + +func (e *csvEncoder) Encode(originalNode *yaml.Node) error { + // node must be a sequence + node := unwrapDoc(originalNode) + if node.Kind != yaml.SequenceNode { + return fmt.Errorf("csv encoding only works for arrays, got: %v", node.Tag) + } else if len(node.Content) == 0 { + return nil + } + if node.Content[0].Kind == yaml.ScalarNode { + return e.encodeRow(node.Content) + } + + for i, child := range node.Content { + + if child.Kind != yaml.SequenceNode { + return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag) + } + err := e.encodeRow(child.Content) + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/yqlib/encoder_csv_test.go b/pkg/yqlib/encoder_csv_test.go index b29b1e03..1db2859e 100644 --- a/pkg/yqlib/encoder_csv_test.go +++ b/pkg/yqlib/encoder_csv_test.go @@ -31,6 +31,13 @@ func yamlToCsv(sampleYaml string, separator rune) string { var sampleYaml = `["apple", apple2, "comma, in, value", "new line", 3, 3.40, true, "tab here"]` +var sampleYamlArray = "[" + sampleYaml + ", [bob, cat, meow, puss]]" + +func TestCsvEncoderEmptyArray(t *testing.T) { + var actualCsv = yamlToCsv(`[]`, ',') + test.AssertResult(t, "", actualCsv) +} + func TestCsvEncoder(t *testing.T) { var expectedCsv = `apple,apple2,"comma, in, value",new line,3,3.40,true,tab here` @@ -38,6 +45,12 @@ func TestCsvEncoder(t *testing.T) { test.AssertResult(t, expectedCsv, actualCsv) } +func TestCsvEncoderArrayOfArrays(t *testing.T) { + var actualCsv = yamlToCsv(sampleYamlArray, ',') + var expectedCsv = "apple,apple2,\"comma, in, value\",new line,3,3.40,true,tab here\nbob,cat,meow,puss" + test.AssertResult(t, expectedCsv, actualCsv) +} + func TestTsvEncoder(t *testing.T) { var expectedCsv = `apple apple2 comma, in, value new line 3 3.40 true "tab here"` diff --git a/pkg/yqlib/expression_tokeniser.go b/pkg/yqlib/expression_tokeniser.go index 6383efeb..562ddccd 100644 --- a/pkg/yqlib/expression_tokeniser.go +++ b/pkg/yqlib/expression_tokeniser.go @@ -325,10 +325,12 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`to_json\([0-9]+\)`), encodeWithIndent(JsonOutputFormat)) lexer.Add([]byte(`toyaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2})) - lexer.Add([]byte(`@yaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 0})) + // 0 indent doesn't work with yaml. + lexer.Add([]byte(`@yaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2})) lexer.Add([]byte(`tojson`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JsonOutputFormat, indent: 2})) lexer.Add([]byte(`toprops`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2})) + lexer.Add([]byte(`@props`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2})) lexer.Add([]byte(`to_yaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2})) lexer.Add([]byte(`to_json`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JsonOutputFormat, indent: 2})) diff --git a/pkg/yqlib/operator_encoder_decoder.go b/pkg/yqlib/operator_encoder_decoder.go index 328a8e67..d4d61b3f 100644 --- a/pkg/yqlib/operator_encoder_decoder.go +++ b/pkg/yqlib/operator_encoder_decoder.go @@ -56,7 +56,9 @@ func encodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre } // dont print a new line when printing json on a single line. - if preferences.format == JsonOutputFormat && preferences.indent == 0 { + if (preferences.format == JsonOutputFormat && preferences.indent == 0) || + preferences.format == CsvOutputFormat || + preferences.format == TsvOutputFormat { stringValue = chomper.ReplaceAllString(stringValue, "") } diff --git a/pkg/yqlib/operator_encoder_decoder_test.go b/pkg/yqlib/operator_encoder_decoder_test.go index 775590ec..60726e1a 100644 --- a/pkg/yqlib/operator_encoder_decoder_test.go +++ b/pkg/yqlib/operator_encoder_decoder_test.go @@ -7,6 +7,70 @@ import ( var prefix = "D0, P[], (doc)::a:\n cool:\n bob: dylan\n" var encoderDecoderOperatorScenarios = []expressionScenario{ + { + description: "Encode value as json string", + document: `{a: {cool: "thing"}}`, + expression: `.b = (.a | to_json)`, + expected: []string{ + `D0, P[], (doc)::{a: {cool: "thing"}, b: "{\n \"cool\": \"thing\"\n}\n"} +`, + }, + }, + { + description: "Encode value as json string, on one line", + subdescription: "Pass in a 0 indent to print json on a single line.", + document: `{a: {cool: "thing"}}`, + expression: `.b = (.a | to_json(0))`, + expected: []string{ + `D0, P[], (doc)::{a: {cool: "thing"}, b: '{"cool":"thing"}'} +`, + }, + }, + { + description: "Encode value as json string, on one line shorthand", + subdescription: "Pass in a 0 indent to print json on a single line.", + document: `{a: {cool: "thing"}}`, + expression: `.b = (.a | @json)`, + expected: []string{ + `D0, P[], (doc)::{a: {cool: "thing"}, b: '{"cool":"thing"}'} +`, + }, + }, + { + description: "Decode a json encoded string", + subdescription: "Keep in mind JSON is a subset of YAML. If you want idiomatic yaml, pipe through the style operator to clear out the JSON styling.", + document: `a: '{"cool":"thing"}'`, + expression: `.a | from_json | ... style=""`, + expected: []string{ + "D0, P[a], (!!map)::cool: thing\n", + }, + }, + { + skipDoc: true, + document: `{a: {cool: "thing"}}`, + expression: `.b = (.a | to_props)`, + expected: []string{ + `D0, P[], (doc)::{a: {cool: "thing"}, b: "cool = thing\n"} +`, + }, + }, + { + description: "Encode value as props string", + document: `{a: {cool: "thing"}}`, + expression: `.b = (.a | @props)`, + expected: []string{ + `D0, P[], (doc)::{a: {cool: "thing"}, b: "cool = thing\n"} +`, + }, + }, + { + skipDoc: true, + document: "a:\n cool:\n bob: dylan", + expression: `.b = (.a | @yaml)`, + expected: []string{ + prefix + "b: |\n cool:\n bob: dylan\n", + }, + }, { description: "Encode value as yaml string", subdescription: "Indent defaults to 2", @@ -32,34 +96,6 @@ var encoderDecoderOperatorScenarios = []expressionScenario{ expression: `.b = (.a | to_yaml)`, expected: []string{ `D0, P[], (doc)::{a: {cool: "thing"}, b: "{cool: \"thing\"}\n"} -`, - }, - }, - { - description: "Encode value as json string", - document: `{a: {cool: "thing"}}`, - expression: `.b = (.a | to_json)`, - expected: []string{ - `D0, P[], (doc)::{a: {cool: "thing"}, b: "{\n \"cool\": \"thing\"\n}\n"} -`, - }, - }, - { - description: "Encode value as json string, on one line", - subdescription: "Pass in a 0 indent to print json on a single line.", - document: `{a: {cool: "thing"}}`, - expression: `.b = (.a | to_json(0))`, - expected: []string{ - `D0, P[], (doc)::{a: {cool: "thing"}, b: '{"cool":"thing"}'} -`, - }, - }, - { - description: "Encode value as props string", - document: `{a: {cool: "thing"}}`, - expression: `.b = (.a | to_props)`, - expected: []string{ - `D0, P[], (doc)::{a: {cool: "thing"}, b: "cool = thing\n"} `, }, }, @@ -98,6 +134,32 @@ var encoderDecoderOperatorScenarios = []expressionScenario{ "D0, P[], (doc)::a: 'foo: cat'\n", }, }, + { + description: "Encode array of scalars as csv string", + subdescription: "Scalars are strings, numbers and booleans.", + document: `[cat, "thing1,thing2", true, 3.40]`, + expression: `@csv`, + expected: []string{ + "D0, P[], (!!str)::cat,\"thing1,thing2\",true,3.40\n", + }, + }, + { + description: "Encode array of arrays as csv string", + document: `[[cat, "thing1,thing2", true, 3.40], [dog, thing3, false, 12]]`, + expression: `@csv`, + expected: []string{ + "D0, P[], (!!str)::cat,\"thing1,thing2\",true,3.40\ndog,thing3,false,12\n", + }, + }, + { + description: "Encode array of array scalars as tsv string", + subdescription: "Scalars are strings, numbers and booleans.", + document: `[[cat, "thing1,thing2", true, 3.40], [dog, thing3, false, 12]]`, + expression: `@tsv`, + expected: []string{ + "D0, P[], (!!str)::cat\tthing1,thing2\ttrue\t3.40\ndog\tthing3\tfalse\t12\n", + }, + }, { skipDoc: true, dontFormatInputForDoc: true,