suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2024-11-12 13:48:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added encoder tests

Browse Source
This commit is contained in:
Mike Farah 2021-12-02 12:11:15 +11:00
parent f62cfe5ec9
commit df5128fa36
7 changed files with 320 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

227
pkg/yqlib/doc/encode-decode.md
View File

@ -6,6 +6,111 @@ Note that you can optionally pass an indent value to the encode functions (see b
These operators are useful to process yaml documents that have stringified embeded yaml/json/props in them.
| Format | Decode (from string) | Encode (to string) |
| --- | -- | --|
| Yaml | from_yaml | to_yaml(i)/@yaml |
| JSON | from_json | to_json(i)/@json |
| Properties | | to_props/@props |
| CSV | | to_csv/@csv |
| TSV | | to_tsv/@tsv |
CSV and TSV format both accept either a single array or scalars (representing a single row), or an array of array of scalars (representing multiple rows).
## Encode value as json string
Given a sample.yml file of:
```yaml
a:
cool: thing
```
then
```bash
yq eval '.b = (.a | to_json)' sample.yml
```
will output
```yaml
a:
cool: thing
b: |
{
"cool": "thing"
}
```
## Encode value as json string, on one line
Pass in a 0 indent to print json on a single line.
Given a sample.yml file of:
```yaml
a:
cool: thing
```
then
```bash
yq eval '.b = (.a | to_json(0))' sample.yml
```
will output
```yaml
a:
cool: thing
b: '{"cool":"thing"}'
```
## Encode value as json string, on one line shorthand
Pass in a 0 indent to print json on a single line.
Given a sample.yml file of:
```yaml
a:
cool: thing
```
then
```bash
yq eval '.b = (.a | @json)' sample.yml
```
will output
```yaml
a:
cool: thing
b: '{"cool":"thing"}'
```
## Decode a json encoded string
Keep in mind JSON is a subset of YAML. If you want idiomatic yaml, pipe through the style operator to clear out the JSON styling.
Given a sample.yml file of:
```yaml
a: '{"cool":"thing"}'
```
then
```bash
yq eval '.a | from_json | ... style=""' sample.yml
```
will output
```yaml
cool: thing
```
## Encode value as props string
Given a sample.yml file of:
```yaml
a:
cool: thing
```
then
```bash
yq eval '.b = (.a | @props)' sample.yml
```
will output
```yaml
a:
cool: thing
b: |
cool = thing
```
## Encode value as yaml string
Indent defaults to 2
@ -72,63 +177,6 @@ b: |
cool: thing
```
## Encode value as json string
Given a sample.yml file of:
```yaml
a:
cool: thing
```
then
```bash
yq eval '.b = (.a | to_json)' sample.yml
```
will output
```yaml
a:
cool: thing
b: |
{
"cool": "thing"
}
```
## Encode value as json string, on one line
Pass in a 0 indent to print json on a single line.
Given a sample.yml file of:
```yaml
a:
cool: thing
```
then
```bash
yq eval '.b = (.a | to_json(0))' sample.yml
```
will output
```yaml
a:
cool: thing
b: '{"cool":"thing"}'
```
## Encode value as props string
Given a sample.yml file of:
```yaml
a:
cool: thing
```
then
```bash
yq eval '.b = (.a | to_props)' sample.yml
```
will output
```yaml
a:
cool: thing
b: |
cool = thing
```
## Decode a yaml encoded string
Given a sample.yml file of:
```yaml
@ -178,3 +226,68 @@ will output
a: 'foo: cat'
```
## Encode array of scalars as csv string
Scalars are strings, numbers and booleans.
Given a sample.yml file of:
```yaml
- cat
- thing1,thing2
- true
- 3.40
```
then
```bash
yq eval '@csv' sample.yml
```
will output
```yaml
cat,"thing1,thing2",true,3.40
```
## Encode array of arrays as csv string
Given a sample.yml file of:
```yaml
- - cat
- thing1,thing2
- true
- 3.40
- - dog
- thing3
- false
- 12
```
then
```bash
yq eval '@csv' sample.yml
```
will output
```yaml
cat,"thing1,thing2",true,3.40
dog,thing3,false,12
```
## Encode array of array scalars as tsv string
Scalars are strings, numbers and booleans.
Given a sample.yml file of:
```yaml
- - cat
- thing1,thing2
- true
- 3.40
- - dog
- thing3
- false
- 12
```
then
```bash
yq eval '@tsv' sample.yml
```
will output
```yaml
cat thing1,thing2 true 3.40
dog thing3 false 12
```

13
pkg/yqlib/doc/headers/encode-decode.md
View File

@ -5,3 +5,16 @@ Encode operators will take the piped in object structure and encode it as a stri
Note that you can optionally pass an indent value to the encode functions (see below).
These operators are useful to process yaml documents that have stringified embeded yaml/json/props in them.
| Format | Decode (from string) | Encode (to string) |
| --- | -- | --|
| Yaml | from_yaml | to_yaml(i)/@yaml |
| JSON | from_json | to_json(i)/@json |
| Properties | | to_props/@props |
| CSV | | to_csv/@csv |
| TSV | | to_tsv/@tsv |
CSV and TSV format both accept either a single array or scalars (representing a single row), or an array of array of scalars (representing multiple rows).

37
pkg/yqlib/encoder_csv.go
View File

@ -18,16 +18,10 @@ func NewCsvEncoder(destination io.Writer, separator rune) Encoder {
return &csvEncoder{csvWriter}
}
func (e *csvEncoder) Encode(originalNode *yaml.Node) error {
// node must be a sequence
node := unwrapDoc(originalNode)
if node.Kind != yaml.SequenceNode {
return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), got: %v", node.Tag)
}
func (e *csvEncoder) encodeRow(contents []*yaml.Node) error {
stringValues := make([]string, len(contents))
stringValues := make([]string, len(node.Content))
for i, child := range node.Content {
for i, child := range contents {
if child.Kind != yaml.ScalarNode {
return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag)
@ -36,3 +30,28 @@ func (e *csvEncoder) Encode(originalNode *yaml.Node) error {
}
return e.destination.Write(stringValues)
}
func (e *csvEncoder) Encode(originalNode *yaml.Node) error {
// node must be a sequence
node := unwrapDoc(originalNode)
if node.Kind != yaml.SequenceNode {
return fmt.Errorf("csv encoding only works for arrays, got: %v", node.Tag)
} else if len(node.Content) == 0 {
return nil
}
if node.Content[0].Kind == yaml.ScalarNode {
return e.encodeRow(node.Content)
}
for i, child := range node.Content {
if child.Kind != yaml.SequenceNode {
return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag)
}
err := e.encodeRow(child.Content)
if err != nil {
return err
}
}
return nil
}

13
pkg/yqlib/encoder_csv_test.go
View File

@ -31,6 +31,13 @@ func yamlToCsv(sampleYaml string, separator rune) string {
var sampleYaml = `["apple", apple2, "comma, in, value", "new
line", 3, 3.40, true, "tab here"]`
var sampleYamlArray = "[" + sampleYaml + ", [bob, cat, meow, puss]]"
func TestCsvEncoderEmptyArray(t *testing.T) {
var actualCsv = yamlToCsv(`[]`, ',')
test.AssertResult(t, "", actualCsv)
}
func TestCsvEncoder(t *testing.T) {
var expectedCsv = `apple,apple2,"comma, in, value",new line,3,3.40,true,tab here`
@ -38,6 +45,12 @@ func TestCsvEncoder(t *testing.T) {
test.AssertResult(t, expectedCsv, actualCsv)
}
func TestCsvEncoderArrayOfArrays(t *testing.T) {
var actualCsv = yamlToCsv(sampleYamlArray, ',')
var expectedCsv = "apple,apple2,\"comma, in, value\",new line,3,3.40,true,tab here\nbob,cat,meow,puss"
test.AssertResult(t, expectedCsv, actualCsv)
}
func TestTsvEncoder(t *testing.T) {
var expectedCsv = `apple apple2 comma, in, value new line 3 3.40 true "tab here"`

4
pkg/yqlib/expression_tokeniser.go
View File

@ -325,10 +325,12 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`to_json\([0-9]+\)`), encodeWithIndent(JsonOutputFormat))
lexer.Add([]byte(`toyaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2}))
lexer.Add([]byte(`@yaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 0}))
// 0 indent doesn't work with yaml.
lexer.Add([]byte(`@yaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2}))
lexer.Add([]byte(`tojson`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JsonOutputFormat, indent: 2}))
lexer.Add([]byte(`toprops`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2}))
lexer.Add([]byte(`@props`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2}))
lexer.Add([]byte(`to_yaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2}))
lexer.Add([]byte(`to_json`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JsonOutputFormat, indent: 2}))

4
pkg/yqlib/operator_encoder_decoder.go
View File

@ -56,7 +56,9 @@ func encodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
}
// dont print a new line when printing json on a single line.
if preferences.format == JsonOutputFormat && preferences.indent == 0 {
if (preferences.format == JsonOutputFormat && preferences.indent == 0) ||
preferences.format == CsvOutputFormat ||
preferences.format == TsvOutputFormat {
stringValue = chomper.ReplaceAllString(stringValue, "")
}

118
pkg/yqlib/operator_encoder_decoder_test.go
View File

@ -7,6 +7,70 @@ import (
var prefix = "D0, P[], (doc)::a:\n cool:\n bob: dylan\n"
var encoderDecoderOperatorScenarios = []expressionScenario{
{
description: "Encode value as json string",
document: `{a: {cool: "thing"}}`,
expression: `.b = (.a | to_json)`,
expected: []string{
`D0, P[], (doc)::{a: {cool: "thing"}, b: "{\n \"cool\": \"thing\"\n}\n"}
`,
},
},
{
description: "Encode value as json string, on one line",
subdescription: "Pass in a 0 indent to print json on a single line.",
document: `{a: {cool: "thing"}}`,
expression: `.b = (.a | to_json(0))`,
expected: []string{
`D0, P[], (doc)::{a: {cool: "thing"}, b: '{"cool":"thing"}'}
`,
},
},
{
description: "Encode value as json string, on one line shorthand",
subdescription: "Pass in a 0 indent to print json on a single line.",
document: `{a: {cool: "thing"}}`,
expression: `.b = (.a | @json)`,
expected: []string{
`D0, P[], (doc)::{a: {cool: "thing"}, b: '{"cool":"thing"}'}
`,
},
},
{
description: "Decode a json encoded string",
subdescription: "Keep in mind JSON is a subset of YAML. If you want idiomatic yaml, pipe through the style operator to clear out the JSON styling.",
document: `a: '{"cool":"thing"}'`,
expression: `.a | from_json | ... style=""`,
expected: []string{
"D0, P[a], (!!map)::cool: thing\n",
},
},
{
skipDoc: true,
document: `{a: {cool: "thing"}}`,
expression: `.b = (.a | to_props)`,
expected: []string{
`D0, P[], (doc)::{a: {cool: "thing"}, b: "cool = thing\n"}
`,
},
},
{
description: "Encode value as props string",
document: `{a: {cool: "thing"}}`,
expression: `.b = (.a | @props)`,
expected: []string{
`D0, P[], (doc)::{a: {cool: "thing"}, b: "cool = thing\n"}
`,
},
},
{
skipDoc: true,
document: "a:\n cool:\n bob: dylan",
expression: `.b = (.a | @yaml)`,
expected: []string{
prefix + "b: |\n cool:\n bob: dylan\n",
},
},
{
description: "Encode value as yaml string",
subdescription: "Indent defaults to 2",
@ -32,34 +96,6 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
expression: `.b = (.a | to_yaml)`,
expected: []string{
`D0, P[], (doc)::{a: {cool: "thing"}, b: "{cool: \"thing\"}\n"}
`,
},
},
{
description: "Encode value as json string",
document: `{a: {cool: "thing"}}`,
expression: `.b = (.a | to_json)`,
expected: []string{
`D0, P[], (doc)::{a: {cool: "thing"}, b: "{\n \"cool\": \"thing\"\n}\n"}
`,
},
},
{
description: "Encode value as json string, on one line",
subdescription: "Pass in a 0 indent to print json on a single line.",
document: `{a: {cool: "thing"}}`,
expression: `.b = (.a | to_json(0))`,
expected: []string{
`D0, P[], (doc)::{a: {cool: "thing"}, b: '{"cool":"thing"}'}
`,
},
},
{
description: "Encode value as props string",
document: `{a: {cool: "thing"}}`,
expression: `.b = (.a | to_props)`,
expected: []string{
`D0, P[], (doc)::{a: {cool: "thing"}, b: "cool = thing\n"}
`,
},
},
@ -98,6 +134,32 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
"D0, P[], (doc)::a: 'foo: cat'\n",
},
},
{
description: "Encode array of scalars as csv string",
subdescription: "Scalars are strings, numbers and booleans.",
document: `[cat, "thing1,thing2", true, 3.40]`,
expression: `@csv`,
expected: []string{
"D0, P[], (!!str)::cat,\"thing1,thing2\",true,3.40\n",
},
},
{
description: "Encode array of arrays as csv string",
document: `[[cat, "thing1,thing2", true, 3.40], [dog, thing3, false, 12]]`,
expression: `@csv`,
expected: []string{
"D0, P[], (!!str)::cat,\"thing1,thing2\",true,3.40\ndog,thing3,false,12\n",
},
},
{
description: "Encode array of array scalars as tsv string",
subdescription: "Scalars are strings, numbers and booleans.",
document: `[[cat, "thing1,thing2", true, 3.40], [dog, thing3, false, 12]]`,
expression: `@tsv`,
expected: []string{
"D0, P[], (!!str)::cat\tthing1,thing2\ttrue\t3.40\ndog\tthing3\tfalse\t12\n",
},
},
{
skipDoc: true,
dontFormatInputForDoc: true,