Added encoder tests

pkg/yqlib/doc/encode-decode.md

@@ -6,6 +6,111 @@ Note that you can optionally pass an indent value to the encode functions (see b
 
 These operators are useful to process yaml documents that have stringified embeded yaml/json/props in them.
 
+
+| Format | Decode (from string) | Encode (to string) |
+| --- | -- | --|
+| Yaml | from_yaml | to_yaml(i)/@yaml |
+| JSON | from_json | to_json(i)/@json |
+| Properties |  | to_props/@props |
+| CSV |  | to_csv/@csv |
+| TSV |  | to_tsv/@tsv |
+
+
+CSV and TSV format both accept either a single array or scalars (representing a single row), or an array of array of scalars (representing multiple rows). 
+
+
+## Encode value as json string
+Given a sample.yml file of:
+```yaml
+a:
+  cool: thing
+```
+then
+```bash
+yq eval '.b = (.a | to_json)' sample.yml
+```
+will output
+```yaml
+a:
+  cool: thing
+b: |
+  {
+    "cool": "thing"
+  }
+```
+
+## Encode value as json string, on one line
+Pass in a 0 indent to print json on a single line.
+
+Given a sample.yml file of:
+```yaml
+a:
+  cool: thing
+```
+then
+```bash
+yq eval '.b = (.a | to_json(0))' sample.yml
+```
+will output
+```yaml
+a:
+  cool: thing
+b: '{"cool":"thing"}'
+```
+
+## Encode value as json string, on one line shorthand
+Pass in a 0 indent to print json on a single line.
+
+Given a sample.yml file of:
+```yaml
+a:
+  cool: thing
+```
+then
+```bash
+yq eval '.b = (.a | @json)' sample.yml
+```
+will output
+```yaml
+a:
+  cool: thing
+b: '{"cool":"thing"}'
+```
+
+## Decode a json encoded string
+Keep in mind JSON is a subset of YAML. If you want idiomatic yaml, pipe through the style operator to clear out the JSON styling.
+
+Given a sample.yml file of:
+```yaml
+a: '{"cool":"thing"}'
+```
+then
+```bash
+yq eval '.a | from_json | ... style=""' sample.yml
+```
+will output
+```yaml
+cool: thing
+```
+
+## Encode value as props string
+Given a sample.yml file of:
+```yaml
+a:
+  cool: thing
+```
+then
+```bash
+yq eval '.b = (.a | @props)' sample.yml
+```
+will output
+```yaml
+a:
+  cool: thing
+b: |
+  cool = thing
+```
+
 ## Encode value as yaml string
 Indent defaults to 2
 
@@ -72,63 +177,6 @@ b: |
   cool: thing
 ```
 
-## Encode value as json string
-Given a sample.yml file of:
-```yaml
-a:
-  cool: thing
-```
-then
-```bash
-yq eval '.b = (.a | to_json)' sample.yml
-```
-will output
-```yaml
-a:
-  cool: thing
-b: |
-  {
-    "cool": "thing"
-  }
-```
-
-## Encode value as json string, on one line
-Pass in a 0 indent to print json on a single line.
-
-Given a sample.yml file of:
-```yaml
-a:
-  cool: thing
-```
-then
-```bash
-yq eval '.b = (.a | to_json(0))' sample.yml
-```
-will output
-```yaml
-a:
-  cool: thing
-b: '{"cool":"thing"}'
-```
-
-## Encode value as props string
-Given a sample.yml file of:
-```yaml
-a:
-  cool: thing
-```
-then
-```bash
-yq eval '.b = (.a | to_props)' sample.yml
-```
-will output
-```yaml
-a:
-  cool: thing
-b: |
-  cool = thing
-```
-
 ## Decode a yaml encoded string
 Given a sample.yml file of:
 ```yaml
@@ -178,3 +226,68 @@ will output
 a: 'foo: cat'
 ```
 
+## Encode array of scalars as csv string
+Scalars are strings, numbers and booleans.
+
+Given a sample.yml file of:
+```yaml
+- cat
+- thing1,thing2
+- true
+- 3.40
+```
+then
+```bash
+yq eval '@csv' sample.yml
+```
+will output
+```yaml
+cat,"thing1,thing2",true,3.40
+```
+
+## Encode array of arrays as csv string
+Given a sample.yml file of:
+```yaml
+- - cat
+  - thing1,thing2
+  - true
+  - 3.40
+- - dog
+  - thing3
+  - false
+  - 12
+```
+then
+```bash
+yq eval '@csv' sample.yml
+```
+will output
+```yaml
+cat,"thing1,thing2",true,3.40
+dog,thing3,false,12
+```
+
+## Encode array of array scalars as tsv string
+Scalars are strings, numbers and booleans.
+
+Given a sample.yml file of:
+```yaml
+- - cat
+  - thing1,thing2
+  - true
+  - 3.40
+- - dog
+  - thing3
+  - false
+  - 12
+```
+then
+```bash
+yq eval '@tsv' sample.yml
+```
+will output
+```yaml
+cat	thing1,thing2	true	3.40
+dog	thing3	false	12
+```
+

pkg/yqlib/doc/headers/encode-decode.md

@@ -5,3 +5,16 @@ Encode operators will take the piped in object structure and encode it as a stri
 Note that you can optionally pass an indent value to the encode functions (see below).
 
 These operators are useful to process yaml documents that have stringified embeded yaml/json/props in them.
+
+
+| Format | Decode (from string) | Encode (to string) |
+| --- | -- | --|
+| Yaml | from_yaml | to_yaml(i)/@yaml |
+| JSON | from_json | to_json(i)/@json |
+| Properties |  | to_props/@props |
+| CSV |  | to_csv/@csv |
+| TSV |  | to_tsv/@tsv |
+
+
+CSV and TSV format both accept either a single array or scalars (representing a single row), or an array of array of scalars (representing multiple rows). 
+

pkg/yqlib/encoder_csv.go

@@ -18,16 +18,10 @@ func NewCsvEncoder(destination io.Writer, separator rune) Encoder {
 	return &csvEncoder{csvWriter}
 }
 
-func (e *csvEncoder) Encode(originalNode *yaml.Node) error {
+func (e *csvEncoder) encodeRow(contents []*yaml.Node) error {
-	// node must be a sequence
+	stringValues := make([]string, len(contents))
-	node := unwrapDoc(originalNode)
-	if node.Kind != yaml.SequenceNode {
-		return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), got: %v", node.Tag)
-	}
 
-	stringValues := make([]string, len(node.Content))
+	for i, child := range contents {
-
-	for i, child := range node.Content {
 
 		if child.Kind != yaml.ScalarNode {
 			return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag)
@@ -36,3 +30,28 @@ func (e *csvEncoder) Encode(originalNode *yaml.Node) error {
 	}
 	return e.destination.Write(stringValues)
 }
+
+func (e *csvEncoder) Encode(originalNode *yaml.Node) error {
+	// node must be a sequence
+	node := unwrapDoc(originalNode)
+	if node.Kind != yaml.SequenceNode {
+		return fmt.Errorf("csv encoding only works for arrays, got: %v", node.Tag)
+	} else if len(node.Content) == 0 {
+		return nil
+	}
+	if node.Content[0].Kind == yaml.ScalarNode {
+		return e.encodeRow(node.Content)
+	}
+
+	for i, child := range node.Content {
+
+		if child.Kind != yaml.SequenceNode {
+			return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag)
+		}
+		err := e.encodeRow(child.Content)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/yqlib/encoder_csv_test.go

@@ -31,6 +31,13 @@ func yamlToCsv(sampleYaml string, separator rune) string {
 var sampleYaml = `["apple", apple2, "comma, in, value", "new
 line", 3, 3.40, true, "tab	here"]`
 
+var sampleYamlArray = "[" + sampleYaml + ", [bob, cat, meow, puss]]"
+
+func TestCsvEncoderEmptyArray(t *testing.T) {
+	var actualCsv = yamlToCsv(`[]`, ',')
+	test.AssertResult(t, "", actualCsv)
+}
+
 func TestCsvEncoder(t *testing.T) {
 	var expectedCsv = `apple,apple2,"comma, in, value",new line,3,3.40,true,tab	here`
 
@@ -38,6 +45,12 @@ func TestCsvEncoder(t *testing.T) {
 	test.AssertResult(t, expectedCsv, actualCsv)
 }
 
+func TestCsvEncoderArrayOfArrays(t *testing.T) {
+	var actualCsv = yamlToCsv(sampleYamlArray, ',')
+	var expectedCsv = "apple,apple2,\"comma, in, value\",new line,3,3.40,true,tab	here\nbob,cat,meow,puss"
+	test.AssertResult(t, expectedCsv, actualCsv)
+}
+
 func TestTsvEncoder(t *testing.T) {
 
 	var expectedCsv = `apple	apple2	comma, in, value	new line	3	3.40	true	"tab	here"`

pkg/yqlib/expression_tokeniser.go

@@ -325,10 +325,12 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`to_json\([0-9]+\)`), encodeWithIndent(JsonOutputFormat))
 
 	lexer.Add([]byte(`toyaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2}))
-	lexer.Add([]byte(`@yaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 0}))
+	// 0 indent doesn't work with yaml.
+	lexer.Add([]byte(`@yaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2}))
 
 	lexer.Add([]byte(`tojson`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JsonOutputFormat, indent: 2}))
 	lexer.Add([]byte(`toprops`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2}))
+	lexer.Add([]byte(`@props`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2}))
 
 	lexer.Add([]byte(`to_yaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2}))
 	lexer.Add([]byte(`to_json`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JsonOutputFormat, indent: 2}))

pkg/yqlib/operator_encoder_decoder.go

@@ -56,7 +56,9 @@ func encodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 		}
 
 		// dont print a new line when printing json on a single line.
-		if preferences.format == JsonOutputFormat && preferences.indent == 0 {
+		if (preferences.format == JsonOutputFormat && preferences.indent == 0) ||
+			preferences.format == CsvOutputFormat ||
+			preferences.format == TsvOutputFormat {
 			stringValue = chomper.ReplaceAllString(stringValue, "")
 		}
 

pkg/yqlib/operator_encoder_decoder_test.go

@@ -7,6 +7,70 @@ import (
 var prefix = "D0, P[], (doc)::a:\n    cool:\n        bob: dylan\n"
 
 var encoderDecoderOperatorScenarios = []expressionScenario{
+	{
+		description: "Encode value as json string",
+		document:    `{a: {cool: "thing"}}`,
+		expression:  `.b = (.a | to_json)`,
+		expected: []string{
+			`D0, P[], (doc)::{a: {cool: "thing"}, b: "{\n  \"cool\": \"thing\"\n}\n"}
+`,
+		},
+	},
+	{
+		description:    "Encode value as json string, on one line",
+		subdescription: "Pass in a 0 indent to print json on a single line.",
+		document:       `{a: {cool: "thing"}}`,
+		expression:     `.b = (.a | to_json(0))`,
+		expected: []string{
+			`D0, P[], (doc)::{a: {cool: "thing"}, b: '{"cool":"thing"}'}
+`,
+		},
+	},
+	{
+		description:    "Encode value as json string, on one line shorthand",
+		subdescription: "Pass in a 0 indent to print json on a single line.",
+		document:       `{a: {cool: "thing"}}`,
+		expression:     `.b = (.a | @json)`,
+		expected: []string{
+			`D0, P[], (doc)::{a: {cool: "thing"}, b: '{"cool":"thing"}'}
+`,
+		},
+	},
+	{
+		description:    "Decode a json encoded string",
+		subdescription: "Keep in mind JSON is a subset of YAML. If you want idiomatic yaml, pipe through the style operator to clear out the JSON styling.",
+		document:       `a: '{"cool":"thing"}'`,
+		expression:     `.a | from_json | ... style=""`,
+		expected: []string{
+			"D0, P[a], (!!map)::cool: thing\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: {cool: "thing"}}`,
+		expression: `.b = (.a | to_props)`,
+		expected: []string{
+			`D0, P[], (doc)::{a: {cool: "thing"}, b: "cool = thing\n"}
+`,
+		},
+	},
+	{
+		description: "Encode value as props string",
+		document:    `{a: {cool: "thing"}}`,
+		expression:  `.b = (.a | @props)`,
+		expected: []string{
+			`D0, P[], (doc)::{a: {cool: "thing"}, b: "cool = thing\n"}
+`,
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a:\n  cool:\n    bob: dylan",
+		expression: `.b = (.a | @yaml)`,
+		expected: []string{
+			prefix + "b: |\n    cool:\n      bob: dylan\n",
+		},
+	},
 	{
 		description:    "Encode value as yaml string",
 		subdescription: "Indent defaults to 2",
@@ -32,34 +96,6 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 		expression:     `.b = (.a | to_yaml)`,
 		expected: []string{
 			`D0, P[], (doc)::{a: {cool: "thing"}, b: "{cool: \"thing\"}\n"}
-`,
-		},
-	},
-	{
-		description: "Encode value as json string",
-		document:    `{a: {cool: "thing"}}`,
-		expression:  `.b = (.a | to_json)`,
-		expected: []string{
-			`D0, P[], (doc)::{a: {cool: "thing"}, b: "{\n  \"cool\": \"thing\"\n}\n"}
-`,
-		},
-	},
-	{
-		description:    "Encode value as json string, on one line",
-		subdescription: "Pass in a 0 indent to print json on a single line.",
-		document:       `{a: {cool: "thing"}}`,
-		expression:     `.b = (.a | to_json(0))`,
-		expected: []string{
-			`D0, P[], (doc)::{a: {cool: "thing"}, b: '{"cool":"thing"}'}
-`,
-		},
-	},
-	{
-		description: "Encode value as props string",
-		document:    `{a: {cool: "thing"}}`,
-		expression:  `.b = (.a | to_props)`,
-		expected: []string{
-			`D0, P[], (doc)::{a: {cool: "thing"}, b: "cool = thing\n"}
 `,
 		},
 	},
@@ -98,6 +134,32 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: 'foo: cat'\n",
 		},
 	},
+	{
+		description:    "Encode array of scalars as csv string",
+		subdescription: "Scalars are strings, numbers and booleans.",
+		document:       `[cat, "thing1,thing2", true, 3.40]`,
+		expression:     `@csv`,
+		expected: []string{
+			"D0, P[], (!!str)::cat,\"thing1,thing2\",true,3.40\n",
+		},
+	},
+	{
+		description: "Encode array of arrays as csv string",
+		document:    `[[cat, "thing1,thing2", true, 3.40], [dog, thing3, false, 12]]`,
+		expression:  `@csv`,
+		expected: []string{
+			"D0, P[], (!!str)::cat,\"thing1,thing2\",true,3.40\ndog,thing3,false,12\n",
+		},
+	},
+	{
+		description:    "Encode array of array scalars as tsv string",
+		subdescription: "Scalars are strings, numbers and booleans.",
+		document:       `[[cat, "thing1,thing2", true, 3.40], [dog, thing3, false, 12]]`,
+		expression:     `@tsv`,
+		expected: []string{
+			"D0, P[], (!!str)::cat\tthing1,thing2\ttrue\t3.40\ndog\tthing3\tfalse\t12\n",
+		},
+	},
 	{
 		skipDoc:               true,
 		dontFormatInputForDoc: true,