Bug fix

* WIP: adding CSV decoder

* Adding CSV decoder

* Added CSV roundtrip

* Fixing from review

.gitignore

@@ -56,4 +56,7 @@ _build
 debian/files
 
 # intellij
-/.idea
+/.idea
+
+# vscode
+.vscode

README.md

@@ -3,7 +3,7 @@
 ![Build](https://github.com/mikefarah/yq/workflows/Build/badge.svg)  ![Docker Pulls](https://img.shields.io/docker/pulls/mikefarah/yq.svg) ![Github Releases (by Release)](https://img.shields.io/github/downloads/mikefarah/yq/total.svg) ![Go Report](https://goreportcard.com/badge/github.com/mikefarah/yq) ![CodeQL](https://github.com/mikefarah/yq/workflows/CodeQL/badge.svg)
 
 
-a lightweight and portable command-line YAML, JSON and XML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json and xml. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.
+a lightweight and portable command-line YAML, JSON and XML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json, xml, properties, csv and tsv. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.
 
 yq is written in go - so you can download a dependency free binary for your platform and you are good to go! If you prefer there are a variety of package managers that can be used as well as Docker and Podman, all listed below.
 
@@ -207,6 +207,14 @@ RUN apk add --no-cache tzdata
 USER yq
 ```
 
+#### Podman with SELinux
+
+If you are using podman with SELinux, you will need to set the shared volume flag `:z` on the volume mount:
+
+```
+-v "${PWD}":/workdir:z
+```
+
 ### GitHub Action
 ```
   - name: Set foobar to cool
@@ -290,10 +298,10 @@ Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq)
 - Keeps yaml formatting and comments when updating (though there are issues with whitespace)
 - [Decode/Encode base64 data](https://mikefarah.gitbook.io/yq/operators/encode-decode)
 - [Load content from other files](https://mikefarah.gitbook.io/yq/operators/load)
-- [Convert to/from json](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert)
+- [Convert to/from json/ndjson](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert)
 - [Convert to/from xml](https://mikefarah.gitbook.io/yq/v/v4.x/usage/xml)
 - [Convert to/from properties](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties)
-- [Convert to csv/tsv](https://mikefarah.gitbook.io/yq/usage/csv-tsv)
+- [Convert to/from csv/tsv](https://mikefarah.gitbook.io/yq/usage/csv-tsv)
 - [General shell completion scripts (bash/zsh/fish/powershell)](https://mikefarah.gitbook.io/yq/v/v4.x/commands/shell-completion)
 - [Reduce](https://mikefarah.gitbook.io/yq/operators/reduce) to merge multiple files or sum an array or other fancy things.
 - [Github Action](https://mikefarah.gitbook.io/yq/usage/github-action) to use in your automated pipeline (thanks @devorbitus)

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.26.1"
+	Version = "4.27.2"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

pkg/yqlib/decoder_json.go

@@ -22,13 +22,13 @@ func (dec *jsonDecoder) Init(reader io.Reader) {
 
 func (dec *jsonDecoder) Decode(rootYamlNode *yaml.Node) error {
 
-	var dataBucket interface{}
+	var dataBucket orderedMap
 	log.Debug("going to decode")
 	err := dec.decoder.Decode(&dataBucket)
 	if err != nil {
 		return err
 	}
-	node, err := dec.convertToYamlNode(dataBucket)
+	node, err := dec.convertToYamlNode(&dataBucket)
 
 	if err != nil {
 		return err
@@ -38,39 +38,36 @@ func (dec *jsonDecoder) Decode(rootYamlNode *yaml.Node) error {
 	return nil
 }
 
-func (dec *jsonDecoder) convertToYamlNode(data interface{}) (*yaml.Node, error) {
-	switch data := data.(type) {
-	case nil:
-		return createScalarNode(nil, "null"), nil
-	case float64, float32:
-		// json decoder returns ints as float.
-		return parseSnippet(fmt.Sprintf("%v", data))
-	case int, int64, int32, string, bool:
-		return createScalarNode(data, fmt.Sprintf("%v", data)), nil
-	case map[string]interface{}:
-		return dec.parseMap(data)
-	case []interface{}:
-		return dec.parseArray(data)
-	default:
-		return nil, fmt.Errorf("unrecognised type :(")
+func (dec *jsonDecoder) convertToYamlNode(data *orderedMap) (*yaml.Node, error) {
+	if data.kv == nil {
+		switch rawData := data.altVal.(type) {
+		case nil:
+			return createScalarNode(nil, "null"), nil
+		case float64, float32:
+			// json decoder returns ints as float.
+			return parseSnippet(fmt.Sprintf("%v", rawData))
+		case int, int64, int32, string, bool:
+			return createScalarNode(rawData, fmt.Sprintf("%v", rawData)), nil
+		case []*orderedMap:
+			return dec.parseArray(rawData)
+		default:
+			return nil, fmt.Errorf("unrecognised type :( %v", rawData)
+		}
 	}
-}
-
-func (dec *jsonDecoder) parseMap(dataMap map[string]interface{}) (*yaml.Node, error) {
 
 	var yamlMap = &yaml.Node{Kind: yaml.MappingNode}
-
-	for key, value := range dataMap {
-		yamlValue, err := dec.convertToYamlNode(value)
+	for _, keyValuePair := range data.kv {
+		yamlValue, err := dec.convertToYamlNode(&keyValuePair.V)
 		if err != nil {
 			return nil, err
 		}
-		yamlMap.Content = append(yamlMap.Content, createScalarNode(key, fmt.Sprintf("%v", key)), yamlValue)
+		yamlMap.Content = append(yamlMap.Content, createScalarNode(keyValuePair.K, keyValuePair.K), yamlValue)
 	}
 	return yamlMap, nil
+
 }
 
-func (dec *jsonDecoder) parseArray(dataArray []interface{}) (*yaml.Node, error) {
+func (dec *jsonDecoder) parseArray(dataArray []*orderedMap) (*yaml.Node, error) {
 
 	var yamlMap = &yaml.Node{Kind: yaml.SequenceNode}
 

pkg/yqlib/doc/operators/encode-decode.md

@@ -9,12 +9,12 @@ These operators are useful to process yaml documents that have stringified embed
 
 | Format | Decode (from string) | Encode (to string) |
 | --- | -- | --|
-| Yaml | from_yaml | to_yaml(i)/@yaml |
-| JSON | from_json | to_json(i)/@json |
+| Yaml | from_yaml/@yamld | to_yaml(i)/@yaml |
+| JSON | from_json/@jsond | to_json(i)/@json |
 | Properties | from_props/@propsd  | to_props/@props |
 | CSV | from_csv/@csvd | to_csv/@csv |
 | TSV | from_tsv/@tsvd | to_tsv/@tsv |
-| XML | from_xml | to_xml(i)/@xml |
+| XML | from_xml/@xmld | to_xml(i)/@xml |
 | Base64 | @base64d | @base64 |
 
 

pkg/yqlib/doc/operators/headers/encode-decode.md

@@ -9,12 +9,12 @@ These operators are useful to process yaml documents that have stringified embed
 
 | Format | Decode (from string) | Encode (to string) |
 | --- | -- | --|
-| Yaml | from_yaml | to_yaml(i)/@yaml |
-| JSON | from_json | to_json(i)/@json |
+| Yaml | from_yaml/@yamld | to_yaml(i)/@yaml |
+| JSON | from_json/@jsond | to_json(i)/@json |
 | Properties | from_props/@propsd  | to_props/@props |
 | CSV | from_csv/@csvd | to_csv/@csv |
 | TSV | from_tsv/@tsvd | to_tsv/@tsv |
-| XML | from_xml | to_xml(i)/@xml |
+| XML | from_xml/@xmld | to_xml(i)/@xml |
 | Base64 | @base64d | @base64 |
 
 

pkg/yqlib/doc/usage/convert.md

@@ -1,10 +1,9 @@
 # JSON
 
-Encode and decode to and from JSON. Note that, unless you have multiple JSON documents in a single file, YAML is a _superset_ of JSON - so `yq` can read any json file without doing anything special.
+Encode and decode to and from JSON. Supports multiple JSON documents in a single file (e.g. NDJSON).
 
-If you do have mulitple JSON documents in a single file (e.g. NDJSON) then you will need to explicity use the json parser `-p=json`.
+Note that YAML is a superset of (single document) JSON - so you don't have to use the JSON parser to read JSON when there is only one JSON document in the input. You will probably want to pretty print the result in this case, to get idiomatic YAML styling.
 
-This means you don't need to 'convert' a JSON file to YAML - however if you want idiomatic YAML styling, then you can use the `-P/--prettyPrint` flag, see examples below.
 
 {% hint style="warning" %}
 Note that versions prior to 4.18 require the 'eval/e' command to be specified. 

pkg/yqlib/doc/usage/headers/convert.md

@@ -1,7 +1,6 @@
 # JSON
 
-Encode and decode to and from JSON. Note that, unless you have multiple JSON documents in a single file, YAML is a _superset_ of JSON - so `yq` can read any json file without doing anything special.
+Encode and decode to and from JSON. Supports multiple JSON documents in a single file (e.g. NDJSON).
 
-If you do have mulitple JSON documents in a single file (e.g. NDJSON) then you will need to explicity use the json parser `-p=json`.
+Note that YAML is a superset of (single document) JSON - so you don't have to use the JSON parser to read JSON when there is only one JSON document in the input. You will probably want to pretty print the result in this case, to get idiomatic YAML styling.
 
-This means you don't need to 'convert' a JSON file to YAML - however if you want idiomatic YAML styling, then you can use the `-P/--prettyPrint` flag, see examples below.

pkg/yqlib/encoder.go

@@ -66,8 +66,13 @@ func (o *orderedMap) UnmarshalJSON(data []byte) error {
 		}
 		return nil
 	case '[':
-		var arr []orderedMap
-		return json.Unmarshal(data, &arr)
+		var res []*orderedMap
+		if err := json.Unmarshal(data, &res); err != nil {
+			return err
+		}
+		o.altVal = res
+		o.kv = nil
+		return nil
 	}
 
 	return json.Unmarshal(data, &o.altVal)

pkg/yqlib/json_test.go

@@ -22,6 +22,13 @@ const sampleNdJson = `{"this": "is a multidoc json file"}
 {"a number": 4}
 `
 
+const sampleNdJsonKey = `{"a": "first", "b": "next", "ab": "last"}`
+
+const expectedJsonKeysInOrder = `a: first
+b: next
+ab: last
+`
+
 const expectedNdJsonYaml = `this: is a multidoc json file
 ---
 each:
@@ -157,6 +164,13 @@ var jsonScenarios = []formatScenario{
 		expected:     expectedNdJsonYaml,
 		scenarioType: "decode-ndjson",
 	},
+	{
+		description:  "Decode NDJSON, maintain key order",
+		skipDoc:      true,
+		input:        sampleNdJsonKey,
+		expected:     expectedJsonKeysInOrder,
+		scenarioType: "decode-ndjson",
+	},
 	{
 		description:  "numbers",
 		skipDoc:      true,
@@ -164,6 +178,20 @@ var jsonScenarios = []formatScenario{
 		expected:     "- 3\n- 3\n- 3.1\n- -1\n",
 		scenarioType: "decode-ndjson",
 	},
+	{
+		description:  "number single",
+		skipDoc:      true,
+		input:        "3",
+		expected:     "3\n",
+		scenarioType: "decode-ndjson",
+	},
+	{
+		description:  "empty string",
+		skipDoc:      true,
+		input:        `""`,
+		expected:     "\"\"\n",
+		scenarioType: "decode-ndjson",
+	},
 	{
 		description:  "strings",
 		skipDoc:      true,

pkg/yqlib/stream_evaluator.go

@@ -110,7 +110,7 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr
 			Node:      &dataBucket,
 			FileIndex: s.fileIndex,
 		}
-		//move document comments into candidate node
+		// move document comments into candidate node
 		// otherwise unwrap drops them.
 		candidateNode.TrailingContent = dataBucket.FootComment
 		dataBucket.FootComment = ""

pkg/yqlib/string_evaluator.go

@@ -0,0 +1,85 @@
+package yqlib
+
+import (
+	"bytes"
+	"container/list"
+	"errors"
+	"fmt"
+	"io"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+type StringEvaluator interface {
+	Evaluate(expression string, input string, encoder Encoder, leadingContentPreProcessing bool, decoder Decoder) (string, error)
+}
+
+type stringEvaluator struct {
+	treeNavigator DataTreeNavigator
+	fileIndex     int
+}
+
+func NewStringEvaluator() StringEvaluator {
+	return &stringEvaluator{
+		treeNavigator: NewDataTreeNavigator(),
+	}
+}
+
+func (s *stringEvaluator) Evaluate(expression string, input string, encoder Encoder, leadingContentPreProcessing bool, decoder Decoder) (string, error) {
+
+	// Use bytes.Buffer for output of string
+	out := new(bytes.Buffer)
+	printer := NewPrinter(encoder, NewSinglePrinterWriter(out))
+
+	InitExpressionParser()
+	node, err := ExpressionParser.ParseExpression(expression)
+	if err != nil {
+		return "", err
+	}
+
+	reader, leadingContent, err := readString(input, leadingContentPreProcessing)
+	if err != nil {
+		return "", err
+	}
+
+	var currentIndex uint
+	decoder.Init(reader)
+	for {
+		var dataBucket yaml.Node
+		errorReading := decoder.Decode(&dataBucket)
+
+		if errors.Is(errorReading, io.EOF) {
+			s.fileIndex = s.fileIndex + 1
+			return out.String(), nil
+		} else if errorReading != nil {
+			return "", fmt.Errorf("bad input '%v': %w", input, errorReading)
+		}
+
+		candidateNode := &CandidateNode{
+			Document:  currentIndex,
+			Node:      &dataBucket,
+			FileIndex: s.fileIndex,
+		}
+		// move document comments into candidate node
+		// otherwise unwrap drops them.
+		candidateNode.TrailingContent = dataBucket.FootComment
+		dataBucket.FootComment = ""
+
+		if currentIndex == 0 {
+			candidateNode.LeadingContent = leadingContent
+		}
+		inputList := list.New()
+		inputList.PushBack(candidateNode)
+
+		result, errorParsing := s.treeNavigator.GetMatchingNodes(Context{MatchingNodes: inputList}, node)
+		if errorParsing != nil {
+			return "", errorParsing
+		}
+		err = printer.PrintResults(result.MatchingNodes)
+
+		if err != nil {
+			return "", err
+		}
+		currentIndex = currentIndex + 1
+	}
+}

pkg/yqlib/string_evaluator_test.go

@@ -0,0 +1,30 @@
+package yqlib
+
+import (
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+func TestStringEvaluator_Evaluate_Nominal(t *testing.T) {
+	expected_output := `` +
+		`yq` + "\n" +
+		`---` + "\n" +
+		`jq` + "\n"
+	expression := ".[].name"
+	input := `` +
+		` - name: yq` + "\n" +
+		`   description: yq is a portable command-line YAML, JSON and XML processor` + "\n" +
+		`---` + "\n" +
+		` - name: jq` + "\n" +
+		`   description: Command-line JSON processor` + "\n"
+	encoder := NewYamlEncoder(2, true, true, true)
+	decoder := NewYamlDecoder()
+
+	result, err := NewStringEvaluator().Evaluate(expression, input, encoder, true, decoder)
+	if err != nil {
+		t.Error(err)
+	}
+
+	test.AssertResult(t, expected_output, result)
+}

pkg/yqlib/utils.go

@@ -33,6 +33,14 @@ func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, s
 	return processReadStream(reader)
 }
 
+func readString(input string, leadingContentPreProcessing bool) (io.Reader, string, error) {
+	reader := bufio.NewReader(strings.NewReader(input))
+	if !leadingContentPreProcessing {
+		return reader, "", nil
+	}
+	return processReadStream(reader)
+}
+
 func writeString(writer io.Writer, txt string) error {
 	_, errorWriting := writer.Write([]byte(txt))
 	return errorWriting

release_notes.txt

@@ -1,5 +1,10 @@
+4.27.2:
+  - Fixed JSON decoder to maintain object key order.
+
 4.27.1:
   - Added 'json' decoder for support for multiple JSON documents in a single file (e.g. NDJSON)
+  - Added 'csv' decoding, array of objects encoding, and round-triping
+  - New StringEvaluator when using yq as a lib (thanks @leviliangtw)
   - Fixed XML decoding issue (#1284)
 
 4.26.1:

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.26.1'
+version: '4.27.2'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.