Using new lexer by alecthomas!

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.18.4 as builder
+FROM golang:1.18.3 as builder
 
 WORKDIR /go/src/mikefarah/yq
 

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM golang:1.18.4
+FROM golang:1.18.3
 
 COPY scripts/devtools.sh /opt/devtools.sh
 

acceptance_tests/inputs-format.sh

@@ -3,8 +3,6 @@
 setUp() {
   rm test*.yml 2>/dev/null || true
   rm test*.properties 2>/dev/null || true
-  rm test*.csv 2>/dev/null || true
-  rm test*.tsv 2>/dev/null || true
   rm test*.xml 2>/dev/null || true
 }
 
@@ -42,51 +40,6 @@ EOM
   assertEquals "$expected" "$X"
 }
 
-testInputCSV() {
-  cat >test.csv <<EOL
-fruit,yumLevel
-apple,5
-banana,4
-EOL
-
-  read -r -d '' expected << EOM
-- fruit: apple
-  yumLevel: 5
-- fruit: banana
-  yumLevel: 4
-EOM
-
-  X=$(./yq e -p=csv test.csv)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea -p=csv test.csv)
-  assertEquals "$expected" "$X"
-}
-
-testInputTSV() {
-  cat >test.tsv <<EOL
-fruit	yumLevel
-apple	5
-banana	4
-EOL
-
-  read -r -d '' expected << EOM
-- fruit: apple
-  yumLevel: 5
-- fruit: banana
-  yumLevel: 4
-EOM
-
-  X=$(./yq e -p=t test.tsv)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea -p=t test.tsv)
-  assertEquals "$expected" "$X"
-}
-
-
-
-
 testInputXml() {
   cat >test.yml <<EOL
 <cat legs="4">BiBi</cat>

acceptance_tests/output-format.sh

@@ -102,48 +102,6 @@ EOM
   assertEquals "$expected" "$X"
 }
 
-testOutputCSV() {
-  cat >test.yml <<EOL
-- fruit: apple
-  yumLevel: 5
-- fruit: banana
-  yumLevel: 4
-EOL
-
-  read -r -d '' expected << EOM
-fruit,yumLevel
-apple,5
-banana,4
-EOM
-
-  X=$(./yq -o=c test.yml)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea -o=csv test.yml)
-  assertEquals "$expected" "$X"
-}
-
-testOutputTSV() {
-  cat >test.yml <<EOL
-- fruit: apple
-  yumLevel: 5
-- fruit: banana
-  yumLevel: 4
-EOL
-
-  read -r -d '' expected << EOM
-fruit	yumLevel
-apple	5
-banana	4
-EOM
-
-  X=$(./yq -o=t test.yml)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea -o=tsv test.yml)
-  assertEquals "$expected" "$X"
-}
-
 testOutputXml() {
   cat >test.yml <<EOL
 a: {b: {c: ["cat"]}}

cmd/utils.go

@@ -66,12 +66,6 @@ func configureDecoder() (yqlib.Decoder, error) {
 		return yqlib.NewXMLDecoder(xmlAttributePrefix, xmlContentName, xmlStrictMode, xmlKeepNamespace, xmlUseRawToken), nil
 	case yqlib.PropertiesInputFormat:
 		return yqlib.NewPropertiesDecoder(), nil
-	case yqlib.JsonInputFormat:
-		return yqlib.NewJSONDecoder(), nil
-	case yqlib.CSVObjectInputFormat:
-		return yqlib.NewCSVObjectDecoder(','), nil
-	case yqlib.TSVObjectInputFormat:
-		return yqlib.NewCSVObjectDecoder('\t'), nil
 	}
 
 	return yqlib.NewYamlDecoder(), nil
@@ -97,7 +91,7 @@ func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yq
 func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder {
 	switch format {
 	case yqlib.JSONOutputFormat:
-		return yqlib.NewJSONEncoder(indent, colorsEnabled)
+		return yqlib.NewJONEncoder(indent, colorsEnabled)
 	case yqlib.PropsOutputFormat:
 		return yqlib.NewPropertiesEncoder(unwrapScalar)
 	case yqlib.CSVOutputFormat:

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.26.1"
+	Version = "4.25.3"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

examples/sample_objects.csv

@@ -1,3 +0,0 @@
-name,numberOfCats,likesApples,height
-Gary,1,true,168.8
-Samantha's Rabbit,2,false,-188.8

go.mod

@@ -6,7 +6,6 @@ require (
 	github.com/alecthomas/repr v0.1.0
 	github.com/elliotchance/orderedmap v1.4.0
 	github.com/fatih/color v1.13.0
-	github.com/goccy/go-json v0.9.10
 	github.com/goccy/go-yaml v1.9.5
 	github.com/jinzhu/copier v0.3.5
 	github.com/magiconair/properties v1.8.6

go.sum

@@ -17,8 +17,6 @@ github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvSc
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
-github.com/goccy/go-json v0.9.10 h1:hCeNmprSNLB8B8vQKWl6DpuH0t60oEs+TAk9a7CScKc=
-github.com/goccy/go-json v0.9.10/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/goccy/go-yaml v1.9.5 h1:Eh/+3uk9kLxG4koCX6lRMAPS1OaMSAi+FJcya0INdB0=
 github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=

pkg/yqlib/csv_test.go

@@ -1,273 +0,0 @@
-package yqlib
-
-import (
-	"bufio"
-	"fmt"
-	"testing"
-
-	"github.com/mikefarah/yq/v4/test"
-)
-
-const csvSimple = `name,numberOfCats,likesApples,height
-Gary,1,true,168.8
-Samantha's Rabbit,2,false,-188.8
-`
-
-const expectedUpdatedSimpleCsv = `name,numberOfCats,likesApples,height
-Gary,3,true,168.8
-Samantha's Rabbit,2,false,-188.8
-`
-
-const csvSimpleShort = `Name,Number of Cats
-Gary,1
-Samantha's Rabbit,2
-`
-
-const tsvSimple = `name	numberOfCats	likesApples	height
-Gary	1	true	168.8
-Samantha's Rabbit	2	false	-188.8
-`
-
-const expectedYamlFromCSV = `- name: Gary
-  numberOfCats: 1
-  likesApples: true
-  height: 168.8
-- name: Samantha's Rabbit
-  numberOfCats: 2
-  likesApples: false
-  height: -188.8
-`
-
-const expectedYamlFromCSVMissingData = `- name: Gary
-  numberOfCats: 1
-  height: 168.8
-- name: Samantha's Rabbit
-  height: -188.8
-  likesApples: false
-`
-
-const csvSimpleMissingData = `name,numberOfCats,height
-Gary,1,168.8
-Samantha's Rabbit,,-188.8
-`
-
-const csvTestSimpleYaml = `- [i, like, csv]
-- [because, excel, is, cool]`
-
-const expectedSimpleCsv = `i,like,csv
-because,excel,is,cool
-`
-
-const tsvTestExpectedSimpleCsv = `i	like	csv
-because	excel	is	cool
-`
-
-var csvScenarios = []formatScenario{
-	{
-		description:  "Encode CSV simple",
-		input:        csvTestSimpleYaml,
-		expected:     expectedSimpleCsv,
-		scenarioType: "encode-csv",
-	},
-	{
-		description:  "Encode TSV simple",
-		input:        csvTestSimpleYaml,
-		expected:     tsvTestExpectedSimpleCsv,
-		scenarioType: "encode-tsv",
-	},
-	{
-		description:  "Encode Empty",
-		skipDoc:      true,
-		input:        `[]`,
-		expected:     "",
-		scenarioType: "encode-csv",
-	},
-	{
-		description:  "Comma in value",
-		skipDoc:      true,
-		input:        `["comma, in, value", things]`,
-		expected:     "\"comma, in, value\",things\n",
-		scenarioType: "encode-csv",
-	},
-	{
-		description:  "Encode array of objects to csv",
-		input:        expectedYamlFromCSV,
-		expected:     csvSimple,
-		scenarioType: "encode-csv",
-	},
-	{
-		description:    "Encode array of objects to custom csv format",
-		subdescription: "Add the header row manually, then the we convert each object into an array of values - resulting in an array of arrays. Pick the columns and call the header whatever you like.",
-		input:          expectedYamlFromCSV,
-		expected:       csvSimpleShort,
-		expression:     `[["Name", "Number of Cats"]] +  [.[] | [.name, .numberOfCats ]]`,
-		scenarioType:   "encode-csv",
-	},
-	{
-		description:    "Encode array of objects to csv - missing fields behaviour",
-		subdescription: "First entry is used to determine the headers, and it is missing 'likesApples', so it is not included in the csv. Second entry does not have 'numberOfCats' so that is blank",
-		input:          expectedYamlFromCSVMissingData,
-		expected:       csvSimpleMissingData,
-		scenarioType:   "encode-csv",
-	},
-	{
-		description:    "Parse CSV into an array of objects",
-		subdescription: "First row is assumed to be the header row.",
-		input:          csvSimple,
-		expected:       expectedYamlFromCSV,
-		scenarioType:   "decode-csv-object",
-	},
-	{
-		description:    "Parse TSV into an array of objects",
-		subdescription: "First row is assumed to be the header row.",
-		input:          tsvSimple,
-		expected:       expectedYamlFromCSV,
-		scenarioType:   "decode-tsv-object",
-	},
-	{
-		description:  "Round trip",
-		input:        csvSimple,
-		expected:     expectedUpdatedSimpleCsv,
-		expression:   `(.[] | select(.name == "Gary") | .numberOfCats) = 3`,
-		scenarioType: "roundtrip-csv",
-	},
-}
-
-func testCSVScenario(t *testing.T, s formatScenario) {
-	switch s.scenarioType {
-	case "encode-csv":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder(',')), s.description)
-	case "encode-tsv":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder('\t')), s.description)
-	case "decode-csv-object":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewYamlEncoder(2, false, true, true)), s.description)
-	case "decode-tsv-object":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder('\t'), NewYamlEncoder(2, false, true, true)), s.description)
-	case "roundtrip-csv":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewCsvEncoder(',')), s.description)
-	default:
-		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
-	}
-}
-
-func documentCSVDecodeObjectScenario(t *testing.T, w *bufio.Writer, s formatScenario, formatType string) {
-	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
-
-	if s.subdescription != "" {
-		writeOrPanic(w, s.subdescription)
-		writeOrPanic(w, "\n\n")
-	}
-
-	writeOrPanic(w, fmt.Sprintf("Given a sample.%v file of:\n", formatType))
-	writeOrPanic(w, fmt.Sprintf("```%v\n%v\n```\n", formatType, s.input))
-
-	writeOrPanic(w, "then\n")
-	writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=%v sample.%v\n```\n", formatType, formatType))
-	writeOrPanic(w, "will output\n")
-
-	separator := ','
-	if formatType == "tsv" {
-		separator = '\t'
-	}
-
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n",
-		processFormatScenario(s, NewCSVObjectDecoder(separator), NewYamlEncoder(s.indent, false, true, true))),
-	)
-}
-
-func documentCSVEncodeScenario(w *bufio.Writer, s formatScenario, formatType string) {
-	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
-
-	if s.subdescription != "" {
-		writeOrPanic(w, s.subdescription)
-		writeOrPanic(w, "\n\n")
-	}
-
-	writeOrPanic(w, "Given a sample.yml file of:\n")
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input))
-
-	writeOrPanic(w, "then\n")
-
-	expression := s.expression
-
-	if expression != "" {
-		writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=%v '%v' sample.yml\n```\n", formatType, expression))
-	} else {
-		writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=%v sample.yml\n```\n", formatType))
-	}
-	writeOrPanic(w, "will output\n")
-
-	separator := ','
-	if formatType == "tsv" {
-		separator = '\t'
-	}
-
-	writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
-		processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder(separator))),
-	)
-}
-
-func documentCSVRoundTripScenario(w *bufio.Writer, s formatScenario, formatType string) {
-	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
-
-	if s.subdescription != "" {
-		writeOrPanic(w, s.subdescription)
-		writeOrPanic(w, "\n\n")
-	}
-
-	writeOrPanic(w, fmt.Sprintf("Given a sample.%v file of:\n", formatType))
-	writeOrPanic(w, fmt.Sprintf("```%v\n%v\n```\n", formatType, s.input))
-
-	writeOrPanic(w, "then\n")
-
-	expression := s.expression
-
-	if expression != "" {
-		writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=%v -o=%v '%v' sample.%v\n```\n", formatType, formatType, expression, formatType))
-	} else {
-		writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=%v -o=%v sample.%v\n```\n", formatType, formatType, formatType))
-	}
-	writeOrPanic(w, "will output\n")
-
-	separator := ','
-	if formatType == "tsv" {
-		separator = '\t'
-	}
-
-	writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
-		processFormatScenario(s, NewCSVObjectDecoder(separator), NewCsvEncoder(separator))),
-	)
-}
-
-func documentCSVScenario(t *testing.T, w *bufio.Writer, i interface{}) {
-	s := i.(formatScenario)
-	if s.skipDoc {
-		return
-	}
-	switch s.scenarioType {
-	case "encode-csv":
-		documentCSVEncodeScenario(w, s, "csv")
-	case "encode-tsv":
-		documentCSVEncodeScenario(w, s, "tsv")
-	case "decode-csv-object":
-		documentCSVDecodeObjectScenario(t, w, s, "csv")
-	case "decode-tsv-object":
-		documentCSVDecodeObjectScenario(t, w, s, "tsv")
-	case "roundtrip-csv":
-		documentCSVRoundTripScenario(w, s, "csv")
-
-	default:
-		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
-	}
-}
-
-func TestCSVScenarios(t *testing.T) {
-	for _, tt := range csvScenarios {
-		testCSVScenario(t, tt)
-	}
-	genericScenarios := make([]interface{}, len(csvScenarios))
-	for i, s := range csvScenarios {
-		genericScenarios[i] = s
-	}
-	documentScenarios(t, "usage", "csv-tsv", genericScenarios, documentCSVScenario)
-}

pkg/yqlib/decoder.go

@@ -14,9 +14,6 @@ const (
 	XMLInputFormat
 	PropertiesInputFormat
 	Base64InputFormat
-	JsonInputFormat
-	CSVObjectInputFormat
-	TSVObjectInputFormat
 )
 
 type Decoder interface {
@@ -32,12 +29,6 @@ func InputFormatFromString(format string) (InputFormat, error) {
 		return XMLInputFormat, nil
 	case "props", "p":
 		return PropertiesInputFormat, nil
-	case "json", "ndjson", "j":
-		return JsonInputFormat, nil
-	case "csv", "c":
-		return CSVObjectInputFormat, nil
-	case "tsv", "t":
-		return TSVObjectInputFormat, nil
 	default:
 		return 0, fmt.Errorf("unknown format '%v' please use [yaml|xml|props]", format)
 	}

pkg/yqlib/decoder_csv_object.go

@@ -1,77 +0,0 @@
-package yqlib
-
-import (
-	"encoding/csv"
-	"errors"
-	"io"
-
-	yaml "gopkg.in/yaml.v3"
-)
-
-type csvObjectDecoder struct {
-	separator rune
-	reader    csv.Reader
-	finished  bool
-}
-
-func NewCSVObjectDecoder(separator rune) Decoder {
-	return &csvObjectDecoder{separator: separator}
-}
-
-func (dec *csvObjectDecoder) Init(reader io.Reader) {
-	dec.reader = *csv.NewReader(reader)
-	dec.reader.Comma = dec.separator
-	dec.finished = false
-}
-
-func (dec *csvObjectDecoder) convertToYamlNode(content string) *yaml.Node {
-	node, err := parseSnippet(content)
-	if err != nil {
-		return createScalarNode(content, content)
-	}
-	return node
-}
-
-func (dec *csvObjectDecoder) createObject(headerRow []string, contentRow []string) *yaml.Node {
-	objectNode := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
-
-	for i, header := range headerRow {
-		objectNode.Content = append(
-			objectNode.Content,
-			createScalarNode(header, header),
-			dec.convertToYamlNode(contentRow[i]))
-	}
-	return objectNode
-}
-
-func (dec *csvObjectDecoder) Decode(rootYamlNode *yaml.Node) error {
-	if dec.finished {
-		return io.EOF
-	}
-	headerRow, err := dec.reader.Read()
-	log.Debugf(": headerRow%v", headerRow)
-	if err != nil {
-		return err
-	}
-
-	rootArray := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
-
-	contentRow, err := dec.reader.Read()
-
-	for err == nil && len(contentRow) > 0 {
-		log.Debugf("Adding contentRow: %v", contentRow)
-		rootArray.Content = append(rootArray.Content, dec.createObject(headerRow, contentRow))
-		contentRow, err = dec.reader.Read()
-		log.Debugf("Read next contentRow: %v, %v", contentRow, err)
-	}
-	if !errors.Is(err, io.EOF) {
-		return err
-	}
-
-	log.Debugf("finished, contentRow%v", contentRow)
-	log.Debugf("err: %v", err)
-
-	rootYamlNode.Kind = yaml.DocumentNode
-	rootYamlNode.Content = []*yaml.Node{rootArray}
-	return nil
-}

pkg/yqlib/decoder_json.go

@@ -1,85 +0,0 @@
-package yqlib
-
-import (
-	"fmt"
-	"io"
-
-	"github.com/goccy/go-json"
-	yaml "gopkg.in/yaml.v3"
-)
-
-type jsonDecoder struct {
-	decoder json.Decoder
-}
-
-func NewJSONDecoder() Decoder {
-	return &jsonDecoder{}
-}
-
-func (dec *jsonDecoder) Init(reader io.Reader) {
-	dec.decoder = *json.NewDecoder(reader)
-}
-
-func (dec *jsonDecoder) Decode(rootYamlNode *yaml.Node) error {
-
-	var dataBucket interface{}
-	log.Debug("going to decode")
-	err := dec.decoder.Decode(&dataBucket)
-	if err != nil {
-		return err
-	}
-	node, err := dec.convertToYamlNode(dataBucket)
-
-	if err != nil {
-		return err
-	}
-	rootYamlNode.Kind = yaml.DocumentNode
-	rootYamlNode.Content = []*yaml.Node{node}
-	return nil
-}
-
-func (dec *jsonDecoder) convertToYamlNode(data interface{}) (*yaml.Node, error) {
-	switch data := data.(type) {
-	case nil:
-		return createScalarNode(nil, "null"), nil
-	case float64, float32:
-		// json decoder returns ints as float.
-		return parseSnippet(fmt.Sprintf("%v", data))
-	case int, int64, int32, string, bool:
-		return createScalarNode(data, fmt.Sprintf("%v", data)), nil
-	case map[string]interface{}:
-		return dec.parseMap(data)
-	case []interface{}:
-		return dec.parseArray(data)
-	default:
-		return nil, fmt.Errorf("unrecognised type :(")
-	}
-}
-
-func (dec *jsonDecoder) parseMap(dataMap map[string]interface{}) (*yaml.Node, error) {
-
-	var yamlMap = &yaml.Node{Kind: yaml.MappingNode}
-
-	for key, value := range dataMap {
-		yamlValue, err := dec.convertToYamlNode(value)
-		if err != nil {
-			return nil, err
-		}
-		yamlMap.Content = append(yamlMap.Content, createScalarNode(key, fmt.Sprintf("%v", key)), yamlValue)
-	}
-	return yamlMap, nil
-}
-
-func (dec *jsonDecoder) parseArray(dataArray []interface{}) (*yaml.Node, error) {
-
-	var yamlMap = &yaml.Node{Kind: yaml.SequenceNode}
-
-	for _, value := range dataArray {
-		yamlValue, err := dec.convertToYamlNode(value)
-		if err != nil {
-			return nil, err
-		}
-		yamlMap.Content = append(yamlMap.Content, yamlValue)
-	}
-	return yamlMap, nil
-}

pkg/yqlib/decoder_xml.go

@@ -43,7 +43,7 @@ func (dec *xmlDecoder) Init(reader io.Reader) {
 }
 
 func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yaml.Node, error) {
-	yamlNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+	yamlNode := &yaml.Node{Kind: yaml.SequenceNode}
 	for _, child := range nodes {
 		yamlChild, err := dec.convertToYamlNode(child)
 		if err != nil {
@@ -64,7 +64,7 @@ func (dec *xmlDecoder) processComment(c string) string {
 
 func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
 	log.Debug("createMap: headC: %v, footC: %v", n.HeadComment, n.FootComment)
-	yamlNode := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
+	yamlNode := &yaml.Node{Kind: yaml.MappingNode}
 
 	if len(n.Data) > 0 {
 		label := dec.contentName

pkg/yqlib/doc/operators/encode-decode.md

@@ -11,14 +11,14 @@ These operators are useful to process yaml documents that have stringified embed
 | --- | -- | --|
 | Yaml | from_yaml | to_yaml(i)/@yaml |
 | JSON | from_json | to_json(i)/@json |
-| Properties | from_props/@propsd  | to_props/@props |
-| CSV | from_csv/@csvd | to_csv/@csv |
-| TSV | from_tsv/@tsvd | to_tsv/@tsv |
+| Properties | from_props  | to_props/@props |
+| CSV |  | to_csv/@csv |
+| TSV |  | to_tsv/@tsv |
 | XML | from_xml | to_xml(i)/@xml |
 | Base64 | @base64d | @base64 |
 
 
-See CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) for accepted formats.
+CSV and TSV format both accept either a single array or scalars (representing a single row), or an array of array of scalars (representing multiple rows). 
 
 XML uses the `--xml-attribute-prefix` and `xml-content-name` flags to identify attributes and content fields.
 
@@ -132,7 +132,7 @@ a: |-
 ```
 then
 ```bash
-yq '.a |= @propsd' sample.yml
+yq '.a |= from_props' sample.yml
 ```
 will output
 ```yaml
@@ -141,42 +141,6 @@ a:
   dogs: cool as well
 ```
 
-## Decode csv encoded string
-Given a sample.yml file of:
-```yaml
-a: |-
-  cats,dogs
-  great,cool as well
-```
-then
-```bash
-yq '.a |= @csvd' sample.yml
-```
-will output
-```yaml
-a:
-  - cats: great
-    dogs: cool as well
-```
-
-## Decode tsv encoded string
-Given a sample.yml file of:
-```yaml
-a: |-
-  cats	dogs
-  great	cool as well
-```
-then
-```bash
-yq '.a |= @tsvd' sample.yml
-```
-will output
-```yaml
-a:
-  - cats: great
-    dogs: cool as well
-```
-
 ## Encode value as yaml string
 Indent defaults to 2
 

pkg/yqlib/doc/operators/headers/encode-decode.md

@@ -11,14 +11,14 @@ These operators are useful to process yaml documents that have stringified embed
 | --- | -- | --|
 | Yaml | from_yaml | to_yaml(i)/@yaml |
 | JSON | from_json | to_json(i)/@json |
-| Properties | from_props/@propsd  | to_props/@props |
-| CSV | from_csv/@csvd | to_csv/@csv |
-| TSV | from_tsv/@tsvd | to_tsv/@tsv |
+| Properties | from_props  | to_props/@props |
+| CSV |  | to_csv/@csv |
+| TSV |  | to_tsv/@tsv |
 | XML | from_xml | to_xml(i)/@xml |
 | Base64 | @base64d | @base64 |
 
 
-See CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) for accepted formats.
+CSV and TSV format both accept either a single array or scalars (representing a single row), or an array of array of scalars (representing multiple rows). 
 
 XML uses the `--xml-attribute-prefix` and `xml-content-name` flags to identify attributes and content fields.
 

pkg/yqlib/doc/usage/convert.md

@@ -1,8 +1,6 @@
 # JSON
 
-Encode and decode to and from JSON. Note that, unless you have multiple JSON documents in a single file, YAML is a _superset_ of JSON - so `yq` can read any json file without doing anything special.
-
-If you do have mulitple JSON documents in a single file (e.g. NDJSON) then you will need to explicity use the json parser `-p=json`.
+Encode and decode to and from JSON. Note that YAML is a _superset_ of JSON - so `yq` can read any json file without doing anything special.
 
 This means you don't need to 'convert' a JSON file to YAML - however if you want idiomatic YAML styling, then you can use the `-P/--prettyPrint` flag, see examples below.
 
@@ -132,129 +130,3 @@ will output
 {"whatever":"cat"}
 ```
 
-## Roundtrip NDJSON
-Unfortunately the json encoder strips leading spaces of values.
-
-Given a sample.json file of:
-```json
-{"this": "is a multidoc json file"}
-{"each": ["line is a valid json document"]}
-{"a number": 4}
-
-```
-then
-```bash
-yq -p=json -o=json -I=0 sample.json
-```
-will output
-```yaml
-{"this":"is a multidoc json file"}
-{"each":["line is a valid json document"]}
-{"a number":4}
-```
-
-## Roundtrip multi-document JSON
-The NDJSON parser can also handle multiple multi-line json documents in a single file!
-
-Given a sample.json file of:
-```json
-{
-	"this": "is a multidoc json file"
-}
-{
-	"it": [
-		"has",
-		"consecutive",
-		"json documents"
-	]
-}
-{
-	"a number": 4
-}
-
-```
-then
-```bash
-yq -p=json -o=json -I=2 sample.json
-```
-will output
-```yaml
-{
-  "this": "is a multidoc json file"
-}
-{
-  "it": [
-    "has",
-    "consecutive",
-    "json documents"
-  ]
-}
-{
-  "a number": 4
-}
-```
-
-## Update a specific document in a multi-document json
-Documents are indexed by the `documentIndex` or `di` operator.
-
-Given a sample.json file of:
-```json
-{"this": "is a multidoc json file"}
-{"each": ["line is a valid json document"]}
-{"a number": 4}
-
-```
-then
-```bash
-yq -p=json -o=json -I=0 '(select(di == 1) | .each ) += "cool"' sample.json
-```
-will output
-```yaml
-{"this":"is a multidoc json file"}
-{"each":["line is a valid json document","cool"]}
-{"a number":4}
-```
-
-## Find and update a specific document in a multi-document json
-Use expressions as you normally would.
-
-Given a sample.json file of:
-```json
-{"this": "is a multidoc json file"}
-{"each": ["line is a valid json document"]}
-{"a number": 4}
-
-```
-then
-```bash
-yq -p=json -o=json -I=0 '(select(has("each")) | .each ) += "cool"' sample.json
-```
-will output
-```yaml
-{"this":"is a multidoc json file"}
-{"each":["line is a valid json document","cool"]}
-{"a number":4}
-```
-
-## Decode NDJSON
-Given a sample.json file of:
-```json
-{"this": "is a multidoc json file"}
-{"each": ["line is a valid json document"]}
-{"a number": 4}
-
-```
-then
-```bash
-yq -p=json sample.json
-```
-will output
-```yaml
-this: is a multidoc json file
----
-each:
-    - line is a valid json document
----
-a number: 4
-```
-

pkg/yqlib/doc/usage/csv-tsv.md

@@ -1,214 +0,0 @@
-# CSV
-Encode/Decode/Roundtrip CSV and TSV files.
-
-## Encode 
-Currently supports arrays of homogenous flat objects, that is: no nesting and it assumes the _first_ object has all the keys required:
-
-```yaml
-- name: Bobo
-  type: dog
-- name: Fifi
-  type: cat
-```
-
-As well as arrays of arrays of scalars (strings/numbers/booleans):
-
-```yaml
-- [Bobo, dog]
-- [Fifi, cat]
-```
-
-## Decode
-Decode assumes the first CSV/TSV row is the header row, and all rows beneath are the entries.
-The data will be coded into an array of objects, using the header rows as keys.
-
-```csv
-name,type
-Bobo,dog
-Fifi,cat
-```
-
-
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
-## Encode CSV simple
-Given a sample.yml file of:
-```yaml
-- [i, like, csv]
-- [because, excel, is, cool]
-```
-then
-```bash
-yq -o=csv sample.yml
-```
-will output
-```csv
-i,like,csv
-because,excel,is,cool
-```
-
-## Encode TSV simple
-Given a sample.yml file of:
-```yaml
-- [i, like, csv]
-- [because, excel, is, cool]
-```
-then
-```bash
-yq -o=tsv sample.yml
-```
-will output
-```tsv
-i	like	csv
-because	excel	is	cool
-```
-
-## Encode array of objects to csv
-Given a sample.yml file of:
-```yaml
-- name: Gary
-  numberOfCats: 1
-  likesApples: true
-  height: 168.8
-- name: Samantha's Rabbit
-  numberOfCats: 2
-  likesApples: false
-  height: -188.8
-
-```
-then
-```bash
-yq -o=csv sample.yml
-```
-will output
-```csv
-name,numberOfCats,likesApples,height
-Gary,1,true,168.8
-Samantha's Rabbit,2,false,-188.8
-```
-
-## Encode array of objects to custom csv format
-Add the header row manually, then the we convert each object into an array of values - resulting in an array of arrays. Pick the columns and call the header whatever you like.
-
-Given a sample.yml file of:
-```yaml
-- name: Gary
-  numberOfCats: 1
-  likesApples: true
-  height: 168.8
-- name: Samantha's Rabbit
-  numberOfCats: 2
-  likesApples: false
-  height: -188.8
-
-```
-then
-```bash
-yq -o=csv '[["Name", "Number of Cats"]] +  [.[] | [.name, .numberOfCats ]]' sample.yml
-```
-will output
-```csv
-Name,Number of Cats
-Gary,1
-Samantha's Rabbit,2
-```
-
-## Encode array of objects to csv - missing fields behaviour
-First entry is used to determine the headers, and it is missing 'likesApples', so it is not included in the csv. Second entry does not have 'numberOfCats' so that is blank
-
-Given a sample.yml file of:
-```yaml
-- name: Gary
-  numberOfCats: 1
-  height: 168.8
-- name: Samantha's Rabbit
-  height: -188.8
-  likesApples: false
-
-```
-then
-```bash
-yq -o=csv sample.yml
-```
-will output
-```csv
-name,numberOfCats,height
-Gary,1,168.8
-Samantha's Rabbit,,-188.8
-```
-
-## Parse CSV into an array of objects
-First row is assumed to be the header row.
-
-Given a sample.csv file of:
-```csv
-name,numberOfCats,likesApples,height
-Gary,1,true,168.8
-Samantha's Rabbit,2,false,-188.8
-
-```
-then
-```bash
-yq -p=csv sample.csv
-```
-will output
-```yaml
-- name: Gary
-  numberOfCats: 1
-  likesApples: true
-  height: 168.8
-- name: Samantha's Rabbit
-  numberOfCats: 2
-  likesApples: false
-  height: -188.8
-```
-
-## Parse TSV into an array of objects
-First row is assumed to be the header row.
-
-Given a sample.tsv file of:
-```tsv
-name	numberOfCats	likesApples	height
-Gary	1	true	168.8
-Samantha's Rabbit	2	false	-188.8
-
-```
-then
-```bash
-yq -p=tsv sample.tsv
-```
-will output
-```yaml
-- name: Gary
-  numberOfCats: 1
-  likesApples: true
-  height: 168.8
-- name: Samantha's Rabbit
-  numberOfCats: 2
-  likesApples: false
-  height: -188.8
-```
-
-## Round trip
-Given a sample.csv file of:
-```csv
-name,numberOfCats,likesApples,height
-Gary,1,true,168.8
-Samantha's Rabbit,2,false,-188.8
-
-```
-then
-```bash
-yq -p=csv -o=csv '(.[] | select(.name == "Gary") | .numberOfCats) = 3' sample.csv
-```
-will output
-```csv
-name,numberOfCats,likesApples,height
-Gary,3,true,168.8
-Samantha's Rabbit,2,false,-188.8
-```
-

pkg/yqlib/doc/usage/headers/convert.md

@@ -1,7 +1,5 @@
 # JSON
 
-Encode and decode to and from JSON. Note that, unless you have multiple JSON documents in a single file, YAML is a _superset_ of JSON - so `yq` can read any json file without doing anything special.
-
-If you do have mulitple JSON documents in a single file (e.g. NDJSON) then you will need to explicity use the json parser `-p=json`.
+Encode and decode to and from JSON. Note that YAML is a _superset_ of JSON - so `yq` can read any json file without doing anything special.
 
 This means you don't need to 'convert' a JSON file to YAML - however if you want idiomatic YAML styling, then you can use the `-P/--prettyPrint` flag, see examples below.

pkg/yqlib/doc/usage/headers/csv-tsv.md

@@ -1,30 +0,0 @@
-# CSV
-Encode/Decode/Roundtrip CSV and TSV files.
-
-## Encode 
-Currently supports arrays of homogenous flat objects, that is: no nesting and it assumes the _first_ object has all the keys required:
-
-```yaml
-- name: Bobo
-  type: dog
-- name: Fifi
-  type: cat
-```
-
-As well as arrays of arrays of scalars (strings/numbers/booleans):
-
-```yaml
-- [Bobo, dog]
-- [Fifi, cat]
-```
-
-## Decode
-Decode assumes the first CSV/TSV row is the header row, and all rows beneath are the entries.
-The data will be coded into an array of objects, using the header rows as keys.
-
-```csv
-name,type
-Bobo,dog
-Fifi,cat
-```
-

pkg/yqlib/doc/usage/headers/properties.md

@@ -1,5 +1,5 @@
 # Properties
 
-Encode/Decode/Roundtrip to/from a property file. Line comments on value nodes will be copied across.
+Encode to a property file (decode not yet supported). Line comments on value nodes will be copied across.
 
 By default, empty maps and arrays are not encoded - see below for an example on how to encode a value for these.

pkg/yqlib/doc/usage/properties.md

@@ -1,6 +1,6 @@
 # Properties
 
-Encode/Decode/Roundtrip to/from a property file. Line comments on value nodes will be copied across.
+Encode to a property file (decode not yet supported). Line comments on value nodes will be copied across.
 
 By default, empty maps and arrays are not encoded - see below for an example on how to encode a value for these.
 

pkg/yqlib/encoder_csv.go

@@ -13,7 +13,7 @@ type csvEncoder struct {
 }
 
 func NewCsvEncoder(separator rune) Encoder {
-	return &csvEncoder{separator: separator}
+	return &csvEncoder{separator}
 }
 
 func (e *csvEncoder) CanHandleAliases() bool {
@@ -41,67 +41,6 @@ func (e *csvEncoder) encodeRow(csvWriter *csv.Writer, contents []*yaml.Node) err
 	return csvWriter.Write(stringValues)
 }
 
-func (e *csvEncoder) encodeArrays(csvWriter *csv.Writer, content []*yaml.Node) error {
-	for i, child := range content {
-
-		if child.Kind != yaml.SequenceNode {
-			return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag)
-		}
-		err := e.encodeRow(csvWriter, child.Content)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (e *csvEncoder) extractHeader(child *yaml.Node) ([]*yaml.Node, error) {
-	if child.Kind != yaml.MappingNode {
-		return nil, fmt.Errorf("csv object encoding only works for arrays of flat objects (string key => string/numbers/boolean value), child[0] is a %v", child.Tag)
-	}
-	mapKeys := getMapKeys(child)
-	return mapKeys.Content, nil
-}
-
-func (e *csvEncoder) createChildRow(child *yaml.Node, headers []*yaml.Node) []*yaml.Node {
-	childRow := make([]*yaml.Node, 0)
-	for _, header := range headers {
-		keyIndex := findKeyInMap(child, header)
-		value := createScalarNode(nil, "")
-		if keyIndex != -1 {
-			value = child.Content[keyIndex+1]
-		}
-		childRow = append(childRow, value)
-	}
-	return childRow
-
-}
-
-func (e *csvEncoder) encodeObjects(csvWriter *csv.Writer, content []*yaml.Node) error {
-	headers, err := e.extractHeader(content[0])
-	if err != nil {
-		return nil
-	}
-
-	err = e.encodeRow(csvWriter, headers)
-	if err != nil {
-		return nil
-	}
-
-	for i, child := range content {
-		if child.Kind != yaml.MappingNode {
-			return fmt.Errorf("csv object encoding only works for arrays of flat objects (string key => string/numbers/boolean value), child[%v] is a %v", i, child.Tag)
-		}
-		row := e.createChildRow(child, headers)
-		err = e.encodeRow(csvWriter, row)
-		if err != nil {
-			return err
-		}
-
-	}
-	return nil
-}
-
 func (e *csvEncoder) Encode(writer io.Writer, originalNode *yaml.Node) error {
 	csvWriter := csv.NewWriter(writer)
 	csvWriter.Comma = e.separator
@@ -117,10 +56,15 @@ func (e *csvEncoder) Encode(writer io.Writer, originalNode *yaml.Node) error {
 		return e.encodeRow(csvWriter, node.Content)
 	}
 
-	if node.Content[0].Kind == yaml.MappingNode {
-		return e.encodeObjects(csvWriter, node.Content)
+	for i, child := range node.Content {
+
+		if child.Kind != yaml.SequenceNode {
+			return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag)
+		}
+		err := e.encodeRow(csvWriter, child.Content)
+		if err != nil {
+			return err
+		}
 	}
-
-	return e.encodeArrays(csvWriter, node.Content)
-
+	return nil
 }

pkg/yqlib/encoder_csv_test.go

@@ -0,0 +1,60 @@
+package yqlib
+
+import (
+	"bufio"
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+func yamlToCsv(sampleYaml string, separator rune) string {
+	var output bytes.Buffer
+	writer := bufio.NewWriter(&output)
+
+	var jsonEncoder = NewCsvEncoder(separator)
+	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
+	if err != nil {
+		panic(err)
+	}
+	node := inputs.Front().Value.(*CandidateNode).Node
+	err = jsonEncoder.Encode(writer, node)
+	if err != nil {
+		panic(err)
+	}
+	writer.Flush()
+
+	return strings.TrimSuffix(output.String(), "\n")
+}
+
+var sampleYaml = `["apple", apple2, "comma, in, value", "new
+line", 3, 3.40, true, "tab	here"]`
+
+var sampleYamlArray = "[" + sampleYaml + ", [bob, cat, meow, puss]]"
+
+func TestCsvEncoderEmptyArray(t *testing.T) {
+	var actualCsv = yamlToCsv(`[]`, ',')
+	test.AssertResult(t, "", actualCsv)
+}
+
+func TestCsvEncoder(t *testing.T) {
+	var expectedCsv = `apple,apple2,"comma, in, value",new line,3,3.40,true,tab	here`
+
+	var actualCsv = yamlToCsv(sampleYaml, ',')
+	test.AssertResult(t, expectedCsv, actualCsv)
+}
+
+func TestCsvEncoderArrayOfArrays(t *testing.T) {
+	var actualCsv = yamlToCsv(sampleYamlArray, ',')
+	var expectedCsv = "apple,apple2,\"comma, in, value\",new line,3,3.40,true,tab	here\nbob,cat,meow,puss"
+	test.AssertResult(t, expectedCsv, actualCsv)
+}
+
+func TestTsvEncoder(t *testing.T) {
+
+	var expectedCsv = `apple	apple2	comma, in, value	new line	3	3.40	true	"tab	here"`
+
+	var actualCsv = yamlToCsv(sampleYaml, '\t')
+	test.AssertResult(t, expectedCsv, actualCsv)
+}

pkg/yqlib/encoder_json.go

@@ -2,9 +2,9 @@ package yqlib
 
 import (
 	"bytes"
+	"encoding/json"
 	"io"
 
-	"github.com/goccy/go-json"
 	yaml "gopkg.in/yaml.v3"
 )
 
@@ -28,7 +28,7 @@ func mapKeysToStrings(node *yaml.Node) {
 	}
 }
 
-func NewJSONEncoder(indent int, colorise bool) Encoder {
+func NewJONEncoder(indent int, colorise bool) Encoder {
 	var indentString = ""
 
 	for index := 0; index < indent; index++ {

pkg/yqlib/encoder_test.go

@@ -13,7 +13,7 @@ func yamlToJSON(sampleYaml string, indent int) string {
 	var output bytes.Buffer
 	writer := bufio.NewWriter(&output)
 
-	var jsonEncoder = NewJSONEncoder(indent, false)
+	var jsonEncoder = NewJONEncoder(indent, false)
 	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
 		panic(err)

pkg/yqlib/json_test.go

@@ -17,59 +17,6 @@ b:
         - 4
 `
 
-const sampleNdJson = `{"this": "is a multidoc json file"}
-{"each": ["line is a valid json document"]}
-{"a number": 4}
-`
-
-const expectedNdJsonYaml = `this: is a multidoc json file
----
-each:
-  - line is a valid json document
----
-a number: 4
-`
-
-const expectedRoundTripSampleNdJson = `{"this":"is a multidoc json file"}
-{"each":["line is a valid json document"]}
-{"a number":4}
-`
-
-const expectedUpdatedMultilineJson = `{"this":"is a multidoc json file"}
-{"each":["line is a valid json document","cool"]}
-{"a number":4}
-`
-
-const sampleMultiLineJson = `{
-	"this": "is a multidoc json file"
-}
-{
-	"it": [
-		"has",
-		"consecutive",
-		"json documents"
-	]
-}
-{
-	"a number": 4
-}
-`
-
-const roundTripMultiLineJson = `{
-  "this": "is a multidoc json file"
-}
-{
-  "it": [
-    "has",
-    "consecutive",
-    "json documents"
-  ]
-}
-{
-  "a number": 4
-}
-`
-
 var jsonScenarios = []formatScenario{
 	{
 		description:    "Parse json: simple",
@@ -121,120 +68,6 @@ var jsonScenarios = []formatScenario{
 		expected:       "{\"stuff\":\"cool\"}\n{\"whatever\":\"cat\"}\n",
 		scenarioType:   "encode",
 	},
-	{
-		description:    "Roundtrip NDJSON",
-		subdescription: "Unfortunately the json encoder strips leading spaces of values.",
-		input:          sampleNdJson,
-		expected:       expectedRoundTripSampleNdJson,
-		scenarioType:   "roundtrip-ndjson",
-	},
-	{
-		description:    "Roundtrip multi-document JSON",
-		subdescription: "The NDJSON parser can also handle multiple multi-line json documents in a single file!",
-		input:          sampleMultiLineJson,
-		expected:       roundTripMultiLineJson,
-		scenarioType:   "roundtrip-multi",
-	},
-	{
-		description:    "Update a specific document in a multi-document json",
-		subdescription: "Documents are indexed by the `documentIndex` or `di` operator.",
-		input:          sampleNdJson,
-		expected:       expectedUpdatedMultilineJson,
-		expression:     `(select(di == 1) | .each ) += "cool"`,
-		scenarioType:   "roundtrip-ndjson",
-	},
-	{
-		description:    "Find and update a specific document in a multi-document json",
-		subdescription: "Use expressions as you normally would.",
-		input:          sampleNdJson,
-		expected:       expectedUpdatedMultilineJson,
-		expression:     `(select(has("each")) | .each ) += "cool"`,
-		scenarioType:   "roundtrip-ndjson",
-	},
-	{
-		description:  "Decode NDJSON",
-		input:        sampleNdJson,
-		expected:     expectedNdJsonYaml,
-		scenarioType: "decode-ndjson",
-	},
-	{
-		description:  "numbers",
-		skipDoc:      true,
-		input:        "[3, 3.0, 3.1, -1]",
-		expected:     "- 3\n- 3\n- 3.1\n- -1\n",
-		scenarioType: "decode-ndjson",
-	},
-	{
-		description:  "strings",
-		skipDoc:      true,
-		input:        `["", "cat"]`,
-		expected:     "- \"\"\n- cat\n",
-		scenarioType: "decode-ndjson",
-	},
-	{
-		description:  "null",
-		skipDoc:      true,
-		input:        `null`,
-		expected:     "null\n",
-		scenarioType: "decode-ndjson",
-	},
-	{
-		description:  "booleans",
-		skipDoc:      true,
-		input:        `[true, false]`,
-		expected:     "- true\n- false\n",
-		scenarioType: "decode-ndjson",
-	},
-}
-
-func documentRoundtripNdJsonScenario(w *bufio.Writer, s formatScenario, indent int) {
-	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
-
-	if s.subdescription != "" {
-		writeOrPanic(w, s.subdescription)
-		writeOrPanic(w, "\n\n")
-	}
-
-	writeOrPanic(w, "Given a sample.json file of:\n")
-	writeOrPanic(w, fmt.Sprintf("```json\n%v\n```\n", s.input))
-
-	writeOrPanic(w, "then\n")
-
-	expression := s.expression
-	if expression != "" {
-		writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=json -o=json -I=%v '%v' sample.json\n```\n", indent, expression))
-	} else {
-		writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=json -o=json -I=%v sample.json\n```\n", indent))
-	}
-
-	writeOrPanic(w, "will output\n")
-
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(indent, false))))
-}
-
-func documentDecodeNdJsonScenario(w *bufio.Writer, s formatScenario) {
-	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
-
-	if s.subdescription != "" {
-		writeOrPanic(w, s.subdescription)
-		writeOrPanic(w, "\n\n")
-	}
-
-	writeOrPanic(w, "Given a sample.json file of:\n")
-	writeOrPanic(w, fmt.Sprintf("```json\n%v\n```\n", s.input))
-
-	writeOrPanic(w, "then\n")
-
-	expression := s.expression
-	if expression != "" {
-		writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=json '%v' sample.json\n```\n", expression))
-	} else {
-		writeOrPanic(w, "```bash\nyq -p=json sample.json\n```\n")
-	}
-
-	writeOrPanic(w, "will output\n")
-
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(s.indent, false, true, true))))
 }
 
 func decodeJSON(t *testing.T, jsonString string) *CandidateNode {
@@ -265,16 +98,10 @@ func decodeJSON(t *testing.T, jsonString string) *CandidateNode {
 func testJSONScenario(t *testing.T, s formatScenario) {
 	switch s.scenarioType {
 	case "encode", "decode":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewJSONEncoder(s.indent, false)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewJONEncoder(s.indent, false)), s.description)
 	case "":
 		var actual = resultToString(t, decodeJSON(t, s.input))
 		test.AssertResultWithContext(t, s.expected, actual, s.description)
-	case "decode-ndjson":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(2, false, true, true)), s.description)
-	case "roundtrip-ndjson":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(0, false)), s.description)
-	case "roundtrip-multi":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(2, false)), s.description)
 
 	default:
 		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
@@ -320,12 +147,6 @@ func documentJSONScenario(t *testing.T, w *bufio.Writer, i interface{}) {
 		documentJSONDecodeScenario(t, w, s)
 	case "encode":
 		documentJSONEncodeScenario(w, s)
-	case "decode-ndjson":
-		documentDecodeNdJsonScenario(w, s)
-	case "roundtrip-ndjson":
-		documentRoundtripNdJsonScenario(w, s, 0)
-	case "roundtrip-multi":
-		documentRoundtripNdJsonScenario(w, s, 2)
 
 	default:
 		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
@@ -357,7 +178,7 @@ func documentJSONEncodeScenario(w *bufio.Writer, s formatScenario) {
 	}
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewJSONEncoder(s.indent, false))))
+	writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewJONEncoder(s.indent, false))))
 }
 
 func TestJSONScenarios(t *testing.T) {

pkg/yqlib/lexer_participle.go

@@ -67,10 +67,7 @@ var participleYqRules = []*participleYqRule{
 	{"XMLEncode", `to_?xml`, encodeWithIndent(XMLOutputFormat, 2), 0},
 	{"XMLEncodeNoIndent", `@xml`, encodeWithIndent(XMLOutputFormat, 0), 0},
 
-	{"CSVDecode", `from_?csv|@csvd`, decodeOp(CSVObjectInputFormat), 0},
 	{"CSVEncode", `to_?csv|@csv`, encodeWithIndent(CSVOutputFormat, 0), 0},
-
-	{"TSVDecode", `from_?tsv|@tsvd`, decodeOp(TSVObjectInputFormat), 0},
 	{"TSVEncode", `to_?tsv|@tsv`, encodeWithIndent(TSVOutputFormat, 0), 0},
 
 	{"Base64d", `@base64d`, decodeOp(Base64InputFormat), 0},

pkg/yqlib/lexer_participle_test.go

@@ -523,6 +523,7 @@ var participleLexerScenarios = []participleLexerScenario{
 }
 
 func TestParticipleLexer(t *testing.T) {
+	log.Errorf("TestParticiple")
 	lexer := newParticipleLexer()
 
 	for _, scenario := range participleLexerScenarios {

pkg/yqlib/lib.go

@@ -205,10 +205,10 @@ func findInArray(array *yaml.Node, item *yaml.Node) int {
 	return -1
 }
 
-func findKeyInMap(dataMap *yaml.Node, item *yaml.Node) int {
+func findKeyInMap(array *yaml.Node, item *yaml.Node) int {
 
-	for index := 0; index < len(dataMap.Content); index = index + 2 {
-		if recursiveNodeEqual(dataMap.Content[index], item) {
+	for index := 0; index < len(array.Content); index = index + 2 {
+		if recursiveNodeEqual(array.Content[index], item) {
 			return index
 		}
 	}
@@ -240,28 +240,20 @@ func guessTagFromCustomType(node *yaml.Node) string {
 		log.Warning("node has no value to guess the type with")
 		return node.Tag
 	}
-	dataBucket, errorReading := parseSnippet(node.Value)
 
+	decoder := NewYamlDecoder()
+	decoder.Init(strings.NewReader(node.Value))
+	var dataBucket yaml.Node
+	errorReading := decoder.Decode(&dataBucket)
 	if errorReading != nil {
 		log.Warning("could not guess underlying tag type %v", errorReading)
 		return node.Tag
 	}
-	guessedTag := unwrapDoc(dataBucket).Tag
+	guessedTag := unwrapDoc(&dataBucket).Tag
 	log.Info("im guessing the tag %v is a %v", node.Tag, guessedTag)
 	return guessedTag
 }
 
-func parseSnippet(value string) (*yaml.Node, error) {
-	decoder := NewYamlDecoder()
-	decoder.Init(strings.NewReader(value))
-	var dataBucket yaml.Node
-	err := decoder.Decode(&dataBucket)
-	if len(dataBucket.Content) == 0 {
-		return nil, fmt.Errorf("bad data")
-	}
-	return dataBucket.Content[0], err
-}
-
 func recursiveNodeEqual(lhs *yaml.Node, rhs *yaml.Node) bool {
 	if lhs.Kind != rhs.Kind {
 		return false

pkg/yqlib/operator_encoder_decoder.go

@@ -13,7 +13,7 @@ import (
 func configureEncoder(format PrinterOutputFormat, indent int) Encoder {
 	switch format {
 	case JSONOutputFormat:
-		return NewJSONEncoder(indent, false)
+		return NewJONEncoder(indent, false)
 	case PropsOutputFormat:
 		return NewPropertiesEncoder(true)
 	case CSVOutputFormat:
@@ -114,10 +114,6 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 		decoder = NewBase64Decoder()
 	case PropertiesInputFormat:
 		decoder = NewPropertiesDecoder()
-	case CSVObjectInputFormat:
-		decoder = NewCSVObjectDecoder(',')
-	case TSVObjectInputFormat:
-		decoder = NewCSVObjectDecoder('\t')
 	}
 
 	var results = list.New()

pkg/yqlib/operator_encoder_decoder_test.go

@@ -66,27 +66,11 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 	{
 		description: "Decode props encoded string",
 		document:    `a: "cats=great\ndogs=cool as well"`,
-		expression:  `.a |= @propsd`,
+		expression:  `.a |= from_props`,
 		expected: []string{
 			"D0, P[], (doc)::a:\n    cats: great\n    dogs: cool as well\n",
 		},
 	},
-	{
-		description: "Decode csv encoded string",
-		document:    `a: "cats,dogs\ngreat,cool as well"`,
-		expression:  `.a |= @csvd`,
-		expected: []string{
-			"D0, P[], (doc)::a:\n    - cats: great\n      dogs: cool as well\n",
-		},
-	},
-	{
-		description: "Decode tsv encoded string",
-		document: `a: "cats	dogs\ngreat	cool as well"`,
-		expression: `.a |= @tsvd`,
-		expected: []string{
-			"D0, P[], (doc)::a:\n    - cats: great\n      dogs: cool as well\n",
-		},
-	},
 	{
 		skipDoc:    true,
 		document:   "a:\n  cool:\n    bob: dylan",

pkg/yqlib/operator_pick.go

@@ -67,9 +67,9 @@ func pickOperator(d *dataTreeNavigator, context Context, expressionNode *Express
 		node := unwrapDoc(candidate.Node)
 
 		var replacement *yaml.Node
-		if node.Kind == yaml.MappingNode {
+		if node.Tag == "!!map" {
 			replacement = pickMap(node, indicesToPick)
-		} else if node.Kind == yaml.SequenceNode {
+		} else if node.Tag == "!!seq" {
 			replacement, err = pickSequence(node, indicesToPick)
 			if err != nil {
 				return Context{}, err

pkg/yqlib/operator_pick_test.go

@@ -14,15 +14,6 @@ var pickOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::myMap: {hamster: squeek, cat: meow}\n",
 		},
 	},
-	{
-		description: "Pick keys from map",
-		skipDoc:     true,
-		document:    "!things myMap: {cat: meow, dog: bark, thing: hamster, hamster: squeek}\n",
-		expression:  `.myMap |= pick(["hamster", "cat", "goat"])`,
-		expected: []string{
-			"D0, P[], (doc)::!things myMap: {hamster: squeek, cat: meow}\n",
-		},
-	},
 	{
 		description: "Pick keys from map with comments",
 		skipDoc:     true,

pkg/yqlib/printer_test.go

@@ -314,7 +314,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	// note printDocSeparators is true, it should still not print document separators
 	// when outputing JSON.
-	printer := NewPrinter(NewJSONEncoder(0, false), NewSinglePrinterWriter(writer))
+	printer := NewPrinter(NewJONEncoder(0, false), NewSinglePrinterWriter(writer))
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {

pkg/yqlib/xml_test.go

@@ -292,22 +292,6 @@ var xmlScenarios = []formatScenario{
 		expected:     "<cat>purrs</cat>\n",
 		scenarioType: "encode",
 	},
-	{
-		description:  "includes map tags",
-		skipDoc:      true,
-		input:        "<cat>purrs</cat>\n",
-		expression:   `tag`,
-		expected:     "!!map\n",
-		scenarioType: "decode",
-	},
-	{
-		description:  "includes array tags",
-		skipDoc:      true,
-		input:        "<cat>purrs</cat><cat>purrs</cat>\n",
-		expression:   `.cat | tag`,
-		expected:     "!!seq\n",
-		scenarioType: "decode",
-	},
 	{
 		description:  "Encode xml: array",
 		input:        "pets:\n  cat:\n    - purrs\n    - meows",

release_notes.txt

@@ -1,15 +1,3 @@
-4.27.1:
-  - Added 'json' decoder for support for multiple JSON documents in a single file (e.g. NDJSON)
-  - Fixed XML decoding issue (#1284)
-
-4.26.1:
-  - Switched to new expression parser (#1264)
-  - Don't clobber anchor when adding nodes (#1269)
-  - New error operator for custom validation (#1259)
-  - Added support for --wrapScalar=false in properties encoder (#1241) Thanks @dcarbone
-  - Fix error on multiple assign (#1257) Thanks @care0717
-  - Bumped dependency versions
-
 4.25.4:
   - Fixed panic when using multiply assign on multiple documents #1256 Thanks @care0717
 

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.26.1'
+version: '4.25.3'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.