Added csv separator flag #1950

acceptance_tests/inputs-format.sh

@@ -63,6 +63,27 @@ EOM
   assertEquals "$expected" "$X"
 }
 
+testInputCSVCustomSeparator() {
+  cat >test.csv <<EOL
+fruit;yumLevel
+apple;5
+banana;4
+EOL
+
+  read -r -d '' expected << EOM
+- fruit: apple
+  yumLevel: 5
+- fruit: banana
+  yumLevel: 4
+EOM
+
+  X=$(./yq -p=csv --csv-separator ";" test.csv)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -p=csv --csv-separator ";" test.csv)
+  assertEquals "$expected" "$X"
+}
+
 testInputCSVNoAuto() {
   cat >test.csv <<EOL
 thing1

acceptance_tests/output-format.sh

@@ -198,6 +198,27 @@ EOM
   assertEquals "$expected" "$X"
 }
 
+testOutputCSVCustomSeparator() {
+  cat >test.yml <<EOL
+- fruit: apple
+  yumLevel: 5
+- fruit: banana
+  yumLevel: 4
+EOL
+
+  read -r -d '' expected << EOM
+fruit;yumLevel
+apple;5
+banana;4
+EOM
+
+  X=$(./yq -oc --csv-separator ";" test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -o=csv --csv-separator ";" test.yml)
+  assertEquals "$expected" "$X"
+}
+
 testOutputTSV() {
   cat >test.yml <<EOL
 - fruit: apple

cmd/root.go

@@ -1,13 +1,42 @@
 package cmd
 
 import (
+	"fmt"
 	"os"
+	"strings"
 
 	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"github.com/spf13/cobra"
 	logging "gopkg.in/op/go-logging.v1"
 )
 
+type runeValue rune
+
+func newRuneVar(p *rune) *runeValue {
+	return (*runeValue)(p)
+}
+
+func (r *runeValue) String() string {
+	return string(*r)
+}
+
+func (r *runeValue) Set(rawVal string) error {
+	val := strings.ReplaceAll(rawVal, "\\n", "\n")
+	val = strings.ReplaceAll(val, "\\t", "\t")
+	val = strings.ReplaceAll(val, "\\r", "\r")
+	val = strings.ReplaceAll(val, "\\f", "\f")
+	val = strings.ReplaceAll(val, "\\v", "\v")
+	if len(val) != 1 {
+		return fmt.Errorf("[%v] is not a valid character. Must be length 1 was %v", val, len(val))
+	}
+	*r = runeValue(rune(val[0]))
+	return nil
+}
+
+func (r *runeValue) Type() string {
+	return "char"
+}
+
 func New() *cobra.Command {
 	var rootCmd = &cobra.Command{
 		Use:   "yq",
@@ -82,6 +111,8 @@ yq -P -oy sample.json
 	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipDirectives, "xml-skip-directives", yqlib.ConfiguredXMLPreferences.SkipDirectives, "skip over directives (e.g. <!DOCTYPE thing cat>)")
 
 	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredCsvPreferences.AutoParse, "csv-auto-parse", yqlib.ConfiguredCsvPreferences.AutoParse, "parse CSV YAML/JSON values")
+	rootCmd.PersistentFlags().Var(newRuneVar(&yqlib.ConfiguredCsvPreferences.Separator), "csv-separator", "CSV Separator character")
+
 	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredTsvPreferences.AutoParse, "tsv-auto-parse", yqlib.ConfiguredTsvPreferences.AutoParse, "parse TSV YAML/JSON values")
 
 	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredLuaPreferences.DocPrefix, "lua-prefix", yqlib.ConfiguredLuaPreferences.DocPrefix, "prefix")

cmd/utils.go

@@ -189,9 +189,9 @@ func createEncoder(format yqlib.PrinterOutputFormat) (yqlib.Encoder, error) {
 	case yqlib.PropsOutputFormat:
 		return yqlib.NewPropertiesEncoder(unwrapScalar), nil
 	case yqlib.CSVOutputFormat:
-		return yqlib.NewCsvEncoder(','), nil
+		return yqlib.NewCsvEncoder(yqlib.ConfiguredCsvPreferences), nil
 	case yqlib.TSVOutputFormat:
-		return yqlib.NewCsvEncoder('\t'), nil
+		return yqlib.NewCsvEncoder(yqlib.ConfiguredTsvPreferences), nil
 	case yqlib.YamlOutputFormat:
 		return yqlib.NewYamlEncoder(indent, colorsEnabled, yqlib.ConfiguredYamlPreferences), nil
 	case yqlib.XMLOutputFormat:

pkg/yqlib/csv_test.go

@@ -206,9 +206,9 @@ var csvScenarios = []formatScenario{
 func testCSVScenario(t *testing.T, s formatScenario) {
 	switch s.scenarioType {
 	case "encode-csv":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(',')), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(ConfiguredCsvPreferences)), s.description)
 	case "encode-tsv":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder('\t')), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(ConfiguredTsvPreferences)), s.description)
 	case "decode-csv":
 		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredCsvPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
 	case "decode-csv-no-auto":
@@ -216,7 +216,7 @@ func testCSVScenario(t *testing.T, s formatScenario) {
 	case "decode-tsv-object":
 		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredTsvPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
 	case "roundtrip-csv":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredCsvPreferences), NewCsvEncoder(',')), s.description)
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredCsvPreferences), NewCsvEncoder(ConfiguredCsvPreferences)), s.description)
 	default:
 		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
 	}
@@ -298,9 +298,10 @@ func documentCSVEncodeScenario(w *bufio.Writer, s formatScenario, formatType str
 	if formatType == "tsv" {
 		separator = '\t'
 	}
-
+	csvPrefs := NewDefaultCsvPreferences()
+	csvPrefs.Separator = separator
 	writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
-		mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(separator))),
+		mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(csvPrefs))),
 	)
 }
 
@@ -331,8 +332,11 @@ func documentCSVRoundTripScenario(w *bufio.Writer, s formatScenario, formatType
 		separator = '\t'
 	}
 
+	csvPrefs := NewDefaultCsvPreferences()
+	csvPrefs.Separator = separator
+
 	writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
-		mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: true}), NewCsvEncoder(separator))),
+		mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: true}), NewCsvEncoder(csvPrefs))),
 	)
 }
 

pkg/yqlib/encoder_csv.go

@@ -10,8 +10,8 @@ type csvEncoder struct {
 	separator rune
 }
 
-func NewCsvEncoder(separator rune) Encoder {
-	return &csvEncoder{separator: separator}
+func NewCsvEncoder(prefs CsvPreferences) Encoder {
+	return &csvEncoder{separator: prefs.Separator}
 }
 
 func (e *csvEncoder) CanHandleAliases() bool {

pkg/yqlib/operator_encoder_decoder.go

@@ -16,9 +16,9 @@ func configureEncoder(format PrinterOutputFormat, indent int) Encoder {
 	case PropsOutputFormat:
 		return NewPropertiesEncoder(true)
 	case CSVOutputFormat:
-		return NewCsvEncoder(',')
+		return NewCsvEncoder(ConfiguredCsvPreferences)
 	case TSVOutputFormat:
-		return NewCsvEncoder('\t')
+		return NewCsvEncoder(ConfiguredTsvPreferences)
 	case YamlOutputFormat:
 		return NewYamlEncoder(indent, false, ConfiguredYamlPreferences)
 	case XMLOutputFormat: