From 2865022cf8a18a9210aa87f63ce1d2f56c1b08c8 Mon Sep 17 00:00:00 2001 From: Ryan Drew Date: Mon, 19 Feb 2024 16:57:44 -0700 Subject: [PATCH] Add --properties-separator option (#1951) This commit adds the --properties-separator option, which lets users specify the separator used between keys and values in the properties output format. This is done by adjusting the value of github.com/magiconair/properties#Properties.WriteSeparator at encode time. Some refactoring of the properties encoder unit tests was done to make it easier to write unit tests that include different separator values. Fixes: #1864 Signed-off-by: Ryan Drew --- cmd/root.go | 2 + cmd/utils.go | 2 +- pkg/yqlib/doc/usage/properties.md | 30 ++++ pkg/yqlib/encoder_properties.go | 5 +- pkg/yqlib/encoder_properties_test.go | 218 ++++++++++++++++++-------- pkg/yqlib/operator_encoder_decoder.go | 2 +- pkg/yqlib/properties.go | 13 ++ pkg/yqlib/properties_test.go | 12 +- 8 files changed, 208 insertions(+), 76 deletions(-) create mode 100644 pkg/yqlib/properties.go diff --git a/cmd/root.go b/cmd/root.go index 05dc4485..3a6d0b03 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -120,6 +120,8 @@ yq -P -oy sample.json rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.UnquotedKeys, "lua-unquoted", yqlib.ConfiguredLuaPreferences.UnquotedKeys, "output unquoted string keys (e.g. {foo=\"bar\"})") rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.Globals, "lua-globals", yqlib.ConfiguredLuaPreferences.Globals, "output keys as top-level global variables") + rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredPropertiesPreferences.KeyValueSeparator, "properties-separator", yqlib.ConfiguredPropertiesPreferences.KeyValueSeparator, "separator to use between keys and values") + rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.") rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)") diff --git a/cmd/utils.go b/cmd/utils.go index 91f95f81..0a13c31f 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -187,7 +187,7 @@ func createEncoder(format yqlib.PrinterOutputFormat) (yqlib.Encoder, error) { case yqlib.JSONOutputFormat: return yqlib.NewJSONEncoder(indent, colorsEnabled, unwrapScalar), nil case yqlib.PropsOutputFormat: - return yqlib.NewPropertiesEncoder(unwrapScalar), nil + return yqlib.NewPropertiesEncoder(unwrapScalar, yqlib.ConfiguredPropertiesPreferences), nil case yqlib.CSVOutputFormat: return yqlib.NewCsvEncoder(yqlib.ConfiguredCsvPreferences), nil case yqlib.TSVOutputFormat: diff --git a/pkg/yqlib/doc/usage/properties.md b/pkg/yqlib/doc/usage/properties.md index f3d01235..70758dad 100644 --- a/pkg/yqlib/doc/usage/properties.md +++ b/pkg/yqlib/doc/usage/properties.md @@ -120,6 +120,36 @@ emptyArray = emptyMap = ``` +## Encode properties: use custom separator +Provide a custom key-value separator using the `--properties-separator` flag. + +Given a sample.yml file of: +```yaml +# block comments come through +person: # neither do comments on maps + name: Mike Wazowski # comments on values appear + pets: + - cat # comments on array values appear + food: [pizza] # comments on arrays do not +emptyArray: [] +emptyMap: [] + +``` +then +```bash +yq -o props --properties-separator=";" sample.yml +``` +will output +```properties +# block comments come through +# comments on values appear +person.name;Mike Wazowski + +# comments on array values appear +person.pets.0;cat +person.food.0;pizza +``` + ## Decode properties Given a sample.properties file of: ```properties diff --git a/pkg/yqlib/encoder_properties.go b/pkg/yqlib/encoder_properties.go index 74826fd8..49d0e007 100644 --- a/pkg/yqlib/encoder_properties.go +++ b/pkg/yqlib/encoder_properties.go @@ -12,11 +12,13 @@ import ( type propertiesEncoder struct { unwrapScalar bool + prefs PropertiesPreferences } -func NewPropertiesEncoder(unwrapScalar bool) Encoder { +func NewPropertiesEncoder(unwrapScalar bool, prefs PropertiesPreferences) Encoder { return &propertiesEncoder{ unwrapScalar: unwrapScalar, + prefs: prefs, } } @@ -69,6 +71,7 @@ func (pe *propertiesEncoder) Encode(writer io.Writer, node *CandidateNode) error mapKeysToStrings(node) p := properties.NewProperties() + p.WriteSeparator = pe.prefs.KeyValueSeparator err := pe.doEncode(p, node, "", nil) if err != nil { return err diff --git a/pkg/yqlib/encoder_properties_test.go b/pkg/yqlib/encoder_properties_test.go index 70138fe1..ffa3c202 100644 --- a/pkg/yqlib/encoder_properties_test.go +++ b/pkg/yqlib/encoder_properties_test.go @@ -3,17 +3,60 @@ package yqlib import ( "bufio" "bytes" + "fmt" "strings" "testing" "github.com/mikefarah/yq/v4/test" ) -func yamlToProps(sampleYaml string, unwrapScalar bool) string { +type keyValuePair struct { + key string + value string + comment string +} + +func (kv *keyValuePair) String(unwrap bool, sep string) string { + builder := strings.Builder{} + + if kv.comment != "" { + builder.WriteString(kv.comment) + builder.WriteString("\n") + } + + builder.WriteString(kv.key) + builder.WriteString(sep) + + if unwrap { + builder.WriteString(kv.value) + } else { + builder.WriteString("\"") + builder.WriteString(kv.value) + builder.WriteString("\"") + } + + return builder.String() +} + +type testProperties struct { + pairs []keyValuePair +} + +func (tp *testProperties) String(unwrap bool, sep string) string { + kvs := []string{} + + for _, kv := range tp.pairs { + kvs = append(kvs, kv.String(unwrap, sep)) + } + + return strings.Join(kvs, "\n") +} + +func yamlToProps(sampleYaml string, unwrapScalar bool, separator string) string { var output bytes.Buffer writer := bufio.NewWriter(&output) - var propsEncoder = NewPropertiesEncoder(unwrapScalar) + var propsEncoder = NewPropertiesEncoder(unwrapScalar, PropertiesPreferences{KeyValueSeparator: separator}) inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) @@ -28,80 +71,97 @@ func yamlToProps(sampleYaml string, unwrapScalar bool) string { return strings.TrimSuffix(output.String(), "\n") } -func TestPropertiesEncoderSimple_Unwrapped(t *testing.T) { +func doTest(t *testing.T, sampleYaml string, props testProperties, testUnwrapped, testWrapped bool) { + wraps := []bool{} + if testUnwrapped { + wraps = append(wraps, true) + } + if testWrapped { + wraps = append(wraps, false) + } + + for _, unwrap := range wraps { + fmt.Println(props) + fmt.Println(unwrap) + for _, sep := range []string{" = ", ";", "=", " "} { + var actualProps = yamlToProps(sampleYaml, unwrap, sep) + test.AssertResult(t, props.String(unwrap, sep), actualProps) + } + } +} + +func TestPropertiesEncoderSimple(t *testing.T) { var sampleYaml = `a: 'bob cool'` - var expectedProps = `a = bob cool` - var actualProps = yamlToProps(sampleYaml, true) - test.AssertResult(t, expectedProps, actualProps) + doTest( + t, sampleYaml, + testProperties{ + pairs: []keyValuePair{ + { + key: "a", + value: "bob cool", + }, + }, + }, + true, true, + ) } -func TestPropertiesEncoderSimple_Wrapped(t *testing.T) { - var sampleYaml = `a: 'bob cool'` - - var expectedProps = `a = "bob cool"` - var actualProps = yamlToProps(sampleYaml, false) - test.AssertResult(t, expectedProps, actualProps) -} - -func TestPropertiesEncoderSimpleWithComments_Unwrapped(t *testing.T) { +func TestPropertiesEncoderSimpleWithComments(t *testing.T) { var sampleYaml = `a: 'bob cool' # line` - var expectedProps = `# line -a = bob cool` - var actualProps = yamlToProps(sampleYaml, true) - test.AssertResult(t, expectedProps, actualProps) + doTest( + t, sampleYaml, + testProperties{ + pairs: []keyValuePair{ + { + key: "a", + value: "bob cool", + comment: "# line", + }, + }, + }, + true, true, + ) } -func TestPropertiesEncoderSimpleWithComments_Wrapped(t *testing.T) { - var sampleYaml = `a: 'bob cool' # line` - - var expectedProps = `# line -a = "bob cool"` - var actualProps = yamlToProps(sampleYaml, false) - test.AssertResult(t, expectedProps, actualProps) -} - -func TestPropertiesEncoderDeep_Unwrapped(t *testing.T) { +func TestPropertiesEncoderDeep(t *testing.T) { var sampleYaml = `a: b: "bob cool" ` - var expectedProps = `a.b = bob cool` - var actualProps = yamlToProps(sampleYaml, true) - test.AssertResult(t, expectedProps, actualProps) + doTest( + t, sampleYaml, + testProperties{ + pairs: []keyValuePair{ + { + key: "a.b", + value: "bob cool", + }, + }, + }, + true, true, + ) } -func TestPropertiesEncoderDeep_Wrapped(t *testing.T) { - var sampleYaml = `a: - b: "bob cool" -` - - var expectedProps = `a.b = "bob cool"` - var actualProps = yamlToProps(sampleYaml, false) - test.AssertResult(t, expectedProps, actualProps) -} - -func TestPropertiesEncoderDeepWithComments_Unwrapped(t *testing.T) { +func TestPropertiesEncoderDeepWithComments(t *testing.T) { var sampleYaml = `a: # a thing b: "bob cool" # b thing ` - var expectedProps = `# b thing -a.b = bob cool` - var actualProps = yamlToProps(sampleYaml, true) - test.AssertResult(t, expectedProps, actualProps) -} - -func TestPropertiesEncoderDeepWithComments_Wrapped(t *testing.T) { - var sampleYaml = `a: # a thing - b: "bob cool" # b thing -` - - var expectedProps = `# b thing -a.b = "bob cool"` - var actualProps = yamlToProps(sampleYaml, false) - test.AssertResult(t, expectedProps, actualProps) + doTest( + t, sampleYaml, + testProperties{ + pairs: []keyValuePair{ + { + key: "a.b", + value: "bob cool", + comment: "# b thing", + }, + }, + }, + true, true, + ) } func TestPropertiesEncoderArray_Unwrapped(t *testing.T) { @@ -109,19 +169,43 @@ func TestPropertiesEncoderArray_Unwrapped(t *testing.T) { b: [{c: dog}, {c: cat}] ` - var expectedProps = `a.b.0.c = dog -a.b.1.c = cat` - var actualProps = yamlToProps(sampleYaml, true) - test.AssertResult(t, expectedProps, actualProps) + doTest( + t, sampleYaml, + testProperties{ + pairs: []keyValuePair{ + { + key: "a.b.0.c", + value: "dog", + }, + { + key: "a.b.1.c", + value: "cat", + }, + }, + }, + true, false, + ) } func TestPropertiesEncoderArray_Wrapped(t *testing.T) { var sampleYaml = `a: - b: [{c: dog named jim}, {c: cat named jam}] + b: [{c: dog named jim}, {c: cat named jim}] ` - var expectedProps = `a.b.0.c = "dog named jim" -a.b.1.c = "cat named jam"` - var actualProps = yamlToProps(sampleYaml, false) - test.AssertResult(t, expectedProps, actualProps) + doTest( + t, sampleYaml, + testProperties{ + pairs: []keyValuePair{ + { + key: "a.b.0.c", + value: "dog named jim", + }, + { + key: "a.b.1.c", + value: "cat named jim", + }, + }, + }, + false, true, + ) } diff --git a/pkg/yqlib/operator_encoder_decoder.go b/pkg/yqlib/operator_encoder_decoder.go index f59c8702..c68a2d3d 100644 --- a/pkg/yqlib/operator_encoder_decoder.go +++ b/pkg/yqlib/operator_encoder_decoder.go @@ -14,7 +14,7 @@ func configureEncoder(format PrinterOutputFormat, indent int) Encoder { case JSONOutputFormat: return NewJSONEncoder(indent, false, false) case PropsOutputFormat: - return NewPropertiesEncoder(true) + return NewPropertiesEncoder(true, ConfiguredPropertiesPreferences) case CSVOutputFormat: return NewCsvEncoder(ConfiguredCsvPreferences) case TSVOutputFormat: diff --git a/pkg/yqlib/properties.go b/pkg/yqlib/properties.go new file mode 100644 index 00000000..b1920b12 --- /dev/null +++ b/pkg/yqlib/properties.go @@ -0,0 +1,13 @@ +package yqlib + +type PropertiesPreferences struct { + KeyValueSeparator string +} + +func NewDefaultPropertiesPreferences() PropertiesPreferences { + return PropertiesPreferences{ + KeyValueSeparator: " = ", + } +} + +var ConfiguredPropertiesPreferences = NewDefaultPropertiesPreferences() diff --git a/pkg/yqlib/properties_test.go b/pkg/yqlib/properties_test.go index 43293428..15d69dfe 100644 --- a/pkg/yqlib/properties_test.go +++ b/pkg/yqlib/properties_test.go @@ -272,7 +272,7 @@ func documentUnwrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) } writeOrPanic(w, "will output\n") - writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true)))) + writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true, ConfiguredPropertiesPreferences)))) } func documentWrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) { @@ -297,7 +297,7 @@ func documentWrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) { } writeOrPanic(w, "will output\n") - writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false)))) + writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false, ConfiguredPropertiesPreferences)))) } func documentDecodePropertyScenario(w *bufio.Writer, s formatScenario) { @@ -347,7 +347,7 @@ func documentRoundTripPropertyScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, "will output\n") - writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", mustProcessFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(true)))) + writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", mustProcessFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(true, ConfiguredPropertiesPreferences)))) } func documentPropertyScenario(_ *testing.T, w *bufio.Writer, i interface{}) { @@ -374,13 +374,13 @@ func TestPropertyScenarios(t *testing.T) { for _, s := range propertyScenarios { switch s.scenarioType { case "": - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true)), s.description) + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true, ConfiguredPropertiesPreferences)), s.description) case "decode": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description) case "encode-wrapped": - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false)), s.description) + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false, ConfiguredPropertiesPreferences)), s.description) case "roundtrip": - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(true)), s.description) + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(true, ConfiguredPropertiesPreferences)), s.description) default: panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))