From 4ba96d902b24a63a4da2124ec6c44ad2446ebdca Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Tue, 17 Aug 2021 09:54:12 +1000 Subject: [PATCH] Fixed document header/footer comment handling when merging --- pkg/yqlib/doc/String Operators.md | 39 +++++++++++++ pkg/yqlib/operator_multiply.go | 23 +++++++- pkg/yqlib/operator_multiply_test.go | 88 +++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 1 deletion(-) diff --git a/pkg/yqlib/doc/String Operators.md b/pkg/yqlib/doc/String Operators.md index b3fa0bdd..0e2c9746 100644 --- a/pkg/yqlib/doc/String Operators.md +++ b/pkg/yqlib/doc/String Operators.md @@ -3,6 +3,45 @@ ## RegEx This uses golangs native regex functions under the hood - See https://github.com/google/re2/wiki/Syntax for the supported syntax. + +# String blocks, bash and newlines +Bash is notorious for chomping on precious trailing newline characters, making it tricky to set strings with newlines properly. In particular, the `$( exp )` _will trim trailing newlines_. + +For instance to get this yaml: + +``` +a: | + cat +``` + +Using `$( exp )` wont work, as it will trim the trailing new line. + +``` +m=$(echo "cat\n") yq e -n '.a = strenv(m)' +a: cat +``` + +However, using printf works: +``` +printf -v m "cat\n" ; m="$m" yq e -n '.a = strenv(m)' +a: | + cat +``` + +As well as having multiline expressions: +``` +m="cat +" yq e -n '.a = strenv(m)' +a: | + cat +``` + +Similarly, if you're trying to set the content from a file, and want a trailing new line: + +``` +IFS= read -rd '' output < <(cat my_file) +output=$output ./yq e '.data.values = strenv(output)' first.yml +``` ## Join strings Given a sample.yml file of: ```yaml diff --git a/pkg/yqlib/operator_multiply.go b/pkg/yqlib/operator_multiply.go index 8d273066..d7af255e 100644 --- a/pkg/yqlib/operator_multiply.go +++ b/pkg/yqlib/operator_multiply.go @@ -20,8 +20,29 @@ func multiplyOperator(d *dataTreeNavigator, context Context, expressionNode *Exp return crossFunction(d, context, expressionNode, multiply(expressionNode.Operation.Preferences.(multiplyPreferences)), false) } +func getNewBlankNode(lhs *yaml.Node, rhs *yaml.Node) *yaml.Node { + + blankNode := &yaml.Node{} + + if lhs.HeadComment != "" { + blankNode.HeadComment = lhs.HeadComment + } else if rhs.HeadComment != "" { + blankNode.HeadComment = rhs.HeadComment + } + + if lhs.FootComment != "" { + blankNode.FootComment = lhs.FootComment + } else if rhs.FootComment != "" { + blankNode.FootComment = rhs.FootComment + } + + return blankNode +} + func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { + // need to do this before unWrapping the potential document node + newBlankNode := getNewBlankNode(lhs.Node, rhs.Node) lhs.Node = unwrapDoc(lhs.Node) rhs.Node = unwrapDoc(rhs.Node) log.Debugf("Multipling LHS: %v", lhs.Node.Tag) @@ -30,7 +51,7 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode || (lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) { - var newBlank = lhs.CreateChild(nil, &yaml.Node{}) + var newBlank = lhs.CreateChild(nil, newBlankNode) log.Debugf("merge - merge lhs into blank") var newThing, err = mergeObjects(d, context.WritableClone(), newBlank, lhs, multiplyPreferences{}) if err != nil { diff --git a/pkg/yqlib/operator_multiply_test.go b/pkg/yqlib/operator_multiply_test.go index c805e108..c1c93adf 100644 --- a/pkg/yqlib/operator_multiply_test.go +++ b/pkg/yqlib/operator_multiply_test.go @@ -35,6 +35,31 @@ We then need to update the first array. We will use the relative update (|=) bec We set the current element of the first array as $cur. Now we multiply (merge) $cur with the matching entry in $two, by passing $two through a select filter. ` +var docWithHeader = ` +# here + +a: apple +` + +var nodeWithHeader = ` +# here +a: apple +` + +var docNoComments = ` +b: banana +` + +var docWithFooter = ` +a: apple + +# footer +` + +var nodeWithFooter = ` +a: apple +# footer` + var multiplyOperatorScenarios = []expressionScenario{ { skipDoc: true, @@ -44,6 +69,69 @@ var multiplyOperatorScenarios = []expressionScenario{ "D0, P[], (!!map)::sample:\n - &a\n - !!merge <<: *a\n", }, }, + { + skipDoc: true, + document: docWithHeader, + document2: docNoComments, + expression: `select(fi == 0) * select(fi == 1)`, + expected: []string{ + "D0, P[], (!!map)::# here\na: apple\nb: banana\n", + }, + }, + { + skipDoc: true, + document: nodeWithHeader, + document2: docNoComments, + expression: `select(fi == 0) * select(fi == 1)`, + expected: []string{ + "D0, P[], (!!map)::# here\na: apple\nb: banana\n", + }, + }, + { + skipDoc: true, + document: docNoComments, + document2: docWithHeader, + expression: `select(fi == 0) * select(fi == 1)`, + expected: []string{ + "D0, P[], (!!map)::# here\nb: banana\na: apple\n", + }, + }, + { + skipDoc: true, + document: docNoComments, + document2: nodeWithHeader, + expression: `select(fi == 0) * select(fi == 1)`, + expected: []string{ + "D0, P[], (!!map)::b: banana\n# here\na: apple\n", + }, + }, + { + skipDoc: true, + document: docWithFooter, + document2: docNoComments, + expression: `select(fi == 0) * select(fi == 1)`, + expected: []string{ + "D0, P[], (!!map)::a: apple\nb: banana\n\n# footer\n", + }, + }, + { + skipDoc: true, + document: nodeWithFooter, + document2: docNoComments, + expression: `select(fi == 0) * select(fi == 1)`, + expected: []string{ // not sure why there's an extra newline *shrug* + "D0, P[], (!!map)::a: apple\n# footer\n\nb: banana\n", + }, + }, + { + skipDoc: true, + document: docNoComments, + document2: docWithFooter, + expression: `select(fi == 0) * select(fi == 1)`, + expected: []string{ + "D0, P[], (!!map)::b: banana\na: apple\n\n# footer\n", + }, + }, { description: "Multiply integers", expression: `3 * 4`,