From 49b8eafaf53ca631f37187e6d8511c3ca491eba8 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Tue, 6 Jun 2023 12:10:28 -0700 Subject: [PATCH] Comment processing fixes --- examples/sample.yaml | 13 ++----------- pkg/yqlib/candidate_node.go | 15 ++++++--------- pkg/yqlib/decoder_yaml.go | 4 ---- pkg/yqlib/doc/operators/comment-operators.md | 1 - pkg/yqlib/encoder_yaml.go | 13 +++++++++++++ pkg/yqlib/operator_comments.go | 11 +---------- pkg/yqlib/operator_comments_test.go | 20 +++++++++++++++++++- pkg/yqlib/operator_entries.go | 11 ++++++++--- pkg/yqlib/operator_multiply.go | 11 ++++------- pkg/yqlib/operator_pick.go | 1 - pkg/yqlib/operator_reverse.go | 2 +- pkg/yqlib/operator_sort.go | 2 +- pkg/yqlib/operator_unique.go | 2 +- pkg/yqlib/printer.go | 4 ---- 14 files changed, 56 insertions(+), 54 deletions(-) diff --git a/examples/sample.yaml b/examples/sample.yaml index 9b925d75..8e7fa374 100644 --- a/examples/sample.yaml +++ b/examples/sample.yaml @@ -1,11 +1,2 @@ -# Some doc - -a: true -b: - c: 2 - d: [3, 4, 5] - e: - - name: fred - value: 3 - - name: sam - value: 4 \ No newline at end of file + # things +a: apple \ No newline at end of file diff --git a/pkg/yqlib/candidate_node.go b/pkg/yqlib/candidate_node.go index c5b296ee..3c7b54a2 100644 --- a/pkg/yqlib/candidate_node.go +++ b/pkg/yqlib/candidate_node.go @@ -70,8 +70,7 @@ type CandidateNode struct { Parent *CandidateNode // parent node Key *CandidateNode // node key, if this is a value from a map (or index in an array) - LeadingContent string - TrailingContent string + LeadingContent string document uint // the document index of this node filename string @@ -312,10 +311,12 @@ func (n *CandidateNode) CopyAsReplacement(replacement *CandidateNode) *Candidate return newCopy } -func (n *CandidateNode) CreateReplacementWithDocWrappers(kind Kind, tag string, style Style) *CandidateNode { +func (n *CandidateNode) CreateReplacementWithComments(kind Kind, tag string, style Style) *CandidateNode { replacement := n.CreateReplacement(kind, tag, "") replacement.LeadingContent = n.LeadingContent - replacement.TrailingContent = n.TrailingContent + replacement.HeadComment = n.HeadComment + replacement.LineComment = n.LineComment + replacement.FootComment = n.FootComment replacement.Style = style return replacement } @@ -356,8 +357,7 @@ func (n *CandidateNode) doCopy(cloneContent bool) *CandidateNode { Parent: n.Parent, Key: copyKey, - LeadingContent: n.LeadingContent, - TrailingContent: n.TrailingContent, + LeadingContent: n.LeadingContent, document: n.document, filename: n.filename, @@ -428,9 +428,6 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignP if other.FootComment != "" { n.FootComment = other.FootComment } - if other.TrailingContent != "" { - n.TrailingContent = other.TrailingContent - } if other.HeadComment != "" { n.HeadComment = other.HeadComment } diff --git a/pkg/yqlib/decoder_yaml.go b/pkg/yqlib/decoder_yaml.go index 42cc3a4d..a92a197a 100644 --- a/pkg/yqlib/decoder_yaml.go +++ b/pkg/yqlib/decoder_yaml.go @@ -135,10 +135,6 @@ func (dec *yamlDecoder) Decode() (*CandidateNode, error) { } dec.readAnything = true dec.documentIndex++ - // move document comments into candidate node - // otherwise unwrap drops them. - candidateNode.TrailingContent = candidateNode.FootComment - candidateNode.FootComment = "" return &candidateNode, nil } diff --git a/pkg/yqlib/doc/operators/comment-operators.md b/pkg/yqlib/doc/operators/comment-operators.md index 2c0e99e7..414fff83 100644 --- a/pkg/yqlib/doc/operators/comment-operators.md +++ b/pkg/yqlib/doc/operators/comment-operators.md @@ -221,7 +221,6 @@ yq '. foot_comment=.a' sample.yml will output ```yaml a: cat - # cat ``` diff --git a/pkg/yqlib/encoder_yaml.go b/pkg/yqlib/encoder_yaml.go index 64ea9498..24ecda5b 100644 --- a/pkg/yqlib/encoder_yaml.go +++ b/pkg/yqlib/encoder_yaml.go @@ -5,6 +5,7 @@ import ( "bytes" "errors" "io" + "regexp" "strings" yaml "gopkg.in/yaml.v3" @@ -41,6 +42,8 @@ func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) err // log.Debug("headcommentwas [%v]", content) reader := bufio.NewReader(strings.NewReader(content)) + var commentLineRegEx = regexp.MustCompile(`^\s*#`) + for { readline, errReading := reader.ReadString('\n') @@ -54,6 +57,9 @@ func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) err } } else { + if len(readline) > 0 && readline != "\n" && readline[0] != '%' && !commentLineRegEx.MatchString(readline) { + readline = "# " + readline + } if err := writeString(writer, readline); err != nil { return err } @@ -95,10 +101,17 @@ func (ye *yamlEncoder) Encode(writer io.Writer, node *CandidateNode) error { return err } + trailingContent := target.FootComment + target.FootComment = "" + if err := encoder.Encode(target); err != nil { return err } + if err := ye.PrintLeadingContent(destination, trailingContent); err != nil { + return err + } + if ye.colorise { return colorizeAndPrint(tempBuffer.Bytes(), writer) } diff --git a/pkg/yqlib/operator_comments.go b/pkg/yqlib/operator_comments.go index b6aa135a..09c56df4 100644 --- a/pkg/yqlib/operator_comments.go +++ b/pkg/yqlib/operator_comments.go @@ -64,15 +64,8 @@ func assignCommentsOperator(d *dataTreeNavigator, context Context, expressionNod candidate.HeadComment = comment candidate.LeadingContent = "" // clobber the leading content, if there was any. } - // if preferences.FootComment && candidate.Kind == DocumentNode && comment != "" { - // log.Debugf("AssignComments - setting line comment to %v", comment) - // candidate.TrailingContent = "# " + comment - // } else if preferences.FootComment && candidate.Kind == DocumentNode { - // log.Debugf("AssignComments - setting line comment to %v", comment) - // candidate.TrailingContent = comment - if preferences.FootComment { //&& candidate.Kind != DocumentNode { + if preferences.FootComment { candidate.FootComment = comment - candidate.TrailingContent = "" } } @@ -112,8 +105,6 @@ func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode * comment = chompRegexp.ReplaceAllString(comment, "") } else if preferences.HeadComment { comment = candidate.HeadComment - } else if preferences.FootComment && candidate.TrailingContent != "" { - comment = candidate.TrailingContent } else if preferences.FootComment { comment = candidate.FootComment } diff --git a/pkg/yqlib/operator_comments_test.go b/pkg/yqlib/operator_comments_test.go index 8d32a309..c4fda05f 100644 --- a/pkg/yqlib/operator_comments_test.go +++ b/pkg/yqlib/operator_comments_test.go @@ -162,7 +162,7 @@ var commentOperatorScenarios = []expressionScenario{ document: `a: cat`, expression: `. foot_comment=.a`, expected: []string{ - "D0, P[], (!!map)::a: cat\n\n# cat\n", + "D0, P[], (!!map)::a: cat\n# cat\n", }, }, { @@ -260,6 +260,24 @@ var commentOperatorScenarios = []expressionScenario{ "D0, P[], (!!str)::have a great day\nno really\n", }, }, + { + description: "leading spaces", + skipDoc: true, + document: " # hi", + expression: `.`, + expected: []string{ + "D0, P[], (!!null):: # hi\n\n", + }, + }, + { + description: "directive", + skipDoc: true, + document: "%YAML 1.1\n# hi\n", + expression: `.`, + expected: []string{ + "D0, P[], (!!null)::%YAML 1.1\n# hi\n\n", + }, + }, } func TestCommentOperatorScenarios(t *testing.T) { diff --git a/pkg/yqlib/operator_entries.go b/pkg/yqlib/operator_entries.go index 7fe83e7d..d52552a8 100644 --- a/pkg/yqlib/operator_entries.go +++ b/pkg/yqlib/operator_entries.go @@ -17,7 +17,7 @@ func entrySeqFor(key *CandidateNode, value *CandidateNode) *CandidateNode { } func toEntriesFromMap(candidateNode *CandidateNode) *CandidateNode { - var sequence = candidateNode.CreateReplacementWithDocWrappers(SequenceNode, "!!seq", 0) + var sequence = candidateNode.CreateReplacementWithComments(SequenceNode, "!!seq", 0) var contents = candidateNode.unwrapDocument().Content for index := 0; index < len(contents); index = index + 2 { @@ -30,7 +30,7 @@ func toEntriesFromMap(candidateNode *CandidateNode) *CandidateNode { } func toEntriesfromSeq(candidateNode *CandidateNode) *CandidateNode { - var sequence = candidateNode.CreateReplacementWithDocWrappers(SequenceNode, "!!seq", 0) + var sequence = candidateNode.CreateReplacementWithComments(SequenceNode, "!!seq", 0) var contents = candidateNode.unwrapDocument().Content for index := 0; index < len(contents); index = index + 1 { @@ -158,8 +158,13 @@ func withEntriesOperator(d *dataTreeNavigator, context Context, expressionNode * if err != nil { return Context{}, err } + log.Debug("candidate %v", NodeToString(candidate)) + log.Debug("candidate leading content: %v", candidate.LeadingContent) collected.LeadingContent = candidate.LeadingContent - collected.TrailingContent = candidate.TrailingContent + log.Debug("candidate FootComment: [%v]", candidate.FootComment) + + collected.HeadComment = candidate.HeadComment + collected.FootComment = candidate.FootComment log.Debugf("**** collected %v", collected.LeadingContent) diff --git a/pkg/yqlib/operator_multiply.go b/pkg/yqlib/operator_multiply.go index c69262ad..20f1bc7e 100644 --- a/pkg/yqlib/operator_multiply.go +++ b/pkg/yqlib/operator_multiply.go @@ -33,28 +33,26 @@ func multiplyOperator(d *dataTreeNavigator, context Context, expressionNode *Exp return crossFunction(d, context, expressionNode, multiply(expressionNode.Operation.Preferences.(multiplyPreferences)), false) } -func getComments(lhs *CandidateNode, rhs *CandidateNode) (leadingContent string, headComment string, footComment string, trailingContent string) { +func getComments(lhs *CandidateNode, rhs *CandidateNode) (leadingContent string, headComment string, footComment string) { leadingContent = rhs.LeadingContent headComment = rhs.HeadComment footComment = rhs.FootComment - trailingContent = rhs.TrailingContent if lhs.HeadComment != "" || lhs.LeadingContent != "" { headComment = lhs.HeadComment leadingContent = lhs.LeadingContent } - if lhs.FootComment != "" || lhs.TrailingContent != "" { + if lhs.FootComment != "" { footComment = lhs.FootComment - trailingContent = lhs.TrailingContent } - return leadingContent, headComment, footComment, trailingContent + return leadingContent, headComment, footComment } func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { // need to do this before unWrapping the potential document node - leadingContent, headComment, footComment, trailingContent := getComments(lhs, rhs) + leadingContent, headComment, footComment := getComments(lhs, rhs) lhs = lhs.unwrapDocument() rhs = rhs.unwrapDocument() log.Debugf("Multiplying LHS: %v", NodeToString(lhs)) @@ -74,7 +72,6 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex newBlank.LeadingContent = leadingContent newBlank.HeadComment = headComment newBlank.FootComment = footComment - newBlank.TrailingContent = trailingContent return mergeObjects(d, context.WritableClone(), newBlank, rhs, preferences) } diff --git a/pkg/yqlib/operator_pick.go b/pkg/yqlib/operator_pick.go index 72fbdfa8..8fe46b2f 100644 --- a/pkg/yqlib/operator_pick.go +++ b/pkg/yqlib/operator_pick.go @@ -78,7 +78,6 @@ func pickOperator(d *dataTreeNavigator, context Context, expressionNode *Express } replacement.LeadingContent = candidate.LeadingContent - replacement.TrailingContent = candidate.TrailingContent results.PushBack(replacement) } diff --git a/pkg/yqlib/operator_reverse.go b/pkg/yqlib/operator_reverse.go index 1a28c75f..a101299a 100644 --- a/pkg/yqlib/operator_reverse.go +++ b/pkg/yqlib/operator_reverse.go @@ -17,7 +17,7 @@ func reverseOperator(d *dataTreeNavigator, context Context, expressionNode *Expr return context, fmt.Errorf("node at path [%v] is not an array (it's a %v)", candidate.GetNicePath(), candidate.GetNiceTag()) } - reverseList := candidate.CreateReplacementWithDocWrappers(SequenceNode, "!!seq", candidateNode.Style) + reverseList := candidate.CreateReplacementWithComments(SequenceNode, "!!seq", candidateNode.Style) reverseContent := make([]*CandidateNode, len(candidateNode.Content)) for i, originalNode := range candidateNode.Content { diff --git a/pkg/yqlib/operator_sort.go b/pkg/yqlib/operator_sort.go index 0a1c5ed8..dcd9837a 100644 --- a/pkg/yqlib/operator_sort.go +++ b/pkg/yqlib/operator_sort.go @@ -45,7 +45,7 @@ func sortByOperator(d *dataTreeNavigator, context Context, expressionNode *Expre sort.Stable(sortableArray) - sortedList := candidate.CreateReplacementWithDocWrappers(SequenceNode, "!!seq", candidateNode.Style) + sortedList := candidate.CreateReplacementWithComments(SequenceNode, "!!seq", candidateNode.Style) for _, sortedNode := range sortableArray { sortedList.AddChild(sortedNode.Node) diff --git a/pkg/yqlib/operator_unique.go b/pkg/yqlib/operator_unique.go index 83838996..4828f8c4 100644 --- a/pkg/yqlib/operator_unique.go +++ b/pkg/yqlib/operator_unique.go @@ -49,7 +49,7 @@ func uniqueBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionN newMatches.Set(keyValue, child) } } - resultNode := candidate.CreateReplacementWithDocWrappers(SequenceNode, "!!seq", candidateNode.Style) + resultNode := candidate.CreateReplacementWithComments(SequenceNode, "!!seq", candidateNode.Style) for el := newMatches.Front(); el != nil; el = el.Next() { resultNode.AddChild(el.Value.(*CandidateNode)) } diff --git a/pkg/yqlib/printer.go b/pkg/yqlib/printer.go index b4e57672..6222d602 100644 --- a/pkg/yqlib/printer.go +++ b/pkg/yqlib/printer.go @@ -166,10 +166,6 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error { return err } - if err := p.encoder.PrintLeadingContent(destination, mappedDoc.TrailingContent); err != nil { - return err - } - if p.nulSepOutput { removeLastEOL(tempBuffer) tempBufferBytes := tempBuffer.Bytes()