Comment processing fixes

examples/sample.yaml

@@ -1,11 +1,2 @@
-# Some doc
+ # things
-
+a: apple
-a: true
-b:
-  c: 2
-  d: [3, 4, 5]
-  e:
-    - name: fred
-      value: 3
-    - name: sam
-      value: 4

pkg/yqlib/candidate_node.go

@@ -70,8 +70,7 @@ type CandidateNode struct {
 	Parent *CandidateNode // parent node
 	Key    *CandidateNode // node key, if this is a value from a map (or index in an array)
 
-	LeadingContent  string
+	LeadingContent string
-	TrailingContent string
 
 	document uint // the document index of this node
 	filename string
@@ -312,10 +311,12 @@ func (n *CandidateNode) CopyAsReplacement(replacement *CandidateNode) *Candidate
 	return newCopy
 }
 
-func (n *CandidateNode) CreateReplacementWithDocWrappers(kind Kind, tag string, style Style) *CandidateNode {
+func (n *CandidateNode) CreateReplacementWithComments(kind Kind, tag string, style Style) *CandidateNode {
 	replacement := n.CreateReplacement(kind, tag, "")
 	replacement.LeadingContent = n.LeadingContent
-	replacement.TrailingContent = n.TrailingContent
+	replacement.HeadComment = n.HeadComment
+	replacement.LineComment = n.LineComment
+	replacement.FootComment = n.FootComment
 	replacement.Style = style
 	return replacement
 }
@@ -356,8 +357,7 @@ func (n *CandidateNode) doCopy(cloneContent bool) *CandidateNode {
 		Parent: n.Parent,
 		Key:    copyKey,
 
-		LeadingContent:  n.LeadingContent,
+		LeadingContent: n.LeadingContent,
-		TrailingContent: n.TrailingContent,
 
 		document:  n.document,
 		filename:  n.filename,
@@ -428,9 +428,6 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignP
 	if other.FootComment != "" {
 		n.FootComment = other.FootComment
 	}
-	if other.TrailingContent != "" {
-		n.TrailingContent = other.TrailingContent
-	}
 	if other.HeadComment != "" {
 		n.HeadComment = other.HeadComment
 	}

pkg/yqlib/decoder_yaml.go

@@ -135,10 +135,6 @@ func (dec *yamlDecoder) Decode() (*CandidateNode, error) {
 	}
 	dec.readAnything = true
 	dec.documentIndex++
-	// move document comments into candidate node
-	// otherwise unwrap drops them.
-	candidateNode.TrailingContent = candidateNode.FootComment
-	candidateNode.FootComment = ""
 	return &candidateNode, nil
 }
 

pkg/yqlib/doc/operators/comment-operators.md

@@ -221,7 +221,6 @@ yq '. foot_comment=.a' sample.yml
 will output
 ```yaml
 a: cat
-
 # cat
 ```
 

pkg/yqlib/encoder_yaml.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"errors"
 	"io"
+	"regexp"
 	"strings"
 
 	yaml "gopkg.in/yaml.v3"
@@ -41,6 +42,8 @@ func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) err
 	// log.Debug("headcommentwas [%v]", content)
 	reader := bufio.NewReader(strings.NewReader(content))
 
+	var commentLineRegEx = regexp.MustCompile(`^\s*#`)
+
 	for {
 
 		readline, errReading := reader.ReadString('\n')
@@ -54,6 +57,9 @@ func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) err
 			}
 
 		} else {
+			if len(readline) > 0 && readline != "\n" && readline[0] != '%' && !commentLineRegEx.MatchString(readline) {
+				readline = "# " + readline
+			}
 			if err := writeString(writer, readline); err != nil {
 				return err
 			}
@@ -95,10 +101,17 @@ func (ye *yamlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 		return err
 	}
 
+	trailingContent := target.FootComment
+	target.FootComment = ""
+
 	if err := encoder.Encode(target); err != nil {
 		return err
 	}
 
+	if err := ye.PrintLeadingContent(destination, trailingContent); err != nil {
+		return err
+	}
+
 	if ye.colorise {
 		return colorizeAndPrint(tempBuffer.Bytes(), writer)
 	}

pkg/yqlib/operator_comments.go

@@ -64,15 +64,8 @@ func assignCommentsOperator(d *dataTreeNavigator, context Context, expressionNod
 			candidate.HeadComment = comment
 			candidate.LeadingContent = "" // clobber the leading content, if there was any.
 		}
-		// if preferences.FootComment && candidate.Kind == DocumentNode && comment != "" {
+		if preferences.FootComment {
-		// 	log.Debugf("AssignComments - setting line comment to %v", comment)
-		// 	candidate.TrailingContent = "# " + comment
-		// } else if preferences.FootComment && candidate.Kind == DocumentNode {
-		// 	log.Debugf("AssignComments - setting line comment to %v", comment)
-		// 	candidate.TrailingContent = comment
-		if preferences.FootComment { //&& candidate.Kind != DocumentNode {
 			candidate.FootComment = comment
-			candidate.TrailingContent = ""
 		}
 
 	}
@@ -112,8 +105,6 @@ func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *
 			comment = chompRegexp.ReplaceAllString(comment, "")
 		} else if preferences.HeadComment {
 			comment = candidate.HeadComment
-		} else if preferences.FootComment && candidate.TrailingContent != "" {
-			comment = candidate.TrailingContent
 		} else if preferences.FootComment {
 			comment = candidate.FootComment
 		}

pkg/yqlib/operator_comments_test.go

@@ -162,7 +162,7 @@ var commentOperatorScenarios = []expressionScenario{
 		document:    `a: cat`,
 		expression:  `. foot_comment=.a`,
 		expected: []string{
-			"D0, P[], (!!map)::a: cat\n\n# cat\n",
+			"D0, P[], (!!map)::a: cat\n# cat\n",
 		},
 	},
 	{
@@ -260,6 +260,24 @@ var commentOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!str)::have a great day\nno really\n",
 		},
 	},
+	{
+		description: "leading spaces",
+		skipDoc:     true,
+		document:    " # hi",
+		expression:  `.`,
+		expected: []string{
+			"D0, P[], (!!null):: # hi\n\n",
+		},
+	},
+	{
+		description: "directive",
+		skipDoc:     true,
+		document:    "%YAML 1.1\n# hi\n",
+		expression:  `.`,
+		expected: []string{
+			"D0, P[], (!!null)::%YAML 1.1\n# hi\n\n",
+		},
+	},
 }
 
 func TestCommentOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_entries.go

@@ -17,7 +17,7 @@ func entrySeqFor(key *CandidateNode, value *CandidateNode) *CandidateNode {
 }
 
 func toEntriesFromMap(candidateNode *CandidateNode) *CandidateNode {
-	var sequence = candidateNode.CreateReplacementWithDocWrappers(SequenceNode, "!!seq", 0)
+	var sequence = candidateNode.CreateReplacementWithComments(SequenceNode, "!!seq", 0)
 
 	var contents = candidateNode.unwrapDocument().Content
 	for index := 0; index < len(contents); index = index + 2 {
@@ -30,7 +30,7 @@ func toEntriesFromMap(candidateNode *CandidateNode) *CandidateNode {
 }
 
 func toEntriesfromSeq(candidateNode *CandidateNode) *CandidateNode {
-	var sequence = candidateNode.CreateReplacementWithDocWrappers(SequenceNode, "!!seq", 0)
+	var sequence = candidateNode.CreateReplacementWithComments(SequenceNode, "!!seq", 0)
 
 	var contents = candidateNode.unwrapDocument().Content
 	for index := 0; index < len(contents); index = index + 1 {
@@ -158,8 +158,13 @@ func withEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *
 		if err != nil {
 			return Context{}, err
 		}
+		log.Debug("candidate %v", NodeToString(candidate))
+		log.Debug("candidate leading content: %v", candidate.LeadingContent)
 		collected.LeadingContent = candidate.LeadingContent
-		collected.TrailingContent = candidate.TrailingContent
+		log.Debug("candidate FootComment: [%v]", candidate.FootComment)
+
+		collected.HeadComment = candidate.HeadComment
+		collected.FootComment = candidate.FootComment
 
 		log.Debugf("**** collected %v", collected.LeadingContent)
 

pkg/yqlib/operator_multiply.go

@@ -33,28 +33,26 @@ func multiplyOperator(d *dataTreeNavigator, context Context, expressionNode *Exp
 	return crossFunction(d, context, expressionNode, multiply(expressionNode.Operation.Preferences.(multiplyPreferences)), false)
 }
 
-func getComments(lhs *CandidateNode, rhs *CandidateNode) (leadingContent string, headComment string, footComment string, trailingContent string) {
+func getComments(lhs *CandidateNode, rhs *CandidateNode) (leadingContent string, headComment string, footComment string) {
 	leadingContent = rhs.LeadingContent
 	headComment = rhs.HeadComment
 	footComment = rhs.FootComment
-	trailingContent = rhs.TrailingContent
 	if lhs.HeadComment != "" || lhs.LeadingContent != "" {
 		headComment = lhs.HeadComment
 		leadingContent = lhs.LeadingContent
 	}
 
-	if lhs.FootComment != "" || lhs.TrailingContent != "" {
+	if lhs.FootComment != "" {
 		footComment = lhs.FootComment
-		trailingContent = lhs.TrailingContent
 	}
 
-	return leadingContent, headComment, footComment, trailingContent
+	return leadingContent, headComment, footComment
 }
 
 func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 		// need to do this before unWrapping the potential document node
-		leadingContent, headComment, footComment, trailingContent := getComments(lhs, rhs)
+		leadingContent, headComment, footComment := getComments(lhs, rhs)
 		lhs = lhs.unwrapDocument()
 		rhs = rhs.unwrapDocument()
 		log.Debugf("Multiplying LHS: %v", NodeToString(lhs))
@@ -74,7 +72,6 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
 			newBlank.LeadingContent = leadingContent
 			newBlank.HeadComment = headComment
 			newBlank.FootComment = footComment
-			newBlank.TrailingContent = trailingContent
 
 			return mergeObjects(d, context.WritableClone(), newBlank, rhs, preferences)
 		}

pkg/yqlib/operator_pick.go

@@ -78,7 +78,6 @@ func pickOperator(d *dataTreeNavigator, context Context, expressionNode *Express
 		}
 
 		replacement.LeadingContent = candidate.LeadingContent
-		replacement.TrailingContent = candidate.TrailingContent
 		results.PushBack(replacement)
 	}
 

pkg/yqlib/operator_reverse.go

@@ -17,7 +17,7 @@ func reverseOperator(d *dataTreeNavigator, context Context, expressionNode *Expr
 			return context, fmt.Errorf("node at path [%v] is not an array (it's a %v)", candidate.GetNicePath(), candidate.GetNiceTag())
 		}
 
-		reverseList := candidate.CreateReplacementWithDocWrappers(SequenceNode, "!!seq", candidateNode.Style)
+		reverseList := candidate.CreateReplacementWithComments(SequenceNode, "!!seq", candidateNode.Style)
 		reverseContent := make([]*CandidateNode, len(candidateNode.Content))
 
 		for i, originalNode := range candidateNode.Content {

pkg/yqlib/operator_sort.go

@@ -45,7 +45,7 @@ func sortByOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 
 		sort.Stable(sortableArray)
 
-		sortedList := candidate.CreateReplacementWithDocWrappers(SequenceNode, "!!seq", candidateNode.Style)
+		sortedList := candidate.CreateReplacementWithComments(SequenceNode, "!!seq", candidateNode.Style)
 
 		for _, sortedNode := range sortableArray {
 			sortedList.AddChild(sortedNode.Node)

pkg/yqlib/operator_unique.go

@@ -49,7 +49,7 @@ func uniqueBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionN
 				newMatches.Set(keyValue, child)
 			}
 		}
-		resultNode := candidate.CreateReplacementWithDocWrappers(SequenceNode, "!!seq", candidateNode.Style)
+		resultNode := candidate.CreateReplacementWithComments(SequenceNode, "!!seq", candidateNode.Style)
 		for el := newMatches.Front(); el != nil; el = el.Next() {
 			resultNode.AddChild(el.Value.(*CandidateNode))
 		}

pkg/yqlib/printer.go

@@ -166,10 +166,6 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 			return err
 		}
 
-		if err := p.encoder.PrintLeadingContent(destination, mappedDoc.TrailingContent); err != nil {
-			return err
-		}
-
 		if p.nulSepOutput {
 			removeLastEOL(tempBuffer)
 			tempBufferBytes := tempBuffer.Bytes()