Load file acceptance test

.github/dependabot.yml

@@ -8,12 +8,15 @@ updates:
   - package-ecosystem: docker
     directory: /
     schedule:
+      day: thursday
       interval: weekly
   - package-ecosystem: github-actions
     directory: /
     schedule:
+      day: thursday
       interval: weekly
   - package-ecosystem: gomod
     directory: /
     schedule:
+      day: thursday
       interval: weekly

acceptance_tests/load-file.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+testLoadFileNotExist() {
+  result=$(./yq e -n 'load("cat.yml")' 2>&1)
+  assertEquals 1 $?
+  assertEquals "Error: Failed to load cat.yml: open cat.yml: no such file or directory" "$result"
+}
+
+testLoadFileExpNotExist() {
+  result=$(./yq e -n 'load(.a)' 2>&1)
+  assertEquals 1 $?
+  assertEquals "Error: Filename expression returned nil" "$result"
+}
+
+testStrLoadFileNotExist() {
+  result=$(./yq e -n 'strload("cat.yml")' 2>&1)
+  assertEquals 1 $?
+  assertEquals "Error: Failed to load cat.yml: open cat.yml: no such file or directory" "$result"
+}
+
+testStrLoadFileExpNotExist() {
+  result=$(./yq e -n 'strload(.a)' 2>&1)
+  assertEquals 1 $?
+  assertEquals "Error: Filename expression returned nil" "$result"
+}
+
+source ./scripts/shunit2

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.14.1"
+	Version = "4.14.2"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

examples/data1.yaml

@@ -1,4 +1,2 @@
-- name: bob
-  age: 23
-- name: tim
-  age: 17
+a:
+  include: 'data2.yaml'

examples/data2.yaml

@@ -1,3 +1,2 @@
-#b1 
-b: 2
-#b2
+c:
+  d: hamster

examples/thing.yml

@@ -0,0 +1,2 @@
+a: apple is included
+b: cool.

github-action/Dockerfile

@@ -1,4 +1,4 @@
-FROM mikefarah/yq:4.14.1
+FROM mikefarah/yq:4.14.2
 
 COPY entrypoint.sh /entrypoint.sh
 

pkg/yqlib/all_at_once_evaluator.go

@@ -71,14 +71,15 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
 
 	if allDocuments.Len() == 0 {
 		candidateNode := &CandidateNode{
-			Document:  0,
-			Filename:  "",
-			Node:      &yaml.Node{Kind: yaml.DocumentNode, HeadComment: firstFileLeadingContent, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
-			FileIndex: 0,
+			Document:       0,
+			Filename:       "",
+			Node:           &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
+			FileIndex:      0,
+			LeadingContent: firstFileLeadingContent,
 		}
 		allDocuments.PushBack(candidateNode)
 	} else {
-		allDocuments.Front().Value.(*CandidateNode).Node.HeadComment = firstFileLeadingContent
+		allDocuments.Front().Value.(*CandidateNode).LeadingContent = firstFileLeadingContent
 	}
 
 	matches, err := e.EvaluateCandidateNodes(expression, allDocuments)

pkg/yqlib/all_at_once_evaluator_test.go

@@ -35,6 +35,6 @@ func TestAllAtOnceEvaluateNodes(t *testing.T) {
 	for _, tt := range evaluateNodesScenario {
 		node := test.ParseData(tt.document)
 		list, _ := evaluator.EvaluateNodes(tt.expression, &node)
-		test.AssertResultComplex(t, tt.expected, resultsToString(list))
+		test.AssertResultComplex(t, tt.expected, resultsToString(t, list))
 	}
 }

pkg/yqlib/candidate_node.go

@@ -9,10 +9,13 @@ import (
 )
 
 type CandidateNode struct {
-	Node      *yaml.Node     // the actual node
-	Parent    *CandidateNode // parent node
-	Path      []interface{}  /// the path we took to get to this node
-	Document  uint           // the document index of this node
+	Node   *yaml.Node     // the actual node
+	Parent *CandidateNode // parent node
+
+	LeadingContent string
+
+	Path      []interface{} /// the path we took to get to this node
+	Document  uint          // the document index of this node
 	Filename  string
 	FileIndex int
 	// when performing op against all nodes given, this will treat all the nodes as one

pkg/yqlib/doc/comment-operators.md

@@ -139,6 +139,26 @@ will output
 welcome!
 ```
 
+## 
+Given a sample.yml file of:
+```yaml
+# welcome!
+---
+# bob
+a: cat # meow
+
+# have a great day
+```
+then
+```bash
+yq eval 'headComment' sample.yml
+```
+will output
+```yaml
+welcome!
+bob
+```
+
 ## Get foot comment
 Given a sample.yml file of:
 ```yaml
@@ -147,6 +167,7 @@ Given a sample.yml file of:
 a: cat # meow
 
 # have a great day
+# no really
 ```
 then
 ```bash
@@ -155,5 +176,6 @@ yq eval '. | footComment' sample.yml
 will output
 ```yaml
 have a great day
+no really
 ```
 

pkg/yqlib/doc/headers/load.md

@@ -0,0 +1,13 @@
+# Load
+
+The `load`/`strload` operator allows you to load in content from another file referenced in your yaml document.
+
+Note that you can use string operators like `+` and `sub` to modify the value in the yaml file to a path that exists in your system.
+
+
+Lets say there is a file `../../examples/thing.yml`:
+
+```yaml
+a: apple is included
+b: cool
+```

pkg/yqlib/doc/load.md

@@ -0,0 +1,93 @@
+# Load
+
+The `load`/`strload` operator allows you to load in content from another file referenced in your yaml document.
+
+Note that you can use string operators like `+` and `sub` to modify the value in the yaml file to a path that exists in your system.
+
+
+Lets say there is a file `../../examples/thing.yml`:
+
+```yaml
+a: apple is included
+b: cool
+```
+
+## Simple example
+Given a sample.yml file of:
+```yaml
+myFile: ../../examples/thing.yml
+```
+then
+```bash
+yq eval 'load(.myFile)' sample.yml
+```
+will output
+```yaml
+a: apple is included
+b: cool.
+```
+
+## Replace node with referenced file
+Note that you can modify the filename in the load operator if needed.
+
+Given a sample.yml file of:
+```yaml
+something:
+  file: thing.yml
+```
+then
+```bash
+yq eval '.something |= load("../../examples/" + .file)' sample.yml
+```
+will output
+```yaml
+something:
+  a: apple is included
+  b: cool.
+```
+
+## Replace _all_ nodes with referenced file
+Recursively match all the nodes (`..`) and then filter the ones that have a 'file' attribute. 
+
+Given a sample.yml file of:
+```yaml
+something:
+  file: thing.yml
+over:
+  here:
+    - file: thing.yml
+```
+then
+```bash
+yq eval '(.. | select(has("file"))) |= load("../../examples/" + .file)' sample.yml
+```
+will output
+```yaml
+something:
+  a: apple is included
+  b: cool.
+over:
+  here:
+    - a: apple is included
+      b: cool.
+```
+
+## Replace node with referenced file as string
+This will work for any text based file
+
+Given a sample.yml file of:
+```yaml
+something:
+  file: thing.yml
+```
+then
+```bash
+yq eval '.something |= strload("../../examples/" + .file)' sample.yml
+```
+will output
+```yaml
+something: |-
+  a: apple is included
+  b: cool.
+```
+

pkg/yqlib/expression_tokeniser.go

@@ -336,6 +336,8 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`from_json`), opToken(decodeOpType))
 
 	lexer.Add([]byte(`sortKeys`), opToken(sortKeysOpType))
+	lexer.Add([]byte(`load`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false}))
+	lexer.Add([]byte(`strload`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: true}))
 	lexer.Add([]byte(`select`), opToken(selectOpType))
 	lexer.Add([]byte(`has`), opToken(hasOpType))
 	lexer.Add([]byte(`unique`), opToken(uniqueOpType))

pkg/yqlib/lib.go

@@ -95,6 +95,8 @@ var captureOpType = &operationType{Type: "CAPTURE", NumArgs: 1, Precedence: 50,
 var testOpType = &operationType{Type: "TEST", NumArgs: 1, Precedence: 50, Handler: testOperator}
 var splitStringOpType = &operationType{Type: "SPLIT", NumArgs: 1, Precedence: 50, Handler: splitStringOperator}
 
+var loadOpType = &operationType{Type: "LOAD", NumArgs: 1, Precedence: 50, Handler: loadYamlOperator}
+
 var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 50, Handler: keysOperator}
 
 var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator}

pkg/yqlib/operator_comments.go

@@ -1,8 +1,12 @@
 package yqlib
 
 import (
+	// "bufio"
+	// "bytes"
+	"bufio"
+	"bytes"
 	"container/list"
-	"strings"
+	"regexp"
 
 	yaml "gopkg.in/yaml.v3"
 )
@@ -57,6 +61,7 @@ func assignCommentsOperator(d *dataTreeNavigator, context Context, expressionNod
 		}
 		if preferences.HeadComment {
 			candidate.Node.HeadComment = comment
+			candidate.LeadingContent = "" // clobber the leading content, if there was any.
 		}
 		if preferences.FootComment {
 			candidate.Node.FootComment = comment
@@ -68,6 +73,9 @@ func assignCommentsOperator(d *dataTreeNavigator, context Context, expressionNod
 
 func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	preferences := expressionNode.Operation.Preferences.(commentOpPreferences)
+	var startCommentCharaterRegExp = regexp.MustCompile(`^# `)
+	var subsequentCommentCharaterRegExp = regexp.MustCompile(`\n# `)
+
 	log.Debugf("GetComments operator!")
 	var results = list.New()
 
@@ -76,12 +84,25 @@ func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *
 		comment := ""
 		if preferences.LineComment {
 			comment = candidate.Node.LineComment
+		} else if preferences.HeadComment && candidate.LeadingContent != "" {
+			var chompRegexp = regexp.MustCompile(`\n$`)
+			var output bytes.Buffer
+			var writer = bufio.NewWriter(&output)
+			if err := processLeadingContent(candidate, writer, false, YamlOutputFormat); err != nil {
+				return Context{}, err
+			}
+			if err := writer.Flush(); err != nil {
+				return Context{}, err
+			}
+			comment = output.String()
+			comment = chompRegexp.ReplaceAllString(comment, "")
 		} else if preferences.HeadComment {
 			comment = candidate.Node.HeadComment
 		} else if preferences.FootComment {
 			comment = candidate.Node.FootComment
 		}
-		comment = strings.Replace(comment, "# ", "", 1)
+		comment = startCommentCharaterRegExp.ReplaceAllString(comment, "")
+		comment = subsequentCommentCharaterRegExp.ReplaceAllString(comment, "\n")
 
 		node := &yaml.Node{Kind: yaml.ScalarNode, Value: comment, Tag: "!!str"}
 		result := candidate.CreateChild(nil, node)

pkg/yqlib/operator_comments_test.go

@@ -112,13 +112,22 @@ var commentOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!str)::welcome!\n",
 		},
 	},
+	{
+		skipDoc:               false,
+		dontFormatInputForDoc: true,
+		document:              "# welcome!\n---\n# bob\na: cat # meow\n\n# have a great day",
+		expression:            `headComment`,
+		expected: []string{
+			"D0, P[], (!!str)::welcome!\nbob\n",
+		},
+	},
 	{
 		description:           "Get foot comment",
 		dontFormatInputForDoc: true,
-		document:              "# welcome!\n\na: cat # meow\n\n# have a great day",
+		document:              "# welcome!\n\na: cat # meow\n\n# have a great day\n# no really",
 		expression:            `. | footComment`,
 		expected: []string{
-			"D0, P[], (!!str)::have a great day\n",
+			"D0, P[], (!!str)::have a great day\nno really\n",
 		},
 	},
 }

pkg/yqlib/operator_equals_test.go

@@ -96,7 +96,7 @@ var equalsOperatorScenarios = []expressionScenario{
 		document:   "{a: { b: {things: \"\"}, f: [1], g: [] }}",
 		expression: ".. | select(. == \"\")",
 		expected: []string{
-			"D0, P[a b things], (!!str)::\"\"\n",
+			"D0, P[a b things], (!!str)::\n",
 		},
 	},
 	{

pkg/yqlib/operator_load.go

@@ -0,0 +1,97 @@
+package yqlib
+
+import (
+	"bufio"
+	"container/list"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"gopkg.in/yaml.v3"
+)
+
+type loadPrefs struct {
+	loadAsString bool
+}
+
+func loadString(filename string) (*CandidateNode, error) {
+	// ignore CWE-22 gosec issue - that's more targetted for http based apps that run in a public directory,
+	// and ensuring that it's not possible to give a path to a file outside that directory.
+
+	filebytes, err := ioutil.ReadFile(filename) // #nosec
+	if err != nil {
+		return nil, err
+	}
+
+	return &CandidateNode{Node: &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: string(filebytes)}}, nil
+}
+
+func loadYaml(filename string) (*CandidateNode, error) {
+
+	file, err := os.Open(filename) // #nosec
+	if err != nil {
+		return nil, err
+	}
+	reader := bufio.NewReader(file)
+
+	documents, err := readDocuments(reader, filename, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	if documents.Len() == 0 {
+		// return null candidate
+		return &CandidateNode{Node: &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!null"}}, nil
+	} else if documents.Len() == 1 {
+		return documents.Front().Value.(*CandidateNode), nil
+
+	} else {
+		sequenceNode := &CandidateNode{Node: &yaml.Node{Kind: yaml.SequenceNode}}
+		for doc := documents.Front(); doc != nil; doc = doc.Next() {
+			sequenceNode.Node.Content = append(sequenceNode.Node.Content, doc.Value.(*CandidateNode).Node)
+		}
+		return sequenceNode, nil
+	}
+}
+
+func loadYamlOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	log.Debugf("loadYamlOperator")
+
+	loadPrefs := expressionNode.Operation.Preferences.(loadPrefs)
+
+	// need to evaluate the 1st parameter against the context
+	// and return the data accordingly.
+
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+
+		rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
+		if err != nil {
+			return Context{}, err
+		}
+		if rhs.MatchingNodes.Front() == nil {
+			return Context{}, fmt.Errorf("Filename expression returned nil")
+		}
+		nameCandidateNode := rhs.MatchingNodes.Front().Value.(*CandidateNode)
+
+		filename := nameCandidateNode.Node.Value
+
+		var contentsCandidate *CandidateNode
+
+		if loadPrefs.loadAsString {
+			contentsCandidate, err = loadString(filename)
+		} else {
+			contentsCandidate, err = loadYaml(filename)
+		}
+		if err != nil {
+			return Context{}, fmt.Errorf("Failed to load %v: %w", filename, err)
+		}
+
+		results.PushBack(contentsCandidate)
+
+	}
+
+	return context.ChildContext(results), nil
+}

pkg/yqlib/operator_load_test.go

@@ -0,0 +1,50 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var loadScenarios = []expressionScenario{
+	{
+		description: "Simple example",
+		document:    `{myFile: "../../examples/thing.yml"}`,
+		expression:  `load(.myFile)`,
+		expected: []string{
+			"D0, P[], (doc)::a: apple is included\nb: cool.\n",
+		},
+	},
+	{
+		description:    "Replace node with referenced file",
+		subdescription: "Note that you can modify the filename in the load operator if needed.",
+		document:       `{something: {file: "thing.yml"}}`,
+		expression:     `.something |= load("../../examples/" + .file)`,
+		expected: []string{
+			"D0, P[], (doc)::{something: {a: apple is included, b: cool.}}\n",
+		},
+	},
+	{
+		description:    "Replace _all_ nodes with referenced file",
+		subdescription: "Recursively match all the nodes (`..`) and then filter the ones that have a 'file' attribute. ",
+		document:       `{something: {file: "thing.yml"}, over: {here: [{file: "thing.yml"}]}}`,
+		expression:     `(.. | select(has("file"))) |= load("../../examples/" + .file)`,
+		expected: []string{
+			"D0, P[], (!!map)::{something: {a: apple is included, b: cool.}, over: {here: [{a: apple is included, b: cool.}]}}\n",
+		},
+	},
+	{
+		description:    "Replace node with referenced file as string",
+		subdescription: "This will work for any text based file",
+		document:       `{something: {file: "thing.yml"}}`,
+		expression:     `.something |= strload("../../examples/" + .file)`,
+		expected: []string{
+			"D0, P[], (doc)::{something: \"a: apple is included\\nb: cool.\"}\n",
+		},
+	},
+}
+
+func TestLoadScenarios(t *testing.T) {
+	for _, tt := range loadScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "load", loadScenarios)
+}

pkg/yqlib/operator_multiply.go

@@ -21,29 +21,25 @@ func multiplyOperator(d *dataTreeNavigator, context Context, expressionNode *Exp
 	return crossFunction(d, context, expressionNode, multiply(expressionNode.Operation.Preferences.(multiplyPreferences)), false)
 }
 
-func getNewBlankNode(lhs *yaml.Node, rhs *yaml.Node) *yaml.Node {
-
-	blankNode := &yaml.Node{}
-
-	if lhs.HeadComment != "" {
-		blankNode.HeadComment = lhs.HeadComment
-	} else if rhs.HeadComment != "" {
-		blankNode.HeadComment = rhs.HeadComment
+func getComments(lhs *CandidateNode, rhs *CandidateNode) (leadingContent string, headComment string, footComment string) {
+	leadingContent = rhs.LeadingContent
+	headComment = rhs.Node.HeadComment
+	footComment = rhs.Node.FootComment
+	if lhs.Node.HeadComment != "" || lhs.LeadingContent != "" {
+		headComment = lhs.Node.HeadComment
+		leadingContent = lhs.LeadingContent
 	}
 
-	if lhs.FootComment != "" {
-		blankNode.FootComment = lhs.FootComment
-	} else if rhs.FootComment != "" {
-		blankNode.FootComment = rhs.FootComment
+	if lhs.Node.FootComment != "" {
+		footComment = lhs.Node.FootComment
 	}
-
-	return blankNode
+	return leadingContent, headComment, footComment
 }
 
 func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 		// need to do this before unWrapping the potential document node
-		newBlankNode := getNewBlankNode(lhs.Node, rhs.Node)
+		leadingContent, headComment, footComment := getComments(lhs, rhs)
 		lhs.Node = unwrapDoc(lhs.Node)
 		rhs.Node = unwrapDoc(rhs.Node)
 		log.Debugf("Multipling LHS: %v", lhs.Node.Tag)
@@ -56,15 +52,10 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
 			if err != nil {
 				return nil, err
 			}
-			newBlank.Node.HeadComment = newBlankNode.HeadComment
-			newBlank.Node.FootComment = newBlankNode.FootComment
+			newBlank.LeadingContent = leadingContent
+			newBlank.Node.HeadComment = headComment
+			newBlank.Node.FootComment = footComment
 
-			// var newBlank = lhs.CreateChild(nil, newBlankNode)
-			// log.Debugf("merge - merge lhs into blank")
-			// var newThing, err = mergeObjects(d, context.WritableClone(), newBlank, lhs, multiplyPreferences{})
-			// if err != nil {
-			// 	return nil, err
-			// }
 			return mergeObjects(d, context.WritableClone(), &newBlank, rhs, preferences)
 		} else if lhs.Node.Tag == "!!int" && rhs.Node.Tag == "!!int" {
 			return multiplyIntegers(lhs, rhs)

pkg/yqlib/operator_multiply_test.go

@@ -35,33 +35,28 @@ We then need to update the first array. We will use the relative update (|=) bec
 We set the current element of the first array as $cur. Now we multiply (merge) $cur with the matching entry in $two, by passing $two through a select filter.
 `
 
-var docWithHeader = `
-# here
+var docWithHeader = `# here
 
 a: apple
 `
 
-var nodeWithHeader = `
-# here
-a: apple
+var nodeWithHeader = `node:
+  # here
+  a: apple
 `
 
-var docNoComments = `
-b: banana
+var docNoComments = `b: banana
 `
 
-var docWithFooter = `
-a: apple
+var docWithFooter = `a: apple
 
 # footer
 `
 
-var nodeWithFooter = `
-a: apple
+var nodeWithFooter = `a: apple
 # footer`
 
-var document = `
-a: &cat {name: cat}
+var document = `a: &cat {name: cat}
 b: {name: dog}
 c: 
   <<: *cat
@@ -89,9 +84,9 @@ var multiplyOperatorScenarios = []expressionScenario{
 		skipDoc:    true,
 		document:   nodeWithHeader,
 		document2:  docNoComments,
-		expression: `select(fi == 0) * select(fi == 1)`,
+		expression: `(select(fi == 0) | .node) * select(fi == 1)`,
 		expected: []string{
-			"D0, P[], (!!map)::# here\na: apple\nb: banana\n",
+			"D0, P[node], (!!map)::# here\na: apple\nb: banana\n",
 		},
 	},
 	{
@@ -107,7 +102,7 @@ var multiplyOperatorScenarios = []expressionScenario{
 		skipDoc:    true,
 		document:   docNoComments,
 		document2:  nodeWithHeader,
-		expression: `select(fi == 0) * select(fi == 1)`,
+		expression: `select(fi == 0) * (select(fi == 1) | .node)`,
 		expected: []string{
 			"D0, P[], (!!map)::b: banana\n# here\na: apple\n",
 		},

pkg/yqlib/operator_recursive_descent_test.go

@@ -219,7 +219,7 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
 			"D0, P[foobarList], (!!map)::b: foobarList_b\n!!merge <<: [*foo, *bar]\nc: foobarList_c\n",
 			"D0, P[foobarList b], (!!str)::b\n",
 			"D0, P[foobarList b], (!!str)::foobarList_b\n",
-			"D0, P[foobarList <<], (!!merge)::!!merge <<\n",
+			"D0, P[foobarList <<], (!!merge)::<<\n",
 			"D0, P[foobarList <<], (!!seq)::[*foo, *bar]\n",
 			"D0, P[foobarList << 0], (alias)::*foo\n",
 			"D0, P[foobarList << 1], (alias)::*bar\n",

pkg/yqlib/operator_style_test.go

@@ -142,8 +142,8 @@ e: >-
 		document:   `a: cat`,
 		expression: `.. | style`,
 		expected: []string{
-			"D0, P[], (!!str)::\"\"\n",
-			"D0, P[a], (!!str)::\"\"\n",
+			"D0, P[], (!!str)::\n",
+			"D0, P[a], (!!str)::\n",
 		},
 	},
 }

pkg/yqlib/operator_tag_test.go

@@ -10,12 +10,12 @@ var tagOperatorScenarios = []expressionScenario{
 		document:    `{a: cat, b: 5, c: 3.2, e: true, f: []}`,
 		expression:  `.. | tag`,
 		expected: []string{
-			"D0, P[], (!!str)::'!!map'\n",
-			"D0, P[a], (!!str)::'!!str'\n",
-			"D0, P[b], (!!str)::'!!int'\n",
-			"D0, P[c], (!!str)::'!!float'\n",
-			"D0, P[e], (!!str)::'!!bool'\n",
-			"D0, P[f], (!!str)::'!!seq'\n",
+			"D0, P[], (!!str)::!!map\n",
+			"D0, P[a], (!!str)::!!str\n",
+			"D0, P[b], (!!str)::!!int\n",
+			"D0, P[c], (!!str)::!!float\n",
+			"D0, P[e], (!!str)::!!bool\n",
+			"D0, P[f], (!!str)::!!seq\n",
 		},
 	},
 	{
@@ -23,7 +23,7 @@ var tagOperatorScenarios = []expressionScenario{
 		document:   `{a: cat, b: 5, c: 3.2, e: true, f: []}`,
 		expression: `tag`,
 		expected: []string{
-			"D0, P[], (!!str)::'!!map'\n",
+			"D0, P[], (!!str)::!!map\n",
 		},
 	},
 	{

pkg/yqlib/operator_value_test.go

@@ -125,13 +125,13 @@ var valueOperatorScenarios = []expressionScenario{
 		document:   ``,
 		expression: `"1.3"`,
 		expected: []string{
-			"D0, P[], (!!str)::\"1.3\"\n",
+			"D0, P[], (!!str)::1.3\n",
 		},
 	}, {
 		document:   ``,
 		expression: `"true"`,
 		expected: []string{
-			"D0, P[], (!!str)::\"true\"\n",
+			"D0, P[], (!!str)::true\n",
 		},
 	}, {
 		document:   ``,

pkg/yqlib/operators_test.go

@@ -26,6 +26,20 @@ type expressionScenario struct {
 	dontFormatInputForDoc bool // dont format input doc for documentation generation
 }
 
+func readDocumentWithLeadingContent(content string, fakefilename string, fakeFileIndex int) (*list.List, error) {
+	reader, firstFileLeadingContent, err := processReadStream(bufio.NewReader(strings.NewReader(content)))
+	if err != nil {
+		return nil, err
+	}
+
+	inputs, err := readDocuments(reader, fakefilename, fakeFileIndex)
+	if err != nil {
+		return nil, err
+	}
+	inputs.Front().Value.(*CandidateNode).LeadingContent = firstFileLeadingContent
+	return inputs, nil
+}
+
 func testScenario(t *testing.T, s *expressionScenario) {
 	var err error
 
@@ -37,15 +51,17 @@ func testScenario(t *testing.T, s *expressionScenario) {
 	inputs := list.New()
 
 	if s.document != "" {
-		inputs, err = readDocuments(strings.NewReader(s.document), "sample.yml", 0)
+		inputs, err = readDocumentWithLeadingContent(s.document, "sample.yml", 0)
+
 		if err != nil {
 			t.Error(err, s.document, s.expression)
 			return
 		}
+
 		if s.document2 != "" {
-			moreInputs, err := readDocuments(strings.NewReader(s.document2), "another.yml", 1)
+			moreInputs, err := readDocumentWithLeadingContent(s.document2, "another.yml", 1)
 			if err != nil {
-				t.Error(err, s.document, s.expression)
+				t.Error(err, s.document2, s.expression)
 				return
 			}
 			inputs.PushBackList(moreInputs)
@@ -71,14 +87,31 @@ func testScenario(t *testing.T, s *expressionScenario) {
 		t.Error(fmt.Errorf("%v: %v", err, s.expression))
 		return
 	}
-	test.AssertResultComplexWithContext(t, s.expected, resultsToString(context.MatchingNodes), fmt.Sprintf("desc: %v\nexp: %v\ndoc: %v", s.description, s.expression, s.document))
+	test.AssertResultComplexWithContext(t, s.expected, resultsToString(t, context.MatchingNodes), fmt.Sprintf("desc: %v\nexp: %v\ndoc: %v", s.description, s.expression, s.document))
 }
 
-func resultsToString(results *list.List) []string {
+func resultsToString(t *testing.T, results *list.List) []string {
 	var pretty []string = make([]string, 0)
+
 	for el := results.Front(); el != nil; el = el.Next() {
 		n := el.Value.(*CandidateNode)
-		pretty = append(pretty, NodeToString(n))
+		var valueBuffer bytes.Buffer
+		printer := NewPrinterWithSingleWriter(bufio.NewWriter(&valueBuffer), YamlOutputFormat, true, false, 4, true)
+
+		err := printer.PrintResults(n.AsList())
+		if err != nil {
+			t.Error(err)
+			return nil
+		}
+
+		tag := n.Node.Tag
+		if n.Node.Kind == yaml.DocumentNode {
+			tag = "doc"
+		} else if n.Node.Kind == yaml.AliasNode {
+			tag = "alias"
+		}
+		output := fmt.Sprintf(`D%v, P%v, (%v)::%v`, n.Document, n.Path, tag, valueBuffer.String())
+		pretty = append(pretty, output)
 	}
 	return pretty
 }
@@ -227,13 +260,14 @@ func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formatt
 	inputs := list.New()
 
 	if s.document != "" {
-		inputs, err = readDocuments(strings.NewReader(formattedDoc), "sample.yml", 0)
+
+		inputs, err = readDocumentWithLeadingContent(formattedDoc, "sample.yml", 0)
 		if err != nil {
 			t.Error(err, s.document, s.expression)
 			return
 		}
 		if s.document2 != "" {
-			moreInputs, err := readDocuments(strings.NewReader(formattedDoc2), "another.yml", 1)
+			moreInputs, err := readDocumentWithLeadingContent(formattedDoc2, "another.yml", 1)
 			if err != nil {
 				t.Error(err, s.document, s.expression)
 				return

pkg/yqlib/printer.go

@@ -5,7 +5,7 @@ import (
 	"container/list"
 	"fmt"
 	"io"
-	"strings"
+	"regexp"
 
 	yaml "gopkg.in/yaml.v3"
 )
@@ -84,7 +84,7 @@ func (p *resultsPrinter) printNode(node *yaml.Node, writer io.Writer) error {
 
 	var encoder Encoder
 	if node.Kind == yaml.ScalarNode && p.unwrapScalar && p.outputFormat == YamlOutputFormat {
-		return p.writeString(writer, node.Value+"\n")
+		return writeString(writer, node.Value+"\n")
 	}
 
 	if p.outputFormat == JsonOutputFormat {
@@ -97,54 +97,6 @@ func (p *resultsPrinter) printNode(node *yaml.Node, writer io.Writer) error {
 	return encoder.Encode(node)
 }
 
-func (p *resultsPrinter) writeString(writer io.Writer, txt string) error {
-	_, errorWriting := writer.Write([]byte(txt))
-	return errorWriting
-}
-
-func (p *resultsPrinter) processLeadingContent(mappedDoc *CandidateNode, writer io.Writer) error {
-	if strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$") {
-		log.Debug("headcommentwas %v", mappedDoc.Node.HeadComment)
-		log.Debug("finished headcomment")
-		reader := bufio.NewReader(strings.NewReader(mappedDoc.Node.HeadComment))
-		mappedDoc.Node.HeadComment = ""
-
-		for {
-
-			readline, errReading := reader.ReadString('\n')
-			if errReading != nil && errReading != io.EOF {
-				return errReading
-			}
-			if strings.Contains(readline, "$yqLeadingContent$") {
-				// skip this
-
-			} else if strings.Contains(readline, "$yqDocSeperator$") {
-				if p.printDocSeparators {
-					if err := p.writeString(writer, "---\n"); err != nil {
-						return err
-					}
-				}
-			} else if p.outputFormat == YamlOutputFormat {
-				if err := p.writeString(writer, readline); err != nil {
-					return err
-				}
-			}
-
-			if errReading == io.EOF {
-				if readline != "" {
-					// the last comment we read didn't have a new line, put one in
-					if err := p.writeString(writer, "\n"); err != nil {
-						return err
-					}
-				}
-				break
-			}
-		}
-
-	}
-	return nil
-}
-
 func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 	log.Debug("PrintResults for %v matches", matchingNodes.Len())
 
@@ -180,16 +132,17 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 			return errorWriting
 		}
 
-		commentStartsWithSeparator := strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$\n$yqDocSeperator$")
+		commentsStartWithSepExp := regexp.MustCompile(`^\$yqDocSeperator\$`)
+		commentStartsWithSeparator := commentsStartWithSepExp.MatchString(mappedDoc.LeadingContent)
 
 		if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators && !commentStartsWithSeparator {
 			log.Debug("-- writing doc sep")
-			if err := p.writeString(writer, "---\n"); err != nil {
+			if err := writeString(writer, "---\n"); err != nil {
 				return err
 			}
 		}
 
-		if err := p.processLeadingContent(mappedDoc, writer); err != nil {
+		if err := processLeadingContent(mappedDoc, writer, p.printDocSeparators, p.outputFormat); err != nil {
 			return err
 		}
 

pkg/yqlib/printer_test.go

@@ -82,15 +82,15 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
 	}
 
 	el := inputs.Front()
-	el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n"
+	el.Value.(*CandidateNode).LeadingContent = "# go cats\n$yqDocSeperator$\n"
 	sample1 := nodeToList(el.Value.(*CandidateNode))
 
 	el = el.Next()
-	el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n"
+	el.Value.(*CandidateNode).LeadingContent = "$yqDocSeperator$\n"
 	sample2 := nodeToList(el.Value.(*CandidateNode))
 
 	el = el.Next()
-	el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n# cool\n"
+	el.Value.(*CandidateNode).LeadingContent = "$yqDocSeperator$\n# cool\n"
 	sample3 := nodeToList(el.Value.(*CandidateNode))
 
 	err = printer.PrintResults(sample1)
@@ -174,21 +174,21 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
 	elNode := el.Value.(*CandidateNode)
 	elNode.Document = 0
 	elNode.FileIndex = 0
-	elNode.Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n"
+	elNode.LeadingContent = "# go cats\n$yqDocSeperator$\n"
 	sample1 := nodeToList(elNode)
 
 	el = el.Next()
 	elNode = el.Value.(*CandidateNode)
 	elNode.Document = 0
 	elNode.FileIndex = 1
-	elNode.Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n"
+	elNode.LeadingContent = "$yqDocSeperator$\n"
 	sample2 := nodeToList(elNode)
 
 	el = el.Next()
 	elNode = el.Value.(*CandidateNode)
 	elNode.Document = 0
 	elNode.FileIndex = 2
-	elNode.Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n# cool\n"
+	elNode.LeadingContent = "$yqDocSeperator$\n# cool\n"
 	sample3 := nodeToList(elNode)
 
 	err = printer.PrintResults(sample1)
@@ -239,7 +239,7 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) {
 		panic(err)
 	}
 
-	inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n"
+	inputs.Front().Value.(*CandidateNode).LeadingContent = "# go cats\n$yqDocSeperator$\n"
 
 	err = printer.PrintResults(inputs)
 	if err != nil {
@@ -267,7 +267,7 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) {
 	if err != nil {
 		panic(err)
 	}
-	inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n"
+	inputs.Front().Value.(*CandidateNode).LeadingContent = "$yqDocSeperator$\n"
 	err = printer.PrintResults(inputs)
 	if err != nil {
 		panic(err)
@@ -321,7 +321,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
 		panic(err)
 	}
 
-	inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# ignore this\n"
+	inputs.Front().Value.(*CandidateNode).LeadingContent = "# ignore this\n"
 
 	err = printer.PrintResults(inputs)
 	if err != nil {

pkg/yqlib/stream_evaluator.go

@@ -33,10 +33,11 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadin
 		return err
 	}
 	candidateNode := &CandidateNode{
-		Document:  0,
-		Filename:  "",
-		Node:      &yaml.Node{Kind: yaml.DocumentNode, HeadComment: leadingContent, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
-		FileIndex: 0,
+		Document:       0,
+		Filename:       "",
+		Node:           &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
+		FileIndex:      0,
+		LeadingContent: leadingContent,
 	}
 	inputList := list.New()
 	inputList.PushBack(candidateNode)
@@ -100,15 +101,16 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr
 		} else if errorReading != nil {
 			return currentIndex, errorReading
 		}
-		if currentIndex == 0 {
-			dataBucket.HeadComment = leadingContent
-		}
+
 		candidateNode := &CandidateNode{
 			Document:  currentIndex,
 			Filename:  filename,
 			Node:      &dataBucket,
 			FileIndex: s.fileIndex,
 		}
+		if currentIndex == 0 {
+			candidateNode.LeadingContent = leadingContent
+		}
 		inputList := list.New()
 		inputList.PushBack(candidateNode)
 

pkg/yqlib/utils.go

@@ -31,10 +31,51 @@ func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, s
 	return processReadStream(reader)
 }
 
+func writeString(writer io.Writer, txt string) error {
+	_, errorWriting := writer.Write([]byte(txt))
+	return errorWriting
+}
+
+func processLeadingContent(mappedDoc *CandidateNode, writer io.Writer, printDocSeparators bool, outputFormat PrinterOutputFormat) error {
+	log.Debug("headcommentwas %v", mappedDoc.LeadingContent)
+	log.Debug("finished headcomment")
+	reader := bufio.NewReader(strings.NewReader(mappedDoc.LeadingContent))
+
+	for {
+
+		readline, errReading := reader.ReadString('\n')
+		if errReading != nil && errReading != io.EOF {
+			return errReading
+		}
+		if strings.Contains(readline, "$yqDocSeperator$") {
+			if printDocSeparators {
+				if err := writeString(writer, "---\n"); err != nil {
+					return err
+				}
+			}
+		} else if outputFormat == YamlOutputFormat {
+			if err := writeString(writer, readline); err != nil {
+				return err
+			}
+		}
+
+		if errReading == io.EOF {
+			if readline != "" {
+				// the last comment we read didn't have a new line, put one in
+				if err := writeString(writer, "\n"); err != nil {
+					return err
+				}
+			}
+			break
+		}
+	}
+
+	return nil
+}
+
 func processReadStream(reader *bufio.Reader) (io.Reader, string, error) {
 	var commentLineRegEx = regexp.MustCompile(`^\s*#`)
 	var sb strings.Builder
-	sb.WriteString("$yqLeadingContent$\n")
 	for {
 		peekBytes, err := reader.Peek(3)
 		if err == io.EOF {

release_instructions.txt

@@ -1,9 +1,9 @@
 - update release notes
+- check dependabot
 - increment version in version.go
 - increment version in snapcraft.yaml
 - increment version in github-action/Dockerfile
 - make sure local build passes
-- run ./scripts/secure.sh (manual because docker platforms)
 - run ./scripts/copy-docs.sh (and commit the changed in the yq-book branch)
 - commit version update changes
 - tag git with same version number

release_notes.txt

@@ -1,3 +1,7 @@
+4.14.2:
+  - Fixed header preprocessing issue (#1000)
+  - Bumped version dependencies
+
 4.14.1:
   - Added group_by operator
   - Added encode/decode operators (toyaml, fromjson etc) (#974)

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.14.1'
+version: '4.14.2'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.