eval-reduce wip

acceptance_tests/empty.sh

@@ -8,7 +8,27 @@ EOL
 
 testEmptyEval() {
   X=$(./yq e test.yml)
+  expected=$(cat test.yml)
   assertEquals 0 $?
+  assertEquals "$expected" "$X"
+}
+
+testEmptyEvalNoNewLine() {
+  echo -n "#comment" >test.yml
+  X=$(./yq e test.yml)
+  expected=$(cat test.yml)
+  assertEquals 0 $?
+  assertEquals "$expected" "$X"
+}
+
+testEmptyEvalNoNewLineWithExpression() {
+  echo -n "# comment" >test.yml
+  X=$(./yq e '.apple = "tree"' test.yml)
+  read -r -d '' expected << EOM
+# comment
+apple: tree
+EOM
+  assertEquals "$expected" "$X"
 }
 
 testEmptyEvalPipe() {

acceptance_tests/leading-seperator.sh

@@ -36,6 +36,73 @@ testLeadingSeperatorPipeIntoEvalSeq() {
   assertEquals "$expected" "$X"
 }
 
+testLeadingSeperatorExtractField() {
+  X=$(./yq e '.a' - < test.yml)
+  assertEquals "test" "$X"
+}
+
+testLeadingSeperatorExtractFieldWithCommentsAfterSep() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+EOL
+  X=$(./yq e '.a' test.yml)
+  assertEquals "test" "$X"
+}
+
+testLeadingSeperatorExtractFieldWithCommentsBeforeSep() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+---
+a: test
+EOL
+  X=$(./yq e '.a' test.yml)
+  assertEquals "test" "$X"
+}
+
+
+testLeadingSeperatorExtractFieldMultiDoc() {
+  cat >test.yml <<EOL
+---
+a: test
+---
+a: test2
+EOL
+
+  read -r -d '' expected << EOM
+test
+---
+test2
+EOM
+  X=$(./yq e '.a' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorExtractFieldMultiDocWithComments() {
+  cat >test.yml <<EOL
+# here
+---
+# there
+a: test
+# whereever
+---
+# you are
+a: test2
+# woop
+EOL
+
+  read -r -d '' expected << EOM
+test
+---
+test2
+EOM
+  X=$(./yq e '.a' test.yml)
+  assertEquals "$expected" "$X"
+}
+
 
 testLeadingSeperatorEvalSeq() {
   X=$(./yq e test.yml)

cmd/evaluate_all_command.go

@@ -34,7 +34,8 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.
 ## Evaluate All ##
 This command loads _all_ yaml documents of _all_ yaml files and runs expression once
 Useful when you need to run an expression across several yaml documents or files (like merge).
-Note that it consumes more memory than eval.
+If you're just merging entire multiple files together, you may want to consider eval-reduce as it's faster.
+Note that it consumes more memory than eval and eval-reduce.
 `,
 		RunE: evaluateAll,
 	}
@@ -98,10 +99,7 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 		args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename()
 
 		if frontMatter == "process" {
-			reader, err := os.Open(frontMatterHandler.GetContentFilename()) // #nosec
-			if err != nil {
-				return err
-			}
+			reader := frontMatterHandler.GetContentReader()
 			printer.SetAppendix(reader)
 			defer yqlib.SafelyCloseReader(reader)
 		}

cmd/evaluate_reduce_command.go

@@ -0,0 +1,112 @@
+package cmd
+
+import (
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/mikefarah/yq/v4/pkg/yqlib"
+	"github.com/spf13/cobra"
+)
+
+func createEvaluateReduceCommand() *cobra.Command {
+	var cmdEvalReduce = &cobra.Command{
+		Use:     "eval-reduce [reduce expression] [yaml_file1]...",
+		Aliases: []string{"er"},
+		Short:   "Runs a reduce expression sequentially against each document of each file given. More memory efficient than using eval-all if you can get away with it.",
+		Example: `
+# Merge f2.yml into f1.yml (inplace)
+yq eval-reduce --inplace '{} ; . * $doc' f1.yml f2.yml
+`,
+		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
+See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.
+
+## Evaluate Reduce ##
+This command runs the reduce expression against each document of each file given, accumulating the results.
+It is most useful when merging multiple files (but isn't as flexible as using eval-all with ireduce, as you
+can only merge the top level nodes).
+`,
+		RunE: evaluateReduce,
+	}
+	return cmdEvalReduce
+}
+func evaluateReduce(cmd *cobra.Command, args []string) error {
+	cmd.SilenceUsage = true
+	// 2+ args, [0] = expression, file the rest
+
+	var err error
+
+	out := cmd.OutOrStdout()
+
+	fileInfo, _ := os.Stdout.Stat()
+
+	if forceColor || (!forceNoColor && (fileInfo.Mode()&os.ModeCharDevice) != 0) {
+		colorsEnabled = true
+	}
+
+	firstFileIndex := -1
+	if !nullInput && len(args) == 1 {
+		firstFileIndex = 0
+	} else if len(args) > 1 {
+		firstFileIndex = 1
+	}
+
+	if writeInplace && (firstFileIndex == -1) {
+		return fmt.Errorf("Write inplace flag only applicable when giving an expression and at least one file")
+	}
+
+	if writeInplace {
+		// only use colors if its forced
+		colorsEnabled = forceColor
+		writeInPlaceHandler := yqlib.NewWriteInPlaceHandler(args[firstFileIndex])
+		out, err = writeInPlaceHandler.CreateTempFile()
+		if err != nil {
+			return err
+		}
+		// need to indirectly call the function so  that completedSuccessfully is
+		// passed when we finish execution as opposed to now
+		defer func() { writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully) }()
+	}
+
+	if nullInput && len(args) > 1 {
+		return errors.New("Cannot pass files in when using null-input flag")
+	}
+
+	printer := yqlib.NewPrinter(out, outputToJSON, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
+
+	if frontMatter != "" {
+		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])
+		err = frontMatterHandler.Split()
+		if err != nil {
+			return err
+		}
+		args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename()
+
+		if frontMatter == "process" {
+			reader := frontMatterHandler.GetContentReader()
+			printer.SetAppendix(reader)
+			defer yqlib.SafelyCloseReader(reader)
+		}
+		defer frontMatterHandler.CleanUp()
+	}
+
+	reduceEvaluator := yqlib.NewReduceEvaluator()
+	switch len(args) {
+	case 0:
+		cmd.Println(cmd.UsageString())
+		return nil
+	case 1:
+		cmd.Println(cmd.UsageString())
+		return nil
+	default:
+		err = reduceEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer, leadingContentPreProcessing)
+	}
+
+	completedSuccessfully = err == nil
+
+	if err == nil && exitStatus && !printer.PrintedAnything() {
+		return errors.New("no matches found")
+	}
+
+	return err
+}

cmd/evalute_sequence_command.go

@@ -112,10 +112,7 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 		args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename()
 
 		if frontMatter == "process" {
-			reader, err := os.Open(frontMatterHandler.GetContentFilename()) // #nosec
-			if err != nil {
-				return err
-			}
+			reader := frontMatterHandler.GetContentReader()
 			printer.SetAppendix(reader)
 			defer yqlib.SafelyCloseReader(reader)
 		}

cmd/root.go

@@ -59,6 +59,7 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 	rootCmd.AddCommand(
 		createEvaluateSequenceCommand(),
 		createEvaluateAllCommand(),
+		createEvaluateReduceCommand(),
 		completionCmd,
 	)
 	return rootCmd

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.11.0"
+	Version = "4.11.1"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

examples/data1.yaml

@@ -1,4 +1,3 @@
+a: apple
 ---
-# hi peeps
-# cool
-a: test
+a2: fish

examples/data2.yaml

@@ -1,7 +1 @@
-a: other # better than the original
-b: [3, 4]
-c:
-  toast: leave
-  test: 1
-  tell: 1
-  tasty.taco: cool
+b: doc2

examples/front-matter.yaml

@@ -2,4 +2,5 @@
 a: apple
 b: bannana
 ---
-<h1>I like {{a}} and {{b}} </h1>
+hello there
+apples: great

github-action/Dockerfile

@@ -1,4 +1,4 @@
-FROM mikefarah/yq:4.11.0
+FROM mikefarah/yq:4.11.1
 
 COPY entrypoint.sh /entrypoint.sh
 

pkg/yqlib/context.go

@@ -30,6 +30,7 @@ func (n *Context) GetVariable(name string) *list.List {
 	if n.Variables == nil {
 		return nil
 	}
+	log.Debug("GetVariable - %v to %v", name, NodesToString(n.Variables[name]))
 	return n.Variables[name]
 }
 
@@ -37,6 +38,7 @@ func (n *Context) SetVariable(name string, value *list.List) {
 	if n.Variables == nil {
 		n.Variables = make(map[string]*list.List)
 	}
+	log.Debug("SetVariable - %v to %v", name, NodesToString(value))
 	n.Variables[name] = value
 }
 

pkg/yqlib/front_matter.go

@@ -9,47 +9,48 @@ import (
 type frontMatterHandler interface {
 	Split() error
 	GetYamlFrontMatterFilename() string
-	GetContentFilename() string
+	GetContentReader() io.Reader
 	CleanUp()
 }
 
 type frontMatterHandlerImpl struct {
 	originalFilename        string
 	yamlFrontMatterFilename string
-	contentFilename         string
+	contentReader           io.Reader
 }
 
 func NewFrontMatterHandler(originalFilename string) frontMatterHandler {
-	return &frontMatterHandlerImpl{originalFilename, "", ""}
+	return &frontMatterHandlerImpl{originalFilename, "", nil}
 }
 
 func (f *frontMatterHandlerImpl) GetYamlFrontMatterFilename() string {
 	return f.yamlFrontMatterFilename
 }
 
-func (f *frontMatterHandlerImpl) GetContentFilename() string {
-	return f.contentFilename
+func (f *frontMatterHandlerImpl) GetContentReader() io.Reader {
+	return f.contentReader
 }
 
 func (f *frontMatterHandlerImpl) CleanUp() {
 	tryRemoveFile(f.yamlFrontMatterFilename)
-	tryRemoveFile(f.contentFilename)
 }
 
 // Splits the given file by yaml front matter
 // yaml content will be saved to first temporary file
 // remaining content will be saved to second temporary file
 func (f *frontMatterHandlerImpl) Split() error {
-	var reader io.Reader
+	var reader *bufio.Reader
 	var err error
 	if f.originalFilename == "-" {
 		reader = bufio.NewReader(os.Stdin)
 	} else {
-		reader, err = os.Open(f.originalFilename) // #nosec
+		file, err := os.Open(f.originalFilename) // #nosec
 		if err != nil {
 			return err
 		}
+		reader = bufio.NewReader(file)
 	}
+	f.contentReader = reader
 
 	yamlTempFile, err := createTempFile()
 	if err != nil {
@@ -58,39 +59,35 @@ func (f *frontMatterHandlerImpl) Split() error {
 	f.yamlFrontMatterFilename = yamlTempFile.Name()
 	log.Debug("yamlTempFile: %v", yamlTempFile.Name())
 
-	contentTempFile, err := createTempFile()
-	if err != nil {
-		return err
-	}
-	f.contentFilename = contentTempFile.Name()
-	log.Debug("contentTempFile: %v", contentTempFile.Name())
-
-	scanner := bufio.NewScanner(reader)
-
 	lineCount := 0
-	yamlContentBlock := true
 
-	for scanner.Scan() {
-		line := scanner.Text()
-
-		if lineCount > 0 && line == "---" {
-			//we've finished reading the yaml content
-			yamlContentBlock = false
-		}
-		if yamlContentBlock {
-			_, err = yamlTempFile.Write([]byte(line + "\n"))
-		} else {
-			_, err = contentTempFile.Write([]byte(line + "\n"))
-		}
-		if err != nil {
+	for {
+		peekBytes, err := reader.Peek(3)
+		if err == io.EOF {
+			// we've finished reading the yaml content..I guess
+			break
+		} else if err != nil {
 			return err
 		}
+		if lineCount > 0 && string(peekBytes) == "---" {
+			// we've finished reading the yaml content..
+			break
+		}
+		line, errReading := reader.ReadString('\n')
 		lineCount = lineCount + 1
+		if errReading != nil && errReading != io.EOF {
+			return errReading
+		}
+
+		_, errWriting := yamlTempFile.Write([]byte(line))
+
+		if errWriting != nil {
+			return errWriting
+		}
 	}
 
 	safelyCloseFile(yamlTempFile)
-	safelyCloseFile(contentTempFile)
 
-	return scanner.Err()
+	return nil
 
 }

pkg/yqlib/front_matter_test.go

@@ -60,8 +60,11 @@ yaml: doc
 
 	test.AssertResult(t, expectedYamlFm, yamlFm)
 
-	content := readFile(fmHandler.GetContentFilename())
-	test.AssertResult(t, expectedContent, content)
+	contentBytes, err := ioutil.ReadAll(fmHandler.GetContentReader())
+	if err != nil {
+		panic(err)
+	}
+	test.AssertResult(t, expectedContent, string(contentBytes))
 
 	tryRemoveFile(file)
 	fmHandler.CleanUp()
@@ -94,8 +97,11 @@ yaml: doc
 
 	test.AssertResult(t, expectedYamlFm, yamlFm)
 
-	content := readFile(fmHandler.GetContentFilename())
-	test.AssertResult(t, expectedContent, content)
+	contentBytes, err := ioutil.ReadAll(fmHandler.GetContentReader())
+	if err != nil {
+		panic(err)
+	}
+	test.AssertResult(t, expectedContent, string(contentBytes))
 
 	tryRemoveFile(file)
 	fmHandler.CleanUp()
@@ -125,8 +131,11 @@ yaml: doc
 
 	test.AssertResult(t, expectedYamlFm, yamlFm)
 
-	content := readFile(fmHandler.GetContentFilename())
-	test.AssertResult(t, expectedContent, content)
+	contentBytes, err := ioutil.ReadAll(fmHandler.GetContentReader())
+	if err != nil {
+		panic(err)
+	}
+	test.AssertResult(t, expectedContent, string(contentBytes))
 
 	tryRemoveFile(file)
 	fmHandler.CleanUp()

pkg/yqlib/lib.go

@@ -112,7 +112,7 @@ type Operation struct {
 	OperationType *operationType
 	Value         interface{}
 	StringValue   string
-	CandidateNode *CandidateNode // used for Value Path elements
+	ValueNodes    *list.List // used for Value Path elements
 	Preferences   interface{}
 	UpdateAssign  bool // used for assign ops, when true it means we evaluate the rhs given the lhs
 }
@@ -138,12 +138,13 @@ func createScalarNode(value interface{}, stringValue string) *yaml.Node {
 
 func createValueOperation(value interface{}, stringValue string) *Operation {
 	var node *yaml.Node = createScalarNode(value, stringValue)
-
+	list := list.New()
+	list.PushBack(&CandidateNode{Node: node})
 	return &Operation{
 		OperationType: valueOpType,
 		Value:         value,
 		StringValue:   stringValue,
-		CandidateNode: &CandidateNode{Node: node},
+		ValueNodes:    list,
 	}
 }
 

pkg/yqlib/operator_env.go

@@ -13,7 +13,8 @@ type envOpPreferences struct {
 }
 
 func envOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	envName := expressionNode.Operation.CandidateNode.Node.Value
+	envNameNode := expressionNode.Operation.ValueNodes.Front().Value.(*CandidateNode)
+	envName := envNameNode.Node.Value
 	log.Debug("EnvOperator, env name:", envName)
 
 	rawValue := os.Getenv(envName)

pkg/yqlib/operator_multiply.go

@@ -31,7 +31,7 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
 			(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
 
 			var newBlank = lhs.CreateChild(nil, &yaml.Node{})
-
+			log.Debugf("merge - merge lhs into blank")
 			var newThing, err = mergeObjects(d, context.WritableClone(), newBlank, lhs, multiplyPreferences{})
 			if err != nil {
 				return nil, err
@@ -83,12 +83,13 @@ func multiplyIntegers(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, e
 }
 
 func mergeObjects(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode, preferences multiplyPreferences) (*CandidateNode, error) {
-	shouldAppendArrays := preferences.AppendArrays
 	var results = list.New()
 
-	// shouldn't recurse arrays if appending
-	prefs := recursiveDescentPreferences{RecurseArray: !shouldAppendArrays,
+	// only need to recurse the array if we are doing a deep merge
+	prefs := recursiveDescentPreferences{RecurseArray: preferences.DeepMergeArrays,
 		TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: true}}
+	log.Debugf("merge - preferences.DeepMergeArrays %v", preferences.DeepMergeArrays)
+	log.Debugf("merge - preferences.AppendArrays %v", preferences.AppendArrays)
 	err := recursiveDecent(d, results, context.SingleChildContext(rhs), prefs)
 	if err != nil {
 		return nil, err
@@ -118,16 +119,26 @@ func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom
 	log.Debugf("merge - applyAssignment lhs %v, rhs: %v", lhs.GetKey(), rhs.GetKey())
 
 	lhsPath := rhs.Path[pathIndexToStartFrom:]
+	log.Debugf("merge - lhsPath %v", lhsPath)
 
 	assignmentOp := &Operation{OperationType: assignAttributesOpType}
 	if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode {
 		assignmentOp.OperationType = addAssignOpType
+		log.Debugf("merge - assignmentOp.OperationType = addAssignOpType")
 	} else if !preferences.DeepMergeArrays && rhs.Node.Kind == yaml.SequenceNode ||
 		(rhs.Node.Kind == yaml.ScalarNode || rhs.Node.Kind == yaml.AliasNode) {
 		assignmentOp.OperationType = assignOpType
 		assignmentOp.UpdateAssign = false
+		log.Debugf("merge - rhs.Node.Kind == yaml.SequenceNode: %v", rhs.Node.Kind == yaml.SequenceNode)
+		log.Debugf("merge - rhs.Node.Kind == yaml.ScalarNode: %v", rhs.Node.Kind == yaml.ScalarNode)
+		log.Debugf("merge - rhs.Node.Kind == yaml.AliasNode: %v", rhs.Node.Kind == yaml.AliasNode)
+		log.Debugf("merge - assignmentOp.OperationType = assignOpType, no updateassign")
+	} else {
+		log.Debugf("merge - assignmentOp := &Operation{OperationType: assignAttributesOpType}")
 	}
-	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs}
+	valueNodes := list.New()
+	valueNodes.PushBack(rhs)
+	rhsOp := &Operation{OperationType: valueOpType, ValueNodes: valueNodes}
 
 	assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath, preferences.TraversePrefs, rhs.IsMapKey), Rhs: &ExpressionNode{Operation: rhsOp}}
 

pkg/yqlib/operator_multiply_test.go

@@ -24,6 +24,11 @@ list2:
     - "123"
 `
 
+var mergeArrayWithAnchors = `sample:
+- &a
+- <<: *a
+`
+
 var mergeArraysObjectKeysText = `It's a complex command, the trickyness comes from needing to have the right context in the expressions.
 First we save the second array into a variable '$two' which lets us reference it later.
 We then need to update the first array. We will use the relative update (|=) because we need to update relative to the current element of the array in the LHS in the RHS expression. 
@@ -31,6 +36,14 @@ We set the current element of the first array as $cur. Now we multiply (merge) $
 `
 
 var multiplyOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   mergeArrayWithAnchors,
+		expression: `. * .`,
+		expected: []string{
+			"D0, P[], (!!map)::sample:\n    - &a\n    - !!merge <<: *a\n",
+		},
+	},
 	{
 		description: "Multiply integers",
 		expression:  `3 * 4`,

pkg/yqlib/operator_reduce.go

@@ -24,7 +24,7 @@ func reduceOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 	arrayExpNode := expressionNode.Lhs.Lhs
 	array, err := d.GetMatchingNodes(context, arrayExpNode)
 
-	log.Debugf("array of %v things", array.MatchingNodes.Len())
+	log.Debugf("reducing %v", NodesToString(array.MatchingNodes))
 
 	if err != nil {
 		return Context{}, err
@@ -39,6 +39,8 @@ func reduceOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 		return Context{}, err
 	}
 
+	log.Debugf("initialised with %v", NodesToString(accum.MatchingNodes))
+
 	log.Debugf("with variable %v", variableName)
 
 	blockExp := expressionNode.Rhs.Rhs

pkg/yqlib/operator_value.go

@@ -1,6 +1,5 @@
 package yqlib
 
 func valueOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	log.Debug("value = %v", expressionNode.Operation.CandidateNode.Node.Value)
-	return context.SingleChildContext(expressionNode.Operation.CandidateNode), nil
+	return context.ChildContext(expressionNode.Operation.ValueNodes), nil
 }

pkg/yqlib/operators.go

@@ -23,7 +23,9 @@ func compoundAssignFunction(d *dataTreeNavigator, context Context, expressionNod
 
 	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
-		valueOp.CandidateNode = candidate
+		valueNodes := list.New()
+		valueNodes.PushBack(candidate)
+		valueOp.ValueNodes = valueNodes
 		valueExpression := &ExpressionNode{Operation: valueOp}
 
 		assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: valueExpression, Rhs: calculation(valueExpression, expressionNode.Rhs)}
@@ -83,7 +85,7 @@ func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *Expressi
 	if err != nil {
 		return Context{}, err
 	}
-	log.Debugf("crossFunction LHS len: %v", lhs.MatchingNodes.Len())
+	log.Debugf("crossFunction LHS %v", NodesToString(lhs.MatchingNodes))
 
 	rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
 

pkg/yqlib/printer.go

@@ -28,7 +28,6 @@ type resultsPrinter struct {
 	previousFileIndex  int
 	printedMatches     bool
 	treeNavigator      DataTreeNavigator
-	preambleReader     io.Reader
 	appendixReader     io.Reader
 }
 
@@ -45,10 +44,6 @@ func NewPrinter(writer io.Writer, outputToJSON bool, unwrapScalar bool, colorsEn
 	}
 }
 
-func (p *resultsPrinter) SetPreamble(reader io.Reader) {
-	p.preambleReader = reader
-}
-
 func (p *resultsPrinter) SetAppendix(reader io.Reader) {
 	p.appendixReader = reader
 }
@@ -152,6 +147,12 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 				}
 
 				if errReading == io.EOF {
+					if readline != "" {
+						// the last comment we read didn't have a new line, put one in
+						if err := p.writeString(bufferedWriter, "\n"); err != nil {
+							return err
+						}
+					}
 					break
 				}
 			}

pkg/yqlib/reduce_evaluator.go

@@ -0,0 +1,142 @@
+package yqlib
+
+import (
+	"container/list"
+	"fmt"
+	"io"
+	"os"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+type ReduceEvaluator interface {
+	EvaluateFiles(reduceExpression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error
+}
+
+type reduceEvaluator struct {
+	treeNavigator DataTreeNavigator
+	treeCreator   ExpressionParser
+	reduceLhs     *ExpressionNode
+	fileIndex     int
+}
+
+func NewReduceEvaluator() ReduceEvaluator {
+	treeCreator := NewExpressionParser()
+	reduceLhs, err := treeCreator.ParseExpression(". as $doc")
+	if err != nil {
+		panic(err)
+	}
+	return &reduceEvaluator{treeNavigator: NewDataTreeNavigator(), treeCreator: treeCreator, reduceLhs: reduceLhs}
+}
+
+func (r *reduceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error {
+
+	node, err := r.treeCreator.ParseExpression(expression)
+	if err != nil {
+		return err
+	}
+	log.Debug("node %v", node.Operation.toString())
+
+	if node.Operation.OperationType != blockOpType {
+		return fmt.Errorf("Invalid reduce expression - expected '<initialValue>; <block that uses $doc>' got '%v'", expression)
+	}
+
+	currentValue := node.Lhs
+	reduceExp := node.Rhs
+	firstLeadingContent := ""
+
+	log.Debug("initialValue %v", currentValue.Operation.toString())
+
+	log.Debug("reduce Exp %v", reduceExp.Operation.toString())
+
+	for index, filename := range filenames {
+		reader, leadingContent, err := readStream(filename, leadingContentPreProcessing)
+
+		if index == 0 {
+			firstLeadingContent = leadingContent
+		}
+
+		if err != nil {
+			return err
+		}
+
+		currentValue, err = r.ReduceFile(filename, leadingContent, reader, currentValue, reduceExp)
+		if err != nil {
+			return err
+		}
+
+		switch reader := reader.(type) {
+		case *os.File:
+			safelyCloseFile(reader)
+		}
+	}
+
+	result := currentValue.Operation.ValueNodes
+
+	if result.Len() > 0 {
+		result.Front().Value.(*CandidateNode).Node.HeadComment = firstLeadingContent
+	}
+
+	printer.PrintResults(result)
+
+	return nil
+}
+
+func (r *reduceEvaluator) createReduceOp(initialValue *ExpressionNode, reduceExp *ExpressionNode) *ExpressionNode {
+	reduceBlock := &ExpressionNode{
+		Operation: &Operation{OperationType: blockOpType},
+		Lhs:       initialValue,
+		Rhs:       reduceExp,
+	}
+
+	return &ExpressionNode{
+		Operation: &Operation{OperationType: reduceOpType},
+		Lhs:       r.reduceLhs,
+		Rhs:       reduceBlock,
+	}
+}
+
+func (r *reduceEvaluator) ReduceFile(filename string, leadingContent string, reader io.Reader, initialValue *ExpressionNode, reduceExp *ExpressionNode) (*ExpressionNode, error) {
+
+	var currentIndex uint
+	var currentValue = initialValue
+	decoder := yaml.NewDecoder(reader)
+	for {
+		var dataBucket yaml.Node
+		errorReading := decoder.Decode(&dataBucket)
+
+		if errorReading == io.EOF {
+			r.fileIndex = r.fileIndex + 1
+			return currentValue, nil
+		} else if errorReading != nil {
+			return currentValue, errorReading
+		}
+		candidateNode := &CandidateNode{
+			Document:  currentIndex,
+			Filename:  filename,
+			Node:      &dataBucket,
+			FileIndex: r.fileIndex,
+		}
+		inputList := list.New()
+		inputList.PushBack(candidateNode)
+
+		reduceOp := r.createReduceOp(currentValue, reduceExp)
+		// log.Debug("reduce - currentValueBefore: %v", NodesToString(currentValue.Operation.ValueNodes))
+
+		result, errorParsing := r.treeNavigator.GetMatchingNodes(Context{MatchingNodes: inputList}, reduceOp)
+		if errorParsing != nil {
+			return currentValue, errorParsing
+		}
+
+		currentValue = &ExpressionNode{
+			Operation: &Operation{
+				OperationType: valueOpType,
+				ValueNodes:    result.MatchingNodes,
+			},
+		}
+
+		log.Debug("reduce - currentValueAfter: %v", NodesToString(currentValue.Operation.ValueNodes))
+
+		currentIndex = currentIndex + 1
+	}
+}

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.11.0'
+version: '4.11.1'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.