eval-reduce wip

README.md

@@ -7,39 +7,12 @@ a lightweight and portable command-line YAML processor. `yq` uses [jq](https://g
 
 yq is written in go - so you can download a dependency free binary for your platform and you are good to go! If you prefer there are a variety of package managers that can be used as well as docker, all listed below.
 
-## Quick Usage Guide
+## V4 released!
+V4 is now officially released, it's quite different from V3 (sorry for the migration), however it is much more similar to ```jq```, using a similar expression syntax and therefore support much more complex functionality! 
 
-Read a value:
+If you've been using v3 and want/need to upgrade, checkout the [upgrade guide](https://mikefarah.gitbook.io/yq/v/v4.x/upgrading-from-v3).
 
-```bash
-yq e '.a.b[0].c' file.yaml
-```
-
-Update a yaml file, inplace
-```bash
-yq e -i '.a.b[0].c = "cool"' file.yaml
-```
-
-Update using environment variables
-```bash
-NAME=mike yq e -i '.a.b[0].c = strenv(NAME)' file.yaml
-```
-
-Merge multiple files
-```
-yq ea '. as $item ireduce ({}; . * $item )' file1.yml file2.yml ...
-```
-
-Multiple updates to a yaml file
-```bash
-yq e -i '
-  .a.b[0].c = "cool" |
-  .x.y.z = "foobar" |
-  .person.name = strenv(NAME)
-' file.yaml
-```
-
-See the [documentation](https://mikefarah.gitbook.io/yq/) for more.
+Support for v3 will cease August 2021, until then, critical bug and security fixes will still get applied if required.
 
 ## Install
 
@@ -147,16 +120,6 @@ RUN apk add bash
 USER yq
 ```
 
-### GitHub Action
-```
-  - name: Set foobar to cool
-    uses: mikefarah/yq@master
-    with:
-      cmd: yq eval -i '.foo.bar = "cool"' 'config.yml'
-```
-
-See https://mikefarah.gitbook.io/yq/usage/github-action for more.
-
 ### Go Get:
 ```
 GO111MODULE=on go get github.com/mikefarah/yq/v4
@@ -260,6 +223,13 @@ Flags:
 
 Use "yq [command] --help" for more information about a command.
 ```
+
+Simple Example:
+
+```bash
+yq e '.a.b | length' f1.yml f2.yml 
+```
+
 ## Known Issues / Missing Features
 - `yq` attempts to preserve comment positions and whitespace as much as possible, but it does not handle all scenarios (see https://github.com/go-yaml/yaml/tree/v3 for details)
 - Powershell has its own...opinions: https://mikefarah.gitbook.io/yq/usage/tips-and-tricks#quotes-in-windows-powershell

cmd/constant.go

@@ -5,7 +5,6 @@ var unwrapScalar = true
 
 var writeInplace = false
 var outputToJSON = false
-var outputFormat = "yaml"
 
 var exitStatus = false
 var forceColor = false

cmd/evaluate_all_command.go

@@ -34,7 +34,8 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.
 ## Evaluate All ##
 This command loads _all_ yaml documents of _all_ yaml files and runs expression once
 Useful when you need to run an expression across several yaml documents or files (like merge).
-Note that it consumes more memory than eval.
+If you're just merging entire multiple files together, you may want to consider eval-reduce as it's faster.
+Note that it consumes more memory than eval and eval-reduce.
 `,
 		RunE: evaluateAll,
 	}
@@ -86,17 +87,8 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 	if nullInput && len(args) > 1 {
 		return errors.New("Cannot pass files in when using null-input flag")
 	}
-	// backwards compatibilty
-	if outputToJSON {
-		outputFormat = "json"
-	}
 
-	format, err := yqlib.OutputFormatFromString(outputFormat)
-	if err != nil {
-		return err
-	}
-
-	printer := yqlib.NewPrinter(out, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
+	printer := yqlib.NewPrinter(out, outputToJSON, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
 
 	if frontMatter != "" {
 		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])

cmd/evaluate_reduce_command.go

@@ -0,0 +1,112 @@
+package cmd
+
+import (
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/mikefarah/yq/v4/pkg/yqlib"
+	"github.com/spf13/cobra"
+)
+
+func createEvaluateReduceCommand() *cobra.Command {
+	var cmdEvalReduce = &cobra.Command{
+		Use:     "eval-reduce [reduce expression] [yaml_file1]...",
+		Aliases: []string{"er"},
+		Short:   "Runs a reduce expression sequentially against each document of each file given. More memory efficient than using eval-all if you can get away with it.",
+		Example: `
+# Merge f2.yml into f1.yml (inplace)
+yq eval-reduce --inplace '{} ; . * $doc' f1.yml f2.yml
+`,
+		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
+See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.
+
+## Evaluate Reduce ##
+This command runs the reduce expression against each document of each file given, accumulating the results.
+It is most useful when merging multiple files (but isn't as flexible as using eval-all with ireduce, as you
+can only merge the top level nodes).
+`,
+		RunE: evaluateReduce,
+	}
+	return cmdEvalReduce
+}
+func evaluateReduce(cmd *cobra.Command, args []string) error {
+	cmd.SilenceUsage = true
+	// 2+ args, [0] = expression, file the rest
+
+	var err error
+
+	out := cmd.OutOrStdout()
+
+	fileInfo, _ := os.Stdout.Stat()
+
+	if forceColor || (!forceNoColor && (fileInfo.Mode()&os.ModeCharDevice) != 0) {
+		colorsEnabled = true
+	}
+
+	firstFileIndex := -1
+	if !nullInput && len(args) == 1 {
+		firstFileIndex = 0
+	} else if len(args) > 1 {
+		firstFileIndex = 1
+	}
+
+	if writeInplace && (firstFileIndex == -1) {
+		return fmt.Errorf("Write inplace flag only applicable when giving an expression and at least one file")
+	}
+
+	if writeInplace {
+		// only use colors if its forced
+		colorsEnabled = forceColor
+		writeInPlaceHandler := yqlib.NewWriteInPlaceHandler(args[firstFileIndex])
+		out, err = writeInPlaceHandler.CreateTempFile()
+		if err != nil {
+			return err
+		}
+		// need to indirectly call the function so  that completedSuccessfully is
+		// passed when we finish execution as opposed to now
+		defer func() { writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully) }()
+	}
+
+	if nullInput && len(args) > 1 {
+		return errors.New("Cannot pass files in when using null-input flag")
+	}
+
+	printer := yqlib.NewPrinter(out, outputToJSON, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
+
+	if frontMatter != "" {
+		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])
+		err = frontMatterHandler.Split()
+		if err != nil {
+			return err
+		}
+		args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename()
+
+		if frontMatter == "process" {
+			reader := frontMatterHandler.GetContentReader()
+			printer.SetAppendix(reader)
+			defer yqlib.SafelyCloseReader(reader)
+		}
+		defer frontMatterHandler.CleanUp()
+	}
+
+	reduceEvaluator := yqlib.NewReduceEvaluator()
+	switch len(args) {
+	case 0:
+		cmd.Println(cmd.UsageString())
+		return nil
+	case 1:
+		cmd.Println(cmd.UsageString())
+		return nil
+	default:
+		err = reduceEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer, leadingContentPreProcessing)
+	}
+
+	completedSuccessfully = err == nil
+
+	if err == nil && exitStatus && !printer.PrintedAnything() {
+		return errors.New("no matches found")
+	}
+
+	return err
+}

cmd/evalute_sequence_command.go

@@ -95,17 +95,7 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 		defer func() { writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully) }()
 	}
 
-	// backwards compatibilty
-	if outputToJSON {
-		outputFormat = "json"
-	}
-
-	format, err := yqlib.OutputFormatFromString(outputFormat)
-	if err != nil {
-		return err
-	}
-
-	printer := yqlib.NewPrinter(out, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
+	printer := yqlib.NewPrinter(out, outputToJSON, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
 
 	streamEvaluator := yqlib.NewStreamEvaluator()
 

cmd/root.go

@@ -41,14 +41,7 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 	}
 
 	rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "verbose mode")
-
-	rootCmd.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "(deprecated) output as json. Set indent to 0 to print json in one line.")
-	err := rootCmd.PersistentFlags().MarkDeprecated("tojson", "please use -t=json instead")
-	if err != nil {
-		panic(err)
-	}
-
-	rootCmd.PersistentFlags().StringVarP(&outputFormat, "to-type", "t", "yaml", "[yaml|json|props] output format type.")
+	rootCmd.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "output as json. Set indent to 0 to print json in one line.")
 	rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating yaml docs from scratch.")
 	rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)")
 
@@ -66,6 +59,7 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 	rootCmd.AddCommand(
 		createEvaluateSequenceCommand(),
 		createEvaluateAllCommand(),
+		createEvaluateReduceCommand(),
 		completionCmd,
 	)
 	return rootCmd

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.11.2"
+	Version = "4.11.1"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

examples/data1.yaml

@@ -1,4 +1,3 @@
+a: apple
 ---
-# hi peeps
-# cool
-a: test
+a2: fish

examples/data2.yaml

@@ -1,7 +1 @@
-a: other # better than the original
-b: [3, 4]
-c:
-  toast: leave
-  test: 1
-  tell: 1
-  tasty.taco: cool
+b: doc2

github-action/Dockerfile

@@ -1,4 +1,4 @@
-FROM mikefarah/yq:4.11.2
+FROM mikefarah/yq:4.11.1
 
 COPY entrypoint.sh /entrypoint.sh
 

github-action/entrypoint.sh

@@ -1,5 +1,5 @@
 #!/bin/sh -l
-set -e
+
 echo "::debug::\$cmd: $1"
 RESULT=$(eval "$1")
 echo "::debug::\$RESULT: $RESULT"

go.mod

@@ -5,7 +5,6 @@ require (
 	github.com/fatih/color v1.10.0
 	github.com/goccy/go-yaml v1.8.9
 	github.com/jinzhu/copier v0.2.8
-	github.com/magiconair/properties v1.8.5
 	github.com/spf13/cobra v1.1.3
 	github.com/timtadh/data-structures v0.5.3 // indirect
 	github.com/timtadh/lexmachine v0.2.2

go.sum

@@ -119,8 +119,6 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls=
-github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=

pkg/yqlib/context.go

@@ -30,6 +30,7 @@ func (n *Context) GetVariable(name string) *list.List {
 	if n.Variables == nil {
 		return nil
 	}
+	log.Debug("GetVariable - %v to %v", name, NodesToString(n.Variables[name]))
 	return n.Variables[name]
 }
 
@@ -37,6 +38,7 @@ func (n *Context) SetVariable(name string, value *list.List) {
 	if n.Variables == nil {
 		n.Variables = make(map[string]*list.List)
 	}
+	log.Debug("SetVariable - %v to %v", name, NodesToString(value))
 	n.Variables[name] = value
 }
 

pkg/yqlib/encoder_properties.go

@@ -1,80 +0,0 @@
-package yqlib
-
-import (
-	"fmt"
-	"io"
-	"strings"
-
-	"github.com/magiconair/properties"
-	yaml "gopkg.in/yaml.v3"
-)
-
-type propertiesEncoder struct {
-	destination io.Writer
-}
-
-func NewPropertiesEncoder(destination io.Writer) Encoder {
-	return &propertiesEncoder{destination}
-}
-
-func (pe *propertiesEncoder) Encode(node *yaml.Node) error {
-	mapKeysToStrings(node)
-	p := properties.NewProperties()
-	err := pe.doEncode(p, node, "")
-	if err != nil {
-		return err
-	}
-
-	_, err = p.WriteComment(pe.destination, "#", properties.UTF8)
-	return err
-}
-
-func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yaml.Node, path string) error {
-	p.SetComment(path,
-		strings.Replace(node.HeadComment, "#", "", 1)+
-			strings.Replace(node.LineComment, "#", "", 1))
-	switch node.Kind {
-	case yaml.ScalarNode:
-		_, _, err := p.Set(path, node.Value)
-		return err
-	case yaml.DocumentNode:
-		return pe.doEncode(p, node.Content[0], path)
-	case yaml.SequenceNode:
-		return pe.encodeArray(p, node.Content, path)
-	case yaml.MappingNode:
-		return pe.encodeMap(p, node.Content, path)
-	case yaml.AliasNode:
-		return pe.doEncode(p, node.Alias, path)
-	default:
-		return fmt.Errorf("Unsupported node %v", node.Tag)
-	}
-}
-
-func (pe *propertiesEncoder) appendPath(path string, key interface{}) string {
-	if path == "" {
-		return fmt.Sprintf("%v", key)
-	}
-	return fmt.Sprintf("%v.%v", path, key)
-}
-
-func (pe *propertiesEncoder) encodeArray(p *properties.Properties, kids []*yaml.Node, path string) error {
-	for index, child := range kids {
-		err := pe.doEncode(p, child, pe.appendPath(path, index))
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (pe *propertiesEncoder) encodeMap(p *properties.Properties, kids []*yaml.Node, path string) error {
-	for index := 0; index < len(kids); index = index + 2 {
-		key := kids[index]
-		value := kids[index+1]
-		err := pe.doEncode(p, value, pe.appendPath(path, key.Value))
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}

pkg/yqlib/encoder_properties_test.go

@@ -1,78 +0,0 @@
-package yqlib
-
-import (
-	"bufio"
-	"bytes"
-	"strings"
-	"testing"
-
-	"github.com/mikefarah/yq/v4/test"
-)
-
-func yamlToProps(sampleYaml string) string {
-	var output bytes.Buffer
-	writer := bufio.NewWriter(&output)
-
-	var propsEncoder = NewPropertiesEncoder(writer)
-	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0)
-	if err != nil {
-		panic(err)
-	}
-	node := inputs.Front().Value.(*CandidateNode).Node
-	err = propsEncoder.Encode(node)
-	if err != nil {
-		panic(err)
-	}
-	writer.Flush()
-
-	return strings.TrimSuffix(output.String(), "\n")
-}
-
-func TestPropertiesEncoderSimple(t *testing.T) {
-	var sampleYaml = `a: 'bob cool'`
-
-	var expectedJson = `a = bob cool`
-	var actualProps = yamlToProps(sampleYaml)
-	test.AssertResult(t, expectedJson, actualProps)
-}
-
-func TestPropertiesEncoderSimpleWithComments(t *testing.T) {
-	var sampleYaml = `a: 'bob cool' # line`
-
-	var expectedJson = `# line
-a = bob cool`
-	var actualProps = yamlToProps(sampleYaml)
-	test.AssertResult(t, expectedJson, actualProps)
-}
-
-func TestPropertiesEncoderDeep(t *testing.T) {
-	var sampleYaml = `a: 
-  b: "bob cool"
-`
-
-	var expectedJson = `a.b = bob cool`
-	var actualProps = yamlToProps(sampleYaml)
-	test.AssertResult(t, expectedJson, actualProps)
-}
-
-func TestPropertiesEncoderDeepWithComments(t *testing.T) {
-	var sampleYaml = `a:  # a thing
-  b: "bob cool" # b thing
-`
-
-	var expectedJson = `# b thing
-a.b = bob cool`
-	var actualProps = yamlToProps(sampleYaml)
-	test.AssertResult(t, expectedJson, actualProps)
-}
-
-func TestPropertiesEncoderArray(t *testing.T) {
-	var sampleYaml = `a: 
-  b: [{c: dog}, {c: cat}]
-`
-
-	var expectedJson = `a.b.0.c = dog
-a.b.1.c = cat`
-	var actualProps = yamlToProps(sampleYaml)
-	test.AssertResult(t, expectedJson, actualProps)
-}

pkg/yqlib/lib.go

@@ -112,7 +112,7 @@ type Operation struct {
 	OperationType *operationType
 	Value         interface{}
 	StringValue   string
-	CandidateNode *CandidateNode // used for Value Path elements
+	ValueNodes    *list.List // used for Value Path elements
 	Preferences   interface{}
 	UpdateAssign  bool // used for assign ops, when true it means we evaluate the rhs given the lhs
 }
@@ -138,12 +138,13 @@ func createScalarNode(value interface{}, stringValue string) *yaml.Node {
 
 func createValueOperation(value interface{}, stringValue string) *Operation {
 	var node *yaml.Node = createScalarNode(value, stringValue)
-
+	list := list.New()
+	list.PushBack(&CandidateNode{Node: node})
 	return &Operation{
 		OperationType: valueOpType,
 		Value:         value,
 		StringValue:   stringValue,
-		CandidateNode: &CandidateNode{Node: node},
+		ValueNodes:    list,
 	}
 }
 

pkg/yqlib/operator_env.go

@@ -13,7 +13,8 @@ type envOpPreferences struct {
 }
 
 func envOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	envName := expressionNode.Operation.CandidateNode.Node.Value
+	envNameNode := expressionNode.Operation.ValueNodes.Front().Value.(*CandidateNode)
+	envName := envNameNode.Node.Value
 	log.Debug("EnvOperator, env name:", envName)
 
 	rawValue := os.Getenv(envName)

pkg/yqlib/operator_multiply.go

@@ -136,7 +136,9 @@ func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom
 	} else {
 		log.Debugf("merge - assignmentOp := &Operation{OperationType: assignAttributesOpType}")
 	}
-	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs}
+	valueNodes := list.New()
+	valueNodes.PushBack(rhs)
+	rhsOp := &Operation{OperationType: valueOpType, ValueNodes: valueNodes}
 
 	assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath, preferences.TraversePrefs, rhs.IsMapKey), Rhs: &ExpressionNode{Operation: rhsOp}}
 

pkg/yqlib/operator_reduce.go

@@ -24,7 +24,7 @@ func reduceOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 	arrayExpNode := expressionNode.Lhs.Lhs
 	array, err := d.GetMatchingNodes(context, arrayExpNode)
 
-	log.Debugf("array of %v things", array.MatchingNodes.Len())
+	log.Debugf("reducing %v", NodesToString(array.MatchingNodes))
 
 	if err != nil {
 		return Context{}, err
@@ -39,6 +39,8 @@ func reduceOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 		return Context{}, err
 	}
 
+	log.Debugf("initialised with %v", NodesToString(accum.MatchingNodes))
+
 	log.Debugf("with variable %v", variableName)
 
 	blockExp := expressionNode.Rhs.Rhs

pkg/yqlib/operator_value.go

@@ -1,6 +1,5 @@
 package yqlib
 
 func valueOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	log.Debug("value = %v", expressionNode.Operation.CandidateNode.Node.Value)
-	return context.SingleChildContext(expressionNode.Operation.CandidateNode), nil
+	return context.ChildContext(expressionNode.Operation.ValueNodes), nil
 }

pkg/yqlib/operators.go

@@ -23,7 +23,9 @@ func compoundAssignFunction(d *dataTreeNavigator, context Context, expressionNod
 
 	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
-		valueOp.CandidateNode = candidate
+		valueNodes := list.New()
+		valueNodes.PushBack(candidate)
+		valueOp.ValueNodes = valueNodes
 		valueExpression := &ExpressionNode{Operation: valueOp}
 
 		assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: valueExpression, Rhs: calculation(valueExpression, expressionNode.Rhs)}
@@ -83,7 +85,7 @@ func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *Expressi
 	if err != nil {
 		return Context{}, err
 	}
-	log.Debugf("crossFunction LHS len: %v", lhs.MatchingNodes.Len())
+	log.Debugf("crossFunction LHS %v", NodesToString(lhs.MatchingNodes))
 
 	rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
 

pkg/yqlib/operators_test.go

@@ -107,7 +107,7 @@ func copyFromHeader(title string, out *os.File) error {
 
 func formatYaml(yaml string, filename string) string {
 	var output bytes.Buffer
-	printer := NewPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinter(bufio.NewWriter(&output), false, true, false, 2, true)
 
 	node, err := NewExpressionParser().ParseExpression(".. style= \"\"")
 	if err != nil {
@@ -216,7 +216,7 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) {
 func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formattedDoc string, formattedDoc2 string) {
 	var output bytes.Buffer
 	var err error
-	printer := NewPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinter(bufio.NewWriter(&output), false, true, false, 2, true)
 
 	node, err := NewExpressionParser().ParseExpression(s.expression)
 	if err != nil {

pkg/yqlib/printer.go

@@ -3,7 +3,6 @@ package yqlib
 import (
 	"bufio"
 	"container/list"
-	"fmt"
 	"io"
 	"strings"
 
@@ -17,29 +16,8 @@ type Printer interface {
 	SetAppendix(reader io.Reader)
 }
 
-type PrinterOutputFormat uint32
-
-const (
-	YamlOutputFormat = 1 << iota
-	JsonOutputFormat
-	PropsOutputFormat
-)
-
-func OutputFormatFromString(format string) (PrinterOutputFormat, error) {
-	switch format {
-	case "yaml":
-		return YamlOutputFormat, nil
-	case "json":
-		return JsonOutputFormat, nil
-	case "props":
-		return PropsOutputFormat, nil
-	default:
-		return 0, fmt.Errorf("Unknown fromat '%v' please use [yaml|json|props]", format)
-	}
-}
-
 type resultsPrinter struct {
-	outputFormat       PrinterOutputFormat
+	outputToJSON       bool
 	unwrapScalar       bool
 	colorsEnabled      bool
 	indent             int
@@ -53,14 +31,14 @@ type resultsPrinter struct {
 	appendixReader     io.Reader
 }
 
-func NewPrinter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
+func NewPrinter(writer io.Writer, outputToJSON bool, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
 	return &resultsPrinter{
 		writer:             writer,
-		outputFormat:       outputFormat,
+		outputToJSON:       outputToJSON,
 		unwrapScalar:       unwrapScalar,
 		colorsEnabled:      colorsEnabled,
 		indent:             indent,
-		printDocSeparators: outputFormat == YamlOutputFormat && printDocSeparators,
+		printDocSeparators: !outputToJSON && printDocSeparators,
 		firstTimePrinting:  true,
 		treeNavigator:      NewDataTreeNavigator(),
 	}
@@ -79,14 +57,11 @@ func (p *resultsPrinter) printNode(node *yaml.Node, writer io.Writer) error {
 		(node.Tag != "!!bool" || node.Value != "false"))
 
 	var encoder Encoder
-	if node.Kind == yaml.ScalarNode && p.unwrapScalar && p.outputFormat == YamlOutputFormat {
+	if node.Kind == yaml.ScalarNode && p.unwrapScalar && !p.outputToJSON {
 		return p.writeString(writer, node.Value+"\n")
 	}
-
-	if p.outputFormat == JsonOutputFormat {
+	if p.outputToJSON {
 		encoder = NewJsonEncoder(writer, p.indent)
-	} else if p.outputFormat == PropsOutputFormat {
-		encoder = NewPropertiesEncoder(writer)
 	} else {
 		encoder = NewYamlEncoder(writer, p.indent, p.colorsEnabled)
 	}
@@ -107,7 +82,7 @@ func (p *resultsPrinter) safelyFlush(writer *bufio.Writer) {
 
 func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 	log.Debug("PrintResults for %v matches", matchingNodes.Len())
-	if p.outputFormat != YamlOutputFormat {
+	if p.outputToJSON {
 		explodeOp := Operation{OperationType: explodeOpType}
 		explodeNode := ExpressionNode{Operation: &explodeOp}
 		context, err := p.treeNavigator.GetMatchingNodes(Context{MatchingNodes: matchingNodes}, &explodeNode)
@@ -165,7 +140,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 							return err
 						}
 					}
-				} else if p.outputFormat == YamlOutputFormat {
+				} else if !p.outputToJSON {
 					if err := p.writeString(bufferedWriter, readline); err != nil {
 						return err
 					}
@@ -183,7 +158,6 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 			}
 
 		}
-
 		if err := p.printNode(mappedDoc.Node, bufferedWriter); err != nil {
 			return err
 		}
@@ -191,7 +165,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 		p.previousDocIndex = mappedDoc.Document
 	}
 
-	if p.appendixReader != nil && p.outputFormat == YamlOutputFormat {
+	if p.appendixReader != nil && !p.outputToJSON {
 		log.Debug("Piping appendix reader...")
 		betterReader := bufio.NewReader(p.appendixReader)
 		_, err := io.Copy(bufferedWriter, betterReader)

pkg/yqlib/printer_test.go

@@ -36,7 +36,7 @@ func nodeToList(candidate *CandidateNode) *list.List {
 func TestPrinterMultipleDocsInSequence(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinter(writer, false, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -74,7 +74,7 @@ func TestPrinterMultipleDocsInSequence(t *testing.T) {
 func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinter(writer, false, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -116,7 +116,7 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
 func TestPrinterMultipleFilesInSequence(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinter(writer, false, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -163,7 +163,7 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) {
 func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinter(writer, false, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -213,7 +213,7 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
 func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinter(writer, false, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -232,7 +232,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinter(writer, false, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -261,7 +261,7 @@ a: coconut
 func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinter(writer, false, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -287,7 +287,7 @@ a: coconut
 func TestPrinterScalarWithLeadingCont(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinter(writer, false, true, false, 2, true)
 
 	node, err := NewExpressionParser().ParseExpression(".a")
 	if err != nil {
@@ -314,7 +314,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	// note printDocSeparators is true, it should still not print document separators
 	// when outputing JSON.
-	printer := NewPrinter(writer, JsonOutputFormat, true, false, 0, true)
+	printer := NewPrinter(writer, true, true, false, 0, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {

pkg/yqlib/reduce_evaluator.go

@@ -0,0 +1,142 @@
+package yqlib
+
+import (
+	"container/list"
+	"fmt"
+	"io"
+	"os"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+type ReduceEvaluator interface {
+	EvaluateFiles(reduceExpression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error
+}
+
+type reduceEvaluator struct {
+	treeNavigator DataTreeNavigator
+	treeCreator   ExpressionParser
+	reduceLhs     *ExpressionNode
+	fileIndex     int
+}
+
+func NewReduceEvaluator() ReduceEvaluator {
+	treeCreator := NewExpressionParser()
+	reduceLhs, err := treeCreator.ParseExpression(". as $doc")
+	if err != nil {
+		panic(err)
+	}
+	return &reduceEvaluator{treeNavigator: NewDataTreeNavigator(), treeCreator: treeCreator, reduceLhs: reduceLhs}
+}
+
+func (r *reduceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error {
+
+	node, err := r.treeCreator.ParseExpression(expression)
+	if err != nil {
+		return err
+	}
+	log.Debug("node %v", node.Operation.toString())
+
+	if node.Operation.OperationType != blockOpType {
+		return fmt.Errorf("Invalid reduce expression - expected '<initialValue>; <block that uses $doc>' got '%v'", expression)
+	}
+
+	currentValue := node.Lhs
+	reduceExp := node.Rhs
+	firstLeadingContent := ""
+
+	log.Debug("initialValue %v", currentValue.Operation.toString())
+
+	log.Debug("reduce Exp %v", reduceExp.Operation.toString())
+
+	for index, filename := range filenames {
+		reader, leadingContent, err := readStream(filename, leadingContentPreProcessing)
+
+		if index == 0 {
+			firstLeadingContent = leadingContent
+		}
+
+		if err != nil {
+			return err
+		}
+
+		currentValue, err = r.ReduceFile(filename, leadingContent, reader, currentValue, reduceExp)
+		if err != nil {
+			return err
+		}
+
+		switch reader := reader.(type) {
+		case *os.File:
+			safelyCloseFile(reader)
+		}
+	}
+
+	result := currentValue.Operation.ValueNodes
+
+	if result.Len() > 0 {
+		result.Front().Value.(*CandidateNode).Node.HeadComment = firstLeadingContent
+	}
+
+	printer.PrintResults(result)
+
+	return nil
+}
+
+func (r *reduceEvaluator) createReduceOp(initialValue *ExpressionNode, reduceExp *ExpressionNode) *ExpressionNode {
+	reduceBlock := &ExpressionNode{
+		Operation: &Operation{OperationType: blockOpType},
+		Lhs:       initialValue,
+		Rhs:       reduceExp,
+	}
+
+	return &ExpressionNode{
+		Operation: &Operation{OperationType: reduceOpType},
+		Lhs:       r.reduceLhs,
+		Rhs:       reduceBlock,
+	}
+}
+
+func (r *reduceEvaluator) ReduceFile(filename string, leadingContent string, reader io.Reader, initialValue *ExpressionNode, reduceExp *ExpressionNode) (*ExpressionNode, error) {
+
+	var currentIndex uint
+	var currentValue = initialValue
+	decoder := yaml.NewDecoder(reader)
+	for {
+		var dataBucket yaml.Node
+		errorReading := decoder.Decode(&dataBucket)
+
+		if errorReading == io.EOF {
+			r.fileIndex = r.fileIndex + 1
+			return currentValue, nil
+		} else if errorReading != nil {
+			return currentValue, errorReading
+		}
+		candidateNode := &CandidateNode{
+			Document:  currentIndex,
+			Filename:  filename,
+			Node:      &dataBucket,
+			FileIndex: r.fileIndex,
+		}
+		inputList := list.New()
+		inputList.PushBack(candidateNode)
+
+		reduceOp := r.createReduceOp(currentValue, reduceExp)
+		// log.Debug("reduce - currentValueBefore: %v", NodesToString(currentValue.Operation.ValueNodes))
+
+		result, errorParsing := r.treeNavigator.GetMatchingNodes(Context{MatchingNodes: inputList}, reduceOp)
+		if errorParsing != nil {
+			return currentValue, errorParsing
+		}
+
+		currentValue = &ExpressionNode{
+			Operation: &Operation{
+				OperationType: valueOpType,
+				ValueNodes:    result.MatchingNodes,
+			},
+		}
+
+		log.Debug("reduce - currentValueAfter: %v", NodesToString(currentValue.Operation.ValueNodes))
+
+		currentIndex = currentIndex + 1
+	}
+}

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.11.2'
+version: '4.11.1'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.