Added properties encoder test

README.md

@@ -7,12 +7,39 @@ a lightweight and portable command-line YAML processor. `yq` uses [jq](https://g
 
 yq is written in go - so you can download a dependency free binary for your platform and you are good to go! If you prefer there are a variety of package managers that can be used as well as docker, all listed below.
 
-## V4 released!
-V4 is now officially released, it's quite different from V3 (sorry for the migration), however it is much more similar to ```jq```, using a similar expression syntax and therefore support much more complex functionality! 
+## Quick Usage Guide
 
-If you've been using v3 and want/need to upgrade, checkout the [upgrade guide](https://mikefarah.gitbook.io/yq/v/v4.x/upgrading-from-v3).
+Read a value:
 
-Support for v3 will cease August 2021, until then, critical bug and security fixes will still get applied if required.
+```bash
+yq e '.a.b[0].c' file.yaml
+```
+
+Update a yaml file, inplace
+```bash
+yq e -i '.a.b[0].c = "cool"' file.yaml
+```
+
+Update using environment variables
+```bash
+NAME=mike yq e -i '.a.b[0].c = strenv(NAME)' file.yaml
+```
+
+Merge multiple files
+```
+yq ea '. as $item ireduce ({}; . * $item )' file1.yml file2.yml ...
+```
+
+Multiple updates to a yaml file
+```bash
+yq e -i '
+  .a.b[0].c = "cool" |
+  .x.y.z = "foobar" |
+  .person.name = strenv(NAME)
+' file.yaml
+```
+
+See the [documentation](https://mikefarah.gitbook.io/yq/) for more.
 
 ## Install
 
@@ -120,6 +147,16 @@ RUN apk add bash
 USER yq
 ```
 
+### GitHub Action
+```
+  - name: Set foobar to cool
+    uses: mikefarah/yq@master
+    with:
+      cmd: yq eval -i '.foo.bar = "cool"' 'config.yml'
+```
+
+See https://mikefarah.gitbook.io/yq/usage/github-action for more.
+
 ### Go Get:
 ```
 GO111MODULE=on go get github.com/mikefarah/yq/v4
@@ -223,13 +260,6 @@ Flags:
 
 Use "yq [command] --help" for more information about a command.
 ```
-
-Simple Example:
-
-```bash
-yq e '.a.b | length' f1.yml f2.yml 
-```
-
 ## Known Issues / Missing Features
 - `yq` attempts to preserve comment positions and whitespace as much as possible, but it does not handle all scenarios (see https://github.com/go-yaml/yaml/tree/v3 for details)
 - Powershell has its own...opinions: https://mikefarah.gitbook.io/yq/usage/tips-and-tricks#quotes-in-windows-powershell

acceptance_tests/empty.sh

@@ -8,7 +8,27 @@ EOL
 
 testEmptyEval() {
   X=$(./yq e test.yml)
+  expected=$(cat test.yml)
   assertEquals 0 $?
+  assertEquals "$expected" "$X"
+}
+
+testEmptyEvalNoNewLine() {
+  echo -n "#comment" >test.yml
+  X=$(./yq e test.yml)
+  expected=$(cat test.yml)
+  assertEquals 0 $?
+  assertEquals "$expected" "$X"
+}
+
+testEmptyEvalNoNewLineWithExpression() {
+  echo -n "# comment" >test.yml
+  X=$(./yq e '.apple = "tree"' test.yml)
+  read -r -d '' expected << EOM
+# comment
+apple: tree
+EOM
+  assertEquals "$expected" "$X"
 }
 
 testEmptyEvalPipe() {

acceptance_tests/leading-seperator.sh

@@ -36,6 +36,73 @@ testLeadingSeperatorPipeIntoEvalSeq() {
   assertEquals "$expected" "$X"
 }
 
+testLeadingSeperatorExtractField() {
+  X=$(./yq e '.a' - < test.yml)
+  assertEquals "test" "$X"
+}
+
+testLeadingSeperatorExtractFieldWithCommentsAfterSep() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+EOL
+  X=$(./yq e '.a' test.yml)
+  assertEquals "test" "$X"
+}
+
+testLeadingSeperatorExtractFieldWithCommentsBeforeSep() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+---
+a: test
+EOL
+  X=$(./yq e '.a' test.yml)
+  assertEquals "test" "$X"
+}
+
+
+testLeadingSeperatorExtractFieldMultiDoc() {
+  cat >test.yml <<EOL
+---
+a: test
+---
+a: test2
+EOL
+
+  read -r -d '' expected << EOM
+test
+---
+test2
+EOM
+  X=$(./yq e '.a' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorExtractFieldMultiDocWithComments() {
+  cat >test.yml <<EOL
+# here
+---
+# there
+a: test
+# whereever
+---
+# you are
+a: test2
+# woop
+EOL
+
+  read -r -d '' expected << EOM
+test
+---
+test2
+EOM
+  X=$(./yq e '.a' test.yml)
+  assertEquals "$expected" "$X"
+}
+
 
 testLeadingSeperatorEvalSeq() {
   X=$(./yq e test.yml)

cmd/constant.go

@@ -5,6 +5,7 @@ var unwrapScalar = true
 
 var writeInplace = false
 var outputToJSON = false
+var outputFormat = "yaml"
 
 var exitStatus = false
 var forceColor = false

cmd/evaluate_all_command.go

@@ -86,8 +86,17 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 	if nullInput && len(args) > 1 {
 		return errors.New("Cannot pass files in when using null-input flag")
 	}
+	// backwards compatibilty
+	if outputToJSON {
+		outputFormat = "json"
+	}
 
-	printer := yqlib.NewPrinter(out, outputToJSON, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
+	format, err := yqlib.OutputFormatFromString(outputFormat)
+	if err != nil {
+		return err
+	}
+
+	printer := yqlib.NewPrinter(out, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
 
 	if frontMatter != "" {
 		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])
@@ -98,10 +107,7 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 		args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename()
 
 		if frontMatter == "process" {
-			reader, err := os.Open(frontMatterHandler.GetContentFilename()) // #nosec
-			if err != nil {
-				return err
-			}
+			reader := frontMatterHandler.GetContentReader()
 			printer.SetAppendix(reader)
 			defer yqlib.SafelyCloseReader(reader)
 		}

cmd/evalute_sequence_command.go

@@ -95,7 +95,17 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 		defer func() { writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully) }()
 	}
 
-	printer := yqlib.NewPrinter(out, outputToJSON, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
+	// backwards compatibilty
+	if outputToJSON {
+		outputFormat = "json"
+	}
+
+	format, err := yqlib.OutputFormatFromString(outputFormat)
+	if err != nil {
+		return err
+	}
+
+	printer := yqlib.NewPrinter(out, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
 
 	streamEvaluator := yqlib.NewStreamEvaluator()
 
@@ -112,10 +122,7 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 		args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename()
 
 		if frontMatter == "process" {
-			reader, err := os.Open(frontMatterHandler.GetContentFilename()) // #nosec
-			if err != nil {
-				return err
-			}
+			reader := frontMatterHandler.GetContentReader()
 			printer.SetAppendix(reader)
 			defer yqlib.SafelyCloseReader(reader)
 		}

cmd/root.go

@@ -41,7 +41,14 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 	}
 
 	rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "verbose mode")
-	rootCmd.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "output as json. Set indent to 0 to print json in one line.")
+
+	rootCmd.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "(deprecated) output as json. Set indent to 0 to print json in one line.")
+	err := rootCmd.PersistentFlags().MarkDeprecated("tojson", "please use -t=json instead")
+	if err != nil {
+		panic(err)
+	}
+
+	rootCmd.PersistentFlags().StringVarP(&outputFormat, "to-type", "t", "yaml", "[yaml|json|props] output format type.")
 	rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating yaml docs from scratch.")
 	rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)")
 

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.11.0"
+	Version = "4.11.2"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

examples/front-matter.yaml

@@ -2,4 +2,5 @@
 a: apple
 b: bannana
 ---
-<h1>I like {{a}} and {{b}} </h1>
+hello there
+apples: great

github-action/Dockerfile

@@ -1,4 +1,4 @@
-FROM mikefarah/yq:4.11.0
+FROM mikefarah/yq:4.11.2
 
 COPY entrypoint.sh /entrypoint.sh
 

github-action/entrypoint.sh

@@ -1,5 +1,5 @@
 #!/bin/sh -l
-
+set -e
 echo "::debug::\$cmd: $1"
 RESULT=$(eval "$1")
 echo "::debug::\$RESULT: $RESULT"

go.mod

@@ -5,6 +5,7 @@ require (
 	github.com/fatih/color v1.10.0
 	github.com/goccy/go-yaml v1.8.9
 	github.com/jinzhu/copier v0.2.8
+	github.com/magiconair/properties v1.8.5
 	github.com/spf13/cobra v1.1.3
 	github.com/timtadh/data-structures v0.5.3 // indirect
 	github.com/timtadh/lexmachine v0.2.2

go.sum

@@ -119,6 +119,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls=
+github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=

pkg/yqlib/encoder_properties.go

@@ -0,0 +1,80 @@
+package yqlib
+
+import (
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/magiconair/properties"
+	yaml "gopkg.in/yaml.v3"
+)
+
+type propertiesEncoder struct {
+	destination io.Writer
+}
+
+func NewPropertiesEncoder(destination io.Writer) Encoder {
+	return &propertiesEncoder{destination}
+}
+
+func (pe *propertiesEncoder) Encode(node *yaml.Node) error {
+	mapKeysToStrings(node)
+	p := properties.NewProperties()
+	err := pe.doEncode(p, node, "")
+	if err != nil {
+		return err
+	}
+
+	_, err = p.WriteComment(pe.destination, "#", properties.UTF8)
+	return err
+}
+
+func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yaml.Node, path string) error {
+	p.SetComment(path,
+		strings.Replace(node.HeadComment, "#", "", 1)+
+			strings.Replace(node.LineComment, "#", "", 1))
+	switch node.Kind {
+	case yaml.ScalarNode:
+		_, _, err := p.Set(path, node.Value)
+		return err
+	case yaml.DocumentNode:
+		return pe.doEncode(p, node.Content[0], path)
+	case yaml.SequenceNode:
+		return pe.encodeArray(p, node.Content, path)
+	case yaml.MappingNode:
+		return pe.encodeMap(p, node.Content, path)
+	case yaml.AliasNode:
+		return pe.doEncode(p, node.Alias, path)
+	default:
+		return fmt.Errorf("Unsupported node %v", node.Tag)
+	}
+}
+
+func (pe *propertiesEncoder) appendPath(path string, key interface{}) string {
+	if path == "" {
+		return fmt.Sprintf("%v", key)
+	}
+	return fmt.Sprintf("%v.%v", path, key)
+}
+
+func (pe *propertiesEncoder) encodeArray(p *properties.Properties, kids []*yaml.Node, path string) error {
+	for index, child := range kids {
+		err := pe.doEncode(p, child, pe.appendPath(path, index))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (pe *propertiesEncoder) encodeMap(p *properties.Properties, kids []*yaml.Node, path string) error {
+	for index := 0; index < len(kids); index = index + 2 {
+		key := kids[index]
+		value := kids[index+1]
+		err := pe.doEncode(p, value, pe.appendPath(path, key.Value))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/yqlib/encoder_properties_test.go

@@ -0,0 +1,78 @@
+package yqlib
+
+import (
+	"bufio"
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+func yamlToProps(sampleYaml string) string {
+	var output bytes.Buffer
+	writer := bufio.NewWriter(&output)
+
+	var propsEncoder = NewPropertiesEncoder(writer)
+	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0)
+	if err != nil {
+		panic(err)
+	}
+	node := inputs.Front().Value.(*CandidateNode).Node
+	err = propsEncoder.Encode(node)
+	if err != nil {
+		panic(err)
+	}
+	writer.Flush()
+
+	return strings.TrimSuffix(output.String(), "\n")
+}
+
+func TestPropertiesEncoderSimple(t *testing.T) {
+	var sampleYaml = `a: 'bob cool'`
+
+	var expectedJson = `a = bob cool`
+	var actualProps = yamlToProps(sampleYaml)
+	test.AssertResult(t, expectedJson, actualProps)
+}
+
+func TestPropertiesEncoderSimpleWithComments(t *testing.T) {
+	var sampleYaml = `a: 'bob cool' # line`
+
+	var expectedJson = `# line
+a = bob cool`
+	var actualProps = yamlToProps(sampleYaml)
+	test.AssertResult(t, expectedJson, actualProps)
+}
+
+func TestPropertiesEncoderDeep(t *testing.T) {
+	var sampleYaml = `a: 
+  b: "bob cool"
+`
+
+	var expectedJson = `a.b = bob cool`
+	var actualProps = yamlToProps(sampleYaml)
+	test.AssertResult(t, expectedJson, actualProps)
+}
+
+func TestPropertiesEncoderDeepWithComments(t *testing.T) {
+	var sampleYaml = `a:  # a thing
+  b: "bob cool" # b thing
+`
+
+	var expectedJson = `# b thing
+a.b = bob cool`
+	var actualProps = yamlToProps(sampleYaml)
+	test.AssertResult(t, expectedJson, actualProps)
+}
+
+func TestPropertiesEncoderArray(t *testing.T) {
+	var sampleYaml = `a: 
+  b: [{c: dog}, {c: cat}]
+`
+
+	var expectedJson = `a.b.0.c = dog
+a.b.1.c = cat`
+	var actualProps = yamlToProps(sampleYaml)
+	test.AssertResult(t, expectedJson, actualProps)
+}

pkg/yqlib/front_matter.go

@@ -9,47 +9,48 @@ import (
 type frontMatterHandler interface {
 	Split() error
 	GetYamlFrontMatterFilename() string
-	GetContentFilename() string
+	GetContentReader() io.Reader
 	CleanUp()
 }
 
 type frontMatterHandlerImpl struct {
 	originalFilename        string
 	yamlFrontMatterFilename string
-	contentFilename         string
+	contentReader           io.Reader
 }
 
 func NewFrontMatterHandler(originalFilename string) frontMatterHandler {
-	return &frontMatterHandlerImpl{originalFilename, "", ""}
+	return &frontMatterHandlerImpl{originalFilename, "", nil}
 }
 
 func (f *frontMatterHandlerImpl) GetYamlFrontMatterFilename() string {
 	return f.yamlFrontMatterFilename
 }
 
-func (f *frontMatterHandlerImpl) GetContentFilename() string {
-	return f.contentFilename
+func (f *frontMatterHandlerImpl) GetContentReader() io.Reader {
+	return f.contentReader
 }
 
 func (f *frontMatterHandlerImpl) CleanUp() {
 	tryRemoveFile(f.yamlFrontMatterFilename)
-	tryRemoveFile(f.contentFilename)
 }
 
 // Splits the given file by yaml front matter
 // yaml content will be saved to first temporary file
 // remaining content will be saved to second temporary file
 func (f *frontMatterHandlerImpl) Split() error {
-	var reader io.Reader
+	var reader *bufio.Reader
 	var err error
 	if f.originalFilename == "-" {
 		reader = bufio.NewReader(os.Stdin)
 	} else {
-		reader, err = os.Open(f.originalFilename) // #nosec
+		file, err := os.Open(f.originalFilename) // #nosec
 		if err != nil {
 			return err
 		}
+		reader = bufio.NewReader(file)
 	}
+	f.contentReader = reader
 
 	yamlTempFile, err := createTempFile()
 	if err != nil {
@@ -58,39 +59,35 @@ func (f *frontMatterHandlerImpl) Split() error {
 	f.yamlFrontMatterFilename = yamlTempFile.Name()
 	log.Debug("yamlTempFile: %v", yamlTempFile.Name())
 
-	contentTempFile, err := createTempFile()
-	if err != nil {
-		return err
-	}
-	f.contentFilename = contentTempFile.Name()
-	log.Debug("contentTempFile: %v", contentTempFile.Name())
-
-	scanner := bufio.NewScanner(reader)
-
 	lineCount := 0
-	yamlContentBlock := true
 
-	for scanner.Scan() {
-		line := scanner.Text()
-
-		if lineCount > 0 && line == "---" {
-			//we've finished reading the yaml content
-			yamlContentBlock = false
-		}
-		if yamlContentBlock {
-			_, err = yamlTempFile.Write([]byte(line + "\n"))
-		} else {
-			_, err = contentTempFile.Write([]byte(line + "\n"))
-		}
-		if err != nil {
+	for {
+		peekBytes, err := reader.Peek(3)
+		if err == io.EOF {
+			// we've finished reading the yaml content..I guess
+			break
+		} else if err != nil {
 			return err
 		}
+		if lineCount > 0 && string(peekBytes) == "---" {
+			// we've finished reading the yaml content..
+			break
+		}
+		line, errReading := reader.ReadString('\n')
 		lineCount = lineCount + 1
+		if errReading != nil && errReading != io.EOF {
+			return errReading
+		}
+
+		_, errWriting := yamlTempFile.Write([]byte(line))
+
+		if errWriting != nil {
+			return errWriting
+		}
 	}
 
 	safelyCloseFile(yamlTempFile)
-	safelyCloseFile(contentTempFile)
 
-	return scanner.Err()
+	return nil
 
 }

pkg/yqlib/front_matter_test.go

@@ -60,8 +60,11 @@ yaml: doc
 
 	test.AssertResult(t, expectedYamlFm, yamlFm)
 
-	content := readFile(fmHandler.GetContentFilename())
-	test.AssertResult(t, expectedContent, content)
+	contentBytes, err := ioutil.ReadAll(fmHandler.GetContentReader())
+	if err != nil {
+		panic(err)
+	}
+	test.AssertResult(t, expectedContent, string(contentBytes))
 
 	tryRemoveFile(file)
 	fmHandler.CleanUp()
@@ -94,8 +97,11 @@ yaml: doc
 
 	test.AssertResult(t, expectedYamlFm, yamlFm)
 
-	content := readFile(fmHandler.GetContentFilename())
-	test.AssertResult(t, expectedContent, content)
+	contentBytes, err := ioutil.ReadAll(fmHandler.GetContentReader())
+	if err != nil {
+		panic(err)
+	}
+	test.AssertResult(t, expectedContent, string(contentBytes))
 
 	tryRemoveFile(file)
 	fmHandler.CleanUp()
@@ -125,8 +131,11 @@ yaml: doc
 
 	test.AssertResult(t, expectedYamlFm, yamlFm)
 
-	content := readFile(fmHandler.GetContentFilename())
-	test.AssertResult(t, expectedContent, content)
+	contentBytes, err := ioutil.ReadAll(fmHandler.GetContentReader())
+	if err != nil {
+		panic(err)
+	}
+	test.AssertResult(t, expectedContent, string(contentBytes))
 
 	tryRemoveFile(file)
 	fmHandler.CleanUp()

pkg/yqlib/operator_multiply.go

@@ -31,7 +31,7 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
 			(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
 
 			var newBlank = lhs.CreateChild(nil, &yaml.Node{})
-
+			log.Debugf("merge - merge lhs into blank")
 			var newThing, err = mergeObjects(d, context.WritableClone(), newBlank, lhs, multiplyPreferences{})
 			if err != nil {
 				return nil, err
@@ -83,12 +83,13 @@ func multiplyIntegers(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, e
 }
 
 func mergeObjects(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode, preferences multiplyPreferences) (*CandidateNode, error) {
-	shouldAppendArrays := preferences.AppendArrays
 	var results = list.New()
 
-	// shouldn't recurse arrays if appending
-	prefs := recursiveDescentPreferences{RecurseArray: !shouldAppendArrays,
+	// only need to recurse the array if we are doing a deep merge
+	prefs := recursiveDescentPreferences{RecurseArray: preferences.DeepMergeArrays,
 		TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: true}}
+	log.Debugf("merge - preferences.DeepMergeArrays %v", preferences.DeepMergeArrays)
+	log.Debugf("merge - preferences.AppendArrays %v", preferences.AppendArrays)
 	err := recursiveDecent(d, results, context.SingleChildContext(rhs), prefs)
 	if err != nil {
 		return nil, err
@@ -118,14 +119,22 @@ func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom
 	log.Debugf("merge - applyAssignment lhs %v, rhs: %v", lhs.GetKey(), rhs.GetKey())
 
 	lhsPath := rhs.Path[pathIndexToStartFrom:]
+	log.Debugf("merge - lhsPath %v", lhsPath)
 
 	assignmentOp := &Operation{OperationType: assignAttributesOpType}
 	if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode {
 		assignmentOp.OperationType = addAssignOpType
+		log.Debugf("merge - assignmentOp.OperationType = addAssignOpType")
 	} else if !preferences.DeepMergeArrays && rhs.Node.Kind == yaml.SequenceNode ||
 		(rhs.Node.Kind == yaml.ScalarNode || rhs.Node.Kind == yaml.AliasNode) {
 		assignmentOp.OperationType = assignOpType
 		assignmentOp.UpdateAssign = false
+		log.Debugf("merge - rhs.Node.Kind == yaml.SequenceNode: %v", rhs.Node.Kind == yaml.SequenceNode)
+		log.Debugf("merge - rhs.Node.Kind == yaml.ScalarNode: %v", rhs.Node.Kind == yaml.ScalarNode)
+		log.Debugf("merge - rhs.Node.Kind == yaml.AliasNode: %v", rhs.Node.Kind == yaml.AliasNode)
+		log.Debugf("merge - assignmentOp.OperationType = assignOpType, no updateassign")
+	} else {
+		log.Debugf("merge - assignmentOp := &Operation{OperationType: assignAttributesOpType}")
 	}
 	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs}
 

pkg/yqlib/operator_multiply_test.go

@@ -24,6 +24,11 @@ list2:
     - "123"
 `
 
+var mergeArrayWithAnchors = `sample:
+- &a
+- <<: *a
+`
+
 var mergeArraysObjectKeysText = `It's a complex command, the trickyness comes from needing to have the right context in the expressions.
 First we save the second array into a variable '$two' which lets us reference it later.
 We then need to update the first array. We will use the relative update (|=) because we need to update relative to the current element of the array in the LHS in the RHS expression. 
@@ -31,6 +36,14 @@ We set the current element of the first array as $cur. Now we multiply (merge) $
 `
 
 var multiplyOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   mergeArrayWithAnchors,
+		expression: `. * .`,
+		expected: []string{
+			"D0, P[], (!!map)::sample:\n    - &a\n    - !!merge <<: *a\n",
+		},
+	},
 	{
 		description: "Multiply integers",
 		expression:  `3 * 4`,

pkg/yqlib/operators_test.go

@@ -107,7 +107,7 @@ func copyFromHeader(title string, out *os.File) error {
 
 func formatYaml(yaml string, filename string) string {
 	var output bytes.Buffer
-	printer := NewPrinter(bufio.NewWriter(&output), false, true, false, 2, true)
+	printer := NewPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
 
 	node, err := NewExpressionParser().ParseExpression(".. style= \"\"")
 	if err != nil {
@@ -216,7 +216,7 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) {
 func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formattedDoc string, formattedDoc2 string) {
 	var output bytes.Buffer
 	var err error
-	printer := NewPrinter(bufio.NewWriter(&output), false, true, false, 2, true)
+	printer := NewPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
 
 	node, err := NewExpressionParser().ParseExpression(s.expression)
 	if err != nil {

pkg/yqlib/printer.go

@@ -3,6 +3,7 @@ package yqlib
 import (
 	"bufio"
 	"container/list"
+	"fmt"
 	"io"
 	"strings"
 
@@ -16,8 +17,29 @@ type Printer interface {
 	SetAppendix(reader io.Reader)
 }
 
+type PrinterOutputFormat uint32
+
+const (
+	YamlOutputFormat = 1 << iota
+	JsonOutputFormat
+	PropsOutputFormat
+)
+
+func OutputFormatFromString(format string) (PrinterOutputFormat, error) {
+	switch format {
+	case "yaml":
+		return YamlOutputFormat, nil
+	case "json":
+		return JsonOutputFormat, nil
+	case "props":
+		return PropsOutputFormat, nil
+	default:
+		return 0, fmt.Errorf("Unknown fromat '%v' please use [yaml|json|props]", format)
+	}
+}
+
 type resultsPrinter struct {
-	outputToJSON       bool
+	outputFormat       PrinterOutputFormat
 	unwrapScalar       bool
 	colorsEnabled      bool
 	indent             int
@@ -28,27 +50,22 @@ type resultsPrinter struct {
 	previousFileIndex  int
 	printedMatches     bool
 	treeNavigator      DataTreeNavigator
-	preambleReader     io.Reader
 	appendixReader     io.Reader
 }
 
-func NewPrinter(writer io.Writer, outputToJSON bool, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
+func NewPrinter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
 	return &resultsPrinter{
 		writer:             writer,
-		outputToJSON:       outputToJSON,
+		outputFormat:       outputFormat,
 		unwrapScalar:       unwrapScalar,
 		colorsEnabled:      colorsEnabled,
 		indent:             indent,
-		printDocSeparators: !outputToJSON && printDocSeparators,
+		printDocSeparators: outputFormat == YamlOutputFormat && printDocSeparators,
 		firstTimePrinting:  true,
 		treeNavigator:      NewDataTreeNavigator(),
 	}
 }
 
-func (p *resultsPrinter) SetPreamble(reader io.Reader) {
-	p.preambleReader = reader
-}
-
 func (p *resultsPrinter) SetAppendix(reader io.Reader) {
 	p.appendixReader = reader
 }
@@ -62,11 +79,14 @@ func (p *resultsPrinter) printNode(node *yaml.Node, writer io.Writer) error {
 		(node.Tag != "!!bool" || node.Value != "false"))
 
 	var encoder Encoder
-	if node.Kind == yaml.ScalarNode && p.unwrapScalar && !p.outputToJSON {
+	if node.Kind == yaml.ScalarNode && p.unwrapScalar && p.outputFormat == YamlOutputFormat {
 		return p.writeString(writer, node.Value+"\n")
 	}
-	if p.outputToJSON {
+
+	if p.outputFormat == JsonOutputFormat {
 		encoder = NewJsonEncoder(writer, p.indent)
+	} else if p.outputFormat == PropsOutputFormat {
+		encoder = NewPropertiesEncoder(writer)
 	} else {
 		encoder = NewYamlEncoder(writer, p.indent, p.colorsEnabled)
 	}
@@ -87,7 +107,7 @@ func (p *resultsPrinter) safelyFlush(writer *bufio.Writer) {
 
 func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 	log.Debug("PrintResults for %v matches", matchingNodes.Len())
-	if p.outputToJSON {
+	if p.outputFormat != YamlOutputFormat {
 		explodeOp := Operation{OperationType: explodeOpType}
 		explodeNode := ExpressionNode{Operation: &explodeOp}
 		context, err := p.treeNavigator.GetMatchingNodes(Context{MatchingNodes: matchingNodes}, &explodeNode)
@@ -145,18 +165,25 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 							return err
 						}
 					}
-				} else if !p.outputToJSON {
+				} else if p.outputFormat == YamlOutputFormat {
 					if err := p.writeString(bufferedWriter, readline); err != nil {
 						return err
 					}
 				}
 
 				if errReading == io.EOF {
+					if readline != "" {
+						// the last comment we read didn't have a new line, put one in
+						if err := p.writeString(bufferedWriter, "\n"); err != nil {
+							return err
+						}
+					}
 					break
 				}
 			}
 
 		}
+
 		if err := p.printNode(mappedDoc.Node, bufferedWriter); err != nil {
 			return err
 		}
@@ -164,7 +191,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 		p.previousDocIndex = mappedDoc.Document
 	}
 
-	if p.appendixReader != nil && !p.outputToJSON {
+	if p.appendixReader != nil && p.outputFormat == YamlOutputFormat {
 		log.Debug("Piping appendix reader...")
 		betterReader := bufio.NewReader(p.appendixReader)
 		_, err := io.Copy(bufferedWriter, betterReader)

pkg/yqlib/printer_test.go

@@ -36,7 +36,7 @@ func nodeToList(candidate *CandidateNode) *list.List {
 func TestPrinterMultipleDocsInSequence(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, false, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -74,7 +74,7 @@ func TestPrinterMultipleDocsInSequence(t *testing.T) {
 func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, false, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -116,7 +116,7 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
 func TestPrinterMultipleFilesInSequence(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, false, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -163,7 +163,7 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) {
 func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, false, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -213,7 +213,7 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
 func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, false, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -232,7 +232,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, false, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -261,7 +261,7 @@ a: coconut
 func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, false, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -287,7 +287,7 @@ a: coconut
 func TestPrinterScalarWithLeadingCont(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, false, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	node, err := NewExpressionParser().ParseExpression(".a")
 	if err != nil {
@@ -314,7 +314,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	// note printDocSeparators is true, it should still not print document separators
 	// when outputing JSON.
-	printer := NewPrinter(writer, true, true, false, 0, true)
+	printer := NewPrinter(writer, JsonOutputFormat, true, false, 0, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.11.0'
+version: '4.11.2'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.