refining

cmd/constant.go

@@ -4,29 +4,16 @@ import (
 	logging "gopkg.in/op/go-logging.v1"
 )
 
-var customTag = ""
-var printMode = "v"
-var printLength = false
 var unwrapScalar = true
-var customStyle = ""
-var anchorName = ""
-var makeAlias = false
 var writeInplace = false
-var writeScript = ""
-var sourceYamlFile = ""
 var outputToJSON = false
 var exitStatus = false
-var explodeAnchors = false
 var forceColor = false
 var forceNoColor = false
 var colorsEnabled = false
-var defaultValue = ""
 var indent = 2
 var printDocSeparators = true
-var overwriteFlag = false
+var nullInput = false
-var autoCreateFlag = true
-var arrayMergeStrategyFlag = "update"
-var commentsMergeStrategyFlag = "setWhenBlank"
 var verbose = false
 var version = false
 var shellCompletion = ""

cmd/evalute_sequence_command.go

@@ -0,0 +1,66 @@
+package cmd
+
+import (
+	"container/list"
+	"os"
+
+	"github.com/mikefarah/yq/v4/pkg/yqlib"
+	"github.com/spf13/cobra"
+)
+
+func createEvaluateSequenceCommand() *cobra.Command {
+	var cmdEvalSequence = &cobra.Command{
+		Use:     "eval-seq [expression] [yaml_file1]...",
+		Aliases: []string{"es"},
+		Short:   "Apply expression to each document in each yaml file given in sequence",
+		Example: `
+yq es '.a.b | length' file1.yml file2.yml
+yq es < sample.yaml
+yq es -n '{"a": "b"}'
+`,
+		Long: "Evaluate Sequence:\nIterate over each yaml document, apply the expression and print the results, in sequence.",
+		RunE: evaluateSequence,
+	}
+	return cmdEvalSequence
+}
+func evaluateSequence(cmd *cobra.Command, args []string) error {
+	// 0 args, read std in
+	// 1 arg, null input, process expression
+	// 1 arg, read file in sequence
+	// 2+ args, [0] = expression, file the rest
+
+	var matchingNodes *list.List
+	var err error
+	stat, _ := os.Stdin.Stat()
+	pipingStdIn := (stat.Mode() & os.ModeCharDevice) == 0
+
+	switch len(args) {
+	case 0:
+		if pipingStdIn {
+			matchingNodes, err = yqlib.Evaluate("-", "")
+		} else {
+			cmd.Println(cmd.UsageString())
+			return nil
+		}
+	case 1:
+		if nullInput {
+			matchingNodes, err = yqlib.EvaluateExpression(args[0])
+		} else {
+			matchingNodes, err = yqlib.Evaluate(args[0], "")
+		}
+	}
+	cmd.SilenceUsage = true
+	if err != nil {
+		return err
+	}
+	out := cmd.OutOrStdout()
+
+	fileInfo, _ := os.Stdout.Stat()
+
+	if forceColor || (!forceNoColor && (fileInfo.Mode()&os.ModeCharDevice) != 0) {
+		colorsEnabled = true
+	}
+	printer := yqlib.NewPrinter(outputToJSON, unwrapScalar, colorsEnabled, indent, printDocSeparators)
+
+	return printer.PrintResults(matchingNodes, out)
+}

cmd/root.go

@@ -1,11 +1,9 @@
 package cmd
 
 import (
-	"errors"
 	"fmt"
 	"os"
 
-	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"github.com/spf13/cobra"
 	logging "gopkg.in/op/go-logging.v1"
 )
@@ -34,51 +32,9 @@ func New() *cobra.Command {
 					return fmt.Errorf("Unknown variant %v", shellCompletion)
 				}
 			}
-			// if len(args) == 0 {
+			cmd.Println(cmd.UsageString())
-			// 	cmd.Println(cmd.UsageString())
+			return nil
-			// 	return nil
-			// }
-			cmd.SilenceUsage = true
 
-			var treeCreator = yqlib.NewPathTreeCreator()
-
-			expression := ""
-			if len(args) > 0 {
-				expression = args[0]
-			}
-
-			pathNode, err := treeCreator.ParsePath(expression)
-			if err != nil {
-				return err
-			}
-
-			if outputToJSON {
-				explodeOp := yqlib.Operation{OperationType: yqlib.Explode}
-				explodeNode := yqlib.PathTreeNode{Operation: &explodeOp}
-				pipeOp := yqlib.Operation{OperationType: yqlib.Pipe}
-				pathNode = &yqlib.PathTreeNode{Operation: &pipeOp, Lhs: pathNode, Rhs: &explodeNode}
-			}
-
-			matchingNodes, err := yqlib.Evaluate("-", pathNode)
-			if err != nil {
-				return err
-			}
-
-			if exitStatus && matchingNodes.Len() == 0 {
-				cmd.SilenceUsage = true
-				return errors.New("No matches found")
-			}
-
-			out := cmd.OutOrStdout()
-
-			fileInfo, _ := os.Stdout.Stat()
-
-			if forceColor || (!forceNoColor && (fileInfo.Mode()&os.ModeCharDevice) != 0) {
-				colorsEnabled = true
-			}
-			printer := yqlib.NewPrinter(outputToJSON, unwrapScalar, colorsEnabled, indent, printDocSeparators)
-
-			return printer.PrintResults(matchingNodes, out)
 		},
 		PersistentPreRun: func(cmd *cobra.Command, args []string) {
 			cmd.SetOut(cmd.OutOrStdout())
@@ -100,6 +56,8 @@ func New() *cobra.Command {
 
 	rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "verbose mode")
 	rootCmd.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "output as json. Set indent to 0 to print json in one line.")
+	rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating yaml docs from scratch.")
+
 	rootCmd.PersistentFlags().IntVarP(&indent, "indent", "I", 2, "sets indent level for output")
 	rootCmd.Flags().BoolVarP(&version, "version", "V", false, "Print version information and quit")
 
@@ -107,10 +65,6 @@ func New() *cobra.Command {
 
 	rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors")
 	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
-
+	rootCmd.AddCommand(createEvaluateSequenceCommand())
-	// rootCmd.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)")
-	rootCmd.PersistentFlags().StringVarP(&printMode, "printMode", "p", "v", "print mode (v (values, default), p (paths), pv (path and value pairs)")
-	rootCmd.PersistentFlags().StringVarP(&defaultValue, "defaultValue", "D", "", "default value printed when there are no results")
-
 	return rootCmd
 }

pkg/yqlib/data_tree_navigator_test.go

@@ -4,8 +4,6 @@ import (
 	"container/list"
 )
 
-var treeCreator = NewPathTreeCreator()
-
 func resultsToString(results *list.List) []string {
 	var pretty []string = make([]string, 0)
 	for el := results.Front(); el != nil; el = el.Next() {

pkg/yqlib/document_index_operator_test.go

@@ -22,6 +22,15 @@ var documentIndexScenarios = []expressionScenario{
 			"D1, P[], (doc)::a: frog\n",
 		},
 	},
+	{
+		description: "Print Document Index with matches",
+		document:    "a: cat\n---\na: frog\n",
+		expression:  `.a | {"match": ., "doc": (. | documentIndex)}`,
+		expected: []string{
+			"D0, P[], (!!map)::match: cat\ndoc: 0\n",
+			"D1, P[], (!!map)::match: frog\ndoc: 1\n",
+		},
+	},
 }
 
 func TestDocumentIndexScenarios(t *testing.T) {

pkg/yqlib/lib.go

@@ -18,7 +18,11 @@ type OperationType struct {
 
 // operators TODO:
 // - generator doc from operator tests
-// - documentIndex - retrieves document index, can be used with select
+// - slurp - stdin, read in sequence, vs read all
+// - write in place
+// - get path operator (like doc index)
+// - get file index op (like doc index)
+// - get file name op (like doc index)
 // - mergeAppend (merges and appends arrays)
 // - mergeEmpty (sets only if the document is empty, do I do that now?)
 // - updateTag - not recursive

pkg/yqlib/operators_test.go

@@ -3,6 +3,7 @@ package yqlib
 import (
 	"bufio"
 	"bytes"
+	"container/list"
 	"fmt"
 	"os"
 	"strings"
@@ -20,15 +21,16 @@ type expressionScenario struct {
 }
 
 func testScenario(t *testing.T, s *expressionScenario) {
-	node, errPath := treeCreator.ParsePath(s.expression)
+	var results *list.List
-	if errPath != nil {
+	var err error
-		t.Error(errPath)
+	if s.document != "" {
-		return
+		results, err = EvaluateStream("sample.yaml", strings.NewReader(s.document), s.expression)
+	} else {
+		results, err = EvaluateExpression(s.expression)
 	}
-	results, errNav := EvaluateStream("sample.yaml", strings.NewReader(s.document), node)
 
-	if errNav != nil {
+	if err != nil {
-		t.Error(errNav)
+		t.Error(err)
 		return
 	}
 	test.AssertResultComplexWithContext(t, s.expected, resultsToString(results), fmt.Sprintf("exp: %v\ndoc: %v", s.expression, s.document))
@@ -66,13 +68,15 @@ func documentScenarios(t *testing.T, title string, scenarios []expressionScenari
 
 			w.WriteString(fmt.Sprintf("Result\n"))
 
-			node, errPath := treeCreator.ParsePath(s.expression)
-			if errPath != nil {
-				t.Error(errPath)
-				return
-			}
 			var output bytes.Buffer
-			results, err := EvaluateStream("sample.yaml", strings.NewReader(s.document), node)
+			var results *list.List
+			var err error
+			if s.document != "" {
+				results, err = EvaluateStream("sample.yaml", strings.NewReader(s.document), s.expression)
+			} else {
+				results, err = EvaluateExpression(s.expression)
+			}
+
 			printer.PrintResults(results, bufio.NewWriter(&output))
 
 			w.WriteString(fmt.Sprintf("```yaml\n%v```\n", output.String()))

pkg/yqlib/printer.go

@@ -43,6 +43,15 @@ func (p *resultsPrinter) writeString(writer io.Writer, txt string) error {
 }
 
 func (p *resultsPrinter) PrintResults(matchingNodes *list.List, writer io.Writer) error {
+	var err error
+	if p.outputToJSON {
+		explodeOp := Operation{OperationType: Explode}
+		explodeNode := PathTreeNode{Operation: &explodeOp}
+		matchingNodes, err = treeNavigator.GetMatchingNodes(matchingNodes, &explodeNode)
+		if err != nil {
+			return err
+		}
+	}
 
 	bufferedWriter := bufio.NewWriter(writer)
 	defer safelyFlush(bufferedWriter)

pkg/yqlib/utils.go

@@ -3,7 +3,6 @@ package yqlib
 import (
 	"bufio"
 	"container/list"
-	"errors"
 	"io"
 	"os"
 
@@ -11,12 +10,9 @@ import (
 )
 
 var treeNavigator = NewDataTreeNavigator(NavigationPrefs{})
+var treeCreator = NewPathTreeCreator()
 
 func readStream(filename string) (io.Reader, error) {
-	if filename == "" {
-		return nil, errors.New("Must provide filename")
-	}
-
 	var stream io.Reader
 	if filename == "-" {
 		stream = bufio.NewReader(os.Stdin)
@@ -31,7 +27,20 @@ func readStream(filename string) (io.Reader, error) {
 	return stream, nil
 }
 
-func EvaluateStream(filename string, reader io.Reader, node *PathTreeNode) (*list.List, error) {
+func EvaluateExpression(expression string) (*list.List, error) {
+	node, err := treeCreator.ParsePath(expression)
+	if err != nil {
+		return nil, err
+	}
+	return treeNavigator.GetMatchingNodes(list.New(), node)
+}
+
+func EvaluateStream(filename string, reader io.Reader, expression string) (*list.List, error) {
+	node, err := treeCreator.ParsePath(expression)
+	if err != nil {
+		return nil, err
+	}
+
 	var matchingNodes = list.New()
 
 	var currentIndex uint = 0
@@ -63,13 +72,13 @@ func EvaluateStream(filename string, reader io.Reader, node *PathTreeNode) (*lis
 	}
 }
 
-func Evaluate(filename string, node *PathTreeNode) (*list.List, error) {
+func Evaluate(filename string, expression string) (*list.List, error) {
 
 	var reader, err = readStream(filename)
 	if err != nil {
 		return nil, err
 	}
-	return EvaluateStream(filename, reader, node)
+	return EvaluateStream(filename, reader, expression)
 
 }