From 2edf64182bc65cb65831f2180959a9826cbb2398 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 6 Nov 2020 14:37:01 +1100 Subject: [PATCH] refining --- cmd/constant.go | 15 +----- cmd/evalute_sequence_command.go | 66 +++++++++++++++++++++++ cmd/root.go | 56 ++----------------- pkg/yqlib/data_tree_navigator_test.go | 2 - pkg/yqlib/document_index_operator_test.go | 9 ++++ pkg/yqlib/lib.go | 6 ++- pkg/yqlib/operators_test.go | 30 ++++++----- pkg/yqlib/printer.go | 9 ++++ pkg/yqlib/utils.go | 25 ++++++--- 9 files changed, 129 insertions(+), 89 deletions(-) create mode 100644 cmd/evalute_sequence_command.go diff --git a/cmd/constant.go b/cmd/constant.go index 45400f76..4a17d53e 100644 --- a/cmd/constant.go +++ b/cmd/constant.go @@ -4,29 +4,16 @@ import ( logging "gopkg.in/op/go-logging.v1" ) -var customTag = "" -var printMode = "v" -var printLength = false var unwrapScalar = true -var customStyle = "" -var anchorName = "" -var makeAlias = false var writeInplace = false -var writeScript = "" -var sourceYamlFile = "" var outputToJSON = false var exitStatus = false -var explodeAnchors = false var forceColor = false var forceNoColor = false var colorsEnabled = false -var defaultValue = "" var indent = 2 var printDocSeparators = true -var overwriteFlag = false -var autoCreateFlag = true -var arrayMergeStrategyFlag = "update" -var commentsMergeStrategyFlag = "setWhenBlank" +var nullInput = false var verbose = false var version = false var shellCompletion = "" diff --git a/cmd/evalute_sequence_command.go b/cmd/evalute_sequence_command.go new file mode 100644 index 00000000..fd41bf79 --- /dev/null +++ b/cmd/evalute_sequence_command.go @@ -0,0 +1,66 @@ +package cmd + +import ( + "container/list" + "os" + + "github.com/mikefarah/yq/v4/pkg/yqlib" + "github.com/spf13/cobra" +) + +func createEvaluateSequenceCommand() *cobra.Command { + var cmdEvalSequence = &cobra.Command{ + Use: "eval-seq [expression] [yaml_file1]...", + Aliases: []string{"es"}, + Short: "Apply expression to each document in each yaml file given in sequence", + Example: ` +yq es '.a.b | length' file1.yml file2.yml +yq es < sample.yaml +yq es -n '{"a": "b"}' +`, + Long: "Evaluate Sequence:\nIterate over each yaml document, apply the expression and print the results, in sequence.", + RunE: evaluateSequence, + } + return cmdEvalSequence +} +func evaluateSequence(cmd *cobra.Command, args []string) error { + // 0 args, read std in + // 1 arg, null input, process expression + // 1 arg, read file in sequence + // 2+ args, [0] = expression, file the rest + + var matchingNodes *list.List + var err error + stat, _ := os.Stdin.Stat() + pipingStdIn := (stat.Mode() & os.ModeCharDevice) == 0 + + switch len(args) { + case 0: + if pipingStdIn { + matchingNodes, err = yqlib.Evaluate("-", "") + } else { + cmd.Println(cmd.UsageString()) + return nil + } + case 1: + if nullInput { + matchingNodes, err = yqlib.EvaluateExpression(args[0]) + } else { + matchingNodes, err = yqlib.Evaluate(args[0], "") + } + } + cmd.SilenceUsage = true + if err != nil { + return err + } + out := cmd.OutOrStdout() + + fileInfo, _ := os.Stdout.Stat() + + if forceColor || (!forceNoColor && (fileInfo.Mode()&os.ModeCharDevice) != 0) { + colorsEnabled = true + } + printer := yqlib.NewPrinter(outputToJSON, unwrapScalar, colorsEnabled, indent, printDocSeparators) + + return printer.PrintResults(matchingNodes, out) +} diff --git a/cmd/root.go b/cmd/root.go index 17602f65..cae90a58 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,11 +1,9 @@ package cmd import ( - "errors" "fmt" "os" - "github.com/mikefarah/yq/v4/pkg/yqlib" "github.com/spf13/cobra" logging "gopkg.in/op/go-logging.v1" ) @@ -34,51 +32,9 @@ func New() *cobra.Command { return fmt.Errorf("Unknown variant %v", shellCompletion) } } - // if len(args) == 0 { - // cmd.Println(cmd.UsageString()) - // return nil - // } - cmd.SilenceUsage = true + cmd.Println(cmd.UsageString()) + return nil - var treeCreator = yqlib.NewPathTreeCreator() - - expression := "" - if len(args) > 0 { - expression = args[0] - } - - pathNode, err := treeCreator.ParsePath(expression) - if err != nil { - return err - } - - if outputToJSON { - explodeOp := yqlib.Operation{OperationType: yqlib.Explode} - explodeNode := yqlib.PathTreeNode{Operation: &explodeOp} - pipeOp := yqlib.Operation{OperationType: yqlib.Pipe} - pathNode = &yqlib.PathTreeNode{Operation: &pipeOp, Lhs: pathNode, Rhs: &explodeNode} - } - - matchingNodes, err := yqlib.Evaluate("-", pathNode) - if err != nil { - return err - } - - if exitStatus && matchingNodes.Len() == 0 { - cmd.SilenceUsage = true - return errors.New("No matches found") - } - - out := cmd.OutOrStdout() - - fileInfo, _ := os.Stdout.Stat() - - if forceColor || (!forceNoColor && (fileInfo.Mode()&os.ModeCharDevice) != 0) { - colorsEnabled = true - } - printer := yqlib.NewPrinter(outputToJSON, unwrapScalar, colorsEnabled, indent, printDocSeparators) - - return printer.PrintResults(matchingNodes, out) }, PersistentPreRun: func(cmd *cobra.Command, args []string) { cmd.SetOut(cmd.OutOrStdout()) @@ -100,6 +56,8 @@ func New() *cobra.Command { rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "verbose mode") rootCmd.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "output as json. Set indent to 0 to print json in one line.") + rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating yaml docs from scratch.") + rootCmd.PersistentFlags().IntVarP(&indent, "indent", "I", 2, "sets indent level for output") rootCmd.Flags().BoolVarP(&version, "version", "V", false, "Print version information and quit") @@ -107,10 +65,6 @@ func New() *cobra.Command { rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors") rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors") - - // rootCmd.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)") - rootCmd.PersistentFlags().StringVarP(&printMode, "printMode", "p", "v", "print mode (v (values, default), p (paths), pv (path and value pairs)") - rootCmd.PersistentFlags().StringVarP(&defaultValue, "defaultValue", "D", "", "default value printed when there are no results") - + rootCmd.AddCommand(createEvaluateSequenceCommand()) return rootCmd } diff --git a/pkg/yqlib/data_tree_navigator_test.go b/pkg/yqlib/data_tree_navigator_test.go index d4b18b32..1e2d7b91 100644 --- a/pkg/yqlib/data_tree_navigator_test.go +++ b/pkg/yqlib/data_tree_navigator_test.go @@ -4,8 +4,6 @@ import ( "container/list" ) -var treeCreator = NewPathTreeCreator() - func resultsToString(results *list.List) []string { var pretty []string = make([]string, 0) for el := results.Front(); el != nil; el = el.Next() { diff --git a/pkg/yqlib/document_index_operator_test.go b/pkg/yqlib/document_index_operator_test.go index 1fbea1be..2a4a18bb 100644 --- a/pkg/yqlib/document_index_operator_test.go +++ b/pkg/yqlib/document_index_operator_test.go @@ -22,6 +22,15 @@ var documentIndexScenarios = []expressionScenario{ "D1, P[], (doc)::a: frog\n", }, }, + { + description: "Print Document Index with matches", + document: "a: cat\n---\na: frog\n", + expression: `.a | {"match": ., "doc": (. | documentIndex)}`, + expected: []string{ + "D0, P[], (!!map)::match: cat\ndoc: 0\n", + "D1, P[], (!!map)::match: frog\ndoc: 1\n", + }, + }, } func TestDocumentIndexScenarios(t *testing.T) { diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 6982af1a..044e5648 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -18,7 +18,11 @@ type OperationType struct { // operators TODO: // - generator doc from operator tests -// - documentIndex - retrieves document index, can be used with select +// - slurp - stdin, read in sequence, vs read all +// - write in place +// - get path operator (like doc index) +// - get file index op (like doc index) +// - get file name op (like doc index) // - mergeAppend (merges and appends arrays) // - mergeEmpty (sets only if the document is empty, do I do that now?) // - updateTag - not recursive diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index 692d4ea8..7c271ad6 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -3,6 +3,7 @@ package yqlib import ( "bufio" "bytes" + "container/list" "fmt" "os" "strings" @@ -20,15 +21,16 @@ type expressionScenario struct { } func testScenario(t *testing.T, s *expressionScenario) { - node, errPath := treeCreator.ParsePath(s.expression) - if errPath != nil { - t.Error(errPath) - return + var results *list.List + var err error + if s.document != "" { + results, err = EvaluateStream("sample.yaml", strings.NewReader(s.document), s.expression) + } else { + results, err = EvaluateExpression(s.expression) } - results, errNav := EvaluateStream("sample.yaml", strings.NewReader(s.document), node) - if errNav != nil { - t.Error(errNav) + if err != nil { + t.Error(err) return } test.AssertResultComplexWithContext(t, s.expected, resultsToString(results), fmt.Sprintf("exp: %v\ndoc: %v", s.expression, s.document)) @@ -66,13 +68,15 @@ func documentScenarios(t *testing.T, title string, scenarios []expressionScenari w.WriteString(fmt.Sprintf("Result\n")) - node, errPath := treeCreator.ParsePath(s.expression) - if errPath != nil { - t.Error(errPath) - return - } var output bytes.Buffer - results, err := EvaluateStream("sample.yaml", strings.NewReader(s.document), node) + var results *list.List + var err error + if s.document != "" { + results, err = EvaluateStream("sample.yaml", strings.NewReader(s.document), s.expression) + } else { + results, err = EvaluateExpression(s.expression) + } + printer.PrintResults(results, bufio.NewWriter(&output)) w.WriteString(fmt.Sprintf("```yaml\n%v```\n", output.String())) diff --git a/pkg/yqlib/printer.go b/pkg/yqlib/printer.go index 9aabd7b3..280669b4 100644 --- a/pkg/yqlib/printer.go +++ b/pkg/yqlib/printer.go @@ -43,6 +43,15 @@ func (p *resultsPrinter) writeString(writer io.Writer, txt string) error { } func (p *resultsPrinter) PrintResults(matchingNodes *list.List, writer io.Writer) error { + var err error + if p.outputToJSON { + explodeOp := Operation{OperationType: Explode} + explodeNode := PathTreeNode{Operation: &explodeOp} + matchingNodes, err = treeNavigator.GetMatchingNodes(matchingNodes, &explodeNode) + if err != nil { + return err + } + } bufferedWriter := bufio.NewWriter(writer) defer safelyFlush(bufferedWriter) diff --git a/pkg/yqlib/utils.go b/pkg/yqlib/utils.go index 459a918a..af104efa 100644 --- a/pkg/yqlib/utils.go +++ b/pkg/yqlib/utils.go @@ -3,7 +3,6 @@ package yqlib import ( "bufio" "container/list" - "errors" "io" "os" @@ -11,12 +10,9 @@ import ( ) var treeNavigator = NewDataTreeNavigator(NavigationPrefs{}) +var treeCreator = NewPathTreeCreator() func readStream(filename string) (io.Reader, error) { - if filename == "" { - return nil, errors.New("Must provide filename") - } - var stream io.Reader if filename == "-" { stream = bufio.NewReader(os.Stdin) @@ -31,7 +27,20 @@ func readStream(filename string) (io.Reader, error) { return stream, nil } -func EvaluateStream(filename string, reader io.Reader, node *PathTreeNode) (*list.List, error) { +func EvaluateExpression(expression string) (*list.List, error) { + node, err := treeCreator.ParsePath(expression) + if err != nil { + return nil, err + } + return treeNavigator.GetMatchingNodes(list.New(), node) +} + +func EvaluateStream(filename string, reader io.Reader, expression string) (*list.List, error) { + node, err := treeCreator.ParsePath(expression) + if err != nil { + return nil, err + } + var matchingNodes = list.New() var currentIndex uint = 0 @@ -63,13 +72,13 @@ func EvaluateStream(filename string, reader io.Reader, node *PathTreeNode) (*lis } } -func Evaluate(filename string, node *PathTreeNode) (*list.List, error) { +func Evaluate(filename string, expression string) (*list.List, error) { var reader, err = readStream(filename) if err != nil { return nil, err } - return EvaluateStream(filename, reader, node) + return EvaluateStream(filename, reader, expression) }