diff --git a/cmd/evaluate_all_command.go b/cmd/evaluate_all_command.go index 312688e4..460f0cb7 100644 --- a/cmd/evaluate_all_command.go +++ b/cmd/evaluate_all_command.go @@ -40,23 +40,23 @@ func evaluateAll(cmd *cobra.Command, args []string) error { colorsEnabled = true } printer := yqlib.NewPrinter(out, outputToJSON, unwrapScalar, colorsEnabled, indent, !noDocSeparators) - + allAtOnceEvaluator := yqlib.NewAllAtOnceEvaluator() switch len(args) { case 0: if pipingStdIn { - err = yqlib.EvaluateAllFileStreams("", []string{"-"}, printer) + err = allAtOnceEvaluator.EvaluateFiles("", []string{"-"}, printer) } else { cmd.Println(cmd.UsageString()) return nil } case 1: if nullInput { - err = yqlib.EvaluateAllFileStreams(args[0], []string{}, printer) + err = allAtOnceEvaluator.EvaluateFiles(args[0], []string{}, printer) } else { - err = yqlib.EvaluateAllFileStreams("", []string{args[0]}, printer) + err = allAtOnceEvaluator.EvaluateFiles("", []string{args[0]}, printer) } default: - err = yqlib.EvaluateAllFileStreams(args[0], args[1:], printer) + err = allAtOnceEvaluator.EvaluateFiles(args[0], args[1:], printer) } cmd.SilenceUsage = true diff --git a/cmd/evalute_sequence_command.go b/cmd/evalute_sequence_command.go index 191293b0..56125fbb 100644 --- a/cmd/evalute_sequence_command.go +++ b/cmd/evalute_sequence_command.go @@ -41,22 +41,25 @@ func evaluateSequence(cmd *cobra.Command, args []string) error { } printer := yqlib.NewPrinter(out, outputToJSON, unwrapScalar, colorsEnabled, indent, !noDocSeparators) + streamEvaluator := yqlib.NewStreamEvaluator() + allAtOnceEvaluator := yqlib.NewAllAtOnceEvaluator() + switch len(args) { case 0: if pipingStdIn { - err = yqlib.EvaluateFileStreamsSequence("", []string{"-"}, printer) + err = streamEvaluator.EvaluateFiles("", []string{"-"}, printer) } else { cmd.Println(cmd.UsageString()) return nil } case 1: if nullInput { - err = yqlib.EvaluateAllFileStreams(args[0], []string{}, printer) + err = allAtOnceEvaluator.EvaluateFiles(args[0], []string{}, printer) } else { - err = yqlib.EvaluateFileStreamsSequence("", []string{args[0]}, printer) + err = streamEvaluator.EvaluateFiles("", []string{args[0]}, printer) } default: - err = yqlib.EvaluateFileStreamsSequence(args[0], args[1:], printer) + err = streamEvaluator.EvaluateFiles(args[0], args[1:], printer) } cmd.SilenceUsage = true diff --git a/pkg/yqlib/all_at_once_evaluator.go b/pkg/yqlib/all_at_once_evaluator.go new file mode 100644 index 00000000..4082c7e2 --- /dev/null +++ b/pkg/yqlib/all_at_once_evaluator.go @@ -0,0 +1,45 @@ +package yqlib + +import "container/list" + +/** + Loads all yaml documents of all files given into memory, then runs the given expression once. +**/ +type Evaluator interface { + EvaluateFiles(expression string, filenames []string, printer Printer) error +} + +type allAtOnceEvaluator struct { + treeNavigator DataTreeNavigator + treeCreator PathTreeCreator +} + +func NewAllAtOnceEvaluator() Evaluator { + return &allAtOnceEvaluator{treeNavigator: NewDataTreeNavigator(), treeCreator: NewPathTreeCreator()} +} + +func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer) error { + fileIndex := 0 + node, err := treeCreator.ParsePath(expression) + if err != nil { + return err + } + var allDocuments *list.List = list.New() + for _, filename := range filenames { + reader, err := readStream(filename) + if err != nil { + return err + } + fileDocuments, err := readDocuments(reader, filename, fileIndex) + if err != nil { + return err + } + allDocuments.PushBackList(fileDocuments) + fileIndex = fileIndex + 1 + } + matches, err := treeNavigator.GetMatchingNodes(allDocuments, node) + if err != nil { + return err + } + return printer.PrintResults(matches) +} diff --git a/pkg/yqlib/data_tree_navigator.go b/pkg/yqlib/data_tree_navigator.go index 78ff4410..f8acc73f 100644 --- a/pkg/yqlib/data_tree_navigator.go +++ b/pkg/yqlib/data_tree_navigator.go @@ -5,17 +5,9 @@ import ( "container/list" - "gopkg.in/op/go-logging.v1" + logging "gopkg.in/op/go-logging.v1" ) -type dataTreeNavigator struct { - navigationPrefs NavigationPrefs -} - -type NavigationPrefs struct { - FollowAlias bool -} - type DataTreeNavigator interface { // given a list of CandidateEntities and a pathNode, // this will process the list against the given pathNode and return @@ -23,8 +15,11 @@ type DataTreeNavigator interface { GetMatchingNodes(matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) } -func NewDataTreeNavigator(navigationPrefs NavigationPrefs) DataTreeNavigator { - return &dataTreeNavigator{navigationPrefs} +type dataTreeNavigator struct { +} + +func NewDataTreeNavigator() DataTreeNavigator { + return &dataTreeNavigator{} } func (d *dataTreeNavigator) GetMatchingNodes(matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) { diff --git a/pkg/yqlib/doc/File Operators.md b/pkg/yqlib/doc/File Operators.md index bf122b63..a6edf7ba 100644 --- a/pkg/yqlib/doc/File Operators.md +++ b/pkg/yqlib/doc/File Operators.md @@ -29,6 +29,6 @@ yq eval 'fileIndex' sample.yml ``` will output ```yaml -73 +0 ``` diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index c9713b00..6c929f6a 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -90,7 +90,8 @@ func formatYaml(yaml string) string { if err != nil { panic(err) } - err = EvaluateStream("sample.yaml", strings.NewReader(yaml), node, printer) + streamEvaluator := NewStreamEvaluator() + err = streamEvaluator.Evaluate("sample.yaml", strings.NewReader(yaml), node, printer) if err != nil { panic(err) } @@ -161,12 +162,14 @@ func documentScenarios(t *testing.T, title string, scenarios []expressionScenari if err != nil { t.Error(err) } - err = EvaluateStream("sample.yaml", strings.NewReader(formattedDoc), node, printer) + streamEvaluator := NewStreamEvaluator() + err = streamEvaluator.Evaluate("sample.yaml", strings.NewReader(formattedDoc), node, printer) if err != nil { t.Error(err) } } else { - err = EvaluateAllFileStreams(s.expression, []string{}, printer) + allAtOnceEvaluator := NewAllAtOnceEvaluator() + err = allAtOnceEvaluator.EvaluateFiles(s.expression, []string{}, printer) if err != nil { t.Error(err) } diff --git a/pkg/yqlib/stream_evaluator.go b/pkg/yqlib/stream_evaluator.go new file mode 100644 index 00000000..d4c22f19 --- /dev/null +++ b/pkg/yqlib/stream_evaluator.go @@ -0,0 +1,85 @@ +package yqlib + +import ( + "container/list" + "io" + "os" + + yaml "gopkg.in/yaml.v3" +) + +type StreamEvaluator interface { + Evaluate(filename string, reader io.Reader, node *PathTreeNode, printer Printer) error + EvaluateFiles(expression string, filenames []string, printer Printer) error +} + +type streamEvaluator struct { + treeNavigator DataTreeNavigator + treeCreator PathTreeCreator + fileIndex int +} + +func NewStreamEvaluator() StreamEvaluator { + return &streamEvaluator{treeNavigator: NewDataTreeNavigator(), treeCreator: NewPathTreeCreator()} +} + +func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer) error { + + node, err := treeCreator.ParsePath(expression) + if err != nil { + return err + } + + for _, filename := range filenames { + reader, err := readStream(filename) + if err != nil { + return err + } + err = s.Evaluate(filename, reader, node, printer) + if err != nil { + return err + } + + switch reader := reader.(type) { + case *os.File: + safelyCloseFile(reader) + } + } + return nil +} + +func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *PathTreeNode, printer Printer) error { + + var currentIndex uint + + decoder := yaml.NewDecoder(reader) + for { + var dataBucket yaml.Node + errorReading := decoder.Decode(&dataBucket) + + if errorReading == io.EOF { + s.fileIndex = s.fileIndex + 1 + return nil + } else if errorReading != nil { + return errorReading + } + candidateNode := &CandidateNode{ + Document: currentIndex, + Filename: filename, + Node: &dataBucket, + FileIndex: s.fileIndex, + } + inputList := list.New() + inputList.PushBack(candidateNode) + + matches, errorParsing := treeNavigator.GetMatchingNodes(inputList, node) + if errorParsing != nil { + return errorParsing + } + err := printer.PrintResults(matches) + if err != nil { + return err + } + currentIndex = currentIndex + 1 + } +} diff --git a/pkg/yqlib/utils.go b/pkg/yqlib/utils.go index 4833094b..d46cdd77 100644 --- a/pkg/yqlib/utils.go +++ b/pkg/yqlib/utils.go @@ -11,11 +11,9 @@ import ( //TODO: convert to interface + struct -var treeNavigator = NewDataTreeNavigator(NavigationPrefs{}) +var treeNavigator = NewDataTreeNavigator() var treeCreator = NewPathTreeCreator() -var fileIndex = 0 - func readStream(filename string) (io.Reader, error) { if filename == "-" { return bufio.NewReader(os.Stdin), nil @@ -24,42 +22,6 @@ func readStream(filename string) (io.Reader, error) { } } -func EvaluateStream(filename string, reader io.Reader, node *PathTreeNode, printer Printer) error { - - var currentIndex uint = 0 - - decoder := yaml.NewDecoder(reader) - for { - var dataBucket yaml.Node - errorReading := decoder.Decode(&dataBucket) - - if errorReading == io.EOF { - fileIndex = fileIndex + 1 - return nil - } else if errorReading != nil { - return errorReading - } - candidateNode := &CandidateNode{ - Document: currentIndex, - Filename: filename, - Node: &dataBucket, - FileIndex: fileIndex, - } - inputList := list.New() - inputList.PushBack(candidateNode) - - matches, errorParsing := treeNavigator.GetMatchingNodes(inputList, node) - if errorParsing != nil { - return errorParsing - } - err := printer.PrintResults(matches) - if err != nil { - return err - } - currentIndex = currentIndex + 1 - } -} - func readDocuments(reader io.Reader, filename string, fileIndex int) (*list.List, error) { decoder := yaml.NewDecoder(reader) inputList := list.New() @@ -91,57 +53,6 @@ func readDocuments(reader io.Reader, filename string, fileIndex int) (*list.List } } -func EvaluateAllFileStreams(expression string, filenames []string, printer Printer) error { - fileIndex := 0 - node, err := treeCreator.ParsePath(expression) - if err != nil { - return err - } - var allDocuments *list.List = list.New() - for _, filename := range filenames { - reader, err := readStream(filename) - if err != nil { - return err - } - fileDocuments, err := readDocuments(reader, filename, fileIndex) - if err != nil { - return err - } - allDocuments.PushBackList(fileDocuments) - fileIndex = fileIndex + 1 - } - matches, err := treeNavigator.GetMatchingNodes(allDocuments, node) - if err != nil { - return err - } - return printer.PrintResults(matches) -} - -func EvaluateFileStreamsSequence(expression string, filenames []string, printer Printer) error { - - node, err := treeCreator.ParsePath(expression) - if err != nil { - return err - } - - for _, filename := range filenames { - reader, err := readStream(filename) - if err != nil { - return err - } - err = EvaluateStream(filename, reader, node, printer) - if err != nil { - return err - } - - switch reader := reader.(type) { - case *os.File: - safelyCloseFile(reader) - } - } - return nil -} - // func safelyRenameFile(from string, to string) { // if renameError := os.Rename(from, to); renameError != nil { // log.Debugf("Error renaming from %v to %v, attempting to copy contents", from, to)