suyiiyii
/
yq
mirror of https://github.com/mikefarah/yq.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Extracted out evaluators

This commit is contained in:
Mike Farah 2020-11-22 11:56:28 +11:00
parent e451119014
commit fc3af441e5
8 changed files with 156 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
cmd/evaluate_all_command.go
View File

@ -40,23 +40,23 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
colorsEnabled = true
}
printer := yqlib.NewPrinter(out, outputToJSON, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
allAtOnceEvaluator := yqlib.NewAllAtOnceEvaluator()
switch len(args) {
case 0:
if pipingStdIn {
err = yqlib.EvaluateAllFileStreams("", []string{"-"}, printer)
err = allAtOnceEvaluator.EvaluateFiles("", []string{"-"}, printer)
} else {
cmd.Println(cmd.UsageString())
return nil
}
case 1:
if nullInput {
err = yqlib.EvaluateAllFileStreams(args[0], []string{}, printer)
err = allAtOnceEvaluator.EvaluateFiles(args[0], []string{}, printer)
} else {
err = yqlib.EvaluateAllFileStreams("", []string{args[0]}, printer)
err = allAtOnceEvaluator.EvaluateFiles("", []string{args[0]}, printer)
}
default:
err = yqlib.EvaluateAllFileStreams(args[0], args[1:], printer)
err = allAtOnceEvaluator.EvaluateFiles(args[0], args[1:], printer)
}
cmd.SilenceUsage = true

11
cmd/evalute_sequence_command.go
View File

@ -41,22 +41,25 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
}
printer := yqlib.NewPrinter(out, outputToJSON, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
streamEvaluator := yqlib.NewStreamEvaluator()
allAtOnceEvaluator := yqlib.NewAllAtOnceEvaluator()
switch len(args) {
case 0:
if pipingStdIn {
err = yqlib.EvaluateFileStreamsSequence("", []string{"-"}, printer)
err = streamEvaluator.EvaluateFiles("", []string{"-"}, printer)
} else {
cmd.Println(cmd.UsageString())
return nil
}
case 1:
if nullInput {
err = yqlib.EvaluateAllFileStreams(args[0], []string{}, printer)
err = allAtOnceEvaluator.EvaluateFiles(args[0], []string{}, printer)
} else {
err = yqlib.EvaluateFileStreamsSequence("", []string{args[0]}, printer)
err = streamEvaluator.EvaluateFiles("", []string{args[0]}, printer)
}
default:
err = yqlib.EvaluateFileStreamsSequence(args[0], args[1:], printer)
err = streamEvaluator.EvaluateFiles(args[0], args[1:], printer)
}
cmd.SilenceUsage = true

45
pkg/yqlib/all_at_once_evaluator.go Normal file
View File

@ -0,0 +1,45 @@
package yqlib
import "container/list"
/**
Loads all yaml documents of all files given into memory, then runs the given expression once.
**/
type Evaluator interface {
EvaluateFiles(expression string, filenames []string, printer Printer) error
}
type allAtOnceEvaluator struct {
treeNavigator DataTreeNavigator
treeCreator PathTreeCreator
}
func NewAllAtOnceEvaluator() Evaluator {
return &allAtOnceEvaluator{treeNavigator: NewDataTreeNavigator(), treeCreator: NewPathTreeCreator()}
}
func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer) error {
fileIndex := 0
node, err := treeCreator.ParsePath(expression)
if err != nil {
return err
}
var allDocuments *list.List = list.New()
for _, filename := range filenames {
reader, err := readStream(filename)
if err != nil {
return err
}
fileDocuments, err := readDocuments(reader, filename, fileIndex)
if err != nil {
return err
}
allDocuments.PushBackList(fileDocuments)
fileIndex = fileIndex + 1
}
matches, err := treeNavigator.GetMatchingNodes(allDocuments, node)
if err != nil {
return err
}
return printer.PrintResults(matches)
}

17
pkg/yqlib/data_tree_navigator.go
View File

@ -5,17 +5,9 @@ import (
"container/list"
"gopkg.in/op/go-logging.v1"
logging "gopkg.in/op/go-logging.v1"
)
type dataTreeNavigator struct {
navigationPrefs NavigationPrefs
}
type NavigationPrefs struct {
FollowAlias bool
}
type DataTreeNavigator interface {
// given a list of CandidateEntities and a pathNode,
// this will process the list against the given pathNode and return
@ -23,8 +15,11 @@ type DataTreeNavigator interface {
GetMatchingNodes(matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error)
}
func NewDataTreeNavigator(navigationPrefs NavigationPrefs) DataTreeNavigator {
return &dataTreeNavigator{navigationPrefs}
type dataTreeNavigator struct {
}
func NewDataTreeNavigator() DataTreeNavigator {
return &dataTreeNavigator{}
}
func (d *dataTreeNavigator) GetMatchingNodes(matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {

2
pkg/yqlib/doc/File Operators.md
View File

@ -29,6 +29,6 @@ yq eval 'fileIndex' sample.yml
```
will output
```yaml
73
0
```

9
pkg/yqlib/operators_test.go
View File

@ -90,7 +90,8 @@ func formatYaml(yaml string) string {
if err != nil {
panic(err)
}
err = EvaluateStream("sample.yaml", strings.NewReader(yaml), node, printer)
streamEvaluator := NewStreamEvaluator()
err = streamEvaluator.Evaluate("sample.yaml", strings.NewReader(yaml), node, printer)
if err != nil {
panic(err)
}
@ -161,12 +162,14 @@ func documentScenarios(t *testing.T, title string, scenarios []expressionScenari
if err != nil {
t.Error(err)
}
err = EvaluateStream("sample.yaml", strings.NewReader(formattedDoc), node, printer)
streamEvaluator := NewStreamEvaluator()
err = streamEvaluator.Evaluate("sample.yaml", strings.NewReader(formattedDoc), node, printer)
if err != nil {
t.Error(err)
}
} else {
err = EvaluateAllFileStreams(s.expression, []string{}, printer)
allAtOnceEvaluator := NewAllAtOnceEvaluator()
err = allAtOnceEvaluator.EvaluateFiles(s.expression, []string{}, printer)
if err != nil {
t.Error(err)
}

85
pkg/yqlib/stream_evaluator.go Normal file
View File

@ -0,0 +1,85 @@
package yqlib
import (
"container/list"
"io"
"os"
yaml "gopkg.in/yaml.v3"
)
type StreamEvaluator interface {
Evaluate(filename string, reader io.Reader, node *PathTreeNode, printer Printer) error
EvaluateFiles(expression string, filenames []string, printer Printer) error
}
type streamEvaluator struct {
treeNavigator DataTreeNavigator
treeCreator PathTreeCreator
fileIndex int
}
func NewStreamEvaluator() StreamEvaluator {
return &streamEvaluator{treeNavigator: NewDataTreeNavigator(), treeCreator: NewPathTreeCreator()}
}
func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer) error {
node, err := treeCreator.ParsePath(expression)
if err != nil {
return err
}
for _, filename := range filenames {
reader, err := readStream(filename)
if err != nil {
return err
}
err = s.Evaluate(filename, reader, node, printer)
if err != nil {
return err
}
switch reader := reader.(type) {
case *os.File:
safelyCloseFile(reader)
}
}
return nil
}
func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *PathTreeNode, printer Printer) error {
var currentIndex uint
decoder := yaml.NewDecoder(reader)
for {
var dataBucket yaml.Node
errorReading := decoder.Decode(&dataBucket)
if errorReading == io.EOF {
s.fileIndex = s.fileIndex + 1
return nil
} else if errorReading != nil {
return errorReading
}
candidateNode := &CandidateNode{
Document: currentIndex,
Filename: filename,
Node: &dataBucket,
FileIndex: s.fileIndex,
}
inputList := list.New()
inputList.PushBack(candidateNode)
matches, errorParsing := treeNavigator.GetMatchingNodes(inputList, node)
if errorParsing != nil {
return errorParsing
}
err := printer.PrintResults(matches)
if err != nil {
return err
}
currentIndex = currentIndex + 1
}
}

91
pkg/yqlib/utils.go
View File

@ -11,11 +11,9 @@ import (
//TODO: convert to interface + struct
var treeNavigator = NewDataTreeNavigator(NavigationPrefs{})
var treeNavigator = NewDataTreeNavigator()
var treeCreator = NewPathTreeCreator()
var fileIndex = 0
func readStream(filename string) (io.Reader, error) {
if filename == "-" {
return bufio.NewReader(os.Stdin), nil
@ -24,42 +22,6 @@ func readStream(filename string) (io.Reader, error) {
}
}
func EvaluateStream(filename string, reader io.Reader, node *PathTreeNode, printer Printer) error {
var currentIndex uint = 0
decoder := yaml.NewDecoder(reader)
for {
var dataBucket yaml.Node
errorReading := decoder.Decode(&dataBucket)
if errorReading == io.EOF {
fileIndex = fileIndex + 1
return nil
} else if errorReading != nil {
return errorReading
}
candidateNode := &CandidateNode{
Document: currentIndex,
Filename: filename,
Node: &dataBucket,
FileIndex: fileIndex,
}
inputList := list.New()
inputList.PushBack(candidateNode)
matches, errorParsing := treeNavigator.GetMatchingNodes(inputList, node)
if errorParsing != nil {
return errorParsing
}
err := printer.PrintResults(matches)
if err != nil {
return err
}
currentIndex = currentIndex + 1
}
}
func readDocuments(reader io.Reader, filename string, fileIndex int) (*list.List, error) {
decoder := yaml.NewDecoder(reader)
inputList := list.New()
@ -91,57 +53,6 @@ func readDocuments(reader io.Reader, filename string, fileIndex int) (*list.List
}
}
func EvaluateAllFileStreams(expression string, filenames []string, printer Printer) error {
fileIndex := 0
node, err := treeCreator.ParsePath(expression)
if err != nil {
return err
}
var allDocuments *list.List = list.New()
for _, filename := range filenames {
reader, err := readStream(filename)
if err != nil {
return err
}
fileDocuments, err := readDocuments(reader, filename, fileIndex)
if err != nil {
return err
}
allDocuments.PushBackList(fileDocuments)
fileIndex = fileIndex + 1
}
matches, err := treeNavigator.GetMatchingNodes(allDocuments, node)
if err != nil {
return err
}
return printer.PrintResults(matches)
}
func EvaluateFileStreamsSequence(expression string, filenames []string, printer Printer) error {
node, err := treeCreator.ParsePath(expression)
if err != nil {
return err
}
for _, filename := range filenames {
reader, err := readStream(filename)
if err != nil {
return err
}
err = EvaluateStream(filename, reader, node, printer)
if err != nil {
return err
}
switch reader := reader.(type) {
case *os.File:
safelyCloseFile(reader)
}
}
return nil
}
// func safelyRenameFile(from string, to string) {
// if renameError := os.Rename(from, to); renameError != nil {
// log.Debugf("Error renaming from %v to %v, attempting to copy contents", from, to)