diff --git a/go.sum b/go.sum index 05588769..1e9dabdf 100644 --- a/go.sum +++ b/go.sum @@ -281,6 +281,7 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc h1:NCy3Ohtk6Iny5V/reW2Ktypo4zIpWBdRJ1uFMjBxdg8= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= diff --git a/pkg/yqlib/all_at_once_evaluator.go b/pkg/yqlib/all_at_once_evaluator.go index 4082c7e2..4efd6afd 100644 --- a/pkg/yqlib/all_at_once_evaluator.go +++ b/pkg/yqlib/all_at_once_evaluator.go @@ -2,11 +2,13 @@ package yqlib import "container/list" -/** - Loads all yaml documents of all files given into memory, then runs the given expression once. -**/ +// A yaml expression evaluator that runs the expression once against all files/nodes in memory. type Evaluator interface { EvaluateFiles(expression string, filenames []string, printer Printer) error + + // Runs the expression once against the list of candidate nodes, returns the + // resulting nodes. + EvaluateNodes(expression string, inputCandidateNodes *list.List) (*list.List, error) } type allAtOnceEvaluator struct { @@ -18,12 +20,17 @@ func NewAllAtOnceEvaluator() Evaluator { return &allAtOnceEvaluator{treeNavigator: NewDataTreeNavigator(), treeCreator: NewPathTreeCreator()} } -func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer) error { - fileIndex := 0 +func (e *allAtOnceEvaluator) EvaluateNodes(expression string, inputCandidates *list.List) (*list.List, error) { node, err := treeCreator.ParsePath(expression) if err != nil { - return err + return nil, err } + return treeNavigator.GetMatchingNodes(inputCandidates, node) +} + +func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer) error { + fileIndex := 0 + var allDocuments *list.List = list.New() for _, filename := range filenames { reader, err := readStream(filename) @@ -37,7 +44,7 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string allDocuments.PushBackList(fileDocuments) fileIndex = fileIndex + 1 } - matches, err := treeNavigator.GetMatchingNodes(allDocuments, node) + matches, err := e.EvaluateNodes(expression, allDocuments) if err != nil { return err } diff --git a/pkg/yqlib/path_parse_test.go b/pkg/yqlib/path_parse_test.go index 44c81cca..d077559a 100644 --- a/pkg/yqlib/path_parse_test.go +++ b/pkg/yqlib/path_parse_test.go @@ -164,8 +164,8 @@ var pathTests = []struct { }, } -var tokeniser = NewPathTokeniser() -var postFixer = NewPathPostFixer() +var tokeniser = newPathTokeniser() +var postFixer = newPathPostFixer() func TestPathParsing(t *testing.T) { for _, tt := range pathTests { diff --git a/pkg/yqlib/path_postfix.go b/pkg/yqlib/path_postfix.go index a26adbcc..cb56bdad 100644 --- a/pkg/yqlib/path_postfix.go +++ b/pkg/yqlib/path_postfix.go @@ -6,38 +6,38 @@ import ( logging "gopkg.in/op/go-logging.v1" ) -type PathPostFixer interface { - ConvertToPostfix([]*Token) ([]*Operation, error) +type pathPostFixerInterface interface { + ConvertToPostfix([]*token) ([]*Operation, error) } type pathPostFixer struct { } -func NewPathPostFixer() PathPostFixer { +func newPathPostFixer() pathPostFixerInterface { return &pathPostFixer{} } -func popOpToResult(opStack []*Token, result []*Operation) ([]*Token, []*Operation) { - var newOp *Token +func popOpToResult(opStack []*token, result []*Operation) ([]*token, []*Operation) { + var newOp *token opStack, newOp = opStack[0:len(opStack)-1], opStack[len(opStack)-1] return opStack, append(result, newOp.Operation) } -func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*Operation, error) { +func (p *pathPostFixer) ConvertToPostfix(infixTokens []*token) ([]*Operation, error) { var result []*Operation // surround the whole thing with quotes - var opStack = []*Token{&Token{TokenType: OpenBracket}} - var tokens = append(infixTokens, &Token{TokenType: CloseBracket}) + var opStack = []*token{&token{TokenType: OpenBracket}} + var tokens = append(infixTokens, &token{TokenType: CloseBracket}) - for _, token := range tokens { - log.Debugf("postfix processing token %v, %v", token.toString(), token.Operation) - switch token.TokenType { + for _, currentToken := range tokens { + log.Debugf("postfix processing currentToken %v, %v", currentToken.toString(), currentToken.Operation) + switch currentToken.TokenType { case OpenBracket, OpenCollect, OpenCollectObject: - opStack = append(opStack, token) + opStack = append(opStack, currentToken) case CloseCollect, CloseCollectObject: - var opener TokenType = OpenCollect + var opener tokenType = OpenCollect var collectOperator *OperationType = Collect - if token.TokenType == CloseCollectObject { + if currentToken.TokenType == CloseCollectObject { opener = OpenCollectObject collectOperator = CollectObject } @@ -56,8 +56,8 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*Operation, er // now we should have [] as the last element on the opStack, get rid of it opStack = opStack[0 : len(opStack)-1] //and append a collect to the opStack - opStack = append(opStack, &Token{TokenType: OperationToken, Operation: &Operation{OperationType: ShortPipe}}) - opStack = append(opStack, &Token{TokenType: OperationToken, Operation: &Operation{OperationType: collectOperator}}) + opStack = append(opStack, &token{TokenType: OperationToken, Operation: &Operation{OperationType: ShortPipe}}) + opStack = append(opStack, &token{TokenType: OperationToken, Operation: &Operation{OperationType: collectOperator}}) case CloseBracket: for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != OpenBracket { opStack, result = popOpToResult(opStack, result) @@ -69,7 +69,7 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*Operation, er opStack = opStack[0 : len(opStack)-1] default: - var currentPrecedence = token.Operation.OperationType.Precedence + var currentPrecedence = currentToken.Operation.OperationType.Precedence // pop off higher precedent operators onto the result for len(opStack) > 0 && opStack[len(opStack)-1].TokenType == OperationToken && @@ -77,14 +77,14 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*Operation, er opStack, result = popOpToResult(opStack, result) } // add this operator to the opStack - opStack = append(opStack, token) + opStack = append(opStack, currentToken) } } if log.IsEnabledFor(logging.DEBUG) { log.Debugf("PostFix Result:") - for _, token := range result { - log.Debugf("> %v", token.toString()) + for _, currentToken := range result { + log.Debugf("> %v", currentToken.toString()) } } diff --git a/pkg/yqlib/path_tokeniser.go b/pkg/yqlib/path_tokeniser.go index a25d78c7..950d97bb 100644 --- a/pkg/yqlib/path_tokeniser.go +++ b/pkg/yqlib/path_tokeniser.go @@ -12,7 +12,7 @@ func skip(*lex.Scanner, *machines.Match) (interface{}, error) { return nil, nil } -type TokenType uint32 +type tokenType uint32 const ( OperationToken = 1 << iota @@ -25,15 +25,15 @@ const ( TraverseArrayCollect ) -type Token struct { - TokenType TokenType +type token struct { + TokenType tokenType Operation *Operation AssignOperation *Operation // e.g. tag (GetTag) op becomes AssignTag if '=' follows it CheckForPostTraverse bool // e.g. [1]cat should really be [1].cat } -func (t *Token) toString() string { +func (t *token) toString() string { if t.TokenType == OperationToken { log.Debug("toString, its an op") return t.Operation.toString() @@ -66,7 +66,7 @@ func pathToken(wrapped bool) lex.Action { } log.Debug("PathToken %v", value) op := &Operation{OperationType: TraversePath, Value: value, StringValue: value} - return &Token{TokenType: OperationToken, Operation: op, CheckForPostTraverse: true}, nil + return &token{TokenType: OperationToken, Operation: op, CheckForPostTraverse: true}, nil } } @@ -80,7 +80,7 @@ func documentToken() lex.Action { } log.Debug("documentToken %v", string(m.Bytes)) op := &Operation{OperationType: DocumentFilter, Value: number, StringValue: numberString} - return &Token{TokenType: OperationToken, Operation: op, CheckForPostTraverse: true}, nil + return &token{TokenType: OperationToken, Operation: op, CheckForPostTraverse: true}, nil } } @@ -97,7 +97,7 @@ func assignOpToken(updateAssign bool) lex.Action { log.Debug("assignOpToken %v", string(m.Bytes)) value := string(m.Bytes) op := &Operation{OperationType: Assign, Value: Assign.Type, StringValue: value, UpdateAssign: updateAssign} - return &Token{TokenType: OperationToken, Operation: op}, nil + return &token{TokenType: OperationToken, Operation: op}, nil } } @@ -110,7 +110,7 @@ func opTokenWithPrefs(op *OperationType, assignOpType *OperationType, preference if assignOpType != nil { assign = &Operation{OperationType: assignOpType, Value: assignOpType.Type, StringValue: value, Preferences: preferences} } - return &Token{TokenType: OperationToken, Operation: op, AssignOperation: assign}, nil + return &token{TokenType: OperationToken, Operation: op, AssignOperation: assign}, nil } } @@ -125,13 +125,13 @@ func assignAllCommentsOp(updateAssign bool) lex.Action { UpdateAssign: updateAssign, Preferences: &CommentOpPreferences{LineComment: true, HeadComment: true, FootComment: true}, } - return &Token{TokenType: OperationToken, Operation: op}, nil + return &token{TokenType: OperationToken, Operation: op}, nil } } -func literalToken(pType TokenType, checkForPost bool) lex.Action { +func literalToken(pType tokenType, checkForPost bool) lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { - return &Token{TokenType: pType, CheckForPostTraverse: checkForPost}, nil + return &token{TokenType: pType, CheckForPostTraverse: checkForPost}, nil } } @@ -147,7 +147,7 @@ func numberValue() lex.Action { return nil, errParsingInt } - return &Token{TokenType: OperationToken, Operation: CreateValueOperation(number, numberString)}, nil + return &token{TokenType: OperationToken, Operation: CreateValueOperation(number, numberString)}, nil } } @@ -158,13 +158,13 @@ func floatValue() lex.Action { if errParsingInt != nil { return nil, errParsingInt } - return &Token{TokenType: OperationToken, Operation: CreateValueOperation(number, numberString)}, nil + return &token{TokenType: OperationToken, Operation: CreateValueOperation(number, numberString)}, nil } } func booleanValue(val bool) lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { - return &Token{TokenType: OperationToken, Operation: CreateValueOperation(val, string(m.Bytes))}, nil + return &token{TokenType: OperationToken, Operation: CreateValueOperation(val, string(m.Bytes))}, nil } } @@ -174,7 +174,7 @@ func stringValue(wrapped bool) lex.Action { if wrapped { value = unwrap(value) } - return &Token{TokenType: OperationToken, Operation: CreateValueOperation(value, value)}, nil + return &token{TokenType: OperationToken, Operation: CreateValueOperation(value, value)}, nil } } @@ -196,20 +196,20 @@ func envOp(strenv bool) lex.Action { envOperation.OperationType = EnvOp envOperation.Preferences = preferences - return &Token{TokenType: OperationToken, Operation: envOperation}, nil + return &token{TokenType: OperationToken, Operation: envOperation}, nil } } func nullValue() lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { - return &Token{TokenType: OperationToken, Operation: CreateValueOperation(nil, string(m.Bytes))}, nil + return &token{TokenType: OperationToken, Operation: CreateValueOperation(nil, string(m.Bytes))}, nil } } func selfToken() lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { op := &Operation{OperationType: SelfReference} - return &Token{TokenType: OperationToken, Operation: op}, nil + return &token{TokenType: OperationToken, Operation: op}, nil } } @@ -307,15 +307,15 @@ func initLexer() (*lex.Lexer, error) { return lexer, nil } -type PathTokeniser interface { - Tokenise(path string) ([]*Token, error) +type pathTokeniserInterface interface { + Tokenise(path string) ([]*token, error) } type pathTokeniser struct { lexer *lex.Lexer } -func NewPathTokeniser() PathTokeniser { +func newPathTokeniser() pathTokeniserInterface { var lexer, err = initLexer() if err != nil { panic(err) @@ -323,25 +323,25 @@ func NewPathTokeniser() PathTokeniser { return &pathTokeniser{lexer} } -func (p *pathTokeniser) Tokenise(path string) ([]*Token, error) { +func (p *pathTokeniser) Tokenise(path string) ([]*token, error) { scanner, err := p.lexer.Scanner([]byte(path)) if err != nil { return nil, fmt.Errorf("Parsing expression: %v", err) } - var tokens []*Token + var tokens []*token for tok, err, eof := scanner.Next(); !eof; tok, err, eof = scanner.Next() { if tok != nil { - token := tok.(*Token) - log.Debugf("Tokenising %v", token.toString()) - tokens = append(tokens, token) + currentToken := tok.(*token) + log.Debugf("Tokenising %v", currentToken.toString()) + tokens = append(tokens, currentToken) } if err != nil { return nil, fmt.Errorf("Parsing expression: %v", err) } } - var postProcessedTokens = make([]*Token, 0) + var postProcessedTokens = make([]*token, 0) skipNextToken := false @@ -356,51 +356,51 @@ func (p *pathTokeniser) Tokenise(path string) ([]*Token, error) { return postProcessedTokens, nil } -func (p *pathTokeniser) handleToken(tokens []*Token, index int, postProcessedTokens []*Token) (tokensAccum []*Token, skipNextToken bool) { +func (p *pathTokeniser) handleToken(tokens []*token, index int, postProcessedTokens []*token) (tokensAccum []*token, skipNextToken bool) { skipNextToken = false - token := tokens[index] + currentToken := tokens[index] - if token.TokenType == TraverseArrayCollect { - //need to put a traverse array then a collect token - // do this by adding traverse then converting token to collect + if currentToken.TokenType == TraverseArrayCollect { + //need to put a traverse array then a collect currentToken + // do this by adding traverse then converting currentToken to collect op := &Operation{OperationType: TraverseArray, StringValue: "TRAVERSE_ARRAY"} - postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op}) + postProcessedTokens = append(postProcessedTokens, &token{TokenType: OperationToken, Operation: op}) - token = &Token{TokenType: OpenCollect} + currentToken = &token{TokenType: OpenCollect} } - if index != len(tokens)-1 && token.AssignOperation != nil && + if index != len(tokens)-1 && currentToken.AssignOperation != nil && tokens[index+1].TokenType == OperationToken && tokens[index+1].Operation.OperationType == Assign { - token.Operation = token.AssignOperation - token.Operation.UpdateAssign = tokens[index+1].Operation.UpdateAssign + currentToken.Operation = currentToken.AssignOperation + currentToken.Operation.UpdateAssign = tokens[index+1].Operation.UpdateAssign skipNextToken = true } - postProcessedTokens = append(postProcessedTokens, token) + postProcessedTokens = append(postProcessedTokens, currentToken) - if index != len(tokens)-1 && token.CheckForPostTraverse && + if index != len(tokens)-1 && currentToken.CheckForPostTraverse && tokens[index+1].TokenType == OperationToken && tokens[index+1].Operation.OperationType == TraversePath { op := &Operation{OperationType: ShortPipe, Value: "PIPE"} - postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op}) + postProcessedTokens = append(postProcessedTokens, &token{TokenType: OperationToken, Operation: op}) } - if index != len(tokens)-1 && token.CheckForPostTraverse && + if index != len(tokens)-1 && currentToken.CheckForPostTraverse && tokens[index+1].TokenType == OpenCollect { op := &Operation{OperationType: ShortPipe, Value: "PIPE"} - postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op}) + postProcessedTokens = append(postProcessedTokens, &token{TokenType: OperationToken, Operation: op}) op = &Operation{OperationType: TraverseArray} - postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op}) + postProcessedTokens = append(postProcessedTokens, &token{TokenType: OperationToken, Operation: op}) } - if index != len(tokens)-1 && token.CheckForPostTraverse && + if index != len(tokens)-1 && currentToken.CheckForPostTraverse && tokens[index+1].TokenType == TraverseArrayCollect { op := &Operation{OperationType: ShortPipe, Value: "PIPE"} - postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op}) + postProcessedTokens = append(postProcessedTokens, &token{TokenType: OperationToken, Operation: op}) } return postProcessedTokens, skipNextToken diff --git a/pkg/yqlib/path_tree.go b/pkg/yqlib/path_tree.go index 4ea6f42f..b2a752ff 100644 --- a/pkg/yqlib/path_tree.go +++ b/pkg/yqlib/path_tree.go @@ -5,8 +5,8 @@ import ( "strings" ) -var myPathTokeniser = NewPathTokeniser() -var myPathPostfixer = NewPathPostFixer() +var myPathTokeniser = newPathTokeniser() +var myPathPostfixer = newPathPostFixer() type PathTreeNode struct { Operation *Operation diff --git a/pkg/yqlib/stream_evaluator.go b/pkg/yqlib/stream_evaluator.go index b2fdb9e1..b0faaf02 100644 --- a/pkg/yqlib/stream_evaluator.go +++ b/pkg/yqlib/stream_evaluator.go @@ -8,6 +8,9 @@ import ( yaml "gopkg.in/yaml.v3" ) +// A yaml expression evaluator that runs the expression multiple times for each given yaml document. +// Uses less memory than loading all documents and running the expression once, but this cannot process +// cross document expressions. type StreamEvaluator interface { Evaluate(filename string, reader io.Reader, node *PathTreeNode, printer Printer) error EvaluateFiles(expression string, filenames []string, printer Printer) error