From d18a6963f69810a6c72eaef7240fe2bfc26647f1 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 21 May 2021 14:18:24 +1000 Subject: [PATCH] Fixes nested array indexing #824 --- pkg/yqlib/expression_postfix.go | 8 +++++++- pkg/yqlib/expression_processing_test.go | 10 ++++++++++ pkg/yqlib/expression_tokeniser.go | 16 ++++++++++++++-- pkg/yqlib/operator_traverse_path_test.go | 16 ++++++++++++++++ 4 files changed, 47 insertions(+), 3 deletions(-) diff --git a/pkg/yqlib/expression_postfix.go b/pkg/yqlib/expression_postfix.go index 101dfe29..cdfa36e4 100644 --- a/pkg/yqlib/expression_postfix.go +++ b/pkg/yqlib/expression_postfix.go @@ -54,7 +54,7 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope opStack = opStack[0 : len(opStack)-1] log.Debugf("deleteing open bracket from opstack") - //and append a collect to the opStack + //and append a collect to the result // hack - see if there's the optional traverse flag // on the close op - move it to the collect op. // allows for .["cat"]? @@ -68,6 +68,12 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope result = append(result, &Operation{OperationType: shortPipeOpType}) log.Debugf("put shortpipe onto the result") + //traverseArrayCollect is a sneaky op that needs to be included too + //when closing a [] + if len(opStack) > 0 && opStack[len(opStack)-1].Operation != nil && opStack[len(opStack)-1].Operation.OperationType == traverseArrayOpType { + opStack, result = popOpToResult(opStack, result) + } + case closeBracket: for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != openBracket { opStack, result = popOpToResult(opStack, result) diff --git a/pkg/yqlib/expression_processing_test.go b/pkg/yqlib/expression_processing_test.go index aac09934..2b2f7060 100644 --- a/pkg/yqlib/expression_processing_test.go +++ b/pkg/yqlib/expression_processing_test.go @@ -12,6 +12,16 @@ var pathTests = []struct { expectedTokens []interface{} expectedPostFix []interface{} }{ + { + `.[0]`, + append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"), + append(make([]interface{}, 0), "SELF", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"), + }, + { + `.[0][1]`, + append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]", "TRAVERSE_ARRAY", "[", "1 (int64)", "]"), + append(make([]interface{}, 0), "SELF", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "1 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"), + }, { `"\""`, append(make([]interface{}, 0), "\" (string)"), diff --git a/pkg/yqlib/expression_tokeniser.go b/pkg/yqlib/expression_tokeniser.go index e509c848..0018489b 100644 --- a/pkg/yqlib/expression_tokeniser.go +++ b/pkg/yqlib/expression_tokeniser.go @@ -411,16 +411,19 @@ func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postPr skipNextToken = false currentToken := tokens[index] + log.Debug("processing %v", currentToken.toString(true)) + if currentToken.TokenType == traverseArrayCollect { //need to put a traverse array then a collect currentToken // do this by adding traverse then converting currentToken to collect if index == 0 || tokens[index-1].TokenType != operationToken || tokens[index-1].Operation.OperationType != traversePathOpType { + log.Debug(" adding self") op := &Operation{OperationType: selfReferenceOpType, StringValue: "SELF"} postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op}) } - + log.Debug(" adding traverse array") op := &Operation{OperationType: traverseArrayOpType, StringValue: "TRAVERSE_ARRAY"} postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op}) @@ -431,17 +434,19 @@ func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postPr if index != len(tokens)-1 && currentToken.AssignOperation != nil && tokens[index+1].TokenType == operationToken && tokens[index+1].Operation.OperationType == assignOpType { + log.Debug(" its an update assign") currentToken.Operation = currentToken.AssignOperation currentToken.Operation.UpdateAssign = tokens[index+1].Operation.UpdateAssign skipNextToken = true } + log.Debug(" adding token to the fixed list") postProcessedTokens = append(postProcessedTokens, currentToken) if index != len(tokens)-1 && ((currentToken.TokenType == openCollect && tokens[index+1].TokenType == closeCollect) || (currentToken.TokenType == openCollectObject && tokens[index+1].TokenType == closeCollectObject)) { - + log.Debug(" adding empty") op := &Operation{OperationType: emptyOpType, StringValue: "EMPTY"} postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op}) } @@ -449,12 +454,19 @@ func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postPr if index != len(tokens)-1 && currentToken.CheckForPostTraverse && tokens[index+1].TokenType == operationToken && tokens[index+1].Operation.OperationType == traversePathOpType { + log.Debug(" adding pipe because the next thing is traverse") op := &Operation{OperationType: shortPipeOpType, Value: "PIPE"} postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op}) } if index != len(tokens)-1 && currentToken.CheckForPostTraverse && tokens[index+1].TokenType == openCollect { + // if tokens[index].TokenType == closeCollect { + // log.Debug(" adding pipe because next is opencollect") + // op := &Operation{OperationType: shortPipeOpType, Value: "PIPE"} + // postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op}) + // } + log.Debug(" adding traverArray because next is opencollect") op := &Operation{OperationType: traverseArrayOpType} postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op}) } diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go index 7b8f4432..ce0928de 100644 --- a/pkg/yqlib/operator_traverse_path_test.go +++ b/pkg/yqlib/operator_traverse_path_test.go @@ -27,6 +27,22 @@ foobar: ` var traversePathOperatorScenarios = []expressionScenario{ + { + skipDoc: true, + document: `[[1]]`, + expression: `.[0][0]`, + expected: []string{ + "D0, P[0 0], (!!int)::1\n", + }, + }, + { + skipDoc: true, + document: `[[[1]]]`, + expression: `.[0][0][0]`, + expected: []string{ + "D0, P[0 0 0], (!!int)::1\n", + }, + }, { description: "Simple map navigation", document: `{a: {b: apple}}`,