Fixes nested array indexing #824

pkg/yqlib/expression_postfix.go

@@ -54,7 +54,7 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
 			opStack = opStack[0 : len(opStack)-1]
 			log.Debugf("deleteing open bracket from opstack")
 
-			//and append a collect to the opStack
+			//and append a collect to the result
 			// hack - see if there's the optional traverse flag
 			// on the close op - move it to the collect op.
 			// allows for .["cat"]?
@@ -68,6 +68,12 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
 			result = append(result, &Operation{OperationType: shortPipeOpType})
 			log.Debugf("put shortpipe onto the result")
 
+			//traverseArrayCollect is a sneaky op that needs to be included too
+			//when closing a []
+			if len(opStack) > 0 && opStack[len(opStack)-1].Operation != nil && opStack[len(opStack)-1].Operation.OperationType == traverseArrayOpType {
+				opStack, result = popOpToResult(opStack, result)
+			}
+
 		case closeBracket:
 			for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != openBracket {
 				opStack, result = popOpToResult(opStack, result)

pkg/yqlib/expression_processing_test.go

@@ -12,6 +12,16 @@ var pathTests = []struct {
 	expectedTokens  []interface{}
 	expectedPostFix []interface{}
 }{
+	{
+		`.[0]`,
+		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
+		append(make([]interface{}, 0), "SELF", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
+	{
+		`.[0][1]`,
+		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]", "TRAVERSE_ARRAY", "[", "1 (int64)", "]"),
+		append(make([]interface{}, 0), "SELF", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "1 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
 	{
 		`"\""`,
 		append(make([]interface{}, 0), "\" (string)"),

pkg/yqlib/expression_tokeniser.go

@@ -411,16 +411,19 @@ func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postPr
 	skipNextToken = false
 	currentToken := tokens[index]
 
+	log.Debug("processing %v", currentToken.toString(true))
+
 	if currentToken.TokenType == traverseArrayCollect {
 		//need to put a traverse array then a collect currentToken
 		// do this by adding traverse then converting currentToken to collect
 
 		if index == 0 || tokens[index-1].TokenType != operationToken ||
 			tokens[index-1].Operation.OperationType != traversePathOpType {
+			log.Debug("  adding self")
 			op := &Operation{OperationType: selfReferenceOpType, StringValue: "SELF"}
 			postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 		}
-
+		log.Debug("  adding traverse array")
 		op := &Operation{OperationType: traverseArrayOpType, StringValue: "TRAVERSE_ARRAY"}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 
@@ -431,17 +434,19 @@ func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postPr
 	if index != len(tokens)-1 && currentToken.AssignOperation != nil &&
 		tokens[index+1].TokenType == operationToken &&
 		tokens[index+1].Operation.OperationType == assignOpType {
+		log.Debug("  its an update assign")
 		currentToken.Operation = currentToken.AssignOperation
 		currentToken.Operation.UpdateAssign = tokens[index+1].Operation.UpdateAssign
 		skipNextToken = true
 	}
 
+	log.Debug("  adding token to the fixed list")
 	postProcessedTokens = append(postProcessedTokens, currentToken)
 
 	if index != len(tokens)-1 &&
 		((currentToken.TokenType == openCollect && tokens[index+1].TokenType == closeCollect) ||
 			(currentToken.TokenType == openCollectObject && tokens[index+1].TokenType == closeCollectObject)) {
-
+		log.Debug("  adding empty")
 		op := &Operation{OperationType: emptyOpType, StringValue: "EMPTY"}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 	}
@@ -449,12 +454,19 @@ func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postPr
 	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
 		tokens[index+1].TokenType == operationToken &&
 		tokens[index+1].Operation.OperationType == traversePathOpType {
+		log.Debug("  adding pipe because the next thing is traverse")
 		op := &Operation{OperationType: shortPipeOpType, Value: "PIPE"}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 	}
 	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
 		tokens[index+1].TokenType == openCollect {
 
+		// if tokens[index].TokenType == closeCollect {
+		// 	log.Debug("  adding pipe because next is opencollect")
+		// 	op := &Operation{OperationType: shortPipeOpType, Value: "PIPE"}
+		// 	postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
+		// }
+		log.Debug("  adding traverArray because next is opencollect")
 		op := &Operation{OperationType: traverseArrayOpType}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 	}

pkg/yqlib/operator_traverse_path_test.go

@@ -27,6 +27,22 @@ foobar:
 `
 
 var traversePathOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   `[[1]]`,
+		expression: `.[0][0]`,
+		expected: []string{
+			"D0, P[0 0], (!!int)::1\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[[[1]]]`,
+		expression: `.[0][0][0]`,
+		expected: []string{
+			"D0, P[0 0 0], (!!int)::1\n",
+		},
+	},
 	{
 		description: "Simple map navigation",
 		document:    `{a: {b: apple}}`,