From 2ed5b2ff596afc1a9ab02076f581f8f8f269ee23 Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Thu, 25 Feb 2021 16:47:55 +1100
Subject: [PATCH] Improved lexer performance!

---
 pkg/yqlib/expression_parser.go    | 11 +++++------
 pkg/yqlib/expression_tokeniser.go |  2 +-
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/pkg/yqlib/expression_parser.go b/pkg/yqlib/expression_parser.go
index ad3c969d..7306e1e8 100644
--- a/pkg/yqlib/expression_parser.go
+++ b/pkg/yqlib/expression_parser.go
@@ -5,9 +5,6 @@ import (
 	"strings"
 )
 
-var myPathTokeniser = newExpressionTokeniser()
-var myPathPostfixer = newExpressionPostFixer()
-
 type ExpressionNode struct {
 	Operation *Operation
 	Lhs       *ExpressionNode
@@ -19,19 +16,21 @@ type ExpressionParser interface {
 }
 
 type expressionParserImpl struct {
+	pathTokeniser expressionTokeniser
+	pathPostFixer expressionPostFixer
 }
 
 func NewExpressionParser() ExpressionParser {
-	return &expressionParserImpl{}
+	return &expressionParserImpl{newExpressionTokeniser(), newExpressionPostFixer()}
 }
 
 func (p *expressionParserImpl) ParseExpression(expression string) (*ExpressionNode, error) {
-	tokens, err := myPathTokeniser.Tokenise(expression)
+	tokens, err := p.pathTokeniser.Tokenise(expression)
 	if err != nil {
 		return nil, err
 	}
 	var Operations []*Operation
-	Operations, err = myPathPostfixer.ConvertToPostfix(tokens)
+	Operations, err = p.pathPostFixer.ConvertToPostfix(tokens)
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/yqlib/expression_tokeniser.go b/pkg/yqlib/expression_tokeniser.go
index 372476b3..fb22e6b8 100644
--- a/pkg/yqlib/expression_tokeniser.go
+++ b/pkg/yqlib/expression_tokeniser.go
@@ -328,7 +328,7 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`\$[a-zA-Z_-0-9]+`), getVariableOpToken())
 	lexer.Add([]byte(`as`), opToken(assignVariableOpType))
 
-	err := lexer.Compile()
+	err := lexer.CompileNFA()
 	if err != nil {
 		return nil, err
 	}