eval-reduce wip

pkg/yqlib/context.go

@@ -30,6 +30,7 @@ func (n *Context) GetVariable(name string) *list.List {
 	if n.Variables == nil {
 		return nil
 	}
+	log.Debug("GetVariable - %v to %v", name, NodesToString(n.Variables[name]))
 	return n.Variables[name]
 }
 
@@ -37,6 +38,7 @@ func (n *Context) SetVariable(name string, value *list.List) {
 	if n.Variables == nil {
 		n.Variables = make(map[string]*list.List)
 	}
+	log.Debug("SetVariable - %v to %v", name, NodesToString(value))
 	n.Variables[name] = value
 }
 

pkg/yqlib/lib.go

@@ -112,7 +112,7 @@ type Operation struct {
 	OperationType *operationType
 	Value         interface{}
 	StringValue   string
-	CandidateNode *CandidateNode // used for Value Path elements
+	ValueNodes    *list.List // used for Value Path elements
 	Preferences   interface{}
 	UpdateAssign  bool // used for assign ops, when true it means we evaluate the rhs given the lhs
 }
@@ -138,12 +138,13 @@ func createScalarNode(value interface{}, stringValue string) *yaml.Node {
 
 func createValueOperation(value interface{}, stringValue string) *Operation {
 	var node *yaml.Node = createScalarNode(value, stringValue)
-
+	list := list.New()
+	list.PushBack(&CandidateNode{Node: node})
 	return &Operation{
 		OperationType: valueOpType,
 		Value:         value,
 		StringValue:   stringValue,
-		CandidateNode: &CandidateNode{Node: node},
+		ValueNodes:    list,
 	}
 }
 

pkg/yqlib/operator_env.go

@@ -13,7 +13,8 @@ type envOpPreferences struct {
 }
 
 func envOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	envName := expressionNode.Operation.CandidateNode.Node.Value
+	envNameNode := expressionNode.Operation.ValueNodes.Front().Value.(*CandidateNode)
+	envName := envNameNode.Node.Value
 	log.Debug("EnvOperator, env name:", envName)
 
 	rawValue := os.Getenv(envName)

pkg/yqlib/operator_multiply.go

@@ -136,7 +136,9 @@ func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom
 	} else {
 		log.Debugf("merge - assignmentOp := &Operation{OperationType: assignAttributesOpType}")
 	}
-	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs}
+	valueNodes := list.New()
+	valueNodes.PushBack(rhs)
+	rhsOp := &Operation{OperationType: valueOpType, ValueNodes: valueNodes}
 
 	assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath, preferences.TraversePrefs, rhs.IsMapKey), Rhs: &ExpressionNode{Operation: rhsOp}}
 

pkg/yqlib/operator_reduce.go

@@ -24,7 +24,7 @@ func reduceOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 	arrayExpNode := expressionNode.Lhs.Lhs
 	array, err := d.GetMatchingNodes(context, arrayExpNode)
 
-	log.Debugf("array of %v things", array.MatchingNodes.Len())
+	log.Debugf("reducing %v", NodesToString(array.MatchingNodes))
 
 	if err != nil {
 		return Context{}, err
@@ -39,6 +39,8 @@ func reduceOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 		return Context{}, err
 	}
 
+	log.Debugf("initialised with %v", NodesToString(accum.MatchingNodes))
+
 	log.Debugf("with variable %v", variableName)
 
 	blockExp := expressionNode.Rhs.Rhs

pkg/yqlib/operator_value.go

@@ -1,6 +1,5 @@
 package yqlib
 
 func valueOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	log.Debug("value = %v", expressionNode.Operation.CandidateNode.Node.Value)
-	return context.SingleChildContext(expressionNode.Operation.CandidateNode), nil
+	return context.ChildContext(expressionNode.Operation.ValueNodes), nil
 }

pkg/yqlib/operators.go

@@ -23,7 +23,9 @@ func compoundAssignFunction(d *dataTreeNavigator, context Context, expressionNod
 
 	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
-		valueOp.CandidateNode = candidate
+		valueNodes := list.New()
+		valueNodes.PushBack(candidate)
+		valueOp.ValueNodes = valueNodes
 		valueExpression := &ExpressionNode{Operation: valueOp}
 
 		assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: valueExpression, Rhs: calculation(valueExpression, expressionNode.Rhs)}
@@ -83,7 +85,7 @@ func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *Expressi
 	if err != nil {
 		return Context{}, err
 	}
-	log.Debugf("crossFunction LHS len: %v", lhs.MatchingNodes.Len())
+	log.Debugf("crossFunction LHS %v", NodesToString(lhs.MatchingNodes))
 
 	rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
 

pkg/yqlib/reduce_evaluator.go

@@ -0,0 +1,142 @@
+package yqlib
+
+import (
+	"container/list"
+	"fmt"
+	"io"
+	"os"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+type ReduceEvaluator interface {
+	EvaluateFiles(reduceExpression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error
+}
+
+type reduceEvaluator struct {
+	treeNavigator DataTreeNavigator
+	treeCreator   ExpressionParser
+	reduceLhs     *ExpressionNode
+	fileIndex     int
+}
+
+func NewReduceEvaluator() ReduceEvaluator {
+	treeCreator := NewExpressionParser()
+	reduceLhs, err := treeCreator.ParseExpression(". as $doc")
+	if err != nil {
+		panic(err)
+	}
+	return &reduceEvaluator{treeNavigator: NewDataTreeNavigator(), treeCreator: treeCreator, reduceLhs: reduceLhs}
+}
+
+func (r *reduceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error {
+
+	node, err := r.treeCreator.ParseExpression(expression)
+	if err != nil {
+		return err
+	}
+	log.Debug("node %v", node.Operation.toString())
+
+	if node.Operation.OperationType != blockOpType {
+		return fmt.Errorf("Invalid reduce expression - expected '<initialValue>; <block that uses $doc>' got '%v'", expression)
+	}
+
+	currentValue := node.Lhs
+	reduceExp := node.Rhs
+	firstLeadingContent := ""
+
+	log.Debug("initialValue %v", currentValue.Operation.toString())
+
+	log.Debug("reduce Exp %v", reduceExp.Operation.toString())
+
+	for index, filename := range filenames {
+		reader, leadingContent, err := readStream(filename, leadingContentPreProcessing)
+
+		if index == 0 {
+			firstLeadingContent = leadingContent
+		}
+
+		if err != nil {
+			return err
+		}
+
+		currentValue, err = r.ReduceFile(filename, leadingContent, reader, currentValue, reduceExp)
+		if err != nil {
+			return err
+		}
+
+		switch reader := reader.(type) {
+		case *os.File:
+			safelyCloseFile(reader)
+		}
+	}
+
+	result := currentValue.Operation.ValueNodes
+
+	if result.Len() > 0 {
+		result.Front().Value.(*CandidateNode).Node.HeadComment = firstLeadingContent
+	}
+
+	printer.PrintResults(result)
+
+	return nil
+}
+
+func (r *reduceEvaluator) createReduceOp(initialValue *ExpressionNode, reduceExp *ExpressionNode) *ExpressionNode {
+	reduceBlock := &ExpressionNode{
+		Operation: &Operation{OperationType: blockOpType},
+		Lhs:       initialValue,
+		Rhs:       reduceExp,
+	}
+
+	return &ExpressionNode{
+		Operation: &Operation{OperationType: reduceOpType},
+		Lhs:       r.reduceLhs,
+		Rhs:       reduceBlock,
+	}
+}
+
+func (r *reduceEvaluator) ReduceFile(filename string, leadingContent string, reader io.Reader, initialValue *ExpressionNode, reduceExp *ExpressionNode) (*ExpressionNode, error) {
+
+	var currentIndex uint
+	var currentValue = initialValue
+	decoder := yaml.NewDecoder(reader)
+	for {
+		var dataBucket yaml.Node
+		errorReading := decoder.Decode(&dataBucket)
+
+		if errorReading == io.EOF {
+			r.fileIndex = r.fileIndex + 1
+			return currentValue, nil
+		} else if errorReading != nil {
+			return currentValue, errorReading
+		}
+		candidateNode := &CandidateNode{
+			Document:  currentIndex,
+			Filename:  filename,
+			Node:      &dataBucket,
+			FileIndex: r.fileIndex,
+		}
+		inputList := list.New()
+		inputList.PushBack(candidateNode)
+
+		reduceOp := r.createReduceOp(currentValue, reduceExp)
+		// log.Debug("reduce - currentValueBefore: %v", NodesToString(currentValue.Operation.ValueNodes))
+
+		result, errorParsing := r.treeNavigator.GetMatchingNodes(Context{MatchingNodes: inputList}, reduceOp)
+		if errorParsing != nil {
+			return currentValue, errorParsing
+		}
+
+		currentValue = &ExpressionNode{
+			Operation: &Operation{
+				OperationType: valueOpType,
+				ValueNodes:    result.MatchingNodes,
+			},
+		}
+
+		log.Debug("reduce - currentValueAfter: %v", NodesToString(currentValue.Operation.ValueNodes))
+
+		currentIndex = currentIndex + 1
+	}
+}