Added File operators

pkg/yqlib/doc/File Operators.md

@@ -0,0 +1,34 @@
+The file operator is used to filter based on filename. This is most often used with merge when needing to merge specific files together.
+
+```bash
+yq eval 'filename == "file1.yaml" * fileIndex == 0' file1.yaml file2.yaml
+```
+## Examples
+### Get filename
+Given a sample.yml file of:
+```yaml
+'': null
+```
+then
+```bash
+yq eval 'filename' sample.yml
+```
+will output
+```yaml
+sample.yaml
+```
+
+### Get file index
+Given a sample.yml file of:
+```yaml
+'': null
+```
+then
+```bash
+yq eval 'fileIndex' sample.yml
+```
+will output
+```yaml
+73
+```
+

pkg/yqlib/doc/headers/File Operators.md

@@ -0,0 +1,5 @@
+The file operator is used to filter based on filename. This is most often used with merge when needing to merge specific files together.
+
+```bash
+yq eval 'filename == "file1.yaml" * fileIndex == 0' file1.yaml file2.yaml
+```

pkg/yqlib/encoder_test.go

@@ -32,7 +32,7 @@ func TestJsonEncoderPreservesObjectOrder(t *testing.T) {
 	writer := bufio.NewWriter(&output)
 
 	var jsonEncoder = NewJsonEncoder(writer, 2)
-	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml")
+	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0)
 	if err != nil {
 		panic(err)
 	}

pkg/yqlib/lib.go

@@ -51,8 +51,8 @@ var GetStyle = &OperationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Han
 var GetTag = &OperationType{Type: "GET_TAG", NumArgs: 0, Precedence: 50, Handler: GetTagOperator}
 var GetComment = &OperationType{Type: "GET_COMMENT", NumArgs: 0, Precedence: 50, Handler: GetCommentsOperator}
 var GetDocumentIndex = &OperationType{Type: "GET_DOCUMENT_INDEX", NumArgs: 0, Precedence: 50, Handler: GetDocumentIndexOperator}
-var GetFilename = &OperationType{Type: "GET_FILENAME", NumArgs: 0, Precedence: 50, Handler: GetFilename}
+var GetFilename = &OperationType{Type: "GET_FILENAME", NumArgs: 0, Precedence: 50, Handler: GetFilenameOperator}
-var GetFileIndex = &OperationType{Type: "GET_FILE_INDEX", NumArgs: 0, Precedence: 50, Handler: GetFileIndex}
+var GetFileIndex = &OperationType{Type: "GET_FILE_INDEX", NumArgs: 0, Precedence: 50, Handler: GetFileIndexOperator}
 
 var Explode = &OperationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: ExplodeOperator}
 

pkg/yqlib/operator_file.go

@@ -7,7 +7,7 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
-func GetFilename(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func GetFilenameOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
 	log.Debugf("GetFilename")
 
 	var results = list.New()
@@ -22,7 +22,7 @@ func GetFilename(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathT
 	return results, nil
 }
 
-func GetFileIndex(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func GetFileIndexOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
 	log.Debugf("GetFileIndex")
 
 	var results = list.New()

pkg/yqlib/operator_file_test.go

@@ -0,0 +1,31 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var fileOperatorScenarios = []expressionScenario{
+	{
+		description: "Get filename",
+		document:    `{}`,
+		expression:  `filename`,
+		expected: []string{
+			"D0, P[], (!!str)::sample.yml\n",
+		},
+	},
+	{
+		description: "Get file index",
+		document:    `{}`,
+		expression:  `fileIndex`,
+		expected: []string{
+			"D0, P[], (!!int)::0\n",
+		},
+	},
+}
+
+func TestFileOperatorsScenarios(t *testing.T) {
+	for _, tt := range fileOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "File Operators", fileOperatorScenarios)
+}

pkg/yqlib/operator_multilpy.go

@@ -5,7 +5,7 @@ import (
 
 	"container/list"
 
-	"gopkg.in/yaml.v3"
+	yaml "gopkg.in/yaml.v3"
 )
 
 type CrossFunctionCalculation func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error)
@@ -15,12 +15,14 @@ func crossFunction(d *dataTreeNavigator, matchingNodes *list.List, pathNode *Pat
 	if err != nil {
 		return nil, err
 	}
+	log.Debugf("crossFunction LHS len: %v", lhs.Len())
 
 	rhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
 
 	if err != nil {
 		return nil, err
 	}
+	log.Debugf("crossFunction RHS len: %v", rhs.Len())
 
 	var results = list.New()
 
@@ -28,6 +30,7 @@ func crossFunction(d *dataTreeNavigator, matchingNodes *list.List, pathNode *Pat
 		lhsCandidate := el.Value.(*CandidateNode)
 
 		for rightEl := rhs.Front(); rightEl != nil; rightEl = rightEl.Next() {
+			log.Debugf("Applying calc")
 			rhsCandidate := rightEl.Value.(*CandidateNode)
 			resultCandidate, err := calculation(d, lhsCandidate, rhsCandidate)
 			if err != nil {
@@ -48,8 +51,8 @@ func MultiplyOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *
 func multiply(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	lhs.Node = UnwrapDoc(lhs.Node)
 	rhs.Node = UnwrapDoc(rhs.Node)
-	log.Debugf("Multipling LHS: %v", NodeToString(lhs))
+	log.Debugf("Multipling LHS: %v", lhs.Node.Tag)
-	log.Debugf("-          RHS: %v", NodeToString(rhs))
+	log.Debugf("-          RHS: %v", rhs.Node.Tag)
 
 	if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode ||
 		(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
@@ -67,7 +70,7 @@ func multiply(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*Ca
 		return mergeObjects(d, newThing, rhs)
 
 	}
-	return nil, fmt.Errorf("Cannot multiply %v with %v", NodeToString(lhs), NodeToString(rhs))
+	return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag)
 }
 
 func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {

pkg/yqlib/operators_test.go

@@ -35,7 +35,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
 	inputs := list.New()
 
 	if s.document != "" {
-		inputs, err = readDocuments(strings.NewReader(s.document), "sample.yml")
+		inputs, err = readDocuments(strings.NewReader(s.document), "sample.yml", 0)
 		if err != nil {
 			t.Error(err)
 			return

pkg/yqlib/path_tokeniser.go

@@ -204,6 +204,8 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`style`), opAssignableToken(GetStyle, AssignStyle))
 
 	lexer.Add([]byte(`tag`), opAssignableToken(GetTag, AssignTag))
+	lexer.Add([]byte(`filename`), opToken(GetFilename))
+	lexer.Add([]byte(`fileIndex`), opToken(GetFileIndex))
 
 	lexer.Add([]byte(`lineComment`), opTokenWithPrefs(GetComment, AssignComment, &CommentOpPreferences{LineComment: true}))
 

pkg/yqlib/printer_test.go

@@ -21,7 +21,7 @@ func TestPrinterMultipleDocsInSequence(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	printer := NewPrinter(writer, false, true, false, 2, true)
 
-	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml")
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
 		panic(err)
 	}
@@ -60,7 +60,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	printer := NewPrinter(writer, false, true, false, 2, true)
 
-	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml")
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
 		panic(err)
 	}
@@ -79,7 +79,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	printer := NewPrinter(writer, true, true, false, 0, false)
 
-	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml")
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
 		panic(err)
 	}

pkg/yqlib/utils.go

@@ -9,9 +9,13 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
+//TODO: convert to interface + struct
+
 var treeNavigator = NewDataTreeNavigator(NavigationPrefs{})
 var treeCreator = NewPathTreeCreator()
 
+var fileIndex = 0
+
 func readStream(filename string) (io.Reader, error) {
 	if filename == "-" {
 		return bufio.NewReader(os.Stdin), nil
@@ -30,6 +34,7 @@ func EvaluateStream(filename string, reader io.Reader, node *PathTreeNode, print
 		errorReading := decoder.Decode(&dataBucket)
 
 		if errorReading == io.EOF {
+			fileIndex = fileIndex + 1
 			return nil
 		} else if errorReading != nil {
 			return errorReading
@@ -38,6 +43,7 @@ func EvaluateStream(filename string, reader io.Reader, node *PathTreeNode, print
 			Document:  currentIndex,
 			Filename:  filename,
 			Node:      &dataBucket,
+			FileIndex: fileIndex,
 		}
 		inputList := list.New()
 		inputList.PushBack(candidateNode)
@@ -54,7 +60,7 @@ func EvaluateStream(filename string, reader io.Reader, node *PathTreeNode, print
 	}
 }
 
-func readDocuments(reader io.Reader, filename string) (*list.List, error) {
+func readDocuments(reader io.Reader, filename string, fileIndex int) (*list.List, error) {
 	decoder := yaml.NewDecoder(reader)
 	inputList := list.New()
 	var currentIndex uint = 0
@@ -76,6 +82,7 @@ func readDocuments(reader io.Reader, filename string) (*list.List, error) {
 			Document:  currentIndex,
 			Filename:  filename,
 			Node:      &dataBucket,
+			FileIndex: fileIndex,
 		}
 
 		inputList.PushBack(candidateNode)
@@ -85,6 +92,7 @@ func readDocuments(reader io.Reader, filename string) (*list.List, error) {
 }
 
 func EvaluateAllFileStreams(expression string, filenames []string, printer Printer) error {
+	fileIndex := 0
 	node, err := treeCreator.ParsePath(expression)
 	if err != nil {
 		return err
@@ -95,11 +103,12 @@ func EvaluateAllFileStreams(expression string, filenames []string, printer Print
 		if err != nil {
 			return err
 		}
-		fileDocuments, err := readDocuments(reader, filename)
+		fileDocuments, err := readDocuments(reader, filename, fileIndex)
 		if err != nil {
 			return err
 		}
 		allDocuments.PushBackList(fileDocuments)
+		fileIndex = fileIndex + 1
 	}
 	matches, err := treeNavigator.GetMatchingNodes(allDocuments, node)
 	if err != nil {