From d38caf6bc2c4fdf21318b2f920c8e9ba2b691559 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 20 Nov 2020 22:57:32 +1100 Subject: [PATCH] Added File operators --- pkg/yqlib/doc/File Operators.md | 34 +++++++++++++++++++++++++ pkg/yqlib/doc/headers/File Operators.md | 5 ++++ pkg/yqlib/encoder_test.go | 2 +- pkg/yqlib/lib.go | 4 +-- pkg/yqlib/operator_file.go | 4 +-- pkg/yqlib/operator_file_test.go | 31 ++++++++++++++++++++++ pkg/yqlib/operator_multilpy.go | 11 +++++--- pkg/yqlib/operators_test.go | 2 +- pkg/yqlib/path_tokeniser.go | 2 ++ pkg/yqlib/printer_test.go | 6 ++--- pkg/yqlib/utils.go | 25 ++++++++++++------ 11 files changed, 105 insertions(+), 21 deletions(-) create mode 100644 pkg/yqlib/doc/File Operators.md create mode 100644 pkg/yqlib/doc/headers/File Operators.md create mode 100644 pkg/yqlib/operator_file_test.go diff --git a/pkg/yqlib/doc/File Operators.md b/pkg/yqlib/doc/File Operators.md new file mode 100644 index 00000000..9ecc52fd --- /dev/null +++ b/pkg/yqlib/doc/File Operators.md @@ -0,0 +1,34 @@ +The file operator is used to filter based on filename. This is most often used with merge when needing to merge specific files together. + +```bash +yq eval 'filename == "file1.yaml" * fileIndex == 0' file1.yaml file2.yaml +``` +## Examples +### Get filename +Given a sample.yml file of: +```yaml +'': null +``` +then +```bash +yq eval 'filename' sample.yml +``` +will output +```yaml +sample.yaml +``` + +### Get file index +Given a sample.yml file of: +```yaml +'': null +``` +then +```bash +yq eval 'fileIndex' sample.yml +``` +will output +```yaml +73 +``` + diff --git a/pkg/yqlib/doc/headers/File Operators.md b/pkg/yqlib/doc/headers/File Operators.md new file mode 100644 index 00000000..6f836113 --- /dev/null +++ b/pkg/yqlib/doc/headers/File Operators.md @@ -0,0 +1,5 @@ +The file operator is used to filter based on filename. This is most often used with merge when needing to merge specific files together. + +```bash +yq eval 'filename == "file1.yaml" * fileIndex == 0' file1.yaml file2.yaml +``` \ No newline at end of file diff --git a/pkg/yqlib/encoder_test.go b/pkg/yqlib/encoder_test.go index 867ad40f..d7211734 100644 --- a/pkg/yqlib/encoder_test.go +++ b/pkg/yqlib/encoder_test.go @@ -32,7 +32,7 @@ func TestJsonEncoderPreservesObjectOrder(t *testing.T) { writer := bufio.NewWriter(&output) var jsonEncoder = NewJsonEncoder(writer, 2) - inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml") + inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0) if err != nil { panic(err) } diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 0c93f086..4cd7b597 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -51,8 +51,8 @@ var GetStyle = &OperationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Han var GetTag = &OperationType{Type: "GET_TAG", NumArgs: 0, Precedence: 50, Handler: GetTagOperator} var GetComment = &OperationType{Type: "GET_COMMENT", NumArgs: 0, Precedence: 50, Handler: GetCommentsOperator} var GetDocumentIndex = &OperationType{Type: "GET_DOCUMENT_INDEX", NumArgs: 0, Precedence: 50, Handler: GetDocumentIndexOperator} -var GetFilename = &OperationType{Type: "GET_FILENAME", NumArgs: 0, Precedence: 50, Handler: GetFilename} -var GetFileIndex = &OperationType{Type: "GET_FILE_INDEX", NumArgs: 0, Precedence: 50, Handler: GetFileIndex} +var GetFilename = &OperationType{Type: "GET_FILENAME", NumArgs: 0, Precedence: 50, Handler: GetFilenameOperator} +var GetFileIndex = &OperationType{Type: "GET_FILE_INDEX", NumArgs: 0, Precedence: 50, Handler: GetFileIndexOperator} var Explode = &OperationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: ExplodeOperator} diff --git a/pkg/yqlib/operator_file.go b/pkg/yqlib/operator_file.go index ad0653b4..23331951 100644 --- a/pkg/yqlib/operator_file.go +++ b/pkg/yqlib/operator_file.go @@ -7,7 +7,7 @@ import ( yaml "gopkg.in/yaml.v3" ) -func GetFilename(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) { +func GetFilenameOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) { log.Debugf("GetFilename") var results = list.New() @@ -22,7 +22,7 @@ func GetFilename(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathT return results, nil } -func GetFileIndex(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) { +func GetFileIndexOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) { log.Debugf("GetFileIndex") var results = list.New() diff --git a/pkg/yqlib/operator_file_test.go b/pkg/yqlib/operator_file_test.go new file mode 100644 index 00000000..7117426c --- /dev/null +++ b/pkg/yqlib/operator_file_test.go @@ -0,0 +1,31 @@ +package yqlib + +import ( + "testing" +) + +var fileOperatorScenarios = []expressionScenario{ + { + description: "Get filename", + document: `{}`, + expression: `filename`, + expected: []string{ + "D0, P[], (!!str)::sample.yml\n", + }, + }, + { + description: "Get file index", + document: `{}`, + expression: `fileIndex`, + expected: []string{ + "D0, P[], (!!int)::0\n", + }, + }, +} + +func TestFileOperatorsScenarios(t *testing.T) { + for _, tt := range fileOperatorScenarios { + testScenario(t, &tt) + } + documentScenarios(t, "File Operators", fileOperatorScenarios) +} diff --git a/pkg/yqlib/operator_multilpy.go b/pkg/yqlib/operator_multilpy.go index 9f25962e..ac7b456e 100644 --- a/pkg/yqlib/operator_multilpy.go +++ b/pkg/yqlib/operator_multilpy.go @@ -5,7 +5,7 @@ import ( "container/list" - "gopkg.in/yaml.v3" + yaml "gopkg.in/yaml.v3" ) type CrossFunctionCalculation func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) @@ -15,12 +15,14 @@ func crossFunction(d *dataTreeNavigator, matchingNodes *list.List, pathNode *Pat if err != nil { return nil, err } + log.Debugf("crossFunction LHS len: %v", lhs.Len()) rhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs) if err != nil { return nil, err } + log.Debugf("crossFunction RHS len: %v", rhs.Len()) var results = list.New() @@ -28,6 +30,7 @@ func crossFunction(d *dataTreeNavigator, matchingNodes *list.List, pathNode *Pat lhsCandidate := el.Value.(*CandidateNode) for rightEl := rhs.Front(); rightEl != nil; rightEl = rightEl.Next() { + log.Debugf("Applying calc") rhsCandidate := rightEl.Value.(*CandidateNode) resultCandidate, err := calculation(d, lhsCandidate, rhsCandidate) if err != nil { @@ -48,8 +51,8 @@ func MultiplyOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode * func multiply(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { lhs.Node = UnwrapDoc(lhs.Node) rhs.Node = UnwrapDoc(rhs.Node) - log.Debugf("Multipling LHS: %v", NodeToString(lhs)) - log.Debugf("- RHS: %v", NodeToString(rhs)) + log.Debugf("Multipling LHS: %v", lhs.Node.Tag) + log.Debugf("- RHS: %v", rhs.Node.Tag) if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode || (lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) { @@ -67,7 +70,7 @@ func multiply(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*Ca return mergeObjects(d, newThing, rhs) } - return nil, fmt.Errorf("Cannot multiply %v with %v", NodeToString(lhs), NodeToString(rhs)) + return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag) } func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index 8af48869..c9713b00 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -35,7 +35,7 @@ func testScenario(t *testing.T, s *expressionScenario) { inputs := list.New() if s.document != "" { - inputs, err = readDocuments(strings.NewReader(s.document), "sample.yml") + inputs, err = readDocuments(strings.NewReader(s.document), "sample.yml", 0) if err != nil { t.Error(err) return diff --git a/pkg/yqlib/path_tokeniser.go b/pkg/yqlib/path_tokeniser.go index 115b6d5b..1fc0ed3b 100644 --- a/pkg/yqlib/path_tokeniser.go +++ b/pkg/yqlib/path_tokeniser.go @@ -204,6 +204,8 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`style`), opAssignableToken(GetStyle, AssignStyle)) lexer.Add([]byte(`tag`), opAssignableToken(GetTag, AssignTag)) + lexer.Add([]byte(`filename`), opToken(GetFilename)) + lexer.Add([]byte(`fileIndex`), opToken(GetFileIndex)) lexer.Add([]byte(`lineComment`), opTokenWithPrefs(GetComment, AssignComment, &CommentOpPreferences{LineComment: true})) diff --git a/pkg/yqlib/printer_test.go b/pkg/yqlib/printer_test.go index 699a8d05..f54af3c3 100644 --- a/pkg/yqlib/printer_test.go +++ b/pkg/yqlib/printer_test.go @@ -21,7 +21,7 @@ func TestPrinterMultipleDocsInSequence(t *testing.T) { var writer = bufio.NewWriter(&output) printer := NewPrinter(writer, false, true, false, 2, true) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml") + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0) if err != nil { panic(err) } @@ -60,7 +60,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) { var writer = bufio.NewWriter(&output) printer := NewPrinter(writer, false, true, false, 2, true) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml") + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0) if err != nil { panic(err) } @@ -79,7 +79,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) { var writer = bufio.NewWriter(&output) printer := NewPrinter(writer, true, true, false, 0, false) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml") + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0) if err != nil { panic(err) } diff --git a/pkg/yqlib/utils.go b/pkg/yqlib/utils.go index 3b7360b8..4833094b 100644 --- a/pkg/yqlib/utils.go +++ b/pkg/yqlib/utils.go @@ -9,9 +9,13 @@ import ( yaml "gopkg.in/yaml.v3" ) +//TODO: convert to interface + struct + var treeNavigator = NewDataTreeNavigator(NavigationPrefs{}) var treeCreator = NewPathTreeCreator() +var fileIndex = 0 + func readStream(filename string) (io.Reader, error) { if filename == "-" { return bufio.NewReader(os.Stdin), nil @@ -30,14 +34,16 @@ func EvaluateStream(filename string, reader io.Reader, node *PathTreeNode, print errorReading := decoder.Decode(&dataBucket) if errorReading == io.EOF { + fileIndex = fileIndex + 1 return nil } else if errorReading != nil { return errorReading } candidateNode := &CandidateNode{ - Document: currentIndex, - Filename: filename, - Node: &dataBucket, + Document: currentIndex, + Filename: filename, + Node: &dataBucket, + FileIndex: fileIndex, } inputList := list.New() inputList.PushBack(candidateNode) @@ -54,7 +60,7 @@ func EvaluateStream(filename string, reader io.Reader, node *PathTreeNode, print } } -func readDocuments(reader io.Reader, filename string) (*list.List, error) { +func readDocuments(reader io.Reader, filename string, fileIndex int) (*list.List, error) { decoder := yaml.NewDecoder(reader) inputList := list.New() var currentIndex uint = 0 @@ -73,9 +79,10 @@ func readDocuments(reader io.Reader, filename string) (*list.List, error) { return nil, errorReading } candidateNode := &CandidateNode{ - Document: currentIndex, - Filename: filename, - Node: &dataBucket, + Document: currentIndex, + Filename: filename, + Node: &dataBucket, + FileIndex: fileIndex, } inputList.PushBack(candidateNode) @@ -85,6 +92,7 @@ func readDocuments(reader io.Reader, filename string) (*list.List, error) { } func EvaluateAllFileStreams(expression string, filenames []string, printer Printer) error { + fileIndex := 0 node, err := treeCreator.ParsePath(expression) if err != nil { return err @@ -95,11 +103,12 @@ func EvaluateAllFileStreams(expression string, filenames []string, printer Print if err != nil { return err } - fileDocuments, err := readDocuments(reader, filename) + fileDocuments, err := readDocuments(reader, filename, fileIndex) if err != nil { return err } allDocuments.PushBackList(fileDocuments) + fileIndex = fileIndex + 1 } matches, err := treeNavigator.GetMatchingNodes(allDocuments, node) if err != nil {