suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2024-12-19 20:19:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added File operators

Browse Source
This commit is contained in:
Mike Farah 2020-11-20 22:57:32 +11:00
parent 4e385a1b93
commit d38caf6bc2
11 changed files with 105 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
pkg/yqlib/doc/File Operators.md Normal file
View File

@ -0,0 +1,34 @@
The file operator is used to filter based on filename. This is most often used with merge when needing to merge specific files together.
```bash
yq eval 'filename == "file1.yaml" * fileIndex == 0' file1.yaml file2.yaml
```
## Examples
### Get filename
Given a sample.yml file of:
```yaml
'': null
```
then
```bash
yq eval 'filename' sample.yml
```
will output
```yaml
sample.yaml
```
### Get file index
Given a sample.yml file of:
```yaml
'': null
```
then
```bash
yq eval 'fileIndex' sample.yml
```
will output
```yaml
73
```

5
pkg/yqlib/doc/headers/File Operators.md Normal file
View File

@ -0,0 +1,5 @@
The file operator is used to filter based on filename. This is most often used with merge when needing to merge specific files together.
```bash
yq eval 'filename == "file1.yaml" * fileIndex == 0' file1.yaml file2.yaml
```

2
pkg/yqlib/encoder_test.go
View File

@ -32,7 +32,7 @@ func TestJsonEncoderPreservesObjectOrder(t *testing.T) {
writer := bufio.NewWriter(&output)
var jsonEncoder = NewJsonEncoder(writer, 2)
inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml")
inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0)
if err != nil {
panic(err)
}

4
pkg/yqlib/lib.go
View File

@ -51,8 +51,8 @@ var GetStyle = &OperationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Han
var GetTag = &OperationType{Type: "GET_TAG", NumArgs: 0, Precedence: 50, Handler: GetTagOperator}
var GetComment = &OperationType{Type: "GET_COMMENT", NumArgs: 0, Precedence: 50, Handler: GetCommentsOperator}
var GetDocumentIndex = &OperationType{Type: "GET_DOCUMENT_INDEX", NumArgs: 0, Precedence: 50, Handler: GetDocumentIndexOperator}
var GetFilename = &OperationType{Type: "GET_FILENAME", NumArgs: 0, Precedence: 50, Handler: GetFilename}
var GetFileIndex = &OperationType{Type: "GET_FILE_INDEX", NumArgs: 0, Precedence: 50, Handler: GetFileIndex}
var GetFilename = &OperationType{Type: "GET_FILENAME", NumArgs: 0, Precedence: 50, Handler: GetFilenameOperator}
var GetFileIndex = &OperationType{Type: "GET_FILE_INDEX", NumArgs: 0, Precedence: 50, Handler: GetFileIndexOperator}
var Explode = &OperationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: ExplodeOperator}

4
pkg/yqlib/operator_file.go
View File

@ -7,7 +7,7 @@ import (
yaml "gopkg.in/yaml.v3"
)
func GetFilename(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
func GetFilenameOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
log.Debugf("GetFilename")
var results = list.New()
@ -22,7 +22,7 @@ func GetFilename(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathT
return results, nil
}
func GetFileIndex(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
func GetFileIndexOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
log.Debugf("GetFileIndex")
var results = list.New()

31
pkg/yqlib/operator_file_test.go Normal file
View File

@ -0,0 +1,31 @@
package yqlib
import (
"testing"
)
var fileOperatorScenarios = []expressionScenario{
{
description: "Get filename",
document: `{}`,
expression: `filename`,
expected: []string{
"D0, P[], (!!str)::sample.yml\n",
},
},
{
description: "Get file index",
document: `{}`,
expression: `fileIndex`,
expected: []string{
"D0, P[], (!!int)::0\n",
},
},
}
func TestFileOperatorsScenarios(t *testing.T) {
for _, tt := range fileOperatorScenarios {
testScenario(t, &tt)
}
documentScenarios(t, "File Operators", fileOperatorScenarios)
}

11
pkg/yqlib/operator_multilpy.go
View File

@ -5,7 +5,7 @@ import (
"container/list"
"gopkg.in/yaml.v3"
yaml "gopkg.in/yaml.v3"
)
type CrossFunctionCalculation func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error)
@ -15,12 +15,14 @@ func crossFunction(d *dataTreeNavigator, matchingNodes *list.List, pathNode *Pat
if err != nil {
return nil, err
}
log.Debugf("crossFunction LHS len: %v", lhs.Len())
rhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
if err != nil {
return nil, err
}
log.Debugf("crossFunction RHS len: %v", rhs.Len())
var results = list.New()
@ -28,6 +30,7 @@ func crossFunction(d *dataTreeNavigator, matchingNodes *list.List, pathNode *Pat
lhsCandidate := el.Value.(*CandidateNode)
for rightEl := rhs.Front(); rightEl != nil; rightEl = rightEl.Next() {
log.Debugf("Applying calc")
rhsCandidate := rightEl.Value.(*CandidateNode)
resultCandidate, err := calculation(d, lhsCandidate, rhsCandidate)
if err != nil {
@ -48,8 +51,8 @@ func MultiplyOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *
func multiply(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
lhs.Node = UnwrapDoc(lhs.Node)
rhs.Node = UnwrapDoc(rhs.Node)
log.Debugf("Multipling LHS: %v", NodeToString(lhs))
log.Debugf("- RHS: %v", NodeToString(rhs))
log.Debugf("Multipling LHS: %v", lhs.Node.Tag)
log.Debugf("- RHS: %v", rhs.Node.Tag)
if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode ||
(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
@ -67,7 +70,7 @@ func multiply(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*Ca
return mergeObjects(d, newThing, rhs)
}
return nil, fmt.Errorf("Cannot multiply %v with %v", NodeToString(lhs), NodeToString(rhs))
return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag)
}
func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {

2
pkg/yqlib/operators_test.go
View File

@ -35,7 +35,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
inputs := list.New()
if s.document != "" {
inputs, err = readDocuments(strings.NewReader(s.document), "sample.yml")
inputs, err = readDocuments(strings.NewReader(s.document), "sample.yml", 0)
if err != nil {
t.Error(err)
return

2
pkg/yqlib/path_tokeniser.go
View File

@ -204,6 +204,8 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`style`), opAssignableToken(GetStyle, AssignStyle))
lexer.Add([]byte(`tag`), opAssignableToken(GetTag, AssignTag))
lexer.Add([]byte(`filename`), opToken(GetFilename))
lexer.Add([]byte(`fileIndex`), opToken(GetFileIndex))
lexer.Add([]byte(`lineComment`), opTokenWithPrefs(GetComment, AssignComment, &CommentOpPreferences{LineComment: true}))

6
pkg/yqlib/printer_test.go
View File

@ -21,7 +21,7 @@ func TestPrinterMultipleDocsInSequence(t *testing.T) {
var writer = bufio.NewWriter(&output)
printer := NewPrinter(writer, false, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml")
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
if err != nil {
panic(err)
}
@ -60,7 +60,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
var writer = bufio.NewWriter(&output)
printer := NewPrinter(writer, false, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml")
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
if err != nil {
panic(err)
}
@ -79,7 +79,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
var writer = bufio.NewWriter(&output)
printer := NewPrinter(writer, true, true, false, 0, false)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml")
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
if err != nil {
panic(err)
}

25
pkg/yqlib/utils.go
View File

@ -9,9 +9,13 @@ import (
yaml "gopkg.in/yaml.v3"
)
//TODO: convert to interface + struct
var treeNavigator = NewDataTreeNavigator(NavigationPrefs{})
var treeCreator = NewPathTreeCreator()
var fileIndex = 0
func readStream(filename string) (io.Reader, error) {
if filename == "-" {
return bufio.NewReader(os.Stdin), nil
@ -30,14 +34,16 @@ func EvaluateStream(filename string, reader io.Reader, node *PathTreeNode, print
errorReading := decoder.Decode(&dataBucket)
if errorReading == io.EOF {
fileIndex = fileIndex + 1
return nil
} else if errorReading != nil {
return errorReading
}
candidateNode := &CandidateNode{
Document: currentIndex,
Filename: filename,
Node: &dataBucket,
Document: currentIndex,
Filename: filename,
Node: &dataBucket,
FileIndex: fileIndex,
}
inputList := list.New()
inputList.PushBack(candidateNode)
@ -54,7 +60,7 @@ func EvaluateStream(filename string, reader io.Reader, node *PathTreeNode, print
}
}
func readDocuments(reader io.Reader, filename string) (*list.List, error) {
func readDocuments(reader io.Reader, filename string, fileIndex int) (*list.List, error) {
decoder := yaml.NewDecoder(reader)
inputList := list.New()
var currentIndex uint = 0
@ -73,9 +79,10 @@ func readDocuments(reader io.Reader, filename string) (*list.List, error) {
return nil, errorReading
}
candidateNode := &CandidateNode{
Document: currentIndex,
Filename: filename,
Node: &dataBucket,
Document: currentIndex,
Filename: filename,
Node: &dataBucket,
FileIndex: fileIndex,
}
inputList.PushBack(candidateNode)
@ -85,6 +92,7 @@ func readDocuments(reader io.Reader, filename string) (*list.List, error) {
}
func EvaluateAllFileStreams(expression string, filenames []string, printer Printer) error {
fileIndex := 0
node, err := treeCreator.ParsePath(expression)
if err != nil {
return err
@ -95,11 +103,12 @@ func EvaluateAllFileStreams(expression string, filenames []string, printer Print
if err != nil {
return err
}
fileDocuments, err := readDocuments(reader, filename)
fileDocuments, err := readDocuments(reader, filename, fileIndex)
if err != nil {
return err
}
allDocuments.PushBackList(fileDocuments)
fileIndex = fileIndex + 1
}
matches, err := treeNavigator.GetMatchingNodes(allDocuments, node)
if err != nil {