mirror of
https://github.com/mikefarah/yq.git
synced 2024-12-19 20:19:04 +00:00
Added File operators
This commit is contained in:
parent
4e385a1b93
commit
d38caf6bc2
34
pkg/yqlib/doc/File Operators.md
Normal file
34
pkg/yqlib/doc/File Operators.md
Normal file
@ -0,0 +1,34 @@
|
|||||||
|
The file operator is used to filter based on filename. This is most often used with merge when needing to merge specific files together.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
yq eval 'filename == "file1.yaml" * fileIndex == 0' file1.yaml file2.yaml
|
||||||
|
```
|
||||||
|
## Examples
|
||||||
|
### Get filename
|
||||||
|
Given a sample.yml file of:
|
||||||
|
```yaml
|
||||||
|
'': null
|
||||||
|
```
|
||||||
|
then
|
||||||
|
```bash
|
||||||
|
yq eval 'filename' sample.yml
|
||||||
|
```
|
||||||
|
will output
|
||||||
|
```yaml
|
||||||
|
sample.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
### Get file index
|
||||||
|
Given a sample.yml file of:
|
||||||
|
```yaml
|
||||||
|
'': null
|
||||||
|
```
|
||||||
|
then
|
||||||
|
```bash
|
||||||
|
yq eval 'fileIndex' sample.yml
|
||||||
|
```
|
||||||
|
will output
|
||||||
|
```yaml
|
||||||
|
73
|
||||||
|
```
|
||||||
|
|
5
pkg/yqlib/doc/headers/File Operators.md
Normal file
5
pkg/yqlib/doc/headers/File Operators.md
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
The file operator is used to filter based on filename. This is most often used with merge when needing to merge specific files together.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
yq eval 'filename == "file1.yaml" * fileIndex == 0' file1.yaml file2.yaml
|
||||||
|
```
|
@ -32,7 +32,7 @@ func TestJsonEncoderPreservesObjectOrder(t *testing.T) {
|
|||||||
writer := bufio.NewWriter(&output)
|
writer := bufio.NewWriter(&output)
|
||||||
|
|
||||||
var jsonEncoder = NewJsonEncoder(writer, 2)
|
var jsonEncoder = NewJsonEncoder(writer, 2)
|
||||||
inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml")
|
inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
@ -51,8 +51,8 @@ var GetStyle = &OperationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Han
|
|||||||
var GetTag = &OperationType{Type: "GET_TAG", NumArgs: 0, Precedence: 50, Handler: GetTagOperator}
|
var GetTag = &OperationType{Type: "GET_TAG", NumArgs: 0, Precedence: 50, Handler: GetTagOperator}
|
||||||
var GetComment = &OperationType{Type: "GET_COMMENT", NumArgs: 0, Precedence: 50, Handler: GetCommentsOperator}
|
var GetComment = &OperationType{Type: "GET_COMMENT", NumArgs: 0, Precedence: 50, Handler: GetCommentsOperator}
|
||||||
var GetDocumentIndex = &OperationType{Type: "GET_DOCUMENT_INDEX", NumArgs: 0, Precedence: 50, Handler: GetDocumentIndexOperator}
|
var GetDocumentIndex = &OperationType{Type: "GET_DOCUMENT_INDEX", NumArgs: 0, Precedence: 50, Handler: GetDocumentIndexOperator}
|
||||||
var GetFilename = &OperationType{Type: "GET_FILENAME", NumArgs: 0, Precedence: 50, Handler: GetFilename}
|
var GetFilename = &OperationType{Type: "GET_FILENAME", NumArgs: 0, Precedence: 50, Handler: GetFilenameOperator}
|
||||||
var GetFileIndex = &OperationType{Type: "GET_FILE_INDEX", NumArgs: 0, Precedence: 50, Handler: GetFileIndex}
|
var GetFileIndex = &OperationType{Type: "GET_FILE_INDEX", NumArgs: 0, Precedence: 50, Handler: GetFileIndexOperator}
|
||||||
|
|
||||||
var Explode = &OperationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: ExplodeOperator}
|
var Explode = &OperationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: ExplodeOperator}
|
||||||
|
|
||||||
|
@ -7,7 +7,7 @@ import (
|
|||||||
yaml "gopkg.in/yaml.v3"
|
yaml "gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
func GetFilename(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
|
func GetFilenameOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
|
||||||
log.Debugf("GetFilename")
|
log.Debugf("GetFilename")
|
||||||
|
|
||||||
var results = list.New()
|
var results = list.New()
|
||||||
@ -22,7 +22,7 @@ func GetFilename(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathT
|
|||||||
return results, nil
|
return results, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func GetFileIndex(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
|
func GetFileIndexOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
|
||||||
log.Debugf("GetFileIndex")
|
log.Debugf("GetFileIndex")
|
||||||
|
|
||||||
var results = list.New()
|
var results = list.New()
|
||||||
|
31
pkg/yqlib/operator_file_test.go
Normal file
31
pkg/yqlib/operator_file_test.go
Normal file
@ -0,0 +1,31 @@
|
|||||||
|
package yqlib
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
var fileOperatorScenarios = []expressionScenario{
|
||||||
|
{
|
||||||
|
description: "Get filename",
|
||||||
|
document: `{}`,
|
||||||
|
expression: `filename`,
|
||||||
|
expected: []string{
|
||||||
|
"D0, P[], (!!str)::sample.yml\n",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
description: "Get file index",
|
||||||
|
document: `{}`,
|
||||||
|
expression: `fileIndex`,
|
||||||
|
expected: []string{
|
||||||
|
"D0, P[], (!!int)::0\n",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFileOperatorsScenarios(t *testing.T) {
|
||||||
|
for _, tt := range fileOperatorScenarios {
|
||||||
|
testScenario(t, &tt)
|
||||||
|
}
|
||||||
|
documentScenarios(t, "File Operators", fileOperatorScenarios)
|
||||||
|
}
|
@ -5,7 +5,7 @@ import (
|
|||||||
|
|
||||||
"container/list"
|
"container/list"
|
||||||
|
|
||||||
"gopkg.in/yaml.v3"
|
yaml "gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
type CrossFunctionCalculation func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error)
|
type CrossFunctionCalculation func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error)
|
||||||
@ -15,12 +15,14 @@ func crossFunction(d *dataTreeNavigator, matchingNodes *list.List, pathNode *Pat
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
log.Debugf("crossFunction LHS len: %v", lhs.Len())
|
||||||
|
|
||||||
rhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
|
rhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
log.Debugf("crossFunction RHS len: %v", rhs.Len())
|
||||||
|
|
||||||
var results = list.New()
|
var results = list.New()
|
||||||
|
|
||||||
@ -28,6 +30,7 @@ func crossFunction(d *dataTreeNavigator, matchingNodes *list.List, pathNode *Pat
|
|||||||
lhsCandidate := el.Value.(*CandidateNode)
|
lhsCandidate := el.Value.(*CandidateNode)
|
||||||
|
|
||||||
for rightEl := rhs.Front(); rightEl != nil; rightEl = rightEl.Next() {
|
for rightEl := rhs.Front(); rightEl != nil; rightEl = rightEl.Next() {
|
||||||
|
log.Debugf("Applying calc")
|
||||||
rhsCandidate := rightEl.Value.(*CandidateNode)
|
rhsCandidate := rightEl.Value.(*CandidateNode)
|
||||||
resultCandidate, err := calculation(d, lhsCandidate, rhsCandidate)
|
resultCandidate, err := calculation(d, lhsCandidate, rhsCandidate)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -48,8 +51,8 @@ func MultiplyOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *
|
|||||||
func multiply(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
|
func multiply(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
|
||||||
lhs.Node = UnwrapDoc(lhs.Node)
|
lhs.Node = UnwrapDoc(lhs.Node)
|
||||||
rhs.Node = UnwrapDoc(rhs.Node)
|
rhs.Node = UnwrapDoc(rhs.Node)
|
||||||
log.Debugf("Multipling LHS: %v", NodeToString(lhs))
|
log.Debugf("Multipling LHS: %v", lhs.Node.Tag)
|
||||||
log.Debugf("- RHS: %v", NodeToString(rhs))
|
log.Debugf("- RHS: %v", rhs.Node.Tag)
|
||||||
|
|
||||||
if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode ||
|
if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode ||
|
||||||
(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
|
(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
|
||||||
@ -67,7 +70,7 @@ func multiply(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*Ca
|
|||||||
return mergeObjects(d, newThing, rhs)
|
return mergeObjects(d, newThing, rhs)
|
||||||
|
|
||||||
}
|
}
|
||||||
return nil, fmt.Errorf("Cannot multiply %v with %v", NodeToString(lhs), NodeToString(rhs))
|
return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag)
|
||||||
}
|
}
|
||||||
|
|
||||||
func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
|
func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
|
||||||
|
@ -35,7 +35,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
|
|||||||
inputs := list.New()
|
inputs := list.New()
|
||||||
|
|
||||||
if s.document != "" {
|
if s.document != "" {
|
||||||
inputs, err = readDocuments(strings.NewReader(s.document), "sample.yml")
|
inputs, err = readDocuments(strings.NewReader(s.document), "sample.yml", 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Error(err)
|
t.Error(err)
|
||||||
return
|
return
|
||||||
|
@ -204,6 +204,8 @@ func initLexer() (*lex.Lexer, error) {
|
|||||||
lexer.Add([]byte(`style`), opAssignableToken(GetStyle, AssignStyle))
|
lexer.Add([]byte(`style`), opAssignableToken(GetStyle, AssignStyle))
|
||||||
|
|
||||||
lexer.Add([]byte(`tag`), opAssignableToken(GetTag, AssignTag))
|
lexer.Add([]byte(`tag`), opAssignableToken(GetTag, AssignTag))
|
||||||
|
lexer.Add([]byte(`filename`), opToken(GetFilename))
|
||||||
|
lexer.Add([]byte(`fileIndex`), opToken(GetFileIndex))
|
||||||
|
|
||||||
lexer.Add([]byte(`lineComment`), opTokenWithPrefs(GetComment, AssignComment, &CommentOpPreferences{LineComment: true}))
|
lexer.Add([]byte(`lineComment`), opTokenWithPrefs(GetComment, AssignComment, &CommentOpPreferences{LineComment: true}))
|
||||||
|
|
||||||
|
@ -21,7 +21,7 @@ func TestPrinterMultipleDocsInSequence(t *testing.T) {
|
|||||||
var writer = bufio.NewWriter(&output)
|
var writer = bufio.NewWriter(&output)
|
||||||
printer := NewPrinter(writer, false, true, false, 2, true)
|
printer := NewPrinter(writer, false, true, false, 2, true)
|
||||||
|
|
||||||
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml")
|
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
@ -60,7 +60,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
|
|||||||
var writer = bufio.NewWriter(&output)
|
var writer = bufio.NewWriter(&output)
|
||||||
printer := NewPrinter(writer, false, true, false, 2, true)
|
printer := NewPrinter(writer, false, true, false, 2, true)
|
||||||
|
|
||||||
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml")
|
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
@ -79,7 +79,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
|
|||||||
var writer = bufio.NewWriter(&output)
|
var writer = bufio.NewWriter(&output)
|
||||||
printer := NewPrinter(writer, true, true, false, 0, false)
|
printer := NewPrinter(writer, true, true, false, 0, false)
|
||||||
|
|
||||||
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml")
|
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
@ -9,9 +9,13 @@ import (
|
|||||||
yaml "gopkg.in/yaml.v3"
|
yaml "gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
//TODO: convert to interface + struct
|
||||||
|
|
||||||
var treeNavigator = NewDataTreeNavigator(NavigationPrefs{})
|
var treeNavigator = NewDataTreeNavigator(NavigationPrefs{})
|
||||||
var treeCreator = NewPathTreeCreator()
|
var treeCreator = NewPathTreeCreator()
|
||||||
|
|
||||||
|
var fileIndex = 0
|
||||||
|
|
||||||
func readStream(filename string) (io.Reader, error) {
|
func readStream(filename string) (io.Reader, error) {
|
||||||
if filename == "-" {
|
if filename == "-" {
|
||||||
return bufio.NewReader(os.Stdin), nil
|
return bufio.NewReader(os.Stdin), nil
|
||||||
@ -30,14 +34,16 @@ func EvaluateStream(filename string, reader io.Reader, node *PathTreeNode, print
|
|||||||
errorReading := decoder.Decode(&dataBucket)
|
errorReading := decoder.Decode(&dataBucket)
|
||||||
|
|
||||||
if errorReading == io.EOF {
|
if errorReading == io.EOF {
|
||||||
|
fileIndex = fileIndex + 1
|
||||||
return nil
|
return nil
|
||||||
} else if errorReading != nil {
|
} else if errorReading != nil {
|
||||||
return errorReading
|
return errorReading
|
||||||
}
|
}
|
||||||
candidateNode := &CandidateNode{
|
candidateNode := &CandidateNode{
|
||||||
Document: currentIndex,
|
Document: currentIndex,
|
||||||
Filename: filename,
|
Filename: filename,
|
||||||
Node: &dataBucket,
|
Node: &dataBucket,
|
||||||
|
FileIndex: fileIndex,
|
||||||
}
|
}
|
||||||
inputList := list.New()
|
inputList := list.New()
|
||||||
inputList.PushBack(candidateNode)
|
inputList.PushBack(candidateNode)
|
||||||
@ -54,7 +60,7 @@ func EvaluateStream(filename string, reader io.Reader, node *PathTreeNode, print
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func readDocuments(reader io.Reader, filename string) (*list.List, error) {
|
func readDocuments(reader io.Reader, filename string, fileIndex int) (*list.List, error) {
|
||||||
decoder := yaml.NewDecoder(reader)
|
decoder := yaml.NewDecoder(reader)
|
||||||
inputList := list.New()
|
inputList := list.New()
|
||||||
var currentIndex uint = 0
|
var currentIndex uint = 0
|
||||||
@ -73,9 +79,10 @@ func readDocuments(reader io.Reader, filename string) (*list.List, error) {
|
|||||||
return nil, errorReading
|
return nil, errorReading
|
||||||
}
|
}
|
||||||
candidateNode := &CandidateNode{
|
candidateNode := &CandidateNode{
|
||||||
Document: currentIndex,
|
Document: currentIndex,
|
||||||
Filename: filename,
|
Filename: filename,
|
||||||
Node: &dataBucket,
|
Node: &dataBucket,
|
||||||
|
FileIndex: fileIndex,
|
||||||
}
|
}
|
||||||
|
|
||||||
inputList.PushBack(candidateNode)
|
inputList.PushBack(candidateNode)
|
||||||
@ -85,6 +92,7 @@ func readDocuments(reader io.Reader, filename string) (*list.List, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func EvaluateAllFileStreams(expression string, filenames []string, printer Printer) error {
|
func EvaluateAllFileStreams(expression string, filenames []string, printer Printer) error {
|
||||||
|
fileIndex := 0
|
||||||
node, err := treeCreator.ParsePath(expression)
|
node, err := treeCreator.ParsePath(expression)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@ -95,11 +103,12 @@ func EvaluateAllFileStreams(expression string, filenames []string, printer Print
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
fileDocuments, err := readDocuments(reader, filename)
|
fileDocuments, err := readDocuments(reader, filename, fileIndex)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
allDocuments.PushBackList(fileDocuments)
|
allDocuments.PushBackList(fileDocuments)
|
||||||
|
fileIndex = fileIndex + 1
|
||||||
}
|
}
|
||||||
matches, err := treeNavigator.GetMatchingNodes(allDocuments, node)
|
matches, err := treeNavigator.GetMatchingNodes(allDocuments, node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
Loading…
Reference in New Issue
Block a user