diff --git a/acceptance_tests/load-file.sh b/acceptance_tests/load-file.sh new file mode 100755 index 00000000..9fd00661 --- /dev/null +++ b/acceptance_tests/load-file.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +setUp() { + rm test*.yml || true +} + +testLoadFileNotExist() { +} + +testStrLoadFileNotExist() { +} \ No newline at end of file diff --git a/examples/data1.yaml b/examples/data1.yaml index fa32f314..f504f397 100644 --- a/examples/data1.yaml +++ b/examples/data1.yaml @@ -1,8 +1,2 @@ -# welcome! # cat -# bob ---- -# bob - -a: cat # meow - -# have a great day \ No newline at end of file +a: + include: 'data2.yaml' diff --git a/examples/thing.yml b/examples/thing.yml new file mode 100644 index 00000000..582ebae0 --- /dev/null +++ b/examples/thing.yml @@ -0,0 +1,2 @@ +a: apple is included +b: cool. \ No newline at end of file diff --git a/pkg/yqlib/doc/headers/load.md b/pkg/yqlib/doc/headers/load.md new file mode 100644 index 00000000..86264ef4 --- /dev/null +++ b/pkg/yqlib/doc/headers/load.md @@ -0,0 +1,13 @@ +# Load + +The `load`/`strload` operator allows you to load in content from another file referenced in your yaml document. + +Note that you can use string operators like `+` and `sub` to modify the value in the yaml file to a path that exists in your system. + + +Lets say there is a file `../../examples/thing.yml`: + +```yaml +a: apple is included +b: cool +``` diff --git a/pkg/yqlib/doc/load.md b/pkg/yqlib/doc/load.md new file mode 100644 index 00000000..de225df6 --- /dev/null +++ b/pkg/yqlib/doc/load.md @@ -0,0 +1,93 @@ +# Load + +The `load`/`strload` operator allows you to load in content from another file referenced in your yaml document. + +Note that you can use string operators like `+` and `sub` to modify the value in the yaml file to a path that exists in your system. + + +Lets say there is a file `../../examples/thing.yml`: + +```yaml +a: apple is included +b: cool +``` + +## Simple example +Given a sample.yml file of: +```yaml +myFile: ../../examples/thing.yml +``` +then +```bash +yq eval 'load(.myFile)' sample.yml +``` +will output +```yaml +a: apple is included +b: cool. +``` + +## Replace node with referenced file +Note that you can modify the filename in the load operator if needed. + +Given a sample.yml file of: +```yaml +something: + file: thing.yml +``` +then +```bash +yq eval '.something |= load("../../examples/" + .file)' sample.yml +``` +will output +```yaml +something: + a: apple is included + b: cool. +``` + +## Replace _all_ nodes with referenced file +Recursively match all the nodes (`..`) and then filter the ones that have a 'file' attribute. + +Given a sample.yml file of: +```yaml +something: + file: thing.yml +over: + here: + - file: thing.yml +``` +then +```bash +yq eval '(.. | select(has("file"))) |= load("../../examples/" + .file)' sample.yml +``` +will output +```yaml +something: + a: apple is included + b: cool. +over: + here: + - a: apple is included + b: cool. +``` + +## Replace node with referenced file as string +This will work for any text based file + +Given a sample.yml file of: +```yaml +something: + file: thing.yml +``` +then +```bash +yq eval '.something |= strload("../../examples/" + .file)' sample.yml +``` +will output +```yaml +something: |- + a: apple is included + b: cool. +``` + diff --git a/pkg/yqlib/expression_tokeniser.go b/pkg/yqlib/expression_tokeniser.go index 0b1edd28..a7eaab1c 100644 --- a/pkg/yqlib/expression_tokeniser.go +++ b/pkg/yqlib/expression_tokeniser.go @@ -336,6 +336,8 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`from_json`), opToken(decodeOpType)) lexer.Add([]byte(`sortKeys`), opToken(sortKeysOpType)) + lexer.Add([]byte(`load`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false})) + lexer.Add([]byte(`strload`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: true})) lexer.Add([]byte(`select`), opToken(selectOpType)) lexer.Add([]byte(`has`), opToken(hasOpType)) lexer.Add([]byte(`unique`), opToken(uniqueOpType)) diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index e0d4b36b..f63fa4cc 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -95,6 +95,8 @@ var captureOpType = &operationType{Type: "CAPTURE", NumArgs: 1, Precedence: 50, var testOpType = &operationType{Type: "TEST", NumArgs: 1, Precedence: 50, Handler: testOperator} var splitStringOpType = &operationType{Type: "SPLIT", NumArgs: 1, Precedence: 50, Handler: splitStringOperator} +var loadOpType = &operationType{Type: "LOAD", NumArgs: 1, Precedence: 50, Handler: loadYamlOperator} + var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 50, Handler: keysOperator} var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator} diff --git a/pkg/yqlib/operator_load.go b/pkg/yqlib/operator_load.go new file mode 100644 index 00000000..2c3be798 --- /dev/null +++ b/pkg/yqlib/operator_load.go @@ -0,0 +1,97 @@ +package yqlib + +import ( + "bufio" + "container/list" + "fmt" + "io/ioutil" + "os" + + "gopkg.in/yaml.v3" +) + +type loadPrefs struct { + loadAsString bool +} + +func loadString(filename string) (*CandidateNode, error) { + // ignore CWE-22 gosec issue - that's more targetted for http based apps that run in a public directory, + // and ensuring that it's not possible to give a path to a file outside that directory. + + filebytes, err := ioutil.ReadFile(filename) // #nosec + if err != nil { + return nil, err + } + + return &CandidateNode{Node: &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: string(filebytes)}}, nil +} + +func loadYaml(filename string) (*CandidateNode, error) { + + file, err := os.Open(filename) // #nosec + if err != nil { + return nil, err + } + reader := bufio.NewReader(file) + + documents, err := readDocuments(reader, filename, 0) + if err != nil { + return nil, err + } + + if documents.Len() == 0 { + // return null candidate + return &CandidateNode{Node: &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!null"}}, nil + } else if documents.Len() == 1 { + return documents.Front().Value.(*CandidateNode), nil + + } else { + sequenceNode := &CandidateNode{Node: &yaml.Node{Kind: yaml.SequenceNode}} + for doc := documents.Front(); doc != nil; doc = doc.Next() { + sequenceNode.Node.Content = append(sequenceNode.Node.Content, doc.Value.(*CandidateNode).Node) + } + return sequenceNode, nil + } +} + +func loadYamlOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { + log.Debugf("loadYamlOperator") + + loadPrefs := expressionNode.Operation.Preferences.(loadPrefs) + + // need to evaluate the 1st parameter against the context + // and return the data accordingly. + + var results = list.New() + + for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + + rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs) + if err != nil { + return Context{}, err + } + if rhs.MatchingNodes.Front() == nil { + return Context{}, fmt.Errorf("Filename expression returned nil") + } + nameCandidateNode := rhs.MatchingNodes.Front().Value.(*CandidateNode) + + filename := nameCandidateNode.Node.Value + + var contentsCandidate *CandidateNode + + if loadPrefs.loadAsString { + contentsCandidate, err = loadString(filename) + } else { + contentsCandidate, err = loadYaml(filename) + } + if err != nil { + return Context{}, fmt.Errorf("Failed to load %v: %w", filename, err) + } + + results.PushBack(contentsCandidate) + + } + + return context.ChildContext(results), nil +} diff --git a/pkg/yqlib/operator_load_test.go b/pkg/yqlib/operator_load_test.go new file mode 100644 index 00000000..4fe690f1 --- /dev/null +++ b/pkg/yqlib/operator_load_test.go @@ -0,0 +1,50 @@ +package yqlib + +import ( + "testing" +) + +var loadScenarios = []expressionScenario{ + { + description: "Simple example", + document: `{myFile: "../../examples/thing.yml"}`, + expression: `load(.myFile)`, + expected: []string{ + "D0, P[], (doc)::a: apple is included\nb: cool.\n", + }, + }, + { + description: "Replace node with referenced file", + subdescription: "Note that you can modify the filename in the load operator if needed.", + document: `{something: {file: "thing.yml"}}`, + expression: `.something |= load("../../examples/" + .file)`, + expected: []string{ + "D0, P[], (doc)::{something: {a: apple is included, b: cool.}}\n", + }, + }, + { + description: "Replace _all_ nodes with referenced file", + subdescription: "Recursively match all the nodes (`..`) and then filter the ones that have a 'file' attribute. ", + document: `{something: {file: "thing.yml"}, over: {here: [{file: "thing.yml"}]}}`, + expression: `(.. | select(has("file"))) |= load("../../examples/" + .file)`, + expected: []string{ + "D0, P[], (!!map)::{something: {a: apple is included, b: cool.}, over: {here: [{a: apple is included, b: cool.}]}}\n", + }, + }, + { + description: "Replace node with referenced file as string", + subdescription: "This will work for any text based file", + document: `{something: {file: "thing.yml"}}`, + expression: `.something |= strload("../../examples/" + .file)`, + expected: []string{ + "D0, P[], (doc)::{something: \"a: apple is included\\nb: cool.\"}\n", + }, + }, +} + +func TestLoadScenarios(t *testing.T) { + for _, tt := range loadScenarios { + testScenario(t, &tt) + } + documentScenarios(t, "load", loadScenarios) +}