Load file acceptance test

acceptance_tests/load-file.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+testLoadFileNotExist() {
+  result=$(./yq e -n 'load("cat.yml")' 2>&1)
+  assertEquals 1 $?
+  assertEquals "Error: Failed to load cat.yml: open cat.yml: no such file or directory" "$result"
+}
+
+testLoadFileExpNotExist() {
+  result=$(./yq e -n 'load(.a)' 2>&1)
+  assertEquals 1 $?
+  assertEquals "Error: Filename expression returned nil" "$result"
+}
+
+testStrLoadFileNotExist() {
+  result=$(./yq e -n 'strload("cat.yml")' 2>&1)
+  assertEquals 1 $?
+  assertEquals "Error: Failed to load cat.yml: open cat.yml: no such file or directory" "$result"
+}
+
+testStrLoadFileExpNotExist() {
+  result=$(./yq e -n 'strload(.a)' 2>&1)
+  assertEquals 1 $?
+  assertEquals "Error: Filename expression returned nil" "$result"
+}
+
+source ./scripts/shunit2

examples/data1.yaml

@@ -1,8 +1,2 @@
-# welcome! # cat
+a:
-# bob
+  include: 'data2.yaml'
----
-# bob
-
-a: cat # meow
-
-# have a great day

examples/thing.yml

@@ -0,0 +1,2 @@
+a: apple is included
+b: cool.

pkg/yqlib/doc/headers/load.md

@@ -0,0 +1,13 @@
+# Load
+
+The `load`/`strload` operator allows you to load in content from another file referenced in your yaml document.
+
+Note that you can use string operators like `+` and `sub` to modify the value in the yaml file to a path that exists in your system.
+
+
+Lets say there is a file `../../examples/thing.yml`:
+
+```yaml
+a: apple is included
+b: cool
+```

pkg/yqlib/doc/load.md

@@ -0,0 +1,93 @@
+# Load
+
+The `load`/`strload` operator allows you to load in content from another file referenced in your yaml document.
+
+Note that you can use string operators like `+` and `sub` to modify the value in the yaml file to a path that exists in your system.
+
+
+Lets say there is a file `../../examples/thing.yml`:
+
+```yaml
+a: apple is included
+b: cool
+```
+
+## Simple example
+Given a sample.yml file of:
+```yaml
+myFile: ../../examples/thing.yml
+```
+then
+```bash
+yq eval 'load(.myFile)' sample.yml
+```
+will output
+```yaml
+a: apple is included
+b: cool.
+```
+
+## Replace node with referenced file
+Note that you can modify the filename in the load operator if needed.
+
+Given a sample.yml file of:
+```yaml
+something:
+  file: thing.yml
+```
+then
+```bash
+yq eval '.something |= load("../../examples/" + .file)' sample.yml
+```
+will output
+```yaml
+something:
+  a: apple is included
+  b: cool.
+```
+
+## Replace _all_ nodes with referenced file
+Recursively match all the nodes (`..`) and then filter the ones that have a 'file' attribute. 
+
+Given a sample.yml file of:
+```yaml
+something:
+  file: thing.yml
+over:
+  here:
+    - file: thing.yml
+```
+then
+```bash
+yq eval '(.. | select(has("file"))) |= load("../../examples/" + .file)' sample.yml
+```
+will output
+```yaml
+something:
+  a: apple is included
+  b: cool.
+over:
+  here:
+    - a: apple is included
+      b: cool.
+```
+
+## Replace node with referenced file as string
+This will work for any text based file
+
+Given a sample.yml file of:
+```yaml
+something:
+  file: thing.yml
+```
+then
+```bash
+yq eval '.something |= strload("../../examples/" + .file)' sample.yml
+```
+will output
+```yaml
+something: |-
+  a: apple is included
+  b: cool.
+```
+

pkg/yqlib/expression_tokeniser.go

@@ -336,6 +336,8 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`from_json`), opToken(decodeOpType))
 
 	lexer.Add([]byte(`sortKeys`), opToken(sortKeysOpType))
+	lexer.Add([]byte(`load`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false}))
+	lexer.Add([]byte(`strload`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: true}))
 	lexer.Add([]byte(`select`), opToken(selectOpType))
 	lexer.Add([]byte(`has`), opToken(hasOpType))
 	lexer.Add([]byte(`unique`), opToken(uniqueOpType))

pkg/yqlib/lib.go

@@ -95,6 +95,8 @@ var captureOpType = &operationType{Type: "CAPTURE", NumArgs: 1, Precedence: 50,
 var testOpType = &operationType{Type: "TEST", NumArgs: 1, Precedence: 50, Handler: testOperator}
 var splitStringOpType = &operationType{Type: "SPLIT", NumArgs: 1, Precedence: 50, Handler: splitStringOperator}
 
+var loadOpType = &operationType{Type: "LOAD", NumArgs: 1, Precedence: 50, Handler: loadYamlOperator}
+
 var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 50, Handler: keysOperator}
 
 var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator}

pkg/yqlib/operator_load.go

@@ -0,0 +1,97 @@
+package yqlib
+
+import (
+	"bufio"
+	"container/list"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"gopkg.in/yaml.v3"
+)
+
+type loadPrefs struct {
+	loadAsString bool
+}
+
+func loadString(filename string) (*CandidateNode, error) {
+	// ignore CWE-22 gosec issue - that's more targetted for http based apps that run in a public directory,
+	// and ensuring that it's not possible to give a path to a file outside that directory.
+
+	filebytes, err := ioutil.ReadFile(filename) // #nosec
+	if err != nil {
+		return nil, err
+	}
+
+	return &CandidateNode{Node: &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: string(filebytes)}}, nil
+}
+
+func loadYaml(filename string) (*CandidateNode, error) {
+
+	file, err := os.Open(filename) // #nosec
+	if err != nil {
+		return nil, err
+	}
+	reader := bufio.NewReader(file)
+
+	documents, err := readDocuments(reader, filename, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	if documents.Len() == 0 {
+		// return null candidate
+		return &CandidateNode{Node: &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!null"}}, nil
+	} else if documents.Len() == 1 {
+		return documents.Front().Value.(*CandidateNode), nil
+
+	} else {
+		sequenceNode := &CandidateNode{Node: &yaml.Node{Kind: yaml.SequenceNode}}
+		for doc := documents.Front(); doc != nil; doc = doc.Next() {
+			sequenceNode.Node.Content = append(sequenceNode.Node.Content, doc.Value.(*CandidateNode).Node)
+		}
+		return sequenceNode, nil
+	}
+}
+
+func loadYamlOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	log.Debugf("loadYamlOperator")
+
+	loadPrefs := expressionNode.Operation.Preferences.(loadPrefs)
+
+	// need to evaluate the 1st parameter against the context
+	// and return the data accordingly.
+
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+
+		rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
+		if err != nil {
+			return Context{}, err
+		}
+		if rhs.MatchingNodes.Front() == nil {
+			return Context{}, fmt.Errorf("Filename expression returned nil")
+		}
+		nameCandidateNode := rhs.MatchingNodes.Front().Value.(*CandidateNode)
+
+		filename := nameCandidateNode.Node.Value
+
+		var contentsCandidate *CandidateNode
+
+		if loadPrefs.loadAsString {
+			contentsCandidate, err = loadString(filename)
+		} else {
+			contentsCandidate, err = loadYaml(filename)
+		}
+		if err != nil {
+			return Context{}, fmt.Errorf("Failed to load %v: %w", filename, err)
+		}
+
+		results.PushBack(contentsCandidate)
+
+	}
+
+	return context.ChildContext(results), nil
+}

pkg/yqlib/operator_load_test.go

@@ -0,0 +1,50 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var loadScenarios = []expressionScenario{
+	{
+		description: "Simple example",
+		document:    `{myFile: "../../examples/thing.yml"}`,
+		expression:  `load(.myFile)`,
+		expected: []string{
+			"D0, P[], (doc)::a: apple is included\nb: cool.\n",
+		},
+	},
+	{
+		description:    "Replace node with referenced file",
+		subdescription: "Note that you can modify the filename in the load operator if needed.",
+		document:       `{something: {file: "thing.yml"}}`,
+		expression:     `.something |= load("../../examples/" + .file)`,
+		expected: []string{
+			"D0, P[], (doc)::{something: {a: apple is included, b: cool.}}\n",
+		},
+	},
+	{
+		description:    "Replace _all_ nodes with referenced file",
+		subdescription: "Recursively match all the nodes (`..`) and then filter the ones that have a 'file' attribute. ",
+		document:       `{something: {file: "thing.yml"}, over: {here: [{file: "thing.yml"}]}}`,
+		expression:     `(.. | select(has("file"))) |= load("../../examples/" + .file)`,
+		expected: []string{
+			"D0, P[], (!!map)::{something: {a: apple is included, b: cool.}, over: {here: [{a: apple is included, b: cool.}]}}\n",
+		},
+	},
+	{
+		description:    "Replace node with referenced file as string",
+		subdescription: "This will work for any text based file",
+		document:       `{something: {file: "thing.yml"}}`,
+		expression:     `.something |= strload("../../examples/" + .file)`,
+		expected: []string{
+			"D0, P[], (doc)::{something: \"a: apple is included\\nb: cool.\"}\n",
+		},
+	},
+}
+
+func TestLoadScenarios(t *testing.T) {
+	for _, tt := range loadScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "load", loadScenarios)
+}