merged from master

CONTRIBUTING.md

@@ -12,7 +12,7 @@
 
 # Documentation
 
-The documentation is a bit of a mixed bag (sorry in advanced, I do plan on simplifying it...) - with some parts automatically generated and stiched together and some statically defined.
+The documentation is a bit of a mixed bag (sorry in advance, I do plan on simplifying it...) - with some parts automatically generated and stiched together and some statically defined.
 
 Documentation is written in markdown, and is published in the 'gitbook' branch.
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.20.3 as builder
+FROM golang:1.20.4 as builder
 
 WORKDIR /go/src/mikefarah/yq
 

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM golang:1.20.3
+FROM golang:1.20.4
 
 COPY scripts/devtools.sh /opt/devtools.sh
 

cmd/utils.go

@@ -195,6 +195,8 @@ func createEncoder(format yqlib.PrinterOutputFormat) (yqlib.Encoder, error) {
 		return yqlib.NewXMLEncoder(indent, yqlib.ConfiguredXMLPreferences), nil
 	case yqlib.TomlOutputFormat:
 		return yqlib.NewTomlEncoder(), nil
+	case yqlib.ShellVariablesOutputFormat:
+		return yqlib.NewShellVariablesEncoder(), nil
 	}
 	return nil, fmt.Errorf("invalid encoder: %v", format)
 }

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
 	golang.org/x/net v0.9.0
+	golang.org/x/text v0.9.0
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -25,7 +26,6 @@ require (
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
 	golang.org/x/sys v0.7.0 // indirect
-	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 )
 

pkg/yqlib/candidiate_node_json.go

@@ -93,6 +93,11 @@ func (o *CandidateNode) UnmarshalJSON(data []byte) error {
 		}
 		// now we put the children into the content, and set a key value for them
 		for i, child := range children {
+
+			if child == nil {
+				// need to represent it as a null scalar
+				child = createScalarNode(nil, "null")
+			}
 			childKey := o.CreateChild()
 			childKey.Kind = ScalarNode
 			childKey.Tag = "!!int"

pkg/yqlib/decoder_json.go

@@ -3,7 +3,6 @@
 package yqlib
 
 import (
-	"fmt"
 	"io"
 
 	"github.com/goccy/go-json"
@@ -29,58 +28,9 @@ func (dec *jsonDecoder) Decode() (*CandidateNode, error) {
 	if err != nil {
 		return nil, err
 	}
-	// log.Debug("convert to yaml")
-	// node, err := dec.convertToYamlNode(&dataBucket)
-	// log.Debug("done, %w", err)
-	// if err != nil {
-	// 	return nil, err
-	// }
 
 	return &CandidateNode{
 		Kind:    DocumentNode,
 		Content: []*CandidateNode{&dataBucket},
 	}, nil
 }
-
-func (dec *jsonDecoder) convertToYamlNode(data *orderedMap) (*CandidateNode, error) {
-	if data.kv == nil {
-		switch rawData := data.altVal.(type) {
-		case nil:
-			return createScalarNode(nil, "null"), nil
-		case float64, float32:
-			// json decoder returns ints as float.
-			return parseSnippet(fmt.Sprintf("%v", rawData))
-		case int, int64, int32, string, bool:
-			return createScalarNode(rawData, fmt.Sprintf("%v", rawData)), nil
-		case []*orderedMap:
-			return dec.parseArray(rawData)
-		default:
-			return nil, fmt.Errorf("unrecognised type :( %v", rawData)
-		}
-	}
-
-	var yamlMap = &CandidateNode{Kind: MappingNode, Tag: "!!map"}
-	for _, keyValuePair := range data.kv {
-		yamlValue, err := dec.convertToYamlNode(&keyValuePair.V)
-		if err != nil {
-			return nil, err
-		}
-		yamlMap.Content = append(yamlMap.Content, createScalarNode(keyValuePair.K, keyValuePair.K), yamlValue)
-	}
-	return yamlMap, nil
-
-}
-
-func (dec *jsonDecoder) parseArray(dataArray []*orderedMap) (*CandidateNode, error) {
-
-	var yamlMap = &CandidateNode{Kind: SequenceNode, Tag: "!!seq"}
-
-	for _, value := range dataArray {
-		yamlValue, err := dec.convertToYamlNode(value)
-		if err != nil {
-			return nil, err
-		}
-		yamlMap.Content = append(yamlMap.Content, yamlValue)
-	}
-	return yamlMap, nil
-}

pkg/yqlib/doc/usage/shellvariables.md

@@ -0,0 +1,86 @@
+
+## Encode shell variables
+Note that comments are dropped and values will be enclosed in single quotes as needed.
+
+Given a sample.yml file of:
+```yaml
+# comment
+name: Mike Wazowski
+eyes:
+  color: turquoise
+  number: 1
+friends:
+  - James P. Sullivan
+  - Celia Mae
+```
+then
+```bash
+yq -o=shell sample.yml
+```
+will output
+```sh
+name='Mike Wazowski'
+eyes_color=turquoise
+eyes_number=1
+friends_0='James P. Sullivan'
+friends_1='Celia Mae'
+```
+
+## Encode shell variables: illegal variable names as key.
+Keys that would be illegal as variable keys are adapted.
+
+Given a sample.yml file of:
+```yaml
+ascii_=_symbols: replaced with _
+"ascii_	_controls": dropped (this example uses \t)
+nonascii_א_characters: dropped
+effrot_expeñded_tò_preserve_accented_latin_letters: moderate (via unicode NFKD)
+
+```
+then
+```bash
+yq -o=shell sample.yml
+```
+will output
+```sh
+ascii___symbols='replaced with _'
+ascii__controls='dropped (this example uses \t)'
+nonascii__characters=dropped
+effrot_expended_to_preserve_accented_latin_letters='moderate (via unicode NFKD)'
+```
+
+## Encode shell variables: empty values, arrays and maps
+Empty values are encoded to empty variables, but empty arrays and maps are skipped.
+
+Given a sample.yml file of:
+```yaml
+empty:
+  value:
+  array: []
+  map:   {}
+```
+then
+```bash
+yq -o=shell sample.yml
+```
+will output
+```sh
+empty_value=
+```
+
+## Encode shell variables: single quotes in values
+Single quotes in values are encoded as '"'"' (close single quote, double-quoted single quote, open single quote).
+
+Given a sample.yml file of:
+```yaml
+name: Miles O'Brien
+```
+then
+```bash
+yq -o=shell sample.yml
+```
+will output
+```sh
+name='Miles O'"'"'Brien'
+```
+

pkg/yqlib/encoder_shellvariables.go

@@ -0,0 +1,152 @@
+package yqlib
+
+import (
+	"fmt"
+	"io"
+	"strings"
+	"unicode/utf8"
+
+	"golang.org/x/text/unicode/norm"
+)
+
+type shellVariablesEncoder struct {
+}
+
+func NewShellVariablesEncoder() Encoder {
+	return &shellVariablesEncoder{}
+}
+
+func (pe *shellVariablesEncoder) CanHandleAliases() bool {
+	return false
+}
+
+func (pe *shellVariablesEncoder) PrintDocumentSeparator(_ io.Writer) error {
+	return nil
+}
+
+func (pe *shellVariablesEncoder) PrintLeadingContent(_ io.Writer, _ string) error {
+	return nil
+}
+
+func (pe *shellVariablesEncoder) Encode(writer io.Writer, node *CandidateNode) error {
+
+	mapKeysToStrings(node)
+	err := pe.doEncode(&writer, node, "")
+	if err != nil {
+		return err
+	}
+
+	return err
+}
+
+func (pe *shellVariablesEncoder) doEncode(w *io.Writer, node *CandidateNode, path string) error {
+
+	// Note this drops all comments.
+
+	switch node.Kind {
+	case ScalarNode:
+		nonemptyPath := path
+		if path == "" {
+			// We can't assign an empty variable "=somevalue" because that would error out if sourced in a shell,
+			// nor can we use "_" as a variable name ($_ is a special shell variable that can't be assigned)...
+			// let's just pick a fallback key to use if we are encoding a single scalar
+			nonemptyPath = "value"
+		}
+		_, err := io.WriteString(*w, nonemptyPath+"="+quoteValue(node.Value)+"\n")
+		return err
+	case DocumentNode:
+		return pe.doEncode(w, node.Content[0], path)
+	case SequenceNode:
+		for index, child := range node.Content {
+			err := pe.doEncode(w, child, appendPath(path, index))
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	case MappingNode:
+		for index := 0; index < len(node.Content); index = index + 2 {
+			key := node.Content[index]
+			value := node.Content[index+1]
+			err := pe.doEncode(w, value, appendPath(path, key.Value))
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	case AliasNode:
+		return pe.doEncode(w, node.Alias, path)
+	default:
+		return fmt.Errorf("Unsupported node %v", node.Tag)
+	}
+}
+
+func appendPath(cookedPath string, rawKey interface{}) string {
+
+	// Shell variable names must match
+	//    [a-zA-Z_]+[a-zA-Z0-9_]*
+	//
+	// While this is not mandated by POSIX, which is quite lenient, it is
+	// what shells (for example busybox ash *) allow in practice.
+	//
+	// Since yaml names can contain basically any character, we will process them according to these steps:
+	//
+	//     1. apply unicode compatibility decomposition NFKD (this will convert accented
+	//        letters to letters followed by accents, split ligatures, replace exponents
+	//        with the corresponding digit, etc.
+	//
+	//     2. discard non-ASCII characters as well as ASCII control characters (ie. anything
+	//        with code point < 32 or > 126), this will eg. discard accents but keep the base
+	//        unaccented letter because of NFKD above
+	//
+	//     3. replace all non-alphanumeric characters with _
+	//
+	// Moreover, for the root key only, we will prepend an underscore if what results from the steps above
+	// does not start with [a-zA-Z_] (ie. if the root key starts with a digit).
+	//
+	// Note this is NOT a 1:1 mapping.
+	//
+	// (*) see endofname.c from https://git.busybox.net/busybox/tag/?h=1_36_0
+
+	// XXX empty strings
+
+	key := strings.Map(func(r rune) rune {
+		if isAlphaNumericOrUnderscore(r) {
+			return r
+		} else if r < 32 || 126 < r {
+			return -1
+		}
+		return '_'
+	}, norm.NFKD.String(fmt.Sprintf("%v", rawKey)))
+
+	if cookedPath == "" {
+		firstRune, _ := utf8.DecodeRuneInString(key)
+		if !isAlphaOrUnderscore(firstRune) {
+			return "_" + key
+		}
+		return key
+	}
+	return cookedPath + "_" + key
+}
+
+func quoteValue(value string) string {
+	needsQuoting := false
+	for _, r := range value {
+		if !isAlphaNumericOrUnderscore(r) {
+			needsQuoting = true
+			break
+		}
+	}
+	if needsQuoting {
+		return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'"
+	}
+	return value
+}
+
+func isAlphaOrUnderscore(r rune) bool {
+	return ('a' <= r && r <= 'z') || ('A' <= r && r <= 'Z') || r == '_'
+}
+
+func isAlphaNumericOrUnderscore(r rune) bool {
+	return isAlphaOrUnderscore(r) || ('0' <= r && r <= '9')
+}

pkg/yqlib/encoder_shellvariables_test.go

@@ -0,0 +1,93 @@
+package yqlib
+
+import (
+	"bufio"
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+func assertEncodesTo(t *testing.T, yaml string, shellvars string) {
+	var output bytes.Buffer
+	writer := bufio.NewWriter(&output)
+
+	var encoder = NewShellVariablesEncoder()
+	inputs, err := readDocuments(strings.NewReader(yaml), "test.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
+	if err != nil {
+		panic(err)
+	}
+	node := inputs.Front().Value.(*CandidateNode)
+	err = encoder.Encode(writer, node)
+	if err != nil {
+		panic(err)
+	}
+	writer.Flush()
+
+	test.AssertResult(t, shellvars, strings.TrimSuffix(output.String(), "\n"))
+}
+
+func TestShellVariablesEncoderNonquoting(t *testing.T) {
+	assertEncodesTo(t, "a: alice", "a=alice")
+}
+
+func TestShellVariablesEncoderQuoting(t *testing.T) {
+	assertEncodesTo(t, "a: Lewis Carroll", "a='Lewis Carroll'")
+}
+
+func TestShellVariablesEncoderQuotesQuoting(t *testing.T) {
+	assertEncodesTo(t, "a: Lewis Carroll's Alice", "a='Lewis Carroll'\"'\"'s Alice'")
+}
+
+func TestShellVariablesEncoderStripComments(t *testing.T) {
+	assertEncodesTo(t, "a: Alice # comment", "a=Alice")
+}
+
+func TestShellVariablesEncoderMap(t *testing.T) {
+	assertEncodesTo(t, "a:\n b: Lewis\n c: Carroll", "a_b=Lewis\na_c=Carroll")
+}
+
+func TestShellVariablesEncoderArray_Unwrapped(t *testing.T) {
+	assertEncodesTo(t, "a: [{n: Alice}, {n: Bob}]", "a_0_n=Alice\na_1_n=Bob")
+}
+
+func TestShellVariablesEncoderKeyNonPrintable(t *testing.T) {
+	assertEncodesTo(t, `"be\all": ring!`, "bell='ring!'")
+}
+
+func TestShellVariablesEncoderKeyPrintableNonAlphaNumeric(t *testing.T) {
+	assertEncodesTo(t, `"b-e l=l": ring!`, "b_e_l_l='ring!'")
+}
+
+func TestShellVariablesEncoderKeyPrintableNonAscii(t *testing.T) {
+	assertEncodesTo(t, `"b\u00e9ll": ring!`, "bell='ring!'")
+}
+
+func TestShellVariablesEncoderRootKeyStartingWithDigit(t *testing.T) {
+	assertEncodesTo(t, "1a: onea", "_1a=onea")
+}
+
+func TestShellVariablesEncoderRootKeyStartingWithUnderscore(t *testing.T) {
+	assertEncodesTo(t, "_key: value", "_key=value")
+}
+
+func TestShellVariablesEncoderChildStartingWithUnderscore(t *testing.T) {
+	assertEncodesTo(t, "root:\n _child: value", "root__child=value")
+}
+
+func TestShellVariablesEncoderEmptyValue(t *testing.T) {
+	assertEncodesTo(t, "empty:", "empty=")
+}
+
+func TestShellVariablesEncoderEmptyArray(t *testing.T) {
+	assertEncodesTo(t, "empty: []", "")
+}
+
+func TestShellVariablesEncoderEmptyMap(t *testing.T) {
+	assertEncodesTo(t, "empty: {}", "")
+}
+
+func TestShellVariablesEncoderScalarNode(t *testing.T) {
+	assertEncodesTo(t, "some string", "value='some string'")
+}

pkg/yqlib/json_test.go

@@ -80,6 +80,13 @@ const roundTripMultiLineJson = `{
 `
 
 var jsonScenarios = []formatScenario{
+	{
+		description:  "array null",
+		skipDoc:      true,
+		input:        "[null]",
+		scenarioType: "roundtrip-ndjson",
+		expected:     "[null]\n",
+	},
 	{
 		description:  "set tags",
 		skipDoc:      true,

pkg/yqlib/printer.go

@@ -30,6 +30,7 @@ const (
 	UriOutputFormat
 	ShOutputFormat
 	TomlOutputFormat
+	ShellVariablesOutputFormat
 )
 
 func OutputFormatFromString(format string) (PrinterOutputFormat, error) {
@@ -48,8 +49,10 @@ func OutputFormatFromString(format string) (PrinterOutputFormat, error) {
 		return XMLOutputFormat, nil
 	case "toml":
 		return TomlOutputFormat, nil
+	case "shell", "s", "sh":
+		return ShellVariablesOutputFormat, nil
 	default:
-		return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml]", format)
+		return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml|toml|shell]", format)
 	}
 }
 

pkg/yqlib/shellvariables_test.go

@@ -0,0 +1,98 @@
+package yqlib
+
+import (
+	"bufio"
+	"fmt"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+var shellVariablesScenarios = []formatScenario{
+	{
+		description:    "Encode shell variables",
+		subdescription: "Note that comments are dropped and values will be enclosed in single quotes as needed.",
+		input: "" +
+			"# comment" + "\n" +
+			"name: Mike Wazowski" + "\n" +
+			"eyes:" + "\n" +
+			"  color: turquoise" + "\n" +
+			"  number: 1" + "\n" +
+			"friends:" + "\n" +
+			"  - James P. Sullivan" + "\n" +
+			"  - Celia Mae",
+		expected: "" +
+			"name='Mike Wazowski'" + "\n" +
+			"eyes_color=turquoise" + "\n" +
+			"eyes_number=1" + "\n" +
+			"friends_0='James P. Sullivan'" + "\n" +
+			"friends_1='Celia Mae'" + "\n",
+	},
+	{
+		description:    "Encode shell variables: illegal variable names as key.",
+		subdescription: "Keys that would be illegal as variable keys are adapted.",
+		input: "" +
+			"ascii_=_symbols: replaced with _" + "\n" +
+			"\"ascii_\t_controls\": dropped (this example uses \\t)" + "\n" +
+			"nonascii_\u05d0_characters: dropped" + "\n" +
+			"effrot_expe\u00f1ded_t\u00f2_preserve_accented_latin_letters: moderate (via unicode NFKD)" + "\n",
+		expected: "" +
+			"ascii___symbols='replaced with _'" + "\n" +
+			"ascii__controls='dropped (this example uses \\t)'" + "\n" +
+			"nonascii__characters=dropped" + "\n" +
+			"effrot_expended_to_preserve_accented_latin_letters='moderate (via unicode NFKD)'" + "\n",
+	},
+	{
+		description:    "Encode shell variables: empty values, arrays and maps",
+		subdescription: "Empty values are encoded to empty variables, but empty arrays and maps are skipped.",
+		input:          "empty:\n  value:\n  array: []\n  map:   {}",
+		expected:       "empty_value=" + "\n",
+	},
+	{
+		description:    "Encode shell variables: single quotes in values",
+		subdescription: "Single quotes in values are encoded as '\"'\"' (close single quote, double-quoted single quote, open single quote).",
+		input:          "name: Miles O'Brien",
+		expected:       `name='Miles O'"'"'Brien'` + "\n",
+	},
+}
+
+func TestShellVariableScenarios(t *testing.T) {
+	for _, s := range shellVariablesScenarios {
+		//fmt.Printf("\t<%s> <%s>\n", s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder()))
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder()), s.description)
+	}
+	genericScenarios := make([]interface{}, len(shellVariablesScenarios))
+	for i, s := range shellVariablesScenarios {
+		genericScenarios[i] = s
+	}
+	documentScenarios(t, "usage", "shellvariables", genericScenarios, documentShellVaraibleScenario)
+}
+
+func documentShellVaraibleScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
+	s := i.(formatScenario)
+	if s.skipDoc {
+		return
+	}
+	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
+
+	if s.subdescription != "" {
+		writeOrPanic(w, s.subdescription)
+		writeOrPanic(w, "\n\n")
+	}
+
+	writeOrPanic(w, "Given a sample.yml file of:\n")
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input))
+
+	writeOrPanic(w, "then\n")
+
+	expression := s.expression
+
+	if expression != "" {
+		writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=shell '%v' sample.yml\n```\n", expression))
+	} else {
+		writeOrPanic(w, "```bash\nyq -o=shell sample.yml\n```\n")
+	}
+	writeOrPanic(w, "will output\n")
+
+	writeOrPanic(w, fmt.Sprintf("```sh\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder())))
+}

pkg/yqlib/utils.go

@@ -15,7 +15,7 @@ func readStream(filename string) (io.Reader, error) {
 		reader = bufio.NewReader(os.Stdin)
 	} else {
 		// ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory,
-		// and ensuring that it's not possible to give a path to a file outside thar directory.
+		// and ensuring that it's not possible to give a path to a file outside that directory.
 		file, err := os.Open(filename) // #nosec
 		if err != nil {
 			return nil, err