diff --git a/cmd/utils.go b/cmd/utils.go index e54cfb79..7d34c723 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -195,6 +195,8 @@ func createEncoder(format yqlib.PrinterOutputFormat) (yqlib.Encoder, error) { return yqlib.NewXMLEncoder(indent, yqlib.ConfiguredXMLPreferences), nil case yqlib.TomlOutputFormat: return yqlib.NewTomlEncoder(), nil + case yqlib.ShellVariablesOutputFormat: + return yqlib.NewShellVariablesEncoder(), nil } return nil, fmt.Errorf("invalid encoder: %v", format) } diff --git a/pkg/yqlib/encoder_shellvariables.go b/pkg/yqlib/encoder_shellvariables.go new file mode 100644 index 00000000..bc50e022 --- /dev/null +++ b/pkg/yqlib/encoder_shellvariables.go @@ -0,0 +1,153 @@ +package yqlib + +import ( + "fmt" + "io" + "strings" + "unicode/utf8" + + "golang.org/x/text/unicode/norm" + yaml "gopkg.in/yaml.v3" +) + +type shellVariablesEncoder struct { +} + +func NewShellVariablesEncoder() Encoder { + return &shellVariablesEncoder{} +} + +func (pe *shellVariablesEncoder) CanHandleAliases() bool { + return false +} + +func (pe *shellVariablesEncoder) PrintDocumentSeparator(_ io.Writer) error { + return nil +} + +func (pe *shellVariablesEncoder) PrintLeadingContent(_ io.Writer, _ string) error { + return nil +} + +func (pe *shellVariablesEncoder) Encode(writer io.Writer, node *yaml.Node) error { + + mapKeysToStrings(node) + err := pe.doEncode(&writer, node, "", nil) + if err != nil { + return err + } + + return err +} + +func (pe *shellVariablesEncoder) doEncode(w *io.Writer, node *yaml.Node, path string, _ *yaml.Node) error { + + // Note this drops all comments. + + switch node.Kind { + case yaml.ScalarNode: + nonemptyPath := path + if path == "" { + // We can't assign an empty variable "=somevalue" because that would error out if sourced in a shell, + // nor can we use "_" as a variable name ($_ is a special shell variable that can't be assigned)... + // let's just pick a fallback key to use if we are encoding a single scalar + nonemptyPath = "value" + } + _, err := io.WriteString(*w, nonemptyPath+"="+quoteValue(node.Value)+"\n") + return err + case yaml.DocumentNode: + return pe.doEncode(w, node.Content[0], path, node) + case yaml.SequenceNode: + for index, child := range node.Content { + err := pe.doEncode(w, child, appendPath(path, index), nil) + if err != nil { + return err + } + } + return nil + case yaml.MappingNode: + for index := 0; index < len(node.Content); index = index + 2 { + key := node.Content[index] + value := node.Content[index+1] + err := pe.doEncode(w, value, appendPath(path, key.Value), key) + if err != nil { + return err + } + } + return nil + case yaml.AliasNode: + return pe.doEncode(w, node.Alias, path, nil) + default: + return fmt.Errorf("Unsupported node %v", node.Tag) + } +} + +func appendPath(cookedPath string, rawKey interface{}) string { + + // Shell variable names must match + // [a-zA-Z_]+[a-zA-Z0-9_]* + // + // While this is not mandated by POSIX, which is quite lenient, it is + // what shells (for example busybox ash *) allow in practice. + // + // Since yaml names can contain basically any character, we will process them according to these steps: + // + // 1. apply unicode compatibility decomposition NFKD (this will convert accented + // letters to letters followed by accents, split ligatures, replace exponents + // with the corresponding digit, etc. + // + // 2. discard non-ASCII characters as well as ASCII control characters (ie. anything + // with code point < 32 or > 126), this will eg. discard accents but keep the base + // unaccented letter because of NFKD above + // + // 3. replace all non-alphanumeric characters with _ + // + // Moreover, for the root key only, we will prepend an underscore if what results from the steps above + // does not start with [a-zA-Z_] (ie. if the root key starts with a digit). + // + // Note this is NOT a 1:1 mapping. + // + // (*) see endofname.c from https://git.busybox.net/busybox/tag/?h=1_36_0 + + // XXX empty strings + + key := strings.Map(func(r rune) rune { + if isAlphaNumericOrUnderscore(r) { + return r + } else if r < 32 || 126 < r { + return -1 + } + return '_' + }, norm.NFKD.String(fmt.Sprintf("%v", rawKey))) + + if cookedPath == "" { + firstRune, _ := utf8.DecodeRuneInString(key) + if !isAlphaOrUnderscore(firstRune) { + return "_" + key + } + return key + } + return cookedPath + "_" + key +} + +func quoteValue(value string) string { + needsQuoting := false + for _, r := range value { + if !isAlphaNumericOrUnderscore(r) { + needsQuoting = true + break + } + } + if needsQuoting { + return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'" + } + return value +} + +func isAlphaOrUnderscore(r rune) bool { + return ('a' <= r && r <= 'z') || ('A' <= r && r <= 'Z') +} + +func isAlphaNumericOrUnderscore(r rune) bool { + return isAlphaOrUnderscore(r) || ('0' <= r && r <= '9') +} diff --git a/pkg/yqlib/encoder_shellvariables_test.go b/pkg/yqlib/encoder_shellvariables_test.go new file mode 100644 index 00000000..a098f3af --- /dev/null +++ b/pkg/yqlib/encoder_shellvariables_test.go @@ -0,0 +1,77 @@ +package yqlib + +import ( + "bufio" + "bytes" + "strings" + "testing" + + "github.com/mikefarah/yq/v4/test" +) + +func assertEncodesTo(t *testing.T, yaml string, shellvars string) { + var output bytes.Buffer + writer := bufio.NewWriter(&output) + + var encoder = NewShellVariablesEncoder() + inputs, err := readDocuments(strings.NewReader(yaml), "test.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) + if err != nil { + panic(err) + } + node := inputs.Front().Value.(*CandidateNode).Node + err = encoder.Encode(writer, node) + if err != nil { + panic(err) + } + writer.Flush() + + test.AssertResult(t, shellvars, strings.TrimSuffix(output.String(), "\n")) +} + +func TestShellVariablesEncoderNonquoting(t *testing.T) { + assertEncodesTo(t, "a: alice", "a=alice") +} + +func TestShellVariablesEncoderQuoting(t *testing.T) { + assertEncodesTo(t, "a: Lewis Carroll", "a='Lewis Carroll'") +} + +func TestShellVariablesEncoderQuotesQuoting(t *testing.T) { + assertEncodesTo(t, "a: Lewis Carroll's Alice", "a='Lewis Carroll'\"'\"'s Alice'") +} + +func TestShellVariablesEncoderStripComments(t *testing.T) { + assertEncodesTo(t, "a: Alice # comment", "a=Alice") +} + +func TestShellVariablesEncoderMap(t *testing.T) { + assertEncodesTo(t, "a:\n b: Lewis\n c: Carroll", "a_b=Lewis\na_c=Carroll") +} + +func TestShellVariablesEncoderArray_Unwrapped(t *testing.T) { + assertEncodesTo(t, "a: [{n: Alice}, {n: Bob}]", "a_0_n=Alice\na_1_n=Bob") +} + +func TestShellVariablesEncoderKeyNonPrintable(t *testing.T) { + assertEncodesTo(t, `"be\all": ring!`, "bell='ring!'") +} + +func TestShellVariablesEncoderKeyPrintableNonAlphaNumeric(t *testing.T) { + assertEncodesTo(t, `"b-e l=l": ring!`, "b_e_l_l='ring!'") +} + +func TestShellVariablesEncoderKeyPrintableNonAscii(t *testing.T) { + assertEncodesTo(t, `"bèll": ring!`, "bell='ring!'") +} + +func TestShellVariablesEncoderRootKeyStartingWithDigit(t *testing.T) { + assertEncodesTo(t, "1a: onea", "_1a=onea") +} + +func TestShellVariablesEncoderEmptyValue(t *testing.T) { + assertEncodesTo(t, "empty:", "empty=") +} + +func TestShellVariablesEncoderScalarNode(t *testing.T) { + assertEncodesTo(t, "some string", "value='some string'") +} diff --git a/pkg/yqlib/printer.go b/pkg/yqlib/printer.go index a2d9f4ed..9cd65753 100644 --- a/pkg/yqlib/printer.go +++ b/pkg/yqlib/printer.go @@ -32,6 +32,7 @@ const ( UriOutputFormat ShOutputFormat TomlOutputFormat + ShellVariablesOutputFormat ) func OutputFormatFromString(format string) (PrinterOutputFormat, error) { @@ -50,8 +51,10 @@ func OutputFormatFromString(format string) (PrinterOutputFormat, error) { return XMLOutputFormat, nil case "toml": return TomlOutputFormat, nil + case "shell", "s", "sh": + return ShellVariablesOutputFormat, nil default: - return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml]", format) + return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml|toml|shell]", format) } }