From d302d75c77b6fe18fe3c25210659d3e2ed01e788 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 11 Aug 2023 04:56:49 +0200 Subject: [PATCH] Implement basic Lua output support (#1745) * Implement basic Lua output support Ref #1700 Basic but working serialization to Lua tables. * Escape larger set of characters in Lua output Started with a minimum of replacements, this should be more complete, tho not all substitutions are strictly required in Lua. * Print simple keys unquoted in Lua output String keys that satisfy the requirements for variable names can be used as keys without quotes in tables. * Quote Lua keywords in table keys Keywords are not valid as unquoted keys, thus must be quoted * Make output of unquoted Lua table keys optional Generally safer and simpler to not do it. * Hook up settings for Lua output * Allow special characters in Lua prefix and suffix --lua-suffix='});^M' didn't work, so taking this approach instead * Panic on unhandled YAML Kind in Lua encoder * Handle YAML case varied booleans in Lua encoder * Handle special-case numbers in Lua encoder * Reject unhandled scalar Tags in Lua encoder * Add note about how Lua nil is unsuitable as table key Could add some context tracking in the future to allow rejecting nil in a table key context. * Return error instead of panic in Lua encoder * Add initial test for Lua encoder Boilerplate mostly copied from toml_test.go * Additional Lua output tests * Generate Lua encoder documentation Mostly just for the boilerplate * Convert octal for Lua output Lua doesn't have the 0oNNN syntax for octal integers, only decimal and hexadecimal, hence those can be passed trough as is while octal needs special treatment. * Implement indentation in in Lua output * Respect string Style in Lua encoder Lua has 'single', "double" and [[ long ]] strings. * Expand Lua examples * Output line comments in Lua output * Implement Lua globals output mode --- cmd/root.go | 5 + cmd/utils.go | 2 + pkg/yqlib/doc/usage/lua.md | 144 ++++++++++++++++ pkg/yqlib/encoder_lua.go | 328 +++++++++++++++++++++++++++++++++++++ pkg/yqlib/lua.go | 19 +++ pkg/yqlib/lua_test.go | 257 +++++++++++++++++++++++++++++ pkg/yqlib/printer.go | 5 +- 7 files changed, 759 insertions(+), 1 deletion(-) create mode 100644 pkg/yqlib/doc/usage/lua.md create mode 100644 pkg/yqlib/encoder_lua.go create mode 100644 pkg/yqlib/lua.go create mode 100644 pkg/yqlib/lua_test.go diff --git a/cmd/root.go b/cmd/root.go index 96f9815d..45cb9de6 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -78,6 +78,11 @@ yq -P sample.json rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipProcInst, "xml-skip-proc-inst", yqlib.ConfiguredXMLPreferences.SkipProcInst, "skip over process instructions (e.g. )") rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipDirectives, "xml-skip-directives", yqlib.ConfiguredXMLPreferences.SkipDirectives, "skip over directives (e.g. )") + rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredLuaPreferences.DocPrefix, "lua-prefix", yqlib.ConfiguredLuaPreferences.DocPrefix, "prefix") + rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredLuaPreferences.DocSuffix, "lua-suffix", yqlib.ConfiguredLuaPreferences.DocSuffix, "suffix") + rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.UnquotedKeys, "lua-unquoted", yqlib.ConfiguredLuaPreferences.UnquotedKeys, "output unquoted string keys (e.g. {foo=\"bar\"})") + rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.Globals, "lua-globals", yqlib.ConfiguredLuaPreferences.Globals, "output keys as top-level global variables") + rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.") rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)") diff --git a/cmd/utils.go b/cmd/utils.go index 7d34c723..a50f7048 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -197,6 +197,8 @@ func createEncoder(format yqlib.PrinterOutputFormat) (yqlib.Encoder, error) { return yqlib.NewTomlEncoder(), nil case yqlib.ShellVariablesOutputFormat: return yqlib.NewShellVariablesEncoder(), nil + case yqlib.LuaOutputFormat: + return yqlib.NewLuaEncoder(yqlib.ConfiguredLuaPreferences), nil } return nil, fmt.Errorf("invalid encoder: %v", format) } diff --git a/pkg/yqlib/doc/usage/lua.md b/pkg/yqlib/doc/usage/lua.md new file mode 100644 index 00000000..32141707 --- /dev/null +++ b/pkg/yqlib/doc/usage/lua.md @@ -0,0 +1,144 @@ + +## Basic example +Given a sample.yml file of: +```yaml +--- +country: Australia # this place +cities: +- Sydney +- Melbourne +- Brisbane +- Perth +``` +then +```bash +yq -o=lua '.' sample.yml +``` +will output +```lua +return { + ["country"] = "Australia"; -- this place + ["cities"] = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", + }; +}; +``` + +## Unquoted keys +Uses the `--lua-unquoted` option to produce a nicer-looking output. + +Given a sample.yml file of: +```yaml +--- +country: Australia # this place +cities: +- Sydney +- Melbourne +- Brisbane +- Perth +``` +then +```bash +yq -o=lua '.' sample.yml +``` +will output +```lua +return { + country = "Australia"; -- this place + cities = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", + }; +}; +``` + +## Globals +Uses the `--lua-globals` option to export the values into the global scope. + +Given a sample.yml file of: +```yaml +--- +country: Australia # this place +cities: +- Sydney +- Melbourne +- Brisbane +- Perth +``` +then +```bash +yq -o=lua '.' sample.yml +``` +will output +```lua +country = "Australia"; -- this place +cities = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", +}; +``` + +## Elaborate example +Given a sample.yml file of: +```yaml +--- +hello: world +tables: + like: this + keys: values + ? look: non-string keys + : True +numbers: + - decimal: 12345 + - hex: 0x7fabc123 + - octal: 0o30 + - float: 123.45 + - infinity: .inf + - not: .nan + +``` +then +```bash +yq -o=lua '.' sample.yml +``` +will output +```lua +return { + ["hello"] = "world"; + ["tables"] = { + ["like"] = "this"; + ["keys"] = "values"; + [{ + ["look"] = "non-string keys"; + }] = true; + }; + ["numbers"] = { + { + ["decimal"] = 12345; + }, + { + ["hex"] = 0x7fabc123; + }, + { + ["octal"] = 24; + }, + { + ["float"] = 123.45; + }, + { + ["infinity"] = (1/0); + }, + { + ["not"] = (0/0); + }, + }; +}; +``` + diff --git a/pkg/yqlib/encoder_lua.go b/pkg/yqlib/encoder_lua.go new file mode 100644 index 00000000..898b3855 --- /dev/null +++ b/pkg/yqlib/encoder_lua.go @@ -0,0 +1,328 @@ +package yqlib + +import ( + "fmt" + "io" + "strings" + + yaml "gopkg.in/yaml.v3" +) + +type luaEncoder struct { + docPrefix string + docSuffix string + indent int + indentStr string + unquoted bool + globals bool + escape *strings.Replacer +} + +func (le *luaEncoder) CanHandleAliases() bool { + return false +} + +func NewLuaEncoder(prefs LuaPreferences) Encoder { + escape := strings.NewReplacer( + "\000", "\\000", + "\001", "\\001", + "\002", "\\002", + "\003", "\\003", + "\004", "\\004", + "\005", "\\005", + "\006", "\\006", + "\007", "\\a", + "\010", "\\b", + "\011", "\\t", + "\012", "\\n", + "\013", "\\v", + "\014", "\\f", + "\015", "\\r", + "\016", "\\014", + "\017", "\\015", + "\020", "\\016", + "\021", "\\017", + "\022", "\\018", + "\023", "\\019", + "\024", "\\020", + "\025", "\\021", + "\026", "\\022", + "\027", "\\023", + "\030", "\\024", + "\031", "\\025", + "\032", "\\026", + "\033", "\\027", + "\034", "\\028", + "\035", "\\029", + "\036", "\\030", + "\037", "\\031", + "\"", "\\\"", + "'", "\\'", + "\\", "\\\\", + "\177", "\\127", + ) + unescape := strings.NewReplacer( + "\\'", "'", + "\\\"", "\"", + "\\n", "\n", + "\\r", "\r", + "\\t", "\t", + "\\\\", "\\", + ) + return &luaEncoder{unescape.Replace(prefs.DocPrefix), unescape.Replace(prefs.DocSuffix), 0, "\t", prefs.UnquotedKeys, prefs.Globals, escape} +} + +func (le *luaEncoder) PrintDocumentSeparator(writer io.Writer) error { + return nil +} + +func (le *luaEncoder) PrintLeadingContent(writer io.Writer, content string) error { + return nil +} + +func (le *luaEncoder) encodeString(writer io.Writer, node *yaml.Node) error { + quote := "\"" + switch node.Style { + case yaml.LiteralStyle, yaml.FoldedStyle, yaml.FlowStyle: + for i := 0; i < 10; i++ { + if !strings.Contains(node.Value, "]"+strings.Repeat("=", i)+"]") { + err := writeString(writer, "["+strings.Repeat("=", i)+"[\n") + if err != nil { + return err + } + err = writeString(writer, node.Value) + if err != nil { + return err + } + return writeString(writer, "]"+strings.Repeat("=", i)+"]") + } + } + case yaml.SingleQuotedStyle: + quote = "'" + + // falltrough to regular ol' string + } + return writeString(writer, quote+le.escape.Replace(node.Value)+quote) +} + +func (le *luaEncoder) writeIndent(writer io.Writer) error { + if le.indentStr == "" { + return nil + } + err := writeString(writer, "\n") + if err != nil { + return err + } + return writeString(writer, strings.Repeat(le.indentStr, le.indent)) +} + +func (le *luaEncoder) encodeArray(writer io.Writer, node *yaml.Node) error { + err := writeString(writer, "{") + if err != nil { + return err + } + le.indent++ + for _, child := range node.Content { + err = le.writeIndent(writer) + if err != nil { + return err + } + err := le.Encode(writer, child) + if err != nil { + return err + } + err = writeString(writer, ",") + if err != nil { + return err + } + if child.LineComment != "" { + sansPrefix, _ := strings.CutPrefix(child.LineComment, "#") + err = writeString(writer, " --"+sansPrefix) + if err != nil { + return err + } + } + } + le.indent-- + if len(node.Content) != 0 { + err = le.writeIndent(writer) + if err != nil { + return err + } + } + return writeString(writer, "}") +} + +func needsQuoting(s string) bool { + // known keywords as of Lua 5.4 + switch s { + case "do", "and", "else", "break", + "if", "end", "goto", "false", + "in", "for", "then", "local", + "or", "nil", "true", "until", + "elseif", "function", "not", + "repeat", "return", "while": + return true + } + // [%a_][%w_]* + for i, c := range s { + if i == 0 { + if !((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_') { + return true + } + } else { + if !((c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_') { + return true + } + } + } + return false +} + +func (le *luaEncoder) encodeMap(writer io.Writer, node *yaml.Node, global bool) error { + if !global { + err := writeString(writer, "{") + if err != nil { + return err + } + le.indent++ + } + for i, child := range node.Content { + if (i % 2) == 1 { + // value + err := le.Encode(writer, child) + if err != nil { + return err + } + err = writeString(writer, ";") + if err != nil { + return err + } + } else { + // key + if !global || i > 0 { + err := le.writeIndent(writer) + if err != nil { + return err + } + } + if (le.unquoted || global) && child.Tag == "!!str" && !needsQuoting(child.Value) { + err := writeString(writer, child.Value+" = ") + if err != nil { + return err + } + } else { + if global { + // This only works in Lua 5.2+ + err := writeString(writer, "_ENV") + if err != nil { + return err + } + } + err := writeString(writer, "[") + if err != nil { + return err + } + err = le.encodeAny(writer, child) + if err != nil { + return err + } + err = writeString(writer, "] = ") + if err != nil { + return err + } + } + } + if child.LineComment != "" { + sansPrefix, _ := strings.CutPrefix(child.LineComment, "#") + err := writeString(writer, strings.Repeat(" ", i%2)+"--"+sansPrefix) + if err != nil { + return err + } + if (i % 2) == 0 { + // newline and indent after comments on keys + err = le.writeIndent(writer) + if err != nil { + return err + } + } + } + } + if global { + return writeString(writer, "\n") + } + le.indent-- + if len(node.Content) != 0 { + err := le.writeIndent(writer) + if err != nil { + return err + } + } + return writeString(writer, "}") +} + +func (le *luaEncoder) encodeAny(writer io.Writer, node *yaml.Node) error { + switch node.Kind { + case yaml.SequenceNode: + return le.encodeArray(writer, node) + case yaml.MappingNode: + return le.encodeMap(writer, node, false) + case yaml.ScalarNode: + switch node.Tag { + case "!!str": + return le.encodeString(writer, node) + case "!!null": + // TODO reject invalid use as a table key + return writeString(writer, "nil") + case "!!bool": + // Yaml 1.2 has case variation e.g. True, FALSE etc but Lua only has + // lower case + return writeString(writer, strings.ToLower(node.Value)) + case "!!int": + if strings.HasPrefix(node.Value, "0o") { + var octalValue int + err := node.Decode(&octalValue) + if err != nil { + return err + } + return writeString(writer, fmt.Sprintf("%d", octalValue)) + } + return writeString(writer, strings.ToLower(node.Value)) + case "!!float": + switch strings.ToLower(node.Value) { + case ".inf": + return writeString(writer, "(1/0)") + case "-.inf": + return writeString(writer, "(-1/0)") + case ".nan": + return writeString(writer, "(0/0)") + default: + return writeString(writer, node.Value) + } + default: + return fmt.Errorf("Lua encoder NYI -- %s", node.ShortTag()) + } + case yaml.DocumentNode: + if le.globals { + if node.Content[0].Kind != yaml.MappingNode { + return fmt.Errorf("--lua-global requires a top level MappingNode") + } + return le.encodeMap(writer, node.Content[0], true) + } + err := writeString(writer, le.docPrefix) + if err != nil { + return err + } + err = le.encodeAny(writer, node.Content[0]) + if err != nil { + return err + } + return writeString(writer, le.docSuffix) + default: + return fmt.Errorf("Lua encoder NYI -- %s", node.ShortTag()) + } +} + +func (le *luaEncoder) Encode(writer io.Writer, node *yaml.Node) error { + return le.encodeAny(writer, node) +} diff --git a/pkg/yqlib/lua.go b/pkg/yqlib/lua.go new file mode 100644 index 00000000..ba63e634 --- /dev/null +++ b/pkg/yqlib/lua.go @@ -0,0 +1,19 @@ +package yqlib + +type LuaPreferences struct { + DocPrefix string + DocSuffix string + UnquotedKeys bool + Globals bool +} + +func NewDefaultLuaPreferences() LuaPreferences { + return LuaPreferences{ + DocPrefix: "return ", + DocSuffix: ";\n", + UnquotedKeys: false, + Globals: false, + } +} + +var ConfiguredLuaPreferences = NewDefaultLuaPreferences() diff --git a/pkg/yqlib/lua_test.go b/pkg/yqlib/lua_test.go new file mode 100644 index 00000000..3f8133d6 --- /dev/null +++ b/pkg/yqlib/lua_test.go @@ -0,0 +1,257 @@ +package yqlib + +import ( + "bufio" + "fmt" + "testing" + + "github.com/mikefarah/yq/v4/test" +) + +var luaScenarios = []formatScenario{ + { + description: "Basic example", + scenarioType: "encode", + input: `--- +country: Australia # this place +cities: +- Sydney +- Melbourne +- Brisbane +- Perth`, + expected: `return { + ["country"] = "Australia"; -- this place + ["cities"] = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", + }; +}; +`, + }, + { + description: "Unquoted keys", + subdescription: "Uses the `--lua-unquoted` option to produce a nicer-looking output.", + scenarioType: "unquoted-encode", + input: `--- +country: Australia # this place +cities: +- Sydney +- Melbourne +- Brisbane +- Perth`, + expected: `return { + country = "Australia"; -- this place + cities = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", + }; +}; +`, + }, + { + description: "Globals", + subdescription: "Uses the `--lua-globals` option to export the values into the global scope.", + scenarioType: "globals-encode", + input: `--- +country: Australia # this place +cities: +- Sydney +- Melbourne +- Brisbane +- Perth`, + expected: `country = "Australia"; -- this place +cities = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", +}; +`, + }, + { + description: "Elaborate example", + input: `--- +hello: world +tables: + like: this + keys: values + ? look: non-string keys + : True +numbers: + - decimal: 12345 + - hex: 0x7fabc123 + - octal: 0o30 + - float: 123.45 + - infinity: .inf + - not: .nan +`, + expected: `return { + ["hello"] = "world"; + ["tables"] = { + ["like"] = "this"; + ["keys"] = "values"; + [{ + ["look"] = "non-string keys"; + }] = true; + }; + ["numbers"] = { + { + ["decimal"] = 12345; + }, + { + ["hex"] = 0x7fabc123; + }, + { + ["octal"] = 24; + }, + { + ["float"] = 123.45; + }, + { + ["infinity"] = (1/0); + }, + { + ["not"] = (0/0); + }, + }; +}; +`, + scenarioType: "encode", + }, + { + skipDoc: true, + description: "Sequence", + input: "- a\n- b\n- c\n", + expected: "return {\n\t\"a\",\n\t\"b\",\n\t\"c\",\n};\n", + scenarioType: "encode", + }, + { + skipDoc: true, + description: "Mapping", + input: "a: b\nc:\n d: e\nf: 0\n", + expected: "return {\n\t[\"a\"] = \"b\";\n\t[\"c\"] = {\n\t\t[\"d\"] = \"e\";\n\t};\n\t[\"f\"] = 0;\n};\n", + scenarioType: "encode", + }, + { + skipDoc: true, + description: "Scalar str", + input: "str: |\n foo\n bar\nanother: 'single'\nand: \"double\"", + expected: "return {\n\t[\"str\"] = [[\nfoo\nbar\n]];\n\t[\"another\"] = 'single';\n\t[\"and\"] = \"double\";\n};\n", + scenarioType: "encode", + }, + { + skipDoc: true, + description: "Scalar null", + input: "x: null\n", + expected: "return {\n\t[\"x\"] = nil;\n};\n", + scenarioType: "encode", + }, + { + skipDoc: true, + description: "Scalar int", + input: "- 1\n- 2\n- 0x10\n- 0o30\n- -999\n", + expected: "return {\n\t1,\n\t2,\n\t0x10,\n\t24,\n\t-999,\n};\n", + scenarioType: "encode", + }, + { + skipDoc: true, + description: "Scalar float", + input: "- 1.0\n- 3.14\n- 1e100\n- .Inf\n- .NAN\n", + expected: "return {\n\t1.0,\n\t3.14,\n\t1e100,\n\t(1/0),\n\t(0/0),\n};\n", + scenarioType: "encode", + }, +} + +func testLuaScenario(t *testing.T, s formatScenario) { + switch s.scenarioType { + case "encode": + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(ConfiguredLuaPreferences)), s.description) + case "unquoted-encode": + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(LuaPreferences{ + DocPrefix: "return ", + DocSuffix: ";\n", + UnquotedKeys: true, + Globals: false, + })), s.description) + case "globals-encode": + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(LuaPreferences{ + DocPrefix: "return ", + DocSuffix: ";\n", + UnquotedKeys: false, + Globals: true, + })), s.description) + default: + panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) + } +} + +func documentLuaScenario(t *testing.T, w *bufio.Writer, i interface{}) { + s := i.(formatScenario) + + if s.skipDoc { + return + } + switch s.scenarioType { + case "encode", "unquoted-encode", "globals-encode": + documentLuaEncodeScenario(w, s) + default: + panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) + } +} + +func documentLuaEncodeScenario(w *bufio.Writer, s formatScenario) { + writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) + + if s.subdescription != "" { + writeOrPanic(w, s.subdescription) + writeOrPanic(w, "\n\n") + } + + prefs := ConfiguredLuaPreferences + switch s.scenarioType { + case "unquoted-encode": + prefs = LuaPreferences{ + DocPrefix: "return ", + DocSuffix: ";\n", + UnquotedKeys: true, + Globals: false, + } + case "globals-encode": + prefs = LuaPreferences{ + DocPrefix: "return ", + DocSuffix: ";\n", + UnquotedKeys: false, + Globals: true, + } + } + writeOrPanic(w, "Given a sample.yml file of:\n") + writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input)) + + writeOrPanic(w, "then\n") + switch s.scenarioType { + case "unquoted-encode": + writeOrPanic(w, "```bash\nyq -o=lua --lua-unquoted '.' sample.yml\n```\n") + case "globals-encode": + writeOrPanic(w, "```bash\nyq -o=lua --lua-globals '.' sample.yml\n```\n") + default: + writeOrPanic(w, "```bash\nyq -o=lua '.' sample.yml\n```\n") + } + writeOrPanic(w, "will output\n") + + writeOrPanic(w, fmt.Sprintf("```lua\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(prefs)))) +} + +func TestLuaScenarios(t *testing.T) { + for _, tt := range luaScenarios { + testLuaScenario(t, tt) + } + genericScenarios := make([]interface{}, len(luaScenarios)) + for i, s := range luaScenarios { + genericScenarios[i] = s + } + documentScenarios(t, "usage", "lua", genericScenarios, documentLuaScenario) +} diff --git a/pkg/yqlib/printer.go b/pkg/yqlib/printer.go index 9cd65753..c430fcfe 100644 --- a/pkg/yqlib/printer.go +++ b/pkg/yqlib/printer.go @@ -33,6 +33,7 @@ const ( ShOutputFormat TomlOutputFormat ShellVariablesOutputFormat + LuaOutputFormat ) func OutputFormatFromString(format string) (PrinterOutputFormat, error) { @@ -53,8 +54,10 @@ func OutputFormatFromString(format string) (PrinterOutputFormat, error) { return TomlOutputFormat, nil case "shell", "s", "sh": return ShellVariablesOutputFormat, nil + case "lua", "l": + return LuaOutputFormat, nil default: - return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml|toml|shell]", format) + return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml|toml|shell|lua]", format) } }