Implement basic Lua output support (#1745)

* Implement basic Lua output support

Ref #1700

Basic but working serialization to Lua tables.

* Escape larger set of characters in Lua output

Started with a minimum of replacements, this should be more complete,
tho not all substitutions are strictly required in Lua.

* Print simple keys unquoted in Lua output

String keys that satisfy the requirements for variable names can be used
as keys without quotes in tables.

* Quote Lua keywords in table keys

Keywords are not valid as unquoted keys, thus must be quoted

* Make output of unquoted Lua table keys optional

Generally safer and simpler to not do it.

* Hook up settings for Lua output

* Allow special characters in Lua prefix and suffix

--lua-suffix='});^M' didn't work, so taking this approach instead

* Panic on unhandled YAML Kind in Lua encoder

* Handle YAML case varied booleans in Lua encoder

* Handle special-case numbers in Lua encoder

* Reject unhandled scalar Tags in Lua encoder

* Add note about how Lua nil is unsuitable as table key

Could add some context tracking in the future to allow rejecting nil in
a table key context.

* Return error instead of panic in Lua encoder

* Add initial test for Lua encoder

Boilerplate mostly copied from toml_test.go

* Additional Lua output tests

* Generate Lua encoder documentation

Mostly just for the boilerplate

* Convert octal for Lua output

Lua doesn't have the 0oNNN syntax for octal integers, only decimal and
hexadecimal, hence those can be passed trough as is while octal needs
special treatment.

* Implement indentation in in Lua output

* Respect string Style in Lua encoder

Lua has 'single', "double" and [[ long ]] strings.

* Expand Lua examples

* Output line comments in Lua output

* Implement Lua globals output mode
This commit is contained in:
Kim Alvefur 2023-08-11 04:56:49 +02:00 committed by GitHub
parent 9b4082919b
commit d302d75c77
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 759 additions and 1 deletions

View File

@ -78,6 +78,11 @@ yq -P sample.json
rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipProcInst, "xml-skip-proc-inst", yqlib.ConfiguredXMLPreferences.SkipProcInst, "skip over process instructions (e.g. <?xml version=\"1\"?>)") rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipProcInst, "xml-skip-proc-inst", yqlib.ConfiguredXMLPreferences.SkipProcInst, "skip over process instructions (e.g. <?xml version=\"1\"?>)")
rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipDirectives, "xml-skip-directives", yqlib.ConfiguredXMLPreferences.SkipDirectives, "skip over directives (e.g. <!DOCTYPE thing cat>)") rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipDirectives, "xml-skip-directives", yqlib.ConfiguredXMLPreferences.SkipDirectives, "skip over directives (e.g. <!DOCTYPE thing cat>)")
rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredLuaPreferences.DocPrefix, "lua-prefix", yqlib.ConfiguredLuaPreferences.DocPrefix, "prefix")
rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredLuaPreferences.DocSuffix, "lua-suffix", yqlib.ConfiguredLuaPreferences.DocSuffix, "suffix")
rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.UnquotedKeys, "lua-unquoted", yqlib.ConfiguredLuaPreferences.UnquotedKeys, "output unquoted string keys (e.g. {foo=\"bar\"})")
rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.Globals, "lua-globals", yqlib.ConfiguredLuaPreferences.Globals, "output keys as top-level global variables")
rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.") rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.")
rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)") rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)")

View File

@ -197,6 +197,8 @@ func createEncoder(format yqlib.PrinterOutputFormat) (yqlib.Encoder, error) {
return yqlib.NewTomlEncoder(), nil return yqlib.NewTomlEncoder(), nil
case yqlib.ShellVariablesOutputFormat: case yqlib.ShellVariablesOutputFormat:
return yqlib.NewShellVariablesEncoder(), nil return yqlib.NewShellVariablesEncoder(), nil
case yqlib.LuaOutputFormat:
return yqlib.NewLuaEncoder(yqlib.ConfiguredLuaPreferences), nil
} }
return nil, fmt.Errorf("invalid encoder: %v", format) return nil, fmt.Errorf("invalid encoder: %v", format)
} }

144
pkg/yqlib/doc/usage/lua.md Normal file
View File

@ -0,0 +1,144 @@
## Basic example
Given a sample.yml file of:
```yaml
---
country: Australia # this place
cities:
- Sydney
- Melbourne
- Brisbane
- Perth
```
then
```bash
yq -o=lua '.' sample.yml
```
will output
```lua
return {
["country"] = "Australia"; -- this place
["cities"] = {
"Sydney",
"Melbourne",
"Brisbane",
"Perth",
};
};
```
## Unquoted keys
Uses the `--lua-unquoted` option to produce a nicer-looking output.
Given a sample.yml file of:
```yaml
---
country: Australia # this place
cities:
- Sydney
- Melbourne
- Brisbane
- Perth
```
then
```bash
yq -o=lua '.' sample.yml
```
will output
```lua
return {
country = "Australia"; -- this place
cities = {
"Sydney",
"Melbourne",
"Brisbane",
"Perth",
};
};
```
## Globals
Uses the `--lua-globals` option to export the values into the global scope.
Given a sample.yml file of:
```yaml
---
country: Australia # this place
cities:
- Sydney
- Melbourne
- Brisbane
- Perth
```
then
```bash
yq -o=lua '.' sample.yml
```
will output
```lua
country = "Australia"; -- this place
cities = {
"Sydney",
"Melbourne",
"Brisbane",
"Perth",
};
```
## Elaborate example
Given a sample.yml file of:
```yaml
---
hello: world
tables:
like: this
keys: values
? look: non-string keys
: True
numbers:
- decimal: 12345
- hex: 0x7fabc123
- octal: 0o30
- float: 123.45
- infinity: .inf
- not: .nan
```
then
```bash
yq -o=lua '.' sample.yml
```
will output
```lua
return {
["hello"] = "world";
["tables"] = {
["like"] = "this";
["keys"] = "values";
[{
["look"] = "non-string keys";
}] = true;
};
["numbers"] = {
{
["decimal"] = 12345;
},
{
["hex"] = 0x7fabc123;
},
{
["octal"] = 24;
},
{
["float"] = 123.45;
},
{
["infinity"] = (1/0);
},
{
["not"] = (0/0);
},
};
};
```

328
pkg/yqlib/encoder_lua.go Normal file
View File

@ -0,0 +1,328 @@
package yqlib
import (
"fmt"
"io"
"strings"
yaml "gopkg.in/yaml.v3"
)
type luaEncoder struct {
docPrefix string
docSuffix string
indent int
indentStr string
unquoted bool
globals bool
escape *strings.Replacer
}
func (le *luaEncoder) CanHandleAliases() bool {
return false
}
func NewLuaEncoder(prefs LuaPreferences) Encoder {
escape := strings.NewReplacer(
"\000", "\\000",
"\001", "\\001",
"\002", "\\002",
"\003", "\\003",
"\004", "\\004",
"\005", "\\005",
"\006", "\\006",
"\007", "\\a",
"\010", "\\b",
"\011", "\\t",
"\012", "\\n",
"\013", "\\v",
"\014", "\\f",
"\015", "\\r",
"\016", "\\014",
"\017", "\\015",
"\020", "\\016",
"\021", "\\017",
"\022", "\\018",
"\023", "\\019",
"\024", "\\020",
"\025", "\\021",
"\026", "\\022",
"\027", "\\023",
"\030", "\\024",
"\031", "\\025",
"\032", "\\026",
"\033", "\\027",
"\034", "\\028",
"\035", "\\029",
"\036", "\\030",
"\037", "\\031",
"\"", "\\\"",
"'", "\\'",
"\\", "\\\\",
"\177", "\\127",
)
unescape := strings.NewReplacer(
"\\'", "'",
"\\\"", "\"",
"\\n", "\n",
"\\r", "\r",
"\\t", "\t",
"\\\\", "\\",
)
return &luaEncoder{unescape.Replace(prefs.DocPrefix), unescape.Replace(prefs.DocSuffix), 0, "\t", prefs.UnquotedKeys, prefs.Globals, escape}
}
func (le *luaEncoder) PrintDocumentSeparator(writer io.Writer) error {
return nil
}
func (le *luaEncoder) PrintLeadingContent(writer io.Writer, content string) error {
return nil
}
func (le *luaEncoder) encodeString(writer io.Writer, node *yaml.Node) error {
quote := "\""
switch node.Style {
case yaml.LiteralStyle, yaml.FoldedStyle, yaml.FlowStyle:
for i := 0; i < 10; i++ {
if !strings.Contains(node.Value, "]"+strings.Repeat("=", i)+"]") {
err := writeString(writer, "["+strings.Repeat("=", i)+"[\n")
if err != nil {
return err
}
err = writeString(writer, node.Value)
if err != nil {
return err
}
return writeString(writer, "]"+strings.Repeat("=", i)+"]")
}
}
case yaml.SingleQuotedStyle:
quote = "'"
// falltrough to regular ol' string
}
return writeString(writer, quote+le.escape.Replace(node.Value)+quote)
}
func (le *luaEncoder) writeIndent(writer io.Writer) error {
if le.indentStr == "" {
return nil
}
err := writeString(writer, "\n")
if err != nil {
return err
}
return writeString(writer, strings.Repeat(le.indentStr, le.indent))
}
func (le *luaEncoder) encodeArray(writer io.Writer, node *yaml.Node) error {
err := writeString(writer, "{")
if err != nil {
return err
}
le.indent++
for _, child := range node.Content {
err = le.writeIndent(writer)
if err != nil {
return err
}
err := le.Encode(writer, child)
if err != nil {
return err
}
err = writeString(writer, ",")
if err != nil {
return err
}
if child.LineComment != "" {
sansPrefix, _ := strings.CutPrefix(child.LineComment, "#")
err = writeString(writer, " --"+sansPrefix)
if err != nil {
return err
}
}
}
le.indent--
if len(node.Content) != 0 {
err = le.writeIndent(writer)
if err != nil {
return err
}
}
return writeString(writer, "}")
}
func needsQuoting(s string) bool {
// known keywords as of Lua 5.4
switch s {
case "do", "and", "else", "break",
"if", "end", "goto", "false",
"in", "for", "then", "local",
"or", "nil", "true", "until",
"elseif", "function", "not",
"repeat", "return", "while":
return true
}
// [%a_][%w_]*
for i, c := range s {
if i == 0 {
if !((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_') {
return true
}
} else {
if !((c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_') {
return true
}
}
}
return false
}
func (le *luaEncoder) encodeMap(writer io.Writer, node *yaml.Node, global bool) error {
if !global {
err := writeString(writer, "{")
if err != nil {
return err
}
le.indent++
}
for i, child := range node.Content {
if (i % 2) == 1 {
// value
err := le.Encode(writer, child)
if err != nil {
return err
}
err = writeString(writer, ";")
if err != nil {
return err
}
} else {
// key
if !global || i > 0 {
err := le.writeIndent(writer)
if err != nil {
return err
}
}
if (le.unquoted || global) && child.Tag == "!!str" && !needsQuoting(child.Value) {
err := writeString(writer, child.Value+" = ")
if err != nil {
return err
}
} else {
if global {
// This only works in Lua 5.2+
err := writeString(writer, "_ENV")
if err != nil {
return err
}
}
err := writeString(writer, "[")
if err != nil {
return err
}
err = le.encodeAny(writer, child)
if err != nil {
return err
}
err = writeString(writer, "] = ")
if err != nil {
return err
}
}
}
if child.LineComment != "" {
sansPrefix, _ := strings.CutPrefix(child.LineComment, "#")
err := writeString(writer, strings.Repeat(" ", i%2)+"--"+sansPrefix)
if err != nil {
return err
}
if (i % 2) == 0 {
// newline and indent after comments on keys
err = le.writeIndent(writer)
if err != nil {
return err
}
}
}
}
if global {
return writeString(writer, "\n")
}
le.indent--
if len(node.Content) != 0 {
err := le.writeIndent(writer)
if err != nil {
return err
}
}
return writeString(writer, "}")
}
func (le *luaEncoder) encodeAny(writer io.Writer, node *yaml.Node) error {
switch node.Kind {
case yaml.SequenceNode:
return le.encodeArray(writer, node)
case yaml.MappingNode:
return le.encodeMap(writer, node, false)
case yaml.ScalarNode:
switch node.Tag {
case "!!str":
return le.encodeString(writer, node)
case "!!null":
// TODO reject invalid use as a table key
return writeString(writer, "nil")
case "!!bool":
// Yaml 1.2 has case variation e.g. True, FALSE etc but Lua only has
// lower case
return writeString(writer, strings.ToLower(node.Value))
case "!!int":
if strings.HasPrefix(node.Value, "0o") {
var octalValue int
err := node.Decode(&octalValue)
if err != nil {
return err
}
return writeString(writer, fmt.Sprintf("%d", octalValue))
}
return writeString(writer, strings.ToLower(node.Value))
case "!!float":
switch strings.ToLower(node.Value) {
case ".inf":
return writeString(writer, "(1/0)")
case "-.inf":
return writeString(writer, "(-1/0)")
case ".nan":
return writeString(writer, "(0/0)")
default:
return writeString(writer, node.Value)
}
default:
return fmt.Errorf("Lua encoder NYI -- %s", node.ShortTag())
}
case yaml.DocumentNode:
if le.globals {
if node.Content[0].Kind != yaml.MappingNode {
return fmt.Errorf("--lua-global requires a top level MappingNode")
}
return le.encodeMap(writer, node.Content[0], true)
}
err := writeString(writer, le.docPrefix)
if err != nil {
return err
}
err = le.encodeAny(writer, node.Content[0])
if err != nil {
return err
}
return writeString(writer, le.docSuffix)
default:
return fmt.Errorf("Lua encoder NYI -- %s", node.ShortTag())
}
}
func (le *luaEncoder) Encode(writer io.Writer, node *yaml.Node) error {
return le.encodeAny(writer, node)
}

19
pkg/yqlib/lua.go Normal file
View File

@ -0,0 +1,19 @@
package yqlib
type LuaPreferences struct {
DocPrefix string
DocSuffix string
UnquotedKeys bool
Globals bool
}
func NewDefaultLuaPreferences() LuaPreferences {
return LuaPreferences{
DocPrefix: "return ",
DocSuffix: ";\n",
UnquotedKeys: false,
Globals: false,
}
}
var ConfiguredLuaPreferences = NewDefaultLuaPreferences()

257
pkg/yqlib/lua_test.go Normal file
View File

@ -0,0 +1,257 @@
package yqlib
import (
"bufio"
"fmt"
"testing"
"github.com/mikefarah/yq/v4/test"
)
var luaScenarios = []formatScenario{
{
description: "Basic example",
scenarioType: "encode",
input: `---
country: Australia # this place
cities:
- Sydney
- Melbourne
- Brisbane
- Perth`,
expected: `return {
["country"] = "Australia"; -- this place
["cities"] = {
"Sydney",
"Melbourne",
"Brisbane",
"Perth",
};
};
`,
},
{
description: "Unquoted keys",
subdescription: "Uses the `--lua-unquoted` option to produce a nicer-looking output.",
scenarioType: "unquoted-encode",
input: `---
country: Australia # this place
cities:
- Sydney
- Melbourne
- Brisbane
- Perth`,
expected: `return {
country = "Australia"; -- this place
cities = {
"Sydney",
"Melbourne",
"Brisbane",
"Perth",
};
};
`,
},
{
description: "Globals",
subdescription: "Uses the `--lua-globals` option to export the values into the global scope.",
scenarioType: "globals-encode",
input: `---
country: Australia # this place
cities:
- Sydney
- Melbourne
- Brisbane
- Perth`,
expected: `country = "Australia"; -- this place
cities = {
"Sydney",
"Melbourne",
"Brisbane",
"Perth",
};
`,
},
{
description: "Elaborate example",
input: `---
hello: world
tables:
like: this
keys: values
? look: non-string keys
: True
numbers:
- decimal: 12345
- hex: 0x7fabc123
- octal: 0o30
- float: 123.45
- infinity: .inf
- not: .nan
`,
expected: `return {
["hello"] = "world";
["tables"] = {
["like"] = "this";
["keys"] = "values";
[{
["look"] = "non-string keys";
}] = true;
};
["numbers"] = {
{
["decimal"] = 12345;
},
{
["hex"] = 0x7fabc123;
},
{
["octal"] = 24;
},
{
["float"] = 123.45;
},
{
["infinity"] = (1/0);
},
{
["not"] = (0/0);
},
};
};
`,
scenarioType: "encode",
},
{
skipDoc: true,
description: "Sequence",
input: "- a\n- b\n- c\n",
expected: "return {\n\t\"a\",\n\t\"b\",\n\t\"c\",\n};\n",
scenarioType: "encode",
},
{
skipDoc: true,
description: "Mapping",
input: "a: b\nc:\n d: e\nf: 0\n",
expected: "return {\n\t[\"a\"] = \"b\";\n\t[\"c\"] = {\n\t\t[\"d\"] = \"e\";\n\t};\n\t[\"f\"] = 0;\n};\n",
scenarioType: "encode",
},
{
skipDoc: true,
description: "Scalar str",
input: "str: |\n foo\n bar\nanother: 'single'\nand: \"double\"",
expected: "return {\n\t[\"str\"] = [[\nfoo\nbar\n]];\n\t[\"another\"] = 'single';\n\t[\"and\"] = \"double\";\n};\n",
scenarioType: "encode",
},
{
skipDoc: true,
description: "Scalar null",
input: "x: null\n",
expected: "return {\n\t[\"x\"] = nil;\n};\n",
scenarioType: "encode",
},
{
skipDoc: true,
description: "Scalar int",
input: "- 1\n- 2\n- 0x10\n- 0o30\n- -999\n",
expected: "return {\n\t1,\n\t2,\n\t0x10,\n\t24,\n\t-999,\n};\n",
scenarioType: "encode",
},
{
skipDoc: true,
description: "Scalar float",
input: "- 1.0\n- 3.14\n- 1e100\n- .Inf\n- .NAN\n",
expected: "return {\n\t1.0,\n\t3.14,\n\t1e100,\n\t(1/0),\n\t(0/0),\n};\n",
scenarioType: "encode",
},
}
func testLuaScenario(t *testing.T, s formatScenario) {
switch s.scenarioType {
case "encode":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(ConfiguredLuaPreferences)), s.description)
case "unquoted-encode":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(LuaPreferences{
DocPrefix: "return ",
DocSuffix: ";\n",
UnquotedKeys: true,
Globals: false,
})), s.description)
case "globals-encode":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(LuaPreferences{
DocPrefix: "return ",
DocSuffix: ";\n",
UnquotedKeys: false,
Globals: true,
})), s.description)
default:
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
}
}
func documentLuaScenario(t *testing.T, w *bufio.Writer, i interface{}) {
s := i.(formatScenario)
if s.skipDoc {
return
}
switch s.scenarioType {
case "encode", "unquoted-encode", "globals-encode":
documentLuaEncodeScenario(w, s)
default:
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
}
}
func documentLuaEncodeScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
if s.subdescription != "" {
writeOrPanic(w, s.subdescription)
writeOrPanic(w, "\n\n")
}
prefs := ConfiguredLuaPreferences
switch s.scenarioType {
case "unquoted-encode":
prefs = LuaPreferences{
DocPrefix: "return ",
DocSuffix: ";\n",
UnquotedKeys: true,
Globals: false,
}
case "globals-encode":
prefs = LuaPreferences{
DocPrefix: "return ",
DocSuffix: ";\n",
UnquotedKeys: false,
Globals: true,
}
}
writeOrPanic(w, "Given a sample.yml file of:\n")
writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input))
writeOrPanic(w, "then\n")
switch s.scenarioType {
case "unquoted-encode":
writeOrPanic(w, "```bash\nyq -o=lua --lua-unquoted '.' sample.yml\n```\n")
case "globals-encode":
writeOrPanic(w, "```bash\nyq -o=lua --lua-globals '.' sample.yml\n```\n")
default:
writeOrPanic(w, "```bash\nyq -o=lua '.' sample.yml\n```\n")
}
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```lua\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(prefs))))
}
func TestLuaScenarios(t *testing.T) {
for _, tt := range luaScenarios {
testLuaScenario(t, tt)
}
genericScenarios := make([]interface{}, len(luaScenarios))
for i, s := range luaScenarios {
genericScenarios[i] = s
}
documentScenarios(t, "usage", "lua", genericScenarios, documentLuaScenario)
}

View File

@ -33,6 +33,7 @@ const (
ShOutputFormat ShOutputFormat
TomlOutputFormat TomlOutputFormat
ShellVariablesOutputFormat ShellVariablesOutputFormat
LuaOutputFormat
) )
func OutputFormatFromString(format string) (PrinterOutputFormat, error) { func OutputFormatFromString(format string) (PrinterOutputFormat, error) {
@ -53,8 +54,10 @@ func OutputFormatFromString(format string) (PrinterOutputFormat, error) {
return TomlOutputFormat, nil return TomlOutputFormat, nil
case "shell", "s", "sh": case "shell", "s", "sh":
return ShellVariablesOutputFormat, nil return ShellVariablesOutputFormat, nil
case "lua", "l":
return LuaOutputFormat, nil
default: default:
return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml|toml|shell]", format) return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml|toml|shell|lua]", format)
} }
} }