Add support for Lua input (#1810)

cmd/utils.go

@@ -130,6 +130,8 @@ func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
 
 func createDecoder(format yqlib.InputFormat, evaluateTogether bool) (yqlib.Decoder, error) {
 	switch format {
+	case yqlib.LuaInputFormat:
+		return yqlib.NewLuaDecoder(yqlib.ConfiguredLuaPreferences), nil
 	case yqlib.XMLInputFormat:
 		return yqlib.NewXMLDecoder(yqlib.ConfiguredXMLPreferences), nil
 	case yqlib.PropertiesInputFormat:

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
+	github.com/yuin/gopher-lua v1.1.0
 	golang.org/x/net v0.15.0
 	golang.org/x/text v0.13.0
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473

go.sum

@@ -50,6 +50,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE=
+github.com/yuin/gopher-lua v1.1.0/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
 golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
 golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

pkg/yqlib/decoder.go

@@ -18,6 +18,7 @@ const (
 	TSVObjectInputFormat
 	TomlInputFormat
 	UriInputFormat
+	LuaInputFormat
 )
 
 type Decoder interface {
@@ -41,6 +42,8 @@ func InputFormatFromString(format string) (InputFormat, error) {
 		return TSVObjectInputFormat, nil
 	case "toml":
 		return TomlInputFormat, nil
+	case "lua", "l":
+		return LuaInputFormat, nil
 	default:
 		return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml|toml]", format)
 	}

pkg/yqlib/decoder_lua.go

@@ -0,0 +1,167 @@
+package yqlib
+
+import (
+	"fmt"
+	"io"
+	"math"
+
+	lua "github.com/yuin/gopher-lua"
+	yaml "gopkg.in/yaml.v3"
+)
+
+type luaDecoder struct {
+	reader   io.Reader
+	finished bool
+	prefs    LuaPreferences
+}
+
+func NewLuaDecoder(prefs LuaPreferences) Decoder {
+	return &luaDecoder{
+		prefs: prefs,
+	}
+}
+
+func (dec *luaDecoder) Init(reader io.Reader) error {
+	dec.reader = reader
+	return nil
+}
+
+func (dec *luaDecoder) convertToYamlNode(ls *lua.LState, lv lua.LValue) *yaml.Node {
+	switch lv.Type() {
+	case lua.LTNil:
+		return &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "!!null",
+			Value: "",
+		}
+	case lua.LTBool:
+		return &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "!!bool",
+			Value: lv.String(),
+		}
+	case lua.LTNumber:
+		n := float64(lua.LVAsNumber(lv))
+		// various special case floats
+		if math.IsNaN(n) {
+			return &yaml.Node{
+				Kind:  yaml.ScalarNode,
+				Tag:   "!!float",
+				Value: ".nan",
+			}
+		}
+		if math.IsInf(n, 1) {
+			return &yaml.Node{
+				Kind:  yaml.ScalarNode,
+				Tag:   "!!float",
+				Value: ".inf",
+			}
+		}
+		if math.IsInf(n, -1) {
+			return &yaml.Node{
+				Kind:  yaml.ScalarNode,
+				Tag:   "!!float",
+				Value: "-.inf",
+			}
+		}
+
+		// does it look like an integer?
+		if n == float64(int(n)) {
+			return &yaml.Node{
+				Kind:  yaml.ScalarNode,
+				Tag:   "!!int",
+				Value: lv.String(),
+			}
+		}
+
+		return &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "!!float",
+			Value: lv.String(),
+		}
+	case lua.LTString:
+		return &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "!!str",
+			Value: lv.String(),
+		}
+	case lua.LTFunction:
+		return &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "tag:lua.org,2006,function",
+			Value: lv.String(),
+		}
+	case lua.LTTable:
+		// Simultaneously create a sequence and a map, pick which one to return
+		// based on whether all keys were consecutive integers
+		i := 1
+		yaml_sequence := &yaml.Node{
+			Kind: yaml.SequenceNode,
+			Tag:  "!!seq",
+		}
+		yaml_map := &yaml.Node{
+			Kind: yaml.MappingNode,
+			Tag:  "!!map",
+		}
+		t := lv.(*lua.LTable)
+		k, v := ls.Next(t, lua.LNil)
+		for k != lua.LNil {
+			if ki, ok := k.(lua.LNumber); i != 0 && ok && math.Mod(float64(ki), 1) == 0 && int(ki) == i {
+				i++
+			} else {
+				i = 0
+			}
+			yaml_map.Content = append(yaml_map.Content, dec.convertToYamlNode(ls, k))
+			yv := dec.convertToYamlNode(ls, v)
+			yaml_map.Content = append(yaml_map.Content, yv)
+			if i != 0 {
+				yaml_sequence.Content = append(yaml_sequence.Content, yv)
+			}
+			k, v = ls.Next(t, k)
+		}
+		if i != 0 {
+			return yaml_sequence
+		}
+		return yaml_map
+	default:
+		return &yaml.Node{
+			Kind:        yaml.ScalarNode,
+			LineComment: fmt.Sprintf("Unhandled Lua type: %s", lv.Type().String()),
+			Tag:         "!!null",
+			Value:       lv.String(),
+		}
+	}
+}
+
+func (dec *luaDecoder) decideTopLevelNode(ls *lua.LState) *yaml.Node {
+	if ls.GetTop() == 0 {
+		// no items were explicitly returned, encode the globals table instead
+		return dec.convertToYamlNode(ls, ls.Get(lua.GlobalsIndex))
+	}
+	return dec.convertToYamlNode(ls, ls.Get(1))
+}
+
+func (dec *luaDecoder) Decode() (*CandidateNode, error) {
+	if dec.finished {
+		return nil, io.EOF
+	}
+	ls := lua.NewState(lua.Options{SkipOpenLibs: true})
+	defer ls.Close()
+	fn, err := ls.Load(dec.reader, "@input")
+	if err != nil {
+		return nil, err
+	}
+	ls.Push(fn)
+	err = ls.PCall(0, lua.MultRet, nil)
+	if err != nil {
+		return nil, err
+	}
+	firstNode := dec.decideTopLevelNode(ls)
+	dec.finished = true
+	return &CandidateNode{
+		Node: &yaml.Node{
+			Kind:    yaml.DocumentNode,
+			Content: []*yaml.Node{firstNode},
+		},
+	}, nil
+}

pkg/yqlib/doc/usage/lua.md

@@ -1,5 +1,33 @@
 
-## Basic example
+## Basic input example
+Given a sample.lua file of:
+```lua
+return {
+	["country"] = "Australia"; -- this place
+	["cities"] = {
+		"Sydney",
+		"Melbourne",
+		"Brisbane",
+		"Perth",
+	};
+};
+
+```
+then
+```bash
+yq -oy '.' sample.lua
+```
+will output
+```yaml
+country: Australia
+cities:
+  - Sydney
+  - Melbourne
+  - Brisbane
+  - Perth
+```
+
+## Basic output example
 Given a sample.yml file of:
 ```yaml
 ---

pkg/yqlib/lua_test.go

@@ -10,7 +10,27 @@ import (
 
 var luaScenarios = []formatScenario{
 	{
-		description:  "Basic example",
+		description: "Basic input example",
+		input: `return {
+	["country"] = "Australia"; -- this place
+	["cities"] = {
+		"Sydney",
+		"Melbourne",
+		"Brisbane",
+		"Perth",
+	};
+};
+`,
+		expected: `country: Australia
+cities:
+    - Sydney
+    - Melbourne
+    - Brisbane
+    - Perth
+`,
+	},
+	{
+		description:  "Basic output example",
 		scenarioType: "encode",
 		input: `---
 country: Australia # this place
@@ -28,6 +48,31 @@ cities:
 		"Perth",
 	};
 };
+`,
+	},
+	{
+		description:  "Basic roundtrip",
+		skipDoc:      true,
+		scenarioType: "roundtrip",
+		input: `return {
+	["country"] = "Australia"; -- this place
+	["cities"] = {
+		"Sydney",
+		"Melbourne",
+		"Brisbane",
+		"Perth",
+	};
+};
+`,
+		expected: `return {
+	["country"] = "Australia";
+	["cities"] = {
+		"Sydney",
+		"Melbourne",
+		"Brisbane",
+		"Perth",
+	};
+};
 `,
 	},
 	{
@@ -168,8 +213,12 @@ numbers:
 
 func testLuaScenario(t *testing.T, s formatScenario) {
 	switch s.scenarioType {
+	case "", "decode":
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewYamlEncoder(4, false, ConfiguredYamlPreferences)), s.description)
 	case "encode":
 		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(ConfiguredLuaPreferences)), s.description)
+	case "roundtrip":
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewLuaEncoder(ConfiguredLuaPreferences)), s.description)
 	case "unquoted-encode":
 		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(LuaPreferences{
 			DocPrefix:    "return ",
@@ -196,13 +245,39 @@ func documentLuaScenario(t *testing.T, w *bufio.Writer, i interface{}) {
 		return
 	}
 	switch s.scenarioType {
+	case "", "decode":
+		documentLuaDecodeScenario(w, s)
 	case "encode", "unquoted-encode", "globals-encode":
 		documentLuaEncodeScenario(w, s)
+	case "roundtrip":
+		documentLuaRoundTripScenario(w, s)
 	default:
 		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
 	}
 }
 
+func documentLuaDecodeScenario(w *bufio.Writer, s formatScenario) {
+	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
+
+	if s.subdescription != "" {
+		writeOrPanic(w, s.subdescription)
+		writeOrPanic(w, "\n\n")
+	}
+
+	writeOrPanic(w, "Given a sample.lua file of:\n")
+	writeOrPanic(w, fmt.Sprintf("```lua\n%v\n```\n", s.input))
+
+	writeOrPanic(w, "then\n")
+	expression := s.expression
+	if expression == "" {
+		expression = "."
+	}
+	writeOrPanic(w, fmt.Sprintf("```bash\nyq -oy '%v' sample.lua\n```\n", expression))
+	writeOrPanic(w, "will output\n")
+
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences))))
+}
+
 func documentLuaEncodeScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
@@ -245,6 +320,24 @@ func documentLuaEncodeScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, fmt.Sprintf("```lua\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(prefs))))
 }
 
+func documentLuaRoundTripScenario(w *bufio.Writer, s formatScenario) {
+	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
+
+	if s.subdescription != "" {
+		writeOrPanic(w, s.subdescription)
+		writeOrPanic(w, "\n\n")
+	}
+
+	writeOrPanic(w, "Given a sample.lua file of:\n")
+	writeOrPanic(w, fmt.Sprintf("```lua\n%v\n```\n", s.input))
+
+	writeOrPanic(w, "then\n")
+	writeOrPanic(w, "```bash\nyq '.' sample.lua\n```\n")
+	writeOrPanic(w, "will output\n")
+
+	writeOrPanic(w, fmt.Sprintf("```lua\n%v```\n\n", mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewLuaEncoder(ConfiguredLuaPreferences))))
+}
+
 func TestLuaScenarios(t *testing.T) {
 	for _, tt := range luaScenarios {
 		testLuaScenario(t, tt)

project-words.txt

@@ -126,6 +126,7 @@ Lexer
 libdistro
 lindex
 linecomment
+LVAs
 magiconair
 mapvalues
 Mier
@@ -135,6 +136,7 @@ minishift
 mipsle
 mitchellh
 mktemp
+Mult
 multidoc
 multimaint
 myenv
@@ -244,4 +246,5 @@ xmld
 xyzzy
 yamld
 yqlib
+yuin
 zabbix