suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2025-01-26 08:25:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add support for Lua input (#1810)

Browse Source
This commit is contained in:
Kim Alvefur 2023-10-03 07:00:51 +02:00 committed by GitHub
parent ee900ec997
commit 5fa41624c9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 301 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/utils.go
View File

@ -130,6 +130,8 @@ func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
func createDecoder(format yqlib.InputFormat, evaluateTogether bool) (yqlib.Decoder, error) {
switch format {
case yqlib.LuaInputFormat:
return yqlib.NewLuaDecoder(yqlib.ConfiguredLuaPreferences), nil
case yqlib.XMLInputFormat:
return yqlib.NewXMLDecoder(yqlib.ConfiguredXMLPreferences), nil
case yqlib.PropertiesInputFormat:

1
go.mod
View File

@ -15,6 +15,7 @@ require (
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
github.com/spf13/cobra v1.7.0
github.com/spf13/pflag v1.0.5
github.com/yuin/gopher-lua v1.1.0
golang.org/x/net v0.15.0
golang.org/x/text v0.13.0
gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473

2
go.sum
View File

@ -50,6 +50,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE=
github.com/yuin/gopher-lua v1.1.0/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

3
pkg/yqlib/decoder.go
View File

@ -18,6 +18,7 @@ const (
TSVObjectInputFormat
TomlInputFormat
UriInputFormat
LuaInputFormat
)
type Decoder interface {
@ -41,6 +42,8 @@ func InputFormatFromString(format string) (InputFormat, error) {
return TSVObjectInputFormat, nil
case "toml":
return TomlInputFormat, nil
case "lua", "l":
return LuaInputFormat, nil
default:
return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml|toml]", format)
}

167
pkg/yqlib/decoder_lua.go Normal file
View File

@ -0,0 +1,167 @@
package yqlib
import (
"fmt"
"io"
"math"
lua "github.com/yuin/gopher-lua"
yaml "gopkg.in/yaml.v3"
)
type luaDecoder struct {
reader io.Reader
finished bool
prefs LuaPreferences
}
func NewLuaDecoder(prefs LuaPreferences) Decoder {
return &luaDecoder{
prefs: prefs,
}
}
func (dec *luaDecoder) Init(reader io.Reader) error {
dec.reader = reader
return nil
}
func (dec *luaDecoder) convertToYamlNode(ls *lua.LState, lv lua.LValue) *yaml.Node {
switch lv.Type() {
case lua.LTNil:
return &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!null",
Value: "",
}
case lua.LTBool:
return &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!bool",
Value: lv.String(),
}
case lua.LTNumber:
n := float64(lua.LVAsNumber(lv))
// various special case floats
if math.IsNaN(n) {
return &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!float",
Value: ".nan",
}
}
if math.IsInf(n, 1) {
return &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!float",
Value: ".inf",
}
}
if math.IsInf(n, -1) {
return &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!float",
Value: "-.inf",
}
}
// does it look like an integer?
if n == float64(int(n)) {
return &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!int",
Value: lv.String(),
}
}
return &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!float",
Value: lv.String(),
}
case lua.LTString:
return &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!str",
Value: lv.String(),
}
case lua.LTFunction:
return &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "tag:lua.org,2006,function",
Value: lv.String(),
}
case lua.LTTable:
// Simultaneously create a sequence and a map, pick which one to return
// based on whether all keys were consecutive integers
i := 1
yaml_sequence := &yaml.Node{
Kind: yaml.SequenceNode,
Tag: "!!seq",
}
yaml_map := &yaml.Node{
Kind: yaml.MappingNode,
Tag: "!!map",
}
t := lv.(*lua.LTable)
k, v := ls.Next(t, lua.LNil)
for k != lua.LNil {
if ki, ok := k.(lua.LNumber); i != 0 && ok && math.Mod(float64(ki), 1) == 0 && int(ki) == i {
i++
} else {
i = 0
}
yaml_map.Content = append(yaml_map.Content, dec.convertToYamlNode(ls, k))
yv := dec.convertToYamlNode(ls, v)
yaml_map.Content = append(yaml_map.Content, yv)
if i != 0 {
yaml_sequence.Content = append(yaml_sequence.Content, yv)
}
k, v = ls.Next(t, k)
}
if i != 0 {
return yaml_sequence
}
return yaml_map
default:
return &yaml.Node{
Kind: yaml.ScalarNode,
LineComment: fmt.Sprintf("Unhandled Lua type: %s", lv.Type().String()),
Tag: "!!null",
Value: lv.String(),
}
}
}
func (dec *luaDecoder) decideTopLevelNode(ls *lua.LState) *yaml.Node {
if ls.GetTop() == 0 {
// no items were explicitly returned, encode the globals table instead
return dec.convertToYamlNode(ls, ls.Get(lua.GlobalsIndex))
}
return dec.convertToYamlNode(ls, ls.Get(1))
}
func (dec *luaDecoder) Decode() (*CandidateNode, error) {
if dec.finished {
return nil, io.EOF
}
ls := lua.NewState(lua.Options{SkipOpenLibs: true})
defer ls.Close()
fn, err := ls.Load(dec.reader, "@input")
if err != nil {
return nil, err
}
ls.Push(fn)
err = ls.PCall(0, lua.MultRet, nil)
if err != nil {
return nil, err
}
firstNode := dec.decideTopLevelNode(ls)
dec.finished = true
return &CandidateNode{
Node: &yaml.Node{
Kind: yaml.DocumentNode,
Content: []*yaml.Node{firstNode},
},
}, nil
}

30
pkg/yqlib/doc/usage/lua.md
View File

@ -1,5 +1,33 @@
## Basic example
## Basic input example
Given a sample.lua file of:
```lua
return {
["country"] = "Australia"; -- this place
["cities"] = {
"Sydney",
"Melbourne",
"Brisbane",
"Perth",
};
};
```
then
```bash
yq -oy '.' sample.lua
```
will output
```yaml
country: Australia
cities:
- Sydney
- Melbourne
- Brisbane
- Perth
```
## Basic output example
Given a sample.yml file of:
```yaml
---

95
pkg/yqlib/lua_test.go
View File

@ -10,7 +10,27 @@ import (
var luaScenarios = []formatScenario{
{
description: "Basic example",
description: "Basic input example",
input: `return {
["country"] = "Australia"; -- this place
["cities"] = {
"Sydney",
"Melbourne",
"Brisbane",
"Perth",
};
};
`,
expected: `country: Australia
cities:
- Sydney
- Melbourne
- Brisbane
- Perth
`,
},
{
description: "Basic output example",
scenarioType: "encode",
input: `---
country: Australia # this place
@ -28,6 +48,31 @@ cities:
"Perth",
};
};
`,
},
{
description: "Basic roundtrip",
skipDoc: true,
scenarioType: "roundtrip",
input: `return {
["country"] = "Australia"; -- this place
["cities"] = {
"Sydney",
"Melbourne",
"Brisbane",
"Perth",
};
};
`,
expected: `return {
["country"] = "Australia";
["cities"] = {
"Sydney",
"Melbourne",
"Brisbane",
"Perth",
};
};
`,
},
{
@ -168,8 +213,12 @@ numbers:
func testLuaScenario(t *testing.T, s formatScenario) {
switch s.scenarioType {
case "", "decode":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewYamlEncoder(4, false, ConfiguredYamlPreferences)), s.description)
case "encode":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(ConfiguredLuaPreferences)), s.description)
case "roundtrip":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewLuaEncoder(ConfiguredLuaPreferences)), s.description)
case "unquoted-encode":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(LuaPreferences{
DocPrefix: "return ",
@ -196,13 +245,39 @@ func documentLuaScenario(t *testing.T, w *bufio.Writer, i interface{}) {
return
}
switch s.scenarioType {
case "", "decode":
documentLuaDecodeScenario(w, s)
case "encode", "unquoted-encode", "globals-encode":
documentLuaEncodeScenario(w, s)
case "roundtrip":
documentLuaRoundTripScenario(w, s)
default:
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
}
}
func documentLuaDecodeScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
if s.subdescription != "" {
writeOrPanic(w, s.subdescription)
writeOrPanic(w, "\n\n")
}
writeOrPanic(w, "Given a sample.lua file of:\n")
writeOrPanic(w, fmt.Sprintf("```lua\n%v\n```\n", s.input))
writeOrPanic(w, "then\n")
expression := s.expression
if expression == "" {
expression = "."
}
writeOrPanic(w, fmt.Sprintf("```bash\nyq -oy '%v' sample.lua\n```\n", expression))
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences))))
}
func documentLuaEncodeScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
@ -245,6 +320,24 @@ func documentLuaEncodeScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("```lua\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(prefs))))
}
func documentLuaRoundTripScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
if s.subdescription != "" {
writeOrPanic(w, s.subdescription)
writeOrPanic(w, "\n\n")
}
writeOrPanic(w, "Given a sample.lua file of:\n")
writeOrPanic(w, fmt.Sprintf("```lua\n%v\n```\n", s.input))
writeOrPanic(w, "then\n")
writeOrPanic(w, "```bash\nyq '.' sample.lua\n```\n")
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```lua\n%v```\n\n", mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewLuaEncoder(ConfiguredLuaPreferences))))
}
func TestLuaScenarios(t *testing.T) {
for _, tt := range luaScenarios {
testLuaScenario(t, tt)

3
project-words.txt
View File

@ -126,6 +126,7 @@ Lexer
libdistro
lindex
linecomment
LVAs
magiconair
mapvalues
Mier
@ -135,6 +136,7 @@ minishift
mipsle
mitchellh
mktemp
Mult
multidoc
multimaint
myenv
@ -244,4 +246,5 @@ xmld
xyzzy
yamld
yqlib
yuin
zabbix