mirror of
https://github.com/mikefarah/yq.git
synced 2024-12-19 20:19:04 +00:00
Draft: Toml (#1439)
* toml wip * wip * Fixed auto parsing toml * Added build flag not to include toml * Parse toml docs and tests * minor updates
This commit is contained in:
Mike Farah
GitHub
parent
47f4ddc910
commit
7103b78d38
2
.gitignore
vendored
2
.gitignore
vendored
@ -41,6 +41,8 @@ yq*.snap
|
||||
|
||||
test.yml
|
||||
test*.yml
|
||||
test*.xml
|
||||
test*.toml
|
||||
test*.yaml
|
||||
0.yml
|
||||
1.yml
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
|
||||
setUp() {
|
||||
rm test*.yml 2>/dev/null || true
|
||||
rm test*.toml 2>/dev/null || true
|
||||
rm test*.tfstate 2>/dev/null || true
|
||||
rm test*.json 2>/dev/null || true
|
||||
rm test*.properties 2>/dev/null || true
|
||||
@ -30,6 +31,26 @@ EOM
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testInputToml() {
|
||||
cat >test.toml <<EOL
|
||||
[owner]
|
||||
name = "Tom Preston-Werner"
|
||||
dob = 1979-05-27T07:32:00-08:00
|
||||
EOL
|
||||
|
||||
read -r -d '' expected << EOM
|
||||
owner:
|
||||
name: Tom Preston-Werner
|
||||
dob: 1979-05-27T07:32:00-08:00
|
||||
EOM
|
||||
|
||||
X=$(./yq -oy test.toml)
|
||||
assertEquals "$expected" "$X"
|
||||
|
||||
X=$(./yq ea -oy test.toml)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testInputTfstate() {
|
||||
cat >test.tfstate <<EOL
|
||||
{ "mike" : { "things": "cool" } }
|
||||
|
||||
@ -65,8 +65,8 @@ yq -P sample.json
|
||||
panic(err)
|
||||
}
|
||||
|
||||
rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "auto", "[auto|a|yaml|y|json|j|props|p|xml|x] output format type.")
|
||||
rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "auto", "[auto|a|yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.")
|
||||
rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "auto", "[auto|a|yaml|y|json|j|props|p|xml|x|tsv|t|csv|c] output format type.")
|
||||
rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "auto", "[auto|a|yaml|y|props|p|xml|x|tsv|t|csv|c|toml] parse format for input. Note that json is a subset of yaml.")
|
||||
|
||||
rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", yqlib.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes")
|
||||
rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", yqlib.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).")
|
||||
|
||||
@ -75,6 +75,9 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
|
||||
}
|
||||
} else if isAutomaticOutputFormat() {
|
||||
// automatic input worked, we can do it for output too unless specified
|
||||
if inputFormat == "toml" {
|
||||
return "", nil, fmt.Errorf("toml is not yet supported as an output format. Please specify another output format using the [--output-format/-o] flag")
|
||||
}
|
||||
outputFormat = inputFormat
|
||||
}
|
||||
} else if isAutomaticOutputFormat() {
|
||||
@ -137,6 +140,8 @@ func createDecoder(format yqlib.InputFormat, evaluateTogether bool) (yqlib.Decod
|
||||
return yqlib.NewCSVObjectDecoder(','), nil
|
||||
case yqlib.TSVObjectInputFormat:
|
||||
return yqlib.NewCSVObjectDecoder('\t'), nil
|
||||
case yqlib.TomlInputFormat:
|
||||
return yqlib.NewTomlDecoder(), nil
|
||||
case yqlib.YamlInputFormat:
|
||||
prefs := yqlib.ConfiguredYamlPreferences
|
||||
prefs.EvaluateTogether = evaluateTogether
|
||||
|
||||
26
examples/sample.toml
Normal file
26
examples/sample.toml
Normal file
@ -0,0 +1,26 @@
|
||||
|
||||
|
||||
# This is a TOML document
|
||||
|
||||
title = "TOML Example"
|
||||
|
||||
[owner]
|
||||
name = "Tom Preston-Werner"
|
||||
dob = 1979-05-27T07:32:00-08:00
|
||||
|
||||
[database]
|
||||
enabled = true
|
||||
ports = [ 8000, 8001, 8002 ]
|
||||
data = [ ["delta", "phi"], [3.14] ]
|
||||
temp_targets = { cpu = 79.5, case = 72.0 }
|
||||
|
||||
[servers]
|
||||
|
||||
[servers.alpha]
|
||||
ip = "10.0.0.1"
|
||||
role = "frontend"
|
||||
|
||||
[servers.beta]
|
||||
ip = "10.0.0.2"
|
||||
role = "backend"
|
||||
|
||||
1
go.mod
1
go.mod
@ -11,6 +11,7 @@ require (
|
||||
github.com/goccy/go-yaml v1.10.0
|
||||
github.com/jinzhu/copier v0.3.5
|
||||
github.com/magiconair/properties v1.8.7
|
||||
github.com/pelletier/go-toml/v2 v2.0.6
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
|
||||
github.com/spf13/cobra v1.6.1
|
||||
github.com/spf13/pflag v1.0.5
|
||||
|
||||
12
go.sum
12
go.sum
@ -6,8 +6,9 @@ github.com/alecthomas/participle/v2 v2.0.0-beta.5/go.mod h1:RC764t6n4L8D8ITAJv0q
|
||||
github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk=
|
||||
github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
|
||||
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
|
||||
github.com/elliotchance/orderedmap v1.5.0 h1:1IsExUsjv5XNBD3ZdC7jkAAqLWOOKdbPTmkHx63OsBg=
|
||||
@ -36,6 +37,8 @@ github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU=
|
||||
github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
|
||||
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
@ -49,9 +52,14 @@ github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUq
|
||||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
|
||||
@ -4,6 +4,7 @@ import (
|
||||
"fmt"
|
||||
|
||||
logging "gopkg.in/op/go-logging.v1"
|
||||
yaml "gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type DataTreeNavigator interface {
|
||||
@ -11,6 +12,8 @@ type DataTreeNavigator interface {
|
||||
// this will process the against the given expressionNode and return
|
||||
// a new context of matching candidates
|
||||
GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error)
|
||||
|
||||
DeeplyAssign(context Context, path []interface{}, rhsNode *yaml.Node) error
|
||||
}
|
||||
|
||||
type dataTreeNavigator struct {
|
||||
@ -20,6 +23,27 @@ func NewDataTreeNavigator() DataTreeNavigator {
|
||||
return &dataTreeNavigator{}
|
||||
}
|
||||
|
||||
func (d *dataTreeNavigator) DeeplyAssign(context Context, path []interface{}, rhsNode *yaml.Node) error {
|
||||
|
||||
rhsCandidateNode := &CandidateNode{
|
||||
Path: path,
|
||||
Node: rhsNode,
|
||||
}
|
||||
|
||||
assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
|
||||
|
||||
rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}
|
||||
|
||||
assignmentOpNode := &ExpressionNode{
|
||||
Operation: assignmentOp,
|
||||
LHS: createTraversalTree(path, traversePreferences{}, false),
|
||||
RHS: &ExpressionNode{Operation: rhsOp},
|
||||
}
|
||||
|
||||
_, err := d.GetMatchingNodes(context, assignmentOpNode)
|
||||
return err
|
||||
}
|
||||
|
||||
func (d *dataTreeNavigator) GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error) {
|
||||
if expressionNode == nil {
|
||||
log.Debugf("getMatchingNodes - nothing to do")
|
||||
|
||||
@ -16,6 +16,7 @@ const (
|
||||
JsonInputFormat
|
||||
CSVObjectInputFormat
|
||||
TSVObjectInputFormat
|
||||
TomlInputFormat
|
||||
UriInputFormat
|
||||
)
|
||||
|
||||
@ -38,8 +39,10 @@ func InputFormatFromString(format string) (InputFormat, error) {
|
||||
return CSVObjectInputFormat, nil
|
||||
case "tsv", "t":
|
||||
return TSVObjectInputFormat, nil
|
||||
case "toml":
|
||||
return TomlInputFormat, nil
|
||||
default:
|
||||
return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml]", format)
|
||||
return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml|toml]", format)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -95,23 +95,7 @@ func (dec *propertiesDecoder) applyProperty(context Context, properties *propert
|
||||
|
||||
rhsNode.Tag = guessTagFromCustomType(rhsNode)
|
||||
|
||||
rhsCandidateNode := &CandidateNode{
|
||||
Path: path,
|
||||
Node: rhsNode,
|
||||
}
|
||||
|
||||
assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
|
||||
|
||||
rhsOp := &Operation{OperationType: referenceOpType, CandidateNode: rhsCandidateNode}
|
||||
|
||||
assignmentOpNode := &ExpressionNode{
|
||||
Operation: assignmentOp,
|
||||
LHS: createTraversalTree(path, traversePreferences{}, false),
|
||||
RHS: &ExpressionNode{Operation: rhsOp},
|
||||
}
|
||||
|
||||
_, err := dec.d.GetMatchingNodes(context, assignmentOpNode)
|
||||
return err
|
||||
return dec.d.DeeplyAssign(context, path, rhsNode)
|
||||
}
|
||||
|
||||
func (dec *propertiesDecoder) Decode() (*CandidateNode, error) {
|
||||
|
||||
@ -3,6 +3,7 @@ package yqlib
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@ -62,7 +63,7 @@ func mustProcessFormatScenario(s formatScenario, decoder Decoder, encoder Encode
|
||||
|
||||
result, err := processFormatScenario(s, decoder, encoder)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
panic(fmt.Errorf("Bad scenario %v: %w", s.description, err))
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
369
pkg/yqlib/decoder_toml.go
Normal file
369
pkg/yqlib/decoder_toml.go
Normal file
@ -0,0 +1,369 @@
|
||||
//go:build !yq_notoml
|
||||
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
toml "github.com/pelletier/go-toml/v2/unstable"
|
||||
yaml "gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type tomlDecoder struct {
|
||||
parser toml.Parser
|
||||
finished bool
|
||||
d DataTreeNavigator
|
||||
rootMap *CandidateNode
|
||||
}
|
||||
|
||||
func NewTomlDecoder() Decoder {
|
||||
return &tomlDecoder{
|
||||
finished: false,
|
||||
d: NewDataTreeNavigator(),
|
||||
}
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) Init(reader io.Reader) error {
|
||||
dec.parser = toml.Parser{}
|
||||
buf := new(bytes.Buffer)
|
||||
_, err := buf.ReadFrom(reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dec.parser.Reset(buf.Bytes())
|
||||
dec.rootMap = &CandidateNode{
|
||||
Node: &yaml.Node{
|
||||
Kind: yaml.MappingNode,
|
||||
Tag: "!!map",
|
||||
}}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) getFullPath(tomlNode *toml.Node) []interface{} {
|
||||
path := make([]interface{}, 0)
|
||||
for {
|
||||
path = append(path, string(tomlNode.Data))
|
||||
tomlNode = tomlNode.Next()
|
||||
if tomlNode == nil {
|
||||
return path
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) processKeyValueIntoMap(rootMap *CandidateNode, tomlNode *toml.Node) error {
|
||||
value := tomlNode.Value()
|
||||
path := dec.getFullPath(value.Next())
|
||||
log.Debug("!!!processKeyValueIntoMap: %v", path)
|
||||
|
||||
valueNode, err := dec.decodeNode(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
context := Context{}
|
||||
context = context.SingleChildContext(rootMap)
|
||||
|
||||
return dec.d.DeeplyAssign(context, path, valueNode)
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) decodeKeyValuesIntoMap(rootMap *CandidateNode, tomlNode *toml.Node) (bool, error) {
|
||||
log.Debug("!! DECODE_KV_INTO_MAP -- processing first (current) entry")
|
||||
if err := dec.processKeyValueIntoMap(rootMap, tomlNode); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
for dec.parser.NextExpression() {
|
||||
nextItem := dec.parser.Expression()
|
||||
log.Debug("!! DECODE_KV_INTO_MAP -- next exp, its a %v", nextItem.Kind)
|
||||
|
||||
if nextItem.Kind == toml.KeyValue {
|
||||
if err := dec.processKeyValueIntoMap(rootMap, nextItem); err != nil {
|
||||
return false, err
|
||||
}
|
||||
} else {
|
||||
// run out of key values
|
||||
log.Debug("! DECODE_KV_INTO_MAP - ok we are done in decodeKeyValuesIntoMap, gota a %v", nextItem.Kind)
|
||||
log.Debug("! DECODE_KV_INTO_MAP - processAgainstCurrentExp = true!")
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
log.Debug("! DECODE_KV_INTO_MAP - no more things to read in")
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) createInlineTableMap(tomlNode *toml.Node) (*yaml.Node, error) {
|
||||
content := make([]*yaml.Node, 0)
|
||||
log.Debug("!! createInlineTableMap")
|
||||
|
||||
iterator := tomlNode.Children()
|
||||
for iterator.Next() {
|
||||
child := iterator.Node()
|
||||
if child.Kind != toml.KeyValue {
|
||||
return nil, fmt.Errorf("only keyvalue pairs are supported in inlinetables, got %v instead", child.Kind)
|
||||
}
|
||||
|
||||
keyValues := &CandidateNode{
|
||||
Node: &yaml.Node{
|
||||
Kind: yaml.MappingNode,
|
||||
Tag: "!!map",
|
||||
},
|
||||
}
|
||||
|
||||
if err := dec.processKeyValueIntoMap(keyValues, child); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
content = append(content, keyValues.Node.Content...)
|
||||
}
|
||||
|
||||
return &yaml.Node{
|
||||
Kind: yaml.MappingNode,
|
||||
Tag: "!!map",
|
||||
Content: content,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) createArray(tomlNode *toml.Node) (*yaml.Node, error) {
|
||||
content := make([]*yaml.Node, 0)
|
||||
iterator := tomlNode.Children()
|
||||
for iterator.Next() {
|
||||
child := iterator.Node()
|
||||
yamlNode, err := dec.decodeNode(child)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
content = append(content, yamlNode)
|
||||
}
|
||||
|
||||
return &yaml.Node{
|
||||
Kind: yaml.SequenceNode,
|
||||
Tag: "!!seq",
|
||||
Content: content,
|
||||
}, nil
|
||||
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) createStringScalar(tomlNode *toml.Node) (*yaml.Node, error) {
|
||||
content := string(tomlNode.Data)
|
||||
return createScalarNode(content, content), nil
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) createBoolScalar(tomlNode *toml.Node) (*yaml.Node, error) {
|
||||
content := string(tomlNode.Data)
|
||||
return createScalarNode(content == "true", content), nil
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) createIntegerScalar(tomlNode *toml.Node) (*yaml.Node, error) {
|
||||
content := string(tomlNode.Data)
|
||||
_, num, err := parseInt64(content)
|
||||
return createScalarNode(num, content), err
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) createDateTimeScalar(tomlNode *toml.Node) (*yaml.Node, error) {
|
||||
content := string(tomlNode.Data)
|
||||
val, err := parseDateTime(time.RFC3339, content)
|
||||
return createScalarNode(val, content), err
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) createFloatScalar(tomlNode *toml.Node) (*yaml.Node, error) {
|
||||
content := string(tomlNode.Data)
|
||||
num, err := strconv.ParseFloat(content, 64)
|
||||
return createScalarNode(num, content), err
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) decodeNode(tomlNode *toml.Node) (*yaml.Node, error) {
|
||||
switch tomlNode.Kind {
|
||||
case toml.Key, toml.String:
|
||||
return dec.createStringScalar(tomlNode)
|
||||
case toml.Bool:
|
||||
return dec.createBoolScalar(tomlNode)
|
||||
case toml.Integer:
|
||||
return dec.createIntegerScalar(tomlNode)
|
||||
case toml.DateTime:
|
||||
return dec.createDateTimeScalar(tomlNode)
|
||||
case toml.Float:
|
||||
return dec.createFloatScalar(tomlNode)
|
||||
case toml.Array:
|
||||
return dec.createArray(tomlNode)
|
||||
case toml.InlineTable:
|
||||
return dec.createInlineTableMap(tomlNode)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported type %v", tomlNode.Kind)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) Decode() (*CandidateNode, error) {
|
||||
if dec.finished {
|
||||
return nil, io.EOF
|
||||
}
|
||||
//
|
||||
// toml library likes to panic
|
||||
var deferredError error
|
||||
defer func() { //catch or finally
|
||||
if r := recover(); r != nil {
|
||||
var ok bool
|
||||
deferredError, ok = r.(error)
|
||||
if !ok {
|
||||
deferredError = fmt.Errorf("pkg: %v", r)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
log.Debug("ok here we go")
|
||||
var runAgainstCurrentExp = false
|
||||
var err error
|
||||
for runAgainstCurrentExp || dec.parser.NextExpression() {
|
||||
|
||||
if runAgainstCurrentExp {
|
||||
log.Debug("running against current exp")
|
||||
}
|
||||
|
||||
currentNode := dec.parser.Expression()
|
||||
|
||||
log.Debug("currentNode: %v ", currentNode.Kind)
|
||||
runAgainstCurrentExp, err = dec.processTopLevelNode(currentNode)
|
||||
if err != nil {
|
||||
return dec.rootMap, err
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
err = dec.parser.Error()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// must have finished
|
||||
dec.finished = true
|
||||
|
||||
if len(dec.rootMap.Node.Content) == 0 {
|
||||
return nil, io.EOF
|
||||
}
|
||||
|
||||
return &CandidateNode{
|
||||
Node: &yaml.Node{
|
||||
Kind: yaml.DocumentNode,
|
||||
Content: []*yaml.Node{dec.rootMap.Node},
|
||||
},
|
||||
}, deferredError
|
||||
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) processTopLevelNode(currentNode *toml.Node) (bool, error) {
|
||||
var runAgainstCurrentExp bool
|
||||
var err error
|
||||
log.Debug("!!!!!!!!!!!!Going to process %v state is current %v", currentNode.Kind, NodeToString(dec.rootMap))
|
||||
if currentNode.Kind == toml.Table {
|
||||
runAgainstCurrentExp, err = dec.processTable((currentNode))
|
||||
} else if currentNode.Kind == toml.ArrayTable {
|
||||
runAgainstCurrentExp, err = dec.processArrayTable((currentNode))
|
||||
} else {
|
||||
runAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(dec.rootMap, currentNode)
|
||||
}
|
||||
|
||||
log.Debug("!!!!!!!!!!!!DONE Processing state is now %v", NodeToString(dec.rootMap))
|
||||
return runAgainstCurrentExp, err
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) processTable(currentNode *toml.Node) (bool, error) {
|
||||
log.Debug("!!! processing table")
|
||||
fullPath := dec.getFullPath(currentNode.Child())
|
||||
log.Debug("!!!fullpath: %v", fullPath)
|
||||
|
||||
hasValue := dec.parser.NextExpression()
|
||||
if !hasValue {
|
||||
return false, fmt.Errorf("error retrieving table %v value: %w", fullPath, dec.parser.Error())
|
||||
}
|
||||
|
||||
tableNodeValue := &CandidateNode{
|
||||
Node: &yaml.Node{
|
||||
Kind: yaml.MappingNode,
|
||||
Tag: "!!map",
|
||||
},
|
||||
}
|
||||
|
||||
tableValue := dec.parser.Expression()
|
||||
if tableValue.Kind != toml.KeyValue {
|
||||
log.Debug("got an empty table, returning")
|
||||
return true, nil
|
||||
}
|
||||
|
||||
runAgainstCurrentExp, err := dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue)
|
||||
log.Debugf("table node err: %w", err)
|
||||
if err != nil && !errors.Is(io.EOF, err) {
|
||||
return false, err
|
||||
}
|
||||
c := Context{}
|
||||
|
||||
c = c.SingleChildContext(dec.rootMap)
|
||||
err = dec.d.DeeplyAssign(c, fullPath, tableNodeValue.Node)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return runAgainstCurrentExp, nil
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) arrayAppend(context Context, path []interface{}, rhsNode *yaml.Node) error {
|
||||
rhsCandidateNode := &CandidateNode{
|
||||
Path: path,
|
||||
Node: &yaml.Node{
|
||||
Kind: yaml.SequenceNode,
|
||||
Tag: "!!seq",
|
||||
Content: []*yaml.Node{rhsNode},
|
||||
},
|
||||
}
|
||||
|
||||
assignmentOp := &Operation{OperationType: addAssignOpType}
|
||||
|
||||
rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}
|
||||
|
||||
assignmentOpNode := &ExpressionNode{
|
||||
Operation: assignmentOp,
|
||||
LHS: createTraversalTree(path, traversePreferences{}, false),
|
||||
RHS: &ExpressionNode{Operation: rhsOp},
|
||||
}
|
||||
|
||||
_, err := dec.d.GetMatchingNodes(context, assignmentOpNode)
|
||||
return err
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) processArrayTable(currentNode *toml.Node) (bool, error) {
|
||||
log.Debug("!!! processing table")
|
||||
fullPath := dec.getFullPath(currentNode.Child())
|
||||
log.Debug("!!!fullpath: %v", fullPath)
|
||||
|
||||
// need to use the array append exp to add another entry to
|
||||
// this array: fullpath += [ thing ]
|
||||
|
||||
hasValue := dec.parser.NextExpression()
|
||||
if !hasValue {
|
||||
return false, fmt.Errorf("error retrieving table %v value: %w", fullPath, dec.parser.Error())
|
||||
}
|
||||
|
||||
tableNodeValue := &CandidateNode{
|
||||
Node: &yaml.Node{
|
||||
Kind: yaml.MappingNode,
|
||||
Tag: "!!map",
|
||||
},
|
||||
}
|
||||
|
||||
tableValue := dec.parser.Expression()
|
||||
runAgainstCurrentExp, err := dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue)
|
||||
log.Debugf("table node err: %w", err)
|
||||
if err != nil && !errors.Is(io.EOF, err) {
|
||||
return false, err
|
||||
}
|
||||
c := Context{}
|
||||
|
||||
c = c.SingleChildContext(dec.rootMap)
|
||||
|
||||
// += function
|
||||
err = dec.arrayAppend(c, fullPath, tableNodeValue.Node)
|
||||
|
||||
return runAgainstCurrentExp, err
|
||||
}
|
||||
4
pkg/yqlib/doc/usage/headers/toml.md
Normal file
4
pkg/yqlib/doc/usage/headers/toml.md
Normal file
@ -0,0 +1,4 @@
|
||||
# TOML
|
||||
|
||||
Decode from TOML. Note that `yq` does not yet support outputting in TOML format (and therefore it cannot roundtrip)
|
||||
|
||||
111
pkg/yqlib/doc/usage/toml.md
Normal file
111
pkg/yqlib/doc/usage/toml.md
Normal file
@ -0,0 +1,111 @@
|
||||
# TOML
|
||||
|
||||
Decode from TOML. Note that `yq` does not yet support outputting in TOML format (and therefore it cannot roundtrip)
|
||||
|
||||
|
||||
## Parse: Simple
|
||||
Given a sample.toml file of:
|
||||
```toml
|
||||
A = "hello"
|
||||
B = 12
|
||||
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq -oy '.' sample.toml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
A: hello
|
||||
B: 12
|
||||
```
|
||||
|
||||
## Parse: Deep paths
|
||||
Given a sample.toml file of:
|
||||
```toml
|
||||
person.name = "hello"
|
||||
person.address = "12 cat st"
|
||||
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq -oy '.' sample.toml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
person:
|
||||
name: hello
|
||||
address: 12 cat st
|
||||
```
|
||||
|
||||
## Parse: inline table
|
||||
Given a sample.toml file of:
|
||||
```toml
|
||||
name = { first = "Tom", last = "Preston-Werner" }
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq -oy '.' sample.toml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
name:
|
||||
first: Tom
|
||||
last: Preston-Werner
|
||||
```
|
||||
|
||||
## Parse: Array Table
|
||||
Given a sample.toml file of:
|
||||
```toml
|
||||
|
||||
[owner.contact]
|
||||
name = "Tom Preston-Werner"
|
||||
age = 36
|
||||
|
||||
[[owner.addresses]]
|
||||
street = "first street"
|
||||
suburb = "ok"
|
||||
|
||||
[[owner.addresses]]
|
||||
street = "second street"
|
||||
suburb = "nice"
|
||||
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq -oy '.' sample.toml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
owner:
|
||||
contact:
|
||||
name: Tom Preston-Werner
|
||||
age: 36
|
||||
addresses:
|
||||
- street: first street
|
||||
suburb: ok
|
||||
- street: second street
|
||||
suburb: nice
|
||||
```
|
||||
|
||||
## Parse: with header
|
||||
Given a sample.toml file of:
|
||||
```toml
|
||||
|
||||
[servers]
|
||||
|
||||
[servers.alpha]
|
||||
ip = "10.0.0.1"
|
||||
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq -oy '.' sample.toml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
servers:
|
||||
alpha:
|
||||
ip: 10.0.0.1
|
||||
```
|
||||
|
||||
7
pkg/yqlib/no_toml.go
Normal file
7
pkg/yqlib/no_toml.go
Normal file
@ -0,0 +1,7 @@
|
||||
//go:build yq_notoml
|
||||
|
||||
package yqlib
|
||||
|
||||
func NewTomlDecoder() Decoder {
|
||||
return nil
|
||||
}
|
||||
239
pkg/yqlib/toml_test.go
Normal file
239
pkg/yqlib/toml_test.go
Normal file
@ -0,0 +1,239 @@
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/mikefarah/yq/v4/test"
|
||||
)
|
||||
|
||||
var sampleTable = `
|
||||
var = "x"
|
||||
|
||||
[owner.contact]
|
||||
name = "Tom Preston-Werner"
|
||||
age = 36
|
||||
`
|
||||
|
||||
var sampleTableExpected = `var: x
|
||||
owner:
|
||||
contact:
|
||||
name: Tom Preston-Werner
|
||||
age: 36
|
||||
`
|
||||
|
||||
var sampleArrayTable = `
|
||||
[owner.contact]
|
||||
name = "Tom Preston-Werner"
|
||||
age = 36
|
||||
|
||||
[[owner.addresses]]
|
||||
street = "first street"
|
||||
suburb = "ok"
|
||||
|
||||
[[owner.addresses]]
|
||||
street = "second street"
|
||||
suburb = "nice"
|
||||
`
|
||||
|
||||
var sampleArrayTableExpected = `owner:
|
||||
contact:
|
||||
name: Tom Preston-Werner
|
||||
age: 36
|
||||
addresses:
|
||||
- street: first street
|
||||
suburb: ok
|
||||
- street: second street
|
||||
suburb: nice
|
||||
`
|
||||
|
||||
var sampleWithHeader = `
|
||||
[servers]
|
||||
|
||||
[servers.alpha]
|
||||
ip = "10.0.0.1"
|
||||
`
|
||||
|
||||
var expectedSampleWithHeader = `servers:
|
||||
alpha:
|
||||
ip: 10.0.0.1
|
||||
`
|
||||
|
||||
var tomlScenarios = []formatScenario{
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "blank",
|
||||
input: "",
|
||||
expected: "",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "datetime",
|
||||
input: "A = 1979-05-27T07:32:00-08:00",
|
||||
expected: "A: 1979-05-27T07:32:00-08:00\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "blank",
|
||||
input: `A = "hello`,
|
||||
expectedError: `bad file 'sample.yml': basic string not terminated by "`,
|
||||
scenarioType: "decode-error",
|
||||
},
|
||||
{
|
||||
description: "Parse: Simple",
|
||||
input: "A = \"hello\"\nB = 12\n",
|
||||
expected: "A: hello\nB: 12\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "Parse: Deep paths",
|
||||
input: "person.name = \"hello\"\nperson.address = \"12 cat st\"\n",
|
||||
expected: "person:\n name: hello\n address: 12 cat st\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
input: `A.B = "hello"`,
|
||||
expected: "A:\n B: hello\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "bool",
|
||||
input: `A = true`,
|
||||
expected: "A: true\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "bool false",
|
||||
input: `A = false `,
|
||||
expected: "A: false\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "number",
|
||||
input: `A = 3 `,
|
||||
expected: "A: 3\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "number",
|
||||
input: `A = 0xDEADBEEF`,
|
||||
expression: " .A += 1",
|
||||
expected: "A: 0xDEADBEF0\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "float",
|
||||
input: `A = 6.626e-34`,
|
||||
expected: "A: 6.626e-34\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "empty arraY",
|
||||
input: `A = []`,
|
||||
expected: "A: []\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "array",
|
||||
input: `A = ["hello", ["world", "again"]]`,
|
||||
expected: "A:\n - hello\n - - world\n - again\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "Parse: inline table",
|
||||
input: `name = { first = "Tom", last = "Preston-Werner" }`,
|
||||
expected: "name:\n first: Tom\n last: Preston-Werner\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
input: sampleTable,
|
||||
expected: sampleTableExpected,
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "Parse: Array Table",
|
||||
input: sampleArrayTable,
|
||||
expected: sampleArrayTableExpected,
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "Parse: with header",
|
||||
input: sampleWithHeader,
|
||||
expected: expectedSampleWithHeader,
|
||||
scenarioType: "decode",
|
||||
},
|
||||
}
|
||||
|
||||
func testTomlScenario(t *testing.T, s formatScenario) {
|
||||
switch s.scenarioType {
|
||||
case "", "decode":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewTomlDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
|
||||
case "decode-error":
|
||||
result, err := processFormatScenario(s, NewTomlDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences))
|
||||
if err == nil {
|
||||
t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result)
|
||||
} else {
|
||||
test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func documentTomlDecodeScenario(w *bufio.Writer, s formatScenario) {
|
||||
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
|
||||
|
||||
if s.subdescription != "" {
|
||||
writeOrPanic(w, s.subdescription)
|
||||
writeOrPanic(w, "\n\n")
|
||||
}
|
||||
|
||||
writeOrPanic(w, "Given a sample.toml file of:\n")
|
||||
writeOrPanic(w, fmt.Sprintf("```toml\n%v\n```\n", s.input))
|
||||
|
||||
writeOrPanic(w, "then\n")
|
||||
expression := s.expression
|
||||
if expression == "" {
|
||||
expression = "."
|
||||
}
|
||||
writeOrPanic(w, fmt.Sprintf("```bash\nyq -oy '%v' sample.toml\n```\n", expression))
|
||||
writeOrPanic(w, "will output\n")
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewTomlDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences))))
|
||||
}
|
||||
|
||||
func documentTomlScenario(t *testing.T, w *bufio.Writer, i interface{}) {
|
||||
s := i.(formatScenario)
|
||||
|
||||
if s.skipDoc {
|
||||
return
|
||||
}
|
||||
switch s.scenarioType {
|
||||
case "", "decode":
|
||||
documentTomlDecodeScenario(w, s)
|
||||
|
||||
default:
|
||||
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTomlScenarios(t *testing.T) {
|
||||
for _, tt := range tomlScenarios {
|
||||
testTomlScenario(t, tt)
|
||||
}
|
||||
genericScenarios := make([]interface{}, len(tomlScenarios))
|
||||
for i, s := range tomlScenarios {
|
||||
genericScenarios[i] = s
|
||||
}
|
||||
documentScenarios(t, "usage", "toml", genericScenarios, documentTomlScenario)
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user