Draft: Toml (#1439)

* toml wip

* wip

* Fixed auto parsing toml

* Added build flag not to include toml

* Parse toml docs and tests

* minor updates
This commit is contained in:
Mike Farah 2023-03-26 00:59:15 +01:00 committed by GitHub
parent 47f4ddc910
commit 7103b78d38
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 828 additions and 23 deletions

2
.gitignore vendored
View File

@ -41,6 +41,8 @@ yq*.snap
test.yml test.yml
test*.yml test*.yml
test*.xml
test*.toml
test*.yaml test*.yaml
0.yml 0.yml
1.yml 1.yml

View File

@ -2,6 +2,7 @@
setUp() { setUp() {
rm test*.yml 2>/dev/null || true rm test*.yml 2>/dev/null || true
rm test*.toml 2>/dev/null || true
rm test*.tfstate 2>/dev/null || true rm test*.tfstate 2>/dev/null || true
rm test*.json 2>/dev/null || true rm test*.json 2>/dev/null || true
rm test*.properties 2>/dev/null || true rm test*.properties 2>/dev/null || true
@ -30,6 +31,26 @@ EOM
assertEquals "$expected" "$X" assertEquals "$expected" "$X"
} }
testInputToml() {
cat >test.toml <<EOL
[owner]
name = "Tom Preston-Werner"
dob = 1979-05-27T07:32:00-08:00
EOL
read -r -d '' expected << EOM
owner:
name: Tom Preston-Werner
dob: 1979-05-27T07:32:00-08:00
EOM
X=$(./yq -oy test.toml)
assertEquals "$expected" "$X"
X=$(./yq ea -oy test.toml)
assertEquals "$expected" "$X"
}
testInputTfstate() { testInputTfstate() {
cat >test.tfstate <<EOL cat >test.tfstate <<EOL
{ "mike" : { "things": "cool" } } { "mike" : { "things": "cool" } }

View File

@ -65,8 +65,8 @@ yq -P sample.json
panic(err) panic(err)
} }
rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "auto", "[auto|a|yaml|y|json|j|props|p|xml|x] output format type.") rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "auto", "[auto|a|yaml|y|json|j|props|p|xml|x|tsv|t|csv|c] output format type.")
rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "auto", "[auto|a|yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.") rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "auto", "[auto|a|yaml|y|props|p|xml|x|tsv|t|csv|c|toml] parse format for input. Note that json is a subset of yaml.")
rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", yqlib.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes") rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", yqlib.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes")
rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", yqlib.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).") rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", yqlib.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).")

View File

@ -75,6 +75,9 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
} }
} else if isAutomaticOutputFormat() { } else if isAutomaticOutputFormat() {
// automatic input worked, we can do it for output too unless specified // automatic input worked, we can do it for output too unless specified
if inputFormat == "toml" {
return "", nil, fmt.Errorf("toml is not yet supported as an output format. Please specify another output format using the [--output-format/-o] flag")
}
outputFormat = inputFormat outputFormat = inputFormat
} }
} else if isAutomaticOutputFormat() { } else if isAutomaticOutputFormat() {
@ -137,6 +140,8 @@ func createDecoder(format yqlib.InputFormat, evaluateTogether bool) (yqlib.Decod
return yqlib.NewCSVObjectDecoder(','), nil return yqlib.NewCSVObjectDecoder(','), nil
case yqlib.TSVObjectInputFormat: case yqlib.TSVObjectInputFormat:
return yqlib.NewCSVObjectDecoder('\t'), nil return yqlib.NewCSVObjectDecoder('\t'), nil
case yqlib.TomlInputFormat:
return yqlib.NewTomlDecoder(), nil
case yqlib.YamlInputFormat: case yqlib.YamlInputFormat:
prefs := yqlib.ConfiguredYamlPreferences prefs := yqlib.ConfiguredYamlPreferences
prefs.EvaluateTogether = evaluateTogether prefs.EvaluateTogether = evaluateTogether

26
examples/sample.toml Normal file
View File

@ -0,0 +1,26 @@
# This is a TOML document
title = "TOML Example"
[owner]
name = "Tom Preston-Werner"
dob = 1979-05-27T07:32:00-08:00
[database]
enabled = true
ports = [ 8000, 8001, 8002 ]
data = [ ["delta", "phi"], [3.14] ]
temp_targets = { cpu = 79.5, case = 72.0 }
[servers]
[servers.alpha]
ip = "10.0.0.1"
role = "frontend"
[servers.beta]
ip = "10.0.0.2"
role = "backend"

1
go.mod
View File

@ -11,6 +11,7 @@ require (
github.com/goccy/go-yaml v1.10.0 github.com/goccy/go-yaml v1.10.0
github.com/jinzhu/copier v0.3.5 github.com/jinzhu/copier v0.3.5
github.com/magiconair/properties v1.8.7 github.com/magiconair/properties v1.8.7
github.com/pelletier/go-toml/v2 v2.0.6
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
github.com/spf13/cobra v1.6.1 github.com/spf13/cobra v1.6.1
github.com/spf13/pflag v1.0.5 github.com/spf13/pflag v1.0.5

12
go.sum
View File

@ -6,8 +6,9 @@ github.com/alecthomas/participle/v2 v2.0.0-beta.5/go.mod h1:RC764t6n4L8D8ITAJv0q
github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk= github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk=
github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
github.com/elliotchance/orderedmap v1.5.0 h1:1IsExUsjv5XNBD3ZdC7jkAAqLWOOKdbPTmkHx63OsBg= github.com/elliotchance/orderedmap v1.5.0 h1:1IsExUsjv5XNBD3ZdC7jkAAqLWOOKdbPTmkHx63OsBg=
@ -36,6 +37,8 @@ github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU=
github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@ -49,9 +52,14 @@ github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUq
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=

View File

@ -4,6 +4,7 @@ import (
"fmt" "fmt"
logging "gopkg.in/op/go-logging.v1" logging "gopkg.in/op/go-logging.v1"
yaml "gopkg.in/yaml.v3"
) )
type DataTreeNavigator interface { type DataTreeNavigator interface {
@ -11,6 +12,8 @@ type DataTreeNavigator interface {
// this will process the against the given expressionNode and return // this will process the against the given expressionNode and return
// a new context of matching candidates // a new context of matching candidates
GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error) GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error)
DeeplyAssign(context Context, path []interface{}, rhsNode *yaml.Node) error
} }
type dataTreeNavigator struct { type dataTreeNavigator struct {
@ -20,6 +23,27 @@ func NewDataTreeNavigator() DataTreeNavigator {
return &dataTreeNavigator{} return &dataTreeNavigator{}
} }
func (d *dataTreeNavigator) DeeplyAssign(context Context, path []interface{}, rhsNode *yaml.Node) error {
rhsCandidateNode := &CandidateNode{
Path: path,
Node: rhsNode,
}
assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}
assignmentOpNode := &ExpressionNode{
Operation: assignmentOp,
LHS: createTraversalTree(path, traversePreferences{}, false),
RHS: &ExpressionNode{Operation: rhsOp},
}
_, err := d.GetMatchingNodes(context, assignmentOpNode)
return err
}
func (d *dataTreeNavigator) GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error) { func (d *dataTreeNavigator) GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error) {
if expressionNode == nil { if expressionNode == nil {
log.Debugf("getMatchingNodes - nothing to do") log.Debugf("getMatchingNodes - nothing to do")

View File

@ -16,6 +16,7 @@ const (
JsonInputFormat JsonInputFormat
CSVObjectInputFormat CSVObjectInputFormat
TSVObjectInputFormat TSVObjectInputFormat
TomlInputFormat
UriInputFormat UriInputFormat
) )
@ -38,8 +39,10 @@ func InputFormatFromString(format string) (InputFormat, error) {
return CSVObjectInputFormat, nil return CSVObjectInputFormat, nil
case "tsv", "t": case "tsv", "t":
return TSVObjectInputFormat, nil return TSVObjectInputFormat, nil
case "toml":
return TomlInputFormat, nil
default: default:
return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml]", format) return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml|toml]", format)
} }
} }

View File

@ -95,23 +95,7 @@ func (dec *propertiesDecoder) applyProperty(context Context, properties *propert
rhsNode.Tag = guessTagFromCustomType(rhsNode) rhsNode.Tag = guessTagFromCustomType(rhsNode)
rhsCandidateNode := &CandidateNode{ return dec.d.DeeplyAssign(context, path, rhsNode)
Path: path,
Node: rhsNode,
}
assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
rhsOp := &Operation{OperationType: referenceOpType, CandidateNode: rhsCandidateNode}
assignmentOpNode := &ExpressionNode{
Operation: assignmentOp,
LHS: createTraversalTree(path, traversePreferences{}, false),
RHS: &ExpressionNode{Operation: rhsOp},
}
_, err := dec.d.GetMatchingNodes(context, assignmentOpNode)
return err
} }
func (dec *propertiesDecoder) Decode() (*CandidateNode, error) { func (dec *propertiesDecoder) Decode() (*CandidateNode, error) {

View File

@ -3,6 +3,7 @@ package yqlib
import ( import (
"bufio" "bufio"
"bytes" "bytes"
"fmt"
"strings" "strings"
) )
@ -62,7 +63,7 @@ func mustProcessFormatScenario(s formatScenario, decoder Decoder, encoder Encode
result, err := processFormatScenario(s, decoder, encoder) result, err := processFormatScenario(s, decoder, encoder)
if err != nil { if err != nil {
panic(err) panic(fmt.Errorf("Bad scenario %v: %w", s.description, err))
} }
return result return result

369
pkg/yqlib/decoder_toml.go Normal file
View File

@ -0,0 +1,369 @@
//go:build !yq_notoml
package yqlib
import (
"bytes"
"errors"
"fmt"
"io"
"strconv"
"time"
toml "github.com/pelletier/go-toml/v2/unstable"
yaml "gopkg.in/yaml.v3"
)
type tomlDecoder struct {
parser toml.Parser
finished bool
d DataTreeNavigator
rootMap *CandidateNode
}
func NewTomlDecoder() Decoder {
return &tomlDecoder{
finished: false,
d: NewDataTreeNavigator(),
}
}
func (dec *tomlDecoder) Init(reader io.Reader) error {
dec.parser = toml.Parser{}
buf := new(bytes.Buffer)
_, err := buf.ReadFrom(reader)
if err != nil {
return err
}
dec.parser.Reset(buf.Bytes())
dec.rootMap = &CandidateNode{
Node: &yaml.Node{
Kind: yaml.MappingNode,
Tag: "!!map",
}}
return nil
}
func (dec *tomlDecoder) getFullPath(tomlNode *toml.Node) []interface{} {
path := make([]interface{}, 0)
for {
path = append(path, string(tomlNode.Data))
tomlNode = tomlNode.Next()
if tomlNode == nil {
return path
}
}
}
func (dec *tomlDecoder) processKeyValueIntoMap(rootMap *CandidateNode, tomlNode *toml.Node) error {
value := tomlNode.Value()
path := dec.getFullPath(value.Next())
log.Debug("!!!processKeyValueIntoMap: %v", path)
valueNode, err := dec.decodeNode(value)
if err != nil {
return err
}
context := Context{}
context = context.SingleChildContext(rootMap)
return dec.d.DeeplyAssign(context, path, valueNode)
}
func (dec *tomlDecoder) decodeKeyValuesIntoMap(rootMap *CandidateNode, tomlNode *toml.Node) (bool, error) {
log.Debug("!! DECODE_KV_INTO_MAP -- processing first (current) entry")
if err := dec.processKeyValueIntoMap(rootMap, tomlNode); err != nil {
return false, err
}
for dec.parser.NextExpression() {
nextItem := dec.parser.Expression()
log.Debug("!! DECODE_KV_INTO_MAP -- next exp, its a %v", nextItem.Kind)
if nextItem.Kind == toml.KeyValue {
if err := dec.processKeyValueIntoMap(rootMap, nextItem); err != nil {
return false, err
}
} else {
// run out of key values
log.Debug("! DECODE_KV_INTO_MAP - ok we are done in decodeKeyValuesIntoMap, gota a %v", nextItem.Kind)
log.Debug("! DECODE_KV_INTO_MAP - processAgainstCurrentExp = true!")
return true, nil
}
}
log.Debug("! DECODE_KV_INTO_MAP - no more things to read in")
return false, nil
}
func (dec *tomlDecoder) createInlineTableMap(tomlNode *toml.Node) (*yaml.Node, error) {
content := make([]*yaml.Node, 0)
log.Debug("!! createInlineTableMap")
iterator := tomlNode.Children()
for iterator.Next() {
child := iterator.Node()
if child.Kind != toml.KeyValue {
return nil, fmt.Errorf("only keyvalue pairs are supported in inlinetables, got %v instead", child.Kind)
}
keyValues := &CandidateNode{
Node: &yaml.Node{
Kind: yaml.MappingNode,
Tag: "!!map",
},
}
if err := dec.processKeyValueIntoMap(keyValues, child); err != nil {
return nil, err
}
content = append(content, keyValues.Node.Content...)
}
return &yaml.Node{
Kind: yaml.MappingNode,
Tag: "!!map",
Content: content,
}, nil
}
func (dec *tomlDecoder) createArray(tomlNode *toml.Node) (*yaml.Node, error) {
content := make([]*yaml.Node, 0)
iterator := tomlNode.Children()
for iterator.Next() {
child := iterator.Node()
yamlNode, err := dec.decodeNode(child)
if err != nil {
return nil, err
}
content = append(content, yamlNode)
}
return &yaml.Node{
Kind: yaml.SequenceNode,
Tag: "!!seq",
Content: content,
}, nil
}
func (dec *tomlDecoder) createStringScalar(tomlNode *toml.Node) (*yaml.Node, error) {
content := string(tomlNode.Data)
return createScalarNode(content, content), nil
}
func (dec *tomlDecoder) createBoolScalar(tomlNode *toml.Node) (*yaml.Node, error) {
content := string(tomlNode.Data)
return createScalarNode(content == "true", content), nil
}
func (dec *tomlDecoder) createIntegerScalar(tomlNode *toml.Node) (*yaml.Node, error) {
content := string(tomlNode.Data)
_, num, err := parseInt64(content)
return createScalarNode(num, content), err
}
func (dec *tomlDecoder) createDateTimeScalar(tomlNode *toml.Node) (*yaml.Node, error) {
content := string(tomlNode.Data)
val, err := parseDateTime(time.RFC3339, content)
return createScalarNode(val, content), err
}
func (dec *tomlDecoder) createFloatScalar(tomlNode *toml.Node) (*yaml.Node, error) {
content := string(tomlNode.Data)
num, err := strconv.ParseFloat(content, 64)
return createScalarNode(num, content), err
}
func (dec *tomlDecoder) decodeNode(tomlNode *toml.Node) (*yaml.Node, error) {
switch tomlNode.Kind {
case toml.Key, toml.String:
return dec.createStringScalar(tomlNode)
case toml.Bool:
return dec.createBoolScalar(tomlNode)
case toml.Integer:
return dec.createIntegerScalar(tomlNode)
case toml.DateTime:
return dec.createDateTimeScalar(tomlNode)
case toml.Float:
return dec.createFloatScalar(tomlNode)
case toml.Array:
return dec.createArray(tomlNode)
case toml.InlineTable:
return dec.createInlineTableMap(tomlNode)
default:
return nil, fmt.Errorf("unsupported type %v", tomlNode.Kind)
}
}
func (dec *tomlDecoder) Decode() (*CandidateNode, error) {
if dec.finished {
return nil, io.EOF
}
//
// toml library likes to panic
var deferredError error
defer func() { //catch or finally
if r := recover(); r != nil {
var ok bool
deferredError, ok = r.(error)
if !ok {
deferredError = fmt.Errorf("pkg: %v", r)
}
}
}()
log.Debug("ok here we go")
var runAgainstCurrentExp = false
var err error
for runAgainstCurrentExp || dec.parser.NextExpression() {
if runAgainstCurrentExp {
log.Debug("running against current exp")
}
currentNode := dec.parser.Expression()
log.Debug("currentNode: %v ", currentNode.Kind)
runAgainstCurrentExp, err = dec.processTopLevelNode(currentNode)
if err != nil {
return dec.rootMap, err
}
}
err = dec.parser.Error()
if err != nil {
return nil, err
}
// must have finished
dec.finished = true
if len(dec.rootMap.Node.Content) == 0 {
return nil, io.EOF
}
return &CandidateNode{
Node: &yaml.Node{
Kind: yaml.DocumentNode,
Content: []*yaml.Node{dec.rootMap.Node},
},
}, deferredError
}
func (dec *tomlDecoder) processTopLevelNode(currentNode *toml.Node) (bool, error) {
var runAgainstCurrentExp bool
var err error
log.Debug("!!!!!!!!!!!!Going to process %v state is current %v", currentNode.Kind, NodeToString(dec.rootMap))
if currentNode.Kind == toml.Table {
runAgainstCurrentExp, err = dec.processTable((currentNode))
} else if currentNode.Kind == toml.ArrayTable {
runAgainstCurrentExp, err = dec.processArrayTable((currentNode))
} else {
runAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(dec.rootMap, currentNode)
}
log.Debug("!!!!!!!!!!!!DONE Processing state is now %v", NodeToString(dec.rootMap))
return runAgainstCurrentExp, err
}
func (dec *tomlDecoder) processTable(currentNode *toml.Node) (bool, error) {
log.Debug("!!! processing table")
fullPath := dec.getFullPath(currentNode.Child())
log.Debug("!!!fullpath: %v", fullPath)
hasValue := dec.parser.NextExpression()
if !hasValue {
return false, fmt.Errorf("error retrieving table %v value: %w", fullPath, dec.parser.Error())
}
tableNodeValue := &CandidateNode{
Node: &yaml.Node{
Kind: yaml.MappingNode,
Tag: "!!map",
},
}
tableValue := dec.parser.Expression()
if tableValue.Kind != toml.KeyValue {
log.Debug("got an empty table, returning")
return true, nil
}
runAgainstCurrentExp, err := dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue)
log.Debugf("table node err: %w", err)
if err != nil && !errors.Is(io.EOF, err) {
return false, err
}
c := Context{}
c = c.SingleChildContext(dec.rootMap)
err = dec.d.DeeplyAssign(c, fullPath, tableNodeValue.Node)
if err != nil {
return false, err
}
return runAgainstCurrentExp, nil
}
func (dec *tomlDecoder) arrayAppend(context Context, path []interface{}, rhsNode *yaml.Node) error {
rhsCandidateNode := &CandidateNode{
Path: path,
Node: &yaml.Node{
Kind: yaml.SequenceNode,
Tag: "!!seq",
Content: []*yaml.Node{rhsNode},
},
}
assignmentOp := &Operation{OperationType: addAssignOpType}
rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}
assignmentOpNode := &ExpressionNode{
Operation: assignmentOp,
LHS: createTraversalTree(path, traversePreferences{}, false),
RHS: &ExpressionNode{Operation: rhsOp},
}
_, err := dec.d.GetMatchingNodes(context, assignmentOpNode)
return err
}
func (dec *tomlDecoder) processArrayTable(currentNode *toml.Node) (bool, error) {
log.Debug("!!! processing table")
fullPath := dec.getFullPath(currentNode.Child())
log.Debug("!!!fullpath: %v", fullPath)
// need to use the array append exp to add another entry to
// this array: fullpath += [ thing ]
hasValue := dec.parser.NextExpression()
if !hasValue {
return false, fmt.Errorf("error retrieving table %v value: %w", fullPath, dec.parser.Error())
}
tableNodeValue := &CandidateNode{
Node: &yaml.Node{
Kind: yaml.MappingNode,
Tag: "!!map",
},
}
tableValue := dec.parser.Expression()
runAgainstCurrentExp, err := dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue)
log.Debugf("table node err: %w", err)
if err != nil && !errors.Is(io.EOF, err) {
return false, err
}
c := Context{}
c = c.SingleChildContext(dec.rootMap)
// += function
err = dec.arrayAppend(c, fullPath, tableNodeValue.Node)
return runAgainstCurrentExp, err
}

View File

@ -0,0 +1,4 @@
# TOML
Decode from TOML. Note that `yq` does not yet support outputting in TOML format (and therefore it cannot roundtrip)

111
pkg/yqlib/doc/usage/toml.md Normal file
View File

@ -0,0 +1,111 @@
# TOML
Decode from TOML. Note that `yq` does not yet support outputting in TOML format (and therefore it cannot roundtrip)
## Parse: Simple
Given a sample.toml file of:
```toml
A = "hello"
B = 12
```
then
```bash
yq -oy '.' sample.toml
```
will output
```yaml
A: hello
B: 12
```
## Parse: Deep paths
Given a sample.toml file of:
```toml
person.name = "hello"
person.address = "12 cat st"
```
then
```bash
yq -oy '.' sample.toml
```
will output
```yaml
person:
name: hello
address: 12 cat st
```
## Parse: inline table
Given a sample.toml file of:
```toml
name = { first = "Tom", last = "Preston-Werner" }
```
then
```bash
yq -oy '.' sample.toml
```
will output
```yaml
name:
first: Tom
last: Preston-Werner
```
## Parse: Array Table
Given a sample.toml file of:
```toml
[owner.contact]
name = "Tom Preston-Werner"
age = 36
[[owner.addresses]]
street = "first street"
suburb = "ok"
[[owner.addresses]]
street = "second street"
suburb = "nice"
```
then
```bash
yq -oy '.' sample.toml
```
will output
```yaml
owner:
contact:
name: Tom Preston-Werner
age: 36
addresses:
- street: first street
suburb: ok
- street: second street
suburb: nice
```
## Parse: with header
Given a sample.toml file of:
```toml
[servers]
[servers.alpha]
ip = "10.0.0.1"
```
then
```bash
yq -oy '.' sample.toml
```
will output
```yaml
servers:
alpha:
ip: 10.0.0.1
```

7
pkg/yqlib/no_toml.go Normal file
View File

@ -0,0 +1,7 @@
//go:build yq_notoml
package yqlib
func NewTomlDecoder() Decoder {
return nil
}

239
pkg/yqlib/toml_test.go Normal file
View File

@ -0,0 +1,239 @@
package yqlib
import (
"bufio"
"fmt"
"testing"
"github.com/mikefarah/yq/v4/test"
)
var sampleTable = `
var = "x"
[owner.contact]
name = "Tom Preston-Werner"
age = 36
`
var sampleTableExpected = `var: x
owner:
contact:
name: Tom Preston-Werner
age: 36
`
var sampleArrayTable = `
[owner.contact]
name = "Tom Preston-Werner"
age = 36
[[owner.addresses]]
street = "first street"
suburb = "ok"
[[owner.addresses]]
street = "second street"
suburb = "nice"
`
var sampleArrayTableExpected = `owner:
contact:
name: Tom Preston-Werner
age: 36
addresses:
- street: first street
suburb: ok
- street: second street
suburb: nice
`
var sampleWithHeader = `
[servers]
[servers.alpha]
ip = "10.0.0.1"
`
var expectedSampleWithHeader = `servers:
alpha:
ip: 10.0.0.1
`
var tomlScenarios = []formatScenario{
{
skipDoc: true,
description: "blank",
input: "",
expected: "",
scenarioType: "decode",
},
{
skipDoc: true,
description: "datetime",
input: "A = 1979-05-27T07:32:00-08:00",
expected: "A: 1979-05-27T07:32:00-08:00\n",
scenarioType: "decode",
},
{
skipDoc: true,
description: "blank",
input: `A = "hello`,
expectedError: `bad file 'sample.yml': basic string not terminated by "`,
scenarioType: "decode-error",
},
{
description: "Parse: Simple",
input: "A = \"hello\"\nB = 12\n",
expected: "A: hello\nB: 12\n",
scenarioType: "decode",
},
{
description: "Parse: Deep paths",
input: "person.name = \"hello\"\nperson.address = \"12 cat st\"\n",
expected: "person:\n name: hello\n address: 12 cat st\n",
scenarioType: "decode",
},
{
skipDoc: true,
input: `A.B = "hello"`,
expected: "A:\n B: hello\n",
scenarioType: "decode",
},
{
skipDoc: true,
description: "bool",
input: `A = true`,
expected: "A: true\n",
scenarioType: "decode",
},
{
skipDoc: true,
description: "bool false",
input: `A = false `,
expected: "A: false\n",
scenarioType: "decode",
},
{
skipDoc: true,
description: "number",
input: `A = 3 `,
expected: "A: 3\n",
scenarioType: "decode",
},
{
skipDoc: true,
description: "number",
input: `A = 0xDEADBEEF`,
expression: " .A += 1",
expected: "A: 0xDEADBEF0\n",
scenarioType: "decode",
},
{
skipDoc: true,
description: "float",
input: `A = 6.626e-34`,
expected: "A: 6.626e-34\n",
scenarioType: "decode",
},
{
skipDoc: true,
description: "empty arraY",
input: `A = []`,
expected: "A: []\n",
scenarioType: "decode",
},
{
skipDoc: true,
description: "array",
input: `A = ["hello", ["world", "again"]]`,
expected: "A:\n - hello\n - - world\n - again\n",
scenarioType: "decode",
},
{
description: "Parse: inline table",
input: `name = { first = "Tom", last = "Preston-Werner" }`,
expected: "name:\n first: Tom\n last: Preston-Werner\n",
scenarioType: "decode",
},
{
skipDoc: true,
input: sampleTable,
expected: sampleTableExpected,
scenarioType: "decode",
},
{
description: "Parse: Array Table",
input: sampleArrayTable,
expected: sampleArrayTableExpected,
scenarioType: "decode",
},
{
description: "Parse: with header",
input: sampleWithHeader,
expected: expectedSampleWithHeader,
scenarioType: "decode",
},
}
func testTomlScenario(t *testing.T, s formatScenario) {
switch s.scenarioType {
case "", "decode":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewTomlDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
case "decode-error":
result, err := processFormatScenario(s, NewTomlDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences))
if err == nil {
t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result)
} else {
test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description)
}
}
}
func documentTomlDecodeScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
if s.subdescription != "" {
writeOrPanic(w, s.subdescription)
writeOrPanic(w, "\n\n")
}
writeOrPanic(w, "Given a sample.toml file of:\n")
writeOrPanic(w, fmt.Sprintf("```toml\n%v\n```\n", s.input))
writeOrPanic(w, "then\n")
expression := s.expression
if expression == "" {
expression = "."
}
writeOrPanic(w, fmt.Sprintf("```bash\nyq -oy '%v' sample.toml\n```\n", expression))
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewTomlDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences))))
}
func documentTomlScenario(t *testing.T, w *bufio.Writer, i interface{}) {
s := i.(formatScenario)
if s.skipDoc {
return
}
switch s.scenarioType {
case "", "decode":
documentTomlDecodeScenario(w, s)
default:
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
}
}
func TestTomlScenarios(t *testing.T) {
for _, tt := range tomlScenarios {
testTomlScenario(t, tt)
}
genericScenarios := make([]interface{}, len(tomlScenarios))
for i, s := range tomlScenarios {
genericScenarios[i] = s
}
documentScenarios(t, "usage", "toml", genericScenarios, documentTomlScenario)
}