From 7103b78d38c366b60fa3fbbef0afbec154a8c391 Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Sun, 26 Mar 2023 00:59:15 +0100
Subject: [PATCH] Draft: Toml (#1439)

* toml wip

* wip

* Fixed auto parsing toml

* Added build flag not to include toml

* Parse toml docs and tests

* minor updates
---
 .gitignore                             |   2 +
 acceptance_tests/inputs-format-auto.sh |  21 ++
 cmd/root.go                            |   4 +-
 cmd/utils.go                           |   5 +
 examples/sample.toml                   |  26 ++
 go.mod                                 |   1 +
 go.sum                                 |  12 +-
 pkg/yqlib/data_tree_navigator.go       |  24 ++
 pkg/yqlib/decoder.go                   |   5 +-
 pkg/yqlib/decoder_properties.go        |  18 +-
 pkg/yqlib/decoder_test.go              |   3 +-
 pkg/yqlib/decoder_toml.go              | 369 +++++++++++++++++++++++++
 pkg/yqlib/doc/usage/headers/toml.md    |   4 +
 pkg/yqlib/doc/usage/toml.md            | 111 ++++++++
 pkg/yqlib/no_toml.go                   |   7 +
 pkg/yqlib/toml_test.go                 | 239 ++++++++++++++++
 16 files changed, 828 insertions(+), 23 deletions(-)
 create mode 100644 examples/sample.toml
 create mode 100644 pkg/yqlib/decoder_toml.go
 create mode 100644 pkg/yqlib/doc/usage/headers/toml.md
 create mode 100644 pkg/yqlib/doc/usage/toml.md
 create mode 100644 pkg/yqlib/no_toml.go
 create mode 100644 pkg/yqlib/toml_test.go

diff --git a/.gitignore b/.gitignore
index b159f65d..aef9fba2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -41,6 +41,8 @@ yq*.snap
 
 test.yml
 test*.yml
+test*.xml
+test*.toml
 test*.yaml
 0.yml
 1.yml
diff --git a/acceptance_tests/inputs-format-auto.sh b/acceptance_tests/inputs-format-auto.sh
index a5e6897c..3c241d65 100755
--- a/acceptance_tests/inputs-format-auto.sh
+++ b/acceptance_tests/inputs-format-auto.sh
@@ -2,6 +2,7 @@
 
 setUp() {
   rm test*.yml 2>/dev/null || true
+  rm test*.toml 2>/dev/null || true
   rm test*.tfstate 2>/dev/null || true
   rm test*.json 2>/dev/null || true
   rm test*.properties 2>/dev/null || true
@@ -30,6 +31,26 @@ EOM
   assertEquals "$expected" "$X"
 }
 
+testInputToml() {
+  cat >test.toml <<EOL
+[owner]
+name = "Tom Preston-Werner"
+dob = 1979-05-27T07:32:00-08:00
+EOL
+
+  read -r -d '' expected << EOM
+owner:
+  name: Tom Preston-Werner
+  dob: 1979-05-27T07:32:00-08:00
+EOM
+
+  X=$(./yq -oy test.toml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -oy test.toml)
+  assertEquals "$expected" "$X"
+}
+
 testInputTfstate() {
   cat >test.tfstate <<EOL
 { "mike" : { "things": "cool" } }
diff --git a/cmd/root.go b/cmd/root.go
index 98e52752..f3a00cf2 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -65,8 +65,8 @@ yq -P sample.json
 		panic(err)
 	}
 
-	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "auto", "[auto|a|yaml|y|json|j|props|p|xml|x] output format type.")
-	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "auto", "[auto|a|yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.")
+	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "auto", "[auto|a|yaml|y|json|j|props|p|xml|x|tsv|t|csv|c] output format type.")
+	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "auto", "[auto|a|yaml|y|props|p|xml|x|tsv|t|csv|c|toml] parse format for input. Note that json is a subset of yaml.")
 
 	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", yqlib.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes")
 	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", yqlib.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).")
diff --git a/cmd/utils.go b/cmd/utils.go
index a447f050..6fc67160 100644
--- a/cmd/utils.go
+++ b/cmd/utils.go
@@ -75,6 +75,9 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 			}
 		} else if isAutomaticOutputFormat() {
 			// automatic input worked, we can do it for output too unless specified
+			if inputFormat == "toml" {
+				return "", nil, fmt.Errorf("toml is not yet supported as an output format. Please specify another output format using the [--output-format/-o] flag")
+			}
 			outputFormat = inputFormat
 		}
 	} else if isAutomaticOutputFormat() {
@@ -137,6 +140,8 @@ func createDecoder(format yqlib.InputFormat, evaluateTogether bool) (yqlib.Decod
 		return yqlib.NewCSVObjectDecoder(','), nil
 	case yqlib.TSVObjectInputFormat:
 		return yqlib.NewCSVObjectDecoder('\t'), nil
+	case yqlib.TomlInputFormat:
+		return yqlib.NewTomlDecoder(), nil
 	case yqlib.YamlInputFormat:
 		prefs := yqlib.ConfiguredYamlPreferences
 		prefs.EvaluateTogether = evaluateTogether
diff --git a/examples/sample.toml b/examples/sample.toml
new file mode 100644
index 00000000..1376c81a
--- /dev/null
+++ b/examples/sample.toml
@@ -0,0 +1,26 @@
+
+
+# This is a TOML document
+
+title = "TOML Example"
+
+[owner]
+name = "Tom Preston-Werner"
+dob = 1979-05-27T07:32:00-08:00
+
+[database]
+enabled = true
+ports = [ 8000, 8001, 8002 ]
+data = [ ["delta", "phi"], [3.14] ]
+temp_targets = { cpu = 79.5, case = 72.0 }
+
+[servers]
+
+[servers.alpha]
+ip = "10.0.0.1"
+role = "frontend"
+
+[servers.beta]
+ip = "10.0.0.2"
+role = "backend"
+
diff --git a/go.mod b/go.mod
index 56160980..eaca1f42 100644
--- a/go.mod
+++ b/go.mod
@@ -11,6 +11,7 @@ require (
 	github.com/goccy/go-yaml v1.10.0
 	github.com/jinzhu/copier v0.3.5
 	github.com/magiconair/properties v1.8.7
+	github.com/pelletier/go-toml/v2 v2.0.6
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/pflag v1.0.5
diff --git a/go.sum b/go.sum
index 1b0c528e..44b71463 100644
--- a/go.sum
+++ b/go.sum
@@ -6,8 +6,9 @@ github.com/alecthomas/participle/v2 v2.0.0-beta.5/go.mod h1:RC764t6n4L8D8ITAJv0q
 github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk=
 github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
 github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
 github.com/elliotchance/orderedmap v1.5.0 h1:1IsExUsjv5XNBD3ZdC7jkAAqLWOOKdbPTmkHx63OsBg=
@@ -36,6 +37,8 @@ github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU=
+github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
 github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@@ -49,9 +52,14 @@ github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUq
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
diff --git a/pkg/yqlib/data_tree_navigator.go b/pkg/yqlib/data_tree_navigator.go
index 13659eaa..986c129d 100644
--- a/pkg/yqlib/data_tree_navigator.go
+++ b/pkg/yqlib/data_tree_navigator.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	logging "gopkg.in/op/go-logging.v1"
+	yaml "gopkg.in/yaml.v3"
 )
 
 type DataTreeNavigator interface {
@@ -11,6 +12,8 @@ type DataTreeNavigator interface {
 	// this will process the against the given expressionNode and return
 	// a new context of matching candidates
 	GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error)
+
+	DeeplyAssign(context Context, path []interface{}, rhsNode *yaml.Node) error
 }
 
 type dataTreeNavigator struct {
@@ -20,6 +23,27 @@ func NewDataTreeNavigator() DataTreeNavigator {
 	return &dataTreeNavigator{}
 }
 
+func (d *dataTreeNavigator) DeeplyAssign(context Context, path []interface{}, rhsNode *yaml.Node) error {
+
+	rhsCandidateNode := &CandidateNode{
+		Path: path,
+		Node: rhsNode,
+	}
+
+	assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
+
+	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}
+
+	assignmentOpNode := &ExpressionNode{
+		Operation: assignmentOp,
+		LHS:       createTraversalTree(path, traversePreferences{}, false),
+		RHS:       &ExpressionNode{Operation: rhsOp},
+	}
+
+	_, err := d.GetMatchingNodes(context, assignmentOpNode)
+	return err
+}
+
 func (d *dataTreeNavigator) GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error) {
 	if expressionNode == nil {
 		log.Debugf("getMatchingNodes - nothing to do")
diff --git a/pkg/yqlib/decoder.go b/pkg/yqlib/decoder.go
index 9fba83d3..af5d3ecf 100644
--- a/pkg/yqlib/decoder.go
+++ b/pkg/yqlib/decoder.go
@@ -16,6 +16,7 @@ const (
 	JsonInputFormat
 	CSVObjectInputFormat
 	TSVObjectInputFormat
+	TomlInputFormat
 	UriInputFormat
 )
 
@@ -38,8 +39,10 @@ func InputFormatFromString(format string) (InputFormat, error) {
 		return CSVObjectInputFormat, nil
 	case "tsv", "t":
 		return TSVObjectInputFormat, nil
+	case "toml":
+		return TomlInputFormat, nil
 	default:
-		return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml]", format)
+		return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml|toml]", format)
 	}
 }
 
diff --git a/pkg/yqlib/decoder_properties.go b/pkg/yqlib/decoder_properties.go
index 4e6541a9..e2438ee1 100644
--- a/pkg/yqlib/decoder_properties.go
+++ b/pkg/yqlib/decoder_properties.go
@@ -95,23 +95,7 @@ func (dec *propertiesDecoder) applyProperty(context Context, properties *propert
 
 	rhsNode.Tag = guessTagFromCustomType(rhsNode)
 
-	rhsCandidateNode := &CandidateNode{
-		Path: path,
-		Node: rhsNode,
-	}
-
-	assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
-
-	rhsOp := &Operation{OperationType: referenceOpType, CandidateNode: rhsCandidateNode}
-
-	assignmentOpNode := &ExpressionNode{
-		Operation: assignmentOp,
-		LHS:       createTraversalTree(path, traversePreferences{}, false),
-		RHS:       &ExpressionNode{Operation: rhsOp},
-	}
-
-	_, err := dec.d.GetMatchingNodes(context, assignmentOpNode)
-	return err
+	return dec.d.DeeplyAssign(context, path, rhsNode)
 }
 
 func (dec *propertiesDecoder) Decode() (*CandidateNode, error) {
diff --git a/pkg/yqlib/decoder_test.go b/pkg/yqlib/decoder_test.go
index d3982b29..d14c8a61 100644
--- a/pkg/yqlib/decoder_test.go
+++ b/pkg/yqlib/decoder_test.go
@@ -3,6 +3,7 @@ package yqlib
 import (
 	"bufio"
 	"bytes"
+	"fmt"
 	"strings"
 )
 
@@ -62,7 +63,7 @@ func mustProcessFormatScenario(s formatScenario, decoder Decoder, encoder Encode
 
 	result, err := processFormatScenario(s, decoder, encoder)
 	if err != nil {
-		panic(err)
+		panic(fmt.Errorf("Bad scenario %v: %w", s.description, err))
 	}
 	return result
 
diff --git a/pkg/yqlib/decoder_toml.go b/pkg/yqlib/decoder_toml.go
new file mode 100644
index 00000000..166089ce
--- /dev/null
+++ b/pkg/yqlib/decoder_toml.go
@@ -0,0 +1,369 @@
+//go:build !yq_notoml
+
+package yqlib
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"strconv"
+	"time"
+
+	toml "github.com/pelletier/go-toml/v2/unstable"
+	yaml "gopkg.in/yaml.v3"
+)
+
+type tomlDecoder struct {
+	parser   toml.Parser
+	finished bool
+	d        DataTreeNavigator
+	rootMap  *CandidateNode
+}
+
+func NewTomlDecoder() Decoder {
+	return &tomlDecoder{
+		finished: false,
+		d:        NewDataTreeNavigator(),
+	}
+}
+
+func (dec *tomlDecoder) Init(reader io.Reader) error {
+	dec.parser = toml.Parser{}
+	buf := new(bytes.Buffer)
+	_, err := buf.ReadFrom(reader)
+	if err != nil {
+		return err
+	}
+	dec.parser.Reset(buf.Bytes())
+	dec.rootMap = &CandidateNode{
+		Node: &yaml.Node{
+			Kind: yaml.MappingNode,
+			Tag:  "!!map",
+		}}
+	return nil
+}
+
+func (dec *tomlDecoder) getFullPath(tomlNode *toml.Node) []interface{} {
+	path := make([]interface{}, 0)
+	for {
+		path = append(path, string(tomlNode.Data))
+		tomlNode = tomlNode.Next()
+		if tomlNode == nil {
+			return path
+		}
+	}
+}
+
+func (dec *tomlDecoder) processKeyValueIntoMap(rootMap *CandidateNode, tomlNode *toml.Node) error {
+	value := tomlNode.Value()
+	path := dec.getFullPath(value.Next())
+	log.Debug("!!!processKeyValueIntoMap: %v", path)
+
+	valueNode, err := dec.decodeNode(value)
+	if err != nil {
+		return err
+	}
+	context := Context{}
+	context = context.SingleChildContext(rootMap)
+
+	return dec.d.DeeplyAssign(context, path, valueNode)
+}
+
+func (dec *tomlDecoder) decodeKeyValuesIntoMap(rootMap *CandidateNode, tomlNode *toml.Node) (bool, error) {
+	log.Debug("!! DECODE_KV_INTO_MAP -- processing first (current) entry")
+	if err := dec.processKeyValueIntoMap(rootMap, tomlNode); err != nil {
+		return false, err
+	}
+
+	for dec.parser.NextExpression() {
+		nextItem := dec.parser.Expression()
+		log.Debug("!! DECODE_KV_INTO_MAP -- next exp, its a %v", nextItem.Kind)
+
+		if nextItem.Kind == toml.KeyValue {
+			if err := dec.processKeyValueIntoMap(rootMap, nextItem); err != nil {
+				return false, err
+			}
+		} else {
+			// run out of key values
+			log.Debug("! DECODE_KV_INTO_MAP - ok we are done in decodeKeyValuesIntoMap, gota a %v", nextItem.Kind)
+			log.Debug("! DECODE_KV_INTO_MAP - processAgainstCurrentExp = true!")
+			return true, nil
+		}
+	}
+	log.Debug("! DECODE_KV_INTO_MAP - no more things to read in")
+	return false, nil
+}
+
+func (dec *tomlDecoder) createInlineTableMap(tomlNode *toml.Node) (*yaml.Node, error) {
+	content := make([]*yaml.Node, 0)
+	log.Debug("!! createInlineTableMap")
+
+	iterator := tomlNode.Children()
+	for iterator.Next() {
+		child := iterator.Node()
+		if child.Kind != toml.KeyValue {
+			return nil, fmt.Errorf("only keyvalue pairs are supported in inlinetables, got %v instead", child.Kind)
+		}
+
+		keyValues := &CandidateNode{
+			Node: &yaml.Node{
+				Kind: yaml.MappingNode,
+				Tag:  "!!map",
+			},
+		}
+
+		if err := dec.processKeyValueIntoMap(keyValues, child); err != nil {
+			return nil, err
+		}
+
+		content = append(content, keyValues.Node.Content...)
+	}
+
+	return &yaml.Node{
+		Kind:    yaml.MappingNode,
+		Tag:     "!!map",
+		Content: content,
+	}, nil
+}
+
+func (dec *tomlDecoder) createArray(tomlNode *toml.Node) (*yaml.Node, error) {
+	content := make([]*yaml.Node, 0)
+	iterator := tomlNode.Children()
+	for iterator.Next() {
+		child := iterator.Node()
+		yamlNode, err := dec.decodeNode(child)
+		if err != nil {
+			return nil, err
+		}
+		content = append(content, yamlNode)
+	}
+
+	return &yaml.Node{
+		Kind:    yaml.SequenceNode,
+		Tag:     "!!seq",
+		Content: content,
+	}, nil
+
+}
+
+func (dec *tomlDecoder) createStringScalar(tomlNode *toml.Node) (*yaml.Node, error) {
+	content := string(tomlNode.Data)
+	return createScalarNode(content, content), nil
+}
+
+func (dec *tomlDecoder) createBoolScalar(tomlNode *toml.Node) (*yaml.Node, error) {
+	content := string(tomlNode.Data)
+	return createScalarNode(content == "true", content), nil
+}
+
+func (dec *tomlDecoder) createIntegerScalar(tomlNode *toml.Node) (*yaml.Node, error) {
+	content := string(tomlNode.Data)
+	_, num, err := parseInt64(content)
+	return createScalarNode(num, content), err
+}
+
+func (dec *tomlDecoder) createDateTimeScalar(tomlNode *toml.Node) (*yaml.Node, error) {
+	content := string(tomlNode.Data)
+	val, err := parseDateTime(time.RFC3339, content)
+	return createScalarNode(val, content), err
+}
+
+func (dec *tomlDecoder) createFloatScalar(tomlNode *toml.Node) (*yaml.Node, error) {
+	content := string(tomlNode.Data)
+	num, err := strconv.ParseFloat(content, 64)
+	return createScalarNode(num, content), err
+}
+
+func (dec *tomlDecoder) decodeNode(tomlNode *toml.Node) (*yaml.Node, error) {
+	switch tomlNode.Kind {
+	case toml.Key, toml.String:
+		return dec.createStringScalar(tomlNode)
+	case toml.Bool:
+		return dec.createBoolScalar(tomlNode)
+	case toml.Integer:
+		return dec.createIntegerScalar(tomlNode)
+	case toml.DateTime:
+		return dec.createDateTimeScalar(tomlNode)
+	case toml.Float:
+		return dec.createFloatScalar(tomlNode)
+	case toml.Array:
+		return dec.createArray(tomlNode)
+	case toml.InlineTable:
+		return dec.createInlineTableMap(tomlNode)
+	default:
+		return nil, fmt.Errorf("unsupported type %v", tomlNode.Kind)
+	}
+
+}
+
+func (dec *tomlDecoder) Decode() (*CandidateNode, error) {
+	if dec.finished {
+		return nil, io.EOF
+	}
+	//
+	// toml library likes to panic
+	var deferredError error
+	defer func() { //catch or finally
+		if r := recover(); r != nil {
+			var ok bool
+			deferredError, ok = r.(error)
+			if !ok {
+				deferredError = fmt.Errorf("pkg: %v", r)
+			}
+		}
+	}()
+
+	log.Debug("ok here we go")
+	var runAgainstCurrentExp = false
+	var err error
+	for runAgainstCurrentExp || dec.parser.NextExpression() {
+
+		if runAgainstCurrentExp {
+			log.Debug("running against current exp")
+		}
+
+		currentNode := dec.parser.Expression()
+
+		log.Debug("currentNode: %v ", currentNode.Kind)
+		runAgainstCurrentExp, err = dec.processTopLevelNode(currentNode)
+		if err != nil {
+			return dec.rootMap, err
+		}
+
+	}
+
+	err = dec.parser.Error()
+	if err != nil {
+		return nil, err
+	}
+
+	// must have finished
+	dec.finished = true
+
+	if len(dec.rootMap.Node.Content) == 0 {
+		return nil, io.EOF
+	}
+
+	return &CandidateNode{
+		Node: &yaml.Node{
+			Kind:    yaml.DocumentNode,
+			Content: []*yaml.Node{dec.rootMap.Node},
+		},
+	}, deferredError
+
+}
+
+func (dec *tomlDecoder) processTopLevelNode(currentNode *toml.Node) (bool, error) {
+	var runAgainstCurrentExp bool
+	var err error
+	log.Debug("!!!!!!!!!!!!Going to process %v state is current %v", currentNode.Kind, NodeToString(dec.rootMap))
+	if currentNode.Kind == toml.Table {
+		runAgainstCurrentExp, err = dec.processTable((currentNode))
+	} else if currentNode.Kind == toml.ArrayTable {
+		runAgainstCurrentExp, err = dec.processArrayTable((currentNode))
+	} else {
+		runAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(dec.rootMap, currentNode)
+	}
+
+	log.Debug("!!!!!!!!!!!!DONE Processing state is now %v", NodeToString(dec.rootMap))
+	return runAgainstCurrentExp, err
+}
+
+func (dec *tomlDecoder) processTable(currentNode *toml.Node) (bool, error) {
+	log.Debug("!!! processing table")
+	fullPath := dec.getFullPath(currentNode.Child())
+	log.Debug("!!!fullpath: %v", fullPath)
+
+	hasValue := dec.parser.NextExpression()
+	if !hasValue {
+		return false, fmt.Errorf("error retrieving table %v value: %w", fullPath, dec.parser.Error())
+	}
+
+	tableNodeValue := &CandidateNode{
+		Node: &yaml.Node{
+			Kind: yaml.MappingNode,
+			Tag:  "!!map",
+		},
+	}
+
+	tableValue := dec.parser.Expression()
+	if tableValue.Kind != toml.KeyValue {
+		log.Debug("got an empty table, returning")
+		return true, nil
+	}
+
+	runAgainstCurrentExp, err := dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue)
+	log.Debugf("table node err: %w", err)
+	if err != nil && !errors.Is(io.EOF, err) {
+		return false, err
+	}
+	c := Context{}
+
+	c = c.SingleChildContext(dec.rootMap)
+	err = dec.d.DeeplyAssign(c, fullPath, tableNodeValue.Node)
+	if err != nil {
+		return false, err
+	}
+	return runAgainstCurrentExp, nil
+}
+
+func (dec *tomlDecoder) arrayAppend(context Context, path []interface{}, rhsNode *yaml.Node) error {
+	rhsCandidateNode := &CandidateNode{
+		Path: path,
+		Node: &yaml.Node{
+			Kind:    yaml.SequenceNode,
+			Tag:     "!!seq",
+			Content: []*yaml.Node{rhsNode},
+		},
+	}
+
+	assignmentOp := &Operation{OperationType: addAssignOpType}
+
+	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}
+
+	assignmentOpNode := &ExpressionNode{
+		Operation: assignmentOp,
+		LHS:       createTraversalTree(path, traversePreferences{}, false),
+		RHS:       &ExpressionNode{Operation: rhsOp},
+	}
+
+	_, err := dec.d.GetMatchingNodes(context, assignmentOpNode)
+	return err
+}
+
+func (dec *tomlDecoder) processArrayTable(currentNode *toml.Node) (bool, error) {
+	log.Debug("!!! processing table")
+	fullPath := dec.getFullPath(currentNode.Child())
+	log.Debug("!!!fullpath: %v", fullPath)
+
+	// need to use the array append exp to add another entry to
+	// this array: fullpath += [ thing ]
+
+	hasValue := dec.parser.NextExpression()
+	if !hasValue {
+		return false, fmt.Errorf("error retrieving table %v value: %w", fullPath, dec.parser.Error())
+	}
+
+	tableNodeValue := &CandidateNode{
+		Node: &yaml.Node{
+			Kind: yaml.MappingNode,
+			Tag:  "!!map",
+		},
+	}
+
+	tableValue := dec.parser.Expression()
+	runAgainstCurrentExp, err := dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue)
+	log.Debugf("table node err: %w", err)
+	if err != nil && !errors.Is(io.EOF, err) {
+		return false, err
+	}
+	c := Context{}
+
+	c = c.SingleChildContext(dec.rootMap)
+
+	// += function
+	err = dec.arrayAppend(c, fullPath, tableNodeValue.Node)
+
+	return runAgainstCurrentExp, err
+}
diff --git a/pkg/yqlib/doc/usage/headers/toml.md b/pkg/yqlib/doc/usage/headers/toml.md
new file mode 100644
index 00000000..a02aecf2
--- /dev/null
+++ b/pkg/yqlib/doc/usage/headers/toml.md
@@ -0,0 +1,4 @@
+# TOML
+
+Decode from TOML. Note that `yq` does not yet support outputting in TOML format (and therefore it cannot roundtrip)
+
diff --git a/pkg/yqlib/doc/usage/toml.md b/pkg/yqlib/doc/usage/toml.md
new file mode 100644
index 00000000..b40b827c
--- /dev/null
+++ b/pkg/yqlib/doc/usage/toml.md
@@ -0,0 +1,111 @@
+# TOML
+
+Decode from TOML. Note that `yq` does not yet support outputting in TOML format (and therefore it cannot roundtrip)
+
+
+## Parse: Simple
+Given a sample.toml file of:
+```toml
+A = "hello"
+B = 12
+
+```
+then
+```bash
+yq -oy '.' sample.toml
+```
+will output
+```yaml
+A: hello
+B: 12
+```
+
+## Parse: Deep paths
+Given a sample.toml file of:
+```toml
+person.name = "hello"
+person.address = "12 cat st"
+
+```
+then
+```bash
+yq -oy '.' sample.toml
+```
+will output
+```yaml
+person:
+  name: hello
+  address: 12 cat st
+```
+
+## Parse: inline table
+Given a sample.toml file of:
+```toml
+name = { first = "Tom", last = "Preston-Werner" }
+```
+then
+```bash
+yq -oy '.' sample.toml
+```
+will output
+```yaml
+name:
+  first: Tom
+  last: Preston-Werner
+```
+
+## Parse: Array Table
+Given a sample.toml file of:
+```toml
+
+[owner.contact]
+name = "Tom Preston-Werner"
+age = 36
+
+[[owner.addresses]]
+street = "first street"
+suburb = "ok"
+
+[[owner.addresses]]
+street = "second street"
+suburb = "nice"
+
+```
+then
+```bash
+yq -oy '.' sample.toml
+```
+will output
+```yaml
+owner:
+  contact:
+    name: Tom Preston-Werner
+    age: 36
+  addresses:
+    - street: first street
+      suburb: ok
+    - street: second street
+      suburb: nice
+```
+
+## Parse: with header
+Given a sample.toml file of:
+```toml
+
+[servers]
+
+[servers.alpha]
+ip = "10.0.0.1"
+
+```
+then
+```bash
+yq -oy '.' sample.toml
+```
+will output
+```yaml
+servers:
+  alpha:
+    ip: 10.0.0.1
+```
+
diff --git a/pkg/yqlib/no_toml.go b/pkg/yqlib/no_toml.go
new file mode 100644
index 00000000..3067dae8
--- /dev/null
+++ b/pkg/yqlib/no_toml.go
@@ -0,0 +1,7 @@
+//go:build yq_notoml
+
+package yqlib
+
+func NewTomlDecoder() Decoder {
+	return nil
+}
diff --git a/pkg/yqlib/toml_test.go b/pkg/yqlib/toml_test.go
new file mode 100644
index 00000000..7c0ab3c7
--- /dev/null
+++ b/pkg/yqlib/toml_test.go
@@ -0,0 +1,239 @@
+package yqlib
+
+import (
+	"bufio"
+	"fmt"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+var sampleTable = `
+var = "x"
+
+[owner.contact]
+name = "Tom Preston-Werner"
+age = 36
+`
+
+var sampleTableExpected = `var: x
+owner:
+  contact:
+    name: Tom Preston-Werner
+    age: 36
+`
+
+var sampleArrayTable = `
+[owner.contact]
+name = "Tom Preston-Werner"
+age = 36
+
+[[owner.addresses]]
+street = "first street"
+suburb = "ok"
+
+[[owner.addresses]]
+street = "second street"
+suburb = "nice"
+`
+
+var sampleArrayTableExpected = `owner:
+  contact:
+    name: Tom Preston-Werner
+    age: 36
+  addresses:
+    - street: first street
+      suburb: ok
+    - street: second street
+      suburb: nice
+`
+
+var sampleWithHeader = `
+[servers]
+
+[servers.alpha]
+ip = "10.0.0.1"
+`
+
+var expectedSampleWithHeader = `servers:
+  alpha:
+    ip: 10.0.0.1
+`
+
+var tomlScenarios = []formatScenario{
+	{
+		skipDoc:      true,
+		description:  "blank",
+		input:        "",
+		expected:     "",
+		scenarioType: "decode",
+	},
+	{
+		skipDoc:      true,
+		description:  "datetime",
+		input:        "A = 1979-05-27T07:32:00-08:00",
+		expected:     "A: 1979-05-27T07:32:00-08:00\n",
+		scenarioType: "decode",
+	},
+	{
+		skipDoc:       true,
+		description:   "blank",
+		input:         `A = "hello`,
+		expectedError: `bad file 'sample.yml': basic string not terminated by "`,
+		scenarioType:  "decode-error",
+	},
+	{
+		description:  "Parse: Simple",
+		input:        "A = \"hello\"\nB = 12\n",
+		expected:     "A: hello\nB: 12\n",
+		scenarioType: "decode",
+	},
+	{
+		description:  "Parse: Deep paths",
+		input:        "person.name = \"hello\"\nperson.address = \"12 cat st\"\n",
+		expected:     "person:\n  name: hello\n  address: 12 cat st\n",
+		scenarioType: "decode",
+	},
+	{
+		skipDoc:      true,
+		input:        `A.B = "hello"`,
+		expected:     "A:\n  B: hello\n",
+		scenarioType: "decode",
+	},
+	{
+		skipDoc:      true,
+		description:  "bool",
+		input:        `A = true`,
+		expected:     "A: true\n",
+		scenarioType: "decode",
+	},
+	{
+		skipDoc:      true,
+		description:  "bool false",
+		input:        `A = false `,
+		expected:     "A: false\n",
+		scenarioType: "decode",
+	},
+	{
+		skipDoc:      true,
+		description:  "number",
+		input:        `A = 3 `,
+		expected:     "A: 3\n",
+		scenarioType: "decode",
+	},
+	{
+		skipDoc:      true,
+		description:  "number",
+		input:        `A = 0xDEADBEEF`,
+		expression:   " .A += 1",
+		expected:     "A: 0xDEADBEF0\n",
+		scenarioType: "decode",
+	},
+	{
+		skipDoc:      true,
+		description:  "float",
+		input:        `A = 6.626e-34`,
+		expected:     "A: 6.626e-34\n",
+		scenarioType: "decode",
+	},
+	{
+		skipDoc:      true,
+		description:  "empty arraY",
+		input:        `A = []`,
+		expected:     "A: []\n",
+		scenarioType: "decode",
+	},
+	{
+		skipDoc:      true,
+		description:  "array",
+		input:        `A = ["hello", ["world", "again"]]`,
+		expected:     "A:\n  - hello\n  - - world\n    - again\n",
+		scenarioType: "decode",
+	},
+	{
+		description:  "Parse: inline table",
+		input:        `name = { first = "Tom", last = "Preston-Werner" }`,
+		expected:     "name:\n  first: Tom\n  last: Preston-Werner\n",
+		scenarioType: "decode",
+	},
+	{
+		skipDoc:      true,
+		input:        sampleTable,
+		expected:     sampleTableExpected,
+		scenarioType: "decode",
+	},
+	{
+		description:  "Parse: Array Table",
+		input:        sampleArrayTable,
+		expected:     sampleArrayTableExpected,
+		scenarioType: "decode",
+	},
+	{
+		description:  "Parse: with header",
+		input:        sampleWithHeader,
+		expected:     expectedSampleWithHeader,
+		scenarioType: "decode",
+	},
+}
+
+func testTomlScenario(t *testing.T, s formatScenario) {
+	switch s.scenarioType {
+	case "", "decode":
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewTomlDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
+	case "decode-error":
+		result, err := processFormatScenario(s, NewTomlDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences))
+		if err == nil {
+			t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result)
+		} else {
+			test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description)
+		}
+	}
+}
+
+func documentTomlDecodeScenario(w *bufio.Writer, s formatScenario) {
+	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
+
+	if s.subdescription != "" {
+		writeOrPanic(w, s.subdescription)
+		writeOrPanic(w, "\n\n")
+	}
+
+	writeOrPanic(w, "Given a sample.toml file of:\n")
+	writeOrPanic(w, fmt.Sprintf("```toml\n%v\n```\n", s.input))
+
+	writeOrPanic(w, "then\n")
+	expression := s.expression
+	if expression == "" {
+		expression = "."
+	}
+	writeOrPanic(w, fmt.Sprintf("```bash\nyq -oy '%v' sample.toml\n```\n", expression))
+	writeOrPanic(w, "will output\n")
+
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewTomlDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences))))
+}
+
+func documentTomlScenario(t *testing.T, w *bufio.Writer, i interface{}) {
+	s := i.(formatScenario)
+
+	if s.skipDoc {
+		return
+	}
+	switch s.scenarioType {
+	case "", "decode":
+		documentTomlDecodeScenario(w, s)
+
+	default:
+		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
+	}
+}
+
+func TestTomlScenarios(t *testing.T) {
+	for _, tt := range tomlScenarios {
+		testTomlScenario(t, tt)
+	}
+	genericScenarios := make([]interface{}, len(tomlScenarios))
+	for i, s := range tomlScenarios {
+		genericScenarios[i] = s
+	}
+	documentScenarios(t, "usage", "toml", genericScenarios, documentTomlScenario)
+}