Fixing TOML subarray parsing issue #2581

pkg/yqlib/decoder_toml.go

@@ -430,23 +430,42 @@ func (dec *tomlDecoder) processArrayTable(currentNode *toml.Node) (bool, error)
 // Because TOML. So we'll inject the last index into the path.
 
 func getPathToUse(fullPath []interface{}, dec *tomlDecoder, c Context) ([]interface{}, error) {
-	pathToCheck := fullPath
+	// We need to check the entire path (except the last element), not just the immediate parent,
-	if len(fullPath) >= 1 {
+	// because we may have nested array tables like [[array.subarray.subsubarray]]
-		pathToCheck = fullPath[:len(fullPath)-1]
+	// where both 'array' and 'subarray' are arrays that already exist.
-	}
-	readOp := createTraversalTree(pathToCheck, traversePreferences{DontAutoCreate: true}, false)
 
-	resultContext, err := dec.d.GetMatchingNodes(c, readOp)
+	if len(fullPath) == 0 {
-	if err != nil {
+		return fullPath, nil
-		return nil, err
 	}
-	if resultContext.MatchingNodes.Len() >= 1 {
+
-		match := resultContext.MatchingNodes.Front().Value.(*CandidateNode)
+	resultPath := make([]interface{}, 0, len(fullPath)*2) // preallocate with extra space for indices
-		// path refers to an array, we need to add this to the last element in the array
+
-		if match.Kind == SequenceNode {
+	// Process all segments except the last one
-			fullPath = append(pathToCheck, len(match.Content)-1, fullPath[len(fullPath)-1])
+	for i := 0; i < len(fullPath)-1; i++ {
-			log.Debugf("Adding to end of %v array, using path: %v", pathToCheck, fullPath)
+		resultPath = append(resultPath, fullPath[i])
+
+		// Check if the current path segment points to an array
+		readOp := createTraversalTree(resultPath, traversePreferences{DontAutoCreate: true}, false)
+		resultContext, err := dec.d.GetMatchingNodes(c, readOp)
+		if err != nil {
+			return nil, err
+		}
+
+		if resultContext.MatchingNodes.Len() >= 1 {
+			match := resultContext.MatchingNodes.Front().Value.(*CandidateNode)
+			// If this segment points to an array, we need to add the last index
+			// before continuing with the rest of the path
+			if match.Kind == SequenceNode && len(match.Content) > 0 {
+				lastIndex := len(match.Content) - 1
+				resultPath = append(resultPath, lastIndex)
+				log.Debugf("Path segment %v is an array, injecting index %d", resultPath[:len(resultPath)-1], lastIndex)
+			}
 		}
 	}
-	return fullPath, err
+
+	// Add the last segment
+	resultPath = append(resultPath, fullPath[len(fullPath)-1])
+
+	log.Debugf("getPathToUse: original path %v -> result path %v", fullPath, resultPath)
+	return resultPath, nil
 }

pkg/yqlib/doc/usage/toml.md

@@ -329,3 +329,58 @@ B = 12
 name = "Tom"  # name comment
 ```
 
+## Roundtrip: sample from web
+Given a sample.toml file of:
+```toml
+# This is a TOML document
+title = "TOML Example"
+
+[owner]
+name = "Tom Preston-Werner"
+dob = 1979-05-27T07:32:00-08:00
+
+[database]
+enabled = true
+ports = [8000, 8001, 8002]
+data = [["delta", "phi"], [3.14]]
+temp_targets = { cpu = 79.5, case = 72.0 }
+
+# [servers] yq can't do this one yet
+[servers.alpha]
+ip = "10.0.0.1"
+role = "frontend"
+
+[servers.beta]
+ip = "10.0.0.2"
+role = "backend"
+
+```
+then
+```bash
+yq '.' sample.toml
+```
+will output
+```yaml
+# This is a TOML document
+title = "TOML Example"
+
+[owner]
+name = "Tom Preston-Werner"
+dob = 1979-05-27T07:32:00-08:00
+
+[database]
+enabled = true
+ports = [8000, 8001, 8002]
+data = [["delta", "phi"], [3.14]]
+temp_targets = { cpu = 79.5, case = 72.0 }
+
+# [servers] yq can't do this one yet
+[servers.alpha]
+ip = "10.0.0.1"
+role = "frontend"
+
+[servers.beta]
+ip = "10.0.0.2"
+role = "backend"
+```
+

pkg/yqlib/toml_test.go

@@ -228,31 +228,42 @@ B = 12
 name = "Tom"  # name comment
 `
 
-// var sampleFromWeb = `
+var sampleFromWeb = `# This is a TOML document
-// # This is a TOML document
+title = "TOML Example"
 
-// title = "TOML Example"
+[owner]
+name = "Tom Preston-Werner"
+dob = 1979-05-27T07:32:00-08:00
 
-// [owner]
+[database]
-// name = "Tom Preston-Werner"
+enabled = true
-// dob = 1979-05-27T07:32:00-08:00
+ports = [8000, 8001, 8002]
+data = [["delta", "phi"], [3.14]]
+temp_targets = { cpu = 79.5, case = 72.0 }
 
-// [database]
+# [servers] yq can't do this one yet
-// enabled = true
+[servers.alpha]
-// ports = [8000, 8001, 8002]
+ip = "10.0.0.1"
-// data = [["delta", "phi"], [3.14]]
+role = "frontend"
-// temp_targets = { cpu = 79.5, case = 72.0 }
 
-// [servers]
+[servers.beta]
+ip = "10.0.0.2"
+role = "backend"
+`
 
-// [servers.alpha]
+var subArrays = `
-// ip = "10.0.0.1"
+[[array]]
-// role = "frontend"
 
-// [servers.beta]
+[[array.subarray]]
-// ip = "10.0.0.2"
+
-// role = "backend"
+[[array.subarray.subsubarray]]
-// `
+`
+
+var expectedSubArrays = `array:
+  - subarray:
+      - subsubarray:
+          - {}
+`
 
 var tomlScenarios = []formatScenario{
 	{
@@ -461,6 +472,13 @@ var tomlScenarios = []formatScenario{
 		expected:     expectedMultipleEmptyTables,
 		scenarioType: "decode",
 	},
+	{
+		description:  "subArrays",
+		skipDoc:      true,
+		input:        subArrays,
+		expected:     expectedSubArrays,
+		scenarioType: "decode",
+	},
 	// Roundtrip scenarios
 	{
 		description:  "Roundtrip: inline table attribute",
@@ -532,13 +550,13 @@ var tomlScenarios = []formatScenario{
 		expected:     rtComments,
 		scenarioType: "roundtrip",
 	},
-	// {
+	{
-	// 	description:  "Roundtrip: sample from web",
+		description:  "Roundtrip: sample from web",
-	// 	input:        sampleFromWeb,
+		input:        sampleFromWeb,
-	// 	expression:   ".",
+		expression:   ".",
-	// 	expected:     sampleFromWeb,
+		expected:     sampleFromWeb,
-	// 	scenarioType: "roundtrip",
+		scenarioType: "roundtrip",
-	// },
+	},
 }
 
 func testTomlScenario(t *testing.T, s formatScenario) {

project-words.txt

@@ -293,3 +293,4 @@ buildvcs
 behaviour
 GOFLAGS
 gocache
+subsubarray