From 306dc931a5593df58f0323416a444272a8e27640 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sat, 22 Nov 2025 14:35:07 +1100 Subject: [PATCH] Fixing TOML ArrayTable parsing issues #1758 --- examples/data1.yaml | 11 ++--- examples/sample.toml | 28 ++--------- pkg/yqlib/decoder_toml.go | 54 ++++++++++++++++------ pkg/yqlib/doc/usage/toml.md | 21 +++++++++ pkg/yqlib/toml_test.go | 92 +++++++++++++++++++++++++++++++++++++ 5 files changed, 160 insertions(+), 46 deletions(-) diff --git a/examples/data1.yaml b/examples/data1.yaml index 470fab3d..7b95dbbf 100644 --- a/examples/data1.yaml +++ b/examples/data1.yaml @@ -1,8 +1,3 @@ -# 001 ---- -abc: # 001 -- 1 # one -- 2 # two - ---- -def # 002 \ No newline at end of file +[[fruits]] +[[fruits.varieties]] # nested array of tables +name = "red delicious \ No newline at end of file diff --git a/examples/sample.toml b/examples/sample.toml index 1376c81a..ea5c4f93 100644 --- a/examples/sample.toml +++ b/examples/sample.toml @@ -1,26 +1,6 @@ +[[fruits]] +[animals] -# This is a TOML document - -title = "TOML Example" - -[owner] -name = "Tom Preston-Werner" -dob = 1979-05-27T07:32:00-08:00 - -[database] -enabled = true -ports = [ 8000, 8001, 8002 ] -data = [ ["delta", "phi"], [3.14] ] -temp_targets = { cpu = 79.5, case = 72.0 } - -[servers] - -[servers.alpha] -ip = "10.0.0.1" -role = "frontend" - -[servers.beta] -ip = "10.0.0.2" -role = "backend" - +[[fruits.varieties]] # nested array of tables +name = "red delicious" \ No newline at end of file diff --git a/pkg/yqlib/decoder_toml.go b/pkg/yqlib/decoder_toml.go index 8f90c66e..3299aaf3 100644 --- a/pkg/yqlib/decoder_toml.go +++ b/pkg/yqlib/decoder_toml.go @@ -324,32 +324,58 @@ func (dec *tomlDecoder) arrayAppend(context Context, path []interface{}, rhsNode } func (dec *tomlDecoder) processArrayTable(currentNode *toml.Node) (bool, error) { - log.Debug("Entering processArrayTable") + log.Debug("c") fullPath := dec.getFullPath(currentNode.Child()) log.Debug("Fullpath: %v", fullPath) + c := Context{} + c = c.SingleChildContext(dec.rootMap) + + pathToCheck := fullPath + if len(fullPath) >= 1 { + pathToCheck = fullPath[:len(fullPath)-1] + } + + // if fullPath points to an array of maps rather than a map + // then it should set this element into the _last_ element of that array. + // Because TOML. So we'll inject the last index into the path. + readOp := createTraversalTree(pathToCheck, traversePreferences{DontAutoCreate: true}, false) + + resultContext, err := dec.d.GetMatchingNodes(c, readOp) + if err != nil { + return false, err + } + if resultContext.MatchingNodes.Len() >= 1 { + match := resultContext.MatchingNodes.Front().Value.(*CandidateNode) + // path refers to an array, we need to add this to the last element in the array + if match.Kind == SequenceNode { + fullPath = append(pathToCheck, len(match.Content)-1, fullPath[len(fullPath)-1]) + log.Debugf("Adding to end of %v array, using path: %v", pathToCheck, fullPath) + } + } + // need to use the array append exp to add another entry to // this array: fullpath += [ thing ] - hasValue := dec.parser.NextExpression() - if !hasValue { - return false, fmt.Errorf("error retrieving table %v value: %w", fullPath, dec.parser.Error()) - } tableNodeValue := &CandidateNode{ Kind: MappingNode, Tag: "!!map", } - - tableValue := dec.parser.Expression() - runAgainstCurrentExp, err := dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue) - log.Debugf("table node err: %w", err) - if err != nil && !errors.Is(err, io.EOF) { - return false, err + runAgainstCurrentExp := false + // if the next value is a ArrayTable or Table, then its not part of this declaration (not a key value pair) + // so lets leave that expression for the next round of parsing + if hasValue && (dec.parser.Expression().Kind == toml.ArrayTable || dec.parser.Expression().Kind == toml.Table) { + runAgainstCurrentExp = true + } else if hasValue { + // otherwise, if there is a value, it must be some key value pairs of the + // first object in the array! + tableValue := dec.parser.Expression() + runAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue) + if err != nil && !errors.Is(err, io.EOF) { + return false, err + } } - c := Context{} - - c = c.SingleChildContext(dec.rootMap) // += function err = dec.arrayAppend(c, fullPath, tableNodeValue) diff --git a/pkg/yqlib/doc/usage/toml.md b/pkg/yqlib/doc/usage/toml.md index 72773383..7cc4c375 100644 --- a/pkg/yqlib/doc/usage/toml.md +++ b/pkg/yqlib/doc/usage/toml.md @@ -104,6 +104,27 @@ owner: suburb: nice ``` +## Parse: Array of Array Table +Given a sample.toml file of: +```toml + +[[fruits]] +name = "apple" +[[fruits.varieties]] # nested array of tables +name = "red delicious" +``` +then +```bash +yq -oy '.' sample.toml +``` +will output +```yaml +fruits: + - name: apple + varieties: + - name: red delicious +``` + ## Parse: Empty Table Given a sample.toml file of: ```toml diff --git a/pkg/yqlib/toml_test.go b/pkg/yqlib/toml_test.go index 82c82f13..255a3c8a 100644 --- a/pkg/yqlib/toml_test.go +++ b/pkg/yqlib/toml_test.go @@ -37,6 +37,64 @@ owner: age: 36 ` +var doubleArrayTable = ` +[[fruits]] +name = "apple" +[[fruits.varieties]] # nested array of tables +name = "red delicious"` + +var doubleArrayTableExpected = `fruits: + - name: apple + varieties: + - name: red delicious +` + +var doubleArrayTableMultipleEntries = ` +[[fruits]] +name = "banana" +[[fruits]] +name = "apple" +[[fruits.varieties]] # nested array of tables +name = "red delicious"` + +var doubleArrayTableMultipleEntriesExpected = `fruits: + - name: banana + - name: apple + varieties: + - name: red delicious +` + +var doubleArrayTableNothingAbove = ` +[[fruits.varieties]] # nested array of tables +name = "red delicious"` + +var doubleArrayTableNothingAboveExpected = `fruits: + varieties: + - name: red delicious +` + +var doubleArrayTableEmptyAbove = ` +[[fruits]] +[[fruits.varieties]] # nested array of tables +name = "red delicious"` + +var doubleArrayTableEmptyAboveExpected = `fruits: + - varieties: + - name: red delicious +` + +var emptyArrayTableThenTable = ` +[[fruits]] +[animals] +[[fruits.varieties]] # nested array of tables +name = "red delicious"` + +var emptyArrayTableThenTableExpected = `fruits: + - varieties: + - name: red delicious +animals: {} +` + var sampleArrayTable = ` [owner.contact] name = "Tom Preston-Werner" @@ -249,6 +307,40 @@ var tomlScenarios = []formatScenario{ expected: sampleArrayTableExpected, scenarioType: "decode", }, + { + description: "Parse: Array of Array Table", + input: doubleArrayTable, + expected: doubleArrayTableExpected, + scenarioType: "decode", + }, + { + skipDoc: true, + description: "Parse: Array of Array Table; nothing above", + input: doubleArrayTableNothingAbove, + expected: doubleArrayTableNothingAboveExpected, + scenarioType: "decode", + }, + { + skipDoc: true, + description: "Parse: Array of Array Table; empty above", + input: doubleArrayTableEmptyAbove, + expected: doubleArrayTableEmptyAboveExpected, + scenarioType: "decode", + }, + { + skipDoc: true, + description: "Parse: Array of Array Table; multiple entries", + input: doubleArrayTableMultipleEntries, + expected: doubleArrayTableMultipleEntriesExpected, + scenarioType: "decode", + }, + { + skipDoc: true, + description: "Parse: Array of Array Table; then table; then array table", + input: emptyArrayTableThenTable, + expected: emptyArrayTableThenTableExpected, + scenarioType: "decode", + }, { description: "Parse: Empty Table", input: emptyTable,