From 4e9d5e8e483743940bdfc35a0701af74b4099cc1 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Mon, 15 Dec 2025 11:40:28 +1100 Subject: [PATCH] wip --- pkg/yqlib/decoder_toml.go | 76 +++++++++++++++++++++++-------------- pkg/yqlib/doc/usage/toml.md | 61 ++++++++++++++++++++++++++++- pkg/yqlib/encoder_toml.go | 13 +++++++ pkg/yqlib/toml_test.go | 33 ++++++++++++++++ 4 files changed, 153 insertions(+), 30 deletions(-) diff --git a/pkg/yqlib/decoder_toml.go b/pkg/yqlib/decoder_toml.go index 0b8c107b..567a2361 100644 --- a/pkg/yqlib/decoder_toml.go +++ b/pkg/yqlib/decoder_toml.go @@ -15,11 +15,12 @@ import ( ) type tomlDecoder struct { - parser toml.Parser - finished bool - d DataTreeNavigator - rootMap *CandidateNode - fileBytes []byte + parser toml.Parser + finished bool + d DataTreeNavigator + rootMap *CandidateNode + fileBytes []byte + firstKeyValue bool // Track if this is the first key-value for root comment } func NewTomlDecoder() Decoder { @@ -42,6 +43,7 @@ func (dec *tomlDecoder) Init(reader io.Reader) error { Kind: MappingNode, Tag: "!!map", } + dec.firstKeyValue = true return nil } @@ -68,47 +70,54 @@ func (dec *tomlDecoder) extractLineComment(endPos int) string { } // extractHeadComment extracts comments before a given start position -// Only extracts comments from immediately preceding lines (no blank lines in between) +// Skips whitespace (including blank lines) first, then collects comments func (dec *tomlDecoder) extractHeadComment(startPos int) string { src := dec.fileBytes var comments []string - // Start just before the token and go back to previous newline + // Start just before the token and skip trailing whitespace (including newlines) i := startPos - 1 - for i >= 0 && src[i] != '\n' { + for i >= 0 && (src[i] == ' ' || src[i] == '\t' || src[i] == '\n' || src[i] == '\r') { i-- } - // Now i is at the newline before the current line, or -1 if at start // Keep collecting comment lines going backwards for i >= 0 { - // Move to end of previous line - i-- // skip the newline - if i < 0 { - break - } - - // Find the start of this line + // Find line boundaries: go back to find start, then forward to find end lineEnd := i + // Find the end of this line + for lineEnd < len(src) && src[lineEnd] != '\n' { + lineEnd++ + } + lineEnd-- // Back up from the newline + + // Now find the start of this line for i >= 0 && src[i] != '\n' { i-- } lineStart := i + 1 - line := strings.TrimSpace(string(src[lineStart : lineEnd+1])) + line := strings.TrimRight(string(src[lineStart:lineEnd+1]), " \t\r") + trimmed := strings.TrimSpace(line) // Empty line stops the comment block - if line == "" { + if trimmed == "" { break } // Non-comment line stops the comment block - if !strings.HasPrefix(line, "#") { + if !strings.HasPrefix(trimmed, "#") { break } // Prepend this comment line - comments = append([]string{line}, comments...) + comments = append([]string{trimmed}, comments...) + + // Move to previous line (skip any whitespace/newlines) + i = lineStart - 1 + for i >= 0 && (src[i] == ' ' || src[i] == '\t' || src[i] == '\n' || src[i] == '\r') { + i-- + } } if len(comments) > 0 { @@ -131,28 +140,37 @@ func (dec *tomlDecoder) getFullPath(tomlNode *toml.Node) []interface{} { func (dec *tomlDecoder) processKeyValueIntoMap(rootMap *CandidateNode, tomlNode *toml.Node) error { value := tomlNode.Value() path := dec.getFullPath(value.Next()) - log.Debug("processKeyValueIntoMap: %v", path) valueNode, err := dec.decodeNode(value) if err != nil { return err } - // Extract comments using the value's Raw range (more reliable than KeyValue node) - startPos := int(value.Raw.Offset) - endPos := int(value.Raw.Offset + value.Raw.Length) - + // Extract comments using the KeyValue node's start and value's end + kvStartPos := int(tomlNode.Raw.Offset) + valueEndPos := int(value.Raw.Offset + value.Raw.Length) + + log.Debug("processKeyValueIntoMap: kvStartPos=%d, valueEndPos=%d, firstKeyValue=%v", kvStartPos, valueEndPos, dec.firstKeyValue) + // HeadComment appears before the key-value line - if startPos > 0 { - if headComment := dec.extractHeadComment(startPos); headComment != "" { + // Use kvStartPos + 1 to ensure we look before the key, not at position 0 + headComment := dec.extractHeadComment(kvStartPos + 1) + log.Debug("processKeyValueIntoMap: extracted headComment: %q", headComment) + if headComment != "" { + // For the first key-value, attach head comment to root + if dec.firstKeyValue { + log.Debug("processKeyValueIntoMap: attaching head comment to root") + dec.rootMap.HeadComment = headComment + dec.firstKeyValue = false + } else { valueNode.HeadComment = headComment } } // LineComment appears after the value on the same line - if lineComment := dec.extractLineComment(endPos); lineComment != "" { + if lineComment := dec.extractLineComment(valueEndPos); lineComment != "" { valueNode.LineComment = lineComment } - + context := Context{} context = context.SingleChildContext(rootMap) diff --git a/pkg/yqlib/doc/usage/toml.md b/pkg/yqlib/doc/usage/toml.md index f0aec270..69586385 100644 --- a/pkg/yqlib/doc/usage/toml.md +++ b/pkg/yqlib/doc/usage/toml.md @@ -320,12 +320,71 @@ yq '.' sample.toml ``` will output ```yaml + # This is a comment + A = "hello" # inline comment +# This is a comment B = 12 -# Table comment [person] +# This is a comment name = "Tom" # name comment ``` +## Roundtrip: sample from web +Given a sample.toml file of: +```toml + +# This is a TOML document + +title = "TOML Example" + +[owner] +name = "Tom Preston-Werner" +dob = 1979-05-27T07:32:00-08:00 + +[database] +enabled = true +ports = [8000, 8001, 8002] +data = [["delta", "phi"], [3.14]] +temp_targets = { cpu = 79.5, case = 72.0 } + +[servers] + +[servers.alpha] +ip = "10.0.0.1" +role = "frontend" + +[servers.beta] +ip = "10.0.0.2" +role = "backend" + +``` +then +```bash +yq '.' sample.toml +``` +will output +```yaml +title = "TOML Example" + +[owner] +name = "Tom Preston-Werner" +dob = 1979-05-27T07:32:00-08:00 + +[database] +enabled = true +ports = [8000, 8001, 8002] +data = [["delta", "phi"], [3.14]] +temp_targets = { cpu = 79.5, case = 72.0 } + +[servers.alpha] +ip = "10.0.0.1" +role = "frontend" + +[servers.beta] +ip = "10.0.0.2" +role = "backend" +``` + diff --git a/pkg/yqlib/encoder_toml.go b/pkg/yqlib/encoder_toml.go index e7fc6a30..6b31b52b 100644 --- a/pkg/yqlib/encoder_toml.go +++ b/pkg/yqlib/encoder_toml.go @@ -104,6 +104,19 @@ func (te *tomlEncoder) formatScalar(node *CandidateNode) string { func (te *tomlEncoder) encodeRootMapping(w io.Writer, node *CandidateNode) error { te.wroteRootAttr = false // Reset state + // Write root head comment if present + if node.HeadComment != "" { + if _, err := w.Write([]byte("\n")); err != nil { + return err + } + if err := te.writeComment(w, node.HeadComment); err != nil { + return err + } + if _, err := w.Write([]byte("\n")); err != nil { + return err + } + } + // Preserve existing order by iterating Content for i := 0; i < len(node.Content); i += 2 { keyNode := node.Content[i] diff --git a/pkg/yqlib/toml_test.go b/pkg/yqlib/toml_test.go index 29fa8cd8..f756c630 100644 --- a/pkg/yqlib/toml_test.go +++ b/pkg/yqlib/toml_test.go @@ -225,6 +225,32 @@ B = 12 name = "Tom" # name comment ` +var sampleFromWeb = ` +# This is a TOML document + +title = "TOML Example" + +[owner] +name = "Tom Preston-Werner" +dob = 1979-05-27T07:32:00-08:00 + +[database] +enabled = true +ports = [8000, 8001, 8002] +data = [["delta", "phi"], [3.14]] +temp_targets = { cpu = 79.5, case = 72.0 } + +[servers] + +[servers.alpha] +ip = "10.0.0.1" +role = "frontend" + +[servers.beta] +ip = "10.0.0.2" +role = "backend" +` + var tomlScenarios = []formatScenario{ { skipDoc: true, @@ -503,6 +529,13 @@ var tomlScenarios = []formatScenario{ expected: rtComments, scenarioType: "roundtrip", }, + { + description: "Roundtrip: sample from web", + input: sampleFromWeb, + expression: ".", + expected: sampleFromWeb, + scenarioType: "roundtrip", + }, } func testTomlScenario(t *testing.T, s formatScenario) {