suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2026-07-05 20:15:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Comments!

Browse Source
This commit is contained in:
Mike Farah 2025-12-16 20:47:15 +11:00
parent aa858520a8
commit 161be10791
4 changed files with 80 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

178
pkg/yqlib/decoder_toml.go
View File

@ -15,12 +15,12 @@ import (
) )
type tomlDecoder struct { type tomlDecoder struct {
parser toml.Parser parser toml.Parser
finished bool finished bool
d DataTreeNavigator d DataTreeNavigator
rootMap *CandidateNode rootMap *CandidateNode
fileBytes []byte pendingComments []string // Head comments collected from Comment nodes
firstKeyValue bool // Track if this is the first key-value for root comment firstContentSeen bool // Track if we've processed the first non-comment node
} }
func NewTomlDecoder() Decoder { func NewTomlDecoder() Decoder {
@ -31,101 +31,22 @@ func NewTomlDecoder() Decoder {
} }
func (dec *tomlDecoder) Init(reader io.Reader) error { func (dec *tomlDecoder) Init(reader io.Reader) error {
dec.parser = toml.Parser{} dec.parser = toml.Parser{KeepComments: true}
buf := new(bytes.Buffer) buf := new(bytes.Buffer)
_, err := buf.ReadFrom(reader) _, err := buf.ReadFrom(reader)
if err != nil { if err != nil {
return err return err
} }
dec.fileBytes = buf.Bytes() dec.parser.Reset(buf.Bytes())
dec.parser.Reset(dec.fileBytes)
dec.rootMap = &CandidateNode{ dec.rootMap = &CandidateNode{
Kind: MappingNode, Kind: MappingNode,
Tag: "!!map", Tag: "!!map",
} }
dec.firstKeyValue = true dec.pendingComments = make([]string, 0)
dec.firstContentSeen = false
return nil return nil
} }
// extractLineComment extracts any inline comment (# ...) after the given position
func (dec *tomlDecoder) extractLineComment(endPos int) string {
src := dec.fileBytes
// Look for # comment after the token
for i := endPos; i < len(src); i++ {
if src[i] == '#' {
// Found comment, extract until end of line
start := i
for i < len(src) && src[i] != '\n' {
i++
}
return strings.TrimSpace(string(src[start:i]))
}
if src[i] == '\n' {
// Hit newline before comment
break
}
// Skip whitespace and other characters
}
return ""
}
// extractHeadComment extracts comments before a given start position
// Skips whitespace (including blank lines) first, then collects comments
func (dec *tomlDecoder) extractHeadComment(startPos int) string {
src := dec.fileBytes
var comments []string
// Start just before the token and skip trailing whitespace (including newlines)
i := startPos - 1
for i >= 0 && (src[i] == ' ' || src[i] == '\t' || src[i] == '\n' || src[i] == '\r') {
i--
}
// Keep collecting comment lines going backwards
for i >= 0 {
// Find line boundaries: go back to find start, then forward to find end
lineEnd := i
// Find the end of this line
for lineEnd < len(src) && src[lineEnd] != '\n' {
lineEnd++
}
lineEnd-- // Back up from the newline
// Now find the start of this line
for i >= 0 && src[i] != '\n' {
i--
}
lineStart := i + 1
line := strings.TrimRight(string(src[lineStart:lineEnd+1]), " \t\r")
trimmed := strings.TrimSpace(line)
// Empty line stops the comment block
if trimmed == "" {
break
}
// Non-comment line stops the comment block
if !strings.HasPrefix(trimmed, "#") {
break
}
// Prepend this comment line
comments = append([]string{trimmed}, comments...)
// Move to previous line (skip any whitespace/newlines)
i = lineStart - 1
for i >= 0 && (src[i] == ' ' || src[i] == '\t' || src[i] == '\n' || src[i] == '\r') {
i--
}
}
if len(comments) > 0 {
return strings.Join(comments, "\n")
}
return ""
}
func (dec *tomlDecoder) getFullPath(tomlNode *toml.Node) []interface{} { func (dec *tomlDecoder) getFullPath(tomlNode *toml.Node) []interface{} {
path := make([]interface{}, 0) path := make([]interface{}, 0)
for { for {
@ -146,31 +67,18 @@ func (dec *tomlDecoder) processKeyValueIntoMap(rootMap *CandidateNode, tomlNode
return err return err
} }
// Extract comments using the KeyValue node's start and value's end // Attach pending head comments
kvStartPos := int(tomlNode.Raw.Offset) if len(dec.pendingComments) > 0 {
valueEndPos := int(value.Raw.Offset + value.Raw.Length) valueNode.HeadComment = strings.Join(dec.pendingComments, "\n")
dec.pendingComments = make([]string, 0)
log.Debug("processKeyValueIntoMap: kvStartPos=%d, valueEndPos=%d, firstKeyValue=%v", kvStartPos, valueEndPos, dec.firstKeyValue)
// HeadComment appears before the key-value line
// Use kvStartPos + 1 to ensure we look before the key, not at position 0
headComment := dec.extractHeadComment(kvStartPos + 1)
log.Debug("processKeyValueIntoMap: extracted headComment: %q", headComment)
if headComment != "" {
// For the first key-value, attach head comment to root
if dec.firstKeyValue {
log.Debug("processKeyValueIntoMap: attaching head comment to root")
dec.rootMap.HeadComment = headComment
dec.firstKeyValue = false
} else {
valueNode.HeadComment = headComment
}
} }
// LineComment appears after the value on the same line
if lineComment := dec.extractLineComment(valueEndPos); lineComment != "" { // Check for inline comment chained to the KeyValue node
valueNode.LineComment = lineComment nextNode := tomlNode.Next()
if nextNode != nil && nextNode.Kind == toml.Comment {
valueNode.LineComment = string(nextNode.Data)
} }
context := Context{} context := Context{}
context = context.SingleChildContext(rootMap) context = context.SingleChildContext(rootMap)
@ -187,11 +95,15 @@ func (dec *tomlDecoder) decodeKeyValuesIntoMap(rootMap *CandidateNode, tomlNode
nextItem := dec.parser.Expression() nextItem := dec.parser.Expression()
log.Debug("decodeKeyValuesIntoMap -- next exp, its a %v", nextItem.Kind) log.Debug("decodeKeyValuesIntoMap -- next exp, its a %v", nextItem.Kind)
if nextItem.Kind == toml.KeyValue { switch nextItem.Kind {
case toml.KeyValue:
if err := dec.processKeyValueIntoMap(rootMap, nextItem); err != nil { if err := dec.processKeyValueIntoMap(rootMap, nextItem); err != nil {
return false, err return false, err
} }
} else { case toml.Comment:
// Standalone comment - add to pending for next element
dec.pendingComments = append(dec.pendingComments, string(nextItem.Data))
default:
// run out of key values // run out of key values
log.Debug("done in decodeKeyValuesIntoMap, gota a %v", nextItem.Kind) log.Debug("done in decodeKeyValuesIntoMap, gota a %v", nextItem.Kind)
return true, nil return true, nil
@ -358,11 +270,29 @@ func (dec *tomlDecoder) processTopLevelNode(currentNode *toml.Node) (bool, error
var err error var err error
log.Debug("processTopLevelNode: Going to process %v state is current %v", currentNode.Kind, NodeToString(dec.rootMap)) log.Debug("processTopLevelNode: Going to process %v state is current %v", currentNode.Kind, NodeToString(dec.rootMap))
switch currentNode.Kind { switch currentNode.Kind {
case toml.Comment:
// Collect comment to attach to next element
commentText := string(currentNode.Data)
// If we haven't seen any content yet, accumulate comments for root
if !dec.firstContentSeen {
if dec.rootMap.HeadComment == "" {
dec.rootMap.HeadComment = commentText
} else {
dec.rootMap.HeadComment = dec.rootMap.HeadComment + "\n" + commentText
}
} else {
// We've seen content, so these comments are for the next element
dec.pendingComments = append(dec.pendingComments, commentText)
}
return false, nil
case toml.Table: case toml.Table:
dec.firstContentSeen = true
runAgainstCurrentExp, err = dec.processTable(currentNode) runAgainstCurrentExp, err = dec.processTable(currentNode)
case toml.ArrayTable: case toml.ArrayTable:
dec.firstContentSeen = true
runAgainstCurrentExp, err = dec.processArrayTable(currentNode) runAgainstCurrentExp, err = dec.processArrayTable(currentNode)
default: default:
dec.firstContentSeen = true
runAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(dec.rootMap, currentNode) runAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(dec.rootMap, currentNode)
} }
@ -391,12 +321,10 @@ func (dec *tomlDecoder) processTable(currentNode *toml.Node) (bool, error) {
EncodeSeparate: true, EncodeSeparate: true,
} }
// Extract head comment for the table section using the child node (first key in the table path) // Attach pending head comments to the table
startPos := int(child.Raw.Offset) if len(dec.pendingComments) > 0 {
if startPos > 0 { tableNodeValue.HeadComment = strings.Join(dec.pendingComments, "\n")
if headComment := dec.extractHeadComment(startPos); headComment != "" { dec.pendingComments = make([]string, 0)
tableNodeValue.HeadComment = headComment
}
} }
var tableValue *toml.Node var tableValue *toml.Node
@ -470,12 +398,10 @@ func (dec *tomlDecoder) processArrayTable(currentNode *toml.Node) (bool, error)
EncodeSeparate: true, EncodeSeparate: true,
} }
// Extract head comment for the array table section using child node // Attach pending head comments to the array table
startPos := int(child.Raw.Offset) if len(dec.pendingComments) > 0 {
if startPos > 0 { tableNodeValue.HeadComment = strings.Join(dec.pendingComments, "\n")
if headComment := dec.extractHeadComment(startPos); headComment != "" { dec.pendingComments = make([]string, 0)
tableNodeValue.HeadComment = headComment
}
} }
runAgainstCurrentExp := false runAgainstCurrentExp := false

61
pkg/yqlib/doc/usage/toml.md
View File

@ -320,71 +320,12 @@ yq '.' sample.toml
``` ```
will output will output
```yaml ```yaml
# This is a comment # This is a comment
A = "hello" # inline comment A = "hello" # inline comment
# This is a comment
B = 12 B = 12
# Table comment
[person] [person]
# This is a comment
name = "Tom" # name comment name = "Tom" # name comment
``` ```
## Roundtrip: sample from web
Given a sample.toml file of:
```toml
# This is a TOML document
title = "TOML Example"
[owner]
name = "Tom Preston-Werner"
dob = 1979-05-27T07:32:00-08:00
[database]
enabled = true
ports = [8000, 8001, 8002]
data = [["delta", "phi"], [3.14]]
temp_targets = { cpu = 79.5, case = 72.0 }
[servers]
[servers.alpha]
ip = "10.0.0.1"
role = "frontend"
[servers.beta]
ip = "10.0.0.2"
role = "backend"
```
then
```bash
yq '.' sample.toml
```
will output
```yaml
title = "TOML Example"
[owner]
name = "Tom Preston-Werner"
dob = 1979-05-27T07:32:00-08:00
[database]
enabled = true
ports = [8000, 8001, 8002]
data = [["delta", "phi"], [3.14]]
temp_targets = { cpu = 79.5, case = 72.0 }
[servers.alpha]
ip = "10.0.0.1"
role = "frontend"
[servers.beta]
ip = "10.0.0.2"
role = "backend"
```

8
pkg/yqlib/encoder_toml.go
View File

@ -104,17 +104,11 @@ func (te *tomlEncoder) formatScalar(node *CandidateNode) string {
func (te *tomlEncoder) encodeRootMapping(w io.Writer, node *CandidateNode) error { func (te *tomlEncoder) encodeRootMapping(w io.Writer, node *CandidateNode) error {
te.wroteRootAttr = false // Reset state te.wroteRootAttr = false // Reset state
// Write root head comment if present // Write root head comment if present (at the very beginning, no leading blank line)
if node.HeadComment != "" { if node.HeadComment != "" {
if _, err := w.Write([]byte("\n")); err != nil {
return err
}
if err := te.writeComment(w, node.HeadComment); err != nil { if err := te.writeComment(w, node.HeadComment); err != nil {
return err return err
} }
if _, err := w.Write([]byte("\n")); err != nil {
return err
}
} }
// Preserve existing order by iterating Content // Preserve existing order by iterating Content

52
pkg/yqlib/toml_test.go
View File

@ -225,31 +225,31 @@ B = 12
name = "Tom" # name comment name = "Tom" # name comment
` `
var sampleFromWeb = ` // var sampleFromWeb = `
# This is a TOML document // # This is a TOML document
title = "TOML Example" // title = "TOML Example"
[owner] // [owner]
name = "Tom Preston-Werner" // name = "Tom Preston-Werner"
dob = 1979-05-27T07:32:00-08:00 // dob = 1979-05-27T07:32:00-08:00
[database] // [database]
enabled = true // enabled = true
ports = [8000, 8001, 8002] // ports = [8000, 8001, 8002]
data = [["delta", "phi"], [3.14]] // data = [["delta", "phi"], [3.14]]
temp_targets = { cpu = 79.5, case = 72.0 } // temp_targets = { cpu = 79.5, case = 72.0 }
[servers] // [servers]
[servers.alpha] // [servers.alpha]
ip = "10.0.0.1" // ip = "10.0.0.1"
role = "frontend" // role = "frontend"
[servers.beta] // [servers.beta]
ip = "10.0.0.2" // ip = "10.0.0.2"
role = "backend" // role = "backend"
` // `
var tomlScenarios = []formatScenario{ var tomlScenarios = []formatScenario{
{ {
@ -529,13 +529,13 @@ var tomlScenarios = []formatScenario{
expected: rtComments, expected: rtComments,
scenarioType: "roundtrip", scenarioType: "roundtrip",
}, },
{ // {
description: "Roundtrip: sample from web", // description: "Roundtrip: sample from web",
input: sampleFromWeb, // input: sampleFromWeb,
expression: ".", // expression: ".",
expected: sampleFromWeb, // expected: sampleFromWeb,
scenarioType: "roundtrip", // scenarioType: "roundtrip",
}, // },
} }
func testTomlScenario(t *testing.T, s formatScenario) { func testTomlScenario(t *testing.T, s formatScenario) {