diff --git a/agents.md b/agents.md index 76d1e51f..93de183e 100644 --- a/agents.md +++ b/agents.md @@ -135,6 +135,7 @@ Key methods: - Use the candidate_node style attribute to store style information for round-trip. Ask if this needs to be updated with new styles. - Use build tags for optional compilation - Add comprehensive tests +- Run the specific encoder/decoder test (e.g. _test.go) whenever you make ay changes to the encoder_ or decoder_ - Handle errors gracefully - Add the no build directive, like the xml encoder and decoder, that enables a minimal yq builds. e.g. `//go:build !yq_`. Be sure to also update the build_small-yq.sh and build-tinygo-yq.sh to not include the new format. diff --git a/pkg/yqlib/candidate_node.go b/pkg/yqlib/candidate_node.go index 7dee6303..1e119ae7 100644 --- a/pkg/yqlib/candidate_node.go +++ b/pkg/yqlib/candidate_node.go @@ -97,6 +97,9 @@ type CandidateNode struct { // (e.g. top level cross document merge). This property does not propagate to child nodes. EvaluateTogether bool IsMapKey bool + // For formats like HCL and TOML: indicates that child entries should be emitted as separate blocks/tables + // rather than consolidated into nested mappings (default behaviour) + EncodeSeparate bool } func (n *CandidateNode) CreateChild() *CandidateNode { @@ -407,6 +410,8 @@ func (n *CandidateNode) doCopy(cloneContent bool) *CandidateNode { EvaluateTogether: n.EvaluateTogether, IsMapKey: n.IsMapKey, + + EncodeSeparate: n.EncodeSeparate, } if cloneContent { diff --git a/pkg/yqlib/decoder_hcl.go b/pkg/yqlib/decoder_hcl.go index 1227a812..731f9ff3 100644 --- a/pkg/yqlib/decoder_hcl.go +++ b/pkg/yqlib/decoder_hcl.go @@ -161,8 +161,14 @@ func (dec *hclDecoder) Decode() (*CandidateNode, error) { } // process blocks + // Count blocks by type at THIS level to detect multiple separate blocks + blocksByType := make(map[string]int) for _, block := range body.Blocks { - addBlockToMapping(root, block, dec.fileBytes) + blocksByType[block.Type]++ + } + + for _, block := range body.Blocks { + addBlockToMapping(root, block, dec.fileBytes, blocksByType[block.Type] > 1) } dec.documentIndex++ @@ -187,14 +193,23 @@ func hclBodyToNode(body *hclsyntax.Body, src []byte) *CandidateNode { node.AddKeyValueChild(key, val) } + + // Process nested blocks, counting blocks by type at THIS level + // to detect which block types appear multiple times + blocksByType := make(map[string]int) for _, block := range body.Blocks { - addBlockToMapping(node, block, src) + blocksByType[block.Type]++ + } + + for _, block := range body.Blocks { + addBlockToMapping(node, block, src, blocksByType[block.Type] > 1) } return node } // addBlockToMapping nests block type and labels into the parent mapping, merging children. -func addBlockToMapping(parent *CandidateNode, block *hclsyntax.Block, src []byte) { +// isMultipleBlocksOfType indicates if there are multiple blocks of this type at THIS level +func addBlockToMapping(parent *CandidateNode, block *hclsyntax.Block, src []byte, isMultipleBlocksOfType bool) { bodyNode := hclBodyToNode(block.Body, src) current := parent @@ -208,6 +223,11 @@ func addBlockToMapping(parent *CandidateNode, block *hclsyntax.Block, src []byte } if typeNode == nil { _, typeNode = current.AddKeyValueChild(createStringScalarNode(block.Type), &CandidateNode{Kind: MappingNode}) + // Mark the type node if there are multiple blocks of this type at this level + // This tells the encoder to emit them as separate blocks rather than consolidating them + if isMultipleBlocksOfType { + typeNode.EncodeSeparate = true + } } current = typeNode diff --git a/pkg/yqlib/doc/usage/hcl.md b/pkg/yqlib/doc/usage/hcl.md index a0de9b41..4d636a70 100644 --- a/pkg/yqlib/doc/usage/hcl.md +++ b/pkg/yqlib/doc/usage/hcl.md @@ -175,3 +175,27 @@ message = "Hello, ${name}!" shouty_message = upper(message) ``` +## Roundtrip: Separate blocks with same name. +Given a sample.hcl file of: +```hcl +resource "aws_instance" "web" { + ami = "ami-12345" +} +resource "aws_instance" "db" { + ami = "ami-67890" +} +``` +then +```bash +yq sample.hcl +``` +will output +```hcl +resource "aws_instance" "web" { + ami = "ami-12345" +} +resource "aws_instance" "db" { + ami = "ami-67890" +} +``` + diff --git a/pkg/yqlib/encoder_hcl.go b/pkg/yqlib/encoder_hcl.go index 28ee6305..cc7d5806 100644 --- a/pkg/yqlib/encoder_hcl.go +++ b/pkg/yqlib/encoder_hcl.go @@ -438,6 +438,13 @@ func (he *hclEncoder) encodeBlockIfMapping(body *hclwrite.Body, key string, valu return false } + // If EncodeSeparate is set, emit children as separate blocks regardless of label extraction + if valueNode.EncodeSeparate { + if handled, _ := he.encodeMappingChildrenAsBlocks(body, key, valueNode); handled { + return true + } + } + // Try to extract block labels from a single-entry mapping chain if labels, bodyNode, ok := extractBlockLabels(valueNode); ok { if len(labels) > 1 && mappingChildrenAllMappings(bodyNode) { @@ -519,17 +526,46 @@ func (he *hclEncoder) encodeMappingChildrenAsBlocks(body *hclwrite.Body, blockTy return false, nil } + // Only emit as separate blocks if EncodeSeparate is true + // This allows the encoder to respect the original block structure preserved by the decoder + if !valueNode.EncodeSeparate { + return false, nil + } + for i := 0; i < len(valueNode.Content); i += 2 { childKey := valueNode.Content[i].Value childVal := valueNode.Content[i+1] - labels := []string{childKey} - if extraLabels, bodyNode, ok := extractBlockLabels(childVal); ok { - labels = append(labels, extraLabels...) - childVal = bodyNode - } - block := body.AppendNewBlock(blockType, labels) - if err := he.encodeNodeAttributes(block.Body(), childVal); err != nil { - return true, err + + // Check if this child also represents multiple blocks (all children are mappings) + if mappingChildrenAllMappings(childVal) { + // Recursively emit each grandchild as a separate block with extended labels + for j := 0; j < len(childVal.Content); j += 2 { + grandchildKey := childVal.Content[j].Value + grandchildVal := childVal.Content[j+1] + labels := []string{childKey, grandchildKey} + + // Try to extract additional labels if this is a single-entry chain + if extraLabels, bodyNode, ok := extractBlockLabels(grandchildVal); ok { + labels = append(labels, extraLabels...) + grandchildVal = bodyNode + } + + block := body.AppendNewBlock(blockType, labels) + if err := he.encodeNodeAttributes(block.Body(), grandchildVal); err != nil { + return true, err + } + } + } else { + // Single block with this child as label(s) + labels := []string{childKey} + if extraLabels, bodyNode, ok := extractBlockLabels(childVal); ok { + labels = append(labels, extraLabels...) + childVal = bodyNode + } + block := body.AppendNewBlock(blockType, labels) + if err := he.encodeNodeAttributes(block.Body(), childVal); err != nil { + return true, err + } } } diff --git a/pkg/yqlib/hcl_test.go b/pkg/yqlib/hcl_test.go index b4e3ebf2..b81934d9 100644 --- a/pkg/yqlib/hcl_test.go +++ b/pkg/yqlib/hcl_test.go @@ -345,6 +345,68 @@ var hclFormatScenarios = []formatScenario{ expected: "tags:\n - \"a\"\n - \"b\"\n", scenarioType: "decode", }, + { + description: "roundtrip list of objects", + skipDoc: true, + input: `items = [{ name = "a", value = 1 }, { name = "b", value = 2 }]`, + expected: "items = [{\n name = \"a\"\n value = 1\n }, {\n name = \"b\"\n value = 2\n}]\n", + scenarioType: "roundtrip", + }, + { + description: "roundtrip nested blocks with same name", + skipDoc: true, + input: "database \"primary\" {\n host = \"localhost\"\n port = 5432\n}\ndatabase \"replica\" {\n host = \"replica.local\"\n port = 5433\n}", + expected: "database \"primary\" {\n host = \"localhost\"\n port = 5432\n}\ndatabase \"replica\" {\n host = \"replica.local\"\n port = 5433\n}\n", + scenarioType: "roundtrip", + }, + { + description: "roundtrip mixed nested structure", + skipDoc: true, + input: "servers \"web\" {\n addresses = [\"10.0.1.1\", \"10.0.1.2\"]\n port = 8080\n}", + expected: "servers \"web\" {\n addresses = [\"10.0.1.1\", \"10.0.1.2\"]\n port = 8080\n}\n", + scenarioType: "roundtrip", + }, + { + description: "roundtrip null value", + skipDoc: true, + input: `value = null`, + expected: "value = null\n", + scenarioType: "roundtrip", + }, + { + description: "roundtrip empty list", + skipDoc: true, + input: `items = []`, + expected: "items = []\n", + scenarioType: "roundtrip", + }, + { + description: "roundtrip empty object", + skipDoc: true, + input: `config = {}`, + expected: "config = {}\n", + scenarioType: "roundtrip", + }, + { + description: "Roundtrip: Separate blocks with same name.", + input: "resource \"aws_instance\" \"web\" {\n ami = \"ami-12345\"\n}\nresource \"aws_instance\" \"db\" {\n ami = \"ami-67890\"\n}", + expected: "resource \"aws_instance\" \"web\" {\n ami = \"ami-12345\"\n}\nresource \"aws_instance\" \"db\" {\n ami = \"ami-67890\"\n}\n", + scenarioType: "roundtrip", + }, + { + description: "roundtrip deeply nested structure", + skipDoc: true, + input: "app \"database\" \"primary\" \"connection\" {\n host = \"db.local\"\n port = 5432\n}", + expected: "app \"database\" \"primary\" \"connection\" {\n host = \"db.local\"\n port = 5432\n}\n", + scenarioType: "roundtrip", + }, + { + description: "roundtrip with leading comments", + skipDoc: true, + input: "# Main config\nenabled = true\nport = 8080", + expected: "# Main config\nenabled = true\nport = 8080\n", + scenarioType: "roundtrip", + }, } func testHclScenario(t *testing.T, s formatScenario) {