From 7923d04bdd3602cc1603abfa766ccdca7be27551 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sun, 7 Dec 2025 20:08:29 +1100 Subject: [PATCH] Refining --- examples/sample.hcl | 8 +++ pkg/yqlib/decoder_hcl.go | 29 ++++++--- pkg/yqlib/encoder_hcl.go | 127 ++++++++++++++++++++++++++++++++++++--- pkg/yqlib/hcl_test.go | 36 +++++++++-- project-words.txt | 3 + 5 files changed, 179 insertions(+), 24 deletions(-) create mode 100644 examples/sample.hcl diff --git a/examples/sample.hcl b/examples/sample.hcl new file mode 100644 index 00000000..ec3efc46 --- /dev/null +++ b/examples/sample.hcl @@ -0,0 +1,8 @@ +# Arithmetic with literals and application-provided variables +sum = 1 + addend + +# String interpolation and templates +message = "Hello, ${name}!" + +# Application-provided functions +shouty_message = upper(message) \ No newline at end of file diff --git a/pkg/yqlib/decoder_hcl.go b/pkg/yqlib/decoder_hcl.go index 116d27b6..c2e34856 100644 --- a/pkg/yqlib/decoder_hcl.go +++ b/pkg/yqlib/decoder_hcl.go @@ -128,7 +128,9 @@ func convertHclExprToNode(expr hclsyntax.Expression, src []byte) *CandidateNode // prefer to extract exact source (to avoid extra quoting) when available // Prefer the actual cty string value s := v.AsString() - return createScalarNode(s, s) + node := createScalarNode(s, s) + // Don't set style for regular quoted strings - let YAML handle naturally + return node case v.Type().Equals(cty.Bool): b := v.True() return createScalarNode(b, strconv.FormatBool(b)) @@ -192,8 +194,8 @@ func convertHclExprToNode(expr hclsyntax.Expression, src []byte) *CandidateNode r := item.KeyExpr.Range() start := r.Start.Byte end := r.End.Byte - if start > 0 && end >= start && end <= len(src) { - keyNode := createStringScalarNode(strings.TrimSpace(string(src[start-1 : end]))) + if start >= 0 && end >= start && end <= len(src) { + keyNode := createStringScalarNode(strings.TrimSpace(string(src[start:end]))) valNode := convertHclExprToNode(item.ValueExpr, src) m.AddKeyValueChild(keyNode, valNode) } @@ -206,7 +208,7 @@ func convertHclExprToNode(expr hclsyntax.Expression, src []byte) *CandidateNode } return m case *hclsyntax.TemplateExpr: - // join parts; if single literal, return that string + // Reconstruct template string, preserving ${} syntax for interpolations var parts []string for _, p := range e.Parts { switch lp := p.(type) { @@ -217,18 +219,24 @@ func convertHclExprToNode(expr hclsyntax.Expression, src []byte) *CandidateNode parts = append(parts, lp.Val.GoString()) } default: + // Non-literal expression - reconstruct with ${} wrapper r := p.Range() start := r.Start.Byte end := r.End.Byte - if start > 0 && end >= start && end <= len(src) { - parts = append(parts, strings.TrimSpace(string(src[start-1:end]))) + if start >= 0 && end >= start && end <= len(src) { + exprText := string(src[start:end]) + parts = append(parts, "${"+exprText+"}") } else { - parts = append(parts, fmt.Sprintf("%v", p)) + parts = append(parts, fmt.Sprintf("${%v}", p)) } } } combined := strings.Join(parts, "") - return createScalarNode(combined, combined) + node := createScalarNode(combined, combined) + // Set DoubleQuotedStyle for all templates (which includes all quoted strings in HCL) + // This ensures HCL roundtrips preserve quotes, and YAML properly quotes strings with ${} + node.Style = DoubleQuotedStyle + return node default: // try to evaluate the expression (handles unary, binary ops, etc.) val, diags := expr.Value(nil) @@ -240,8 +248,9 @@ func convertHclExprToNode(expr hclsyntax.Expression, src []byte) *CandidateNode r := expr.Range() start := r.Start.Byte end := r.End.Byte - if start > 0 && end >= start && end <= len(src) { - text := string(src[start-1 : end]) + if start >= 0 && end >= start && end <= len(src) { + text := string(src[start:end]) + // Unquoted identifier - no style return createStringScalarNode(text) } return createStringScalarNode(fmt.Sprintf("%v", expr)) diff --git a/pkg/yqlib/encoder_hcl.go b/pkg/yqlib/encoder_hcl.go index 870cefa2..fd41bc3b 100644 --- a/pkg/yqlib/encoder_hcl.go +++ b/pkg/yqlib/encoder_hcl.go @@ -6,7 +6,10 @@ import ( "fmt" "io" "regexp" + "strings" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" hclwrite "github.com/hashicorp/hcl/v2/hclwrite" "github.com/zclconf/go-cty/cty" ) @@ -55,6 +58,36 @@ func (he *hclEncoder) compactSpacing(input []byte) []byte { return re.ReplaceAll(input, []byte("$1 =")) } +// Helper runes for unquoted identifiers +func isHCLIdentifierStart(r rune) bool { + return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || r == '_' +} + +func isHCLIdentifierPart(r rune) bool { + return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' || r == '-' +} + +// isValidHCLIdentifier checks if a string is a valid HCL identifier (unquoted) +func isValidHCLIdentifier(s string) bool { + if s == "" { + return false + } + // HCL identifiers must start with a letter or underscore + // and contain only letters, digits, underscores, and hyphens + for i, r := range s { + if i == 0 { + if !isHCLIdentifierStart(r) { + return false + } + continue + } + if !isHCLIdentifierPart(r) { + return false + } + } + return true +} + // encodeNode encodes a CandidateNode directly to HCL, preserving style information func (he *hclEncoder) encodeNode(body *hclwrite.Body, node *CandidateNode) error { if node.Kind != MappingNode { @@ -75,11 +108,79 @@ func (he *hclEncoder) encodeNode(body *hclwrite.Body, node *CandidateNode) error } } else { // Render as attribute: key = value - ctyValue, err := nodeToCtyValue(valueNode) - if err != nil { - return err + // Check the style to determine how to encode strings + if valueNode.Kind == ScalarNode && valueNode.Tag == "!!str" { + // Check style: DoubleQuotedStyle means template, no style could be unquoted or regular + // To distinguish unquoted from regular, we check if the value is a valid identifier + if valueNode.Style&DoubleQuotedStyle != 0 && strings.Contains(valueNode.Value, "${") { + // Template string - use raw tokens to preserve ${} syntax + tokens := hclwrite.Tokens{ + {Type: hclsyntax.TokenOQuote, Bytes: []byte{'"'}}, + } + // Parse the string and add tokens + for i := 0; i < len(valueNode.Value); i++ { + if i < len(valueNode.Value)-1 && valueNode.Value[i] == '$' && valueNode.Value[i+1] == '{' { + // Start of template interpolation + tokens = append(tokens, &hclwrite.Token{ + Type: hclsyntax.TokenTemplateInterp, + Bytes: []byte("${"), + }) + i++ // skip the '{' + // Find the matching '}' + start := i + 1 + depth := 1 + for i++; i < len(valueNode.Value) && depth > 0; i++ { + switch valueNode.Value[i] { + case '{': + depth++ + case '}': + depth-- + } + } + i-- // back up to the '}' + interpExpr := valueNode.Value[start:i] + tokens = append(tokens, &hclwrite.Token{ + Type: hclsyntax.TokenIdent, + Bytes: []byte(interpExpr), + }) + tokens = append(tokens, &hclwrite.Token{ + Type: hclsyntax.TokenTemplateSeqEnd, + Bytes: []byte("}"), + }) + } else { + // Regular character + tokens = append(tokens, &hclwrite.Token{ + Type: hclsyntax.TokenQuotedLit, + Bytes: []byte{valueNode.Value[i]}, + }) + } + } + tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenCQuote, Bytes: []byte{'"'}}) + body.SetAttributeRaw(key, tokens) + } else if isValidHCLIdentifier(valueNode.Value) && valueNode.Style == 0 { + // Could be unquoted identifier - but only if it came from HCL originally + // For safety, only use traversal if style is explicitly 0 (not set) + // This avoids treating strings from YAML as unquoted + traversal := hcl.Traversal{ + hcl.TraverseRoot{Name: valueNode.Value}, + } + body.SetAttributeTraversal(key, traversal) + } else { + // Regular quoted string - use cty.Value + ctyValue, err := nodeToCtyValue(valueNode) + if err != nil { + return err + } + body.SetAttributeValue(key, ctyValue) + } + } else { + // Non-string value - use cty.Value + ctyValue, err := nodeToCtyValue(valueNode) + if err != nil { + return err + } + body.SetAttributeValue(key, ctyValue) } - body.SetAttributeValue(key, ctyValue) } } return nil @@ -96,11 +197,21 @@ func (he *hclEncoder) encodeNodeAttributes(body *hclwrite.Body, node *CandidateN valueNode := node.Content[i+1] key := keyNode.Value - ctyValue, err := nodeToCtyValue(valueNode) - if err != nil { - return err + // Check if this is an unquoted identifier (no DoubleQuotedStyle) + if valueNode.Kind == ScalarNode && valueNode.Tag == "!!str" && valueNode.Style&DoubleQuotedStyle == 0 { + // Unquoted identifier - use traversal + traversal := hcl.Traversal{ + hcl.TraverseRoot{Name: valueNode.Value}, + } + body.SetAttributeTraversal(key, traversal) + } else { + // Quoted value or non-string - use cty.Value + ctyValue, err := nodeToCtyValue(valueNode) + if err != nil { + return err + } + body.SetAttributeValue(key, ctyValue) } - body.SetAttributeValue(key, ctyValue) } return nil } diff --git a/pkg/yqlib/hcl_test.go b/pkg/yqlib/hcl_test.go index 1c8777fe..3d17eb5a 100644 --- a/pkg/yqlib/hcl_test.go +++ b/pkg/yqlib/hcl_test.go @@ -10,9 +10,33 @@ var hclFormatScenarios = []formatScenario{ { description: "Simple decode", input: `io_mode = "async"`, + expected: "io_mode: \"async\"\n", + scenarioType: "decode", + }, + { + description: "Simple decode, no quotes", + input: `io_mode = async`, expected: "io_mode: async\n", scenarioType: "decode", }, + { + description: "Simple roundtrip, no quotes", + input: `io_mode = async`, + expected: "io_mode = async\n", + scenarioType: "roundtrip", + }, + { + description: "Template decode", + input: `message = "Hello, ${name}!"`, + expected: "message: \"Hello, ${name}!\"\n", + scenarioType: "decode", + }, + { + description: "Template roundtrip", + input: `message = "Hello, ${name}!"`, + expected: "message = \"Hello, ${name}!\"\n", + scenarioType: "roundtrip", + }, { description: "number attribute", input: `port = 8080`, @@ -34,13 +58,13 @@ var hclFormatScenarios = []formatScenario{ { description: "list of strings", input: `tags = ["a", "b"]`, - expected: "tags:\n - a\n - b\n", + expected: "tags:\n - \"a\"\n - \"b\"\n", scenarioType: "decode", }, { description: "object/map attribute", input: `obj = { a = 1, b = "two" }`, - expected: "obj: {a: 1, b: two}\n", + expected: "obj: {a: 1, b: \"two\"}\n", scenarioType: "decode", }, { @@ -52,7 +76,7 @@ var hclFormatScenarios = []formatScenario{ { description: "multiple attributes", input: "name = \"app\"\nversion = 1\nenabled = true", - expected: "name: app\nversion: 1\nenabled: true\n", + expected: "name: \"app\"\nversion: 1\nenabled: true\n", scenarioType: "decode", }, { @@ -76,19 +100,19 @@ var hclFormatScenarios = []formatScenario{ { description: "nested object", input: `config = { db = { host = "localhost", port = 5432 } }`, - expected: "config: {db: {host: localhost, port: 5432}}\n", + expected: "config: {db: {host: \"localhost\", port: 5432}}\n", scenarioType: "decode", }, { description: "mixed list", input: `values = [1, "two", true]`, - expected: "values:\n - 1\n - two\n - true\n", + expected: "values:\n - 1\n - \"two\"\n - true\n", scenarioType: "decode", }, { description: "block with labels", input: `resource "aws_instance" "example" { ami = "ami-12345" }`, - expected: "resource aws_instance example:\n ami: ami-12345\n", + expected: "resource aws_instance example:\n ami: \"ami-12345\"\n", scenarioType: "decode", }, { diff --git a/project-words.txt b/project-words.txt index 577ab603..afdbc8fd 100644 --- a/project-words.txt +++ b/project-words.txt @@ -189,8 +189,11 @@ risentveber rmescandon Rosey roundtrip +roundtrips Roundtrip roundtripping +Interp +interp runningvms sadface selfupdate