diff --git a/go.mod b/go.mod index af10ea9e..557b22c7 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/go-ini/ini v1.67.0 github.com/goccy/go-json v0.10.5 github.com/goccy/go-yaml v1.19.0 + github.com/hashicorp/hcl/v2 v2.24.0 github.com/jinzhu/copier v0.4.0 github.com/magiconair/properties v1.8.10 github.com/pelletier/go-toml/v2 v2.2.4 @@ -17,6 +18,7 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/yuin/gopher-lua v1.1.1 + github.com/zclconf/go-cty v1.16.3 go.yaml.in/yaml/v4 v4.0.0-rc.3 golang.org/x/net v0.47.0 golang.org/x/text v0.31.0 @@ -24,10 +26,16 @@ require ( ) require ( + github.com/agext/levenshtein v1.2.1 // indirect + github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mitchellh/go-wordwrap v1.0.1 // indirect + golang.org/x/mod v0.29.0 // indirect + golang.org/x/sync v0.18.0 // indirect golang.org/x/sys v0.38.0 // indirect + golang.org/x/tools v0.38.0 // indirect ) go 1.24.0 diff --git a/go.sum b/go.sum index 74816d07..11be9384 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,19 @@ github.com/a8m/envsubst v1.4.3 h1:kDF7paGK8QACWYaQo6KtyYBozY2jhQrTuNNuUxQkhJY= github.com/a8m/envsubst v1.4.3/go.mod h1:4jjHWQlZoaXPoLQUb7H2qT4iLkZDdmEQiOUogdUmqVU= +github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8= +github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/participle/v2 v2.1.4 h1:W/H79S8Sat/krZ3el6sQMvMaahJ+XcM9WSI2naI7w2U= github.com/alecthomas/participle/v2 v2.1.4/go.mod h1:8tqVbpTX20Ru4NfYQgZf4mP18eXPTBViyMWiArNEgGI= github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs= github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= +github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/elliotchance/orderedmap v1.8.0 h1:TrOREecvh3JbS+NCgwposXG5ZTFHtEsQiCGOhPElnMw= @@ -17,10 +22,16 @@ github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= +github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.19.0 h1:EmkZ9RIsX+Uq4DYFowegAuJo8+xdX3T/2dwNPXbxEYE= github.com/goccy/go-yaml v1.19.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/hashicorp/hcl/v2 v2.24.0 h1:2QJdZ454DSsYGoaE6QheQZjtKZSUs9Nh2izTWiwQxvE= +github.com/hashicorp/hcl/v2 v2.24.0/go.mod h1:oGoO1FIQYfn/AgyOhlg9qLC6/nOJPX3qGbkZpYAcqfM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -33,6 +44,8 @@ github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHP github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= @@ -50,16 +63,26 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +github.com/zclconf/go-cty v1.16.3 h1:osr++gw2T61A8KVYHoQiFbFd1Lh3JOCXc/jFLJXKTxk= +github.com/zclconf/go-cty v1.16.3/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= +github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo= +github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go= go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW8s2qTSe3wGBtvo0MbVQG/c5k8RE= gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx0Ydtgjl4cqmbRCsY4/+z4cYDeqwZTk6zog= diff --git a/pkg/yqlib/decoder_hcl.go b/pkg/yqlib/decoder_hcl.go new file mode 100644 index 00000000..395d8508 --- /dev/null +++ b/pkg/yqlib/decoder_hcl.go @@ -0,0 +1,225 @@ +package yqlib + +import ( + "fmt" + "io" + "strconv" + "strings" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/zclconf/go-cty/cty" +) + +type hclDecoder struct { + file *hcl.File + fileBytes []byte + readAnything bool + documentIndex uint +} + +func NewHclDecoder() Decoder { + return &hclDecoder{} +} + +func (dec *hclDecoder) Init(reader io.Reader) error { + data, err := io.ReadAll(reader) + if err != nil { + return err + } + file, diags := hclsyntax.ParseConfig(data, "input.hcl", hcl.Pos{Line: 1, Column: 1}) + if diags != nil && diags.HasErrors() { + return fmt.Errorf("hcl parse error: %w", diags) + } + dec.file = file + dec.fileBytes = data + dec.readAnything = false + dec.documentIndex = 0 + return nil +} + +func (dec *hclDecoder) Decode() (*CandidateNode, error) { + if dec.readAnything { + return nil, io.EOF + } + dec.readAnything = true + + if dec.file == nil { + return nil, fmt.Errorf("no hcl file parsed") + } + + root := &CandidateNode{Kind: MappingNode} + + // process attributes + body := dec.file.Body.(*hclsyntax.Body) + for name, attr := range body.Attributes { + keyNode := createStringScalarNode(name) + valNode := convertHclExprToNode(attr.Expr, dec.fileBytes) + root.AddKeyValueChild(keyNode, valNode) + } + + // process blocks + for _, block := range body.Blocks { + // build a key from type and labels to preserve identity + keyName := block.Type + if len(block.Labels) > 0 { + keyName = keyName + " " + strings.Join(block.Labels, " ") + } + keyNode := createStringScalarNode(keyName) + valueNode := hclBodyToNode(block.Body, dec.fileBytes) + root.AddKeyValueChild(keyNode, valueNode) + } + + dec.documentIndex++ + root.document = dec.documentIndex - 1 + return root, nil +} + +func hclBodyToNode(body *hclsyntax.Body, src []byte) *CandidateNode { + node := &CandidateNode{Kind: MappingNode} + for name, attr := range body.Attributes { + key := createStringScalarNode(name) + val := convertHclExprToNode(attr.Expr, src) + node.AddKeyValueChild(key, val) + } + for _, block := range body.Blocks { + keyName := block.Type + if len(block.Labels) > 0 { + keyName = keyName + " " + strings.Join(block.Labels, " ") + } + key := createStringScalarNode(keyName) + val := hclBodyToNode(block.Body, src) + node.AddKeyValueChild(key, val) + } + return node +} + +func convertHclExprToNode(expr hclsyntax.Expression, src []byte) *CandidateNode { + // handle literal values directly + switch e := expr.(type) { + case *hclsyntax.LiteralValueExpr: + v := e.Val + if v.IsNull() { + return createScalarNode(nil, "") + } + switch { + case v.Type().Equals(cty.String): + // prefer to extract exact source (to avoid extra quoting) when available + // Prefer the actual cty string value + s := v.AsString() + return createScalarNode(s, s) + case v.Type().Equals(cty.Bool): + b := v.True() + return createScalarNode(b, strconv.FormatBool(b)) + case v.Type() == cty.Number: + // represent numbers as float string + bf := v.AsBigFloat() + if bf == nil { + // fallback to string + return createScalarNode(nil, v.GoString()) + } + s := bf.Text('g', -1) + return createScalarNode(0.0, s) + case v.Type().IsTupleType() || v.Type().IsListType() || v.Type().IsSetType(): + seq := &CandidateNode{Kind: SequenceNode} + it := v.ElementIterator() + for it.Next() { + _, val := it.Element() + // convert cty.Value to a node by wrapping in literal expr via string representation + child := convertCtyValueToNode(val) + seq.AddChild(child) + } + return seq + case v.Type().IsMapType() || v.Type().IsObjectType(): + m := &CandidateNode{Kind: MappingNode} + it := v.ElementIterator() + for it.Next() { + key, val := it.Element() + keyStr := key.AsString() + keyNode := createStringScalarNode(keyStr) + valNode := convertCtyValueToNode(val) + m.AddKeyValueChild(keyNode, valNode) + } + return m + default: + // fallback to string + s := v.GoString() + return createScalarNode(nil, s) + } + case *hclsyntax.TemplateExpr: + // join parts; if single literal, return that string + var parts []string + for _, p := range e.Parts { + switch lp := p.(type) { + case *hclsyntax.LiteralValueExpr: + if lp.Val.Type().Equals(cty.String) { + parts = append(parts, lp.Val.AsString()) + } else { + parts = append(parts, lp.Val.GoString()) + } + default: + r := p.Range() + start := r.Start.Byte + end := r.End.Byte + if start > 0 && end >= start && end <= len(src) { + parts = append(parts, strings.TrimSpace(string(src[start-1:end]))) + } else { + parts = append(parts, fmt.Sprintf("%v", p)) + } + } + } + combined := strings.Join(parts, "") + return createScalarNode(combined, combined) + default: + // fallback: extract source text for the expression + r := expr.Range() + start := r.Start.Byte + end := r.End.Byte + if start > 0 && end >= start && end <= len(src) { + text := string(src[start-1 : end]) + return createScalarNode(nil, text) + } + return createScalarNode(nil, fmt.Sprintf("%v", expr)) + } +} + +func convertCtyValueToNode(v cty.Value) *CandidateNode { + if v.IsNull() { + return createScalarNode(nil, "") + } + switch { + case v.Type().Equals(cty.String): + return createScalarNode("", v.AsString()) + case v.Type().Equals(cty.Bool): + b := v.True() + return createScalarNode(b, strconv.FormatBool(b)) + case v.Type() == cty.Number: + bf := v.AsBigFloat() + if bf == nil { + return createScalarNode(nil, v.GoString()) + } + s := bf.Text('g', -1) + return createScalarNode(0.0, s) + case v.Type().IsTupleType() || v.Type().IsListType() || v.Type().IsSetType(): + seq := &CandidateNode{Kind: SequenceNode} + it := v.ElementIterator() + for it.Next() { + _, val := it.Element() + seq.AddChild(convertCtyValueToNode(val)) + } + return seq + case v.Type().IsMapType() || v.Type().IsObjectType(): + m := &CandidateNode{Kind: MappingNode} + it := v.ElementIterator() + for it.Next() { + key, val := it.Element() + keyNode := createStringScalarNode(key.AsString()) + valNode := convertCtyValueToNode(val) + m.AddKeyValueChild(keyNode, valNode) + } + return m + default: + return createScalarNode(nil, v.GoString()) + } + +} diff --git a/pkg/yqlib/hcl_test.go b/pkg/yqlib/hcl_test.go new file mode 100644 index 00000000..490e7c25 --- /dev/null +++ b/pkg/yqlib/hcl_test.go @@ -0,0 +1,28 @@ +package yqlib + +import ( + "testing" + + "github.com/mikefarah/yq/v4/test" +) + +var hclFormatScenarios = []formatScenario{ + { + description: "Simple decode", + input: `io_mode = "async"`, + expected: "io_mode: async\n", + scenarioType: "decode", + }, +} + +func testHclScenario(t *testing.T, s formatScenario) { + if s.scenarioType == "decode" { + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewHclDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description) + } +} + +func TestHclFormatScenarios(t *testing.T) { + for _, tt := range hclFormatScenarios { + testHclScenario(t, tt) + } +} diff --git a/project-words.txt b/project-words.txt index a00398f2..3f913d37 100644 --- a/project-words.txt +++ b/project-words.txt @@ -265,4 +265,8 @@ noprops nosh noshell tinygo -nonexistent \ No newline at end of file +nonexistent +hclsyntax +zclconf +cty +go-cty \ No newline at end of file