mirror of
https://github.com/mikefarah/yq.git
synced 2026-08-25 00:44:43 +08:00
Compare commits
62
Commits
hcl
..
toml_encoder
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
207bec6b29 | ||
|
|
7198d16575 | ||
|
|
5d6c2047cf | ||
|
|
7f60daad20 | ||
|
|
b7cbe59fd7 | ||
|
|
9fa353b123 | ||
|
|
c6ecad1546 | ||
|
|
56eb3655b8 | ||
|
|
1de4ec59f2 | ||
|
|
c132c32731 | ||
|
|
0914121d29 | ||
|
|
aa5134e645 | ||
|
|
4d620bfa26 | ||
|
|
b8d90fd574 | ||
|
|
c1b81f1a03 | ||
|
|
ea40e14fb1 | ||
|
|
b974d973ee | ||
|
|
66ec487792 | ||
|
|
161be10791 | ||
|
|
aa858520a8 | ||
|
|
ac2889c296 | ||
|
|
626624af7b | ||
|
|
b0d2522f80 | ||
|
|
2ee38e15b6 | ||
|
|
4e9d5e8e48 | ||
|
|
1338b521ff | ||
|
|
3a5323824f | ||
|
|
8780172b33 | ||
|
|
5f9bf8d241 | ||
|
|
065b200af9 | ||
|
|
745a7ffb3c | ||
|
|
a305d706d4 | ||
|
|
0671ccd2cc | ||
|
|
4d8cd450bd | ||
|
|
d2d657eacc | ||
|
|
f4fd8c585a | ||
|
|
e4bf8a1e0a | ||
|
|
fd405749f9 | ||
|
|
51ddf8d357 | ||
|
|
77eccfd3db | ||
|
|
554bf5a2f2 | ||
|
|
8162f3a100 | ||
|
|
48707369a0 | ||
|
|
4f72c37de7 | ||
|
|
795f9c954c | ||
|
|
3d35386ad9 | ||
|
|
154a4ace01 | ||
|
|
effdfe1221 | ||
|
|
8af768a015 | ||
|
|
5f3dcb1ccf | ||
|
|
6270c29f54 | ||
|
|
df3101ce53 | ||
|
|
65e79845d4 | ||
|
|
b4d8131197 | ||
|
|
c75a2fad86 | ||
|
|
8d430cf41c | ||
|
|
2e96a28270 | ||
|
|
656f07d0c2 | ||
|
|
1852073f29 | ||
|
|
7d2c774e8f | ||
|
|
69076dfe81 | ||
|
|
9e17cd683f |
@@ -0,0 +1 @@
|
||||
When you find a bug - make sure to include a new test that exposes the bug, as well as the fix for the bug itself.
|
||||
@@ -43,6 +43,7 @@ yq*.snap
|
||||
|
||||
test.yml
|
||||
test*.yml
|
||||
test*.tf
|
||||
test*.xml
|
||||
test*.toml
|
||||
test*.yaml
|
||||
|
||||
@@ -363,6 +363,8 @@ gah install yq
|
||||
- [Load content from other files](https://mikefarah.gitbook.io/yq/operators/load)
|
||||
- [Convert to/from json/ndjson](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert)
|
||||
- [Convert to/from xml](https://mikefarah.gitbook.io/yq/v/v4.x/usage/xml)
|
||||
- [Convert to/from hcl (terraform)](https://mikefarah.gitbook.io/yq/v/v4.x/usage/hcl)
|
||||
- [Convert to/from toml](https://mikefarah.gitbook.io/yq/v/v4.x/usage/toml)
|
||||
- [Convert to/from properties](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties)
|
||||
- [Convert to/from csv/tsv](https://mikefarah.gitbook.io/yq/usage/csv-tsv)
|
||||
- [General shell completion scripts (bash/zsh/fish/powershell)](https://mikefarah.gitbook.io/yq/v/v4.x/commands/shell-completion)
|
||||
@@ -413,7 +415,7 @@ Flags:
|
||||
-h, --help help for yq
|
||||
-I, --indent int sets indent level for output (default 2)
|
||||
-i, --inplace update the file in place of first file given.
|
||||
-p, --input-format string [auto|a|yaml|y|json|j|props|p|csv|c|tsv|t|xml|x|base64|uri|toml|lua|l|ini|i] parse format for input. (default "auto")
|
||||
-p, --input-format string [auto|a|yaml|y|json|j|props|p|csv|c|tsv|t|xml|x|base64|uri|toml|hcl|h|lua|l|ini|i] parse format for input. (default "auto")
|
||||
--lua-globals output keys as top-level global variables
|
||||
--lua-prefix string prefix (default "return ")
|
||||
--lua-suffix string suffix (default ";\n")
|
||||
@@ -422,7 +424,7 @@ Flags:
|
||||
-N, --no-doc Don't print document separators (---)
|
||||
-0, --nul-output Use NUL char to separate values. If unwrap scalar is also set, fail if unwrapped scalar contains NUL char.
|
||||
-n, --null-input Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.
|
||||
-o, --output-format string [auto|a|yaml|y|json|j|props|p|csv|c|tsv|t|xml|x|base64|uri|toml|shell|s|lua|l|ini|i] output format type. (default "auto")
|
||||
-o, --output-format string [auto|a|yaml|y|json|j|props|p|csv|c|tsv|t|xml|x|base64|uri|toml|hcl|h|shell|s|lua|l|ini|i] output format type. (default "auto")
|
||||
-P, --prettyPrint pretty print, shorthand for '... style = ""'
|
||||
--properties-array-brackets use [x] in array paths (e.g. for SpringBoot)
|
||||
--properties-separator string separator to use between keys and values (default " = ")
|
||||
|
||||
@@ -6,6 +6,7 @@ setUp() {
|
||||
rm test*.csv 2>/dev/null || true
|
||||
rm test*.tsv 2>/dev/null || true
|
||||
rm test*.xml 2>/dev/null || true
|
||||
rm test*.tf 2>/dev/null || true
|
||||
}
|
||||
|
||||
testInputProperties() {
|
||||
@@ -255,4 +256,61 @@ EOM
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testInputTerraform() {
|
||||
cat >test.tf <<EOL
|
||||
resource "aws_s3_bucket" "example" {
|
||||
bucket = "my-bucket"
|
||||
tags = {
|
||||
Environment = "Dev"
|
||||
Project = "Test"
|
||||
}
|
||||
}
|
||||
EOL
|
||||
|
||||
read -r -d '' expected << EOM
|
||||
resource "aws_s3_bucket" "example" {
|
||||
bucket = "my-bucket"
|
||||
tags = {
|
||||
Environment = "Dev"
|
||||
Project = "Test"
|
||||
}
|
||||
}
|
||||
EOM
|
||||
|
||||
X=$(./yq test.tf)
|
||||
assertEquals "$expected" "$X"
|
||||
|
||||
X=$(./yq ea test.tf)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testInputTerraformGithubAction() {
|
||||
cat >test.tf <<EOL
|
||||
resource "aws_s3_bucket" "example" {
|
||||
bucket = "my-bucket"
|
||||
|
||||
tags = {
|
||||
Environment = "Dev"
|
||||
Project = "Test"
|
||||
}
|
||||
}
|
||||
EOL
|
||||
|
||||
read -r -d '' expected << EOM
|
||||
resource "aws_s3_bucket" "example" {
|
||||
bucket = "my-bucket"
|
||||
tags = {
|
||||
Environment = "Dev"
|
||||
Project = "Test"
|
||||
}
|
||||
}
|
||||
EOM
|
||||
|
||||
X=$(cat /dev/null | ./yq test.tf)
|
||||
assertEquals "$expected" "$X"
|
||||
|
||||
X=$(cat /dev/null | ./yq ea test.tf)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
source ./scripts/shunit2
|
||||
@@ -1,3 +1,16 @@
|
||||
# General rules
|
||||
✅ **DO:**
|
||||
- You can use ./yq with the `--debug-node-info` flag to get a deeper understanding of the ast.
|
||||
- run ./scripts/format.sh then ./scripts/check.sh to format, then validate linting and spelling
|
||||
- Add comprehensive tests to cover the changes
|
||||
- Run test suite to ensure there is no regression
|
||||
- Use UK english spelling (e.g. Colorisation not Colorization)
|
||||
|
||||
❌ **DON'T:**
|
||||
- Git add or commit
|
||||
|
||||
|
||||
|
||||
# Adding a New Encoder/Decoder
|
||||
|
||||
This guide explains how to add support for a new format (encoder/decoder) to yq without modifying `candidate_node.go`.
|
||||
@@ -135,6 +148,7 @@ Key methods:
|
||||
- Use the candidate_node style attribute to store style information for round-trip. Ask if this needs to be updated with new styles.
|
||||
- Use build tags for optional compilation
|
||||
- Add comprehensive tests
|
||||
- Run the specific encoder/decoder test (e.g. <format>_test.go) whenever you make ay changes to the encoder_<format> or decoder_<format>
|
||||
- Handle errors gracefully
|
||||
- Add the no build directive, like the xml encoder and decoder, that enables a minimal yq builds. e.g. `//go:build !yq_<format>`. Be sure to also update the build_small-yq.sh and build-tinygo-yq.sh to not include the new format.
|
||||
|
||||
|
||||
@@ -206,6 +206,7 @@ func configureEncoder() (yqlib.Encoder, error) {
|
||||
yqlib.ConfiguredYamlPreferences.ColorsEnabled = colorsEnabled
|
||||
yqlib.ConfiguredJSONPreferences.ColorsEnabled = colorsEnabled
|
||||
yqlib.ConfiguredHclPreferences.ColorsEnabled = colorsEnabled
|
||||
yqlib.ConfiguredTomlPreferences.ColorsEnabled = colorsEnabled
|
||||
|
||||
yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators
|
||||
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@ var (
|
||||
GitDescribe string
|
||||
|
||||
// Version is main version number that is being run at the moment.
|
||||
Version = "v4.49.2"
|
||||
Version = "v4.50.1"
|
||||
|
||||
// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
|
||||
// then it means that it is a final release. Otherwise, this is a pre-release
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
# main.tf
|
||||
|
||||
# Define required providers and minimum Terraform version
|
||||
terraform {
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "~> 5.0"
|
||||
}
|
||||
}
|
||||
required_version = ">= 1.2"
|
||||
}
|
||||
|
||||
# Configure the AWS provider
|
||||
provider "aws" {
|
||||
region = var.aws_region
|
||||
}
|
||||
|
||||
# Define an S3 bucket resource
|
||||
resource "aws_s3_bucket" "example_bucket" {
|
||||
bucket = var.bucket_name
|
||||
|
||||
tags = {
|
||||
Environment = "Development"
|
||||
Project = "TerraformExample"
|
||||
}
|
||||
}
|
||||
+24
-4
@@ -1,6 +1,26 @@
|
||||
[[fruits]]
|
||||
|
||||
[animals]
|
||||
|
||||
[[fruits.varieties]] # nested array of tables
|
||||
name = "red delicious"
|
||||
# This is a TOML document
|
||||
|
||||
title = "TOML Example"
|
||||
|
||||
[owner]
|
||||
name = "Tom Preston-Werner"
|
||||
dob = 1979-05-27T07:32:00-08:00
|
||||
|
||||
[database]
|
||||
enabled = true
|
||||
ports = [ 8000, 8001, 8002 ]
|
||||
data = [ ["delta", "phi"], [3.14] ]
|
||||
temp_targets = { cpu = 79.5, case = 72.0 }
|
||||
|
||||
[servers]
|
||||
|
||||
[servers.alpha]
|
||||
ip = "10.0.0.1"
|
||||
role = "frontend"
|
||||
|
||||
[servers.beta]
|
||||
ip = "10.0.0.2"
|
||||
role = "backend"
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ require (
|
||||
github.com/fatih/color v1.18.0
|
||||
github.com/go-ini/ini v1.67.0
|
||||
github.com/goccy/go-json v0.10.5
|
||||
github.com/goccy/go-yaml v1.19.0
|
||||
github.com/goccy/go-yaml v1.19.1
|
||||
github.com/hashicorp/hcl/v2 v2.24.0
|
||||
github.com/jinzhu/copier v0.4.0
|
||||
github.com/magiconair/properties v1.8.10
|
||||
@@ -18,10 +18,10 @@ require (
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/spf13/pflag v1.0.10
|
||||
github.com/yuin/gopher-lua v1.1.1
|
||||
github.com/zclconf/go-cty v1.16.3
|
||||
github.com/zclconf/go-cty v1.17.0
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.3
|
||||
golang.org/x/net v0.47.0
|
||||
golang.org/x/text v0.31.0
|
||||
golang.org/x/net v0.48.0
|
||||
golang.org/x/text v0.32.0
|
||||
gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
|
||||
)
|
||||
|
||||
@@ -33,10 +33,10 @@ require (
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
|
||||
golang.org/x/mod v0.29.0 // indirect
|
||||
golang.org/x/sync v0.18.0 // indirect
|
||||
golang.org/x/sys v0.38.0 // indirect
|
||||
golang.org/x/tools v0.38.0 // indirect
|
||||
golang.org/x/mod v0.30.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.39.0 // indirect
|
||||
golang.org/x/tools v0.39.0 // indirect
|
||||
)
|
||||
|
||||
go 1.24.0
|
||||
|
||||
@@ -26,8 +26,8 @@ github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
|
||||
github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
|
||||
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
||||
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/goccy/go-yaml v1.19.0 h1:EmkZ9RIsX+Uq4DYFowegAuJo8+xdX3T/2dwNPXbxEYE=
|
||||
github.com/goccy/go-yaml v1.19.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
|
||||
github.com/goccy/go-yaml v1.19.1 h1:3rG3+v8pkhRqoQ/88NYNMHYVGYztCOCIZ7UQhu7H+NE=
|
||||
github.com/goccy/go-yaml v1.19.1/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/hashicorp/hcl/v2 v2.24.0 h1:2QJdZ454DSsYGoaE6QheQZjtKZSUs9Nh2izTWiwQxvE=
|
||||
@@ -63,26 +63,26 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
|
||||
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
|
||||
github.com/zclconf/go-cty v1.16.3 h1:osr++gw2T61A8KVYHoQiFbFd1Lh3JOCXc/jFLJXKTxk=
|
||||
github.com/zclconf/go-cty v1.16.3/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE=
|
||||
github.com/zclconf/go-cty v1.17.0 h1:seZvECve6XX4tmnvRzWtJNHdscMtYEx5R7bnnVyd/d0=
|
||||
github.com/zclconf/go-cty v1.17.0/go.mod h1:wqFzcImaLTI6A5HfsRwB0nj5n0MRZFwmey8YoFPPs3U=
|
||||
github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo=
|
||||
github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||
golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
|
||||
golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
|
||||
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
|
||||
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
|
||||
golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
|
||||
golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
|
||||
golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
|
||||
golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
|
||||
golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
|
||||
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
|
||||
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
|
||||
golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
|
||||
golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
|
||||
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
|
||||
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
|
||||
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
|
||||
golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
|
||||
golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW8s2qTSe3wGBtvo0MbVQG/c5k8RE=
|
||||
gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx0Ydtgjl4cqmbRCsY4/+z4cYDeqwZTk6zog=
|
||||
|
||||
@@ -97,6 +97,9 @@ type CandidateNode struct {
|
||||
// (e.g. top level cross document merge). This property does not propagate to child nodes.
|
||||
EvaluateTogether bool
|
||||
IsMapKey bool
|
||||
// For formats like HCL and TOML: indicates that child entries should be emitted as separate blocks/tables
|
||||
// rather than consolidated into nested mappings (default behaviour)
|
||||
EncodeSeparate bool
|
||||
}
|
||||
|
||||
func (n *CandidateNode) CreateChild() *CandidateNode {
|
||||
@@ -407,6 +410,8 @@ func (n *CandidateNode) doCopy(cloneContent bool) *CandidateNode {
|
||||
|
||||
EvaluateTogether: n.EvaluateTogether,
|
||||
IsMapKey: n.IsMapKey,
|
||||
|
||||
EncodeSeparate: n.EncodeSeparate,
|
||||
}
|
||||
|
||||
if cloneContent {
|
||||
@@ -460,6 +465,9 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignP
|
||||
n.Anchor = other.Anchor
|
||||
}
|
||||
|
||||
// Preserve EncodeSeparate flag for format-specific encoding hints
|
||||
n.EncodeSeparate = other.EncodeSeparate
|
||||
|
||||
// merge will pickup the style of the new thing
|
||||
// when autocreating nodes
|
||||
|
||||
|
||||
+29
-59
@@ -64,44 +64,6 @@ func extractLineComment(src []byte, endPos int) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// extractLeadingComments extracts comments from the very beginning of the file.
|
||||
// It returns the comment text and the byte position of the last character in that leading block.
|
||||
func extractLeadingComments(src []byte) (string, int) {
|
||||
var comments []string
|
||||
i := 0
|
||||
|
||||
// Skip leading whitespace
|
||||
for i < len(src) && (src[i] == ' ' || src[i] == '\t' || src[i] == '\n' || src[i] == '\r') {
|
||||
i++
|
||||
}
|
||||
|
||||
lastPos := -1
|
||||
|
||||
// Extract comment lines from the start
|
||||
for i < len(src) && src[i] == '#' {
|
||||
lineStart := i
|
||||
// Find end of line
|
||||
for i < len(src) && src[i] != '\n' {
|
||||
i++
|
||||
}
|
||||
lastPos = i - 1
|
||||
comments = append(comments, strings.TrimSpace(string(src[lineStart:i])))
|
||||
// Skip newline
|
||||
if i < len(src) && src[i] == '\n' {
|
||||
i++
|
||||
}
|
||||
// Skip whitespace between comment lines
|
||||
for i < len(src) && (src[i] == ' ' || src[i] == '\t' || src[i] == '\n' || src[i] == '\r') {
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
if len(comments) > 0 {
|
||||
return strings.Join(comments, "\n"), lastPos
|
||||
}
|
||||
return "", -1
|
||||
}
|
||||
|
||||
// extractHeadComment extracts comments before a given start position
|
||||
func extractHeadComment(src []byte, startPos int) string {
|
||||
var comments []string
|
||||
@@ -174,15 +136,9 @@ func (dec *hclDecoder) Decode() (*CandidateNode, error) {
|
||||
|
||||
root := &CandidateNode{Kind: MappingNode}
|
||||
|
||||
// Extract file-level head comments (comments at the very beginning of the file)
|
||||
leadingComment, _ := extractLeadingComments(dec.fileBytes)
|
||||
leadingUsed := false
|
||||
if leadingComment != "" {
|
||||
root.HeadComment = leadingComment
|
||||
}
|
||||
|
||||
// process attributes in declaration order
|
||||
body := dec.file.Body.(*hclsyntax.Body)
|
||||
firstAttr := true
|
||||
for _, attrWithName := range sortedAttributes(body.Attributes) {
|
||||
keyNode := createStringScalarNode(attrWithName.Name)
|
||||
valNode := convertHclExprToNode(attrWithName.Attr.Expr, dec.fileBytes)
|
||||
@@ -190,17 +146,11 @@ func (dec *hclDecoder) Decode() (*CandidateNode, error) {
|
||||
// Attach comments if any
|
||||
attrRange := attrWithName.Attr.Range()
|
||||
headComment := extractHeadComment(dec.fileBytes, attrRange.Start.Byte)
|
||||
if !leadingUsed && leadingComment != "" {
|
||||
// Avoid double-applying the leading file comment to the first attribute
|
||||
switch headComment {
|
||||
case leadingComment:
|
||||
headComment = ""
|
||||
case "":
|
||||
headComment = leadingComment
|
||||
}
|
||||
leadingUsed = true
|
||||
}
|
||||
if headComment != "" {
|
||||
if firstAttr && headComment != "" {
|
||||
// For the first attribute, apply its head comment to the root
|
||||
root.HeadComment = headComment
|
||||
firstAttr = false
|
||||
} else if headComment != "" {
|
||||
keyNode.HeadComment = headComment
|
||||
}
|
||||
if lineComment := extractLineComment(dec.fileBytes, attrRange.End.Byte); lineComment != "" {
|
||||
@@ -211,8 +161,14 @@ func (dec *hclDecoder) Decode() (*CandidateNode, error) {
|
||||
}
|
||||
|
||||
// process blocks
|
||||
// Count blocks by type at THIS level to detect multiple separate blocks
|
||||
blocksByType := make(map[string]int)
|
||||
for _, block := range body.Blocks {
|
||||
addBlockToMapping(root, block, dec.fileBytes)
|
||||
blocksByType[block.Type]++
|
||||
}
|
||||
|
||||
for _, block := range body.Blocks {
|
||||
addBlockToMapping(root, block, dec.fileBytes, blocksByType[block.Type] > 1)
|
||||
}
|
||||
|
||||
dec.documentIndex++
|
||||
@@ -237,14 +193,23 @@ func hclBodyToNode(body *hclsyntax.Body, src []byte) *CandidateNode {
|
||||
|
||||
node.AddKeyValueChild(key, val)
|
||||
}
|
||||
|
||||
// Process nested blocks, counting blocks by type at THIS level
|
||||
// to detect which block types appear multiple times
|
||||
blocksByType := make(map[string]int)
|
||||
for _, block := range body.Blocks {
|
||||
addBlockToMapping(node, block, src)
|
||||
blocksByType[block.Type]++
|
||||
}
|
||||
|
||||
for _, block := range body.Blocks {
|
||||
addBlockToMapping(node, block, src, blocksByType[block.Type] > 1)
|
||||
}
|
||||
return node
|
||||
}
|
||||
|
||||
// addBlockToMapping nests block type and labels into the parent mapping, merging children.
|
||||
func addBlockToMapping(parent *CandidateNode, block *hclsyntax.Block, src []byte) {
|
||||
// isMultipleBlocksOfType indicates if there are multiple blocks of this type at THIS level
|
||||
func addBlockToMapping(parent *CandidateNode, block *hclsyntax.Block, src []byte, isMultipleBlocksOfType bool) {
|
||||
bodyNode := hclBodyToNode(block.Body, src)
|
||||
current := parent
|
||||
|
||||
@@ -258,6 +223,11 @@ func addBlockToMapping(parent *CandidateNode, block *hclsyntax.Block, src []byte
|
||||
}
|
||||
if typeNode == nil {
|
||||
_, typeNode = current.AddKeyValueChild(createStringScalarNode(block.Type), &CandidateNode{Kind: MappingNode})
|
||||
// Mark the type node if there are multiple blocks of this type at this level
|
||||
// This tells the encoder to emit them as separate blocks rather than consolidating them
|
||||
if isMultipleBlocksOfType {
|
||||
typeNode.EncodeSeparate = true
|
||||
}
|
||||
}
|
||||
current = typeNode
|
||||
|
||||
|
||||
+70
-15
@@ -8,16 +8,19 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
toml "github.com/pelletier/go-toml/v2/unstable"
|
||||
)
|
||||
|
||||
type tomlDecoder struct {
|
||||
parser toml.Parser
|
||||
finished bool
|
||||
d DataTreeNavigator
|
||||
rootMap *CandidateNode
|
||||
parser toml.Parser
|
||||
finished bool
|
||||
d DataTreeNavigator
|
||||
rootMap *CandidateNode
|
||||
pendingComments []string // Head comments collected from Comment nodes
|
||||
firstContentSeen bool // Track if we've processed the first non-comment node
|
||||
}
|
||||
|
||||
func NewTomlDecoder() Decoder {
|
||||
@@ -28,7 +31,7 @@ func NewTomlDecoder() Decoder {
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) Init(reader io.Reader) error {
|
||||
dec.parser = toml.Parser{}
|
||||
dec.parser = toml.Parser{KeepComments: true}
|
||||
buf := new(bytes.Buffer)
|
||||
_, err := buf.ReadFrom(reader)
|
||||
if err != nil {
|
||||
@@ -39,6 +42,8 @@ func (dec *tomlDecoder) Init(reader io.Reader) error {
|
||||
Kind: MappingNode,
|
||||
Tag: "!!map",
|
||||
}
|
||||
dec.pendingComments = make([]string, 0)
|
||||
dec.firstContentSeen = false
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -56,13 +61,24 @@ func (dec *tomlDecoder) getFullPath(tomlNode *toml.Node) []interface{} {
|
||||
func (dec *tomlDecoder) processKeyValueIntoMap(rootMap *CandidateNode, tomlNode *toml.Node) error {
|
||||
value := tomlNode.Value()
|
||||
path := dec.getFullPath(value.Next())
|
||||
log.Debug("processKeyValueIntoMap: %v", path)
|
||||
|
||||
valueNode, err := dec.decodeNode(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Attach pending head comments
|
||||
if len(dec.pendingComments) > 0 {
|
||||
valueNode.HeadComment = strings.Join(dec.pendingComments, "\n")
|
||||
dec.pendingComments = make([]string, 0)
|
||||
}
|
||||
|
||||
// Check for inline comment chained to the KeyValue node
|
||||
nextNode := tomlNode.Next()
|
||||
if nextNode != nil && nextNode.Kind == toml.Comment {
|
||||
valueNode.LineComment = string(nextNode.Data)
|
||||
}
|
||||
|
||||
context := Context{}
|
||||
context = context.SingleChildContext(rootMap)
|
||||
|
||||
@@ -79,11 +95,15 @@ func (dec *tomlDecoder) decodeKeyValuesIntoMap(rootMap *CandidateNode, tomlNode
|
||||
nextItem := dec.parser.Expression()
|
||||
log.Debug("decodeKeyValuesIntoMap -- next exp, its a %v", nextItem.Kind)
|
||||
|
||||
if nextItem.Kind == toml.KeyValue {
|
||||
switch nextItem.Kind {
|
||||
case toml.KeyValue:
|
||||
if err := dec.processKeyValueIntoMap(rootMap, nextItem); err != nil {
|
||||
return false, err
|
||||
}
|
||||
} else {
|
||||
case toml.Comment:
|
||||
// Standalone comment - add to pending for next element
|
||||
dec.pendingComments = append(dec.pendingComments, string(nextItem.Data))
|
||||
default:
|
||||
// run out of key values
|
||||
log.Debug("done in decodeKeyValuesIntoMap, gota a %v", nextItem.Kind)
|
||||
return true, nil
|
||||
@@ -250,11 +270,29 @@ func (dec *tomlDecoder) processTopLevelNode(currentNode *toml.Node) (bool, error
|
||||
var err error
|
||||
log.Debug("processTopLevelNode: Going to process %v state is current %v", currentNode.Kind, NodeToString(dec.rootMap))
|
||||
switch currentNode.Kind {
|
||||
case toml.Comment:
|
||||
// Collect comment to attach to next element
|
||||
commentText := string(currentNode.Data)
|
||||
// If we haven't seen any content yet, accumulate comments for root
|
||||
if !dec.firstContentSeen {
|
||||
if dec.rootMap.HeadComment == "" {
|
||||
dec.rootMap.HeadComment = commentText
|
||||
} else {
|
||||
dec.rootMap.HeadComment = dec.rootMap.HeadComment + "\n" + commentText
|
||||
}
|
||||
} else {
|
||||
// We've seen content, so these comments are for the next element
|
||||
dec.pendingComments = append(dec.pendingComments, commentText)
|
||||
}
|
||||
return false, nil
|
||||
case toml.Table:
|
||||
dec.firstContentSeen = true
|
||||
runAgainstCurrentExp, err = dec.processTable(currentNode)
|
||||
case toml.ArrayTable:
|
||||
dec.firstContentSeen = true
|
||||
runAgainstCurrentExp, err = dec.processArrayTable(currentNode)
|
||||
default:
|
||||
dec.firstContentSeen = true
|
||||
runAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(dec.rootMap, currentNode)
|
||||
}
|
||||
|
||||
@@ -264,7 +302,8 @@ func (dec *tomlDecoder) processTopLevelNode(currentNode *toml.Node) (bool, error
|
||||
|
||||
func (dec *tomlDecoder) processTable(currentNode *toml.Node) (bool, error) {
|
||||
log.Debug("Enter processTable")
|
||||
fullPath := dec.getFullPath(currentNode.Child())
|
||||
child := currentNode.Child()
|
||||
fullPath := dec.getFullPath(child)
|
||||
log.Debug("fullpath: %v", fullPath)
|
||||
|
||||
c := Context{}
|
||||
@@ -276,9 +315,16 @@ func (dec *tomlDecoder) processTable(currentNode *toml.Node) (bool, error) {
|
||||
}
|
||||
|
||||
tableNodeValue := &CandidateNode{
|
||||
Kind: MappingNode,
|
||||
Tag: "!!map",
|
||||
Content: make([]*CandidateNode, 0),
|
||||
Kind: MappingNode,
|
||||
Tag: "!!map",
|
||||
Content: make([]*CandidateNode, 0),
|
||||
EncodeSeparate: true,
|
||||
}
|
||||
|
||||
// Attach pending head comments to the table
|
||||
if len(dec.pendingComments) > 0 {
|
||||
tableNodeValue.HeadComment = strings.Join(dec.pendingComments, "\n")
|
||||
dec.pendingComments = make([]string, 0)
|
||||
}
|
||||
|
||||
var tableValue *toml.Node
|
||||
@@ -330,7 +376,8 @@ func (dec *tomlDecoder) arrayAppend(context Context, path []interface{}, rhsNode
|
||||
|
||||
func (dec *tomlDecoder) processArrayTable(currentNode *toml.Node) (bool, error) {
|
||||
log.Debug("Enter processArrayTable")
|
||||
fullPath := dec.getFullPath(currentNode.Child())
|
||||
child := currentNode.Child()
|
||||
fullPath := dec.getFullPath(child)
|
||||
log.Debug("Fullpath: %v", fullPath)
|
||||
|
||||
c := Context{}
|
||||
@@ -346,9 +393,17 @@ func (dec *tomlDecoder) processArrayTable(currentNode *toml.Node) (bool, error)
|
||||
hasValue := dec.parser.NextExpression()
|
||||
|
||||
tableNodeValue := &CandidateNode{
|
||||
Kind: MappingNode,
|
||||
Tag: "!!map",
|
||||
Kind: MappingNode,
|
||||
Tag: "!!map",
|
||||
EncodeSeparate: true,
|
||||
}
|
||||
|
||||
// Attach pending head comments to the array table
|
||||
if len(dec.pendingComments) > 0 {
|
||||
tableNodeValue.HeadComment = strings.Join(dec.pendingComments, "\n")
|
||||
dec.pendingComments = make([]string, 0)
|
||||
}
|
||||
|
||||
runAgainstCurrentExp := false
|
||||
// if the next value is a ArrayTable or Table, then its not part of this declaration (not a key value pair)
|
||||
// so lets leave that expression for the next round of parsing
|
||||
|
||||
@@ -79,6 +79,46 @@ will output
|
||||
c: cat
|
||||
```
|
||||
|
||||
## Get the top (root) parent
|
||||
Use negative numbers to get the top parents
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
a:
|
||||
b:
|
||||
c: cat
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.a.b.c | parent(-1)' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
a:
|
||||
b:
|
||||
c: cat
|
||||
```
|
||||
|
||||
## Root
|
||||
Alias for parent(-1), returns the top level parent. This is usually the document node.
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
a:
|
||||
b:
|
||||
c: cat
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.a.b.c | root' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
a:
|
||||
b:
|
||||
c: cat
|
||||
```
|
||||
|
||||
## N-th parent
|
||||
You can optionally supply the number of levels to go up for the parent, the default being 1.
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ HCL is commonly used in HashiCorp tools like Terraform for configuration files.
|
||||
- String interpolation and expressions (preserved without quotes)
|
||||
- Comments (leading, head, and line comments)
|
||||
- Nested structures (maps and lists)
|
||||
- Syntax colorization when enabled
|
||||
- Syntax colorisation when enabled
|
||||
|
||||
|
||||
## Parse HCL
|
||||
@@ -175,3 +175,27 @@ message = "Hello, ${name}!"
|
||||
shouty_message = upper(message)
|
||||
```
|
||||
|
||||
## Roundtrip: Separate blocks with same name.
|
||||
Given a sample.hcl file of:
|
||||
```hcl
|
||||
resource "aws_instance" "web" {
|
||||
ami = "ami-12345"
|
||||
}
|
||||
resource "aws_instance" "db" {
|
||||
ami = "ami-67890"
|
||||
}
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq sample.hcl
|
||||
```
|
||||
will output
|
||||
```hcl
|
||||
resource "aws_instance" "web" {
|
||||
ami = "ami-12345"
|
||||
}
|
||||
resource "aws_instance" "db" {
|
||||
ami = "ami-67890"
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -7,5 +7,5 @@ HCL is commonly used in HashiCorp tools like Terraform for configuration files.
|
||||
- String interpolation and expressions (preserved without quotes)
|
||||
- Comments (leading, head, and line comments)
|
||||
- Nested structures (maps and lists)
|
||||
- Syntax colorization when enabled
|
||||
- Syntax colorisation when enabled
|
||||
|
||||
|
||||
@@ -141,3 +141,191 @@ will output
|
||||
dependencies: {}
|
||||
```
|
||||
|
||||
## Roundtrip: inline table attribute
|
||||
Given a sample.toml file of:
|
||||
```toml
|
||||
name = { first = "Tom", last = "Preston-Werner" }
|
||||
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.' sample.toml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
name = { first = "Tom", last = "Preston-Werner" }
|
||||
```
|
||||
|
||||
## Roundtrip: table section
|
||||
Given a sample.toml file of:
|
||||
```toml
|
||||
[owner.contact]
|
||||
name = "Tom"
|
||||
age = 36
|
||||
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.' sample.toml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
[owner.contact]
|
||||
name = "Tom"
|
||||
age = 36
|
||||
```
|
||||
|
||||
## Roundtrip: array of tables
|
||||
Given a sample.toml file of:
|
||||
```toml
|
||||
[[fruits]]
|
||||
name = "apple"
|
||||
[[fruits.varieties]]
|
||||
name = "red delicious"
|
||||
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.' sample.toml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
[[fruits]]
|
||||
name = "apple"
|
||||
[[fruits.varieties]]
|
||||
name = "red delicious"
|
||||
```
|
||||
|
||||
## Roundtrip: arrays and scalars
|
||||
Given a sample.toml file of:
|
||||
```toml
|
||||
A = ["hello", ["world", "again"]]
|
||||
B = 12
|
||||
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.' sample.toml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
A = ["hello", ["world", "again"]]
|
||||
B = 12
|
||||
```
|
||||
|
||||
## Roundtrip: simple
|
||||
Given a sample.toml file of:
|
||||
```toml
|
||||
A = "hello"
|
||||
B = 12
|
||||
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.' sample.toml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
A = "hello"
|
||||
B = 12
|
||||
```
|
||||
|
||||
## Roundtrip: deep paths
|
||||
Given a sample.toml file of:
|
||||
```toml
|
||||
[person]
|
||||
name = "hello"
|
||||
address = "12 cat st"
|
||||
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.' sample.toml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
[person]
|
||||
name = "hello"
|
||||
address = "12 cat st"
|
||||
```
|
||||
|
||||
## Roundtrip: empty array
|
||||
Given a sample.toml file of:
|
||||
```toml
|
||||
A = []
|
||||
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.' sample.toml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
A = []
|
||||
```
|
||||
|
||||
## Roundtrip: sample table
|
||||
Given a sample.toml file of:
|
||||
```toml
|
||||
var = "x"
|
||||
|
||||
[owner.contact]
|
||||
name = "Tom Preston-Werner"
|
||||
age = 36
|
||||
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.' sample.toml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
var = "x"
|
||||
|
||||
[owner.contact]
|
||||
name = "Tom Preston-Werner"
|
||||
age = 36
|
||||
```
|
||||
|
||||
## Roundtrip: empty table
|
||||
Given a sample.toml file of:
|
||||
```toml
|
||||
[dependencies]
|
||||
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.' sample.toml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
[dependencies]
|
||||
```
|
||||
|
||||
## Roundtrip: comments
|
||||
Given a sample.toml file of:
|
||||
```toml
|
||||
# This is a comment
|
||||
A = "hello" # inline comment
|
||||
B = 12
|
||||
|
||||
# Table comment
|
||||
[person]
|
||||
name = "Tom" # name comment
|
||||
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.' sample.toml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
# This is a comment
|
||||
A = "hello" # inline comment
|
||||
B = 12
|
||||
|
||||
# Table comment
|
||||
[person]
|
||||
name = "Tom" # name comment
|
||||
```
|
||||
|
||||
|
||||
+72
-24
@@ -19,6 +19,11 @@ type hclEncoder struct {
|
||||
prefs HclPreferences
|
||||
}
|
||||
|
||||
// commentPathSep is used to join path segments when collecting comments.
|
||||
// It uses a rarely used ASCII control character to avoid collisions with
|
||||
// normal key names (including dots).
|
||||
const commentPathSep = "\x1e"
|
||||
|
||||
// NewHclEncoder creates a new HCL encoder
|
||||
func NewHclEncoder(prefs HclPreferences) Encoder {
|
||||
return &hclEncoder{prefs: prefs}
|
||||
@@ -38,6 +43,9 @@ func (he *hclEncoder) PrintLeadingContent(_ io.Writer, _ string) error {
|
||||
|
||||
func (he *hclEncoder) Encode(writer io.Writer, node *CandidateNode) error {
|
||||
log.Debugf("I need to encode %v", NodeToString(node))
|
||||
if node.Kind == ScalarNode {
|
||||
return writeString(writer, node.Value+"\n")
|
||||
}
|
||||
|
||||
f := hclwrite.NewEmptyFile()
|
||||
body := f.Body()
|
||||
@@ -84,7 +92,7 @@ func (he *hclEncoder) collectComments(node *CandidateNode, prefix string, commen
|
||||
if node.Kind == MappingNode {
|
||||
// Collect root-level head comment if at root (prefix is empty)
|
||||
if prefix == "" && node.HeadComment != "" {
|
||||
commentMap[".head"] = node.HeadComment
|
||||
commentMap[joinCommentPath("__root__", "head")] = node.HeadComment
|
||||
}
|
||||
|
||||
for i := 0; i < len(node.Content); i += 2 {
|
||||
@@ -93,21 +101,18 @@ func (he *hclEncoder) collectComments(node *CandidateNode, prefix string, commen
|
||||
key := keyNode.Value
|
||||
|
||||
// Create a path for this key
|
||||
path := key
|
||||
if prefix != "" {
|
||||
path = prefix + "." + key
|
||||
}
|
||||
path := joinCommentPath(prefix, key)
|
||||
|
||||
// Store comments from the key (head comments appear before the attribute)
|
||||
if keyNode.HeadComment != "" {
|
||||
commentMap[path+".head"] = keyNode.HeadComment
|
||||
commentMap[joinCommentPath(path, "head")] = keyNode.HeadComment
|
||||
}
|
||||
// Store comments from the value (line comments appear after the value)
|
||||
if valueNode.LineComment != "" {
|
||||
commentMap[path+".line"] = valueNode.LineComment
|
||||
commentMap[joinCommentPath(path, "line")] = valueNode.LineComment
|
||||
}
|
||||
if valueNode.FootComment != "" {
|
||||
commentMap[path+".foot"] = valueNode.FootComment
|
||||
commentMap[joinCommentPath(path, "foot")] = valueNode.FootComment
|
||||
}
|
||||
|
||||
// Recurse into nested mappings
|
||||
@@ -118,14 +123,22 @@ func (he *hclEncoder) collectComments(node *CandidateNode, prefix string, commen
|
||||
}
|
||||
}
|
||||
|
||||
// joinCommentPath concatenates path segments using commentPathSep, safely handling empty prefixes.
|
||||
func joinCommentPath(prefix, segment string) string {
|
||||
if prefix == "" {
|
||||
return segment
|
||||
}
|
||||
return prefix + commentPathSep + segment
|
||||
}
|
||||
|
||||
// injectComments adds collected comments back into the HCL output
|
||||
func (he *hclEncoder) injectComments(output []byte, commentMap map[string]string) []byte {
|
||||
// Convert output to string for easier manipulation
|
||||
result := string(output)
|
||||
|
||||
// Root-level head comment (stored as ".head")
|
||||
// Root-level head comment (stored on the synthetic __root__/head path)
|
||||
for path, comment := range commentMap {
|
||||
if path == ".head" {
|
||||
if path == joinCommentPath("__root__", "head") {
|
||||
trimmed := strings.TrimSpace(comment)
|
||||
if trimmed != "" && !strings.HasPrefix(result, trimmed) {
|
||||
result = trimmed + "\n" + result
|
||||
@@ -135,13 +148,13 @@ func (he *hclEncoder) injectComments(output []byte, commentMap map[string]string
|
||||
|
||||
// Attribute head comments: insert above matching assignment
|
||||
for path, comment := range commentMap {
|
||||
parts := strings.Split(path, ".")
|
||||
if len(parts) != 2 {
|
||||
parts := strings.Split(path, commentPathSep)
|
||||
if len(parts) < 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
key := parts[0]
|
||||
commentType := parts[1]
|
||||
commentType := parts[len(parts)-1]
|
||||
key := parts[len(parts)-2]
|
||||
if commentType != "head" || key == "" {
|
||||
continue
|
||||
}
|
||||
@@ -271,7 +284,6 @@ func isHCLIdentifierPart(r rune) bool {
|
||||
return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' || r == '-'
|
||||
}
|
||||
|
||||
// isValidHCLIdentifier checks if a string is a valid HCL identifier (unquoted)
|
||||
func isValidHCLIdentifier(s string) bool {
|
||||
if s == "" {
|
||||
return false
|
||||
@@ -438,6 +450,13 @@ func (he *hclEncoder) encodeBlockIfMapping(body *hclwrite.Body, key string, valu
|
||||
return false
|
||||
}
|
||||
|
||||
// If EncodeSeparate is set, emit children as separate blocks regardless of label extraction
|
||||
if valueNode.EncodeSeparate {
|
||||
if handled, _ := he.encodeMappingChildrenAsBlocks(body, key, valueNode); handled {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// Try to extract block labels from a single-entry mapping chain
|
||||
if labels, bodyNode, ok := extractBlockLabels(valueNode); ok {
|
||||
if len(labels) > 1 && mappingChildrenAllMappings(bodyNode) {
|
||||
@@ -474,7 +493,7 @@ func (he *hclEncoder) encodeBlockIfMapping(body *hclwrite.Body, key string, valu
|
||||
// encodeNode encodes a CandidateNode directly to HCL, preserving style information
|
||||
func (he *hclEncoder) encodeNode(body *hclwrite.Body, node *CandidateNode) error {
|
||||
if node.Kind != MappingNode {
|
||||
return fmt.Errorf("HCL encoder expects a mapping at the root level")
|
||||
return fmt.Errorf("HCL encoder expects a mapping at the root level, got %v", kindToString(node.Kind))
|
||||
}
|
||||
|
||||
for i := 0; i < len(node.Content); i += 2 {
|
||||
@@ -519,17 +538,46 @@ func (he *hclEncoder) encodeMappingChildrenAsBlocks(body *hclwrite.Body, blockTy
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Only emit as separate blocks if EncodeSeparate is true
|
||||
// This allows the encoder to respect the original block structure preserved by the decoder
|
||||
if !valueNode.EncodeSeparate {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
for i := 0; i < len(valueNode.Content); i += 2 {
|
||||
childKey := valueNode.Content[i].Value
|
||||
childVal := valueNode.Content[i+1]
|
||||
labels := []string{childKey}
|
||||
if extraLabels, bodyNode, ok := extractBlockLabels(childVal); ok {
|
||||
labels = append(labels, extraLabels...)
|
||||
childVal = bodyNode
|
||||
}
|
||||
block := body.AppendNewBlock(blockType, labels)
|
||||
if err := he.encodeNodeAttributes(block.Body(), childVal); err != nil {
|
||||
return true, err
|
||||
|
||||
// Check if this child also represents multiple blocks (all children are mappings)
|
||||
if mappingChildrenAllMappings(childVal) {
|
||||
// Recursively emit each grandchild as a separate block with extended labels
|
||||
for j := 0; j < len(childVal.Content); j += 2 {
|
||||
grandchildKey := childVal.Content[j].Value
|
||||
grandchildVal := childVal.Content[j+1]
|
||||
labels := []string{childKey, grandchildKey}
|
||||
|
||||
// Try to extract additional labels if this is a single-entry chain
|
||||
if extraLabels, bodyNode, ok := extractBlockLabels(grandchildVal); ok {
|
||||
labels = append(labels, extraLabels...)
|
||||
grandchildVal = bodyNode
|
||||
}
|
||||
|
||||
block := body.AppendNewBlock(blockType, labels)
|
||||
if err := he.encodeNodeAttributes(block.Body(), grandchildVal); err != nil {
|
||||
return true, err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Single block with this child as label(s)
|
||||
labels := []string{childKey}
|
||||
if extraLabels, bodyNode, ok := extractBlockLabels(childVal); ok {
|
||||
labels = append(labels, extraLabels...)
|
||||
childVal = bodyNode
|
||||
}
|
||||
block := body.AppendNewBlock(blockType, labels)
|
||||
if err := he.encodeNodeAttributes(block.Body(), childVal); err != nil {
|
||||
return true, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+667
-4
@@ -1,22 +1,56 @@
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/fatih/color"
|
||||
)
|
||||
|
||||
type tomlEncoder struct {
|
||||
wroteRootAttr bool // Track if we wrote root-level attributes before tables
|
||||
prefs TomlPreferences
|
||||
}
|
||||
|
||||
func NewTomlEncoder() Encoder {
|
||||
return &tomlEncoder{}
|
||||
return NewTomlEncoderWithPrefs(ConfiguredTomlPreferences)
|
||||
}
|
||||
|
||||
func NewTomlEncoderWithPrefs(prefs TomlPreferences) Encoder {
|
||||
return &tomlEncoder{prefs: prefs}
|
||||
}
|
||||
|
||||
func (te *tomlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
|
||||
if node.Kind == ScalarNode {
|
||||
return writeString(writer, node.Value+"\n")
|
||||
if node.Kind != MappingNode {
|
||||
// For standalone selections, TOML tests expect raw value for scalars
|
||||
if node.Kind == ScalarNode {
|
||||
return writeString(writer, node.Value+"\n")
|
||||
}
|
||||
return fmt.Errorf("TOML encoder expects a mapping at the root level")
|
||||
}
|
||||
return fmt.Errorf("only scalars (e.g. strings, numbers, booleans) are supported for TOML output at the moment. Please use yaml output format (-oy) until the encoder has been fully implemented")
|
||||
|
||||
// Encode to a buffer first if colors are enabled
|
||||
var buf bytes.Buffer
|
||||
var targetWriter io.Writer
|
||||
targetWriter = writer
|
||||
if te.prefs.ColorsEnabled {
|
||||
targetWriter = &buf
|
||||
}
|
||||
|
||||
// Encode a root mapping as a sequence of attributes, tables, and arrays of tables
|
||||
if err := te.encodeRootMapping(targetWriter, node); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if te.prefs.ColorsEnabled {
|
||||
colorized := te.colorizeToml(buf.Bytes())
|
||||
_, err := writer.Write(colorized)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (te *tomlEncoder) PrintDocumentSeparator(_ io.Writer) error {
|
||||
@@ -30,3 +64,632 @@ func (te *tomlEncoder) PrintLeadingContent(_ io.Writer, _ string) error {
|
||||
func (te *tomlEncoder) CanHandleAliases() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// ---- helpers ----
|
||||
|
||||
func (te *tomlEncoder) writeComment(w io.Writer, comment string) error {
|
||||
if comment == "" {
|
||||
return nil
|
||||
}
|
||||
lines := strings.Split(comment, "\n")
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if !strings.HasPrefix(line, "#") {
|
||||
line = "# " + line
|
||||
}
|
||||
if _, err := w.Write([]byte(line + "\n")); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (te *tomlEncoder) formatScalar(node *CandidateNode) string {
|
||||
switch node.Tag {
|
||||
case "!!str":
|
||||
// Quote strings per TOML spec
|
||||
return fmt.Sprintf("%q", node.Value)
|
||||
case "!!bool", "!!int", "!!float":
|
||||
return node.Value
|
||||
case "!!null":
|
||||
// TOML does not have null; encode as empty string
|
||||
return `""`
|
||||
default:
|
||||
return node.Value
|
||||
}
|
||||
}
|
||||
|
||||
func (te *tomlEncoder) encodeRootMapping(w io.Writer, node *CandidateNode) error {
|
||||
te.wroteRootAttr = false // Reset state
|
||||
|
||||
// Write root head comment if present (at the very beginning, no leading blank line)
|
||||
if node.HeadComment != "" {
|
||||
if err := te.writeComment(w, node.HeadComment); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Preserve existing order by iterating Content
|
||||
for i := 0; i < len(node.Content); i += 2 {
|
||||
keyNode := node.Content[i]
|
||||
valNode := node.Content[i+1]
|
||||
if err := te.encodeTopLevelEntry(w, []string{keyNode.Value}, valNode); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// encodeTopLevelEntry encodes a key/value at the root, dispatching to attribute, table, or array-of-tables
|
||||
func (te *tomlEncoder) encodeTopLevelEntry(w io.Writer, path []string, node *CandidateNode) error {
|
||||
switch node.Kind {
|
||||
case ScalarNode:
|
||||
// key = value
|
||||
return te.writeAttribute(w, path[len(path)-1], node)
|
||||
case SequenceNode:
|
||||
// Empty arrays should be encoded as [] attributes
|
||||
if len(node.Content) == 0 {
|
||||
return te.writeArrayAttribute(w, path[len(path)-1], node)
|
||||
}
|
||||
|
||||
// If all items are mappings => array of tables; else => array attribute
|
||||
allMaps := true
|
||||
for _, it := range node.Content {
|
||||
if it.Kind != MappingNode {
|
||||
allMaps = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allMaps {
|
||||
key := path[len(path)-1]
|
||||
for _, it := range node.Content {
|
||||
// [[key]] then body
|
||||
if _, err := w.Write([]byte("[[" + key + "]]\n")); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := te.encodeMappingBodyWithPath(w, []string{key}, it); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
// Regular array attribute
|
||||
return te.writeArrayAttribute(w, path[len(path)-1], node)
|
||||
case MappingNode:
|
||||
// Inline table if not EncodeSeparate, else emit separate tables/arrays of tables for children under this path
|
||||
if !node.EncodeSeparate {
|
||||
// If children contain mappings or arrays of mappings, prefer separate sections
|
||||
if te.hasEncodeSeparateChild(node) || te.hasStructuralChildren(node) {
|
||||
return te.encodeSeparateMapping(w, path, node)
|
||||
}
|
||||
return te.writeInlineTableAttribute(w, path[len(path)-1], node)
|
||||
}
|
||||
return te.encodeSeparateMapping(w, path, node)
|
||||
default:
|
||||
return fmt.Errorf("unsupported node kind for TOML: %v", node.Kind)
|
||||
}
|
||||
}
|
||||
|
||||
func (te *tomlEncoder) writeAttribute(w io.Writer, key string, value *CandidateNode) error {
|
||||
te.wroteRootAttr = true // Mark that we wrote a root attribute
|
||||
|
||||
// Write head comment before the attribute
|
||||
if err := te.writeComment(w, value.HeadComment); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Write the attribute
|
||||
line := key + " = " + te.formatScalar(value)
|
||||
|
||||
// Add line comment if present
|
||||
if value.LineComment != "" {
|
||||
lineComment := strings.TrimSpace(value.LineComment)
|
||||
if !strings.HasPrefix(lineComment, "#") {
|
||||
lineComment = "# " + lineComment
|
||||
}
|
||||
line += " " + lineComment
|
||||
}
|
||||
|
||||
_, err := w.Write([]byte(line + "\n"))
|
||||
return err
|
||||
}
|
||||
|
||||
func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *CandidateNode) error {
|
||||
te.wroteRootAttr = true // Mark that we wrote a root attribute
|
||||
|
||||
// Write head comment before the array
|
||||
if err := te.writeComment(w, seq.HeadComment); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Handle empty arrays
|
||||
if len(seq.Content) == 0 {
|
||||
line := key + " = []"
|
||||
if seq.LineComment != "" {
|
||||
lineComment := strings.TrimSpace(seq.LineComment)
|
||||
if !strings.HasPrefix(lineComment, "#") {
|
||||
lineComment = "# " + lineComment
|
||||
}
|
||||
line += " " + lineComment
|
||||
}
|
||||
_, err := w.Write([]byte(line + "\n"))
|
||||
return err
|
||||
}
|
||||
|
||||
// Join scalars or nested arrays recursively into TOML array syntax
|
||||
items := make([]string, 0, len(seq.Content))
|
||||
for _, it := range seq.Content {
|
||||
switch it.Kind {
|
||||
case ScalarNode:
|
||||
items = append(items, te.formatScalar(it))
|
||||
case SequenceNode:
|
||||
// Nested arrays: encode inline
|
||||
nested, err := te.sequenceToInlineArray(it)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
items = append(items, nested)
|
||||
case MappingNode:
|
||||
// Inline table inside array
|
||||
inline, err := te.mappingToInlineTable(it)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
items = append(items, inline)
|
||||
case AliasNode:
|
||||
return fmt.Errorf("aliases are not supported in TOML")
|
||||
default:
|
||||
return fmt.Errorf("unsupported array item kind: %v", it.Kind)
|
||||
}
|
||||
}
|
||||
|
||||
line := key + " = [" + strings.Join(items, ", ") + "]"
|
||||
|
||||
// Add line comment if present
|
||||
if seq.LineComment != "" {
|
||||
lineComment := strings.TrimSpace(seq.LineComment)
|
||||
if !strings.HasPrefix(lineComment, "#") {
|
||||
lineComment = "# " + lineComment
|
||||
}
|
||||
line += " " + lineComment
|
||||
}
|
||||
|
||||
_, err := w.Write([]byte(line + "\n"))
|
||||
return err
|
||||
}
|
||||
|
||||
func (te *tomlEncoder) sequenceToInlineArray(seq *CandidateNode) (string, error) {
|
||||
items := make([]string, 0, len(seq.Content))
|
||||
for _, it := range seq.Content {
|
||||
switch it.Kind {
|
||||
case ScalarNode:
|
||||
items = append(items, te.formatScalar(it))
|
||||
case SequenceNode:
|
||||
nested, err := te.sequenceToInlineArray(it)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
items = append(items, nested)
|
||||
case MappingNode:
|
||||
inline, err := te.mappingToInlineTable(it)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
items = append(items, inline)
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported array item kind: %v", it.Kind)
|
||||
}
|
||||
}
|
||||
return "[" + strings.Join(items, ", ") + "]", nil
|
||||
}
|
||||
|
||||
func (te *tomlEncoder) mappingToInlineTable(m *CandidateNode) (string, error) {
|
||||
// key = { a = 1, b = "x" }
|
||||
parts := make([]string, 0, len(m.Content)/2)
|
||||
for i := 0; i < len(m.Content); i += 2 {
|
||||
k := m.Content[i].Value
|
||||
v := m.Content[i+1]
|
||||
switch v.Kind {
|
||||
case ScalarNode:
|
||||
parts = append(parts, fmt.Sprintf("%s = %s", k, te.formatScalar(v)))
|
||||
case SequenceNode:
|
||||
// inline array in inline table
|
||||
arr, err := te.sequenceToInlineArray(v)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
parts = append(parts, fmt.Sprintf("%s = %s", k, arr))
|
||||
case MappingNode:
|
||||
// nested inline table
|
||||
inline, err := te.mappingToInlineTable(v)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
parts = append(parts, fmt.Sprintf("%s = %s", k, inline))
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported inline table value kind: %v", v.Kind)
|
||||
}
|
||||
}
|
||||
return "{ " + strings.Join(parts, ", ") + " }", nil
|
||||
}
|
||||
|
||||
func (te *tomlEncoder) writeInlineTableAttribute(w io.Writer, key string, m *CandidateNode) error {
|
||||
inline, err := te.mappingToInlineTable(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = w.Write([]byte(key + " = " + inline + "\n"))
|
||||
return err
|
||||
}
|
||||
|
||||
func (te *tomlEncoder) writeTableHeader(w io.Writer, path []string, m *CandidateNode) error {
|
||||
// Add blank line before table header (or before comment if present) if we wrote root attributes
|
||||
needsBlankLine := te.wroteRootAttr
|
||||
if needsBlankLine {
|
||||
if _, err := w.Write([]byte("\n")); err != nil {
|
||||
return err
|
||||
}
|
||||
te.wroteRootAttr = false // Only add once
|
||||
}
|
||||
|
||||
// Write head comment before the table header
|
||||
if m.HeadComment != "" {
|
||||
if err := te.writeComment(w, m.HeadComment); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Write table header [a.b.c]
|
||||
header := "[" + strings.Join(path, ".") + "]\n"
|
||||
_, err := w.Write([]byte(header))
|
||||
return err
|
||||
}
|
||||
|
||||
// encodeSeparateMapping handles a mapping that should be encoded as table sections.
|
||||
// It emits the table header for this mapping if it has any content, then processes children.
|
||||
func (te *tomlEncoder) encodeSeparateMapping(w io.Writer, path []string, m *CandidateNode) error {
|
||||
// Check if this mapping has any non-mapping, non-array-of-tables children (i.e., attributes)
|
||||
hasAttrs := false
|
||||
for i := 0; i < len(m.Content); i += 2 {
|
||||
v := m.Content[i+1]
|
||||
if v.Kind == ScalarNode {
|
||||
hasAttrs = true
|
||||
break
|
||||
}
|
||||
if v.Kind == SequenceNode {
|
||||
// Check if it's NOT an array of tables
|
||||
allMaps := true
|
||||
for _, it := range v.Content {
|
||||
if it.Kind != MappingNode {
|
||||
allMaps = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if !allMaps {
|
||||
hasAttrs = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If there are attributes or if the mapping is empty, emit the table header
|
||||
if hasAttrs || len(m.Content) == 0 {
|
||||
if err := te.writeTableHeader(w, path, m); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := te.encodeMappingBodyWithPath(w, path, m); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// No attributes, just nested structures - process children
|
||||
for i := 0; i < len(m.Content); i += 2 {
|
||||
k := m.Content[i].Value
|
||||
v := m.Content[i+1]
|
||||
switch v.Kind {
|
||||
case MappingNode:
|
||||
// Emit [path.k]
|
||||
newPath := append(append([]string{}, path...), k)
|
||||
if err := te.writeTableHeader(w, newPath, v); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := te.encodeMappingBodyWithPath(w, newPath, v); err != nil {
|
||||
return err
|
||||
}
|
||||
case SequenceNode:
|
||||
// If sequence of maps, emit [[path.k]] per element
|
||||
allMaps := true
|
||||
for _, it := range v.Content {
|
||||
if it.Kind != MappingNode {
|
||||
allMaps = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allMaps {
|
||||
key := strings.Join(append(append([]string{}, path...), k), ".")
|
||||
for _, it := range v.Content {
|
||||
if _, err := w.Write([]byte("[[" + key + "]]\n")); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := te.encodeMappingBodyWithPath(w, append(append([]string{}, path...), k), it); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Regular array attribute under the current table path
|
||||
if err := te.writeArrayAttribute(w, k, v); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case ScalarNode:
|
||||
// Attributes directly under the current table path
|
||||
if err := te.writeAttribute(w, k, v); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (te *tomlEncoder) hasEncodeSeparateChild(m *CandidateNode) bool {
|
||||
for i := 0; i < len(m.Content); i += 2 {
|
||||
v := m.Content[i+1]
|
||||
if v.Kind == MappingNode && v.EncodeSeparate {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (te *tomlEncoder) hasStructuralChildren(m *CandidateNode) bool {
|
||||
for i := 0; i < len(m.Content); i += 2 {
|
||||
v := m.Content[i+1]
|
||||
// Only consider it structural if mapping has EncodeSeparate or is non-empty
|
||||
if v.Kind == MappingNode && v.EncodeSeparate {
|
||||
return true
|
||||
}
|
||||
if v.Kind == SequenceNode {
|
||||
allMaps := true
|
||||
for _, it := range v.Content {
|
||||
if it.Kind != MappingNode {
|
||||
allMaps = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allMaps {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// encodeMappingBodyWithPath encodes attributes and nested arrays of tables using full dotted path context
|
||||
func (te *tomlEncoder) encodeMappingBodyWithPath(w io.Writer, path []string, m *CandidateNode) error {
|
||||
// First, attributes (scalars and non-map arrays)
|
||||
for i := 0; i < len(m.Content); i += 2 {
|
||||
k := m.Content[i].Value
|
||||
v := m.Content[i+1]
|
||||
switch v.Kind {
|
||||
case ScalarNode:
|
||||
if err := te.writeAttribute(w, k, v); err != nil {
|
||||
return err
|
||||
}
|
||||
case SequenceNode:
|
||||
allMaps := true
|
||||
for _, it := range v.Content {
|
||||
if it.Kind != MappingNode {
|
||||
allMaps = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if !allMaps {
|
||||
if err := te.writeArrayAttribute(w, k, v); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Then, nested arrays of tables with full path
|
||||
for i := 0; i < len(m.Content); i += 2 {
|
||||
k := m.Content[i].Value
|
||||
v := m.Content[i+1]
|
||||
if v.Kind == SequenceNode {
|
||||
allMaps := true
|
||||
for _, it := range v.Content {
|
||||
if it.Kind != MappingNode {
|
||||
allMaps = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allMaps {
|
||||
dotted := strings.Join(append(append([]string{}, path...), k), ".")
|
||||
for _, it := range v.Content {
|
||||
if _, err := w.Write([]byte("[[" + dotted + "]]\n")); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := te.encodeMappingBodyWithPath(w, append(append([]string{}, path...), k), it); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Finally, child mappings that are not marked EncodeSeparate get inlined as attributes
|
||||
for i := 0; i < len(m.Content); i += 2 {
|
||||
k := m.Content[i].Value
|
||||
v := m.Content[i+1]
|
||||
if v.Kind == MappingNode && !v.EncodeSeparate {
|
||||
if err := te.writeInlineTableAttribute(w, k, v); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// colorizeToml applies syntax highlighting to TOML output using fatih/color
|
||||
func (te *tomlEncoder) colorizeToml(input []byte) []byte {
|
||||
toml := string(input)
|
||||
result := strings.Builder{}
|
||||
|
||||
// Force color output (don't check for TTY)
|
||||
color.NoColor = false
|
||||
|
||||
// Create color functions for different token types
|
||||
commentColor := color.New(color.FgHiBlack).SprintFunc()
|
||||
stringColor := color.New(color.FgGreen).SprintFunc()
|
||||
numberColor := color.New(color.FgHiMagenta).SprintFunc()
|
||||
keyColor := color.New(color.FgCyan).SprintFunc()
|
||||
boolColor := color.New(color.FgHiMagenta).SprintFunc()
|
||||
sectionColor := color.New(color.FgYellow, color.Bold).SprintFunc()
|
||||
|
||||
// Simple tokenization for TOML coloring
|
||||
i := 0
|
||||
for i < len(toml) {
|
||||
ch := toml[i]
|
||||
|
||||
// Comments - from # to end of line
|
||||
if ch == '#' {
|
||||
end := i
|
||||
for end < len(toml) && toml[end] != '\n' {
|
||||
end++
|
||||
}
|
||||
result.WriteString(commentColor(toml[i:end]))
|
||||
i = end
|
||||
continue
|
||||
}
|
||||
|
||||
// Table sections - [section] or [[array]]
|
||||
// Only treat '[' as a table section if it appears at the start of the line
|
||||
// (possibly after whitespace). This avoids mis-colouring inline arrays like
|
||||
// "ports = [8000, 8001]" as table sections.
|
||||
if ch == '[' {
|
||||
isSectionHeader := true
|
||||
if i > 0 {
|
||||
isSectionHeader = false
|
||||
j := i - 1
|
||||
for j >= 0 && toml[j] != '\n' {
|
||||
if toml[j] != ' ' && toml[j] != '\t' && toml[j] != '\r' {
|
||||
// Found a non-whitespace character before this '[' on the same line,
|
||||
// so this is not a table header.
|
||||
break
|
||||
}
|
||||
j--
|
||||
}
|
||||
if j < 0 || toml[j] == '\n' {
|
||||
// Reached the start of the string or a newline without encountering
|
||||
// any non-whitespace, so '[' is at the logical start of the line.
|
||||
isSectionHeader = true
|
||||
}
|
||||
}
|
||||
if isSectionHeader {
|
||||
end := i + 1
|
||||
// Check for [[
|
||||
if end < len(toml) && toml[end] == '[' {
|
||||
end++
|
||||
}
|
||||
// Find closing ]
|
||||
for end < len(toml) && toml[end] != ']' {
|
||||
end++
|
||||
}
|
||||
// Include closing ]
|
||||
if end < len(toml) {
|
||||
end++
|
||||
// Check for ]]
|
||||
if end < len(toml) && toml[end] == ']' {
|
||||
end++
|
||||
}
|
||||
}
|
||||
result.WriteString(sectionColor(toml[i:end]))
|
||||
i = end
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Strings - quoted text (double or single quotes)
|
||||
if ch == '"' || ch == '\'' {
|
||||
quote := ch
|
||||
end := i + 1
|
||||
for end < len(toml) {
|
||||
if toml[end] == quote {
|
||||
break
|
||||
}
|
||||
if toml[end] == '\\' && end+1 < len(toml) {
|
||||
// Skip the backslash and the escaped character
|
||||
end += 2
|
||||
continue
|
||||
}
|
||||
end++
|
||||
}
|
||||
if end < len(toml) {
|
||||
end++ // include closing quote
|
||||
}
|
||||
result.WriteString(stringColor(toml[i:end]))
|
||||
i = end
|
||||
continue
|
||||
}
|
||||
|
||||
// Numbers - sequences of digits, possibly with decimal point or minus
|
||||
if (ch >= '0' && ch <= '9') || (ch == '-' && i+1 < len(toml) && toml[i+1] >= '0' && toml[i+1] <= '9') {
|
||||
end := i
|
||||
if ch == '-' {
|
||||
end++
|
||||
}
|
||||
for end < len(toml) {
|
||||
c := toml[end]
|
||||
if (c >= '0' && c <= '9') || c == '.' || c == 'e' || c == 'E' {
|
||||
end++
|
||||
} else if (c == '+' || c == '-') && end > 0 && (toml[end-1] == 'e' || toml[end-1] == 'E') {
|
||||
// Only allow + or - immediately after 'e' or 'E' for scientific notation
|
||||
end++
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
result.WriteString(numberColor(toml[i:end]))
|
||||
i = end
|
||||
continue
|
||||
}
|
||||
|
||||
// Identifiers/keys - alphanumeric + underscore + dash
|
||||
if (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '_' {
|
||||
end := i
|
||||
for end < len(toml) && ((toml[end] >= 'a' && toml[end] <= 'z') ||
|
||||
(toml[end] >= 'A' && toml[end] <= 'Z') ||
|
||||
(toml[end] >= '0' && toml[end] <= '9') ||
|
||||
toml[end] == '_' || toml[end] == '-') {
|
||||
end++
|
||||
}
|
||||
ident := toml[i:end]
|
||||
|
||||
// Check if this is a boolean/null keyword
|
||||
switch ident {
|
||||
case "true", "false":
|
||||
result.WriteString(boolColor(ident))
|
||||
default:
|
||||
// Check if followed by = or whitespace then = (it's a key)
|
||||
j := end
|
||||
for j < len(toml) && (toml[j] == ' ' || toml[j] == '\t') {
|
||||
j++
|
||||
}
|
||||
if j < len(toml) && toml[j] == '=' {
|
||||
result.WriteString(keyColor(ident))
|
||||
} else {
|
||||
result.WriteString(ident) // plain text for other identifiers
|
||||
}
|
||||
}
|
||||
i = end
|
||||
continue
|
||||
}
|
||||
|
||||
// Everything else (whitespace, operators, brackets) - no color
|
||||
result.WriteByte(ch)
|
||||
i++
|
||||
}
|
||||
|
||||
return []byte(result.String())
|
||||
}
|
||||
|
||||
+2
-2
@@ -63,11 +63,11 @@ var ShFormat = &Format{"", nil,
|
||||
}
|
||||
|
||||
var TomlFormat = &Format{"toml", []string{},
|
||||
func() Encoder { return NewTomlEncoder() },
|
||||
func() Encoder { return NewTomlEncoderWithPrefs(ConfiguredTomlPreferences) },
|
||||
func() Decoder { return NewTomlDecoder() },
|
||||
}
|
||||
|
||||
var HclFormat = &Format{"hcl", []string{"h"},
|
||||
var HclFormat = &Format{"hcl", []string{"h", "tf"},
|
||||
func() Encoder { return NewHclEncoder(ConfiguredHclPreferences) },
|
||||
func() Decoder { return NewHclDecoder() },
|
||||
}
|
||||
|
||||
@@ -325,6 +325,14 @@ var hclFormatScenarios = []formatScenario{
|
||||
expected: "# Configuration\nport = 8080\n",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "Roundtrip: extraction",
|
||||
skipDoc: true,
|
||||
input: simpleSample,
|
||||
expression: ".shouty_message",
|
||||
expected: "upper(message)\n",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "Roundtrip: With templates, functions and arithmetic",
|
||||
input: simpleSample,
|
||||
@@ -345,6 +353,124 @@ var hclFormatScenarios = []formatScenario{
|
||||
expected: "tags:\n - \"a\"\n - \"b\"\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "roundtrip list of objects",
|
||||
skipDoc: true,
|
||||
input: `items = [{ name = "a", value = 1 }, { name = "b", value = 2 }]`,
|
||||
expected: "items = [{\n name = \"a\"\n value = 1\n }, {\n name = \"b\"\n value = 2\n}]\n",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "roundtrip nested blocks with same name",
|
||||
skipDoc: true,
|
||||
input: "database \"primary\" {\n host = \"localhost\"\n port = 5432\n}\ndatabase \"replica\" {\n host = \"replica.local\"\n port = 5433\n}",
|
||||
expected: "database \"primary\" {\n host = \"localhost\"\n port = 5432\n}\ndatabase \"replica\" {\n host = \"replica.local\"\n port = 5433\n}\n",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "roundtrip mixed nested structure",
|
||||
skipDoc: true,
|
||||
input: "servers \"web\" {\n addresses = [\"10.0.1.1\", \"10.0.1.2\"]\n port = 8080\n}",
|
||||
expected: "servers \"web\" {\n addresses = [\"10.0.1.1\", \"10.0.1.2\"]\n port = 8080\n}\n",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "roundtrip null value",
|
||||
skipDoc: true,
|
||||
input: `value = null`,
|
||||
expected: "value = null\n",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "roundtrip empty list",
|
||||
skipDoc: true,
|
||||
input: `items = []`,
|
||||
expected: "items = []\n",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "roundtrip empty object",
|
||||
skipDoc: true,
|
||||
input: `config = {}`,
|
||||
expected: "config = {}\n",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "Roundtrip: Separate blocks with same name.",
|
||||
input: "resource \"aws_instance\" \"web\" {\n ami = \"ami-12345\"\n}\nresource \"aws_instance\" \"db\" {\n ami = \"ami-67890\"\n}",
|
||||
expected: "resource \"aws_instance\" \"web\" {\n ami = \"ami-12345\"\n}\nresource \"aws_instance\" \"db\" {\n ami = \"ami-67890\"\n}\n",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "roundtrip deeply nested structure",
|
||||
skipDoc: true,
|
||||
input: "app \"database\" \"primary\" \"connection\" {\n host = \"db.local\"\n port = 5432\n}",
|
||||
expected: "app \"database\" \"primary\" \"connection\" {\n host = \"db.local\"\n port = 5432\n}\n",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "roundtrip with leading comments",
|
||||
skipDoc: true,
|
||||
input: "# Main config\nenabled = true\nport = 8080",
|
||||
expected: "# Main config\nenabled = true\nport = 8080\n",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "Multiple attributes with comments (comment safety with safe path separator)",
|
||||
skipDoc: true,
|
||||
input: "# Database config\ndb_host = \"localhost\"\n# Connection pool\ndb_pool = 10",
|
||||
expected: "# Database config\ndb_host = \"localhost\"\n# Connection pool\ndb_pool = 10\n",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "Nested blocks with head comments",
|
||||
skipDoc: true,
|
||||
input: "service \"api\" {\n # Listen address\n listen = \"0.0.0.0:8080\"\n # TLS enabled\n tls = true\n}",
|
||||
expected: "service \"api\" {\n # Listen address\n listen = \"0.0.0.0:8080\"\n # TLS enabled\n tls = true\n}\n",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "Multiple blocks with EncodeSeparate preservation",
|
||||
skipDoc: true,
|
||||
input: "resource \"aws_s3_bucket\" \"bucket1\" {\n bucket = \"my-bucket-1\"\n}\nresource \"aws_s3_bucket\" \"bucket2\" {\n bucket = \"my-bucket-2\"\n}",
|
||||
expected: "resource \"aws_s3_bucket\" \"bucket1\" {\n bucket = \"my-bucket-1\"\n}\nresource \"aws_s3_bucket\" \"bucket2\" {\n bucket = \"my-bucket-2\"\n}\n",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "Blocks with same name handled separately",
|
||||
skipDoc: true,
|
||||
input: "server \"primary\" { port = 8080 }\nserver \"backup\" { port = 8081 }",
|
||||
expected: "server \"primary\" {\n port = 8080\n}\nserver \"backup\" {\n port = 8081\n}\n",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "Block label with dot roundtrip (commentPathSep)",
|
||||
skipDoc: true,
|
||||
input: "service \"api.service\" {\n port = 8080\n}",
|
||||
expected: "service \"api.service\" {\n port = 8080\n}\n",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "Nested template expression",
|
||||
skipDoc: true,
|
||||
input: `message = "User: ${username}, Role: ${user_role}"`,
|
||||
expected: "message = \"User: ${username}, Role: ${user_role}\"\n",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "Empty object roundtrip",
|
||||
skipDoc: true,
|
||||
input: `obj = {}`,
|
||||
expected: "obj = {}\n",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "Null value in block",
|
||||
skipDoc: true,
|
||||
input: `service { optional_field = null }`,
|
||||
expected: "service {\n optional_field = null\n}\n",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
}
|
||||
|
||||
func testHclScenario(t *testing.T, s formatScenario) {
|
||||
|
||||
+1
-1
@@ -61,7 +61,7 @@ func unwrap(value string) string {
|
||||
}
|
||||
|
||||
func extractNumberParameter(value string) (int, error) {
|
||||
parameterParser := regexp.MustCompile(`.*\(([0-9]+)\)`)
|
||||
parameterParser := regexp.MustCompile(`.*\((-?[0-9]+)\)`)
|
||||
matches := parameterParser.FindStringSubmatch(value)
|
||||
var indent, errParsingInt = parseInt(matches[1])
|
||||
if errParsingInt != nil {
|
||||
|
||||
@@ -57,7 +57,7 @@ var participleYqRules = []*participleYqRule{
|
||||
simpleOp("sort_?keys", sortKeysOpType),
|
||||
|
||||
{"ArrayToMap", "array_?to_?map", expressionOpToken(`(.[] | select(. != null) ) as $i ireduce({}; .[$i | key] = $i)`), 0},
|
||||
|
||||
{"Root", "root", expressionOpToken(`parent(-1)`), 0},
|
||||
{"YamlEncodeWithIndent", `to_?yaml\([0-9]+\)`, encodeParseIndent(YamlFormat), 0},
|
||||
{"XMLEncodeWithIndent", `to_?xml\([0-9]+\)`, encodeParseIndent(XMLFormat), 0},
|
||||
{"JSONEncodeWithIndent", `to_?json\([0-9]+\)`, encodeParseIndent(JSONFormat), 0},
|
||||
@@ -132,7 +132,7 @@ var participleYqRules = []*participleYqRule{
|
||||
simpleOp("split", splitStringOpType),
|
||||
|
||||
simpleOp("parents", getParentsOpType),
|
||||
{"ParentWithLevel", `parent\([0-9]+\)`, parentWithLevel(), 0},
|
||||
{"ParentWithLevel", `parent\(-?[0-9]+\)`, parentWithLevel(), 0},
|
||||
{"ParentWithDefaultLevel", `parent`, parentWithDefaultLevel(), 0},
|
||||
|
||||
simpleOp("keys", keysOpType),
|
||||
|
||||
@@ -35,9 +35,28 @@ func getParentOperator(_ *dataTreeNavigator, context Context, expressionNode *Ex
|
||||
|
||||
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
|
||||
candidate := el.Value.(*CandidateNode)
|
||||
|
||||
// Handle negative levels: count total parents first
|
||||
levelsToGoUp := prefs.Level
|
||||
if prefs.Level < 0 {
|
||||
// Count all parents
|
||||
totalParents := 0
|
||||
temp := candidate.Parent
|
||||
for temp != nil {
|
||||
totalParents++
|
||||
temp = temp.Parent
|
||||
}
|
||||
// Convert negative index to positive
|
||||
// -1 means last parent (root), -2 means second to last, etc.
|
||||
levelsToGoUp = totalParents + prefs.Level + 1
|
||||
if levelsToGoUp < 0 {
|
||||
levelsToGoUp = 0
|
||||
}
|
||||
}
|
||||
|
||||
currentLevel := 0
|
||||
for currentLevel < prefs.Level && candidate != nil {
|
||||
log.Debugf("currentLevel: %v, desired: %v", currentLevel, prefs.Level)
|
||||
for currentLevel < levelsToGoUp && candidate != nil {
|
||||
log.Debugf("currentLevel: %v, desired: %v", currentLevel, levelsToGoUp)
|
||||
log.Debugf("candidate: %v", NodeToString(candidate))
|
||||
candidate = candidate.Parent
|
||||
currentLevel++
|
||||
|
||||
@@ -38,6 +38,67 @@ var parentOperatorScenarios = []expressionScenario{
|
||||
"D0, P[], (!!seq)::- {c: cat}\n- {b: {c: cat}}\n- {a: {b: {c: cat}}}\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Get the top (root) parent",
|
||||
subdescription: "Use negative numbers to get the top parents",
|
||||
document: "a:\n b:\n c: cat\n",
|
||||
expression: `.a.b.c | parent(-1)`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::a:\n b:\n c: cat\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Root",
|
||||
subdescription: "Alias for parent(-1), returns the top level parent. This is usually the document node.",
|
||||
document: "a:\n b:\n c: cat\n",
|
||||
expression: `.a.b.c | root`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::a:\n b:\n c: cat\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "N-th negative",
|
||||
skipDoc: true,
|
||||
document: "a:\n b:\n c: cat\n",
|
||||
expression: `.a.b.c | parent(-2)`,
|
||||
expected: []string{
|
||||
"D0, P[a], (!!map)::b:\n c: cat\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "boundary negative",
|
||||
skipDoc: true,
|
||||
document: "a:\n b:\n c: cat\n",
|
||||
expression: `.a.b.c | parent(-3)`,
|
||||
expected: []string{
|
||||
"D0, P[a b], (!!map)::c: cat\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "large negative",
|
||||
skipDoc: true,
|
||||
document: "a:\n b:\n c: cat\n",
|
||||
expression: `.a.b.c | parent(-10)`,
|
||||
expected: []string{
|
||||
"D0, P[a b c], (!!str)::cat\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "parent zero",
|
||||
skipDoc: true,
|
||||
document: "a:\n b:\n c: cat\n",
|
||||
expression: `.a.b.c | parent(0)`,
|
||||
expected: []string{
|
||||
"D0, P[a b c], (!!str)::cat\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "large positive",
|
||||
skipDoc: true,
|
||||
document: "a:\n b:\n c: cat\n",
|
||||
expression: `.a.b.c | parent(10)`,
|
||||
expected: []string{},
|
||||
},
|
||||
{
|
||||
description: "N-th parent",
|
||||
subdescription: "You can optionally supply the number of levels to go up for the parent, the default being 1.",
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
package yqlib
|
||||
|
||||
type TomlPreferences struct {
|
||||
ColorsEnabled bool
|
||||
}
|
||||
|
||||
func NewDefaultTomlPreferences() TomlPreferences {
|
||||
return TomlPreferences{ColorsEnabled: false}
|
||||
}
|
||||
|
||||
func (p *TomlPreferences) Copy() TomlPreferences {
|
||||
return TomlPreferences{ColorsEnabled: p.ColorsEnabled}
|
||||
}
|
||||
|
||||
var ConfiguredTomlPreferences = NewDefaultTomlPreferences()
|
||||
@@ -3,8 +3,10 @@ package yqlib
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/fatih/color"
|
||||
"github.com/mikefarah/yq/v4/test"
|
||||
)
|
||||
|
||||
@@ -175,6 +177,82 @@ var expectedSampleWithHeader = `servers:
|
||||
ip: 10.0.0.1
|
||||
`
|
||||
|
||||
// Roundtrip fixtures
|
||||
var rtInlineTableAttr = `name = { first = "Tom", last = "Preston-Werner" }
|
||||
`
|
||||
|
||||
var rtTableSection = `[owner.contact]
|
||||
name = "Tom"
|
||||
age = 36
|
||||
`
|
||||
|
||||
var rtArrayOfTables = `[[fruits]]
|
||||
name = "apple"
|
||||
[[fruits.varieties]]
|
||||
name = "red delicious"
|
||||
`
|
||||
|
||||
var rtArraysAndScalars = `A = ["hello", ["world", "again"]]
|
||||
B = 12
|
||||
`
|
||||
|
||||
var rtSimple = `A = "hello"
|
||||
B = 12
|
||||
`
|
||||
|
||||
var rtDeepPaths = `[person]
|
||||
name = "hello"
|
||||
address = "12 cat st"
|
||||
`
|
||||
|
||||
var rtEmptyArray = `A = []
|
||||
`
|
||||
|
||||
var rtSampleTable = `var = "x"
|
||||
|
||||
[owner.contact]
|
||||
name = "Tom Preston-Werner"
|
||||
age = 36
|
||||
`
|
||||
|
||||
var rtEmptyTable = `[dependencies]
|
||||
`
|
||||
|
||||
var rtComments = `# This is a comment
|
||||
A = "hello" # inline comment
|
||||
B = 12
|
||||
|
||||
# Table comment
|
||||
[person]
|
||||
name = "Tom" # name comment
|
||||
`
|
||||
|
||||
// var sampleFromWeb = `
|
||||
// # This is a TOML document
|
||||
|
||||
// title = "TOML Example"
|
||||
|
||||
// [owner]
|
||||
// name = "Tom Preston-Werner"
|
||||
// dob = 1979-05-27T07:32:00-08:00
|
||||
|
||||
// [database]
|
||||
// enabled = true
|
||||
// ports = [8000, 8001, 8002]
|
||||
// data = [["delta", "phi"], [3.14]]
|
||||
// temp_targets = { cpu = 79.5, case = 72.0 }
|
||||
|
||||
// [servers]
|
||||
|
||||
// [servers.alpha]
|
||||
// ip = "10.0.0.1"
|
||||
// role = "frontend"
|
||||
|
||||
// [servers.beta]
|
||||
// ip = "10.0.0.2"
|
||||
// role = "backend"
|
||||
// `
|
||||
|
||||
var tomlScenarios = []formatScenario{
|
||||
{
|
||||
skipDoc: true,
|
||||
@@ -382,6 +460,84 @@ var tomlScenarios = []formatScenario{
|
||||
expected: expectedMultipleEmptyTables,
|
||||
scenarioType: "decode",
|
||||
},
|
||||
// Roundtrip scenarios
|
||||
{
|
||||
description: "Roundtrip: inline table attribute",
|
||||
input: rtInlineTableAttr,
|
||||
expression: ".",
|
||||
expected: rtInlineTableAttr,
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "Roundtrip: table section",
|
||||
input: rtTableSection,
|
||||
expression: ".",
|
||||
expected: rtTableSection,
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "Roundtrip: array of tables",
|
||||
input: rtArrayOfTables,
|
||||
expression: ".",
|
||||
expected: rtArrayOfTables,
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "Roundtrip: arrays and scalars",
|
||||
input: rtArraysAndScalars,
|
||||
expression: ".",
|
||||
expected: rtArraysAndScalars,
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "Roundtrip: simple",
|
||||
input: rtSimple,
|
||||
expression: ".",
|
||||
expected: rtSimple,
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "Roundtrip: deep paths",
|
||||
input: rtDeepPaths,
|
||||
expression: ".",
|
||||
expected: rtDeepPaths,
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "Roundtrip: empty array",
|
||||
input: rtEmptyArray,
|
||||
expression: ".",
|
||||
expected: rtEmptyArray,
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "Roundtrip: sample table",
|
||||
input: rtSampleTable,
|
||||
expression: ".",
|
||||
expected: rtSampleTable,
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "Roundtrip: empty table",
|
||||
input: rtEmptyTable,
|
||||
expression: ".",
|
||||
expected: rtEmptyTable,
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "Roundtrip: comments",
|
||||
input: rtComments,
|
||||
expression: ".",
|
||||
expected: rtComments,
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
// {
|
||||
// description: "Roundtrip: sample from web",
|
||||
// input: sampleFromWeb,
|
||||
// expression: ".",
|
||||
// expected: sampleFromWeb,
|
||||
// scenarioType: "roundtrip",
|
||||
// },
|
||||
}
|
||||
|
||||
func testTomlScenario(t *testing.T, s formatScenario) {
|
||||
@@ -471,3 +627,238 @@ func TestTomlScenarios(t *testing.T) {
|
||||
}
|
||||
documentScenarios(t, "usage", "toml", genericScenarios, documentTomlScenario)
|
||||
}
|
||||
|
||||
// TestTomlColourization tests that colourization correctly distinguishes
|
||||
// between table section headers and inline arrays
|
||||
func TestTomlColourization(t *testing.T) {
|
||||
// Test that inline arrays are not coloured as table sections
|
||||
encoder := &tomlEncoder{prefs: TomlPreferences{ColorsEnabled: true}}
|
||||
|
||||
// Create TOML with both table sections and inline arrays
|
||||
input := []byte(`[database]
|
||||
enabled = true
|
||||
ports = [8000, 8001, 8002]
|
||||
|
||||
[servers]
|
||||
alpha = "test"
|
||||
`)
|
||||
|
||||
result := encoder.colorizeToml(input)
|
||||
resultStr := string(result)
|
||||
|
||||
// The bug would cause the inline array [8000, 8001, 8002] to be
|
||||
// coloured with the section colour (Yellow + Bold) instead of being
|
||||
// left uncoloured or coloured differently.
|
||||
//
|
||||
// To test this, we check that the section colour codes appear only
|
||||
// for actual table sections, not for inline arrays.
|
||||
|
||||
// Get the ANSI codes for section colour (Yellow + Bold)
|
||||
sectionColour := color.New(color.FgYellow, color.Bold).SprintFunc()
|
||||
sampleSection := sectionColour("[database]")
|
||||
|
||||
// Extract just the ANSI codes from the sample
|
||||
// ANSI codes start with \x1b[
|
||||
var ansiStart string
|
||||
for i := 0; i < len(sampleSection); i++ {
|
||||
if sampleSection[i] == '\x1b' {
|
||||
// Find the end of the ANSI sequence (ends with 'm')
|
||||
end := i
|
||||
for end < len(sampleSection) && sampleSection[end] != 'm' {
|
||||
end++
|
||||
}
|
||||
if end < len(sampleSection) {
|
||||
ansiStart = sampleSection[i : end+1]
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Count how many times the section colour appears in the output
|
||||
// It should appear exactly twice: once for [database] and once for [servers]
|
||||
// If it appears more times (e.g., for [8000, 8001, 8002]), that's the bug
|
||||
sectionColourCount := strings.Count(resultStr, ansiStart)
|
||||
|
||||
// We expect exactly 2 occurrences (for [database] and [servers])
|
||||
// The bug would cause more occurrences (e.g., also for [8000)
|
||||
if sectionColourCount != 2 {
|
||||
t.Errorf("Expected section colour to appear exactly 2 times (for [database] and [servers]), but it appeared %d times.\nOutput: %s", sectionColourCount, resultStr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTomlColorisationNumberBug(t *testing.T) {
|
||||
// Save and restore color state
|
||||
oldNoColor := color.NoColor
|
||||
color.NoColor = false
|
||||
defer func() { color.NoColor = oldNoColor }()
|
||||
|
||||
encoder := NewTomlEncoder()
|
||||
tomlEncoder := encoder.(*tomlEncoder)
|
||||
|
||||
// Test case that exposes the bug: "123-+-45" should NOT be colorized as a single number
|
||||
input := "A = 123-+-45\n"
|
||||
result := string(tomlEncoder.colorizeToml([]byte(input)))
|
||||
|
||||
// The bug causes "123-+-45" to be colorized as one token
|
||||
// It should stop at "123" because the next character '-' is not valid in this position
|
||||
if strings.Contains(result, "123-+-45") {
|
||||
// Check if it's colorized as a single token (no color codes in the middle)
|
||||
idx := strings.Index(result, "123-+-45")
|
||||
// Look backwards for color code
|
||||
beforeIdx := idx - 1
|
||||
for beforeIdx >= 0 && result[beforeIdx] != '\x1b' {
|
||||
beforeIdx--
|
||||
}
|
||||
// Look forward for reset code
|
||||
afterIdx := idx + 8 // length of "123-+-45"
|
||||
hasResetAfter := false
|
||||
for afterIdx < len(result) && afterIdx < idx+20 {
|
||||
if result[afterIdx] == '\x1b' {
|
||||
hasResetAfter = true
|
||||
break
|
||||
}
|
||||
afterIdx++
|
||||
}
|
||||
|
||||
if beforeIdx >= 0 && hasResetAfter {
|
||||
// The entire "123-+-45" is wrapped in color codes - this is the bug!
|
||||
t.Errorf("BUG DETECTED: '123-+-45' is incorrectly colorized as a single number")
|
||||
t.Errorf("Expected only '123' to be colorized as a number, but got the entire '123-+-45'")
|
||||
t.Logf("Full output: %q", result)
|
||||
t.Fail()
|
||||
}
|
||||
}
|
||||
|
||||
// Additional test cases for the bug
|
||||
bugTests := []struct {
|
||||
name string
|
||||
input string
|
||||
invalidSequence string
|
||||
description string
|
||||
}{
|
||||
{
|
||||
name: "consecutive minuses",
|
||||
input: "A = 123--45\n",
|
||||
invalidSequence: "123--45",
|
||||
description: "'123--45' should not be colorized as a single number",
|
||||
},
|
||||
{
|
||||
name: "plus in middle",
|
||||
input: "A = 123+45\n",
|
||||
invalidSequence: "123+45",
|
||||
description: "'123+45' should not be colorized as a single number",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range bugTests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := string(tomlEncoder.colorizeToml([]byte(tt.input)))
|
||||
if strings.Contains(result, tt.invalidSequence) {
|
||||
idx := strings.Index(result, tt.invalidSequence)
|
||||
beforeIdx := idx - 1
|
||||
for beforeIdx >= 0 && result[beforeIdx] != '\x1b' {
|
||||
beforeIdx--
|
||||
}
|
||||
afterIdx := idx + len(tt.invalidSequence)
|
||||
hasResetAfter := false
|
||||
for afterIdx < len(result) && afterIdx < idx+20 {
|
||||
if result[afterIdx] == '\x1b' {
|
||||
hasResetAfter = true
|
||||
break
|
||||
}
|
||||
afterIdx++
|
||||
}
|
||||
|
||||
if beforeIdx >= 0 && hasResetAfter {
|
||||
t.Errorf("BUG: %s", tt.description)
|
||||
t.Logf("Full output: %q", result)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Test that valid scientific notation still works
|
||||
validTests := []struct {
|
||||
name string
|
||||
input string
|
||||
}{
|
||||
{"scientific positive", "A = 1.23e+45\n"},
|
||||
{"scientific negative", "A = 6.626e-34\n"},
|
||||
{"scientific uppercase", "A = 1.23E+10\n"},
|
||||
}
|
||||
|
||||
for _, tt := range validTests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := tomlEncoder.colorizeToml([]byte(tt.input))
|
||||
if len(result) == 0 {
|
||||
t.Error("Expected non-empty colorized output")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestTomlStringEscapeColourization tests that string colourization correctly
|
||||
// handles escape sequences, particularly escaped quotes at the end of strings
|
||||
func TestTomlStringEscapeColourization(t *testing.T) {
|
||||
// Save and restore color state
|
||||
oldNoColor := color.NoColor
|
||||
color.NoColor = false
|
||||
defer func() { color.NoColor = oldNoColor }()
|
||||
|
||||
encoder := NewTomlEncoder()
|
||||
tomlEncoder := encoder.(*tomlEncoder)
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
input string
|
||||
description string
|
||||
}{
|
||||
{
|
||||
name: "escaped quote at end",
|
||||
input: `A = "test\""` + "\n",
|
||||
description: "String ending with escaped quote should be colorized correctly",
|
||||
},
|
||||
{
|
||||
name: "escaped backslash then quote",
|
||||
input: `A = "test\\\""` + "\n",
|
||||
description: "String with escaped backslash followed by escaped quote",
|
||||
},
|
||||
{
|
||||
name: "escaped quote in middle",
|
||||
input: `A = "test\"middle"` + "\n",
|
||||
description: "String with escaped quote in the middle should be colorized correctly",
|
||||
},
|
||||
{
|
||||
name: "multiple escaped quotes",
|
||||
input: `A = "\"test\""` + "\n",
|
||||
description: "String with escaped quotes at start and end",
|
||||
},
|
||||
{
|
||||
name: "escaped newline",
|
||||
input: `A = "test\n"` + "\n",
|
||||
description: "String with escaped newline should be colorized correctly",
|
||||
},
|
||||
{
|
||||
name: "single quote with escaped single quote",
|
||||
input: `A = 'test\''` + "\n",
|
||||
description: "Single-quoted string with escaped single quote",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range testCases {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// The test should not panic and should return some output
|
||||
result := tomlEncoder.colorizeToml([]byte(tt.input))
|
||||
if len(result) == 0 {
|
||||
t.Error("Expected non-empty colorized output")
|
||||
}
|
||||
|
||||
// Check that the result contains the input string (with color codes)
|
||||
// At minimum, it should contain "A" and "="
|
||||
resultStr := string(result)
|
||||
if !strings.Contains(resultStr, "A") || !strings.Contains(resultStr, "=") {
|
||||
t.Errorf("Expected output to contain 'A' and '=', got: %q", resultStr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+2
-1
@@ -276,4 +276,5 @@ nohcl
|
||||
zclconf
|
||||
cty
|
||||
go-cty
|
||||
unlabeled
|
||||
unlabeled
|
||||
colorisation
|
||||
@@ -1,3 +1,8 @@
|
||||
4.50.1:
|
||||
- Added HCL support!
|
||||
- Fixing handling of CRLF #2352
|
||||
- Bumped dependencies
|
||||
|
||||
4.49.2:
|
||||
- Fixing escape character bugs :sweat: #2517
|
||||
- Fixing snap release pipeline #2518 Thanks @aalexjo
|
||||
|
||||
+2
-3
@@ -1,5 +1,5 @@
|
||||
name: yq
|
||||
version: 'v4.49.2'
|
||||
version: 'v4.50.1'
|
||||
summary: A lightweight and portable command-line data file processor
|
||||
description: |
|
||||
`yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml, json, xml, csv, properties and TOML files.
|
||||
@@ -22,7 +22,6 @@ platforms:
|
||||
ppc64el:
|
||||
build-on: [ppc64el]
|
||||
build-for: [ppc64el]
|
||||
|
||||
apps:
|
||||
yq:
|
||||
command: bin/yq
|
||||
@@ -33,6 +32,6 @@ parts:
|
||||
build-environment:
|
||||
- CGO_ENABLED: 0
|
||||
source: https://github.com/mikefarah/yq.git
|
||||
source-tag: v4.49.2
|
||||
source-tag: v4.50.1
|
||||
build-snaps:
|
||||
- go/latest/stable
|
||||
|
||||
Reference in New Issue
Block a user