suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2026-07-01 09:51:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add INI support

Browse Source
This commit is contained in:
beliys 2025-05-16 23:05:01 +03:00 committed by Mike Farah
parent b534aa9ee5
commit 3ac203ebb8
11 changed files with 517 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
README.md
View File

@ -3,7 +3,7 @@
![Build](https://github.com/mikefarah/yq/workflows/Build/badge.svg) ![Docker Pulls](https://img.shields.io/docker/pulls/mikefarah/yq.svg) ![Github Releases (by Release)](https://img.shields.io/github/downloads/mikefarah/yq/total.svg) ![Go Report](https://goreportcard.com/badge/github.com/mikefarah/yq) ![CodeQL](https://github.com/mikefarah/yq/workflows/CodeQL/badge.svg)
a lightweight and portable command-line YAML, JSON and XML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json, xml, properties, csv and tsv. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.
a lightweight and portable command-line YAML, JSON, INI and XML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json, xml, ini, properties, csv and tsv. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.
yq is written in go - so you can download a dependency free binary for your platform and you are good to go! If you prefer there are a variety of package managers that can be used as well as Docker and Podman, all listed below.
@ -327,7 +327,7 @@ flox install yq
## Features
- [Detailed documentation with many examples](https://mikefarah.gitbook.io/yq/)
- Written in portable go, so you can download a lovely dependency free binary
- Uses similar syntax as `jq` but works with YAML, [JSON](https://mikefarah.gitbook.io/yq/usage/convert) and [XML](https://mikefarah.gitbook.io/yq/usage/xml) files
- Uses similar syntax as `jq` but works with YAML, INI, [JSON](https://mikefarah.gitbook.io/yq/usage/convert) and [XML](https://mikefarah.gitbook.io/yq/usage/xml) files
- Fully supports multi document yaml files
- Supports yaml [front matter](https://mikefarah.gitbook.io/yq/usage/front-matter) blocks (e.g. jekyll/assemble)
- Colorized yaml output
@ -366,31 +366,52 @@ yq -i '.stuff = "foo"' myfile.yml # update myfile.yml in place
Available Commands:
completion Generate the autocompletion script for the specified shell
eval (default) Apply the expression to each document in each yaml file in sequence
eval-all Loads _all_ yaml documents of _all_ yaml files and runs expression once
help Help about any command
completion Generate the autocompletion script for the specified shell
eval (default) Apply the expression to each document in each yaml file in sequence
eval-all Loads _all_ yaml documents of _all_ yaml files and runs expression once
help Help about any command
Flags:
-C, --colors force print with colors
--csv-auto-parse parse CSV YAML/JSON values (default true)
--csv-separator char CSV Separator character (default ,)
-e, --exit-status set exit status if there are no matches or null or false is returned
--expression string forcibly set the expression argument. Useful when yq argument detection thinks your expression is a file.
--from-file string Load expression from specified file.
-f, --front-matter string (extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact
--header-preprocess Slurp any header comments and separators before processing expression. (default true)
-h, --help help for yq
-I, --indent int sets indent level for output (default 2)
-i, --inplace update the file in place of first file given.
-p, --input-format string [yaml|y|xml|x] parse format for input. Note that json is a subset of yaml. (default "yaml")
-p, --input-format string [auto|a|yaml|y|json|j|props|p|csv|c|tsv|t|xml|x|base64|uri|toml|lua|l|ini|i] parse format for input. (default "auto")
--lua-globals output keys as top-level global variables
--lua-prefix string prefix (default "return ")
--lua-suffix string suffix (default ";\n")
--lua-unquoted output unquoted string keys (e.g. {foo="bar"})
-M, --no-colors force print with no colors
-N, --no-doc Don't print document separators (---)
-0, --nul-output Use NUL char to separate values. If unwrap scalar is also set, fail if unwrapped scalar contains NUL char.
-n, --null-input Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.
-o, --output-format string [yaml|y|json|j|props|p|xml|x] output format type. (default "yaml")
-o, --output-format string [auto|a|yaml|y|json|j|props|p|csv|c|tsv|t|xml|x|base64|uri|toml|shell|s|lua|l|ini|i] output format type. (default "auto")
-P, --prettyPrint pretty print, shorthand for '... style = ""'
-s, --split-exp string print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter.
--unwrapScalar unwrap scalar, print the value with no quotes, colors or comments (default true)
--properties-array-brackets use [x] in array paths (e.g. for SpringBoot)
--properties-separator string separator to use between keys and values (default " = ")
-s, --split-exp string print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter. The necessary directories will be created.
--split-exp-file string Use a file to specify the split-exp expression.
--string-interpolation Toggles strings interpolation of \(exp) (default true)
--tsv-auto-parse parse TSV YAML/JSON values (default true)
-r, --unwrapScalar unwrap scalar, print the value with no quotes, colors or comments. Defaults to true for yaml (default true)
-v, --verbose verbose mode
-V, --version Print version information and quit
--xml-attribute-prefix string prefix for xml attributes (default "+")
--xml-attribute-prefix string prefix for xml attributes (default "+@")
--xml-content-name string name for xml content (if no attribute name is present). (default "+content")
--xml-directive-name string name for xml directives (e.g. <!DOCTYPE thing cat>) (default "+directive")
--xml-keep-namespace enables keeping namespace after parsing attributes (default true)
--xml-proc-inst-prefix string prefix for xml processing instructions (e.g. <?xml version="1"?>) (default "+p_")
--xml-raw-token enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details. (default true)
--xml-skip-directives skip over directives (e.g. <!DOCTYPE thing cat>)
--xml-skip-proc-inst skip over process instructions (e.g. <?xml version="1"?>)
--xml-strict-mode enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.
Use "yq [command] --help" for more information about a command.
```

14
examples/sample.ini Normal file
View File

@ -0,0 +1,14 @@
; This is a INI document
[owner]
name = "Tom Preston-Werner"
dob = 1979-05-27T07:32:00-08:00
[database]
db_host = "localhost"
db_port = 5432
db_user = "postgres"
db_password = ""
db_name = "postgres"

8
examples/sample_no_sections.ini Normal file
View File

@ -0,0 +1,8 @@
; This is a INI document
name = "Tom Preston-Werner"
dob = 1979-05-27T07:32:00-08:00
enabled = true
ip = "10.0.0.1"
role = "frontend"
treads = 4

1
go.mod
View File

@ -7,6 +7,7 @@ require (
github.com/dimchansky/utfbom v1.1.1
github.com/elliotchance/orderedmap v1.8.0
github.com/fatih/color v1.18.0
github.com/go-ini/ini v1.67.0
github.com/goccy/go-json v0.10.5
github.com/goccy/go-yaml v1.17.1
github.com/jinzhu/copier v0.4.0

2
go.sum
View File

@ -15,6 +15,8 @@ github.com/elliotchance/orderedmap v1.8.0 h1:TrOREecvh3JbS+NCgwposXG5ZTFHtEsQiCG
github.com/elliotchance/orderedmap v1.8.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/goccy/go-yaml v1.17.1 h1:LI34wktB2xEE3ONG/2Ar54+/HJVBriAGJ55PHls4YuY=

106
pkg/yqlib/decoder_ini.go Normal file
View File

@ -0,0 +1,106 @@
//go:build !yq_noini
package yqlib
import (
"fmt"
"io"
"github.com/go-ini/ini"
)
type iniDecoder struct {
reader io.Reader
finished bool // Flag to signal completion of processing
}
func NewINIDecoder() Decoder {
return &iniDecoder{
finished: false, // Initialize the flag as false
}
}
func (dec *iniDecoder) Init(reader io.Reader) error {
// Store the reader for use in Decode
dec.reader = reader
return nil
}
func (dec *iniDecoder) Decode() (*CandidateNode, error) {
// If processing is already finished, return io.EOF
if dec.finished {
return nil, io.EOF
}
// Read all content from the stored reader
content, err := io.ReadAll(dec.reader)
if err != nil {
return nil, fmt.Errorf("failed to read INI content: %v", err)
}
// Parse the INI content
cfg, err := ini.Load(content)
if err != nil {
return nil, fmt.Errorf("failed to parse INI content: %v", err)
}
// Create a root CandidateNode as a MappingNode (since INI is key-value based)
root := &CandidateNode{
Kind: MappingNode,
Tag: "!!map",
Value: "",
}
// Process each section in the INI file
for _, section := range cfg.Sections() {
sectionName := section.Name()
if sectionName == ini.DefaultSection {
// For the default section, add key-value pairs directly to the root node
for _, key := range section.Keys() {
keyName := key.Name()
keyValue := key.String()
// Create a key node (scalar for the key name)
keyNode := createStringScalarNode(keyName)
// Create a value node (scalar for the value)
valueNode := createStringScalarNode(keyValue)
// Add key-value pair to the root node
root.AddKeyValueChild(keyNode, valueNode)
}
} else {
// For named sections, create a nested map
sectionNode := &CandidateNode{
Kind: MappingNode,
Tag: "!!map",
Value: "",
}
// Add key-value pairs to the section node
for _, key := range section.Keys() {
keyName := key.Name()
keyValue := key.String()
// Create a key node (scalar for the key name)
keyNode := createStringScalarNode(keyName)
// Create a value node (scalar for the value)
valueNode := createStringScalarNode(keyValue)
// Add key-value pair to the section node
sectionNode.AddKeyValueChild(keyNode, valueNode)
}
// Create a key node for the section name
sectionKeyNode := createStringScalarNode(sectionName)
// Add the section as a nested map to the root node
root.AddKeyValueChild(sectionKeyNode, sectionNode)
}
}
// Set the finished flag to true to prevent further Decode calls
dec.finished = true
// Return the root node
return root, nil
}

115
pkg/yqlib/encoder_ini.go Normal file
View File

@ -0,0 +1,115 @@
//go:build !yq_noini
package yqlib
import (
"bytes"
"fmt"
"io"
"github.com/go-ini/ini"
)
type iniEncoder struct {
indentString string
}
// NewINIEncoder creates a new INI encoder with the specified indent level for formatting.
func NewINIEncoder(indent int) Encoder {
var indentString = ""
for index := 0; index < indent; index++ {
indentString = indentString + " "
}
return &iniEncoder{indentString}
}
// CanHandleAliases indicates whether the encoder supports aliases. INI does not support aliases.
func (ie *iniEncoder) CanHandleAliases() bool {
return false
}
// PrintDocumentSeparator is a no-op since INI does not support multiple documents.
func (ie *iniEncoder) PrintDocumentSeparator(_ io.Writer) error {
return nil
}
// PrintLeadingContent is a no-op since INI does not support leading content or comments at the encoder level.
func (ie *iniEncoder) PrintLeadingContent(_ io.Writer, _ string) error {
return nil
}
// Encode converts a CandidateNode into INI format and writes it to the provided writer.
func (ie *iniEncoder) Encode(writer io.Writer, node *CandidateNode) error {
log.Debugf("I need to encode %v", NodeToString(node))
log.Debugf("kids %v", len(node.Content))
if node.Kind == ScalarNode {
return writeStringINI(writer, node.Value+"\n")
}
// Create a new INI configuration.
cfg := ini.Empty()
if node.Kind == MappingNode {
// Default section for key-value pairs at the root level.
defaultSection, err := cfg.NewSection(ini.DefaultSection)
if err != nil {
return err
}
// Process the node's content.
for i := 0; i < len(node.Content); i += 2 {
keyNode := node.Content[i]
valueNode := node.Content[i+1]
key := keyNode.Value
if valueNode.Kind == ScalarNode {
// Add key-value pair to the default section.
_, err := defaultSection.NewKey(key, valueNode.Value)
if err != nil {
return err
}
} else if valueNode.Kind == MappingNode {
// Create a new section for nested MappingNode.
section, err := cfg.NewSection(key)
if err != nil {
return err
}
// Process nested key-value pairs.
for j := 0; j < len(valueNode.Content); j += 2 {
nestedKeyNode := valueNode.Content[j]
nestedValueNode := valueNode.Content[j+1]
if nestedValueNode.Kind == ScalarNode {
_, err := section.NewKey(nestedKeyNode.Value, nestedValueNode.Value)
if err != nil {
return err
}
} else {
log.Debugf("Skipping nested non-scalar value for key %s: %v", nestedKeyNode.Value, nestedValueNode.Kind)
}
}
} else {
log.Debugf("Skipping non-scalar value for key %s: %v", key, valueNode.Kind)
}
}
} else {
return fmt.Errorf("INI encoder supports only MappingNode at the root level, got %v", node.Kind)
}
// Use a buffer to store the INI output as the library doesn't support direct io.Writer with indent.
var buffer bytes.Buffer
_, err := cfg.WriteToIndent(&buffer, ie.indentString)
if err != nil {
return err
}
// Write the buffer content to the provided writer.
_, err = writer.Write(buffer.Bytes())
return err
}
// writeStringINI is a helper function to write a string to the provided writer for INI encoder.
func writeStringINI(writer io.Writer, content string) error {
_, err := writer.Write([]byte(content))
return err
}

6
pkg/yqlib/format.go
View File

@ -77,6 +77,11 @@ var LuaFormat = &Format{"lua", []string{"l"},
func() Decoder { return NewLuaDecoder(ConfiguredLuaPreferences) },
}
var INIFormat = &Format{"ini", []string{"i"},
func() Encoder { return NewINIEncoder(0) },
func() Decoder { return NewINIDecoder() },
}
var Formats = []*Format{
YamlFormat,
JSONFormat,
@ -90,6 +95,7 @@ var Formats = []*Format{
TomlFormat,
ShellVariablesFormat,
LuaFormat,
INIFormat,
}
func (f *Format) MatchesName(name string) bool {

22
pkg/yqlib/ini.go Normal file
View File

@ -0,0 +1,22 @@
package yqlib
type INIPreferences struct {
Indent int
ColorsEnabled bool
}
func NewDefaultINIPreferences() INIPreferences {
return INIPreferences{
Indent: 2,
ColorsEnabled: false,
}
}
func (p *INIPreferences) Copy() INIPreferences {
return INIPreferences{
Indent: p.Indent,
ColorsEnabled: p.ColorsEnabled,
}
}
var ConfiguredINIPreferences = NewDefaultINIPreferences()

200
pkg/yqlib/ini_test.go Normal file
View File

@ -0,0 +1,200 @@
//go:build !yq_noini
package yqlib
import (
"bufio"
"fmt"
"testing"
"github.com/mikefarah/yq/v4/test"
)
const simpleINIInput = `[section]
key = value
`
const expectedSimpleINIOutput = `[section]
key = value
`
const expectedSimpleINIYaml = `section:
key: value
`
var iniScenarios = []formatScenario{
{
description: "Parse INI: simple",
input: simpleINIInput,
scenarioType: "decode",
expected: expectedSimpleINIYaml,
},
{
description: "Encode INI: simple",
input: `section: {key: value}`,
indent: 0,
expected: expectedSimpleINIOutput,
scenarioType: "encode",
},
{
description: "Roundtrip INI: simple",
input: simpleINIInput,
expected: expectedSimpleINIOutput,
scenarioType: "roundtrip",
indent: 0,
},
{
description: "bad ini",
input: `[section\nkey = value`,
expectedError: `bad file 'sample.yml': failed to parse INI content: unclosed section: [section\nkey = value`,
scenarioType: "decode-error",
},
}
func documentRoundtripINIScenario(w *bufio.Writer, s formatScenario, indent int) {
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
if s.subdescription != "" {
writeOrPanic(w, s.subdescription)
writeOrPanic(w, "\n\n")
}
writeOrPanic(w, "Given a sample.ini file of:\n")
writeOrPanic(w, fmt.Sprintf("```ini\n%v\n```\n", s.input))
writeOrPanic(w, "then\n")
expression := s.expression
if expression != "" {
writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=ini -o=ini -I=%v '%v' sample.ini\n```\n", indent, expression))
} else {
writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=ini -o=ini -I=%v sample.ini\n```\n", indent))
}
writeOrPanic(w, "will output\n")
prefs := ConfiguredINIPreferences.Copy()
prefs.Indent = indent
// Pass prefs.Indent instead of prefs
writeOrPanic(w, fmt.Sprintf("```ini\n%v```\n\n", mustProcessFormatScenario(s, NewINIDecoder(), NewINIEncoder(prefs.Indent))))
}
func documentDecodeINIScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
if s.subdescription != "" {
writeOrPanic(w, s.subdescription)
writeOrPanic(w, "\n\n")
}
writeOrPanic(w, "Given a sample.ini file of:\n")
writeOrPanic(w, fmt.Sprintf("```ini\n%v\n```\n", s.input))
writeOrPanic(w, "then\n")
expression := s.expression
if expression != "" {
writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=ini '%v' sample.ini\n```\n", expression))
} else {
writeOrPanic(w, "```bash\nyq -p=ini sample.ini\n```\n")
}
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewINIDecoder(), NewYamlEncoder(ConfiguredYamlPreferences))))
}
func testINIScenario(t *testing.T, s formatScenario) {
prefs := ConfiguredINIPreferences.Copy()
prefs.Indent = s.indent
switch s.scenarioType {
case "encode":
// Pass prefs.Indent instead of prefs
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewINIEncoder(prefs.Indent)), s.description)
case "decode":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewINIDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
case "roundtrip":
// Pass prefs.Indent instead of prefs
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewINIDecoder(), NewINIEncoder(prefs.Indent)), s.description)
case "decode-error":
// Pass prefs.Indent instead of prefs
result, err := processFormatScenario(s, NewINIDecoder(), NewINIEncoder(prefs.Indent))
if err == nil {
t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result)
} else {
test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description)
}
default:
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
}
}
func documentINIScenario(t *testing.T, w *bufio.Writer, i interface{}) {
s := i.(formatScenario)
if s.skipDoc {
return
}
switch s.scenarioType {
case "encode":
documentINIEncodeScenario(w, s)
case "decode":
documentDecodeINIScenario(w, s)
case "roundtrip":
documentRoundtripINIScenario(w, s, s.indent)
case "decode-error":
// Add handling for decode-error scenario type to prevent panic
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
if s.subdescription != "" {
writeOrPanic(w, s.subdescription)
writeOrPanic(w, "\n\n")
}
writeOrPanic(w, "Given a sample.ini file of:\n")
writeOrPanic(w, fmt.Sprintf("```ini\n%v\n```\n", s.input))
writeOrPanic(w, "then an error is expected:\n")
writeOrPanic(w, fmt.Sprintf("```\n%v\n```\n\n", s.expectedError))
default:
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
}
}
func documentINIEncodeScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
if s.subdescription != "" {
writeOrPanic(w, s.subdescription)
writeOrPanic(w, "\n\n")
}
writeOrPanic(w, "Given a sample.yml file of:\n")
writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input))
writeOrPanic(w, "then\n")
expression := s.expression
if expression == "" {
expression = "."
}
if s.indent == 2 {
writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=ini '%v' sample.yml\n```\n", expression))
} else {
writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=ini -I=%v '%v' sample.yml\n```\n", s.indent, expression))
}
writeOrPanic(w, "will output\n")
prefs := ConfiguredINIPreferences.Copy()
prefs.Indent = s.indent
// Pass prefs.Indent instead of prefs
writeOrPanic(w, fmt.Sprintf("```ini\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewINIEncoder(prefs.Indent))))
}
func TestINIScenarios(t *testing.T) {
for _, tt := range iniScenarios {
testINIScenario(t, tt)
}
genericScenarios := make([]interface{}, len(iniScenarios))
for i, s := range iniScenarios {
genericScenarios[i] = s
}
documentScenarios(t, "usage", "convert", genericScenarios, documentINIScenario)
}

11
pkg/yqlib/no_ini.go Normal file
View File

@ -0,0 +1,11 @@
//go:build yq_noini
package yqlib
func NewINIDecoder() Decoder {
return nil
}
func NewINIEncoder(indent int) Encoder {
return nil
}