diff --git a/README.md b/README.md index 2249ff46..c11fb4b5 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ![Build](https://github.com/mikefarah/yq/workflows/Build/badge.svg) ![Docker Pulls](https://img.shields.io/docker/pulls/mikefarah/yq.svg) ![Github Releases (by Release)](https://img.shields.io/github/downloads/mikefarah/yq/total.svg) ![Go Report](https://goreportcard.com/badge/github.com/mikefarah/yq) ![CodeQL](https://github.com/mikefarah/yq/workflows/CodeQL/badge.svg) -a lightweight and portable command-line YAML, JSON and XML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json, xml, properties, csv and tsv. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously. +a lightweight and portable command-line YAML, JSON, INI and XML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json, xml, ini, properties, csv and tsv. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously. yq is written in go - so you can download a dependency free binary for your platform and you are good to go! If you prefer there are a variety of package managers that can be used as well as Docker and Podman, all listed below. @@ -327,7 +327,7 @@ flox install yq ## Features - [Detailed documentation with many examples](https://mikefarah.gitbook.io/yq/) - Written in portable go, so you can download a lovely dependency free binary -- Uses similar syntax as `jq` but works with YAML, [JSON](https://mikefarah.gitbook.io/yq/usage/convert) and [XML](https://mikefarah.gitbook.io/yq/usage/xml) files +- Uses similar syntax as `jq` but works with YAML, INI, [JSON](https://mikefarah.gitbook.io/yq/usage/convert) and [XML](https://mikefarah.gitbook.io/yq/usage/xml) files - Fully supports multi document yaml files - Supports yaml [front matter](https://mikefarah.gitbook.io/yq/usage/front-matter) blocks (e.g. jekyll/assemble) - Colorized yaml output @@ -366,31 +366,52 @@ yq -i '.stuff = "foo"' myfile.yml # update myfile.yml in place Available Commands: - completion Generate the autocompletion script for the specified shell - eval (default) Apply the expression to each document in each yaml file in sequence - eval-all Loads _all_ yaml documents of _all_ yaml files and runs expression once - help Help about any command + completion Generate the autocompletion script for the specified shell + eval (default) Apply the expression to each document in each yaml file in sequence + eval-all Loads _all_ yaml documents of _all_ yaml files and runs expression once + help Help about any command Flags: -C, --colors force print with colors + --csv-auto-parse parse CSV YAML/JSON values (default true) + --csv-separator char CSV Separator character (default ,) -e, --exit-status set exit status if there are no matches or null or false is returned + --expression string forcibly set the expression argument. Useful when yq argument detection thinks your expression is a file. + --from-file string Load expression from specified file. -f, --front-matter string (extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact --header-preprocess Slurp any header comments and separators before processing expression. (default true) -h, --help help for yq -I, --indent int sets indent level for output (default 2) -i, --inplace update the file in place of first file given. - -p, --input-format string [yaml|y|xml|x] parse format for input. Note that json is a subset of yaml. (default "yaml") + -p, --input-format string [auto|a|yaml|y|json|j|props|p|csv|c|tsv|t|xml|x|base64|uri|toml|lua|l|ini|i] parse format for input. (default "auto") + --lua-globals output keys as top-level global variables + --lua-prefix string prefix (default "return ") + --lua-suffix string suffix (default ";\n") + --lua-unquoted output unquoted string keys (e.g. {foo="bar"}) -M, --no-colors force print with no colors -N, --no-doc Don't print document separators (---) + -0, --nul-output Use NUL char to separate values. If unwrap scalar is also set, fail if unwrapped scalar contains NUL char. -n, --null-input Don't read input, simply evaluate the expression given. Useful for creating docs from scratch. - -o, --output-format string [yaml|y|json|j|props|p|xml|x] output format type. (default "yaml") + -o, --output-format string [auto|a|yaml|y|json|j|props|p|csv|c|tsv|t|xml|x|base64|uri|toml|shell|s|lua|l|ini|i] output format type. (default "auto") -P, --prettyPrint pretty print, shorthand for '... style = ""' - -s, --split-exp string print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter. - --unwrapScalar unwrap scalar, print the value with no quotes, colors or comments (default true) + --properties-array-brackets use [x] in array paths (e.g. for SpringBoot) + --properties-separator string separator to use between keys and values (default " = ") + -s, --split-exp string print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter. The necessary directories will be created. + --split-exp-file string Use a file to specify the split-exp expression. + --string-interpolation Toggles strings interpolation of \(exp) (default true) + --tsv-auto-parse parse TSV YAML/JSON values (default true) + -r, --unwrapScalar unwrap scalar, print the value with no quotes, colors or comments. Defaults to true for yaml (default true) -v, --verbose verbose mode -V, --version Print version information and quit - --xml-attribute-prefix string prefix for xml attributes (default "+") + --xml-attribute-prefix string prefix for xml attributes (default "+@") --xml-content-name string name for xml content (if no attribute name is present). (default "+content") + --xml-directive-name string name for xml directives (e.g. ) (default "+directive") + --xml-keep-namespace enables keeping namespace after parsing attributes (default true) + --xml-proc-inst-prefix string prefix for xml processing instructions (e.g. ) (default "+p_") + --xml-raw-token enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details. (default true) + --xml-skip-directives skip over directives (e.g. ) + --xml-skip-proc-inst skip over process instructions (e.g. ) + --xml-strict-mode enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details. Use "yq [command] --help" for more information about a command. ``` diff --git a/examples/sample.ini b/examples/sample.ini new file mode 100644 index 00000000..bf8d20fa --- /dev/null +++ b/examples/sample.ini @@ -0,0 +1,14 @@ + +; This is a INI document + +[owner] +name = "Tom Preston-Werner" +dob = 1979-05-27T07:32:00-08:00 + +[database] +db_host = "localhost" +db_port = 5432 +db_user = "postgres" +db_password = "" +db_name = "postgres" + diff --git a/examples/sample_no_sections.ini b/examples/sample_no_sections.ini new file mode 100644 index 00000000..ffb94ab2 --- /dev/null +++ b/examples/sample_no_sections.ini @@ -0,0 +1,8 @@ + +; This is a INI document +name = "Tom Preston-Werner" +dob = 1979-05-27T07:32:00-08:00 +enabled = true +ip = "10.0.0.1" +role = "frontend" +treads = 4 diff --git a/go.mod b/go.mod index 21871af9..58a3d7c1 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/dimchansky/utfbom v1.1.1 github.com/elliotchance/orderedmap v1.8.0 github.com/fatih/color v1.18.0 + github.com/go-ini/ini v1.67.0 github.com/goccy/go-json v0.10.5 github.com/goccy/go-yaml v1.17.1 github.com/jinzhu/copier v0.4.0 diff --git a/go.sum b/go.sum index e0bbc021..a565b596 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,8 @@ github.com/elliotchance/orderedmap v1.8.0 h1:TrOREecvh3JbS+NCgwposXG5ZTFHtEsQiCG github.com/elliotchance/orderedmap v1.8.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.17.1 h1:LI34wktB2xEE3ONG/2Ar54+/HJVBriAGJ55PHls4YuY= diff --git a/pkg/yqlib/decoder_ini.go b/pkg/yqlib/decoder_ini.go new file mode 100644 index 00000000..f3761e78 --- /dev/null +++ b/pkg/yqlib/decoder_ini.go @@ -0,0 +1,106 @@ +//go:build !yq_noini + +package yqlib + +import ( + "fmt" + "io" + + "github.com/go-ini/ini" +) + +type iniDecoder struct { + reader io.Reader + finished bool // Flag to signal completion of processing +} + +func NewINIDecoder() Decoder { + return &iniDecoder{ + finished: false, // Initialize the flag as false + } +} + +func (dec *iniDecoder) Init(reader io.Reader) error { + // Store the reader for use in Decode + dec.reader = reader + return nil +} + +func (dec *iniDecoder) Decode() (*CandidateNode, error) { + // If processing is already finished, return io.EOF + if dec.finished { + return nil, io.EOF + } + + // Read all content from the stored reader + content, err := io.ReadAll(dec.reader) + if err != nil { + return nil, fmt.Errorf("failed to read INI content: %v", err) + } + + // Parse the INI content + cfg, err := ini.Load(content) + if err != nil { + return nil, fmt.Errorf("failed to parse INI content: %v", err) + } + + // Create a root CandidateNode as a MappingNode (since INI is key-value based) + root := &CandidateNode{ + Kind: MappingNode, + Tag: "!!map", + Value: "", + } + + // Process each section in the INI file + for _, section := range cfg.Sections() { + sectionName := section.Name() + + if sectionName == ini.DefaultSection { + // For the default section, add key-value pairs directly to the root node + for _, key := range section.Keys() { + keyName := key.Name() + keyValue := key.String() + + // Create a key node (scalar for the key name) + keyNode := createStringScalarNode(keyName) + // Create a value node (scalar for the value) + valueNode := createStringScalarNode(keyValue) + + // Add key-value pair to the root node + root.AddKeyValueChild(keyNode, valueNode) + } + } else { + // For named sections, create a nested map + sectionNode := &CandidateNode{ + Kind: MappingNode, + Tag: "!!map", + Value: "", + } + + // Add key-value pairs to the section node + for _, key := range section.Keys() { + keyName := key.Name() + keyValue := key.String() + + // Create a key node (scalar for the key name) + keyNode := createStringScalarNode(keyName) + // Create a value node (scalar for the value) + valueNode := createStringScalarNode(keyValue) + + // Add key-value pair to the section node + sectionNode.AddKeyValueChild(keyNode, valueNode) + } + + // Create a key node for the section name + sectionKeyNode := createStringScalarNode(sectionName) + // Add the section as a nested map to the root node + root.AddKeyValueChild(sectionKeyNode, sectionNode) + } + } + + // Set the finished flag to true to prevent further Decode calls + dec.finished = true + + // Return the root node + return root, nil +} diff --git a/pkg/yqlib/encoder_ini.go b/pkg/yqlib/encoder_ini.go new file mode 100644 index 00000000..cf743490 --- /dev/null +++ b/pkg/yqlib/encoder_ini.go @@ -0,0 +1,115 @@ +//go:build !yq_noini + +package yqlib + +import ( + "bytes" + "fmt" + "io" + + "github.com/go-ini/ini" +) + +type iniEncoder struct { + indentString string +} + +// NewINIEncoder creates a new INI encoder with the specified indent level for formatting. +func NewINIEncoder(indent int) Encoder { + var indentString = "" + for index := 0; index < indent; index++ { + indentString = indentString + " " + } + return &iniEncoder{indentString} +} + +// CanHandleAliases indicates whether the encoder supports aliases. INI does not support aliases. +func (ie *iniEncoder) CanHandleAliases() bool { + return false +} + +// PrintDocumentSeparator is a no-op since INI does not support multiple documents. +func (ie *iniEncoder) PrintDocumentSeparator(_ io.Writer) error { + return nil +} + +// PrintLeadingContent is a no-op since INI does not support leading content or comments at the encoder level. +func (ie *iniEncoder) PrintLeadingContent(_ io.Writer, _ string) error { + return nil +} + +// Encode converts a CandidateNode into INI format and writes it to the provided writer. +func (ie *iniEncoder) Encode(writer io.Writer, node *CandidateNode) error { + log.Debugf("I need to encode %v", NodeToString(node)) + log.Debugf("kids %v", len(node.Content)) + + if node.Kind == ScalarNode { + return writeStringINI(writer, node.Value+"\n") + } + + // Create a new INI configuration. + cfg := ini.Empty() + + if node.Kind == MappingNode { + // Default section for key-value pairs at the root level. + defaultSection, err := cfg.NewSection(ini.DefaultSection) + if err != nil { + return err + } + + // Process the node's content. + for i := 0; i < len(node.Content); i += 2 { + keyNode := node.Content[i] + valueNode := node.Content[i+1] + key := keyNode.Value + + if valueNode.Kind == ScalarNode { + // Add key-value pair to the default section. + _, err := defaultSection.NewKey(key, valueNode.Value) + if err != nil { + return err + } + } else if valueNode.Kind == MappingNode { + // Create a new section for nested MappingNode. + section, err := cfg.NewSection(key) + if err != nil { + return err + } + // Process nested key-value pairs. + for j := 0; j < len(valueNode.Content); j += 2 { + nestedKeyNode := valueNode.Content[j] + nestedValueNode := valueNode.Content[j+1] + if nestedValueNode.Kind == ScalarNode { + _, err := section.NewKey(nestedKeyNode.Value, nestedValueNode.Value) + if err != nil { + return err + } + } else { + log.Debugf("Skipping nested non-scalar value for key %s: %v", nestedKeyNode.Value, nestedValueNode.Kind) + } + } + } else { + log.Debugf("Skipping non-scalar value for key %s: %v", key, valueNode.Kind) + } + } + } else { + return fmt.Errorf("INI encoder supports only MappingNode at the root level, got %v", node.Kind) + } + + // Use a buffer to store the INI output as the library doesn't support direct io.Writer with indent. + var buffer bytes.Buffer + _, err := cfg.WriteToIndent(&buffer, ie.indentString) + if err != nil { + return err + } + + // Write the buffer content to the provided writer. + _, err = writer.Write(buffer.Bytes()) + return err +} + +// writeStringINI is a helper function to write a string to the provided writer for INI encoder. +func writeStringINI(writer io.Writer, content string) error { + _, err := writer.Write([]byte(content)) + return err +} diff --git a/pkg/yqlib/format.go b/pkg/yqlib/format.go index 40bdf346..81f1b240 100644 --- a/pkg/yqlib/format.go +++ b/pkg/yqlib/format.go @@ -77,6 +77,11 @@ var LuaFormat = &Format{"lua", []string{"l"}, func() Decoder { return NewLuaDecoder(ConfiguredLuaPreferences) }, } +var INIFormat = &Format{"ini", []string{"i"}, + func() Encoder { return NewINIEncoder(0) }, + func() Decoder { return NewINIDecoder() }, +} + var Formats = []*Format{ YamlFormat, JSONFormat, @@ -90,6 +95,7 @@ var Formats = []*Format{ TomlFormat, ShellVariablesFormat, LuaFormat, + INIFormat, } func (f *Format) MatchesName(name string) bool { diff --git a/pkg/yqlib/ini.go b/pkg/yqlib/ini.go new file mode 100644 index 00000000..7c498d9d --- /dev/null +++ b/pkg/yqlib/ini.go @@ -0,0 +1,22 @@ +package yqlib + +type INIPreferences struct { + Indent int + ColorsEnabled bool +} + +func NewDefaultINIPreferences() INIPreferences { + return INIPreferences{ + Indent: 2, + ColorsEnabled: false, + } +} + +func (p *INIPreferences) Copy() INIPreferences { + return INIPreferences{ + Indent: p.Indent, + ColorsEnabled: p.ColorsEnabled, + } +} + +var ConfiguredINIPreferences = NewDefaultINIPreferences() diff --git a/pkg/yqlib/ini_test.go b/pkg/yqlib/ini_test.go new file mode 100644 index 00000000..f201a564 --- /dev/null +++ b/pkg/yqlib/ini_test.go @@ -0,0 +1,200 @@ +//go:build !yq_noini + +package yqlib + +import ( + "bufio" + "fmt" + "testing" + + "github.com/mikefarah/yq/v4/test" +) + +const simpleINIInput = `[section] +key = value +` + +const expectedSimpleINIOutput = `[section] +key = value +` + +const expectedSimpleINIYaml = `section: + key: value +` + +var iniScenarios = []formatScenario{ + { + description: "Parse INI: simple", + input: simpleINIInput, + scenarioType: "decode", + expected: expectedSimpleINIYaml, + }, + { + description: "Encode INI: simple", + input: `section: {key: value}`, + indent: 0, + expected: expectedSimpleINIOutput, + scenarioType: "encode", + }, + { + description: "Roundtrip INI: simple", + input: simpleINIInput, + expected: expectedSimpleINIOutput, + scenarioType: "roundtrip", + indent: 0, + }, + { + description: "bad ini", + input: `[section\nkey = value`, + expectedError: `bad file 'sample.yml': failed to parse INI content: unclosed section: [section\nkey = value`, + scenarioType: "decode-error", + }, +} + +func documentRoundtripINIScenario(w *bufio.Writer, s formatScenario, indent int) { + writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) + + if s.subdescription != "" { + writeOrPanic(w, s.subdescription) + writeOrPanic(w, "\n\n") + } + + writeOrPanic(w, "Given a sample.ini file of:\n") + writeOrPanic(w, fmt.Sprintf("```ini\n%v\n```\n", s.input)) + + writeOrPanic(w, "then\n") + + expression := s.expression + if expression != "" { + writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=ini -o=ini -I=%v '%v' sample.ini\n```\n", indent, expression)) + } else { + writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=ini -o=ini -I=%v sample.ini\n```\n", indent)) + } + + writeOrPanic(w, "will output\n") + prefs := ConfiguredINIPreferences.Copy() + prefs.Indent = indent + + // Pass prefs.Indent instead of prefs + writeOrPanic(w, fmt.Sprintf("```ini\n%v```\n\n", mustProcessFormatScenario(s, NewINIDecoder(), NewINIEncoder(prefs.Indent)))) +} + +func documentDecodeINIScenario(w *bufio.Writer, s formatScenario) { + writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) + + if s.subdescription != "" { + writeOrPanic(w, s.subdescription) + writeOrPanic(w, "\n\n") + } + + writeOrPanic(w, "Given a sample.ini file of:\n") + writeOrPanic(w, fmt.Sprintf("```ini\n%v\n```\n", s.input)) + + writeOrPanic(w, "then\n") + + expression := s.expression + if expression != "" { + writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=ini '%v' sample.ini\n```\n", expression)) + } else { + writeOrPanic(w, "```bash\nyq -p=ini sample.ini\n```\n") + } + + writeOrPanic(w, "will output\n") + + writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewINIDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)))) +} + +func testINIScenario(t *testing.T, s formatScenario) { + prefs := ConfiguredINIPreferences.Copy() + prefs.Indent = s.indent + switch s.scenarioType { + case "encode": + // Pass prefs.Indent instead of prefs + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewINIEncoder(prefs.Indent)), s.description) + case "decode": + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewINIDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description) + case "roundtrip": + // Pass prefs.Indent instead of prefs + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewINIDecoder(), NewINIEncoder(prefs.Indent)), s.description) + case "decode-error": + // Pass prefs.Indent instead of prefs + result, err := processFormatScenario(s, NewINIDecoder(), NewINIEncoder(prefs.Indent)) + if err == nil { + t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result) + } else { + test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description) + } + default: + panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) + } +} + +func documentINIScenario(t *testing.T, w *bufio.Writer, i interface{}) { + s := i.(formatScenario) + if s.skipDoc { + return + } + switch s.scenarioType { + case "encode": + documentINIEncodeScenario(w, s) + case "decode": + documentDecodeINIScenario(w, s) + case "roundtrip": + documentRoundtripINIScenario(w, s, s.indent) + case "decode-error": + // Add handling for decode-error scenario type to prevent panic + writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) + if s.subdescription != "" { + writeOrPanic(w, s.subdescription) + writeOrPanic(w, "\n\n") + } + writeOrPanic(w, "Given a sample.ini file of:\n") + writeOrPanic(w, fmt.Sprintf("```ini\n%v\n```\n", s.input)) + writeOrPanic(w, "then an error is expected:\n") + writeOrPanic(w, fmt.Sprintf("```\n%v\n```\n\n", s.expectedError)) + default: + panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) + } +} + +func documentINIEncodeScenario(w *bufio.Writer, s formatScenario) { + writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) + + if s.subdescription != "" { + writeOrPanic(w, s.subdescription) + writeOrPanic(w, "\n\n") + } + + writeOrPanic(w, "Given a sample.yml file of:\n") + writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input)) + + writeOrPanic(w, "then\n") + + expression := s.expression + if expression == "" { + expression = "." + } + + if s.indent == 2 { + writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=ini '%v' sample.yml\n```\n", expression)) + } else { + writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=ini -I=%v '%v' sample.yml\n```\n", s.indent, expression)) + } + writeOrPanic(w, "will output\n") + prefs := ConfiguredINIPreferences.Copy() + prefs.Indent = s.indent + + // Pass prefs.Indent instead of prefs + writeOrPanic(w, fmt.Sprintf("```ini\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewINIEncoder(prefs.Indent)))) +} + +func TestINIScenarios(t *testing.T) { + for _, tt := range iniScenarios { + testINIScenario(t, tt) + } + genericScenarios := make([]interface{}, len(iniScenarios)) + for i, s := range iniScenarios { + genericScenarios[i] = s + } + documentScenarios(t, "usage", "convert", genericScenarios, documentINIScenario) +} diff --git a/pkg/yqlib/no_ini.go b/pkg/yqlib/no_ini.go new file mode 100644 index 00000000..ef1ae8f8 --- /dev/null +++ b/pkg/yqlib/no_ini.go @@ -0,0 +1,11 @@ +//go:build yq_noini + +package yqlib + +func NewINIDecoder() Decoder { + return nil +} + +func NewINIEncoder(indent int) Encoder { + return nil +}