Updated README, better xml docs

README.md

@@ -3,7 +3,7 @@
 ![Build](https://github.com/mikefarah/yq/workflows/Build/badge.svg)  ![Docker Pulls](https://img.shields.io/docker/pulls/mikefarah/yq.svg) ![Github Releases (by Release)](https://img.shields.io/github/downloads/mikefarah/yq/total.svg) ![Go Report](https://goreportcard.com/badge/github.com/mikefarah/yq)
 
 
-a lightweight and portable command-line YAML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.
+a lightweight and portable command-line YAML, JSON and XML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json and xml. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.
 
 yq is written in go - so you can download a dependency free binary for your platform and you are good to go! If you prefer there are a variety of package managers that can be used as well as Docker and Podman, all listed below.
 
@@ -241,19 +241,21 @@ Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq)
 ## Features
 - [Detailed documentation with many examples](https://mikefarah.gitbook.io/yq/)
 - Written in portable go, so you can download a lovely dependency free binary
-- Uses similar syntax as `jq` but works with YAML and JSON files
+- Uses similar syntax as `jq` but works with YAML, [JSON](https://mikefarah.gitbook.io/yq/usage/convert) and [XML](https://mikefarah.gitbook.io/yq/usage/xml) files
 - Fully supports multi document yaml files
 - Supports yaml [front matter](https://mikefarah.gitbook.io/yq/usage/front-matter) blocks (e.g. jekyll/assemble)
 - Colorized yaml output
-- [Deeply traverse yaml](https://mikefarah.gitbook.io/yq/operators/traverse-read)
+- [Deeply data structures](https://mikefarah.gitbook.io/yq/operators/traverse-read)
-- [Sort yaml by keys](https://mikefarah.gitbook.io/yq/operators/sort-keys)
+- [Sort keys](https://mikefarah.gitbook.io/yq/operators/sort-keys)
 - Manipulate yaml [comments](https://mikefarah.gitbook.io/yq/operators/comment-operators), [styling](https://mikefarah.gitbook.io/yq/operators/style), [tags](https://mikefarah.gitbook.io/yq/operators/tag) and [anchors and aliases](https://mikefarah.gitbook.io/yq/operators/anchor-and-alias-operators).
-- [Update yaml inplace](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags)
+- [Update inplace](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags)
 - [Complex expressions to select and update](https://mikefarah.gitbook.io/yq/operators/select#select-and-update-matching-values-in-map)
 - Keeps yaml formatting and comments when updating (though there are issues with whitespace)
 - [Load content from other files](https://mikefarah.gitbook.io/yq/operators/load)
 - [Convert to/from json](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert)
+- [Convert to/from xml](https://mikefarah.gitbook.io/yq/v/v4.x/usage/xml)
 - [Convert to properties](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties)
+- [Convert to csv/tsv](https://mikefarah.gitbook.io/yq/usage/csv-tsv)
 - [Pipe data in by using '-'](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate)
 - [General shell completion scripts (bash/zsh/fish/powershell)](https://mikefarah.gitbook.io/yq/v/v4.x/commands/shell-completion)
 - [Reduce](https://mikefarah.gitbook.io/yq/operators/reduce) to merge multiple files or sum an array or other fancy things.
@@ -269,6 +271,7 @@ Usage:
   yq [command]
 
 Available Commands:
+  completion       Generate the autocompletion script for the specified shell
   eval             Apply the expression to each document in each yaml file in sequence
   eval-all         Loads _all_ yaml documents of _all_ yaml files and runs expression once
   help             Help about any command
@@ -278,18 +281,22 @@ Flags:
   -C, --colors                        force print with colors
   -e, --exit-status                   set exit status if there are no matches or null or false is returned
   -f, --front-matter string           (extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact
-      --header-preprocess     Slurp any header comments and seperators before processing expression. This is a workaround for go-yaml to persist header content (default true)
+      --header-preprocess             Slurp any header comments and separators before processing expression. (default true)
   -h, --help                          help for yq
   -I, --indent int                    sets indent level for output (default 2)
-  -i, --inplace               update the yaml file inplace of first yaml file given.
+  -i, --inplace                       update the file inplace of first file given.
+  -p, --input-format string           [yaml|y|xml|x] parse format for input. Note that json is a subset of yaml. (default "yaml")
   -M, --no-colors                     force print with no colors
   -N, --no-doc                        Don't print document separators (---)
-  -n, --null-input            Don't read input, simply evaluate the expression given. Useful for creating yaml docs from scratch.
+  -n, --null-input                    Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.
-  -o, --output-format string  [yaml|y|json|j|props|p] output format type. (default "yaml")
+  -o, --output-format string          [yaml|y|json|j|props|p|xml|x] output format type. (default "yaml")
   -P, --prettyPrint                   pretty print, shorthand for '... style = ""'
+  -s, --split-exp string              print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter.
       --unwrapScalar                  unwrap scalar, print the value with no quotes, colors or comments (default true)
   -v, --verbose                       verbose mode
   -V, --version                       Print version information and quit
+      --xml-attribute-prefix string   prefix for xml attributes (default "+")
+      --xml-content-name string       name for xml content (if no attribute name is present). (default "+content")
 
 Use "yq [command] --help" for more information about a command.
 ```

pkg/yqlib/doc/usage/headers/xml.md

@@ -4,16 +4,4 @@ Encode and decode to and from XML. Whitespace is not conserved for round trips -
 
 Consecutive xml nodes with the same name are assumed to be arrays.
 
-All values in XML are assumed to be strings - but you can use `from_yaml` to parse them into their correct types:
+XML content data and attributes are created as fields. This can be controlled by the `'--xml-attribute-prefix` and `--xml-content-name` flags - see below for examples.
-
-
-```
-yq e -p=xml '.myNumberField |= from_yaml' my.xml
-```
-
-
-```xml
-<cat name="tiger">meow</cat>
-```
-
-The content of the node will be set as a field in the map with the key "+content". Use the `--xml-content-name` flag to change this.

pkg/yqlib/doc/usage/xml.md

@@ -4,25 +4,19 @@ Encode and decode to and from XML. Whitespace is not conserved for round trips -
 
 Consecutive xml nodes with the same name are assumed to be arrays.
 
-All values in XML are assumed to be strings - but you can use `from_yaml` to parse them into their correct types:
+XML content data and attributes are created as fields. This can be controlled by the `'--xml-attribute-prefix` and `--xml-content-name` flags - see below for examples.
-
-
-```
-yq e -p=xml '.myNumberField |= from_yaml' my.xml
-```
-
-
-```xml
-<cat name="tiger">meow</cat>
-```
-
-The content of the node will be set as a field in the map with the key "+content". Use the `--xml-content-name` flag to change this.
 
 ## Parse xml: simple
+Notice how all the values are strings, see the next example on how you can fix that.
+
 Given a sample.xml file of:
 ```xml
 <?xml version="1.0" encoding="UTF-8"?>
-<cat>meow</cat>
+<cat>
+  <says>meow</says>
+  <legs>4</legs>
+  <cute>true</cute>
+</cat>
 ```
 then
 ```bash
@@ -30,7 +24,34 @@ yq e -p=xml '.' sample.xml
 ```
 will output
 ```yaml
-cat: meow
+cat:
+  says: meow
+  legs: "4"
+  cute: "true"
+```
+
+## Parse xml: number
+All values are assumed to be strings when parsing XML, but you can use the `from_yaml` operator on all the strings values to autoparse into the correct type.
+
+Given a sample.xml file of:
+```xml
+<?xml version="1.0" encoding="UTF-8"?>
+<cat>
+  <says>meow</says>
+  <legs>4</legs>
+  <cute>true</cute>
+</cat>
+```
+then
+```bash
+yq e -p=xml ' (.. | select(tag == "!!str")) |= from_yaml' sample.xml
+```
+will output
+```yaml
+cat:
+  says: meow
+  legs: 4
+  cute: true
 ```
 
 ## Parse xml: array
@@ -39,8 +60,8 @@ Consecutive nodes with identical xml names are assumed to be arrays.
 Given a sample.xml file of:
 ```xml
 <?xml version="1.0" encoding="UTF-8"?>
-<animal>1</animal>
+<animal>cat</animal>
-<animal>2</animal>
+<animal>goat</animal>
 ```
 then
 ```bash
@@ -49,8 +70,8 @@ yq e -p=xml '.' sample.xml
 will output
 ```yaml
 animal:
-  - "1"
+  - cat
-  - "2"
+  - goat
 ```
 
 ## Parse xml: attributes
@@ -75,7 +96,7 @@ cat:
 ```
 
 ## Parse xml: attributes with content
-Content is added as a field, using the default content name of '+content'. Use `--xml-content-name` to set your own.
+Content is added as a field, using the default content name of `+content`. Use `--xml-content-name` to set your own.
 
 Given a sample.xml file of:
 ```xml

pkg/yqlib/xml_test.go

@@ -11,17 +11,39 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
-func decodeXml(t *testing.T, xml string) *CandidateNode {
+func decodeXml(t *testing.T, s formatScenario) *CandidateNode {
 	decoder := NewXmlDecoder("+", "+content")
 
-	decoder.Init(strings.NewReader(xml))
+	decoder.Init(strings.NewReader(s.input))
 
 	node := &yaml.Node{}
 	err := decoder.Decode(node)
 	if err != nil {
-		t.Error(err, "fail to decode", xml)
+		t.Error(err, "fail to decode", s.input)
 	}
-	return &CandidateNode{Node: node}
+
+	expression := s.expression
+	if expression == "" {
+		expression = "."
+	}
+
+	exp, err := NewExpressionParser().ParseExpression(expression)
+
+	if err != nil {
+		t.Error(err)
+		return nil
+	}
+
+	candidateNode := CandidateNode{Node: node}
+
+	context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: candidateNode.AsList()}, exp)
+
+	if err != nil {
+		t.Error(err)
+		return nil
+	}
+
+	return context.MatchingNodes.Front().Value.(*CandidateNode)
 }
 
 func processXmlScenario(s formatScenario) string {
@@ -208,14 +230,22 @@ var expectedXmlWithComments = `<!-- above_cat inline_cat --><cat><!-- above_arra
 var xmlScenarios = []formatScenario{
 	{
 		description:    "Parse xml: simple",
-		input:       "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat>meow</cat>",
+		subdescription: "Notice how all the values are strings, see the next example on how you can fix that.",
-		expected:    "D0, P[], (doc)::cat: meow\n",
+		input:          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat>\n  <says>meow</says>\n  <legs>4</legs>\n  <cute>true</cute>\n</cat>",
+		expected:       "D0, P[], (doc)::cat:\n    says: meow\n    legs: \"4\"\n    cute: \"true\"\n",
+	},
+	{
+		description:    "Parse xml: number",
+		subdescription: "All values are assumed to be strings when parsing XML, but you can use the `from_yaml` operator on all the strings values to autoparse into the correct type.",
+		input:          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat>\n  <says>meow</says>\n  <legs>4</legs>\n  <cute>true</cute>\n</cat>",
+		expression:     " (.. | select(tag == \"!!str\")) |= from_yaml",
+		expected:       "D0, P[], ()::cat:\n    says: meow\n    legs: 4\n    cute: true\n",
 	},
 	{
 		description:    "Parse xml: array",
 		subdescription: "Consecutive nodes with identical xml names are assumed to be arrays.",
-		input:          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<animal>1</animal>\n<animal>2</animal>",
+		input:          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<animal>cat</animal>\n<animal>goat</animal>",
-		expected:       "D0, P[], (doc)::animal:\n    - \"1\"\n    - \"2\"\n",
+		expected:       "D0, P[], (doc)::animal:\n    - cat\n    - goat\n",
 	},
 	{
 		description:    "Parse xml: attributes",
@@ -225,7 +255,7 @@ var xmlScenarios = []formatScenario{
 	},
 	{
 		description:    "Parse xml: attributes with content",
-		subdescription: "Content is added as a field, using the default content name of '+content'. Use `--xml-content-name` to set your own.",
+		subdescription: "Content is added as a field, using the default content name of `+content`. Use `--xml-content-name` to set your own.",
 		input:          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">meow</cat>",
 		expected:       "D0, P[], (doc)::cat:\n    +content: meow\n    +legs: \"4\"\n",
 	},
@@ -302,7 +332,7 @@ func testXmlScenario(t *testing.T, s formatScenario) {
 	if s.scenarioType == "encode" || s.scenarioType == "roundtrip" {
 		test.AssertResultWithContext(t, s.expected, processXmlScenario(s), s.description)
 	} else {
-		var actual = resultToString(t, decodeXml(t, s.input))
+		var actual = resultToString(t, decodeXml(t, s))
 		test.AssertResultWithContext(t, s.expected, actual, s.description)
 	}
 }
@@ -335,13 +365,17 @@ func documentXmlDecodeScenario(t *testing.T, w *bufio.Writer, s formatScenario)
 	writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input))
 
 	writeOrPanic(w, "then\n")
-	writeOrPanic(w, "```bash\nyq e -p=xml '.' sample.xml\n```\n")
+	expression := s.expression
+	if expression == "" {
+		expression = "."
+	}
+	writeOrPanic(w, fmt.Sprintf("```bash\nyq e -p=xml '%v' sample.xml\n```\n", expression))
 	writeOrPanic(w, "will output\n")
 
 	var output bytes.Buffer
 	printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
 
-	node := decodeXml(t, s.input)
+	node := decodeXml(t, s)
 
 	err := printer.PrintResults(node.AsList())
 	if err != nil {