From 6f24e878aa1011a52d94180fcc9a2fdc5d2dec00 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sat, 22 Jan 2022 12:35:33 +1100 Subject: [PATCH] Updated README, better xml docs --- README.md | 47 +++++++++++++---------- pkg/yqlib/doc/usage/headers/xml.md | 14 +------ pkg/yqlib/doc/usage/xml.md | 61 ++++++++++++++++++++---------- pkg/yqlib/xml_test.go | 60 ++++++++++++++++++++++------- 4 files changed, 116 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 10302385..03a4aede 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ![Build](https://github.com/mikefarah/yq/workflows/Build/badge.svg) ![Docker Pulls](https://img.shields.io/docker/pulls/mikefarah/yq.svg) ![Github Releases (by Release)](https://img.shields.io/github/downloads/mikefarah/yq/total.svg) ![Go Report](https://goreportcard.com/badge/github.com/mikefarah/yq) -a lightweight and portable command-line YAML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously. +a lightweight and portable command-line YAML, JSON and XML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json and xml. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously. yq is written in go - so you can download a dependency free binary for your platform and you are good to go! If you prefer there are a variety of package managers that can be used as well as Docker and Podman, all listed below. @@ -241,19 +241,21 @@ Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq) ## Features - [Detailed documentation with many examples](https://mikefarah.gitbook.io/yq/) - Written in portable go, so you can download a lovely dependency free binary -- Uses similar syntax as `jq` but works with YAML and JSON files +- Uses similar syntax as `jq` but works with YAML, [JSON](https://mikefarah.gitbook.io/yq/usage/convert) and [XML](https://mikefarah.gitbook.io/yq/usage/xml) files - Fully supports multi document yaml files - Supports yaml [front matter](https://mikefarah.gitbook.io/yq/usage/front-matter) blocks (e.g. jekyll/assemble) - Colorized yaml output -- [Deeply traverse yaml](https://mikefarah.gitbook.io/yq/operators/traverse-read) -- [Sort yaml by keys](https://mikefarah.gitbook.io/yq/operators/sort-keys) +- [Deeply data structures](https://mikefarah.gitbook.io/yq/operators/traverse-read) +- [Sort keys](https://mikefarah.gitbook.io/yq/operators/sort-keys) - Manipulate yaml [comments](https://mikefarah.gitbook.io/yq/operators/comment-operators), [styling](https://mikefarah.gitbook.io/yq/operators/style), [tags](https://mikefarah.gitbook.io/yq/operators/tag) and [anchors and aliases](https://mikefarah.gitbook.io/yq/operators/anchor-and-alias-operators). -- [Update yaml inplace](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags) +- [Update inplace](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags) - [Complex expressions to select and update](https://mikefarah.gitbook.io/yq/operators/select#select-and-update-matching-values-in-map) - Keeps yaml formatting and comments when updating (though there are issues with whitespace) - [Load content from other files](https://mikefarah.gitbook.io/yq/operators/load) - [Convert to/from json](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert) +- [Convert to/from xml](https://mikefarah.gitbook.io/yq/v/v4.x/usage/xml) - [Convert to properties](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties) +- [Convert to csv/tsv](https://mikefarah.gitbook.io/yq/usage/csv-tsv) - [Pipe data in by using '-'](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate) - [General shell completion scripts (bash/zsh/fish/powershell)](https://mikefarah.gitbook.io/yq/v/v4.x/commands/shell-completion) - [Reduce](https://mikefarah.gitbook.io/yq/operators/reduce) to merge multiple files or sum an array or other fancy things. @@ -269,27 +271,32 @@ Usage: yq [command] Available Commands: + completion Generate the autocompletion script for the specified shell eval Apply the expression to each document in each yaml file in sequence eval-all Loads _all_ yaml documents of _all_ yaml files and runs expression once help Help about any command shell-completion Generate completion script Flags: - -C, --colors force print with colors - -e, --exit-status set exit status if there are no matches or null or false is returned - -f, --front-matter string (extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact - --header-preprocess Slurp any header comments and seperators before processing expression. This is a workaround for go-yaml to persist header content (default true) - -h, --help help for yq - -I, --indent int sets indent level for output (default 2) - -i, --inplace update the yaml file inplace of first yaml file given. - -M, --no-colors force print with no colors - -N, --no-doc Don't print document separators (---) - -n, --null-input Don't read input, simply evaluate the expression given. Useful for creating yaml docs from scratch. - -o, --output-format string [yaml|y|json|j|props|p] output format type. (default "yaml") - -P, --prettyPrint pretty print, shorthand for '... style = ""' - --unwrapScalar unwrap scalar, print the value with no quotes, colors or comments (default true) - -v, --verbose verbose mode - -V, --version Print version information and quit + -C, --colors force print with colors + -e, --exit-status set exit status if there are no matches or null or false is returned + -f, --front-matter string (extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact + --header-preprocess Slurp any header comments and separators before processing expression. (default true) + -h, --help help for yq + -I, --indent int sets indent level for output (default 2) + -i, --inplace update the file inplace of first file given. + -p, --input-format string [yaml|y|xml|x] parse format for input. Note that json is a subset of yaml. (default "yaml") + -M, --no-colors force print with no colors + -N, --no-doc Don't print document separators (---) + -n, --null-input Don't read input, simply evaluate the expression given. Useful for creating docs from scratch. + -o, --output-format string [yaml|y|json|j|props|p|xml|x] output format type. (default "yaml") + -P, --prettyPrint pretty print, shorthand for '... style = ""' + -s, --split-exp string print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter. + --unwrapScalar unwrap scalar, print the value with no quotes, colors or comments (default true) + -v, --verbose verbose mode + -V, --version Print version information and quit + --xml-attribute-prefix string prefix for xml attributes (default "+") + --xml-content-name string name for xml content (if no attribute name is present). (default "+content") Use "yq [command] --help" for more information about a command. ``` diff --git a/pkg/yqlib/doc/usage/headers/xml.md b/pkg/yqlib/doc/usage/headers/xml.md index 1bf3d01a..fd8e79d6 100644 --- a/pkg/yqlib/doc/usage/headers/xml.md +++ b/pkg/yqlib/doc/usage/headers/xml.md @@ -4,16 +4,4 @@ Encode and decode to and from XML. Whitespace is not conserved for round trips - Consecutive xml nodes with the same name are assumed to be arrays. -All values in XML are assumed to be strings - but you can use `from_yaml` to parse them into their correct types: - - -``` -yq e -p=xml '.myNumberField |= from_yaml' my.xml -``` - - -```xml -meow -``` - -The content of the node will be set as a field in the map with the key "+content". Use the `--xml-content-name` flag to change this. +XML content data and attributes are created as fields. This can be controlled by the `'--xml-attribute-prefix` and `--xml-content-name` flags - see below for examples. diff --git a/pkg/yqlib/doc/usage/xml.md b/pkg/yqlib/doc/usage/xml.md index c18a039e..80317b3a 100644 --- a/pkg/yqlib/doc/usage/xml.md +++ b/pkg/yqlib/doc/usage/xml.md @@ -4,25 +4,19 @@ Encode and decode to and from XML. Whitespace is not conserved for round trips - Consecutive xml nodes with the same name are assumed to be arrays. -All values in XML are assumed to be strings - but you can use `from_yaml` to parse them into their correct types: - - -``` -yq e -p=xml '.myNumberField |= from_yaml' my.xml -``` - - -```xml -meow -``` - -The content of the node will be set as a field in the map with the key "+content". Use the `--xml-content-name` flag to change this. +XML content data and attributes are created as fields. This can be controlled by the `'--xml-attribute-prefix` and `--xml-content-name` flags - see below for examples. ## Parse xml: simple +Notice how all the values are strings, see the next example on how you can fix that. + Given a sample.xml file of: ```xml -meow + + meow + 4 + true + ``` then ```bash @@ -30,7 +24,34 @@ yq e -p=xml '.' sample.xml ``` will output ```yaml -cat: meow +cat: + says: meow + legs: "4" + cute: "true" +``` + +## Parse xml: number +All values are assumed to be strings when parsing XML, but you can use the `from_yaml` operator on all the strings values to autoparse into the correct type. + +Given a sample.xml file of: +```xml + + + meow + 4 + true + +``` +then +```bash +yq e -p=xml ' (.. | select(tag == "!!str")) |= from_yaml' sample.xml +``` +will output +```yaml +cat: + says: meow + legs: 4 + cute: true ``` ## Parse xml: array @@ -39,8 +60,8 @@ Consecutive nodes with identical xml names are assumed to be arrays. Given a sample.xml file of: ```xml -1 -2 +cat +goat ``` then ```bash @@ -49,8 +70,8 @@ yq e -p=xml '.' sample.xml will output ```yaml animal: - - "1" - - "2" + - cat + - goat ``` ## Parse xml: attributes @@ -75,7 +96,7 @@ cat: ``` ## Parse xml: attributes with content -Content is added as a field, using the default content name of '+content'. Use `--xml-content-name` to set your own. +Content is added as a field, using the default content name of `+content`. Use `--xml-content-name` to set your own. Given a sample.xml file of: ```xml diff --git a/pkg/yqlib/xml_test.go b/pkg/yqlib/xml_test.go index b45a68df..049ed314 100644 --- a/pkg/yqlib/xml_test.go +++ b/pkg/yqlib/xml_test.go @@ -11,17 +11,39 @@ import ( yaml "gopkg.in/yaml.v3" ) -func decodeXml(t *testing.T, xml string) *CandidateNode { +func decodeXml(t *testing.T, s formatScenario) *CandidateNode { decoder := NewXmlDecoder("+", "+content") - decoder.Init(strings.NewReader(xml)) + decoder.Init(strings.NewReader(s.input)) node := &yaml.Node{} err := decoder.Decode(node) if err != nil { - t.Error(err, "fail to decode", xml) + t.Error(err, "fail to decode", s.input) } - return &CandidateNode{Node: node} + + expression := s.expression + if expression == "" { + expression = "." + } + + exp, err := NewExpressionParser().ParseExpression(expression) + + if err != nil { + t.Error(err) + return nil + } + + candidateNode := CandidateNode{Node: node} + + context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: candidateNode.AsList()}, exp) + + if err != nil { + t.Error(err) + return nil + } + + return context.MatchingNodes.Front().Value.(*CandidateNode) } func processXmlScenario(s formatScenario) string { @@ -207,15 +229,23 @@ var expectedXmlWithComments = `