suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2024-12-19 20:19:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

xml wip

Browse Source
This commit is contained in:
Mike Farah 2022-01-15 11:25:56 +11:00
parent 4c47f43251
commit deee3c1f60
8 changed files with 385 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/utils.go
View File

@ -52,7 +52,7 @@ func configureDecoder() (yqlib.Decoder, error) {
}
switch yqlibInputFormat {
case yqlib.XmlInputFormat:
return yqlib.NewXmlDecoder(), nil
return yqlib.NewXmlDecoder(xmlAttributePrefix, xmlContentName), nil
}
return yqlib.NewYamlDecoder(), nil
}
@ -87,7 +87,7 @@ func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder {
case yqlib.YamlOutputFormat:
return yqlib.NewYamlEncoder(indent, colorsEnabled, !noDocSeparators, unwrapScalar)
case yqlib.XmlOutputFormat:
return yqlib.NewXmlEncoder(indent)
return yqlib.NewXmlEncoder(indent, xmlAttributePrefix, xmlContentName)
}
panic("invalid encoder")
}

1
go.mod
View File

@ -6,6 +6,7 @@ require (
github.com/goccy/go-yaml v1.9.5
github.com/jinzhu/copier v0.3.4
github.com/magiconair/properties v1.8.5
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
github.com/spf13/cobra v1.3.0
github.com/timtadh/lexmachine v0.2.2
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d

2
go.sum
View File

@ -292,6 +292,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=

41
pkg/yqlib/decoder_xml.go
View File

@ -34,15 +34,15 @@ func InputFormatFromString(format string) (InputFormat, error) {
type xmlDecoder struct {
reader io.Reader
attributePrefix string
contentPrefix string
contentName string
finished bool
}
func NewXmlDecoder(attributePrefix string, contentPrefix string) Decoder {
if contentPrefix == "" {
contentPrefix = "content"
func NewXmlDecoder(attributePrefix string, contentName string) Decoder {
if contentName == "" {
contentName = "content"
}
return &xmlDecoder{attributePrefix: attributePrefix, contentPrefix: contentPrefix, finished: false}
return &xmlDecoder{attributePrefix: attributePrefix, contentName: contentName, finished: false}
}
func (dec *xmlDecoder) Init(reader io.Reader) {
@ -75,7 +75,7 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
yamlNode := &yaml.Node{Kind: yaml.MappingNode}
if len(n.Data) > 0 {
label := dec.contentPrefix
label := dec.contentName
labelNode := createScalarNode(label, label)
labelNode.HeadComment = dec.processComment(n.HeadComment)
labelNode.FootComment = dec.processComment(n.FootComment)
@ -94,9 +94,9 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
}
if i == len(n.Children)-1 {
labelNode.FootComment = dec.processComment(n.FootComment)
}
// if i == len(n.Children)-1 {
labelNode.FootComment = dec.processComment(keyValuePair.FootComment)
// }
log.Debug("len of children in %v is %v", label, len(children))
if len(children) > 1 {
@ -167,8 +167,9 @@ type xmlNode struct {
}
type xmlChildrenKv struct {
K string
V []*xmlNode
K string
V []*xmlNode
FootComment string
}
// AddChild appends a node to the list of children
@ -255,8 +256,8 @@ func (dec *xmlDecoder) decodeXml(root *xmlNode) error {
commentStr := string(xml.CharData(se))
if elem.state == "started" {
log.Debug("got a foot comment for %v: [%v]", elem.label, commentStr)
elem.n.FootComment = joinFilter([]string{elem.n.FootComment, commentStr})
applyFootComment(elem, commentStr)
} else if elem.state == "chardata" {
log.Debug("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
elem.n.LineComment = joinFilter([]string{elem.n.LineComment, commentStr})
@ -271,6 +272,20 @@ func (dec *xmlDecoder) decodeXml(root *xmlNode) error {
return nil
}
func applyFootComment(elem *element, commentStr string) {
// first lets try to put the comment on the last child
if len(elem.n.Children) > 0 {
lastChildIndex := len(elem.n.Children) - 1
childKv := elem.n.Children[lastChildIndex]
log.Debug("got a foot comment for %v: [%v]", childKv.K, commentStr)
childKv.FootComment = joinFilter([]string{elem.n.FootComment, commentStr})
} else {
log.Debug("got a foot comment for %v: [%v]", elem.label, commentStr)
elem.n.FootComment = joinFilter([]string{elem.n.FootComment, commentStr})
}
}
func joinFilter(rawStrings []string) string {
stringsToJoin := make([]string, 0)
for _, str := range rawStrings {

227
pkg/yqlib/doc/usage/xml.md
View File

@ -22,6 +22,229 @@ XML nodes that have attributes then plain content, e.g:
The content of the node will be set as a field in the map with the key "+content". Use the `--xml-content-name` flag to change this.
## Parse xml: simple
Given a sample.xml file of:
```xml
<?xml version="1.0" encoding="UTF-8"?>
<cat>meow</cat>
```
then
```bash
yq e -p=xml '.' sample.xml
```
will output
```yaml
cat: meow
```
## Parse xml: array
Consecutive nodes with identical xml names are assumed to be arrays.
Given a sample.xml file of:
```xml
<?xml version="1.0" encoding="UTF-8"?>
<animal>1</animal>
<animal>2</animal>
```
then
```bash
yq e -p=xml '.' sample.xml
```
will output
```yaml
animal:
- "1"
- "2"
```
## Parse xml: attributes
Attributes are converted to fields, with the attribute prefix.
Given a sample.xml file of:
```xml
<?xml version="1.0" encoding="UTF-8"?>
<cat legs="4">
<legs>7</legs>
</cat>
```
then
```bash
yq e -p=xml '.' sample.xml
```
will output
```yaml
cat:
+legs: "4"
legs: "7"
```
## Parse xml: attributes with content
Content is added as a field, using the content name
Given a sample.xml file of:
```xml
<?xml version="1.0" encoding="UTF-8"?>
<cat legs="4">meow</cat>
```
then
```bash
yq e -p=xml '.' sample.xml
```
will output
```yaml
cat:
+content: meow
+legs: "4"
```
## Parse xml: with comments
A best attempt is made to preserve comments.
Given a sample.xml file of:
```xml
<!-- before cat -->
<cat>
<!-- in cat before -->
<x>3<!-- multi
line comment
for x --></x>
<!-- before y -->
<y>
<!-- in y before -->
<d><!-- in d before -->z<!-- in d after --></d>
<!-- in y after -->
</y>
<!-- in_cat_after -->
</cat>
<!-- after cat -->
```
then
```bash
yq e -p=xml '.' sample.xml
```
will output
```yaml
# before cat
cat:
# in cat before
x: "3" # multi
# line comment
# for x
# before y
y:
# in y before
# in d before
d: z # in d after
# in y after
# in_cat_after
# after cat
```
## Encode xml: simple
Given a sample.yml file of:
```yaml
cat: purrs
```
then
```bash
yq e -o=xml '.' sample.yml
```
will output
```xml
<cat>purrs</cat>
```
## Encode xml: array
Given a sample.yml file of:
```yaml
pets:
cat:
- purrs
- meows
```
then
```bash
yq e -o=xml '.' sample.yml
```
will output
```xml
<pets>
<cat>purrs</cat>
<cat>meows</cat>
</pets>
```
## Encode xml: attributes
Fields with the matching xml-attribute-prefix are assumed to be attributes.
Given a sample.yml file of:
```yaml
cat:
+name: tiger
meows: true
```
then
```bash
yq e -o=xml '.' sample.yml
```
will output
```xml
<cat name="tiger">
<meows>true</meows>
</cat>
```
## Encode xml: attributes with content
Fields with the matching xml-content-name is assumed to be content.
Given a sample.yml file of:
```yaml
cat:
+name: tiger
+content: cool
```
then
```bash
yq e -o=xml '.' sample.yml
```
will output
```xml
<cat name="tiger">cool</cat>
```
## Encode xml: comments
A best attempt is made to copy comments to xml.
Given a sample.yml file of:
```yaml
# above_cat
cat: # inline_cat
# above_array
array: # inline_array
- val1 # inline_val1
# above_val2
- val2 # inline_val2
# below_cat
```
then
```bash
yq e -o=xml '.' sample.yml
```
will output
```xml
<!-- above_cat inline_cat --><cat><!-- above_array inline_array -->
<array>val1<!-- inline_val1 --></array>
<array><!-- above_val2 -->val2<!-- inline_val2 --></array>
</cat><!-- below_cat -->
```
## Round trip: with comments
A best effort is made, but comment positions and white space are not preserved perfectly.
@ -58,12 +281,14 @@ cat:
x: "3" # multi
# line comment
# for x
# before y
y:
# in y before
# in d before
d: z # in d after
# in y after
# before y in_cat_after
# in_cat_after
# after cat
```

12
pkg/yqlib/encoder_xml.go
View File

@ -168,16 +168,18 @@ func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) erro
}
func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
e.encodeComment(encoder, headAndLineComment(node))
if err := e.encodeComment(encoder, headAndLineComment(node)); err != nil {
return err
}
for i := 0; i < len(node.Content); i++ {
value := node.Content[i]
err := e.doEncode(encoder, value, start.Copy())
if err != nil {
if err := e.doEncode(encoder, value, start.Copy()); err != nil {
return err
}
}
e.encodeComment(encoder, footComment(node))
return nil
return e.encodeComment(encoder, footComment(node))
}
func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {

185
pkg/yqlib/xml_test.go
View File

@ -105,13 +105,14 @@ cat:
y:
# in y before
# in d before
d:
# in d before
z:
sweet: cool
# in d after
# in y after
# after cat
+sweet: cool
# in d after
# in y after
# in_cat_after
# after cat
`
var inputXmlWithCommentsWithArray = `
@ -124,7 +125,8 @@ for x --></x>
<!-- before y -->
<y>
<!-- in y before -->
<d><!-- in d before --><z/><!-- in d after --></d>
<d><!-- in d before --><z sweet="cool"/><!-- in d after --></d>
<d><!-- in d2 before --><z sweet="cool2"/><!-- in d2 after --></d>
<!-- in y after -->
</y>
@ -133,6 +135,30 @@ for x --></x>
<!-- after cat -->
`
var expectedDecodeYamlWithArray = `D0, P[], (doc)::# before cat
cat:
# in cat before
x: "3" # multi
# line comment
# for x
# before y
y:
# in y before
d:
- # in d before
z:
+sweet: cool
# in d after
- # in d2 before
z:
+sweet: cool2
# in d2 after
# in y after
# in_cat_after
# after cat
`
var expectedDecodeYamlWithComments = `D0, P[], (doc)::# before cat
cat:
# in cat before
@ -146,7 +172,7 @@ cat:
# in d before
d: z # in d after
# in y after
# in_cat_after
# after cat
`
@ -178,36 +204,36 @@ var expectedXmlWithComments = `<!-- above_cat inline_cat --><cat><!-- above_arra
`
var xmlScenarios = []xmlScenario{
// {
// description: "Parse xml: simple",
// input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat>meow</cat>",
// expected: "D0, P[], (doc)::cat: meow\n",
// },
// {
// description: "Parse xml: array",
// subdescription: "Consecutive nodes with identical xml names are assumed to be arrays.",
// input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<animal>1</animal>\n<animal>2</animal>",
// expected: "D0, P[], (doc)::animal:\n - \"1\"\n - \"2\"\n",
// },
// {
// description: "Parse xml: attributes",
// subdescription: "Attributes are converted to fields, with the attribute prefix.",
// input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">\n <legs>7</legs>\n</cat>",
// expected: "D0, P[], (doc)::cat:\n +legs: \"4\"\n legs: \"7\"\n",
// },
// {
// description: "Parse xml: attributes with content",
// subdescription: "Content is added as a field, using the content name",
// input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">meow</cat>",
// expected: "D0, P[], (doc)::cat:\n +content: meow\n +legs: \"4\"\n",
// },
// {
// description: "Parse xml: with comments",
// subdescription: "A best attempt is made to preserve comments.",
// input: inputXmlWithComments,
// expected: expectedDecodeYamlWithComments,
// scenarioType: "decode",
// },
{
description: "Parse xml: simple",
input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat>meow</cat>",
expected: "D0, P[], (doc)::cat: meow\n",
},
{
description: "Parse xml: array",
subdescription: "Consecutive nodes with identical xml names are assumed to be arrays.",
input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<animal>1</animal>\n<animal>2</animal>",
expected: "D0, P[], (doc)::animal:\n - \"1\"\n - \"2\"\n",
},
{
description: "Parse xml: attributes",
subdescription: "Attributes are converted to fields, with the attribute prefix.",
input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">\n <legs>7</legs>\n</cat>",
expected: "D0, P[], (doc)::cat:\n +legs: \"4\"\n legs: \"7\"\n",
},
{
description: "Parse xml: attributes with content",
subdescription: "Content is added as a field, using the content name",
input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">meow</cat>",
expected: "D0, P[], (doc)::cat:\n +content: meow\n +legs: \"4\"\n",
},
{
description: "Parse xml: with comments",
subdescription: "A best attempt is made to preserve comments.",
input: inputXmlWithComments,
expected: expectedDecodeYamlWithComments,
scenarioType: "decode",
},
{
description: "Parse xml: with comments subchild",
skipDoc: true,
@ -215,45 +241,52 @@ var xmlScenarios = []xmlScenario{
expected: expectedDecodeYamlWithSubChild,
scenarioType: "decode",
},
// {
// description: "Encode xml: simple",
// input: "cat: purrs",
// expected: "<cat>purrs</cat>\n",
// scenarioType: "encode",
// },
// {
// description: "Encode xml: array",
// input: "pets:\n cat:\n - purrs\n - meows",
// expected: "<pets>\n <cat>purrs</cat>\n <cat>meows</cat>\n</pets>\n",
// scenarioType: "encode",
// },
// {
// description: "Encode xml: attributes",
// subdescription: "Fields with the matching xml-attribute-prefix are assumed to be attributes.",
// input: "cat:\n +name: tiger\n meows: true\n",
// expected: "<cat name=\"tiger\">\n <meows>true</meows>\n</cat>\n",
// scenarioType: "encode",
// },
// {
// skipDoc: true,
// input: "cat:\n ++name: tiger\n meows: true\n",
// expected: "<cat +name=\"tiger\">\n <meows>true</meows>\n</cat>\n",
// scenarioType: "encode",
// },
// {
// description: "Encode xml: attributes with content",
// subdescription: "Fields with the matching xml-content-name is assumed to be content.",
// input: "cat:\n +name: tiger\n +content: cool\n",
// expected: "<cat name=\"tiger\">cool</cat>\n",
// scenarioType: "encode",
// },
// {
// description: "Encode xml: comments",
// subdescription: "A best attempt is made to copy comments to xml.",
// input: yamlWithComments,
// expected: expectedXmlWithComments,
// scenarioType: "encode",
// },
{
description: "Parse xml: with comments array",
skipDoc: true,
input: inputXmlWithCommentsWithArray,
expected: expectedDecodeYamlWithArray,
scenarioType: "decode",
},
{
description: "Encode xml: simple",
input: "cat: purrs",
expected: "<cat>purrs</cat>\n",
scenarioType: "encode",
},
{
description: "Encode xml: array",
input: "pets:\n cat:\n - purrs\n - meows",
expected: "<pets>\n <cat>purrs</cat>\n <cat>meows</cat>\n</pets>\n",
scenarioType: "encode",
},
{
description: "Encode xml: attributes",
subdescription: "Fields with the matching xml-attribute-prefix are assumed to be attributes.",
input: "cat:\n +name: tiger\n meows: true\n",
expected: "<cat name=\"tiger\">\n <meows>true</meows>\n</cat>\n",
scenarioType: "encode",
},
{
skipDoc: true,
input: "cat:\n ++name: tiger\n meows: true\n",
expected: "<cat +name=\"tiger\">\n <meows>true</meows>\n</cat>\n",
scenarioType: "encode",
},
{
description: "Encode xml: attributes with content",
subdescription: "Fields with the matching xml-content-name is assumed to be content.",
input: "cat:\n +name: tiger\n +content: cool\n",
expected: "<cat name=\"tiger\">cool</cat>\n",
scenarioType: "encode",
},
{
description: "Encode xml: comments",
subdescription: "A best attempt is made to copy comments to xml.",
input: yamlWithComments,
expected: expectedXmlWithComments,
scenarioType: "encode",
},
{
description: "Round trip: with comments",
subdescription: "A best effort is made, but comment positions and white space are not preserved perfectly.",

12
test/utils.go
View File

@ -1,6 +1,7 @@
package test
import (
"bufio"
"bytes"
"fmt"
"os"
@ -8,6 +9,8 @@ import (
"strings"
"testing"
"github.com/pkg/diff"
"github.com/pkg/diff/write"
"github.com/spf13/cobra"
yaml "gopkg.in/yaml.v3"
)
@ -63,9 +66,14 @@ func AssertResultComplexWithContext(t *testing.T, expectedValue interface{}, act
func AssertResultWithContext(t *testing.T, expectedValue interface{}, actualValue interface{}, context interface{}) {
t.Helper()
opts := []write.Option{write.TerminalColor()}
if expectedValue != actualValue {
t.Error(context)
t.Error(": expected <\n", strings.ReplaceAll(fmt.Sprintf("%v", expectedValue), " ", "@"), ">\n but got <\n", strings.ReplaceAll(fmt.Sprintf("%v", actualValue), " ", "@"), ">\n")
// t.Error(": expected <\n", expectedValue, ">\n but got <\n", actualValue, ">\n")
var differenceBuffer bytes.Buffer
if err := diff.Text("expected", "actual", expectedValue, actualValue, bufio.NewWriter(&differenceBuffer), opts...); err != nil {
t.Error(err)
} else {
t.Error(differenceBuffer.String())
}
}
}