xml wip

cmd/utils.go

@@ -52,7 +52,7 @@ func configureDecoder() (yqlib.Decoder, error) {
 	}
 	switch yqlibInputFormat {
 	case yqlib.XmlInputFormat:
-		return yqlib.NewXmlDecoder(), nil
+		return yqlib.NewXmlDecoder(xmlAttributePrefix, xmlContentName), nil
 	}
 	return yqlib.NewYamlDecoder(), nil
 }
@@ -87,7 +87,7 @@ func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder {
 	case yqlib.YamlOutputFormat:
 		return yqlib.NewYamlEncoder(indent, colorsEnabled, !noDocSeparators, unwrapScalar)
 	case yqlib.XmlOutputFormat:
-		return yqlib.NewXmlEncoder(indent)
+		return yqlib.NewXmlEncoder(indent, xmlAttributePrefix, xmlContentName)
 	}
 	panic("invalid encoder")
 }

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/goccy/go-yaml v1.9.5
 	github.com/jinzhu/copier v0.3.4
 	github.com/magiconair/properties v1.8.5
+	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
 	github.com/spf13/cobra v1.3.0
 	github.com/timtadh/lexmachine v0.2.2
 	golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d

go.sum

@@ -292,6 +292,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=

pkg/yqlib/decoder_xml.go

@@ -34,15 +34,15 @@ func InputFormatFromString(format string) (InputFormat, error) {
 type xmlDecoder struct {
 	reader          io.Reader
 	attributePrefix string
-	contentPrefix   string
+	contentName     string
 	finished        bool
 }
 
-func NewXmlDecoder(attributePrefix string, contentPrefix string) Decoder {
+func NewXmlDecoder(attributePrefix string, contentName string) Decoder {
-	if contentPrefix == "" {
+	if contentName == "" {
-		contentPrefix = "content"
+		contentName = "content"
 	}
-	return &xmlDecoder{attributePrefix: attributePrefix, contentPrefix: contentPrefix, finished: false}
+	return &xmlDecoder{attributePrefix: attributePrefix, contentName: contentName, finished: false}
 }
 
 func (dec *xmlDecoder) Init(reader io.Reader) {
@@ -75,7 +75,7 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
 	yamlNode := &yaml.Node{Kind: yaml.MappingNode}
 
 	if len(n.Data) > 0 {
-		label := dec.contentPrefix
+		label := dec.contentName
 		labelNode := createScalarNode(label, label)
 		labelNode.HeadComment = dec.processComment(n.HeadComment)
 		labelNode.FootComment = dec.processComment(n.FootComment)
@@ -94,9 +94,9 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
 
 		}
 
-		if i == len(n.Children)-1 {
+		// if i == len(n.Children)-1 {
-			labelNode.FootComment = dec.processComment(n.FootComment)
+		labelNode.FootComment = dec.processComment(keyValuePair.FootComment)
-		}
+		// }
 
 		log.Debug("len of children in %v is %v", label, len(children))
 		if len(children) > 1 {
@@ -167,8 +167,9 @@ type xmlNode struct {
 }
 
 type xmlChildrenKv struct {
-	K string
+	K           string
-	V []*xmlNode
+	V           []*xmlNode
+	FootComment string
 }
 
 // AddChild appends a node to the list of children
@@ -255,8 +256,8 @@ func (dec *xmlDecoder) decodeXml(root *xmlNode) error {
 
 			commentStr := string(xml.CharData(se))
 			if elem.state == "started" {
-				log.Debug("got a foot comment for %v: [%v]", elem.label, commentStr)
+				applyFootComment(elem, commentStr)
-				elem.n.FootComment = joinFilter([]string{elem.n.FootComment, commentStr})
+
 			} else if elem.state == "chardata" {
 				log.Debug("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
 				elem.n.LineComment = joinFilter([]string{elem.n.LineComment, commentStr})
@@ -271,6 +272,20 @@ func (dec *xmlDecoder) decodeXml(root *xmlNode) error {
 	return nil
 }
 
+func applyFootComment(elem *element, commentStr string) {
+
+	// first lets try to put the comment on the last child
+	if len(elem.n.Children) > 0 {
+		lastChildIndex := len(elem.n.Children) - 1
+		childKv := elem.n.Children[lastChildIndex]
+		log.Debug("got a foot comment for %v: [%v]", childKv.K, commentStr)
+		childKv.FootComment = joinFilter([]string{elem.n.FootComment, commentStr})
+	} else {
+		log.Debug("got a foot comment for %v: [%v]", elem.label, commentStr)
+		elem.n.FootComment = joinFilter([]string{elem.n.FootComment, commentStr})
+	}
+}
+
 func joinFilter(rawStrings []string) string {
 	stringsToJoin := make([]string, 0)
 	for _, str := range rawStrings {

pkg/yqlib/doc/usage/xml.md

@@ -22,6 +22,229 @@ XML nodes that have attributes then plain content, e.g:
 
 The content of the node will be set as a field in the map with the key "+content". Use the `--xml-content-name` flag to change this.
 
+## Parse xml: simple
+Given a sample.xml file of:
+```xml
+<?xml version="1.0" encoding="UTF-8"?>
+<cat>meow</cat>
+```
+then
+```bash
+yq e -p=xml '.' sample.xml
+```
+will output
+```yaml
+cat: meow
+```
+
+## Parse xml: array
+Consecutive nodes with identical xml names are assumed to be arrays.
+
+Given a sample.xml file of:
+```xml
+<?xml version="1.0" encoding="UTF-8"?>
+<animal>1</animal>
+<animal>2</animal>
+```
+then
+```bash
+yq e -p=xml '.' sample.xml
+```
+will output
+```yaml
+animal:
+  - "1"
+  - "2"
+```
+
+## Parse xml: attributes
+Attributes are converted to fields, with the attribute prefix.
+
+Given a sample.xml file of:
+```xml
+<?xml version="1.0" encoding="UTF-8"?>
+<cat legs="4">
+  <legs>7</legs>
+</cat>
+```
+then
+```bash
+yq e -p=xml '.' sample.xml
+```
+will output
+```yaml
+cat:
+  +legs: "4"
+  legs: "7"
+```
+
+## Parse xml: attributes with content
+Content is added as a field, using the content name
+
+Given a sample.xml file of:
+```xml
+<?xml version="1.0" encoding="UTF-8"?>
+<cat legs="4">meow</cat>
+```
+then
+```bash
+yq e -p=xml '.' sample.xml
+```
+will output
+```yaml
+cat:
+  +content: meow
+  +legs: "4"
+```
+
+## Parse xml: with comments
+A best attempt is made to preserve comments.
+
+Given a sample.xml file of:
+```xml
+
+<!-- before cat -->
+<cat>
+	<!-- in cat before -->
+	<x>3<!-- multi
+line comment 
+for x --></x>
+	<!-- before y -->
+	<y>
+		<!-- in y before -->
+		<d><!-- in d before -->z<!-- in d after --></d>
+		
+		<!-- in y after -->
+	</y>
+	<!-- in_cat_after -->
+</cat>
+<!-- after cat -->
+
+```
+then
+```bash
+yq e -p=xml '.' sample.xml
+```
+will output
+```yaml
+# before cat
+cat:
+  # in cat before
+  x: "3" # multi
+  # line comment 
+  # for x
+  # before y
+
+  y:
+    # in y before
+    # in d before
+    d: z # in d after
+    # in y after
+  # in_cat_after
+# after cat
+```
+
+## Encode xml: simple
+Given a sample.yml file of:
+```yaml
+cat: purrs
+```
+then
+```bash
+yq e -o=xml '.' sample.yml
+```
+will output
+```xml
+<cat>purrs</cat>
+```
+
+## Encode xml: array
+Given a sample.yml file of:
+```yaml
+pets:
+  cat:
+    - purrs
+    - meows
+```
+then
+```bash
+yq e -o=xml '.' sample.yml
+```
+will output
+```xml
+<pets>
+  <cat>purrs</cat>
+  <cat>meows</cat>
+</pets>
+```
+
+## Encode xml: attributes
+Fields with the matching xml-attribute-prefix are assumed to be attributes.
+
+Given a sample.yml file of:
+```yaml
+cat:
+  +name: tiger
+  meows: true
+
+```
+then
+```bash
+yq e -o=xml '.' sample.yml
+```
+will output
+```xml
+<cat name="tiger">
+  <meows>true</meows>
+</cat>
+```
+
+## Encode xml: attributes with content
+Fields with the matching xml-content-name is assumed to be content.
+
+Given a sample.yml file of:
+```yaml
+cat:
+  +name: tiger
+  +content: cool
+
+```
+then
+```bash
+yq e -o=xml '.' sample.yml
+```
+will output
+```xml
+<cat name="tiger">cool</cat>
+```
+
+## Encode xml: comments
+A best attempt is made to copy comments to xml.
+
+Given a sample.yml file of:
+```yaml
+# above_cat
+cat: # inline_cat
+  # above_array
+  array: # inline_array
+    - val1 # inline_val1
+    # above_val2
+    - val2 # inline_val2
+# below_cat
+
+```
+then
+```bash
+yq e -o=xml '.' sample.yml
+```
+will output
+```xml
+<!-- above_cat inline_cat --><cat><!-- above_array inline_array -->
+  <array>val1<!-- inline_val1 --></array>
+  <array><!-- above_val2 -->val2<!-- inline_val2 --></array>
+</cat><!-- below_cat -->
+```
+
 ## Round trip: with comments
 A best effort is made, but comment positions and white space are not preserved perfectly.
 
@@ -58,12 +281,14 @@ cat:
   x: "3" # multi
   # line comment 
   # for x
+  # before y
+
   y:
     # in y before
     # in d before
     d: z # in d after
     # in y after
-  # before y   in_cat_after
+  # in_cat_after
 # after cat
 ```
 

pkg/yqlib/encoder_xml.go

@@ -168,16 +168,18 @@ func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) erro
 }
 
 func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
-	e.encodeComment(encoder, headAndLineComment(node))
+
+	if err := e.encodeComment(encoder, headAndLineComment(node)); err != nil {
+		return err
+	}
+
 	for i := 0; i < len(node.Content); i++ {
 		value := node.Content[i]
-		err := e.doEncode(encoder, value, start.Copy())
+		if err := e.doEncode(encoder, value, start.Copy()); err != nil {
-		if err != nil {
 			return err
 		}
 	}
-	e.encodeComment(encoder, footComment(node))
+	return e.encodeComment(encoder, footComment(node))
-	return nil
 }
 
 func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {

pkg/yqlib/xml_test.go

@@ -105,13 +105,14 @@ cat:
 
     y:
         # in y before
-        # in d before
         d:
+            # in d before
             z:
-                sweet: cool
+                +sweet: cool
-								# in d after
+            # in d after
-        				# in y after
+        # in y after
-								# after cat
+    # in_cat_after
+# after cat
 `
 
 var inputXmlWithCommentsWithArray = `
@@ -124,7 +125,8 @@ for x --></x>
 	<!-- before y -->
 	<y>
 		<!-- in y before -->
-		<d><!-- in d before --><z/><!-- in d after --></d>
+		<d><!-- in d before --><z sweet="cool"/><!-- in d after --></d>
+        <d><!-- in d2 before --><z sweet="cool2"/><!-- in d2 after --></d>
 		
 		<!-- in y after -->
 	</y>
@@ -133,6 +135,30 @@ for x --></x>
 <!-- after cat -->
 `
 
+var expectedDecodeYamlWithArray = `D0, P[], (doc)::# before cat
+cat:
+    # in cat before
+    x: "3" # multi
+    # line comment 
+    # for x
+    # before y
+
+    y:
+        # in y before
+        d:
+            - # in d before
+              z:
+                +sweet: cool
+              # in d after
+            - # in d2 before
+              z:
+                +sweet: cool2
+              # in d2 after
+        # in y after
+    # in_cat_after
+# after cat
+`
+
 var expectedDecodeYamlWithComments = `D0, P[], (doc)::# before cat
 cat:
     # in cat before
@@ -146,7 +172,7 @@ cat:
         # in d before
         d: z # in d after
         # in y after
-
+    # in_cat_after
 # after cat
 `
 
@@ -178,36 +204,36 @@ var expectedXmlWithComments = `<!-- above_cat inline_cat --><cat><!-- above_arra
 `
 
 var xmlScenarios = []xmlScenario{
-	// {
+	{
-	// 	description: "Parse xml: simple",
+		description: "Parse xml: simple",
-	// 	input:       "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat>meow</cat>",
+		input:       "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat>meow</cat>",
-	// 	expected:    "D0, P[], (doc)::cat: meow\n",
+		expected:    "D0, P[], (doc)::cat: meow\n",
-	// },
+	},
-	// {
+	{
-	// 	description:    "Parse xml: array",
+		description:    "Parse xml: array",
-	// 	subdescription: "Consecutive nodes with identical xml names are assumed to be arrays.",
+		subdescription: "Consecutive nodes with identical xml names are assumed to be arrays.",
-	// 	input:          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<animal>1</animal>\n<animal>2</animal>",
+		input:          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<animal>1</animal>\n<animal>2</animal>",
-	// 	expected:       "D0, P[], (doc)::animal:\n    - \"1\"\n    - \"2\"\n",
+		expected:       "D0, P[], (doc)::animal:\n    - \"1\"\n    - \"2\"\n",
-	// },
+	},
-	// {
+	{
-	// 	description:    "Parse xml: attributes",
+		description:    "Parse xml: attributes",
-	// 	subdescription: "Attributes are converted to fields, with the attribute prefix.",
+		subdescription: "Attributes are converted to fields, with the attribute prefix.",
-	// 	input:          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">\n  <legs>7</legs>\n</cat>",
+		input:          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">\n  <legs>7</legs>\n</cat>",
-	// 	expected:       "D0, P[], (doc)::cat:\n    +legs: \"4\"\n    legs: \"7\"\n",
+		expected:       "D0, P[], (doc)::cat:\n    +legs: \"4\"\n    legs: \"7\"\n",
-	// },
+	},
-	// {
+	{
-	// 	description:    "Parse xml: attributes with content",
+		description:    "Parse xml: attributes with content",
-	// 	subdescription: "Content is added as a field, using the content name",
+		subdescription: "Content is added as a field, using the content name",
-	// 	input:          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">meow</cat>",
+		input:          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">meow</cat>",
-	// 	expected:       "D0, P[], (doc)::cat:\n    +content: meow\n    +legs: \"4\"\n",
+		expected:       "D0, P[], (doc)::cat:\n    +content: meow\n    +legs: \"4\"\n",
-	// },
+	},
-	// {
+	{
-	// 	description:    "Parse xml: with comments",
+		description:    "Parse xml: with comments",
-	// 	subdescription: "A best attempt is made to preserve comments.",
+		subdescription: "A best attempt is made to preserve comments.",
-	// 	input:          inputXmlWithComments,
+		input:          inputXmlWithComments,
-	// 	expected:       expectedDecodeYamlWithComments,
+		expected:       expectedDecodeYamlWithComments,
-	// 	scenarioType:   "decode",
+		scenarioType:   "decode",
-	// },
+	},
 	{
 		description:  "Parse xml: with comments subchild",
 		skipDoc:      true,
@@ -215,45 +241,52 @@ var xmlScenarios = []xmlScenario{
 		expected:     expectedDecodeYamlWithSubChild,
 		scenarioType: "decode",
 	},
-	// {
+	{
-	// 	description:  "Encode xml: simple",
+		description:  "Parse xml: with comments array",
-	// 	input:        "cat: purrs",
+		skipDoc:      true,
-	// 	expected:     "<cat>purrs</cat>\n",
+		input:        inputXmlWithCommentsWithArray,
-	// 	scenarioType: "encode",
+		expected:     expectedDecodeYamlWithArray,
-	// },
+		scenarioType: "decode",
-	// {
+	},
-	// 	description:  "Encode xml: array",
+	{
-	// 	input:        "pets:\n  cat:\n    - purrs\n    - meows",
+		description:  "Encode xml: simple",
-	// 	expected:     "<pets>\n  <cat>purrs</cat>\n  <cat>meows</cat>\n</pets>\n",
+		input:        "cat: purrs",
-	// 	scenarioType: "encode",
+		expected:     "<cat>purrs</cat>\n",
-	// },
+		scenarioType: "encode",
-	// {
+	},
-	// 	description:    "Encode xml: attributes",
+	{
-	// 	subdescription: "Fields with the matching xml-attribute-prefix are assumed to be attributes.",
+		description:  "Encode xml: array",
-	// 	input:          "cat:\n  +name: tiger\n  meows: true\n",
+		input:        "pets:\n  cat:\n    - purrs\n    - meows",
-	// 	expected:       "<cat name=\"tiger\">\n  <meows>true</meows>\n</cat>\n",
+		expected:     "<pets>\n  <cat>purrs</cat>\n  <cat>meows</cat>\n</pets>\n",
-	// 	scenarioType:   "encode",
+		scenarioType: "encode",
-	// },
+	},
-	// {
+	{
-	// 	skipDoc:      true,
+		description:    "Encode xml: attributes",
-	// 	input:        "cat:\n  ++name: tiger\n  meows: true\n",
+		subdescription: "Fields with the matching xml-attribute-prefix are assumed to be attributes.",
-	// 	expected:     "<cat +name=\"tiger\">\n  <meows>true</meows>\n</cat>\n",
+		input:          "cat:\n  +name: tiger\n  meows: true\n",
-	// 	scenarioType: "encode",
+		expected:       "<cat name=\"tiger\">\n  <meows>true</meows>\n</cat>\n",
-	// },
+		scenarioType:   "encode",
-	// {
+	},
-	// 	description:    "Encode xml: attributes with content",
+	{
-	// 	subdescription: "Fields with the matching xml-content-name is assumed to be content.",
+		skipDoc:      true,
-	// 	input:          "cat:\n  +name: tiger\n  +content: cool\n",
+		input:        "cat:\n  ++name: tiger\n  meows: true\n",
-	// 	expected:       "<cat name=\"tiger\">cool</cat>\n",
+		expected:     "<cat +name=\"tiger\">\n  <meows>true</meows>\n</cat>\n",
-	// 	scenarioType:   "encode",
+		scenarioType: "encode",
-	// },
+	},
-	// {
+	{
-	// 	description:    "Encode xml: comments",
+		description:    "Encode xml: attributes with content",
-	// 	subdescription: "A best attempt is made to copy comments to xml.",
+		subdescription: "Fields with the matching xml-content-name is assumed to be content.",
-	// 	input:          yamlWithComments,
+		input:          "cat:\n  +name: tiger\n  +content: cool\n",
-	// 	expected:       expectedXmlWithComments,
+		expected:       "<cat name=\"tiger\">cool</cat>\n",
-	// 	scenarioType:   "encode",
+		scenarioType:   "encode",
-	// },
+	},
+	{
+		description:    "Encode xml: comments",
+		subdescription: "A best attempt is made to copy comments to xml.",
+		input:          yamlWithComments,
+		expected:       expectedXmlWithComments,
+		scenarioType:   "encode",
+	},
 	{
 		description:    "Round trip: with comments",
 		subdescription: "A best effort is made, but comment positions and white space are not preserved perfectly.",

test/utils.go

@@ -1,6 +1,7 @@
 package test
 
 import (
+	"bufio"
 	"bytes"
 	"fmt"
 	"os"
@@ -8,6 +9,8 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/pkg/diff"
+	"github.com/pkg/diff/write"
 	"github.com/spf13/cobra"
 	yaml "gopkg.in/yaml.v3"
 )
@@ -63,9 +66,14 @@ func AssertResultComplexWithContext(t *testing.T, expectedValue interface{}, act
 
 func AssertResultWithContext(t *testing.T, expectedValue interface{}, actualValue interface{}, context interface{}) {
 	t.Helper()
+	opts := []write.Option{write.TerminalColor()}
 	if expectedValue != actualValue {
 		t.Error(context)
-		t.Error(": expected <\n", strings.ReplaceAll(fmt.Sprintf("%v", expectedValue), " ", "@"), ">\n but got <\n", strings.ReplaceAll(fmt.Sprintf("%v", actualValue), " ", "@"), ">\n")
+		var differenceBuffer bytes.Buffer
-		// t.Error(": expected <\n", expectedValue, ">\n but got <\n", actualValue, ">\n")
+		if err := diff.Text("expected", "actual", expectedValue, actualValue, bufio.NewWriter(&differenceBuffer), opts...); err != nil {
+			t.Error(err)
+		} else {
+			t.Error(differenceBuffer.String())
+		}
 	}
 }