yq/pkg/yqlib/xml_test.go

package yqlib

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"strings"
	"testing"

	"github.com/mikefarah/yq/v4/test"
	yaml "gopkg.in/yaml.v3"
)

func decodeXML(t *testing.T, s formatScenario) *CandidateNode {
	decoder := NewXMLDecoder("+", "+content")

	decoder.Init(strings.NewReader(s.input))

	node := &yaml.Node{}
	err := decoder.Decode(node)
	if err != nil && !errors.Is(err, io.EOF) {
		t.Error(err, "fail to decode", s.input)
	}

	expression := s.expression
	if expression == "" {
		expression = "."
	}

	exp, err := getExpressionParser().ParseExpression(expression)

	if err != nil {
		t.Error(err)
		return nil
	}

	candidateNode := CandidateNode{Node: node}

	context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: candidateNode.AsList()}, exp)

	if err != nil {
		t.Error(err)
		return nil
	}

	return context.MatchingNodes.Front().Value.(*CandidateNode)
}

func processXMLScenario(s formatScenario) string {
	var output bytes.Buffer
	writer := bufio.NewWriter(&output)

	var encoder = NewXMLEncoder(2, "+", "+content")

	var decoder = NewYamlDecoder()
	if s.scenarioType == "roundtrip" {
		decoder = NewXMLDecoder("+", "+content")
	}

	inputs, err := readDocuments(strings.NewReader(s.input), "sample.yml", 0, decoder)
	if err != nil {
		panic(err)
	}
	node := inputs.Front().Value.(*CandidateNode).Node
	err = encoder.Encode(writer, node)
	if err != nil {
		panic(err)
	}
	writer.Flush()

	return output.String()
}

type formatScenario struct {
	input          string
	indent         int
	expression     string
	expected       string
	description    string
	subdescription string
	skipDoc        bool
	scenarioType   string
}

var inputXMLWithComments = `
<!-- before cat -->
<cat>
	<!-- in cat before -->
	<x>3<!-- multi
line comment 
for x --></x>
	<!-- before y -->
	<y>
		<!-- in y before -->
		<d><!-- in d before -->z<!-- in d after --></d>
		
		<!-- in y after -->
	</y>
	<!-- in_cat_after -->
</cat>
<!-- after cat -->
`
var inputXMLWithCommentsWithSubChild = `
<!-- before cat -->
<cat>
	<!-- in cat before -->
	<x>3<!-- multi
line comment 
for x --></x>
	<!-- before y -->
	<y>
		<!-- in y before -->
		<d><!-- in d before --><z sweet="cool"/><!-- in d after --></d>
		
		<!-- in y after -->
	</y>
	<!-- in_cat_after -->
</cat>
<!-- after cat -->
`

var expectedDecodeYamlWithSubChild = `D0, P[], (doc)::# before cat
cat:
    # in cat before
    x: "3" # multi
    # line comment 
    # for x
    # before y

    y:
        # in y before
        d:
            # in d before
            z:
                +sweet: cool
            # in d after
        # in y after
    # in_cat_after
# after cat
`

var inputXMLWithCommentsWithArray = `
<!-- before cat -->
<cat>
	<!-- in cat before -->
	<x>3<!-- multi
line comment 
for x --></x>
	<!-- before y -->
	<y>
		<!-- in y before -->
		<d><!-- in d before --><z sweet="cool"/><!-- in d after --></d>
        <d><!-- in d2 before --><z sweet="cool2"/><!-- in d2 after --></d>
		
		<!-- in y after -->
	</y>
	<!-- in_cat_after -->
</cat>
<!-- after cat -->
`

var expectedDecodeYamlWithArray = `D0, P[], (doc)::# before cat
cat:
    # in cat before
    x: "3" # multi
    # line comment 
    # for x
    # before y

    y:
        # in y before
        d:
            - # in d before
              z:
                +sweet: cool
              # in d after
            - # in d2 before
              z:
                +sweet: cool2
              # in d2 after
        # in y after
    # in_cat_after
# after cat
`

var expectedDecodeYamlWithComments = `D0, P[], (doc)::# before cat
cat:
    # in cat before
    x: "3" # multi
    # line comment 
    # for x
    # before y

    y:
        # in y before
        # in d before
        d: z # in d after
        # in y after
    # in_cat_after
# after cat
`

var expectedRoundtripXMLWithComments = `<!-- before cat --><cat><!-- in cat before -->
  <x>3<!-- multi
line comment 
for x --></x><!-- before y -->
  <y><!-- in y before
in d before -->
    <d>z<!-- in d after --></d><!-- in y after -->
  </y><!-- in_cat_after -->
</cat><!-- after cat -->
`

var yamlWithComments = `# above_cat
cat: # inline_cat
  # above_array
  array: # inline_array
    - val1 # inline_val1
    # above_val2
    - val2 # inline_val2
# below_cat
`

var expectedXMLWithComments = `<!-- above_cat inline_cat --><cat><!-- above_array inline_array -->
  <array>val1<!-- inline_val1 --></array>
  <array><!-- above_val2 -->val2<!-- inline_val2 --></array>
</cat><!-- below_cat -->
`

var xmlScenarios = []formatScenario{
	{
		description:    "Parse xml: simple",
		subdescription: "Notice how all the values are strings, see the next example on how you can fix that.",
		input:          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat>\n  <says>meow</says>\n  <legs>4</legs>\n  <cute>true</cute>\n</cat>",
		expected:       "D0, P[], (doc)::cat:\n    says: meow\n    legs: \"4\"\n    cute: \"true\"\n",
	},
	{
		description:    "Parse xml: number",
		subdescription: "All values are assumed to be strings when parsing XML, but you can use the `from_yaml` operator on all the strings values to autoparse into the correct type.",
		input:          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat>\n  <says>meow</says>\n  <legs>4</legs>\n  <cute>true</cute>\n</cat>",
		expression:     " (.. | select(tag == \"!!str\")) |= from_yaml",
		expected:       "D0, P[], ()::cat:\n    says: meow\n    legs: 4\n    cute: true\n",
	},
	{
		description:    "Parse xml: array",
		subdescription: "Consecutive nodes with identical xml names are assumed to be arrays.",
		input:          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<animal>cat</animal>\n<animal>goat</animal>",
		expected:       "D0, P[], (doc)::animal:\n    - cat\n    - goat\n",
	},
	{
		description:    "Parse xml: attributes",
		subdescription: "Attributes are converted to fields, with the default attribute prefix '+'. Use '--xml-attribute-prefix` to set your own.",
		input:          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">\n  <legs>7</legs>\n</cat>",
		expected:       "D0, P[], (doc)::cat:\n    +legs: \"4\"\n    legs: \"7\"\n",
	},
	{
		description:    "Parse xml: attributes with content",
		subdescription: "Content is added as a field, using the default content name of `+content`. Use `--xml-content-name` to set your own.",
		input:          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">meow</cat>",
		expected:       "D0, P[], (doc)::cat:\n    +content: meow\n    +legs: \"4\"\n",
	},
	{
		description:    "Parse xml: with comments",
		subdescription: "A best attempt is made to preserve comments.",
		input:          inputXMLWithComments,
		expected:       expectedDecodeYamlWithComments,
		scenarioType:   "decode",
	},
	{
		description:  "Empty doc",
		skipDoc:      true,
		input:        "",
		expected:     "D0, P[], ()::null\n",
		scenarioType: "decode",
	},
	{
		description:  "Empty single node",
		skipDoc:      true,
		input:        "<a/>",
		expected:     "D0, P[], (doc)::a:\n",
		scenarioType: "decode",
	},
	{
		description:  "Empty close node",
		skipDoc:      true,
		input:        "<a></a>",
		expected:     "D0, P[], (doc)::a:\n",
		scenarioType: "decode",
	},
	{
		description:  "Nested empty",
		skipDoc:      true,
		input:        "<a><b/></a>",
		expected:     "D0, P[], (doc)::a:\n    b:\n",
		scenarioType: "decode",
	},
	{
		description:  "Parse xml: with comments subchild",
		skipDoc:      true,
		input:        inputXMLWithCommentsWithSubChild,
		expected:     expectedDecodeYamlWithSubChild,
		scenarioType: "decode",
	},
	{
		description:  "Parse xml: with comments array",
		skipDoc:      true,
		input:        inputXMLWithCommentsWithArray,
		expected:     expectedDecodeYamlWithArray,
		scenarioType: "decode",
	},
	{
		description:  "Encode xml: simple",
		input:        "cat: purrs",
		expected:     "<cat>purrs</cat>\n",
		scenarioType: "encode",
	},
	{
		description:  "Encode xml: array",
		input:        "pets:\n  cat:\n    - purrs\n    - meows",
		expected:     "<pets>\n  <cat>purrs</cat>\n  <cat>meows</cat>\n</pets>\n",
		scenarioType: "encode",
	},
	{
		description:    "Encode xml: attributes",
		subdescription: "Fields with the matching xml-attribute-prefix are assumed to be attributes.",
		input:          "cat:\n  +name: tiger\n  meows: true\n",
		expected:       "<cat name=\"tiger\">\n  <meows>true</meows>\n</cat>\n",
		scenarioType:   "encode",
	},
	{
		skipDoc:      true,
		input:        "cat:\n  ++name: tiger\n  meows: true\n",
		expected:     "<cat +name=\"tiger\">\n  <meows>true</meows>\n</cat>\n",
		scenarioType: "encode",
	},
	{
		description:    "Encode xml: attributes with content",
		subdescription: "Fields with the matching xml-content-name is assumed to be content.",
		input:          "cat:\n  +name: tiger\n  +content: cool\n",
		expected:       "<cat name=\"tiger\">cool</cat>\n",
		scenarioType:   "encode",
	},
	{
		description:    "Encode xml: comments",
		subdescription: "A best attempt is made to copy comments to xml.",
		input:          yamlWithComments,
		expected:       expectedXMLWithComments,
		scenarioType:   "encode",
	},
	{
		description:    "Round trip: with comments",
		subdescription: "A best effort is made, but comment positions and white space are not preserved perfectly.",
		input:          inputXMLWithComments,
		expected:       expectedRoundtripXMLWithComments,
		scenarioType:   "roundtrip",
	},
}

func testXMLScenario(t *testing.T, s formatScenario) {
	if s.scenarioType == "encode" || s.scenarioType == "roundtrip" {
		test.AssertResultWithContext(t, s.expected, processXMLScenario(s), s.description)
	} else {
		var actual = resultToString(t, decodeXML(t, s))
		test.AssertResultWithContext(t, s.expected, actual, s.description)
	}
}

func documentXMLScenario(t *testing.T, w *bufio.Writer, i interface{}) {
	s := i.(formatScenario)

	if s.skipDoc {
		return
	}
	if s.scenarioType == "encode" {
		documentXMLEncodeScenario(w, s)
	} else if s.scenarioType == "roundtrip" {
		documentXMLRoundTripScenario(w, s)
	} else {
		documentXMLDecodeScenario(t, w, s)
	}

}

func documentXMLDecodeScenario(t *testing.T, w *bufio.Writer, s formatScenario) {
	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))

	if s.subdescription != "" {
		writeOrPanic(w, s.subdescription)
		writeOrPanic(w, "\n\n")
	}

	writeOrPanic(w, "Given a sample.xml file of:\n")
	writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input))

	writeOrPanic(w, "then\n")
	expression := s.expression
	if expression == "" {
		expression = "."
	}
	writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=xml '%v' sample.xml\n```\n", expression))
	writeOrPanic(w, "will output\n")

	var output bytes.Buffer
	printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)

	node := decodeXML(t, s)

	err := printer.PrintResults(node.AsList())
	if err != nil {
		t.Error(err)
		return
	}

	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", output.String()))
}

func documentXMLEncodeScenario(w *bufio.Writer, s formatScenario) {
	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))

	if s.subdescription != "" {
		writeOrPanic(w, s.subdescription)
		writeOrPanic(w, "\n\n")
	}

	writeOrPanic(w, "Given a sample.yml file of:\n")
	writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input))

	writeOrPanic(w, "then\n")
	writeOrPanic(w, "```bash\nyq -o=xml '.' sample.yml\n```\n")
	writeOrPanic(w, "will output\n")

	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processXMLScenario(s)))
}

func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) {
	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))

	if s.subdescription != "" {
		writeOrPanic(w, s.subdescription)
		writeOrPanic(w, "\n\n")
	}

	writeOrPanic(w, "Given a sample.xml file of:\n")
	writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input))

	writeOrPanic(w, "then\n")
	writeOrPanic(w, "```bash\nyq -p=xml -o=xml '.' sample.xml\n```\n")
	writeOrPanic(w, "will output\n")

	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processXMLScenario(s)))
}

func TestXMLScenarios(t *testing.T) {
	for _, tt := range xmlScenarios {
		testXMLScenario(t, tt)
	}
	genericScenarios := make([]interface{}, len(xmlScenarios))
	for i, s := range xmlScenarios {
		genericScenarios[i] = s
	}
	documentScenarios(t, "usage", "xml", genericScenarios, documentXMLScenario)
}