mirror of
https://github.com/mikefarah/yq.git
synced 2026-09-05 00:54:51 +08:00
wip
This commit is contained in:
Mike Farah
@@ -0,0 +1,408 @@
|
||||
//go:build !yq_noxml
|
||||
|
||||
package xml
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/mikefarah/yq/v4/pkg/yqlib"
|
||||
"golang.org/x/net/html/charset"
|
||||
)
|
||||
|
||||
type xmlDecoder struct {
|
||||
reader io.Reader
|
||||
readAnything bool
|
||||
finished bool
|
||||
prefs XmlPreferences
|
||||
}
|
||||
|
||||
func NewXMLDecoder(prefs XmlPreferences) yqlib.Decoder {
|
||||
return &xmlDecoder{
|
||||
finished: false,
|
||||
prefs: prefs,
|
||||
}
|
||||
}
|
||||
|
||||
func (dec *xmlDecoder) Init(reader io.Reader) error {
|
||||
dec.reader = reader
|
||||
dec.readAnything = false
|
||||
dec.finished = false
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yqlib..CandidateNode, error) {
|
||||
yamlNode := &yqlib..CandidateNode{Kind: SequenceNode, Tag: "!!seq"}
|
||||
for _, child := range nodes {
|
||||
yamlChild, err := dec.convertToYamlNode(child)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
yamlNode.AddChild(yamlChild)
|
||||
}
|
||||
|
||||
return yamlNode, nil
|
||||
}
|
||||
|
||||
var decoderCommentPrefix = regexp.MustCompile(`(^|\n)([[:alpha:]])`)
|
||||
|
||||
func (dec *xmlDecoder) processComment(c string) string {
|
||||
if c == "" {
|
||||
return ""
|
||||
}
|
||||
//need to replace "cat " with "# cat"
|
||||
// "\ncat\n" with "\n cat\n"
|
||||
// ensure non-empty comments starting with newline have a space in front
|
||||
|
||||
replacement := decoderCommentPrefix.ReplaceAllString(c, "$1 $2")
|
||||
replacement = "#" + strings.ReplaceAll(strings.TrimRight(replacement, " "), "\n", "\n#")
|
||||
return replacement
|
||||
}
|
||||
|
||||
func (dec *xmlDecoder) createMap(n *xmlNode) (*yqlib..CandidateNode, error) {
|
||||
log.Debug("createMap: headC: %v, lineC: %v, footC: %v", n.HeadComment, n.LineComment, n.FootComment)
|
||||
yamlNode := &yqlib..CandidateNode{Kind: MappingNode, Tag: "!!map"}
|
||||
|
||||
if len(n.Data) > 0 {
|
||||
log.Debugf("creating content node for map: %v", dec.prefs.ContentName)
|
||||
label := dec.prefs.ContentName
|
||||
labelNode := createScalarNode(label, label)
|
||||
labelNode.HeadComment = dec.processComment(n.HeadComment)
|
||||
labelNode.LineComment = dec.processComment(n.LineComment)
|
||||
labelNode.FootComment = dec.processComment(n.FootComment)
|
||||
yamlNode.AddKeyValueChild(labelNode, dec.createValueNodeFromData(n.Data))
|
||||
}
|
||||
|
||||
for i, keyValuePair := range n.Children {
|
||||
label := keyValuePair.K
|
||||
children := keyValuePair.V
|
||||
labelNode := createScalarNode(label, label)
|
||||
var valueNode *yqlib..CandidateNode
|
||||
var err error
|
||||
|
||||
if i == 0 {
|
||||
log.Debugf("head comment here")
|
||||
labelNode.HeadComment = dec.processComment(n.HeadComment)
|
||||
|
||||
}
|
||||
log.Debugf("label=%v, i=%v, keyValuePair.FootComment: %v", label, i, keyValuePair.FootComment)
|
||||
labelNode.FootComment = dec.processComment(keyValuePair.FootComment)
|
||||
|
||||
log.Debug("len of children in %v is %v", label, len(children))
|
||||
if len(children) > 1 {
|
||||
valueNode, err = dec.createSequence(children)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
// comment hack for maps of scalars
|
||||
// if the value is a scalar, the head comment of the scalar needs to go on the key?
|
||||
// add tests for <z/> as well as multiple <ds> of inputXmlWithComments > yaml
|
||||
if len(children[0].Children) == 0 && children[0].HeadComment != "" {
|
||||
if len(children[0].Data) > 0 {
|
||||
|
||||
log.Debug("scalar comment hack, currentlabel [%v]", labelNode.HeadComment)
|
||||
labelNode.HeadComment = joinComments([]string{labelNode.HeadComment, strings.TrimSpace(children[0].HeadComment)}, "\n")
|
||||
children[0].HeadComment = ""
|
||||
} else {
|
||||
// child is null, put the headComment as a linecomment for reasons
|
||||
children[0].LineComment = children[0].HeadComment
|
||||
children[0].HeadComment = ""
|
||||
}
|
||||
}
|
||||
valueNode, err = dec.convertToYamlNode(children[0])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
yamlNode.AddKeyValueChild(labelNode, valueNode)
|
||||
}
|
||||
|
||||
return yamlNode, nil
|
||||
}
|
||||
|
||||
func (dec *xmlDecoder) createValueNodeFromData(values []string) *yqlib..CandidateNode {
|
||||
switch len(values) {
|
||||
case 0:
|
||||
return createScalarNode(nil, "")
|
||||
case 1:
|
||||
return createScalarNode(values[0], values[0])
|
||||
default:
|
||||
content := make([]*yqlib..CandidateNode, 0)
|
||||
for _, value := range values {
|
||||
content = append(content, createScalarNode(value, value))
|
||||
}
|
||||
return &yqlib..CandidateNode{
|
||||
Kind: SequenceNode,
|
||||
Tag: "!!seq",
|
||||
Content: content,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yqlib..CandidateNode, error) {
|
||||
if len(n.Children) > 0 {
|
||||
return dec.createMap(n)
|
||||
}
|
||||
|
||||
scalar := dec.createValueNodeFromData(n.Data)
|
||||
|
||||
log.Debug("scalar (%v), headC: %v, lineC: %v, footC: %v", scalar.Tag, n.HeadComment, n.LineComment, n.FootComment)
|
||||
scalar.HeadComment = dec.processComment(n.HeadComment)
|
||||
scalar.LineComment = dec.processComment(n.LineComment)
|
||||
if scalar.Tag == "!!seq" {
|
||||
scalar.Content[0].HeadComment = scalar.LineComment
|
||||
scalar.LineComment = ""
|
||||
}
|
||||
|
||||
scalar.FootComment = dec.processComment(n.FootComment)
|
||||
|
||||
return scalar, nil
|
||||
}
|
||||
|
||||
func (dec *xmlDecoder) Decode() (*yqlib..CandidateNode, error) {
|
||||
if dec.finished {
|
||||
return nil, io.EOF
|
||||
}
|
||||
root := &xmlNode{}
|
||||
// cant use xj - it doesn't keep map order.
|
||||
err := dec.decodeXML(root)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
firstNode, err := dec.convertToYamlNode(root)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
} else if firstNode.Tag == "!!null" {
|
||||
dec.finished = true
|
||||
if dec.readAnything {
|
||||
return nil, io.EOF
|
||||
}
|
||||
}
|
||||
dec.readAnything = true
|
||||
dec.finished = true
|
||||
|
||||
return firstNode, nil
|
||||
}
|
||||
|
||||
type xmlNode struct {
|
||||
Children []*xmlChildrenKv
|
||||
HeadComment string
|
||||
FootComment string
|
||||
LineComment string
|
||||
Data []string
|
||||
}
|
||||
|
||||
type xmlChildrenKv struct {
|
||||
K string
|
||||
V []*xmlNode
|
||||
FootComment string
|
||||
}
|
||||
|
||||
// AddChild appends a node to the list of children
|
||||
func (n *xmlNode) AddChild(s string, c *xmlNode) {
|
||||
|
||||
if n.Children == nil {
|
||||
n.Children = make([]*xmlChildrenKv, 0)
|
||||
}
|
||||
log.Debug("looking for %s", s)
|
||||
// see if we can find an existing entry to add to
|
||||
for _, childEntry := range n.Children {
|
||||
if childEntry.K == s {
|
||||
log.Debug("found it, appending an entry%s", s)
|
||||
childEntry.V = append(childEntry.V, c)
|
||||
log.Debug("yay len of children in %v is %v", s, len(childEntry.V))
|
||||
return
|
||||
}
|
||||
}
|
||||
log.Debug("not there, making a new one %s", s)
|
||||
n.Children = append(n.Children, &xmlChildrenKv{K: s, V: []*xmlNode{c}})
|
||||
}
|
||||
|
||||
type element struct {
|
||||
parent *element
|
||||
n *xmlNode
|
||||
label string
|
||||
state string
|
||||
}
|
||||
|
||||
// this code is heavily based on https://github.com/basgys/goxml2json
|
||||
// main changes are to decode into a structure that preserves the original order
|
||||
// of the map keys.
|
||||
func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
|
||||
xmlDec := xml.NewDecoder(dec.reader)
|
||||
xmlDec.Strict = dec.prefs.StrictMode
|
||||
// That will convert the charset if the provided XML is non-UTF-8
|
||||
xmlDec.CharsetReader = charset.NewReaderLabel
|
||||
|
||||
started := false
|
||||
|
||||
// Create first element from the root node
|
||||
elem := &element{
|
||||
parent: nil,
|
||||
n: root,
|
||||
}
|
||||
|
||||
getToken := func() (xml.Token, error) {
|
||||
if dec.prefs.UseRawToken {
|
||||
return xmlDec.RawToken()
|
||||
}
|
||||
return xmlDec.Token()
|
||||
}
|
||||
|
||||
for {
|
||||
t, e := getToken()
|
||||
if e != nil && !errors.Is(e, io.EOF) {
|
||||
return e
|
||||
}
|
||||
if t == nil {
|
||||
break
|
||||
}
|
||||
|
||||
switch se := t.(type) {
|
||||
case xml.StartElement:
|
||||
log.Debug("start element %v", se.Name.Local)
|
||||
elem.state = "started"
|
||||
// Build new a new current element and link it to its parent
|
||||
elem = &element{
|
||||
parent: elem,
|
||||
n: &xmlNode{},
|
||||
label: se.Name.Local,
|
||||
}
|
||||
|
||||
// Extract attributes as children
|
||||
for _, a := range se.Attr {
|
||||
if dec.prefs.KeepNamespace {
|
||||
if a.Name.Space != "" {
|
||||
a.Name.Local = a.Name.Space + ":" + a.Name.Local
|
||||
}
|
||||
}
|
||||
elem.n.AddChild(dec.prefs.AttributePrefix+a.Name.Local, &xmlNode{Data: []string{a.Value}})
|
||||
}
|
||||
case xml.CharData:
|
||||
|
||||
// Extract XML data (if any)
|
||||
newBit := trimNonGraphic(string(se))
|
||||
if !started && len(newBit) > 0 {
|
||||
return fmt.Errorf("invalid XML: Encountered chardata [%v] outside of XML node", newBit)
|
||||
}
|
||||
|
||||
if len(newBit) > 0 {
|
||||
elem.n.Data = append(elem.n.Data, newBit)
|
||||
elem.state = "chardata"
|
||||
log.Debug("chardata [%v] for %v", elem.n.Data, elem.label)
|
||||
}
|
||||
case xml.EndElement:
|
||||
if elem == nil {
|
||||
log.Debug("no element, probably bad xml")
|
||||
continue
|
||||
}
|
||||
log.Debug("end element %v", elem.label)
|
||||
elem.state = "finished"
|
||||
// And add it to its parent list
|
||||
if elem.parent != nil {
|
||||
elem.parent.n.AddChild(elem.label, elem.n)
|
||||
}
|
||||
|
||||
// Then change the current element to its parent
|
||||
elem = elem.parent
|
||||
case xml.Comment:
|
||||
|
||||
commentStr := string(xml.CharData(se))
|
||||
if elem.state == "started" {
|
||||
applyFootComment(elem, commentStr)
|
||||
|
||||
} else if elem.state == "chardata" {
|
||||
log.Debug("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
|
||||
elem.n.LineComment = joinComments([]string{elem.n.LineComment, commentStr}, " ")
|
||||
} else {
|
||||
log.Debug("got a head comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
|
||||
elem.n.HeadComment = joinComments([]string{elem.n.HeadComment, commentStr}, " ")
|
||||
}
|
||||
|
||||
case xml.ProcInst:
|
||||
if !dec.prefs.SkipProcInst {
|
||||
elem.n.AddChild(dec.prefs.ProcInstPrefix+se.Target, &xmlNode{Data: []string{string(se.Inst)}})
|
||||
}
|
||||
case xml.Directive:
|
||||
if !dec.prefs.SkipDirectives {
|
||||
elem.n.AddChild(dec.prefs.DirectiveName, &xmlNode{Data: []string{string(se)}})
|
||||
}
|
||||
}
|
||||
started = true
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func applyFootComment(elem *element, commentStr string) {
|
||||
|
||||
// first lets try to put the comment on the last child
|
||||
if len(elem.n.Children) > 0 {
|
||||
lastChildIndex := len(elem.n.Children) - 1
|
||||
childKv := elem.n.Children[lastChildIndex]
|
||||
log.Debug("got a foot comment, putting on last child for %v: [%v]", childKv.K, commentStr)
|
||||
// if it's an array of scalars, put the foot comment on the scalar itself
|
||||
if len(childKv.V) > 0 && len(childKv.V[0].Children) == 0 {
|
||||
nodeToUpdate := childKv.V[len(childKv.V)-1]
|
||||
nodeToUpdate.FootComment = joinComments([]string{nodeToUpdate.FootComment, commentStr}, " ")
|
||||
} else {
|
||||
childKv.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, " ")
|
||||
}
|
||||
} else {
|
||||
log.Debug("got a foot comment for %v: [%v]", elem.label, commentStr)
|
||||
elem.n.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, " ")
|
||||
}
|
||||
}
|
||||
|
||||
func joinComments(rawStrings []string, joinStr string) string {
|
||||
stringsToJoin := make([]string, 0)
|
||||
for _, str := range rawStrings {
|
||||
if str != "" {
|
||||
stringsToJoin = append(stringsToJoin, str)
|
||||
}
|
||||
}
|
||||
return strings.Join(stringsToJoin, joinStr)
|
||||
}
|
||||
|
||||
// trimNonGraphic returns a slice of the string s, with all leading and trailing
|
||||
// non graphic characters and spaces removed.
|
||||
//
|
||||
// Graphic characters include letters, marks, numbers, punctuation, symbols,
|
||||
// and spaces, from categories L, M, N, P, S, Zs.
|
||||
// Spacing characters are set by category Z and property Pattern_White_Space.
|
||||
func trimNonGraphic(s string) string {
|
||||
if s == "" {
|
||||
return s
|
||||
}
|
||||
|
||||
var first *int
|
||||
var last int
|
||||
for i, r := range []rune(s) {
|
||||
if !unicode.IsGraphic(r) || unicode.IsSpace(r) {
|
||||
continue
|
||||
}
|
||||
|
||||
if first == nil {
|
||||
f := i // copy i
|
||||
first = &f
|
||||
last = i
|
||||
} else {
|
||||
last = i
|
||||
}
|
||||
}
|
||||
|
||||
// If first is nil, it means there are no graphic characters
|
||||
if first == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
return string([]rune(s)[*first : last+1])
|
||||
}
|
||||
@@ -0,0 +1,337 @@
|
||||
//go:build !yq_noxml
|
||||
|
||||
package xml
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/mikefarah/yq/v4/pkg/yqlib"
|
||||
)
|
||||
|
||||
type xmlEncoder struct {
|
||||
indentString string
|
||||
writer io.Writer
|
||||
prefs XmlPreferences
|
||||
leadingContent string
|
||||
}
|
||||
|
||||
func NewXMLEncoder(prefs XmlPreferences) yqlib.Encoder {
|
||||
var indentString = ""
|
||||
|
||||
for index := 0; index < prefs.Indent; index++ {
|
||||
indentString = indentString + " "
|
||||
}
|
||||
return &xmlEncoder{indentString, nil, prefs, ""}
|
||||
}
|
||||
|
||||
func (e *xmlEncoder) CanHandleAliases() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (e *xmlEncoder) PrintDocumentSeparator(_ io.Writer) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *xmlEncoder) PrintLeadingContent(_ io.Writer, content string) error {
|
||||
e.leadingContent = content
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *xmlEncoder) Encode(writer io.Writer, node *yqlib.CandidateNode) error {
|
||||
encoder := xml.NewEncoder(writer)
|
||||
// hack so we can manually add newlines to procInst and directives
|
||||
e.writer = writer
|
||||
encoder.Indent("", e.indentString)
|
||||
var newLine xml.CharData = []byte("\n")
|
||||
|
||||
if node.Tag == "!!map" {
|
||||
// make sure <?xml .. ?> processing instructions are encoded first
|
||||
for i := 0; i < len(node.Content); i += 2 {
|
||||
key := node.Content[i]
|
||||
value := node.Content[i+1]
|
||||
|
||||
if key.Value == (e.prefs.ProcInstPrefix + "xml") {
|
||||
name := strings.Replace(key.Value, e.prefs.ProcInstPrefix, "", 1)
|
||||
procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)}
|
||||
if err := encoder.EncodeToken(procInst); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := e.writer.Write([]byte("\n")); err != nil {
|
||||
log.Warning("Unable to write newline, skipping: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if e.leadingContent != "" {
|
||||
|
||||
// remove first and last newlines if present
|
||||
err := e.encodeComment(encoder, e.leadingContent)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = encoder.EncodeToken(newLine)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
switch node.Kind {
|
||||
case yqlib.MappingNode:
|
||||
err := e.encodeTopLevelMap(encoder, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
case yqlib.ScalarNode:
|
||||
var charData xml.CharData = []byte(node.Value)
|
||||
err := encoder.EncodeToken(charData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return encoder.Flush()
|
||||
default:
|
||||
return fmt.Errorf("cannot encode %v to XML - only maps can be encoded", node.Tag)
|
||||
}
|
||||
|
||||
return encoder.EncodeToken(newLine)
|
||||
|
||||
}
|
||||
|
||||
func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yqlib.CandidateNode) error {
|
||||
err := e.encodeComment(encoder, headAndLineComment(node))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for i := 0; i < len(node.Content); i += 2 {
|
||||
key := node.Content[i]
|
||||
value := node.Content[i+1]
|
||||
|
||||
start := xml.StartElement{Name: xml.Name{Local: key.Value}}
|
||||
log.Debugf("comments of key %v", key.Value)
|
||||
err := e.encodeComment(encoder, headAndLineComment(key))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if headAndLineComment(key) != "" {
|
||||
var newLine xml.CharData = []byte("\n")
|
||||
err = encoder.EncodeToken(newLine)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if key.Value == (e.prefs.ProcInstPrefix + "xml") { //nolint
|
||||
// dont double process these.
|
||||
} else if strings.HasPrefix(key.Value, e.prefs.ProcInstPrefix) {
|
||||
name := strings.Replace(key.Value, e.prefs.ProcInstPrefix, "", 1)
|
||||
procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)}
|
||||
if err := encoder.EncodeToken(procInst); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := e.writer.Write([]byte("\n")); err != nil {
|
||||
log.Warning("Unable to write newline, skipping: %w", err)
|
||||
}
|
||||
} else if key.Value == e.prefs.DirectiveName {
|
||||
var directive xml.Directive = []byte(value.Value)
|
||||
if err := encoder.EncodeToken(directive); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := e.writer.Write([]byte("\n")); err != nil {
|
||||
log.Warning("Unable to write newline, skipping: %w", err)
|
||||
}
|
||||
} else {
|
||||
|
||||
log.Debugf("recursing")
|
||||
|
||||
err = e.doEncode(encoder, value, start)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
err = e.encodeComment(encoder, footComment(key))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return e.encodeComment(encoder, footComment(node))
|
||||
}
|
||||
|
||||
func (e *xmlEncoder) encodeStart(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
|
||||
err := encoder.EncodeToken(start)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return e.encodeComment(encoder, headComment(node))
|
||||
}
|
||||
|
||||
func (e *xmlEncoder) encodeEnd(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
|
||||
err := encoder.EncodeToken(start.End())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return e.encodeComment(encoder, footComment(node))
|
||||
}
|
||||
|
||||
func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
|
||||
switch node.Kind {
|
||||
case MappingNode:
|
||||
return e.encodeMap(encoder, node, start)
|
||||
case SequenceNode:
|
||||
return e.encodeArray(encoder, node, start)
|
||||
case ScalarNode:
|
||||
err := e.encodeStart(encoder, node, start)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var charData xml.CharData = []byte(node.Value)
|
||||
err = encoder.EncodeToken(charData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = e.encodeComment(encoder, lineComment(node)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return e.encodeEnd(encoder, node, start)
|
||||
}
|
||||
return fmt.Errorf("unsupported type %v", node.Tag)
|
||||
}
|
||||
|
||||
var xmlEncodeMultilineCommentRegex = regexp.MustCompile(`(^|\n) *# ?(.*)`)
|
||||
var xmlEncodeSingleLineCommentRegex = regexp.MustCompile(`^\s*#(.*)\n?`)
|
||||
var chompRegexp = regexp.MustCompile(`\n$`)
|
||||
|
||||
func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) error {
|
||||
if commentStr != "" {
|
||||
log.Debugf("got comment [%v]", commentStr)
|
||||
// multi line string
|
||||
if len(commentStr) > 2 && strings.Contains(commentStr[1:len(commentStr)-1], "\n") {
|
||||
commentStr = chompRegexp.ReplaceAllString(commentStr, "")
|
||||
log.Debugf("chompRegexp [%v]", commentStr)
|
||||
commentStr = xmlEncodeMultilineCommentRegex.ReplaceAllString(commentStr, "$1$2")
|
||||
log.Debugf("processed multiline [%v]", commentStr)
|
||||
// if the first line is non blank, add a space
|
||||
if commentStr[0] != '\n' && commentStr[0] != ' ' {
|
||||
commentStr = " " + commentStr
|
||||
}
|
||||
|
||||
} else {
|
||||
commentStr = xmlEncodeSingleLineCommentRegex.ReplaceAllString(commentStr, "$1")
|
||||
}
|
||||
|
||||
if !strings.HasSuffix(commentStr, " ") && !strings.HasSuffix(commentStr, "\n") {
|
||||
commentStr = commentStr + " "
|
||||
log.Debugf("added suffix [%v]", commentStr)
|
||||
}
|
||||
log.Debugf("encoding comment [%v]", commentStr)
|
||||
|
||||
var comment xml.Comment = []byte(commentStr)
|
||||
err := encoder.EncodeToken(comment)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
|
||||
|
||||
if err := e.encodeComment(encoder, headAndLineComment(node)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for i := 0; i < len(node.Content); i++ {
|
||||
value := node.Content[i]
|
||||
if err := e.doEncode(encoder, value, start.Copy()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return e.encodeComment(encoder, footComment(node))
|
||||
}
|
||||
|
||||
func (e *xmlEncoder) isAttribute(name string) bool {
|
||||
return strings.HasPrefix(name, e.prefs.AttributePrefix) &&
|
||||
name != e.prefs.ContentName &&
|
||||
name != e.prefs.DirectiveName &&
|
||||
!strings.HasPrefix(name, e.prefs.ProcInstPrefix)
|
||||
}
|
||||
|
||||
func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
|
||||
log.Debug("its a map")
|
||||
|
||||
//first find all the attributes and put them on the start token
|
||||
for i := 0; i < len(node.Content); i += 2 {
|
||||
key := node.Content[i]
|
||||
value := node.Content[i+1]
|
||||
|
||||
if e.isAttribute(key.Value) {
|
||||
if value.Kind == ScalarNode {
|
||||
attributeName := strings.Replace(key.Value, e.prefs.AttributePrefix, "", 1)
|
||||
start.Attr = append(start.Attr, xml.Attr{Name: xml.Name{Local: attributeName}, Value: value.Value})
|
||||
} else {
|
||||
return fmt.Errorf("cannot use %v as attribute, only scalars are supported", value.Tag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
err := e.encodeStart(encoder, node, start)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
//now we encode non attribute tokens
|
||||
for i := 0; i < len(node.Content); i += 2 {
|
||||
key := node.Content[i]
|
||||
value := node.Content[i+1]
|
||||
|
||||
err := e.encodeComment(encoder, headAndLineComment(key))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.HasPrefix(key.Value, e.prefs.ProcInstPrefix) {
|
||||
name := strings.Replace(key.Value, e.prefs.ProcInstPrefix, "", 1)
|
||||
procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)}
|
||||
if err := encoder.EncodeToken(procInst); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if key.Value == e.prefs.DirectiveName {
|
||||
var directive xml.Directive = []byte(value.Value)
|
||||
if err := encoder.EncodeToken(directive); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if key.Value == e.prefs.ContentName {
|
||||
// directly encode the contents
|
||||
err = e.encodeComment(encoder, headAndLineComment(value))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var charData xml.CharData = []byte(value.Value)
|
||||
err = encoder.EncodeToken(charData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = e.encodeComment(encoder, footComment(value))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else if !e.isAttribute(key.Value) {
|
||||
start := xml.StartElement{Name: xml.Name{Local: key.Value}}
|
||||
err := e.doEncode(encoder, value, start)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
err = e.encodeComment(encoder, footComment(key))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return e.encodeEnd(encoder, node, start)
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package xml
|
||||
|
||||
import "github.com/mikefarah/yq/v4/pkg/yqlib"
|
||||
|
||||
type XmlPreferences struct {
|
||||
Indent int
|
||||
AttributePrefix string
|
||||
ContentName string
|
||||
StrictMode bool
|
||||
KeepNamespace bool
|
||||
UseRawToken bool
|
||||
ProcInstPrefix string
|
||||
DirectiveName string
|
||||
SkipProcInst bool
|
||||
SkipDirectives bool
|
||||
}
|
||||
|
||||
func NewDefaultXmlPreferences() XmlPreferences {
|
||||
return XmlPreferences{
|
||||
Indent: 2,
|
||||
AttributePrefix: "+@",
|
||||
ContentName: "+content",
|
||||
StrictMode: false,
|
||||
KeepNamespace: true,
|
||||
UseRawToken: true,
|
||||
ProcInstPrefix: "+p_",
|
||||
DirectiveName: "+directive",
|
||||
SkipProcInst: false,
|
||||
SkipDirectives: false,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *XmlPreferences) Copy() XmlPreferences {
|
||||
return XmlPreferences{
|
||||
Indent: p.Indent,
|
||||
AttributePrefix: p.AttributePrefix,
|
||||
ContentName: p.ContentName,
|
||||
StrictMode: p.StrictMode,
|
||||
KeepNamespace: p.KeepNamespace,
|
||||
UseRawToken: p.UseRawToken,
|
||||
ProcInstPrefix: p.ProcInstPrefix,
|
||||
DirectiveName: p.DirectiveName,
|
||||
SkipProcInst: p.SkipProcInst,
|
||||
SkipDirectives: p.SkipDirectives,
|
||||
}
|
||||
}
|
||||
|
||||
var ConfiguredXMLPreferences = NewDefaultXmlPreferences()
|
||||
|
||||
var XMLFormat = &yqlib.Format{"xml", []string{"x"},
|
||||
func() yqlib.Encoder { return NewXMLEncoder(ConfiguredXMLPreferences) },
|
||||
func() yqlib.Decoder { return NewXMLDecoder(ConfiguredXMLPreferences) },
|
||||
}
|
||||
|
||||
var xmlYqRules = []*yqlib.ParticipleYqRule{
|
||||
{"XMLEncodeWithIndent", `to_?xml\([0-9]+\)`, encodeParseIndent(XMLFormat), 0},
|
||||
{"XmlDecode", `from_?xml|@xmld`, decodeOp(XMLFormat), 0},
|
||||
{"XMLEncode", `to_?xml`, encodeWithIndent(XMLFormat, 2), 0},
|
||||
{"XMLEncodeNoIndent", `@xml`, encodeWithIndent(XMLFormat, 0), 0},
|
||||
{"LoadXML", `load_?xml|xml_?load`, loadOp(NewXMLDecoder(ConfiguredXMLPreferences), false), 0},
|
||||
}
|
||||
|
||||
func RegisterXmlFormat() {
|
||||
yqlib.RegisterFormat(XMLFormat)
|
||||
yqlib.RegisterRules(xmlYqRules)
|
||||
|
||||
}
|
||||
@@ -0,0 +1,827 @@
|
||||
//go:build !yq_noxml
|
||||
|
||||
package xml
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/mikefarah/yq/v4/pkg/yqlib"
|
||||
"github.com/mikefarah/yq/v4/test"
|
||||
)
|
||||
|
||||
const yamlInputWithProcInstAndHeadComment = `# cats
|
||||
+p_xml: version="1.0"
|
||||
this: is some xml`
|
||||
|
||||
const expectedXmlProcInstAndHeadComment = `<?xml version="1.0"?>
|
||||
<!-- cats -->
|
||||
<this>is some xml</this>
|
||||
`
|
||||
|
||||
const xmlProcInstAndHeadCommentBlock = `<?xml version="1.0"?>
|
||||
<!--
|
||||
cats
|
||||
-->
|
||||
<this>is some xml</this>
|
||||
`
|
||||
|
||||
const expectedYamlProcInstAndHeadCommentBlock = `#
|
||||
# cats
|
||||
#
|
||||
+p_xml: version="1.0"
|
||||
this: is some xml
|
||||
`
|
||||
|
||||
const inputXMLWithComments = `
|
||||
<!-- before cat -->
|
||||
<cat>
|
||||
<!-- in cat before -->
|
||||
<x>3<!-- multi
|
||||
line comment
|
||||
for x --></x>
|
||||
<!-- before y -->
|
||||
<y>
|
||||
<!-- in y before -->
|
||||
<d><!-- in d before -->z<!-- in d after --></d>
|
||||
|
||||
<!-- in y after -->
|
||||
</y>
|
||||
<!-- in_cat_after -->
|
||||
</cat>
|
||||
<!-- after cat -->
|
||||
`
|
||||
const inputXMLWithCommentsWithSubChild = `
|
||||
<!-- before cat -->
|
||||
<cat>
|
||||
<!-- in cat before -->
|
||||
<x>3<!-- multi
|
||||
line comment
|
||||
for x --></x>
|
||||
<!-- before y -->
|
||||
<y>
|
||||
<!-- in y before -->
|
||||
<d><!-- in d before --><z sweet="cool"/><!-- in d after --></d>
|
||||
|
||||
<!-- in y after -->
|
||||
</y>
|
||||
<!-- in_cat_after -->
|
||||
</cat>
|
||||
<!-- after cat -->
|
||||
`
|
||||
|
||||
const expectedDecodeYamlWithSubChild = `# before cat
|
||||
cat:
|
||||
# in cat before
|
||||
x: "3" # multi
|
||||
# line comment
|
||||
# for x
|
||||
# before y
|
||||
|
||||
y:
|
||||
# in y before
|
||||
d:
|
||||
# in d before
|
||||
z:
|
||||
+@sweet: cool
|
||||
# in d after
|
||||
# in y after
|
||||
# in_cat_after
|
||||
# after cat
|
||||
`
|
||||
|
||||
const inputXMLWithCommentsWithArray = `
|
||||
<!-- before cat -->
|
||||
<cat>
|
||||
<!-- in cat before -->
|
||||
<x>3<!-- multi
|
||||
line comment
|
||||
for x --></x>
|
||||
<!-- before y -->
|
||||
<y>
|
||||
<!-- in y before -->
|
||||
<d><!-- in d before --><z sweet="cool"/><!-- in d after --></d>
|
||||
<d><!-- in d2 before --><z sweet="cool2"/><!-- in d2 after --></d>
|
||||
|
||||
<!-- in y after -->
|
||||
</y>
|
||||
<!-- in_cat_after -->
|
||||
</cat>
|
||||
<!-- after cat -->
|
||||
`
|
||||
|
||||
const expectedDecodeYamlWithArray = `# before cat
|
||||
cat:
|
||||
# in cat before
|
||||
x: "3" # multi
|
||||
# line comment
|
||||
# for x
|
||||
# before y
|
||||
|
||||
y:
|
||||
# in y before
|
||||
d:
|
||||
- # in d before
|
||||
z:
|
||||
+@sweet: cool
|
||||
# in d after
|
||||
- # in d2 before
|
||||
z:
|
||||
+@sweet: cool2
|
||||
# in d2 after
|
||||
# in y after
|
||||
# in_cat_after
|
||||
# after cat
|
||||
`
|
||||
|
||||
const expectedDecodeYamlWithComments = `# before cat
|
||||
cat:
|
||||
# in cat before
|
||||
x: "3" # multi
|
||||
# line comment
|
||||
# for x
|
||||
# before y
|
||||
|
||||
y:
|
||||
# in y before
|
||||
# in d before
|
||||
d: z # in d after
|
||||
# in y after
|
||||
# in_cat_after
|
||||
# after cat
|
||||
`
|
||||
|
||||
const expectedRoundtripXMLWithComments = `<!-- before cat -->
|
||||
<cat><!-- in cat before -->
|
||||
<x>3<!-- multi
|
||||
line comment
|
||||
for x --></x><!-- before y -->
|
||||
<y><!-- in y before
|
||||
in d before -->
|
||||
<d>z<!-- in d after --></d><!-- in y after -->
|
||||
</y><!-- in_cat_after -->
|
||||
</cat><!-- after cat -->
|
||||
`
|
||||
|
||||
const yamlWithComments = `#
|
||||
# header comment
|
||||
# above_cat
|
||||
#
|
||||
cat: # inline_cat
|
||||
# above_array
|
||||
array: # inline_array
|
||||
- val1 # inline_val1
|
||||
# above_val2
|
||||
- val2 # inline_val2
|
||||
# below_cat
|
||||
`
|
||||
|
||||
const expectedXMLWithComments = `<!--
|
||||
header comment
|
||||
above_cat
|
||||
-->
|
||||
<!-- inline_cat -->
|
||||
<cat><!-- above_array inline_array -->
|
||||
<array>val1<!-- inline_val1 --></array>
|
||||
<array><!-- above_val2 -->val2<!-- inline_val2 --></array>
|
||||
</cat><!-- below_cat -->
|
||||
`
|
||||
|
||||
const inputXMLWithNamespacedAttr = `<?xml version="1.0"?>
|
||||
<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url"></map>
|
||||
`
|
||||
|
||||
const expectedYAMLWithNamespacedAttr = `+p_xml: version="1.0"
|
||||
map:
|
||||
+@xmlns: some-namespace
|
||||
+@xmlns:xsi: some-instance
|
||||
+@xsi:schemaLocation: some-url
|
||||
`
|
||||
|
||||
const expectedYAMLWithRawNamespacedAttr = `+p_xml: version="1.0"
|
||||
map:
|
||||
+@xmlns: some-namespace
|
||||
+@xmlns:xsi: some-instance
|
||||
+@xsi:schemaLocation: some-url
|
||||
`
|
||||
|
||||
const expectedYAMLWithoutRawNamespacedAttr = `+p_xml: version="1.0"
|
||||
map:
|
||||
+@xmlns: some-namespace
|
||||
+@xmlns:xsi: some-instance
|
||||
+@some-instance:schemaLocation: some-url
|
||||
`
|
||||
|
||||
const xmlWithCustomDtd = `
|
||||
<?xml version="1.0"?>
|
||||
<!DOCTYPE root [
|
||||
<!ENTITY writer "Blah.">
|
||||
<!ENTITY copyright "Blah">
|
||||
]>
|
||||
<root>
|
||||
<item>&writer;©right;</item>
|
||||
</root>`
|
||||
|
||||
const expectedDtd = `<?xml version="1.0"?>
|
||||
<!DOCTYPE root [
|
||||
<!ENTITY writer "Blah.">
|
||||
<!ENTITY copyright "Blah">
|
||||
]>
|
||||
<root>
|
||||
<item>&writer;&copyright;</item>
|
||||
</root>
|
||||
`
|
||||
|
||||
const expectedSkippedDtd = `<?xml version="1.0"?>
|
||||
<root>
|
||||
<item>&writer;&copyright;</item>
|
||||
</root>
|
||||
`
|
||||
|
||||
const xmlWithProcInstAndDirectives = `<?xml version="1.0"?>
|
||||
<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
|
||||
<apple>
|
||||
<?coolioo version="1.0"?>
|
||||
<!CATYPE meow purr puss >
|
||||
<b>things</b>
|
||||
</apple>
|
||||
`
|
||||
|
||||
const yamlWithProcInstAndDirectives = `+p_xml: version="1.0"
|
||||
+directive: 'DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" '
|
||||
apple:
|
||||
+p_coolioo: version="1.0"
|
||||
+directive: 'CATYPE meow purr puss '
|
||||
b: things
|
||||
`
|
||||
|
||||
const expectedXmlWithProcInstAndDirectives = `<?xml version="1.0"?>
|
||||
<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
|
||||
<apple><?coolioo version="1.0"?><!CATYPE meow purr puss >
|
||||
<b>things</b>
|
||||
</apple>
|
||||
`
|
||||
|
||||
var xmlScenarios = []yqlib.FormatScenario{
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "bad xml",
|
||||
input: `<?xml version="1.0" encoding="UTF-8"?></Child></Root>`,
|
||||
expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\n",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
input: " <root>value<!-- comment--> </root>",
|
||||
expected: "root: value # comment\n",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
input: "value<root>value</root>",
|
||||
expectedError: "bad file 'sample.yml': invalid XML: Encountered chardata [value] outside of XML node",
|
||||
scenarioType: "decode-error",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
input: "<root><!-- comment-->value</root>",
|
||||
expected: "# comment\nroot: value\n",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
input: "<root> <!-- comment--></root>",
|
||||
expected: "root: # comment\n",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
input: "<root>value<!-- comment-->anotherValue </root>",
|
||||
expected: "root:\n # comment\n - value\n - anotherValue\n",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
input: "<root><cats><cat>quick</cat><cat>soft</cat><!-- kitty_comment--><cat>squishy</cat></cats></root>",
|
||||
expected: "root:\n cats:\n cat:\n - quick\n - soft\n # kitty_comment\n\n - squishy\n",
|
||||
},
|
||||
{
|
||||
description: "ProcInst with head comment",
|
||||
skipDoc: true,
|
||||
input: yamlInputWithProcInstAndHeadComment,
|
||||
expected: expectedXmlProcInstAndHeadComment,
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
description: "Scalar roundtrip",
|
||||
skipDoc: true,
|
||||
input: "<mike>cat</mike>",
|
||||
expression: ".mike",
|
||||
expected: "cat",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "ProcInst with head comment round trip",
|
||||
skipDoc: true,
|
||||
input: expectedXmlProcInstAndHeadComment,
|
||||
expected: expectedXmlProcInstAndHeadComment,
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "ProcInst with block head comment to yaml",
|
||||
skipDoc: true,
|
||||
input: xmlProcInstAndHeadCommentBlock,
|
||||
expected: expectedYamlProcInstAndHeadCommentBlock,
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "ProcInst with block head comment from yaml",
|
||||
skipDoc: true,
|
||||
input: expectedYamlProcInstAndHeadCommentBlock,
|
||||
expected: xmlProcInstAndHeadCommentBlock,
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
description: "ProcInst with head comment round trip block",
|
||||
skipDoc: true,
|
||||
input: xmlProcInstAndHeadCommentBlock,
|
||||
expected: xmlProcInstAndHeadCommentBlock,
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "Parse xml: simple",
|
||||
subdescription: "Notice how all the values are strings, see the next example on how you can fix that.",
|
||||
input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat>\n <says>meow</says>\n <legs>4</legs>\n <cute>true</cute>\n</cat>",
|
||||
expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n says: meow\n legs: \"4\"\n cute: \"true\"\n",
|
||||
},
|
||||
{
|
||||
description: "Parse xml: number",
|
||||
subdescription: "All values are assumed to be strings when parsing XML, but you can use the `from_yaml` operator on all the strings values to autoparse into the correct type.",
|
||||
input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat>\n <says>meow</says>\n <legs>4</legs>\n <cute>true</cute>\n</cat>",
|
||||
expression: " (.. | select(tag == \"!!str\")) |= from_yaml",
|
||||
expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n says: meow\n legs: 4\n cute: true\n",
|
||||
},
|
||||
{
|
||||
description: "Parse xml: array",
|
||||
subdescription: "Consecutive nodes with identical xml names are assumed to be arrays.",
|
||||
input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<animal>cat</animal>\n<animal>goat</animal>",
|
||||
expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\nanimal:\n - cat\n - goat\n",
|
||||
},
|
||||
{
|
||||
description: "Parse xml: force as an array",
|
||||
subdescription: "In XML, if your array has a single item, then yq doesn't know its an array. This is how you can consistently force it to be an array. This handles the 3 scenarios of having nothing in the array, having a single item and having multiple.",
|
||||
input: "<zoo><animal>cat</animal></zoo>",
|
||||
expression: ".zoo.animal |= ([] + .)",
|
||||
expected: "zoo:\n animal:\n - cat\n",
|
||||
},
|
||||
{
|
||||
description: "Parse xml: force all as an array",
|
||||
input: "<zoo><thing><frog>boing</frog></thing></zoo>",
|
||||
expression: ".. |= [] + .",
|
||||
expected: "- zoo:\n - thing:\n - frog:\n - boing\n",
|
||||
},
|
||||
{
|
||||
description: "Parse xml: attributes",
|
||||
subdescription: "Attributes are converted to fields, with the default attribute prefix '+'. Use '--xml-attribute-prefix` to set your own.",
|
||||
input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">\n <legs>7</legs>\n</cat>",
|
||||
expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n +@legs: \"4\"\n legs: \"7\"\n",
|
||||
},
|
||||
{
|
||||
description: "Parse xml: attributes with content",
|
||||
subdescription: "Content is added as a field, using the default content name of `+content`. Use `--xml-content-name` to set your own.",
|
||||
input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">meow</cat>",
|
||||
expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n +content: meow\n +@legs: \"4\"\n",
|
||||
},
|
||||
{
|
||||
description: "Parse xml: content split between comments/children",
|
||||
subdescription: "Multiple content texts are collected into a sequence.",
|
||||
input: "<root> value <!-- comment-->anotherValue <a>frog</a> cool!</root>",
|
||||
expected: "root:\n +content: # comment\n - value\n - anotherValue\n - cool!\n a: frog\n",
|
||||
},
|
||||
{
|
||||
description: "Parse xml: custom dtd",
|
||||
subdescription: "DTD entities are processed as directives.",
|
||||
input: xmlWithCustomDtd,
|
||||
expected: expectedDtd,
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "Roundtrip with name spaced attributes",
|
||||
skipDoc: true,
|
||||
input: inputXMLWithNamespacedAttr,
|
||||
expected: inputXMLWithNamespacedAttr,
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "Parse xml: skip custom dtd",
|
||||
subdescription: "DTDs are directives, skip over directives to skip DTDs.",
|
||||
input: xmlWithCustomDtd,
|
||||
expected: expectedSkippedDtd,
|
||||
scenarioType: "roundtrip-skip-directives",
|
||||
},
|
||||
{
|
||||
description: "Parse xml: with comments",
|
||||
subdescription: "A best attempt is made to preserve comments.",
|
||||
input: inputXMLWithComments,
|
||||
expected: expectedDecodeYamlWithComments,
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "Empty doc",
|
||||
skipDoc: true,
|
||||
input: "",
|
||||
expected: "\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "Empty single node",
|
||||
skipDoc: true,
|
||||
input: "<a/>",
|
||||
expected: "a:\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "Empty close node",
|
||||
skipDoc: true,
|
||||
input: "<a></a>",
|
||||
expected: "a:\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "Nested empty",
|
||||
skipDoc: true,
|
||||
input: "<a><b/></a>",
|
||||
expected: "a:\n b:\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "Parse xml: with comments subchild",
|
||||
skipDoc: true,
|
||||
input: inputXMLWithCommentsWithSubChild,
|
||||
expected: expectedDecodeYamlWithSubChild,
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "Parse xml: with comments array",
|
||||
skipDoc: true,
|
||||
input: inputXMLWithCommentsWithArray,
|
||||
expected: expectedDecodeYamlWithArray,
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "Parse xml: keep attribute namespace",
|
||||
subdescription: fmt.Sprintf(`Defaults to %v`, ConfiguredXMLPreferences.KeepNamespace),
|
||||
skipDoc: false,
|
||||
input: inputXMLWithNamespacedAttr,
|
||||
expected: expectedYAMLWithNamespacedAttr,
|
||||
scenarioType: "decode-keep-ns",
|
||||
},
|
||||
{
|
||||
description: "Parse xml: keep raw attribute namespace",
|
||||
skipDoc: true,
|
||||
input: inputXMLWithNamespacedAttr,
|
||||
expected: expectedYAMLWithRawNamespacedAttr,
|
||||
scenarioType: "decode-raw-token",
|
||||
},
|
||||
{
|
||||
description: "Parse xml: keep raw attribute namespace",
|
||||
subdescription: fmt.Sprintf(`Defaults to %v`, ConfiguredXMLPreferences.UseRawToken),
|
||||
skipDoc: false,
|
||||
input: inputXMLWithNamespacedAttr,
|
||||
expected: expectedYAMLWithoutRawNamespacedAttr,
|
||||
scenarioType: "decode-raw-token-off",
|
||||
},
|
||||
{
|
||||
description: "Encode xml: simple",
|
||||
input: "cat: purrs",
|
||||
expected: "<cat>purrs</cat>\n",
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
description: "includes map tags",
|
||||
skipDoc: true,
|
||||
input: "<cat>purrs</cat>\n",
|
||||
expression: `tag`,
|
||||
expected: "!!map\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "includes array tags",
|
||||
skipDoc: true,
|
||||
input: "<cat>purrs</cat><cat>purrs</cat>\n",
|
||||
expression: `.cat | tag`,
|
||||
expected: "!!seq\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "Encode xml: array",
|
||||
input: "pets:\n cat:\n - purrs\n - meows",
|
||||
expected: "<pets>\n <cat>purrs</cat>\n <cat>meows</cat>\n</pets>\n",
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
description: "Encode xml: attributes",
|
||||
subdescription: "Fields with the matching xml-attribute-prefix are assumed to be attributes.",
|
||||
input: "cat:\n +@name: tiger\n meows: true\n",
|
||||
expected: "<cat name=\"tiger\">\n <meows>true</meows>\n</cat>\n",
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
description: "double prefix",
|
||||
skipDoc: true,
|
||||
input: "cat:\n +@+@name: tiger\n meows: true\n",
|
||||
expected: "<cat +@name=\"tiger\">\n <meows>true</meows>\n</cat>\n",
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
description: "arrays cannot be encoded",
|
||||
skipDoc: true,
|
||||
input: "[cat, dog, fish]",
|
||||
expectedError: "cannot encode !!seq to XML - only maps can be encoded",
|
||||
scenarioType: "encode-error",
|
||||
},
|
||||
{
|
||||
description: "arrays cannot be encoded - 2",
|
||||
skipDoc: true,
|
||||
input: "[cat, dog]",
|
||||
expectedError: "cannot encode !!seq to XML - only maps can be encoded",
|
||||
scenarioType: "encode-error",
|
||||
},
|
||||
{
|
||||
description: "Encode xml: attributes with content",
|
||||
subdescription: "Fields with the matching xml-content-name is assumed to be content.",
|
||||
input: "cat:\n +@name: tiger\n +content: cool\n",
|
||||
expected: "<cat name=\"tiger\">cool</cat>\n",
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
description: "round trip multiline 1",
|
||||
skipDoc: true,
|
||||
input: "<x><!-- cats --></x>\n",
|
||||
expected: "<x><!-- cats --></x>\n",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "round trip multiline 2",
|
||||
skipDoc: true,
|
||||
input: "<x><!--\n cats\n --></x>\n",
|
||||
expected: "<x><!--\ncats\n--></x>\n",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "round trip multiline 3",
|
||||
skipDoc: true,
|
||||
input: "<x><!--\n\tcats\n --></x>\n",
|
||||
expected: "<x><!--\n\tcats\n--></x>\n",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "round trip multiline 4",
|
||||
skipDoc: true,
|
||||
input: "<x><!--\n\tcats\n\tdogs\n--></x>\n",
|
||||
expected: "<x><!--\n\tcats\n\tdogs\n--></x>\n",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "round trip multiline 5",
|
||||
skipDoc: true, // pity spaces aren't kept atm.
|
||||
input: "<x><!--\ncats\ndogs\n--></x>\n",
|
||||
expected: "<x><!--\ncats\ndogs\n--></x>\n",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "Encode xml: comments",
|
||||
subdescription: "A best attempt is made to copy comments to xml.",
|
||||
input: yamlWithComments,
|
||||
expected: expectedXMLWithComments,
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
description: "Encode: doctype and xml declaration",
|
||||
subdescription: "Use the special xml names to add/modify proc instructions and directives.",
|
||||
input: yamlWithProcInstAndDirectives,
|
||||
expected: expectedXmlWithProcInstAndDirectives,
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
description: "Round trip: with comments",
|
||||
subdescription: "A best effort is made, but comment positions and white space are not preserved perfectly.",
|
||||
input: inputXMLWithComments,
|
||||
expected: expectedRoundtripXMLWithComments,
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "Roundtrip: with doctype and declaration",
|
||||
subdescription: "yq parses XML proc instructions and directives into nodes.\nUnfortunately the underlying XML parser loses whitespace information.",
|
||||
input: xmlWithProcInstAndDirectives,
|
||||
expected: expectedXmlWithProcInstAndDirectives,
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
}
|
||||
|
||||
func testXMLScenario(t *testing.T, s yqlib.FormatScenario) {
|
||||
switch s.scenarioType {
|
||||
case "", "decode":
|
||||
yamlPrefs := yqlib.ConfiguredYamlPreferences.Copy()
|
||||
yamlPrefs.Indent = 4
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(yamlPrefs)), s.description)
|
||||
case "encode":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(ConfiguredXMLPreferences)), s.description)
|
||||
case "roundtrip":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(ConfiguredXMLPreferences)), s.description)
|
||||
case "decode-keep-ns":
|
||||
prefs := NewDefaultXmlPreferences()
|
||||
prefs.KeepNamespace = true
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
|
||||
case "decode-raw-token":
|
||||
prefs := NewDefaultXmlPreferences()
|
||||
prefs.UseRawToken = true
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
|
||||
case "decode-raw-token-off":
|
||||
prefs := NewDefaultXmlPreferences()
|
||||
prefs.UseRawToken = false
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
|
||||
case "roundtrip-skip-directives":
|
||||
prefs := NewDefaultXmlPreferences()
|
||||
prefs.SkipDirectives = true
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(prefs)), s.description)
|
||||
case "decode-error":
|
||||
result, err := processFormatScenario(s, NewXMLDecoder(NewDefaultXmlPreferences()), NewYamlEncoder(ConfiguredYamlPreferences))
|
||||
if err == nil {
|
||||
t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result)
|
||||
} else {
|
||||
test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description)
|
||||
}
|
||||
case "encode-error":
|
||||
result, err := processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(NewDefaultXmlPreferences()))
|
||||
if err == nil {
|
||||
t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result)
|
||||
} else {
|
||||
test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description)
|
||||
}
|
||||
|
||||
default:
|
||||
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
|
||||
}
|
||||
}
|
||||
|
||||
func documentXMLScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
|
||||
s := i.(FormatScenario)
|
||||
|
||||
if s.skipDoc {
|
||||
return
|
||||
}
|
||||
switch s.scenarioType {
|
||||
case "", "decode":
|
||||
documentXMLDecodeScenario(w, s)
|
||||
case "encode":
|
||||
documentXMLEncodeScenario(w, s)
|
||||
case "roundtrip":
|
||||
documentXMLRoundTripScenario(w, s)
|
||||
case "decode-keep-ns":
|
||||
documentXMLDecodeKeepNsScenario(w, s)
|
||||
case "decode-raw-token-off":
|
||||
documentXMLDecodeKeepNsRawTokenScenario(w, s)
|
||||
case "roundtrip-skip-directives":
|
||||
documentXMLSkipDirectivesScenario(w, s)
|
||||
|
||||
default:
|
||||
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
|
||||
}
|
||||
}
|
||||
|
||||
func documentXMLDecodeScenario(w *bufio.Writer, s FormatScenario) {
|
||||
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
|
||||
|
||||
if s.subdescription != "" {
|
||||
writeOrPanic(w, s.subdescription)
|
||||
writeOrPanic(w, "\n\n")
|
||||
}
|
||||
|
||||
writeOrPanic(w, "Given a sample.xml file of:\n")
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input))
|
||||
|
||||
writeOrPanic(w, "then\n")
|
||||
expression := s.expression
|
||||
if expression == "" {
|
||||
expression = "."
|
||||
}
|
||||
writeOrPanic(w, fmt.Sprintf("```bash\nyq -oy '%v' sample.xml\n```\n", expression))
|
||||
writeOrPanic(w, "will output\n")
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(ConfiguredYamlPreferences))))
|
||||
}
|
||||
|
||||
func documentXMLDecodeKeepNsScenario(w *bufio.Writer, s FormatScenario) {
|
||||
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
|
||||
|
||||
if s.subdescription != "" {
|
||||
writeOrPanic(w, s.subdescription)
|
||||
writeOrPanic(w, "\n\n")
|
||||
}
|
||||
|
||||
writeOrPanic(w, "Given a sample.xml file of:\n")
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input))
|
||||
|
||||
writeOrPanic(w, "then\n")
|
||||
writeOrPanic(w, "```bash\nyq --xml-keep-namespace=false '.' sample.xml\n```\n")
|
||||
writeOrPanic(w, "will output\n")
|
||||
prefs := NewDefaultXmlPreferences()
|
||||
prefs.KeepNamespace = false
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(prefs))))
|
||||
|
||||
prefsWithout := NewDefaultXmlPreferences()
|
||||
prefs.KeepNamespace = true
|
||||
writeOrPanic(w, "instead of\n")
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(prefsWithout))))
|
||||
}
|
||||
|
||||
func documentXMLDecodeKeepNsRawTokenScenario(w *bufio.Writer, s FormatScenario) {
|
||||
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
|
||||
|
||||
if s.subdescription != "" {
|
||||
writeOrPanic(w, s.subdescription)
|
||||
writeOrPanic(w, "\n\n")
|
||||
}
|
||||
|
||||
writeOrPanic(w, "Given a sample.xml file of:\n")
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input))
|
||||
|
||||
writeOrPanic(w, "then\n")
|
||||
writeOrPanic(w, "```bash\nyq --xml-raw-token=false '.' sample.xml\n```\n")
|
||||
writeOrPanic(w, "will output\n")
|
||||
|
||||
prefs := NewDefaultXmlPreferences()
|
||||
prefs.UseRawToken = false
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(prefs))))
|
||||
|
||||
prefsWithout := NewDefaultXmlPreferences()
|
||||
prefsWithout.UseRawToken = true
|
||||
|
||||
writeOrPanic(w, "instead of\n")
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(prefsWithout))))
|
||||
}
|
||||
|
||||
func documentXMLEncodeScenario(w *bufio.Writer, s FormatScenario) {
|
||||
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
|
||||
|
||||
if s.subdescription != "" {
|
||||
writeOrPanic(w, s.subdescription)
|
||||
writeOrPanic(w, "\n\n")
|
||||
}
|
||||
|
||||
writeOrPanic(w, "Given a sample.yml file of:\n")
|
||||
writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input))
|
||||
|
||||
writeOrPanic(w, "then\n")
|
||||
writeOrPanic(w, "```bash\nyq -o=xml sample.yml\n```\n")
|
||||
writeOrPanic(w, "will output\n")
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(ConfiguredXMLPreferences))))
|
||||
}
|
||||
|
||||
func documentXMLRoundTripScenario(w *bufio.Writer, s FormatScenario) {
|
||||
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
|
||||
|
||||
if s.subdescription != "" {
|
||||
writeOrPanic(w, s.subdescription)
|
||||
writeOrPanic(w, "\n\n")
|
||||
}
|
||||
|
||||
writeOrPanic(w, "Given a sample.xml file of:\n")
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input))
|
||||
|
||||
writeOrPanic(w, "then\n")
|
||||
writeOrPanic(w, "```bash\nyq '.' sample.xml\n```\n")
|
||||
writeOrPanic(w, "will output\n")
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(ConfiguredXMLPreferences))))
|
||||
}
|
||||
|
||||
func documentXMLSkipDirectivesScenario(w *bufio.Writer, s FormatScenario) {
|
||||
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
|
||||
|
||||
if s.subdescription != "" {
|
||||
writeOrPanic(w, s.subdescription)
|
||||
writeOrPanic(w, "\n\n")
|
||||
}
|
||||
|
||||
writeOrPanic(w, "Given a sample.xml file of:\n")
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input))
|
||||
|
||||
writeOrPanic(w, "then\n")
|
||||
writeOrPanic(w, "```bash\nyq --xml-skip-directives '.' sample.xml\n```\n")
|
||||
writeOrPanic(w, "will output\n")
|
||||
prefs := NewDefaultXmlPreferences()
|
||||
prefs.SkipDirectives = true
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(prefs))))
|
||||
}
|
||||
|
||||
func TestXMLScenarios(t *testing.T) {
|
||||
for _, tt := range xmlScenarios {
|
||||
testXMLScenario(t, tt)
|
||||
}
|
||||
genericScenarios := make([]interface{}, len(xmlScenarios))
|
||||
for i, s := range xmlScenarios {
|
||||
genericScenarios[i] = s
|
||||
}
|
||||
documentScenarios(t, "usage", "xml", genericScenarios, documentXMLScenario)
|
||||
}
|
||||
Reference in New Issue
Block a user