mirror of
https://github.com/mikefarah/yq.git
synced 2024-12-19 20:19:04 +00:00
Added XML encoding/decoding
This commit is contained in:
parent
606ef91cc6
commit
be36a0ef4b
@ -142,4 +142,25 @@ EOM
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testOutputXmComplex() {
|
||||
cat >test.yml <<EOL
|
||||
a: {b: {c: ["cat", "dog"], +f: meow}}
|
||||
EOL
|
||||
|
||||
read -r -d '' expected << EOM
|
||||
<a>
|
||||
<b f="meow">
|
||||
<c>cat</c>
|
||||
<c>dog</c>
|
||||
</b>
|
||||
</a>
|
||||
EOM
|
||||
|
||||
X=$(./yq e --output-format=x test.yml)
|
||||
assertEquals "$expected" "$X"
|
||||
|
||||
X=$(./yq ea --output-format=x test.yml)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
source ./scripts/shunit2
|
@ -85,8 +85,9 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
|
||||
}
|
||||
|
||||
printerWriter := configurePrinterWriter(format, out)
|
||||
encoder := configureEncoder(format)
|
||||
|
||||
printer := yqlib.NewPrinter(printerWriter, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
|
||||
printer := yqlib.NewPrinter(encoder, printerWriter)
|
||||
|
||||
if frontMatter != "" {
|
||||
frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])
|
||||
|
@ -93,8 +93,9 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
|
||||
}
|
||||
|
||||
printerWriter := configurePrinterWriter(format, out)
|
||||
encoder := configureEncoder(format)
|
||||
|
||||
printer := yqlib.NewPrinter(printerWriter, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
|
||||
printer := yqlib.NewPrinter(encoder, printerWriter)
|
||||
|
||||
decoder, err := configureDecoder()
|
||||
if err != nil {
|
||||
|
18
cmd/utils.go
18
cmd/utils.go
@ -73,3 +73,21 @@ func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) yql
|
||||
}
|
||||
return printerWriter
|
||||
}
|
||||
|
||||
func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder {
|
||||
switch format {
|
||||
case yqlib.JsonOutputFormat:
|
||||
return yqlib.NewJsonEncoder(indent)
|
||||
case yqlib.PropsOutputFormat:
|
||||
return yqlib.NewPropertiesEncoder()
|
||||
case yqlib.CsvOutputFormat:
|
||||
return yqlib.NewCsvEncoder(',')
|
||||
case yqlib.TsvOutputFormat:
|
||||
return yqlib.NewCsvEncoder('\t')
|
||||
case yqlib.YamlOutputFormat:
|
||||
return yqlib.NewYamlEncoder(indent, colorsEnabled, !noDocSeparators, unwrapScalar)
|
||||
case yqlib.XmlOutputFormat:
|
||||
return yqlib.NewXmlEncoder(indent, xmlAttributePrefix, xmlContentName)
|
||||
}
|
||||
panic("invalid encoder")
|
||||
}
|
||||
|
@ -1,15 +1 @@
|
||||
strings:
|
||||
- a: banana
|
||||
- a: cat
|
||||
- a: apple
|
||||
|
||||
numbers:
|
||||
- a: 12
|
||||
- a: 13
|
||||
- a: 120
|
||||
|
||||
|
||||
obj:
|
||||
- a: {cat: "adog"}
|
||||
- a: {cat: "doga"}
|
||||
- a: apple
|
||||
cat: purrs
|
||||
|
@ -1,2 +1,14 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<cat legs="4">BiBi</cat>
|
||||
<!-- before cat -->
|
||||
<cat>
|
||||
<!-- in cat before -->
|
||||
<x>3<!-- multi
|
||||
line comment
|
||||
for x --></x>
|
||||
<y>
|
||||
<!-- in y before -->
|
||||
<d><!-- in d before -->4<!-- in d after --></d>
|
||||
<!-- in y after -->
|
||||
</y>
|
||||
<!-- in_cat_after -->
|
||||
</cat>
|
||||
<!-- after cat -->
|
1
go.mod
1
go.mod
@ -6,6 +6,7 @@ require (
|
||||
github.com/goccy/go-yaml v1.9.5
|
||||
github.com/jinzhu/copier v0.3.4
|
||||
github.com/magiconair/properties v1.8.5
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
|
||||
github.com/spf13/cobra v1.3.0
|
||||
github.com/timtadh/lexmachine v0.2.2
|
||||
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
|
||||
|
2
go.sum
2
go.sum
@ -292,6 +292,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
|
||||
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
|
||||
|
@ -4,6 +4,9 @@ import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
// "strings"
|
||||
"unicode"
|
||||
|
||||
"golang.org/x/net/html/charset"
|
||||
@ -31,15 +34,15 @@ func InputFormatFromString(format string) (InputFormat, error) {
|
||||
type xmlDecoder struct {
|
||||
reader io.Reader
|
||||
attributePrefix string
|
||||
contentPrefix string
|
||||
contentName string
|
||||
finished bool
|
||||
}
|
||||
|
||||
func NewXmlDecoder(attributePrefix string, contentPrefix string) Decoder {
|
||||
if contentPrefix == "" {
|
||||
contentPrefix = "content"
|
||||
func NewXmlDecoder(attributePrefix string, contentName string) Decoder {
|
||||
if contentName == "" {
|
||||
contentName = "content"
|
||||
}
|
||||
return &xmlDecoder{attributePrefix: attributePrefix, contentPrefix: contentPrefix, finished: false}
|
||||
return &xmlDecoder{attributePrefix: attributePrefix, contentName: contentName, finished: false}
|
||||
}
|
||||
|
||||
func (dec *xmlDecoder) Init(reader io.Reader) {
|
||||
@ -60,20 +63,41 @@ func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yaml.Node, error) {
|
||||
return yamlNode, nil
|
||||
}
|
||||
|
||||
func (dec *xmlDecoder) processComment(c string) string {
|
||||
if c == "" {
|
||||
return ""
|
||||
}
|
||||
return "#" + strings.TrimRight(c, " ")
|
||||
}
|
||||
|
||||
func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
|
||||
yamlNode := &yaml.Node{Kind: yaml.MappingNode, HeadComment: n.Comment}
|
||||
log.Debug("createMap: headC: %v, footC: %v", n.HeadComment, n.FootComment)
|
||||
yamlNode := &yaml.Node{Kind: yaml.MappingNode}
|
||||
|
||||
if len(n.Data) > 0 {
|
||||
label := dec.contentPrefix
|
||||
yamlNode.Content = append(yamlNode.Content, createScalarNode(label, label), createScalarNode(n.Data, n.Data))
|
||||
label := dec.contentName
|
||||
labelNode := createScalarNode(label, label)
|
||||
labelNode.HeadComment = dec.processComment(n.HeadComment)
|
||||
labelNode.FootComment = dec.processComment(n.FootComment)
|
||||
yamlNode.Content = append(yamlNode.Content, labelNode, createScalarNode(n.Data, n.Data))
|
||||
}
|
||||
|
||||
for _, keyValuePair := range n.Children {
|
||||
for i, keyValuePair := range n.Children {
|
||||
label := keyValuePair.K
|
||||
children := keyValuePair.V
|
||||
labelNode := createScalarNode(label, label)
|
||||
var valueNode *yaml.Node
|
||||
var err error
|
||||
|
||||
if i == 0 {
|
||||
labelNode.HeadComment = dec.processComment(n.HeadComment)
|
||||
|
||||
}
|
||||
|
||||
// if i == len(n.Children)-1 {
|
||||
labelNode.FootComment = dec.processComment(keyValuePair.FootComment)
|
||||
// }
|
||||
|
||||
log.Debug("len of children in %v is %v", label, len(children))
|
||||
if len(children) > 1 {
|
||||
valueNode, err = dec.createSequence(children)
|
||||
@ -81,6 +105,13 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
// comment hack for maps of scalars
|
||||
// if the value is a scalar, the head comment of the scalar needs to go on the key?
|
||||
// add tests for <z/> as well as multiple <ds> of inputXmlWithComments > yaml
|
||||
if len(children[0].Children) == 0 && children[0].HeadComment != "" {
|
||||
labelNode.HeadComment = labelNode.HeadComment + "\n" + strings.TrimSpace(children[0].HeadComment)
|
||||
children[0].HeadComment = ""
|
||||
}
|
||||
valueNode, err = dec.convertToYamlNode(children[0])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -97,7 +128,11 @@ func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yaml.Node, error) {
|
||||
return dec.createMap(n)
|
||||
}
|
||||
scalar := createScalarNode(n.Data, n.Data)
|
||||
scalar.HeadComment = n.Comment
|
||||
log.Debug("scalar headC: %v, footC: %v", n.HeadComment, n.FootComment)
|
||||
scalar.HeadComment = dec.processComment(n.HeadComment)
|
||||
scalar.LineComment = dec.processComment(n.LineComment)
|
||||
scalar.FootComment = dec.processComment(n.FootComment)
|
||||
|
||||
return scalar, nil
|
||||
}
|
||||
|
||||
@ -124,14 +159,17 @@ func (dec *xmlDecoder) Decode(rootYamlNode *yaml.Node) error {
|
||||
}
|
||||
|
||||
type xmlNode struct {
|
||||
Children []*xmlChildrenKv
|
||||
Comment string
|
||||
Data string
|
||||
Children []*xmlChildrenKv
|
||||
HeadComment string
|
||||
FootComment string
|
||||
LineComment string
|
||||
Data string
|
||||
}
|
||||
|
||||
type xmlChildrenKv struct {
|
||||
K string
|
||||
V []*xmlNode
|
||||
K string
|
||||
V []*xmlNode
|
||||
FootComment string
|
||||
}
|
||||
|
||||
// AddChild appends a node to the list of children
|
||||
@ -158,6 +196,7 @@ type element struct {
|
||||
parent *element
|
||||
n *xmlNode
|
||||
label string
|
||||
state string
|
||||
}
|
||||
|
||||
// this code is heavily based on https://github.com/basgys/goxml2json
|
||||
@ -183,6 +222,8 @@ func (dec *xmlDecoder) decodeXml(root *xmlNode) error {
|
||||
|
||||
switch se := t.(type) {
|
||||
case xml.StartElement:
|
||||
log.Debug("start element %v", se.Name.Local)
|
||||
elem.state = "started"
|
||||
// Build new a new current element and link it to its parent
|
||||
elem = &element{
|
||||
parent: elem,
|
||||
@ -197,7 +238,13 @@ func (dec *xmlDecoder) decodeXml(root *xmlNode) error {
|
||||
case xml.CharData:
|
||||
// Extract XML data (if any)
|
||||
elem.n.Data = trimNonGraphic(string(se))
|
||||
if elem.n.Data != "" {
|
||||
elem.state = "chardata"
|
||||
log.Debug("chardata [%v] for %v", elem.n.Data, elem.label)
|
||||
}
|
||||
case xml.EndElement:
|
||||
log.Debug("end element %v", elem.label)
|
||||
elem.state = "finished"
|
||||
// And add it to its parent list
|
||||
if elem.parent != nil {
|
||||
elem.parent.n.AddChild(elem.label, elem.n)
|
||||
@ -206,13 +253,49 @@ func (dec *xmlDecoder) decodeXml(root *xmlNode) error {
|
||||
// Then change the current element to its parent
|
||||
elem = elem.parent
|
||||
case xml.Comment:
|
||||
elem.n.Comment = trimNonGraphic(string(xml.CharData(se)))
|
||||
|
||||
commentStr := string(xml.CharData(se))
|
||||
if elem.state == "started" {
|
||||
applyFootComment(elem, commentStr)
|
||||
|
||||
} else if elem.state == "chardata" {
|
||||
log.Debug("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
|
||||
elem.n.LineComment = joinFilter([]string{elem.n.LineComment, commentStr})
|
||||
} else {
|
||||
log.Debug("got a head comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
|
||||
elem.n.HeadComment = joinFilter([]string{elem.n.HeadComment, commentStr})
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func applyFootComment(elem *element, commentStr string) {
|
||||
|
||||
// first lets try to put the comment on the last child
|
||||
if len(elem.n.Children) > 0 {
|
||||
lastChildIndex := len(elem.n.Children) - 1
|
||||
childKv := elem.n.Children[lastChildIndex]
|
||||
log.Debug("got a foot comment for %v: [%v]", childKv.K, commentStr)
|
||||
childKv.FootComment = joinFilter([]string{elem.n.FootComment, commentStr})
|
||||
} else {
|
||||
log.Debug("got a foot comment for %v: [%v]", elem.label, commentStr)
|
||||
elem.n.FootComment = joinFilter([]string{elem.n.FootComment, commentStr})
|
||||
}
|
||||
}
|
||||
|
||||
func joinFilter(rawStrings []string) string {
|
||||
stringsToJoin := make([]string, 0)
|
||||
for _, str := range rawStrings {
|
||||
if str != "" {
|
||||
stringsToJoin = append(stringsToJoin, str)
|
||||
}
|
||||
}
|
||||
return strings.Join(stringsToJoin, " ")
|
||||
}
|
||||
|
||||
// trimNonGraphic returns a slice of the string s, with all leading and trailing
|
||||
// non graphic characters and spaces removed.
|
||||
//
|
||||
|
@ -2,8 +2,6 @@
|
||||
|
||||
Encode and decode to and from XML. Whitespace is not conserved for round trips - but the order of the fields are.
|
||||
|
||||
As yaml does not have the concept of attributes, xml attributes are converted to regular fields with a prefix to prevent clobbering. This defaults to "+", use the `--xml-attribute-prefix` to change.
|
||||
|
||||
Consecutive xml nodes with the same name are assumed to be arrays.
|
||||
|
||||
All values in XML are assumed to be strings - but you can use `from_yaml` to parse them into their correct types:
|
||||
@ -14,8 +12,6 @@ yq e -p=xml '.myNumberField |= from_yaml' my.xml
|
||||
```
|
||||
|
||||
|
||||
XML nodes that have attributes then plain content, e.g:
|
||||
|
||||
```xml
|
||||
<cat name="tiger">meow</cat>
|
||||
```
|
||||
|
@ -2,8 +2,6 @@
|
||||
|
||||
Encode and decode to and from XML. Whitespace is not conserved for round trips - but the order of the fields are.
|
||||
|
||||
As yaml does not have the concept of attributes, xml attributes are converted to regular fields with a prefix to prevent clobbering. This defaults to "+", use the `--xml-attribute-prefix` to change.
|
||||
|
||||
Consecutive xml nodes with the same name are assumed to be arrays.
|
||||
|
||||
All values in XML are assumed to be strings - but you can use `from_yaml` to parse them into their correct types:
|
||||
@ -14,8 +12,6 @@ yq e -p=xml '.myNumberField |= from_yaml' my.xml
|
||||
```
|
||||
|
||||
|
||||
XML nodes that have attributes then plain content, e.g:
|
||||
|
||||
```xml
|
||||
<cat name="tiger">meow</cat>
|
||||
```
|
||||
@ -58,7 +54,7 @@ animal:
|
||||
```
|
||||
|
||||
## Parse xml: attributes
|
||||
Attributes are converted to fields, with the attribute prefix.
|
||||
Attributes are converted to fields, with the default attribute prefix '+'. Use '--xml-attribute-prefix` to set your own.
|
||||
|
||||
Given a sample.xml file of:
|
||||
```xml
|
||||
@ -79,7 +75,7 @@ cat:
|
||||
```
|
||||
|
||||
## Parse xml: attributes with content
|
||||
Content is added as a field, using the content name
|
||||
Content is added as a field, using the default content name of '+content'. Use `--xml-content-name` to set your own.
|
||||
|
||||
Given a sample.xml file of:
|
||||
```xml
|
||||
@ -97,6 +93,53 @@ cat:
|
||||
+legs: "4"
|
||||
```
|
||||
|
||||
## Parse xml: with comments
|
||||
A best attempt is made to preserve comments.
|
||||
|
||||
Given a sample.xml file of:
|
||||
```xml
|
||||
|
||||
<!-- before cat -->
|
||||
<cat>
|
||||
<!-- in cat before -->
|
||||
<x>3<!-- multi
|
||||
line comment
|
||||
for x --></x>
|
||||
<!-- before y -->
|
||||
<y>
|
||||
<!-- in y before -->
|
||||
<d><!-- in d before -->z<!-- in d after --></d>
|
||||
|
||||
<!-- in y after -->
|
||||
</y>
|
||||
<!-- in_cat_after -->
|
||||
</cat>
|
||||
<!-- after cat -->
|
||||
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq e -p=xml '.' sample.xml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
# before cat
|
||||
cat:
|
||||
# in cat before
|
||||
x: "3" # multi
|
||||
# line comment
|
||||
# for x
|
||||
# before y
|
||||
|
||||
y:
|
||||
# in y before
|
||||
# in d before
|
||||
d: z # in d after
|
||||
# in y after
|
||||
# in_cat_after
|
||||
# after cat
|
||||
```
|
||||
|
||||
## Encode xml: simple
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
@ -108,7 +151,8 @@ yq e -o=xml '.' sample.yml
|
||||
```
|
||||
will output
|
||||
```xml
|
||||
<cat>purrs</cat>```
|
||||
<cat>purrs</cat>
|
||||
```
|
||||
|
||||
## Encode xml: array
|
||||
Given a sample.yml file of:
|
||||
@ -127,7 +171,8 @@ will output
|
||||
<pets>
|
||||
<cat>purrs</cat>
|
||||
<cat>meows</cat>
|
||||
</pets>```
|
||||
</pets>
|
||||
```
|
||||
|
||||
## Encode xml: attributes
|
||||
Fields with the matching xml-attribute-prefix are assumed to be attributes.
|
||||
@ -147,7 +192,8 @@ will output
|
||||
```xml
|
||||
<cat name="tiger">
|
||||
<meows>true</meows>
|
||||
</cat>```
|
||||
</cat>
|
||||
```
|
||||
|
||||
## Encode xml: attributes with content
|
||||
Fields with the matching xml-content-name is assumed to be content.
|
||||
@ -165,5 +211,74 @@ yq e -o=xml '.' sample.yml
|
||||
```
|
||||
will output
|
||||
```xml
|
||||
<cat name="tiger">cool</cat>```
|
||||
<cat name="tiger">cool</cat>
|
||||
```
|
||||
|
||||
## Encode xml: comments
|
||||
A best attempt is made to copy comments to xml.
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
# above_cat
|
||||
cat: # inline_cat
|
||||
# above_array
|
||||
array: # inline_array
|
||||
- val1 # inline_val1
|
||||
# above_val2
|
||||
- val2 # inline_val2
|
||||
# below_cat
|
||||
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq e -o=xml '.' sample.yml
|
||||
```
|
||||
will output
|
||||
```xml
|
||||
<!-- above_cat inline_cat --><cat><!-- above_array inline_array -->
|
||||
<array>val1<!-- inline_val1 --></array>
|
||||
<array><!-- above_val2 -->val2<!-- inline_val2 --></array>
|
||||
</cat><!-- below_cat -->
|
||||
```
|
||||
|
||||
## Round trip: with comments
|
||||
A best effort is made, but comment positions and white space are not preserved perfectly.
|
||||
|
||||
Given a sample.xml file of:
|
||||
```xml
|
||||
|
||||
<!-- before cat -->
|
||||
<cat>
|
||||
<!-- in cat before -->
|
||||
<x>3<!-- multi
|
||||
line comment
|
||||
for x --></x>
|
||||
<!-- before y -->
|
||||
<y>
|
||||
<!-- in y before -->
|
||||
<d><!-- in d before -->z<!-- in d after --></d>
|
||||
|
||||
<!-- in y after -->
|
||||
</y>
|
||||
<!-- in_cat_after -->
|
||||
</cat>
|
||||
<!-- after cat -->
|
||||
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq e -p=xml -o=xml '.' sample.xml
|
||||
```
|
||||
will output
|
||||
```xml
|
||||
<!-- before cat --><cat><!-- in cat before -->
|
||||
<x>3<!-- multi
|
||||
line comment
|
||||
for x --></x><!-- before y -->
|
||||
<y><!-- in y before
|
||||
in d before -->
|
||||
<d>z<!-- in d after --></d><!-- in y after -->
|
||||
</y><!-- in_cat_after -->
|
||||
</cat><!-- after cat -->
|
||||
```
|
||||
|
||||
|
@ -11,92 +11,10 @@ import (
|
||||
)
|
||||
|
||||
type Encoder interface {
|
||||
Encode(node *yaml.Node) error
|
||||
}
|
||||
|
||||
type yamlEncoder struct {
|
||||
destination io.Writer
|
||||
indent int
|
||||
colorise bool
|
||||
firstDoc bool
|
||||
}
|
||||
|
||||
func NewYamlEncoder(destination io.Writer, indent int, colorise bool) Encoder {
|
||||
if indent < 0 {
|
||||
indent = 0
|
||||
}
|
||||
return &yamlEncoder{destination, indent, colorise, true}
|
||||
}
|
||||
|
||||
func (ye *yamlEncoder) Encode(node *yaml.Node) error {
|
||||
|
||||
destination := ye.destination
|
||||
tempBuffer := bytes.NewBuffer(nil)
|
||||
if ye.colorise {
|
||||
destination = tempBuffer
|
||||
}
|
||||
|
||||
var encoder = yaml.NewEncoder(destination)
|
||||
|
||||
encoder.SetIndent(ye.indent)
|
||||
// TODO: work out if the first doc had a separator or not.
|
||||
if ye.firstDoc {
|
||||
ye.firstDoc = false
|
||||
} else if _, err := destination.Write([]byte("---\n")); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := encoder.Encode(node); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if ye.colorise {
|
||||
return colorizeAndPrint(tempBuffer.Bytes(), ye.destination)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type jsonEncoder struct {
|
||||
encoder *json.Encoder
|
||||
}
|
||||
|
||||
func mapKeysToStrings(node *yaml.Node) {
|
||||
|
||||
if node.Kind == yaml.MappingNode {
|
||||
for index, child := range node.Content {
|
||||
if index%2 == 0 { // its a map key
|
||||
child.Tag = "!!str"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, child := range node.Content {
|
||||
mapKeysToStrings(child)
|
||||
}
|
||||
}
|
||||
|
||||
func NewJsonEncoder(destination io.Writer, indent int) Encoder {
|
||||
var encoder = json.NewEncoder(destination)
|
||||
encoder.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
|
||||
|
||||
var indentString = ""
|
||||
|
||||
for index := 0; index < indent; index++ {
|
||||
indentString = indentString + " "
|
||||
}
|
||||
encoder.SetIndent("", indentString)
|
||||
return &jsonEncoder{encoder}
|
||||
}
|
||||
|
||||
func (je *jsonEncoder) Encode(node *yaml.Node) error {
|
||||
var dataBucket orderedMap
|
||||
// firstly, convert all map keys to strings
|
||||
mapKeysToStrings(node)
|
||||
errorDecoding := node.Decode(&dataBucket)
|
||||
if errorDecoding != nil {
|
||||
return errorDecoding
|
||||
}
|
||||
return je.encoder.Encode(dataBucket)
|
||||
Encode(writer io.Writer, node *yaml.Node) error
|
||||
PrintDocumentSeparator(writer io.Writer) error
|
||||
PrintLeadingContent(writer io.Writer, content string) error
|
||||
CanHandleAliases() bool
|
||||
}
|
||||
|
||||
// orderedMap allows to marshal and unmarshal JSON and YAML values keeping the
|
||||
|
@ -9,16 +9,26 @@ import (
|
||||
)
|
||||
|
||||
type csvEncoder struct {
|
||||
destination csv.Writer
|
||||
separator rune
|
||||
}
|
||||
|
||||
func NewCsvEncoder(destination io.Writer, separator rune) Encoder {
|
||||
csvWriter := *csv.NewWriter(destination)
|
||||
csvWriter.Comma = separator
|
||||
return &csvEncoder{csvWriter}
|
||||
func NewCsvEncoder(separator rune) Encoder {
|
||||
return &csvEncoder{separator}
|
||||
}
|
||||
|
||||
func (e *csvEncoder) encodeRow(contents []*yaml.Node) error {
|
||||
func (e *csvEncoder) CanHandleAliases() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (e *csvEncoder) PrintDocumentSeparator(writer io.Writer) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *csvEncoder) PrintLeadingContent(writer io.Writer, content string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *csvEncoder) encodeRow(csvWriter *csv.Writer, contents []*yaml.Node) error {
|
||||
stringValues := make([]string, len(contents))
|
||||
|
||||
for i, child := range contents {
|
||||
@ -28,10 +38,13 @@ func (e *csvEncoder) encodeRow(contents []*yaml.Node) error {
|
||||
}
|
||||
stringValues[i] = child.Value
|
||||
}
|
||||
return e.destination.Write(stringValues)
|
||||
return csvWriter.Write(stringValues)
|
||||
}
|
||||
|
||||
func (e *csvEncoder) Encode(originalNode *yaml.Node) error {
|
||||
func (e *csvEncoder) Encode(writer io.Writer, originalNode *yaml.Node) error {
|
||||
csvWriter := csv.NewWriter(writer)
|
||||
csvWriter.Comma = e.separator
|
||||
|
||||
// node must be a sequence
|
||||
node := unwrapDoc(originalNode)
|
||||
if node.Kind != yaml.SequenceNode {
|
||||
@ -40,7 +53,7 @@ func (e *csvEncoder) Encode(originalNode *yaml.Node) error {
|
||||
return nil
|
||||
}
|
||||
if node.Content[0].Kind == yaml.ScalarNode {
|
||||
return e.encodeRow(node.Content)
|
||||
return e.encodeRow(csvWriter, node.Content)
|
||||
}
|
||||
|
||||
for i, child := range node.Content {
|
||||
@ -48,7 +61,7 @@ func (e *csvEncoder) Encode(originalNode *yaml.Node) error {
|
||||
if child.Kind != yaml.SequenceNode {
|
||||
return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag)
|
||||
}
|
||||
err := e.encodeRow(child.Content)
|
||||
err := e.encodeRow(csvWriter, child.Content)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -13,13 +13,13 @@ func yamlToCsv(sampleYaml string, separator rune) string {
|
||||
var output bytes.Buffer
|
||||
writer := bufio.NewWriter(&output)
|
||||
|
||||
var jsonEncoder = NewCsvEncoder(writer, separator)
|
||||
var jsonEncoder = NewCsvEncoder(separator)
|
||||
inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
node := inputs.Front().Value.(*CandidateNode).Node
|
||||
err = jsonEncoder.Encode(node)
|
||||
err = jsonEncoder.Encode(writer, node)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
64
pkg/yqlib/encoder_json.go
Normal file
64
pkg/yqlib/encoder_json.go
Normal file
@ -0,0 +1,64 @@
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
|
||||
yaml "gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type jsonEncoder struct {
|
||||
indentString string
|
||||
}
|
||||
|
||||
func mapKeysToStrings(node *yaml.Node) {
|
||||
|
||||
if node.Kind == yaml.MappingNode {
|
||||
for index, child := range node.Content {
|
||||
if index%2 == 0 { // its a map key
|
||||
child.Tag = "!!str"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, child := range node.Content {
|
||||
mapKeysToStrings(child)
|
||||
}
|
||||
}
|
||||
|
||||
func NewJsonEncoder(indent int) Encoder {
|
||||
var indentString = ""
|
||||
|
||||
for index := 0; index < indent; index++ {
|
||||
indentString = indentString + " "
|
||||
}
|
||||
|
||||
return &jsonEncoder{indentString}
|
||||
}
|
||||
|
||||
func (je *jsonEncoder) CanHandleAliases() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (je *jsonEncoder) PrintDocumentSeparator(writer io.Writer) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (je *jsonEncoder) PrintLeadingContent(writer io.Writer, content string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (je *jsonEncoder) Encode(writer io.Writer, node *yaml.Node) error {
|
||||
var encoder = json.NewEncoder(writer)
|
||||
encoder.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
|
||||
encoder.SetIndent("", je.indentString)
|
||||
|
||||
var dataBucket orderedMap
|
||||
// firstly, convert all map keys to strings
|
||||
mapKeysToStrings(node)
|
||||
errorDecoding := node.Decode(&dataBucket)
|
||||
if errorDecoding != nil {
|
||||
return errorDecoding
|
||||
}
|
||||
return encoder.Encode(dataBucket)
|
||||
}
|
@ -1,6 +1,8 @@
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
@ -10,14 +12,54 @@ import (
|
||||
)
|
||||
|
||||
type propertiesEncoder struct {
|
||||
destination io.Writer
|
||||
}
|
||||
|
||||
func NewPropertiesEncoder(destination io.Writer) Encoder {
|
||||
return &propertiesEncoder{destination}
|
||||
func NewPropertiesEncoder() Encoder {
|
||||
return &propertiesEncoder{}
|
||||
}
|
||||
|
||||
func (pe *propertiesEncoder) Encode(node *yaml.Node) error {
|
||||
func (pe *propertiesEncoder) CanHandleAliases() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (pe *propertiesEncoder) PrintDocumentSeparator(writer io.Writer) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content string) error {
|
||||
reader := bufio.NewReader(strings.NewReader(content))
|
||||
for {
|
||||
|
||||
readline, errReading := reader.ReadString('\n')
|
||||
if errReading != nil && !errors.Is(errReading, io.EOF) {
|
||||
return errReading
|
||||
}
|
||||
if strings.Contains(readline, "$yqDocSeperator$") {
|
||||
|
||||
if err := pe.PrintDocumentSeparator(writer); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
} else {
|
||||
if err := writeString(writer, readline); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if errors.Is(errReading, io.EOF) {
|
||||
if readline != "" {
|
||||
// the last comment we read didn't have a new line, put one in
|
||||
if err := writeString(writer, "\n"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (pe *propertiesEncoder) Encode(writer io.Writer, node *yaml.Node) error {
|
||||
mapKeysToStrings(node)
|
||||
p := properties.NewProperties()
|
||||
err := pe.doEncode(p, node, "")
|
||||
@ -25,14 +67,12 @@ func (pe *propertiesEncoder) Encode(node *yaml.Node) error {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = p.WriteComment(pe.destination, "#", properties.UTF8)
|
||||
_, err = p.WriteComment(writer, "#", properties.UTF8)
|
||||
return err
|
||||
}
|
||||
|
||||
func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yaml.Node, path string) error {
|
||||
p.SetComment(path,
|
||||
strings.Replace(node.HeadComment, "#", "", 1)+
|
||||
strings.Replace(node.LineComment, "#", "", 1))
|
||||
p.SetComment(path, headAndLineComment(node))
|
||||
switch node.Kind {
|
||||
case yaml.ScalarNode:
|
||||
_, _, err := p.Set(path, node.Value)
|
||||
|
@ -13,13 +13,13 @@ func yamlToProps(sampleYaml string) string {
|
||||
var output bytes.Buffer
|
||||
writer := bufio.NewWriter(&output)
|
||||
|
||||
var propsEncoder = NewPropertiesEncoder(writer)
|
||||
var propsEncoder = NewPropertiesEncoder()
|
||||
inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
node := inputs.Front().Value.(*CandidateNode).Node
|
||||
err = propsEncoder.Encode(node)
|
||||
err = propsEncoder.Encode(writer, node)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
@ -13,13 +13,13 @@ func yamlToJson(sampleYaml string, indent int) string {
|
||||
var output bytes.Buffer
|
||||
writer := bufio.NewWriter(&output)
|
||||
|
||||
var jsonEncoder = NewJsonEncoder(writer, indent)
|
||||
var jsonEncoder = NewJsonEncoder(indent)
|
||||
inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
node := inputs.Front().Value.(*CandidateNode).Node
|
||||
err = jsonEncoder.Encode(node)
|
||||
err = jsonEncoder.Encode(writer, node)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
@ -9,89 +9,157 @@ import (
|
||||
yaml "gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
var XmlPreferences = xmlPreferences{AttributePrefix: "+", ContentName: "+content"}
|
||||
|
||||
type xmlEncoder struct {
|
||||
xmlEncoder *xml.Encoder
|
||||
attributePrefix string
|
||||
contentName string
|
||||
indentString string
|
||||
}
|
||||
|
||||
func NewXmlEncoder(writer io.Writer, indent int, attributePrefix string, contentName string) Encoder {
|
||||
encoder := xml.NewEncoder(writer)
|
||||
func NewXmlEncoder(indent int, attributePrefix string, contentName string) Encoder {
|
||||
var indentString = ""
|
||||
|
||||
for index := 0; index < indent; index++ {
|
||||
indentString = indentString + " "
|
||||
}
|
||||
encoder.Indent("", indentString)
|
||||
return &xmlEncoder{encoder, attributePrefix, contentName}
|
||||
}
|
||||
func (e *xmlEncoder) Encode(node *yaml.Node) error {
|
||||
switch node.Kind {
|
||||
case yaml.MappingNode:
|
||||
err := e.encodeTopLevelMap(node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var charData xml.CharData = []byte("\n")
|
||||
err = e.xmlEncoder.EncodeToken(charData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return e.xmlEncoder.Flush()
|
||||
case yaml.DocumentNode:
|
||||
return e.Encode(unwrapDoc(node))
|
||||
case yaml.ScalarNode:
|
||||
var charData xml.CharData = []byte(node.Value)
|
||||
err := e.xmlEncoder.EncodeToken(charData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return e.xmlEncoder.Flush()
|
||||
}
|
||||
return fmt.Errorf("unsupported type %v", node.Tag)
|
||||
return &xmlEncoder{attributePrefix, contentName, indentString}
|
||||
}
|
||||
|
||||
func (e *xmlEncoder) encodeTopLevelMap(node *yaml.Node) error {
|
||||
func (e *xmlEncoder) CanHandleAliases() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (e *xmlEncoder) PrintDocumentSeparator(writer io.Writer) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *xmlEncoder) PrintLeadingContent(writer io.Writer, content string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
|
||||
encoder := xml.NewEncoder(writer)
|
||||
encoder.Indent("", e.indentString)
|
||||
|
||||
switch node.Kind {
|
||||
case yaml.MappingNode:
|
||||
err := e.encodeTopLevelMap(encoder, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
case yaml.DocumentNode:
|
||||
log.Debugf("ENCODING DOCUMENT NODE")
|
||||
err := e.encodeComment(encoder, headAndLineComment(node))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Debugf("OK NOW THE ACTUAL")
|
||||
err = e.encodeTopLevelMap(encoder, unwrapDoc(node))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = e.encodeComment(encoder, footComment(node))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
case yaml.ScalarNode:
|
||||
var charData xml.CharData = []byte(node.Value)
|
||||
err := encoder.EncodeToken(charData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return encoder.Flush()
|
||||
default:
|
||||
return fmt.Errorf("unsupported type %v", node.Tag)
|
||||
}
|
||||
var charData xml.CharData = []byte("\n")
|
||||
return encoder.EncodeToken(charData)
|
||||
|
||||
}
|
||||
|
||||
func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node) error {
|
||||
err := e.encodeComment(encoder, headAndLineComment(node))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for i := 0; i < len(node.Content); i += 2 {
|
||||
key := node.Content[i]
|
||||
value := node.Content[i+1]
|
||||
|
||||
start := xml.StartElement{Name: xml.Name{Local: key.Value}}
|
||||
err := e.doEncode(value, start)
|
||||
log.Debugf("comments of key %v", key.Value)
|
||||
err := e.encodeComment(encoder, headAndLineComment(key))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debugf("recursing")
|
||||
|
||||
err = e.doEncode(encoder, value, start)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = e.encodeComment(encoder, footComment(key))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return e.encodeComment(encoder, footComment(node))
|
||||
}
|
||||
|
||||
func (e *xmlEncoder) doEncode(node *yaml.Node, start xml.StartElement) error {
|
||||
func (e *xmlEncoder) encodeStart(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
|
||||
err := encoder.EncodeToken(start)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return e.encodeComment(encoder, headComment(node))
|
||||
}
|
||||
|
||||
func (e *xmlEncoder) encodeEnd(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
|
||||
err := encoder.EncodeToken(start.End())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return e.encodeComment(encoder, footComment(node))
|
||||
}
|
||||
|
||||
func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
|
||||
switch node.Kind {
|
||||
case yaml.MappingNode:
|
||||
return e.encodeMap(node, start)
|
||||
return e.encodeMap(encoder, node, start)
|
||||
case yaml.SequenceNode:
|
||||
return e.encodeArray(node, start)
|
||||
return e.encodeArray(encoder, node, start)
|
||||
case yaml.ScalarNode:
|
||||
err := e.xmlEncoder.EncodeToken(start)
|
||||
err := e.encodeStart(encoder, node, start)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var charData xml.CharData = []byte(node.Value)
|
||||
err = e.xmlEncoder.EncodeToken(charData)
|
||||
|
||||
err = encoder.EncodeToken(charData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return e.xmlEncoder.EncodeToken(start.End())
|
||||
|
||||
if err = e.encodeComment(encoder, lineComment(node)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return e.encodeEnd(encoder, node, start)
|
||||
}
|
||||
return fmt.Errorf("unsupported type %v", node.Tag)
|
||||
}
|
||||
|
||||
func (e *xmlEncoder) encodeArray(node *yaml.Node, start xml.StartElement) error {
|
||||
for i := 0; i < len(node.Content); i++ {
|
||||
value := node.Content[i]
|
||||
err := e.doEncode(value, start.Copy())
|
||||
func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) error {
|
||||
if commentStr != "" {
|
||||
log.Debugf("encoding comment %v", commentStr)
|
||||
if !strings.HasSuffix(commentStr, " ") {
|
||||
commentStr = commentStr + " "
|
||||
}
|
||||
|
||||
var comment xml.Comment = []byte(commentStr)
|
||||
err := encoder.EncodeToken(comment)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -99,7 +167,23 @@ func (e *xmlEncoder) encodeArray(node *yaml.Node, start xml.StartElement) error
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *xmlEncoder) encodeMap(node *yaml.Node, start xml.StartElement) error {
|
||||
func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
|
||||
|
||||
if err := e.encodeComment(encoder, headAndLineComment(node)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for i := 0; i < len(node.Content); i++ {
|
||||
value := node.Content[i]
|
||||
if err := e.doEncode(encoder, value, start.Copy()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return e.encodeComment(encoder, footComment(node))
|
||||
}
|
||||
|
||||
func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
|
||||
log.Debug("its a map")
|
||||
|
||||
//first find all the attributes and put them on the start token
|
||||
for i := 0; i < len(node.Content); i += 2 {
|
||||
@ -116,7 +200,7 @@ func (e *xmlEncoder) encodeMap(node *yaml.Node, start xml.StartElement) error {
|
||||
}
|
||||
}
|
||||
|
||||
err := e.xmlEncoder.EncodeToken(start)
|
||||
err := e.encodeStart(encoder, node, start)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -126,21 +210,38 @@ func (e *xmlEncoder) encodeMap(node *yaml.Node, start xml.StartElement) error {
|
||||
key := node.Content[i]
|
||||
value := node.Content[i+1]
|
||||
|
||||
err := e.encodeComment(encoder, headAndLineComment(key))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(key.Value, e.attributePrefix) && key.Value != e.contentName {
|
||||
start := xml.StartElement{Name: xml.Name{Local: key.Value}}
|
||||
err := e.doEncode(value, start)
|
||||
err := e.doEncode(encoder, value, start)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else if key.Value == e.contentName {
|
||||
// directly encode the contents
|
||||
err = e.encodeComment(encoder, headAndLineComment(value))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var charData xml.CharData = []byte(value.Value)
|
||||
err = e.xmlEncoder.EncodeToken(charData)
|
||||
err = encoder.EncodeToken(charData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = e.encodeComment(encoder, footComment(value))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
err = e.encodeComment(encoder, footComment(key))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return e.xmlEncoder.EncodeToken(start.End())
|
||||
return e.encodeEnd(encoder, node, start)
|
||||
}
|
||||
|
101
pkg/yqlib/encoder_yaml.go
Normal file
101
pkg/yqlib/encoder_yaml.go
Normal file
@ -0,0 +1,101 @@
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
yaml "gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type yamlEncoder struct {
|
||||
indent int
|
||||
colorise bool
|
||||
printDocSeparators bool
|
||||
unwrapScalar bool
|
||||
}
|
||||
|
||||
func NewYamlEncoder(indent int, colorise bool, printDocSeparators bool, unwrapScalar bool) Encoder {
|
||||
if indent < 0 {
|
||||
indent = 0
|
||||
}
|
||||
return &yamlEncoder{indent, colorise, printDocSeparators, unwrapScalar}
|
||||
}
|
||||
|
||||
func (ye *yamlEncoder) CanHandleAliases() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (ye *yamlEncoder) PrintDocumentSeparator(writer io.Writer) error {
|
||||
if ye.printDocSeparators {
|
||||
log.Debug("-- writing doc sep")
|
||||
if err := writeString(writer, "---\n"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) error {
|
||||
// log.Debug("headcommentwas [%v]", content)
|
||||
reader := bufio.NewReader(strings.NewReader(content))
|
||||
|
||||
for {
|
||||
|
||||
readline, errReading := reader.ReadString('\n')
|
||||
if errReading != nil && !errors.Is(errReading, io.EOF) {
|
||||
return errReading
|
||||
}
|
||||
if strings.Contains(readline, "$yqDocSeperator$") {
|
||||
|
||||
if err := ye.PrintDocumentSeparator(writer); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
} else {
|
||||
if err := writeString(writer, readline); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if errors.Is(errReading, io.EOF) {
|
||||
if readline != "" {
|
||||
// the last comment we read didn't have a new line, put one in
|
||||
if err := writeString(writer, "\n"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ye *yamlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
|
||||
|
||||
if node.Kind == yaml.ScalarNode && ye.unwrapScalar {
|
||||
return writeString(writer, node.Value+"\n")
|
||||
}
|
||||
|
||||
destination := writer
|
||||
tempBuffer := bytes.NewBuffer(nil)
|
||||
if ye.colorise {
|
||||
destination = tempBuffer
|
||||
}
|
||||
|
||||
var encoder = yaml.NewEncoder(destination)
|
||||
|
||||
encoder.SetIndent(ye.indent)
|
||||
|
||||
if err := encoder.Encode(node); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if ye.colorise {
|
||||
return colorizeAndPrint(tempBuffer.Bytes(), writer)
|
||||
}
|
||||
return nil
|
||||
}
|
@ -18,8 +18,6 @@ type xmlPreferences struct {
|
||||
ContentName string
|
||||
}
|
||||
|
||||
var XmlPreferences = xmlPreferences{AttributePrefix: "+", ContentName: "+content"}
|
||||
|
||||
var log = logging.MustGetLogger("yq-lib")
|
||||
|
||||
// GetLogger returns the yq logger instance.
|
||||
@ -236,6 +234,22 @@ func createScalarNode(value interface{}, stringValue string) *yaml.Node {
|
||||
return node
|
||||
}
|
||||
|
||||
func headAndLineComment(node *yaml.Node) string {
|
||||
return headComment(node) + lineComment(node)
|
||||
}
|
||||
|
||||
func headComment(node *yaml.Node) string {
|
||||
return strings.Replace(node.HeadComment, "#", "", 1)
|
||||
}
|
||||
|
||||
func lineComment(node *yaml.Node) string {
|
||||
return strings.Replace(node.LineComment, "#", "", 1)
|
||||
}
|
||||
|
||||
func footComment(node *yaml.Node) string {
|
||||
return strings.Replace(node.FootComment, "#", "", 1)
|
||||
}
|
||||
|
||||
func createValueOperation(value interface{}, stringValue string) *Operation {
|
||||
var node *yaml.Node = createScalarNode(value, stringValue)
|
||||
|
||||
|
@ -86,7 +86,8 @@ func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *
|
||||
var chompRegexp = regexp.MustCompile(`\n$`)
|
||||
var output bytes.Buffer
|
||||
var writer = bufio.NewWriter(&output)
|
||||
if err := processLeadingContent(candidate, writer, false, YamlOutputFormat); err != nil {
|
||||
var encoder = NewYamlEncoder(2, false, false, false)
|
||||
if err := encoder.PrintLeadingContent(writer, candidate.LeadingContent); err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
if err := writer.Flush(); err != nil {
|
||||
|
@ -10,11 +10,31 @@ import (
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func yamlToString(candidate *CandidateNode, prefs encoderPreferences) (string, error) {
|
||||
func configureEncoder(format PrinterOutputFormat, indent int) Encoder {
|
||||
switch format {
|
||||
case JsonOutputFormat:
|
||||
return NewJsonEncoder(indent)
|
||||
case PropsOutputFormat:
|
||||
return NewPropertiesEncoder()
|
||||
case CsvOutputFormat:
|
||||
return NewCsvEncoder(',')
|
||||
case TsvOutputFormat:
|
||||
return NewCsvEncoder('\t')
|
||||
case YamlOutputFormat:
|
||||
return NewYamlEncoder(indent, false, true, true)
|
||||
case XmlOutputFormat:
|
||||
return NewXmlEncoder(indent, XmlPreferences.AttributePrefix, XmlPreferences.ContentName)
|
||||
}
|
||||
panic("invalid encoder")
|
||||
}
|
||||
|
||||
func encodeToString(candidate *CandidateNode, prefs encoderPreferences) (string, error) {
|
||||
var output bytes.Buffer
|
||||
log.Debug("printing with indent: %v", prefs.indent)
|
||||
|
||||
printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), prefs.format, true, false, prefs.indent, true)
|
||||
encoder := configureEncoder(prefs.format, prefs.indent)
|
||||
|
||||
printer := NewPrinter(encoder, NewSinglePrinterWriter(bufio.NewWriter(&output)))
|
||||
err := printer.PrintResults(candidate.AsList())
|
||||
return output.String(), err
|
||||
}
|
||||
@ -36,7 +56,7 @@ func encodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
|
||||
|
||||
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
|
||||
candidate := el.Value.(*CandidateNode)
|
||||
stringValue, err := yamlToString(candidate, preferences)
|
||||
stringValue, err := encodeToString(candidate, preferences)
|
||||
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
|
@ -26,6 +26,10 @@ type expressionScenario struct {
|
||||
dontFormatInputForDoc bool // dont format input doc for documentation generation
|
||||
}
|
||||
|
||||
func NewSimpleYamlPrinter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
|
||||
return NewPrinter(NewYamlEncoder(indent, colorsEnabled, printDocSeparators, unwrapScalar), NewSinglePrinterWriter(writer))
|
||||
}
|
||||
|
||||
func readDocumentWithLeadingContent(content string, fakefilename string, fakeFileIndex int) (*list.List, error) {
|
||||
reader, firstFileLeadingContent, err := processReadStream(bufio.NewReader(strings.NewReader(content)))
|
||||
if err != nil {
|
||||
@ -92,7 +96,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
|
||||
|
||||
func resultToString(t *testing.T, n *CandidateNode) string {
|
||||
var valueBuffer bytes.Buffer
|
||||
printer := NewPrinterWithSingleWriter(bufio.NewWriter(&valueBuffer), YamlOutputFormat, true, false, 4, true)
|
||||
printer := NewSimpleYamlPrinter(bufio.NewWriter(&valueBuffer), YamlOutputFormat, true, false, 4, true)
|
||||
|
||||
err := printer.PrintResults(n.AsList())
|
||||
if err != nil {
|
||||
@ -145,7 +149,7 @@ func copyFromHeader(folder string, title string, out *os.File) error {
|
||||
|
||||
func formatYaml(yaml string, filename string) string {
|
||||
var output bytes.Buffer
|
||||
printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
|
||||
printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
|
||||
|
||||
node, err := NewExpressionParser().ParseExpression(".. style= \"\"")
|
||||
if err != nil {
|
||||
@ -268,7 +272,7 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) {
|
||||
func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formattedDoc string, formattedDoc2 string) {
|
||||
var output bytes.Buffer
|
||||
var err error
|
||||
printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
|
||||
printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
|
||||
|
||||
node, err := NewExpressionParser().ParseExpression(s.expression)
|
||||
if err != nil {
|
||||
|
@ -48,34 +48,22 @@ func OutputFormatFromString(format string) (PrinterOutputFormat, error) {
|
||||
}
|
||||
|
||||
type resultsPrinter struct {
|
||||
outputFormat PrinterOutputFormat
|
||||
unwrapScalar bool
|
||||
colorsEnabled bool
|
||||
indent int
|
||||
printDocSeparators bool
|
||||
printerWriter PrinterWriter
|
||||
firstTimePrinting bool
|
||||
previousDocIndex uint
|
||||
previousFileIndex int
|
||||
printedMatches bool
|
||||
treeNavigator DataTreeNavigator
|
||||
appendixReader io.Reader
|
||||
encoder Encoder
|
||||
printerWriter PrinterWriter
|
||||
firstTimePrinting bool
|
||||
previousDocIndex uint
|
||||
previousFileIndex int
|
||||
printedMatches bool
|
||||
treeNavigator DataTreeNavigator
|
||||
appendixReader io.Reader
|
||||
}
|
||||
|
||||
func NewPrinterWithSingleWriter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
|
||||
return NewPrinter(NewSinglePrinterWriter(writer), outputFormat, unwrapScalar, colorsEnabled, indent, printDocSeparators)
|
||||
}
|
||||
|
||||
func NewPrinter(printerWriter PrinterWriter, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
|
||||
func NewPrinter(encoder Encoder, printerWriter PrinterWriter) Printer {
|
||||
return &resultsPrinter{
|
||||
printerWriter: printerWriter,
|
||||
outputFormat: outputFormat,
|
||||
unwrapScalar: unwrapScalar,
|
||||
colorsEnabled: colorsEnabled,
|
||||
indent: indent,
|
||||
printDocSeparators: outputFormat == YamlOutputFormat && printDocSeparators,
|
||||
firstTimePrinting: true,
|
||||
treeNavigator: NewDataTreeNavigator(),
|
||||
encoder: encoder,
|
||||
printerWriter: printerWriter,
|
||||
firstTimePrinting: true,
|
||||
treeNavigator: NewDataTreeNavigator(),
|
||||
}
|
||||
}
|
||||
|
||||
@ -90,28 +78,7 @@ func (p *resultsPrinter) PrintedAnything() bool {
|
||||
func (p *resultsPrinter) printNode(node *yaml.Node, writer io.Writer) error {
|
||||
p.printedMatches = p.printedMatches || (node.Tag != "!!null" &&
|
||||
(node.Tag != "!!bool" || node.Value != "false"))
|
||||
|
||||
var encoder Encoder
|
||||
if node.Kind == yaml.ScalarNode && p.unwrapScalar && p.outputFormat == YamlOutputFormat {
|
||||
return writeString(writer, node.Value+"\n")
|
||||
}
|
||||
|
||||
switch p.outputFormat {
|
||||
case JsonOutputFormat:
|
||||
encoder = NewJsonEncoder(writer, p.indent)
|
||||
case PropsOutputFormat:
|
||||
encoder = NewPropertiesEncoder(writer)
|
||||
case CsvOutputFormat:
|
||||
encoder = NewCsvEncoder(writer, ',')
|
||||
case TsvOutputFormat:
|
||||
encoder = NewCsvEncoder(writer, '\t')
|
||||
case YamlOutputFormat:
|
||||
encoder = NewYamlEncoder(writer, p.indent, p.colorsEnabled)
|
||||
case XmlOutputFormat:
|
||||
encoder = NewXmlEncoder(writer, p.indent, XmlPreferences.AttributePrefix, XmlPreferences.ContentName)
|
||||
}
|
||||
|
||||
return encoder.Encode(node)
|
||||
return p.encoder.Encode(writer, node)
|
||||
}
|
||||
|
||||
func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
|
||||
@ -122,7 +89,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
if p.outputFormat != YamlOutputFormat {
|
||||
if !p.encoder.CanHandleAliases() {
|
||||
explodeOp := Operation{OperationType: explodeOpType}
|
||||
explodeNode := ExpressionNode{Operation: &explodeOp}
|
||||
context, err := p.treeNavigator.GetMatchingNodes(Context{MatchingNodes: matchingNodes}, &explodeNode)
|
||||
@ -142,7 +109,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
|
||||
for el := matchingNodes.Front(); el != nil; el = el.Next() {
|
||||
|
||||
mappedDoc := el.Value.(*CandidateNode)
|
||||
log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v, printDocSeparators: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document, p.printDocSeparators)
|
||||
log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document)
|
||||
|
||||
writer, errorWriting := p.printerWriter.GetWriter(mappedDoc)
|
||||
if errorWriting != nil {
|
||||
@ -152,14 +119,13 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
|
||||
commentsStartWithSepExp := regexp.MustCompile(`^\$yqDocSeperator\$`)
|
||||
commentStartsWithSeparator := commentsStartWithSepExp.MatchString(mappedDoc.LeadingContent)
|
||||
|
||||
if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators && !commentStartsWithSeparator {
|
||||
log.Debug("-- writing doc sep")
|
||||
if err := writeString(writer, "---\n"); err != nil {
|
||||
if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && !commentStartsWithSeparator {
|
||||
if err := p.encoder.PrintDocumentSeparator(writer); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := processLeadingContent(mappedDoc, writer, p.printDocSeparators, p.outputFormat); err != nil {
|
||||
if err := p.encoder.PrintLeadingContent(writer, mappedDoc.LeadingContent); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -171,9 +137,11 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
|
||||
if err := writer.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Debugf("done printing results")
|
||||
}
|
||||
|
||||
if p.appendixReader != nil && p.outputFormat == YamlOutputFormat {
|
||||
// what happens if I remove output format check?
|
||||
if p.appendixReader != nil {
|
||||
writer, err := p.printerWriter.GetWriter(nil)
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -33,10 +33,10 @@ func nodeToList(candidate *CandidateNode) *list.List {
|
||||
return elMap
|
||||
}
|
||||
|
||||
func TestPrinterMultipleDocsInSequence(t *testing.T) {
|
||||
func TestPrinterMultipleDocsInSequenceOnly(t *testing.T) {
|
||||
var output bytes.Buffer
|
||||
var writer = bufio.NewWriter(&output)
|
||||
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
|
||||
printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
|
||||
|
||||
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
|
||||
if err != nil {
|
||||
@ -74,7 +74,7 @@ func TestPrinterMultipleDocsInSequence(t *testing.T) {
|
||||
func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
|
||||
var output bytes.Buffer
|
||||
var writer = bufio.NewWriter(&output)
|
||||
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
|
||||
printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
|
||||
|
||||
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
|
||||
if err != nil {
|
||||
@ -116,7 +116,7 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
|
||||
func TestPrinterMultipleFilesInSequence(t *testing.T) {
|
||||
var output bytes.Buffer
|
||||
var writer = bufio.NewWriter(&output)
|
||||
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
|
||||
printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
|
||||
|
||||
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
|
||||
if err != nil {
|
||||
@ -163,7 +163,7 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) {
|
||||
func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
|
||||
var output bytes.Buffer
|
||||
var writer = bufio.NewWriter(&output)
|
||||
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
|
||||
printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
|
||||
|
||||
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
|
||||
if err != nil {
|
||||
@ -213,7 +213,7 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
|
||||
func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
|
||||
var output bytes.Buffer
|
||||
var writer = bufio.NewWriter(&output)
|
||||
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
|
||||
printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
|
||||
|
||||
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
|
||||
if err != nil {
|
||||
@ -232,7 +232,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
|
||||
func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) {
|
||||
var output bytes.Buffer
|
||||
var writer = bufio.NewWriter(&output)
|
||||
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
|
||||
printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
|
||||
|
||||
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
|
||||
if err != nil {
|
||||
@ -261,7 +261,7 @@ a: coconut
|
||||
func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) {
|
||||
var output bytes.Buffer
|
||||
var writer = bufio.NewWriter(&output)
|
||||
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
|
||||
printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
|
||||
|
||||
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
|
||||
if err != nil {
|
||||
@ -287,7 +287,7 @@ a: coconut
|
||||
func TestPrinterScalarWithLeadingCont(t *testing.T) {
|
||||
var output bytes.Buffer
|
||||
var writer = bufio.NewWriter(&output)
|
||||
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
|
||||
printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
|
||||
|
||||
node, err := NewExpressionParser().ParseExpression(".a")
|
||||
if err != nil {
|
||||
@ -314,7 +314,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
|
||||
var writer = bufio.NewWriter(&output)
|
||||
// note printDocSeparators is true, it should still not print document separators
|
||||
// when outputing JSON.
|
||||
printer := NewPrinterWithSingleWriter(writer, JsonOutputFormat, true, false, 0, true)
|
||||
printer := NewPrinter(NewJsonEncoder(0), NewSinglePrinterWriter(writer))
|
||||
|
||||
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
|
||||
if err != nil {
|
||||
|
@ -38,43 +38,6 @@ func writeString(writer io.Writer, txt string) error {
|
||||
return errorWriting
|
||||
}
|
||||
|
||||
func processLeadingContent(mappedDoc *CandidateNode, writer io.Writer, printDocSeparators bool, outputFormat PrinterOutputFormat) error {
|
||||
log.Debug("headcommentwas %v", mappedDoc.LeadingContent)
|
||||
log.Debug("finished headcomment")
|
||||
reader := bufio.NewReader(strings.NewReader(mappedDoc.LeadingContent))
|
||||
|
||||
for {
|
||||
|
||||
readline, errReading := reader.ReadString('\n')
|
||||
if errReading != nil && !errors.Is(errReading, io.EOF) {
|
||||
return errReading
|
||||
}
|
||||
if strings.Contains(readline, "$yqDocSeperator$") {
|
||||
if printDocSeparators {
|
||||
if err := writeString(writer, "---\n"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
} else if outputFormat == YamlOutputFormat {
|
||||
if err := writeString(writer, readline); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if errors.Is(errReading, io.EOF) {
|
||||
if readline != "" {
|
||||
// the last comment we read didn't have a new line, put one in
|
||||
if err := writeString(writer, "\n"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func processReadStream(reader *bufio.Reader) (io.Reader, string, error) {
|
||||
var commentLineRegEx = regexp.MustCompile(`^\s*#`)
|
||||
var sb strings.Builder
|
||||
|
@ -24,23 +24,29 @@ func decodeXml(t *testing.T, xml string) *CandidateNode {
|
||||
return &CandidateNode{Node: node}
|
||||
}
|
||||
|
||||
func yamlToXml(sampleYaml string, indent int) string {
|
||||
func processScenario(s xmlScenario) string {
|
||||
var output bytes.Buffer
|
||||
writer := bufio.NewWriter(&output)
|
||||
|
||||
var encoder = NewXmlEncoder(writer, indent, "+", "+content")
|
||||
inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
|
||||
var encoder = NewXmlEncoder(2, "+", "+content")
|
||||
|
||||
var decoder = NewYamlDecoder()
|
||||
if s.scenarioType == "roundtrip" {
|
||||
decoder = NewXmlDecoder("+", "+content")
|
||||
}
|
||||
|
||||
inputs, err := readDocuments(strings.NewReader(s.input), "sample.yml", 0, decoder)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
node := inputs.Front().Value.(*CandidateNode).Node
|
||||
err = encoder.Encode(node)
|
||||
err = encoder.Encode(writer, node)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
writer.Flush()
|
||||
|
||||
return strings.TrimSuffix(output.String(), "\n")
|
||||
return output.String()
|
||||
}
|
||||
|
||||
type xmlScenario struct {
|
||||
@ -49,9 +55,154 @@ type xmlScenario struct {
|
||||
description string
|
||||
subdescription string
|
||||
skipDoc bool
|
||||
encodeScenario bool
|
||||
scenarioType string
|
||||
}
|
||||
|
||||
var inputXmlWithComments = `
|
||||
<!-- before cat -->
|
||||
<cat>
|
||||
<!-- in cat before -->
|
||||
<x>3<!-- multi
|
||||
line comment
|
||||
for x --></x>
|
||||
<!-- before y -->
|
||||
<y>
|
||||
<!-- in y before -->
|
||||
<d><!-- in d before -->z<!-- in d after --></d>
|
||||
|
||||
<!-- in y after -->
|
||||
</y>
|
||||
<!-- in_cat_after -->
|
||||
</cat>
|
||||
<!-- after cat -->
|
||||
`
|
||||
var inputXmlWithCommentsWithSubChild = `
|
||||
<!-- before cat -->
|
||||
<cat>
|
||||
<!-- in cat before -->
|
||||
<x>3<!-- multi
|
||||
line comment
|
||||
for x --></x>
|
||||
<!-- before y -->
|
||||
<y>
|
||||
<!-- in y before -->
|
||||
<d><!-- in d before --><z sweet="cool"/><!-- in d after --></d>
|
||||
|
||||
<!-- in y after -->
|
||||
</y>
|
||||
<!-- in_cat_after -->
|
||||
</cat>
|
||||
<!-- after cat -->
|
||||
`
|
||||
|
||||
var expectedDecodeYamlWithSubChild = `D0, P[], (doc)::# before cat
|
||||
cat:
|
||||
# in cat before
|
||||
x: "3" # multi
|
||||
# line comment
|
||||
# for x
|
||||
# before y
|
||||
|
||||
y:
|
||||
# in y before
|
||||
d:
|
||||
# in d before
|
||||
z:
|
||||
+sweet: cool
|
||||
# in d after
|
||||
# in y after
|
||||
# in_cat_after
|
||||
# after cat
|
||||
`
|
||||
|
||||
var inputXmlWithCommentsWithArray = `
|
||||
<!-- before cat -->
|
||||
<cat>
|
||||
<!-- in cat before -->
|
||||
<x>3<!-- multi
|
||||
line comment
|
||||
for x --></x>
|
||||
<!-- before y -->
|
||||
<y>
|
||||
<!-- in y before -->
|
||||
<d><!-- in d before --><z sweet="cool"/><!-- in d after --></d>
|
||||
<d><!-- in d2 before --><z sweet="cool2"/><!-- in d2 after --></d>
|
||||
|
||||
<!-- in y after -->
|
||||
</y>
|
||||
<!-- in_cat_after -->
|
||||
</cat>
|
||||
<!-- after cat -->
|
||||
`
|
||||
|
||||
var expectedDecodeYamlWithArray = `D0, P[], (doc)::# before cat
|
||||
cat:
|
||||
# in cat before
|
||||
x: "3" # multi
|
||||
# line comment
|
||||
# for x
|
||||
# before y
|
||||
|
||||
y:
|
||||
# in y before
|
||||
d:
|
||||
- # in d before
|
||||
z:
|
||||
+sweet: cool
|
||||
# in d after
|
||||
- # in d2 before
|
||||
z:
|
||||
+sweet: cool2
|
||||
# in d2 after
|
||||
# in y after
|
||||
# in_cat_after
|
||||
# after cat
|
||||
`
|
||||
|
||||
var expectedDecodeYamlWithComments = `D0, P[], (doc)::# before cat
|
||||
cat:
|
||||
# in cat before
|
||||
x: "3" # multi
|
||||
# line comment
|
||||
# for x
|
||||
# before y
|
||||
|
||||
y:
|
||||
# in y before
|
||||
# in d before
|
||||
d: z # in d after
|
||||
# in y after
|
||||
# in_cat_after
|
||||
# after cat
|
||||
`
|
||||
|
||||
var expectedRoundtripXmlWithComments = `<!-- before cat --><cat><!-- in cat before -->
|
||||
<x>3<!-- multi
|
||||
line comment
|
||||
for x --></x><!-- before y -->
|
||||
<y><!-- in y before
|
||||
in d before -->
|
||||
<d>z<!-- in d after --></d><!-- in y after -->
|
||||
</y><!-- in_cat_after -->
|
||||
</cat><!-- after cat -->
|
||||
`
|
||||
|
||||
var yamlWithComments = `# above_cat
|
||||
cat: # inline_cat
|
||||
# above_array
|
||||
array: # inline_array
|
||||
- val1 # inline_val1
|
||||
# above_val2
|
||||
- val2 # inline_val2
|
||||
# below_cat
|
||||
`
|
||||
|
||||
var expectedXmlWithComments = `<!-- above_cat inline_cat --><cat><!-- above_array inline_array -->
|
||||
<array>val1<!-- inline_val1 --></array>
|
||||
<array><!-- above_val2 -->val2<!-- inline_val2 --></array>
|
||||
</cat><!-- below_cat -->
|
||||
`
|
||||
|
||||
var xmlScenarios = []xmlScenario{
|
||||
{
|
||||
description: "Parse xml: simple",
|
||||
@ -66,53 +217,88 @@ var xmlScenarios = []xmlScenario{
|
||||
},
|
||||
{
|
||||
description: "Parse xml: attributes",
|
||||
subdescription: "Attributes are converted to fields, with the attribute prefix.",
|
||||
subdescription: "Attributes are converted to fields, with the default attribute prefix '+'. Use '--xml-attribute-prefix` to set your own.",
|
||||
input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">\n <legs>7</legs>\n</cat>",
|
||||
expected: "D0, P[], (doc)::cat:\n +legs: \"4\"\n legs: \"7\"\n",
|
||||
},
|
||||
{
|
||||
description: "Parse xml: attributes with content",
|
||||
subdescription: "Content is added as a field, using the content name",
|
||||
subdescription: "Content is added as a field, using the default content name of '+content'. Use `--xml-content-name` to set your own.",
|
||||
input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">meow</cat>",
|
||||
expected: "D0, P[], (doc)::cat:\n +content: meow\n +legs: \"4\"\n",
|
||||
},
|
||||
{
|
||||
description: "Encode xml: simple",
|
||||
input: "cat: purrs",
|
||||
expected: "<cat>purrs</cat>",
|
||||
encodeScenario: true,
|
||||
description: "Parse xml: with comments",
|
||||
subdescription: "A best attempt is made to preserve comments.",
|
||||
input: inputXmlWithComments,
|
||||
expected: expectedDecodeYamlWithComments,
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "Encode xml: array",
|
||||
input: "pets:\n cat:\n - purrs\n - meows",
|
||||
expected: "<pets>\n <cat>purrs</cat>\n <cat>meows</cat>\n</pets>",
|
||||
encodeScenario: true,
|
||||
description: "Parse xml: with comments subchild",
|
||||
skipDoc: true,
|
||||
input: inputXmlWithCommentsWithSubChild,
|
||||
expected: expectedDecodeYamlWithSubChild,
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "Parse xml: with comments array",
|
||||
skipDoc: true,
|
||||
input: inputXmlWithCommentsWithArray,
|
||||
expected: expectedDecodeYamlWithArray,
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "Encode xml: simple",
|
||||
input: "cat: purrs",
|
||||
expected: "<cat>purrs</cat>\n",
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
description: "Encode xml: array",
|
||||
input: "pets:\n cat:\n - purrs\n - meows",
|
||||
expected: "<pets>\n <cat>purrs</cat>\n <cat>meows</cat>\n</pets>\n",
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
description: "Encode xml: attributes",
|
||||
subdescription: "Fields with the matching xml-attribute-prefix are assumed to be attributes.",
|
||||
input: "cat:\n +name: tiger\n meows: true\n",
|
||||
expected: "<cat name=\"tiger\">\n <meows>true</meows>\n</cat>",
|
||||
encodeScenario: true,
|
||||
expected: "<cat name=\"tiger\">\n <meows>true</meows>\n</cat>\n",
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
input: "cat:\n ++name: tiger\n meows: true\n",
|
||||
expected: "<cat +name=\"tiger\">\n <meows>true</meows>\n</cat>",
|
||||
encodeScenario: true,
|
||||
skipDoc: true,
|
||||
input: "cat:\n ++name: tiger\n meows: true\n",
|
||||
expected: "<cat +name=\"tiger\">\n <meows>true</meows>\n</cat>\n",
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
description: "Encode xml: attributes with content",
|
||||
subdescription: "Fields with the matching xml-content-name is assumed to be content.",
|
||||
input: "cat:\n +name: tiger\n +content: cool\n",
|
||||
expected: "<cat name=\"tiger\">cool</cat>",
|
||||
encodeScenario: true,
|
||||
expected: "<cat name=\"tiger\">cool</cat>\n",
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
description: "Encode xml: comments",
|
||||
subdescription: "A best attempt is made to copy comments to xml.",
|
||||
input: yamlWithComments,
|
||||
expected: expectedXmlWithComments,
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
description: "Round trip: with comments",
|
||||
subdescription: "A best effort is made, but comment positions and white space are not preserved perfectly.",
|
||||
input: inputXmlWithComments,
|
||||
expected: expectedRoundtripXmlWithComments,
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
}
|
||||
|
||||
func testXmlScenario(t *testing.T, s *xmlScenario) {
|
||||
if s.encodeScenario {
|
||||
test.AssertResultWithContext(t, s.expected, yamlToXml(s.input, 2), s.description)
|
||||
func testXmlScenario(t *testing.T, s xmlScenario) {
|
||||
if s.scenarioType == "encode" || s.scenarioType == "roundtrip" {
|
||||
test.AssertResultWithContext(t, s.expected, processScenario(s), s.description)
|
||||
} else {
|
||||
var actual = resultToString(t, decodeXml(t, s.input))
|
||||
test.AssertResultWithContext(t, s.expected, actual, s.description)
|
||||
@ -125,8 +311,10 @@ func documentXmlScenario(t *testing.T, w *bufio.Writer, i interface{}) {
|
||||
if s.skipDoc {
|
||||
return
|
||||
}
|
||||
if s.encodeScenario {
|
||||
if s.scenarioType == "encode" {
|
||||
documentXmlEncodeScenario(w, s)
|
||||
} else if s.scenarioType == "roundtrip" {
|
||||
documentXmlRoundTripScenario(w, s)
|
||||
} else {
|
||||
documentXmlDecodeScenario(t, w, s)
|
||||
}
|
||||
@ -149,7 +337,7 @@ func documentXmlDecodeScenario(t *testing.T, w *bufio.Writer, s xmlScenario) {
|
||||
writeOrPanic(w, "will output\n")
|
||||
|
||||
var output bytes.Buffer
|
||||
printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
|
||||
printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
|
||||
|
||||
node := decodeXml(t, s.input)
|
||||
|
||||
@ -177,12 +365,30 @@ func documentXmlEncodeScenario(w *bufio.Writer, s xmlScenario) {
|
||||
writeOrPanic(w, "```bash\nyq e -o=xml '.' sample.yml\n```\n")
|
||||
writeOrPanic(w, "will output\n")
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", yamlToXml(s.input, 2)))
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processScenario(s)))
|
||||
}
|
||||
|
||||
func documentXmlRoundTripScenario(w *bufio.Writer, s xmlScenario) {
|
||||
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
|
||||
|
||||
if s.subdescription != "" {
|
||||
writeOrPanic(w, s.subdescription)
|
||||
writeOrPanic(w, "\n\n")
|
||||
}
|
||||
|
||||
writeOrPanic(w, "Given a sample.xml file of:\n")
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input))
|
||||
|
||||
writeOrPanic(w, "then\n")
|
||||
writeOrPanic(w, "```bash\nyq e -p=xml -o=xml '.' sample.xml\n```\n")
|
||||
writeOrPanic(w, "will output\n")
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processScenario(s)))
|
||||
}
|
||||
|
||||
func TestXmlScenarios(t *testing.T) {
|
||||
for _, tt := range xmlScenarios {
|
||||
testXmlScenario(t, &tt)
|
||||
testXmlScenario(t, tt)
|
||||
}
|
||||
genericScenarios := make([]interface{}, len(xmlScenarios))
|
||||
for i, s := range xmlScenarios {
|
||||
|
@ -1,6 +1,7 @@
|
||||
package test
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
@ -8,6 +9,8 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/pkg/diff"
|
||||
"github.com/pkg/diff/write"
|
||||
"github.com/spf13/cobra"
|
||||
yaml "gopkg.in/yaml.v3"
|
||||
)
|
||||
@ -63,8 +66,14 @@ func AssertResultComplexWithContext(t *testing.T, expectedValue interface{}, act
|
||||
|
||||
func AssertResultWithContext(t *testing.T, expectedValue interface{}, actualValue interface{}, context interface{}) {
|
||||
t.Helper()
|
||||
opts := []write.Option{write.TerminalColor()}
|
||||
if expectedValue != actualValue {
|
||||
t.Error(context)
|
||||
t.Error(": expected <", expectedValue, "> but got <", actualValue, ">")
|
||||
var differenceBuffer bytes.Buffer
|
||||
if err := diff.Text("expected", "actual", expectedValue, actualValue, bufio.NewWriter(&differenceBuffer), opts...); err != nil {
|
||||
t.Error(err)
|
||||
} else {
|
||||
t.Error(differenceBuffer.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user