suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2025-02-28 11:08:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Mike Farah 2021-12-30 15:39:40 +11:00
parent 2c4bd03ec6
commit 277ba1fe5d
23 changed files with 652 additions and 375 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
acceptance_tests/output-format.sh
View File

@ -142,4 +142,24 @@ EOM
assertEquals "$expected" "$X" assertEquals "$expected" "$X"
} }
testOutputXmlMultiPropertiesMultietc() {
cat >test.yml <<EOL
a: {b: {c: ["cat"]}}asd
EOL
read -r -d '' expected << EOM
<a>
<b>
<c>cat</c>
</b>asd
</a>
EOM
X=$(./yq e --output-format=x test.yml)
assertEquals "$expected" "$X"
X=$(./yq ea --output-format=x test.yml)
assertEquals "$expected" "$X"
}
source ./scripts/shunit2 source ./scripts/shunit2

3
cmd/evaluate_all_command.go
View File

@ -81,8 +81,9 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
} }
printerWriter := configurePrinterWriter(format, out) printerWriter := configurePrinterWriter(format, out)
encoder := configureEncoder(format)
printer := yqlib.NewPrinter(printerWriter, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators) printer := yqlib.NewPrinter(encoder, printerWriter)
if frontMatter != "" { if frontMatter != "" {
frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex]) frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])

3
cmd/evalute_sequence_command.go
View File

@ -89,8 +89,9 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
} }
printerWriter := configurePrinterWriter(format, out) printerWriter := configurePrinterWriter(format, out)
encoder := configureEncoder(format)
printer := yqlib.NewPrinter(printerWriter, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators) printer := yqlib.NewPrinter(encoder, printerWriter)
decoder, err := configureDecoder() decoder, err := configureDecoder()
if err != nil { if err != nil {

20
cmd/utils.go
View File

@ -52,7 +52,7 @@ func configureDecoder() (yqlib.Decoder, error) {
} }
switch yqlibInputFormat { switch yqlibInputFormat {
case yqlib.XmlInputFormat: case yqlib.XmlInputFormat:
return yqlib.NewXmlDecoder(xmlAttributePrefix, xmlContentName), nil return yqlib.NewXmlDecoder(), nil
} }
return yqlib.NewYamlDecoder(), nil return yqlib.NewYamlDecoder(), nil
} }
@ -73,3 +73,21 @@ func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) yql
} }
return printerWriter return printerWriter
} }
func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder {
switch format {
case yqlib.JsonOutputFormat:
return yqlib.NewJsonEncoder(indent)
case yqlib.PropsOutputFormat:
return yqlib.NewPropertiesEncoder()
case yqlib.CsvOutputFormat:
return yqlib.NewCsvEncoder(',')
case yqlib.TsvOutputFormat:
return yqlib.NewCsvEncoder('\t')
case yqlib.YamlOutputFormat:
return yqlib.NewYamlEncoder(indent, colorsEnabled, !noDocSeparators, unwrapScalar)
case yqlib.XmlOutputFormat:
return yqlib.NewXmlEncoder(indent)
}
panic("invalid encoder")
}

5
examples/data1.yaml
View File

@ -1,4 +1 @@
#hi cat: purrs
---
# hi
- - fish

218
pkg/yqlib/doc/usage/xml.md
View File

@ -22,6 +22,224 @@ XML nodes that have attributes then plain content, e.g:
The content of the node will be set as a field in the map with the key "+content". Use the `--xml-content-name` flag to change this. The content of the node will be set as a field in the map with the key "+content". Use the `--xml-content-name` flag to change this.
## Parse xml: simple
Given a sample.xml file of:
```xml
<?xml version="1.0" encoding="UTF-8"?>
<cat>meow</cat>
```
then
```bash
yq e -p=xml '.' sample.xml
```
will output
```yaml
cat: meow
```
## Parse xml: array
Consecutive nodes with identical xml names are assumed to be arrays.
Given a sample.xml file of:
```xml
<?xml version="1.0" encoding="UTF-8"?>
<animal>1</animal>
<animal>2</animal>
```
then
```bash
yq e -p=xml '.' sample.xml
```
will output
```yaml
animal:
- "1"
- "2"
```
## Parse xml: attributes
Attributes are converted to fields, with the attribute prefix.
Given a sample.xml file of:
```xml
<?xml version="1.0" encoding="UTF-8"?>
<cat legs="4">
<legs>7</legs>
</cat>
```
then
```bash
yq e -p=xml '.' sample.xml
```
will output
```yaml
cat:
+legs: "4"
legs: "7"
```
## Parse xml: attributes with content
Content is added as a field, using the content name
Given a sample.xml file of:
```xml
<?xml version="1.0" encoding="UTF-8"?>
<cat legs="4">meow</cat>
```
then
```bash
yq e -p=xml '.' sample.xml
```
will output
```yaml
cat:
+content: meow
+legs: "4"
```
## Parse xml: with comments
A best attempt is made to preserve comments.
Given a sample.xml file of:
```xml
<!-- before cat -->
<cat>
<!-- in cat before -->
<x>3<!-- multi
line comment
for x --></x>
<y>
<!-- in y before -->
<d><!-- in d before -->4<!-- in d after --></d>
<!-- in y after -->
</y>
<!-- in_cat_after -->
</cat>
<!-- after cat -->
```
then
```bash
yq e -p=xml '.' sample.xml
```
will output
```yaml
# before cat
cat:
# in cat before
x: "3" # multi
# line comment
# for x
y:
# in y before
d: "4" # in d before in d after
# in y after
# after cat
```
## Encode xml: simple
Given a sample.yml file of:
```yaml
cat: purrs
```
then
```bash
yq e -o=xml '.' sample.yml
```
will output
```xml
<cat>purrs</cat>
```
## Encode xml: array
Given a sample.yml file of:
```yaml
pets:
cat:
- purrs
- meows
```
then
```bash
yq e -o=xml '.' sample.yml
```
will output
```xml
<pets>
<cat>purrs</cat>
<cat>meows</cat>
</pets>
```
## Encode xml: attributes
Fields with the matching xml-attribute-prefix are assumed to be attributes.
Given a sample.yml file of:
```yaml
cat:
+name: tiger
meows: true
```
then
```bash
yq e -o=xml '.' sample.yml
```
will output
```xml
<cat name="tiger">
<meows>true</meows>
</cat>
```
## Encode xml: attributes with content
Fields with the matching xml-content-name is assumed to be content.
Given a sample.yml file of:
```yaml
cat:
+name: tiger
+content: cool
```
then
```bash
yq e -o=xml '.' sample.yml
```
will output
```xml
<cat name="tiger">cool</cat>
```
## Encode xml: comments
A best attempt is made to copy comments to xml.
Given a sample.yml file of:
```yaml
# above_cat
cat: # inline_cat
# above_array
array: # inline_array
- val1 # inline_val1
# above_val2
- val2 # inline_val2
# below_cat
```
then
```bash
yq e -o=xml '.' sample.yml
```
will output
```xml
<!-- above_cat inline_cat--><cat><!-- above_array inline_array-->
<array><!-- inline_val1-->val1</array>
<array><!-- above_val2 inline_val2-->val2</array>
</cat><!-- below_cat-->
```
## Round trip: with comments ## Round trip: with comments
A best effort is made, but comment positions and white space are not preserved perfectly. A best effort is made, but comment positions and white space are not preserved perfectly.

153
pkg/yqlib/encoder.go
View File

@ -1,165 +1,20 @@
package yqlib package yqlib
import ( import (
"bufio"
"bytes" "bytes"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"io" "io"
"strings"
yaml "gopkg.in/yaml.v3" yaml "gopkg.in/yaml.v3"
) )
type Encoder interface { type Encoder interface {
Encode(node *yaml.Node) error Encode(writer io.Writer, node *yaml.Node) error
PrintDocumentSeparator() error PrintDocumentSeparator(writer io.Writer) error
PrintLeadingContent(content string) error PrintLeadingContent(writer io.Writer, content string) error
} CanHandleAliases() bool
type yamlEncoder struct {
destination io.Writer
indent int
colorise bool
firstDoc bool
printDocSeparators bool
unwrapScalar bool
}
func NewYamlEncoder(destination io.Writer, indent int, colorise bool, printDocSeparators bool, unwrapScalar bool) Encoder {
if indent < 0 {
indent = 0
}
return &yamlEncoder{destination, indent, colorise, true, printDocSeparators, unwrapScalar}
}
func (ye *yamlEncoder) PrintDocumentSeparator() error {
if ye.printDocSeparators {
if err := writeString(ye.destination, "---\n"); err != nil {
return err
}
}
return nil
}
func (ye *yamlEncoder) PrintLeadingContent(content string) error {
log.Debug("headcommentwas %v", content)
log.Debug("finished headcomment")
reader := bufio.NewReader(strings.NewReader(content))
for {
readline, errReading := reader.ReadString('\n')
if errReading != nil && !errors.Is(errReading, io.EOF) {
return errReading
}
if strings.Contains(readline, "$yqDocSeperator$") {
if err := ye.PrintDocumentSeparator(); err != nil {
return err
}
} else {
if err := writeString(ye.destination, readline); err != nil {
return err
}
}
if errors.Is(errReading, io.EOF) {
if readline != "" {
// the last comment we read didn't have a new line, put one in
if err := writeString(ye.destination, "\n"); err != nil {
return err
}
}
break
}
}
return nil
}
func (ye *yamlEncoder) Encode(node *yaml.Node) error {
if node.Kind == yaml.ScalarNode && ye.unwrapScalar {
return writeString(ye.destination, node.Value+"\n")
}
destination := ye.destination
tempBuffer := bytes.NewBuffer(nil)
if ye.colorise {
destination = tempBuffer
}
var encoder = yaml.NewEncoder(destination)
encoder.SetIndent(ye.indent)
// TODO: work out if the first doc had a separator or not.
if ye.firstDoc {
ye.firstDoc = false
} else if _, err := destination.Write([]byte("---\n")); err != nil {
return err
}
if err := encoder.Encode(node); err != nil {
return err
}
if ye.colorise {
return colorizeAndPrint(tempBuffer.Bytes(), ye.destination)
}
return nil
}
type jsonEncoder struct {
encoder *json.Encoder
}
func mapKeysToStrings(node *yaml.Node) {
if node.Kind == yaml.MappingNode {
for index, child := range node.Content {
if index%2 == 0 { // its a map key
child.Tag = "!!str"
}
}
}
for _, child := range node.Content {
mapKeysToStrings(child)
}
}
func NewJsonEncoder(destination io.Writer, indent int) Encoder {
var encoder = json.NewEncoder(destination)
encoder.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
var indentString = ""
for index := 0; index < indent; index++ {
indentString = indentString + " "
}
encoder.SetIndent("", indentString)
return &jsonEncoder{encoder}
}
func (je *jsonEncoder) PrintDocumentSeparator() error {
return nil
}
func (je *jsonEncoder) PrintLeadingContent(content string) error {
return nil
}
func (je *jsonEncoder) Encode(node *yaml.Node) error {
var dataBucket orderedMap
// firstly, convert all map keys to strings
mapKeysToStrings(node)
errorDecoding := node.Decode(&dataBucket)
if errorDecoding != nil {
return errorDecoding
}
return je.encoder.Encode(dataBucket)
} }
// orderedMap allows to marshal and unmarshal JSON and YAML values keeping the // orderedMap allows to marshal and unmarshal JSON and YAML values keeping the

29
pkg/yqlib/encoder_csv.go
View File

@ -9,24 +9,26 @@ import (
) )
type csvEncoder struct { type csvEncoder struct {
destination csv.Writer separator rune
} }
func NewCsvEncoder(destination io.Writer, separator rune) Encoder { func NewCsvEncoder(separator rune) Encoder {
csvWriter := *csv.NewWriter(destination) return &csvEncoder{separator}
csvWriter.Comma = separator
return &csvEncoder{csvWriter}
} }
func (e *csvEncoder) PrintDocumentSeparator() error { func (e *csvEncoder) CanHandleAliases() bool {
return false
}
func (e *csvEncoder) PrintDocumentSeparator(writer io.Writer) error {
return nil return nil
} }
func (e *csvEncoder) PrintLeadingContent(content string) error { func (e *csvEncoder) PrintLeadingContent(writer io.Writer, content string) error {
return nil return nil
} }
func (e *csvEncoder) encodeRow(contents []*yaml.Node) error { func (e *csvEncoder) encodeRow(csvWriter *csv.Writer, contents []*yaml.Node) error {
stringValues := make([]string, len(contents)) stringValues := make([]string, len(contents))
for i, child := range contents { for i, child := range contents {
@ -36,10 +38,13 @@ func (e *csvEncoder) encodeRow(contents []*yaml.Node) error {
} }
stringValues[i] = child.Value stringValues[i] = child.Value
} }
return e.destination.Write(stringValues) return csvWriter.Write(stringValues)
} }
func (e *csvEncoder) Encode(originalNode *yaml.Node) error { func (e *csvEncoder) Encode(writer io.Writer, originalNode *yaml.Node) error {
csvWriter := csv.NewWriter(writer)
csvWriter.Comma = e.separator
// node must be a sequence // node must be a sequence
node := unwrapDoc(originalNode) node := unwrapDoc(originalNode)
if node.Kind != yaml.SequenceNode { if node.Kind != yaml.SequenceNode {
@ -48,7 +53,7 @@ func (e *csvEncoder) Encode(originalNode *yaml.Node) error {
return nil return nil
} }
if node.Content[0].Kind == yaml.ScalarNode { if node.Content[0].Kind == yaml.ScalarNode {
return e.encodeRow(node.Content) return e.encodeRow(csvWriter, node.Content)
} }
for i, child := range node.Content { for i, child := range node.Content {
@ -56,7 +61,7 @@ func (e *csvEncoder) Encode(originalNode *yaml.Node) error {
if child.Kind != yaml.SequenceNode { if child.Kind != yaml.SequenceNode {
return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag) return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag)
} }
err := e.encodeRow(child.Content) err := e.encodeRow(csvWriter, child.Content)
if err != nil { if err != nil {
return err return err
} }

4
pkg/yqlib/encoder_csv_test.go
View File

@ -13,13 +13,13 @@ func yamlToCsv(sampleYaml string, separator rune) string {
var output bytes.Buffer var output bytes.Buffer
writer := bufio.NewWriter(&output) writer := bufio.NewWriter(&output)
var jsonEncoder = NewCsvEncoder(writer, separator) var jsonEncoder = NewCsvEncoder(separator)
inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder()) inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
if err != nil { if err != nil {
panic(err) panic(err)
} }
node := inputs.Front().Value.(*CandidateNode).Node node := inputs.Front().Value.(*CandidateNode).Node
err = jsonEncoder.Encode(node) err = jsonEncoder.Encode(writer, node)
if err != nil { if err != nil {
panic(err) panic(err)
} }

64
pkg/yqlib/encoder_json.go Normal file
View File

@ -0,0 +1,64 @@
package yqlib
import (
"encoding/json"
"io"
yaml "gopkg.in/yaml.v3"
)
type jsonEncoder struct {
indentString string
}
func mapKeysToStrings(node *yaml.Node) {
if node.Kind == yaml.MappingNode {
for index, child := range node.Content {
if index%2 == 0 { // its a map key
child.Tag = "!!str"
}
}
}
for _, child := range node.Content {
mapKeysToStrings(child)
}
}
func NewJsonEncoder(indent int) Encoder {
var indentString = ""
for index := 0; index < indent; index++ {
indentString = indentString + " "
}
return &jsonEncoder{indentString}
}
func (je *jsonEncoder) CanHandleAliases() bool {
return false
}
func (je *jsonEncoder) PrintDocumentSeparator(writer io.Writer) error {
return nil
}
func (je *jsonEncoder) PrintLeadingContent(writer io.Writer, content string) error {
return nil
}
func (je *jsonEncoder) Encode(writer io.Writer, node *yaml.Node) error {
var encoder = json.NewEncoder(writer)
encoder.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
encoder.SetIndent("", je.indentString)
var dataBucket orderedMap
// firstly, convert all map keys to strings
mapKeysToStrings(node)
errorDecoding := node.Decode(&dataBucket)
if errorDecoding != nil {
return errorDecoding
}
return encoder.Encode(dataBucket)
}

23
pkg/yqlib/encoder_properties.go
View File

@ -12,18 +12,21 @@ import (
) )
type propertiesEncoder struct { type propertiesEncoder struct {
destination io.Writer
} }
func NewPropertiesEncoder(destination io.Writer) Encoder { func NewPropertiesEncoder() Encoder {
return &propertiesEncoder{destination} return &propertiesEncoder{}
} }
func (e *propertiesEncoder) PrintDocumentSeparator() error { func (pe *propertiesEncoder) CanHandleAliases() bool {
return false
}
func (pe *propertiesEncoder) PrintDocumentSeparator(writer io.Writer) error {
return nil return nil
} }
func (e *propertiesEncoder) PrintLeadingContent(content string) error { func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content string) error {
reader := bufio.NewReader(strings.NewReader(content)) reader := bufio.NewReader(strings.NewReader(content))
for { for {
@ -33,12 +36,12 @@ func (e *propertiesEncoder) PrintLeadingContent(content string) error {
} }
if strings.Contains(readline, "$yqDocSeperator$") { if strings.Contains(readline, "$yqDocSeperator$") {
if err := e.PrintDocumentSeparator(); err != nil { if err := pe.PrintDocumentSeparator(writer); err != nil {
return err return err
} }
} else { } else {
if err := writeString(e.destination, readline); err != nil { if err := writeString(writer, readline); err != nil {
return err return err
} }
} }
@ -46,7 +49,7 @@ func (e *propertiesEncoder) PrintLeadingContent(content string) error {
if errors.Is(errReading, io.EOF) { if errors.Is(errReading, io.EOF) {
if readline != "" { if readline != "" {
// the last comment we read didn't have a new line, put one in // the last comment we read didn't have a new line, put one in
if err := writeString(e.destination, "\n"); err != nil { if err := writeString(writer, "\n"); err != nil {
return err return err
} }
} }
@ -56,7 +59,7 @@ func (e *propertiesEncoder) PrintLeadingContent(content string) error {
return nil return nil
} }
func (pe *propertiesEncoder) Encode(node *yaml.Node) error { func (pe *propertiesEncoder) Encode(writer io.Writer, node *yaml.Node) error {
mapKeysToStrings(node) mapKeysToStrings(node)
p := properties.NewProperties() p := properties.NewProperties()
err := pe.doEncode(p, node, "") err := pe.doEncode(p, node, "")
@ -64,7 +67,7 @@ func (pe *propertiesEncoder) Encode(node *yaml.Node) error {
return err return err
} }
_, err = p.WriteComment(pe.destination, "#", properties.UTF8) _, err = p.WriteComment(writer, "#", properties.UTF8)
return err return err
} }

4
pkg/yqlib/encoder_properties_test.go
View File

@ -13,13 +13,13 @@ func yamlToProps(sampleYaml string) string {
var output bytes.Buffer var output bytes.Buffer
writer := bufio.NewWriter(&output) writer := bufio.NewWriter(&output)
var propsEncoder = NewPropertiesEncoder(writer) var propsEncoder = NewPropertiesEncoder()
inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder()) inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
if err != nil { if err != nil {
panic(err) panic(err)
} }
node := inputs.Front().Value.(*CandidateNode).Node node := inputs.Front().Value.(*CandidateNode).Node
err = propsEncoder.Encode(node) err = propsEncoder.Encode(writer, node)
if err != nil { if err != nil {
panic(err) panic(err)
} }

4
pkg/yqlib/encoder_test.go
View File

@ -13,13 +13,13 @@ func yamlToJson(sampleYaml string, indent int) string {
var output bytes.Buffer var output bytes.Buffer
writer := bufio.NewWriter(&output) writer := bufio.NewWriter(&output)
var jsonEncoder = NewJsonEncoder(writer, indent) var jsonEncoder = NewJsonEncoder(indent)
inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder()) inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
if err != nil { if err != nil {
panic(err) panic(err)
} }
node := inputs.Front().Value.(*CandidateNode).Node node := inputs.Front().Value.(*CandidateNode).Node
err = jsonEncoder.Encode(node) err = jsonEncoder.Encode(writer, node)
if err != nil { if err != nil {
panic(err) panic(err)
} }

93
pkg/yqlib/encoder_xml.go
View File

@ -9,83 +9,90 @@ import (
yaml "gopkg.in/yaml.v3" yaml "gopkg.in/yaml.v3"
) )
var XmlPreferences = xmlPreferences{AttributePrefix: "+", ContentName: "+content"}
type xmlEncoder struct { type xmlEncoder struct {
xmlEncoder *xml.Encoder
attributePrefix string attributePrefix string
contentName string contentName string
indentString string
} }
func NewXmlEncoder(writer io.Writer, indent int, attributePrefix string, contentName string) Encoder { func NewXmlEncoder(indent int, attributePrefix string, contentName string) Encoder {
encoder := xml.NewEncoder(writer)
var indentString = "" var indentString = ""
for index := 0; index < indent; index++ { for index := 0; index < indent; index++ {
indentString = indentString + " " indentString = indentString + " "
} }
encoder.Indent("", indentString) return &xmlEncoder{attributePrefix, contentName, indentString}
return &xmlEncoder{encoder, attributePrefix, contentName}
} }
func (e *xmlEncoder) PrintDocumentSeparator() error { func (e *xmlEncoder) CanHandleAliases() bool {
return false
}
func (e *xmlEncoder) PrintDocumentSeparator(writer io.Writer) error {
return nil return nil
} }
func (e *xmlEncoder) PrintLeadingContent(content string) error { func (e *xmlEncoder) PrintLeadingContent(writer io.Writer, content string) error {
return nil return nil
} }
func (e *xmlEncoder) Encode(node *yaml.Node) error { func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
encoder := xml.NewEncoder(writer)
encoder.Indent("", e.indentString)
switch node.Kind { switch node.Kind {
case yaml.MappingNode: case yaml.MappingNode:
err := e.encodeTopLevelMap(node) err := e.encodeTopLevelMap(encoder, node)
if err != nil { if err != nil {
return err return err
} }
case yaml.DocumentNode: case yaml.DocumentNode:
err := e.encodeComment(headAndLineComment(node)) err := e.encodeComment(encoder, headAndLineComment(node))
if err != nil { if err != nil {
return err return err
} }
// this used to call encode...
err = e.Encode(unwrapDoc(node)) err = e.encodeTopLevelMap(encoder, unwrapDoc(node))
if err != nil { if err != nil {
return err return err
} }
err = e.encodeComment(footComment(node)) err = e.encodeComment(encoder, footComment(node))
if err != nil { if err != nil {
return err return err
} }
case yaml.ScalarNode: case yaml.ScalarNode:
var charData xml.CharData = []byte(node.Value) var charData xml.CharData = []byte(node.Value)
err := e.xmlEncoder.EncodeToken(charData) err := encoder.EncodeToken(charData)
if err != nil { if err != nil {
return err return err
} }
return e.xmlEncoder.Flush() return encoder.Flush()
default: default:
return fmt.Errorf("unsupported type %v", node.Tag) return fmt.Errorf("unsupported type %v", node.Tag)
} }
var charData xml.CharData = []byte("\n") var charData xml.CharData = []byte("\n")
return e.xmlEncoder.EncodeToken(charData) return encoder.EncodeToken(charData)
} }
func (e *xmlEncoder) encodeTopLevelMap(node *yaml.Node) error { func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node) error {
for i := 0; i < len(node.Content); i += 2 { for i := 0; i < len(node.Content); i += 2 {
key := node.Content[i] key := node.Content[i]
value := node.Content[i+1] value := node.Content[i+1]
start := xml.StartElement{Name: xml.Name{Local: key.Value}} start := xml.StartElement{Name: xml.Name{Local: key.Value}}
err := e.encodeComment(headAndLineComment(key)) err := e.encodeComment(encoder, headAndLineComment(key))
if err != nil { if err != nil {
return err return err
} }
err = e.doEncode(value, start) err = e.doEncode(encoder, value, start)
if err != nil { if err != nil {
return err return err
} }
err = e.encodeComment(footComment(key)) err = e.encodeComment(encoder, footComment(key))
if err != nil { if err != nil {
return err return err
} }
@ -93,49 +100,49 @@ func (e *xmlEncoder) encodeTopLevelMap(node *yaml.Node) error {
return nil return nil
} }
func (e *xmlEncoder) encodeStart(node *yaml.Node, start xml.StartElement) error { func (e *xmlEncoder) encodeStart(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
err := e.xmlEncoder.EncodeToken(start) err := encoder.EncodeToken(start)
if err != nil { if err != nil {
return err return err
} }
return e.encodeComment(headAndLineComment(node)) return e.encodeComment(encoder, headAndLineComment(node))
} }
func (e *xmlEncoder) encodeEnd(node *yaml.Node, start xml.StartElement) error { func (e *xmlEncoder) encodeEnd(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
err := e.xmlEncoder.EncodeToken(start.End()) err := encoder.EncodeToken(start.End())
if err != nil { if err != nil {
return err return err
} }
return e.encodeComment(footComment(node)) return e.encodeComment(encoder, footComment(node))
} }
func (e *xmlEncoder) doEncode(node *yaml.Node, start xml.StartElement) error { func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
switch node.Kind { switch node.Kind {
case yaml.MappingNode: case yaml.MappingNode:
return e.encodeMap(node, start) return e.encodeMap(encoder, node, start)
case yaml.SequenceNode: case yaml.SequenceNode:
return e.encodeArray(node, start) return e.encodeArray(encoder, node, start)
case yaml.ScalarNode: case yaml.ScalarNode:
err := e.encodeStart(node, start) err := e.encodeStart(encoder, node, start)
if err != nil { if err != nil {
return err return err
} }
var charData xml.CharData = []byte(node.Value) var charData xml.CharData = []byte(node.Value)
err = e.xmlEncoder.EncodeToken(charData) err = encoder.EncodeToken(charData)
if err != nil { if err != nil {
return err return err
} }
return e.encodeEnd(node, start) return e.encodeEnd(encoder, node, start)
} }
return fmt.Errorf("unsupported type %v", node.Tag) return fmt.Errorf("unsupported type %v", node.Tag)
} }
func (e *xmlEncoder) encodeComment(commentStr string) error { func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) error {
if commentStr != "" { if commentStr != "" {
var comment xml.Comment = []byte(commentStr) var comment xml.Comment = []byte(commentStr)
err := e.xmlEncoder.EncodeToken(comment) err := encoder.EncodeToken(comment)
if err != nil { if err != nil {
return err return err
} }
@ -143,10 +150,10 @@ func (e *xmlEncoder) encodeComment(commentStr string) error {
return nil return nil
} }
func (e *xmlEncoder) encodeArray(node *yaml.Node, start xml.StartElement) error { func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
for i := 0; i < len(node.Content); i++ { for i := 0; i < len(node.Content); i++ {
value := node.Content[i] value := node.Content[i]
err := e.doEncode(value, start.Copy()) err := e.doEncode(encoder, value, start.Copy())
if err != nil { if err != nil {
return err return err
} }
@ -154,7 +161,7 @@ func (e *xmlEncoder) encodeArray(node *yaml.Node, start xml.StartElement) error
return nil return nil
} }
func (e *xmlEncoder) encodeMap(node *yaml.Node, start xml.StartElement) error { func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
//first find all the attributes and put them on the start token //first find all the attributes and put them on the start token
for i := 0; i < len(node.Content); i += 2 { for i := 0; i < len(node.Content); i += 2 {
@ -171,7 +178,7 @@ func (e *xmlEncoder) encodeMap(node *yaml.Node, start xml.StartElement) error {
} }
} }
err := e.encodeStart(node, start) err := e.encodeStart(encoder, node, start)
if err != nil { if err != nil {
return err return err
} }
@ -181,30 +188,30 @@ func (e *xmlEncoder) encodeMap(node *yaml.Node, start xml.StartElement) error {
key := node.Content[i] key := node.Content[i]
value := node.Content[i+1] value := node.Content[i+1]
err := e.encodeComment(headAndLineComment(key)) err := e.encodeComment(encoder, headAndLineComment(key))
if err != nil { if err != nil {
return err return err
} }
if !strings.HasPrefix(key.Value, e.attributePrefix) && key.Value != e.contentName { if !strings.HasPrefix(key.Value, e.attributePrefix) && key.Value != e.contentName {
start := xml.StartElement{Name: xml.Name{Local: key.Value}} start := xml.StartElement{Name: xml.Name{Local: key.Value}}
err := e.doEncode(value, start) err := e.doEncode(encoder, value, start)
if err != nil { if err != nil {
return err return err
} }
} else if key.Value == e.contentName { } else if key.Value == e.contentName {
// directly encode the contents // directly encode the contents
var charData xml.CharData = []byte(value.Value) var charData xml.CharData = []byte(value.Value)
err = e.xmlEncoder.EncodeToken(charData) err = encoder.EncodeToken(charData)
if err != nil { if err != nil {
return err return err
} }
} }
err = e.encodeComment(footComment(key)) err = e.encodeComment(encoder, footComment(key))
if err != nil { if err != nil {
return err return err
} }
} }
return e.encodeEnd(node, start) return e.encodeEnd(encoder, node, start)
} }

101
pkg/yqlib/encoder_yaml.go Normal file
View File

@ -0,0 +1,101 @@
package yqlib
import (
"bufio"
"bytes"
"errors"
"io"
"strings"
yaml "gopkg.in/yaml.v3"
)
type yamlEncoder struct {
indent int
colorise bool
printDocSeparators bool
unwrapScalar bool
}
func NewYamlEncoder(indent int, colorise bool, printDocSeparators bool, unwrapScalar bool) Encoder {
if indent < 0 {
indent = 0
}
return &yamlEncoder{indent, colorise, printDocSeparators, unwrapScalar}
}
func (ye *yamlEncoder) CanHandleAliases() bool {
return true
}
func (ye *yamlEncoder) PrintDocumentSeparator(writer io.Writer) error {
if ye.printDocSeparators {
log.Debug("-- writing doc sep")
if err := writeString(writer, "---\n"); err != nil {
return err
}
}
return nil
}
func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) error {
// log.Debug("headcommentwas [%v]", content)
reader := bufio.NewReader(strings.NewReader(content))
for {
readline, errReading := reader.ReadString('\n')
if errReading != nil && !errors.Is(errReading, io.EOF) {
return errReading
}
if strings.Contains(readline, "$yqDocSeperator$") {
if err := ye.PrintDocumentSeparator(writer); err != nil {
return err
}
} else {
if err := writeString(writer, readline); err != nil {
return err
}
}
if errors.Is(errReading, io.EOF) {
if readline != "" {
// the last comment we read didn't have a new line, put one in
if err := writeString(writer, "\n"); err != nil {
return err
}
}
break
}
}
return nil
}
func (ye *yamlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
if node.Kind == yaml.ScalarNode && ye.unwrapScalar {
return writeString(writer, node.Value+"\n")
}
destination := writer
tempBuffer := bytes.NewBuffer(nil)
if ye.colorise {
destination = tempBuffer
}
var encoder = yaml.NewEncoder(destination)
encoder.SetIndent(ye.indent)
if err := encoder.Encode(node); err != nil {
return err
}
if ye.colorise {
return colorizeAndPrint(tempBuffer.Bytes(), writer)
}
return nil
}

2
pkg/yqlib/lib.go
View File

@ -18,8 +18,6 @@ type xmlPreferences struct {
ContentName string ContentName string
} }
var XmlPreferences = xmlPreferences{AttributePrefix: "+", ContentName: "+content"}
var log = logging.MustGetLogger("yq-lib") var log = logging.MustGetLogger("yq-lib")
// GetLogger returns the yq logger instance. // GetLogger returns the yq logger instance.

4
pkg/yqlib/operator_comments.go
View File

@ -86,8 +86,8 @@ func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *
var chompRegexp = regexp.MustCompile(`\n$`) var chompRegexp = regexp.MustCompile(`\n$`)
var output bytes.Buffer var output bytes.Buffer
var writer = bufio.NewWriter(&output) var writer = bufio.NewWriter(&output)
var encoder = NewYamlEncoder(writer, 2, false, true, false) var encoder = NewYamlEncoder(2, false, false, false)
if err := encoder.PrintLeadingContent(candidate.LeadingContent); err != nil { if err := encoder.PrintLeadingContent(writer, candidate.LeadingContent); err != nil {
return Context{}, err return Context{}, err
} }
if err := writer.Flush(); err != nil { if err := writer.Flush(); err != nil {

26
pkg/yqlib/operator_encoder_decoder.go
View File

@ -10,11 +10,31 @@ import (
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
) )
func yamlToString(candidate *CandidateNode, prefs encoderPreferences) (string, error) { func configureEncoder(format PrinterOutputFormat, indent int) Encoder {
switch format {
case JsonOutputFormat:
return NewJsonEncoder(indent)
case PropsOutputFormat:
return NewPropertiesEncoder()
case CsvOutputFormat:
return NewCsvEncoder(',')
case TsvOutputFormat:
return NewCsvEncoder('\t')
case YamlOutputFormat:
return NewYamlEncoder(indent, false, true, true)
case XmlOutputFormat:
return NewXmlEncoder(indent, XmlPreferences.AttributePrefix, XmlPreferences.ContentName)
}
panic("invalid encoder")
}
func encodeToString(candidate *CandidateNode, prefs encoderPreferences) (string, error) {
var output bytes.Buffer var output bytes.Buffer
log.Debug("printing with indent: %v", prefs.indent) log.Debug("printing with indent: %v", prefs.indent)
printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), prefs.format, true, false, prefs.indent, true) encoder := configureEncoder(prefs.format, prefs.indent)
printer := NewPrinter(encoder, NewSinglePrinterWriter(bufio.NewWriter(&output)))
err := printer.PrintResults(candidate.AsList()) err := printer.PrintResults(candidate.AsList())
return output.String(), err return output.String(), err
} }
@ -36,7 +56,7 @@ func encodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode) candidate := el.Value.(*CandidateNode)
stringValue, err := yamlToString(candidate, preferences) stringValue, err := encodeToString(candidate, preferences)
if err != nil { if err != nil {
return Context{}, err return Context{}, err

10
pkg/yqlib/operators_test.go
View File

@ -26,6 +26,10 @@ type expressionScenario struct {
dontFormatInputForDoc bool // dont format input doc for documentation generation dontFormatInputForDoc bool // dont format input doc for documentation generation
} }
func NewSimpleYamlPrinter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
return NewPrinter(NewYamlEncoder(indent, colorsEnabled, printDocSeparators, unwrapScalar), NewSinglePrinterWriter(writer))
}
func readDocumentWithLeadingContent(content string, fakefilename string, fakeFileIndex int) (*list.List, error) { func readDocumentWithLeadingContent(content string, fakefilename string, fakeFileIndex int) (*list.List, error) {
reader, firstFileLeadingContent, err := processReadStream(bufio.NewReader(strings.NewReader(content))) reader, firstFileLeadingContent, err := processReadStream(bufio.NewReader(strings.NewReader(content)))
if err != nil { if err != nil {
@ -92,7 +96,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
func resultToString(t *testing.T, n *CandidateNode) string { func resultToString(t *testing.T, n *CandidateNode) string {
var valueBuffer bytes.Buffer var valueBuffer bytes.Buffer
printer := NewPrinterWithSingleWriter(bufio.NewWriter(&valueBuffer), YamlOutputFormat, true, false, 4, true) printer := NewSimpleYamlPrinter(bufio.NewWriter(&valueBuffer), YamlOutputFormat, true, false, 4, true)
err := printer.PrintResults(n.AsList()) err := printer.PrintResults(n.AsList())
if err != nil { if err != nil {
@ -145,7 +149,7 @@ func copyFromHeader(folder string, title string, out *os.File) error {
func formatYaml(yaml string, filename string) string { func formatYaml(yaml string, filename string) string {
var output bytes.Buffer var output bytes.Buffer
printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true) printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
node, err := NewExpressionParser().ParseExpression(".. style= \"\"") node, err := NewExpressionParser().ParseExpression(".. style= \"\"")
if err != nil { if err != nil {
@ -268,7 +272,7 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) {
func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formattedDoc string, formattedDoc2 string) { func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formattedDoc string, formattedDoc2 string) {
var output bytes.Buffer var output bytes.Buffer
var err error var err error
printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true) printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
node, err := NewExpressionParser().ParseExpression(s.expression) node, err := NewExpressionParser().ParseExpression(s.expression)
if err != nil { if err != nil {

46
pkg/yqlib/printer.go
View File

@ -48,12 +48,7 @@ func OutputFormatFromString(format string) (PrinterOutputFormat, error) {
} }
type resultsPrinter struct { type resultsPrinter struct {
outputFormat PrinterOutputFormat
encoder Encoder encoder Encoder
unwrapScalar bool
colorsEnabled bool
indent int
printDocSeparators bool
printerWriter PrinterWriter printerWriter PrinterWriter
firstTimePrinting bool firstTimePrinting bool
previousDocIndex uint previousDocIndex uint
@ -63,18 +58,10 @@ type resultsPrinter struct {
appendixReader io.Reader appendixReader io.Reader
} }
func NewPrinterWithSingleWriter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer { func NewPrinter(encoder Encoder, printerWriter PrinterWriter) Printer {
return NewPrinter(NewSinglePrinterWriter(writer), outputFormat, unwrapScalar, colorsEnabled, indent, printDocSeparators)
}
func NewPrinter(printerWriter PrinterWriter, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
return &resultsPrinter{ return &resultsPrinter{
encoder: encoder,
printerWriter: printerWriter, printerWriter: printerWriter,
outputFormat: outputFormat,
unwrapScalar: unwrapScalar,
colorsEnabled: colorsEnabled,
indent: indent,
printDocSeparators: outputFormat == YamlOutputFormat && printDocSeparators,
firstTimePrinting: true, firstTimePrinting: true,
treeNavigator: NewDataTreeNavigator(), treeNavigator: NewDataTreeNavigator(),
} }
@ -91,23 +78,7 @@ func (p *resultsPrinter) PrintedAnything() bool {
func (p *resultsPrinter) printNode(node *yaml.Node, writer io.Writer) error { func (p *resultsPrinter) printNode(node *yaml.Node, writer io.Writer) error {
p.printedMatches = p.printedMatches || (node.Tag != "!!null" && p.printedMatches = p.printedMatches || (node.Tag != "!!null" &&
(node.Tag != "!!bool" || node.Value != "false")) (node.Tag != "!!bool" || node.Value != "false"))
return p.encoder.Encode(writer, node)
// switch p.outputFormat {
// case JsonOutputFormat:
// encoder = NewJsonEncoder(writer, p.indent)
// case PropsOutputFormat:
// encoder = NewPropertiesEncoder(writer)
// case CsvOutputFormat:
// encoder = NewCsvEncoder(writer, ',')
// case TsvOutputFormat:
// encoder = NewCsvEncoder(writer, '\t')
// case YamlOutputFormat:
// encoder = NewYamlEncoder(writer, p.indent, p.colorsEnabled)
// case XmlOutputFormat:
// encoder = NewXmlEncoder(writer, p.indent, XmlPreferences.AttributePrefix, XmlPreferences.ContentName)
// }
return p.encoder.Encode(node)
} }
func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error { func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
@ -118,7 +89,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
return nil return nil
} }
if p.outputFormat != YamlOutputFormat { if !p.encoder.CanHandleAliases() {
explodeOp := Operation{OperationType: explodeOpType} explodeOp := Operation{OperationType: explodeOpType}
explodeNode := ExpressionNode{Operation: &explodeOp} explodeNode := ExpressionNode{Operation: &explodeOp}
context, err := p.treeNavigator.GetMatchingNodes(Context{MatchingNodes: matchingNodes}, &explodeNode) context, err := p.treeNavigator.GetMatchingNodes(Context{MatchingNodes: matchingNodes}, &explodeNode)
@ -129,6 +100,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
} }
if p.firstTimePrinting { if p.firstTimePrinting {
log.Debugf("its my first time *blush*")
node := matchingNodes.Front().Value.(*CandidateNode) node := matchingNodes.Front().Value.(*CandidateNode)
p.previousDocIndex = node.Document p.previousDocIndex = node.Document
p.previousFileIndex = node.FileIndex p.previousFileIndex = node.FileIndex
@ -138,7 +110,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
for el := matchingNodes.Front(); el != nil; el = el.Next() { for el := matchingNodes.Front(); el != nil; el = el.Next() {
mappedDoc := el.Value.(*CandidateNode) mappedDoc := el.Value.(*CandidateNode)
log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v, printDocSeparators: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document, p.printDocSeparators) log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document)
writer, errorWriting := p.printerWriter.GetWriter(mappedDoc) writer, errorWriting := p.printerWriter.GetWriter(mappedDoc)
if errorWriting != nil { if errorWriting != nil {
@ -149,13 +121,12 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
commentStartsWithSeparator := commentsStartWithSepExp.MatchString(mappedDoc.LeadingContent) commentStartsWithSeparator := commentsStartWithSepExp.MatchString(mappedDoc.LeadingContent)
if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && !commentStartsWithSeparator { if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && !commentStartsWithSeparator {
log.Debug("-- writing doc sep") if err := p.encoder.PrintDocumentSeparator(writer); err != nil {
if err := p.encoder.PrintDocumentSeparator(); err != nil {
return err return err
} }
} }
if err := p.encoder.PrintLeadingContent(mappedDoc.LeadingContent); err != nil { if err := p.encoder.PrintLeadingContent(writer, mappedDoc.LeadingContent); err != nil {
return err return err
} }
@ -167,6 +138,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
if err := writer.Flush(); err != nil { if err := writer.Flush(); err != nil {
return err return err
} }
log.Debugf("done printing results")
} }
// what happens if I remove output format check? // what happens if I remove output format check?

20
pkg/yqlib/printer_test.go
View File

@ -33,10 +33,10 @@ func nodeToList(candidate *CandidateNode) *list.List {
return elMap return elMap
} }
func TestPrinterMultipleDocsInSequence(t *testing.T) { func TestPrinterMultipleDocsInSequenceOnly(t *testing.T) {
var output bytes.Buffer var output bytes.Buffer
var writer = bufio.NewWriter(&output) var writer = bufio.NewWriter(&output)
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
if err != nil { if err != nil {
@ -74,7 +74,7 @@ func TestPrinterMultipleDocsInSequence(t *testing.T) {
func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) { func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
var output bytes.Buffer var output bytes.Buffer
var writer = bufio.NewWriter(&output) var writer = bufio.NewWriter(&output)
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
if err != nil { if err != nil {
@ -116,7 +116,7 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
func TestPrinterMultipleFilesInSequence(t *testing.T) { func TestPrinterMultipleFilesInSequence(t *testing.T) {
var output bytes.Buffer var output bytes.Buffer
var writer = bufio.NewWriter(&output) var writer = bufio.NewWriter(&output)
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
if err != nil { if err != nil {
@ -163,7 +163,7 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) {
func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) { func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
var output bytes.Buffer var output bytes.Buffer
var writer = bufio.NewWriter(&output) var writer = bufio.NewWriter(&output)
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
if err != nil { if err != nil {
@ -213,7 +213,7 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
func TestPrinterMultipleDocsInSinglePrint(t *testing.T) { func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
var output bytes.Buffer var output bytes.Buffer
var writer = bufio.NewWriter(&output) var writer = bufio.NewWriter(&output)
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
if err != nil { if err != nil {
@ -232,7 +232,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) { func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) {
var output bytes.Buffer var output bytes.Buffer
var writer = bufio.NewWriter(&output) var writer = bufio.NewWriter(&output)
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
if err != nil { if err != nil {
@ -261,7 +261,7 @@ a: coconut
func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) { func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) {
var output bytes.Buffer var output bytes.Buffer
var writer = bufio.NewWriter(&output) var writer = bufio.NewWriter(&output)
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
if err != nil { if err != nil {
@ -287,7 +287,7 @@ a: coconut
func TestPrinterScalarWithLeadingCont(t *testing.T) { func TestPrinterScalarWithLeadingCont(t *testing.T) {
var output bytes.Buffer var output bytes.Buffer
var writer = bufio.NewWriter(&output) var writer = bufio.NewWriter(&output)
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
node, err := NewExpressionParser().ParseExpression(".a") node, err := NewExpressionParser().ParseExpression(".a")
if err != nil { if err != nil {
@ -314,7 +314,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
var writer = bufio.NewWriter(&output) var writer = bufio.NewWriter(&output)
// note printDocSeparators is true, it should still not print document separators // note printDocSeparators is true, it should still not print document separators
// when outputing JSON. // when outputing JSON.
printer := NewPrinterWithSingleWriter(writer, JsonOutputFormat, true, false, 0, true) printer := NewPrinter(NewJsonEncoder(0), NewSinglePrinterWriter(writer))
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
if err != nil { if err != nil {

7
pkg/yqlib/utils.go
View File

@ -70,13 +70,6 @@ func processReadStream(reader *bufio.Reader) (io.Reader, string, error) {
} }
} }
func explodeNodes(nodes *list.List) (*list.List, error) {
explodeOp := Operation{OperationType: explodeOpType}
explodeNode := ExpressionNode{Operation: &explodeOp}
context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: nodes}, &explodeNode)
return context.MatchingNodes, err
}
func readDocuments(reader io.Reader, filename string, fileIndex int, decoder Decoder) (*list.List, error) { func readDocuments(reader io.Reader, filename string, fileIndex int, decoder Decoder) (*list.List, error) {
decoder.Init(reader) decoder.Init(reader)
inputList := list.New() inputList := list.New()

146
pkg/yqlib/xml_test.go
View File

@ -28,7 +28,7 @@ func processScenario(s xmlScenario) string {
var output bytes.Buffer var output bytes.Buffer
writer := bufio.NewWriter(&output) writer := bufio.NewWriter(&output)
var encoder = NewXmlEncoder(writer, 2, "+", "+content") var encoder = NewXmlEncoder(2, "+", "+content")
var decoder = NewYamlDecoder() var decoder = NewYamlDecoder()
if s.scenarioType == "roundtrip" { if s.scenarioType == "roundtrip" {
@ -40,13 +40,13 @@ func processScenario(s xmlScenario) string {
panic(err) panic(err)
} }
node := inputs.Front().Value.(*CandidateNode).Node node := inputs.Front().Value.(*CandidateNode).Node
err = encoder.Encode(node) err = encoder.Encode(writer, node)
if err != nil { if err != nil {
panic(err) panic(err)
} }
writer.Flush() writer.Flush()
return strings.TrimSuffix(output.String(), "\n") return output.String()
} }
type xmlScenario struct { type xmlScenario struct {
@ -116,75 +116,75 @@ var expectedXmlWithComments = `<!-- above_cat inline_cat--><cat><!-- above_array
` `
var xmlScenarios = []xmlScenario{ var xmlScenarios = []xmlScenario{
// { {
// description: "Parse xml: simple", description: "Parse xml: simple",
// input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat>meow</cat>", input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat>meow</cat>",
// expected: "D0, P[], (doc)::cat: meow\n", expected: "D0, P[], (doc)::cat: meow\n",
// }, },
// { {
// description: "Parse xml: array", description: "Parse xml: array",
// subdescription: "Consecutive nodes with identical xml names are assumed to be arrays.", subdescription: "Consecutive nodes with identical xml names are assumed to be arrays.",
// input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<animal>1</animal>\n<animal>2</animal>", input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<animal>1</animal>\n<animal>2</animal>",
// expected: "D0, P[], (doc)::animal:\n - \"1\"\n - \"2\"\n", expected: "D0, P[], (doc)::animal:\n - \"1\"\n - \"2\"\n",
// }, },
// { {
// description: "Parse xml: attributes", description: "Parse xml: attributes",
// subdescription: "Attributes are converted to fields, with the attribute prefix.", subdescription: "Attributes are converted to fields, with the attribute prefix.",
// input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">\n <legs>7</legs>\n</cat>", input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">\n <legs>7</legs>\n</cat>",
// expected: "D0, P[], (doc)::cat:\n +legs: \"4\"\n legs: \"7\"\n", expected: "D0, P[], (doc)::cat:\n +legs: \"4\"\n legs: \"7\"\n",
// }, },
// { {
// description: "Parse xml: attributes with content", description: "Parse xml: attributes with content",
// subdescription: "Content is added as a field, using the content name", subdescription: "Content is added as a field, using the content name",
// input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">meow</cat>", input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">meow</cat>",
// expected: "D0, P[], (doc)::cat:\n +content: meow\n +legs: \"4\"\n", expected: "D0, P[], (doc)::cat:\n +content: meow\n +legs: \"4\"\n",
// }, },
// { {
// description: "Parse xml: with comments", description: "Parse xml: with comments",
// subdescription: "A best attempt is made to preserve comments.", subdescription: "A best attempt is made to preserve comments.",
// input: inputXmlWithComments, input: inputXmlWithComments,
// expected: expectedDecodeYamlWithComments, expected: expectedDecodeYamlWithComments,
// scenarioType: "decode", scenarioType: "decode",
// }, },
// { {
// description: "Encode xml: simple", description: "Encode xml: simple",
// input: "cat: purrs", input: "cat: purrs",
// expected: "<cat>purrs</cat>\n", expected: "<cat>purrs</cat>\n",
// scenarioType: "encode", scenarioType: "encode",
// }, },
// { {
// description: "Encode xml: array", description: "Encode xml: array",
// input: "pets:\n cat:\n - purrs\n - meows", input: "pets:\n cat:\n - purrs\n - meows",
// expected: "<pets>\n <cat>purrs</cat>\n <cat>meows</cat>\n</pets>\n", expected: "<pets>\n <cat>purrs</cat>\n <cat>meows</cat>\n</pets>\n",
// scenarioType: "encode", scenarioType: "encode",
// }, },
// { {
// description: "Encode xml: attributes", description: "Encode xml: attributes",
// subdescription: "Fields with the matching xml-attribute-prefix are assumed to be attributes.", subdescription: "Fields with the matching xml-attribute-prefix are assumed to be attributes.",
// input: "cat:\n +name: tiger\n meows: true\n", input: "cat:\n +name: tiger\n meows: true\n",
// expected: "<cat name=\"tiger\">\n <meows>true</meows>\n</cat>\n", expected: "<cat name=\"tiger\">\n <meows>true</meows>\n</cat>\n",
// scenarioType: "encode", scenarioType: "encode",
// }, },
// { {
// skipDoc: true, skipDoc: true,
// input: "cat:\n ++name: tiger\n meows: true\n", input: "cat:\n ++name: tiger\n meows: true\n",
// expected: "<cat +name=\"tiger\">\n <meows>true</meows>\n</cat>\n", expected: "<cat +name=\"tiger\">\n <meows>true</meows>\n</cat>\n",
// scenarioType: "encode", scenarioType: "encode",
// }, },
// { {
// description: "Encode xml: attributes with content", description: "Encode xml: attributes with content",
// subdescription: "Fields with the matching xml-content-name is assumed to be content.", subdescription: "Fields with the matching xml-content-name is assumed to be content.",
// input: "cat:\n +name: tiger\n +content: cool\n", input: "cat:\n +name: tiger\n +content: cool\n",
// expected: "<cat name=\"tiger\">cool</cat>\n", expected: "<cat name=\"tiger\">cool</cat>\n",
// scenarioType: "encode", scenarioType: "encode",
// }, },
// { {
// description: "Encode xml: comments", description: "Encode xml: comments",
// subdescription: "A best attempt is made to copy comments to xml.", subdescription: "A best attempt is made to copy comments to xml.",
// input: yamlWithComments, input: yamlWithComments,
// expected: expectedXmlWithComments, expected: expectedXmlWithComments,
// scenarioType: "encode", scenarioType: "encode",
// }, },
{ {
description: "Round trip: with comments", description: "Round trip: with comments",
subdescription: "A best effort is made, but comment positions and white space are not preserved perfectly.", subdescription: "A best effort is made, but comment positions and white space are not preserved perfectly.",
@ -233,7 +233,7 @@ func documentXmlDecodeScenario(t *testing.T, w *bufio.Writer, s xmlScenario) {
writeOrPanic(w, "will output\n") writeOrPanic(w, "will output\n")
var output bytes.Buffer var output bytes.Buffer
printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true) printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
node := decodeXml(t, s.input) node := decodeXml(t, s.input)