suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2025-02-28 11:08:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Mike Farah 2021-12-30 15:39:40 +11:00
parent 2c4bd03ec6
commit 277ba1fe5d
23 changed files with 652 additions and 375 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
acceptance_tests/output-format.sh
View File

@ -142,4 +142,24 @@ EOM
assertEquals "$expected" "$X" assertEquals "$expected" "$X"
} }
testOutputXmlMultiPropertiesMultietc() {
cat >test.yml <<EOL
a: {b: {c: ["cat"]}}asd
EOL
read -r -d '' expected << EOM
<a>
<b>
<c>cat</c>
</b>asd
</a>
EOM
X=$(./yq e --output-format=x test.yml)
assertEquals "$expected" "$X"
X=$(./yq ea --output-format=x test.yml)
assertEquals "$expected" "$X"
}
source ./scripts/shunit2 source ./scripts/shunit2

3
cmd/evaluate_all_command.go
View File

@ -81,8 +81,9 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
} }
printerWriter := configurePrinterWriter(format, out) printerWriter := configurePrinterWriter(format, out)
encoder := configureEncoder(format)
printer := yqlib.NewPrinter(printerWriter, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators) printer := yqlib.NewPrinter(encoder, printerWriter)
if frontMatter != "" { if frontMatter != "" {
frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex]) frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])

3
cmd/evalute_sequence_command.go
View File

@ -89,8 +89,9 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
} }
printerWriter := configurePrinterWriter(format, out) printerWriter := configurePrinterWriter(format, out)
encoder := configureEncoder(format)
printer := yqlib.NewPrinter(printerWriter, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators) printer := yqlib.NewPrinter(encoder, printerWriter)
decoder, err := configureDecoder() decoder, err := configureDecoder()
if err != nil { if err != nil {

20
cmd/utils.go
View File

@ -52,7 +52,7 @@ func configureDecoder() (yqlib.Decoder, error) {
} }
switch yqlibInputFormat { switch yqlibInputFormat {
case yqlib.XmlInputFormat: case yqlib.XmlInputFormat:
return yqlib.NewXmlDecoder(xmlAttributePrefix, xmlContentName), nil return yqlib.NewXmlDecoder(), nil
} }
return yqlib.NewYamlDecoder(), nil return yqlib.NewYamlDecoder(), nil
} }
@ -73,3 +73,21 @@ func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) yql
} }
return printerWriter return printerWriter
} }
func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder {
switch format {
case yqlib.JsonOutputFormat:
return yqlib.NewJsonEncoder(indent)
case yqlib.PropsOutputFormat:
return yqlib.NewPropertiesEncoder()
case yqlib.CsvOutputFormat:
return yqlib.NewCsvEncoder(',')
case yqlib.TsvOutputFormat:
return yqlib.NewCsvEncoder('\t')
case yqlib.YamlOutputFormat:
return yqlib.NewYamlEncoder(indent, colorsEnabled, !noDocSeparators, unwrapScalar)
case yqlib.XmlOutputFormat:
return yqlib.NewXmlEncoder(indent)
}
panic("invalid encoder")
}

5
examples/data1.yaml
View File

@ -1,4 +1 @@
#hi cat: purrs
---
# hi
- - fish

218
pkg/yqlib/doc/usage/xml.md
View File

@ -22,6 +22,224 @@ XML nodes that have attributes then plain content, e.g:
The content of the node will be set as a field in the map with the key "+content". Use the `--xml-content-name` flag to change this. The content of the node will be set as a field in the map with the key "+content". Use the `--xml-content-name` flag to change this.
## Parse xml: simple
Given a sample.xml file of:
```xml
<?xml version="1.0" encoding="UTF-8"?>
<cat>meow</cat>
```
then
```bash
yq e -p=xml '.' sample.xml
```
will output
```yaml
cat: meow
```
## Parse xml: array
Consecutive nodes with identical xml names are assumed to be arrays.
Given a sample.xml file of:
```xml
<?xml version="1.0" encoding="UTF-8"?>
<animal>1</animal>
<animal>2</animal>
```
then
```bash
yq e -p=xml '.' sample.xml
```
will output
```yaml
animal:
- "1"
- "2"
```
## Parse xml: attributes
Attributes are converted to fields, with the attribute prefix.
Given a sample.xml file of:
```xml
<?xml version="1.0" encoding="UTF-8"?>
<cat legs="4">
<legs>7</legs>
</cat>
```
then
```bash
yq e -p=xml '.' sample.xml
```
will output
```yaml
cat:
+legs: "4"
legs: "7"
```
## Parse xml: attributes with content
Content is added as a field, using the content name
Given a sample.xml file of:
```xml
<?xml version="1.0" encoding="UTF-8"?>
<cat legs="4">meow</cat>
```
then
```bash
yq e -p=xml '.' sample.xml
```
will output
```yaml
cat:
+content: meow
+legs: "4"
```
## Parse xml: with comments
A best attempt is made to preserve comments.
Given a sample.xml file of:
```xml
<!-- before cat -->
<cat>
<!-- in cat before -->
<x>3<!-- multi
line comment
for x --></x>
<y>
<!-- in y before -->
<d><!-- in d before -->4<!-- in d after --></d>
<!-- in y after -->
</y>
<!-- in_cat_after -->
</cat>
<!-- after cat -->
```
then
```bash
yq e -p=xml '.' sample.xml
```
will output
```yaml
# before cat
cat:
# in cat before
x: "3" # multi
# line comment
# for x
y:
# in y before
d: "4" # in d before in d after
# in y after
# after cat
```
## Encode xml: simple
Given a sample.yml file of:
```yaml
cat: purrs
```
then
```bash
yq e -o=xml '.' sample.yml
```
will output
```xml
<cat>purrs</cat>
```
## Encode xml: array
Given a sample.yml file of:
```yaml
pets:
cat:
- purrs
- meows
```
then
```bash
yq e -o=xml '.' sample.yml
```
will output
```xml
<pets>
<cat>purrs</cat>
<cat>meows</cat>
</pets>
```
## Encode xml: attributes
Fields with the matching xml-attribute-prefix are assumed to be attributes.
Given a sample.yml file of:
```yaml
cat:
+name: tiger
meows: true
```
then
```bash
yq e -o=xml '.' sample.yml
```
will output
```xml
<cat name="tiger">
<meows>true</meows>
</cat>
```
## Encode xml: attributes with content
Fields with the matching xml-content-name is assumed to be content.
Given a sample.yml file of:
```yaml
cat:
+name: tiger
+content: cool
```
then
```bash
yq e -o=xml '.' sample.yml
```
will output
```xml
<cat name="tiger">cool</cat>
```
## Encode xml: comments
A best attempt is made to copy comments to xml.
Given a sample.yml file of:
```yaml
# above_cat
cat: # inline_cat
# above_array
array: # inline_array
- val1 # inline_val1
# above_val2
- val2 # inline_val2
# below_cat
```
then
```bash
yq e -o=xml '.' sample.yml
```
will output
```xml
<!-- above_cat inline_cat--><cat><!-- above_array inline_array-->
<array><!-- inline_val1-->val1</array>
<array><!-- above_val2 inline_val2-->val2</array>
</cat><!-- below_cat-->
```
## Round trip: with comments ## Round trip: with comments
A best effort is made, but comment positions and white space are not preserved perfectly. A best effort is made, but comment positions and white space are not preserved perfectly.

153
pkg/yqlib/encoder.go
View File

@ -1,165 +1,20 @@
package yqlib package yqlib
import ( import (
"bufio"
"bytes" "bytes"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"io" "io"
"strings"
yaml "gopkg.in/yaml.v3" yaml "gopkg.in/yaml.v3"
) )
type Encoder interface { type Encoder interface {
Encode(node *yaml.Node) error Encode(writer io.Writer, node *yaml.Node) error
PrintDocumentSeparator() error PrintDocumentSeparator(writer io.Writer) error
PrintLeadingContent(content string) error PrintLeadingContent(writer io.Writer, content string) error
} CanHandleAliases() bool
type yamlEncoder struct {
destination io.Writer
indent int
colorise bool
firstDoc bool
printDocSeparators bool
unwrapScalar bool
}
func NewYamlEncoder(destination io.Writer, indent int, colorise bool, printDocSeparators bool, unwrapScalar bool) Encoder {
if indent < 0 {
indent = 0
}
return &yamlEncoder{destination, indent, colorise, true, printDocSeparators, unwrapScalar}
}
func (ye *yamlEncoder) PrintDocumentSeparator() error {
if ye.printDocSeparators {
if err := writeString(ye.destination, "---\n"); err != nil {
return err
}
}
return nil
}
func (ye *yamlEncoder) PrintLeadingContent(content string) error {
log.Debug("headcommentwas %v", content)
log.Debug("finished headcomment")
reader := bufio.NewReader(strings.NewReader(content))
for {
readline, errReading := reader.ReadString('\n')
if errReading != nil && !errors.Is(errReading, io.EOF) {
return errReading
}
if strings.Contains(readline, "$yqDocSeperator$") {
if err := ye.PrintDocumentSeparator(); err != nil {
return err
}
} else {
if err := writeString(ye.destination, readline); err != nil {
return err
}
}
if errors.Is(errReading, io.EOF) {
if readline != "" {
// the last comment we read didn't have a new line, put one in
if err := writeString(ye.destination, "\n"); err != nil {
return err
}
}
break
}
}
return nil
}
func (ye *yamlEncoder) Encode(node *yaml.Node) error {
if node.Kind == yaml.ScalarNode && ye.unwrapScalar {
return writeString(ye.destination, node.Value+"\n")
}
destination := ye.destination
tempBuffer := bytes.NewBuffer(nil)
if ye.colorise {
destination = tempBuffer
}
var encoder = yaml.NewEncoder(destination)
encoder.SetIndent(ye.indent)
// TODO: work out if the first doc had a separator or not.
if ye.firstDoc {
ye.firstDoc = false
} else if _, err := destination.Write([]byte("---\n")); err != nil {
return err
}
if err := encoder.Encode(node); err != nil {
return err
}
if ye.colorise {
return colorizeAndPrint(tempBuffer.Bytes(), ye.destination)
}
return nil
}
type jsonEncoder struct {
encoder *json.Encoder
}
func mapKeysToStrings(node *yaml.Node) {
if node.Kind == yaml.MappingNode {
for index, child := range node.Content {
if index%2 == 0 { // its a map key
child.Tag = "!!str"
}
}
}
for _, child := range node.Content {
mapKeysToStrings(child)
}
}
func NewJsonEncoder(destination io.Writer, indent int) Encoder {
var encoder = json.NewEncoder(destination)
encoder.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
var indentString = ""
for index := 0; index < indent; index++ {
indentString = indentString + " "
}
encoder.SetIndent("", indentString)
return &jsonEncoder{encoder}
}
func (je *jsonEncoder) PrintDocumentSeparator() error {
return nil
}
func (je *jsonEncoder) PrintLeadingContent(content string) error {
return nil
}
func (je *jsonEncoder) Encode(node *yaml.Node) error {
var dataBucket orderedMap
// firstly, convert all map keys to strings
mapKeysToStrings(node)
errorDecoding := node.Decode(&dataBucket)
if errorDecoding != nil {
return errorDecoding
}
return je.encoder.Encode(dataBucket)
} }
// orderedMap allows to marshal and unmarshal JSON and YAML values keeping the // orderedMap allows to marshal and unmarshal JSON and YAML values keeping the

29
pkg/yqlib/encoder_csv.go
View File

@ -9,24 +9,26 @@ import (
) )
type csvEncoder struct { type csvEncoder struct {
destination csv.Writer separator rune
} }
func NewCsvEncoder(destination io.Writer, separator rune) Encoder { func NewCsvEncoder(separator rune) Encoder {
csvWriter := *csv.NewWriter(destination) return &csvEncoder{separator}
csvWriter.Comma = separator
return &csvEncoder{csvWriter}
} }
func (e *csvEncoder) PrintDocumentSeparator() error { func (e *csvEncoder) CanHandleAliases() bool {
return false
}
func (e *csvEncoder) PrintDocumentSeparator(writer io.Writer) error {
return nil return nil
} }
func (e *csvEncoder) PrintLeadingContent(content string) error { func (e *csvEncoder) PrintLeadingContent(writer io.Writer, content string) error {
return nil return nil
} }
func (e *csvEncoder) encodeRow(contents []*yaml.Node) error { func (e *csvEncoder) encodeRow(csvWriter *csv.Writer, contents []*yaml.Node) error {
stringValues := make([]string, len(contents)) stringValues := make([]string, len(contents))
for i, child := range contents { for i, child := range contents {
@ -36,10 +38,13 @@ func (e *csvEncoder) encodeRow(contents []*yaml.Node) error {
} }
stringValues[i] = child.Value stringValues[i] = child.Value
} }
return e.destination.Write(stringValues) return csvWriter.Write(stringValues)
} }
func (e *csvEncoder) Encode(originalNode *yaml.Node) error { func (e *csvEncoder) Encode(writer io.Writer, originalNode *yaml.Node) error {
csvWriter := csv.NewWriter(writer)
csvWriter.Comma = e.separator
// node must be a sequence // node must be a sequence
node := unwrapDoc(originalNode) node := unwrapDoc(originalNode)
if node.Kind != yaml.SequenceNode { if node.Kind != yaml.SequenceNode {
@ -48,7 +53,7 @@ func (e *csvEncoder) Encode(originalNode *yaml.Node) error {
return nil return nil
} }
if node.Content[0].Kind == yaml.ScalarNode { if node.Content[0].Kind == yaml.ScalarNode {
return e.encodeRow(node.Content) return e.encodeRow(csvWriter, node.Content)
} }
for i, child := range node.Content { for i, child := range node.Content {
@ -56,7 +61,7 @@ func (e *csvEncoder) Encode(originalNode *yaml.Node) error {
if child.Kind != yaml.SequenceNode { if child.Kind != yaml.SequenceNode {
return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag) return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag)
} }
err := e.encodeRow(child.Content) err := e.encodeRow(csvWriter, child.Content)
if err != nil { if err != nil {
return err return err
} }

4
pkg/yqlib/encoder_csv_test.go
View File

@ -13,13 +13,13 @@ func yamlToCsv(sampleYaml string, separator rune) string {
var output bytes.Buffer var output bytes.Buffer
writer := bufio.NewWriter(&output) writer := bufio.NewWriter(&output)
var jsonEncoder = NewCsvEncoder(writer, separator) var jsonEncoder = NewCsvEncoder(separator)
inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder()) inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
if err != nil { if err != nil {
panic(err) panic(err)
} }
node := inputs.Front().Value.(*CandidateNode).Node node := inputs.Front().Value.(*CandidateNode).Node
err = jsonEncoder.Encode(node) err = jsonEncoder.Encode(writer, node)
if err != nil { if err != nil {
panic(err) panic(err)
} }

64
pkg/yqlib/encoder_json.go Normal file
View File

@ -0,0 +1,64 @@
package yqlib
import (
"encoding/json"
"io"
yaml "gopkg.in/yaml.v3"
)
type jsonEncoder struct {
indentString string
}
func mapKeysToStrings(node *yaml.Node) {
if node.Kind == yaml.MappingNode {
for index, child := range node.Content {
if index%2 == 0 { // its a map key
child.Tag = "!!str"
}
}
}
for _, child := range node.Content {
mapKeysToStrings(child)
}
}
func NewJsonEncoder(indent int) Encoder {
var indentString = ""
for index := 0; index < indent; index++ {
indentString = indentString + " "
}
return &jsonEncoder{indentString}
}
func (je *jsonEncoder) CanHandleAliases() bool {
return false
}
func (je *jsonEncoder) PrintDocumentSeparator(writer io.Writer) error {
return nil
}
func (je *jsonEncoder) PrintLeadingContent(writer io.Writer, content string) error {
return nil
}
func (je *jsonEncoder) Encode(writer io.Writer, node *yaml.Node) error {
var encoder = json.NewEncoder(writer)
encoder.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
encoder.SetIndent("", je.indentString)
var dataBucket orderedMap
// firstly, convert all map keys to strings
mapKeysToStrings(node)
errorDecoding := node.Decode(&dataBucket)
if errorDecoding != nil {
return errorDecoding
}
return encoder.Encode(dataBucket)
}

23
pkg/yqlib/encoder_properties.go
View File

@ -12,18 +12,21 @@ import (
) )
type propertiesEncoder struct { type propertiesEncoder struct {
destination io.Writer
} }
func NewPropertiesEncoder(destination io.Writer) Encoder { func NewPropertiesEncoder() Encoder {
return &propertiesEncoder{destination} return &propertiesEncoder{}
} }
func (e *propertiesEncoder) PrintDocumentSeparator() error { func (pe *propertiesEncoder) CanHandleAliases() bool {
return false
}
func (pe *propertiesEncoder) PrintDocumentSeparator(writer io.Writer) error {
return nil return nil
} }
func (e *propertiesEncoder) PrintLeadingContent(content string) error { func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content string) error {
reader := bufio.NewReader(strings.NewReader(content)) reader := bufio.NewReader(strings.NewReader(content))
for { for {
@ -33,12 +36,12 @@ func (e *propertiesEncoder) PrintLeadingContent(content string) error {
} }
if strings.Contains(readline, "$yqDocSeperator$") { if strings.Contains(readline, "$yqDocSeperator$") {
if err := e.PrintDocumentSeparator(); err != nil { if err := pe.PrintDocumentSeparator(writer); err != nil {
return err return err
} }
} else { } else {
if err := writeString(e.destination, readline); err != nil { if err := writeString(writer, readline); err != nil {
return err return err
} }
} }
@ -46,7 +49,7 @@ func (e *propertiesEncoder) PrintLeadingContent(content string) error {
if errors.Is(errReading, io.EOF) { if errors.Is(errReading, io.EOF) {
if readline != "" { if readline != "" {
// the last comment we read didn't have a new line, put one in // the last comment we read didn't have a new line, put one in
if err := writeString(e.destination, "\n"); err != nil { if err := writeString(writer, "\n"); err != nil {
return err return err
} }
} }
@ -56,7 +59,7 @@ func (e *propertiesEncoder) PrintLeadingContent(content string) error {
return nil return nil
} }
func (pe *propertiesEncoder) Encode(node *yaml.Node) error { func (pe *propertiesEncoder) Encode(writer io.Writer, node *yaml.Node) error {
mapKeysToStrings(node) mapKeysToStrings(node)
p := properties.NewProperties() p := properties.NewProperties()
err := pe.doEncode(p, node, "") err := pe.doEncode(p, node, "")
@ -64,7 +67,7 @@ func (pe *propertiesEncoder) Encode(node *yaml.Node) error {
return err return err
} }
_, err = p.WriteComment(pe.destination, "#", properties.UTF8) _, err = p.WriteComment(writer, "#", properties.UTF8)
return err return err
} }

4
pkg/yqlib/encoder_properties_test.go
View File

@ -13,13 +13,13 @@ func yamlToProps(sampleYaml string) string {
var output bytes.Buffer var output bytes.Buffer
writer := bufio.NewWriter(&output) writer := bufio.NewWriter(&output)
var propsEncoder = NewPropertiesEncoder(writer) var propsEncoder = NewPropertiesEncoder()
inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder()) inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
if err != nil { if err != nil {
panic(err) panic(err)
} }
node := inputs.Front().Value.(*CandidateNode).Node node := inputs.Front().Value.(*CandidateNode).Node
err = propsEncoder.Encode(node) err = propsEncoder.Encode(writer, node)
if err != nil { if err != nil {
panic(err) panic(err)
} }

4
pkg/yqlib/encoder_test.go
View File

@ -13,13 +13,13 @@ func yamlToJson(sampleYaml string, indent int) string {
var output bytes.Buffer var output bytes.Buffer
writer := bufio.NewWriter(&output) writer := bufio.NewWriter(&output)
var jsonEncoder = NewJsonEncoder(writer, indent) var jsonEncoder = NewJsonEncoder(indent)
inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder()) inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
if err != nil { if err != nil {
panic(err) panic(err)
} }
node := inputs.Front().Value.(*CandidateNode).Node node := inputs.Front().Value.(*CandidateNode).Node
err = jsonEncoder.Encode(node) err = jsonEncoder.Encode(writer, node)
if err != nil { if err != nil {
panic(err) panic(err)
} }

93
pkg/yqlib/encoder_xml.go
View File

@ -9,83 +9,90 @@ import (
yaml "gopkg.in/yaml.v3" yaml "gopkg.in/yaml.v3"
) )
var XmlPreferences = xmlPreferences{AttributePrefix: "+", ContentName: "+content"}
type xmlEncoder struct { type xmlEncoder struct {
xmlEncoder *xml.Encoder
attributePrefix string attributePrefix string
contentName string contentName string
indentString string
} }
func NewXmlEncoder(writer io.Writer, indent int, attributePrefix string, contentName string) Encoder { func NewXmlEncoder(indent int, attributePrefix string, contentName string) Encoder {
encoder := xml.NewEncoder(writer)
var indentString = "" var indentString = ""
for index := 0; index < indent; index++ { for index := 0; index < indent; index++ {
indentString = indentString + " " indentString = indentString + " "
} }
encoder.Indent("", indentString) return &xmlEncoder{attributePrefix, contentName, indentString}
return &xmlEncoder{encoder, attributePrefix, contentName}
} }
func (e *xmlEncoder) PrintDocumentSeparator() error { func (e *xmlEncoder) CanHandleAliases() bool {
return false
}
func (e *xmlEncoder) PrintDocumentSeparator(writer io.Writer) error {
return nil return nil
} }
func (e *xmlEncoder) PrintLeadingContent(content string) error { func (e *xmlEncoder) PrintLeadingContent(writer io.Writer, content string) error {
return nil return nil
} }
func (e *xmlEncoder) Encode(node *yaml.Node) error { func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
encoder := xml.NewEncoder(writer)
encoder.Indent("", e.indentString)
switch node.Kind { switch node.Kind {
case yaml.MappingNode: case yaml.MappingNode:
err := e.encodeTopLevelMap(node) err := e.encodeTopLevelMap(encoder, node)
if err != nil { if err != nil {
return err return err
} }
case yaml.DocumentNode: case yaml.DocumentNode:
err := e.encodeComment(headAndLineComment(node)) err := e.encodeComment(encoder, headAndLineComment(node))
if err != nil { if err != nil {
return err return err
} }
// this used to call encode...
err = e.Encode(unwrapDoc(node)) err = e.encodeTopLevelMap(encoder, unwrapDoc(node))
if err != nil { if err != nil {
return err return err
} }
err = e.encodeComment(footComment(node)) err = e.encodeComment(encoder, footComment(node))
if err != nil { if err != nil {
return err return err
} }
case yaml.ScalarNode: case yaml.ScalarNode:
var charData xml.CharData = []byte(node.Value) var charData xml.CharData = []byte(node.Value)
err := e.xmlEncoder.EncodeToken(charData) err := encoder.EncodeToken(charData)
if err != nil { if err != nil {
return err return err
} }
return e.xmlEncoder.Flush() return encoder.Flush()
default: default:
return fmt.Errorf("unsupported type %v", node.Tag) return fmt.Errorf("unsupported type %v", node.Tag)
} }
var charData xml.CharData = []byte("\n") var charData xml.CharData = []byte("\n")
return e.xmlEncoder.EncodeToken(charData) return encoder.EncodeToken(charData)
} }
func (e *xmlEncoder) encodeTopLevelMap(node *yaml.Node) error { func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node) error {
for i := 0; i < len(node.Content); i += 2 { for i := 0; i < len(node.Content); i += 2 {
key := node.Content[i] key := node.Content[i]
value := node.Content[i+1] value := node.Content[i+1]
start := xml.StartElement{Name: xml.Name{Local: key.Value}} start := xml.StartElement{Name: xml.Name{Local: key.Value}}
err := e.encodeComment(headAndLineComment(key)) err := e.encodeComment(encoder, headAndLineComment(key))
if err != nil { if err != nil {
return err return err
} }
err = e.doEncode(value, start) err = e.doEncode(encoder, value, start)
if err != nil { if err != nil {
return err return err
} }
err = e.encodeComment(footComment(key)) err = e.encodeComment(encoder, footComment(key))
if err != nil { if err != nil {
return err return err
} }
@ -93,49 +100,49 @@ func (e *xmlEncoder) encodeTopLevelMap(node *yaml.Node) error {
return nil return nil
} }
func (e *xmlEncoder) encodeStart(node *yaml.Node, start xml.StartElement) error { func (e *xmlEncoder) encodeStart(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
err := e.xmlEncoder.EncodeToken(start) err := encoder.EncodeToken(start)
if err != nil { if err != nil {
return err return err
} }
return e.encodeComment(headAndLineComment(node)) return e.encodeComment(encoder, headAndLineComment(node))
} }
func (e *xmlEncoder) encodeEnd(node *yaml.Node, start xml.StartElement) error { func (e *xmlEncoder) encodeEnd(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
err := e.xmlEncoder.EncodeToken(start.End()) err := encoder.EncodeToken(start.End())
if err != nil { if err != nil {
return err return err
} }
return e.encodeComment(footComment(node)) return e.encodeComment(encoder, footComment(node))
} }
func (e *xmlEncoder) doEncode(node *yaml.Node, start xml.StartElement) error { func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
switch node.Kind { switch node.Kind {
case yaml.MappingNode: case yaml.MappingNode:
return e.encodeMap(node, start) return e.encodeMap(encoder, node, start)
case yaml.SequenceNode: case yaml.SequenceNode:
return e.encodeArray(node, start) return e.encodeArray(encoder, node, start)
case yaml.ScalarNode: case yaml.ScalarNode:
err := e.encodeStart(node, start) err := e.encodeStart(encoder, node, start)
if err != nil { if err != nil {
return err return err
} }
var charData xml.CharData = []byte(node.Value) var charData xml.CharData = []byte(node.Value)
err = e.xmlEncoder.EncodeToken(charData) err = encoder.EncodeToken(charData)
if err != nil { if err != nil {
return err return err
} }
return e.encodeEnd(node, start) return e.encodeEnd(encoder, node, start)
} }
return fmt.Errorf("unsupported type %v", node.Tag) return fmt.Errorf("unsupported type %v", node.Tag)
} }
func (e *xmlEncoder) encodeComment(commentStr string) error { func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) error {
if commentStr != "" { if commentStr != "" {
var comment xml.Comment = []byte(commentStr) var comment xml.Comment = []byte(commentStr)
err := e.xmlEncoder.EncodeToken(comment) err := encoder.EncodeToken(comment)
if err != nil { if err != nil {
return err return err
} }
@ -143,10 +150,10 @@ func (e *xmlEncoder) encodeComment(commentStr string) error {
return nil return nil
} }
func (e *xmlEncoder) encodeArray(node *yaml.Node, start xml.StartElement) error { func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
for i := 0; i < len(node.Content); i++ { for i := 0; i < len(node.Content); i++ {
value := node.Content[i] value := node.Content[i]
err := e.doEncode(value, start.Copy()) err := e.doEncode(encoder, value, start.Copy())
if err != nil { if err != nil {
return err return err
} }
@ -154,7 +161,7 @@ func (e *xmlEncoder) encodeArray(node *yaml.Node, start xml.StartElement) error
return nil return nil
} }
func (e *xmlEncoder) encodeMap(node *yaml.Node, start xml.StartElement) error { func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
//first find all the attributes and put them on the start token //first find all the attributes and put them on the start token
for i := 0; i < len(node.Content); i += 2 { for i := 0; i < len(node.Content); i += 2 {
@ -171,7 +178,7 @@ func (e *xmlEncoder) encodeMap(node *yaml.Node, start xml.StartElement) error {
} }
} }
err := e.encodeStart(node, start) err := e.encodeStart(encoder, node, start)
if err != nil { if err != nil {
return err return err
} }
@ -181,30 +188,30 @@ func (e *xmlEncoder) encodeMap(node *yaml.Node, start xml.StartElement) error {
key := node.Content[i] key := node.Content[i]
value := node.Content[i+1] value := node.Content[i+1]
err := e.encodeComment(headAndLineComment(key)) err := e.encodeComment(encoder, headAndLineComment(key))
if err != nil { if err != nil {
return err return err
} }
if !strings.HasPrefix(key.Value, e.attributePrefix) && key.Value != e.contentName { if !strings.HasPrefix(key.Value, e.attributePrefix) && key.Value != e.contentName {
start := xml.StartElement{Name: xml.Name{Local: key.Value}} start := xml.StartElement{Name: xml.Name{Local: key.Value}}
err := e.doEncode(value, start) err := e.doEncode(encoder, value, start)
if err != nil { if err != nil {
return err return err
} }
} else if key.Value == e.contentName { } else if key.Value == e.contentName {
// directly encode the contents // directly encode the contents
var charData xml.CharData = []byte(value.Value) var charData xml.CharData = []byte(value.Value)
err = e.xmlEncoder.EncodeToken(charData) err = encoder.EncodeToken(charData)
if err != nil { if err != nil {
return err return err
} }
} }
err = e.encodeComment(footComment(key)) err = e.encodeComment(encoder, footComment(key))
if err != nil { if err != nil {
return err return err
} }
} }
return e.encodeEnd(node, start) return e.encodeEnd(encoder, node, start)
} }

101
pkg/yqlib/encoder_yaml.go Normal file
View File

@ -0,0 +1,101 @@
package yqlib
import (
"bufio"
"bytes"
"errors"
"io"
"strings"
yaml "gopkg.in/yaml.v3"
)
type yamlEncoder struct {
indent int
colorise bool
printDocSeparators bool
unwrapScalar bool
}
func NewYamlEncoder(indent int, colorise bool, printDocSeparators bool, unwrapScalar bool) Encoder {
if indent < 0 {
indent = 0
}
return &yamlEncoder{indent, colorise, printDocSeparators, unwrapScalar}
}
func (ye *yamlEncoder) CanHandleAliases() bool {
return true
}
func (ye *yamlEncoder) PrintDocumentSeparator(writer io.Writer) error {
if ye.printDocSeparators {
log.Debug("-- writing doc sep")
if err := writeString(writer, "---\n"); err != nil {
return err
}
}
return nil
}
func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) error {
// log.Debug("headcommentwas [%v]", content)
reader := bufio.NewReader(strings.NewReader(content))
for {
readline, errReading := reader.ReadString('\n')
if errReading != nil && !errors.Is(errReading, io.EOF) {
return errReading
}
if strings.Contains(readline, "$yqDocSeperator$") {
if err := ye.PrintDocumentSeparator(writer); err != nil {
return err
}
} else {
if err := writeString(writer, readline); err != nil {
return err
}
}
if errors.Is(errReading, io.EOF) {
if readline != "" {
// the last comment we read didn't have a new line, put one in
if err := writeString(writer, "\n"); err != nil {
return err
}
}
break
}
}
return nil
}
func (ye *yamlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
if node.Kind == yaml.ScalarNode && ye.unwrapScalar {
return writeString(writer, node.Value+"\n")
}
destination := writer
tempBuffer := bytes.NewBuffer(nil)
if ye.colorise {
destination = tempBuffer
}
var encoder = yaml.NewEncoder(destination)
encoder.SetIndent(ye.indent)
if err := encoder.Encode(node); err != nil {
return err
}
if ye.colorise {
return colorizeAndPrint(tempBuffer.Bytes(), writer)
}
return nil
}

2
pkg/yqlib/lib.go
View File

@ -18,8 +18,6 @@ type xmlPreferences struct {
ContentName string ContentName string
} }
var XmlPreferences = xmlPreferences{AttributePrefix: "+", ContentName: "+content"}
var log = logging.MustGetLogger("yq-lib") var log = logging.MustGetLogger("yq-lib")
// GetLogger returns the yq logger instance. // GetLogger returns the yq logger instance.

4
pkg/yqlib/operator_comments.go
View File

@ -86,8 +86,8 @@ func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *
var chompRegexp = regexp.MustCompile(`\n$`) var chompRegexp = regexp.MustCompile(`\n$`)
var output bytes.Buffer var output bytes.Buffer
var writer = bufio.NewWriter(&output) var writer = bufio.NewWriter(&output)
var encoder = NewYamlEncoder(writer, 2, false, true, false) var encoder = NewYamlEncoder(2, false, false, false)
if err := encoder.PrintLeadingContent(candidate.LeadingContent); err != nil { if err := encoder.PrintLeadingContent(writer, candidate.LeadingContent); err != nil {
return Context{}, err return Context{}, err
} }
if err := writer.Flush(); err != nil { if err := writer.Flush(); err != nil {

26
pkg/yqlib/operator_encoder_decoder.go
View File

@ -10,11 +10,31 @@ import (
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
) )
func yamlToString(candidate *CandidateNode, prefs encoderPreferences) (string, error) { func configureEncoder(format PrinterOutputFormat, indent int) Encoder {
switch format {
case JsonOutputFormat:
return NewJsonEncoder(indent)
case PropsOutputFormat:
return NewPropertiesEncoder()
case CsvOutputFormat:
return NewCsvEncoder(',')
case TsvOutputFormat:
return NewCsvEncoder('\t')
case YamlOutputFormat:
return NewYamlEncoder(indent, false, true, true)
case XmlOutputFormat:
return NewXmlEncoder(indent, XmlPreferences.AttributePrefix, XmlPreferences.ContentName)
}
panic("invalid encoder")
}
func encodeToString(candidate *CandidateNode, prefs encoderPreferences) (string, error) {
var output bytes.Buffer var output bytes.Buffer
log.Debug("printing with indent: %v", prefs.indent) log.Debug("printing with indent: %v", prefs.indent)
printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), prefs.format, true, false, prefs.indent, true) encoder := configureEncoder(prefs.format, prefs.indent)
printer := NewPrinter(encoder, NewSinglePrinterWriter(bufio.NewWriter(&output)))
err := printer.PrintResults(candidate.AsList()) err := printer.PrintResults(candidate.AsList())
return output.String(), err return output.String(), err
} }
@ -36,7 +56,7 @@ func encodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode) candidate := el.Value.(*CandidateNode)
stringValue, err := yamlToString(candidate, preferences) stringValue, err := encodeToString(candidate, preferences)
if err != nil { if err != nil {
return Context{}, err return Context{}, err

10
pkg/yqlib/operators_test.go
View File

@ -26,6 +26,10 @@ type expressionScenario struct {
dontFormatInputForDoc bool // dont format input doc for documentation generation dontFormatInputForDoc bool // dont format input doc for documentation generation
} }
func NewSimpleYamlPrinter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
return NewPrinter(NewYamlEncoder(indent, colorsEnabled, printDocSeparators, unwrapScalar), NewSinglePrinterWriter(writer))
}
func readDocumentWithLeadingContent(content string, fakefilename string, fakeFileIndex int) (*list.List, error) { func readDocumentWithLeadingContent(content string, fakefilename string, fakeFileIndex int) (*list.List, error) {
reader, firstFileLeadingContent, err := processReadStream(bufio.NewReader(strings.NewReader(content))) reader, firstFileLeadingContent, err := processReadStream(bufio.NewReader(strings.NewReader(content)))
if err != nil { if err != nil {
@ -92,7 +96,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
func resultToString(t *testing.T, n *CandidateNode) string { func resultToString(t *testing.T, n *CandidateNode) string {
var valueBuffer bytes.Buffer var valueBuffer bytes.Buffer
printer := NewPrinterWithSingleWriter(bufio.NewWriter(&valueBuffer), YamlOutputFormat, true, false, 4, true) printer := NewSimpleYamlPrinter(bufio.NewWriter(&valueBuffer), YamlOutputFormat, true, false, 4, true)
err := printer.PrintResults(n.AsList()) err := printer.PrintResults(n.AsList())
if err != nil { if err != nil {
@ -145,7 +149,7 @@ func copyFromHeader(folder string, title string, out *os.File) error {
func formatYaml(yaml string, filename string) string { func formatYaml(yaml string, filename string) string {
var output bytes.Buffer var output bytes.Buffer
printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true) printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
node, err := NewExpressionParser().ParseExpression(".. style= \"\"") node, err := NewExpressionParser().ParseExpression(".. style= \"\"")
if err != nil { if err != nil {
@ -268,7 +272,7 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) {
func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formattedDoc string, formattedDoc2 string) { func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formattedDoc string, formattedDoc2 string) {
var output bytes.Buffer var output bytes.Buffer
var err error var err error
printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true) printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
node, err := NewExpressionParser().ParseExpression(s.expression) node, err := NewExpressionParser().ParseExpression(s.expression)
if err != nil { if err != nil {

68
pkg/yqlib/printer.go
View File

@ -48,35 +48,22 @@ func OutputFormatFromString(format string) (PrinterOutputFormat, error) {
} }
type resultsPrinter struct { type resultsPrinter struct {
outputFormat PrinterOutputFormat encoder Encoder
encoder Encoder printerWriter PrinterWriter
unwrapScalar bool firstTimePrinting bool
colorsEnabled bool previousDocIndex uint
indent int previousFileIndex int
printDocSeparators bool printedMatches bool
printerWriter PrinterWriter treeNavigator DataTreeNavigator
firstTimePrinting bool appendixReader io.Reader
previousDocIndex uint
previousFileIndex int
printedMatches bool
treeNavigator DataTreeNavigator
appendixReader io.Reader
} }
func NewPrinterWithSingleWriter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer { func NewPrinter(encoder Encoder, printerWriter PrinterWriter) Printer {
return NewPrinter(NewSinglePrinterWriter(writer), outputFormat, unwrapScalar, colorsEnabled, indent, printDocSeparators)
}
func NewPrinter(printerWriter PrinterWriter, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
return &resultsPrinter{ return &resultsPrinter{
printerWriter: printerWriter, encoder: encoder,
outputFormat: outputFormat, printerWriter: printerWriter,
unwrapScalar: unwrapScalar, firstTimePrinting: true,
colorsEnabled: colorsEnabled, treeNavigator: NewDataTreeNavigator(),
indent: indent,
printDocSeparators: outputFormat == YamlOutputFormat && printDocSeparators,
firstTimePrinting: true,
treeNavigator: NewDataTreeNavigator(),
} }
} }
@ -91,23 +78,7 @@ func (p *resultsPrinter) PrintedAnything() bool {
func (p *resultsPrinter) printNode(node *yaml.Node, writer io.Writer) error { func (p *resultsPrinter) printNode(node *yaml.Node, writer io.Writer) error {
p.printedMatches = p.printedMatches || (node.Tag != "!!null" && p.printedMatches = p.printedMatches || (node.Tag != "!!null" &&
(node.Tag != "!!bool" || node.Value != "false")) (node.Tag != "!!bool" || node.Value != "false"))
return p.encoder.Encode(writer, node)
// switch p.outputFormat {
// case JsonOutputFormat:
// encoder = NewJsonEncoder(writer, p.indent)
// case PropsOutputFormat:
// encoder = NewPropertiesEncoder(writer)
// case CsvOutputFormat:
// encoder = NewCsvEncoder(writer, ',')
// case TsvOutputFormat:
// encoder = NewCsvEncoder(writer, '\t')
// case YamlOutputFormat:
// encoder = NewYamlEncoder(writer, p.indent, p.colorsEnabled)
// case XmlOutputFormat:
// encoder = NewXmlEncoder(writer, p.indent, XmlPreferences.AttributePrefix, XmlPreferences.ContentName)
// }
return p.encoder.Encode(node)
} }
func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error { func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
@ -118,7 +89,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
return nil return nil
} }
if p.outputFormat != YamlOutputFormat { if !p.encoder.CanHandleAliases() {
explodeOp := Operation{OperationType: explodeOpType} explodeOp := Operation{OperationType: explodeOpType}
explodeNode := ExpressionNode{Operation: &explodeOp} explodeNode := ExpressionNode{Operation: &explodeOp}
context, err := p.treeNavigator.GetMatchingNodes(Context{MatchingNodes: matchingNodes}, &explodeNode) context, err := p.treeNavigator.GetMatchingNodes(Context{MatchingNodes: matchingNodes}, &explodeNode)
@ -129,6 +100,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
} }
if p.firstTimePrinting { if p.firstTimePrinting {
log.Debugf("its my first time *blush*")
node := matchingNodes.Front().Value.(*CandidateNode) node := matchingNodes.Front().Value.(*CandidateNode)
p.previousDocIndex = node.Document p.previousDocIndex = node.Document
p.previousFileIndex = node.FileIndex p.previousFileIndex = node.FileIndex
@ -138,7 +110,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
for el := matchingNodes.Front(); el != nil; el = el.Next() { for el := matchingNodes.Front(); el != nil; el = el.Next() {
mappedDoc := el.Value.(*CandidateNode) mappedDoc := el.Value.(*CandidateNode)
log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v, printDocSeparators: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document, p.printDocSeparators) log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document)
writer, errorWriting := p.printerWriter.GetWriter(mappedDoc) writer, errorWriting := p.printerWriter.GetWriter(mappedDoc)
if errorWriting != nil { if errorWriting != nil {
@ -149,13 +121,12 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
commentStartsWithSeparator := commentsStartWithSepExp.MatchString(mappedDoc.LeadingContent) commentStartsWithSeparator := commentsStartWithSepExp.MatchString(mappedDoc.LeadingContent)
if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && !commentStartsWithSeparator { if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && !commentStartsWithSeparator {
log.Debug("-- writing doc sep") if err := p.encoder.PrintDocumentSeparator(writer); err != nil {
if err := p.encoder.PrintDocumentSeparator(); err != nil {
return err return err
} }
} }
if err := p.encoder.PrintLeadingContent(mappedDoc.LeadingContent); err != nil { if err := p.encoder.PrintLeadingContent(writer, mappedDoc.LeadingContent); err != nil {
return err return err
} }
@ -167,6 +138,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
if err := writer.Flush(); err != nil { if err := writer.Flush(); err != nil {
return err return err
} }
log.Debugf("done printing results")
} }
// what happens if I remove output format check? // what happens if I remove output format check?

20
pkg/yqlib/printer_test.go
View File

@ -33,10 +33,10 @@ func nodeToList(candidate *CandidateNode) *list.List {
return elMap return elMap
} }
func TestPrinterMultipleDocsInSequence(t *testing.T) { func TestPrinterMultipleDocsInSequenceOnly(t *testing.T) {
var output bytes.Buffer var output bytes.Buffer
var writer = bufio.NewWriter(&output) var writer = bufio.NewWriter(&output)
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
if err != nil { if err != nil {
@ -74,7 +74,7 @@ func TestPrinterMultipleDocsInSequence(t *testing.T) {
func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) { func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
var output bytes.Buffer var output bytes.Buffer
var writer = bufio.NewWriter(&output) var writer = bufio.NewWriter(&output)
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
if err != nil { if err != nil {
@ -116,7 +116,7 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
func TestPrinterMultipleFilesInSequence(t *testing.T) { func TestPrinterMultipleFilesInSequence(t *testing.T) {
var output bytes.Buffer var output bytes.Buffer
var writer = bufio.NewWriter(&output) var writer = bufio.NewWriter(&output)
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
if err != nil { if err != nil {
@ -163,7 +163,7 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) {
func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) { func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
var output bytes.Buffer var output bytes.Buffer
var writer = bufio.NewWriter(&output) var writer = bufio.NewWriter(&output)
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
if err != nil { if err != nil {
@ -213,7 +213,7 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
func TestPrinterMultipleDocsInSinglePrint(t *testing.T) { func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
var output bytes.Buffer var output bytes.Buffer
var writer = bufio.NewWriter(&output) var writer = bufio.NewWriter(&output)
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
if err != nil { if err != nil {
@ -232,7 +232,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) { func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) {
var output bytes.Buffer var output bytes.Buffer
var writer = bufio.NewWriter(&output) var writer = bufio.NewWriter(&output)
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
if err != nil { if err != nil {
@ -261,7 +261,7 @@ a: coconut
func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) { func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) {
var output bytes.Buffer var output bytes.Buffer
var writer = bufio.NewWriter(&output) var writer = bufio.NewWriter(&output)
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
if err != nil { if err != nil {
@ -287,7 +287,7 @@ a: coconut
func TestPrinterScalarWithLeadingCont(t *testing.T) { func TestPrinterScalarWithLeadingCont(t *testing.T) {
var output bytes.Buffer var output bytes.Buffer
var writer = bufio.NewWriter(&output) var writer = bufio.NewWriter(&output)
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
node, err := NewExpressionParser().ParseExpression(".a") node, err := NewExpressionParser().ParseExpression(".a")
if err != nil { if err != nil {
@ -314,7 +314,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
var writer = bufio.NewWriter(&output) var writer = bufio.NewWriter(&output)
// note printDocSeparators is true, it should still not print document separators // note printDocSeparators is true, it should still not print document separators
// when outputing JSON. // when outputing JSON.
printer := NewPrinterWithSingleWriter(writer, JsonOutputFormat, true, false, 0, true) printer := NewPrinter(NewJsonEncoder(0), NewSinglePrinterWriter(writer))
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
if err != nil { if err != nil {

7
pkg/yqlib/utils.go
View File

@ -70,13 +70,6 @@ func processReadStream(reader *bufio.Reader) (io.Reader, string, error) {
} }
} }
func explodeNodes(nodes *list.List) (*list.List, error) {
explodeOp := Operation{OperationType: explodeOpType}
explodeNode := ExpressionNode{Operation: &explodeOp}
context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: nodes}, &explodeNode)
return context.MatchingNodes, err
}
func readDocuments(reader io.Reader, filename string, fileIndex int, decoder Decoder) (*list.List, error) { func readDocuments(reader io.Reader, filename string, fileIndex int, decoder Decoder) (*list.List, error) {
decoder.Init(reader) decoder.Init(reader)
inputList := list.New() inputList := list.New()

146
pkg/yqlib/xml_test.go
View File

@ -28,7 +28,7 @@ func processScenario(s xmlScenario) string {
var output bytes.Buffer var output bytes.Buffer
writer := bufio.NewWriter(&output) writer := bufio.NewWriter(&output)
var encoder = NewXmlEncoder(writer, 2, "+", "+content") var encoder = NewXmlEncoder(2, "+", "+content")
var decoder = NewYamlDecoder() var decoder = NewYamlDecoder()
if s.scenarioType == "roundtrip" { if s.scenarioType == "roundtrip" {
@ -40,13 +40,13 @@ func processScenario(s xmlScenario) string {
panic(err) panic(err)
} }
node := inputs.Front().Value.(*CandidateNode).Node node := inputs.Front().Value.(*CandidateNode).Node
err = encoder.Encode(node) err = encoder.Encode(writer, node)
if err != nil { if err != nil {
panic(err) panic(err)
} }
writer.Flush() writer.Flush()
return strings.TrimSuffix(output.String(), "\n") return output.String()
} }
type xmlScenario struct { type xmlScenario struct {
@ -116,75 +116,75 @@ var expectedXmlWithComments = `<!-- above_cat inline_cat--><cat><!-- above_array
` `
var xmlScenarios = []xmlScenario{ var xmlScenarios = []xmlScenario{
// { {
// description: "Parse xml: simple", description: "Parse xml: simple",
// input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat>meow</cat>", input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat>meow</cat>",
// expected: "D0, P[], (doc)::cat: meow\n", expected: "D0, P[], (doc)::cat: meow\n",
// }, },
// { {
// description: "Parse xml: array", description: "Parse xml: array",
// subdescription: "Consecutive nodes with identical xml names are assumed to be arrays.", subdescription: "Consecutive nodes with identical xml names are assumed to be arrays.",
// input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<animal>1</animal>\n<animal>2</animal>", input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<animal>1</animal>\n<animal>2</animal>",
// expected: "D0, P[], (doc)::animal:\n - \"1\"\n - \"2\"\n", expected: "D0, P[], (doc)::animal:\n - \"1\"\n - \"2\"\n",
// }, },
// { {
// description: "Parse xml: attributes", description: "Parse xml: attributes",
// subdescription: "Attributes are converted to fields, with the attribute prefix.", subdescription: "Attributes are converted to fields, with the attribute prefix.",
// input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">\n <legs>7</legs>\n</cat>", input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">\n <legs>7</legs>\n</cat>",
// expected: "D0, P[], (doc)::cat:\n +legs: \"4\"\n legs: \"7\"\n", expected: "D0, P[], (doc)::cat:\n +legs: \"4\"\n legs: \"7\"\n",
// }, },
// { {
// description: "Parse xml: attributes with content", description: "Parse xml: attributes with content",
// subdescription: "Content is added as a field, using the content name", subdescription: "Content is added as a field, using the content name",
// input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">meow</cat>", input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">meow</cat>",
// expected: "D0, P[], (doc)::cat:\n +content: meow\n +legs: \"4\"\n", expected: "D0, P[], (doc)::cat:\n +content: meow\n +legs: \"4\"\n",
// }, },
// { {
// description: "Parse xml: with comments", description: "Parse xml: with comments",
// subdescription: "A best attempt is made to preserve comments.", subdescription: "A best attempt is made to preserve comments.",
// input: inputXmlWithComments, input: inputXmlWithComments,
// expected: expectedDecodeYamlWithComments, expected: expectedDecodeYamlWithComments,
// scenarioType: "decode", scenarioType: "decode",
// }, },
// { {
// description: "Encode xml: simple", description: "Encode xml: simple",
// input: "cat: purrs", input: "cat: purrs",
// expected: "<cat>purrs</cat>\n", expected: "<cat>purrs</cat>\n",
// scenarioType: "encode", scenarioType: "encode",
// }, },
// { {
// description: "Encode xml: array", description: "Encode xml: array",
// input: "pets:\n cat:\n - purrs\n - meows", input: "pets:\n cat:\n - purrs\n - meows",
// expected: "<pets>\n <cat>purrs</cat>\n <cat>meows</cat>\n</pets>\n", expected: "<pets>\n <cat>purrs</cat>\n <cat>meows</cat>\n</pets>\n",
// scenarioType: "encode", scenarioType: "encode",
// }, },
// { {
// description: "Encode xml: attributes", description: "Encode xml: attributes",
// subdescription: "Fields with the matching xml-attribute-prefix are assumed to be attributes.", subdescription: "Fields with the matching xml-attribute-prefix are assumed to be attributes.",
// input: "cat:\n +name: tiger\n meows: true\n", input: "cat:\n +name: tiger\n meows: true\n",
// expected: "<cat name=\"tiger\">\n <meows>true</meows>\n</cat>\n", expected: "<cat name=\"tiger\">\n <meows>true</meows>\n</cat>\n",
// scenarioType: "encode", scenarioType: "encode",
// }, },
// { {
// skipDoc: true, skipDoc: true,
// input: "cat:\n ++name: tiger\n meows: true\n", input: "cat:\n ++name: tiger\n meows: true\n",
// expected: "<cat +name=\"tiger\">\n <meows>true</meows>\n</cat>\n", expected: "<cat +name=\"tiger\">\n <meows>true</meows>\n</cat>\n",
// scenarioType: "encode", scenarioType: "encode",
// }, },
// { {
// description: "Encode xml: attributes with content", description: "Encode xml: attributes with content",
// subdescription: "Fields with the matching xml-content-name is assumed to be content.", subdescription: "Fields with the matching xml-content-name is assumed to be content.",
// input: "cat:\n +name: tiger\n +content: cool\n", input: "cat:\n +name: tiger\n +content: cool\n",
// expected: "<cat name=\"tiger\">cool</cat>\n", expected: "<cat name=\"tiger\">cool</cat>\n",
// scenarioType: "encode", scenarioType: "encode",
// }, },
// { {
// description: "Encode xml: comments", description: "Encode xml: comments",
// subdescription: "A best attempt is made to copy comments to xml.", subdescription: "A best attempt is made to copy comments to xml.",
// input: yamlWithComments, input: yamlWithComments,
// expected: expectedXmlWithComments, expected: expectedXmlWithComments,
// scenarioType: "encode", scenarioType: "encode",
// }, },
{ {
description: "Round trip: with comments", description: "Round trip: with comments",
subdescription: "A best effort is made, but comment positions and white space are not preserved perfectly.", subdescription: "A best effort is made, but comment positions and white space are not preserved perfectly.",
@ -233,7 +233,7 @@ func documentXmlDecodeScenario(t *testing.T, w *bufio.Writer, s xmlScenario) {
writeOrPanic(w, "will output\n") writeOrPanic(w, "will output\n")
var output bytes.Buffer var output bytes.Buffer
printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true) printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
node := decodeXml(t, s.input) node := decodeXml(t, s.input)