diff --git a/acceptance_tests/output-format.sh b/acceptance_tests/output-format.sh index 9f2e79d9..1ba77ffe 100755 --- a/acceptance_tests/output-format.sh +++ b/acceptance_tests/output-format.sh @@ -142,4 +142,24 @@ EOM assertEquals "$expected" "$X" } +testOutputXmlMultiPropertiesMultietc() { + cat >test.yml < + + cat + asd + +EOM + + X=$(./yq e --output-format=x test.yml) + assertEquals "$expected" "$X" + + X=$(./yq ea --output-format=x test.yml) + assertEquals "$expected" "$X" +} + source ./scripts/shunit2 \ No newline at end of file diff --git a/cmd/evaluate_all_command.go b/cmd/evaluate_all_command.go index c52e2526..c01b02bb 100644 --- a/cmd/evaluate_all_command.go +++ b/cmd/evaluate_all_command.go @@ -81,8 +81,9 @@ func evaluateAll(cmd *cobra.Command, args []string) error { } printerWriter := configurePrinterWriter(format, out) + encoder := configureEncoder(format) - printer := yqlib.NewPrinter(printerWriter, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators) + printer := yqlib.NewPrinter(encoder, printerWriter) if frontMatter != "" { frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex]) diff --git a/cmd/evalute_sequence_command.go b/cmd/evalute_sequence_command.go index 2a371ed4..5ef46805 100644 --- a/cmd/evalute_sequence_command.go +++ b/cmd/evalute_sequence_command.go @@ -89,8 +89,9 @@ func evaluateSequence(cmd *cobra.Command, args []string) error { } printerWriter := configurePrinterWriter(format, out) + encoder := configureEncoder(format) - printer := yqlib.NewPrinter(printerWriter, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators) + printer := yqlib.NewPrinter(encoder, printerWriter) decoder, err := configureDecoder() if err != nil { diff --git a/cmd/utils.go b/cmd/utils.go index 7ab518a4..7688cad0 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -52,7 +52,7 @@ func configureDecoder() (yqlib.Decoder, error) { } switch yqlibInputFormat { case yqlib.XmlInputFormat: - return yqlib.NewXmlDecoder(xmlAttributePrefix, xmlContentName), nil + return yqlib.NewXmlDecoder(), nil } return yqlib.NewYamlDecoder(), nil } @@ -73,3 +73,21 @@ func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) yql } return printerWriter } + +func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder { + switch format { + case yqlib.JsonOutputFormat: + return yqlib.NewJsonEncoder(indent) + case yqlib.PropsOutputFormat: + return yqlib.NewPropertiesEncoder() + case yqlib.CsvOutputFormat: + return yqlib.NewCsvEncoder(',') + case yqlib.TsvOutputFormat: + return yqlib.NewCsvEncoder('\t') + case yqlib.YamlOutputFormat: + return yqlib.NewYamlEncoder(indent, colorsEnabled, !noDocSeparators, unwrapScalar) + case yqlib.XmlOutputFormat: + return yqlib.NewXmlEncoder(indent) + } + panic("invalid encoder") +} diff --git a/examples/data1.yaml b/examples/data1.yaml index 38071207..1bf3a5cb 100644 --- a/examples/data1.yaml +++ b/examples/data1.yaml @@ -1,4 +1 @@ -#hi ---- -# hi -- - fish \ No newline at end of file +cat: purrs diff --git a/pkg/yqlib/doc/usage/xml.md b/pkg/yqlib/doc/usage/xml.md index 408b28d7..f3a31666 100644 --- a/pkg/yqlib/doc/usage/xml.md +++ b/pkg/yqlib/doc/usage/xml.md @@ -22,6 +22,224 @@ XML nodes that have attributes then plain content, e.g: The content of the node will be set as a field in the map with the key "+content". Use the `--xml-content-name` flag to change this. +## Parse xml: simple +Given a sample.xml file of: +```xml + +meow +``` +then +```bash +yq e -p=xml '.' sample.xml +``` +will output +```yaml +cat: meow +``` + +## Parse xml: array +Consecutive nodes with identical xml names are assumed to be arrays. + +Given a sample.xml file of: +```xml + +1 +2 +``` +then +```bash +yq e -p=xml '.' sample.xml +``` +will output +```yaml +animal: + - "1" + - "2" +``` + +## Parse xml: attributes +Attributes are converted to fields, with the attribute prefix. + +Given a sample.xml file of: +```xml + + + 7 + +``` +then +```bash +yq e -p=xml '.' sample.xml +``` +will output +```yaml +cat: + +legs: "4" + legs: "7" +``` + +## Parse xml: attributes with content +Content is added as a field, using the content name + +Given a sample.xml file of: +```xml + +meow +``` +then +```bash +yq e -p=xml '.' sample.xml +``` +will output +```yaml +cat: + +content: meow + +legs: "4" +``` + +## Parse xml: with comments +A best attempt is made to preserve comments. + +Given a sample.xml file of: +```xml + + + + + 3 + + + 4 + + + + + + +``` +then +```bash +yq e -p=xml '.' sample.xml +``` +will output +```yaml +# before cat +cat: + # in cat before + x: "3" # multi + # line comment + # for x + y: + # in y before + d: "4" # in d before in d after + # in y after + +# after cat +``` + +## Encode xml: simple +Given a sample.yml file of: +```yaml +cat: purrs +``` +then +```bash +yq e -o=xml '.' sample.yml +``` +will output +```xml +purrs +``` + +## Encode xml: array +Given a sample.yml file of: +```yaml +pets: + cat: + - purrs + - meows +``` +then +```bash +yq e -o=xml '.' sample.yml +``` +will output +```xml + + purrs + meows + +``` + +## Encode xml: attributes +Fields with the matching xml-attribute-prefix are assumed to be attributes. + +Given a sample.yml file of: +```yaml +cat: + +name: tiger + meows: true + +``` +then +```bash +yq e -o=xml '.' sample.yml +``` +will output +```xml + + true + +``` + +## Encode xml: attributes with content +Fields with the matching xml-content-name is assumed to be content. + +Given a sample.yml file of: +```yaml +cat: + +name: tiger + +content: cool + +``` +then +```bash +yq e -o=xml '.' sample.yml +``` +will output +```xml +cool +``` + +## Encode xml: comments +A best attempt is made to copy comments to xml. + +Given a sample.yml file of: +```yaml +# above_cat +cat: # inline_cat + # above_array + array: # inline_array + - val1 # inline_val1 + # above_val2 + - val2 # inline_val2 +# below_cat + +``` +then +```bash +yq e -o=xml '.' sample.yml +``` +will output +```xml + + val1 + val2 + +``` + ## Round trip: with comments A best effort is made, but comment positions and white space are not preserved perfectly. diff --git a/pkg/yqlib/encoder.go b/pkg/yqlib/encoder.go index b1d83d61..d0d5af86 100644 --- a/pkg/yqlib/encoder.go +++ b/pkg/yqlib/encoder.go @@ -1,165 +1,20 @@ package yqlib import ( - "bufio" "bytes" "encoding/json" "errors" "fmt" "io" - "strings" yaml "gopkg.in/yaml.v3" ) type Encoder interface { - Encode(node *yaml.Node) error - PrintDocumentSeparator() error - PrintLeadingContent(content string) error -} - -type yamlEncoder struct { - destination io.Writer - indent int - colorise bool - firstDoc bool - printDocSeparators bool - unwrapScalar bool -} - -func NewYamlEncoder(destination io.Writer, indent int, colorise bool, printDocSeparators bool, unwrapScalar bool) Encoder { - if indent < 0 { - indent = 0 - } - return &yamlEncoder{destination, indent, colorise, true, printDocSeparators, unwrapScalar} -} -func (ye *yamlEncoder) PrintDocumentSeparator() error { - if ye.printDocSeparators { - if err := writeString(ye.destination, "---\n"); err != nil { - return err - } - } - return nil -} - -func (ye *yamlEncoder) PrintLeadingContent(content string) error { - log.Debug("headcommentwas %v", content) - log.Debug("finished headcomment") - reader := bufio.NewReader(strings.NewReader(content)) - - for { - - readline, errReading := reader.ReadString('\n') - if errReading != nil && !errors.Is(errReading, io.EOF) { - return errReading - } - if strings.Contains(readline, "$yqDocSeperator$") { - - if err := ye.PrintDocumentSeparator(); err != nil { - return err - } - - } else { - if err := writeString(ye.destination, readline); err != nil { - return err - } - } - - if errors.Is(errReading, io.EOF) { - if readline != "" { - // the last comment we read didn't have a new line, put one in - if err := writeString(ye.destination, "\n"); err != nil { - return err - } - } - break - } - } - - return nil -} - -func (ye *yamlEncoder) Encode(node *yaml.Node) error { - - if node.Kind == yaml.ScalarNode && ye.unwrapScalar { - return writeString(ye.destination, node.Value+"\n") - } - - destination := ye.destination - tempBuffer := bytes.NewBuffer(nil) - if ye.colorise { - destination = tempBuffer - } - - var encoder = yaml.NewEncoder(destination) - - encoder.SetIndent(ye.indent) - // TODO: work out if the first doc had a separator or not. - if ye.firstDoc { - ye.firstDoc = false - } else if _, err := destination.Write([]byte("---\n")); err != nil { - return err - } - - if err := encoder.Encode(node); err != nil { - return err - } - - if ye.colorise { - return colorizeAndPrint(tempBuffer.Bytes(), ye.destination) - } - return nil -} - -type jsonEncoder struct { - encoder *json.Encoder -} - -func mapKeysToStrings(node *yaml.Node) { - - if node.Kind == yaml.MappingNode { - for index, child := range node.Content { - if index%2 == 0 { // its a map key - child.Tag = "!!str" - } - } - } - - for _, child := range node.Content { - mapKeysToStrings(child) - } -} - -func NewJsonEncoder(destination io.Writer, indent int) Encoder { - var encoder = json.NewEncoder(destination) - encoder.SetEscapeHTML(false) // do not escape html chars e.g. &, <, > - - var indentString = "" - - for index := 0; index < indent; index++ { - indentString = indentString + " " - } - encoder.SetIndent("", indentString) - return &jsonEncoder{encoder} -} - -func (je *jsonEncoder) PrintDocumentSeparator() error { - return nil -} - -func (je *jsonEncoder) PrintLeadingContent(content string) error { - return nil -} - -func (je *jsonEncoder) Encode(node *yaml.Node) error { - var dataBucket orderedMap - // firstly, convert all map keys to strings - mapKeysToStrings(node) - errorDecoding := node.Decode(&dataBucket) - if errorDecoding != nil { - return errorDecoding - } - return je.encoder.Encode(dataBucket) + Encode(writer io.Writer, node *yaml.Node) error + PrintDocumentSeparator(writer io.Writer) error + PrintLeadingContent(writer io.Writer, content string) error + CanHandleAliases() bool } // orderedMap allows to marshal and unmarshal JSON and YAML values keeping the diff --git a/pkg/yqlib/encoder_csv.go b/pkg/yqlib/encoder_csv.go index 2f16dbb5..2e321fa5 100644 --- a/pkg/yqlib/encoder_csv.go +++ b/pkg/yqlib/encoder_csv.go @@ -9,24 +9,26 @@ import ( ) type csvEncoder struct { - destination csv.Writer + separator rune } -func NewCsvEncoder(destination io.Writer, separator rune) Encoder { - csvWriter := *csv.NewWriter(destination) - csvWriter.Comma = separator - return &csvEncoder{csvWriter} +func NewCsvEncoder(separator rune) Encoder { + return &csvEncoder{separator} } -func (e *csvEncoder) PrintDocumentSeparator() error { +func (e *csvEncoder) CanHandleAliases() bool { + return false +} + +func (e *csvEncoder) PrintDocumentSeparator(writer io.Writer) error { return nil } -func (e *csvEncoder) PrintLeadingContent(content string) error { +func (e *csvEncoder) PrintLeadingContent(writer io.Writer, content string) error { return nil } -func (e *csvEncoder) encodeRow(contents []*yaml.Node) error { +func (e *csvEncoder) encodeRow(csvWriter *csv.Writer, contents []*yaml.Node) error { stringValues := make([]string, len(contents)) for i, child := range contents { @@ -36,10 +38,13 @@ func (e *csvEncoder) encodeRow(contents []*yaml.Node) error { } stringValues[i] = child.Value } - return e.destination.Write(stringValues) + return csvWriter.Write(stringValues) } -func (e *csvEncoder) Encode(originalNode *yaml.Node) error { +func (e *csvEncoder) Encode(writer io.Writer, originalNode *yaml.Node) error { + csvWriter := csv.NewWriter(writer) + csvWriter.Comma = e.separator + // node must be a sequence node := unwrapDoc(originalNode) if node.Kind != yaml.SequenceNode { @@ -48,7 +53,7 @@ func (e *csvEncoder) Encode(originalNode *yaml.Node) error { return nil } if node.Content[0].Kind == yaml.ScalarNode { - return e.encodeRow(node.Content) + return e.encodeRow(csvWriter, node.Content) } for i, child := range node.Content { @@ -56,7 +61,7 @@ func (e *csvEncoder) Encode(originalNode *yaml.Node) error { if child.Kind != yaml.SequenceNode { return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag) } - err := e.encodeRow(child.Content) + err := e.encodeRow(csvWriter, child.Content) if err != nil { return err } diff --git a/pkg/yqlib/encoder_csv_test.go b/pkg/yqlib/encoder_csv_test.go index 816cbfe4..9e636c98 100644 --- a/pkg/yqlib/encoder_csv_test.go +++ b/pkg/yqlib/encoder_csv_test.go @@ -13,13 +13,13 @@ func yamlToCsv(sampleYaml string, separator rune) string { var output bytes.Buffer writer := bufio.NewWriter(&output) - var jsonEncoder = NewCsvEncoder(writer, separator) + var jsonEncoder = NewCsvEncoder(separator) inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder()) if err != nil { panic(err) } node := inputs.Front().Value.(*CandidateNode).Node - err = jsonEncoder.Encode(node) + err = jsonEncoder.Encode(writer, node) if err != nil { panic(err) } diff --git a/pkg/yqlib/encoder_json.go b/pkg/yqlib/encoder_json.go new file mode 100644 index 00000000..cfe96e9e --- /dev/null +++ b/pkg/yqlib/encoder_json.go @@ -0,0 +1,64 @@ +package yqlib + +import ( + "encoding/json" + "io" + + yaml "gopkg.in/yaml.v3" +) + +type jsonEncoder struct { + indentString string +} + +func mapKeysToStrings(node *yaml.Node) { + + if node.Kind == yaml.MappingNode { + for index, child := range node.Content { + if index%2 == 0 { // its a map key + child.Tag = "!!str" + } + } + } + + for _, child := range node.Content { + mapKeysToStrings(child) + } +} + +func NewJsonEncoder(indent int) Encoder { + var indentString = "" + + for index := 0; index < indent; index++ { + indentString = indentString + " " + } + + return &jsonEncoder{indentString} +} + +func (je *jsonEncoder) CanHandleAliases() bool { + return false +} + +func (je *jsonEncoder) PrintDocumentSeparator(writer io.Writer) error { + return nil +} + +func (je *jsonEncoder) PrintLeadingContent(writer io.Writer, content string) error { + return nil +} + +func (je *jsonEncoder) Encode(writer io.Writer, node *yaml.Node) error { + var encoder = json.NewEncoder(writer) + encoder.SetEscapeHTML(false) // do not escape html chars e.g. &, <, > + encoder.SetIndent("", je.indentString) + + var dataBucket orderedMap + // firstly, convert all map keys to strings + mapKeysToStrings(node) + errorDecoding := node.Decode(&dataBucket) + if errorDecoding != nil { + return errorDecoding + } + return encoder.Encode(dataBucket) +} diff --git a/pkg/yqlib/encoder_properties.go b/pkg/yqlib/encoder_properties.go index 36c86a2e..287b707f 100644 --- a/pkg/yqlib/encoder_properties.go +++ b/pkg/yqlib/encoder_properties.go @@ -12,18 +12,21 @@ import ( ) type propertiesEncoder struct { - destination io.Writer } -func NewPropertiesEncoder(destination io.Writer) Encoder { - return &propertiesEncoder{destination} +func NewPropertiesEncoder() Encoder { + return &propertiesEncoder{} } -func (e *propertiesEncoder) PrintDocumentSeparator() error { +func (pe *propertiesEncoder) CanHandleAliases() bool { + return false +} + +func (pe *propertiesEncoder) PrintDocumentSeparator(writer io.Writer) error { return nil } -func (e *propertiesEncoder) PrintLeadingContent(content string) error { +func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content string) error { reader := bufio.NewReader(strings.NewReader(content)) for { @@ -33,12 +36,12 @@ func (e *propertiesEncoder) PrintLeadingContent(content string) error { } if strings.Contains(readline, "$yqDocSeperator$") { - if err := e.PrintDocumentSeparator(); err != nil { + if err := pe.PrintDocumentSeparator(writer); err != nil { return err } } else { - if err := writeString(e.destination, readline); err != nil { + if err := writeString(writer, readline); err != nil { return err } } @@ -46,7 +49,7 @@ func (e *propertiesEncoder) PrintLeadingContent(content string) error { if errors.Is(errReading, io.EOF) { if readline != "" { // the last comment we read didn't have a new line, put one in - if err := writeString(e.destination, "\n"); err != nil { + if err := writeString(writer, "\n"); err != nil { return err } } @@ -56,7 +59,7 @@ func (e *propertiesEncoder) PrintLeadingContent(content string) error { return nil } -func (pe *propertiesEncoder) Encode(node *yaml.Node) error { +func (pe *propertiesEncoder) Encode(writer io.Writer, node *yaml.Node) error { mapKeysToStrings(node) p := properties.NewProperties() err := pe.doEncode(p, node, "") @@ -64,7 +67,7 @@ func (pe *propertiesEncoder) Encode(node *yaml.Node) error { return err } - _, err = p.WriteComment(pe.destination, "#", properties.UTF8) + _, err = p.WriteComment(writer, "#", properties.UTF8) return err } diff --git a/pkg/yqlib/encoder_properties_test.go b/pkg/yqlib/encoder_properties_test.go index f82af12a..97e1c62b 100644 --- a/pkg/yqlib/encoder_properties_test.go +++ b/pkg/yqlib/encoder_properties_test.go @@ -13,13 +13,13 @@ func yamlToProps(sampleYaml string) string { var output bytes.Buffer writer := bufio.NewWriter(&output) - var propsEncoder = NewPropertiesEncoder(writer) + var propsEncoder = NewPropertiesEncoder() inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder()) if err != nil { panic(err) } node := inputs.Front().Value.(*CandidateNode).Node - err = propsEncoder.Encode(node) + err = propsEncoder.Encode(writer, node) if err != nil { panic(err) } diff --git a/pkg/yqlib/encoder_test.go b/pkg/yqlib/encoder_test.go index 576cfb95..e21fe695 100644 --- a/pkg/yqlib/encoder_test.go +++ b/pkg/yqlib/encoder_test.go @@ -13,13 +13,13 @@ func yamlToJson(sampleYaml string, indent int) string { var output bytes.Buffer writer := bufio.NewWriter(&output) - var jsonEncoder = NewJsonEncoder(writer, indent) + var jsonEncoder = NewJsonEncoder(indent) inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder()) if err != nil { panic(err) } node := inputs.Front().Value.(*CandidateNode).Node - err = jsonEncoder.Encode(node) + err = jsonEncoder.Encode(writer, node) if err != nil { panic(err) } diff --git a/pkg/yqlib/encoder_xml.go b/pkg/yqlib/encoder_xml.go index ff307409..329c57fa 100644 --- a/pkg/yqlib/encoder_xml.go +++ b/pkg/yqlib/encoder_xml.go @@ -9,83 +9,90 @@ import ( yaml "gopkg.in/yaml.v3" ) +var XmlPreferences = xmlPreferences{AttributePrefix: "+", ContentName: "+content"} + type xmlEncoder struct { - xmlEncoder *xml.Encoder attributePrefix string contentName string + indentString string } -func NewXmlEncoder(writer io.Writer, indent int, attributePrefix string, contentName string) Encoder { - encoder := xml.NewEncoder(writer) +func NewXmlEncoder(indent int, attributePrefix string, contentName string) Encoder { var indentString = "" for index := 0; index < indent; index++ { indentString = indentString + " " } - encoder.Indent("", indentString) - return &xmlEncoder{encoder, attributePrefix, contentName} + return &xmlEncoder{attributePrefix, contentName, indentString} } -func (e *xmlEncoder) PrintDocumentSeparator() error { +func (e *xmlEncoder) CanHandleAliases() bool { + return false +} + +func (e *xmlEncoder) PrintDocumentSeparator(writer io.Writer) error { return nil } -func (e *xmlEncoder) PrintLeadingContent(content string) error { +func (e *xmlEncoder) PrintLeadingContent(writer io.Writer, content string) error { return nil } -func (e *xmlEncoder) Encode(node *yaml.Node) error { +func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error { + encoder := xml.NewEncoder(writer) + encoder.Indent("", e.indentString) + switch node.Kind { case yaml.MappingNode: - err := e.encodeTopLevelMap(node) + err := e.encodeTopLevelMap(encoder, node) if err != nil { return err } case yaml.DocumentNode: - err := e.encodeComment(headAndLineComment(node)) + err := e.encodeComment(encoder, headAndLineComment(node)) if err != nil { return err } - - err = e.Encode(unwrapDoc(node)) + // this used to call encode... + err = e.encodeTopLevelMap(encoder, unwrapDoc(node)) if err != nil { return err } - err = e.encodeComment(footComment(node)) + err = e.encodeComment(encoder, footComment(node)) if err != nil { return err } case yaml.ScalarNode: var charData xml.CharData = []byte(node.Value) - err := e.xmlEncoder.EncodeToken(charData) + err := encoder.EncodeToken(charData) if err != nil { return err } - return e.xmlEncoder.Flush() + return encoder.Flush() default: return fmt.Errorf("unsupported type %v", node.Tag) } var charData xml.CharData = []byte("\n") - return e.xmlEncoder.EncodeToken(charData) + return encoder.EncodeToken(charData) } -func (e *xmlEncoder) encodeTopLevelMap(node *yaml.Node) error { +func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node) error { for i := 0; i < len(node.Content); i += 2 { key := node.Content[i] value := node.Content[i+1] start := xml.StartElement{Name: xml.Name{Local: key.Value}} - err := e.encodeComment(headAndLineComment(key)) + err := e.encodeComment(encoder, headAndLineComment(key)) if err != nil { return err } - err = e.doEncode(value, start) + err = e.doEncode(encoder, value, start) if err != nil { return err } - err = e.encodeComment(footComment(key)) + err = e.encodeComment(encoder, footComment(key)) if err != nil { return err } @@ -93,49 +100,49 @@ func (e *xmlEncoder) encodeTopLevelMap(node *yaml.Node) error { return nil } -func (e *xmlEncoder) encodeStart(node *yaml.Node, start xml.StartElement) error { - err := e.xmlEncoder.EncodeToken(start) +func (e *xmlEncoder) encodeStart(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error { + err := encoder.EncodeToken(start) if err != nil { return err } - return e.encodeComment(headAndLineComment(node)) + return e.encodeComment(encoder, headAndLineComment(node)) } -func (e *xmlEncoder) encodeEnd(node *yaml.Node, start xml.StartElement) error { - err := e.xmlEncoder.EncodeToken(start.End()) +func (e *xmlEncoder) encodeEnd(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error { + err := encoder.EncodeToken(start.End()) if err != nil { return err } - return e.encodeComment(footComment(node)) + return e.encodeComment(encoder, footComment(node)) } -func (e *xmlEncoder) doEncode(node *yaml.Node, start xml.StartElement) error { +func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error { switch node.Kind { case yaml.MappingNode: - return e.encodeMap(node, start) + return e.encodeMap(encoder, node, start) case yaml.SequenceNode: - return e.encodeArray(node, start) + return e.encodeArray(encoder, node, start) case yaml.ScalarNode: - err := e.encodeStart(node, start) + err := e.encodeStart(encoder, node, start) if err != nil { return err } var charData xml.CharData = []byte(node.Value) - err = e.xmlEncoder.EncodeToken(charData) + err = encoder.EncodeToken(charData) if err != nil { return err } - return e.encodeEnd(node, start) + return e.encodeEnd(encoder, node, start) } return fmt.Errorf("unsupported type %v", node.Tag) } -func (e *xmlEncoder) encodeComment(commentStr string) error { +func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) error { if commentStr != "" { var comment xml.Comment = []byte(commentStr) - err := e.xmlEncoder.EncodeToken(comment) + err := encoder.EncodeToken(comment) if err != nil { return err } @@ -143,10 +150,10 @@ func (e *xmlEncoder) encodeComment(commentStr string) error { return nil } -func (e *xmlEncoder) encodeArray(node *yaml.Node, start xml.StartElement) error { +func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error { for i := 0; i < len(node.Content); i++ { value := node.Content[i] - err := e.doEncode(value, start.Copy()) + err := e.doEncode(encoder, value, start.Copy()) if err != nil { return err } @@ -154,7 +161,7 @@ func (e *xmlEncoder) encodeArray(node *yaml.Node, start xml.StartElement) error return nil } -func (e *xmlEncoder) encodeMap(node *yaml.Node, start xml.StartElement) error { +func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error { //first find all the attributes and put them on the start token for i := 0; i < len(node.Content); i += 2 { @@ -171,7 +178,7 @@ func (e *xmlEncoder) encodeMap(node *yaml.Node, start xml.StartElement) error { } } - err := e.encodeStart(node, start) + err := e.encodeStart(encoder, node, start) if err != nil { return err } @@ -181,30 +188,30 @@ func (e *xmlEncoder) encodeMap(node *yaml.Node, start xml.StartElement) error { key := node.Content[i] value := node.Content[i+1] - err := e.encodeComment(headAndLineComment(key)) + err := e.encodeComment(encoder, headAndLineComment(key)) if err != nil { return err } if !strings.HasPrefix(key.Value, e.attributePrefix) && key.Value != e.contentName { start := xml.StartElement{Name: xml.Name{Local: key.Value}} - err := e.doEncode(value, start) + err := e.doEncode(encoder, value, start) if err != nil { return err } } else if key.Value == e.contentName { // directly encode the contents var charData xml.CharData = []byte(value.Value) - err = e.xmlEncoder.EncodeToken(charData) + err = encoder.EncodeToken(charData) if err != nil { return err } } - err = e.encodeComment(footComment(key)) + err = e.encodeComment(encoder, footComment(key)) if err != nil { return err } } - return e.encodeEnd(node, start) + return e.encodeEnd(encoder, node, start) } diff --git a/pkg/yqlib/encoder_yaml.go b/pkg/yqlib/encoder_yaml.go new file mode 100644 index 00000000..39142c19 --- /dev/null +++ b/pkg/yqlib/encoder_yaml.go @@ -0,0 +1,101 @@ +package yqlib + +import ( + "bufio" + "bytes" + "errors" + "io" + "strings" + + yaml "gopkg.in/yaml.v3" +) + +type yamlEncoder struct { + indent int + colorise bool + printDocSeparators bool + unwrapScalar bool +} + +func NewYamlEncoder(indent int, colorise bool, printDocSeparators bool, unwrapScalar bool) Encoder { + if indent < 0 { + indent = 0 + } + return &yamlEncoder{indent, colorise, printDocSeparators, unwrapScalar} +} + +func (ye *yamlEncoder) CanHandleAliases() bool { + return true +} + +func (ye *yamlEncoder) PrintDocumentSeparator(writer io.Writer) error { + if ye.printDocSeparators { + log.Debug("-- writing doc sep") + if err := writeString(writer, "---\n"); err != nil { + return err + } + } + return nil +} + +func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) error { + // log.Debug("headcommentwas [%v]", content) + reader := bufio.NewReader(strings.NewReader(content)) + + for { + + readline, errReading := reader.ReadString('\n') + if errReading != nil && !errors.Is(errReading, io.EOF) { + return errReading + } + if strings.Contains(readline, "$yqDocSeperator$") { + + if err := ye.PrintDocumentSeparator(writer); err != nil { + return err + } + + } else { + if err := writeString(writer, readline); err != nil { + return err + } + } + + if errors.Is(errReading, io.EOF) { + if readline != "" { + // the last comment we read didn't have a new line, put one in + if err := writeString(writer, "\n"); err != nil { + return err + } + } + break + } + } + + return nil +} + +func (ye *yamlEncoder) Encode(writer io.Writer, node *yaml.Node) error { + + if node.Kind == yaml.ScalarNode && ye.unwrapScalar { + return writeString(writer, node.Value+"\n") + } + + destination := writer + tempBuffer := bytes.NewBuffer(nil) + if ye.colorise { + destination = tempBuffer + } + + var encoder = yaml.NewEncoder(destination) + + encoder.SetIndent(ye.indent) + + if err := encoder.Encode(node); err != nil { + return err + } + + if ye.colorise { + return colorizeAndPrint(tempBuffer.Bytes(), writer) + } + return nil +} diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index e1fb18b6..00c6c190 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -18,8 +18,6 @@ type xmlPreferences struct { ContentName string } -var XmlPreferences = xmlPreferences{AttributePrefix: "+", ContentName: "+content"} - var log = logging.MustGetLogger("yq-lib") // GetLogger returns the yq logger instance. diff --git a/pkg/yqlib/operator_comments.go b/pkg/yqlib/operator_comments.go index 7b11c1c0..50a88a82 100644 --- a/pkg/yqlib/operator_comments.go +++ b/pkg/yqlib/operator_comments.go @@ -86,8 +86,8 @@ func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode * var chompRegexp = regexp.MustCompile(`\n$`) var output bytes.Buffer var writer = bufio.NewWriter(&output) - var encoder = NewYamlEncoder(writer, 2, false, true, false) - if err := encoder.PrintLeadingContent(candidate.LeadingContent); err != nil { + var encoder = NewYamlEncoder(2, false, false, false) + if err := encoder.PrintLeadingContent(writer, candidate.LeadingContent); err != nil { return Context{}, err } if err := writer.Flush(); err != nil { diff --git a/pkg/yqlib/operator_encoder_decoder.go b/pkg/yqlib/operator_encoder_decoder.go index ea7c51ee..940c87c4 100644 --- a/pkg/yqlib/operator_encoder_decoder.go +++ b/pkg/yqlib/operator_encoder_decoder.go @@ -10,11 +10,31 @@ import ( "gopkg.in/yaml.v3" ) -func yamlToString(candidate *CandidateNode, prefs encoderPreferences) (string, error) { +func configureEncoder(format PrinterOutputFormat, indent int) Encoder { + switch format { + case JsonOutputFormat: + return NewJsonEncoder(indent) + case PropsOutputFormat: + return NewPropertiesEncoder() + case CsvOutputFormat: + return NewCsvEncoder(',') + case TsvOutputFormat: + return NewCsvEncoder('\t') + case YamlOutputFormat: + return NewYamlEncoder(indent, false, true, true) + case XmlOutputFormat: + return NewXmlEncoder(indent, XmlPreferences.AttributePrefix, XmlPreferences.ContentName) + } + panic("invalid encoder") +} + +func encodeToString(candidate *CandidateNode, prefs encoderPreferences) (string, error) { var output bytes.Buffer log.Debug("printing with indent: %v", prefs.indent) - printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), prefs.format, true, false, prefs.indent, true) + encoder := configureEncoder(prefs.format, prefs.indent) + + printer := NewPrinter(encoder, NewSinglePrinterWriter(bufio.NewWriter(&output))) err := printer.PrintResults(candidate.AsList()) return output.String(), err } @@ -36,7 +56,7 @@ func encodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) - stringValue, err := yamlToString(candidate, preferences) + stringValue, err := encodeToString(candidate, preferences) if err != nil { return Context{}, err diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index c99c63a7..cc697cf0 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -26,6 +26,10 @@ type expressionScenario struct { dontFormatInputForDoc bool // dont format input doc for documentation generation } +func NewSimpleYamlPrinter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer { + return NewPrinter(NewYamlEncoder(indent, colorsEnabled, printDocSeparators, unwrapScalar), NewSinglePrinterWriter(writer)) +} + func readDocumentWithLeadingContent(content string, fakefilename string, fakeFileIndex int) (*list.List, error) { reader, firstFileLeadingContent, err := processReadStream(bufio.NewReader(strings.NewReader(content))) if err != nil { @@ -92,7 +96,7 @@ func testScenario(t *testing.T, s *expressionScenario) { func resultToString(t *testing.T, n *CandidateNode) string { var valueBuffer bytes.Buffer - printer := NewPrinterWithSingleWriter(bufio.NewWriter(&valueBuffer), YamlOutputFormat, true, false, 4, true) + printer := NewSimpleYamlPrinter(bufio.NewWriter(&valueBuffer), YamlOutputFormat, true, false, 4, true) err := printer.PrintResults(n.AsList()) if err != nil { @@ -145,7 +149,7 @@ func copyFromHeader(folder string, title string, out *os.File) error { func formatYaml(yaml string, filename string) string { var output bytes.Buffer - printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true) + printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true) node, err := NewExpressionParser().ParseExpression(".. style= \"\"") if err != nil { @@ -268,7 +272,7 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) { func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formattedDoc string, formattedDoc2 string) { var output bytes.Buffer var err error - printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true) + printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true) node, err := NewExpressionParser().ParseExpression(s.expression) if err != nil { diff --git a/pkg/yqlib/printer.go b/pkg/yqlib/printer.go index d5fedccc..82517414 100644 --- a/pkg/yqlib/printer.go +++ b/pkg/yqlib/printer.go @@ -48,35 +48,22 @@ func OutputFormatFromString(format string) (PrinterOutputFormat, error) { } type resultsPrinter struct { - outputFormat PrinterOutputFormat - encoder Encoder - unwrapScalar bool - colorsEnabled bool - indent int - printDocSeparators bool - printerWriter PrinterWriter - firstTimePrinting bool - previousDocIndex uint - previousFileIndex int - printedMatches bool - treeNavigator DataTreeNavigator - appendixReader io.Reader + encoder Encoder + printerWriter PrinterWriter + firstTimePrinting bool + previousDocIndex uint + previousFileIndex int + printedMatches bool + treeNavigator DataTreeNavigator + appendixReader io.Reader } -func NewPrinterWithSingleWriter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer { - return NewPrinter(NewSinglePrinterWriter(writer), outputFormat, unwrapScalar, colorsEnabled, indent, printDocSeparators) -} - -func NewPrinter(printerWriter PrinterWriter, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer { +func NewPrinter(encoder Encoder, printerWriter PrinterWriter) Printer { return &resultsPrinter{ - printerWriter: printerWriter, - outputFormat: outputFormat, - unwrapScalar: unwrapScalar, - colorsEnabled: colorsEnabled, - indent: indent, - printDocSeparators: outputFormat == YamlOutputFormat && printDocSeparators, - firstTimePrinting: true, - treeNavigator: NewDataTreeNavigator(), + encoder: encoder, + printerWriter: printerWriter, + firstTimePrinting: true, + treeNavigator: NewDataTreeNavigator(), } } @@ -91,23 +78,7 @@ func (p *resultsPrinter) PrintedAnything() bool { func (p *resultsPrinter) printNode(node *yaml.Node, writer io.Writer) error { p.printedMatches = p.printedMatches || (node.Tag != "!!null" && (node.Tag != "!!bool" || node.Value != "false")) - - // switch p.outputFormat { - // case JsonOutputFormat: - // encoder = NewJsonEncoder(writer, p.indent) - // case PropsOutputFormat: - // encoder = NewPropertiesEncoder(writer) - // case CsvOutputFormat: - // encoder = NewCsvEncoder(writer, ',') - // case TsvOutputFormat: - // encoder = NewCsvEncoder(writer, '\t') - // case YamlOutputFormat: - // encoder = NewYamlEncoder(writer, p.indent, p.colorsEnabled) - // case XmlOutputFormat: - // encoder = NewXmlEncoder(writer, p.indent, XmlPreferences.AttributePrefix, XmlPreferences.ContentName) - // } - - return p.encoder.Encode(node) + return p.encoder.Encode(writer, node) } func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error { @@ -118,7 +89,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error { return nil } - if p.outputFormat != YamlOutputFormat { + if !p.encoder.CanHandleAliases() { explodeOp := Operation{OperationType: explodeOpType} explodeNode := ExpressionNode{Operation: &explodeOp} context, err := p.treeNavigator.GetMatchingNodes(Context{MatchingNodes: matchingNodes}, &explodeNode) @@ -129,6 +100,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error { } if p.firstTimePrinting { + log.Debugf("its my first time *blush*") node := matchingNodes.Front().Value.(*CandidateNode) p.previousDocIndex = node.Document p.previousFileIndex = node.FileIndex @@ -138,7 +110,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error { for el := matchingNodes.Front(); el != nil; el = el.Next() { mappedDoc := el.Value.(*CandidateNode) - log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v, printDocSeparators: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document, p.printDocSeparators) + log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document) writer, errorWriting := p.printerWriter.GetWriter(mappedDoc) if errorWriting != nil { @@ -149,13 +121,12 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error { commentStartsWithSeparator := commentsStartWithSepExp.MatchString(mappedDoc.LeadingContent) if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && !commentStartsWithSeparator { - log.Debug("-- writing doc sep") - if err := p.encoder.PrintDocumentSeparator(); err != nil { + if err := p.encoder.PrintDocumentSeparator(writer); err != nil { return err } } - if err := p.encoder.PrintLeadingContent(mappedDoc.LeadingContent); err != nil { + if err := p.encoder.PrintLeadingContent(writer, mappedDoc.LeadingContent); err != nil { return err } @@ -167,6 +138,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error { if err := writer.Flush(); err != nil { return err } + log.Debugf("done printing results") } // what happens if I remove output format check? diff --git a/pkg/yqlib/printer_test.go b/pkg/yqlib/printer_test.go index d3172906..d23f9b96 100644 --- a/pkg/yqlib/printer_test.go +++ b/pkg/yqlib/printer_test.go @@ -33,10 +33,10 @@ func nodeToList(candidate *CandidateNode) *list.List { return elMap } -func TestPrinterMultipleDocsInSequence(t *testing.T) { +func TestPrinterMultipleDocsInSequenceOnly(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) - printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) + printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) if err != nil { @@ -74,7 +74,7 @@ func TestPrinterMultipleDocsInSequence(t *testing.T) { func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) - printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) + printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) if err != nil { @@ -116,7 +116,7 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) { func TestPrinterMultipleFilesInSequence(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) - printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) + printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) if err != nil { @@ -163,7 +163,7 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) { func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) - printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) + printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) if err != nil { @@ -213,7 +213,7 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) { func TestPrinterMultipleDocsInSinglePrint(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) - printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) + printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) if err != nil { @@ -232,7 +232,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) { func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) - printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) + printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) if err != nil { @@ -261,7 +261,7 @@ a: coconut func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) - printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) + printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) if err != nil { @@ -287,7 +287,7 @@ a: coconut func TestPrinterScalarWithLeadingCont(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) - printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) + printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) node, err := NewExpressionParser().ParseExpression(".a") if err != nil { @@ -314,7 +314,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) { var writer = bufio.NewWriter(&output) // note printDocSeparators is true, it should still not print document separators // when outputing JSON. - printer := NewPrinterWithSingleWriter(writer, JsonOutputFormat, true, false, 0, true) + printer := NewPrinter(NewJsonEncoder(0), NewSinglePrinterWriter(writer)) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) if err != nil { diff --git a/pkg/yqlib/utils.go b/pkg/yqlib/utils.go index 3914e967..47871f05 100644 --- a/pkg/yqlib/utils.go +++ b/pkg/yqlib/utils.go @@ -70,13 +70,6 @@ func processReadStream(reader *bufio.Reader) (io.Reader, string, error) { } } -func explodeNodes(nodes *list.List) (*list.List, error) { - explodeOp := Operation{OperationType: explodeOpType} - explodeNode := ExpressionNode{Operation: &explodeOp} - context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: nodes}, &explodeNode) - return context.MatchingNodes, err -} - func readDocuments(reader io.Reader, filename string, fileIndex int, decoder Decoder) (*list.List, error) { decoder.Init(reader) inputList := list.New() diff --git a/pkg/yqlib/xml_test.go b/pkg/yqlib/xml_test.go index 91b4ca2b..72de0fa6 100644 --- a/pkg/yqlib/xml_test.go +++ b/pkg/yqlib/xml_test.go @@ -28,7 +28,7 @@ func processScenario(s xmlScenario) string { var output bytes.Buffer writer := bufio.NewWriter(&output) - var encoder = NewXmlEncoder(writer, 2, "+", "+content") + var encoder = NewXmlEncoder(2, "+", "+content") var decoder = NewYamlDecoder() if s.scenarioType == "roundtrip" { @@ -40,13 +40,13 @@ func processScenario(s xmlScenario) string { panic(err) } node := inputs.Front().Value.(*CandidateNode).Node - err = encoder.Encode(node) + err = encoder.Encode(writer, node) if err != nil { panic(err) } writer.Flush() - return strings.TrimSuffix(output.String(), "\n") + return output.String() } type xmlScenario struct { @@ -116,75 +116,75 @@ var expectedXmlWithComments = `