Split printer

This commit is contained in:
Mike Farah 2021-10-29 14:14:39 +11:00
parent 65fd001575
commit 0b2688c0f1
18 changed files with 385 additions and 71 deletions

6
.gitignore vendored
View File

@ -38,8 +38,12 @@ parts/
prime/
.snapcraft/
yq*.snap
test.yml
test2.yml
test*.yml
0.yml
1.yml
2.yml
# man page

View File

@ -1,7 +1,7 @@
#!/bin/bash
setUp() {
rm -f test.yml
rm test*.yml || true
}
testBasicEvalRoundTrip() {

View File

@ -1,6 +1,7 @@
#!/bin/bash
setUp() {
rm test*.yml || true
cat >test.yml <<EOL
# comment
EOL

View File

@ -1,6 +1,7 @@
#!/bin/bash
setUp() {
rm test*.yml || true
cat >test.yml <<EOL
---
a: apple

View File

@ -1,6 +1,7 @@
#!/bin/bash
setUp() {
rm test*.yml || true
cat >test.yml <<EOL
---
a: test

View File

@ -1,5 +1,9 @@
#!/bin/bash
setUp() {
rm test*.yml || true
}
testOutputJsonDeprecated() {
cat >test.yml <<EOL
a: {b: ["cat"]}

View File

@ -1,5 +1,8 @@
#!/bin/bash
setUp() {
rm test*.yml || true
}
testPrettyPrintWithBooleans() {
cat >test.yml <<EOL

157
acceptance_tests/split-printer.sh Executable file
View File

@ -0,0 +1,157 @@
#!/bin/bash
setUp() {
rm test*.yml || true
}
testBasicSplitWithName() {
cat >test.yml <<EOL
a: test_doc1
---
a: test_doc2
EOL
./yq e test.yml -s ".a"
doc1=$(cat test_doc1.yml)
assertEquals "a: test_doc1" "$doc1"
doc2=$(cat test_doc2.yml)
read -r -d '' expectedDoc2 << EOM
---
a: test_doc2
EOM
assertEquals "$expectedDoc2" "$doc2"
}
testBasicSplitWithNameEvalAll() {
cat >test.yml <<EOL
a: test_doc1
---
a: test_doc2
EOL
./yq ea test.yml -s ".a"
doc1=$(cat test_doc1.yml)
assertEquals "a: test_doc1" "$doc1"
doc2=$(cat test_doc2.yml)
read -r -d '' expectedDoc2 << EOM
---
a: test_doc2
EOM
assertEquals "$expectedDoc2" "$doc2"
}
testBasicSplitWithIndex() {
cat >test.yml <<EOL
a: test_doc1
---
a: test_doc2
EOL
./yq e test.yml -s '"test_" + $index'
doc1=$(cat test_0.yml)
assertEquals "a: test_doc1" "$doc1"
doc2=$(cat test_1.yml)
read -r -d '' expectedDoc2 << EOM
---
a: test_doc2
EOM
assertEquals "$expectedDoc2" "$doc2"
}
testBasicSplitWithIndexEvalAll() {
cat >test.yml <<EOL
a: test_doc1
---
a: test_doc2
EOL
./yq ea test.yml -s '"test_" + $index'
doc1=$(cat test_0.yml)
assertEquals "a: test_doc1" "$doc1"
doc2=$(cat test_1.yml)
read -r -d '' expectedDoc2 << EOM
---
a: test_doc2
EOM
assertEquals "$expectedDoc2" "$doc2"
}
testArraySplitWithNameNoSeparators() {
cat >test.yml <<EOL
- name: test_fred
age: 35
- name: test_catherine
age: 37
EOL
./yq e --no-doc -s ".name" ".[]" test.yml
doc1=$(cat test_fred.yml)
read -r -d '' expectedDoc1 << EOM
name: test_fred
age: 35
EOM
assertEquals "$expectedDoc1" "$doc1"
doc2=$(cat test_catherine.yml)
read -r -d '' expectedDoc2 << EOM
name: test_catherine
age: 37
EOM
assertEquals "$expectedDoc2" "$doc2"
}
testArraySplitWithNameNoSeparatorsEvalAll() {
cat >test.yml <<EOL
- name: test_fred
age: 35
- name: test_catherine
age: 37
EOL
cat >test2.yml <<EOL
- name: test_mike
age: 564
EOL
./yq ea --no-doc -s ".name" ".[]" test.yml test2.yml
doc1=$(cat test_fred.yml)
read -r -d '' expectedDoc1 << EOM
name: test_fred
age: 35
EOM
assertEquals "$expectedDoc1" "$doc1"
doc2=$(cat test_catherine.yml)
read -r -d '' expectedDoc2 << EOM
name: test_catherine
age: 37
EOM
assertEquals "$expectedDoc2" "$doc2"
doc3=$(cat test_mike.yml)
read -r -d '' expectedDoc3 << EOM
name: test_mike
age: 564
EOM
assertEquals "$expectedDoc3" "$doc3"
}
source ./scripts/shunit2

View File

@ -21,4 +21,6 @@ var prettyPrint = false
// can be either "" (off), "extract" or "process"
var frontMatter = ""
var splitFileExp = ""
var completedSuccessfully = false

View File

@ -70,6 +70,10 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
return fmt.Errorf("Write inplace flag only applicable when giving an expression and at least one file")
}
if writeInplace && splitFileExp != "" {
return fmt.Errorf("Write inplace cannot be used with split file")
}
if writeInplace {
// only use colors if its forced
colorsEnabled = forceColor
@ -96,7 +100,9 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
return err
}
printer := yqlib.NewPrinter(out, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
printerWriter := configurePrinterWriter(format, out)
printer := yqlib.NewPrinter(printerWriter, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
if frontMatter != "" {
frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])

View File

@ -83,6 +83,10 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
return fmt.Errorf("Write inplace flag only applicable when giving an expression and at least one file")
}
if writeInplace && splitFileExp != "" {
return fmt.Errorf("Write inplace cannot be used with split file")
}
if writeInplace {
// only use colors if its forced
colorsEnabled = forceColor
@ -106,7 +110,9 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
return err
}
printer := yqlib.NewPrinter(out, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
printerWriter := configurePrinterWriter(format, out)
printer := yqlib.NewPrinter(printerWriter, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
streamEvaluator := yqlib.NewStreamEvaluator()

24
cmd/printer_utils.go Normal file
View File

@ -0,0 +1,24 @@
package cmd
import (
"io"
"github.com/mikefarah/yq/v4/pkg/yqlib"
)
func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) yqlib.PrinterWriter {
var printerWriter yqlib.PrinterWriter
if splitFileExp != "" {
colorsEnabled = forceColor
splitExp, err := yqlib.NewExpressionParser().ParseExpression(splitFileExp)
if err != nil {
return nil
}
printerWriter = yqlib.NewMultiPrinterWriter(splitExp, format)
} else {
printerWriter = yqlib.NewSinglePrinterWriter(out)
}
return printerWriter
}

View File

@ -63,6 +63,9 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression. This is a workaround for go-yaml to persist header content properly.")
rootCmd.PersistentFlags().StringVarP(&splitFileExp, "split-exp", "s", "", "print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter.")
rootCmd.AddCommand(
createEvaluateSequenceCommand(),
createEvaluateAllCommand(),

View File

@ -13,7 +13,8 @@ import (
func yamlToString(candidate *CandidateNode, prefs encoderPreferences) (string, error) {
var output bytes.Buffer
log.Debug("printing with indent: %v", prefs.indent)
printer := NewPrinter(bufio.NewWriter(&output), prefs.format, true, false, prefs.indent, true)
printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), prefs.format, true, false, prefs.indent, true)
err := printer.PrintResults(candidate.AsList())
return output.String(), err
}

View File

@ -107,7 +107,7 @@ func copyFromHeader(title string, out *os.File) error {
func formatYaml(yaml string, filename string) string {
var output bytes.Buffer
printer := NewPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
node, err := NewExpressionParser().ParseExpression(".. style= \"\"")
if err != nil {
@ -216,7 +216,7 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) {
func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formattedDoc string, formattedDoc2 string) {
var output bytes.Buffer
var err error
printer := NewPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
node, err := NewExpressionParser().ParseExpression(s.expression)
if err != nil {

View File

@ -44,7 +44,7 @@ type resultsPrinter struct {
colorsEnabled bool
indent int
printDocSeparators bool
writer io.Writer
printerWriter PrinterWriter
firstTimePrinting bool
previousDocIndex uint
previousFileIndex int
@ -53,9 +53,13 @@ type resultsPrinter struct {
appendixReader io.Reader
}
func NewPrinter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
func NewPrinterWithSingleWriter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
return NewPrinter(NewSinglePrinterWriter(writer), outputFormat, unwrapScalar, colorsEnabled, indent, printDocSeparators)
}
func NewPrinter(printerWriter PrinterWriter, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
return &resultsPrinter{
writer: writer,
printerWriter: printerWriter,
outputFormat: outputFormat,
unwrapScalar: unwrapScalar,
colorsEnabled: colorsEnabled,
@ -98,15 +102,57 @@ func (p *resultsPrinter) writeString(writer io.Writer, txt string) error {
return errorWriting
}
func (p *resultsPrinter) safelyFlush(writer *bufio.Writer) {
if err := writer.Flush(); err != nil {
log.Error("Error flushing writer!")
log.Error(err.Error())
func (p *resultsPrinter) processLeadingContent(mappedDoc *CandidateNode, writer io.Writer) error {
if strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$") {
log.Debug("headcommentwas %v", mappedDoc.Node.HeadComment)
log.Debug("finished headcomment")
reader := bufio.NewReader(strings.NewReader(mappedDoc.Node.HeadComment))
mappedDoc.Node.HeadComment = ""
for {
readline, errReading := reader.ReadString('\n')
if errReading != nil && errReading != io.EOF {
return errReading
}
if strings.Contains(readline, "$yqLeadingContent$") {
// skip this
} else if strings.Contains(readline, "$yqDocSeperator$") {
if p.printDocSeparators {
if err := p.writeString(writer, "---\n"); err != nil {
return err
}
}
} else if p.outputFormat == YamlOutputFormat {
if err := p.writeString(writer, readline); err != nil {
return err
}
}
if errReading == io.EOF {
if readline != "" {
// the last comment we read didn't have a new line, put one in
if err := p.writeString(writer, "\n"); err != nil {
return err
}
}
break
}
}
}
return nil
}
func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
log.Debug("PrintResults for %v matches", matchingNodes.Len())
if matchingNodes.Len() == 0 {
log.Debug("no matching results, nothing to print")
return nil
}
if p.outputFormat != YamlOutputFormat {
explodeOp := Operation{OperationType: explodeOpType}
explodeNode := ExpressionNode{Operation: &explodeOp}
@ -117,13 +163,6 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
matchingNodes = context.MatchingNodes
}
bufferedWriter := bufio.NewWriter(p.writer)
defer p.safelyFlush(bufferedWriter)
if matchingNodes.Len() == 0 {
log.Debug("no matching results, nothing to print")
return nil
}
if p.firstTimePrinting {
node := matchingNodes.Front().Value.(*CandidateNode)
p.previousDocIndex = node.Document
@ -132,72 +171,53 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
}
for el := matchingNodes.Front(); el != nil; el = el.Next() {
mappedDoc := el.Value.(*CandidateNode)
log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v, printDocSeparators: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document, p.printDocSeparators)
writer, errorWriting := p.printerWriter.GetWriter(mappedDoc)
if errorWriting != nil {
return errorWriting
}
commentStartsWithSeparator := strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$\n$yqDocSeperator$")
if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators && !commentStartsWithSeparator {
log.Debug("-- writing doc sep")
if err := p.writeString(bufferedWriter, "---\n"); err != nil {
if err := p.writeString(writer, "---\n"); err != nil {
return err
}
}
if strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$") {
log.Debug("headcommentwas %v", mappedDoc.Node.HeadComment)
log.Debug("finished headcomment")
reader := bufio.NewReader(strings.NewReader(mappedDoc.Node.HeadComment))
mappedDoc.Node.HeadComment = ""
for {
readline, errReading := reader.ReadString('\n')
if errReading != nil && errReading != io.EOF {
return errReading
}
if strings.Contains(readline, "$yqLeadingContent$") {
// skip this
} else if strings.Contains(readline, "$yqDocSeperator$") {
if p.printDocSeparators {
if err := p.writeString(bufferedWriter, "---\n"); err != nil {
return err
}
}
} else if p.outputFormat == YamlOutputFormat {
if err := p.writeString(bufferedWriter, readline); err != nil {
return err
}
}
if errReading == io.EOF {
if readline != "" {
// the last comment we read didn't have a new line, put one in
if err := p.writeString(bufferedWriter, "\n"); err != nil {
return err
}
}
break
}
}
if err := p.processLeadingContent(mappedDoc, writer); err != nil {
return err
}
if err := p.printNode(mappedDoc.Node, bufferedWriter); err != nil {
if err := p.printNode(mappedDoc.Node, writer); err != nil {
return err
}
p.previousDocIndex = mappedDoc.Document
if err := writer.Flush(); err != nil {
return err
}
}
if p.appendixReader != nil && p.outputFormat == YamlOutputFormat {
writer, err := p.printerWriter.GetWriter(nil)
if err != nil {
return err
}
log.Debug("Piping appendix reader...")
betterReader := bufio.NewReader(p.appendixReader)
_, err := io.Copy(bufferedWriter, betterReader)
_, err = io.Copy(writer, betterReader)
if err != nil {
return err
}
if err := writer.Flush(); err != nil {
return err
}
}
return nil

View File

@ -36,7 +36,7 @@ func nodeToList(candidate *CandidateNode) *list.List {
func TestPrinterMultipleDocsInSequence(t *testing.T) {
var output bytes.Buffer
var writer = bufio.NewWriter(&output)
printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
if err != nil {
@ -74,7 +74,7 @@ func TestPrinterMultipleDocsInSequence(t *testing.T) {
func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
var output bytes.Buffer
var writer = bufio.NewWriter(&output)
printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
if err != nil {
@ -116,7 +116,7 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
func TestPrinterMultipleFilesInSequence(t *testing.T) {
var output bytes.Buffer
var writer = bufio.NewWriter(&output)
printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
if err != nil {
@ -163,7 +163,7 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) {
func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
var output bytes.Buffer
var writer = bufio.NewWriter(&output)
printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
if err != nil {
@ -213,7 +213,7 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
var output bytes.Buffer
var writer = bufio.NewWriter(&output)
printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
if err != nil {
@ -232,7 +232,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) {
var output bytes.Buffer
var writer = bufio.NewWriter(&output)
printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
if err != nil {
@ -261,7 +261,7 @@ a: coconut
func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) {
var output bytes.Buffer
var writer = bufio.NewWriter(&output)
printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
if err != nil {
@ -287,7 +287,7 @@ a: coconut
func TestPrinterScalarWithLeadingCont(t *testing.T) {
var output bytes.Buffer
var writer = bufio.NewWriter(&output)
printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
node, err := NewExpressionParser().ParseExpression(".a")
if err != nil {
@ -314,7 +314,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
var writer = bufio.NewWriter(&output)
// note printDocSeparators is true, it should still not print document separators
// when outputing JSON.
printer := NewPrinter(writer, JsonOutputFormat, true, false, 0, true)
printer := NewPrinterWithSingleWriter(writer, JsonOutputFormat, true, false, 0, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
if err != nil {

View File

@ -0,0 +1,81 @@
package yqlib
import (
"bufio"
"fmt"
"io"
"os"
"gopkg.in/yaml.v3"
)
type PrinterWriter interface {
GetWriter(node *CandidateNode) (*bufio.Writer, error)
}
type singlePrinterWriter struct {
bufferedWriter *bufio.Writer
}
func NewSinglePrinterWriter(writer io.Writer) PrinterWriter {
return &singlePrinterWriter{
bufferedWriter: bufio.NewWriter(writer),
}
}
func (sp *singlePrinterWriter) GetWriter(node *CandidateNode) (*bufio.Writer, error) {
return sp.bufferedWriter, nil
}
type multiPrintWriter struct {
treeNavigator DataTreeNavigator
nameExpression *ExpressionNode
extension string
index int
}
func NewMultiPrinterWriter(expression *ExpressionNode, format PrinterOutputFormat) PrinterWriter {
extension := "yml"
switch format {
case JsonOutputFormat:
extension = "json"
case PropsOutputFormat:
extension = "properties"
}
return &multiPrintWriter{
nameExpression: expression,
extension: extension,
treeNavigator: NewDataTreeNavigator(),
index: 0,
}
}
func (sp *multiPrintWriter) GetWriter(node *CandidateNode) (*bufio.Writer, error) {
name := ""
indexVariableNode := yaml.Node{Kind: yaml.ScalarNode, Tag: "!!int", Value: fmt.Sprintf("%v", sp.index)}
indexVariableCandidate := CandidateNode{Node: &indexVariableNode}
context := Context{MatchingNodes: node.AsList()}
context.SetVariable("index", indexVariableCandidate.AsList())
result, err := sp.treeNavigator.GetMatchingNodes(context, sp.nameExpression)
if err != nil {
return nil, err
}
if result.MatchingNodes.Len() > 0 {
name = result.MatchingNodes.Front().Value.(*CandidateNode).Node.Value
}
name = fmt.Sprintf("%v.%v", name, sp.extension)
f, err := os.Create(name)
if err != nil {
return nil, err
}
sp.index = sp.index + 1
return bufio.NewWriter(f), nil
}