suyiiyii
/
yq
mirror of https://github.com/mikefarah/yq.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Better way of processing leading content

This commit is contained in:
Mike Farah 2021-07-20 10:19:55 +10:00
parent bbebebe30c
commit 4e628327c4
9 changed files with 92 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
acceptance_tests/leading-seperator.sh
View File

@ -248,8 +248,7 @@ a: test
b: sane
EOM
X=$(./yq e --header-preprocess=false '... comments=""' test.yml)
X=$(./yq e '... comments=""' test.yml)
assertEquals "$expected" "$X"
}
@ -265,9 +264,7 @@ a: test
b: sane
EOL
# it will be hard to remove that top level separator
read -r -d '' expected << EOM
---
# hi peeps
# cool
a: test

2
cmd/root.go
View File

@ -55,7 +55,7 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors")
rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and seperators before processing expression. This is a workaround for go-yaml to persist header content. You will want this off if you want to remove leading comments.")
rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and seperators before processing expression. This is a workaround for go-yaml to persist header content. This flag will be removed once this feature has been out in the wild for a while.")
rootCmd.AddCommand(
createEvaluateSequenceCommand(),
createEvaluateAllCommand(),

6
examples/data1.yaml
View File

@ -1,8 +1,4 @@
---
# hi peeps
# cool
a: test
---
# this is another doc
# great
b: sane
a: test

6
pkg/yqlib/all_at_once_evaluator.go
View File

@ -73,15 +73,17 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
candidateNode := &CandidateNode{
Document: 0,
Filename: "",
Node: &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode},
Node: &yaml.Node{Kind: yaml.DocumentNode, HeadComment: firstFileLeadingContent, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
FileIndex: 0,
}
allDocuments.PushBack(candidateNode)
} else {
allDocuments.Front().Value.(*CandidateNode).Node.HeadComment = firstFileLeadingContent
}
matches, err := e.EvaluateCandidateNodes(expression, allDocuments)
if err != nil {
return err
}
return printer.PrintResults(matches, firstFileLeadingContent)
return printer.PrintResults(matches)
}

2
pkg/yqlib/operators_test.go
View File

@ -256,7 +256,7 @@ func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formatt
t.Error(err, s.expression)
}
err = printer.PrintResults(context.MatchingNodes, "")
err = printer.PrintResults(context.MatchingNodes)
if err != nil {
t.Error(err, s.expression)
}

55
pkg/yqlib/printer.go
View File

@ -10,7 +10,7 @@ import (
)
type Printer interface {
PrintResults(matchingNodes *list.List, leadingContent string) error
PrintResults(matchingNodes *list.List) error
PrintedAnything() bool
//e.g. when given a front-matter doc, like jekyll
SetAppendix(reader io.Reader)
@ -85,7 +85,7 @@ func (p *resultsPrinter) safelyFlush(writer *bufio.Writer) {
}
}
func (p *resultsPrinter) PrintResults(matchingNodes *list.List, leadingContent string) error {
func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
log.Debug("PrintResults for %v matches", matchingNodes.Len())
if p.outputToJSON {
explodeOp := Operation{OperationType: explodeOpType}
@ -101,9 +101,6 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List, leadingContent s
defer p.safelyFlush(bufferedWriter)
if matchingNodes.Len() == 0 {
if err := p.writeString(bufferedWriter, leadingContent); err != nil {
return err
}
log.Debug("no matching results, nothing to print")
return nil
}
@ -114,32 +111,52 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List, leadingContent s
p.firstTimePrinting = false
}
printedLead := false
for el := matchingNodes.Front(); el != nil; el = el.Next() {
mappedDoc := el.Value.(*CandidateNode)
log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v, printDocSeparators: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document, p.printDocSeparators)
if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators &&
(printedLead || !strings.HasPrefix(leadingContent, "---")) {
commentStartsWithSeparator := strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$\n$yqDocSeperator$")
if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators && !commentStartsWithSeparator {
log.Debug("-- writing doc sep")
if err := p.writeString(bufferedWriter, "---\n"); err != nil {
return err
}
}
if !printedLead {
// dont print leading comments and seperator if:
// - we are print json; or
// - we are printing an unwrapped scalar node
if !p.outputToJSON && (mappedDoc.Node.Kind != yaml.ScalarNode || !p.unwrapScalar) {
// we want to print this after the seperator logic
if err := p.writeString(bufferedWriter, leadingContent); err != nil {
return err
if strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$") {
log.Debug("headcommentwas %v", mappedDoc.Node.HeadComment)
log.Debug("finished headcomment")
reader := bufio.NewReader(strings.NewReader(mappedDoc.Node.HeadComment))
mappedDoc.Node.HeadComment = ""
for {
readline, errReading := reader.ReadString('\n')
if errReading != nil && errReading != io.EOF {
return errReading
}
if strings.Contains(readline, "$yqLeadingContent$") {
// skip this
} else if strings.Contains(readline, "$yqDocSeperator$") {
if p.printDocSeparators {
if err := p.writeString(bufferedWriter, "---\n"); err != nil {
return err
}
}
} else if !p.outputToJSON {
if err := p.writeString(bufferedWriter, readline); err != nil {
return err
}
}
if errReading == io.EOF {
break
}
}
printedLead = true
}
}
if err := p.printNode(mappedDoc.Node, bufferedWriter); err != nil {
return err
}

44
pkg/yqlib/printer_test.go
View File

@ -52,17 +52,17 @@ func TestPrinterMultipleDocsInSequence(t *testing.T) {
el = el.Next()
sample3 := nodeToList(el.Value.(*CandidateNode))
err = printer.PrintResults(sample1, "")
err = printer.PrintResults(sample1)
if err != nil {
panic(err)
}
err = printer.PrintResults(sample2, "")
err = printer.PrintResults(sample2)
if err != nil {
panic(err)
}
err = printer.PrintResults(sample3, "")
err = printer.PrintResults(sample3)
if err != nil {
panic(err)
}
@ -82,25 +82,28 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
}
el := inputs.Front()
el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n"
sample1 := nodeToList(el.Value.(*CandidateNode))
el = el.Next()
el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n"
sample2 := nodeToList(el.Value.(*CandidateNode))
el = el.Next()
el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n# cool\n"
sample3 := nodeToList(el.Value.(*CandidateNode))
err = printer.PrintResults(sample1, "# go cats\n---\n")
err = printer.PrintResults(sample1)
if err != nil {
panic(err)
}
err = printer.PrintResults(sample2, "---\n")
err = printer.PrintResults(sample2)
if err != nil {
panic(err)
}
err = printer.PrintResults(sample3, "---\n# cool\n")
err = printer.PrintResults(sample3)
if err != nil {
panic(err)
}
@ -138,17 +141,17 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) {
elNode.FileIndex = 2
sample3 := nodeToList(elNode)
err = printer.PrintResults(sample1, "")
err = printer.PrintResults(sample1)
if err != nil {
panic(err)
}
err = printer.PrintResults(sample2, "")
err = printer.PrintResults(sample2)
if err != nil {
panic(err)
}
err = printer.PrintResults(sample3, "")
err = printer.PrintResults(sample3)
if err != nil {
panic(err)
}
@ -171,31 +174,34 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
elNode := el.Value.(*CandidateNode)
elNode.Document = 0
elNode.FileIndex = 0
elNode.Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n"
sample1 := nodeToList(elNode)
el = el.Next()
elNode = el.Value.(*CandidateNode)
elNode.Document = 0
elNode.FileIndex = 1
elNode.Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n"
sample2 := nodeToList(elNode)
el = el.Next()
elNode = el.Value.(*CandidateNode)
elNode.Document = 0
elNode.FileIndex = 2
elNode.Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n# cool\n"
sample3 := nodeToList(elNode)
err = printer.PrintResults(sample1, "# go cats\n---\n")
err = printer.PrintResults(sample1)
if err != nil {
panic(err)
}
err = printer.PrintResults(sample2, "---\n")
err = printer.PrintResults(sample2)
if err != nil {
panic(err)
}
err = printer.PrintResults(sample3, "---\n# cool\n")
err = printer.PrintResults(sample3)
if err != nil {
panic(err)
}
@ -214,7 +220,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
panic(err)
}
err = printer.PrintResults(inputs, "")
err = printer.PrintResults(inputs)
if err != nil {
panic(err)
}
@ -233,7 +239,9 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) {
panic(err)
}
err = printer.PrintResults(inputs, "# go cats\n---\n")
inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n"
err = printer.PrintResults(inputs)
if err != nil {
panic(err)
}
@ -259,8 +267,8 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) {
if err != nil {
panic(err)
}
err = printer.PrintResults(inputs, "---\n")
inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n"
err = printer.PrintResults(inputs)
if err != nil {
panic(err)
}
@ -313,7 +321,9 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
panic(err)
}
err = printer.PrintResults(inputs, "# ignore this")
inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# ignore this\n"
err = printer.PrintResults(inputs)
if err != nil {
panic(err)
}

14
pkg/yqlib/stream_evaluator.go
View File

@ -35,7 +35,7 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadin
candidateNode := &CandidateNode{
Document: 0,
Filename: "",
Node: &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode},
Node: &yaml.Node{Kind: yaml.DocumentNode, HeadComment: leadingContent, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
FileIndex: 0,
}
inputList := list.New()
@ -45,7 +45,7 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadin
if errorParsing != nil {
return errorParsing
}
return printer.PrintResults(result.MatchingNodes, leadingContent)
return printer.PrintResults(result.MatchingNodes)
}
func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error {
@ -100,6 +100,9 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr
} else if errorReading != nil {
return currentIndex, errorReading
}
if currentIndex == 0 {
dataBucket.HeadComment = leadingContent
}
candidateNode := &CandidateNode{
Document: currentIndex,
Filename: filename,
@ -113,12 +116,7 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr
if errorParsing != nil {
return currentIndex, errorParsing
}
var err error
if currentIndex == 0 {
err = printer.PrintResults(result.MatchingNodes, leadingContent)
} else {
err = printer.PrintResults(result.MatchingNodes, "")
}
err := printer.PrintResults(result.MatchingNodes)
if err != nil {
return currentIndex, err

20
pkg/yqlib/utils.go
View File

@ -12,7 +12,6 @@ import (
)
func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, string, error) {
var commentLineRegEx = regexp.MustCompile(`^\s*#`)
var reader *bufio.Reader
if filename == "-" {
reader = bufio.NewReader(os.Stdin)
@ -25,21 +24,33 @@ func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, s
}
reader = bufio.NewReader(file)
}
var sb strings.Builder
if !leadingContentPreProcessing {
return reader, "", nil
}
return processReadStream(reader)
}
func processReadStream(reader *bufio.Reader) (io.Reader, string, error) {
var commentLineRegEx = regexp.MustCompile(`^\s*#`)
var sb strings.Builder
sb.WriteString("$yqLeadingContent$\n")
for {
peekBytes, err := reader.Peek(3)
if err == io.EOF {
// EOF are handled else where..
return reader, sb.String(), nil
} else if err != nil {
return reader, sb.String(), err
} else if string(peekBytes) == "---" || commentLineRegEx.MatchString(string(peekBytes)) {
} else if string(peekBytes) == "---" {
_, err := reader.ReadString('\n')
sb.WriteString("$yqDocSeperator$\n")
if err == io.EOF {
return reader, sb.String(), nil
} else if err != nil {
return reader, sb.String(), err
}
} else if commentLineRegEx.MatchString(string(peekBytes)) {
line, err := reader.ReadString('\n')
sb.WriteString(line)
if err == io.EOF {
@ -51,7 +62,6 @@ func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, s
return reader, sb.String(), nil
}
}
}
func readDocuments(reader io.Reader, filename string, fileIndex int) (*list.List, error) {