From ffd3616a07645a1535363c817c9e7333f8c6e59b Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 12 Nov 2021 15:02:28 +1100 Subject: [PATCH] wip --- examples/data1.yaml | 12 ++++--- examples/data2.yaml | 5 ++- pkg/yqlib/all_at_once_evaluator.go | 11 +++--- pkg/yqlib/candidate_node.go | 11 +++--- pkg/yqlib/doc/comment-operators.md | 22 ++++++++++++ pkg/yqlib/operator_comments.go | 25 +++++++++++-- pkg/yqlib/operator_comments_test.go | 13 +++++-- pkg/yqlib/operators_test.go | 27 +++++++++++--- pkg/yqlib/printer.go | 56 +++-------------------------- pkg/yqlib/stream_evaluator.go | 16 +++++---- pkg/yqlib/utils.go | 48 +++++++++++++++++++++++++ 11 files changed, 162 insertions(+), 84 deletions(-) diff --git a/examples/data1.yaml b/examples/data1.yaml index e940b64d..fa32f314 100644 --- a/examples/data1.yaml +++ b/examples/data1.yaml @@ -1,4 +1,8 @@ -- name: bob - age: 23 -- name: tim - age: 17 \ No newline at end of file +# welcome! # cat +# bob +--- +# bob + +a: cat # meow + +# have a great day \ No newline at end of file diff --git a/examples/data2.yaml b/examples/data2.yaml index 7eaac718..f4313254 100644 --- a/examples/data2.yaml +++ b/examples/data2.yaml @@ -1,3 +1,2 @@ -#b1 -b: 2 -#b2 \ No newline at end of file +c: + d: hamster \ No newline at end of file diff --git a/pkg/yqlib/all_at_once_evaluator.go b/pkg/yqlib/all_at_once_evaluator.go index e691f242..9c79a035 100644 --- a/pkg/yqlib/all_at_once_evaluator.go +++ b/pkg/yqlib/all_at_once_evaluator.go @@ -71,14 +71,15 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string if allDocuments.Len() == 0 { candidateNode := &CandidateNode{ - Document: 0, - Filename: "", - Node: &yaml.Node{Kind: yaml.DocumentNode, HeadComment: firstFileLeadingContent, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}}, - FileIndex: 0, + Document: 0, + Filename: "", + Node: &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}}, + FileIndex: 0, + LeadingContent: firstFileLeadingContent, } allDocuments.PushBack(candidateNode) } else { - allDocuments.Front().Value.(*CandidateNode).Node.HeadComment = firstFileLeadingContent + allDocuments.Front().Value.(*CandidateNode).LeadingContent = firstFileLeadingContent } matches, err := e.EvaluateCandidateNodes(expression, allDocuments) diff --git a/pkg/yqlib/candidate_node.go b/pkg/yqlib/candidate_node.go index f1e092bb..f685f679 100644 --- a/pkg/yqlib/candidate_node.go +++ b/pkg/yqlib/candidate_node.go @@ -9,10 +9,13 @@ import ( ) type CandidateNode struct { - Node *yaml.Node // the actual node - Parent *CandidateNode // parent node - Path []interface{} /// the path we took to get to this node - Document uint // the document index of this node + Node *yaml.Node // the actual node + Parent *CandidateNode // parent node + + LeadingContent string + + Path []interface{} /// the path we took to get to this node + Document uint // the document index of this node Filename string FileIndex int // when performing op against all nodes given, this will treat all the nodes as one diff --git a/pkg/yqlib/doc/comment-operators.md b/pkg/yqlib/doc/comment-operators.md index 507a6a7f..6086b20a 100644 --- a/pkg/yqlib/doc/comment-operators.md +++ b/pkg/yqlib/doc/comment-operators.md @@ -139,6 +139,26 @@ will output welcome! ``` +## +Given a sample.yml file of: +```yaml +# welcome! +--- +# bob +a: cat # meow + +# have a great day +``` +then +```bash +yq eval 'headComment' sample.yml +``` +will output +```yaml +welcome! +bob +``` + ## Get foot comment Given a sample.yml file of: ```yaml @@ -147,6 +167,7 @@ Given a sample.yml file of: a: cat # meow # have a great day +# no really ``` then ```bash @@ -155,5 +176,6 @@ yq eval '. | footComment' sample.yml will output ```yaml have a great day +no really ``` diff --git a/pkg/yqlib/operator_comments.go b/pkg/yqlib/operator_comments.go index 6ece8f29..c58a84bb 100644 --- a/pkg/yqlib/operator_comments.go +++ b/pkg/yqlib/operator_comments.go @@ -1,8 +1,12 @@ package yqlib import ( + // "bufio" + // "bytes" + "bufio" + "bytes" "container/list" - "strings" + "regexp" yaml "gopkg.in/yaml.v3" ) @@ -57,6 +61,7 @@ func assignCommentsOperator(d *dataTreeNavigator, context Context, expressionNod } if preferences.HeadComment { candidate.Node.HeadComment = comment + candidate.LeadingContent = "" // clobber the leading content, if there was any. } if preferences.FootComment { candidate.Node.FootComment = comment @@ -68,6 +73,9 @@ func assignCommentsOperator(d *dataTreeNavigator, context Context, expressionNod func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { preferences := expressionNode.Operation.Preferences.(commentOpPreferences) + var startCommentCharaterRegExp = regexp.MustCompile(`^# `) + var subsequentCommentCharaterRegExp = regexp.MustCompile(`\n# `) + log.Debugf("GetComments operator!") var results = list.New() @@ -76,12 +84,25 @@ func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode * comment := "" if preferences.LineComment { comment = candidate.Node.LineComment + } else if preferences.HeadComment && candidate.LeadingContent != "" { + var chompRegexp = regexp.MustCompile(`\n$`) + var output bytes.Buffer + var writer = bufio.NewWriter(&output) + if err := processLeadingContent(candidate, writer, false, YamlOutputFormat); err != nil { + return Context{}, err + } + if err := writer.Flush(); err != nil { + return Context{}, err + } + comment = output.String() + comment = chompRegexp.ReplaceAllString(comment, "") } else if preferences.HeadComment { comment = candidate.Node.HeadComment } else if preferences.FootComment { comment = candidate.Node.FootComment } - comment = strings.Replace(comment, "# ", "", 1) + comment = startCommentCharaterRegExp.ReplaceAllString(comment, "") + comment = subsequentCommentCharaterRegExp.ReplaceAllString(comment, "\n") node := &yaml.Node{Kind: yaml.ScalarNode, Value: comment, Tag: "!!str"} result := candidate.CreateChild(nil, node) diff --git a/pkg/yqlib/operator_comments_test.go b/pkg/yqlib/operator_comments_test.go index 14b1d83e..f951ed0b 100644 --- a/pkg/yqlib/operator_comments_test.go +++ b/pkg/yqlib/operator_comments_test.go @@ -112,13 +112,22 @@ var commentOperatorScenarios = []expressionScenario{ "D0, P[], (!!str)::welcome!\n", }, }, + { + skipDoc: false, + dontFormatInputForDoc: true, + document: "# welcome!\n---\n# bob\na: cat # meow\n\n# have a great day", + expression: `headComment`, + expected: []string{ + "D0, P[], (!!str)::|-\n welcome!\n bob\n", + }, + }, { description: "Get foot comment", dontFormatInputForDoc: true, - document: "# welcome!\n\na: cat # meow\n\n# have a great day", + document: "# welcome!\n\na: cat # meow\n\n# have a great day\n# no really", expression: `. | footComment`, expected: []string{ - "D0, P[], (!!str)::have a great day\n", + "D0, P[], (!!str)::have a great day\nno really\n", }, }, } diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index af7b1d7a..e784f319 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -26,6 +26,20 @@ type expressionScenario struct { dontFormatInputForDoc bool // dont format input doc for documentation generation } +func readDocumentWithLeadingContent(content string, fakefilename string, fakeFileIndex int) (*list.List, error) { + reader, firstFileLeadingContent, err := processReadStream(bufio.NewReader(strings.NewReader(content))) + if err != nil { + return nil, err + } + + inputs, err := readDocuments(reader, fakefilename, fakeFileIndex) + if err != nil { + return nil, err + } + inputs.Front().Value.(*CandidateNode).LeadingContent = firstFileLeadingContent + return inputs, nil +} + func testScenario(t *testing.T, s *expressionScenario) { var err error @@ -37,15 +51,17 @@ func testScenario(t *testing.T, s *expressionScenario) { inputs := list.New() if s.document != "" { - inputs, err = readDocuments(strings.NewReader(s.document), "sample.yml", 0) + inputs, err = readDocumentWithLeadingContent(s.document, "sample.yml", 0) + if err != nil { t.Error(err, s.document, s.expression) return } + if s.document2 != "" { - moreInputs, err := readDocuments(strings.NewReader(s.document2), "another.yml", 1) + moreInputs, err := readDocumentWithLeadingContent(s.document2, "another.yml", 1) if err != nil { - t.Error(err, s.document, s.expression) + t.Error(err, s.document2, s.expression) return } inputs.PushBackList(moreInputs) @@ -227,13 +243,14 @@ func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formatt inputs := list.New() if s.document != "" { - inputs, err = readDocuments(strings.NewReader(formattedDoc), "sample.yml", 0) + + inputs, err = readDocumentWithLeadingContent(formattedDoc, "sample.yml", 0) if err != nil { t.Error(err, s.document, s.expression) return } if s.document2 != "" { - moreInputs, err := readDocuments(strings.NewReader(formattedDoc2), "another.yml", 1) + moreInputs, err := readDocumentWithLeadingContent(formattedDoc2, "another.yml", 1) if err != nil { t.Error(err, s.document, s.expression) return diff --git a/pkg/yqlib/printer.go b/pkg/yqlib/printer.go index 3ac4bcee..71783fe3 100644 --- a/pkg/yqlib/printer.go +++ b/pkg/yqlib/printer.go @@ -84,7 +84,7 @@ func (p *resultsPrinter) printNode(node *yaml.Node, writer io.Writer) error { var encoder Encoder if node.Kind == yaml.ScalarNode && p.unwrapScalar && p.outputFormat == YamlOutputFormat { - return p.writeString(writer, node.Value+"\n") + return writeString(writer, node.Value+"\n") } if p.outputFormat == JsonOutputFormat { @@ -97,54 +97,6 @@ func (p *resultsPrinter) printNode(node *yaml.Node, writer io.Writer) error { return encoder.Encode(node) } -func (p *resultsPrinter) writeString(writer io.Writer, txt string) error { - _, errorWriting := writer.Write([]byte(txt)) - return errorWriting -} - -func (p *resultsPrinter) processLeadingContent(mappedDoc *CandidateNode, writer io.Writer) error { - if strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$") { - log.Debug("headcommentwas %v", mappedDoc.Node.HeadComment) - log.Debug("finished headcomment") - reader := bufio.NewReader(strings.NewReader(mappedDoc.Node.HeadComment)) - mappedDoc.Node.HeadComment = "" - - for { - - readline, errReading := reader.ReadString('\n') - if errReading != nil && errReading != io.EOF { - return errReading - } - if strings.Contains(readline, "$yqLeadingContent$") { - // skip this - - } else if strings.Contains(readline, "$yqDocSeperator$") { - if p.printDocSeparators { - if err := p.writeString(writer, "---\n"); err != nil { - return err - } - } - } else if p.outputFormat == YamlOutputFormat { - if err := p.writeString(writer, readline); err != nil { - return err - } - } - - if errReading == io.EOF { - if readline != "" { - // the last comment we read didn't have a new line, put one in - if err := p.writeString(writer, "\n"); err != nil { - return err - } - } - break - } - } - - } - return nil -} - func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error { log.Debug("PrintResults for %v matches", matchingNodes.Len()) @@ -180,16 +132,16 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error { return errorWriting } - commentStartsWithSeparator := strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$\n$yqDocSeperator$") + commentStartsWithSeparator := strings.Contains(mappedDoc.LeadingContent, "$yqLeadingContent$\n$yqDocSeperator$") if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators && !commentStartsWithSeparator { log.Debug("-- writing doc sep") - if err := p.writeString(writer, "---\n"); err != nil { + if err := writeString(writer, "---\n"); err != nil { return err } } - if err := p.processLeadingContent(mappedDoc, writer); err != nil { + if err := processLeadingContent(mappedDoc, writer, p.printDocSeparators, p.outputFormat); err != nil { return err } diff --git a/pkg/yqlib/stream_evaluator.go b/pkg/yqlib/stream_evaluator.go index 84cebc2c..71380121 100644 --- a/pkg/yqlib/stream_evaluator.go +++ b/pkg/yqlib/stream_evaluator.go @@ -33,10 +33,11 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadin return err } candidateNode := &CandidateNode{ - Document: 0, - Filename: "", - Node: &yaml.Node{Kind: yaml.DocumentNode, HeadComment: leadingContent, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}}, - FileIndex: 0, + Document: 0, + Filename: "", + Node: &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}}, + FileIndex: 0, + LeadingContent: leadingContent, } inputList := list.New() inputList.PushBack(candidateNode) @@ -100,15 +101,16 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr } else if errorReading != nil { return currentIndex, errorReading } - if currentIndex == 0 { - dataBucket.HeadComment = leadingContent - } + candidateNode := &CandidateNode{ Document: currentIndex, Filename: filename, Node: &dataBucket, FileIndex: s.fileIndex, } + if currentIndex == 0 { + candidateNode.LeadingContent = leadingContent + } inputList := list.New() inputList.PushBack(candidateNode) diff --git a/pkg/yqlib/utils.go b/pkg/yqlib/utils.go index 4927e1f6..c38c3c43 100644 --- a/pkg/yqlib/utils.go +++ b/pkg/yqlib/utils.go @@ -31,6 +31,54 @@ func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, s return processReadStream(reader) } +func writeString(writer io.Writer, txt string) error { + _, errorWriting := writer.Write([]byte(txt)) + return errorWriting +} + +func processLeadingContent(mappedDoc *CandidateNode, writer io.Writer, printDocSeparators bool, outputFormat PrinterOutputFormat) error { + if strings.Contains(mappedDoc.LeadingContent, "$yqLeadingContent$") { + log.Debug("headcommentwas %v", mappedDoc.LeadingContent) + log.Debug("finished headcomment") + reader := bufio.NewReader(strings.NewReader(mappedDoc.LeadingContent)) + mappedDoc.Node.HeadComment = "" + + for { + + readline, errReading := reader.ReadString('\n') + if errReading != nil && errReading != io.EOF { + return errReading + } + if strings.Contains(readline, "$yqLeadingContent$") { + // skip this + + } else if strings.Contains(readline, "$yqDocSeperator$") { + if printDocSeparators { + if err := writeString(writer, "---\n"); err != nil { + return err + } + } + } else if outputFormat == YamlOutputFormat { + if err := writeString(writer, readline); err != nil { + return err + } + } + + if errReading == io.EOF { + if readline != "" { + // the last comment we read didn't have a new line, put one in + if err := writeString(writer, "\n"); err != nil { + return err + } + } + break + } + } + + } + return nil +} + func processReadStream(reader *bufio.Reader) (io.Reader, string, error) { var commentLineRegEx = regexp.MustCompile(`^\s*#`) var sb strings.Builder