suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2024-12-19 20:19:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Mike Farah 2021-11-12 15:02:28 +11:00
parent bff4c9e586
commit d912d7d178
11 changed files with 162 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
examples/data1.yaml
View File

@ -1,4 +1,8 @@
- name: bob
age: 23
- name: tim
age: 17
# welcome! # cat
# bob
---
# bob
a: cat # meow
# have a great day

5
examples/data2.yaml
View File

@ -1,3 +1,2 @@
#b1
b: 2
#b2
c:
d: hamster

5
pkg/yqlib/all_at_once_evaluator.go
View File

@ -73,12 +73,13 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
candidateNode := &CandidateNode{
Document: 0,
Filename: "",
Node: &yaml.Node{Kind: yaml.DocumentNode, HeadComment: firstFileLeadingContent, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
Node: &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
FileIndex: 0,
LeadingContent: firstFileLeadingContent,
}
allDocuments.PushBack(candidateNode)
} else {
allDocuments.Front().Value.(*CandidateNode).Node.HeadComment = firstFileLeadingContent
allDocuments.Front().Value.(*CandidateNode).LeadingContent = firstFileLeadingContent
}
matches, err := e.EvaluateCandidateNodes(expression, allDocuments)

3
pkg/yqlib/candidate_node.go
View File

@ -11,6 +11,9 @@ import (
type CandidateNode struct {
Node *yaml.Node // the actual node
Parent *CandidateNode // parent node
LeadingContent string
Path []interface{} /// the path we took to get to this node
Document uint // the document index of this node
Filename string

22
pkg/yqlib/doc/comment-operators.md
View File

@ -139,6 +139,26 @@ will output
welcome!
```
##
Given a sample.yml file of:
```yaml
# welcome!
---
# bob
a: cat # meow
# have a great day
```
then
```bash
yq eval 'headComment' sample.yml
```
will output
```yaml
welcome!
bob
```
## Get foot comment
Given a sample.yml file of:
```yaml
@ -147,6 +167,7 @@ Given a sample.yml file of:
a: cat # meow
# have a great day
# no really
```
then
```bash
@ -155,5 +176,6 @@ yq eval '. | footComment' sample.yml
will output
```yaml
have a great day
no really
```

25
pkg/yqlib/operator_comments.go
View File

@ -1,8 +1,12 @@
package yqlib
import (
// "bufio"
// "bytes"
"bufio"
"bytes"
"container/list"
"strings"
"regexp"
yaml "gopkg.in/yaml.v3"
)
@ -57,6 +61,7 @@ func assignCommentsOperator(d *dataTreeNavigator, context Context, expressionNod
}
if preferences.HeadComment {
candidate.Node.HeadComment = comment
candidate.LeadingContent = "" // clobber the leading content, if there was any.
}
if preferences.FootComment {
candidate.Node.FootComment = comment
@ -68,6 +73,9 @@ func assignCommentsOperator(d *dataTreeNavigator, context Context, expressionNod
func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
preferences := expressionNode.Operation.Preferences.(commentOpPreferences)
var startCommentCharaterRegExp = regexp.MustCompile(`^# `)
var subsequentCommentCharaterRegExp = regexp.MustCompile(`\n# `)
log.Debugf("GetComments operator!")
var results = list.New()
@ -76,12 +84,25 @@ func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *
comment := ""
if preferences.LineComment {
comment = candidate.Node.LineComment
} else if preferences.HeadComment && candidate.LeadingContent != "" {
var chompRegexp = regexp.MustCompile(`\n$`)
var output bytes.Buffer
var writer = bufio.NewWriter(&output)
if err := processLeadingContent(candidate, writer, false, YamlOutputFormat); err != nil {
return Context{}, err
}
if err := writer.Flush(); err != nil {
return Context{}, err
}
comment = output.String()
comment = chompRegexp.ReplaceAllString(comment, "")
} else if preferences.HeadComment {
comment = candidate.Node.HeadComment
} else if preferences.FootComment {
comment = candidate.Node.FootComment
}
comment = strings.Replace(comment, "# ", "", 1)
comment = startCommentCharaterRegExp.ReplaceAllString(comment, "")
comment = subsequentCommentCharaterRegExp.ReplaceAllString(comment, "\n")
node := &yaml.Node{Kind: yaml.ScalarNode, Value: comment, Tag: "!!str"}
result := candidate.CreateChild(nil, node)

13
pkg/yqlib/operator_comments_test.go
View File

@ -112,13 +112,22 @@ var commentOperatorScenarios = []expressionScenario{
"D0, P[], (!!str)::welcome!\n",
},
},
{
skipDoc: false,
dontFormatInputForDoc: true,
document: "# welcome!\n---\n# bob\na: cat # meow\n\n# have a great day",
expression: `headComment`,
expected: []string{
"D0, P[], (!!str)::|-\n welcome!\n bob\n",
},
},
{
description: "Get foot comment",
dontFormatInputForDoc: true,
document: "# welcome!\n\na: cat # meow\n\n# have a great day",
document: "# welcome!\n\na: cat # meow\n\n# have a great day\n# no really",
expression: `. | footComment`,
expected: []string{
"D0, P[], (!!str)::have a great day\n",
"D0, P[], (!!str)::have a great day\nno really\n",
},
},
}

27
pkg/yqlib/operators_test.go
View File

@ -26,6 +26,20 @@ type expressionScenario struct {
dontFormatInputForDoc bool // dont format input doc for documentation generation
}
func readDocumentWithLeadingContent(content string, fakefilename string, fakeFileIndex int) (*list.List, error) {
reader, firstFileLeadingContent, err := processReadStream(bufio.NewReader(strings.NewReader(content)))
if err != nil {
return nil, err
}
inputs, err := readDocuments(reader, fakefilename, fakeFileIndex)
if err != nil {
return nil, err
}
inputs.Front().Value.(*CandidateNode).LeadingContent = firstFileLeadingContent
return inputs, nil
}
func testScenario(t *testing.T, s *expressionScenario) {
var err error
@ -37,15 +51,17 @@ func testScenario(t *testing.T, s *expressionScenario) {
inputs := list.New()
if s.document != "" {
inputs, err = readDocuments(strings.NewReader(s.document), "sample.yml", 0)
inputs, err = readDocumentWithLeadingContent(s.document, "sample.yml", 0)
if err != nil {
t.Error(err, s.document, s.expression)
return
}
if s.document2 != "" {
moreInputs, err := readDocuments(strings.NewReader(s.document2), "another.yml", 1)
moreInputs, err := readDocumentWithLeadingContent(s.document2, "another.yml", 1)
if err != nil {
t.Error(err, s.document, s.expression)
t.Error(err, s.document2, s.expression)
return
}
inputs.PushBackList(moreInputs)
@ -227,13 +243,14 @@ func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formatt
inputs := list.New()
if s.document != "" {
inputs, err = readDocuments(strings.NewReader(formattedDoc), "sample.yml", 0)
inputs, err = readDocumentWithLeadingContent(formattedDoc, "sample.yml", 0)
if err != nil {
t.Error(err, s.document, s.expression)
return
}
if s.document2 != "" {
moreInputs, err := readDocuments(strings.NewReader(formattedDoc2), "another.yml", 1)
moreInputs, err := readDocumentWithLeadingContent(formattedDoc2, "another.yml", 1)
if err != nil {
t.Error(err, s.document, s.expression)
return

56
pkg/yqlib/printer.go
View File

@ -84,7 +84,7 @@ func (p *resultsPrinter) printNode(node *yaml.Node, writer io.Writer) error {
var encoder Encoder
if node.Kind == yaml.ScalarNode && p.unwrapScalar && p.outputFormat == YamlOutputFormat {
return p.writeString(writer, node.Value+"\n")
return writeString(writer, node.Value+"\n")
}
if p.outputFormat == JsonOutputFormat {
@ -97,54 +97,6 @@ func (p *resultsPrinter) printNode(node *yaml.Node, writer io.Writer) error {
return encoder.Encode(node)
}
func (p *resultsPrinter) writeString(writer io.Writer, txt string) error {
_, errorWriting := writer.Write([]byte(txt))
return errorWriting
}
func (p *resultsPrinter) processLeadingContent(mappedDoc *CandidateNode, writer io.Writer) error {
if strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$") {
log.Debug("headcommentwas %v", mappedDoc.Node.HeadComment)
log.Debug("finished headcomment")
reader := bufio.NewReader(strings.NewReader(mappedDoc.Node.HeadComment))
mappedDoc.Node.HeadComment = ""
for {
readline, errReading := reader.ReadString('\n')
if errReading != nil && errReading != io.EOF {
return errReading
}
if strings.Contains(readline, "$yqLeadingContent$") {
// skip this
} else if strings.Contains(readline, "$yqDocSeperator$") {
if p.printDocSeparators {
if err := p.writeString(writer, "---\n"); err != nil {
return err
}
}
} else if p.outputFormat == YamlOutputFormat {
if err := p.writeString(writer, readline); err != nil {
return err
}
}
if errReading == io.EOF {
if readline != "" {
// the last comment we read didn't have a new line, put one in
if err := p.writeString(writer, "\n"); err != nil {
return err
}
}
break
}
}
}
return nil
}
func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
log.Debug("PrintResults for %v matches", matchingNodes.Len())
@ -180,16 +132,16 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
return errorWriting
}
commentStartsWithSeparator := strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$\n$yqDocSeperator$")
commentStartsWithSeparator := strings.Contains(mappedDoc.LeadingContent, "$yqLeadingContent$\n$yqDocSeperator$")
if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators && !commentStartsWithSeparator {
log.Debug("-- writing doc sep")
if err := p.writeString(writer, "---\n"); err != nil {
if err := writeString(writer, "---\n"); err != nil {
return err
}
}
if err := p.processLeadingContent(mappedDoc, writer); err != nil {
if err := processLeadingContent(mappedDoc, writer, p.printDocSeparators, p.outputFormat); err != nil {
return err
}

10
pkg/yqlib/stream_evaluator.go
View File

@ -35,8 +35,9 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadin
candidateNode := &CandidateNode{
Document: 0,
Filename: "",
Node: &yaml.Node{Kind: yaml.DocumentNode, HeadComment: leadingContent, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
Node: &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
FileIndex: 0,
LeadingContent: leadingContent,
}
inputList := list.New()
inputList.PushBack(candidateNode)
@ -100,15 +101,16 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr
} else if errorReading != nil {
return currentIndex, errorReading
}
if currentIndex == 0 {
dataBucket.HeadComment = leadingContent
}
candidateNode := &CandidateNode{
Document: currentIndex,
Filename: filename,
Node: &dataBucket,
FileIndex: s.fileIndex,
}
if currentIndex == 0 {
candidateNode.LeadingContent = leadingContent
}
inputList := list.New()
inputList.PushBack(candidateNode)

48
pkg/yqlib/utils.go
View File

@ -31,6 +31,54 @@ func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, s
return processReadStream(reader)
}
func writeString(writer io.Writer, txt string) error {
_, errorWriting := writer.Write([]byte(txt))
return errorWriting
}
func processLeadingContent(mappedDoc *CandidateNode, writer io.Writer, printDocSeparators bool, outputFormat PrinterOutputFormat) error {
if strings.Contains(mappedDoc.LeadingContent, "$yqLeadingContent$") {
log.Debug("headcommentwas %v", mappedDoc.LeadingContent)
log.Debug("finished headcomment")
reader := bufio.NewReader(strings.NewReader(mappedDoc.LeadingContent))
mappedDoc.Node.HeadComment = ""
for {
readline, errReading := reader.ReadString('\n')
if errReading != nil && errReading != io.EOF {
return errReading
}
if strings.Contains(readline, "$yqLeadingContent$") {
// skip this
} else if strings.Contains(readline, "$yqDocSeperator$") {
if printDocSeparators {
if err := writeString(writer, "---\n"); err != nil {
return err
}
}
} else if outputFormat == YamlOutputFormat {
if err := writeString(writer, readline); err != nil {
return err
}
}
if errReading == io.EOF {
if readline != "" {
// the last comment we read didn't have a new line, put one in
if err := writeString(writer, "\n"); err != nil {
return err
}
}
break
}
}
}
return nil
}
func processReadStream(reader *bufio.Reader) (io.Reader, string, error) {
var commentLineRegEx = regexp.MustCompile(`^\s*#`)
var sb strings.Builder