calculate document,filename and fileindex by parent

This commit is contained in:
Mike Farah 2023-05-05 15:19:58 +10:00
parent 68df67f550
commit 0ca8a13e36
25 changed files with 141 additions and 146 deletions

View File

@ -1,3 +1,3 @@
{name: Mike}
---
# cool
a: hello #things
{name: Bob}

View File

@ -64,11 +64,11 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
if allDocuments.Len() == 0 {
candidateNode := &CandidateNode{
Document: 0,
Filename: "",
document: 0,
filename: "",
Kind: DocumentNode,
Content: []*CandidateNode{createScalarNode(nil, "")},
FileIndex: 0,
fileIndex: 0,
LeadingContent: "",
}
allDocuments.PushBack(candidateNode)

View File

@ -74,13 +74,13 @@ type CandidateNode struct {
LeadingContent string
TrailingContent string
Document uint // the document index of this node
Filename string
document uint // the document index of this node
filename string
Line int
Column int
FileIndex int
fileIndex int
// when performing op against all nodes given, this will treat all the nodes as one
// (e.g. top level cross document merge). This property does not propagate to child nodes.
EvaluateTogether bool
@ -89,13 +89,44 @@ type CandidateNode struct {
func (n *CandidateNode) CreateChild() *CandidateNode {
return &CandidateNode{
Parent: n,
Document: n.Document,
Filename: n.Filename,
FileIndex: n.FileIndex,
Parent: n,
}
}
func (n *CandidateNode) SetDocument(idx uint) {
n.document = idx
}
func (n *CandidateNode) GetDocument() uint {
// defer to parent
if n.Parent != nil {
return n.Parent.GetDocument()
}
return n.document
}
func (n *CandidateNode) SetFilename(name string) {
n.filename = name
}
func (n *CandidateNode) GetFilename() string {
if n.Parent != nil {
return n.Parent.GetFilename()
}
return n.filename
}
func (n *CandidateNode) SetFileIndex(idx int) {
n.fileIndex = idx
}
func (n *CandidateNode) GetFileIndex() int {
if n.Parent != nil {
return n.Parent.GetFileIndex()
}
return n.fileIndex
}
func (n *CandidateNode) GetKey() string {
keyPrefix := ""
if n.IsMapKey {
@ -105,7 +136,7 @@ func (n *CandidateNode) GetKey() string {
if n.Key != nil {
key = n.Key.Value
}
return fmt.Sprintf("%v%v - %v", keyPrefix, n.Document, key)
return fmt.Sprintf("%v%v - %v", keyPrefix, n.GetDocument(), key)
}
func (n *CandidateNode) unwrapDocument() *CandidateNode {
@ -192,9 +223,6 @@ func (n *CandidateNode) AddKeyValueChild(rawKey *CandidateNode, rawValue *Candid
func (n *CandidateNode) SetParent(parent *CandidateNode) {
n.Parent = parent
n.Document = parent.Document
n.Filename = parent.Filename
n.FileIndex = parent.FileIndex
}
func (n *CandidateNode) AddChildren(children []*CandidateNode) {
@ -282,13 +310,6 @@ func (n *CandidateNode) CopyAsReplacement(replacement *CandidateNode) *Candidate
newCopy.Key = n.Key
newCopy.IsMapKey = n.IsMapKey
log.Debugf("n path: %v", n.GetNicePath())
log.Debugf("HERE**************")
log.Debugf("newCopy path: %v", newCopy.GetNicePath())
newCopy.Document = n.Document
newCopy.Filename = n.Filename
newCopy.FileIndex = n.FileIndex
return newCopy
}
@ -340,13 +361,13 @@ func (n *CandidateNode) doCopy(cloneContent bool) *CandidateNode {
LeadingContent: n.LeadingContent,
TrailingContent: n.TrailingContent,
Document: n.Document,
Filename: n.Filename,
document: n.document,
filename: n.filename,
fileIndex: n.fileIndex,
Line: n.Line,
Column: n.Column,
FileIndex: n.FileIndex,
EvaluateTogether: n.EvaluateTogether,
IsMapKey: n.IsMapKey,
}

View File

@ -40,6 +40,18 @@ var valueRepScenarios = []valueRepScenario{
},
}
func TestCandidateNodeChildWhenParentUpdated(t *testing.T) {
parent := CandidateNode{}
child := parent.CreateChild()
parent.SetDocument(1)
parent.SetFileIndex(2)
parent.SetFilename("meow")
test.AssertResultWithContext(t, "meow", child.GetFilename(), "filename")
test.AssertResultWithContext(t, 2, child.GetFileIndex(), "fileindex")
test.AssertResultWithContext(t, uint(1), child.GetDocument(), "document index")
}
func TestCandidateNodeGetValueRepScenarios(t *testing.T) {
for _, tt := range valueRepScenarios {
node := CandidateNode{Value: tt.input, Tag: tt.tag}

View File

@ -105,9 +105,6 @@ func (o *CandidateNode) UnmarshalJSON(data []byte) error {
childKey.IsMapKey = true
child.Parent = o
child.Document = o.Document
child.FileIndex = o.FileIndex
child.Filename = o.Filename
child.Key = childKey
o.Content = append(o.Content, child)
}

View File

@ -119,7 +119,7 @@ func (dec *yamlDecoder) Decode() (*CandidateNode, error) {
return nil, err
}
candidateNode := CandidateNode{Document: dec.documentIndex}
candidateNode := CandidateNode{document: dec.documentIndex}
err = candidateNode.UnmarshalYAML(&yamlNode, make(map[string]*CandidateNode))
if err != nil {
return nil, err
@ -140,11 +140,11 @@ func (dec *yamlDecoder) Decode() (*CandidateNode, error) {
func (dec *yamlDecoder) blankNodeWithComment() *CandidateNode {
return &CandidateNode{
Document: 0,
Filename: "",
document: 0,
filename: "",
Kind: DocumentNode,
Content: []*CandidateNode{createScalarNode(nil, "")},
FileIndex: 0,
fileIndex: 0,
LeadingContent: dec.leadingContent,
}
}

View File

@ -54,7 +54,6 @@ yq eval-all 'file_index' sample.yml another.yml
will output
```yaml
0
---
1
```

View File

@ -6,9 +6,7 @@ Filters an array (or map values) by the expression given. Equivalent to doing `m
## Filter array
Given a sample.yml file of:
```yaml
- 1
- 2
- 3
[1, 2, 3]
```
then
```bash
@ -16,19 +14,13 @@ yq 'filter(. < 3)' sample.yml
```
will output
```yaml
- 1
- 2
[1, 2]
```
## Filter map values
Given a sample.yml file of:
```yaml
c:
things: cool
frog: yes
d:
things: hot
frog: false
{c: {things: cool, frog: yes}, d: {things: hot, frog: false}}
```
then
```bash
@ -36,7 +28,6 @@ yq 'filter(.things == "cool")' sample.yml
```
will output
```yaml
- things: cool
frog: yes
[{things: cool, frog: yes}]
```

View File

@ -6,9 +6,7 @@ Recursively flattens all arrays
Given a sample.yml file of:
```yaml
- 1
- - 2
- - - 3
[1, [2], [[3]]]
```
then
```bash
@ -16,17 +14,13 @@ yq 'flatten' sample.yml
```
will output
```yaml
- 1
- 2
- 3
[1, 2, 3]
```
## Flatten with depth of one
Given a sample.yml file of:
```yaml
- 1
- - 2
- - - 3
[1, [2], [[3]]]
```
then
```bash
@ -34,15 +28,13 @@ yq 'flatten(1)' sample.yml
```
will output
```yaml
- 1
- 2
- - 3
[1, 2, [3]]
```
## Flatten empty array
Given a sample.yml file of:
```yaml
- []
[[]]
```
then
```bash
@ -56,8 +48,7 @@ will output
## Flatten array of objects
Given a sample.yml file of:
```yaml
- foo: bar
- - foo: baz
[{foo: bar}, [{foo: baz}]]
```
then
```bash
@ -65,7 +56,6 @@ yq 'flatten' sample.yml
```
will output
```yaml
- foo: bar
- foo: baz
[{foo: bar}, {foo: baz}]
```

View File

@ -5,12 +5,7 @@ This is used to group items in an array by an expression.
## Group by field
Given a sample.yml file of:
```yaml
- foo: 1
bar: 10
- foo: 3
bar: 100
- foo: 1
bar: 1
[{foo: 1, bar: 10}, {foo: 3, bar: 100}, {foo: 1, bar: 1}]
```
then
```bash
@ -18,25 +13,15 @@ yq 'group_by(.foo)' sample.yml
```
will output
```yaml
- - foo: 1
bar: 10
- foo: 1
bar: 1
- - foo: 3
bar: 100
- - [{foo: 1, bar: 10}, {foo: 3, bar: 100}, {foo: 1, bar: 1}]
- [{foo: 1, bar: 10}, {foo: 3, bar: 100}, {foo: 1, bar: 1}]
- - [{foo: 1, bar: 10}, {foo: 3, bar: 100}, {foo: 1, bar: 1}]
```
## Group by field, with nuls
Given a sample.yml file of:
```yaml
- cat: dog
- foo: 1
bar: 10
- foo: 3
bar: 100
- no: foo for you
- foo: 1
bar: 1
[{cat: dog}, {foo: 1, bar: 10}, {foo: 3, bar: 100}, {no: foo for you}, {foo: 1, bar: 1}]
```
then
```bash
@ -44,13 +29,10 @@ yq 'group_by(.foo)' sample.yml
```
will output
```yaml
- - cat: dog
- no: foo for you
- - foo: 1
bar: 10
- foo: 1
bar: 1
- - foo: 3
bar: 100
- - [{cat: dog}, {foo: 1, bar: 10}, {foo: 3, bar: 100}, {no: foo for you}, {foo: 1, bar: 1}]
- [{cat: dog}, {foo: 1, bar: 10}, {foo: 3, bar: 100}, {no: foo for you}, {foo: 1, bar: 1}]
- - [{cat: dog}, {foo: 1, bar: 10}, {foo: 3, bar: 100}, {no: foo for you}, {foo: 1, bar: 1}]
- [{cat: dog}, {foo: 1, bar: 10}, {foo: 3, bar: 100}, {no: foo for you}, {foo: 1, bar: 1}]
- - [{cat: dog}, {foo: 1, bar: 10}, {foo: 3, bar: 100}, {no: foo for you}, {foo: 1, bar: 1}]
```

View File

@ -434,7 +434,7 @@ func NodeToString(node *CandidateNode) string {
if valueToUse == "" {
valueToUse = fmt.Sprintf("%v kids", len(node.Content))
}
return fmt.Sprintf(`D%v, P%v, %v (%v)::%v`, node.Document, node.GetNicePath(), KindString(node.Kind), tag, valueToUse)
return fmt.Sprintf(`D%v, P%v, %v (%v)::%v`, node.GetDocument(), node.GetNicePath(), KindString(node.Kind), tag, valueToUse)
}
func KindString(kind Kind) string {
switch kind {

View File

@ -10,8 +10,6 @@ func createMapOperator(d *dataTreeNavigator, context Context, expressionNode *Ex
//each matchingNodes entry should turn into a sequence of keys to create.
//then collect object should do a cross function of the same index sequence for all matches.
var document uint
sequences := list.New()
if context.MatchingNodes.Len() > 0 {
@ -33,7 +31,6 @@ func createMapOperator(d *dataTreeNavigator, context Context, expressionNode *Ex
}
node := listToNodeSeq(sequences)
node.Document = document
return context.SingleChildContext(node), nil
@ -41,10 +38,15 @@ func createMapOperator(d *dataTreeNavigator, context Context, expressionNode *Ex
func sequenceFor(d *dataTreeNavigator, context Context, matchingNode *CandidateNode, expressionNode *ExpressionNode) (*CandidateNode, error) {
var document uint
var filename string
var fileIndex int
var matches = list.New()
if matchingNode != nil {
document = matchingNode.Document
document = matchingNode.GetDocument()
filename = matchingNode.GetFilename()
fileIndex = matchingNode.GetFileIndex()
matches.PushBack(matchingNode)
}
@ -58,7 +60,9 @@ func sequenceFor(d *dataTreeNavigator, context Context, matchingNode *CandidateN
node.AddKeyValueChild(lhs, rhs)
node.Document = document
node.document = document
node.fileIndex = fileIndex
node.filename = filename
return node, nil
}, false)
@ -68,7 +72,9 @@ func sequenceFor(d *dataTreeNavigator, context Context, matchingNode *CandidateN
}
innerList := listToNodeSeq(mapPairs.MatchingNodes)
innerList.Style = FlowStyle
innerList.Document = document
innerList.document = document
innerList.fileIndex = fileIndex
innerList.filename = filename
return innerList, nil
}

View File

@ -87,10 +87,8 @@ func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *Expre
Value: formattedTimeStr,
}
}
node.Document = candidate.Document
node.Parent = candidate.Parent
node.Key = candidate.Key
node.FileIndex = candidate.FileIndex
results.PushBack(node)
}

View File

@ -10,7 +10,7 @@ func getDocumentIndexOperator(d *dataTreeNavigator, context Context, expressionN
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
scalar := candidate.CreateReplacement(ScalarNode, "!!int", fmt.Sprintf("%v", candidate.Document))
scalar := candidate.CreateReplacement(ScalarNode, "!!int", fmt.Sprintf("%v", candidate.GetDocument()))
results.PushBack(scalar)
}
return context.ChildContext(results), nil

View File

@ -153,9 +153,6 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
node := decodedNode.unwrapDocument()
node.Key = candidate.Key
node.Parent = candidate.Parent
node.Document = candidate.Document
node.FileIndex = candidate.FileIndex
node.Filename = candidate.Filename
results.PushBack(node)
}

View File

@ -35,7 +35,9 @@ func envOperator(d *dataTreeNavigator, context Context, expressionNode *Expressi
return Context{}, fmt.Errorf("value for env variable '%v' not provided in env()", envName)
} else {
decoder := NewYamlDecoder(ConfiguredYamlPreferences)
decoder.Init(strings.NewReader(rawValue))
if err := decoder.Init(strings.NewReader(rawValue)); err != nil {
return Context{}, err
}
result, err := decoder.Decode()
if err != nil {

View File

@ -12,7 +12,7 @@ func getFilenameOperator(d *dataTreeNavigator, context Context, expressionNode *
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
result := candidate.CreateReplacement(ScalarNode, "!!str", candidate.Filename)
result := candidate.CreateReplacement(ScalarNode, "!!str", candidate.GetFilename())
results.PushBack(result)
}
@ -26,7 +26,7 @@ func getFileIndexOperator(d *dataTreeNavigator, context Context, expressionNode
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
result := candidate.CreateReplacement(ScalarNode, "!!int", fmt.Sprintf("%v", candidate.FileIndex))
result := candidate.CreateReplacement(ScalarNode, "!!int", fmt.Sprintf("%v", candidate.GetFileIndex()))
results.PushBack(result)
}

View File

@ -44,7 +44,7 @@ var fileOperatorScenarios = []expressionScenario{
document: "a: cat\nb: dog",
expression: `.. lineComment |= filename`,
expected: []string{
"D0, P[], (!!map)::a: cat # sample.yml\nb: dog # sample.yml\n",
"D0, P[], (doc)::a: cat # sample.yml\nb: dog # sample.yml\n",
},
},
}

View File

@ -6,7 +6,7 @@ func splitDocumentOperator(d *dataTreeNavigator, context Context, expressionNode
var index uint
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
candidate.Document = index
candidate.SetDocument(index)
index = index + 1
}

View File

@ -31,7 +31,7 @@ type expressionScenario struct {
}
func TestMain(m *testing.M) {
logging.SetLevel(logging.DEBUG, "")
logging.SetLevel(logging.ERROR, "")
Now = func() time.Time {
return time.Date(2021, time.May, 19, 1, 2, 3, 4, time.UTC)
}
@ -79,11 +79,11 @@ func testScenario(t *testing.T, s *expressionScenario) {
}
} else {
candidateNode := &CandidateNode{
Document: 0,
Filename: "",
document: 0,
filename: "",
Tag: "!!null",
Kind: ScalarNode,
FileIndex: 0,
fileIndex: 0,
}
inputs.PushBack(candidateNode)
@ -145,7 +145,7 @@ func resultToString(t *testing.T, n *CandidateNode) string {
} else if n.Kind == AliasNode {
tag = "alias"
}
return fmt.Sprintf(`D%v, P%v, (%v)::%v`, n.Document, n.GetPath(), tag, valueBuffer.String())
return fmt.Sprintf(`D%v, P%v, (%v)::%v`, n.GetDocument(), n.GetPath(), tag, valueBuffer.String())
}
func resultsToString(t *testing.T, results *list.List) []string {
@ -351,11 +351,11 @@ func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formatt
}
} else {
candidateNode := &CandidateNode{
Document: 0,
Filename: "",
document: 0,
filename: "",
Tag: "!!null",
Kind: ScalarNode,
FileIndex: 0,
fileIndex: 0,
}
inputs.PushBack(candidateNode)

View File

@ -128,15 +128,15 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
if p.firstTimePrinting {
node := matchingNodes.Front().Value.(*CandidateNode)
p.previousDocIndex = node.Document
p.previousFileIndex = node.FileIndex
p.previousDocIndex = node.GetDocument()
p.previousFileIndex = node.GetFileIndex()
p.firstTimePrinting = false
}
for el := matchingNodes.Front(); el != nil; el = el.Next() {
mappedDoc := el.Value.(*CandidateNode)
log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document)
log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v", p.firstTimePrinting, p.previousDocIndex)
log.Debug("%v", NodeToString(mappedDoc))
writer, errorWriting := p.printerWriter.GetWriter(mappedDoc)
if errorWriting != nil {
@ -146,7 +146,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
commentsStartWithSepExp := regexp.MustCompile(`^\$yqDocSeperator\$`)
commentStartsWithSeparator := commentsStartWithSepExp.MatchString(mappedDoc.LeadingContent)
if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && !commentStartsWithSeparator {
if (p.previousDocIndex != mappedDoc.GetDocument() || p.previousFileIndex != mappedDoc.GetFileIndex()) && !commentStartsWithSeparator {
if err := p.encoder.PrintDocumentSeparator(writer); err != nil {
return err
}
@ -186,7 +186,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
}
}
p.previousDocIndex = mappedDoc.Document
p.previousDocIndex = mappedDoc.GetDocument()
if err := writer.Flush(); err != nil {
return err
}

View File

@ -125,20 +125,20 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) {
el := inputs.Front()
elNode := el.Value.(*CandidateNode)
elNode.Document = 0
elNode.FileIndex = 0
elNode.document = 0
elNode.fileIndex = 0
sample1 := nodeToList(elNode)
el = el.Next()
elNode = el.Value.(*CandidateNode)
elNode.Document = 0
elNode.FileIndex = 1
elNode.document = 0
elNode.fileIndex = 1
sample2 := nodeToList(elNode)
el = el.Next()
elNode = el.Value.(*CandidateNode)
elNode.Document = 0
elNode.FileIndex = 2
elNode.document = 0
elNode.fileIndex = 2
sample3 := nodeToList(elNode)
err = printer.PrintResults(sample1)
@ -172,22 +172,22 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
el := inputs.Front()
elNode := el.Value.(*CandidateNode)
elNode.Document = 0
elNode.FileIndex = 0
elNode.document = 0
elNode.fileIndex = 0
elNode.LeadingContent = "# go cats\n$yqDocSeperator$\n"
sample1 := nodeToList(elNode)
el = el.Next()
elNode = el.Value.(*CandidateNode)
elNode.Document = 0
elNode.FileIndex = 1
elNode.document = 0
elNode.fileIndex = 1
elNode.LeadingContent = "$yqDocSeperator$\n"
sample2 := nodeToList(elNode)
el = el.Next()
elNode = el.Value.(*CandidateNode)
elNode.Document = 0
elNode.FileIndex = 2
elNode.document = 0
elNode.fileIndex = 2
elNode.LeadingContent = "$yqDocSeperator$\n# cool\n"
sample3 := nodeToList(elNode)

View File

@ -32,11 +32,11 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error
return err
}
candidateNode := &CandidateNode{
Document: 0,
Filename: "",
document: 0,
filename: "",
Kind: DocumentNode,
Content: []*CandidateNode{createScalarNode(nil, "")},
FileIndex: 0,
fileIndex: 0,
}
inputList := list.New()
inputList.PushBack(candidateNode)
@ -97,9 +97,9 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr
} else if errorReading != nil {
return currentIndex, fmt.Errorf("bad file '%v': %w", filename, errorReading)
}
candidateNode.Document = currentIndex
candidateNode.Filename = filename
candidateNode.FileIndex = s.fileIndex
candidateNode.document = currentIndex
candidateNode.filename = filename
candidateNode.fileIndex = s.fileIndex
inputList := list.New()
inputList.PushBack(candidateNode)

View File

@ -53,8 +53,8 @@ func (s *stringEvaluator) Evaluate(expression string, input string, encoder Enco
} else if errorReading != nil {
return "", fmt.Errorf("bad input '%v': %w", input, errorReading)
}
candidateNode.Document = currentIndex
candidateNode.FileIndex = s.fileIndex
candidateNode.document = currentIndex
candidateNode.fileIndex = s.fileIndex
inputList := list.New()
inputList.PushBack(candidateNode)

View File

@ -51,9 +51,9 @@ func readDocuments(reader io.Reader, filename string, fileIndex int, decoder Dec
} else if errorReading != nil {
return nil, fmt.Errorf("bad file '%v': %w", filename, errorReading)
}
candidateNode.Document = currentIndex
candidateNode.Filename = filename
candidateNode.FileIndex = fileIndex
candidateNode.document = currentIndex
candidateNode.filename = filename
candidateNode.fileIndex = fileIndex
candidateNode.EvaluateTogether = true
inputList.PushBack(candidateNode)