Fixed loading yaml with header issue #1445

This commit is contained in:
Mike Farah 2022-11-25 12:05:56 +11:00
parent 91dc315fdb
commit 4478bd14c9
6 changed files with 51 additions and 8 deletions

4
examples/small.yaml Normal file
View File

@ -0,0 +1,4 @@
---
# comment
# about things
a: cat

View File

@ -2,6 +2,7 @@ package yqlib
import (
"bufio"
"bytes"
"errors"
"io"
"regexp"
@ -12,9 +13,13 @@ import (
type yamlDecoder struct {
decoder yaml.Decoder
// work around of various parsing issues by yaml.v3 with document headers
prefs YamlPreferences
// work around of various parsing issues by yaml.v3 with document headers
leadingContent string
bufferRead bytes.Buffer
readAnything bool
firstFile bool
}
@ -59,6 +64,7 @@ func (dec *yamlDecoder) processReadStream(reader *bufio.Reader) (io.Reader, stri
func (dec *yamlDecoder) Init(reader io.Reader) error {
readerToUse := reader
leadingContent := ""
dec.bufferRead = bytes.Buffer{}
var err error
// if we 'evaluating together' - we only process the leading content
// of the first file - this ensures comments from subsequent files are
@ -68,6 +74,12 @@ func (dec *yamlDecoder) Init(reader io.Reader) error {
if err != nil {
return err
}
} else if !dec.prefs.LeadingContentPreProcessing {
// if we're not process the leading content
// keep a copy of what we've read. This is incase its a
// doc with only comments - the decoder will return nothing
// then we can read the comments from bufferRead
readerToUse = io.TeeReader(reader, &dec.bufferRead)
}
dec.leadingContent = leadingContent
dec.readAnything = false
@ -78,13 +90,20 @@ func (dec *yamlDecoder) Init(reader io.Reader) error {
func (dec *yamlDecoder) Decode() (*CandidateNode, error) {
var dataBucket yaml.Node
err := dec.decoder.Decode(&dataBucket)
if errors.Is(err, io.EOF) && dec.leadingContent != "" && !dec.readAnything {
// force returning an empty node with a comment.
dec.readAnything = true
return dec.blankNodeWithComment(), nil
} else if errors.Is(err, io.EOF) && !dec.prefs.LeadingContentPreProcessing && !dec.readAnything {
// didn't find any yaml,
// check the tee buffer, maybe there were comments
dec.readAnything = true
dec.leadingContent = dec.bufferRead.String()
if dec.leadingContent != "" {
return dec.blankNodeWithComment(), nil
}
return nil, err
} else if err != nil {
return nil, err
}
@ -97,6 +116,7 @@ func (dec *yamlDecoder) Decode() (*CandidateNode, error) {
candidateNode.LeadingContent = dec.leadingContent
dec.leadingContent = ""
}
dec.readAnything = true
// move document comments into candidate node
// otherwise unwrap drops them.
candidateNode.TrailingContent = dataBucket.FootComment

View File

@ -86,7 +86,7 @@ var participleYqRules = []*participleYqRule{
{"LoadString", `load_?str|str_?load`, loadOp(nil, true), 0},
{"LoadYaml", `load`, loadOp(NewYamlDecoder(ConfiguredYamlPreferences), false), 0},
{"LoadYaml", `load`, loadOp(NewYamlDecoder(LoadYamlPreferences), false), 0},
{"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0},

View File

@ -9,6 +9,13 @@ import (
"gopkg.in/yaml.v3"
)
var LoadYamlPreferences = YamlPreferences{
LeadingContentPreProcessing: false,
PrintDocSeparators: true,
UnwrapScalar: true,
EvaluateTogether: false,
}
type loadPrefs struct {
loadAsString bool
decoder Decoder
@ -43,7 +50,10 @@ func loadYaml(filename string, decoder Decoder) (*CandidateNode, error) {
// return null candidate
return &CandidateNode{Node: &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!null"}}, nil
} else if documents.Len() == 1 {
return documents.Front().Value.(*CandidateNode), nil
candidate := documents.Front().Value.(*CandidateNode)
log.Debug("first comment:", candidate.LeadingContent)
// candidate.Node.Content[0].Content[0].HeadComment = candidate.LeadingContent
return candidate, nil
} else {
sequenceNode := &CandidateNode{Node: &yaml.Node{Kind: yaml.SequenceNode}}

View File

@ -13,6 +13,15 @@ var loadScenarios = []expressionScenario{
"D0, P[], (doc)::# comment\n\n",
},
},
{
skipDoc: true,
description: "Load file with a header comment into an array",
document: `- "../../examples/small.yaml"`,
expression: `.[] |= load(.)`,
expected: []string{
"D0, P[], (doc)::- # comment\n # about things\n a: cat\n",
},
},
{
skipDoc: true,
description: "Load empty file with no comment",

View File

@ -31,7 +31,7 @@ type expressionScenario struct {
}
func TestMain(m *testing.M) {
logging.SetLevel(logging.ERROR, "")
logging.SetLevel(logging.DEBUG, "")
Now = func() time.Time {
return time.Date(2021, time.May, 19, 1, 2, 3, 4, time.UTC)
}