From 4478bd14c9a19a8eda18f63379332d88ca1f768a Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 25 Nov 2022 12:05:56 +1100 Subject: [PATCH] Fixed loading yaml with header issue #1445 --- examples/small.yaml | 4 ++++ pkg/yqlib/decoder_yaml.go | 30 +++++++++++++++++++++++++----- pkg/yqlib/lexer_participle.go | 2 +- pkg/yqlib/operator_load.go | 12 +++++++++++- pkg/yqlib/operator_load_test.go | 9 +++++++++ pkg/yqlib/operators_test.go | 2 +- 6 files changed, 51 insertions(+), 8 deletions(-) create mode 100644 examples/small.yaml diff --git a/examples/small.yaml b/examples/small.yaml new file mode 100644 index 00000000..f28fe164 --- /dev/null +++ b/examples/small.yaml @@ -0,0 +1,4 @@ +--- +# comment +# about things +a: cat \ No newline at end of file diff --git a/pkg/yqlib/decoder_yaml.go b/pkg/yqlib/decoder_yaml.go index e18ad827..b4591e61 100644 --- a/pkg/yqlib/decoder_yaml.go +++ b/pkg/yqlib/decoder_yaml.go @@ -2,6 +2,7 @@ package yqlib import ( "bufio" + "bytes" "errors" "io" "regexp" @@ -12,11 +13,15 @@ import ( type yamlDecoder struct { decoder yaml.Decoder + + prefs YamlPreferences + // work around of various parsing issues by yaml.v3 with document headers - prefs YamlPreferences leadingContent string - readAnything bool - firstFile bool + bufferRead bytes.Buffer + + readAnything bool + firstFile bool } func NewYamlDecoder(prefs YamlPreferences) Decoder { @@ -59,6 +64,7 @@ func (dec *yamlDecoder) processReadStream(reader *bufio.Reader) (io.Reader, stri func (dec *yamlDecoder) Init(reader io.Reader) error { readerToUse := reader leadingContent := "" + dec.bufferRead = bytes.Buffer{} var err error // if we 'evaluating together' - we only process the leading content // of the first file - this ensures comments from subsequent files are @@ -68,6 +74,12 @@ func (dec *yamlDecoder) Init(reader io.Reader) error { if err != nil { return err } + } else if !dec.prefs.LeadingContentPreProcessing { + // if we're not process the leading content + // keep a copy of what we've read. This is incase its a + // doc with only comments - the decoder will return nothing + // then we can read the comments from bufferRead + readerToUse = io.TeeReader(reader, &dec.bufferRead) } dec.leadingContent = leadingContent dec.readAnything = false @@ -78,13 +90,20 @@ func (dec *yamlDecoder) Init(reader io.Reader) error { func (dec *yamlDecoder) Decode() (*CandidateNode, error) { var dataBucket yaml.Node - err := dec.decoder.Decode(&dataBucket) if errors.Is(err, io.EOF) && dec.leadingContent != "" && !dec.readAnything { // force returning an empty node with a comment. dec.readAnything = true return dec.blankNodeWithComment(), nil - + } else if errors.Is(err, io.EOF) && !dec.prefs.LeadingContentPreProcessing && !dec.readAnything { + // didn't find any yaml, + // check the tee buffer, maybe there were comments + dec.readAnything = true + dec.leadingContent = dec.bufferRead.String() + if dec.leadingContent != "" { + return dec.blankNodeWithComment(), nil + } + return nil, err } else if err != nil { return nil, err } @@ -97,6 +116,7 @@ func (dec *yamlDecoder) Decode() (*CandidateNode, error) { candidateNode.LeadingContent = dec.leadingContent dec.leadingContent = "" } + dec.readAnything = true // move document comments into candidate node // otherwise unwrap drops them. candidateNode.TrailingContent = dataBucket.FootComment diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go index 41cf7fd7..14ea14e8 100644 --- a/pkg/yqlib/lexer_participle.go +++ b/pkg/yqlib/lexer_participle.go @@ -86,7 +86,7 @@ var participleYqRules = []*participleYqRule{ {"LoadString", `load_?str|str_?load`, loadOp(nil, true), 0}, - {"LoadYaml", `load`, loadOp(NewYamlDecoder(ConfiguredYamlPreferences), false), 0}, + {"LoadYaml", `load`, loadOp(NewYamlDecoder(LoadYamlPreferences), false), 0}, {"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0}, diff --git a/pkg/yqlib/operator_load.go b/pkg/yqlib/operator_load.go index eda63b72..51af1994 100644 --- a/pkg/yqlib/operator_load.go +++ b/pkg/yqlib/operator_load.go @@ -9,6 +9,13 @@ import ( "gopkg.in/yaml.v3" ) +var LoadYamlPreferences = YamlPreferences{ + LeadingContentPreProcessing: false, + PrintDocSeparators: true, + UnwrapScalar: true, + EvaluateTogether: false, +} + type loadPrefs struct { loadAsString bool decoder Decoder @@ -43,7 +50,10 @@ func loadYaml(filename string, decoder Decoder) (*CandidateNode, error) { // return null candidate return &CandidateNode{Node: &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!null"}}, nil } else if documents.Len() == 1 { - return documents.Front().Value.(*CandidateNode), nil + candidate := documents.Front().Value.(*CandidateNode) + log.Debug("first comment:", candidate.LeadingContent) + // candidate.Node.Content[0].Content[0].HeadComment = candidate.LeadingContent + return candidate, nil } else { sequenceNode := &CandidateNode{Node: &yaml.Node{Kind: yaml.SequenceNode}} diff --git a/pkg/yqlib/operator_load_test.go b/pkg/yqlib/operator_load_test.go index 67a73e15..b27e8f70 100644 --- a/pkg/yqlib/operator_load_test.go +++ b/pkg/yqlib/operator_load_test.go @@ -13,6 +13,15 @@ var loadScenarios = []expressionScenario{ "D0, P[], (doc)::# comment\n\n", }, }, + { + skipDoc: true, + description: "Load file with a header comment into an array", + document: `- "../../examples/small.yaml"`, + expression: `.[] |= load(.)`, + expected: []string{ + "D0, P[], (doc)::- # comment\n # about things\n a: cat\n", + }, + }, { skipDoc: true, description: "Load empty file with no comment", diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index 79757895..f3363dc6 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -31,7 +31,7 @@ type expressionScenario struct { } func TestMain(m *testing.M) { - logging.SetLevel(logging.ERROR, "") + logging.SetLevel(logging.DEBUG, "") Now = func() time.Time { return time.Date(2021, time.May, 19, 1, 2, 3, 4, time.UTC) }