From 64525c3d8243bbc5949860aa6a253deab8c729e3 Mon Sep 17 00:00:00 2001 From: StressTestor <212606152+StressTestor@users.noreply.github.com> Date: Wed, 17 Jun 2026 21:26:33 -0600 Subject: [PATCH] fix: skip UTF-8 BOM when processing front matter --front-matter=process wrapped the file in a bufio.Reader without skipping a leading UTF-8 BOM, so a BOM before the opening --- meant the separator was not recognised and the opening --- was dropped from the output (data loss). Skip the BOM with utfbom.Skip before wrapping the reader, matching the CSV object decoder. --- pkg/yqlib/front_matter.go | 8 ++++++-- pkg/yqlib/front_matter_test.go | 36 ++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/pkg/yqlib/front_matter.go b/pkg/yqlib/front_matter.go index 4c56aabd..60e63228 100644 --- a/pkg/yqlib/front_matter.go +++ b/pkg/yqlib/front_matter.go @@ -5,6 +5,8 @@ import ( "errors" "io" "os" + + "github.com/dimchansky/utfbom" ) type frontMatterHandler interface { @@ -43,13 +45,15 @@ func (f *frontMatterHandlerImpl) Split() error { var reader *bufio.Reader var err error if f.originalFilename == "-" { - reader = bufio.NewReader(os.Stdin) + cleanReader, _ := utfbom.Skip(os.Stdin) + reader = bufio.NewReader(cleanReader) } else { file, err := os.Open(f.originalFilename) // #nosec if err != nil { return err } - reader = bufio.NewReader(file) + cleanReader, _ := utfbom.Skip(file) + reader = bufio.NewReader(cleanReader) } f.contentReader = reader diff --git a/pkg/yqlib/front_matter_test.go b/pkg/yqlib/front_matter_test.go index 65c17c50..dde2f3e2 100644 --- a/pkg/yqlib/front_matter_test.go +++ b/pkg/yqlib/front_matter_test.go @@ -157,6 +157,42 @@ Some content fmHandler.CleanUp() } +func TestFrontMatterSplitWithBOM(t *testing.T) { + // Regression test for https://github.com/mikefarah/yq/issues/2496 + // A UTF-8 BOM before the opening --- must be skipped, otherwise the + // separator isn't recognised and the opening --- is lost. + file := createTestFile("\ufeff---\na: apple\nb: banana\n---\nnot a\nyaml: doc\n") + + expectedYamlFm := `--- +a: apple +b: banana +` + + expectedContent := `--- +not a +yaml: doc +` + + fmHandler := NewFrontMatterHandler(file) + err := fmHandler.Split() + if err != nil { + panic(err) + } + + yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename()) + + test.AssertResult(t, expectedYamlFm, yamlFm) + + contentBytes, err := io.ReadAll(fmHandler.GetContentReader()) + if err != nil { + panic(err) + } + test.AssertResult(t, expectedContent, string(contentBytes)) + + tryRemoveTempFile(file) + fmHandler.CleanUp() +} + func TestFrontMatterSplitWithArray(t *testing.T) { file := createTestFile(`[1,2,3] ---