From 33ec66cfdd277170f680f3919adde0f8efdf785a Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Sat, 8 Oct 2022 13:12:12 +1100
Subject: [PATCH] Fixed handling of UTF8 encoded CSVs #1373

---
 acceptance_tests/inputs-format.sh | 14 ++++++++++++++
 go.mod                            |  1 +
 go.sum                            |  2 ++
 pkg/yqlib/decoder_csv_object.go   |  5 ++++-
 utf8.csv                          |  3 +++
 5 files changed, 24 insertions(+), 1 deletion(-)
 create mode 100644 utf8.csv

diff --git a/acceptance_tests/inputs-format.sh b/acceptance_tests/inputs-format.sh
index 31598f9f..2173678e 100755
--- a/acceptance_tests/inputs-format.sh
+++ b/acceptance_tests/inputs-format.sh
@@ -63,6 +63,20 @@ EOM
   assertEquals "$expected" "$X"
 }
 
+testInputCSVUTF8() {
+  read -r -d '' expected << EOM
+- id: 1
+  first: john
+  last: smith
+- id: 1
+  first: jane
+  last: smith
+EOM
+
+  X=$(./yq -p=csv utf8.csv)
+  assertEquals "$expected" "$X"
+}
+
 testInputTSV() {
   cat >test.tsv <<EOL
 fruit	yumLevel
diff --git a/go.mod b/go.mod
index 34efaa40..fcf07274 100644
--- a/go.mod
+++ b/go.mod
@@ -4,6 +4,7 @@ require (
 	github.com/a8m/envsubst v1.3.0
 	github.com/alecthomas/participle/v2 v2.0.0-beta.5
 	github.com/alecthomas/repr v0.1.0
+	github.com/dimchansky/utfbom v1.1.1
 	github.com/elliotchance/orderedmap v1.5.0
 	github.com/fatih/color v1.13.0
 	github.com/goccy/go-json v0.9.11
diff --git a/go.sum b/go.sum
index 6b8fd518..c2ec4a36 100644
--- a/go.sum
+++ b/go.sum
@@ -8,6 +8,8 @@ github.com/alecthomas/repr v0.1.0/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygv
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
+github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
 github.com/elliotchance/orderedmap v1.5.0 h1:1IsExUsjv5XNBD3ZdC7jkAAqLWOOKdbPTmkHx63OsBg=
 github.com/elliotchance/orderedmap v1.5.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
diff --git a/pkg/yqlib/decoder_csv_object.go b/pkg/yqlib/decoder_csv_object.go
index 20ccb6a9..c2b8f164 100644
--- a/pkg/yqlib/decoder_csv_object.go
+++ b/pkg/yqlib/decoder_csv_object.go
@@ -5,6 +5,7 @@ import (
 	"errors"
 	"io"
 
+	"github.com/dimchansky/utfbom"
 	yaml "gopkg.in/yaml.v3"
 )
 
@@ -19,7 +20,9 @@ func NewCSVObjectDecoder(separator rune) Decoder {
 }
 
 func (dec *csvObjectDecoder) Init(reader io.Reader) {
-	dec.reader = *csv.NewReader(reader)
+	cleanReader, enc := utfbom.Skip(reader)
+	log.Debugf("Detected encoding: %s\n", enc)
+	dec.reader = *csv.NewReader(cleanReader)
 	dec.reader.Comma = dec.separator
 	dec.finished = false
 }
diff --git a/utf8.csv b/utf8.csv
new file mode 100644
index 00000000..d1503b81
--- /dev/null
+++ b/utf8.csv
@@ -0,0 +1,3 @@
+id,first,last
+1,john,smith
+1,jane,smith