Fixed handling of UTF8 encoded CSVs #1373

acceptance_tests/inputs-format.sh

@@ -63,6 +63,20 @@ EOM
   assertEquals "$expected" "$X"
 }
 
+testInputCSVUTF8() {
+  read -r -d '' expected << EOM
+- id: 1
+  first: john
+  last: smith
+- id: 1
+  first: jane
+  last: smith
+EOM
+
+  X=$(./yq -p=csv utf8.csv)
+  assertEquals "$expected" "$X"
+}
+
 testInputTSV() {
   cat >test.tsv <<EOL
 fruit	yumLevel

go.mod

@@ -4,6 +4,7 @@ require (
 	github.com/a8m/envsubst v1.3.0
 	github.com/alecthomas/participle/v2 v2.0.0-beta.5
 	github.com/alecthomas/repr v0.1.0
+	github.com/dimchansky/utfbom v1.1.1
 	github.com/elliotchance/orderedmap v1.5.0
 	github.com/fatih/color v1.13.0
 	github.com/goccy/go-json v0.9.11

go.sum

@@ -8,6 +8,8 @@ github.com/alecthomas/repr v0.1.0/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygv
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
+github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
 github.com/elliotchance/orderedmap v1.5.0 h1:1IsExUsjv5XNBD3ZdC7jkAAqLWOOKdbPTmkHx63OsBg=
 github.com/elliotchance/orderedmap v1.5.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=

pkg/yqlib/decoder_csv_object.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"io"
 
+	"github.com/dimchansky/utfbom"
 	yaml "gopkg.in/yaml.v3"
 )
 
@@ -19,7 +20,9 @@ func NewCSVObjectDecoder(separator rune) Decoder {
 }
 
 func (dec *csvObjectDecoder) Init(reader io.Reader) {
-	dec.reader = *csv.NewReader(reader)
+	cleanReader, enc := utfbom.Skip(reader)
+	log.Debugf("Detected encoding: %s\n", enc)
+	dec.reader = *csv.NewReader(cleanReader)
 	dec.reader.Comma = dec.separator
 	dec.finished = false
 }

utf8.csv

@@ -0,0 +1,3 @@
+id,first,last
+1,john,smith
+1,jane,smith