will now strip out BOM

README.md

@@ -3,7 +3,7 @@
 
 ftcsv, a fairly fast csv library written in pure Lua. It's been tested with LuaJIT 2.0/2.1 and Lua 5.1, 5.2, and 5.3
 
-It works well for CSVs that can easily be fully loaded into memory (easily up to a hundred MB). Currently, there isn't a "large" file mode with proper readers and writers for ingesting CSVs in bulk with a fixed amount of memory. It correctly handles both `\n` (LF), `\r` (CR) and `\r\n` (CRLF) line endings (ie it should work with Unix, Mac OS 9, and Windows line endings) and has UTF-8 support.
+It works well for CSVs that can easily be fully loaded into memory (easily up to a hundred MB). Currently, there isn't a "large" file mode with proper readers and writers for ingesting CSVs in bulk with a fixed amount of memory. It correctly handles both `\n` (LF), `\r` (CR) and `\r\n` (CRLF) line endings (ie it should work with Unix, Mac OS 9, and Windows line endings), strips out the utf BOM (if it exists), and has UTF-8 support.
 
 
 

ftcsv.lua

@@ -347,7 +347,13 @@ function ftcsv.parse(inputFile, delimiter, options)
     end
 
     -- parse through the headers!
-    local headerField, i = parseString(inputString, inputLength, delimiter, 1)
+    local startLine = 1
+
+    -- check for BOM
+    if string.byte(inputString, 1) == 239 and string.byte(inputString, 2) == 187 and string.byte(inputString, 3) == 191 then
+        startLine = 4
+    end
+    local headerField, i = parseString(inputString, inputLength, delimiter, startLine)
     i = i + 1 -- start at the next char
 
     -- make sure a header isn't empty

spec/csvs/bom-os9.csv

@@ -0,0 +1 @@
+a,b,c
1,2,3
4,5,ʤ

spec/csvs/os9.csv

@@ -0,0 +1 @@
+a,b,c
1,2,3
4,5,ʤ

spec/json/bom-os9.json

@@ -0,0 +1,12 @@
+[
+  {
+    "a": "1",
+    "b": "2",
+    "c": "3"
+  },
+  {
+    "a": "4",
+    "b": "5",
+    "c": "ʤ"
+  }
+]

spec/json/os9.json

@@ -0,0 +1,12 @@
+[
+  {
+    "a": "1",
+    "b": "2",
+    "c": "3"
+  },
+  {
+    "a": "4",
+    "b": "5",
+    "c": "ʤ"
+  }
+]

spec/parse_encode_spec.lua

@@ -10,6 +10,7 @@ local function loadFile(textFile)
 end
 
 local files = {
+	"bom-os9",
 	"comma_in_quotes",
 	"correctness",
 	"empty",
@@ -22,6 +23,7 @@ local files = {
 	"json_no_newline",
 	"newlines",
 	"newlines_crlf",
+	"os9",
 	"quotes_and_newlines",
 	"quotes_non_escaped",
 	"simple",