major vanilla lua speedups

ftcsv.lua

@@ -27,7 +27,7 @@ local ftcsv = {
     ]]
 }
 
--- lua 5.1 compat
+-- lua 5.1 load compat
 local M = {}
 if type(jit) == 'table' or _ENV then
     M.load = _G.load
@@ -35,17 +35,12 @@ else
     M.load = loadstring
 end
 
--- load an entire file into memory
+-- luajit specific speedups
-local function loadFile(textFile)
+-- luajit performs faster with iterating over string.byte,
-    local file = io.open(textFile, "r")
+-- whereas vanilla lua performs faster with string.find
-    if not file then error("File not found at " .. textFile) end
+if type(jit) == 'table' then
-    local allLines = file:read("*all")
-    file:close()
-    return allLines
-end
-
     -- finds the end of an escape sequence
-local function findClosingQuote(i, inputLength, inputString, quote, doubleQuoteEscape)
+    function M.findClosingQuote(i, inputLength, inputString, quote, doubleQuoteEscape)
         -- local doubleQuoteEscape = doubleQuoteEscape
         local currentChar, nextChar = string.byte(inputString, i), nil
         while i <= inputLength do
@@ -61,6 +56,7 @@ local function findClosingQuote(i, inputLength, inputString, quote, doubleQuoteE
 
             -- identifies the escape toggle
             elseif currentChar == quote and nextChar ~= quote then
+                -- print("exiting", i-1)
                 return i-1, doubleQuoteEscape
             else
                 i = i + 1
@@ -69,6 +65,40 @@ local function findClosingQuote(i, inputLength, inputString, quote, doubleQuoteE
         end
     end
 
+else
+    -- vanilla lua closing quote finder
+    function M.findClosingQuote(i, inputLength, inputString, quote, doubleQuoteEscape)
+        local firstCharIndex = 1
+        local firstChar, iChar = nil, nil
+        repeat
+            firstCharIndex, i = inputString:find('".?', i+1)
+            firstChar = string.byte(inputString, firstCharIndex)
+            iChar = string.byte(inputString, i)
+            -- nextChar = string.byte(inputString, i+1)
+            -- print("HI", offset, i)
+            -- print(firstChar, iChar)
+            if firstChar == quote and iChar == quote then
+                doubleQuoteEscape = true
+            end
+        until iChar ~= quote
+        if i == nil then
+            return inputLength-1, doubleQuoteEscape
+        end
+        -- print("exiting", i-2)
+        return i-2, doubleQuoteEscape
+    end
+
+end
+
+-- load an entire file into memory
+local function loadFile(textFile)
+    local file = io.open(textFile, "r")
+    if not file then error("File not found at " .. textFile) end
+    local allLines = file:read("*all")
+    file:close()
+    return allLines
+end
+
 -- creates a new field and adds it to the main table
 local function createNewField(inputString, quote, fieldStart, i, line, fieldNum, doubleQuoteEscape, fieldsToKeep)
     -- print(lineNum, fieldNum, fieldStart, i-1)
@@ -193,7 +223,7 @@ function ftcsv.parse(inputFile, delimiter, options)
             elseif currentChar == quote and nextChar ~= quote then
                 -- print("ESCAPE TOGGLE")
                 fieldStart = i + 1
-                i, doubleQuoteEscape = findClosingQuote(i+1, inputLength, inputString, quote, doubleQuoteEscape)
+                i, doubleQuoteEscape = M.findClosingQuote(i+1, inputLength, inputString, quote, doubleQuoteEscape)
                 -- print("I VALUE", i, doubleQuoteEscape)
                 skipChar = 1
             -- end

spec/parse_encode_spec.lua

@@ -34,6 +34,7 @@ describe("csv decode", function()
 			local json = loadFile("spec/json/" .. value .. ".json")
 			json = cjson.decode(json)
 			local parse = ftcsv.parse("spec/csvs/" .. value .. ".csv", ",")
+			assert.are.same(#json, #parse)
 			assert.are.same(json, parse)
 		end)
 	end