From e5b52d0eb141c4ac09fa9992139cc16d31489e7f Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Thu, 2 Nov 2017 13:52:08 +0000 Subject: [PATCH] Added rowTable option to ftcsv.parse() rowTable makes ftcsv.parse() store rows in an existing table, which allows merging of multiple CSV files. --- README.md | 13 +++++++++++++ ftcsv.lua | 19 ++++++++++++------- spec/feature_spec.lua | 18 +++++++++++++++++- 3 files changed, 42 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index fd90ef9..c7c9401 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,19 @@ ftcsv.parse("apple,banana,carrot", ",", {loadFromString=true, headers=false}) In the above example, the first field becomes 'a', the second field becomes 'b' and so on. + - `rowTable` + + If you want to merge multiple CSV files, you can set `rowTable` to be an existing table to which ftcsv will append new rows. + + Note: headers are not compared between multiple datasets. If there is a mismatch between headers of different CSV files, then rows from each CSV file will have missing fields. + + ```lua + local options = {loadFromString=true, rename={["a"] = "d", ["b"] = "e", ["c"] = "f"}} + local actual = ftcsv.parse("a,b,c\r\napple,banana,carrot", ",", options) + options.rowTable = actual + ftcsv.parse("a,b,c\r\ndamson,elderberry,fig", ",", options) + ``` + For all tested examples, take a look in /spec/feature_spec.lua diff --git a/ftcsv.lua b/ftcsv.lua index af4abec..1f9a6aa 100644 --- a/ftcsv.lua +++ b/ftcsv.lua @@ -122,7 +122,7 @@ local function createField(inputString, quote, fieldStart, i, doubleQuoteEscape) end -- main function used to parse -local function parseString(inputString, inputLength, delimiter, i, headerField, fieldsToKeep) +local function parseString(outResults, inputString, inputLength, delimiter, i, headerField, fieldsToKeep) -- keep track of my chars! local currentChar, nextChar = sbyte(inputString, i), nil @@ -141,7 +141,6 @@ local function parseString(inputString, inputLength, delimiter, i, headerField, local delimiterByte = sbyte(delimiter) local assignValue - local outResults -- the headers haven't been set yet. -- aka this is the first run! if headerField == nil then @@ -153,8 +152,9 @@ local function parseString(inputString, inputLength, delimiter, i, headerField, end else -- print("this is for magic") - outResults = {} - outResults[1] = {} + outResults = outResults or {} + lineNum = #outResults + 1 + outResults[lineNum] = {} assignValue = function() if not pcall(function() outResults[lineNum][headerField[fieldNum]] = field @@ -300,6 +300,7 @@ function ftcsv.parse(inputFile, delimiter, options) local fieldsToKeep = nil local loadFromString = false local headerFunc + local rowTable = {} if options then if options.headers ~= nil then assert(type(options.headers) == "boolean", "ftcsv only takes the boolean 'true' or 'false' for the optional parameter 'headers' (default 'true'). You passed in '" .. tostring(options.headers) .. "' of type '" .. type(options.headers) .. "'.") @@ -330,6 +331,10 @@ function ftcsv.parse(inputFile, delimiter, options) assert(type(options.headerFunc) == "function", "ftcsv only takes a function value for optional parameter 'headerFunc'. You passed in '" .. tostring(options.headerFunc) .. "' of type '" .. type(options.headerFunc) .. "'.") headerFunc = options.headerFunc end + if options.rowTable ~= nil then + assert(type(options.rowTable) == "table", "ftcsv only takes in a table for the optional parameter 'rowTable'. You passed in '" .. tostring(options.rowTable) .. "' of type '" .. type(options.rowTable) .. "'.") + rowTable = options.rowTable + end end -- handle input via string or file! @@ -347,7 +352,7 @@ function ftcsv.parse(inputFile, delimiter, options) end -- parse through the headers! - local headerField, i = parseString(inputString, inputLength, delimiter, 1) + local headerField, i = parseString(nil, inputString, inputLength, delimiter, 1) i = i + 1 -- start at the next char -- make sure a header isn't empty @@ -389,8 +394,8 @@ function ftcsv.parse(inputFile, delimiter, options) end end - local output = parseString(inputString, inputLength, delimiter, i, headerField, fieldsToKeep) - return output + rowTable = parseString(rowTable, inputString, inputLength, delimiter, i, headerField, fieldsToKeep) + return rowTable end -- a function that delimits " to "", used by the writer diff --git a/spec/feature_spec.lua b/spec/feature_spec.lua index 8d5b75e..35a5ea0 100644 --- a/spec/feature_spec.lua +++ b/spec/feature_spec.lua @@ -21,6 +21,22 @@ describe("csv features", function() assert.are.same(expected, actual) end) + it("should handle merging multiple datasets", function() + local expected = {} + expected[1] = {} + expected[1].a = "apple" + expected[1].b = "banana" + expected[1].c = "carrot" + expected[2] = {} + expected[2].b = "banana" + expected[2].c = "carrot" + expected[2].d = "damson" + expected[2].e = "elderberry" + local actual = ftcsv.parse("a,b,c\napple,banana,carrot", ",", {loadFromString=true}) + ftcsv.parse("b,c,d,e\nbanana,carrot,damson,elderberry", ",", {loadFromString=true, rowTable=actual}) + assert.are.same(expected, actual) + end) + it("should handle renaming a field", function() local expected = {} expected[1] = {} @@ -191,4 +207,4 @@ describe("csv features", function() assert.are.same(expected, actual) end) -end) \ No newline at end of file +end)