diff --git a/.github/workflows/test-and-coverage.yml b/.github/workflows/test-and-coverage.yml index db5914f..667172b 100644 --- a/.github/workflows/test-and-coverage.yml +++ b/.github/workflows/test-and-coverage.yml @@ -12,26 +12,27 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - luaVersion: [ "5.4", "5.3", "5.2", "5.1", "luajit", "luajit-openresty" ] + lua: [lua=5.1, lua=5.2, lua=5.3, lua=5.4, luajit=@v2.0, luajit=@v2.1] steps: - uses: actions/checkout@master - - name: Setup ‘lua’ - uses: leafo/gh-actions-lua@v9 - with: - luaVersion: ${{ matrix.luaVersion }} - - name: Setup ‘luarocks’ - uses: leafo/gh-actions-luarocks@v4 + - name: Install libreadline + run: sudo apt-get install -y libreadline-dev + + - name: Install Lua (${{ matrix.lua }}) + run: | + pip install git+https://github.com/luarocks/hererocks + hererocks -r^ --${{ matrix.lua }} lua_install + echo lua_install/bin >> $GITHUB_PATH - - name: install depedencies + - name: Install depedencies run: | luarocks install busted luarocks install lua-cjson luarocks install luacov luarocks install luacov-coveralls - - - name: run unit tests with coverage + - name: Run unit tests with coverage run: busted --verbose --coverage - name: Report test coverage diff --git a/README.md b/README.md index 9af2688..1b9e3be 100644 --- a/README.md +++ b/README.md @@ -48,80 +48,80 @@ end The options are the same for `parseLine` and `parse`, with the exception of `loadFromString` and `bufferSize`. `loadFromString` only works with `parse` and `bufferSize` can only be specified for `parseLine`. The following are optional parameters passed in via the third argument as a table. - - `delimiter` +- `delimiter` - If your file doesn't use the comma character as the delimiter, you can specify your own. It is limited to one character and defaults to `,` - ```lua - ftcsv.parse("a>b>c\r\n1,2,3", {loadFromString=true, delimiter=">"}) - ``` + If your file doesn't use the comma character as the delimiter, you can specify your own. It is limited to one character and defaults to `,` + ```lua + ftcsv.parse("a>b>c\r\n1,2,3", {loadFromString=true, delimiter=">"}) + ``` - - `loadFromString` +- `loadFromString` - If you want to load a csv from a string instead of a file, set `loadFromString` to `true` (default: `false`) - ```lua - ftcsv.parse("a,b,c\r\n1,2,3", {loadFromString=true}) - ``` + If you want to load a csv from a string instead of a file, set `loadFromString` to `true` (default: `false`) + ```lua + ftcsv.parse("a,b,c\r\n1,2,3", {loadFromString=true}) + ``` - - `rename` +- `rename` - If you want to rename a field, you can set `rename` to change the field names. The below example will change the headers from `a,b,c` to `d,e,f` + If you want to rename a field, you can set `rename` to change the field names. The below example will change the headers from `a,b,c` to `d,e,f` - Note: You can rename two fields to the same value, ftcsv will keep the field that appears latest in the line. + Note: You can rename two fields to the same value, ftcsv will keep the field that appears latest in the line. - ```lua - local options = {loadFromString=true, rename={["a"] = "d", ["b"] = "e", ["c"] = "f"}} - local actual = ftcsv.parse("a,b,c\r\napple,banana,carrot", options) - ``` + ```lua + local options = {loadFromString=true, rename={["a"] = "d", ["b"] = "e", ["c"] = "f"}} + local actual = ftcsv.parse("a,b,c\r\napple,banana,carrot", options) + ``` - - `fieldsToKeep` +- `fieldsToKeep` - If you only want to keep certain fields from the CSV, send them in as a table-list and it should parse a little faster and use less memory. + If you only want to keep certain fields from the CSV, send them in as a table-list and it should parse a little faster and use less memory. - Note: If you want to keep a renamed field, put the new name of the field in `fieldsToKeep`: + Note: If you want to keep a renamed field, put the new name of the field in `fieldsToKeep`: - ```lua - local options = {loadFromString=true, fieldsToKeep={"a","f"}, rename={["c"] = "f"}} - local actual = ftcsv.parse("a,b,c\r\napple,banana,carrot\r\n", options) - ``` + ```lua + local options = {loadFromString=true, fieldsToKeep={"a","f"}, rename={["c"] = "f"}} + local actual = ftcsv.parse("a,b,c\r\napple,banana,carrot\r\n", options) + ``` - Also Note: If you apply a function to the headers via headerFunc, and want to select fields from fieldsToKeep, you need to have what the post-modified header would be in fieldsToKeep. + Also Note: If you apply a function to the headers via headerFunc, and want to select fields from fieldsToKeep, you need to have what the post-modified header would be in fieldsToKeep. - - `ignoreQuotes` +- `ignoreQuotes` - If `ignoreQuotes` is `true`, it will leave all quotes in the final parsed output. This is useful in situations where the fields aren't quoted, but contain quotes, or if the CSV didn't handle quotes correctly and you're trying to parse it. + If `ignoreQuotes` is `true`, it will leave all quotes in the final parsed output. This is useful in situations where the fields aren't quoted, but contain quotes, or if the CSV didn't handle quotes correctly and you're trying to parse it. - ```lua - local options = {loadFromString=true, ignoreQuotes=true} - local actual = ftcsv.parse('a,b,c\n"apple,banana,carrot', options) - ``` + ```lua + local options = {loadFromString=true, ignoreQuotes=true} + local actual = ftcsv.parse('a,b,c\n"apple,banana,carrot', options) + ``` - - `headerFunc` +- `headerFunc` - Applies a function to every field in the header. If you are using `rename`, the function is applied after the rename. + Applies a function to every field in the header. If you are using `rename`, the function is applied after the rename. - Ex: making all fields uppercase - ```lua - local options = {loadFromString=true, headerFunc=string.upper} - local actual = ftcsv.parse("a,b,c\napple,banana,carrot", options) - ``` + Ex: making all fields uppercase + ```lua + local options = {loadFromString=true, headerFunc=string.upper} + local actual = ftcsv.parse("a,b,c\napple,banana,carrot", options) + ``` - - `headers` +- `headers` - Set `headers` to `false` if the file you are reading doesn't have any headers. This will cause ftcsv to create indexed tables rather than a key-value tables for the output. + Set `headers` to `false` if the file you are reading doesn't have any headers. This will cause ftcsv to create indexed tables rather than a key-value tables for the output. - ```lua - local options = {loadFromString=true, headers=false, delimiter=">"} - local actual = ftcsv.parse("apple>banana>carrot\ndiamond>emerald>pearl", options) - ``` + ```lua + local options = {loadFromString=true, headers=false, delimiter=">"} + local actual = ftcsv.parse("apple>banana>carrot\ndiamond>emerald>pearl", options) + ``` - Note: Header-less files can still use the `rename` option and after a field has been renamed, it can specified as a field to keep. The `rename` syntax changes a little bit: + Note: Header-less files can still use the `rename` option and after a field has been renamed, it can specified as a field to keep. The `rename` syntax changes a little bit: - ```lua - local options = {loadFromString=true, headers=false, rename={"a","b","c"}, fieldsToKeep={"a","b"}, delimiter=">"} - local actual = ftcsv.parse("apple>banana>carrot\ndiamond>emerald>pearl", options) - ``` + ```lua + local options = {loadFromString=true, headers=false, rename={"a","b","c"}, fieldsToKeep={"a","b"}, delimiter=">"} + local actual = ftcsv.parse("apple>banana>carrot\ndiamond>emerald>pearl", options) + ``` - In the above example, the first field becomes 'a', the second field becomes 'b' and so on. + In the above example, the first field becomes 'a', the second field becomes 'b' and so on. For all tested examples, take a look in /spec/feature_spec.lua @@ -147,29 +147,39 @@ file:close() ``` ### Options - - `delimiter` +- `delimiter` - by default the encoder uses a `,` as a delimiter. The delimiter can be changed by setting a value for `delimiter` + by default the encoder uses a `,` as a delimiter. The delimiter can be changed by setting a value for `delimiter` - ```lua - local output = ftcsv.encode(everyUser, {delimiter="\t"}) - ``` + ```lua + local output = ftcsv.encode(everyUser, {delimiter="\t"}) + ``` + +- `fieldsToKeep` + + if `fieldsToKeep` is set in the encode process, only the fields specified will be written out to a file. The `fieldsToKeep` will be written out in the order that is specified. + + ```lua + local output = ftcsv.encode(everyUser, {fieldsToKeep={"Name", "Phone", "City"}}) + ``` + +- `onlyRequiredQuotes` - - `fieldsToKeep` + if `onlyRequiredQuotes` is set to `true`, the output will only include quotes around fields that are quotes, have newlines, or contain the delimter. - if `fieldsToKeep` is set in the encode process, only the fields specified will be written out to a file. The `fieldsToKeep` will be written out in the order that is specified. + ```lua + local output = ftcsv.encode(everyUser, {onlyRequiredQuotes=true}) + ``` - ```lua - local output = ftcsv.encode(everyUser, {fieldsToKeep={"Name", "Phone", "City"}}) - ``` +- `encodeNilAs` - - `onlyRequiredQuotes` + by default a `nil` value in a table will be encoded as the string `"nil"`. The value a `nil` value in the a table can be set with `encodeNilAs`. - if `onlyRequiredQuotes` is set to `true`, the output will only include quotes around fields that are quotes, have newlines, or contain the delimter. + ```lua + local output = ftcsv.encode(everyUser, {encodeNilAs=""}) -- for setting nil to the empty string + local output = ftcsv.encode(everyUser, {encodeNilAs=0}) -- for setting it to 0 + ``` - ```lua - local output = ftcsv.encode(everyUser, {onlyRequiredQuotes=true}) - ``` ## Error Handling diff --git a/ftcsv-1.4.0-1.rockspec b/ftcsv-1.5.0-1.rockspec similarity index 95% rename from ftcsv-1.4.0-1.rockspec rename to ftcsv-1.5.0-1.rockspec index a6cbf7c..2bb553e 100644 --- a/ftcsv-1.4.0-1.rockspec +++ b/ftcsv-1.5.0-1.rockspec @@ -1,9 +1,9 @@ package = "ftcsv" -version = "1.4.0-1" +version = "1.5.0-1" source = { url = "git://github.com/FourierTransformer/ftcsv.git", - tag = "1.4.0" + tag = "1.5.0" } description = { diff --git a/ftcsv.lua b/ftcsv.lua index e0e8db0..400bef4 100644 --- a/ftcsv.lua +++ b/ftcsv.lua @@ -1,11 +1,11 @@ local ftcsv = { - _VERSION = 'ftcsv 1.4.0', + _VERSION = 'ftcsv 1.5.0', _DESCRIPTION = 'CSV library for Lua', _URL = 'https://github.com/FourierTransformer/ftcsv', _LICENSE = [[ The MIT License (MIT) - Copyright (c) 2016-2023 Shakil Thakur + Copyright (c) 2016-2025 Fourier Transformer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -658,19 +658,33 @@ end -- The ENCODER code is below here -- This could be broken out, but is kept here for portability +local function generateCustomToString(valueToConvertNilTo) + local newReturnValue = tostring(valueToConvertNilTo) + local generatedFunction = function(field) + if type(field) == "nil" then + return newReturnValue + else + return tostring(field) + end + end + return generatedFunction +end -local function delimitField(field) - field = tostring(field) - if field:find('"') then - return field:gsub('"', '""') - else - return field +local function generateDelimitField(customToString) + local delimitField = function(field) + field = customToString(field) + if field:find('"') then + return field:gsub('"', '""') + else + return field + end end + return delimitField end -local function generateDelimitAndQuoteField(delimiter) +local function generateDelimitAndQuoteField(delimiter, customToString) local generatedFunction = function(field) - field = tostring(field) + field = customToString(field) if field:find('"') then return '"' .. field:gsub('"', '""') .. '"' elseif field:find('[\n' .. delimiter .. ']') then @@ -722,10 +736,15 @@ local function csvLineGenerator(inputTable, delimiter, headers, options) arguments.t = inputTable -- we want to use the same delimitField throughout, -- so we're just going to pass it in + + local toStringToUse = tostring + if options and options.encodeNilAs ~= nil then + toStringToUse = generateCustomToString(options.encodeNilAs) + end if options and options.onlyRequiredQuotes == true then - arguments.delimitField = generateDelimitAndQuoteField(delimiter) + arguments.delimitField = generateDelimitAndQuoteField(delimiter, toStringToUse) else - arguments.delimitField = delimitField + arguments.delimitField = generateDelimitField(toStringToUse) end return luaCompatibility.load(outputFunc), arguments, 0 @@ -752,9 +771,9 @@ end local function escapeHeadersForOutput(headers, delimiter, options) local escapedHeaders = {} - local delimitField = delimitField + local delimitField = generateDelimitField(tostring) if options and options.onlyRequiredQuotes == true then - delimitField = generateDelimitAndQuoteField(delimiter) + delimitField = generateDelimitAndQuoteField(delimiter, tostring) end for i = 1, #headers do escapedHeaders[i] = delimitField(headers[i]) diff --git a/spec/feature_spec.lua b/spec/feature_spec.lua index 70fb71d..c5e6d92 100644 --- a/spec/feature_spec.lua +++ b/spec/feature_spec.lua @@ -444,7 +444,7 @@ describe("csv features", function() it("should handle encoding files (str test)", function() local expected = '"a","b","c","d"\r\n"1","","foo","""quoted"""\r\n' - output = ftcsv.encode({ + local output = ftcsv.encode({ { a = 1, b = '', c = 'foo', d = '"quoted"' }; }, ',') assert.are.same(expected, output) @@ -452,7 +452,7 @@ describe("csv features", function() it("should handle encoding files (str test) with other delimiter", function() local expected = '"a">"b">"c">"d"\r\n"1">"">"foo">"""quoted"""\r\n' - output = ftcsv.encode({ + local output = ftcsv.encode({ { a = 1, b = '', c = 'foo', d = '"quoted"' }; }, '>') assert.are.same(expected, output) @@ -460,7 +460,7 @@ describe("csv features", function() it("should handle encoding files without quotes (str test)", function() local expected = 'a,b,c,d\r\n1,,"fo,o","""quoted"""\r\n' - output = ftcsv.encode({ + local output = ftcsv.encode({ { a = 1, b = '', c = 'fo,o', d = '"quoted"' }; }, ',', {onlyRequiredQuotes=true}) assert.are.same(expected, output) @@ -468,7 +468,7 @@ describe("csv features", function() it("should handle encoding files without quotes with other delimiter (str test)", function() local expected = 'a>b>c>d\r\n1>>fo,o>"""quoted"""\r\n' - output = ftcsv.encode({ + local output = ftcsv.encode({ { a = 1, b = '', c = 'fo,o', d = '"quoted"' }; }, '>', {onlyRequiredQuotes=true}) assert.are.same(expected, output) @@ -476,12 +476,82 @@ describe("csv features", function() it("should handle encoding files without quotes with certain fields to keep (str test)", function() local expected = "b,c\r\n,foo\r\n" - output = ftcsv.encode({ + local output = ftcsv.encode({ { a = 1, b = '', c = 'foo', d = '"quoted"' }; }, ',', {onlyRequiredQuotes=true, fieldsToKeep={"b", "c"}}) assert.are.same(expected, output) end) + it("should handle encoding files without nil conversion", function() + local expected = '"f1","f2","f3"\r\n"a","b","c"\r\n"d","e","nil"\r\n"nil","nil","f"\r\n' + local output = ftcsv.encode({ + {f1 = "a", f2 = "b", f3 = "c"}, + {f1 = "d", f2 = "e",}, + {f3 = "f"}, + }) + assert.are.same(expected, output) + end) + + it("should handle encoding files with nil conversion to empty string", function() + local expected = '"f1","f2","f3"\r\n"a","b","c"\r\n"d","e",""\r\n"","","f"\r\n' + local output = ftcsv.encode({ + {f1 = "a", f2 = "b", f3 = "c"}, + {f1 = "d", f2 = "e",}, + {f3 = "f"}, + }, {encodeNilAs=""}) + assert.are.same(expected, output) + end) + + it("should handle encoding files with nil conversion to number", function() + local expected = '"f1","f2","f3"\r\n"a","b","c"\r\n"d","e","0"\r\n"0","0","f"\r\n' + local output = ftcsv.encode({ + {f1 = "a", f2 = "b", f3 = "c"}, + {f1 = "d", f2 = "e",}, + {f3 = "f"}, + }, {encodeNilAs=0}) + assert.are.same(expected, output) + end) + + it("should handle encoding files with nil conversion to number while only quoting required field", function() + local expected = 'f1,f2,f3\r\na,b,c\r\nd,e,0\r\n0,0,f\r\n' + local output = ftcsv.encode({ + {f1 = "a", f2 = "b", f3 = "c"}, + {f1 = "d", f2 = "e",}, + {f3 = "f"}, + }, {encodeNilAs=0, onlyRequiredQuotes=true}) + assert.are.same(expected, output) + end) + + it("should handle encoding files with nil conversion to non-specified delimiter while only quoting required field", function() + local expected = 'f1,f2,f3\r\na,b,c\r\nd,e,","\r\n",",",",f\r\n' + local output = ftcsv.encode({ + {f1 = "a", f2 = "b", f3 = "c"}, + {f1 = "d", f2 = "e",}, + {f3 = "f"}, + }, {encodeNilAs=",", onlyRequiredQuotes=true}) + assert.are.same(expected, output) + end) + + it("should handle encoding files with nil conversion to specified delimiter while only quoting required field", function() + local expected = 'f1|f2|f3\r\na|b|c\r\nd|e|,\r\n,|,|f\r\n' + local output = ftcsv.encode({ + {f1 = "a", f2 = "b", f3 = "c"}, + {f1 = "d", f2 = "e",}, + {f3 = "f"}, + }, {encodeNilAs=",", onlyRequiredQuotes=true, delimiter="|"}) + assert.are.same(expected, output) + end) + + it("should handle encoding files to delimiter with nil conversion to specified delimiter while only quoting required field", function() + local expected = 'f1|f2|f3\r\na|b|c\r\nd|e|"|"\r\n"|"|"|"|f\r\n' + local output = ftcsv.encode({ + {f1 = "a", f2 = "b", f3 = "c"}, + {f1 = "d", f2 = "e",}, + {f3 = "f"}, + }, {encodeNilAs="|", onlyRequiredQuotes=true, delimiter="|"}) + assert.are.same(expected, output) + end) + it("should handle headers attempting to escape", function() local expected = {} expected[1] = {}