diff --git a/lib/ash_csv/data_layer.ex b/lib/ash_csv/data_layer.ex index 91d254d..392ad53 100644 --- a/lib/ash_csv/data_layer.ex +++ b/lib/ash_csv/data_layer.ex @@ -209,10 +209,7 @@ defmodule AshCsv.DataLayer do else case dump_row(resource, changeset) do {:ok, row} -> - lines = - [row] - |> CSV.encode(separator: separator(resource)) - |> Enum.to_list() + iodata = csv_module(resource).dump_to_iodata([row]) result = if File.exists?(file(resource)) do @@ -232,7 +229,7 @@ defmodule AshCsv.DataLayer do {:halt, {:error, error}} :ok -> - case write_result(resource, lines) do + case write_result(resource, iodata) do :ok -> new_results = if options.return_records? do @@ -276,10 +273,10 @@ defmodule AshCsv.DataLayer do end # sobelow_skip ["Traversal"] - defp write_result(resource, lines, retry? \\ false) do + defp write_result(resource, iodata, retry? \\ false) do resource |> file() - |> File.write(lines, [:append]) + |> File.write(iodata, [:append]) |> case do :ok -> :ok @@ -289,7 +286,7 @@ defmodule AshCsv.DataLayer do {:error, :enoent} -> if create?(resource) do - write_result(resource, lines, true) + write_result(resource, iodata, true) else {:error, "Error while writing to CSV: #{inspect(:enoent)}"} end @@ -419,21 +416,18 @@ defmodule AshCsv.DataLayer do end) |> case do {:ok, rows} -> - lines = - rows - |> CSV.encode(separator: separator(resource)) - |> Enum.to_list() + iodata = csv_module(resource).dump_to_iodata(rows) - lines = + iodata = if header?(resource) do - [header(resource) | lines] + [header(resource), iodata] else - lines + iodata end resource |> file() - |> File.write(lines, [:write]) + |> File.write(iodata, [:write]) |> case do :ok -> :ok @@ -479,10 +473,7 @@ defmodule AshCsv.DataLayer do end) |> case do {:ok, rows} -> - lines = - rows - |> CSV.encode(separator: separator(resource)) - |> Enum.to_list() + iodata = csv_module(resource).dump_to_iodata(rows) if File.exists?(file(resource)) do :ok @@ -496,16 +487,16 @@ defmodule AshCsv.DataLayer do end end - lines = + iodata = if header?(resource) do - [header(resource) | lines] + [header(resource), iodata] else - lines + iodata end resource |> file() - |> File.write(lines, [:write]) + |> File.write(iodata, [:write]) |> case do :ok -> {:ok, struct(changeset.data, changeset.attributes)} @@ -563,16 +554,12 @@ defmodule AshCsv.DataLayer do file |> File.stream!() |> Stream.drop(amount_to_drop) - |> CSV.decode(separator: separator(resource)) - |> Stream.map(fn - {:error, error} -> - throw({:error, error}) - - {:ok, row} -> - case cast_stored(resource, row) do - {:ok, casted} -> casted - {:error, error} -> throw({:error, error}) - end + |> csv_module(resource).parse_stream(skip_headers: false) + |> Stream.map(fn row -> + case cast_stored(resource, row) do + {:ok, casted} -> casted + {:error, error} -> throw({:error, error}) + end end) |> filter_stream(domain, filter) |> sort_stream(resource, domain, sort) @@ -583,20 +570,16 @@ defmodule AshCsv.DataLayer do file |> File.stream!() |> Stream.drop(amount_to_drop) - |> CSV.decode(separator: separator(resource)) - |> Stream.map(fn - {:error, error} -> - throw({:error, error}) - - {:ok, row} -> - row - end) + |> csv_module(resource).parse_stream(skip_headers: false) |> Enum.to_list() end end) {:ok, results} rescue + e in NimbleCSV.ParseError -> + {:error, Exception.message(e)} + e in File.Error -> if e.reason == :enoent && !retry? do file = file(resource) @@ -652,10 +635,7 @@ defmodule AshCsv.DataLayer do case row do {:ok, row} -> - lines = - [Enum.reverse(row)] - |> CSV.encode(separator: separator(resource)) - |> Enum.to_list() + iodata = csv_module(resource).dump_to_iodata([Enum.reverse(row)]) result = if File.exists?(file(resource)) do @@ -677,7 +657,7 @@ defmodule AshCsv.DataLayer do :ok -> resource |> file() - |> File.write(lines, [:append]) + |> File.write(iodata, [:append]) |> case do :ok -> {:ok, struct(resource, changeset.attributes)} @@ -719,4 +699,8 @@ defmodule AshCsv.DataLayer do "" end end + + defp csv_module(resource) do + AshCsv.DataLayer.Info.csv_module(resource) + end end diff --git a/lib/ash_csv/data_layer/transformers/build_parser.ex b/lib/ash_csv/data_layer/transformers/build_parser.ex index 14bcbdc..8f512ce 100644 --- a/lib/ash_csv/data_layer/transformers/build_parser.ex +++ b/lib/ash_csv/data_layer/transformers/build_parser.ex @@ -8,6 +8,16 @@ defmodule AshCsv.DataLayer.Transformers.BuildParser do def transform(dsl) do columns = AshCsv.DataLayer.Info.columns(dsl) + separator = AshCsv.DataLayer.Info.separator(dsl) || ?, + + separator_string = + try do + <> + rescue + _ -> + raise ArgumentError, + "Invalid separator value: #{inspect(separator)}. Expected a valid UTF-8 character." + end func_args = Enum.map(columns, fn name -> @@ -83,10 +93,13 @@ defmodule AshCsv.DataLayer.Transformers.BuildParser do map = {:%{}, [], Enum.map(columns, fn column -> {column, {column, [], Elixir}} end)} + resource_module = Spark.Dsl.Transformer.get_persisted(dsl, :module) + csv_module = AshCsv.DataLayer.Info.csv_module(resource_module) + struct = {:struct, [], [ - Spark.Dsl.Transformer.get_persisted(dsl, :module), + resource_module, map ]} @@ -95,7 +108,13 @@ defmodule AshCsv.DataLayer.Transformers.BuildParser do dsl, [], quote do - def ash_csv_dump_row(unquote(map)) do + # Define the NimbleCSV parser + NimbleCSV.define(unquote(csv_module), + separator: unquote(separator_string), + line_separator: "\n" + ) + + def ash_csv_dump_row(unquote(map)) do {:ok, unquote(dump_fields)} catch {:error, error} -> diff --git a/lib/ash_csv/info.ex b/lib/ash_csv/info.ex index 476c1cf..6bbf3ce 100644 --- a/lib/ash_csv/info.ex +++ b/lib/ash_csv/info.ex @@ -28,4 +28,8 @@ defmodule AshCsv.DataLayer.Info do def create?(resource) do Extension.get_opt(resource, [:csv], :create?, nil, true) end + + def csv_module(resource) do + Module.concat([resource, AshCsvNimbleCSV]) + end end diff --git a/mix.exs b/mix.exs index 5c3fcf1..84dfd1a 100644 --- a/mix.exs +++ b/mix.exs @@ -112,7 +112,7 @@ defmodule AshCsv.MixProject do defp deps do [ {:ash, ash_version("~> 3.0")}, - {:csv, "~> 3.0"}, + {:nimble_csv, "~> 1.2"}, {:igniter, "~> 0.5", only: [:dev, :test]}, {:ex_doc, "~> 0.37-rc", only: [:dev, :test], runtime: false}, {:ex_check, "~> 0.12", only: [:dev, :test]}, diff --git a/mix.lock b/mix.lock index b7abe85..2dbd214 100644 --- a/mix.lock +++ b/mix.lock @@ -28,6 +28,7 @@ "makeup_erlang": {:hex, :makeup_erlang, "1.0.3", "4252d5d4098da7415c390e847c814bad3764c94a814a0b4245176215615e1035", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "953297c02582a33411ac6208f2c6e55f0e870df7f80da724ed613f10e6706afd"}, "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"}, + "nimble_csv": {:hex, :nimble_csv, "1.3.0", "b7f998dc62b222bce9596e46f028c7a5af04cb5dde6df2ea197c583227c54971", [:mix], [], "hexpm", "41ccdc18f7c8f8bb06e84164fc51635321e80d5a3b450761c4997d620925d619"}, "mix_audit": {:hex, :mix_audit, "2.1.5", "c0f77cee6b4ef9d97e37772359a187a166c7a1e0e08b50edf5bf6959dfe5a016", [:make, :mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:yaml_elixir, "~> 2.11", [hex: :yaml_elixir, repo: "hexpm", optional: false]}], "hexpm", "87f9298e21da32f697af535475860dc1d3617a010e0b418d2ec6142bc8b42d69"}, "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"},