Issue #3 Changing CSV to NimbleCSV

lib/ash_csv/data_layer.ex

@@ -209,10 +209,7 @@ defmodule AshCsv.DataLayer do
             else
               case dump_row(resource, changeset) do
                 {:ok, row} ->
-                  lines =
-                    [row]
-                    |> CSV.encode(separator: separator(resource))
-                    |> Enum.to_list()
+                  iodata = csv_module(resource).dump_to_iodata([row])
 
                   result =
                     if File.exists?(file(resource)) do
@@ -232,7 +229,7 @@ defmodule AshCsv.DataLayer do
                       {:halt, {:error, error}}
 
                     :ok ->
-                      case write_result(resource, lines) do
+                      case write_result(resource, iodata) do
                         :ok ->
                           new_results =
                             if options.return_records? do
@@ -276,10 +273,10 @@ defmodule AshCsv.DataLayer do
   end
 
   # sobelow_skip ["Traversal"]
-  defp write_result(resource, lines, retry? \\ false) do
+  defp write_result(resource, iodata, retry? \\ false) do
     resource
     |> file()
-    |> File.write(lines, [:append])
+    |> File.write(iodata, [:append])
     |> case do
       :ok ->
         :ok
@@ -289,7 +286,7 @@ defmodule AshCsv.DataLayer do
 
       {:error, :enoent} ->
         if create?(resource) do
-          write_result(resource, lines, true)
+          write_result(resource, iodata, true)
         else
           {:error, "Error while writing to CSV: #{inspect(:enoent)}"}
         end
@@ -419,21 +416,18 @@ defmodule AshCsv.DataLayer do
     end)
     |> case do
       {:ok, rows} ->
-        lines =
-          rows
-          |> CSV.encode(separator: separator(resource))
-          |> Enum.to_list()
+        iodata = csv_module(resource).dump_to_iodata(rows)
 
-        lines =
+        iodata =
           if header?(resource) do
-            [header(resource) | lines]
+            [header(resource), iodata]
           else
-            lines
+            iodata
           end
 
         resource
         |> file()
-        |> File.write(lines, [:write])
+        |> File.write(iodata, [:write])
         |> case do
           :ok ->
             :ok
@@ -479,10 +473,7 @@ defmodule AshCsv.DataLayer do
     end)
     |> case do
       {:ok, rows} ->
-        lines =
-          rows
-          |> CSV.encode(separator: separator(resource))
-          |> Enum.to_list()
+        iodata = csv_module(resource).dump_to_iodata(rows)
 
         if File.exists?(file(resource)) do
           :ok
@@ -496,16 +487,16 @@ defmodule AshCsv.DataLayer do
           end
         end
 
-        lines =
+        iodata =
           if header?(resource) do
-            [header(resource) | lines]
+            [header(resource), iodata]
           else
-            lines
+            iodata
           end
 
         resource
         |> file()
-        |> File.write(lines, [:write])
+        |> File.write(iodata, [:write])
         |> case do
           :ok ->
             {:ok, struct(changeset.data, changeset.attributes)}
@@ -563,16 +554,12 @@ defmodule AshCsv.DataLayer do
           file
           |> File.stream!()
           |> Stream.drop(amount_to_drop)
-          |> CSV.decode(separator: separator(resource))
-          |> Stream.map(fn
-            {:error, error} ->
-              throw({:error, error})
-
-            {:ok, row} ->
-              case cast_stored(resource, row) do
-                {:ok, casted} -> casted
-                {:error, error} -> throw({:error, error})
-              end
+          |> csv_module(resource).parse_stream(skip_headers: false)
+          |> Stream.map(fn row ->
+            case cast_stored(resource, row) do
+              {:ok, casted} -> casted
+              {:error, error} -> throw({:error, error})
+            end
           end)
           |> filter_stream(domain, filter)
           |> sort_stream(resource, domain, sort)
@@ -583,20 +570,16 @@ defmodule AshCsv.DataLayer do
           file
           |> File.stream!()
           |> Stream.drop(amount_to_drop)
-          |> CSV.decode(separator: separator(resource))
-          |> Stream.map(fn
-            {:error, error} ->
-              throw({:error, error})
-
-            {:ok, row} ->
-              row
-          end)
+          |> csv_module(resource).parse_stream(skip_headers: false)
           |> Enum.to_list()
         end
       end)
 
     {:ok, results}
   rescue
+    e in NimbleCSV.ParseError ->
+      {:error, Exception.message(e)}
+
     e in File.Error ->
       if e.reason == :enoent && !retry? do
         file = file(resource)
@@ -652,10 +635,7 @@ defmodule AshCsv.DataLayer do
 
       case row do
         {:ok, row} ->
-          lines =
-            [Enum.reverse(row)]
-            |> CSV.encode(separator: separator(resource))
-            |> Enum.to_list()
+          iodata = csv_module(resource).dump_to_iodata([Enum.reverse(row)])
 
           result =
             if File.exists?(file(resource)) do
@@ -677,7 +657,7 @@ defmodule AshCsv.DataLayer do
             :ok ->
               resource
               |> file()
-              |> File.write(lines, [:append])
+              |> File.write(iodata, [:append])
               |> case do
                 :ok ->
                   {:ok, struct(resource, changeset.attributes)}
@@ -719,4 +699,8 @@ defmodule AshCsv.DataLayer do
       ""
     end
   end
+
+  defp csv_module(resource) do
+    AshCsv.DataLayer.Info.csv_module(resource)
+  end
 end

lib/ash_csv/data_layer/transformers/build_parser.ex

@@ -8,6 +8,16 @@
 
   def transform(dsl) do
     columns = AshCsv.DataLayer.Info.columns(dsl)
+    separator = AshCsv.DataLayer.Info.separator(dsl) || ?,
+
+    separator_string =
+      try do
+        <<separator::utf8>>
+      rescue
+        _ ->
+          raise ArgumentError,
+            "Invalid separator value: #{inspect(separator)}. Expected a valid UTF-8 character."
+      end
 
     func_args =
       Enum.map(columns, fn name ->
@@ -83,10 +93,13 @@
 
     map = {:%{}, [], Enum.map(columns, fn column -> {column, {column, [], Elixir}} end)}
 
+    resource_module = Spark.Dsl.Transformer.get_persisted(dsl, :module)
+    csv_module = AshCsv.DataLayer.Info.csv_module(resource_module)
+
     struct =
       {:struct, [],
        [
-         Spark.Dsl.Transformer.get_persisted(dsl, :module),
+         resource_module,
          map
        ]}
 
@@ -95,7 +108,13 @@
        dsl,
        [],
        quote do
-         def ash_csv_dump_row(unquote(map)) do
+        # Define the NimbleCSV parser
+        NimbleCSV.define(unquote(csv_module),
+          separator: unquote(separator_string),
+          line_separator: "\n"
+        )
+
+        def ash_csv_dump_row(unquote(map)) do
            {:ok, unquote(dump_fields)}
          catch
            {:error, error} ->

lib/ash_csv/info.ex

@@ -28,4 +28,8 @@ defmodule AshCsv.DataLayer.Info do
   def create?(resource) do
     Extension.get_opt(resource, [:csv], :create?, nil, true)
   end
+
+  def csv_module(resource) do
+    Module.concat([resource, AshCsvNimbleCSV])
+  end
 end

mix.exs

@@ -112,7 +112,7 @@ defmodule AshCsv.MixProject do
   defp deps do
     [
       {:ash, ash_version("~> 3.0")},
-      {:csv, "~> 3.0"},
+      {:nimble_csv, "~> 1.2"},
       {:igniter, "~> 0.5", only: [:dev, :test]},
       {:ex_doc, "~> 0.37-rc", only: [:dev, :test], runtime: false},
       {:ex_check, "~> 0.12", only: [:dev, :test]},

mix.lock

@@ -28,6 +28,7 @@
   "makeup_erlang": {:hex, :makeup_erlang, "1.0.3", "4252d5d4098da7415c390e847c814bad3764c94a814a0b4245176215615e1035", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "953297c02582a33411ac6208f2c6e55f0e870df7f80da724ed613f10e6706afd"},
   "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"},
   "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"},
+  "nimble_csv": {:hex, :nimble_csv, "1.3.0", "b7f998dc62b222bce9596e46f028c7a5af04cb5dde6df2ea197c583227c54971", [:mix], [], "hexpm", "41ccdc18f7c8f8bb06e84164fc51635321e80d5a3b450761c4997d620925d619"},
   "mix_audit": {:hex, :mix_audit, "2.1.5", "c0f77cee6b4ef9d97e37772359a187a166c7a1e0e08b50edf5bf6959dfe5a016", [:make, :mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:yaml_elixir, "~> 2.11", [hex: :yaml_elixir, repo: "hexpm", optional: false]}], "hexpm", "87f9298e21da32f697af535475860dc1d3617a010e0b418d2ec6142bc8b42d69"},
   "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"},
   "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"},