From 98ee610c29318fd0f028ad8ece1ab561f87c1e5b Mon Sep 17 00:00:00 2001 From: Jacob Atanasio Date: Mon, 2 Feb 2026 13:56:57 -0700 Subject: [PATCH 1/3] Changes to issue #3 --- lib/ash_csv/data_layer.ex | 86 +++++++++---------- .../data_layer/transformers/build_parser.ex | 33 ++++++- mix.exs | 1 + mix.lock | 1 + 4 files changed, 73 insertions(+), 48 deletions(-) diff --git a/lib/ash_csv/data_layer.ex b/lib/ash_csv/data_layer.ex index 91d254d..e804c65 100644 --- a/lib/ash_csv/data_layer.ex +++ b/lib/ash_csv/data_layer.ex @@ -209,10 +209,8 @@ defmodule AshCsv.DataLayer do else case dump_row(resource, changeset) do {:ok, row} -> - lines = - [row] - |> CSV.encode(separator: separator(resource)) - |> Enum.to_list() + # NimbleCSV: encode one row to iodata (replaces CSV.encode(separator: separator(resource)) |> Enum.to_list()) + iodata = csv_module(resource).dump_to_iodata([row]) result = if File.exists?(file(resource)) do @@ -232,7 +230,7 @@ defmodule AshCsv.DataLayer do {:halt, {:error, error}} :ok -> - case write_result(resource, lines) do + case write_result(resource, iodata) do :ok -> new_results = if options.return_records? do @@ -276,10 +274,11 @@ defmodule AshCsv.DataLayer do end # sobelow_skip ["Traversal"] - defp write_result(resource, lines, retry? \\ false) do + # NimbleCSV: param is iodata from dump_to_iodata (was "lines" from CSV.encode |> Enum.to_list()) + defp write_result(resource, iodata, retry? \\ false) do resource |> file() - |> File.write(lines, [:append]) + |> File.write(iodata, [:append]) |> case do :ok -> :ok @@ -289,7 +288,7 @@ defmodule AshCsv.DataLayer do {:error, :enoent} -> if create?(resource) do - write_result(resource, lines, true) + write_result(resource, iodata, true) else {:error, "Error while writing to CSV: #{inspect(:enoent)}"} end @@ -419,21 +418,19 @@ defmodule AshCsv.DataLayer do end) |> case do {:ok, rows} -> - lines = - rows - |> CSV.encode(separator: separator(resource)) - |> Enum.to_list() + # NimbleCSV: encode remaining rows to iodata (replaces CSV.encode(separator: ...) |> Enum.to_list()) + iodata = csv_module(resource).dump_to_iodata(rows) - lines = + iodata = if header?(resource) do - [header(resource) | lines] + [header(resource), iodata] else - lines + iodata end resource |> file() - |> File.write(lines, [:write]) + |> File.write(iodata, [:write]) |> case do :ok -> :ok @@ -479,10 +476,8 @@ defmodule AshCsv.DataLayer do end) |> case do {:ok, rows} -> - lines = - rows - |> CSV.encode(separator: separator(resource)) - |> Enum.to_list() + # NimbleCSV: encode all rows to iodata (replaces CSV.encode(separator: ...) |> Enum.to_list()) + iodata = csv_module(resource).dump_to_iodata(rows) if File.exists?(file(resource)) do :ok @@ -496,16 +491,16 @@ defmodule AshCsv.DataLayer do end end - lines = + iodata = if header?(resource) do - [header(resource) | lines] + [header(resource), iodata] else - lines + iodata end resource |> file() - |> File.write(lines, [:write]) + |> File.write(iodata, [:write]) |> case do :ok -> {:ok, struct(changeset.data, changeset.attributes)} @@ -560,19 +555,16 @@ defmodule AshCsv.DataLayer do |> file() |> then(fn file -> if decode? do + # NimbleCSV: decode stream of lines to stream of rows (replaces CSV.decode(separator: ...)); parse_stream yields rows directly (no {:ok, row} wrapper) file |> File.stream!() |> Stream.drop(amount_to_drop) - |> CSV.decode(separator: separator(resource)) - |> Stream.map(fn - {:error, error} -> - throw({:error, error}) - - {:ok, row} -> - case cast_stored(resource, row) do - {:ok, casted} -> casted - {:error, error} -> throw({:error, error}) - end + |> csv_module(resource).parse_stream(skip_headers: false) + |> Stream.map(fn row -> + case cast_stored(resource, row) do + {:ok, casted} -> casted + {:error, error} -> throw({:error, error}) + end end) |> filter_stream(domain, filter) |> sort_stream(resource, domain, sort) @@ -583,20 +575,17 @@ defmodule AshCsv.DataLayer do file |> File.stream!() |> Stream.drop(amount_to_drop) - |> CSV.decode(separator: separator(resource)) - |> Stream.map(fn - {:error, error} -> - throw({:error, error}) - - {:ok, row} -> - row - end) + |> csv_module(resource).parse_stream(skip_headers: false) |> Enum.to_list() end end) {:ok, results} rescue + # NimbleCSV: parse_stream raises on malformed CSV; convert to {:error, message} for caller + e in NimbleCSV.ParseError -> + {:error, Exception.message(e)} + e in File.Error -> if e.reason == :enoent && !retry? do file = file(resource) @@ -652,10 +641,8 @@ defmodule AshCsv.DataLayer do case row do {:ok, row} -> - lines = - [Enum.reverse(row)] - |> CSV.encode(separator: separator(resource)) - |> Enum.to_list() + # NimbleCSV: encode one row to iodata (replaces CSV.encode(separator: ...) |> Enum.to_list()) + iodata = csv_module(resource).dump_to_iodata([Enum.reverse(row)]) result = if File.exists?(file(resource)) do @@ -677,7 +664,7 @@ defmodule AshCsv.DataLayer do :ok -> resource |> file() - |> File.write(lines, [:append]) + |> File.write(iodata, [:append]) |> case do :ok -> {:ok, struct(resource, changeset.attributes)} @@ -719,4 +706,9 @@ defmodule AshCsv.DataLayer do "" end end + + # NimbleCSV: returns per-resource parser module (from BuildParser); used for dump_to_iodata/1 and parse_stream/1 + defp csv_module(resource) do + resource.ash_csv_csv_module() + end end diff --git a/lib/ash_csv/data_layer/transformers/build_parser.ex b/lib/ash_csv/data_layer/transformers/build_parser.ex index 14bcbdc..654cf72 100644 --- a/lib/ash_csv/data_layer/transformers/build_parser.ex +++ b/lib/ash_csv/data_layer/transformers/build_parser.ex @@ -9,6 +9,17 @@ defmodule AshCsv.DataLayer.Transformers.BuildParser do def transform(dsl) do columns = AshCsv.DataLayer.Info.columns(dsl) + # NimbleCSV: read separator from DSL and convert to string for NimbleCSV.define (expects string, not codepoint) + separator = AshCsv.DataLayer.Info.separator(dsl) || ?, + + separator_string = + try do + <> + rescue + _ -> + "," + end + func_args = Enum.map(columns, fn name -> {name, [], Elixir} @@ -83,10 +94,19 @@ defmodule AshCsv.DataLayer.Transformers.BuildParser do map = {:%{}, [], Enum.map(columns, fn column -> {column, {column, [], Elixir}} end)} + # NimbleCSV: build per-resource parser module name (e.g. MyApp.Post.AshCsvNimbleCSV) for NimbleCSV.define + resource_module = Spark.Dsl.Transformer.get_persisted(dsl, :module) + + csv_module = + Module.concat([ + resource_module, + AshCsvNimbleCSV + ]) + struct = {:struct, [], [ - Spark.Dsl.Transformer.get_persisted(dsl, :module), + resource_module, map ]} @@ -95,6 +115,17 @@ defmodule AshCsv.DataLayer.Transformers.BuildParser do dsl, [], quote do + # NimbleCSV: define per-resource parser at compile time (replaces runtime CSV.encode/decode with separator option) + NimbleCSV.define(unquote(csv_module), + separator: unquote(separator_string), + line_separator: "\n" + ) + + # NimbleCSV: expose parser module so data layer can call dump_to_iodata/1 and parse_stream/1 + def ash_csv_csv_module do + unquote(csv_module) + end + def ash_csv_dump_row(unquote(map)) do {:ok, unquote(dump_fields)} catch diff --git a/mix.exs b/mix.exs index 5c3fcf1..be8e0f5 100644 --- a/mix.exs +++ b/mix.exs @@ -113,6 +113,7 @@ defmodule AshCsv.MixProject do [ {:ash, ash_version("~> 3.0")}, {:csv, "~> 3.0"}, + {:nimble_csv, "~> 1.2"}, {:igniter, "~> 0.5", only: [:dev, :test]}, {:ex_doc, "~> 0.37-rc", only: [:dev, :test], runtime: false}, {:ex_check, "~> 0.12", only: [:dev, :test]}, diff --git a/mix.lock b/mix.lock index 59e1574..b4abfa6 100644 --- a/mix.lock +++ b/mix.lock @@ -28,6 +28,7 @@ "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"}, "mix_audit": {:hex, :mix_audit, "2.1.3", "c70983d5cab5dca923f9a6efe559abfb4ec3f8e87762f02bab00fa4106d17eda", [:make, :mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:yaml_elixir, "~> 2.9", [hex: :yaml_elixir, repo: "hexpm", optional: false]}], "hexpm", "8c3987100b23099aea2f2df0af4d296701efd031affb08d0746b2be9e35988ec"}, + "nimble_csv": {:hex, :nimble_csv, "1.3.0", "b7f998dc62b222bce9596e46f028c7a5af04cb5dde6df2ea197c583227c54971", [:mix], [], "hexpm", "41ccdc18f7c8f8bb06e84164fc51635321e80d5a3b450761c4997d620925d619"}, "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, From 3eec648b3b41b18471b5048707b813ae72ff5785 Mon Sep 17 00:00:00 2001 From: Jacob Atanasio Date: Mon, 2 Feb 2026 14:15:15 -0700 Subject: [PATCH 2/3] Comments for issue #3 --- lib/ash_csv/data_layer.ex | 10 ++-------- lib/ash_csv/data_layer/transformers/build_parser.ex | 6 +----- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/lib/ash_csv/data_layer.ex b/lib/ash_csv/data_layer.ex index e804c65..54deb7c 100644 --- a/lib/ash_csv/data_layer.ex +++ b/lib/ash_csv/data_layer.ex @@ -209,7 +209,7 @@ defmodule AshCsv.DataLayer do else case dump_row(resource, changeset) do {:ok, row} -> - # NimbleCSV: encode one row to iodata (replaces CSV.encode(separator: separator(resource)) |> Enum.to_list()) + # Replace CSV library operations with NimbleCSV operations iodata = csv_module(resource).dump_to_iodata([row]) result = @@ -274,7 +274,6 @@ defmodule AshCsv.DataLayer do end # sobelow_skip ["Traversal"] - # NimbleCSV: param is iodata from dump_to_iodata (was "lines" from CSV.encode |> Enum.to_list()) defp write_result(resource, iodata, retry? \\ false) do resource |> file() @@ -418,7 +417,6 @@ defmodule AshCsv.DataLayer do end) |> case do {:ok, rows} -> - # NimbleCSV: encode remaining rows to iodata (replaces CSV.encode(separator: ...) |> Enum.to_list()) iodata = csv_module(resource).dump_to_iodata(rows) iodata = @@ -476,7 +474,6 @@ defmodule AshCsv.DataLayer do end) |> case do {:ok, rows} -> - # NimbleCSV: encode all rows to iodata (replaces CSV.encode(separator: ...) |> Enum.to_list()) iodata = csv_module(resource).dump_to_iodata(rows) if File.exists?(file(resource)) do @@ -555,7 +552,6 @@ defmodule AshCsv.DataLayer do |> file() |> then(fn file -> if decode? do - # NimbleCSV: decode stream of lines to stream of rows (replaces CSV.decode(separator: ...)); parse_stream yields rows directly (no {:ok, row} wrapper) file |> File.stream!() |> Stream.drop(amount_to_drop) @@ -582,7 +578,6 @@ defmodule AshCsv.DataLayer do {:ok, results} rescue - # NimbleCSV: parse_stream raises on malformed CSV; convert to {:error, message} for caller e in NimbleCSV.ParseError -> {:error, Exception.message(e)} @@ -641,7 +636,6 @@ defmodule AshCsv.DataLayer do case row do {:ok, row} -> - # NimbleCSV: encode one row to iodata (replaces CSV.encode(separator: ...) |> Enum.to_list()) iodata = csv_module(resource).dump_to_iodata([Enum.reverse(row)]) result = @@ -707,7 +701,7 @@ defmodule AshCsv.DataLayer do end end - # NimbleCSV: returns per-resource parser module (from BuildParser); used for dump_to_iodata/1 and parse_stream/1 + # Initialize the NimbleCSV module for the resource defp csv_module(resource) do resource.ash_csv_csv_module() end diff --git a/lib/ash_csv/data_layer/transformers/build_parser.ex b/lib/ash_csv/data_layer/transformers/build_parser.ex index 654cf72..bfd8cb5 100644 --- a/lib/ash_csv/data_layer/transformers/build_parser.ex +++ b/lib/ash_csv/data_layer/transformers/build_parser.ex @@ -8,8 +8,6 @@ defmodule AshCsv.DataLayer.Transformers.BuildParser do def transform(dsl) do columns = AshCsv.DataLayer.Info.columns(dsl) - - # NimbleCSV: read separator from DSL and convert to string for NimbleCSV.define (expects string, not codepoint) separator = AshCsv.DataLayer.Info.separator(dsl) || ?, separator_string = @@ -94,7 +92,6 @@ defmodule AshCsv.DataLayer.Transformers.BuildParser do map = {:%{}, [], Enum.map(columns, fn column -> {column, {column, [], Elixir}} end)} - # NimbleCSV: build per-resource parser module name (e.g. MyApp.Post.AshCsvNimbleCSV) for NimbleCSV.define resource_module = Spark.Dsl.Transformer.get_persisted(dsl, :module) csv_module = @@ -115,13 +112,12 @@ defmodule AshCsv.DataLayer.Transformers.BuildParser do dsl, [], quote do - # NimbleCSV: define per-resource parser at compile time (replaces runtime CSV.encode/decode with separator option) + # Define the NimbleCSV parser NimbleCSV.define(unquote(csv_module), separator: unquote(separator_string), line_separator: "\n" ) - # NimbleCSV: expose parser module so data layer can call dump_to_iodata/1 and parse_stream/1 def ash_csv_csv_module do unquote(csv_module) end From 9c913e8e6311f2875535cc97918afedaa2c6d5dd Mon Sep 17 00:00:00 2001 From: Jacob Atanasio Date: Wed, 4 Feb 2026 13:56:03 -0700 Subject: [PATCH 3/3] adjustments for PR comments --- lib/ash_csv/data_layer.ex | 4 +--- .../data_layer/transformers/build_parser.ex | 24 +++++++------------ lib/ash_csv/info.ex | 4 ++++ mix.exs | 1 - 4 files changed, 13 insertions(+), 20 deletions(-) diff --git a/lib/ash_csv/data_layer.ex b/lib/ash_csv/data_layer.ex index 54deb7c..392ad53 100644 --- a/lib/ash_csv/data_layer.ex +++ b/lib/ash_csv/data_layer.ex @@ -209,7 +209,6 @@ defmodule AshCsv.DataLayer do else case dump_row(resource, changeset) do {:ok, row} -> - # Replace CSV library operations with NimbleCSV operations iodata = csv_module(resource).dump_to_iodata([row]) result = @@ -701,8 +700,7 @@ defmodule AshCsv.DataLayer do end end - # Initialize the NimbleCSV module for the resource defp csv_module(resource) do - resource.ash_csv_csv_module() + AshCsv.DataLayer.Info.csv_module(resource) end end diff --git a/lib/ash_csv/data_layer/transformers/build_parser.ex b/lib/ash_csv/data_layer/transformers/build_parser.ex index bfd8cb5..8f512ce 100644 --- a/lib/ash_csv/data_layer/transformers/build_parser.ex +++ b/lib/ash_csv/data_layer/transformers/build_parser.ex @@ -15,7 +15,8 @@ defmodule AshCsv.DataLayer.Transformers.BuildParser do <> rescue _ -> - "," + raise ArgumentError, + "Invalid separator value: #{inspect(separator)}. Expected a valid UTF-8 character." end func_args = @@ -93,12 +94,7 @@ defmodule AshCsv.DataLayer.Transformers.BuildParser do map = {:%{}, [], Enum.map(columns, fn column -> {column, {column, [], Elixir}} end)} resource_module = Spark.Dsl.Transformer.get_persisted(dsl, :module) - - csv_module = - Module.concat([ - resource_module, - AshCsvNimbleCSV - ]) + csv_module = AshCsv.DataLayer.Info.csv_module(resource_module) struct = {:struct, [], @@ -113,16 +109,12 @@ defmodule AshCsv.DataLayer.Transformers.BuildParser do [], quote do # Define the NimbleCSV parser - NimbleCSV.define(unquote(csv_module), - separator: unquote(separator_string), - line_separator: "\n" - ) - - def ash_csv_csv_module do - unquote(csv_module) - end + NimbleCSV.define(unquote(csv_module), + separator: unquote(separator_string), + line_separator: "\n" + ) - def ash_csv_dump_row(unquote(map)) do + def ash_csv_dump_row(unquote(map)) do {:ok, unquote(dump_fields)} catch {:error, error} -> diff --git a/lib/ash_csv/info.ex b/lib/ash_csv/info.ex index 476c1cf..6bbf3ce 100644 --- a/lib/ash_csv/info.ex +++ b/lib/ash_csv/info.ex @@ -28,4 +28,8 @@ defmodule AshCsv.DataLayer.Info do def create?(resource) do Extension.get_opt(resource, [:csv], :create?, nil, true) end + + def csv_module(resource) do + Module.concat([resource, AshCsvNimbleCSV]) + end end diff --git a/mix.exs b/mix.exs index be8e0f5..84dfd1a 100644 --- a/mix.exs +++ b/mix.exs @@ -112,7 +112,6 @@ defmodule AshCsv.MixProject do defp deps do [ {:ash, ash_version("~> 3.0")}, - {:csv, "~> 3.0"}, {:nimble_csv, "~> 1.2"}, {:igniter, "~> 0.5", only: [:dev, :test]}, {:ex_doc, "~> 0.37-rc", only: [:dev, :test], runtime: false},