Graham McIntire
48833f15be
For large CSVs (e.g. the 34k-row ARRL dump), the synchronous commit
path blocked for minutes and a crash mid-commit left no audit trail.
Refactor to run inserts + enrichment enqueues via Oban.
- New import_runs table: stores preview rows + running counters (total,
processed, imported, refined, error) + status + errors map + timing.
- New :contact_import Oban queue (limit 4) + ContactImportWorker.
Args: {import_run_id, offset, limit}. Each chunk (100 rows) inserts
its slice via CsvImport.commit_rows/1, atomically increments counters
in one update_all, flips status, and broadcasts progress on
'csv_import:<id>' PubSub topic.
- CsvImport.commit_rows/1 extracted from commit/1 (back-compat preserved).
New CsvImport.enqueue/2 persists the run + dispatches N chunk jobs.
Also: submit page copy updated from '902 MHz and up' to '50 MHz and up'
matching the band allowlist expansion done earlier.
LiveView UI for /imports/:id is a separate commit.
25 lines
919 B
Elixir
25 lines
919 B
Elixir
defmodule Microwaveprop.Repo.Migrations.CreateImportRuns do
|
|
use Ecto.Migration
|
|
|
|
def change do
|
|
create table(:import_runs, primary_key: false) do
|
|
add :id, :binary_id, primary_key: true
|
|
add :submitter_email, :string, null: false
|
|
add :rows, :map, null: false
|
|
add :total_rows, :integer, null: false
|
|
add :processed_rows, :integer, null: false, default: 0
|
|
add :imported_count, :integer, null: false, default: 0
|
|
add :refined_count, :integer, null: false, default: 0
|
|
add :error_count, :integer, null: false, default: 0
|
|
add :errors, :map, null: false, default: %{}
|
|
add :status, :string, null: false, default: "pending"
|
|
add :started_at, :utc_datetime_usec
|
|
add :completed_at, :utc_datetime_usec
|
|
|
|
timestamps(type: :utc_datetime_usec)
|
|
end
|
|
|
|
create index(:import_runs, [:submitter_email])
|
|
create index(:import_runs, [:status])
|
|
end
|
|
end
|