defmodule Microwaveprop.Workers.ContactImportWorker do @moduledoc """ Processes one chunk of rows out of an `ImportRun` — the async half of the CSV import flow. Each job receives an `import_run_id`, an `offset`, and a `limit`. The worker loads the run, takes that slice of rows, routes them through `CsvImport.commit_rows/1`, atomically bumps the per-run counters, and broadcasts the updated run on `"csv_import:"` so the submitting LiveView can reflect progress. Counters are updated with a single `UPDATE ... SET col = col + ?` query so concurrent workers finishing at the same time can't clobber each other. The worker that pushes `processed_rows` past `total_rows` flips `status` to `"complete"` and sets `completed_at`; any worker that sees `status = 'pending'` on its first increment flips it to `"running"` and sets `started_at`. """ use Oban.Pro.Worker, queue: :contact_import, max_attempts: 3, unique: [ period: :infinity, fields: [:worker, :args], keys: [:import_run_id, :offset], states: [:available, :scheduled, :executing, :retryable, :suspended, :completed] ] import Ecto.Query alias Microwaveprop.Radio.CsvImport alias Microwaveprop.Radio.ImportRun alias Microwaveprop.Repo require Logger @pubsub Microwaveprop.PubSub @impl Oban.Pro.Worker def process(%Oban.Job{args: %{"import_run_id" => run_id, "offset" => offset, "limit" => limit}}) do case Repo.get(ImportRun, run_id) do nil -> Logger.warning("ContactImportWorker: import_run #{run_id} not found; discarding chunk") :ok %ImportRun{} = run -> process_chunk(run, offset, limit) end end defp process_chunk(run, offset, limit) do all_rows = get_in(run.rows, ["rows"]) || [] chunk = Enum.slice(all_rows, offset, limit) commit_input = build_commit_input(chunk) commit_result = CsvImport.commit_rows(commit_input) updates = build_counter_updates(commit_result, length(chunk)) updated_run = apply_chunk_updates(run.id, updates) _ = broadcast(run.id, updated_run) :ok end defp build_commit_input(chunk) do {valid, refinements} = Enum.reduce(chunk, {[], []}, fn row, {valid, refinements} -> case CsvImport.decode_row(row) do {:valid, decoded} -> {[decoded | valid], refinements} {:refinement, decoded} -> {valid, [decoded | refinements]} end end) %{valid: Enum.reverse(valid), refinements: Enum.reverse(refinements)} end defp build_counter_updates(commit_result, chunk_len) do errors_map = Map.new(commit_result.errors, fn {row_num, messages} -> {to_string(row_num), messages} end) %{ chunk_len: chunk_len, imported_count: length(commit_result.imported), refined_count: length(commit_result.refined), error_count: length(commit_result.errors), errors_map: errors_map } end # Atomic per-chunk update. All counter fields are incremented in one # UPDATE so concurrent chunk-finishers can't clobber each other, and # the status / timestamp transitions are expressed as CASEs off the # pre-increment values. defp apply_chunk_updates(run_id, u) do now = DateTime.utc_now() chunk_len = u.chunk_len imported_count = u.imported_count refined_count = u.refined_count error_count = u.error_count errors_map = u.errors_map query = from(r in ImportRun, where: r.id == ^run_id, update: [ set: [ updated_at: ^now, processed_rows: fragment("? + ?", r.processed_rows, ^chunk_len), imported_count: fragment("? + ?", r.imported_count, ^imported_count), refined_count: fragment("? + ?", r.refined_count, ^refined_count), error_count: fragment("? + ?", r.error_count, ^error_count), status: fragment( """ CASE WHEN ? + ? >= ? THEN 'complete' WHEN ? = 'pending' THEN 'running' ELSE ? END """, r.processed_rows, ^chunk_len, r.total_rows, r.status, r.status ), started_at: fragment( "CASE WHEN ? IS NULL THEN ? ELSE ? END", r.started_at, ^now, r.started_at ), completed_at: fragment( "CASE WHEN ? + ? >= ? THEN ? ELSE ? END", r.processed_rows, ^chunk_len, r.total_rows, ^now, r.completed_at ), errors: fragment("? || ?::jsonb", r.errors, ^errors_map) ] ] ) {1, _} = Repo.update_all(query, []) Repo.get!(ImportRun, run_id) end defp broadcast(run_id, run) do Phoenix.PubSub.broadcast(@pubsub, "csv_import:#{run_id}", {:import_progress, run_id, run}) end end