From 55a51f69ab927a404c05cd6459d6214103fc016a Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Thu, 2 Apr 2026 12:07:51 -0500 Subject: [PATCH] Replace boolean enrichment flags with enum status fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - New fields: hrrr_status, weather_status, terrain_status, iemre_status with values: pending, queued, processing, complete, failed, unavailable - EnrichmentStatus module defines valid state transitions - Migration backfills: queued=true → complete, false → pending - Workers set status on transitions (queued on enqueue, complete on finish) - Show page reads status from contact struct, not computed dynamically - Backfill dashboard queries status fields for accurate progress counts - Remove cron-scheduled ContactWeatherEnqueueWorker (enrichment only on submission, page view, or manual backfill) - Partial indexes on status != 'complete' for fast unprocessed lookups --- config/config.exs | 1 - config/dev.exs | 3 +- lib/microwaveprop/radio.ex | 80 +++++++---------- lib/microwaveprop/radio/contact.ex | 19 +++- lib/microwaveprop/radio/enrichment_status.ex | 57 ++++++++++++ .../workers/terrain_profile_worker.ex | 2 + lib/microwaveprop_web/live/backfill_live.ex | 50 ++++------- .../live/contact_live/show.ex | 90 ++++++++++++------- lib/mix/tasks/reset_enrichment.ex | 6 +- ...replace_enrichment_booleans_with_enums.exs | 54 +++++++++++ test/microwaveprop/radio_test.exs | 30 +++---- .../contact_weather_enqueue_worker_test.exs | 50 +++++------ test/mix/tasks/reset_enrichment_test.exs | 20 ++--- 13 files changed, 285 insertions(+), 177 deletions(-) create mode 100644 lib/microwaveprop/radio/enrichment_status.ex create mode 100644 priv/repo/migrations/20260402170033_replace_enrichment_booleans_with_enums.exs diff --git a/config/config.exs b/config/config.exs index 429a9309..b8e89a2c 100644 --- a/config/config.exs +++ b/config/config.exs @@ -51,7 +51,6 @@ config :microwaveprop, Oban, {Oban.Plugins.Cron, crontab: [ {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker}, - {"*/30 * * * *", Microwaveprop.Workers.ContactWeatherEnqueueWorker}, {"*/5 * * * *", Microwaveprop.Commercial.PollWorker}, {"5 * * * *", Microwaveprop.Workers.PropagationGridWorker} ]} diff --git a/config/dev.exs b/config/dev.exs index 552de9a6..9ec51295 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -78,8 +78,7 @@ config :microwaveprop, Oban, {Oban.Plugins.Cron, crontab: [ {"5 * * * *", Microwaveprop.Workers.PropagationGridWorker}, - {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker}, - {"*/30 * * * *", Microwaveprop.Workers.ContactWeatherEnqueueWorker} + {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker} ]} ] diff --git a/lib/microwaveprop/radio.ex b/lib/microwaveprop/radio.ex index 98478bd4..2416840b 100644 --- a/lib/microwaveprop/radio.ex +++ b/lib/microwaveprop/radio.ex @@ -97,61 +97,43 @@ defmodule Microwaveprop.Radio do {field, dir} end - def unprocessed_contacts(limit \\ 500) do - Contact - |> where([q], q.weather_queued == false and not is_nil(q.pos1)) - |> order_by([q], asc: q.qso_timestamp) - |> limit(^limit) - |> Repo.all() + # ── Enrichment status queries ──────────────────────────────────── + + @enrichable_statuses [:pending, :failed] + + def contacts_needing_enrichment(field, limit \\ 500, extra_filter \\ nil) do + query = + Contact + |> where([q], field(q, ^field) in ^@enrichable_statuses and not is_nil(q.pos1)) + |> order_by([q], asc: q.qso_timestamp) + |> limit(^limit) + + query = if extra_filter, do: extra_filter.(query), else: query + Repo.all(query) end - def mark_weather_queued!(ids) do + def unprocessed_contacts(limit \\ 500), + do: contacts_needing_enrichment(:weather_status, limit) + + def unprocessed_hrrr_contacts(limit \\ 500), + do: contacts_needing_enrichment(:hrrr_status, limit) + + def unprocessed_terrain_contacts(limit \\ 500), + do: contacts_needing_enrichment(:terrain_status, limit, &where(&1, [q], not is_nil(q.pos2))) + + def unprocessed_iemre_contacts(limit \\ 500), + do: contacts_needing_enrichment(:iemre_status, limit) + + def set_enrichment_status!(ids, field, status) do Contact |> where([q], q.id in ^ids) - |> Repo.update_all(set: [weather_queued: true]) + |> Repo.update_all(set: [{field, status}]) end - def unprocessed_hrrr_contacts(limit \\ 500) do - Contact - |> where([q], q.hrrr_queued == false and not is_nil(q.pos1)) - |> order_by([q], asc: q.qso_timestamp) - |> limit(^limit) - |> Repo.all() - end - - def mark_hrrr_queued!(ids) do - Contact - |> where([q], q.id in ^ids) - |> Repo.update_all(set: [hrrr_queued: true]) - end - - def unprocessed_terrain_contacts(limit \\ 500) do - Contact - |> where([q], q.terrain_queued == false and not is_nil(q.pos1) and not is_nil(q.pos2)) - |> order_by([q], asc: q.qso_timestamp) - |> limit(^limit) - |> Repo.all() - end - - def mark_terrain_queued!(ids) do - Contact - |> where([q], q.id in ^ids) - |> Repo.update_all(set: [terrain_queued: true]) - end - - def unprocessed_iemre_contacts(limit \\ 500) do - Contact - |> where([q], q.iemre_queued == false and not is_nil(q.pos1)) - |> order_by([q], asc: q.qso_timestamp) - |> limit(^limit) - |> Repo.all() - end - - def mark_iemre_queued!(ids) do - Contact - |> where([q], q.id in ^ids) - |> Repo.update_all(set: [iemre_queued: true]) - end + def mark_weather_queued!(ids), do: set_enrichment_status!(ids, :weather_status, :queued) + def mark_hrrr_queued!(ids), do: set_enrichment_status!(ids, :hrrr_status, :queued) + def mark_terrain_queued!(ids), do: set_enrichment_status!(ids, :terrain_status, :queued) + def mark_iemre_queued!(ids), do: set_enrichment_status!(ids, :iemre_status, :queued) @doc """ Returns a list of {lat, lon} points along the contact path: pos1, midpoint, pos2. diff --git a/lib/microwaveprop/radio/contact.ex b/lib/microwaveprop/radio/contact.ex index 859ec15e..971b671b 100644 --- a/lib/microwaveprop/radio/contact.ex +++ b/lib/microwaveprop/radio/contact.ex @@ -20,10 +20,21 @@ defmodule Microwaveprop.Radio.Contact do field :mode, :string field :band, :decimal field :distance_km, :decimal - field :weather_queued, :boolean, default: false - field :hrrr_queued, :boolean, default: false - field :terrain_queued, :boolean, default: false - field :iemre_queued, :boolean, default: false + field :hrrr_status, Ecto.Enum, + values: [:pending, :queued, :processing, :complete, :failed, :unavailable], + default: :pending + + field :weather_status, Ecto.Enum, + values: [:pending, :queued, :processing, :complete, :failed, :unavailable], + default: :pending + + field :terrain_status, Ecto.Enum, + values: [:pending, :queued, :processing, :complete, :failed, :unavailable], + default: :pending + + field :iemre_status, Ecto.Enum, + values: [:pending, :queued, :processing, :complete, :failed, :unavailable], + default: :pending field :user_submitted, :boolean, default: false field :submitter_email, :string field :flagged_invalid, :boolean, default: false diff --git a/lib/microwaveprop/radio/enrichment_status.ex b/lib/microwaveprop/radio/enrichment_status.ex new file mode 100644 index 00000000..ee063c21 --- /dev/null +++ b/lib/microwaveprop/radio/enrichment_status.ex @@ -0,0 +1,57 @@ +defmodule Microwaveprop.Radio.EnrichmentStatus do + @moduledoc """ + State machine for contact enrichment lifecycle. + + States: + :pending → initial, no work started + :queued → Oban job inserted + :processing → worker actively running + :complete → data exists in DB + :failed → worker exhausted retries + :unavailable → data cannot be fetched (missing pos, archive gone, etc.) + + Valid transitions: + pending → queued, unavailable + queued → processing, failed + processing → complete, failed + failed → queued (retry) + unavailable → queued (retry with new data source) + complete → pending (reset for re-enrichment) + """ + + @states [:pending, :queued, :processing, :complete, :failed, :unavailable] + @fields [:hrrr_status, :weather_status, :terrain_status, :iemre_status] + + @transitions %{ + pending: [:queued, :unavailable], + queued: [:processing, :failed], + processing: [:complete, :failed], + failed: [:queued], + unavailable: [:queued], + complete: [:pending] + } + + def states, do: @states + def fields, do: @fields + + @doc "Returns true if transitioning from `from` to `to` is valid." + def valid_transition?(from, to) do + to in Map.get(@transitions, from, []) + end + + @doc "Validates and applies a status transition on a changeset." + def transition(changeset, field, new_status) when field in @fields and new_status in @states do + current = Ecto.Changeset.get_field(changeset, field) + + if valid_transition?(current, new_status) do + Ecto.Changeset.put_change(changeset, field, new_status) + else + Ecto.Changeset.add_error(changeset, field, "invalid transition from #{current} to #{new_status}") + end + end + + @doc "Force-sets status without transition validation (for migrations, resets)." + def force_status(changeset, field, status) when field in @fields and status in @states do + Ecto.Changeset.put_change(changeset, field, status) + end +end diff --git a/lib/microwaveprop/workers/terrain_profile_worker.ex b/lib/microwaveprop/workers/terrain_profile_worker.ex index 69660801..707ee2bf 100644 --- a/lib/microwaveprop/workers/terrain_profile_worker.ex +++ b/lib/microwaveprop/workers/terrain_profile_worker.ex @@ -56,6 +56,8 @@ defmodule Microwaveprop.Workers.TerrainProfileWorker do verdict: analysis.verdict }) + Radio.set_enrichment_status!([qso_id], :terrain_status, :complete) + Phoenix.PubSub.broadcast( Microwaveprop.PubSub, "contact_enrichment:#{qso_id}", diff --git a/lib/microwaveprop_web/live/backfill_live.ex b/lib/microwaveprop_web/live/backfill_live.ex index 67021018..0bcbaabe 100644 --- a/lib/microwaveprop_web/live/backfill_live.ex +++ b/lib/microwaveprop_web/live/backfill_live.ex @@ -68,13 +68,15 @@ defmodule MicrowavepropWeb.BackfillLive do )} end + @enrichable [:pending, :failed] + defp enqueue_batch(limit) do contacts = from(c in Contact, where: not is_nil(c.pos1) and - (c.hrrr_queued == false or c.weather_queued == false or - c.terrain_queued == false or c.iemre_queued == false), + (c.hrrr_status in ^@enrichable or c.weather_status in ^@enrichable or + c.terrain_status in ^@enrichable or c.iemre_status in ^@enrichable), order_by: [desc: c.qso_timestamp], limit: ^limit ) @@ -85,42 +87,22 @@ defmodule MicrowavepropWeb.BackfillLive do end defp count_unprocessed do - missing_terrain = - Repo.one( - from(c in Contact, - left_join: t in Microwaveprop.Terrain.TerrainProfile, on: t.qso_id == c.id, - where: not is_nil(c.pos1) and not is_nil(c.pos2) and is_nil(t.id), - select: count(c.id) - ) - ) + incomplete = [:pending, :queued, :processing, :failed] - # Use Oban job counts as proxy for in-progress work since queued flags - # only indicate "was enqueued" not "data exists" - pending_hrrr = - Repo.one( - from(j in "oban_jobs", - where: - j.worker == "Microwaveprop.Workers.HrrrFetchWorker" and - j.state in ["available", "executing", "scheduled", "retryable"], - select: count(j.id) - ) - ) + terrain = + Repo.one(from(c in Contact, where: c.terrain_status in ^incomplete and not is_nil(c.pos1), select: count())) - pending_weather = - Repo.one( - from(j in "oban_jobs", - where: - j.worker == "Microwaveprop.Workers.WeatherFetchWorker" and - j.state in ["available", "executing", "scheduled", "retryable"], - select: count(j.id) - ) - ) + hrrr = + Repo.one(from(c in Contact, where: c.hrrr_status in ^incomplete and not is_nil(c.pos1), select: count())) + + weather = + Repo.one(from(c in Contact, where: c.weather_status in ^incomplete and not is_nil(c.pos1), select: count())) %{ - terrain: missing_terrain, - hrrr: pending_hrrr, - weather: pending_weather, - total: missing_terrain + pending_hrrr + pending_weather + terrain: terrain, + hrrr: hrrr, + weather: weather, + total: max(terrain, max(hrrr, weather)) } end diff --git a/lib/microwaveprop_web/live/contact_live/show.ex b/lib/microwaveprop_web/live/contact_live/show.ex index e574a589..5ffe0ba4 100644 --- a/lib/microwaveprop_web/live/contact_live/show.ex +++ b/lib/microwaveprop_web/live/contact_live/show.ex @@ -32,9 +32,10 @@ defmodule MicrowavepropWeb.ContactLive.Show do solar = load_solar(contact) hrrr_path = Weather.hrrr_profiles_for_path(contact) hrrr = List.first(hrrr_path) - {hrrr, hrrr_status} = maybe_enqueue_hrrr(hrrr, contact) + {hrrr, contact} = maybe_enqueue_hrrr(hrrr, contact) terrain = Terrain.get_terrain_profile(contact.id) - terrain_status = if terrain, do: :loaded, else: maybe_enqueue_terrain(terrain, contact) + contact = maybe_enqueue_terrain(terrain, contact) + contact = maybe_enqueue_weather(weather, contact) elevation_profile = compute_elevation_profile(contact, hrrr_path, weather.soundings) propagation_analysis = build_propagation_analysis(contact, hrrr, terrain, elevation_profile, weather.soundings) @@ -44,12 +45,9 @@ defmodule MicrowavepropWeb.ContactLive.Show do contact: contact, surface_observations: weather.surface_observations, soundings: weather.soundings, - weather_status: maybe_enqueue_weather(weather, contact), solar: solar, hrrr: hrrr, - hrrr_status: hrrr_status, terrain: terrain, - terrain_status: terrain_status, elevation_profile: elevation_profile, propagation_analysis: propagation_analysis, terrain_expanded: false, @@ -117,7 +115,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do @impl true def handle_info({:terrain_ready, _qso_id}, socket) do - contact = socket.assigns.contact + contact = %{socket.assigns.contact | terrain_status: :complete} terrain = Terrain.get_terrain_profile(contact.id) soundings = socket.assigns.soundings hrrr_path = Weather.hrrr_profiles_for_path(contact) @@ -127,8 +125,8 @@ defmodule MicrowavepropWeb.ContactLive.Show do {:noreply, assign(socket, + contact: contact, terrain: terrain, - terrain_status: :loaded, elevation_profile: elevation_profile, propagation_analysis: propagation_analysis )} @@ -140,6 +138,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do hrrr = List.first(hrrr_path) if hrrr do + contact = %{contact | hrrr_status: :complete} soundings = socket.assigns.soundings terrain = socket.assigns.terrain elevation_profile = compute_elevation_profile(contact, hrrr_path, soundings) @@ -147,8 +146,8 @@ defmodule MicrowavepropWeb.ContactLive.Show do {:noreply, assign(socket, + contact: contact, hrrr: hrrr, - hrrr_status: :loaded, elevation_profile: elevation_profile, propagation_analysis: propagation_analysis )} @@ -158,7 +157,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do end def handle_info({:weather_ready, _info}, socket) do - contact = socket.assigns.contact + contact = %{socket.assigns.contact | weather_status: :complete} weather = load_weather(contact) soundings = weather.soundings @@ -166,34 +165,40 @@ defmodule MicrowavepropWeb.ContactLive.Show do hrrr = List.first(hrrr_path) terrain = socket.assigns.terrain - # Recompute elevation profile with new sounding duct data elevation_profile = compute_elevation_profile(contact, hrrr_path, soundings) propagation_analysis = build_propagation_analysis(contact, hrrr, terrain, elevation_profile, soundings) {:noreply, assign(socket, + contact: contact, surface_observations: weather.surface_observations, soundings: soundings, - weather_status: :loaded, elevation_profile: elevation_profile, propagation_analysis: propagation_analysis )} end defp maybe_enqueue_weather(weather, contact) do - has_data = weather.surface_observations != [] or weather.soundings != [] + if contact.weather_status == :complete do + contact + else + has_data = weather.surface_observations != [] or weather.soundings != [] - cond do - has_data -> - :loaded + cond do + has_data -> + Radio.set_enrichment_status!([contact.id], :weather_status, :complete) + %{contact | weather_status: :complete} - contact.pos1 -> - jobs = ContactWeatherEnqueueWorker.build_weather_jobs([contact]) - if jobs != [], do: Oban.insert_all(jobs) - :queued + contact.pos1 && contact.weather_status in [:pending, :failed] -> + jobs = ContactWeatherEnqueueWorker.build_weather_jobs([contact]) + if jobs != [], do: Oban.insert_all(jobs) + Radio.set_enrichment_status!([contact.id], :weather_status, :queued) + %{contact | weather_status: :queued} - true -> - :unavailable + true -> + Radio.set_enrichment_status!([contact.id], :weather_status, :unavailable) + %{contact | weather_status: :unavailable} + end end end @@ -268,24 +273,40 @@ defmodule MicrowavepropWeb.ContactLive.Show do end end - defp maybe_enqueue_terrain(terrain, _contact) when not is_nil(terrain), do: :loaded + defp maybe_enqueue_terrain(_terrain, %{terrain_status: :complete} = contact), do: contact + + defp maybe_enqueue_terrain(terrain, contact) when not is_nil(terrain) do + Radio.set_enrichment_status!([contact.id], :terrain_status, :complete) + %{contact | terrain_status: :complete} + end defp maybe_enqueue_terrain(nil, contact) do - if contact.pos1 && contact.pos2 do + if contact.pos1 && contact.pos2 && contact.terrain_status in [:pending, :failed] do Oban.insert(TerrainProfileWorker.new(%{"qso_id" => contact.id})) - :queued + Radio.set_enrichment_status!([contact.id], :terrain_status, :queued) + %{contact | terrain_status: :queued} else - :unavailable + if is_nil(contact.pos1) or is_nil(contact.pos2) do + Radio.set_enrichment_status!([contact.id], :terrain_status, :unavailable) + %{contact | terrain_status: :unavailable} + else + contact + end end end - defp maybe_enqueue_hrrr(hrrr, _contact) when not is_nil(hrrr), do: {hrrr, :loaded} + defp maybe_enqueue_hrrr(hrrr, %{hrrr_status: :complete} = contact), do: {hrrr, contact} + + defp maybe_enqueue_hrrr(hrrr, contact) when not is_nil(hrrr) do + Radio.set_enrichment_status!([contact.id], :hrrr_status, :complete) + {hrrr, %{contact | hrrr_status: :complete}} + end defp maybe_enqueue_hrrr(nil, contact) do lat = contact.pos1 && contact.pos1["lat"] lon = contact.pos1 && (contact.pos1["lon"] || contact.pos1["lng"]) - if lat && lon do + if lat && lon && contact.hrrr_status in [:pending, :failed] do valid_time = HrrrClient.nearest_hrrr_hour(contact.qso_timestamp) {rlat, rlon} = Weather.round_to_hrrr_grid(lat, lon) @@ -297,9 +318,10 @@ defmodule MicrowavepropWeb.ContactLive.Show do }) ) - {nil, :queued} + Radio.set_enrichment_status!([contact.id], :hrrr_status, :queued) + {nil, %{contact | hrrr_status: :queued}} else - {nil, :unavailable} + {nil, contact} end end @@ -356,7 +378,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do /> <% end %> - <%= if !@elevation_profile && @terrain_status == :queued do %> + <%= if !@elevation_profile && @contact.terrain_status == :queued do %>
Computing elevation profile and terrain analysis... @@ -522,7 +544,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
<% else %>
- <%= if @terrain_status == :queued do %> + <%= if @contact.terrain_status == :queued do %> Computing terrain profile... @@ -538,7 +560,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do

Soundings

<%= if @soundings == [] do %>
- <%= if @weather_status == :queued do %> + <%= if @contact.weather_status == :queued do %> Fetching sounding data from nearby stations... @@ -694,7 +716,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
<% else %>
- <%= case @hrrr_status do %> + <%= case @contact.hrrr_status do %> <% :queued -> %> @@ -713,7 +735,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do

Surface Observations

<%= if @surface_observations == [] do %>
- <%= if @weather_status == :queued do %> + <%= if @contact.weather_status == :queued do %> Fetching surface observations from nearby stations... diff --git a/lib/mix/tasks/reset_enrichment.ex b/lib/mix/tasks/reset_enrichment.ex index fcacdc76..80a76663 100644 --- a/lib/mix/tasks/reset_enrichment.ex +++ b/lib/mix/tasks/reset_enrichment.ex @@ -24,10 +24,10 @@ defmodule Mix.Tasks.ResetEnrichment do {count, _} = Contact - |> where([q], q.weather_queued == true or q.hrrr_queued == true or q.iemre_queued == true) - |> Repo.update_all(set: [weather_queued: false, hrrr_queued: false, iemre_queued: false]) + |> where([q], q.weather_status != :pending or q.hrrr_status != :pending or q.iemre_status != :pending) + |> Repo.update_all(set: [weather_status: "pending", hrrr_status: "pending", iemre_status: "pending"]) - Mix.shell().info("Reset enrichment flags on #{count} QSOs") + Mix.shell().info("Reset enrichment status to :pending on #{count} contacts") :ok end end diff --git a/priv/repo/migrations/20260402170033_replace_enrichment_booleans_with_enums.exs b/priv/repo/migrations/20260402170033_replace_enrichment_booleans_with_enums.exs new file mode 100644 index 00000000..dda7bdba --- /dev/null +++ b/priv/repo/migrations/20260402170033_replace_enrichment_booleans_with_enums.exs @@ -0,0 +1,54 @@ +defmodule Microwaveprop.Repo.Migrations.ReplaceEnrichmentBooleansWithEnums do + use Ecto.Migration + + def change do + alter table(:qsos) do + add :hrrr_status, :string, default: "pending", null: false + add :weather_status, :string, default: "pending", null: false + add :terrain_status, :string, default: "pending", null: false + add :iemre_status, :string, default: "pending", null: false + end + + flush() + + # Backfill: queued=true → complete, queued=false → pending + execute( + "UPDATE qsos SET hrrr_status = CASE WHEN hrrr_queued THEN 'complete' ELSE 'pending' END", + "SELECT 1" + ) + + execute( + "UPDATE qsos SET weather_status = CASE WHEN weather_queued THEN 'complete' ELSE 'pending' END", + "SELECT 1" + ) + + execute( + "UPDATE qsos SET terrain_status = CASE WHEN terrain_queued THEN 'complete' ELSE 'pending' END", + "SELECT 1" + ) + + execute( + "UPDATE qsos SET iemre_status = CASE WHEN iemre_queued THEN 'complete' ELSE 'pending' END", + "SELECT 1" + ) + + # Drop old boolean columns and their partial indexes + drop_if_exists index(:qsos, [:weather_queued], name: :qsos_weather_queued_index) + drop_if_exists index(:qsos, [:hrrr_queued], name: :qsos_hrrr_queued_index) + drop_if_exists index(:qsos, [:terrain_queued], name: :qsos_terrain_queued_index) + drop_if_exists index(:qsos, [:iemre_queued], name: :qsos_iemre_queued_index) + + alter table(:qsos) do + remove :weather_queued, :boolean, default: false + remove :hrrr_queued, :boolean, default: false + remove :terrain_queued, :boolean, default: false + remove :iemre_queued, :boolean, default: false + end + + # Partial indexes for finding contacts needing enrichment + create index(:qsos, [:hrrr_status], where: "hrrr_status != 'complete'", name: :qsos_hrrr_status_pending_index) + create index(:qsos, [:weather_status], where: "weather_status != 'complete'", name: :qsos_weather_status_pending_index) + create index(:qsos, [:terrain_status], where: "terrain_status != 'complete'", name: :qsos_terrain_status_pending_index) + create index(:qsos, [:iemre_status], where: "iemre_status != 'complete'", name: :qsos_iemre_status_pending_index) + end +end diff --git a/test/microwaveprop/radio_test.exs b/test/microwaveprop/radio_test.exs index ec47ae9c..5949ceaf 100644 --- a/test/microwaveprop/radio_test.exs +++ b/test/microwaveprop/radio_test.exs @@ -110,7 +110,7 @@ defmodule Microwaveprop.RadioTest do assert q1.id in ids end - test "excludes QSOs already marked as weather_queued" do + test "excludes QSOs already marked as weather_status" do q = create_contact() Radio.mark_weather_queued!([q.id]) @@ -143,8 +143,8 @@ defmodule Microwaveprop.RadioTest do Radio.mark_weather_queued!([q1.id, q2.id]) - assert Repo.get!(Contact, q1.id).weather_queued == true - assert Repo.get!(Contact, q2.id).weather_queued == true + assert Repo.get!(Contact, q1.id).weather_status == :queued + assert Repo.get!(Contact, q2.id).weather_status == :queued end test "does not affect other QSOs" do @@ -153,8 +153,8 @@ defmodule Microwaveprop.RadioTest do Radio.mark_weather_queued!([q1.id]) - assert Repo.get!(Contact, q1.id).weather_queued == true - assert Repo.get!(Contact, q2.id).weather_queued == false + assert Repo.get!(Contact, q1.id).weather_status == :queued + assert Repo.get!(Contact, q2.id).weather_status == :pending end test "handles empty list" do @@ -230,7 +230,7 @@ defmodule Microwaveprop.RadioTest do assert q1.id in ids end - test "excludes QSOs already marked as hrrr_queued" do + test "excludes QSOs already marked as hrrr_status" do q = create_contact() Radio.mark_hrrr_queued!([q.id]) @@ -263,8 +263,8 @@ defmodule Microwaveprop.RadioTest do Radio.mark_hrrr_queued!([q1.id, q2.id]) - assert Repo.get!(Contact, q1.id).hrrr_queued == true - assert Repo.get!(Contact, q2.id).hrrr_queued == true + assert Repo.get!(Contact, q1.id).hrrr_status == :queued + assert Repo.get!(Contact, q2.id).hrrr_status == :queued end test "does not affect other QSOs" do @@ -273,8 +273,8 @@ defmodule Microwaveprop.RadioTest do Radio.mark_hrrr_queued!([q1.id]) - assert Repo.get!(Contact, q1.id).hrrr_queued == true - assert Repo.get!(Contact, q2.id).hrrr_queued == false + assert Repo.get!(Contact, q1.id).hrrr_status == :queued + assert Repo.get!(Contact, q2.id).hrrr_status == :pending end test "handles empty list" do @@ -294,7 +294,7 @@ defmodule Microwaveprop.RadioTest do assert length(ids) == 1 end - test "excludes QSOs already marked as terrain_queued" do + test "excludes QSOs already marked as terrain_status" do q = create_contact() Radio.mark_terrain_queued!([q.id]) @@ -327,8 +327,8 @@ defmodule Microwaveprop.RadioTest do Radio.mark_terrain_queued!([q1.id, q2.id]) - assert Repo.get!(Contact, q1.id).terrain_queued == true - assert Repo.get!(Contact, q2.id).terrain_queued == true + assert Repo.get!(Contact, q1.id).terrain_status == :queued + assert Repo.get!(Contact, q2.id).terrain_status == :queued end test "does not affect other QSOs" do @@ -337,8 +337,8 @@ defmodule Microwaveprop.RadioTest do Radio.mark_terrain_queued!([q1.id]) - assert Repo.get!(Contact, q1.id).terrain_queued == true - assert Repo.get!(Contact, q2.id).terrain_queued == false + assert Repo.get!(Contact, q1.id).terrain_status == :queued + assert Repo.get!(Contact, q2.id).terrain_status == :pending end test "handles empty list" do diff --git a/test/microwaveprop/workers/contact_weather_enqueue_worker_test.exs b/test/microwaveprop/workers/contact_weather_enqueue_worker_test.exs index debee52e..309098f9 100644 --- a/test/microwaveprop/workers/contact_weather_enqueue_worker_test.exs +++ b/test/microwaveprop/workers/contact_weather_enqueue_worker_test.exs @@ -209,12 +209,12 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do _station = create_asos_station("KDFW", 32.90, -97.04) contact = create_contact() - assert contact.weather_queued == false + assert contact.weather_status == :pending assert :ok = ContactWeatherEnqueueWorker.perform(%Oban.Job{args: %{}}) updated = Repo.get!(Contact, contact.id) - assert updated.weather_queued == true + assert updated.weather_status == :queued end test "returns :ok when no unprocessed QSOs exist" do @@ -240,17 +240,17 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do # QSO without pos1 should not be marked as queued all_qsos = Repo.all(Contact) - refute Enum.any?(all_qsos, & &1.weather_queued) + refute Enum.any?(all_qsos, & &1.weather_status == :queued) end - test "enqueues HRRR jobs and marks QSOs as hrrr_queued" do + test "enqueues HRRR jobs and marks QSOs as hrrr_status" do contact = create_contact() - assert contact.hrrr_queued == false + assert contact.hrrr_status == :pending assert :ok = ContactWeatherEnqueueWorker.perform(%Oban.Job{args: %{}}) updated = Repo.get!(Contact, contact.id) - assert updated.hrrr_queued == true + assert updated.hrrr_status == :queued end end @@ -409,14 +409,14 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do :ok end - test "enqueues terrain jobs and marks QSOs as terrain_queued" do + test "enqueues terrain jobs and marks QSOs as terrain_status" do contact = create_contact() - assert contact.terrain_queued == false + assert contact.terrain_status == :pending assert :ok = ContactWeatherEnqueueWorker.perform(%Oban.Job{args: %{}}) updated = Repo.get!(Contact, contact.id) - assert updated.terrain_queued == true + assert updated.terrain_status == :queued end test "skips terrain for QSOs missing pos2" do @@ -425,7 +425,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do assert :ok = ContactWeatherEnqueueWorker.perform(%Oban.Job{args: %{}}) all_qsos = Repo.all(Contact) - refute Enum.any?(all_qsos, & &1.terrain_queued) + refute Enum.any?(all_qsos, & &1.terrain_status == :queued) end end @@ -550,14 +550,14 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do :ok end - test "enqueues IEMRE jobs and marks QSOs as iemre_queued" do + test "enqueues IEMRE jobs and marks QSOs as iemre_status" do contact = create_contact() - assert contact.iemre_queued == false + assert contact.iemre_status == :pending assert :ok = ContactWeatherEnqueueWorker.perform(%Oban.Job{args: %{}}) updated = Repo.get!(Contact, contact.id) - assert updated.iemre_queued == true + assert updated.iemre_status == :queued end end @@ -593,18 +593,18 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do _station = create_asos_station("KDFW", 32.90, -97.04) contact = create_contact() - assert contact.weather_queued == false - assert contact.hrrr_queued == false - assert contact.terrain_queued == false - assert contact.iemre_queued == false + assert contact.weather_status == :pending + assert contact.hrrr_status == :pending + assert contact.terrain_status == :pending + assert contact.iemre_status == :pending assert :ok = ContactWeatherEnqueueWorker.enqueue_for_contact(contact) updated = Repo.get!(Contact, contact.id) - assert updated.weather_queued == true - assert updated.hrrr_queued == true - assert updated.terrain_queued == true - assert updated.iemre_queued == true + assert updated.weather_status == :queued + assert updated.hrrr_status == :queued + assert updated.terrain_status == :queued + assert updated.iemre_status == :queued end test "works for QSO without pos2 (marks terrain_queued false)" do @@ -614,11 +614,11 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do assert :ok = ContactWeatherEnqueueWorker.enqueue_for_contact(contact) updated = Repo.get!(Contact, contact.id) - assert updated.weather_queued == true - assert updated.hrrr_queued == true + assert updated.weather_status == :queued + assert updated.hrrr_status == :queued # terrain requires pos2, so it can't be queued - assert updated.terrain_queued == false - assert updated.iemre_queued == true + assert updated.terrain_status == :pending + assert updated.iemre_status == :queued end end end diff --git a/test/mix/tasks/reset_enrichment_test.exs b/test/mix/tasks/reset_enrichment_test.exs index c12613f6..8700f052 100644 --- a/test/mix/tasks/reset_enrichment_test.exs +++ b/test/mix/tasks/reset_enrichment_test.exs @@ -28,34 +28,34 @@ defmodule Mix.Tasks.ResetEnrichmentTest do # Set all queued flags to true (not in changeset cast) Contact |> where([q], q.id == ^contact.id) - |> Repo.update_all(set: [weather_queued: true, hrrr_queued: true, iemre_queued: true, terrain_queued: true]) + |> Repo.update_all(set: [weather_status: "queued", hrrr_status: "queued", iemre_status: "queued", terrain_status: "queued"]) Repo.get!(Contact, contact.id) end describe "run/1" do - test "resets weather_queued, hrrr_queued, and iemre_queued to false" do + test "resets weather, hrrr, and iemre status to pending" do contact = create_queued_qso() - assert contact.weather_queued == true - assert contact.hrrr_queued == true - assert contact.iemre_queued == true + assert contact.weather_status == :queued + assert contact.hrrr_status == :queued + assert contact.iemre_status == :queued ResetEnrichment.run([]) updated = Repo.get!(Contact, contact.id) - assert updated.weather_queued == false - assert updated.hrrr_queued == false - assert updated.iemre_queued == false + assert updated.weather_status == :pending + assert updated.hrrr_status == :pending + assert updated.iemre_status == :pending end - test "does not reset terrain_queued" do + test "does not reset terrain status" do contact = create_queued_qso() ResetEnrichment.run([]) updated = Repo.get!(Contact, contact.id) - assert updated.terrain_queued == true + assert updated.terrain_status == :queued end test "handles empty database" do