diff --git a/assets/js/propagation_map_hook.ts b/assets/js/propagation_map_hook.ts index c0c48676..b31a68a8 100644 --- a/assets/js/propagation_map_hook.ts +++ b/assets/js/propagation_map_hook.ts @@ -979,10 +979,14 @@ export const PropagationMap: Record & { renderScores(this: PropagationMapHook, scores: ScorePoint[]) { this.scores = scores - if (this.scoreOverlay) { - this.map.removeLayer(this.scoreOverlay) + + if (scores.length === 0) { + if (this.scoreOverlay) { + this.map.removeLayer(this.scoreOverlay) + this.scoreOverlay = null + } + return } - if (scores.length === 0) return if (!this.scoreGrid) { this.scoreGrid = new ScoreGrid() @@ -991,7 +995,15 @@ export const PropagationMap: Record & { } scores.forEach((s) => this.scoreGrid.put(s.lat, s.lon, s.score)) - // Pre-calculate RGBA strings for scores 0-100 + if (this.scoreOverlay) { + // Layer already on the map — just redraw existing tiles against the + // updated grid data. Avoids the remove/add flash that caused the + // overlay to blink during the initial load pipeline. + this.scoreOverlay.redraw() + return + } + + // First render — build the GridLayer once and reuse it for every update. const colorCache = new Array(101) for (let i = 0; i <= 100; i++) { const c = this.scoreColorRGB(i) diff --git a/config/test.exs b/config/test.exs index b1a8cf5f..3666539b 100644 --- a/config/test.exs +++ b/config/test.exs @@ -42,6 +42,8 @@ config :microwaveprop, nexrad_req_options: [plug: {Req.Test, Microwaveprop.Weath config :microwaveprop, solar_req_options: [plug: {Req.Test, Microwaveprop.Weather.SolarClient}] config :microwaveprop, srtm_req_options: [plug: {Req.Test, Microwaveprop.Terrain.Srtm}, retry: false] config :microwaveprop, start_freshness_monitor: false +config :microwaveprop, cache_contact_count: false +config :microwaveprop, cache_contact_map: false # Initialize plugs at runtime for faster test compilation config :phoenix, :plug_init_mode, :runtime diff --git a/lib/microwaveprop/application.ex b/lib/microwaveprop/application.ex index 91934501..4e267ecd 100644 --- a/lib/microwaveprop/application.ex +++ b/lib/microwaveprop/application.ex @@ -14,6 +14,7 @@ defmodule Microwaveprop.Application do Microwaveprop.Repo, {Cluster.Supervisor, [topologies, [name: Microwaveprop.ClusterSupervisor]]}, {Phoenix.PubSub, name: Microwaveprop.PubSub}, + Microwaveprop.Cache, Microwaveprop.Propagation.ScoreCache, Microwaveprop.Weather.NexradCache, {Oban, Application.fetch_env!(:microwaveprop, Oban)}, diff --git a/lib/microwaveprop/cache.ex b/lib/microwaveprop/cache.ex new file mode 100644 index 00000000..f3a7cea9 --- /dev/null +++ b/lib/microwaveprop/cache.ex @@ -0,0 +1,71 @@ +defmodule Microwaveprop.Cache do + @moduledoc """ + Tiny ETS-backed TTL cache for values that are expensive to compute but + tolerate short staleness. Used for things like `Repo.aggregate` counts that + would otherwise run on every page load. + + Not a replacement for `Microwaveprop.Propagation.ScoreCache` or + `Microwaveprop.Weather.NexradCache` — those have bespoke invalidation logic + driven by PubSub. This module is for generic time-boxed memoization. + """ + use GenServer + + @table :microwaveprop_cache + + @spec start_link(keyword()) :: GenServer.on_start() + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, name: __MODULE__) + end + + @doc """ + Return the cached value for `key` if it's still fresh, otherwise invoke + `fun`, store the result with `ttl_ms` lifetime, and return it. + """ + @spec fetch_or_store(term(), non_neg_integer(), (-> value)) :: value when value: term() + def fetch_or_store(key, ttl_ms, fun) when is_function(fun, 0) do + now = System.monotonic_time(:millisecond) + + case :ets.lookup(@table, key) do + [{_, value, expires_at}] when expires_at > now -> + value + + _ -> + value = fun.() + :ets.insert(@table, {key, value, now + ttl_ms}) + value + end + end + + @doc "Insert `value` directly, overwriting any existing entry for `key`." + @spec put(term(), term(), integer()) :: :ok + def put(key, value, ttl_ms) do + :ets.insert(@table, {key, value, System.monotonic_time(:millisecond) + ttl_ms}) + :ok + end + + @doc "Remove `key` from the cache, forcing the next fetch to recompute." + @spec invalidate(term()) :: :ok + def invalidate(key) do + :ets.delete(@table, key) + :ok + end + + @spec clear() :: :ok + def clear do + :ets.delete_all_objects(@table) + :ok + end + + @impl true + def init(_opts) do + :ets.new(@table, [ + :set, + :named_table, + :public, + read_concurrency: true, + write_concurrency: true + ]) + + {:ok, %{}} + end +end diff --git a/lib/microwaveprop/radio.ex b/lib/microwaveprop/radio.ex index 604922a3..77663f7a 100644 --- a/lib/microwaveprop/radio.ex +++ b/lib/microwaveprop/radio.ex @@ -4,6 +4,7 @@ defmodule Microwaveprop.Radio do import Ecto.Query alias Microwaveprop.Accounts.User + alias Microwaveprop.Cache alias Microwaveprop.Radio.Contact alias Microwaveprop.Radio.ContactEdit alias Microwaveprop.Radio.Maidenhead @@ -11,6 +12,10 @@ defmodule Microwaveprop.Radio do @per_page 20 @sortable_fields ~w(station1 station2 band mode distance_km qso_timestamp inserted_at)a + @count_cache_key {__MODULE__, :total_contact_count} + @count_cache_ttl_ms 30_000 + @map_payload_cache_key {__MODULE__, :contact_map_payload} + @map_payload_ttl_ms 10 * 60 * 1_000 @type contact_page :: %{ entries: [Contact.t()], @@ -20,6 +25,95 @@ defmodule Microwaveprop.Radio do total_entries: non_neg_integer() } + @doc """ + Pre-encoded JSON payload for `/contacts/map`. Computing this requires loading + every contact with coordinates, deriving formatted rows, deduplicating + reciprocals, and JSON-encoding ~58k rows — ~150-300ms of work that would + otherwise run on every page mount. Cached for 10 minutes, invalidated when + a new contact is inserted. + """ + @spec contact_map_payload() :: %{ + json: iodata(), + count: non_neg_integer(), + bands: [integer()] + } + def contact_map_payload do + if Application.get_env(:microwaveprop, :cache_contact_map, true) do + Cache.fetch_or_store(@map_payload_cache_key, @map_payload_ttl_ms, fn -> + build_contact_map_payload() + end) + else + build_contact_map_payload() + end + end + + defp build_contact_map_payload do + contacts = load_contacts_for_map() + bands = contacts |> Enum.map(fn [_, _, _, _, band | _] -> band end) |> Enum.uniq() |> Enum.sort() + %{json: Jason.encode_to_iodata!(contacts), count: length(contacts), bands: bands} + end + + defp load_contacts_for_map do + from(c in Contact, + where: not is_nil(c.pos1) and not is_nil(c.pos2), + select: %{ + id: c.id, + pos1: c.pos1, + pos2: c.pos2, + band: c.band, + station1: c.station1, + station2: c.station2, + mode: c.mode, + distance_km: c.distance_km, + qso_timestamp: c.qso_timestamp + } + ) + |> Repo.all() + |> Enum.map(&format_map_contact/1) + |> Enum.reject(&is_nil/1) + |> dedup_map_reciprocals() + end + + defp format_map_contact(c) do + lat1 = c.pos1["lat"] + lon1 = c.pos1["lon"] || c.pos1["lng"] + lat2 = c.pos2["lat"] + lon2 = c.pos2["lon"] || c.pos2["lng"] + + if lat1 && lon1 && lat2 && lon2 do + [ + lat1, + lon1, + lat2, + lon2, + format_map_band(c.band), + c.station1, + c.station2, + c.mode, + format_map_distance(c.distance_km), + format_map_timestamp(c.qso_timestamp), + c.id + ] + end + end + + defp format_map_band(nil), do: 0 + defp format_map_band(band), do: Decimal.to_integer(band) + + defp format_map_distance(nil), do: nil + defp format_map_distance(d), do: Decimal.to_float(d) + + defp format_map_timestamp(nil), do: nil + defp format_map_timestamp(ts), do: Calendar.strftime(ts, "%Y-%m-%d %H:%M") + + defp dedup_map_reciprocals(contacts) do + Enum.uniq_by(contacts, fn [_lat1, _lon1, _lat2, _lon2, band, s1, s2, _mode, _dist, ts, _id] -> + stations = Enum.sort([s1 || "", s2 || ""]) + hour = if ts, do: String.slice(ts, 0..12), else: "" + {stations, band, hour} + end) + end + @spec list_contacts(keyword()) :: contact_page() def list_contacts(opts \\ []) do page = max(Keyword.get(opts, :page, 1), 1) @@ -30,7 +124,7 @@ defmodule Microwaveprop.Radio do base_query = maybe_search(Contact, search) - total_entries = Repo.aggregate(base_query, :count) + total_entries = total_entries_for(search, base_query) total_pages = max(ceil(total_entries / @per_page), 1) entries = @@ -95,6 +189,18 @@ defmodule Microwaveprop.Radio do t2 |> NaiveDateTime.truncate(:second) |> Map.take([:year, :month, :day, :hour]) end + defp total_entries_for(search, query) when search in [nil, ""] do + if Application.get_env(:microwaveprop, :cache_contact_count, true) do + Cache.fetch_or_store(@count_cache_key, @count_cache_ttl_ms, fn -> + Repo.aggregate(query, :count) + end) + else + Repo.aggregate(query, :count) + end + end + + defp total_entries_for(_search, query), do: Repo.aggregate(query, :count) + defp maybe_search(query, nil), do: query defp maybe_search(query, ""), do: query @@ -421,12 +527,20 @@ defmodule Microwaveprop.Radio do {{:ok, {lat1, lon1}}, {:ok, {lat2, lon2}}} -> distance = lat1 |> haversine_km(lon1, lat2, lon2) |> round() |> Decimal.new() - changeset - |> Ecto.Changeset.put_change(:pos1, %{"lat" => lat1, "lon" => lon1}) - |> Ecto.Changeset.put_change(:pos2, %{"lat" => lat2, "lon" => lon2}) - |> Ecto.Changeset.put_change(:distance_km, distance) - |> Ecto.Changeset.put_change(:user_submitted, true) - |> Repo.insert() + result = + changeset + |> Ecto.Changeset.put_change(:pos1, %{"lat" => lat1, "lon" => lon1}) + |> Ecto.Changeset.put_change(:pos2, %{"lat" => lat2, "lon" => lon2}) + |> Ecto.Changeset.put_change(:distance_km, distance) + |> Ecto.Changeset.put_change(:user_submitted, true) + |> Repo.insert() + + with {:ok, _} <- result do + Cache.invalidate(@count_cache_key) + Cache.invalidate(@map_payload_cache_key) + end + + result _ -> changeset = Ecto.Changeset.add_error(changeset, :grid1, "could not resolve grid to coordinates") diff --git a/lib/microwaveprop/radio/contact.ex b/lib/microwaveprop/radio/contact.ex index a6f72439..c8614e95 100644 --- a/lib/microwaveprop/radio/contact.ex +++ b/lib/microwaveprop/radio/contact.ex @@ -49,8 +49,8 @@ defmodule Microwaveprop.Radio.Contact do @type t :: %__MODULE__{} - @required_fields ~w(station1 station2 qso_timestamp mode band)a - @optional_fields ~w(grid1 grid2 pos1 pos2 distance_km user_id)a + @required_fields ~w(station1 station2 qso_timestamp band)a + @optional_fields ~w(grid1 grid2 pos1 pos2 distance_km user_id mode)a @spec changeset(t() | Ecto.Changeset.t(), map()) :: Ecto.Changeset.t() def changeset(contact, attrs) do @@ -60,7 +60,7 @@ defmodule Microwaveprop.Radio.Contact do end @submission_fields ~w(station1 station2 qso_timestamp mode band grid1 grid2 submitter_email)a - @submission_required ~w(station1 station2 qso_timestamp mode band grid1 grid2)a + @submission_required ~w(station1 station2 qso_timestamp band grid1 grid2)a @allowed_modes ~w(CW SSB FM FT8 FT4 Q65) @allowed_bands Enum.map(~w(1296 2304 3456 5760 10000 24000 47000 68000 75000 122000 134000 241000), &Decimal.new/1) @@ -74,6 +74,7 @@ defmodule Microwaveprop.Radio.Contact do |> sanitize_callsign(:station2) |> sanitize_grid(:grid1) |> sanitize_grid(:grid2) + |> normalize_blank_mode() |> validate_callsign(:station1) |> validate_callsign(:station2) |> validate_grid_format(:grid1) @@ -86,6 +87,22 @@ defmodule Microwaveprop.Radio.Contact do |> validate_length(:station2, max: 20) end + # Treat empty / whitespace-only mode strings as "not provided" so the column + # ends up NULL instead of failing validate_inclusion. Mode is optional on the + # submission path. + defp normalize_blank_mode(changeset) do + case get_change(changeset, :mode) do + nil -> + changeset + + value -> + case String.trim(value) do + "" -> put_change(changeset, :mode, nil) + trimmed -> put_change(changeset, :mode, trimmed) + end + end + end + # Either a user_id (logged-in submitter) or a submitter_email (anonymous) # must be present so we know who submitted the contact. defp validate_user_or_email(changeset) do diff --git a/lib/microwaveprop/radio/csv_import.ex b/lib/microwaveprop/radio/csv_import.ex index eef52187..e23df1d4 100644 --- a/lib/microwaveprop/radio/csv_import.ex +++ b/lib/microwaveprop/radio/csv_import.ex @@ -29,8 +29,8 @@ defmodule Microwaveprop.Radio.CsvImport do @dedup_window_seconds 3600 - @expected_columns 7 - @columns ~w(station1 station2 grid1 grid2 band mode qso_timestamp) + @columns_with_mode ~w(station1 station2 grid1 grid2 band mode qso_timestamp) + @columns_without_mode ~w(station1 station2 grid1 grid2 band qso_timestamp) @doc """ Parse-and-insert in one shot. Kept for backward compatibility — does NOT @@ -156,19 +156,29 @@ defmodule Microwaveprop.Radio.CsvImport do defp parse_row(line, row_num, submitter_email) do fields = parse_csv_fields(line) - if length(fields) == @expected_columns do - attrs = - @columns - |> Enum.zip(fields) - |> Map.new() - |> Map.put("submitter_email", submitter_email) + case columns_for(length(fields)) do + nil -> + {:invalid, + %{ + row_num: row_num, + messages: ["expected 6 columns (no mode) or 7 columns (with mode), got #{length(fields)}"] + }} - parse_row_with_timestamp(attrs, row_num) - else - {:invalid, %{row_num: row_num, messages: ["expected #{@expected_columns} columns, got #{length(fields)}"]}} + columns -> + attrs = + columns + |> Enum.zip(fields) + |> Map.new() + |> Map.put("submitter_email", submitter_email) + + parse_row_with_timestamp(attrs, row_num) end end + defp columns_for(7), do: @columns_with_mode + defp columns_for(6), do: @columns_without_mode + defp columns_for(_), do: nil + defp parse_row_with_timestamp(attrs, row_num) do case normalize_timestamp(attrs["qso_timestamp"]) do {:ok, iso_timestamp} -> diff --git a/lib/microwaveprop_web/components/core_components.ex b/lib/microwaveprop_web/components/core_components.ex index 24c69632..ac5bacd3 100644 --- a/lib/microwaveprop_web/components/core_components.ex +++ b/lib/microwaveprop_web/components/core_components.ex @@ -235,7 +235,9 @@ defmodule MicrowavepropWeb.CoreComponents do ~H"""