diff --git a/.forgejo/workflows/build-base.yaml b/.forgejo/workflows/build-base.yaml index ef9047bf..74189cc2 100644 --- a/.forgejo/workflows/build-base.yaml +++ b/.forgejo/workflows/build-base.yaml @@ -13,44 +13,79 @@ on: env: REGISTRY: git.mcintire.me - IMAGE_NAME: graham/prop-base + OWNER: graham + IMAGE_NAME: prop-base + DOCKER_CLI_VERSION: '28.5.2' + BUILDX_VERSION: '0.35.0' concurrency: group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: false jobs: build-and-push: name: Build and push base image - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout code - uses: https://code.forgejo.org/actions/checkout@v4 + uses: actions/checkout@v4 - - name: Generate image tag - id: tag + - name: Install Docker CLI and buildx run: | - WGRIB2=$(grep -E '^ARG WGRIB2_VERSION=' Dockerfile.base | head -1 | cut -d= -f2) - G2C=$(grep -E '^ARG G2C_VERSION=' Dockerfile.base | head -1 | cut -d= -f2) - TIMESTAMP=$(date +%s) - TAG="wgrib2-${WGRIB2}-g2c-${G2C}-${TIMESTAMP}" - echo "tag=${TAG}" >> $GITHUB_OUTPUT + set -euo pipefail + curl -fsSL --retry 3 --retry-delay 2 \ + "https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_CLI_VERSION}.tgz" \ + | tar xz --strip-components=1 -C /usr/local/bin docker/docker + # Docker 23+ routes `docker build` through buildx. The static tarball + # ships only the CLI, so the plugin has to be installed separately or + # DOCKER_BUILDKIT=1 fails with "buildx component is missing". + mkdir -p "$HOME/.docker/cli-plugins" + curl -fsSL --retry 3 --retry-delay 2 \ + -o "$HOME/.docker/cli-plugins/docker-buildx" \ + "https://github.com/docker/buildx/releases/download/v${BUILDX_VERSION}/buildx-v${BUILDX_VERSION}.linux-amd64" + chmod +x "$HOME/.docker/cli-plugins/docker-buildx" + docker version + docker buildx version - - name: Build and push base image via Kaniko - uses: http://forgejo:3000/graham/infra/.forgejo/actions/kaniko-build@main - with: - dockerfile: Dockerfile.base - token: ${{ secrets.FORGEJO_TOKEN }} - image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tag: ${{ steps.tag.outputs.tag }} - extra_args: "--build-arg CACHE_BUST=$(date -u +%G-W%V)" + - name: Build and push + env: + REGISTRY_TOKEN: ${{ secrets.FORGEJO_TOKEN }} + run: | + set -euo pipefail - - name: Tag as latest - uses: http://forgejo:3000/graham/infra/.forgejo/actions/kaniko-build@main - with: - dockerfile: Dockerfile.base - token: ${{ secrets.FORGEJO_TOKEN }} - image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tag: latest - extra_args: "" + IMAGE="${REGISTRY}/${OWNER}/${IMAGE_NAME}" + WGRIB2=$(grep -oP '^ARG WGRIB2_VERSION=\K.*' Dockerfile.base | head -1) + G2C=$(grep -oP '^ARG G2C_VERSION=\K.*' Dockerfile.base | head -1) + SHA_TAG="wgrib2-${WGRIB2}-g2c-${G2C}-${GITHUB_SHA::7}" + # ISO year+week, so the weekly cron re-executes the apt layers for + # Debian security updates but same-week reruns reuse the cache. + # This previously travelled via an `extra_args` input the build action + # never declared, and additionally contained a `$(date ...)` that a + # `with:` value never evaluates. It was doubly dead. + CACHE_BUST=$(date -u +%G-W%V) + + echo "$REGISTRY_TOKEN" | docker login "$REGISTRY" -u "$OWNER" --password-stdin + + DOCKER_BUILDKIT=1 docker build \ + --file Dockerfile.base \ + --build-arg "CACHE_BUST=${CACHE_BUST}" \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + --cache-from "${IMAGE}:latest" \ + --tag "${IMAGE}:${SHA_TAG}" \ + --tag "${IMAGE}:latest" \ + . + + docker push "${IMAGE}:${SHA_TAG}" + docker push "${IMAGE}:latest" + + { + echo "### Base image pushed" + echo "" + echo "- \`${IMAGE}:${SHA_TAG}\`" + echo "- \`${IMAGE}:latest\`" + echo "- cache-bust: \`${CACHE_BUST}\`" + } >> "$GITHUB_STEP_SUMMARY" + + docker image rm "${IMAGE}:${SHA_TAG}" "${IMAGE}:latest" 2>/dev/null || true + docker logout "$REGISTRY" || true diff --git a/.forgejo/workflows/build-grid-rs.yaml b/.forgejo/workflows/build-grid-rs.yaml index 44054ce0..e102bb63 100644 --- a/.forgejo/workflows/build-grid-rs.yaml +++ b/.forgejo/workflows/build-grid-rs.yaml @@ -10,7 +10,10 @@ on: env: REGISTRY: git.mcintire.me - IMAGE_NAME: graham/prop-grid-rs + OWNER: graham + IMAGE_NAME: prop-grid-rs + DOCKER_CLI_VERSION: '28.5.2' + BUILDX_VERSION: '0.35.0' concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -19,24 +22,28 @@ concurrency: jobs: build-and-push: name: Test, build, push - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout code - uses: https://code.forgejo.org/actions/checkout@v4 - - - name: Generate image tag - id: tag - run: | - TIMESTAMP=$(date +%s) - SHORT_SHA=$(git rev-parse --short=7 HEAD | cut -c1-7) - TAG="main-${TIMESTAMP}-${SHORT_SHA}" - echo "tag=${TAG}" >> $GITHUB_OUTPUT + uses: actions/checkout@v4 - name: Install Rust toolchain run: | - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable - echo "$HOME/.cargo/bin" >> $GITHUB_PATH + set -euo pipefail + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ + | sh -s -- -y --default-toolchain stable + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + rust/prop_grid_rs/target + key: ${{ runner.os }}-cargo-${{ hashFiles('rust/prop_grid_rs/Cargo.lock') }} + restore-keys: ${{ runner.os }}-cargo- - name: Run clippy and tests working-directory: rust/prop_grid_rs @@ -44,19 +51,51 @@ jobs: cargo clippy --all-targets -- -D warnings cargo test --release - - name: Build and push via Kaniko - uses: http://forgejo:3000/graham/infra/.forgejo/actions/kaniko-build@main - with: - context: rust/prop_grid_rs - token: ${{ secrets.FORGEJO_TOKEN }} - image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tag: ${{ steps.tag.outputs.tag }} + - name: Install Docker CLI and buildx + run: | + set -euo pipefail + curl -fsSL --retry 3 --retry-delay 2 \ + "https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_CLI_VERSION}.tgz" \ + | tar xz --strip-components=1 -C /usr/local/bin docker/docker + # Docker 23+ routes `docker build` through buildx. The static tarball + # ships only the CLI, so the plugin has to be installed separately or + # DOCKER_BUILDKIT=1 fails with "buildx component is missing". + mkdir -p "$HOME/.docker/cli-plugins" + curl -fsSL --retry 3 --retry-delay 2 \ + -o "$HOME/.docker/cli-plugins/docker-buildx" \ + "https://github.com/docker/buildx/releases/download/v${BUILDX_VERSION}/buildx-v${BUILDX_VERSION}.linux-amd64" + chmod +x "$HOME/.docker/cli-plugins/docker-buildx" + docker version + docker buildx version - - name: Tag as latest and main - uses: http://forgejo:3000/graham/infra/.forgejo/actions/kaniko-build@main - with: - context: rust/prop_grid_rs - token: ${{ secrets.FORGEJO_TOKEN }} - image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tag: latest - extra_args: "" + - name: Build and push + env: + REGISTRY_TOKEN: ${{ secrets.FORGEJO_TOKEN }} + run: | + set -euo pipefail + + IMAGE="${REGISTRY}/${OWNER}/${IMAGE_NAME}" + SHA_TAG="main-$(date -u +%s)-${GITHUB_SHA::7}" + + echo "$REGISTRY_TOKEN" | docker login "$REGISTRY" -u "$OWNER" --password-stdin + + DOCKER_BUILDKIT=1 docker build \ + --file rust/prop_grid_rs/Dockerfile \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + --cache-from "${IMAGE}:latest" \ + --tag "${IMAGE}:${SHA_TAG}" \ + --tag "${IMAGE}:latest" \ + rust/prop_grid_rs + + docker push "${IMAGE}:${SHA_TAG}" + docker push "${IMAGE}:latest" + + { + echo "### Image pushed" + echo "" + echo "- \`${IMAGE}:${SHA_TAG}\`" + echo "- \`${IMAGE}:latest\`" + } >> "$GITHUB_STEP_SUMMARY" + + docker image rm "${IMAGE}:${SHA_TAG}" "${IMAGE}:latest" 2>/dev/null || true + docker logout "$REGISTRY" || true diff --git a/.forgejo/workflows/build.yaml b/.forgejo/workflows/build.yaml index 73b50116..09c85e75 100644 --- a/.forgejo/workflows/build.yaml +++ b/.forgejo/workflows/build.yaml @@ -7,7 +7,10 @@ on: env: REGISTRY: git.mcintire.me - IMAGE_NAME: graham/prop + OWNER: graham + IMAGE_NAME: prop + DOCKER_CLI_VERSION: '28.5.2' + BUILDX_VERSION: '0.35.0' concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -16,24 +19,35 @@ concurrency: jobs: build-and-push: name: Build and Push Docker Image - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout code - uses: https://code.forgejo.org/actions/checkout@v4 + uses: actions/checkout@v4 - - name: Generate image tag - id: tag + - name: Install Docker CLI and buildx + # The runner automounts the host Docker socket, so we only need the + # client. No Kaniko, no build action. run: | - TIMESTAMP=$(date +%s) - SHORT_SHA=$(git rev-parse --short=7 HEAD | cut -c1-7) - TAG="main-${TIMESTAMP}-${SHORT_SHA}" - echo "tag=${TAG}" >> $GITHUB_OUTPUT + set -euo pipefail + curl -fsSL --retry 3 --retry-delay 2 \ + "https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_CLI_VERSION}.tgz" \ + | tar xz --strip-components=1 -C /usr/local/bin docker/docker + # Docker 23+ routes `docker build` through buildx. The static tarball + # ships only the CLI, so the plugin has to be installed separately or + # DOCKER_BUILDKIT=1 fails with "buildx component is missing". + mkdir -p "$HOME/.docker/cli-plugins" + curl -fsSL --retry 3 --retry-delay 2 \ + -o "$HOME/.docker/cli-plugins/docker-buildx" \ + "https://github.com/docker/buildx/releases/download/v${BUILDX_VERSION}/buildx-v${BUILDX_VERSION}.linux-amd64" + chmod +x "$HOME/.docker/cli-plugins/docker-buildx" + docker version + docker buildx version - name: Verify compilation run: | docker run --rm \ - -v ${{ github.workspace }}:/app \ + -v "${{ github.workspace }}:/app" \ -w /app \ -e MIX_ENV=prod \ docker.io/hexpm/elixir:1.20.1-erlang-29.0.2-debian-trixie-20260518-slim \ @@ -45,17 +59,35 @@ jobs: mix assets.setup && \ mix compile --warnings-as-errors' - - name: Build and Push via Kaniko - uses: http://forgejo:3000/graham/infra/.forgejo/actions/kaniko-build@main - with: - token: ${{ secrets.FORGEJO_TOKEN }} - image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tag: ${{ steps.tag.outputs.tag }} + - name: Build and push + env: + REGISTRY_TOKEN: ${{ secrets.FORGEJO_TOKEN }} + run: | + set -euo pipefail - - name: Tag as latest - uses: http://forgejo:3000/graham/infra/.forgejo/actions/kaniko-build@main - with: - token: ${{ secrets.FORGEJO_TOKEN }} - image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tag: latest - extra_args: "" + IMAGE="${REGISTRY}/${OWNER}/${IMAGE_NAME}" + SHA_TAG="main-$(date -u +%s)-${GITHUB_SHA::7}" + + echo "$REGISTRY_TOKEN" | docker login "$REGISTRY" -u "$OWNER" --password-stdin + + # One build, both tags. This was previously two invocations of a build + # action, i.e. the whole image was compiled twice just to add `latest`. + DOCKER_BUILDKIT=1 docker build \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + --cache-from "${IMAGE}:latest" \ + --tag "${IMAGE}:${SHA_TAG}" \ + --tag "${IMAGE}:latest" \ + . + + docker push "${IMAGE}:${SHA_TAG}" + docker push "${IMAGE}:latest" + + { + echo "### Image pushed" + echo "" + echo "- \`${IMAGE}:${SHA_TAG}\`" + echo "- \`${IMAGE}:latest\`" + } >> "$GITHUB_STEP_SUMMARY" + + docker image rm "${IMAGE}:${SHA_TAG}" "${IMAGE}:latest" 2>/dev/null || true + docker logout "$REGISTRY" || true diff --git a/CLAUDE.md b/CLAUDE.md index 24fe49df..02123b4b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -21,7 +21,7 @@ Microwaveprop is a Phoenix 1.8 web application for the North Texas Microwave Soc ### Key Data Sources - **HRRR** (High-Resolution Rapid Refresh) — 3 km NWP model from NOAA AWS S3. Surface + pressure level profiles. Analysis + 18-hour forecasts. -- **HRDPS** (High Resolution Deterministic Prediction System) — 2.5 km Canadian NWP model from MSC Datamart. 4×/day at 00/06/12/18Z, 48h forecasts. Coverage capped at 60°N for v1 (SRTM stops there). `HrdpsClient` + scaffolding shipped 2026-04-29; full pipeline activates once the Rust `prop-grid-rs` HRDPS branch ships. Plan: `docs/plans/2026-04-29-hrdps-canadian-prop-grid.md`. +- **HRDPS** (High Resolution Deterministic Prediction System) — 2.5 km Canadian NWP model from MSC Datamart. 4×/day at 00/06/12/18Z, 48h forecasts. Coverage capped at 60°N for v1 (SRTM stops there). Cron active at `config/runtime.exs:273` (`{"5,25,45 * * * *", HrdpsGridWorker}`). `.hrdps.prop` files accumulate on `/data` with no prune coverage — see Known cleanup gaps. - **ASOS** (Automated Surface Observing System) — Surface weather via Iowa Environmental Mesonet. - **RAOB** (Radiosonde) — Upper-air soundings via IEM. - **IEMRE** — Gridded hourly reanalysis at 0.125° resolution. @@ -79,12 +79,14 @@ mix assets.deploy # minified + digest ### Directory Structure - `lib/microwaveprop/` — Business logic (contexts, schemas, workers) - - `propagation/` — Scoring engine: `scorer.ex`, `band_config.ex`, `grid.ex`, `model.ex` (ML), `freshness_monitor.ex` + - `propagation/` — Scoring engine: `scorer.ex`, `band_config.ex`, `grid.ex`, `duct.ex`, `mechanism_classifier.ex`, `freshness_monitor.ex` - `weather/` — Data ingestion: `hrrr_client.ex`, `iem_client.ex`, `sounding_params.ex`, GRIB2 decoder - `terrain/` — Path analysis: `terrain_analysis.ex` (ITU-R P.526-16), `viewshed.ex`, `elevation_client.ex` - `radio/` — QSO schema, Maidenhead grid conversion - `commercial/` — SNMP polling for commercial microwave links - `workers/` — Oban background jobs for all data pipelines + - `backtest/` — Propagation score calibration and comparison tooling (~413 lines) + - `pskr/` — PSK Reporter spot ingestion and calibration sampling (~250-300+ lines/file, 4 files) - `lib/microwaveprop_web/` — Web layer - `live/map_live.ex` — Main propagation map with forecast timeline - `live/submit_live.ex` — QSO submission form @@ -108,7 +110,7 @@ mix assets.deploy # minified + digest | `commercial` | PollWorker | SNMP polling of commercial links | | `solar` | SolarIndexWorker | Daily solar indices | | `enqueue` | QsoWeatherEnqueueWorker | Batch enqueue enrichment jobs (dev cron only) | -| `hrdps` | HrdpsGridWorker | Canadian propagation chain seed (4×/day). Dormant until Rust `prop-grid-rs` HRDPS branch ships — module exists, cron entry deferred. | +| `hrdps` | HrdpsGridWorker | Canadian propagation chain seed (3×/hour at :05,:25,:45). Active — produces `.hrdps.prop` files on `/data/scores/`. | **Production** runs: propagation, commercial, solar, weather, hrrr, terrain, iemre queues. No cron backfill — enrichment is triggered by QSO submission only. @@ -140,13 +142,9 @@ Humidity effect reverses by frequency: beneficial at 10 GHz (refractivity), harm - Dynamic k-factor from HRRR refractivity gradient - Profiles stored per QSO path (58k+ paths analyzed) -### ML Model (Nx/Axon/EXLA) +### ML Model (Nx/Axon/EXLA) — Removed -Skeleton feed-forward network for future propagation prediction: -- 13 features: 8 atmospheric + 4 cyclical temporal + 1 log-frequency -- Architecture: Dense(64) → Dropout(0.2) → Dense(32) → Dropout(0.1) → Dense(1, sigmoid) -- Weights saved to `priv/models/propagation_v1.nx` -- Not yet trained — scaffolding only +The ML model scaffolding (`lib/microwaveprop/propagation/model.ex`) and its Nx/Axon/EXLA deps (`:nx`, `:axon`, `:exla` in `mix.exs` dev/test only) were removed. `priv/models/propagation_v1.nx` is an orphaned artifact with no remaining code path. Future ML work should start from a fresh branch. ### Data Flow @@ -210,7 +208,7 @@ User submits QSO → enqueue_for_qso() → weather/hrrr/terrain/iemre workers ### Testing - Use `start_supervised!/1` for process cleanup - Use `Process.monitor/1` + `assert_receive {:DOWN, ...}` instead of `Process.sleep` -- Use `LazyHTML` selectors for DOM assertions, never raw HTML matching +- Use `html =~ "..."` / `refute html =~ "..."` for DOM assertions in tests (raw string matching is the project's actual pattern — 570+ assertions across 29 test files). `lazy_html` is declared in `mix.exs` (`only: :test`) but not currently used in test assertions. - Test against element IDs defined in templates - Stub HTTP calls with `Req.Test.stub` in test setup - Oban runs inline in test (`testing: :inline`) diff --git a/Dockerfile b/Dockerfile index 1863a5ac..6b209ceb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,45 +1,23 @@ # syntax=docker/dockerfile:1.6 -# Find eligible builder and runner images on Docker Hub. We use Ubuntu/Debian -# instead of Alpine to avoid DNS resolution issues in production. # -# https://hub.docker.com/r/hexpm/elixir/tags?name=ubuntu -# https://hub.docker.com/_/ubuntu/tags +# Builder: elixir-base (prebuilt hexpm/elixir + build-essential + git + hex + rebar) +# https://git.mcintire.me/graham/elixir-base +# Runtime: prop-base (prebuilt wgrib2 + cdo + gdal-bin, see Dockerfile.base) # -# This file is based on these images: -# -# - https://hub.docker.com/r/hexpm/elixir/tags - for the build image -# - git.mcintire.me/gmcintire/prop-base - prebuilt runtime base (see Dockerfile.base) -# - Ex: docker.io/hexpm/elixir:1.19.5-erlang-28.4.1-debian-trixie-20260316-slim -# -ARG ELIXIR_VERSION=1.20.1 -ARG OTP_VERSION=29.0.2 -ARG DEBIAN_VERSION=trixie-20260518-slim - -ARG BUILDER_IMAGE="docker.io/hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-debian-${DEBIAN_VERSION}" +ARG BUILDER_IMAGE="git.mcintire.me/graham/elixir-base:latest" # Prebuilt runtime base contains wgrib2 + cdo + gdal-bin + locale + the -# BEAM runtime apt deps. Built by .forgejo/workflows/build-base.yaml -# only when Dockerfile.base changes, so this Dockerfile pulls a -# ready-made layer instead of recompiling wgrib2 + reinstalling cdo -# (~10 min) on every push. +# BEAM runtime apt deps. ARG BASE_IMAGE="git.mcintire.me/gmcintire/prop-base:latest" # ---- Elixir build stage ---- FROM ${BUILDER_IMAGE} AS builder -# install build dependencies -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ - rm -f /etc/apt/apt.conf.d/docker-clean \ - && apt-get update \ - && apt-get install -y --no-install-recommends build-essential git +# build-essential, git, curl, ca-certificates, hex, and rebar are +# already installed in the elixir-base image. # prepare build dir WORKDIR /app -# install hex + rebar -RUN mix local.hex --force \ - && mix local.rebar --force - # set build ENV ENV MIX_ENV="prod" diff --git a/Dockerfile.base b/Dockerfile.base index a3638078..b3b63a2d 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -1,24 +1,23 @@ # syntax=docker/dockerfile:1.6 # # Pre-built runtime base image for the Elixir app. Contains: -# * Debian trixie-slim with locale set to en_US.UTF-8 -# * Runtime apt deps for the BEAM + Erlang ssl + snmp +# * elixir-base (hexpm/elixir + build-essential + git + hex + rebar) # * `cdo` + `gdal-bin` (~1 GB dep tree, the slowest install) # * `wgrib2` (compiled here from source against NCEPLIBS-g2c v2.3.0) # -# Built by `.forgejo/workflows/build-base.yaml` only when this file or -# the workflow changes. The app `Dockerfile` does `FROM prop-base` so -# everyday code pushes skip the wgrib2 compile and apt-install entirely. +# Built by CI only when this file or the workflow changes. +# The app `Dockerfile` does `FROM prop-base` so everyday code pushes +# skip the wgrib2 compile and apt-install entirely. # # Bumping wgrib2 or g2c: change the ARGs below and push — the base # workflow rebuilds and re-tags `:latest`. The app Dockerfile follows # `:latest`, so the next app push picks it up automatically. -ARG DEBIAN_VERSION=trixie-20260518-slim -ARG RUNNER_IMAGE="docker.io/debian:${DEBIAN_VERSION}" +ARG ELIXIR_BASE_IMAGE="git.mcintire.me/graham/elixir-base:latest" -# ---- wgrib2 build stage (cached independently inside this image) ---- -FROM ${RUNNER_IMAGE} AS wgrib2-builder +# ---- wgrib2 build stage (cached independently, uses raw debian) ---- +ARG DEBIAN_VERSION=trixie-20260518-slim +FROM docker.io/debian:${DEBIAN_VERSION} AS wgrib2-builder ARG WGRIB2_VERSION=3.8.0 # wgrib2 3.6.0 has a memory-corruption bug exposed by HRDPS rotated @@ -68,7 +67,10 @@ RUN wget -q --content-disposition "https://github.com/NOAA-EMC/wgrib2/archive/re && strip /usr/local/bin/wgrib2 # ---- Final runtime base ---- -FROM ${RUNNER_IMAGE} AS base +# elixir-base already carries Elixir + Erlang + BEAM runtime apt deps +# (libstdc++6, openssl, libncurses6, libsctp1, ca-certificates). +# We layer cdo + gdal-bin + wgrib2 deps + locale on top. +FROM ${ELIXIR_BASE_IMAGE} AS base # Weekly cron passes CACHE_BUST= so the apt layers below # re-execute and pick up Debian security patches. Source-controlled @@ -83,18 +85,17 @@ FROM ${RUNNER_IMAGE} AS base # re-trigger the build-base workflow and re-upload the layers. ARG CACHE_BUST=none -# Runtime apt deps. cdo + gdal-bin are split into a second RUN so a -# transient cdo install failure doesn't invalidate the smaller base -# layer above. Referencing `${CACHE_BUST}` in the RUN line changes the -# layer hash week-to-week without altering the install set. +# Extra runtime apt deps: snmp (app), wgrib2 deps (gfortran, aec, zlib, +# png, openjpeg), plus cdo + gdal-bin. +# cdo + gdal-bin are split into a second RUN so a transient cdo install +# failure doesn't invalidate the smaller base layer above. RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ echo "cache-bust=${CACHE_BUST}" \ && rm -f /etc/apt/apt.conf.d/docker-clean \ && apt-get update \ && apt-get install -y --no-install-recommends \ - libstdc++6 openssl libncurses6 locales ca-certificates snmp \ - libgfortran5 libaec0 zlib1g libpng16-16t64 libopenjp2-7 libsctp1 + locales snmp libgfortran5 libaec0 zlib1g libpng16-16t64 libopenjp2-7 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ diff --git a/Makefile b/Makefile index 8823cf40..34acad57 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: precommit format test deps credo +.PHONY: precommit format test deps credo audit hex-audit cargo-audit precommit: @EXIT=0; \ @@ -8,6 +8,17 @@ precommit: $(MAKE) credo || EXIT=1; \ exit $$EXIT +# Security audit targets — run independently (network-dependent, not in precommit). +# Add these to CI pipeline in addition to precommit. + +audit: hex-audit cargo-audit + +hex-audit: + MIX_ENV=test mix hex.audit + +cargo-audit: + cargo audit --file rust/prop_grid_rs/Cargo.lock + format: MIX_ENV=test mix format --check-formatted diff --git a/config/config.exs b/config/config.exs index 6888f641..8d6383b2 100644 --- a/config/config.exs +++ b/config/config.exs @@ -158,6 +158,10 @@ config :microwaveprop, ecto_repos: [Microwaveprop.Repo], generators: [timestamp_type: :utc_datetime, binary_id: true] +# Log-filtered parameters — values matching these keys are scrubbed from +# request logs by Phoenix.Logger. Extends the default ~w(password)[a]. +config :phoenix, :filter_parameters, ~w(password token api_key secret community authorization) + # Use EXLA as default Nx backend for accelerated tensor operations if Mix.env() in [:dev, :test] do config :nx, :default_backend, EXLA.Backend diff --git a/config/runtime.exs b/config/runtime.exs index 90bff9bc..8cf58641 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -42,6 +42,9 @@ end config :microwaveprop, MicrowavepropWeb.Endpoint, http: [port: String.to_integer(System.get_env("PORT", "4000"))], + # WARNING: The signing_salt default is checked into git. Override via + # LIVE_VIEW_SIGNING_SALT env var in production — a leaked default would + # allow forging of LiveView session tokens. live_view: [signing_salt: System.get_env("LIVE_VIEW_SIGNING_SALT", "Gdq36xze")] # MicrowavepropWeb.Plugs.RemoteIp will only honour `Cf-Connecting-Ip` / @@ -388,7 +391,8 @@ if config_env() == :prod do tls_options: [ server_name_indication: String.to_charlist(email_server || ""), versions: [:"tlsv1.2", :"tlsv1.3"], - verify: :verify_none + verify: :verify_peer, + cacerts: :public_key.cacerts_get() ], auth: :always diff --git a/k8s/secret.example.yaml b/k8s/secret.example.yaml new file mode 100644 index 00000000..5efba8eb --- /dev/null +++ b/k8s/secret.example.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Secret +metadata: + name: prop-secrets + namespace: prop +type: Opaque +stringData: + DATABASE_URL: "postgres://CHANGE_ME" + SECRET_KEY_BASE: "CHANGE_ME" + EMAIL_SERVER: "mail.smtp2go.com" + EMAIL_USERNAME: "CHANGE_ME" + EMAIL_PASSWORD: "CHANGE_ME" + ERA5_CDS_API_KEY: "CHANGE_ME" + ERA5_CDS_URL: "https://cds.climate.copernicus.eu/api" + QRZ_USERNAME: "CHANGE_ME" + QRZ_PASSWORD: "CHANGE_ME" + GOOGLE_API_KEY: "CHANGE_ME" + # Prometheus auth token — if set, /metrics requires a matching bearer + # token via MicrowavepropWeb.MetricsPlug. Required in production (plug + # denies all requests when unset in prod). + PROMETHEUS_AUTH_TOKEN: "CHANGE_ME" + # LiveView signing salt — override the checked-in default "Gdq36xze". + # A leaked default allows forging of LiveView session tokens. + LIVE_VIEW_SIGNING_SALT: "CHANGE_ME" + # Valkey/Redis backend for Microwaveprop.Weather.GridCache. Single + # cluster-wide cache replaces per-pod ETS replicas. Unset → ETS fallback. + VALKEY_URL: "redis://CHANGE_ME" diff --git a/lib/microwaveprop/commercial/snmp_client.ex b/lib/microwaveprop/commercial/snmp_client.ex index ab58030c..4bfb76a6 100644 --- a/lib/microwaveprop/commercial/snmp_client.ex +++ b/lib/microwaveprop/commercial/snmp_client.ex @@ -53,6 +53,13 @@ defmodule Microwaveprop.Commercial.SnmpClient do @spec poll_af11x(String.t(), String.t()) :: {:ok, map()} | {:error, term()} def poll_af11x(host, community) do + with :ok <- validate_host(host), + :ok <- validate_community(community) do + do_poll_af11x(host, community) + end + end + + defp do_poll_af11x(host, community) do oids = Map.keys(@af11x_oids) args = ["-v1", "-c", community, "-t", "5", "-r", "1", host | oids] @@ -69,6 +76,13 @@ defmodule Microwaveprop.Commercial.SnmpClient do @spec poll_af60(String.t(), String.t()) :: {:ok, map()} | {:error, term()} def poll_af60(host, community) do + with :ok <- validate_host(host), + :ok <- validate_community(community) do + do_poll_af60(host, community) + end + end + + defp do_poll_af60(host, community) do static_oids = Map.keys(@af60_static_oids) static_args = ["-v1", "-c", community, "-t", "5", "-r", "1", host | static_oids] @@ -125,6 +139,31 @@ defmodule Microwaveprop.Commercial.SnmpClient do # --- Private --- + # Reject community strings that start with "-" (prevents flag injection + # via System.cmd argv) and strings with non-printable characters. + defp validate_community(community) do + if String.starts_with?(community, "-") or + not String.valid?(community) or + community =~ ~r/[^[:print:]]/u do + {:error, :invalid_community} + else + :ok + end + end + + # Reject host values that aren't a valid IP address or hostname. + # Prevents SSRF to unexpected destinations via crafted DB values. + @valid_host_pattern ~r/^[a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?)*$/ + @valid_ip_pattern ~r/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/ + + defp validate_host(host) do + if host =~ @valid_host_pattern or host =~ @valid_ip_pattern do + :ok + else + {:error, :invalid_host} + end + end + defp parse_lines(output) do output |> String.split("\n", trim: true) diff --git a/lib/microwaveprop/format.ex b/lib/microwaveprop/format.ex index e2a94a4c..3c0e8477 100644 --- a/lib/microwaveprop/format.ex +++ b/lib/microwaveprop/format.ex @@ -5,6 +5,60 @@ defmodule Microwaveprop.Format do metric. """ + # ── Propagation score tier definitions ── + + @tier_thresholds [ + {80, "EXCELLENT", "badge badge-sm badge-success", "#059669"}, + {65, "GOOD", "badge badge-sm badge-info", "#0d9488"}, + {50, "MARGINAL", "badge badge-sm badge-warning", "#ca8a04"}, + {33, "POOR", "badge badge-sm badge-error", "#ea580c"} + ] + @fallback_tier {"NEGLIGIBLE", "badge badge-sm badge-ghost", "#dc2626"} + + @doc """ + Human-readable label for a propagation composite score (0-100). + """ + @spec propagation_tier_label(number()) :: String.t() + def propagation_tier_label(score) do + Enum.find_value(@tier_thresholds, elem(@fallback_tier, 0), fn {thresh, label, _badge, _color} -> + score >= thresh && label + end) + end + + @doc """ + CSS badge classes for a propagation composite score (0-100). + Includes "badge badge-sm" prefix so callers can use the result as a + bare `class` value. + """ + @spec propagation_tier_badge_class(number()) :: String.t() + def propagation_tier_badge_class(score) do + Enum.find_value(@tier_thresholds, elem(@fallback_tier, 1), fn {thresh, _label, badge, _color} -> + score >= thresh && badge + end) + end + + @doc """ + Hex color (no # prefix) for a propagation composite score (0-100). + """ + @spec propagation_tier_color(number()) :: String.t() + def propagation_tier_color(score) do + Enum.find_value(@tier_thresholds, elem(@fallback_tier, 2), fn {thresh, _label, _badge, color} -> + score >= thresh && color + end) + end + + @doc """ + Tailwind badge color class for a terrain analysis verdict. + Returns only the color portion ("badge-success", "badge-warning", + etc.) — callers append "badge" / "badge-sm" as needed. + """ + @spec terrain_verdict_class(String.t()) :: String.t() + def terrain_verdict_class("CLEAR"), do: "badge-success" + def terrain_verdict_class("FRESNEL_MINOR"), do: "badge-warning" + def terrain_verdict_class("FRESNEL_PARTIAL"), do: "badge-warning" + def terrain_verdict_class("BLOCKED"), do: "badge-error" + def terrain_verdict_class(_), do: "badge-ghost" + @doc """ Format a distance in km as "X mi (Y km)". Under 10 miles the output carries one decimal so near-LOS paths don't round to zero; longer diff --git a/lib/microwaveprop/propagation/grid_task.ex b/lib/microwaveprop/propagation/grid_task.ex new file mode 100644 index 00000000..80cdbe91 --- /dev/null +++ b/lib/microwaveprop/propagation/grid_task.ex @@ -0,0 +1,38 @@ +defmodule Microwaveprop.Propagation.GridTask do + @moduledoc """ + Schema for the `grid_tasks` table — a cross-language shared queue between + Elixir's `PropagationGridWorker` (seeder) and the Rust `prop-grid-rs` + worker (consumer). + + **This schema does NOT own the table.** The table is created by Ecto + migrations but rows are written by Elixir seeders (`GridTaskEnqueuer`) + and consumed/updated by Rust via direct SQL (`SELECT ... FOR UPDATE + SKIP LOCKED`). No Ecto changeset is provided — the schema exists + purely for compile-time field-name checking on Ecto queries, so that + `from(t in GridTask, where: t.status == "queued")` catches typos + (`t.stauts`) that are invisible in the legacy `from(t in "grid_tasks", ...)` + string-table queries. + """ + + use Ecto.Schema + + @type t :: %__MODULE__{} + + @primary_key {:id, :binary_id, autogenerate: false} + @foreign_key_type :binary_id + + schema "grid_tasks" do + field :run_time, :utc_datetime + field :forecast_hour, :integer + field :valid_time, :utc_datetime + field :status, :string, default: "queued" + field :attempt, :integer, default: 0 + field :claimed_at, :utc_datetime_usec + field :completed_at, :utc_datetime_usec + field :error, :string + field :kind, :string, default: "forecast" + field :source, :string, default: "hrrr" + + timestamps(type: :utc_datetime) + end +end diff --git a/lib/microwaveprop/propagation/notify_listener.ex b/lib/microwaveprop/propagation/notify_listener.ex index c6b05724..e439530c 100644 --- a/lib/microwaveprop/propagation/notify_listener.ex +++ b/lib/microwaveprop/propagation/notify_listener.ex @@ -110,7 +110,7 @@ defmodule Microwaveprop.Propagation.NotifyListener do Public so tests can exercise the path without standing up the Postgrex.Notifications subscriber. """ - @spec handle_propagation_ready(DateTime.t()) :: :ok + @spec handle_propagation_ready(DateTime.t()) :: {:ok, pid()} def handle_propagation_ready(valid_time) do {past, future} = Propagation.hot_cache_window() ScoreCache.prune_outside_window(past, future) @@ -119,13 +119,13 @@ defmodule Microwaveprop.Propagation.NotifyListener do # don't accumulate between pipeline runs. Propagation.retain_scores_window(valid_time) - kickoff_scalar_materialization(valid_time) + {:ok, task_pid} = kickoff_scalar_materialization(valid_time) _ = Phoenix.PubSub.broadcast(@pubsub, @topic, {:propagation_updated, [valid_time]}) Logger.info("NotifyListener: broadcast propagation_updated for valid_time=#{valid_time}") - :ok + {:ok, task_pid} end # Kick off `Weather.materialize_scalar_file/1` in a detached Task so the @@ -135,7 +135,7 @@ defmodule Microwaveprop.Propagation.NotifyListener do # Spawn in a try/rescue so a crash in the materializer does not # propagate through the linked Task and kill the NotifyListener # GenServer. - _ = + {:ok, task_pid} = Task.start(fn -> try do Weather.materialize_scalar_file(valid_time) @@ -145,6 +145,6 @@ defmodule Microwaveprop.Propagation.NotifyListener do end end) - :ok + {:ok, task_pid} end end diff --git a/lib/microwaveprop/propagation/path_analysis.ex b/lib/microwaveprop/propagation/path_analysis.ex new file mode 100644 index 00000000..d75eb6aa --- /dev/null +++ b/lib/microwaveprop/propagation/path_analysis.ex @@ -0,0 +1,601 @@ +defmodule Microwaveprop.Propagation.PathAnalysis do + @moduledoc """ + Path-level analysis for contact display: duct detection, propagation + mechanism explanation, data source summaries, and elevation profiles. + + Wraps Propagation.Duct and Propagation.MechanismClassifier for the + algorithm-heavy work while providing display-ready results for the + ContactLive.Show LiveView. + + Extracted from ContactLive.Show (2026-07) to eliminate ~750 lines of + duplicated duct-detection and mechanism-classification logic. + """ + + alias Microwaveprop.Terrain.ElevationClient + alias Microwaveprop.Terrain.TerrainAnalysis + + require Logger + + # ── Gradient thresholds (centralized — shared with Propagation.Duct) ── + # ITU-R P.453 / P.834 definitions: + # dN/dh < -157 N/km → ducting (M-profile decreases with height) + # dN/dh < -100 N/km → super-refractive + # dN/dh < -79 N/km → enhanced refraction + # dN/dh ≥ -79 N/km → standard atmosphere + @ducting_threshold -157 + @super_refractive_threshold -100 + @enhanced_refraction_threshold -79 + + @earth_radius_m 6_371_000.0 + + # ── Public API ─────────────────────────────────────────────────────── + + @doc """ + Compute the elevation profile and duct layers for a contact path. + + Returns a map with `:points`, `:freq_mhz`, `:dist_km`, `:k_factor`, + `:fwd_az`, `:rev_az`, `:fwd_el`, `:rev_el`, `:verdict`, + `:first_obstruction_km`, and `:ducts`, or nil when coordinates are + unavailable or elevation data cannot be fetched. + """ + @spec compute_elevation_profile(map(), list(), list()) :: map() | nil + def compute_elevation_profile(contact, hrrr_path, soundings) do + with %{"lat" => lat1} <- contact.pos1, + lon1 when is_number(lon1) <- contact.pos1["lon"], + %{"lat" => lat2} <- contact.pos2, + lon2 when is_number(lon2) <- contact.pos2["lon"], + {:ok, profile} <- safe_fetch_elevation(lat1, lon1, lat2, lon2) do + freq_ghz = band_to_ghz(contact.band) + dist_km = Microwaveprop.Geo.haversine_km(lat1, lon1, lat2, lon2) + # Default to 10 ft (3.048 m) AGL when the operator didn't record an + # antenna height — better than pretending the antenna is on the dirt. + default_ht_m = 3.048 + ant_ht_a_m = ft_to_m(contact.height1_ft) || default_ht_m + ant_ht_b_m = ft_to_m(contact.height2_ft) || default_ht_m + analysis = TerrainAnalysis.analyse(profile, dist_km, freq_ghz, ant_ht_a: ant_ht_a_m, ant_ht_b: ant_ht_b_m) + + fwd_az = initial_bearing(lat1, lon1, lat2, lon2) + rev_az = initial_bearing(lat2, lon2, lat1, lon1) + + tx_elev = hd(profile).elev + ant_ht_a_m + rx_elev = List.last(profile).elev + ant_ht_b_m + fwd_el = elevation_angle(tx_elev, rx_elev, dist_km * 1000) + rev_el = elevation_angle(rx_elev, tx_elev, dist_km * 1000) + + first_obs = + case Enum.find(analysis.points, & &1.obstructed) do + nil -> nil + p -> p.dist_km + end + + ducts = + hrrr_path + |> extract_ducts(soundings, tx_elev) + |> merge_nearby_ducts() + |> Enum.sort_by(& &1.strength, :desc) + |> Enum.take(3) + |> mark_likely_duct(tx_elev, rx_elev) + + %{ + points: + Enum.map(analysis.points, fn p -> + %{elev: p.elev, beam: p.beam, r1: p.r1, dist_km: p.dist_km} + end), + freq_mhz: freq_ghz * 1000, + dist_km: dist_km, + k_factor: analysis.k_factor, + fwd_az: fwd_az, + rev_az: rev_az, + fwd_el: fwd_el, + rev_el: rev_el, + verdict: analysis.verdict, + first_obstruction_km: first_obs, + ducts: ducts + } + else + _ -> nil + end + end + + @doc """ + Build a summary map of the data sources available for this path. + """ + @spec build_data_sources(map() | nil, list(), map() | nil, map(), map() | nil) :: map() + def build_data_sources(hrrr, hrrr_path, terrain, weather, elevation_profile) do + %{ + hrrr: build_hrrr_source(hrrr, hrrr_path), + elevation: build_elevation_source(elevation_profile), + terrain: build_terrain_source(terrain), + obs_count: length(weather.surface_observations), + sounding_count: length(weather.soundings) + } + end + + @doc """ + Build the propagation summary text for the contact. + """ + @spec build_summary(map() | nil, map() | nil, map() | nil, list(), float() | nil, integer(), map()) :: + String.t() + def build_summary(terrain, elevation_profile, hrrr, soundings, dist_km, band_mhz, contact) do + terrain_status = terrain_summary(terrain, elevation_profile, contact) + mechanism = propagation_mechanism(terrain, elevation_profile, hrrr, soundings, dist_km) + band_note = band_summary(band_mhz, hrrr) + + [terrain_status, mechanism, band_note] + |> Enum.reject(&is_nil/1) + |> Enum.join(" ") + end + + @doc """ + Build the detail notes list for the propagation analysis panel. + """ + @spec build_details(map() | nil, list(), list(), map() | nil, integer(), map()) :: [String.t()] + def build_details(hrrr, hrrr_path, soundings, elevation_profile, _band_mhz, contact) do + Enum.reject( + [ + antenna_heights_detail(contact), + refractivity_detail(hrrr), + boundary_layer_detail(hrrr), + path_duct_count_detail(hrrr_path), + sounding_ducting_detail(soundings), + duct_layer_detail(elevation_profile) + ], + &is_nil/1 + ) + end + + @doc """ + Human-readable description of a detected duct layer. + """ + @spec duct_description(map()) :: String.t() + def duct_description(%{source: source, base_m_msl: base, top_m_msl: top, strength: strength}) do + base_ft = round(base * 3.28084) + top_ft = round(top * 3.28084) + + src = + case source do + "sounding" -> "sounding-detected" + "inferred" -> "estimated" + _ -> "detected" + end + + "#{src} layer at #{base_ft}-#{top_ft} ft, #{strength} M-units" + end + + def duct_description(_), do: "detected layer" + + @doc """ + Determine the most likely propagation mechanism text for a contact, + considering terrain, ducting conditions, and path length. + """ + @spec propagation_mechanism(map() | nil, map() | nil, map() | nil, list(), float() | nil) :: + String.t() | nil + def propagation_mechanism(terrain, elevation_profile, hrrr, soundings, dist_km) do + blocked? = terrain && terrain.verdict == "BLOCKED" + ducting? = has_ducting?(hrrr, soundings) + enhanced_refraction? = enhanced_refraction?(hrrr) + long_path? = dist_km && dist_km > 100 + + ducts = (elevation_profile && elevation_profile.ducts) || [] + likely_duct = Enum.find(ducts, & &1[:likely]) + + props = %{ + blocked?: blocked?, + ducting?: ducting?, + enhanced_refraction?: enhanced_refraction?, + long_path?: long_path?, + likely_duct: likely_duct, + terrain: terrain, + hrrr: hrrr + } + + if blocked? do + blocked_mechanism(props) + else + clear_path_mechanism(props) + end + end + + @doc """ + Returns true if HRRR or sounding data indicates ducting conditions. + """ + @spec has_ducting?(map() | nil, list()) :: boolean() + def has_ducting?(hrrr, soundings) do + hrrr_ducting = hrrr && hrrr.ducting_detected + sounding_ducting = is_list(soundings) && Enum.any?(soundings, & &1.ducting_detected) + hrrr_ducting || sounding_ducting + end + + @doc """ + Returns true if the HRRR refractivity gradient indicates enhanced + (super-refractive) conditions. + """ + @spec enhanced_refraction?(map() | nil) :: boolean() + def enhanced_refraction?(nil), do: false + + def enhanced_refraction?(hrrr) do + is_number(hrrr.min_refractivity_gradient) && hrrr.min_refractivity_gradient < @super_refractive_threshold + end + + @doc """ + Translates a refractivity gradient (N-units/km) into a human label. + + Uses ITU-R P.834 conventions (same thresholds as Propagation.Duct). + """ + @spec refractivity_label(integer()) :: String.t() + def refractivity_label(g) when g < @ducting_threshold, do: "ducting" + def refractivity_label(g) when g < @super_refractive_threshold, do: "super-refractive" + def refractivity_label(g) when g < @enhanced_refraction_threshold, do: "enhanced" + def refractivity_label(_), do: "normal" + + # ── Duct extraction pipeline ──────────────────────────────────────── + + # Extract duct layers from best available source across all HRRR path profiles: + # 1. HRRR explicit ducts (M-profile detected) from any path point + # 2. Sounding explicit ducts (high vertical resolution, most reliable) + # 3. Inferred from strongest HRRR refractivity gradient along path + defp extract_ducts(hrrr_path, soundings, surface_elev) do + hrrr_ducts = extract_hrrr_ducts(hrrr_path, surface_elev) + + cond do + hrrr_ducts != [] -> + hrrr_ducts + + (sounding_ducts = extract_sounding_ducts(soundings, surface_elev)) != [] -> + sounding_ducts + + true -> + # Use the profile with the strongest (most negative) gradient + best = + hrrr_path + |> Enum.filter(&is_number(&1.min_refractivity_gradient)) + |> Enum.min_by(& &1.min_refractivity_gradient, fn -> nil end) + + infer_duct_from_gradient(best, surface_elev) + end + end + + defp extract_hrrr_ducts(hrrr_path, surface_elev) when is_list(hrrr_path) do + hrrr_path + |> Enum.flat_map(&duct_characteristics_to_maps(&1.duct_characteristics, surface_elev, "hrrr")) + |> Enum.uniq_by(fn d -> {round(d.base_m_msl), round(d.top_m_msl)} end) + end + + defp duct_characteristics_to_maps(ducts, surface_elev, source) when is_list(ducts) and ducts != [] do + Enum.map(ducts, fn d -> + %{ + base_m_msl: surface_elev + (d["base"] || 0), + top_m_msl: surface_elev + (d["top"] || 0), + strength: d["strength"], + source: source + } + end) + end + + defp duct_characteristics_to_maps(_ducts, _surface_elev, _source), do: [] + + defp extract_sounding_ducts(soundings, surface_elev) when is_list(soundings) do + soundings + |> Enum.filter(& &1.ducting_detected) + |> Enum.flat_map(&duct_characteristics_to_maps(&1.duct_characteristics, surface_elev, "sounding")) + |> Enum.uniq_by(fn d -> {round(d.base_m_msl), round(d.top_m_msl)} end) + end + + defp extract_sounding_ducts(_, _), do: [] + + # HRRR pressure levels are too coarse (~250m) to detect thin ducting layers + # via the M-profile method. When min_refractivity_gradient indicates enhanced + # propagation, infer a duct layer within the boundary layer. + defp infer_duct_from_gradient(nil, _surface_elev), do: [] + + defp infer_duct_from_gradient(hrrr, surface_elev) do + grad = hrrr.min_refractivity_gradient + hpbl = hrrr.hpbl_m + + if is_number(grad) and grad < @super_refractive_threshold and is_number(hpbl) and hpbl > 0 do + duct_top = min(hpbl * 0.6, 500) + strength = abs(grad) / 10 + + [ + %{ + base_m_msl: surface_elev, + top_m_msl: surface_elev + duct_top, + strength: Float.round(strength, 1), + source: "inferred" + } + ] + else + [] + end + end + + # Merge duct layers that overlap or are within 100m of each other + defp merge_nearby_ducts(ducts) do + ducts + |> Enum.sort_by(& &1.base_m_msl) + |> Enum.reduce([], fn duct, acc -> + case acc do + [prev | rest] when duct.base_m_msl <= prev.top_m_msl + 100 -> + merged = %{ + prev + | top_m_msl: max(prev.top_m_msl, duct.top_m_msl), + strength: max(prev.strength, duct.strength) + } + + [merged | rest] + + _ -> + [duct | acc] + end + end) + |> Enum.reverse() + end + + # Mark the duct most likely carrying the signal. The signal enters at + # surface level from each end, so the duct whose base is closest to + # the average endpoint elevation is the most probable propagation path. + defp mark_likely_duct([], _tx_elev, _rx_elev), do: [] + + defp mark_likely_duct(ducts, tx_elev, rx_elev) do + avg_elev = (tx_elev + rx_elev) / 2 + + {_, likely_idx} = + ducts + |> Enum.with_index() + |> Enum.min_by(fn {d, _idx} -> + # Distance from average endpoint elevation to the duct layer + # Prefer ducts that the endpoints are actually inside of + if avg_elev >= d.base_m_msl and avg_elev <= d.top_m_msl do + -d.strength + else + min(abs(d.base_m_msl - avg_elev), abs(d.top_m_msl - avg_elev)) + end + end) + + Enum.with_index(ducts, fn d, i -> + Map.put(d, :likely, i == likely_idx) + end) + end + + # ── Elevation fetch helper ────────────────────────────────────────── + + defp safe_fetch_elevation(lat1, lon1, lat2, lon2) do + ElevationClient.fetch_elevation_profile(lat1, lon1, lat2, lon2, 256) + rescue + e -> + Logger.warning("Elevation profile failed: #{Exception.message(e)}") + {:error, :elevation_unavailable} + end + + # ── Conversion helpers ────────────────────────────────────────────── + + defp band_to_ghz(nil), do: 10.0 + + defp band_to_ghz(band) do + band + |> Decimal.to_float() + |> Kernel./(1000) + end + + defp ft_to_m(nil), do: nil + defp ft_to_m(ft) when is_integer(ft), do: ft * 0.3048 + + # ── Geodesy helpers ───────────────────────────────────────────────── + + defp initial_bearing(lat1, lon1, lat2, lon2) do + rlat1 = deg_to_rad(lat1) + rlat2 = deg_to_rad(lat2) + dlon = deg_to_rad(lon2 - lon1) + + y = :math.sin(dlon) * :math.cos(rlat2) + + x = + :math.cos(rlat1) * :math.cos(rlat2) - + :math.cos(rlat1) * :math.sin(rlat2) * :math.cos(dlon) + + bearing = y |> :math.atan2(x) |> rad_to_deg() + Float.round(rem_float(bearing + 360, 360), 1) + end + + defp elevation_angle(h_tx, h_rx, dist_m) do + k_r = 4 / 3 * @earth_radius_m + angle = :math.atan2(h_rx - h_tx, dist_m) - dist_m / (2 * k_r) + Float.round(rad_to_deg(angle), 2) + end + + defp deg_to_rad(deg), do: deg * :math.pi() / 180 + defp rad_to_deg(rad), do: rad * 180 / :math.pi() + defp rem_float(a, b), do: a - Float.floor(a / b) * b + + # ── Data sources building ─────────────────────────────────────────── + + defp build_hrrr_source(nil, _), do: nil + + defp build_hrrr_source(hrrr, hrrr_path) do + levels = if hrrr.profile, do: length(hrrr.profile), else: 0 + + %{ + profile_count: length(hrrr_path), + levels: levels, + valid_time: Calendar.strftime(hrrr.valid_time, "%Y-%m-%d %H:%M UTC"), + lat: :erlang.float_to_binary(hrrr.lat / 1, decimals: 2), + lon: :erlang.float_to_binary(hrrr.lon / 1, decimals: 2) + } + end + + defp build_elevation_source(nil), do: nil + + defp build_elevation_source(ep) do + %{ + samples: length(ep.points), + dist: Microwaveprop.Format.distance_km(ep.dist_km), + source: "SRTM 1-arcsecond", + duct_count: length(ep.ducts) + } + end + + defp build_terrain_source(nil), do: nil + + defp build_terrain_source(terrain) do + %{ + samples: terrain.sample_count, + verdict: terrain.verdict, + max_elev: :erlang.float_to_binary((terrain.max_elevation_m || 0) / 1, decimals: 0), + diffraction: :erlang.float_to_binary((terrain.diffraction_db || 0) / 1, decimals: 1) + } + end + + # ── Terrain summary text ──────────────────────────────────────────── + + defp terrain_summary(nil, nil, _contact), do: "No terrain data available." + + defp terrain_summary(terrain, elevation_profile, contact) do + verdict = (elevation_profile && elevation_profile.verdict) || (terrain && terrain.verdict) + describe_verdict(verdict, elevation_profile, contact) + end + + defp describe_verdict("CLEAR", _, _), do: "Line of sight is clear between stations." + + defp describe_verdict("FRESNEL_MINOR", _, _), do: "Line of sight is clear but with minor Fresnel zone encroachment." + + defp describe_verdict("FRESNEL_PARTIAL", _, _), do: "Line of sight has partial Fresnel zone obstruction." + + defp describe_verdict("BLOCKED", elevation_profile, contact) do + obs_dist = elevation_profile && elevation_profile.first_obstruction_km + blocked_message(obs_dist, contact) + end + + defp describe_verdict(_, _, _), do: nil + + defp blocked_message(nil, _contact), do: "Path is terrain-obstructed." + + defp blocked_message(obs_dist, contact) do + origin = contact && contact.station1 + origin_label = if origin, do: origin, else: "station 1" + + "Path is terrain-obstructed at #{:erlang.float_to_binary(obs_dist * 0.621371, decimals: 1)} mi (#{:erlang.float_to_binary(obs_dist, decimals: 1)} km) from #{origin_label}." + end + + # ── Mechanism text generation ─────────────────────────────────────── + + defp blocked_mechanism(%{ducting?: true, likely_duct: duct}) when not is_nil(duct) do + "Signal likely propagated via atmospheric ducting (#{duct_description(duct)}), bending over the terrain obstruction." + end + + defp blocked_mechanism(%{ducting?: true}) do + "Ducting conditions detected — signal likely propagated through a tropospheric duct, bypassing the terrain obstruction." + end + + defp blocked_mechanism(%{enhanced_refraction?: true, hrrr: hrrr}) do + grad = round(hrrr.min_refractivity_gradient) + + "Enhanced refraction (dN/dh = #{grad} N-units/km) likely bent the signal over the obstruction. Conditions are super-refractive but below full ducting threshold." + end + + defp blocked_mechanism(%{terrain: terrain}) do + diff_db = terrain && terrain.diffraction_db + blocked_diffraction_message(diff_db) + end + + defp blocked_diffraction_message(diff_db) when is_number(diff_db) and diff_db > 0 do + "Path is obstructed with #{:erlang.float_to_binary(diff_db, decimals: 1)} dB diffraction loss. Contact may have been enabled by knife-edge diffraction or tropospheric scatter." + end + + defp blocked_diffraction_message(_) do + "Path is obstructed. Contact likely enabled by tropospheric scatter or transient ducting conditions." + end + + defp clear_path_mechanism(%{ducting?: true, long_path?: true, likely_duct: duct}) when not is_nil(duct) do + "Ducting conditions present (#{duct_description(duct)}) — likely extending range beyond normal line of sight." + end + + defp clear_path_mechanism(%{ducting?: true, long_path?: true}) do + "Ducting conditions present — likely contributing to extended range." + end + + defp clear_path_mechanism(%{enhanced_refraction?: true, long_path?: true}) do + "Enhanced refraction present — may have contributed to extended range." + end + + defp clear_path_mechanism(_), do: nil + + # ── Detail text helpers ───────────────────────────────────────────── + + defp antenna_heights_detail(%{height1_ft: h1, height2_ft: h2} = contact) when is_integer(h1) or is_integer(h2) do + s1 = contact.station1 || "Station 1" + s2 = contact.station2 || "Station 2" + + parts = + Enum.reject( + [if(is_integer(h1), do: "#{s1} @ #{h1} ft AGL"), if(is_integer(h2), do: "#{s2} @ #{h2} ft AGL")], + &is_nil/1 + ) + + "Antenna heights: #{Enum.join(parts, ", ")}." + end + + defp antenna_heights_detail(_), do: nil + + defp path_duct_count_detail(hrrr_path) when is_list(hrrr_path) and hrrr_path != [] do + total = length(hrrr_path) + ducting = Enum.count(hrrr_path, &(&1 && &1.ducting_detected)) + + if ducting > 0 do + "Tropo duct detected at #{ducting}/#{total} HRRR samples along the path." + end + end + + defp path_duct_count_detail(_), do: nil + + defp refractivity_detail(%{min_refractivity_gradient: grad}) when is_number(grad) do + g = round(grad) + label = refractivity_label(g) + "Refractivity gradient: #{g} N-units/km (#{label})." + end + + defp refractivity_detail(_), do: nil + + defp boundary_layer_detail(%{hpbl_m: hpbl}) when is_number(hpbl) do + h = round(hpbl) + note = if h < 300, do: " — shallow boundary layer favors ducting", else: "" + "Boundary layer depth: #{h} m#{note}." + end + + defp boundary_layer_detail(_), do: nil + + defp sounding_ducting_detail(soundings) when is_list(soundings) do + ducting = Enum.filter(soundings, & &1.ducting_detected) + + if ducting == [] do + nil + else + names = Enum.map_join(ducting, ", ", fn s -> s.station.name || s.station.station_code end) + "Ducting detected by soundings at: #{names}." + end + end + + defp sounding_ducting_detail(_), do: nil + + defp duct_layer_detail(%{ducts: ducts}) when is_list(ducts) do + likely = Enum.find(ducts, & &1[:likely]) + if likely, do: "Most likely propagation duct: #{duct_description(likely)}." + end + + defp duct_layer_detail(_), do: nil + + # ── Band-specific summary text ────────────────────────────────────── + + defp band_summary(band_mhz, hrrr) when band_mhz <= 12_000 do + if hrrr && is_number(hrrr.pwat_mm) && hrrr.pwat_mm > 20 do + "At 10 GHz, the elevated moisture (PWAT #{:erlang.float_to_binary(hrrr.pwat_mm, decimals: 1)} mm) enhances refractivity and ducting potential." + end + end + + defp band_summary(band_mhz, hrrr) when band_mhz >= 24_000 do + if hrrr && is_number(hrrr.pwat_mm) && hrrr.pwat_mm > 25 do + "At #{round(band_mhz / 1000)} GHz, high moisture (PWAT #{:erlang.float_to_binary(hrrr.pwat_mm, decimals: 1)} mm) causes significant water vapor absorption." + end + end + + defp band_summary(_, _), do: nil +end diff --git a/lib/microwaveprop/radio.ex b/lib/microwaveprop/radio.ex index e368d82c..6385782f 100644 --- a/lib/microwaveprop/radio.ex +++ b/lib/microwaveprop/radio.ex @@ -1,1278 +1,54 @@ defmodule Microwaveprop.Radio do @moduledoc false - import Ecto.Query - - alias Microwaveprop.Accounts.Scope - alias Microwaveprop.Accounts.User - alias Microwaveprop.Cache - alias Microwaveprop.Radio.BandResolver - alias Microwaveprop.Radio.Contact - alias Microwaveprop.Radio.ContactEdit - alias Microwaveprop.Radio.Maidenhead - alias Microwaveprop.Repo - alias Microwaveprop.Workers.ContactWeatherEnqueueWorker - - @per_page 20 - @max_per_page 200 - @sortable_fields ~w(station1 station2 band mode distance_km qso_timestamp inserted_at)a - @count_cache_key {__MODULE__, :total_contact_count} - @count_cache_ttl_ms 30_000 - @map_payload_cache_key {__MODULE__, :contact_map_payload} - @map_payload_ttl_ms 10 * 60 * 1_000 - - @type contact_page :: %{ - entries: [Contact.t()], - grouped_entries: [{Contact.t(), [Contact.t()]}], - page: pos_integer(), - total_pages: pos_integer(), - total_entries: non_neg_integer() - } - - @doc """ - Pre-encoded JSON payload for `/contacts/map`. Computing this requires loading - every contact with coordinates, deriving formatted rows, deduplicating - reciprocals, and JSON-encoding ~58k rows — ~150-300ms of work that would - otherwise run on every page mount. Cached for 10 minutes, invalidated when - a new contact is inserted. - """ - @spec contact_map_payload() :: %{ - json: iodata(), - count: non_neg_integer(), - bands: [integer()] - } - def contact_map_payload do - if Application.get_env(:microwaveprop, :cache_contact_map, true) do - Cache.fetch_or_store(@map_payload_cache_key, @map_payload_ttl_ms, fn -> - build_contact_map_payload() - end) - else - build_contact_map_payload() - end - end - - defp build_contact_map_payload do - contacts = load_contacts_for_map() - bands = contacts |> Enum.map(fn [_, _, _, _, band | _] -> band end) |> Enum.uniq() |> Enum.sort() - %{json: Jason.encode_to_iodata!(contacts), count: length(contacts), bands: bands} - end - - defp load_contacts_for_map do - from(c in Contact, - where: not is_nil(c.pos1) and not is_nil(c.pos2) and c.private == false, - select: %{ - id: c.id, - pos1: c.pos1, - pos2: c.pos2, - band: c.band, - station1: c.station1, - station2: c.station2, - mode: c.mode, - distance_km: c.distance_km, - qso_timestamp: c.qso_timestamp - } - ) - |> Repo.all() - |> Enum.map(&format_map_contact/1) - |> Enum.reject(&is_nil/1) - |> dedup_map_reciprocals() - end - - defp format_map_contact(c) do - lat1 = c.pos1["lat"] - lon1 = c.pos1["lon"] - lat2 = c.pos2["lat"] - lon2 = c.pos2["lon"] - - if lat1 && lon1 && lat2 && lon2 do - [ - lat1, - lon1, - lat2, - lon2, - format_map_band(c.band), - c.station1, - c.station2, - c.mode, - format_map_distance(c.distance_km), - format_map_timestamp(c.qso_timestamp), - c.id - ] - end - end - - defp format_map_band(nil), do: 0 - defp format_map_band(band), do: Decimal.to_integer(band) - - defp format_map_distance(nil), do: nil - defp format_map_distance(d), do: Decimal.to_float(d) - - defp format_map_timestamp(nil), do: nil - defp format_map_timestamp(ts), do: Calendar.strftime(ts, "%Y-%m-%d %H:%M") - - defp dedup_map_reciprocals(contacts) do - Enum.uniq_by(contacts, fn [_lat1, _lon1, _lat2, _lon2, band, s1, s2, _mode, _dist, ts, _id] -> - stations = Enum.sort([s1 || "", s2 || ""]) - hour = if ts, do: String.slice(ts, 0..12), else: "" - {stations, band, hour} - end) - end - - @doc """ - Returns contacts submitted by the given user, newest first. Used by - the `/u/:callsign` profile page. Private contacts are filtered out - unless the viewer is the profile owner or an admin — anonymous or - third-party viewers must not see private rows. - """ - @spec list_contacts_for_user(User.t(), User.t() | nil) :: [Contact.t()] - def list_contacts_for_user(%User{} = owner, viewer \\ nil) do - Contact - |> where([c], c.user_id == ^owner.id) - |> filter_private_for_viewer(owner, viewer) - |> order_by([c], desc: c.qso_timestamp, desc: c.id) - |> preload(:user) - |> Repo.all() - end - - defp filter_private_for_viewer(query, %User{id: owner_id}, %User{id: viewer_id}) when owner_id == viewer_id, do: query - - defp filter_private_for_viewer(query, _owner, %User{is_admin: true}), do: query - - defp filter_private_for_viewer(query, _owner, _viewer) do - where(query, [c], c.private == false) - end - - @doc """ - Returns the 100 most recent contacts where the given callsign appears - as either `station1` or `station2`, newest first. Matching is - case-insensitive. - - When a `viewer` is provided, owner and admin viewers see private - contacts; other viewers only see non-private contacts. - """ - @spec list_contacts_involving_callsign(String.t() | nil, User.t() | nil) :: [Contact.t()] - def list_contacts_involving_callsign(callsign, viewer \\ nil) - - def list_contacts_involving_callsign(callsign, _viewer) when callsign in [nil, ""], do: [] - - def list_contacts_involving_callsign(callsign, viewer) when is_binary(callsign) do - upcased = String.upcase(callsign) - - Contact - |> where([c], fragment("upper(?)", c.station1) == ^upcased or fragment("upper(?)", c.station2) == ^upcased) - |> filter_private_for_callsign_viewer(viewer) - |> order_by([c], desc: c.qso_timestamp, desc: c.id) - |> limit(100) - |> preload(:user) - |> Repo.all() - end - - defp filter_private_for_callsign_viewer(query, nil), do: where(query, [c], c.private == false) - - defp filter_private_for_callsign_viewer(query, %User{is_admin: true}), do: query - - defp filter_private_for_callsign_viewer(query, %User{callsign: callsign}) do - # If the viewer is a co-station on the contact, let them see it even if private - upcased = String.upcase(callsign) - - where( - query, - [c], - c.private == false or - (c.private == true and - (fragment("upper(?)", c.station1) == ^upcased or fragment("upper(?)", c.station2) == ^upcased)) - ) - end - - @spec list_contacts(keyword()) :: contact_page() - def list_contacts(opts \\ []) do - page = max(Keyword.get(opts, :page, 1), 1) - per_page = clamp_per_page(Keyword.get(opts, :per_page, @per_page)) - offset = (page - 1) * per_page - search = Keyword.get(opts, :search) - scope = Keyword.get(opts, :scope) - - {sort_field, sort_dir} = sort_opts(opts) - - base_query = - Contact - |> maybe_search(search) - |> filter_private(scope) - - total_entries = total_entries_for(search, scope, base_query) - total_pages = max(ceil(total_entries / per_page), 1) - - entries = - base_query - |> order_by([q], [{^sort_dir, field(q, ^sort_field)}, {^sort_dir, q.id}]) - |> limit(^per_page) - |> offset(^offset) - |> Repo.all() - |> Repo.preload(:user) - - %{ - entries: entries, - grouped_entries: group_reciprocals(entries), - page: page, - total_pages: total_pages, - total_entries: total_entries - } - end - - defp clamp_per_page(value) when is_integer(value) and value >= 1, do: min(value, @max_per_page) - - defp clamp_per_page(_), do: @per_page - - @doc """ - Groups reciprocal contacts (same station pair swapped, same band, same timestamp). - Returns a list of `{primary, reciprocals}` tuples where reciprocals is - a (possibly empty) list of matching contacts. - """ - @spec group_reciprocals([Contact.t()]) :: [{Contact.t(), [Contact.t()]}] - def group_reciprocals(contacts) do - contacts - |> Enum.group_by(&reciprocal_key/1) - |> Enum.map(fn {_key, [primary | rest]} -> {primary, rest} end) - end - - defp reciprocal_key(contact) do - stations = Enum.sort([contact.station1, contact.station2]) - {stations, contact.band, reciprocal_hour_key(contact.qso_timestamp)} - end - - defp reciprocal_hour_key(nil), do: nil - - defp reciprocal_hour_key(%_{} = ts) do - Map.take(ts, [:year, :month, :day, :hour]) - end - - # Only cache the unscoped, unsearched total — scope-dependent counts - # differ per-viewer and must not share a cache. - defp total_entries_for(search, nil, query) when search in [nil, ""] do - if Application.get_env(:microwaveprop, :cache_contact_count, true) do - Cache.fetch_or_store(@count_cache_key, @count_cache_ttl_ms, fn -> - Repo.aggregate(query, :count) - end) - else - Repo.aggregate(query, :count) - end - end - - defp total_entries_for(_search, _scope, query), do: Repo.aggregate(query, :count) - - # Visibility filter for scope-aware queries. Admins see everything; - # logged-in users see non-private + their own private; others see only - # non-private. - defp filter_private(query, %Scope{user: %User{is_admin: true}}), do: query - - defp filter_private(query, %Scope{user: %User{id: user_id}}) do - where(query, [c], c.private == false or c.user_id == ^user_id) - end - - defp filter_private(query, _), do: where(query, [c], c.private == false) - - defp maybe_search(query, nil), do: query - defp maybe_search(query, ""), do: query - - defp maybe_search(query, search) do - terms = search |> String.split() |> Enum.map(&String.trim/1) |> Enum.reject(&(&1 == "")) - - case terms do - [a, b] -> - pa = "%" <> String.upcase(a) <> "%" - pb = "%" <> String.upcase(b) <> "%" - - where( - query, - [q], - (ilike(q.station1, ^pa) and ilike(q.station2, ^pb)) or - (ilike(q.station1, ^pb) and ilike(q.station2, ^pa)) - ) - - _ -> - pattern = "%" <> String.upcase(search) <> "%" - where(query, [q], ilike(q.station1, ^pattern) or ilike(q.station2, ^pattern)) - end - end - - defp sort_opts(opts) do - sort_by = Keyword.get(opts, :sort_by, :qso_timestamp) - sort_order = Keyword.get(opts, :sort_order, :desc) - - field = if sort_by in @sortable_fields, do: sort_by, else: :qso_timestamp - dir = if sort_order in [:asc, :desc], do: sort_order, else: :desc - - {field, dir} - end - - # ── Enrichment status queries ──────────────────────────────────── - - @enrichable_statuses [:pending, :failed] - - @spec contacts_needing_enrichment( - atom(), - non_neg_integer(), - (Ecto.Query.t() -> Ecto.Query.t()) | nil - ) :: [Contact.t()] - def contacts_needing_enrichment(field, limit \\ 500, extra_filter \\ nil) do - query = - Contact - |> where([q], field(q, ^field) in ^@enrichable_statuses and not is_nil(q.pos1)) - |> order_by([q], asc: q.qso_timestamp) - |> limit(^limit) - - query = if extra_filter, do: extra_filter.(query), else: query - Repo.all(query) - end - - @spec unprocessed_contacts(non_neg_integer()) :: [Contact.t()] - def unprocessed_contacts(limit \\ 500), do: contacts_needing_enrichment(:weather_status, limit) - - @spec unprocessed_hrrr_contacts(non_neg_integer()) :: [Contact.t()] - def unprocessed_hrrr_contacts(limit \\ 500), do: contacts_needing_enrichment(:hrrr_status, limit) - - @spec unprocessed_terrain_contacts(non_neg_integer()) :: [Contact.t()] - def unprocessed_terrain_contacts(limit \\ 500), - do: contacts_needing_enrichment(:terrain_status, limit, &where(&1, [q], not is_nil(q.pos2))) - - @spec unprocessed_iemre_contacts(non_neg_integer()) :: [Contact.t()] - def unprocessed_iemre_contacts(limit \\ 500), do: contacts_needing_enrichment(:iemre_status, limit) - - @spec unprocessed_radar_contacts(non_neg_integer()) :: [Contact.t()] - def unprocessed_radar_contacts(limit \\ 500), - do: contacts_needing_enrichment(:radar_status, limit, &where(&1, [q], not is_nil(q.pos2))) - - @spec set_enrichment_status!([Ecto.UUID.t()], atom(), atom()) :: {non_neg_integer(), nil | [any()]} - def set_enrichment_status!(ids, field, status) do - Contact - |> where([q], q.id in ^ids and field(q, ^field) != ^status) - |> Repo.update_all(set: [{field, status}]) - end - - @spec mark_weather_queued!([Ecto.UUID.t()]) :: {non_neg_integer(), nil | [any()]} - def mark_weather_queued!(ids), do: set_enrichment_status!(ids, :weather_status, :queued) - - @spec mark_hrrr_queued!([Ecto.UUID.t()]) :: {non_neg_integer(), nil | [any()]} - def mark_hrrr_queued!(ids), do: set_enrichment_status!(ids, :hrrr_status, :queued) - - @spec mark_terrain_queued!([Ecto.UUID.t()]) :: {non_neg_integer(), nil | [any()]} - def mark_terrain_queued!(ids), do: set_enrichment_status!(ids, :terrain_status, :queued) - - @spec mark_iemre_queued!([Ecto.UUID.t()]) :: {non_neg_integer(), nil | [any()]} - def mark_iemre_queued!(ids), do: set_enrichment_status!(ids, :iemre_status, :queued) - - @doc """ - Returns a list of {lat, lon} points along the contact path: pos1, midpoint, pos2. - Returns [pos1] if pos2 is nil, or [] if pos1 is nil. - """ - @spec contact_path_points(Contact.t()) :: [{float(), float()}] - def contact_path_points(%{pos1: nil}), do: [] - - def contact_path_points(%{pos1: pos1, pos2: nil}) do - lat = pos1["lat"] - lon = pos1["lon"] - if lat && lon, do: [{lat, lon}], else: [] - end - - def contact_path_points(%{pos1: pos1, pos2: pos2}) do - build_path_points(extract_latlon(pos1), extract_latlon(pos2)) - end - - defp extract_latlon(%{"lat" => lat} = pos) when is_number(lat) do - lon = pos["lon"] - if lon, do: {lat, lon} - end - - defp extract_latlon(_), do: nil - - defp build_path_points({lat1, lon1}, {lat2, lon2}) do - {mid_lat, mid_lon} = great_circle_midpoint(lat1, lon1, lat2, lon2) - [{lat1, lon1}, {mid_lat, mid_lon}, {lat2, lon2}] - end - - defp build_path_points({lat1, lon1}, _), do: [{lat1, lon1}] - defp build_path_points(_, _), do: [] - - # Spherical-vector midpoint between two lat/lon points. The arithmetic - # mean of two longitudes folds across the anti-meridian (e.g. - # `(179 + -179) / 2 == 0` back at Greenwich), so paths that cross the - # date line otherwise get a bogus center. - defp great_circle_midpoint(lat1, lon1, lat2, lon2) do - rlat1 = deg_to_rad(lat1) - rlat2 = deg_to_rad(lat2) - rlon1 = deg_to_rad(lon1) - dlon = deg_to_rad(lon2 - lon1) - - bx = :math.cos(rlat2) * :math.cos(dlon) - by = :math.cos(rlat2) * :math.sin(dlon) - - mid_lat = - :math.atan2( - :math.sin(rlat1) + :math.sin(rlat2), - :math.sqrt((:math.cos(rlat1) + bx) ** 2 + by ** 2) - ) - - mid_lon = rlon1 + :math.atan2(by, :math.cos(rlat1) + bx) - - {rad_to_deg(mid_lat), normalize_lon(rad_to_deg(mid_lon))} - end - - defp rad_to_deg(rad), do: rad * 180.0 / :math.pi() - - defp normalize_lon(lon) when lon > 180.0, do: lon - 360.0 - defp normalize_lon(lon) when lon < -180.0, do: lon + 360.0 - defp normalize_lon(lon), do: lon - - @spec haversine_km(number(), number(), number(), number()) :: float() - defdelegate haversine_km(lat1, lon1, lat2, lon2), to: Microwaveprop.Geo - - @spec backfill_distances([Contact.t()]) :: Postgrex.Result.t() | nil - def backfill_distances(contacts) do - updates = - for contact <- contacts, - is_nil(contact.distance_km) && contact.pos1 && contact.pos2, - lat1 = contact.pos1["lat"], - lon1 = contact.pos1["lon"], - lat2 = contact.pos2["lat"], - lon2 = contact.pos2["lon"], - lat1 && lon1 && lat2 && lon2 do - dist = lat1 |> haversine_km(lon1, lat2, lon2) |> round() - {contact.id, dist} - end - - execute_distance_updates(updates) - end - - defp execute_distance_updates([]), do: nil - - defp execute_distance_updates(updates) do - # Build a single UPDATE ... FROM (VALUES ...) statement - # Ecto.UUID.dump!/1 converts string UUIDs to the 16-byte binary Postgrex expects - {values_sql, params} = - updates - |> Enum.with_index() - |> Enum.reduce({"", []}, fn {{id, dist}, i}, {sql, params} -> - frag = "($#{i * 2 + 1}::uuid, $#{i * 2 + 2}::numeric)" - sep = if sql == "", do: "", else: ", " - {sql <> sep <> frag, params ++ [Ecto.UUID.dump!(id), dist]} - end) - - Repo.query!( - "UPDATE contacts AS c SET distance_km = v.dist FROM (VALUES #{values_sql}) AS v(id, dist) WHERE c.id = v.id", - params - ) - end - - defp deg_to_rad(deg), do: deg * :math.pi() / 180 - - @dialyzer {:no_match, get_contact!: 1} - @spec get_contact!(term()) :: Contact.t() - def get_contact!(id) do - Contact - |> Repo.get!(id) - |> Repo.preload(:flagged_by_user) - end - - @doc """ - True when the viewer (per `Scope`) is allowed to see `contact`. - Public contacts are always viewable; private contacts only by the - submitter or an admin. - """ - @spec can_view?(Contact.t(), Scope.t() | nil) :: boolean() - def can_view?(%Contact{private: false}, _scope), do: true - def can_view?(%Contact{}, %Scope{user: %User{is_admin: true}}), do: true - def can_view?(%Contact{user_id: user_id}, %Scope{user: %User{id: user_id}}) when not is_nil(user_id), do: true - def can_view?(%Contact{}, _scope), do: false - - @doc """ - Hard-delete a contact and everything attached to it. All FK - references (`terrain_profiles`, `contact_edits`, - `contact_common_volume_radar`) declare `on_delete: :delete_all`, - so a single `Repo.delete!` cascades through. - - Admin-only operation — callers must gate with `is_admin` before - invoking. Irreversible: prefer `toggle_flagged_invalid!/2` when - the data should be hidden but kept in the audit trail. - """ - @spec delete_contact!(Contact.t()) :: Contact.t() - def delete_contact!(%Contact{} = contact), do: Repo.delete!(contact) - - @spec toggle_flagged_invalid!(Contact.t(), User.t() | nil) :: Contact.t() - def toggle_flagged_invalid!(contact, flagger \\ nil) do - new_state = !contact.flagged_invalid - - attrs = - if new_state do - %{ - flagged_invalid: true, - flagged_by_user_id: flagger && flagger.id, - flagged_at: DateTime.truncate(DateTime.utc_now(), :second) - } - else - # Clear attribution on unflag — the next flag (potentially - # by a different admin) gets a fresh record, and the badge - # stops claiming a stale flagger after the contact was - # restored to valid. - %{flagged_invalid: false, flagged_by_user_id: nil, flagged_at: nil} - end - - contact - |> Ecto.Changeset.change(attrs) - |> Repo.update!() - end - - @doc """ - Returns all flagged-invalid contacts, newest first. Admin-only - viewer — the admin contact-edits page uses this to surface flagged - QSOs for review alongside pending edit requests. - """ - @spec list_flagged_contacts() :: [Contact.t()] - def list_flagged_contacts do - Contact - |> where([c], c.flagged_invalid == true) - |> order_by([c], desc: c.inserted_at, desc: c.id) - |> Repo.all() - end - - @spec change_contact(Contact.t(), map()) :: Ecto.Changeset.t() - def change_contact(contact, attrs \\ %{}) do - Contact.submission_changeset(contact, attrs) - end - - @doc """ - Computes pos1/pos2/distance_km from grid squares if missing. - Returns the updated contact, or the original if already computed or grids are missing. - """ - @spec ensure_positions!(Contact.t()) :: Contact.t() - def ensure_positions!(contact) do - needs_pos1 = is_nil(contact.pos1) && contact.grid1 - needs_pos2 = is_nil(contact.pos2) && contact.grid2 - - if needs_pos1 || needs_pos2 do - do_ensure_positions(contact) - else - contact - end - end - - defp do_ensure_positions(contact) do - pos1 = contact.pos1 || latlon_from_grid(contact.grid1) - pos2 = contact.pos2 || latlon_from_grid(contact.grid2) - distance = compute_distance(pos1, pos2, contact.distance_km) - - changes = build_position_changes(contact, pos1, pos2, distance) - - apply_position_changes(contact, changes) - end - - defp compute_distance(pos1, pos2, _existing_distance) when not is_nil(pos1) and not is_nil(pos2) do - lon1 = pos1["lon"] - lon2 = pos2["lon"] - - pos1["lat"] - |> haversine_km(lon1, pos2["lat"], lon2) - |> round() - |> Decimal.new() - end - - defp compute_distance(_pos1, _pos2, existing_distance), do: existing_distance - - defp build_position_changes(contact, pos1, pos2, distance) do - Enum.reduce([pos1: pos1, pos2: pos2, distance_km: distance], %{}, fn {key, val}, acc -> - if val && is_nil(Map.get(contact, key)), do: Map.put(acc, key, val), else: acc - end) - end - - defp apply_position_changes(contact, changes) when changes == %{}, do: contact - - defp apply_position_changes(contact, changes) do - changes = reset_enrichment_statuses(contact, changes) - - contact - |> Ecto.Changeset.change(changes) - |> Repo.update!() - end - - defp reset_enrichment_statuses(contact, changes) do - new_pos1 = Map.get(changes, :pos1) - new_pos2 = Map.get(changes, :pos2) - - changes = - if new_pos1 do - changes - |> maybe_reset_status(:hrrr_status, contact.hrrr_status) - |> maybe_reset_status(:weather_status, contact.weather_status) - |> maybe_reset_status(:iemre_status, contact.iemre_status) - else - changes - end - - if new_pos1 && (new_pos2 || contact.pos2) do - maybe_reset_status(changes, :terrain_status, contact.terrain_status) - else - changes - end - end - - defp maybe_reset_status(changes, field, :unavailable), do: Map.put(changes, field, :pending) - defp maybe_reset_status(changes, field, :complete), do: Map.put(changes, field, :pending) - defp maybe_reset_status(changes, _field, _status), do: changes - - defp latlon_from_grid(nil), do: nil - - defp latlon_from_grid(grid) do - case Maidenhead.to_latlon(grid) do - {:ok, {lat, lon}} -> - %{"lat" => lat, "lon" => lon} - - _ -> - # Try truncating to nearest valid even length (e.g. "FN03c" → "FN03") - truncated = String.slice(grid, 0, div(String.length(grid), 2) * 2) - - case Maidenhead.to_latlon(truncated) do - {:ok, {lat, lon}} -> %{"lat" => lat, "lon" => lon} - _ -> nil - end - end - end - - @dedup_window_seconds 3600 - - @spec create_contact(map(), String.t() | nil) :: - {:ok, Contact.t()} | {:error, Ecto.Changeset.t()} | {:error, :duplicate, Contact.t()} - def create_contact(attrs, user_id \\ nil) do - # Place user_id on the struct BEFORE running submission_changeset so - # validate_user_or_email/1 can see it. Otherwise an authenticated - # caller that omits submitter_email gets a spurious "can't be blank" - # error even though user_id satisfies the ownership invariant. - changeset = Contact.submission_changeset(%Contact{user_id: user_id}, normalize_band_attr(attrs)) - - if changeset.valid? do - insert_validated_contact(changeset) - else - {:error, %{changeset | action: :insert}} - end - end - - # Translate ADIF wavelength labels ("33cm") and raw frequency strings - # ("903.100") into our canonical MHz integer string so every create - # path — manual form, CSV commit, ADIF commit, API, console — agrees - # on what "33cm" means before the changeset's validate_inclusion runs. - # Atom and string keys are both supported so this works regardless of - # where the map came from. - defp normalize_band_attr(attrs) do - cond do - raw = Map.get(attrs, "band") -> - case BandResolver.resolve_as_string(raw) do - nil -> attrs - canonical -> Map.put(attrs, "band", canonical) - end - - raw = Map.get(attrs, :band) -> - case BandResolver.resolve_as_string(raw) do - nil -> attrs - canonical -> Map.put(attrs, :band, canonical) - end - - true -> - attrs - end - end - - defp insert_validated_contact(changeset) do - if existing = find_duplicate_contact(changeset) do - {:error, :duplicate, existing} - else - resolve_grids_and_insert(changeset) - end - end - - defp resolve_grids_and_insert(changeset) do - grid1 = Ecto.Changeset.get_change(changeset, :grid1) - grid2 = Ecto.Changeset.get_change(changeset, :grid2) - - case {Maidenhead.to_latlon(grid1), Maidenhead.to_latlon(grid2)} do - {{:ok, {lat1, lon1}}, {:ok, {lat2, lon2}}} -> - distance = lat1 |> haversine_km(lon1, lat2, lon2) |> round() |> Decimal.new() - - result = - changeset - |> Ecto.Changeset.put_change(:pos1, %{"lat" => lat1, "lon" => lon1}) - |> Ecto.Changeset.put_change(:pos2, %{"lat" => lat2, "lon" => lon2}) - |> Ecto.Changeset.put_change(:distance_km, distance) - |> Ecto.Changeset.put_change(:user_submitted, true) - |> Repo.insert() - - with {:ok, _} <- result do - invalidate_contact_map_caches() - end - - result - - _ -> - changeset = Ecto.Changeset.add_error(changeset, :grid1, "could not resolve grid to coordinates") - {:error, %{changeset | action: :insert}} - end - rescue - _e in Ecto.ConstraintError -> - # Unique constraint violation from a concurrent insert — the - # dedup check at `insert_validated_contact` raced with another - # process. Re-check and return the existing contact. - case find_duplicate_contact(changeset) do - nil -> {:error, :constraint_error} - existing -> {:error, :duplicate, existing} - end - end - - defp find_duplicate_contact(changeset) do - s1 = changeset |> Ecto.Changeset.get_field(:station1) |> to_string() |> String.upcase() - s2 = changeset |> Ecto.Changeset.get_field(:station2) |> to_string() |> String.upcase() - g1 = changeset |> Ecto.Changeset.get_field(:grid1) |> to_string() |> String.upcase() - g2 = changeset |> Ecto.Changeset.get_field(:grid2) |> to_string() |> String.upcase() - band = Ecto.Changeset.get_field(changeset, :band) - ts = Ecto.Changeset.get_field(changeset, :qso_timestamp) - - band_decimal = if is_binary(band), do: Decimal.new(band), else: band - min_ts = DateTime.add(ts, -@dedup_window_seconds, :second) - max_ts = DateTime.add(ts, @dedup_window_seconds, :second) - - Repo.one( - from(c in Contact, - where: - c.band == ^band_decimal and c.qso_timestamp >= ^min_ts and c.qso_timestamp <= ^max_ts and - c.flagged_invalid == false and fragment("LEAST(UPPER(station1), UPPER(station2)) = ?", ^min(s1, s2)) and - fragment("GREATEST(UPPER(station1), UPPER(station2)) = ?", ^max(s1, s2)) and - fragment("LEAST(UPPER(grid1), UPPER(grid2)) = ?", ^min(g1, g2)) and - fragment("GREATEST(UPPER(grid1), UPPER(grid2)) = ?", ^max(g1, g2)) - ) - ) - end - - @refinement_fields ~w(grid1 grid2 mode)a - @refinement_allowed_modes ~w(CW SSB FM FT8 FT4 Q65) - - @doc """ - Apply a grid/mode refinement from an import row to an existing contact. - - `changes` may contain any of `:grid1`, `:grid2`, or `:mode`. Grid changes - trigger recomputation of the corresponding `pos` and the `distance_km`, - and reset all four enrichment status fields to `:pending` so the caller - can re-enqueue the enrichment pipeline. Mode-only changes do not touch - positions or enrichment status. - """ - @spec apply_contact_refinement(Contact.t(), map()) :: - {:ok, Contact.t()} | {:error, Ecto.Changeset.t()} - def apply_contact_refinement(%Contact{} = contact, changes) when is_map(changes) do - cleaned = Map.take(changes, @refinement_fields) - - if cleaned == %{} do - {:ok, contact} - else - do_apply_contact_refinement(contact, cleaned) - end - end - - defp do_apply_contact_refinement(contact, changes) do - changeset = - contact - |> Ecto.Changeset.cast(changes, @refinement_fields) - |> validate_refinement_mode() - |> maybe_recompute_refinement_positions(contact, changes) - - if changeset.valid? do - Repo.update(changeset) - else - {:error, %{changeset | action: :update}} - end - end - - defp validate_refinement_mode(changeset) do - Ecto.Changeset.validate_inclusion(changeset, :mode, @refinement_allowed_modes) - end - - defp maybe_recompute_refinement_positions(changeset, contact, changes) do - new_grid1 = Map.get(changes, :grid1) - new_grid2 = Map.get(changes, :grid2) - - cond do - new_grid1 && new_grid2 -> - put_pos_changes(changeset, new_grid1, new_grid2, :both) - - new_grid1 -> - put_pos_changes(changeset, new_grid1, contact.grid2, :side1) - - new_grid2 -> - put_pos_changes(changeset, contact.grid1, new_grid2, :side2) - - true -> - changeset - end - end - - defp put_pos_changes(changeset, grid1, grid2, sides) do - case {Maidenhead.to_latlon(grid1), Maidenhead.to_latlon(grid2)} do - {{:ok, {lat1, lon1}}, {:ok, {lat2, lon2}}} -> - distance = lat1 |> haversine_km(lon1, lat2, lon2) |> round() |> Decimal.new() - - changeset - |> maybe_put_pos_for_side(:pos1, %{"lat" => lat1, "lon" => lon1}, sides) - |> maybe_put_pos_for_side(:pos2, %{"lat" => lat2, "lon" => lon2}, sides) - |> Ecto.Changeset.put_change(:distance_km, distance) - |> reset_all_enrichment_statuses() - - _ -> - Ecto.Changeset.add_error(changeset, :grid1, "could not resolve grid to coordinates") - end - end - - defp maybe_put_pos_for_side(changeset, :pos1, pos, sides) when sides in [:side1, :both] do - Ecto.Changeset.put_change(changeset, :pos1, pos) - end - - defp maybe_put_pos_for_side(changeset, :pos2, pos, sides) when sides in [:side2, :both] do - Ecto.Changeset.put_change(changeset, :pos2, pos) - end - - defp maybe_put_pos_for_side(changeset, _field, _pos, _sides), do: changeset - - defp reset_all_enrichment_statuses(changeset) do - changeset - |> Ecto.Changeset.put_change(:hrrr_status, :pending) - |> Ecto.Changeset.put_change(:weather_status, :pending) - |> Ecto.Changeset.put_change(:terrain_status, :pending) - |> Ecto.Changeset.put_change(:iemre_status, :pending) - end - - # ── Contact Edits ─────────────────────────────────────────────── - - @doc """ - Create a proposed edit for a contact. Only stores fields that actually - differ from the contact's current values. Normalizes callsigns and grids - to uppercase. - """ - @spec create_contact_edit(Contact.t(), User.t(), map()) :: - {:ok, ContactEdit.t()} | {:error, Ecto.Changeset.t()} - def create_contact_edit(%Contact{} = contact, user, proposed_changes) when is_map(proposed_changes) do - # Normalize and diff against current values - normalized = normalize_proposed(proposed_changes) - diffed = diff_against_contact(contact, normalized) - - attrs = %{ - contact_id: contact.id, - user_id: user.id, - proposed_changes: diffed - } - - %ContactEdit{} - |> ContactEdit.changeset(attrs) - |> Repo.insert() - end - - # Whitelist of keys an owner / pending-edit submitter is allowed - # to propose. Anything not in this set is dropped before the - # changes hit `String.to_existing_atom` + `Ecto.Changeset.change` - # downstream — without this filter a crafted form could mass-assign - # `user_id`, `flagged_invalid`, `inserted_at`, etc. on a contact the - # caller owns. `band` is in the list because admins edit it through - # this same path; `ContactEdit.changeset` rejects `band` for - # non-admin pending edits via its own validation. - @editable_proposed_keys ~w(station1 station2 grid1 grid2 band mode qso_timestamp height1_ft height2_ft private) - - @doc false - @spec normalize_proposed(map()) :: map() - def normalize_proposed(changes) do - changes - |> Map.take(@editable_proposed_keys) - |> normalize_string_field("station1") - |> normalize_string_field("station2") - |> normalize_string_field("grid1") - |> normalize_string_field("grid2") - |> normalize_string_field("mode") - |> normalize_integer_field("height1_ft") - |> normalize_integer_field("height2_ft") - |> normalize_boolean_field("private") - |> normalize_timestamp_field("qso_timestamp") - end - - # The /contacts/:id edit form pre-fills qso_timestamp as - # `"YYYY-MM-DD HH:MM"` via Calendar.strftime, then submits whatever - # the user typed. Coerce to %DateTime{} (UTC) so the diff matches - # against the stored DateTime instead of always reading as a change, - # and so the apply path doesn't crash on a non-ISO string. Drop the - # field when empty or unparseable so a bogus value is treated as - # "no change" rather than raising. - defp normalize_timestamp_field(map, key) do - case Map.get(map, key, :not_provided) do - :not_provided -> map - nil -> Map.delete(map, key) - "" -> Map.delete(map, key) - %DateTime{} = dt -> Map.put(map, key, dt) - val when is_binary(val) -> put_or_drop_timestamp(map, key, parse_timestamp(val)) - _ -> Map.delete(map, key) - end - end - - defp put_or_drop_timestamp(map, key, nil), do: Map.delete(map, key) - defp put_or_drop_timestamp(map, key, %DateTime{} = dt), do: Map.put(map, key, dt) - - defp parse_timestamp(val) do - trimmed = String.trim(val) - - # Accept `"YYYY-MM-DD HH:MM[:SS][Z]"` (form pre-fill, space sep) and - # the strict ISO `"YYYY-MM-DDTHH:MM:SSZ"` form. NaiveDateTime handles - # both separators and an optional seconds field. - iso = trimmed |> String.replace(" ", "T") |> ensure_seconds() |> String.trim_trailing("Z") - - case NaiveDateTime.from_iso8601(iso) do - {:ok, ndt} -> DateTime.from_naive!(ndt, "Etc/UTC") - _ -> nil - end - end - - defp ensure_seconds(s) do - # Add ":00" before any trailing "Z" / end-of-string when only HH:MM - # is present, since NaiveDateTime.from_iso8601 requires seconds. - case Regex.run(~r/^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2})(Z?)$/, s) do - [_, prefix, tail] -> prefix <> ":00" <> tail - _ -> s - end - end - - # HTML checkboxes arrive as "true"/"false" strings; Ecto won't coerce - # those inside a validate_inclusion path, so normalize at the boundary. - defp normalize_boolean_field(map, key) do - case Map.get(map, key, :not_provided) do - :not_provided -> map - nil -> map - val when is_boolean(val) -> Map.put(map, key, val) - "true" -> Map.put(map, key, true) - "false" -> Map.put(map, key, false) - _ -> map - end - end - - defp normalize_string_field(map, key) do - case Map.get(map, key) do - nil -> map - val when is_binary(val) -> Map.put(map, key, val |> String.trim() |> String.upcase()) - val -> Map.put(map, key, val) - end - end - - # Forms deliver integer-shaped inputs as strings. Drop the field entirely - # on empty strings so the diff doesn't register "" != current_value. - defp normalize_integer_field(map, key) do - case Map.get(map, key, :not_provided) do - :not_provided -> - map - - nil -> - map - - val when is_integer(val) -> - map - - "" -> - Map.delete(map, key) - - val when is_binary(val) -> - case Integer.parse(String.trim(val)) do - {int, ""} -> Map.put(map, key, int) - _ -> map - end - - _ -> - map - end - end - - @doc false - @spec diff_against_contact(Contact.t(), map()) :: map() - def diff_against_contact(contact, proposed) do - Enum.reduce(proposed, %{}, fn {key, new_val}, acc -> - current = current_value(contact, key) - - if values_equal?(current, new_val) do - acc - else - Map.put(acc, key, new_val) - end - end) - end - - defp current_value(contact, "station1"), do: contact.station1 - defp current_value(contact, "station2"), do: contact.station2 - defp current_value(contact, "grid1"), do: contact.grid1 - defp current_value(contact, "grid2"), do: contact.grid2 - defp current_value(contact, "mode"), do: contact.mode - - defp current_value(contact, "band") do - if contact.band, do: Decimal.to_integer(contact.band) - end - - defp current_value(contact, "qso_timestamp"), do: contact.qso_timestamp - defp current_value(contact, "height1_ft"), do: contact.height1_ft - defp current_value(contact, "height2_ft"), do: contact.height2_ft - defp current_value(contact, "private"), do: contact.private - defp current_value(_contact, _key), do: nil - - defp values_equal?(a, b) when is_binary(a) and is_binary(b) do - String.upcase(a) == String.upcase(b) - end - - defp values_equal?(a, b) when is_integer(a), do: a == to_integer(b) - defp values_equal?(a, b) when is_integer(b), do: to_integer(a) == b - defp values_equal?(a, b), do: a == b - - defp to_integer(v) when is_integer(v), do: v - defp to_integer(v) when is_binary(v), do: String.to_integer(v) - defp to_integer(%Decimal{} = v), do: Decimal.to_integer(v) - defp to_integer(v), do: v - - @spec list_pending_edits() :: [ContactEdit.t()] - def list_pending_edits do - ContactEdit - |> where([e], e.status == :pending) - |> order_by([e], desc: e.inserted_at) - |> preload([:user, :contact]) - |> Repo.all() - end - - @doc """ - Base query for pending contact edits with `:user` and `:contact` - preloaded. Used as a `live_table` `data_provider` so sort/paginate - can be applied on top. - """ - @spec pending_edits_query() :: Ecto.Query.t() - def pending_edits_query do - from e in ContactEdit, - as: :resource, - where: e.status == :pending, - preload: [:user, :contact] - end - - @spec list_contact_edits(Ecto.UUID.t()) :: [ContactEdit.t()] - def list_contact_edits(contact_id) do - ContactEdit - |> where([e], e.contact_id == ^contact_id) - |> order_by([e], desc: e.inserted_at) - |> preload([:user, :reviewed_by]) - |> Repo.all() - end - - @spec pending_edit_count() :: non_neg_integer() - def pending_edit_count do - ContactEdit - |> where([e], e.status == :pending) - |> Repo.aggregate(:count) - end - - @spec pending_edit_for_user(Ecto.UUID.t(), Ecto.UUID.t()) :: ContactEdit.t() | nil - def pending_edit_for_user(contact_id, user_id) do - ContactEdit - |> where([e], e.contact_id == ^contact_id and e.user_id == ^user_id and e.status == :pending) - |> preload([:user, :contact]) - |> Repo.one() - end - - @spec get_contact_edit(Ecto.UUID.t()) :: ContactEdit.t() | nil - def get_contact_edit(id) do - ContactEdit - |> preload([:user, :contact, :reviewed_by]) - |> Repo.get(id) - end - - @spec approve_edit(ContactEdit.t(), User.t(), String.t() | nil) :: - {:ok, ContactEdit.t()} | {:error, any()} - def approve_edit(%ContactEdit{status: :pending} = edit, admin, note) do - Repo.transaction(fn -> - # Check the contact still exists before proceeding — the FK has - # on_delete: :delete_all so if the contact is gone the edit row is - # also gone, making any update to it a StaleEntryError. - case Repo.get(Contact, edit.contact_id) do - nil -> - Repo.rollback(:contact_deleted) - - contact -> - # Mark edit as approved - {:ok, approved} = - edit - |> ContactEdit.review_changeset(%{ - status: :approved, - admin_note: note, - reviewed_by_id: admin.id, - reviewed_at: DateTime.truncate(DateTime.utc_now(), :second) - }) - |> Repo.update() - - # Apply changes to contact - _ = apply_edit_to_contact(contact, edit.proposed_changes) - Repo.preload(approved, [:user, :contact, :reviewed_by]) - end - end) - end - - @spec reject_edit(ContactEdit.t(), User.t(), String.t() | nil) :: - {:ok, ContactEdit.t()} | {:error, Ecto.Changeset.t()} - def reject_edit(%ContactEdit{status: :pending} = edit, admin, note) do - edit - |> ContactEdit.review_changeset(%{ - status: :rejected, - admin_note: note, - reviewed_by_id: admin.id, - reviewed_at: DateTime.truncate(DateTime.utc_now(), :second) - }) - |> Repo.update() - end - - @doc """ - Whether the given user submitted (owns) this contact. Returns `false` - for anonymous contacts (`user_id == nil`) and for a `nil` user. - """ - @spec owner?(Contact.t(), User.t() | nil) :: boolean() - def owner?(%Contact{user_id: nil}, _user), do: false - def owner?(%Contact{}, nil), do: false - def owner?(%Contact{user_id: uid}, %User{id: uid}), do: true - def owner?(%Contact{}, %User{}), do: false - - @doc """ - Allow the original submitter of a contact to edit it directly without - going through the admin review queue. Reuses the same normalization - and diff logic as `create_contact_edit/3` so owners can only push - meaningful changes (at least one field must actually differ). - - Returns `{:ok, updated_contact}` on success, `{:error, :not_owner}` if - the user did not submit the contact, or `{:error, :no_changes}` if - every proposed value already matches the current contact. - """ - @spec apply_owner_edit(Contact.t(), User.t(), map()) :: - {:ok, Contact.t()} | {:error, :not_owner | :no_changes} - def apply_owner_edit(%Contact{} = contact, %User{} = user, proposed_changes) when is_map(proposed_changes) do - if owner?(contact, user) do - diffed = diff_against_contact(contact, normalize_proposed(proposed_changes)) - - if diffed == %{} do - {:error, :no_changes} - else - {:ok, apply_edit_to_contact(contact, diffed)} - end - else - {:error, :not_owner} - end - end - - @doc "Apply proposed changes directly to a contact (admin use)." - @spec apply_edit_to_contact(Contact.t(), map()) :: Contact.t() - def apply_edit_to_contact(contact, proposed_changes) do - changes = build_contact_changes(contact, proposed_changes) - changes = maybe_recompute_positions(changes, contact, proposed_changes) - changes = maybe_reset_terrain_for_heights(changes, proposed_changes) - - updated = - contact - |> Ecto.Changeset.change(changes) - |> Repo.update!() - - maybe_enqueue_terrain_for_heights(updated, proposed_changes) - - # Public contact-map + count caches are built from `private == false` - # rows. A grid correction, callsign change, flagged_invalid flip, or - # a `private: false -> true` toggle on an already-mapped contact all - # change what /api/contacts/map and /contacts/map should return — - # private flips in particular are a privacy leak until the 10-minute - # TTL expires. Invalidate on every direct update. - invalidate_contact_map_caches() - - updated - end - - defp invalidate_contact_map_caches do - Cache.invalidate(@count_cache_key) - Cache.invalidate(@map_payload_cache_key) - Cache.invalidate(MicrowavepropWeb.ContactMapController.cache_key()) - :ok - end - - defp maybe_enqueue_terrain_for_heights(contact, proposed) do - if Map.has_key?(proposed, "height1_ft") or Map.has_key?(proposed, "height2_ft") do - ContactWeatherEnqueueWorker.enqueue_for_contact(contact, [:terrain]) - end - - :ok - end - - # Antenna heights feed directly into the elevation-profile terrain - # analysis (diffraction loss, clearance verdict). Changing them - # invalidates the stored terrain_profile, so reset the status to - # :pending and let TerrainProfileWorker re-run. - defp maybe_reset_terrain_for_heights(changes, proposed) do - if Map.has_key?(proposed, "height1_ft") or Map.has_key?(proposed, "height2_ft") do - Map.put(changes, :terrain_status, :pending) - else - changes - end - end - - defp maybe_recompute_positions(changes, contact, proposed) do - needs_recompute = - Map.has_key?(proposed, "grid1") or - Map.has_key?(proposed, "grid2") or - Map.has_key?(proposed, "band") or - Map.has_key?(proposed, "qso_timestamp") - - if needs_recompute do - recompute_positions(changes, contact, proposed) - else - changes - end - end - - defp recompute_positions(changes, contact, proposed) do - pos1 = resolve_grid_position(Map.get(proposed, "grid1", contact.grid1)) || contact.pos1 - pos2 = resolve_grid_position(Map.get(proposed, "grid2", contact.grid2)) || contact.pos2 - distance = compute_distance(pos1, pos2) - - changes - |> Map.put(:pos1, pos1) - |> Map.put(:pos2, pos2) - |> Map.put(:distance_km, distance) - |> Map.put(:hrrr_status, :pending) - |> Map.put(:weather_status, :pending) - |> Map.put(:terrain_status, :pending) - |> Map.put(:iemre_status, :pending) - end - - defp compute_distance(pos1, pos2) when not is_nil(pos1) and not is_nil(pos2) do - lon1 = pos1["lon"] - lon2 = pos2["lon"] - pos1["lat"] |> haversine_km(lon1, pos2["lat"], lon2) |> round() |> Decimal.new() - end - - defp compute_distance(_, _), do: nil - - defp build_contact_changes(_contact, proposed) do - Enum.reduce(proposed, %{}, fn - {"band", val}, acc -> - Map.put(acc, :band, Decimal.new(to_string(val))) - - {"qso_timestamp", val}, acc when is_binary(val) -> - {:ok, dt, _} = DateTime.from_iso8601(val) - Map.put(acc, :qso_timestamp, dt) - - {"qso_timestamp", %DateTime{} = dt}, acc -> - Map.put(acc, :qso_timestamp, dt) - - {key, val}, acc -> - Map.put(acc, String.to_existing_atom(key), val) - end) - end - - defp resolve_grid_position(nil), do: nil - - defp resolve_grid_position(grid) do - case Maidenhead.to_latlon(grid) do - {:ok, {lat, lon}} -> %{"lat" => lat, "lon" => lon} - _ -> nil - end - end + alias Microwaveprop.Radio.Contacts + alias Microwaveprop.Radio.Import + + # ── Contacts ── + + defdelegate contact_map_payload(), to: Contacts + defdelegate list_contacts_for_user(owner, viewer \\ nil), to: Contacts + defdelegate list_contacts_involving_callsign(callsign, viewer \\ nil), to: Contacts + defdelegate list_contacts(opts \\ []), to: Contacts + defdelegate group_reciprocals(contacts), to: Contacts + defdelegate contacts_needing_enrichment(field, limit \\ 500, extra_filter \\ nil), to: Contacts + defdelegate unprocessed_contacts(limit \\ 500), to: Contacts + defdelegate unprocessed_hrrr_contacts(limit \\ 500), to: Contacts + defdelegate unprocessed_terrain_contacts(limit \\ 500), to: Contacts + defdelegate unprocessed_iemre_contacts(limit \\ 500), to: Contacts + defdelegate unprocessed_radar_contacts(limit \\ 500), to: Contacts + defdelegate set_enrichment_status!(ids, field, status), to: Contacts + defdelegate mark_weather_queued!(ids), to: Contacts + defdelegate mark_hrrr_queued!(ids), to: Contacts + defdelegate mark_terrain_queued!(ids), to: Contacts + defdelegate mark_iemre_queued!(ids), to: Contacts + defdelegate contact_path_points(contact), to: Contacts + defdelegate haversine_km(lat1, lon1, lat2, lon2), to: Contacts + defdelegate backfill_distances(contacts), to: Contacts + defdelegate get_contact!(id), to: Contacts + defdelegate can_view?(contact, scope), to: Contacts + defdelegate delete_contact!(contact), to: Contacts + defdelegate toggle_flagged_invalid!(contact, flagger \\ nil), to: Contacts + defdelegate list_flagged_contacts(), to: Contacts + defdelegate change_contact(contact, attrs \\ %{}), to: Contacts + defdelegate ensure_positions!(contact), to: Contacts + defdelegate create_contact(attrs, user_id \\ nil), to: Contacts + defdelegate owner?(contact, user), to: Contacts + + # ── Import ── + + defdelegate apply_contact_refinement(contact, changes), to: Import + defdelegate create_contact_edit(contact, user, proposed_changes), to: Import + defdelegate normalize_proposed(changes), to: Import + defdelegate diff_against_contact(contact, proposed), to: Import + defdelegate list_pending_edits(), to: Import + defdelegate pending_edits_query(), to: Import + defdelegate list_contact_edits(contact_id), to: Import + defdelegate pending_edit_count(), to: Import + defdelegate pending_edit_for_user(contact_id, user_id), to: Import + defdelegate get_contact_edit(id), to: Import + defdelegate approve_edit(edit, admin, note), to: Import + defdelegate reject_edit(edit, admin, note), to: Import + defdelegate apply_owner_edit(contact, user, proposed_changes), to: Import + defdelegate apply_edit_to_contact(contact, proposed_changes), to: Import end diff --git a/lib/microwaveprop/radio/adif_import.ex b/lib/microwaveprop/radio/adif_import.ex index 407d55d0..62234cb0 100644 --- a/lib/microwaveprop/radio/adif_import.ex +++ b/lib/microwaveprop/radio/adif_import.ex @@ -15,6 +15,7 @@ defmodule Microwaveprop.Radio.AdifImport do alias Microwaveprop.Radio.TextSanitizer alias Microwaveprop.Repo + @max_import_rows 2_000 @dedup_window_seconds 3600 # Map ADIF MODE (and MODE+SUBMODE combinations) onto the fixed set of modes @@ -97,27 +98,32 @@ defmodule Microwaveprop.Radio.AdifImport do submitter_email: String.t() } - @spec preview(String.t(), String.t()) :: {:ok, preview_result()} | {:error, :no_records} + @spec preview(String.t(), String.t()) :: {:ok, preview_result()} | {:error, :no_records} | {:error, :too_many_rows} def preview(adif_string, submitter_email) do records = parse_adif(adif_string) - if records == [] do - {:error, :no_records} - else - rows = records |> Enum.with_index(1) |> Enum.map(&build_row(&1, submitter_email)) - {valid, invalid} = split_parsed(rows) - existing = load_existing_for_dedup(valid) - {unique, duplicates, refinements} = dedupe(valid, existing) + cond do + records == [] -> + {:error, :no_records} - {:ok, - %{ - valid: unique, - invalid: invalid, - duplicates: duplicates, - refinements: refinements, - total_rows: length(records), - submitter_email: submitter_email - }} + length(records) > @max_import_rows -> + {:error, :too_many_rows} + + true -> + rows = records |> Enum.with_index(1) |> Enum.map(&build_row(&1, submitter_email)) + {valid, invalid} = split_parsed(rows) + existing = load_existing_for_dedup(valid) + {unique, duplicates, refinements} = dedupe(valid, existing) + + {:ok, + %{ + valid: unique, + invalid: invalid, + duplicates: duplicates, + refinements: refinements, + total_rows: length(records), + submitter_email: submitter_email + }} end end diff --git a/lib/microwaveprop/radio/contact.ex b/lib/microwaveprop/radio/contact.ex index d5cf0b24..008ade89 100644 --- a/lib/microwaveprop/radio/contact.ex +++ b/lib/microwaveprop/radio/contact.ex @@ -58,6 +58,7 @@ defmodule Microwaveprop.Radio.Contact do field :user_submitted, :boolean, default: false field :submitter_email, :string + field :submitter_verified, :boolean, default: false field :flagged_invalid, :boolean, default: false field :flagged_at, :utc_datetime field :private, :boolean, default: false diff --git a/lib/microwaveprop/radio/contacts.ex b/lib/microwaveprop/radio/contacts.ex new file mode 100644 index 00000000..fc1014f7 --- /dev/null +++ b/lib/microwaveprop/radio/contacts.ex @@ -0,0 +1,607 @@ +defmodule Microwaveprop.Radio.Contacts do + @moduledoc false + + import Ecto.Query + + alias Microwaveprop.Accounts.Scope + alias Microwaveprop.Accounts.User + alias Microwaveprop.Cache + alias Microwaveprop.Radio.BandResolver + alias Microwaveprop.Radio.Contact + alias Microwaveprop.Radio.Maidenhead + alias Microwaveprop.Repo + + @per_page 20 + @max_per_page 200 + @sortable_fields ~w(station1 station2 band mode distance_km qso_timestamp inserted_at)a + @count_cache_key {__MODULE__, :total_contact_count} + @count_cache_ttl_ms 30_000 + @map_payload_cache_key {__MODULE__, :contact_map_payload} + @map_payload_ttl_ms 10 * 60 * 1_000 + @dedup_window_seconds 3600 + + @type contact_page :: %{ + entries: [Contact.t()], + grouped_entries: [{Contact.t(), [Contact.t()]}], + page: pos_integer(), + total_pages: pos_integer(), + total_entries: non_neg_integer() + } + + # ── Map payload ── + + @spec contact_map_payload() :: %{json: iodata(), count: non_neg_integer(), bands: [integer()]} + def contact_map_payload do + if Application.get_env(:microwaveprop, :cache_contact_map, true) do + Cache.fetch_or_store(@map_payload_cache_key, @map_payload_ttl_ms, fn -> + build_contact_map_payload() + end) + else + build_contact_map_payload() + end + end + + defp build_contact_map_payload do + contacts = load_contacts_for_map() + bands = contacts |> Enum.map(fn [_, _, _, _, band | _] -> band end) |> Enum.uniq() |> Enum.sort() + %{json: Jason.encode_to_iodata!(contacts), count: length(contacts), bands: bands} + end + + defp load_contacts_for_map do + from(c in Contact, + where: not is_nil(c.pos1) and not is_nil(c.pos2) and c.private == false, + select: %{ + id: c.id, + pos1: c.pos1, + pos2: c.pos2, + band: c.band, + station1: c.station1, + station2: c.station2, + mode: c.mode, + distance_km: c.distance_km, + qso_timestamp: c.qso_timestamp + } + ) + |> Repo.all() + |> Enum.map(&format_map_contact/1) + |> Enum.reject(&is_nil/1) + |> dedup_map_reciprocals() + end + + defp format_map_contact(c) do + lat1 = c.pos1["lat"] + lon1 = c.pos1["lon"] + lat2 = c.pos2["lat"] + lon2 = c.pos2["lon"] + + if lat1 && lon1 && lat2 && lon2 do + [ + lat1, + lon1, + lat2, + lon2, + format_map_band(c.band), + c.station1, + c.station2, + c.mode, + format_map_distance(c.distance_km), + format_map_timestamp(c.qso_timestamp), + c.id + ] + end + end + + defp format_map_band(nil), do: 0 + defp format_map_band(band), do: Decimal.to_integer(band) + defp format_map_distance(nil), do: nil + defp format_map_distance(d), do: Decimal.to_float(d) + defp format_map_timestamp(nil), do: nil + defp format_map_timestamp(ts), do: Calendar.strftime(ts, "%Y-%m-%d %H:%M") + + defp dedup_map_reciprocals(contacts), + do: + Enum.uniq_by(contacts, fn [_lat1, _lon1, _lat2, _lon2, band, s1, s2, _mode, _dist, ts, _id] -> + stations = Enum.sort([s1 || "", s2 || ""]) + hour = if ts, do: String.slice(ts, 0..12), else: "" + {stations, band, hour} + end) + + # ── User queries ── + + @spec list_contacts_for_user(User.t(), User.t() | nil) :: [Contact.t()] + def list_contacts_for_user(%User{} = owner, viewer \\ nil) do + Contact + |> where([c], c.user_id == ^owner.id) + |> filter_private_for_viewer(owner, viewer) + |> order_by([c], desc: c.qso_timestamp, desc: c.id) + |> preload(:user) + |> Repo.all() + end + + defp filter_private_for_viewer(query, %User{id: owner_id}, %User{id: viewer_id}) when owner_id == viewer_id, do: query + defp filter_private_for_viewer(query, _owner, %User{is_admin: true}), do: query + defp filter_private_for_viewer(query, _owner, _viewer), do: where(query, [c], c.private == false) + + @spec list_contacts_involving_callsign(String.t() | nil, User.t() | nil) :: [Contact.t()] + def list_contacts_involving_callsign(callsign, viewer \\ nil) + def list_contacts_involving_callsign(callsign, _viewer) when callsign in [nil, ""], do: [] + + def list_contacts_involving_callsign(callsign, viewer) when is_binary(callsign) do + upcased = String.upcase(callsign) + + Contact + |> where([c], fragment("upper(?)", c.station1) == ^upcased or fragment("upper(?)", c.station2) == ^upcased) + |> filter_private_for_callsign_viewer(viewer) + |> order_by([c], desc: c.qso_timestamp, desc: c.id) + |> limit(100) + |> preload(:user) + |> Repo.all() + end + + defp filter_private_for_callsign_viewer(query, nil), do: where(query, [c], c.private == false) + defp filter_private_for_callsign_viewer(query, %User{is_admin: true}), do: query + + defp filter_private_for_callsign_viewer(query, %User{callsign: callsign}) do + upcased = String.upcase(callsign) + + where( + query, + [c], + c.private == false or + (c.private == true and + (fragment("upper(?)", c.station1) == ^upcased or fragment("upper(?)", c.station2) == ^upcased)) + ) + end + + # ── Paginated list ── + + @spec list_contacts(keyword()) :: contact_page() + def list_contacts(opts \\ []) do + page = max(Keyword.get(opts, :page, 1), 1) + per_page = clamp_per_page(Keyword.get(opts, :per_page, @per_page)) + offset = (page - 1) * per_page + search = Keyword.get(opts, :search) + scope = Keyword.get(opts, :scope) + {sort_field, sort_dir} = sort_opts(opts) + + base_query = Contact |> maybe_search(search) |> filter_private(scope) + total_entries = total_entries_for(search, scope, base_query) + total_pages = max(ceil(total_entries / per_page), 1) + + entries = + base_query + |> order_by([q], [{^sort_dir, field(q, ^sort_field)}, {^sort_dir, q.id}]) + |> limit(^per_page) + |> offset(^offset) + |> Repo.all() + |> Repo.preload(:user) + + %{ + entries: entries, + grouped_entries: group_reciprocals(entries), + page: page, + total_pages: total_pages, + total_entries: total_entries + } + end + + defp clamp_per_page(value) when is_integer(value) and value >= 1, do: min(value, @max_per_page) + defp clamp_per_page(_), do: @per_page + + @spec group_reciprocals([Contact.t()]) :: [{Contact.t(), [Contact.t()]}] + def group_reciprocals(contacts) do + contacts |> Enum.group_by(&reciprocal_key/1) |> Enum.map(fn {_key, [primary | rest]} -> {primary, rest} end) + end + + defp reciprocal_key(contact), + do: {Enum.sort([contact.station1, contact.station2]), contact.band, reciprocal_hour_key(contact.qso_timestamp)} + + defp reciprocal_hour_key(nil), do: nil + defp reciprocal_hour_key(%_{} = ts), do: Map.take(ts, [:year, :month, :day, :hour]) + + defp total_entries_for(search, nil, query) when search in [nil, ""] do + if Application.get_env(:microwaveprop, :cache_contact_count, true) do + Cache.fetch_or_store(@count_cache_key, @count_cache_ttl_ms, fn -> Repo.aggregate(query, :count) end) + else + Repo.aggregate(query, :count) + end + end + + defp total_entries_for(_search, _scope, query), do: Repo.aggregate(query, :count) + + defp filter_private(query, %Scope{user: %User{is_admin: true}}), do: query + + defp filter_private(query, %Scope{user: %User{id: user_id}}), + do: where(query, [c], c.private == false or c.user_id == ^user_id) + + defp filter_private(query, _), do: where(query, [c], c.private == false) + + defp maybe_search(query, nil), do: query + defp maybe_search(query, ""), do: query + + defp maybe_search(query, search) do + terms = search |> String.split() |> Enum.map(&String.trim/1) |> Enum.reject(&(&1 == "")) + + case terms do + [a, b] -> + pa = "%" <> String.upcase(a) <> "%" + pb = "%" <> String.upcase(b) <> "%" + + where( + query, + [q], + (ilike(q.station1, ^pa) and ilike(q.station2, ^pb)) or (ilike(q.station1, ^pb) and ilike(q.station2, ^pa)) + ) + + _ -> + pattern = "%" <> String.upcase(search) <> "%" + where(query, [q], ilike(q.station1, ^pattern) or ilike(q.station2, ^pattern)) + end + end + + defp sort_opts(opts) do + sort_by = Keyword.get(opts, :sort_by, :qso_timestamp) + sort_order = Keyword.get(opts, :sort_order, :desc) + field = if sort_by in @sortable_fields, do: sort_by, else: :qso_timestamp + dir = if sort_order in [:asc, :desc], do: sort_order, else: :desc + {field, dir} + end + + # ── Enrichment status ── + + @enrichable_statuses [:pending, :failed] + + @spec contacts_needing_enrichment( + atom(), + non_neg_integer(), + (Ecto.Query.t() -> Ecto.Query.t()) | nil + ) :: [Contact.t()] + def contacts_needing_enrichment(field, limit \\ 500, extra_filter \\ nil) do + query = + Contact + |> where([q], field(q, ^field) in ^@enrichable_statuses and not is_nil(q.pos1)) + |> order_by([q], asc: q.qso_timestamp) + |> limit(^limit) + + query = if extra_filter, do: extra_filter.(query), else: query + Repo.all(query) + end + + @spec unprocessed_contacts(non_neg_integer()) :: [Contact.t()] + def unprocessed_contacts(limit \\ 500), do: contacts_needing_enrichment(:weather_status, limit) + @spec unprocessed_hrrr_contacts(non_neg_integer()) :: [Contact.t()] + def unprocessed_hrrr_contacts(limit \\ 500), do: contacts_needing_enrichment(:hrrr_status, limit) + @spec unprocessed_terrain_contacts(non_neg_integer()) :: [Contact.t()] + def unprocessed_terrain_contacts(limit \\ 500), + do: contacts_needing_enrichment(:terrain_status, limit, &where(&1, [q], not is_nil(q.pos2))) + + @spec unprocessed_iemre_contacts(non_neg_integer()) :: [Contact.t()] + def unprocessed_iemre_contacts(limit \\ 500), do: contacts_needing_enrichment(:iemre_status, limit) + @spec unprocessed_radar_contacts(non_neg_integer()) :: [Contact.t()] + def unprocessed_radar_contacts(limit \\ 500), + do: contacts_needing_enrichment(:radar_status, limit, &where(&1, [q], not is_nil(q.pos2))) + + @spec set_enrichment_status!([Ecto.UUID.t()], atom(), atom()) :: {non_neg_integer(), nil | [any()]} + def set_enrichment_status!(ids, field, status) do + Contact |> where([q], q.id in ^ids and field(q, ^field) != ^status) |> Repo.update_all(set: [{field, status}]) + end + + @spec mark_weather_queued!([Ecto.UUID.t()]) :: {non_neg_integer(), nil | [any()]} + def mark_weather_queued!(ids), do: set_enrichment_status!(ids, :weather_status, :queued) + @spec mark_hrrr_queued!([Ecto.UUID.t()]) :: {non_neg_integer(), nil | [any()]} + def mark_hrrr_queued!(ids), do: set_enrichment_status!(ids, :hrrr_status, :queued) + @spec mark_terrain_queued!([Ecto.UUID.t()]) :: {non_neg_integer(), nil | [any()]} + def mark_terrain_queued!(ids), do: set_enrichment_status!(ids, :terrain_status, :queued) + @spec mark_iemre_queued!([Ecto.UUID.t()]) :: {non_neg_integer(), nil | [any()]} + def mark_iemre_queued!(ids), do: set_enrichment_status!(ids, :iemre_status, :queued) + + # ── Path points ── + + @spec contact_path_points(Contact.t()) :: [{float(), float()}] + def contact_path_points(%{pos1: nil}), do: [] + + def contact_path_points(%{pos1: pos1, pos2: nil}) do + lat = pos1["lat"] + lon = pos1["lon"] + if lat && lon, do: [{lat, lon}], else: [] + end + + def contact_path_points(%{pos1: pos1, pos2: pos2}), do: build_path_points(extract_latlon(pos1), extract_latlon(pos2)) + + defp extract_latlon(%{"lat" => lat} = pos) when is_number(lat), do: if(pos["lon"], do: {lat, pos["lon"]}) + defp extract_latlon(_), do: nil + + defp build_path_points({lat1, lon1}, {lat2, lon2}) do + {mid_lat, mid_lon} = great_circle_midpoint(lat1, lon1, lat2, lon2) + [{lat1, lon1}, {mid_lat, mid_lon}, {lat2, lon2}] + end + + defp build_path_points({lat1, lon1}, _), do: [{lat1, lon1}] + defp build_path_points(_, _), do: [] + + defp great_circle_midpoint(lat1, lon1, lat2, lon2) do + rlat1 = deg_to_rad(lat1) + rlat2 = deg_to_rad(lat2) + rlon1 = deg_to_rad(lon1) + dlon = deg_to_rad(lon2 - lon1) + bx = :math.cos(rlat2) * :math.cos(dlon) + by = :math.cos(rlat2) * :math.sin(dlon) + mid_lat = :math.atan2(:math.sin(rlat1) + :math.sin(rlat2), :math.sqrt((:math.cos(rlat1) + bx) ** 2 + by ** 2)) + mid_lon = rlon1 + :math.atan2(by, :math.cos(rlat1) + bx) + {rad_to_deg(mid_lat), normalize_lon(rad_to_deg(mid_lon))} + end + + defp rad_to_deg(rad), do: rad * 180.0 / :math.pi() + defp normalize_lon(lon) when lon > 180.0, do: lon - 360.0 + defp normalize_lon(lon) when lon < -180.0, do: lon + 360.0 + defp normalize_lon(lon), do: lon + defp deg_to_rad(deg), do: deg * :math.pi() / 180 + + # ── Haversine / Backfill ── + + @spec haversine_km(number(), number(), number(), number()) :: float() + defdelegate haversine_km(lat1, lon1, lat2, lon2), to: Microwaveprop.Geo + + @spec backfill_distances([Contact.t()]) :: Postgrex.Result.t() | nil + def backfill_distances(contacts) do + updates = + for contact <- contacts, + is_nil(contact.distance_km) && contact.pos1 && contact.pos2, + lat1 = contact.pos1["lat"], + lon1 = contact.pos1["lon"], + lat2 = contact.pos2["lat"], + lon2 = contact.pos2["lon"], + lat1 && lon1 && lat2 && lon2 do + dist = lat1 |> haversine_km(lon1, lat2, lon2) |> round() + {contact.id, dist} + end + + execute_distance_updates(updates) + end + + defp execute_distance_updates([]), do: nil + + defp execute_distance_updates(updates) do + {values_sql, params} = + updates + |> Enum.with_index() + |> Enum.reduce({"", []}, fn {{id, dist}, i}, {sql, params} -> + frag = "($#{i * 2 + 1}::uuid, $#{i * 2 + 2}::numeric)" + sep = if sql == "", do: "", else: ", " + {sql <> sep <> frag, params ++ [Ecto.UUID.dump!(id), dist]} + end) + + Repo.query!( + "UPDATE contacts AS c SET distance_km = v.dist FROM (VALUES #{values_sql}) AS v(id, dist) WHERE c.id = v.id", + params + ) + end + + # ── Single contact ops ── + + @spec get_contact!(term()) :: Contact.t() + def get_contact!(id), do: Contact |> Repo.get!(id) |> Repo.preload(:flagged_by_user) + + @spec can_view?(Contact.t(), Scope.t() | nil) :: boolean() + def can_view?(%Contact{private: false}, _scope), do: true + def can_view?(%Contact{}, %Scope{user: %User{is_admin: true}}), do: true + def can_view?(%Contact{user_id: user_id}, %Scope{user: %User{id: user_id}}) when not is_nil(user_id), do: true + def can_view?(%Contact{}, _scope), do: false + + @spec delete_contact!(Contact.t()) :: Contact.t() + def delete_contact!(%Contact{} = contact), do: Repo.delete!(contact) + + @spec toggle_flagged_invalid!(Contact.t(), User.t() | nil) :: Contact.t() + def toggle_flagged_invalid!(contact, flagger \\ nil) do + new_state = !contact.flagged_invalid + + attrs = + if new_state do + %{ + flagged_invalid: true, + flagged_by_user_id: flagger && flagger.id, + flagged_at: DateTime.truncate(DateTime.utc_now(), :second) + } + else + %{flagged_invalid: false, flagged_by_user_id: nil, flagged_at: nil} + end + + contact |> Ecto.Changeset.change(attrs) |> Repo.update!() + end + + @spec list_flagged_contacts() :: [Contact.t()] + def list_flagged_contacts do + Contact |> where([c], c.flagged_invalid == true) |> order_by([c], desc: c.inserted_at, desc: c.id) |> Repo.all() + end + + @spec change_contact(Contact.t(), map()) :: Ecto.Changeset.t() + def change_contact(contact, attrs \\ %{}), do: Contact.submission_changeset(contact, attrs) + + @spec owner?(Contact.t(), User.t() | nil) :: boolean() + def owner?(%Contact{user_id: nil}, _user), do: false + def owner?(%Contact{}, nil), do: false + def owner?(%Contact{user_id: uid}, %User{id: uid}), do: true + def owner?(%Contact{}, %User{}), do: false + + # ── Position resolution ── + + @spec ensure_positions!(Contact.t()) :: Contact.t() + def ensure_positions!(contact) do + needs_pos1 = is_nil(contact.pos1) && contact.grid1 + needs_pos2 = is_nil(contact.pos2) && contact.grid2 + if needs_pos1 || needs_pos2, do: do_ensure_positions(contact), else: contact + end + + defp do_ensure_positions(contact) do + pos1 = contact.pos1 || latlon_from_grid(contact.grid1) + pos2 = contact.pos2 || latlon_from_grid(contact.grid2) + distance = compute_distance(pos1, pos2, contact.distance_km) + changes = build_position_changes(contact, pos1, pos2, distance) + apply_position_changes(contact, changes) + end + + defp compute_distance(pos1, pos2, _existing_distance) when not is_nil(pos1) and not is_nil(pos2) do + lon1 = pos1["lon"] + lon2 = pos2["lon"] + pos1["lat"] |> haversine_km(lon1, pos2["lat"], lon2) |> round() |> Decimal.new() + end + + defp compute_distance(_pos1, _pos2, existing_distance), do: existing_distance + + defp build_position_changes(contact, pos1, pos2, distance) do + Enum.reduce([pos1: pos1, pos2: pos2, distance_km: distance], %{}, fn {key, val}, acc -> + if val && is_nil(Map.get(contact, key)), do: Map.put(acc, key, val), else: acc + end) + end + + defp apply_position_changes(contact, changes) when changes == %{}, do: contact + + defp apply_position_changes(contact, changes) do + changes = reset_enrichment_statuses(contact, changes) + contact |> Ecto.Changeset.change(changes) |> Repo.update!() + end + + defp reset_enrichment_statuses(contact, changes) do + new_pos1 = Map.get(changes, :pos1) + new_pos2 = Map.get(changes, :pos2) + + changes = + if new_pos1 do + changes + |> maybe_reset_status(:hrrr_status, contact.hrrr_status) + |> maybe_reset_status(:weather_status, contact.weather_status) + |> maybe_reset_status(:iemre_status, contact.iemre_status) + else + changes + end + + if new_pos1 && (new_pos2 || contact.pos2), + do: maybe_reset_status(changes, :terrain_status, contact.terrain_status), + else: changes + end + + defp maybe_reset_status(changes, field, :unavailable), do: Map.put(changes, field, :pending) + defp maybe_reset_status(changes, field, :complete), do: Map.put(changes, field, :pending) + defp maybe_reset_status(changes, _field, _status), do: changes + + defp latlon_from_grid(nil), do: nil + + defp latlon_from_grid(grid) do + case Maidenhead.to_latlon(grid) do + {:ok, {lat, lon}} -> + %{"lat" => lat, "lon" => lon} + + _ -> + truncated = String.slice(grid, 0, div(String.length(grid), 2) * 2) + + case Maidenhead.to_latlon(truncated) do + {:ok, {lat, lon}} -> %{"lat" => lat, "lon" => lon} + _ -> nil + end + end + end + + # ── Create contact ── + + @spec create_contact(map(), String.t() | nil) :: + {:ok, Contact.t()} | {:error, Ecto.Changeset.t()} | {:error, :duplicate, Contact.t()} + def create_contact(attrs, user_id \\ nil) do + changeset = + Contact.submission_changeset( + %Contact{user_id: user_id, submitter_verified: user_id != nil}, + normalize_band_attr(attrs) + ) + + if changeset.valid?, do: insert_validated_contact(changeset), else: {:error, %{changeset | action: :insert}} + end + + defp normalize_band_attr(attrs) do + cond do + raw = Map.get(attrs, "band") -> + case BandResolver.resolve_as_string(raw) do + nil -> attrs + canonical -> Map.put(attrs, "band", canonical) + end + + raw = Map.get(attrs, :band) -> + case BandResolver.resolve_as_string(raw) do + nil -> attrs + canonical -> Map.put(attrs, :band, canonical) + end + + true -> + attrs + end + end + + defp insert_validated_contact(changeset) do + if existing = find_duplicate_contact(changeset) do + {:error, :duplicate, existing} + else + resolve_grids_and_insert(changeset) + end + end + + defp resolve_grids_and_insert(changeset) do + grid1 = Ecto.Changeset.get_change(changeset, :grid1) + grid2 = Ecto.Changeset.get_change(changeset, :grid2) + + case {Maidenhead.to_latlon(grid1), Maidenhead.to_latlon(grid2)} do + {{:ok, {lat1, lon1}}, {:ok, {lat2, lon2}}} -> + distance = lat1 |> haversine_km(lon1, lat2, lon2) |> round() |> Decimal.new() + + result = + changeset + |> Ecto.Changeset.put_change(:pos1, %{"lat" => lat1, "lon" => lon1}) + |> Ecto.Changeset.put_change(:pos2, %{"lat" => lat2, "lon" => lon2}) + |> Ecto.Changeset.put_change(:distance_km, distance) + |> Ecto.Changeset.put_change(:user_submitted, true) + |> Repo.insert() + + with {:ok, _} <- result, do: invalidate_contact_map_caches() + result + + _ -> + changeset = Ecto.Changeset.add_error(changeset, :grid1, "could not resolve grid to coordinates") + {:error, %{changeset | action: :insert}} + end + rescue + _e in Ecto.ConstraintError -> + case find_duplicate_contact(changeset) do + nil -> {:error, :constraint_error} + existing -> {:error, :duplicate, existing} + end + end + + defp find_duplicate_contact(changeset) do + s1 = changeset |> Ecto.Changeset.get_field(:station1) |> to_string() |> String.upcase() + s2 = changeset |> Ecto.Changeset.get_field(:station2) |> to_string() |> String.upcase() + g1 = changeset |> Ecto.Changeset.get_field(:grid1) |> to_string() |> String.upcase() + g2 = changeset |> Ecto.Changeset.get_field(:grid2) |> to_string() |> String.upcase() + band = Ecto.Changeset.get_field(changeset, :band) + ts = Ecto.Changeset.get_field(changeset, :qso_timestamp) + band_decimal = if is_binary(band), do: Decimal.new(band), else: band + min_ts = DateTime.add(ts, -@dedup_window_seconds, :second) + max_ts = DateTime.add(ts, @dedup_window_seconds, :second) + + Repo.one( + from(c in Contact, + where: + c.band == ^band_decimal and c.qso_timestamp >= ^min_ts and c.qso_timestamp <= ^max_ts and + c.flagged_invalid == false and fragment("LEAST(UPPER(station1), UPPER(station2)) = ?", ^min(s1, s2)) and + fragment("GREATEST(UPPER(station1), UPPER(station2)) = ?", ^max(s1, s2)) and + fragment("LEAST(UPPER(grid1), UPPER(grid2)) = ?", ^min(g1, g2)) and + fragment("GREATEST(UPPER(grid1), UPPER(grid2)) = ?", ^max(g1, g2)) + ) + ) + end + + @doc false + @spec invalidate_contact_map_caches() :: :ok + def invalidate_contact_map_caches do + Cache.invalidate(@count_cache_key) + Cache.invalidate(@map_payload_cache_key) + Cache.invalidate(MicrowavepropWeb.ContactMapController.cache_key()) + :ok + end +end diff --git a/lib/microwaveprop/radio/csv_import.ex b/lib/microwaveprop/radio/csv_import.ex index beb0bd11..b30cf826 100644 --- a/lib/microwaveprop/radio/csv_import.ex +++ b/lib/microwaveprop/radio/csv_import.ex @@ -32,6 +32,7 @@ defmodule Microwaveprop.Radio.CsvImport do alias Microwaveprop.Workers.ContactImportWorker alias Microwaveprop.Workers.ContactWeatherEnqueueWorker + @max_import_rows 2_000 @dedup_window_seconds 3600 @chunk_size 100 @@ -124,22 +125,27 @@ defmodule Microwaveprop.Radio.CsvImport do {:ok, preview_result()} | {:error, :empty_csv} | {:error, :no_data_rows} + | {:error, :too_many_rows} | {:error, {:missing_required_columns, [String.t()]}} def preview(csv_string, submitter_email) do with {:ok, rows} <- parse_csv(TextSanitizer.sanitize(csv_string), submitter_email) do - {valid, invalid} = split_parsed(rows) - existing = load_existing_for_dedup(valid) - {unique, duplicates, refinements} = dedupe(valid, existing) + if length(rows) > @max_import_rows do + {:error, :too_many_rows} + else + {valid, invalid} = split_parsed(rows) + existing = load_existing_for_dedup(valid) + {unique, duplicates, refinements} = dedupe(valid, existing) - {:ok, - %{ - valid: unique, - invalid: invalid, - duplicates: duplicates, - refinements: refinements, - total_rows: length(rows), - submitter_email: submitter_email - }} + {:ok, + %{ + valid: unique, + invalid: invalid, + duplicates: duplicates, + refinements: refinements, + total_rows: length(rows), + submitter_email: submitter_email + }} + end end end @@ -330,19 +336,25 @@ defmodule Microwaveprop.Radio.CsvImport do def chunk_size, do: @chunk_size defp apply_valid_rows(acc, valid_rows) do - Enum.reduce(valid_rows, acc, fn row, acc -> - case Radio.create_contact(row.attrs) do - {:ok, contact} -> - ContactWeatherEnqueueWorker.enqueue_for_contact(contact) - %{acc | imported: [contact | acc.imported]} + {acc, created} = + Enum.reduce(valid_rows, {acc, []}, fn row, {acc, created} -> + case Radio.create_contact(row.attrs) do + {:ok, contact} -> + {%{acc | imported: [contact | acc.imported]}, [contact | created]} - {:error, :duplicate, _existing} -> - acc + {:error, :duplicate, _existing} -> + {acc, created} - {:error, %Ecto.Changeset{} = changeset} -> - %{acc | errors: [{row.row_num, changeset_error_strings(changeset)} | acc.errors]} - end - end) + {:error, %Ecto.Changeset{} = changeset} -> + {%{acc | errors: [{row.row_num, changeset_error_strings(changeset)} | acc.errors]}, created} + end + end) + + if created != [] do + ContactWeatherEnqueueWorker.enqueue_for_contacts(created) + end + + acc end defp apply_refinements(acc, refinements) do diff --git a/lib/microwaveprop/radio/import.ex b/lib/microwaveprop/radio/import.ex new file mode 100644 index 00000000..4a5147f3 --- /dev/null +++ b/lib/microwaveprop/radio/import.ex @@ -0,0 +1,385 @@ +defmodule Microwaveprop.Radio.Import do + @moduledoc false + + import Ecto.Query + + alias Microwaveprop.Accounts.User + alias Microwaveprop.Radio.Contact + alias Microwaveprop.Radio.ContactEdit + alias Microwaveprop.Radio.Contacts + alias Microwaveprop.Radio.Maidenhead + alias Microwaveprop.Repo + alias Microwaveprop.Workers.ContactWeatherEnqueueWorker + + @refinement_fields ~w(grid1 grid2 mode)a + @refinement_allowed_modes ~w(CW SSB FM FT8 FT4 Q65) + @editable_proposed_keys ~w(station1 station2 grid1 grid2 band mode qso_timestamp height1_ft height2_ft private) + + # ── Import refinement ── + + @spec apply_contact_refinement(Contact.t(), map()) :: {:ok, Contact.t()} | {:error, Ecto.Changeset.t()} + def apply_contact_refinement(%Contact{} = contact, changes) when is_map(changes) do + cleaned = Map.take(changes, @refinement_fields) + if cleaned == %{}, do: {:ok, contact}, else: do_apply_contact_refinement(contact, cleaned) + end + + defp do_apply_contact_refinement(contact, changes) do + changeset = + contact + |> Ecto.Changeset.cast(changes, @refinement_fields) + |> validate_refinement_mode() + |> maybe_recompute_refinement_positions(contact, changes) + + if changeset.valid?, do: Repo.update(changeset), else: {:error, %{changeset | action: :update}} + end + + defp validate_refinement_mode(changeset), + do: Ecto.Changeset.validate_inclusion(changeset, :mode, @refinement_allowed_modes) + + defp maybe_recompute_refinement_positions(changeset, contact, changes) do + new_grid1 = Map.get(changes, :grid1) + new_grid2 = Map.get(changes, :grid2) + + cond do + new_grid1 && new_grid2 -> put_pos_changes(changeset, new_grid1, new_grid2, :both) + new_grid1 -> put_pos_changes(changeset, new_grid1, contact.grid2, :side1) + new_grid2 -> put_pos_changes(changeset, contact.grid1, new_grid2, :side2) + true -> changeset + end + end + + defp put_pos_changes(changeset, grid1, grid2, sides) do + case {Maidenhead.to_latlon(grid1), Maidenhead.to_latlon(grid2)} do + {{:ok, {lat1, lon1}}, {:ok, {lat2, lon2}}} -> + distance = lat1 |> Contacts.haversine_km(lon1, lat2, lon2) |> round() |> Decimal.new() + + changeset + |> maybe_put_pos_for_side(:pos1, %{"lat" => lat1, "lon" => lon1}, sides) + |> maybe_put_pos_for_side(:pos2, %{"lat" => lat2, "lon" => lon2}, sides) + |> Ecto.Changeset.put_change(:distance_km, distance) + |> reset_all_enrichment_statuses() + + _ -> + Ecto.Changeset.add_error(changeset, :grid1, "could not resolve grid to coordinates") + end + end + + defp maybe_put_pos_for_side(changeset, :pos1, pos, sides) when sides in [:side1, :both], + do: Ecto.Changeset.put_change(changeset, :pos1, pos) + + defp maybe_put_pos_for_side(changeset, :pos2, pos, sides) when sides in [:side2, :both], + do: Ecto.Changeset.put_change(changeset, :pos2, pos) + + defp maybe_put_pos_for_side(changeset, _field, _pos, _sides), do: changeset + + defp reset_all_enrichment_statuses(changeset) do + changeset + |> Ecto.Changeset.put_change(:hrrr_status, :pending) + |> Ecto.Changeset.put_change(:weather_status, :pending) + |> Ecto.Changeset.put_change(:terrain_status, :pending) + |> Ecto.Changeset.put_change(:iemre_status, :pending) + end + + # ── Contact edits ── + + @spec create_contact_edit(Contact.t(), User.t(), map()) :: {:ok, ContactEdit.t()} | {:error, Ecto.Changeset.t()} + def create_contact_edit(%Contact{} = contact, user, proposed_changes) when is_map(proposed_changes) do + normalized = normalize_proposed(proposed_changes) + diffed = diff_against_contact(contact, normalized) + attrs = %{contact_id: contact.id, user_id: user.id, proposed_changes: diffed} + %ContactEdit{} |> ContactEdit.changeset(attrs) |> Repo.insert() + end + + @doc false + @spec normalize_proposed(map()) :: map() + def normalize_proposed(changes) do + changes + |> Map.take(@editable_proposed_keys) + |> normalize_string_field("station1") + |> normalize_string_field("station2") + |> normalize_string_field("grid1") + |> normalize_string_field("grid2") + |> normalize_string_field("mode") + |> normalize_integer_field("height1_ft") + |> normalize_integer_field("height2_ft") + |> normalize_boolean_field("private") + |> normalize_timestamp_field("qso_timestamp") + end + + defp normalize_string_field(map, key) do + case Map.get(map, key) do + nil -> map + val when is_binary(val) -> Map.put(map, key, val |> String.trim() |> String.upcase()) + val -> Map.put(map, key, val) + end + end + + defp normalize_integer_field(map, key) do + case Map.get(map, key, :not_provided) do + :not_provided -> + map + + nil -> + map + + val when is_integer(val) -> + map + + "" -> + Map.delete(map, key) + + val when is_binary(val) -> + case Integer.parse(String.trim(val)) do + {int, ""} -> Map.put(map, key, int) + _ -> map + end + + _ -> + map + end + end + + defp normalize_boolean_field(map, key) do + case Map.get(map, key, :not_provided) do + :not_provided -> map + nil -> map + val when is_boolean(val) -> Map.put(map, key, val) + "true" -> Map.put(map, key, true) + "false" -> Map.put(map, key, false) + _ -> map + end + end + + defp normalize_timestamp_field(map, key) do + case Map.get(map, key, :not_provided) do + :not_provided -> map + nil -> Map.delete(map, key) + "" -> Map.delete(map, key) + %DateTime{} = dt -> Map.put(map, key, dt) + val when is_binary(val) -> put_or_drop_timestamp(map, key, parse_timestamp(val)) + _ -> Map.delete(map, key) + end + end + + defp put_or_drop_timestamp(map, key, nil), do: Map.delete(map, key) + defp put_or_drop_timestamp(map, key, %DateTime{} = dt), do: Map.put(map, key, dt) + + defp parse_timestamp(val) do + trimmed = String.trim(val) + iso = trimmed |> String.replace(" ", "T") |> ensure_seconds() |> String.trim_trailing("Z") + + case NaiveDateTime.from_iso8601(iso) do + {:ok, ndt} -> DateTime.from_naive!(ndt, "Etc/UTC") + _ -> nil + end + end + + defp ensure_seconds(s) do + case Regex.run(~r/^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2})(Z?)$/, s) do + [_, prefix, tail] -> prefix <> ":00" <> tail + _ -> s + end + end + + @doc false + @spec diff_against_contact(Contact.t(), map()) :: map() + def diff_against_contact(contact, proposed) do + Enum.reduce(proposed, %{}, fn {key, new_val}, acc -> + current = current_value(contact, key) + if values_equal?(current, new_val), do: acc, else: Map.put(acc, key, new_val) + end) + end + + defp current_value(contact, "station1"), do: contact.station1 + defp current_value(contact, "station2"), do: contact.station2 + defp current_value(contact, "grid1"), do: contact.grid1 + defp current_value(contact, "grid2"), do: contact.grid2 + defp current_value(contact, "mode"), do: contact.mode + defp current_value(contact, "band"), do: if(contact.band, do: Decimal.to_integer(contact.band)) + defp current_value(contact, "qso_timestamp"), do: contact.qso_timestamp + defp current_value(contact, "height1_ft"), do: contact.height1_ft + defp current_value(contact, "height2_ft"), do: contact.height2_ft + defp current_value(contact, "private"), do: contact.private + defp current_value(_contact, _key), do: nil + + defp values_equal?(a, b) when is_binary(a) and is_binary(b), do: String.upcase(a) == String.upcase(b) + defp values_equal?(a, b) when is_integer(a), do: a == to_integer(b) + defp values_equal?(a, b) when is_integer(b), do: to_integer(a) == b + defp values_equal?(a, b), do: a == b + + defp to_integer(v) when is_integer(v), do: v + defp to_integer(v) when is_binary(v), do: String.to_integer(v) + defp to_integer(%Decimal{} = v), do: Decimal.to_integer(v) + defp to_integer(v), do: v + + # ── Pending edits ── + + @spec list_pending_edits() :: [ContactEdit.t()] + def list_pending_edits do + ContactEdit + |> where([e], e.status == :pending) + |> order_by([e], desc: e.inserted_at) + |> preload([:user, :contact]) + |> Repo.all() + end + + @spec pending_edits_query() :: Ecto.Query.t() + def pending_edits_query do + from e in ContactEdit, as: :resource, where: e.status == :pending, preload: [:user, :contact] + end + + @spec list_contact_edits(Ecto.UUID.t()) :: [ContactEdit.t()] + def list_contact_edits(contact_id) do + ContactEdit + |> where([e], e.contact_id == ^contact_id) + |> order_by([e], desc: e.inserted_at) + |> preload([:user, :reviewed_by]) + |> Repo.all() + end + + @spec pending_edit_count() :: non_neg_integer() + def pending_edit_count, do: ContactEdit |> where([e], e.status == :pending) |> Repo.aggregate(:count) + + @spec pending_edit_for_user(Ecto.UUID.t(), Ecto.UUID.t()) :: ContactEdit.t() | nil + def pending_edit_for_user(contact_id, user_id) do + ContactEdit + |> where([e], e.contact_id == ^contact_id and e.user_id == ^user_id and e.status == :pending) + |> preload([:user, :contact]) + |> Repo.one() + end + + @spec get_contact_edit(Ecto.UUID.t()) :: ContactEdit.t() | nil + def get_contact_edit(id), do: ContactEdit |> preload([:user, :contact, :reviewed_by]) |> Repo.get(id) + + # ── Edit approval ── + + @spec approve_edit(ContactEdit.t(), User.t(), String.t() | nil) :: {:ok, ContactEdit.t()} | {:error, any()} + def approve_edit(%ContactEdit{status: :pending} = edit, admin, note) do + Repo.transaction(fn -> + case Repo.get(Contact, edit.contact_id) do + nil -> + Repo.rollback(:contact_deleted) + + contact -> + {:ok, approved} = + edit + |> ContactEdit.review_changeset(%{ + status: :approved, + admin_note: note, + reviewed_by_id: admin.id, + reviewed_at: DateTime.truncate(DateTime.utc_now(), :second) + }) + |> Repo.update() + + _ = apply_edit_to_contact(contact, edit.proposed_changes) + Repo.preload(approved, [:user, :contact, :reviewed_by]) + end + end) + end + + @spec reject_edit(ContactEdit.t(), User.t(), String.t() | nil) :: + {:ok, ContactEdit.t()} | {:error, Ecto.Changeset.t()} + def reject_edit(%ContactEdit{status: :pending} = edit, admin, note) do + edit + |> ContactEdit.review_changeset(%{ + status: :rejected, + admin_note: note, + reviewed_by_id: admin.id, + reviewed_at: DateTime.truncate(DateTime.utc_now(), :second) + }) + |> Repo.update() + end + + # ── Owner edit ── + + @spec apply_owner_edit(Contact.t(), User.t(), map()) :: {:ok, Contact.t()} | {:error, :not_owner | :no_changes} + def apply_owner_edit(%Contact{} = contact, %User{} = user, proposed_changes) when is_map(proposed_changes) do + if Contacts.owner?(contact, user) do + diffed = diff_against_contact(contact, normalize_proposed(proposed_changes)) + if diffed == %{}, do: {:error, :no_changes}, else: {:ok, apply_edit_to_contact(contact, diffed)} + else + {:error, :not_owner} + end + end + + @doc "Apply proposed changes directly to a contact (admin use)." + @spec apply_edit_to_contact(Contact.t(), map()) :: Contact.t() + def apply_edit_to_contact(contact, proposed_changes) do + changes = build_contact_changes(contact, proposed_changes) + changes = maybe_recompute_positions(changes, contact, proposed_changes) + changes = maybe_reset_terrain_for_heights(changes, proposed_changes) + + updated = contact |> Ecto.Changeset.change(changes) |> Repo.update!() + maybe_enqueue_terrain_for_heights(updated, proposed_changes) + Contacts.invalidate_contact_map_caches() + updated + end + + defp maybe_enqueue_terrain_for_heights(contact, proposed) do + if Map.has_key?(proposed, "height1_ft") or Map.has_key?(proposed, "height2_ft") do + ContactWeatherEnqueueWorker.enqueue_for_contact(contact, [:terrain]) + end + + :ok + end + + defp maybe_reset_terrain_for_heights(changes, proposed) do + if Map.has_key?(proposed, "height1_ft") or Map.has_key?(proposed, "height2_ft"), + do: Map.put(changes, :terrain_status, :pending), + else: changes + end + + defp maybe_recompute_positions(changes, contact, proposed) do + needs_recompute = + Map.has_key?(proposed, "grid1") or Map.has_key?(proposed, "grid2") or Map.has_key?(proposed, "band") or + Map.has_key?(proposed, "qso_timestamp") + + if needs_recompute, do: recompute_positions(changes, contact, proposed), else: changes + end + + defp recompute_positions(changes, contact, proposed) do + pos1 = resolve_grid_position(Map.get(proposed, "grid1", contact.grid1)) || contact.pos1 + pos2 = resolve_grid_position(Map.get(proposed, "grid2", contact.grid2)) || contact.pos2 + distance = compute_distance(pos1, pos2) + + changes + |> Map.put(:pos1, pos1) + |> Map.put(:pos2, pos2) + |> Map.put(:distance_km, distance) + |> Map.put(:hrrr_status, :pending) + |> Map.put(:weather_status, :pending) + |> Map.put(:terrain_status, :pending) + |> Map.put(:iemre_status, :pending) + end + + defp compute_distance(pos1, pos2) when not is_nil(pos1) and not is_nil(pos2), + do: pos1["lat"] |> Contacts.haversine_km(pos1["lon"], pos2["lat"], pos2["lon"]) |> round() |> Decimal.new() + + defp compute_distance(_, _), do: nil + + defp build_contact_changes(_contact, proposed) do + Enum.reduce(proposed, %{}, fn + {"band", val}, acc -> + Map.put(acc, :band, Decimal.new(to_string(val))) + + {"qso_timestamp", val}, acc when is_binary(val) -> + {:ok, dt, _} = DateTime.from_iso8601(val) + Map.put(acc, :qso_timestamp, dt) + + {"qso_timestamp", %DateTime{} = dt}, acc -> + Map.put(acc, :qso_timestamp, dt) + + {key, val}, acc -> + Map.put(acc, String.to_existing_atom(key), val) + end) + end + + defp resolve_grid_position(nil), do: nil + + defp resolve_grid_position(grid) do + case Maidenhead.to_latlon(grid) do + {:ok, {lat, lon}} -> %{"lat" => lat, "lon" => lon} + _ -> nil + end + end +end diff --git a/lib/microwaveprop/rover/drive_time.ex b/lib/microwaveprop/rover/drive_time.ex index 1b94ff82..1380d0f7 100644 --- a/lib/microwaveprop/rover/drive_time.ex +++ b/lib/microwaveprop/rover/drive_time.ex @@ -4,21 +4,11 @@ defmodule Microwaveprop.Rover.DriveTime do planner. All inputs are degrees / kilometres; no I/O. """ - @earth_radius_km 6371.0 @avg_speed_kmh 65.0 @spec haversine_km({float(), float()}, {float(), float()}) :: float() def haversine_km({lat1, lon1}, {lat2, lon2}) do - rlat1 = deg_to_rad(lat1) - rlat2 = deg_to_rad(lat2) - dlat = deg_to_rad(lat2 - lat1) - dlon = deg_to_rad(lon2 - lon1) - - a = - :math.sin(dlat / 2) ** 2 + - :math.cos(rlat1) * :math.cos(rlat2) * :math.sin(dlon / 2) ** 2 - - 2 * @earth_radius_km * :math.asin(:math.sqrt(a)) + Microwaveprop.Geo.haversine_km(lat1, lon1, lat2, lon2) end @spec drive_min(float()) :: float() diff --git a/lib/microwaveprop/weather.ex b/lib/microwaveprop/weather.ex index c564ec0b..cca46ee3 100644 --- a/lib/microwaveprop/weather.ex +++ b/lib/microwaveprop/weather.ex @@ -1,433 +1,144 @@ defmodule Microwaveprop.Weather do @moduledoc false - import Ecto.Query - alias Ecto.UUID - alias Microwaveprop.Propagation.Grid - alias Microwaveprop.Propagation.ProfilesFile - alias Microwaveprop.Radio - alias Microwaveprop.Radio.Contact alias Microwaveprop.Repo alias Microwaveprop.Weather.GefsProfile - alias Microwaveprop.Weather.GridCache - alias Microwaveprop.Weather.HrrrNativeProfile - alias Microwaveprop.Weather.HrrrProfile - alias Microwaveprop.Weather.IemClient - alias Microwaveprop.Weather.IemreObservation - alias Microwaveprop.Weather.NarrProfile - alias Microwaveprop.Weather.ProfileLookup - alias Microwaveprop.Weather.ScalarFile - alias Microwaveprop.Weather.SolarIndex - alias Microwaveprop.Weather.Sounding - alias Microwaveprop.Weather.SoundingParams - alias Microwaveprop.Weather.Station - alias Microwaveprop.Weather.SurfaceObservation - alias Microwaveprop.Weather.WeatherLayers + alias Microwaveprop.Weather.Grid + alias Microwaveprop.Weather.Hrrr + alias Microwaveprop.Weather.Iemre + alias Microwaveprop.Weather.Narr + alias Microwaveprop.Weather.Soundings + alias Microwaveprop.Weather.Surface require Logger - # Approximate km per degree latitude - @km_per_deg_lat 111.0 + # ── Delegates to sub-facades ── - # Profile-lookup delegates — implementations live in ProfileLookup - defdelegate find_nearest_hrrr(lat, lon, timestamp), to: ProfileLookup - defdelegate hrrr_profiles_for_path(contact), to: ProfileLookup - defdelegate hrrr_profiles_for_contacts(contacts), to: ProfileLookup - defdelegate find_nearest_native_profile(lat, lon, timestamp), to: ProfileLookup - defdelegate narr_profiles_for_path(contact), to: ProfileLookup - defdelegate find_nearest_narr(lat, lon, timestamp), to: ProfileLookup - defdelegate best_profile_for_contact(contact), to: ProfileLookup - defdelegate hrrr_for_contact(contact), to: ProfileLookup - defdelegate narr_for_contact(contact), to: ProfileLookup - defdelegate hrrr_data_fully_present?(contact), to: ProfileLookup - defdelegate has_hrrr_profile?(lat, lon, valid_time), to: ProfileLookup - defdelegate hrrr_points_present_batch(points), to: ProfileLookup - defdelegate iemre_for_contact(contact), to: ProfileLookup - defdelegate iemre_for_path(contact), to: ProfileLookup - defdelegate find_nearest_iemre(lat, lon, timestamp), to: ProfileLookup - defdelegate round_to_hrrr_grid(lat, lon), to: ProfileLookup - defdelegate round_to_iemre_grid(lat, lon), to: ProfileLookup - defdelegate purge_grid_point_profiles(), to: ProfileLookup + # HRRR + defdelegate find_nearest_hrrr(lat, lon, timestamp), to: Hrrr + defdelegate hrrr_profiles_for_path(contact), to: Hrrr + defdelegate hrrr_profiles_for_contacts(contacts), to: Hrrr + defdelegate find_nearest_native_profile(lat, lon, timestamp), to: Hrrr + defdelegate best_profile_for_contact(contact), to: Hrrr + defdelegate hrrr_for_contact(contact), to: Hrrr + defdelegate hrrr_data_fully_present?(contact), to: Hrrr + defdelegate has_hrrr_profile?(lat, lon, valid_time), to: Hrrr + defdelegate hrrr_points_present_batch(points), to: Hrrr + defdelegate round_to_hrrr_grid(lat, lon), to: Hrrr + defdelegate purge_grid_point_profiles(), to: Hrrr + defdelegate upsert_hrrr_profile(attrs), to: Hrrr + defdelegate upsert_hrrr_profiles_batch(profiles, opts \\ []), to: Hrrr + defdelegate nearest_native_duct_ghz(lat, lon, timestamp), to: Hrrr + defdelegate nearest_native_duct_info(lat, lon, timestamp), to: Hrrr + defdelegate backfill_hrrr_scalars(opts \\ []), to: Hrrr + defdelegate reconcile_hrrr_statuses(), to: Hrrr + defdelegate profiles_along_path(contact), to: Hrrr - @spec find_or_create_station(map()) :: {:ok, Station.t()} | {:error, Ecto.Changeset.t()} - def find_or_create_station(attrs) do - code = attrs[:station_code] || attrs["station_code"] - type = attrs[:station_type] || attrs["station_type"] + # NARR + defdelegate narr_profiles_for_path(contact), to: Narr + defdelegate find_nearest_narr(lat, lon, timestamp), to: Narr + defdelegate narr_for_contact(contact), to: Narr - if code && type do - case Repo.get_by(Station, station_code: code, station_type: type) do - nil -> - %Station{} - |> Station.changeset(attrs) - |> Repo.insert() + # IEMRE + defdelegate iemre_for_contact(contact), to: Iemre + defdelegate iemre_for_path(contact), to: Iemre + defdelegate find_nearest_iemre(lat, lon, timestamp), to: Iemre + defdelegate round_to_iemre_grid(lat, lon), to: Iemre + defdelegate upsert_iemre_observation(attrs), to: Iemre + defdelegate has_iemre_observation?(lat, lon, date), to: Iemre + defdelegate reconcile_iemre_statuses(), to: Iemre - station -> - {:ok, station} - end + # Soundings + defdelegate upsert_sounding(station, attrs), to: Soundings + defdelegate has_sounding?(station_id, observed_at), to: Soundings + defdelegate station_ids_with_soundings(station_ids, sounding_times), to: Soundings + defdelegate sounding_times_around(dt), to: Soundings + defdelegate weather_for_contact(contact_params, opts \\ []), to: Soundings + defdelegate soundings_with_widening_radius(params), to: Soundings + defdelegate nearest_sounding_to(lat, lon, timestamp, opts \\ []), to: Soundings + + # Surface + defdelegate find_or_create_station(attrs), to: Surface + defdelegate upsert_surface_observation(station, attrs), to: Surface + defdelegate upsert_surface_observations(station, rows), to: Surface + defdelegate has_surface_observations?(station_id, start_dt, end_dt), to: Surface + defdelegate station_day_covered?(station_id, date), to: Surface + defdelegate station_day_pairs_covered(pairs), to: Surface + defdelegate station_ids_with_surface_observations(station_ids, start_dt, end_dt), to: Surface + defdelegate nearby_stations(lat, lon, station_type, radius_km), to: Surface + defdelegate sync_stations!(), to: Surface + defdelegate reconcile_weather_statuses(), to: Surface + defdelegate upsert_solar_index(attrs), to: Surface + defdelegate upsert_solar_indices_batch(records), to: Surface + defdelegate get_solar_index(date), to: Surface + defdelegate existing_solar_dates(), to: Surface + + # Grid + defdelegate latest_grid_valid_time(), to: Grid + defdelegate latest_weather_grid(bounds), to: Grid + defdelegate load_weather_grid(bounds), to: Grid + defdelegate available_weather_valid_times(), to: Grid + defdelegate available_hrdps_valid_times(), to: Grid + defdelegate weather_grid_hrdps_at(valid_time, bounds), to: Grid + defdelegate weather_grid_at(valid_time, bounds), to: Grid + defdelegate warm_grid_cache_and_broadcast(valid_time), to: Grid + defdelegate warm_grid_cache_from_latest_profile(), to: Grid + defdelegate materialize_scalar_file(valid_time), to: Grid + defdelegate build_grid_cache_rows(grid_data, valid_time, bounds \\ nil), to: Grid + defdelegate weather_point_detail(lat, lon, valid_time), to: Grid + + # ── Remaining inline: GEFS ── + + @spec upsert_gefs_profile(map()) :: {:ok, GefsProfile.t()} | {:error, Ecto.Changeset.t()} + def upsert_gefs_profile(attrs) do + changeset = GefsProfile.changeset(%GefsProfile{}, attrs) + + if changeset.valid? do + Repo.insert(changeset, + on_conflict: :nothing, + conflict_target: [:lat, :lon, :valid_time] + ) else - %Station{} - |> Station.changeset(attrs) - |> Repo.insert() + {:error, changeset} end end - @spec upsert_surface_observation(Station.t(), map()) :: {:ok, SurfaceObservation.t()} | {:error, Ecto.Changeset.t()} - def upsert_surface_observation(%Station{} = station, attrs) do - attrs = Map.put(attrs, :station_id, station.id) - - %SurfaceObservation{} - |> SurfaceObservation.changeset(attrs) - |> Repo.insert( - on_conflict: - from(s in SurfaceObservation, - update: [ - set: [ - temp_f: fragment("EXCLUDED.temp_f"), - dewpoint_f: fragment("EXCLUDED.dewpoint_f"), - relative_humidity: fragment("EXCLUDED.relative_humidity"), - wind_speed_kts: fragment("EXCLUDED.wind_speed_kts"), - sea_level_pressure_mb: fragment("EXCLUDED.sea_level_pressure_mb"), - sky_condition: fragment("EXCLUDED.sky_condition"), - precip_1h_in: fragment("EXCLUDED.precip_1h_in"), - wx_codes: fragment("EXCLUDED.wx_codes"), - updated_at: fragment("EXCLUDED.updated_at") - ] - ], - where: - s.temp_f != fragment("EXCLUDED.temp_f") or - s.dewpoint_f != fragment("EXCLUDED.dewpoint_f") or - s.relative_humidity != fragment("EXCLUDED.relative_humidity") or - s.wind_speed_kts != fragment("EXCLUDED.wind_speed_kts") or - s.sea_level_pressure_mb != fragment("EXCLUDED.sea_level_pressure_mb") - ), - conflict_target: [:station_id, :observed_at], - returning: true, - stale_error_field: :id + @spec upsert_gefs_profiles_batch([map()]) :: {non_neg_integer(), nil} + def upsert_gefs_profiles_batch(profiles) do + Microwaveprop.Instrument.span( + [:db, :upsert_gefs_profiles], + %{count: length(profiles)}, + fn -> do_upsert_gefs_profiles_batch(profiles) end ) end - @doc """ - Bulk-upsert surface observations for a single station via one - `Repo.insert_all` round-trip. ASOS fetches return 24–288 rows per - station per call — collapsing them into a single statement avoids the - per-row UPDATE-conflict round-trip of `upsert_surface_observation/2`, - which is expensive against the Turing Pi 2 Postgres node. - - Rows missing `observed_at` are dropped (they can't satisfy the - `(station_id, observed_at)` unique index). Returns the - `{count, nil}` tuple from `Repo.insert_all/3`; `count` reflects - affected rows (new + updated-when-changed). Returns `{0, nil}` for - an empty input without touching the DB. - """ - @spec upsert_surface_observations(Station.t(), [map()]) :: {non_neg_integer(), nil} - def upsert_surface_observations(%Station{} = station, rows) when is_list(rows) do + defp do_upsert_gefs_profiles_batch(profiles) do now = DateTime.truncate(DateTime.utc_now(), :second) - entries = - rows - |> Enum.filter(&row_has_observed_at?/1) - |> Enum.map(&surface_observation_entry(&1, station.id, now)) - |> dedupe_last_by_conflict_target() - - case entries do - [] -> - {0, nil} - - _ -> - Repo.insert_all(SurfaceObservation, entries, - on_conflict: - from(s in SurfaceObservation, - update: [ - set: [ - temp_f: fragment("EXCLUDED.temp_f"), - dewpoint_f: fragment("EXCLUDED.dewpoint_f"), - relative_humidity: fragment("EXCLUDED.relative_humidity"), - wind_speed_kts: fragment("EXCLUDED.wind_speed_kts"), - wind_direction_deg: fragment("EXCLUDED.wind_direction_deg"), - sea_level_pressure_mb: fragment("EXCLUDED.sea_level_pressure_mb"), - altimeter_setting: fragment("EXCLUDED.altimeter_setting"), - sky_condition: fragment("EXCLUDED.sky_condition"), - precip_1h_in: fragment("EXCLUDED.precip_1h_in"), - wx_codes: fragment("EXCLUDED.wx_codes"), - updated_at: fragment("EXCLUDED.updated_at") - ] - ], - where: - s.temp_f != fragment("EXCLUDED.temp_f") or - s.dewpoint_f != fragment("EXCLUDED.dewpoint_f") or - s.relative_humidity != fragment("EXCLUDED.relative_humidity") or - s.wind_speed_kts != fragment("EXCLUDED.wind_speed_kts") or - s.sea_level_pressure_mb != fragment("EXCLUDED.sea_level_pressure_mb") - ), - conflict_target: [:station_id, :observed_at] - ) - end - end - - defp row_has_observed_at?(%{observed_at: %DateTime{}}), do: true - defp row_has_observed_at?(%{"observed_at" => %DateTime{}}), do: true - defp row_has_observed_at?(_), do: false - - # IEM ASOS occasionally returns two rows with the same timestamp - # (e.g. routine + special METAR at the same minute). Passing both - # to insert_all triggers Postgres 21000 cardinality_violation. - # Keep the last occurrence — IEM's ordering is chronological, so - # the later row is the final correction. - defp dedupe_last_by_conflict_target(entries) do - entries - |> Enum.reverse() - |> Enum.uniq_by(&{&1.station_id, &1.observed_at}) - |> Enum.reverse() - end - - defp surface_observation_entry(row, station_id, now) do - %{ - id: UUID.generate(), - station_id: station_id, - observed_at: fetch_row(row, :observed_at), - temp_f: fetch_row(row, :temp_f), - dewpoint_f: fetch_row(row, :dewpoint_f), - relative_humidity: fetch_row(row, :relative_humidity), - wind_speed_kts: fetch_row(row, :wind_speed_kts), - wind_direction_deg: fetch_row(row, :wind_direction_deg), - sea_level_pressure_mb: fetch_row(row, :sea_level_pressure_mb), - altimeter_setting: fetch_row(row, :altimeter_setting), - sky_condition: fetch_row(row, :sky_condition), - precip_1h_in: fetch_row(row, :precip_1h_in), - wx_codes: fetch_row(row, :wx_codes), - inserted_at: now, - updated_at: now - } - end - - defp fetch_row(row, key) when is_atom(key) do - case Map.fetch(row, key) do - {:ok, value} -> value - :error -> Map.get(row, Atom.to_string(key)) - end - end - - @spec upsert_sounding(Station.t(), map()) :: {:ok, Sounding.t()} | {:error, Ecto.Changeset.t()} - def upsert_sounding(%Station{} = station, attrs) do - attrs = Map.put(attrs, :station_id, station.id) - - %Sounding{} - |> Sounding.changeset(attrs) - |> Repo.insert( - on_conflict: - from(s in Sounding, - update: [ - set: [ - profile: fragment("EXCLUDED.profile"), - level_count: fragment("EXCLUDED.level_count"), - surface_pressure_mb: fragment("EXCLUDED.surface_pressure_mb"), - surface_temp_c: fragment("EXCLUDED.surface_temp_c"), - surface_dewpoint_c: fragment("EXCLUDED.surface_dewpoint_c"), - surface_refractivity: fragment("EXCLUDED.surface_refractivity"), - min_refractivity_gradient: fragment("EXCLUDED.min_refractivity_gradient"), - boundary_layer_depth_m: fragment("EXCLUDED.boundary_layer_depth_m"), - precipitable_water_mm: fragment("EXCLUDED.precipitable_water_mm"), - k_index: fragment("EXCLUDED.k_index"), - lifted_index: fragment("EXCLUDED.lifted_index"), - ducting_detected: fragment("EXCLUDED.ducting_detected"), - duct_characteristics: fragment("EXCLUDED.duct_characteristics"), - updated_at: fragment("EXCLUDED.updated_at") - ] - ], - where: - s.level_count != fragment("EXCLUDED.level_count") or - s.surface_temp_c != fragment("EXCLUDED.surface_temp_c") or - s.surface_refractivity != fragment("EXCLUDED.surface_refractivity") - ), - conflict_target: [:station_id, :observed_at], - returning: true, - stale_error_field: :id - ) - end - - @spec upsert_solar_index(map()) :: {:ok, SolarIndex.t()} | {:error, Ecto.Changeset.t()} - def upsert_solar_index(attrs) do - %SolarIndex{} - |> SolarIndex.changeset(attrs) - |> Repo.insert( - on_conflict: - from(s in SolarIndex, - update: [ - set: [ - sfi: fragment("EXCLUDED.sfi"), - sfi_adjusted: fragment("EXCLUDED.sfi_adjusted"), - sunspot_number: fragment("EXCLUDED.sunspot_number"), - ap_index: fragment("EXCLUDED.ap_index"), - kp_values: fragment("EXCLUDED.kp_values"), - updated_at: fragment("EXCLUDED.updated_at") - ] - ], - where: - s.sfi != fragment("EXCLUDED.sfi") or - s.ap_index != fragment("EXCLUDED.ap_index") - ), - conflict_target: [:date], - returning: true, - stale_error_field: :id - ) - end - - @spec has_surface_observations?(UUID.t(), DateTime.t(), DateTime.t()) :: boolean() - def has_surface_observations?(station_id, start_dt, end_dt) do - SurfaceObservation - |> where([o], o.station_id == ^station_id) - |> where([o], o.observed_at >= ^start_dt and o.observed_at <= ^end_dt) - |> Repo.exists?() - end - - @doc """ - Flip `contacts.iemre_status` from `:queued` to `:complete` for every - contact whose path points all have an `iemre_observations` row at - the rounded grid cell for the QSO's UTC date. - - Per-contact check (via the existing helpers); kept in Elixir because - `contact_path_points/1` produces 1–3 points depending on whether - `pos2` is set, and rounding to the 0.125° IEMRE grid per point in - pure SQL is awkward. Queue sizes are small (<20 stuck at steady - state). - """ - @spec reconcile_iemre_statuses() :: {:ok, non_neg_integer()} - def reconcile_iemre_statuses do - queued = - Contact - |> where([c], c.iemre_status == :queued and not is_nil(c.pos1)) - |> Repo.all() - - to_complete = - Enum.filter(queued, fn c -> - date = DateTime.to_date(c.qso_timestamp) - - c - |> Radio.contact_path_points() - |> Enum.all?(fn {lat, lon} -> - {rlat, rlon} = round_to_iemre_grid(lat, lon) - has_iemre_observation?(rlat, rlon, date) + profiles + |> Enum.chunk_every(500) + |> Enum.reduce({0, nil}, fn chunk, {total_count, _} -> + entries = + Enum.map(chunk, fn attrs -> + Map.merge(attrs, %{ + id: UUID.generate(), + inserted_at: now, + updated_at: now + }) end) - end) - if to_complete == [] do - {:ok, 0} - else - ids = Enum.map(to_complete, & &1.id) - _ = Radio.set_enrichment_status!(ids, :iemre_status, :complete) - {:ok, length(to_complete)} - end - end + {count, rows} = + Repo.insert_all(GefsProfile, entries, + on_conflict: :nothing, + conflict_target: [:lat, :lon, :valid_time] + ) - @doc """ - Reconcile `contacts.hrrr_status` for every `:queued` contact: - - * flips to `:complete` when every path point has an hrrr_profiles - row at the rounded HRRR hour; - * flips to `:unavailable` when any path point falls outside the - HRRR CONUS grid (pos2 or the great-circle midpoint can drift - into Canada, the mid-Atlantic, or the Pacific even when pos1 is - stateside — the Rust hrrr-point-worker returns no profile for - those points and the contact would otherwise sit `:queued` - forever). - - Returns `{:ok, n}` where `n` is the total number of contacts advanced - (completed + oconus). - """ - @spec reconcile_hrrr_statuses() :: {:ok, non_neg_integer()} - def reconcile_hrrr_statuses do - queued = - Contact - |> where([c], c.hrrr_status == :queued and not is_nil(c.pos1)) - |> Repo.all() - - {oconus, rest} = Enum.split_with(queued, &contact_has_oconus_path_point?/1) - to_complete = Enum.filter(rest, &hrrr_data_fully_present?/1) - - _ = - if oconus != [] do - ids = Enum.map(oconus, & &1.id) - Radio.set_enrichment_status!(ids, :hrrr_status, :unavailable) - end - - _ = - if to_complete != [] do - ids = Enum.map(to_complete, & &1.id) - Radio.set_enrichment_status!(ids, :hrrr_status, :complete) - end - - {:ok, length(oconus) + length(to_complete)} - end - - defp contact_has_oconus_path_point?(contact) do - contact - |> Radio.contact_path_points() - |> Enum.any?(fn {lat, lon} -> - not Grid.contains?(%{"lat" => lat, "lon" => lon}) + {total_count + count, rows} end) end - @doc """ - Flip `contacts.weather_status` from `:queued` to `:complete` for - every contact whose ±2h / 150km window now contains at least one - surface observation. Returns `{:ok, n}` where `n` is the number of - contacts advanced. - - Background: `WeatherFetchWorker` upserts observations but has no - back-pointer to the contacts that triggered the fetch — the same - obs row satisfies many contacts. Previously the only place that - flipped `:queued → :complete` was `MicrowavepropWeb.ContactLive.Show` - on page view, which left thousands of contacts stuck in `:queued` - even after their data had landed. This reconciler closes that loop - as a single SQL UPDATE, invoked from the hourly enqueuer cron. - - Radius encoded as a ±1.5° latitude band; the longitude band is - scaled by `1 / cos(lat)` so the box covers the same physical - east-west distance (~150 km) at every latitude. A fixed 1.5° lon - box collapses to ~110 km at lat 49° and would silently skip - observations the per-contact `weather_for_contact/2` query would - match. - """ - @spec reconcile_weather_statuses() :: {:ok, non_neg_integer()} - def reconcile_weather_statuses do - sql = """ - UPDATE contacts c - SET weather_status = 'complete' - WHERE c.weather_status = 'queued' - AND c.pos1 IS NOT NULL - AND ( - EXISTS ( - SELECT 1 - FROM surface_observations o - JOIN weather_stations s ON s.id = o.station_id - WHERE o.observed_at >= c.qso_timestamp - interval '2 hours' - AND o.observed_at <= c.qso_timestamp + interval '2 hours' - AND s.lat BETWEEN ((c.pos1->>'lat')::float - 1.5) - AND ((c.pos1->>'lat')::float + 1.5) - AND s.lon BETWEEN ((c.pos1->>'lon')::float - 1.5 / GREATEST(cos(radians((c.pos1->>'lat')::float)), 0.01)) - AND ((c.pos1->>'lon')::float + 1.5 / GREATEST(cos(radians((c.pos1->>'lat')::float)), 0.01)) - ) - OR ( - c.pos2 IS NOT NULL - AND EXISTS ( - SELECT 1 - FROM surface_observations o - JOIN weather_stations s ON s.id = o.station_id - WHERE o.observed_at >= c.qso_timestamp - interval '2 hours' - AND o.observed_at <= c.qso_timestamp + interval '2 hours' - AND s.lat BETWEEN ((c.pos2->>'lat')::float - 1.5) - AND ((c.pos2->>'lat')::float + 1.5) - AND s.lon BETWEEN ((c.pos2->>'lon')::float - 1.5 / GREATEST(cos(radians((c.pos2->>'lat')::float)), 0.01)) - AND ((c.pos2->>'lon')::float + 1.5 / GREATEST(cos(radians((c.pos2->>'lat')::float)), 0.01)) - ) - ) - ) - """ - - %{num_rows: n} = Repo.query!(sql) - {:ok, n} - end + # ── DB Maintenance ── @doc """ Kick off `ANALYZE` on every public-schema table that hasn't been @@ -502,1195 +213,4 @@ defmodule Microwaveprop.Weather do escaped = String.replace(name, ~s("), ~s("")) ~s("#{escaped}") end - - @doc """ - Derive `surface_refractivity` / `min_refractivity_gradient` / - `ducting_detected` for every `hrrr_profiles` row where those columns - are NULL but the pressure-level `profile` JSONB is populated. - Required one-off after the Rust hrrr_points worker shipped without - deriving these scalars — existing rows have the raw levels but the - DB scalars are NULL. Idempotent; rows with scalars already set are - skipped by the WHERE clause. - - Batches by `limit` rows per pass to keep transactions short on the - Turing Pi 2 Postgres node. Returns the total number of rows advanced. - """ - @spec backfill_hrrr_scalars(keyword()) :: non_neg_integer() - def backfill_hrrr_scalars(opts \\ []) do - batch = Keyword.get(opts, :batch_size, 500) - max_batches = Keyword.get(opts, :max_batches, 1_000) - - Enum.reduce_while(1..max_batches, 0, fn _, total -> - case backfill_hrrr_batch(batch) do - 0 -> {:halt, total} - n -> {:cont, total + n} - end - end) - end - - defp backfill_hrrr_batch(limit) do - rows = - HrrrProfile - |> where([h], is_nil(h.surface_refractivity)) - |> limit(^limit) - |> select([h], %{id: h.id, profile: h.profile}) - |> Repo.all() - - updates = - Enum.flat_map(rows, fn %{id: id, profile: profile} -> - case SoundingParams.derive(profile || []) do - %{} = derived -> - [ - %{ - id: id, - surface_refractivity: Map.get(derived, :surface_refractivity), - min_refractivity_gradient: Map.get(derived, :min_refractivity_gradient), - ducting_detected: Map.get(derived, :ducting_detected, false), - duct_characteristics: Map.get(derived, :duct_characteristics) - } - ] - - _ -> - [] - end - end) - - now = DateTime.truncate(DateTime.utc_now(), :second) - - _ = - if updates != [] do - {values_sql, params} = - updates - |> Enum.with_index() - |> Enum.reduce({"", []}, &build_values_row(&1, &2, now)) - - Repo.query!( - "UPDATE hrrr_profiles AS h " <> - "SET surface_refractivity = v.surface_refractivity, " <> - "min_refractivity_gradient = v.min_refractivity_gradient, " <> - "ducting_detected = v.ducting_detected, " <> - "duct_characteristics = v.duct_characteristics, " <> - "updated_at = v.updated_at " <> - "FROM (VALUES #{values_sql}) " <> - "AS v(id, surface_refractivity, min_refractivity_gradient, " <> - "ducting_detected, duct_characteristics, updated_at) " <> - "WHERE h.id = v.id", - params - ) - end - - length(updates) - end - - defp build_values_row({u, _idx}, {sql, params}, now) do - base = Enum.count(params) - frag = "($#{base + 1}::uuid, $#{base + 2}, $#{base + 3}, $#{base + 4}, $#{base + 5}, $#{base + 6})" - sep = if sql == "", do: "", else: ", " - - params = - params ++ - [ - UUID.dump!(u.id), - u.surface_refractivity, - u.min_refractivity_gradient, - u.ducting_detected, - u.duct_characteristics || [], - now - ] - - {sql <> sep <> frag, params} - end - - @doc """ - True if the station already has at least one surface observation - anywhere within the given UTC date. Used by the `asos_day` worker - to short-circuit the IEM fetch when the day has already been - ingested by a prior run. - """ - @spec station_day_covered?(UUID.t(), Date.t()) :: boolean() - def station_day_covered?(station_id, date) do - {:ok, start_dt} = DateTime.new(date, ~T[00:00:00], "Etc/UTC") - {:ok, end_dt} = DateTime.new(date, ~T[23:59:59], "Etc/UTC") - has_surface_observations?(station_id, start_dt, end_dt) - end - - @doc """ - Returns the MapSet of `{station_id, date}` tuples already covered - (at least one obs in that UTC day). Used by the enqueuer to avoid - emitting jobs for combinations we've already fetched. - """ - @spec station_day_pairs_covered([{UUID.t(), Date.t()}]) :: - MapSet.t({UUID.t(), Date.t()}) - def station_day_pairs_covered([]), do: MapSet.new() - - def station_day_pairs_covered(pairs) do - station_ids = pairs |> Enum.map(&elem(&1, 0)) |> Enum.uniq() - dates = pairs |> Enum.map(&elem(&1, 1)) |> Enum.uniq() - {:ok, start_dt} = DateTime.new(Enum.min(dates, Date), ~T[00:00:00], "Etc/UTC") - {:ok, end_dt} = DateTime.new(Enum.max(dates, Date), ~T[23:59:59], "Etc/UTC") - - rows = - SurfaceObservation - |> where([o], o.station_id in ^station_ids) - |> where([o], o.observed_at >= ^start_dt and o.observed_at <= ^end_dt) - |> select([o], {o.station_id, fragment("(?::date)", o.observed_at)}) - |> distinct(true) - |> Repo.all() - - MapSet.new(rows) - end - - @doc "Returns a MapSet of station_ids that have surface observations in the time window." - @spec station_ids_with_surface_observations([UUID.t()], DateTime.t(), DateTime.t()) :: MapSet.t(UUID.t()) - def station_ids_with_surface_observations(station_ids, start_dt, end_dt) do - SurfaceObservation - |> where([o], o.station_id in ^station_ids) - |> where([o], o.observed_at >= ^start_dt and o.observed_at <= ^end_dt) - |> select([o], o.station_id) - |> distinct(true) - |> Repo.all() - |> MapSet.new() - end - - @spec has_sounding?(UUID.t(), DateTime.t()) :: boolean() - def has_sounding?(station_id, observed_at) do - Sounding - |> where([s], s.station_id == ^station_id and s.observed_at == ^observed_at) - |> Repo.exists?() - end - - @doc "Returns a MapSet of {station_id, observed_at} tuples that have soundings." - @spec station_ids_with_soundings([UUID.t()], [DateTime.t()]) :: MapSet.t({UUID.t(), DateTime.t()}) - def station_ids_with_soundings(station_ids, sounding_times) do - Sounding - |> where([s], s.station_id in ^station_ids and s.observed_at in ^sounding_times) - |> select([s], {s.station_id, s.observed_at}) - |> distinct(true) - |> Repo.all() - |> MapSet.new() - end - - @doc "Batch upsert solar indices using insert_all in chunks of 500." - @spec upsert_solar_indices_batch([map()]) :: non_neg_integer() - def upsert_solar_indices_batch(records) do - now = DateTime.truncate(DateTime.utc_now(), :second) - - records - |> Enum.chunk_every(500) - |> Enum.reduce(0, fn chunk, acc -> - entries = - Enum.map(chunk, fn attrs -> - %{ - id: UUID.generate(), - date: attrs[:date] || attrs.date, - sfi: attrs[:sfi], - sfi_adjusted: attrs[:sfi_adjusted], - sunspot_number: attrs[:sunspot_number], - ap_index: attrs[:ap_index], - kp_values: attrs[:kp_values], - inserted_at: now, - updated_at: now - } - end) - - {count, _} = - Repo.insert_all(SolarIndex, entries, - on_conflict: - from(s in SolarIndex, - update: [ - set: [ - sfi: fragment("EXCLUDED.sfi"), - sfi_adjusted: fragment("EXCLUDED.sfi_adjusted"), - sunspot_number: fragment("EXCLUDED.sunspot_number"), - ap_index: fragment("EXCLUDED.ap_index"), - kp_values: fragment("EXCLUDED.kp_values"), - updated_at: fragment("EXCLUDED.updated_at") - ] - ], - where: - s.sfi != fragment("EXCLUDED.sfi") or - s.ap_index != fragment("EXCLUDED.ap_index") - ), - conflict_target: [:date] - ) - - acc + count - end) - end - - @spec get_solar_index(Date.t()) :: SolarIndex.t() | nil - def get_solar_index(date) do - Repo.get_by(SolarIndex, date: date) - end - - @spec existing_solar_dates() :: MapSet.t(Date.t()) - def existing_solar_dates do - SolarIndex - |> select([s], s.date) - |> Repo.all() - |> MapSet.new() - end - - @spec sync_stations!() :: :ok - def sync_stations! do - asos = - for s <- - ~w(AK AL AR AZ CA CO CT DE FL GA HI IA ID IL IN KS KY LA MA MD ME MI MN MO MS MT NC ND NE NH NJ NM NV NY OH OK OR PA RI SC SD TN TX UT VA VT WA WI WV WY), - do: "#{s}_ASOS" - - for network <- asos ++ ["RAOB"] do - sync_network(network) - Process.sleep(200) - end - - count = Repo.aggregate(Station, :count) - Logger.info("Weather stations sync complete: #{count} total") - :ok - end - - defp sync_network(network) do - type = if String.contains?(network, "ASOS"), do: "asos", else: "sounding" - - case IemClient.fetch_network(network) do - {:ok, stations} -> - now = DateTime.utc_now() - - entries = - Enum.map(stations, fn s -> - %{ - id: UUID.generate(), - station_code: s.station_code, - station_type: type, - name: s.name, - lat: s.lat, - lon: s.lon, - inserted_at: now, - updated_at: now - } - end) - - Repo.insert_all(Station, entries, - on_conflict: :nothing, - conflict_target: [:station_code, :station_type] - ) - - Logger.info("Synced #{length(stations)} stations from #{network}") - - {:error, e} -> - Logger.warning("Failed to sync #{network}: #{inspect(e)}") - end - end - - @spec nearby_stations(float(), float(), String.t(), number()) :: [Station.t()] - def nearby_stations(lat, lon, station_type, radius_km) do - dlat = radius_km / @km_per_deg_lat - dlon = radius_km / (@km_per_deg_lat * :math.cos(lat * :math.pi() / 180)) - - Station - |> where([s], s.station_type == ^station_type) - |> where( - [s], - s.lat >= ^(lat - dlat) and s.lat <= ^(lat + dlat) and - s.lon >= ^(lon - dlon) and s.lon <= ^(lon + dlon) - ) - |> Repo.all() - end - - @spec sounding_times_around(DateTime.t()) :: [DateTime.t()] - def sounding_times_around(dt) do - date = DateTime.to_date(dt) - - times = - if dt.hour < 12 do - [ - DateTime.new!(Date.add(date, -1), ~T[12:00:00], "Etc/UTC"), - DateTime.new!(date, ~T[00:00:00], "Etc/UTC") - ] - else - [ - DateTime.new!(date, ~T[00:00:00], "Etc/UTC"), - DateTime.new!(date, ~T[12:00:00], "Etc/UTC") - ] - end - - Enum.uniq(times) - end - - @spec weather_for_contact(map(), keyword()) :: %{ - surface_observations: [SurfaceObservation.t()], - soundings: [Sounding.t()] - } - def weather_for_contact(contact_params, opts \\ []) do - lat = contact_params[:lat] || contact_params.lat - lon = contact_params[:lon] || contact_params.lon - timestamp = contact_params[:timestamp] || contact_params.timestamp - - radius_km = Keyword.get(opts, :radius_km, 150) - time_window_hours = Keyword.get(opts, :time_window_hours, 6) - - # Bounding box in degrees - dlat = radius_km / @km_per_deg_lat - dlon = radius_km / (@km_per_deg_lat * :math.cos(lat * :math.pi() / 180)) - - time_start = DateTime.add(timestamp, -time_window_hours * 3600, :second) - time_end = DateTime.add(timestamp, time_window_hours * 3600, :second) - - station_ids = - Station - |> where( - [s], - s.lat >= ^(lat - dlat) and s.lat <= ^(lat + dlat) and - s.lon >= ^(lon - dlon) and s.lon <= ^(lon + dlon) - ) - |> select([s], s.id) - - surface_observations = - SurfaceObservation - |> where([o], o.station_id in subquery(station_ids)) - |> where([o], o.observed_at >= ^time_start and o.observed_at <= ^time_end) - |> preload(:station) - |> Repo.all() - - soundings = - Sounding - |> where([s], s.station_id in subquery(station_ids)) - |> where([s], s.observed_at >= ^time_start and s.observed_at <= ^time_end) - |> preload(:station) - |> Repo.all() - - %{surface_observations: surface_observations, soundings: soundings} - end - - @sounding_search_radii_km [150, 300, 600, 1000] - - @doc """ - Search for soundings in widening radii around the given location, stopping - at the first radius that returns any. Returns `%{soundings, radius_km, - exhausted}` where `exhausted: true` means the widest radius also came up - empty — the caller should trigger a fetch for missing data at that point. - """ - @spec soundings_with_widening_radius(map()) :: %{ - soundings: [Sounding.t()], - radius_km: pos_integer(), - exhausted: boolean() - } - def soundings_with_widening_radius(params) do - Enum.reduce_while(@sounding_search_radii_km, nil, fn radius_km, _acc -> - result = weather_for_contact(params, radius_km: radius_km) - - if result.soundings == [] do - {:cont, %{soundings: [], radius_km: radius_km, exhausted: true}} - else - {:halt, %{soundings: result.soundings, radius_km: radius_km, exhausted: false}} - end - end) - end - - @spec latest_grid_valid_time() :: DateTime.t() | nil - def latest_grid_valid_time do - cond do - vt = GridCache.latest_valid_time() -> vt - vt = ProfilesFile.latest_valid_time() -> vt - true -> latest_grid_valid_time_from_db() - end - end - - # Last-resort fallback for historical data sitting in the legacy - # hrrr_profiles table. PropagationGridWorker no longer writes - # grid-point rows there, so in steady state this returns nil and - # the ProfilesFile fallback above is the real source of truth. - defp latest_grid_valid_time_from_db do - Repo.one( - from(h in HrrrProfile, - where: h.is_grid_point == true, - select: max(h.valid_time) - ) - ) - end - - @doc """ - Cache-only read for the /weather LiveView mount hot path. Returns whatever - is in `GridCache` for the latest valid_time and fires a deduped background - fill task on a miss instead of blocking. The async task broadcasts - `weather:updated` when done, triggering every connected LiveView to refresh. - - Callers that genuinely need synchronous data (tests, scripts) should use - `load_weather_grid/1` instead. - """ - @spec latest_weather_grid(%{optional(String.t()) => float()} | nil) :: [map()] - def latest_weather_grid(bounds) do - case latest_grid_valid_time() do - nil -> - [] - - latest_vt -> - case GridCache.fetch_bounds(latest_vt, bounds) do - {:ok, rows} -> - rows - - :miss -> - kickoff_async_grid_fill(latest_vt) - [] - end - end - end - - @doc """ - Synchronous cache-or-disk read. Blocks for ~1s on a cold cache while - `ProfilesFile.read/1` loads the latest grid from `/data/profiles`. - Used by tests and by `weather_point_detail/3` fallbacks. LiveView - callers should prefer `latest_weather_grid/1`. - """ - @spec load_weather_grid(%{optional(String.t()) => float()} | nil) :: [map()] - def load_weather_grid(bounds) do - case latest_grid_valid_time() do - nil -> - [] - - latest_vt -> - case GridCache.fetch_bounds(latest_vt, bounds) do - {:ok, rows} -> - rows - - :miss -> - full = load_grid_rows_for(latest_vt) - GridCache.put(latest_vt, full) - filter_weather_bounds(full, bounds) - end - end - end - - @doc """ - All persisted weather valid_times sorted ascending. The grid worker - writes a ProfilesFile for every forecast hour (f00..f18 hourly, f21..f48 3-hourly), and the - derived `ScalarFile` mirrors that for any hour that has been - materialized. We union both so the timeline survives an aggressive - retention sweep on either side as long as one artifact remains. - """ - @spec available_weather_valid_times() :: [DateTime.t()] - def available_weather_valid_times do - scalar_times = ScalarFile.list_valid_times() - profile_times = ProfilesFile.list_valid_times() - - (scalar_times ++ profile_times) - |> Enum.uniq() - |> Enum.sort(DateTime) - end - - @doc """ - All persisted HRDPS valid_times (i.e. those with a `.hrdps` - scalar dir on disk) sorted ascending. Backs the `/weather-ca` - timeline. - """ - @spec available_hrdps_valid_times() :: [DateTime.t()] - def available_hrdps_valid_times do - ScalarFile.list_valid_times_hrdps() - end - - @doc """ - HRDPS-only counterpart to `weather_grid_at/2`. Reads from the - `.hrdps` scalar dir and skips HRRR completely. Bypasses GridCache - because the cache mixes HRRR + HRDPS rows by design — caching this - separately would double the per-pod memory budget for marginal benefit - (Canadian viewport reads are infrequent compared to CONUS). - - The /weather (dual-source) timeline picks valid_times from HRRR's - hourly cadence, but HRDPS publishes 4×/day with multi-hour latency, - so the requested time will frequently miss any on-disk HRDPS dir. - Snapping to the nearest available HRDPS time within a 6 h window - keeps the Canadian overlay rendering slightly stale instead of - disappearing entirely. /weather-ca picks times from HRDPS-only listings - so the exact time always matches and the snap is a no-op. - """ - @hrdps_nearest_window_seconds 6 * 3600 - - @spec weather_grid_hrdps_at(DateTime.t(), %{optional(String.t()) => float()} | nil) :: [map()] - def weather_grid_hrdps_at(%DateTime{} = valid_time, bounds) do - ScalarFile.read_bounds_hrdps_nearest(valid_time, bounds, @hrdps_nearest_window_seconds) - end - - @doc """ - Read the weather grid for a specific `valid_time` and bounds. Like - `load_weather_grid/1` but takes the valid_time explicitly so the - timeline can scrub to any forecast hour, not just the analysis hour. - Returns `[]` if no profile file exists for that valid_time. - - Deliberately does NOT write forecast-hour grids back into `GridCache` - on a miss: caching 48 forecast hours × 92k points would add ~300 MB - per pod. The ProfilesFile read is a single ~2 MB ETF decode per scrub, - which is fast enough for a user click. - """ - @spec weather_grid_at(DateTime.t(), %{optional(String.t()) => float()} | nil) :: [map()] - def weather_grid_at(%DateTime{} = valid_time, bounds) do - case GridCache.fetch_bounds(valid_time, bounds) do - {:ok, rows} -> - rows - - :miss -> - # Once a scalar file exists for `valid_time` it's the source of - # truth — even an empty viewport read (e.g. over the ocean) is - # authoritative, so don't fall back to a full ProfilesFile decode - # in that case. - if ScalarFile.exists?(valid_time) do - read_via_scalar(valid_time, bounds) - else - read_and_derive_grid(valid_time, bounds) - end - end - end - - defp read_via_scalar(valid_time, bounds) do - fill_grid_cache_from_scalar(valid_time) - - case GridCache.fetch_bounds(valid_time, bounds) do - {:ok, rows} -> rows - :miss -> ScalarFile.read_bounds(valid_time, bounds) - end - end - - # Cap on cached valid_times. Each entry is a chunked map of the full - # ~92k-cell CONUS grid with 22 fields per cell, ~32 MiB compressed in - # ETS. The previous cap of 24 meant the cache could hoard up to - # ~768 MiB on a single pod against a 6 GiB memory limit (the headroom - # that disappeared in the 2026-05-03 OOM cascade). - # - # 8 covers the current hour plus the next ~7 forecast hours — the - # typical user scrub window. Hours beyond 8 fall back to a disk read - # (`ScalarFile.read_bounds`) which is sub-100 ms per file, so rare - # deep-forecast scrubs pay a one-shot latency hit instead of every - # pod permanently parking ~500 MiB on data the user almost never - # looks at. Worst case ETS spend drops from 768 MiB → 256 MiB. - @grid_cache_valid_time_cap 8 - - # Hydrate GridCache from the on-disk ScalarFile so concurrent viewport - # reads for the same valid_time don't each gunzip+msgpack-decode the - # same chunk files. Only the caller that wins `claim_fill` does the - # work; losers wait briefly for the ETS write to land. - defp fill_grid_cache_from_scalar(valid_time) do - lock_key = {:scalar_to_grid_cache, valid_time} - - if GridCache.claim_fill(lock_key) do - try do - rows = ScalarFile.read_bounds(valid_time, nil) - - if rows != [] do - GridCache.put(valid_time, rows) - _ = GridCache.prune_keep_latest(@grid_cache_valid_time_cap) - end - after - GridCache.release_fill(lock_key) - end - else - wait_for_grid_cache(valid_time) - end - end - - defp wait_for_grid_cache(valid_time) do - Enum.reduce_while(1..50, :miss, fn _, _ -> - case GridCache.fetch(valid_time) do - {:ok, _} -> - {:halt, :ok} - - :miss -> - Process.sleep(20) - {:cont, :miss} - end - end) - end - - # Cold path: ScalarFile didn't have anything for this valid_time, so - # decode the raw ProfilesFile and derive the requested viewport. Kicks - # off a background materialization of the full scalar file so the - # next read hits the cheap path. - defp read_and_derive_grid(valid_time, bounds) do - case ProfilesFile.read(valid_time) do - {:ok, grid_data} -> - # Filter-before-derive: only derive the points inside the - # viewport instead of all 92k CONUS grid points. On a DFW - # viewport that's ~20× less `SoundingParams.derive` work - # per timeline scrub. - rows = build_grid_cache_rows(grid_data, valid_time, bounds) - kickoff_async_scalar_materialize(valid_time, grid_data) - rows - - {:error, _} -> - [] - end - end - - # Materialize the full ScalarFile for `valid_time` once per node, - # asynchronously. Reuses GridCache.claim_fill so concurrent - # `weather_grid_at` callers don't trigger N derivations of the same - # 92k-cell grid. Lock key namespaced so it doesn't collide with the - # GridCache cold-fill claim for the same valid_time. - defp kickoff_async_scalar_materialize(valid_time, grid_data) do - if ScalarFile.exists?(valid_time) do - :ok - else - lock_key = {:scalar_materialize, valid_time} - - _ = - if GridCache.claim_fill(lock_key) do - {:ok, _pid} = - Task.start(fn -> - try do - rows = build_grid_cache_rows(grid_data, valid_time) - ScalarFile.write!(valid_time, rows) - - Logger.info("Weather.scalar_file materialized valid_time=#{valid_time} rows=#{length(rows)}") - rescue - e -> - Logger.error("Weather.scalar_file materialize failed valid_time=#{valid_time} #{inspect(e)}") - after - GridCache.release_fill(lock_key) - end - end) - end - - :ok - end - end - - # Build derived GridCache rows for a valid_time from whichever - # source has data: the persisted ProfilesFile first (hot path in - # steady state), then the legacy hrrr_profiles table (historical - # data only). - defp load_grid_rows_for(valid_time) do - case ProfilesFile.read(valid_time) do - {:ok, grid_data} -> build_grid_cache_rows(grid_data, valid_time) - {:error, _} -> [] - end - end - - defp filter_weather_bounds(rows, nil), do: rows - - defp filter_weather_bounds(rows, %{"south" => s, "north" => n, "west" => w, "east" => e}) do - Enum.filter(rows, fn %{lat: lat, lon: lon} -> - lat >= s and lat <= n and lon >= w and lon <= e - end) - end - - defp kickoff_async_grid_fill(valid_time) do - _ = - if GridCache.claim_fill(valid_time) do - {:ok, _pid} = - Task.start(fn -> - try do - Logger.info("Weather.grid_cache async fill starting for #{valid_time}") - warm_grid_cache_and_broadcast(valid_time) - - _ = - Phoenix.PubSub.broadcast( - Microwaveprop.PubSub, - "weather:updated", - {:weather_updated, valid_time} - ) - - Logger.info("Weather.grid_cache async fill complete for #{valid_time}") - rescue - e -> - Logger.error("Weather.grid_cache async fill failed: #{inspect(e)}") - after - GridCache.release_fill(valid_time) - end - end) - end - - :ok - end - - @doc """ - Eagerly populate the `GridCache` with the full CONUS weather grid for - `valid_time` and broadcast it to every node in the cluster. Used by the cold - cache fill path (`kickoff_async_grid_fill/1`) — prefer - `build_grid_cache_rows/2` inside `PropagationGridWorker`, which already has - the in-memory grid data and avoids the ~20s JSONB round trip. - """ - @spec warm_grid_cache_and_broadcast(DateTime.t()) :: :ok - def warm_grid_cache_and_broadcast(valid_time) do - rows = load_grid_rows_for(valid_time) - GridCache.broadcast_put(valid_time, rows) - persist_scalar_file(valid_time, rows) - :ok - end - - @doc """ - Warm the local `GridCache` from the latest persisted `ProfilesFile` - on pod startup. Makes `/weather` usable immediately after a deploy - instead of waiting for the next hourly PropagationGridWorker run. - Local put only — every node reads the same NFS mount so no need to - broadcast. - """ - @spec warm_grid_cache_from_latest_profile() :: :ok - def warm_grid_cache_from_latest_profile do - case ProfilesFile.latest_valid_time() do - nil -> - :ok - - valid_time -> - try do - rows = load_grid_rows_for(valid_time) - GridCache.put(valid_time, rows) - persist_scalar_file(valid_time, rows) - Logger.info("Weather: warmed GridCache from ProfilesFile for #{valid_time} (#{length(rows)} rows)") - rescue - e -> - Logger.warning("Weather: ProfilesFile warm failed: #{inspect(e)}") - end - - :ok - end - end - - # Persist a derived grid as a ScalarFile so subsequent `/weather` reads - # don't have to re-decode the raw ProfilesFile or re-run - # `SoundingParams.derive` + `WeatherLayers.derive`. Best-effort: an NFS - # write failure is logged but never fatal — the cold-derive path keeps - # working. - defp persist_scalar_file(_valid_time, []), do: :ok - - defp persist_scalar_file(valid_time, rows) do - ScalarFile.write!(valid_time, rows) - :ok - rescue - e -> - Logger.warning("Weather: ScalarFile.write failed valid_time=#{valid_time} #{inspect(e)}") - :ok - end - - @doc """ - Materialize the `ScalarFile` for `valid_time` from the on-disk - `ProfilesFile`. Idempotent — if a scalar file already exists, returns - `:ok` without re-deriving. Used by `NotifyListener` to pre-warm scalar - artifacts the moment the Rust propagation pipeline fires - `propagation_ready`, so the first `/weather` reader of a new forecast - hour never falls back to the slow `ProfilesFile.read/1` + per-cell - derive path. - - Synchronous; callers should run this inside a `Task` if they need - not to block (the listener does). - """ - @spec materialize_scalar_file(DateTime.t()) :: :ok - def materialize_scalar_file(%DateTime{} = valid_time) do - if ScalarFile.exists?(valid_time) do - :ok - else - try do - rows = load_grid_rows_for(valid_time) - persist_scalar_file(valid_time, rows) - Logger.info("Weather.materialize_scalar_file vt=#{valid_time} rows=#{length(rows)}") - :ok - rescue - e -> - Logger.warning("Weather.materialize_scalar_file failed vt=#{valid_time} #{inspect(e)}") - :ok - end - end - end - - @doc """ - Build derived weather grid cache rows directly from an in-memory HRRR - `grid_data` map. Used by the cold-cache fill path after `ProfilesFile.read/1` - returns the grid written by the Rust worker. - - `grid_data` is `%{{lat, lon} => profile_map}` as produced by - `ProfilesFile.read/1`. Each row is pushed through `derive_and_clean/1` - to compute the derived fields consumed by the weather map LiveView. - """ - @spec build_grid_cache_rows( - %{{float(), float()} => map()}, - DateTime.t(), - %{optional(String.t()) => float()} | nil - ) :: [map()] - def build_grid_cache_rows(grid_data, valid_time, bounds \\ nil) do - grid_data - |> filter_grid_data_bounds(bounds) - |> Enum.flat_map(fn {{lat, lon}, profile} -> - build_grid_cache_row(lat, lon, profile, valid_time) - end) - end - - defp filter_grid_data_bounds(grid_data, nil), do: grid_data - - defp filter_grid_data_bounds(grid_data, %{"south" => s, "north" => n, "west" => w, "east" => e}) do - :maps.filter(fn {lat, lon}, _ -> lat >= s and lat <= n and lon >= w and lon <= e end, grid_data) - end - - defp build_grid_cache_row(lat, lon, profile, valid_time) do - temp_c = profile[:surface_temp_c] - - if is_nil(temp_c) or temp_c < -80 or temp_c > 60 do - [] - else - # `SoundingParams.derive` + `WeatherLayers.sort_profile` both call - # `SoundingParams.normalize_profile_entry/1` internally, so we can - # hand either shape (legacy string-keyed or Rust atom-keyed) through - # unchanged. `surface_refractivity` and `min_refractivity_gradient` - # aren't persisted by Rust — they flow through from the per-cell - # sounding derivation; `refractivity_gradient` falls back to the - # `native_min_gradient` scalar the Rust f01..f48 pipeline does write. - sounding = derive_sounding(profile[:profile]) - - row = %{ - lat: lat, - lon: lon, - valid_time: valid_time, - temperature: temp_c, - dewpoint_depression: depression(temp_c, profile[:surface_dewpoint_c]), - bl_height: profile[:hpbl_m], - pwat: profile[:pwat_mm], - refractivity_gradient: - prefer(profile, :min_refractivity_gradient, sounding[:min_refractivity_gradient]) || - profile[:native_min_gradient], - ducting: prefer(profile, :ducting_detected, sounding[:ducting_detected]), - surface_pressure_mb: profile[:surface_pressure_mb], - surface_temp_c: temp_c, - surface_dewpoint_c: profile[:surface_dewpoint_c], - surface_refractivity: prefer(profile, :surface_refractivity, sounding[:surface_refractivity]), - profile: profile[:profile] || [], - duct_characteristics: prefer(profile, :duct_characteristics, sounding[:duct_characteristics]) - } - - [derive_and_clean(row)] - end - end - - # Fetch `key` from `profile` verbatim when present (including `false`, - # `0`, or `[]`); only fall through to the derived value when the key is - # absent. `profile[key] || default` would discard legitimate `false` / - # `0` as if they weren't set. - defp prefer(profile, key, default) do - case Map.fetch(profile, key) do - {:ok, value} -> value - :error -> default - end - end - - defp derive_sounding(profile) when is_list(profile) and length(profile) >= 3 do - SoundingParams.derive(profile) || %{} - end - - defp derive_sounding(_), do: %{} - - defp depression(_, nil), do: nil - defp depression(t, d), do: t - d - - @spec weather_point_detail(float(), float(), DateTime.t()) :: map() | nil - def weather_point_detail(lat, lon, valid_time) do - step = 0.125 - snapped_lat = Float.round(Float.round(lat / step) * step, 3) - snapped_lon = Float.round(Float.round(lon / step) * step, 3) - - case GridCache.fetch_point(valid_time, snapped_lat, snapped_lon) do - {:ok, row} -> row - :miss -> point_detail_off_cache(valid_time, snapped_lat, snapped_lon) - end - end - - defp point_detail_off_cache(valid_time, snapped_lat, snapped_lon) do - case ScalarFile.read_point(valid_time, snapped_lat, snapped_lon) do - {:ok, row} -> row - :miss -> point_detail_from_disk(valid_time, snapped_lat, snapped_lon) - end - end - - defp point_detail_from_disk(valid_time, snapped_lat, snapped_lon) do - case weather_point_detail_from_profiles(valid_time, snapped_lat, snapped_lon) do - nil -> weather_point_detail_from_db(valid_time, snapped_lat, snapped_lon) - row -> row - end - end - - # Derive a single GridCache-shaped row from a persisted ProfilesFile - # entry for `(valid_time, lat, lon)`. Returns nil when the file - # doesn't exist or the point has no profile. - defp weather_point_detail_from_profiles(valid_time, snapped_lat, snapped_lon) do - case ProfilesFile.read_point(valid_time, snapped_lat, snapped_lon) do - nil -> - nil - - profile -> - case build_grid_cache_rows(%{{snapped_lat, snapped_lon} => profile}, valid_time) do - [row] -> row - _ -> nil - end - end - end - - defp weather_point_detail_from_db(valid_time, snapped_lat, snapped_lon) do - from(h in HrrrProfile, - where: h.lat == ^snapped_lat and h.lon == ^snapped_lon and h.valid_time == ^valid_time, - select: %{ - lat: h.lat, - lon: h.lon, - valid_time: h.valid_time, - temperature: h.surface_temp_c, - dewpoint_depression: fragment("? - ?", h.surface_temp_c, h.surface_dewpoint_c), - bl_height: h.hpbl_m, - pwat: h.pwat_mm, - refractivity_gradient: h.min_refractivity_gradient, - ducting: h.ducting_detected, - surface_pressure_mb: h.surface_pressure_mb, - surface_temp_c: h.surface_temp_c, - surface_dewpoint_c: h.surface_dewpoint_c, - surface_refractivity: h.surface_refractivity, - profile: h.profile, - duct_characteristics: h.duct_characteristics - } - ) - |> Repo.one() - |> then(fn - nil -> nil - row -> derive_and_clean(row) - end) - end - - defp derive_and_clean(row) do - derived = WeatherLayers.derive(row) - - row - |> Map.merge(derived) - |> Map.drop([:profile, :duct_characteristics, :surface_temp_c, :surface_dewpoint_c]) - end - - @spec upsert_gefs_profile(map()) :: {:ok, GefsProfile.t()} | {:error, Ecto.Changeset.t()} - def upsert_gefs_profile(attrs) do - changeset = GefsProfile.changeset(%GefsProfile{}, attrs) - - if changeset.valid? do - Repo.insert(changeset, - on_conflict: :nothing, - conflict_target: [:lat, :lon, :valid_time] - ) - else - {:error, changeset} - end - end - - @spec upsert_gefs_profiles_batch([map()]) :: {non_neg_integer(), nil} - def upsert_gefs_profiles_batch(profiles) do - Microwaveprop.Instrument.span( - [:db, :upsert_gefs_profiles], - %{count: length(profiles)}, - fn -> do_upsert_gefs_profiles_batch(profiles) end - ) - end - - defp do_upsert_gefs_profiles_batch(profiles) do - now = DateTime.truncate(DateTime.utc_now(), :second) - - profiles - |> Enum.chunk_every(500) - |> Enum.reduce({0, nil}, fn chunk, {total_count, _} -> - entries = - Enum.map(chunk, fn attrs -> - Map.merge(attrs, %{ - id: UUID.generate(), - inserted_at: now, - updated_at: now - }) - end) - - {count, rows} = - Repo.insert_all(GefsProfile, entries, - on_conflict: :nothing, - conflict_target: [:lat, :lon, :valid_time] - ) - - {total_count + count, rows} - end) - end - - @spec upsert_hrrr_profile(map()) :: {:ok, HrrrProfile.t()} | {:error, Ecto.Changeset.t()} - def upsert_hrrr_profile(attrs) do - %HrrrProfile{} - |> HrrrProfile.changeset(attrs) - |> Repo.insert( - on_conflict: :nothing, - conflict_target: [:lat, :lon, :valid_time] - ) - end - - @spec upsert_hrrr_profiles_batch([map()], keyword()) :: {non_neg_integer(), nil} - def upsert_hrrr_profiles_batch(profiles, _opts \\ []) do - Microwaveprop.Instrument.span( - [:db, :upsert_hrrr_profiles], - %{count: length(profiles)}, - fn -> do_upsert_hrrr_profiles_batch(profiles) end - ) - end - - defp do_upsert_hrrr_profiles_batch(profiles) do - now = DateTime.truncate(DateTime.utc_now(), :second) - - profiles - |> Enum.chunk_every(500) - |> Enum.reduce({0, nil}, fn chunk, {total_count, _} -> - entries = - Enum.map(chunk, fn attrs -> - is_gp = - rem(round(attrs.lat * 1000), 125) == 0 and - rem(round(attrs.lon * 1000), 125) == 0 - - Map.merge(attrs, %{ - id: UUID.generate(), - is_grid_point: is_gp, - inserted_at: now, - updated_at: now - }) - end) - - {count, rows} = - Repo.insert_all(HrrrProfile, entries, - on_conflict: :nothing, - conflict_target: [:lat, :lon, :valid_time] - ) - - {total_count + count, rows} - end) - end - - @doc """ - Returns just `hrrr_native_profiles.best_duct_band_ghz` near the - given cell. Convenience wrapper around `nearest_native_duct_info/3` - for callers that don't need Richardson. - """ - @spec nearest_native_duct_ghz(float(), float(), DateTime.t()) :: float() | nil - def nearest_native_duct_ghz(lat, lon, %DateTime{} = timestamp) do - case nearest_native_duct_info(lat, lon, timestamp) do - {:ok, %{best_duct_band_ghz: ghz}} -> ghz - _ -> nil - end - end - - @doc """ - Returns `{:ok, %{best_duct_band_ghz: ghz, bulk_richardson: r}}` for - the nearest `hrrr_native_profiles` cell to (`lat`, `lon`) at - `timestamp` within ±0.07° / ±1h, or `{:error, :not_found}`. - - Richardson gates the 1.15× refractivity boost in `Scorer` — a duct - band reading under turbulent conditions (high Richardson) is likely - to be mixed out before it supports the target path. The pair is - cheap to fetch together, so callers that score a cell prefer this - over the bare `nearest_native_duct_ghz/3`. - """ - @spec nearest_native_duct_info(float(), float(), DateTime.t()) :: - {:ok, %{best_duct_band_ghz: float() | nil, bulk_richardson: float() | nil}} - | {:error, :not_found} - def nearest_native_duct_info(lat, lon, %DateTime{} = timestamp) do - dlat = 0.07 - dlon = 0.07 - time_start = DateTime.add(timestamp, -3600, :second) - time_end = DateTime.add(timestamp, 3600, :second) - - from(h in HrrrNativeProfile, - where: - h.lat >= ^(lat - dlat) and h.lat <= ^(lat + dlat) and - h.lon >= ^(lon - dlon) and h.lon <= ^(lon + dlon) and - h.valid_time >= ^time_start and h.valid_time <= ^time_end, - order_by: - fragment( - "ABS(? - ?) + ABS(? - ?) + ABS(EXTRACT(EPOCH FROM ? - ?))", - h.lat, - ^lat, - h.lon, - ^lon, - h.valid_time, - ^timestamp - ), - limit: 1, - select: %{best_duct_band_ghz: h.best_duct_band_ghz, bulk_richardson: h.bulk_richardson} - ) - |> Repo.one() - |> case do - nil -> {:error, :not_found} - row -> {:ok, row} - end - end - - @doc """ - Nearest sounding to (`lat`, `lon`) within `radius_km` km and a ±3-hour - window around `timestamp`. Joins `weather_stations` to `soundings` so - the caller gets the raw sounding row (station_id set, derived duct - fields populated) back. - - Returns `{:ok, sounding}` or `{:error, :not_found}`. Use this in the - path calculator to surface the nearest RAOB's `ducting_detected` flag - as an independent check on HRRR's pressure-level duct signal, which - under-reads thin surface ducts. - """ - @spec nearest_sounding_to(float(), float(), DateTime.t(), keyword()) :: - {:ok, Sounding.t()} | {:error, :not_found} - def nearest_sounding_to(lat, lon, timestamp, opts \\ []) do - radius_km = Keyword.get(opts, :radius_km, 300) - hours = Keyword.get(opts, :hours, 3) - - # 1 deg lat ≈ 111 km; lon scaled by cos(lat). - dlat = radius_km / 111.0 - dlon = radius_km / (111.0 * max(0.1, :math.cos(lat * :math.pi() / 180.0))) - time_start = DateTime.add(timestamp, -hours * 3600, :second) - time_end = DateTime.add(timestamp, hours * 3600, :second) - - from(s in Sounding, - join: station in assoc(s, :station), - where: - station.lat >= ^(lat - dlat) and station.lat <= ^(lat + dlat) and - station.lon >= ^(lon - dlon) and station.lon <= ^(lon + dlon) and - s.observed_at >= ^time_start and s.observed_at <= ^time_end, - order_by: - fragment( - "SQRT(POW(? - ?, 2) + POW(? - ?, 2)) + ABS(EXTRACT(EPOCH FROM ? - ?)) / 86400.0", - station.lat, - ^lat, - station.lon, - ^lon, - s.observed_at, - ^timestamp - ), - limit: 1 - ) - |> Repo.one() - |> case do - nil -> {:error, :not_found} - sounding -> {:ok, sounding} - end - end - - @doc "Find all atmospheric profiles along a contact's path, from any source." - @spec profiles_along_path(map()) :: [HrrrProfile.t() | NarrProfile.t()] - def profiles_along_path(contact) do - hrrr_path = hrrr_profiles_for_path(contact) - - if hrrr_path == [] do - narr_profiles_for_path(contact) - else - hrrr_path - end - end - - @doc """ - Delete every `is_grid_point = true` row from `hrrr_profiles`, regardless of - age. Grid-point profiles are historical — the propagation grid now lives in - `/data/scores` binary files, nothing reads `is_grid_point = true` rows - anymore, and forecast runs no longer write them. This purge walks each - partition directly so a single DELETE can't scan the whole parent table. - Preserves QSO-linked rows (`is_grid_point = false`), which remain the data - path for contact enrichment and the `/path` calculator. - """ - @spec upsert_iemre_observation(map()) :: {:ok, IemreObservation.t()} | {:error, Ecto.Changeset.t()} - def upsert_iemre_observation(attrs) do - %IemreObservation{} - |> IemreObservation.changeset(attrs) - |> Repo.insert( - on_conflict: :nothing, - conflict_target: [:lat, :lon, :date] - ) - end - - @spec has_iemre_observation?(float(), float(), Date.t()) :: boolean() - def has_iemre_observation?(lat, lon, date) do - IemreObservation - |> where([i], i.lat == ^lat and i.lon == ^lon and i.date == ^date) - |> Repo.exists?() - end end diff --git a/lib/microwaveprop/weather/grid.ex b/lib/microwaveprop/weather/grid.ex new file mode 100644 index 00000000..aa0f6da3 --- /dev/null +++ b/lib/microwaveprop/weather/grid.ex @@ -0,0 +1,568 @@ +defmodule Microwaveprop.Weather.Grid do + @moduledoc false + + import Ecto.Query + + alias Microwaveprop.Propagation.ProfilesFile + alias Microwaveprop.Repo + alias Microwaveprop.Weather.GridCache + alias Microwaveprop.Weather.HrrrProfile + alias Microwaveprop.Weather.ScalarFile + alias Microwaveprop.Weather.SoundingParams + alias Microwaveprop.Weather.WeatherLayers + + require Logger + + @hrdps_nearest_window_seconds 6 * 3600 + + # Cap on cached valid_times. Each entry is a chunked map of the full + # ~92k-cell CONUS grid with 22 fields per cell, ~32 MiB compressed in + # ETS. The previous cap of 24 meant the cache could hoard up to + # ~768 MiB on a single pod against a 6 GiB memory limit (the headroom + # that disappeared in the 2026-05-03 OOM cascade). + # + # 8 covers the current hour plus the next ~7 forecast hours — the + # typical user scrub window. Hours beyond 8 fall back to a disk read + # (`ScalarFile.read_bounds`) which is sub-100 ms per file, so rare + # deep-forecast scrubs pay a one-shot latency hit instead of every + # pod permanently parking ~500 MiB on data the user almost never + # looks at. Worst case ETS spend drops from 768 MiB → 256 MiB. + @grid_cache_valid_time_cap 8 + + @spec latest_grid_valid_time() :: DateTime.t() | nil + def latest_grid_valid_time do + cond do + vt = GridCache.latest_valid_time() -> vt + vt = ProfilesFile.latest_valid_time() -> vt + true -> latest_grid_valid_time_from_db() + end + end + + # Last-resort fallback for historical data sitting in the legacy + # hrrr_profiles table. PropagationGridWorker no longer writes + # grid-point rows there, so in steady state this returns nil and + # the ProfilesFile fallback above is the real source of truth. + defp latest_grid_valid_time_from_db do + Repo.one( + from(h in HrrrProfile, + where: h.is_grid_point == true, + select: max(h.valid_time) + ) + ) + end + + @doc """ + Cache-only read for the /weather LiveView mount hot path. Returns whatever + is in `GridCache` for the latest valid_time and fires a deduped background + fill task on a miss instead of blocking. The async task broadcasts + `weather:updated` when done, triggering every connected LiveView to refresh. + + Callers that genuinely need synchronous data (tests, scripts) should use + `load_weather_grid/1` instead. + """ + @spec latest_weather_grid(%{optional(String.t()) => float()} | nil) :: [map()] + def latest_weather_grid(bounds) do + case latest_grid_valid_time() do + nil -> + [] + + latest_vt -> + case GridCache.fetch_bounds(latest_vt, bounds) do + {:ok, rows} -> + rows + + :miss -> + kickoff_async_grid_fill(latest_vt) + [] + end + end + end + + @doc """ + Synchronous cache-or-disk read. Blocks for ~1s on a cold cache while + `ProfilesFile.read/1` loads the latest grid from `/data/profiles`. + Used by tests and by `weather_point_detail/3` fallbacks. LiveView + callers should prefer `latest_weather_grid/1`. + """ + @spec load_weather_grid(%{optional(String.t()) => float()} | nil) :: [map()] + def load_weather_grid(bounds) do + case latest_grid_valid_time() do + nil -> + [] + + latest_vt -> + case GridCache.fetch_bounds(latest_vt, bounds) do + {:ok, rows} -> + rows + + :miss -> + full = load_grid_rows_for(latest_vt) + GridCache.put(latest_vt, full) + filter_weather_bounds(full, bounds) + end + end + end + + @doc """ + All persisted weather valid_times sorted ascending. The grid worker + writes a ProfilesFile for every forecast hour (f00..f18 hourly, f21..f48 3-hourly), and the + derived `ScalarFile` mirrors that for any hour that has been + materialized. We union both so the timeline survives an aggressive + retention sweep on either side as long as one artifact remains. + """ + @spec available_weather_valid_times() :: [DateTime.t()] + def available_weather_valid_times do + scalar_times = ScalarFile.list_valid_times() + profile_times = ProfilesFile.list_valid_times() + + (scalar_times ++ profile_times) + |> Enum.uniq() + |> Enum.sort(DateTime) + end + + @doc """ + All persisted HRDPS valid_times (i.e. those with a `.hrdps` + scalar dir on disk) sorted ascending. Backs the `/weather-ca` + timeline. + """ + @spec available_hrdps_valid_times() :: [DateTime.t()] + def available_hrdps_valid_times do + ScalarFile.list_valid_times_hrdps() + end + + @doc """ + HRDPS-only counterpart to `weather_grid_at/2`. Reads from the + `.hrdps` scalar dir and skips HRRR completely. Bypasses GridCache + because the cache mixes HRRR + HRDPS rows by design — caching this + separately would double the per-pod memory budget for marginal benefit + (Canadian viewport reads are infrequent compared to CONUS). + + The /weather (dual-source) timeline picks valid_times from HRRR's + hourly cadence, but HRDPS publishes 4×/day with multi-hour latency, + so the requested time will frequently miss any on-disk HRDPS dir. + Snapping to the nearest available HRDPS time within a 6 h window + keeps the Canadian overlay rendering slightly stale instead of + disappearing entirely. /weather-ca picks times from HRDPS-only listings + so the exact time always matches and the snap is a no-op. + """ + @spec weather_grid_hrdps_at(DateTime.t(), %{optional(String.t()) => float()} | nil) :: [map()] + def weather_grid_hrdps_at(%DateTime{} = valid_time, bounds) do + ScalarFile.read_bounds_hrdps_nearest(valid_time, bounds, @hrdps_nearest_window_seconds) + end + + @doc """ + Read the weather grid for a specific `valid_time` and bounds. Like + `load_weather_grid/1` but takes the valid_time explicitly so the + timeline can scrub to any forecast hour, not just the analysis hour. + Returns `[]` if no profile file exists for that valid_time. + + Deliberately does NOT write forecast-hour grids back into `GridCache` + on a miss: caching 48 forecast hours × 92k points would add ~300 MB + per pod. The ProfilesFile read is a single ~2 MB ETF decode per scrub, + which is fast enough for a user click. + """ + @spec weather_grid_at(DateTime.t(), %{optional(String.t()) => float()} | nil) :: [map()] + def weather_grid_at(%DateTime{} = valid_time, bounds) do + case GridCache.fetch_bounds(valid_time, bounds) do + {:ok, rows} -> + rows + + :miss -> + # Once a scalar file exists for `valid_time` it's the source of + # truth — even an empty viewport read (e.g. over the ocean) is + # authoritative, so don't fall back to a full ProfilesFile decode + # in that case. + if ScalarFile.exists?(valid_time) do + read_via_scalar(valid_time, bounds) + else + read_and_derive_grid(valid_time, bounds) + end + end + end + + defp read_via_scalar(valid_time, bounds) do + fill_grid_cache_from_scalar(valid_time) + + case GridCache.fetch_bounds(valid_time, bounds) do + {:ok, rows} -> rows + :miss -> ScalarFile.read_bounds(valid_time, bounds) + end + end + + # Hydrate GridCache from the on-disk ScalarFile so concurrent viewport + # reads for the same valid_time don't each gunzip+msgpack-decode the + # same chunk files. Only the caller that wins `claim_fill` does the + # work; losers wait briefly for the ETS write to land. + defp fill_grid_cache_from_scalar(valid_time) do + lock_key = {:scalar_to_grid_cache, valid_time} + + if GridCache.claim_fill(lock_key) do + try do + rows = ScalarFile.read_bounds(valid_time, nil) + + if rows != [] do + GridCache.put(valid_time, rows) + _ = GridCache.prune_keep_latest(@grid_cache_valid_time_cap) + end + after + GridCache.release_fill(lock_key) + end + else + wait_for_grid_cache(valid_time) + end + end + + defp wait_for_grid_cache(valid_time) do + Enum.reduce_while(1..50, :miss, fn _, _ -> + case GridCache.fetch(valid_time) do + {:ok, _} -> + {:halt, :ok} + + :miss -> + Process.sleep(20) + {:cont, :miss} + end + end) + end + + # Cold path: ScalarFile didn't have anything for this valid_time, so + # decode the raw ProfilesFile and derive the requested viewport. Kicks + # off a background materialization of the full scalar file so the + # next read hits the cheap path. + defp read_and_derive_grid(valid_time, bounds) do + case ProfilesFile.read(valid_time) do + {:ok, grid_data} -> + # Filter-before-derive: only derive the points inside the + # viewport instead of all 92k CONUS grid points. On a DFW + # viewport that's ~20× less `SoundingParams.derive` work + # per timeline scrub. + rows = build_grid_cache_rows(grid_data, valid_time, bounds) + kickoff_async_scalar_materialize(valid_time, grid_data) + rows + + {:error, _} -> + [] + end + end + + # Materialize the full ScalarFile for `valid_time` once per node, + # asynchronously. Reuses GridCache.claim_fill so concurrent + # `weather_grid_at` callers don't trigger N derivations of the same + # 92k-cell grid. Lock key namespaced so it doesn't collide with the + # GridCache cold-fill claim for the same valid_time. + defp kickoff_async_scalar_materialize(valid_time, grid_data) do + if ScalarFile.exists?(valid_time) do + :ok + else + lock_key = {:scalar_materialize, valid_time} + + _ = + if GridCache.claim_fill(lock_key) do + {:ok, _pid} = + Task.start(fn -> + try do + rows = build_grid_cache_rows(grid_data, valid_time) + ScalarFile.write!(valid_time, rows) + + Logger.info("Weather.scalar_file materialized valid_time=#{valid_time} rows=#{length(rows)}") + rescue + e -> + Logger.error("Weather.scalar_file materialize failed valid_time=#{valid_time} #{inspect(e)}") + after + GridCache.release_fill(lock_key) + end + end) + end + + :ok + end + end + + # Build derived GridCache rows for a valid_time from whichever + # source has data: the persisted ProfilesFile first (hot path in + # steady state), then the legacy hrrr_profiles table (historical + # data only). + defp load_grid_rows_for(valid_time) do + case ProfilesFile.read(valid_time) do + {:ok, grid_data} -> build_grid_cache_rows(grid_data, valid_time) + {:error, _} -> [] + end + end + + defp filter_weather_bounds(rows, nil), do: rows + + defp filter_weather_bounds(rows, %{"south" => s, "north" => n, "west" => w, "east" => e}) do + Enum.filter(rows, fn %{lat: lat, lon: lon} -> + lat >= s and lat <= n and lon >= w and lon <= e + end) + end + + defp kickoff_async_grid_fill(valid_time) do + _ = + if GridCache.claim_fill(valid_time) do + {:ok, _pid} = + Task.start(fn -> + try do + Logger.info("Weather.grid_cache async fill starting for #{valid_time}") + warm_grid_cache_and_broadcast(valid_time) + + _ = + Phoenix.PubSub.broadcast( + Microwaveprop.PubSub, + "weather:updated", + {:weather_updated, valid_time} + ) + + Logger.info("Weather.grid_cache async fill complete for #{valid_time}") + rescue + e -> + Logger.error("Weather.grid_cache async fill failed: #{inspect(e)}") + after + GridCache.release_fill(valid_time) + end + end) + end + + :ok + end + + @doc """ + Eagerly populate the `GridCache` with the full CONUS weather grid for + `valid_time` and broadcast it to every node in the cluster. Used by the cold + cache fill path (`kickoff_async_grid_fill/1`) — prefer + `build_grid_cache_rows/2` inside `PropagationGridWorker`, which already has + the in-memory grid data and avoids the ~20s JSONB round trip. + """ + @spec warm_grid_cache_and_broadcast(DateTime.t()) :: :ok + def warm_grid_cache_and_broadcast(valid_time) do + rows = load_grid_rows_for(valid_time) + GridCache.broadcast_put(valid_time, rows) + persist_scalar_file(valid_time, rows) + :ok + end + + @doc """ + Warm the local `GridCache` from the latest persisted `ProfilesFile` + on pod startup. Makes `/weather` usable immediately after a deploy + instead of waiting for the next hourly PropagationGridWorker run. + Local put only — every node reads the same NFS mount so no need to + broadcast. + """ + @spec warm_grid_cache_from_latest_profile() :: :ok + def warm_grid_cache_from_latest_profile do + case ProfilesFile.latest_valid_time() do + nil -> + :ok + + valid_time -> + try do + rows = load_grid_rows_for(valid_time) + GridCache.put(valid_time, rows) + persist_scalar_file(valid_time, rows) + Logger.info("Weather: warmed GridCache from ProfilesFile for #{valid_time} (#{length(rows)} rows)") + rescue + e -> + Logger.warning("Weather: ProfilesFile warm failed: #{inspect(e)}") + end + + :ok + end + end + + # Persist a derived grid as a ScalarFile so subsequent `/weather` reads + # don't have to re-decode the raw ProfilesFile or re-run + # `SoundingParams.derive` + `WeatherLayers.derive`. Best-effort: an NFS + # write failure is logged but never fatal — the cold-derive path keeps + # working. + defp persist_scalar_file(_valid_time, []), do: :ok + + defp persist_scalar_file(valid_time, rows) do + ScalarFile.write!(valid_time, rows) + :ok + rescue + e -> + Logger.warning("Weather: ScalarFile.write failed valid_time=#{valid_time} #{inspect(e)}") + :ok + end + + @doc """ + Materialize the `ScalarFile` for `valid_time` from the on-disk + `ProfilesFile`. Idempotent — if a scalar file already exists, returns + `:ok` without re-deriving. Used by `NotifyListener` to pre-warm scalar + artifacts the moment the Rust propagation pipeline fires + `propagation_ready`, so the first `/weather` reader of a new forecast + hour never falls back to the slow `ProfilesFile.read/1` + per-cell + derive path. + + Synchronous; callers should run this inside a `Task` if they need + not to block (the listener does). + """ + @spec materialize_scalar_file(DateTime.t()) :: :ok + def materialize_scalar_file(%DateTime{} = valid_time) do + if ScalarFile.exists?(valid_time) do + :ok + else + try do + rows = load_grid_rows_for(valid_time) + persist_scalar_file(valid_time, rows) + Logger.info("Weather.materialize_scalar_file vt=#{valid_time} rows=#{length(rows)}") + :ok + rescue + e -> + Logger.warning("Weather.materialize_scalar_file failed vt=#{valid_time} #{inspect(e)}") + :ok + end + end + end + + @doc """ + Build derived weather grid cache rows directly from an in-memory HRRR + `grid_data` map. Used by the cold-cache fill path after `ProfilesFile.read/1` + returns the grid written by the Rust worker. + + `grid_data` is `%{{lat, lon} => profile_map}` as produced by + `ProfilesFile.read/1`. Each row is pushed through `derive_and_clean/1` + to compute the derived fields consumed by the weather map LiveView. + """ + @spec build_grid_cache_rows( + %{{float(), float()} => map()}, + DateTime.t(), + %{optional(String.t()) => float()} | nil + ) :: [map()] + def build_grid_cache_rows(grid_data, valid_time, bounds \\ nil) do + grid_data + |> filter_grid_data_bounds(bounds) + |> Enum.flat_map(fn {{lat, lon}, profile} -> + build_grid_cache_row(lat, lon, profile, valid_time) + end) + end + + defp filter_grid_data_bounds(grid_data, nil), do: grid_data + + defp filter_grid_data_bounds(grid_data, %{"south" => s, "north" => n, "west" => w, "east" => e}) do + :maps.filter(fn {lat, lon}, _ -> lat >= s and lat <= n and lon >= w and lon <= e end, grid_data) + end + + defp build_grid_cache_row(lat, lon, profile, valid_time) do + temp_c = profile[:surface_temp_c] + + if is_nil(temp_c) or temp_c < -80 or temp_c > 60 do + [] + else + sounding = derive_sounding(profile[:profile]) + + row = %{ + lat: lat, + lon: lon, + valid_time: valid_time, + temperature: temp_c, + dewpoint_depression: depression(temp_c, profile[:surface_dewpoint_c]), + bl_height: profile[:hpbl_m], + pwat: profile[:pwat_mm], + refractivity_gradient: + prefer(profile, :min_refractivity_gradient, sounding[:min_refractivity_gradient]) || + profile[:native_min_gradient], + ducting: prefer(profile, :ducting_detected, sounding[:ducting_detected]), + surface_pressure_mb: profile[:surface_pressure_mb], + surface_temp_c: temp_c, + surface_dewpoint_c: profile[:surface_dewpoint_c], + surface_refractivity: prefer(profile, :surface_refractivity, sounding[:surface_refractivity]), + profile: profile[:profile] || [], + duct_characteristics: prefer(profile, :duct_characteristics, sounding[:duct_characteristics]) + } + + [derive_and_clean(row)] + end + end + + # Fetch `key` from `profile` verbatim when present (including `false`, + # `0`, or `[]`); only fall through to the derived value when the key is + # absent. `profile[key] || default` would discard legitimate `false` / + # `0` as if they weren't set. + defp prefer(profile, key, default) do + case Map.fetch(profile, key) do + {:ok, value} -> value + :error -> default + end + end + + defp derive_sounding(profile) when is_list(profile) and length(profile) >= 3 do + SoundingParams.derive(profile) || %{} + end + + defp derive_sounding(_), do: %{} + + defp depression(_, nil), do: nil + defp depression(t, d), do: t - d + + @spec weather_point_detail(float(), float(), DateTime.t()) :: map() | nil + def weather_point_detail(lat, lon, valid_time) do + step = 0.125 + snapped_lat = Float.round(Float.round(lat / step) * step, 3) + snapped_lon = Float.round(Float.round(lon / step) * step, 3) + + case GridCache.fetch_point(valid_time, snapped_lat, snapped_lon) do + {:ok, row} -> row + :miss -> point_detail_off_cache(valid_time, snapped_lat, snapped_lon) + end + end + + defp point_detail_off_cache(valid_time, snapped_lat, snapped_lon) do + case weather_point_detail_from_profiles(valid_time, snapped_lat, snapped_lon) do + nil -> weather_point_detail_from_db(valid_time, snapped_lat, snapped_lon) + row -> row + end + end + + # Derive a single GridCache-shaped row from a persisted ProfilesFile + # entry for `(valid_time, lat, lon)`. Returns nil when the file + # doesn't exist or the point has no profile. + defp weather_point_detail_from_profiles(valid_time, snapped_lat, snapped_lon) do + case ProfilesFile.read_point(valid_time, snapped_lat, snapped_lon) do + nil -> + nil + + profile -> + case build_grid_cache_rows(%{{snapped_lat, snapped_lon} => profile}, valid_time) do + [row] -> row + _ -> nil + end + end + end + + defp weather_point_detail_from_db(valid_time, snapped_lat, snapped_lon) do + from(h in HrrrProfile, + where: h.lat == ^snapped_lat and h.lon == ^snapped_lon and h.valid_time == ^valid_time, + select: %{ + lat: h.lat, + lon: h.lon, + valid_time: h.valid_time, + temperature: h.surface_temp_c, + dewpoint_depression: fragment("? - ?", h.surface_temp_c, h.surface_dewpoint_c), + bl_height: h.hpbl_m, + pwat: h.pwat_mm, + refractivity_gradient: h.min_refractivity_gradient, + ducting: h.ducting_detected, + surface_pressure_mb: h.surface_pressure_mb, + surface_temp_c: h.surface_temp_c, + surface_dewpoint_c: h.surface_dewpoint_c, + surface_refractivity: h.surface_refractivity, + profile: h.profile, + duct_characteristics: h.duct_characteristics + } + ) + |> Repo.one() + |> then(fn + nil -> nil + row -> derive_and_clean(row) + end) + end + + defp derive_and_clean(row) do + derived = WeatherLayers.derive(row) + + row + |> Map.merge(derived) + |> Map.drop([:profile, :duct_characteristics, :surface_temp_c, :surface_dewpoint_c]) + end +end diff --git a/lib/microwaveprop/weather/hrdps_client.ex b/lib/microwaveprop/weather/hrdps_client.ex index f5edee6d..d0137aad 100644 --- a/lib/microwaveprop/weather/hrdps_client.ex +++ b/lib/microwaveprop/weather/hrdps_client.ex @@ -53,6 +53,9 @@ defmodule Microwaveprop.Weather.HrdpsClient do @pressure_var_kinds [{:tmp, "TMP"}, {:depr, "DEPR"}, {:hgt, "HGT"}] + # All variable atoms are generated from compile-time constants + # (@grid_pressure_levels + @pressure_var_kinds = 21 atoms max) — safe + # to use String.to_atom/1 here; not reachable from user input. @pressure_vars (for level <- @grid_pressure_levels, {kind, msc} <- @pressure_var_kinds do atom = String.to_atom("#{kind}_#{level}mb") diff --git a/lib/microwaveprop/weather/hrrr.ex b/lib/microwaveprop/weather/hrrr.ex new file mode 100644 index 00000000..60a60c39 --- /dev/null +++ b/lib/microwaveprop/weather/hrrr.ex @@ -0,0 +1,306 @@ +defmodule Microwaveprop.Weather.Hrrr do + @moduledoc false + + import Ecto.Query + + alias Ecto.UUID + alias Microwaveprop.Propagation.Grid + alias Microwaveprop.Radio + alias Microwaveprop.Radio.Contact + alias Microwaveprop.Repo + alias Microwaveprop.Weather.HrrrNativeProfile + alias Microwaveprop.Weather.HrrrProfile + alias Microwaveprop.Weather.Narr + alias Microwaveprop.Weather.NarrProfile + alias Microwaveprop.Weather.ProfileLookup + alias Microwaveprop.Weather.SoundingParams + + # ── Delegates to ProfileLookup ── + + defdelegate find_nearest_hrrr(lat, lon, timestamp), to: ProfileLookup + defdelegate hrrr_profiles_for_path(contact), to: ProfileLookup + defdelegate hrrr_profiles_for_contacts(contacts), to: ProfileLookup + defdelegate find_nearest_native_profile(lat, lon, timestamp), to: ProfileLookup + defdelegate best_profile_for_contact(contact), to: ProfileLookup + defdelegate hrrr_for_contact(contact), to: ProfileLookup + defdelegate hrrr_data_fully_present?(contact), to: ProfileLookup + defdelegate has_hrrr_profile?(lat, lon, valid_time), to: ProfileLookup + defdelegate hrrr_points_present_batch(points), to: ProfileLookup + defdelegate round_to_hrrr_grid(lat, lon), to: ProfileLookup + defdelegate purge_grid_point_profiles(), to: ProfileLookup + + # ── HRRR profile persistence ── + + @spec upsert_hrrr_profile(map()) :: {:ok, HrrrProfile.t()} | {:error, Ecto.Changeset.t()} + def upsert_hrrr_profile(attrs) do + %HrrrProfile{} + |> HrrrProfile.changeset(attrs) + |> Repo.insert( + on_conflict: :nothing, + conflict_target: [:lat, :lon, :valid_time] + ) + end + + @spec upsert_hrrr_profiles_batch([map()], keyword()) :: {non_neg_integer(), nil} + def upsert_hrrr_profiles_batch(profiles, _opts \\ []) do + Microwaveprop.Instrument.span( + [:db, :upsert_hrrr_profiles], + %{count: length(profiles)}, + fn -> do_upsert_hrrr_profiles_batch(profiles) end + ) + end + + defp do_upsert_hrrr_profiles_batch(profiles) do + now = DateTime.truncate(DateTime.utc_now(), :second) + + profiles + |> Enum.chunk_every(500) + |> Enum.reduce({0, nil}, fn chunk, {total_count, _} -> + entries = + Enum.map(chunk, fn attrs -> + is_gp = + rem(round(attrs.lat * 1000), 125) == 0 and + rem(round(attrs.lon * 1000), 125) == 0 + + Map.merge(attrs, %{ + id: UUID.generate(), + is_grid_point: is_gp, + inserted_at: now, + updated_at: now + }) + end) + + {count, rows} = + Repo.insert_all(HrrrProfile, entries, + on_conflict: :nothing, + conflict_target: [:lat, :lon, :valid_time] + ) + + {total_count + count, rows} + end) + end + + # ── Native duct info ── + + @doc """ + Returns just `hrrr_native_profiles.best_duct_band_ghz` near the + given cell. Convenience wrapper around `nearest_native_duct_info/3` + for callers that don't need Richardson. + """ + @spec nearest_native_duct_ghz(float(), float(), DateTime.t()) :: float() | nil + def nearest_native_duct_ghz(lat, lon, %DateTime{} = timestamp) do + case nearest_native_duct_info(lat, lon, timestamp) do + {:ok, %{best_duct_band_ghz: ghz}} -> ghz + _ -> nil + end + end + + @doc """ + Returns `{:ok, %{best_duct_band_ghz: ghz, bulk_richardson: r}}` for + the nearest `hrrr_native_profiles` cell to (`lat`, `lon`) at + `timestamp` within ±0.07° / ±1h, or `{:error, :not_found}`. + + Richardson gates the 1.15× refractivity boost in `Scorer` — a duct + band reading under turbulent conditions (high Richardson) is likely + to be mixed out before it supports the target path. The pair is + cheap to fetch together, so callers that score a cell prefer this + over the bare `nearest_native_duct_ghz/3`. + """ + @spec nearest_native_duct_info(float(), float(), DateTime.t()) :: + {:ok, %{best_duct_band_ghz: float() | nil, bulk_richardson: float() | nil}} + | {:error, :not_found} + def nearest_native_duct_info(lat, lon, %DateTime{} = timestamp) do + dlat = 0.07 + dlon = 0.07 + time_start = DateTime.add(timestamp, -3600, :second) + time_end = DateTime.add(timestamp, 3600, :second) + + from(h in HrrrNativeProfile, + where: + h.lat >= ^(lat - dlat) and h.lat <= ^(lat + dlat) and + h.lon >= ^(lon - dlon) and h.lon <= ^(lon + dlon) and + h.valid_time >= ^time_start and h.valid_time <= ^time_end, + order_by: + fragment( + "ABS(? - ?) + ABS(? - ?) + ABS(EXTRACT(EPOCH FROM ? - ?))", + h.lat, + ^lat, + h.lon, + ^lon, + h.valid_time, + ^timestamp + ), + limit: 1, + select: %{best_duct_band_ghz: h.best_duct_band_ghz, bulk_richardson: h.bulk_richardson} + ) + |> Repo.one() + |> case do + nil -> {:error, :not_found} + row -> {:ok, row} + end + end + + # ── Reconciliation ── + + @doc """ + Reconcile `contacts.hrrr_status` for every `:queued` contact: + + * flips to `:complete` when every path point has an hrrr_profiles + row at the rounded HRRR hour; + * flips to `:unavailable` when any path point falls outside the + HRRR CONUS grid (pos2 or the great-circle midpoint can drift + into Canada, the mid-Atlantic, or the Pacific even when pos1 is + stateside — the Rust hrrr-point-worker returns no profile for + those points and the contact would otherwise sit `:queued` + forever). + + Returns `{:ok, n}` where `n` is the total number of contacts advanced + (completed + oconus). + """ + @spec reconcile_hrrr_statuses() :: {:ok, non_neg_integer()} + def reconcile_hrrr_statuses do + queued = + Contact + |> where([c], c.hrrr_status == :queued and not is_nil(c.pos1)) + |> Repo.all() + + {oconus, rest} = Enum.split_with(queued, &contact_has_oconus_path_point?/1) + to_complete = Enum.filter(rest, &hrrr_data_fully_present?/1) + + _ = + if oconus != [] do + ids = Enum.map(oconus, & &1.id) + Radio.set_enrichment_status!(ids, :hrrr_status, :unavailable) + end + + _ = + if to_complete != [] do + ids = Enum.map(to_complete, & &1.id) + Radio.set_enrichment_status!(ids, :hrrr_status, :complete) + end + + {:ok, length(oconus) + length(to_complete)} + end + + defp contact_has_oconus_path_point?(contact) do + contact + |> Radio.contact_path_points() + |> Enum.any?(fn {lat, lon} -> + not Grid.contains?(%{"lat" => lat, "lon" => lon}) + end) + end + + # ── Backfill scalars ── + + @doc """ + Derive `surface_refractivity` / `min_refractivity_gradient` / + `ducting_detected` for every `hrrr_profiles` row where those columns + are NULL but the pressure-level `profile` JSONB is populated. + Required one-off after the Rust hrrr_points worker shipped without + deriving these scalars — existing rows have the raw levels but the + DB scalars are NULL. Idempotent; rows with scalars already set are + skipped by the WHERE clause. + + Batches by `limit` rows per pass to keep transactions short on the + Turing Pi 2 Postgres node. Returns the total number of rows advanced. + """ + @spec backfill_hrrr_scalars(keyword()) :: non_neg_integer() + def backfill_hrrr_scalars(opts \\ []) do + batch = Keyword.get(opts, :batch_size, 500) + max_batches = Keyword.get(opts, :max_batches, 1_000) + + Enum.reduce_while(1..max_batches, 0, fn _, total -> + case backfill_hrrr_batch(batch) do + 0 -> {:halt, total} + n -> {:cont, total + n} + end + end) + end + + defp backfill_hrrr_batch(limit) do + rows = + HrrrProfile + |> where([h], is_nil(h.surface_refractivity)) + |> limit(^limit) + |> select([h], %{id: h.id, profile: h.profile}) + |> Repo.all() + + updates = + Enum.flat_map(rows, fn %{id: id, profile: profile} -> + case SoundingParams.derive(profile || []) do + %{} = derived -> + [ + %{ + id: id, + surface_refractivity: Map.get(derived, :surface_refractivity), + min_refractivity_gradient: Map.get(derived, :min_refractivity_gradient), + ducting_detected: Map.get(derived, :ducting_detected, false), + duct_characteristics: Map.get(derived, :duct_characteristics) + } + ] + + _ -> + [] + end + end) + + now = DateTime.truncate(DateTime.utc_now(), :second) + + _ = + if updates != [] do + {values_sql, params} = + updates + |> Enum.with_index() + |> Enum.reduce({"", []}, &build_values_row(&1, &2, now)) + + Repo.query!( + "UPDATE hrrr_profiles AS h " <> + "SET surface_refractivity = v.surface_refractivity, " <> + "min_refractivity_gradient = v.min_refractivity_gradient, " <> + "ducting_detected = v.ducting_detected, " <> + "duct_characteristics = v.duct_characteristics, " <> + "updated_at = v.updated_at " <> + "FROM (VALUES #{values_sql}) " <> + "AS v(id, surface_refractivity, min_refractivity_gradient, " <> + "ducting_detected, duct_characteristics, updated_at) " <> + "WHERE h.id = v.id", + params + ) + end + + length(updates) + end + + defp build_values_row({u, _idx}, {sql, params}, now) do + base = Enum.count(params) + frag = "($#{base + 1}::uuid, $#{base + 2}, $#{base + 3}, $#{base + 4}, $#{base + 5}, $#{base + 6})" + sep = if sql == "", do: "", else: ", " + + params = + params ++ + [ + UUID.dump!(u.id), + u.surface_refractivity, + u.min_refractivity_gradient, + u.ducting_detected, + u.duct_characteristics || [], + now + ] + + {sql <> sep <> frag, params} + end + + # ── Path profiles ── + + @doc "Find all atmospheric profiles along a contact's path, from any source." + @spec profiles_along_path(map()) :: [HrrrProfile.t() | NarrProfile.t()] + def profiles_along_path(contact) do + hrrr_path = hrrr_profiles_for_path(contact) + + if hrrr_path == [] do + Narr.narr_profiles_for_path(contact) + else + hrrr_path + end + end +end diff --git a/lib/microwaveprop/weather/hrrr_point_enqueuer.ex b/lib/microwaveprop/weather/hrrr_point_enqueuer.ex index 87888d7e..61c2a077 100644 --- a/lib/microwaveprop/weather/hrrr_point_enqueuer.ex +++ b/lib/microwaveprop/weather/hrrr_point_enqueuer.ex @@ -38,67 +38,67 @@ defmodule Microwaveprop.Weather.HrrrPointEnqueuer do def enqueue(groups) when is_map(groups) do now = DateTime.truncate(DateTime.utc_now(), :microsecond) - count = - Enum.reduce(groups, 0, fn {valid_time, points}, acc -> + rows = + Enum.map(groups, fn {valid_time, points} -> valid_time = DateTime.truncate(valid_time, :second) json_points = Enum.map(points, fn {lat, lon} -> %{"lat" => lat, "lon" => lon} end) - - # Union with the existing row's points (JSONB array) — use a - # subquery with jsonb_array_elements to dedupe. Postgres - # handles the set math so the Elixir side stays simple. - # - # Insert uses a wrapping schema query so Postgrex's jsonb - # extension encodes the list directly; raw Repo.query! with - # a binding would fall through to `text->jsonb` cast which - # then stores the JSON as a jsonb *string* and breaks the - # || union on round-trip. id = Ecto.UUID.bingenerate() - _ = - Repo.insert_all( - "hrrr_fetch_tasks", - [ - %{ - id: id, - valid_time: valid_time, - points: json_points, - status: "queued", - attempt: 0, - inserted_at: now, - updated_at: now - } - ], - on_conflict: - from(t in "hrrr_fetch_tasks", - update: [ - set: [ - points: - fragment( - "(SELECT COALESCE(jsonb_agg(DISTINCT p), '[]'::jsonb) FROM jsonb_array_elements(? || EXCLUDED.points) AS p)", - t.points - ), - status: - fragment( - "CASE WHEN ? IN ('done', 'failed') THEN 'queued' ELSE ? END", - t.status, - t.status - ), - attempt: - fragment( - "CASE WHEN ? IN ('done', 'failed') THEN 0 ELSE ? END", - t.status, - t.attempt - ), - updated_at: ^now - ] - ] - ), - conflict_target: [:valid_time] - ) - - acc + 1 + %{ + id: id, + valid_time: valid_time, + points: json_points, + status: "queued", + attempt: 0, + inserted_at: now, + updated_at: now + } end) + count = length(rows) + + # Union with the existing row's points (JSONB array) — use a + # subquery with jsonb_array_elements to dedupe. Postgres + # handles the set math so the Elixir side stays simple. + # + # Insert uses a wrapping schema query so Postgrex's jsonb + # extension encodes the list directly; raw Repo.query! with + # a binding would fall through to `text->jsonb` cast which + # then stores the JSON as a jsonb *string* and breaks the + # || union on round-trip. + if count > 0 do + Repo.insert_all( + "hrrr_fetch_tasks", + rows, + on_conflict: + from(t in "hrrr_fetch_tasks", + update: [ + set: [ + points: + fragment( + "(SELECT COALESCE(jsonb_agg(DISTINCT p), '[]'::jsonb) FROM jsonb_array_elements(? || EXCLUDED.points) AS p)", + t.points + ), + status: + fragment( + "CASE WHEN ? IN ('done', 'failed') THEN 'queued' ELSE ? END", + t.status, + t.status + ), + attempt: + fragment( + "CASE WHEN ? IN ('done', 'failed') THEN 0 ELSE ? END", + t.status, + t.attempt + ), + updated_at: ^now + ] + ] + ), + conflict_target: [:valid_time] + ) + end + {:ok, count} rescue e -> diff --git a/lib/microwaveprop/weather/iemre.ex b/lib/microwaveprop/weather/iemre.ex new file mode 100644 index 00000000..7d75a0cc --- /dev/null +++ b/lib/microwaveprop/weather/iemre.ex @@ -0,0 +1,73 @@ +defmodule Microwaveprop.Weather.Iemre do + @moduledoc false + + import Ecto.Query + + alias Microwaveprop.Radio + alias Microwaveprop.Radio.Contact + alias Microwaveprop.Repo + alias Microwaveprop.Weather.IemreObservation + alias Microwaveprop.Weather.ProfileLookup + + # Delegates to ProfileLookup + defdelegate iemre_for_contact(contact), to: ProfileLookup + defdelegate iemre_for_path(contact), to: ProfileLookup + defdelegate find_nearest_iemre(lat, lon, timestamp), to: ProfileLookup + defdelegate round_to_iemre_grid(lat, lon), to: ProfileLookup + + @spec upsert_iemre_observation(map()) :: {:ok, IemreObservation.t()} | {:error, Ecto.Changeset.t()} + def upsert_iemre_observation(attrs) do + %IemreObservation{} + |> IemreObservation.changeset(attrs) + |> Repo.insert( + on_conflict: :nothing, + conflict_target: [:lat, :lon, :date] + ) + end + + @spec has_iemre_observation?(float(), float(), Date.t()) :: boolean() + def has_iemre_observation?(lat, lon, date) do + IemreObservation + |> where([i], i.lat == ^lat and i.lon == ^lon and i.date == ^date) + |> Repo.exists?() + end + + @doc """ + Flip `contacts.iemre_status` from `:queued` to `:complete` for every + contact whose path points all have an `iemre_observations` row at + the rounded grid cell for the QSO's UTC date. + + Per-contact check (via the existing helpers); kept in Elixir because + `contact_path_points/1` produces 1–3 points depending on whether + `pos2` is set, and rounding to the 0.125° IEMRE grid per point in + pure SQL is awkward. Queue sizes are small (<20 stuck at steady + state). + """ + @spec reconcile_iemre_statuses() :: {:ok, non_neg_integer()} + def reconcile_iemre_statuses do + queued = + Contact + |> where([c], c.iemre_status == :queued and not is_nil(c.pos1)) + |> Repo.all() + + to_complete = + Enum.filter(queued, fn c -> + date = DateTime.to_date(c.qso_timestamp) + + c + |> Radio.contact_path_points() + |> Enum.all?(fn {lat, lon} -> + {rlat, rlon} = round_to_iemre_grid(lat, lon) + has_iemre_observation?(rlat, rlon, date) + end) + end) + + if to_complete == [] do + {:ok, 0} + else + ids = Enum.map(to_complete, & &1.id) + _ = Radio.set_enrichment_status!(ids, :iemre_status, :complete) + {:ok, length(to_complete)} + end + end +end diff --git a/lib/microwaveprop/weather/narr.ex b/lib/microwaveprop/weather/narr.ex new file mode 100644 index 00000000..4e96642c --- /dev/null +++ b/lib/microwaveprop/weather/narr.ex @@ -0,0 +1,9 @@ +defmodule Microwaveprop.Weather.Narr do + @moduledoc false + + alias Microwaveprop.Weather.ProfileLookup + + defdelegate narr_profiles_for_path(contact), to: ProfileLookup + defdelegate find_nearest_narr(lat, lon, timestamp), to: ProfileLookup + defdelegate narr_for_contact(contact), to: ProfileLookup +end diff --git a/lib/microwaveprop/weather/soundings.ex b/lib/microwaveprop/weather/soundings.ex new file mode 100644 index 00000000..8a9543d8 --- /dev/null +++ b/lib/microwaveprop/weather/soundings.ex @@ -0,0 +1,206 @@ +defmodule Microwaveprop.Weather.Soundings do + @moduledoc false + + import Ecto.Query + + alias Microwaveprop.Repo + alias Microwaveprop.Weather.Sounding + alias Microwaveprop.Weather.Station + alias Microwaveprop.Weather.SurfaceObservation + + # Approximate km per degree latitude + @km_per_deg_lat 111.0 + @sounding_search_radii_km [150, 300, 600, 1000] + + @spec upsert_sounding(Station.t(), map()) :: {:ok, Sounding.t()} | {:error, Ecto.Changeset.t()} + def upsert_sounding(%Station{} = station, attrs) do + attrs = Map.put(attrs, :station_id, station.id) + + %Sounding{} + |> Sounding.changeset(attrs) + |> Repo.insert( + on_conflict: + from(s in Sounding, + update: [ + set: [ + profile: fragment("EXCLUDED.profile"), + level_count: fragment("EXCLUDED.level_count"), + surface_pressure_mb: fragment("EXCLUDED.surface_pressure_mb"), + surface_temp_c: fragment("EXCLUDED.surface_temp_c"), + surface_dewpoint_c: fragment("EXCLUDED.surface_dewpoint_c"), + surface_refractivity: fragment("EXCLUDED.surface_refractivity"), + min_refractivity_gradient: fragment("EXCLUDED.min_refractivity_gradient"), + boundary_layer_depth_m: fragment("EXCLUDED.boundary_layer_depth_m"), + precipitable_water_mm: fragment("EXCLUDED.precipitable_water_mm"), + k_index: fragment("EXCLUDED.k_index"), + lifted_index: fragment("EXCLUDED.lifted_index"), + ducting_detected: fragment("EXCLUDED.ducting_detected"), + duct_characteristics: fragment("EXCLUDED.duct_characteristics"), + updated_at: fragment("EXCLUDED.updated_at") + ] + ], + where: + s.level_count != fragment("EXCLUDED.level_count") or + s.surface_temp_c != fragment("EXCLUDED.surface_temp_c") or + s.surface_refractivity != fragment("EXCLUDED.surface_refractivity") + ), + conflict_target: [:station_id, :observed_at], + returning: true, + stale_error_field: :id + ) + end + + @spec has_sounding?(Ecto.UUID.t(), DateTime.t()) :: boolean() + def has_sounding?(station_id, observed_at) do + Sounding + |> where([s], s.station_id == ^station_id and s.observed_at == ^observed_at) + |> Repo.exists?() + end + + @doc "Returns a MapSet of {station_id, observed_at} tuples that have soundings." + @spec station_ids_with_soundings([Ecto.UUID.t()], [DateTime.t()]) :: MapSet.t({Ecto.UUID.t(), DateTime.t()}) + def station_ids_with_soundings(station_ids, sounding_times) do + Sounding + |> where([s], s.station_id in ^station_ids and s.observed_at in ^sounding_times) + |> select([s], {s.station_id, s.observed_at}) + |> distinct(true) + |> Repo.all() + |> MapSet.new() + end + + @spec sounding_times_around(DateTime.t()) :: [DateTime.t()] + def sounding_times_around(dt) do + date = DateTime.to_date(dt) + + times = + if dt.hour < 12 do + [ + DateTime.new!(Date.add(date, -1), ~T[12:00:00], "Etc/UTC"), + DateTime.new!(date, ~T[00:00:00], "Etc/UTC") + ] + else + [ + DateTime.new!(date, ~T[00:00:00], "Etc/UTC"), + DateTime.new!(date, ~T[12:00:00], "Etc/UTC") + ] + end + + Enum.uniq(times) + end + + @spec weather_for_contact(map(), keyword()) :: %{ + surface_observations: [SurfaceObservation.t()], + soundings: [Sounding.t()] + } + def weather_for_contact(contact_params, opts \\ []) do + lat = contact_params[:lat] || contact_params.lat + lon = contact_params[:lon] || contact_params.lon + timestamp = contact_params[:timestamp] || contact_params.timestamp + + radius_km = Keyword.get(opts, :radius_km, 150) + time_window_hours = Keyword.get(opts, :time_window_hours, 6) + + # Bounding box in degrees + dlat = radius_km / @km_per_deg_lat + dlon = radius_km / (@km_per_deg_lat * :math.cos(lat * :math.pi() / 180)) + + time_start = DateTime.add(timestamp, -time_window_hours * 3600, :second) + time_end = DateTime.add(timestamp, time_window_hours * 3600, :second) + + station_ids = + Station + |> where( + [s], + s.lat >= ^(lat - dlat) and s.lat <= ^(lat + dlat) and + s.lon >= ^(lon - dlon) and s.lon <= ^(lon + dlon) + ) + |> select([s], s.id) + + surface_observations = + SurfaceObservation + |> where([o], o.station_id in subquery(station_ids)) + |> where([o], o.observed_at >= ^time_start and o.observed_at <= ^time_end) + |> preload(:station) + |> Repo.all() + + soundings = + Sounding + |> where([s], s.station_id in subquery(station_ids)) + |> where([s], s.observed_at >= ^time_start and s.observed_at <= ^time_end) + |> preload(:station) + |> Repo.all() + + %{surface_observations: surface_observations, soundings: soundings} + end + + @doc """ + Search for soundings in widening radii around the given location, stopping + at the first radius that returns any. Returns `%{soundings, radius_km, + exhausted}` where `exhausted: true` means the widest radius also came up + empty — the caller should trigger a fetch for missing data at that point. + """ + @spec soundings_with_widening_radius(map()) :: %{ + soundings: [Sounding.t()], + radius_km: pos_integer(), + exhausted: boolean() + } + def soundings_with_widening_radius(params) do + Enum.reduce_while(@sounding_search_radii_km, nil, fn radius_km, _acc -> + result = weather_for_contact(params, radius_km: radius_km) + + if result.soundings == [] do + {:cont, %{soundings: [], radius_km: radius_km, exhausted: true}} + else + {:halt, %{soundings: result.soundings, radius_km: radius_km, exhausted: false}} + end + end) + end + + @doc """ + Nearest sounding to (`lat`, `lon`) within `radius_km` km and a ±3-hour + window around `timestamp`. Joins `weather_stations` to `soundings` so + the caller gets the raw sounding row (station_id set, derived duct + fields populated) back. + + Returns `{:ok, sounding}` or `{:error, :not_found}`. Use this in the + path calculator to surface the nearest RAOB's `ducting_detected` flag + as an independent check on HRRR's pressure-level duct signal, which + under-reads thin surface ducts. + """ + @spec nearest_sounding_to(float(), float(), DateTime.t(), keyword()) :: + {:ok, Sounding.t()} | {:error, :not_found} + def nearest_sounding_to(lat, lon, timestamp, opts \\ []) do + radius_km = Keyword.get(opts, :radius_km, 300) + hours = Keyword.get(opts, :hours, 3) + + # 1 deg lat ≈ 111 km; lon scaled by cos(lat). + dlat = radius_km / 111.0 + dlon = radius_km / (111.0 * max(0.1, :math.cos(lat * :math.pi() / 180.0))) + time_start = DateTime.add(timestamp, -hours * 3600, :second) + time_end = DateTime.add(timestamp, hours * 3600, :second) + + from(s in Sounding, + join: station in assoc(s, :station), + where: + station.lat >= ^(lat - dlat) and station.lat <= ^(lat + dlat) and + station.lon >= ^(lon - dlon) and station.lon <= ^(lon + dlon) and + s.observed_at >= ^time_start and s.observed_at <= ^time_end, + order_by: + fragment( + "SQRT(POW(? - ?, 2) + POW(? - ?, 2)) + ABS(EXTRACT(EPOCH FROM ? - ?)) / 86400.0", + station.lat, + ^lat, + station.lon, + ^lon, + s.observed_at, + ^timestamp + ), + limit: 1 + ) + |> Repo.one() + |> case do + nil -> {:error, :not_found} + sounding -> {:ok, sounding} + end + end +end diff --git a/lib/microwaveprop/weather/surface.ex b/lib/microwaveprop/weather/surface.ex new file mode 100644 index 00000000..155a19e8 --- /dev/null +++ b/lib/microwaveprop/weather/surface.ex @@ -0,0 +1,452 @@ +defmodule Microwaveprop.Weather.Surface do + @moduledoc false + + import Ecto.Query + + alias Ecto.UUID + alias Microwaveprop.Repo + alias Microwaveprop.Weather.IemClient + alias Microwaveprop.Weather.SolarIndex + alias Microwaveprop.Weather.Station + alias Microwaveprop.Weather.SurfaceObservation + + require Logger + + # Approximate km per degree latitude + @km_per_deg_lat 111.0 + + @spec find_or_create_station(map()) :: {:ok, Station.t()} | {:error, Ecto.Changeset.t()} + def find_or_create_station(attrs) do + code = attrs[:station_code] || attrs["station_code"] + type = attrs[:station_type] || attrs["station_type"] + + if code && type do + case Repo.get_by(Station, station_code: code, station_type: type) do + nil -> + %Station{} + |> Station.changeset(attrs) + |> Repo.insert() + + station -> + {:ok, station} + end + else + %Station{} + |> Station.changeset(attrs) + |> Repo.insert() + end + end + + @spec upsert_surface_observation(Station.t(), map()) :: {:ok, SurfaceObservation.t()} | {:error, Ecto.Changeset.t()} + def upsert_surface_observation(%Station{} = station, attrs) do + attrs = Map.put(attrs, :station_id, station.id) + + %SurfaceObservation{} + |> SurfaceObservation.changeset(attrs) + |> Repo.insert( + on_conflict: + from(s in SurfaceObservation, + update: [ + set: [ + temp_f: fragment("EXCLUDED.temp_f"), + dewpoint_f: fragment("EXCLUDED.dewpoint_f"), + relative_humidity: fragment("EXCLUDED.relative_humidity"), + wind_speed_kts: fragment("EXCLUDED.wind_speed_kts"), + sea_level_pressure_mb: fragment("EXCLUDED.sea_level_pressure_mb"), + sky_condition: fragment("EXCLUDED.sky_condition"), + precip_1h_in: fragment("EXCLUDED.precip_1h_in"), + wx_codes: fragment("EXCLUDED.wx_codes"), + updated_at: fragment("EXCLUDED.updated_at") + ] + ], + where: + s.temp_f != fragment("EXCLUDED.temp_f") or + s.dewpoint_f != fragment("EXCLUDED.dewpoint_f") or + s.relative_humidity != fragment("EXCLUDED.relative_humidity") or + s.wind_speed_kts != fragment("EXCLUDED.wind_speed_kts") or + s.sea_level_pressure_mb != fragment("EXCLUDED.sea_level_pressure_mb") + ), + conflict_target: [:station_id, :observed_at], + returning: true, + stale_error_field: :id + ) + end + + @doc """ + Bulk-upsert surface observations for a single station via one + `Repo.insert_all` round-trip. ASOS fetches return 24–288 rows per + station per call — collapsing them into a single statement avoids the + per-row UPDATE-conflict round-trip of `upsert_surface_observation/2`, + which is expensive against the Turing Pi 2 Postgres node. + + Rows missing `observed_at` are dropped (they can't satisfy the + `(station_id, observed_at)` unique index). Returns the + `{count, nil}` tuple from `Repo.insert_all/3`; `count` reflects + affected rows (new + updated-when-changed). Returns `{0, nil}` for + an empty input without touching the DB. + """ + @spec upsert_surface_observations(Station.t(), [map()]) :: {non_neg_integer(), nil} + def upsert_surface_observations(%Station{} = station, rows) when is_list(rows) do + now = DateTime.truncate(DateTime.utc_now(), :second) + + entries = + rows + |> Enum.filter(&row_has_observed_at?/1) + |> Enum.map(&surface_observation_entry(&1, station.id, now)) + |> dedupe_last_by_conflict_target() + + case entries do + [] -> + {0, nil} + + _ -> + Repo.insert_all(SurfaceObservation, entries, + on_conflict: + from(s in SurfaceObservation, + update: [ + set: [ + temp_f: fragment("EXCLUDED.temp_f"), + dewpoint_f: fragment("EXCLUDED.dewpoint_f"), + relative_humidity: fragment("EXCLUDED.relative_humidity"), + wind_speed_kts: fragment("EXCLUDED.wind_speed_kts"), + wind_direction_deg: fragment("EXCLUDED.wind_direction_deg"), + sea_level_pressure_mb: fragment("EXCLUDED.sea_level_pressure_mb"), + altimeter_setting: fragment("EXCLUDED.altimeter_setting"), + sky_condition: fragment("EXCLUDED.sky_condition"), + precip_1h_in: fragment("EXCLUDED.precip_1h_in"), + wx_codes: fragment("EXCLUDED.wx_codes"), + updated_at: fragment("EXCLUDED.updated_at") + ] + ], + where: + s.temp_f != fragment("EXCLUDED.temp_f") or + s.dewpoint_f != fragment("EXCLUDED.dewpoint_f") or + s.relative_humidity != fragment("EXCLUDED.relative_humidity") or + s.wind_speed_kts != fragment("EXCLUDED.wind_speed_kts") or + s.sea_level_pressure_mb != fragment("EXCLUDED.sea_level_pressure_mb") + ), + conflict_target: [:station_id, :observed_at] + ) + end + end + + @spec has_surface_observations?(UUID.t(), DateTime.t(), DateTime.t()) :: boolean() + def has_surface_observations?(station_id, start_dt, end_dt) do + SurfaceObservation + |> where([o], o.station_id == ^station_id) + |> where([o], o.observed_at >= ^start_dt and o.observed_at <= ^end_dt) + |> Repo.exists?() + end + + @doc """ + True if the station already has at least one surface observation + anywhere within the given UTC date. Used by the `asos_day` worker + to short-circuit the IEM fetch when the day has already been + ingested by a prior run. + """ + @spec station_day_covered?(UUID.t(), Date.t()) :: boolean() + def station_day_covered?(station_id, date) do + {:ok, start_dt} = DateTime.new(date, ~T[00:00:00], "Etc/UTC") + {:ok, end_dt} = DateTime.new(date, ~T[23:59:59], "Etc/UTC") + has_surface_observations?(station_id, start_dt, end_dt) + end + + @doc """ + Returns the MapSet of `{station_id, date}` tuples already covered + (at least one obs in that UTC day). Used by the enqueuer to avoid + emitting jobs for combinations we've already fetched. + """ + @spec station_day_pairs_covered([{UUID.t(), Date.t()}]) :: + MapSet.t({UUID.t(), Date.t()}) + def station_day_pairs_covered([]), do: MapSet.new() + + def station_day_pairs_covered(pairs) do + station_ids = pairs |> Enum.map(&elem(&1, 0)) |> Enum.uniq() + dates = pairs |> Enum.map(&elem(&1, 1)) |> Enum.uniq() + {:ok, start_dt} = DateTime.new(Enum.min(dates, Date), ~T[00:00:00], "Etc/UTC") + {:ok, end_dt} = DateTime.new(Enum.max(dates, Date), ~T[23:59:59], "Etc/UTC") + + rows = + SurfaceObservation + |> where([o], o.station_id in ^station_ids) + |> where([o], o.observed_at >= ^start_dt and o.observed_at <= ^end_dt) + |> select([o], {o.station_id, fragment("(?::date)", o.observed_at)}) + |> distinct(true) + |> Repo.all() + + MapSet.new(rows) + end + + @doc "Returns a MapSet of station_ids that have surface observations in the time window." + @spec station_ids_with_surface_observations([UUID.t()], DateTime.t(), DateTime.t()) :: MapSet.t(UUID.t()) + def station_ids_with_surface_observations(station_ids, start_dt, end_dt) do + SurfaceObservation + |> where([o], o.station_id in ^station_ids) + |> where([o], o.observed_at >= ^start_dt and o.observed_at <= ^end_dt) + |> select([o], o.station_id) + |> distinct(true) + |> Repo.all() + |> MapSet.new() + end + + @spec nearby_stations(float(), float(), String.t(), number()) :: [Station.t()] + def nearby_stations(lat, lon, station_type, radius_km) do + dlat = radius_km / @km_per_deg_lat + dlon = radius_km / (@km_per_deg_lat * :math.cos(lat * :math.pi() / 180)) + + Station + |> where([s], s.station_type == ^station_type) + |> where( + [s], + s.lat >= ^(lat - dlat) and s.lat <= ^(lat + dlat) and + s.lon >= ^(lon - dlon) and s.lon <= ^(lon + dlon) + ) + |> Repo.all() + end + + @spec sync_stations!() :: :ok + def sync_stations! do + asos = + for s <- + ~w(AK AL AR AZ CA CO CT DE FL GA HI IA ID IL IN KS KY LA MA MD ME MI MN MO MS MT NC ND NE NH NJ NM NV NY OH OK OR PA RI SC SD TN TX UT VA VT WA WI WV WY), + do: "#{s}_ASOS" + + for network <- asos ++ ["RAOB"] do + sync_network(network) + Process.sleep(200) + end + + count = Repo.aggregate(Station, :count) + Logger.info("Weather stations sync complete: #{count} total") + :ok + end + + defp sync_network(network) do + type = if String.contains?(network, "ASOS"), do: "asos", else: "sounding" + + case IemClient.fetch_network(network) do + {:ok, stations} -> + now = DateTime.utc_now() + + entries = + Enum.map(stations, fn s -> + %{ + id: UUID.generate(), + station_code: s.station_code, + station_type: type, + name: s.name, + lat: s.lat, + lon: s.lon, + inserted_at: now, + updated_at: now + } + end) + + Repo.insert_all(Station, entries, + on_conflict: :nothing, + conflict_target: [:station_code, :station_type] + ) + + Logger.info("Synced #{length(stations)} stations from #{network}") + + {:error, e} -> + Logger.warning("Failed to sync #{network}: #{inspect(e)}") + end + end + + @doc """ + Flip `contacts.weather_status` from `:queued` to `:complete` for + every contact whose ±2h / 150km window now contains at least one + surface observation. Returns `{:ok, n}` where `n` is the number of + contacts advanced. + + Background: `WeatherFetchWorker` upserts observations but has no + back-pointer to the contacts that triggered the fetch — the same + obs row satisfies many contacts. Previously the only place that + flipped `:queued → :complete` was `MicrowavepropWeb.ContactLive.Show` + on page view, which left thousands of contacts stuck in `:queued` + even after their data had landed. This reconciler closes that loop + as a single SQL UPDATE, invoked from the hourly enqueuer cron. + + Radius encoded as a ±1.5° latitude band; the longitude band is + scaled by `1 / cos(lat)` so the box covers the same physical + east-west distance (~150 km) at every latitude. A fixed 1.5° lon + box collapses to ~110 km at lat 49° and would silently skip + observations the per-contact `weather_for_contact/2` query would + match. + """ + @spec reconcile_weather_statuses() :: {:ok, non_neg_integer()} + def reconcile_weather_statuses do + sql = """ + UPDATE contacts c + SET weather_status = 'complete' + WHERE c.weather_status = 'queued' + AND c.pos1 IS NOT NULL + AND ( + EXISTS ( + SELECT 1 + FROM surface_observations o + JOIN weather_stations s ON s.id = o.station_id + WHERE o.observed_at >= c.qso_timestamp - interval '2 hours' + AND o.observed_at <= c.qso_timestamp + interval '2 hours' + AND s.lat BETWEEN ((c.pos1->>'lat')::float - 1.5) + AND ((c.pos1->>'lat')::float + 1.5) + AND s.lon BETWEEN ((c.pos1->>'lon')::float - 1.5 / GREATEST(cos(radians((c.pos1->>'lat')::float)), 0.01)) + AND ((c.pos1->>'lon')::float + 1.5 / GREATEST(cos(radians((c.pos1->>'lat')::float)), 0.01)) + ) + OR ( + c.pos2 IS NOT NULL + AND EXISTS ( + SELECT 1 + FROM surface_observations o + JOIN weather_stations s ON s.id = o.station_id + WHERE o.observed_at >= c.qso_timestamp - interval '2 hours' + AND o.observed_at <= c.qso_timestamp + interval '2 hours' + AND s.lat BETWEEN ((c.pos2->>'lat')::float - 1.5) + AND ((c.pos2->>'lat')::float + 1.5) + AND s.lon BETWEEN ((c.pos2->>'lon')::float - 1.5 / GREATEST(cos(radians((c.pos2->>'lat')::float)), 0.01)) + AND ((c.pos2->>'lon')::float + 1.5 / GREATEST(cos(radians((c.pos2->>'lat')::float)), 0.01)) + ) + ) + ) + """ + + %{num_rows: n} = Repo.query!(sql) + {:ok, n} + end + + # ── Solar Index ── + + @spec upsert_solar_index(map()) :: {:ok, SolarIndex.t()} | {:error, Ecto.Changeset.t()} + def upsert_solar_index(attrs) do + %SolarIndex{} + |> SolarIndex.changeset(attrs) + |> Repo.insert( + on_conflict: + from(s in SolarIndex, + update: [ + set: [ + sfi: fragment("EXCLUDED.sfi"), + sfi_adjusted: fragment("EXCLUDED.sfi_adjusted"), + sunspot_number: fragment("EXCLUDED.sunspot_number"), + ap_index: fragment("EXCLUDED.ap_index"), + kp_values: fragment("EXCLUDED.kp_values"), + updated_at: fragment("EXCLUDED.updated_at") + ] + ], + where: + s.sfi != fragment("EXCLUDED.sfi") or + s.ap_index != fragment("EXCLUDED.ap_index") + ), + conflict_target: [:date], + returning: true, + stale_error_field: :id + ) + end + + @doc "Batch upsert solar indices using insert_all in chunks of 500." + @spec upsert_solar_indices_batch([map()]) :: non_neg_integer() + def upsert_solar_indices_batch(records) do + now = DateTime.truncate(DateTime.utc_now(), :second) + + records + |> Enum.chunk_every(500) + |> Enum.reduce(0, fn chunk, acc -> + entries = + Enum.map(chunk, fn attrs -> + %{ + id: UUID.generate(), + date: attrs[:date] || attrs.date, + sfi: attrs[:sfi], + sfi_adjusted: attrs[:sfi_adjusted], + sunspot_number: attrs[:sunspot_number], + ap_index: attrs[:ap_index], + kp_values: attrs[:kp_values], + inserted_at: now, + updated_at: now + } + end) + + {count, _} = + Repo.insert_all(SolarIndex, entries, + on_conflict: + from(s in SolarIndex, + update: [ + set: [ + sfi: fragment("EXCLUDED.sfi"), + sfi_adjusted: fragment("EXCLUDED.sfi_adjusted"), + sunspot_number: fragment("EXCLUDED.sunspot_number"), + ap_index: fragment("EXCLUDED.ap_index"), + kp_values: fragment("EXCLUDED.kp_values"), + updated_at: fragment("EXCLUDED.updated_at") + ] + ], + where: + s.sfi != fragment("EXCLUDED.sfi") or + s.ap_index != fragment("EXCLUDED.ap_index") + ), + conflict_target: [:date] + ) + + acc + count + end) + end + + @spec get_solar_index(Date.t()) :: SolarIndex.t() | nil + def get_solar_index(date) do + Repo.get_by(SolarIndex, date: date) + end + + @spec existing_solar_dates() :: MapSet.t(Date.t()) + def existing_solar_dates do + SolarIndex + |> select([s], s.date) + |> Repo.all() + |> MapSet.new() + end + + # ── Private helpers ── + + defp row_has_observed_at?(%{observed_at: %DateTime{}}), do: true + defp row_has_observed_at?(%{"observed_at" => %DateTime{}}), do: true + defp row_has_observed_at?(_), do: false + + # IEM ASOS occasionally returns two rows with the same timestamp + # (e.g. routine + special METAR at the same minute). Passing both + # to insert_all triggers Postgres 21000 cardinality_violation. + # Keep the last occurrence — IEM's ordering is chronological, so + # the later row is the final correction. + defp dedupe_last_by_conflict_target(entries) do + entries + |> Enum.reverse() + |> Enum.uniq_by(&{&1.station_id, &1.observed_at}) + |> Enum.reverse() + end + + defp surface_observation_entry(row, station_id, now) do + %{ + id: UUID.generate(), + station_id: station_id, + observed_at: fetch_row(row, :observed_at), + temp_f: fetch_row(row, :temp_f), + dewpoint_f: fetch_row(row, :dewpoint_f), + relative_humidity: fetch_row(row, :relative_humidity), + wind_speed_kts: fetch_row(row, :wind_speed_kts), + wind_direction_deg: fetch_row(row, :wind_direction_deg), + sea_level_pressure_mb: fetch_row(row, :sea_level_pressure_mb), + altimeter_setting: fetch_row(row, :altimeter_setting), + sky_condition: fetch_row(row, :sky_condition), + precip_1h_in: fetch_row(row, :precip_1h_in), + wx_codes: fetch_row(row, :wx_codes), + inserted_at: now, + updated_at: now + } + end + + defp fetch_row(row, key) when is_atom(key) do + case Map.fetch(row, key) do + {:ok, value} -> value + :error -> Map.get(row, Atom.to_string(key)) + end + end +end diff --git a/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex b/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex index d8bed27b..70a1117c 100644 --- a/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex +++ b/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex @@ -23,6 +23,81 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do # Oban jobs have ~149 params each; PG limit is 65535 params per query @insert_batch_size 400 + @doc """ + Batch-enqueue all enrichment jobs (weather, HRRR, terrain, radar, + mechanism, IEMRE, NARR) for a list of contacts. Used by CSV/ADIF + import to avoid N+1 per-contact enqueuing — all jobs are built and + inserted in bulk, and enrichment statuses are marked for every + contact in one pass. + """ + @spec enqueue_for_contacts([Contact.t()]) :: :ok + def enqueue_for_contacts(contacts) when is_list(contacts) do + if contacts == [] do + :ok + else + do_enqueue_for_contacts(contacts) + end + end + + defp do_enqueue_for_contacts(contacts) do + contacts = Enum.map(contacts, &Radio.ensure_positions!/1) + {narr_contacts, modern_contacts} = Enum.split_with(contacts, &NarrClient.in_coverage?(&1.qso_timestamp)) + modern_conus = Enum.filter(modern_contacts, &(not is_nil(&1.pos1) and Grid.contains?(&1.pos1))) + + # Build all job types in batch using the existing list-accepting builders + weather_jobs = build_weather_jobs(contacts) + terrain_jobs = build_terrain_jobs(contacts) + radar_jobs = build_radar_jobs(contacts) + mechanism_jobs = build_mechanism_jobs(contacts) + iemre_jobs = build_iemre_jobs(contacts) + narr_jobs = build_narr_jobs(contacts) + + bulk_jobs = weather_jobs ++ terrain_jobs ++ iemre_jobs ++ radar_jobs ++ mechanism_jobs + + if bulk_jobs != [] do + insert_all_chunked(bulk_jobs) + end + + if narr_jobs != [] do + Oban.insert_all(narr_jobs) + end + + # HRRR — only modern CONUS contacts (pre-2014 / OCONUS contacts + # can't use HRRR; they fall through to NARR or :unavailable) + _ = + if modern_conus != [] do + {:ok, _} = HrrrPointEnqueuer.enqueue_for_contacts(modern_conus) + end + + # ── mark enrichment statuses ────────────────────────────────────── + all_ids = Enum.map(contacts, & &1.id) + narr_ids = Enum.map(narr_contacts, & &1.id) + modern_ids = Enum.map(modern_contacts, & &1.id) + conus_ids = Enum.map(modern_conus, & &1.id) + + # Per-contact builders enforce pos1 / pos1+pos2 guards, so + # statuses that track "was work created" align correctly. + _ = Radio.mark_weather_queued!(all_ids) + _ = Radio.mark_iemre_queued!(all_ids) + _ = Radio.mark_terrain_queued!(all_ids) + _ = Radio.set_enrichment_status!(all_ids, :mechanism_status, :queued) + + # Radar is unavailable pre-2014 (archive gap) + _ = if narr_ids != [], do: Radio.set_enrichment_status!(narr_ids, :radar_status, :unavailable) + _ = if modern_ids != [], do: Radio.set_enrichment_status!(modern_ids, :radar_status, :queued) + + # HRRR: queued for modern CONUS; unavailable for everything else + _ = if conus_ids != [], do: Radio.mark_hrrr_queued!(conus_ids) + non_hrrr_ids = all_ids -- conus_ids + + _ = + if non_hrrr_ids != [] do + Radio.set_enrichment_status!(non_hrrr_ids, :hrrr_status, :unavailable) + end + + :ok + end + @doc """ Enqueue all enrichment jobs (weather, HRRR, terrain, IEMRE) for a single contact. Called directly from submission flow — no Oban indirection. diff --git a/lib/microwaveprop_web/controllers/contact_map_controller.ex b/lib/microwaveprop_web/controllers/contact_map_controller.ex index 772d9851..7da78ee5 100644 --- a/lib/microwaveprop_web/controllers/contact_map_controller.ex +++ b/lib/microwaveprop_web/controllers/contact_map_controller.ex @@ -10,6 +10,9 @@ defmodule MicrowavepropWeb.ContactMapController do alias Microwaveprop.Cache alias Microwaveprop.Radio + alias MicrowavepropWeb.Api.RateLimiter + + plug RateLimiter, anon_limit: 10, auth_limit: 60 @cache_key {__MODULE__, :gzipped_payload} @cache_ttl_ms 10 * 60 * 1_000 diff --git a/lib/microwaveprop_web/endpoint.ex b/lib/microwaveprop_web/endpoint.ex index 2844be8a..b2aa7e7d 100644 --- a/lib/microwaveprop_web/endpoint.ex +++ b/lib/microwaveprop_web/endpoint.ex @@ -9,7 +9,8 @@ defmodule MicrowavepropWeb.Endpoint do key: "_microwaveprop_key", signing_salt: "FBet7+Mi", encryption_salt: "fPEhSH1PIkX0MJMU1WL3", - same_site: "Lax" + same_site: "Lax", + secure: Mix.env() == :prod ] socket "/live", Phoenix.LiveView.Socket, diff --git a/lib/microwaveprop_web/live/algo_live.ex b/lib/microwaveprop_web/live/algo_live.ex index 320545c5..b4dd4a82 100644 --- a/lib/microwaveprop_web/live/algo_live.ex +++ b/lib/microwaveprop_web/live/algo_live.ex @@ -15,6 +15,10 @@ defmodule MicrowavepropWeb.AlgoLive do ~H"""
+ <%!-- Safe: @content is pre-rendered from checked-in algo.md at compile + time via @external_resource (not user input). If the rendering + pipeline ever accepts dynamic input, switch to Phoenix.HTML.raw/1 + from a sanitized source or a Phoenix.Component function component. --%> {raw(@content)}
diff --git a/lib/microwaveprop_web/live/contact_live/show.ex b/lib/microwaveprop_web/live/contact_live/show.ex index f010d909..5803c168 100644 --- a/lib/microwaveprop_web/live/contact_live/show.ex +++ b/lib/microwaveprop_web/live/contact_live/show.ex @@ -6,14 +6,13 @@ defmodule MicrowavepropWeb.ContactLive.Show do import MicrowavepropWeb.Components.SkewTChart alias Microwaveprop.Propagation.BandConfig + alias Microwaveprop.Propagation.PathAnalysis alias Microwaveprop.Propagation.RainScatterClassifier alias Microwaveprop.Propagation.Scorer alias Microwaveprop.Radio alias Microwaveprop.Radio.ContactCommonVolumeRadar alias Microwaveprop.Repo alias Microwaveprop.Terrain - alias Microwaveprop.Terrain.ElevationClient - alias Microwaveprop.Terrain.TerrainAnalysis alias Microwaveprop.Weather alias Microwaveprop.Weather.HrrrClient alias Microwaveprop.Weather.HrrrNativeProfile @@ -27,7 +26,6 @@ defmodule MicrowavepropWeb.ContactLive.Show do require Logger - @earth_radius_m 6_371_000.0 # T-Mobile CGNAT range — only enqueue enrichment jobs from this subnet @enqueue_subnet {172, 56, 0, 0} @enqueue_mask 13 @@ -497,7 +495,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do terrain = Terrain.get_terrain_profile(contact.id) soundings = socket.assigns.soundings hrrr_path = socket.assigns[:hrrr_path] || Weather.hrrr_profiles_for_path(contact) - elevation_profile = compute_elevation_profile(contact, hrrr_path, soundings) + elevation_profile = PathAnalysis.compute_elevation_profile(contact, hrrr_path, soundings) propagation_analysis = build_propagation_analysis(contact, hrrr_path, terrain, elevation_profile, soundings) {:noreply, @@ -622,13 +620,13 @@ defmodule MicrowavepropWeb.ContactLive.Show do soundings: soundings } = socket.assigns - elevation_profile = compute_elevation_profile(contact, hrrr_path, soundings) + elevation_profile = PathAnalysis.compute_elevation_profile(contact, hrrr_path, soundings) propagation_analysis = build_propagation_analysis(contact, hrrr_path, terrain, elevation_profile, soundings) data_sources = - build_data_sources( + PathAnalysis.build_data_sources( hrrr, hrrr_path, terrain, @@ -891,77 +889,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do def render(assigns) do ~H""" - <.header> - - {@contact.station1} / {@contact.station2} - - <:subtitle> - {format_band(@contact.band)} - <%= if @contact.mode do %> - · {@contact.mode} - <% end %> - · {Calendar.strftime(@contact.qso_timestamp, "%Y-%m-%d %H:%M UTC")} - <%= if @contact.flagged_invalid do %> - - Flagged invalid{flag_attribution(@contact)} - - <% end %> - <%= if @contact.private do %> - - <.icon name="hero-lock-closed" class="w-3 h-3 mr-1" /> Private - - <% end %> - - <:actions> - <%= if current_user(assigns) do %> - - <% end %> - - - <.link navigate={path_calc_url(@contact)} class="btn btn-sm btn-outline"> - <.icon name="hero-calculator" class="w-4 h-4" /> Open in Path Calculator - - <.link navigate={~p"/contacts"} class="btn btn-sm btn-ghost"> - <.icon name="hero-arrow-left" class="w-4 h-4" /> Back to Contacts - - - + <.contact_header contact={@contact} current_scope={@current_scope} editing={@editing} /> <%= if @pending_edit && !@editing && !admin?(assigns) && !Radio.owner?(@contact, current_user(assigns)) do %>
@@ -970,297 +898,612 @@ defmodule MicrowavepropWeb.ContactLive.Show do
<% end %> - <%= if @editing do %> - <% direct_edit = admin?(assigns) or Radio.owner?(@contact, current_user(assigns)) %> -
-

- {if direct_edit, do: "Edit Contact", else: "Suggest Edit"} -

-

- {if direct_edit, - do: "Change any fields below. Changes will be submitted for review.", - else: "Change any fields below. Only fields you modify will be submitted for review."} -

- <.form - for={@edit_form} - id="edit-form" - phx-change="validate_edit" - phx-submit="submit_edit" - class="space-y-4" - > -
- <.input field={@edit_form[:station1]} type="text" label="Station 1" /> - <.input field={@edit_form[:grid1]} type="text" label="Grid 1" /> - <.input - field={@edit_form[:height1_ft]} - type="number" - label="Height 1 (ft AGL)" - min="0" - max="1000" - placeholder="Optional" - /> - <.input field={@edit_form[:station2]} type="text" label="Station 2" /> - <.input field={@edit_form[:grid2]} type="text" label="Grid 2" /> - <.input - field={@edit_form[:height2_ft]} - type="number" - label="Height 2 (ft AGL)" - min="0" - max="1000" - placeholder="Optional" - /> -
-
- <.input field={@edit_form[:band]} type="select" label="Band" options={@band_options} /> - <.input field={@edit_form[:mode]} type="select" label="Mode" options={@mode_options} /> - <.input - field={@edit_form[:qso_timestamp]} - type="text" - label="Timestamp (UTC, 24h)" - placeholder="YYYY-MM-DD HH:MM" - pattern="\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(:\d{2})?Z?" - /> -
-
- <.input - field={@edit_form[:private]} - type="checkbox" - label="Private — only visible to me and administrators" - /> -
-
- <.button type="submit" class="btn btn-primary btn-sm"> - <.icon name="hero-paper-airplane" class="w-4 h-4" /> - {if direct_edit, do: "Save Changes", else: "Submit for Review"} - - -
- -
- <% end %> + <.edit_form + editing={@editing} + current_scope={@current_scope} + contact={@contact} + edit_form={@edit_form} + band_options={@band_options} + mode_options={@mode_options} + /> - <%= if @contact.pos1 && @contact.pos2 do %> -

- Locations approximate, in the center of the grid squares. - <%= if current_user(assigns) do %> - Have more detailed position information? - - <% else %> - Have more detailed position information? - <.link - navigate={~p"/users/log-in"} - class="link link-hover not-italic text-base-content/70" - > - Log in to suggest an edit - - <% end %> -

-
- <% end %> - - <%= if !@elevation_profile && @contact.terrain_status == :queued do %> -
- - Computing elevation profile and terrain analysis {queue_info(@queue_counts, "terrain")} -
- <% end %> - - <%= if @elevation_profile do %> -
-
- {format_band(@contact.band)} · {@contact.mode} · {format_dist( - @elevation_profile.dist_km - )} · <.grid_link grid={@contact.grid1} /> → - <.grid_link grid={@contact.grid2} /> -
- - - - - - - - - - - -
{@contact.station1} → {@contact.station2}Az: {format_bearing(@elevation_profile.fwd_az)}El: {@elevation_profile.fwd_el}°
{@contact.station2} → {@contact.station1}Az: {format_bearing(@elevation_profile.rev_az)}El: {@elevation_profile.rev_el}°
- - <%= if @elevation_profile.verdict == "BLOCKED" && @elevation_profile.first_obstruction_km do %> -
- Obstructed at {format_dist(@elevation_profile.first_obstruction_km)} -
- <% end %> - <%= if @elevation_profile.verdict == "CLEAR" do %> -
- Line of sight clear -
- <% end %> - -
- -
-
- <% end %> + <.elevation_profile_panel + contact={@contact} + current_scope={@current_scope} + elevation_profile={@elevation_profile} + queue_counts={@queue_counts} + /> <%= if @propagation_analysis do %>
- -

Propagation Analysis

-
-

{@propagation_analysis.summary}

- <%= for detail <- @propagation_analysis.details do %> -

{detail}

- <% end %> -
- - <%= if @propagation_analysis.factors do %> -
- - - - - - - - - - - - <%= for f <- @propagation_analysis.factor_rows do %> - - - - - - - - <% end %> - - - - - - - - -
FactorScoreWeightContributionNotes
{f.name}{f.score}/100{f.weight}%{f.contribution}{f.note}
Composite{@propagation_analysis.composite_score}/100 - - {propagation_tier_label(@propagation_analysis.composite_score)} - -
-
- <% end %> + <.propagation_analysis_panel propagation_analysis={@propagation_analysis} /> <% end %>
-

Propagation Mechanism

-
-
- - {Mechanism.label(@mechanism)} + <.mechanism_panel mechanism={@mechanism} radar={@radar} /> + + <.terrain_profile_panel + terrain={@terrain} + terrain_expanded={@terrain_expanded} + hydration_pending={@hydration_pending} + contact={@contact} + queue_counts={@queue_counts} + /> + +
+ + <.soundings_panel + soundings={@soundings} + hydration_pending={@hydration_pending} + sounding_search_pending?={@sounding_search_pending?} + sounding_search_radius_km={@sounding_search_radius_km} + contact={@contact} + queue_counts={@queue_counts} + expanded_soundings={@expanded_soundings} + /> + +
+ + <.solar_panel solar={@solar} queue_counts={@queue_counts} /> + +
+ + <.atmospheric_profile_panel + hrrr={@hrrr} + narr={@narr} + native_profile={@native_profile} + hrrr_profile_expanded={@hrrr_profile_expanded} + hydration_pending={@hydration_pending} + contact={@contact} + queue_counts={@queue_counts} + /> + +
+ + <.iemre_panel iemre={@iemre} contact={@contact} /> + +
+ + <.surface_observations_panel + surface_observations={@surface_observations} + contact={@contact} + queue_counts={@queue_counts} + /> + + <%= if @data_sources do %> +
+ <.data_sources_panel + data_sources={@data_sources} + surface_observations={@surface_observations} + soundings={@soundings} + solar={@solar} + /> + <% end %> + + """ + end + + # ── Render Components ────────────────────────────────────────── + + attr :contact, :map, required: true + attr :current_scope, :any, required: true + attr :editing, :boolean, required: true + + defp contact_header(assigns) do + ~H""" + <.header> + + {@contact.station1} / {@contact.station2} + + <:subtitle> + {format_band(@contact.band)} + <%= if @contact.mode do %> + · {@contact.mode} + <% end %> + · {Calendar.strftime(@contact.qso_timestamp, "%Y-%m-%d %H:%M UTC")} + <%= if @contact.flagged_invalid do %> + + Flagged invalid{flag_attribution(@contact)} - {Mechanism.explainer(@mechanism)} + <% end %> + <%= if @contact.private do %> + + <.icon name="hero-lock-closed" class="w-3 h-3 mr-1" /> Private + + <% end %> + + <:actions> + <%= if current_user(assigns) do %> + + <% end %> + + + <.link navigate={path_calc_url(@contact)} class="btn btn-sm btn-outline"> + <.icon name="hero-calculator" class="w-4 h-4" /> Open in Path Calculator + + <.link navigate={~p"/contacts"} class="btn btn-sm btn-ghost"> + <.icon name="hero-arrow-left" class="w-4 h-4" /> Back to Contacts + + + + """ + end + + attr :editing, :boolean, required: true + attr :current_scope, :any, required: true + attr :contact, :map, required: true + attr :edit_form, :any + attr :band_options, :list, default: [] + attr :mode_options, :list, default: [] + + defp edit_form(assigns) do + ~H""" + <%= if @editing do %> + <% direct_edit = admin?(assigns) or Radio.owner?(@contact, current_user(assigns)) %> +
+

+ {if direct_edit, do: "Edit Contact", else: "Suggest Edit"} +

+

+ {if direct_edit, + do: "Change any fields below. Changes will be submitted for review.", + else: "Change any fields below. Only fields you modify will be submitted for review."} +

+ <.form + for={@edit_form} + id="edit-form" + phx-change="validate_edit" + phx-submit="submit_edit" + class="space-y-4" + > +
+ <.input field={@edit_form[:station1]} type="text" label="Station 1" /> + <.input field={@edit_form[:grid1]} type="text" label="Grid 1" /> + <.input + field={@edit_form[:height1_ft]} + type="number" + label="Height 1 (ft AGL)" + min="0" + max="1000" + placeholder="Optional" + /> + <.input field={@edit_form[:station2]} type="text" label="Station 2" /> + <.input field={@edit_form[:grid2]} type="text" label="Grid 2" /> + <.input + field={@edit_form[:height2_ft]} + type="number" + label="Height 2 (ft AGL)" + min="0" + max="1000" + placeholder="Optional" + /> +
+
+ <.input field={@edit_form[:band]} type="select" label="Band" options={@band_options} /> + <.input field={@edit_form[:mode]} type="select" label="Mode" options={@mode_options} /> + <.input + field={@edit_form[:qso_timestamp]} + type="text" + label="Timestamp (UTC, 24h)" + placeholder="YYYY-MM-DD HH:MM" + pattern="\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(:\d{2})?Z?" + /> +
+
+ <.input + field={@edit_form[:private]} + type="checkbox" + label="Private — only visible to me and administrators" + /> +
+
+ <.button type="submit" class="btn btn-primary btn-sm"> + <.icon name="hero-paper-airplane" class="w-4 h-4" /> + {if direct_edit, do: "Save Changes", else: "Submit for Review"} + + +
+ +
+ <% end %> + """ + end + + attr :contact, :map, required: true + attr :current_scope, :any, required: true + attr :elevation_profile, :any + attr :queue_counts, :map, default: %{} + + defp elevation_profile_panel(assigns) do + ~H""" + <%= if @contact.pos1 && @contact.pos2 do %> +

+ Locations approximate, in the center of the grid squares. + <%= if current_user(assigns) do %> + Have more detailed position information? + + <% else %> + Have more detailed position information? + <.link + navigate={~p"/users/log-in"} + class="link link-hover not-italic text-base-content/70" + > + Log in to suggest an edit + + <% end %> +

+
+ <% end %> + + <%= if !@elevation_profile && @contact.terrain_status == :queued do %> +
+ + Computing elevation profile and terrain analysis {queue_info(@queue_counts, "terrain")} +
+ <% end %> + + <%= if @elevation_profile do %> +
+
+ {format_band(@contact.band)} · {@contact.mode} · {format_dist( + @elevation_profile.dist_km + )} · <.grid_link grid={@contact.grid1} /> → <.grid_link grid={@contact.grid2} />
- <%= if @radar do %> -
- - Common volume: {format_number(@radar.common_volume_km2, 0)} km² - - <%= if @radar.max_dbz do %> - Max reflectivity: {format_number(@radar.max_dbz, 1)} dBZ + + + + + + + + + + + +
{@contact.station1} → {@contact.station2}Az: {format_bearing(@elevation_profile.fwd_az)}El: {@elevation_profile.fwd_el}°
{@contact.station2} → {@contact.station1}Az: {format_bearing(@elevation_profile.rev_az)}El: {@elevation_profile.rev_el}°
+ + <%= if @elevation_profile.verdict == "BLOCKED" && @elevation_profile.first_obstruction_km do %> +
+ Obstructed at {format_dist(@elevation_profile.first_obstruction_km)} +
+ <% end %> + <%= if @elevation_profile.verdict == "CLEAR" do %> +
+ Line of sight clear +
+ <% end %> + +
+ +
+
+ <% end %> + """ + end + + attr :propagation_analysis, :map, required: true + + defp propagation_analysis_panel(assigns) do + ~H""" +

Propagation Analysis

+
+

{@propagation_analysis.summary}

+ <%= for detail <- @propagation_analysis.details do %> +

{detail}

+ <% end %> +
+ + <%= if @propagation_analysis.factors do %> +
+ + + + + + + + + + + + <%= for f <- @propagation_analysis.factor_rows do %> + + + + + + + <% end %> - Heavy-rain pixels: {@radar.heavy_rain_pixel_count} - <%= if @radar.coverage_pct do %> - Radar coverage: {format_number(@radar.coverage_pct, 0)}% - <% end %> - - Frame: {Calendar.strftime(@radar.observed_at, "%Y-%m-%d %H:%M UTC")} + + + + + + + + +
FactorScoreWeightContributionNotes
{f.name}{f.score}/100{f.weight}%{f.contribution}{f.note}
Composite{@propagation_analysis.composite_score}/100 + + {propagation_tier_label(@propagation_analysis.composite_score)} + +
+
+ <% end %> + """ + end + + attr :mechanism, :any + attr :radar, :any + + defp mechanism_panel(assigns) do + ~H""" +

Propagation Mechanism

+
+
+ + {Mechanism.label(@mechanism)} + + {Mechanism.explainer(@mechanism)} +
+ <%= if @radar do %> +
+ + Common volume: {format_number(@radar.common_volume_km2, 0)} km² + + <%= if @radar.max_dbz do %> + Max reflectivity: {format_number(@radar.max_dbz, 1)} dBZ + <% end %> + Heavy-rain pixels: {@radar.heavy_rain_pixel_count} + <%= if @radar.coverage_pct do %> + Radar coverage: {format_number(@radar.coverage_pct, 0)}% + <% end %> + + Frame: {Calendar.strftime(@radar.observed_at, "%Y-%m-%d %H:%M UTC")} + +
+ <% end %> +
+ """ + end + + attr :terrain, :any + attr :terrain_expanded, :boolean, default: false + attr :hydration_pending, :any, default: MapSet.new() + attr :contact, :map, required: true + attr :queue_counts, :map, default: %{} + + defp terrain_profile_panel(assigns) do + ~H""" +

Terrain Profile

+ <%= if @terrain do %> +
+ + + <%= if @terrain_expanded do %> +
+
+ + + + + + + + + + + + <%= for {pt, idx} <- Enum.with_index(@terrain.path_points) do %> + + + + + + + + <% end %> + +
#LatLonDist (km)Elev (m)
{idx}{format_number(pt["lat"])}{format_number(pt["lon"])}{format_number(pt["dist_km"])}{format_number(pt["elev"])}
+
<% end %>
+ <% else %> +
+ <%= cond do %> + <% MapSet.member?(@hydration_pending, :terrain) -> %> + + Loading terrain profile… + + <% @contact.terrain_status == :queued -> %> + + + Computing terrain profile {queue_info(@queue_counts, "terrain")} + + <% true -> %> + No terrain profile available. + <% end %> +
+ <% end %> + """ + end -

Terrain Profile

- <%= if @terrain do %> -
+ attr :soundings, :list, default: [] + attr :hydration_pending, :any, default: MapSet.new() + attr :sounding_search_pending?, :boolean, default: false + attr :sounding_search_radius_km, :any + attr :contact, :map, required: true + attr :queue_counts, :map, default: %{} + attr :expanded_soundings, :any, default: MapSet.new() + + defp soundings_panel(assigns) do + ~H""" +

Soundings

+ <%= if @soundings == [] do %> +
+ <%= cond do %> + <% MapSet.member?(@hydration_pending, :weather) -> %> + + Loading soundings… + + <% @sounding_search_pending? -> %> + + + Fetching soundings from IEM (search radius {@sounding_search_radius_km} km)… + + <% @contact.weather_status == :queued -> %> + + + Fetching sounding data from nearby stations {queue_info(@queue_counts, "weather")} + + <% true -> %> + No soundings found within 1000 km. + <% end %> +
+ <% else %> + <%= for s <- @soundings do %> +
- <%= if @terrain_expanded do %> + <%= if MapSet.member?(@expanded_soundings, s.id) do %>
- - - - - + + + + + + - <%= for {pt, idx} <- Enum.with_index(@terrain.path_points) do %> + <%= for level <- filter_profile(s.profile) do %> - - - - - + + + + + + <% end %> @@ -1269,439 +1512,366 @@ defmodule MicrowavepropWeb.ContactLive.Show do <% end %> - <% else %> -
- <%= cond do %> - <% MapSet.member?(@hydration_pending, :terrain) -> %> - - Loading terrain profile… - - <% @contact.terrain_status == :queued -> %> - - - Computing terrain profile {queue_info(@queue_counts, "terrain")} - - <% true -> %> - No terrain profile available. - <% end %> -
<% end %> + <% end %> + """ + end -
+ attr :solar, :any + attr :queue_counts, :map, default: %{} -

Soundings

- <%= if @soundings == [] do %> -
- <%= cond do %> - <% MapSet.member?(@hydration_pending, :weather) -> %> - - Loading soundings… - - <% @sounding_search_pending? -> %> - - - Fetching soundings from IEM (search radius {@sounding_search_radius_km} km)… - - <% @contact.weather_status == :queued -> %> - - - Fetching sounding data from nearby stations {queue_info(@queue_counts, "weather")} - - <% true -> %> - No soundings found within 1000 km. - <% end %> -
- <% else %> - <%= for s <- @soundings do %> -
- + + <%= if @hrrr_profile_expanded do %> +
+
+
Surface Temp: {format_number(profile.surface_temp_c)}°C
+
Surface Dewpoint: {format_number(profile.surface_dewpoint_c)}°C
+
Surface Pressure: {format_number(profile.surface_pressure_mb)} mb
+
+ <%= if run_time = Map.get(profile, :run_time) do %> + Run Time: {Calendar.strftime(run_time, "%Y-%m-%d %H:%M UTC")} + <% else %> + Source: {profile_source} reanalysis <% end %>
- <.icon - name={ - if MapSet.member?(@expanded_soundings, s.id), - do: "hero-chevron-up", - else: "hero-chevron-down" - } - class="w-5 h-5" - /> - - - <%= if MapSet.member?(@expanded_soundings, s.id) do %> -
-
-
#LatLonDist (km)Elev (m)Pressure (mb)Height (m)Temp (°C)Dewpoint (°C)Wind Dir (°)Wind Spd (kts)
{idx}{format_number(pt["lat"])}{format_number(pt["lon"])}{format_number(pt["dist_km"])}{format_number(pt["elev"])}{format_number(level["pres"])}{format_number(level["hght"])}{format_number(level["tmpc"])}{format_number(level["dwpc"])}{format_number(level["drct"])}{format_number(level["sknt"])}
- - - - - - - - - - - - <%= for level <- filter_profile(s.profile) do %> - - - - - - - - - <% end %> - -
Pressure (mb)Height (m)Temp (°C)Dewpoint (°C)Wind Dir (°)Wind Spd (kts)
{format_number(level["pres"])}{format_number(level["hght"])}{format_number(level["tmpc"])}{format_number(level["dwpc"])}{format_number(level["drct"])}{format_number(level["sknt"])}
+
+ <%= if skew_t_levels != [] do %> + <%= if @native_profile do %> +
+ Skew-T source: {skew_t_source}
+ <% end %> + <.skew_t_chart profile={skew_t_levels} /> +
+ + + + + + + + + + + <%= for level <- skew_t_levels do %> + + + + + + + <% end %> + +
Pressure (mb)Height (m)Temp (°C)Dewpoint (°C)
{format_number(level["pres"])}{format_number(level["hght"])}{format_number(level["tmpc"])}{format_number(level["dwpc"])}
<% end %>
<% end %> - <% end %> - -
- -

Solar Conditions

- <%= if @solar do %> -
-
SFI: {@solar.sfi || "—"}
-
Sunspot #: {@solar.sunspot_number || "—"}
-
Ap: {@solar.ap_index || "—"}
-
Kp: {format_kp(@solar.kp_values)}
-
- <% else %> -
- - - Fetching solar index data {queue_info(@queue_counts, "solar")} - -
- <% end %> - -
- -

Atmospheric Profile

- <% profile = @hrrr || @narr %> - <% profile_source = - cond do - @hrrr -> "HRRR (3 km)" - @narr -> "NARR (32 km)" - true -> nil - end %> - <% skew_t_levels = - cond do - @native_profile -> - HrrrNativeProfile.to_skew_t_profile(@native_profile) - - profile && profile.profile -> - profile.profile - - true -> - [] - end %> - <% skew_t_source = - cond do - @native_profile -> "HRRR native (50 hybrid levels)" - true -> profile_source - end %> - <%= if profile do %> -
- - - <%= if @hrrr_profile_expanded do %> -
-
-
Surface Temp: {format_number(profile.surface_temp_c)}°C
-
Surface Dewpoint: {format_number(profile.surface_dewpoint_c)}°C
-
Surface Pressure: {format_number(profile.surface_pressure_mb)} mb
-
- <%= if run_time = Map.get(profile, :run_time) do %> - Run Time: {Calendar.strftime(run_time, "%Y-%m-%d %H:%M UTC")} - <% else %> - Source: {profile_source} reanalysis - <% end %> -
-
- <%= if skew_t_levels != [] do %> - <%= if @native_profile do %> -
- Skew-T source: {skew_t_source} -
- <% end %> - <.skew_t_chart profile={skew_t_levels} /> -
- - - - - - - - - - - <%= for level <- skew_t_levels do %> - - - - - - - <% end %> - -
Pressure (mb)Height (m)Temp (°C)Dewpoint (°C)
{format_number(level["pres"])}{format_number(level["hght"])}{format_number(level["tmpc"])}{format_number(level["dwpc"])}
-
- <% end %> -
- <% end %> -
- <% else %> -
- <%= cond do %> - <% atmospheric_loading?(@hydration_pending) -> %> - - Loading atmospheric profile… - - <% @contact.hrrr_status == :queued -> %> - - - Fetching HRRR atmospheric data {queue_info(@queue_counts, "hrrr")} - - <% @contact.hrrr_status == :unavailable -> %> - - - HRRR unavailable — fetching NARR reanalysis {queue_info(@queue_counts, "narr")} - - <% true -> %> - No atmospheric profile available. - <% end %> -
- <% end %> - -
- -

IEMRE Gridded Reanalysis

- <%= if @iemre do %> - <% h = @iemre.nearest_hour %> - <% obs = @iemre.observation %> -
- Grid point: {format_number(obs.lat)}°, {format_number(obs.lon)}° — {obs.date} hour {h[ - "utc_hour" - ] || "—"} UTC -
-
-
Temp: {format_number(h["air_temp_f"])}°F
-
Dewpoint: {format_number(h["dew_point_f"])}°F
-
Sky Cover: {format_number(h["skyc_%"])}%
-
- Wind: - {format_number(wind_speed_from_components(h["uwnd_mps"], h["vwnd_mps"]))} kts -
-
- Precip: {format_number(h["hourly_precip_in"])}" -
-
- Soil Temp: {format_number(h["soil4t_f"])}°F -
-
- <% else %> -
- <%= case @contact.iemre_status do %> - <% :queued -> %> - - Fetching IEMRE data - - <% :unavailable -> %> - IEMRE data unavailable for this contact. - <% _ -> %> - No IEMRE data available. - <% end %> -
- <% end %> - -
- -

Surface Observations

- <%= if @surface_observations == [] do %> -
- <%= if @contact.weather_status == :queued do %> +
+ <% else %> +
+ <%= cond do %> + <% atmospheric_loading?(@hydration_pending) -> %> + + Loading atmospheric profile… + + <% @contact.hrrr_status == :queued -> %> - Fetching surface observations from nearby stations {queue_info(@queue_counts, "weather")}... + Fetching HRRR atmospheric data {queue_info(@queue_counts, "hrrr")} + <% @contact.hrrr_status == :unavailable -> %> + + + HRRR unavailable — fetching NARR reanalysis {queue_info(@queue_counts, "narr")} + + <% true -> %> + No atmospheric profile available. + <% end %> +
+ <% end %> + """ + end + + attr :iemre, :any + attr :contact, :map, required: true + + defp iemre_panel(assigns) do + ~H""" +

IEMRE Gridded Reanalysis

+ <%= if @iemre do %> + <% h = @iemre.nearest_hour %> + <% obs = @iemre.observation %> +
+ Grid point: {format_number(obs.lat)}°, {format_number(obs.lon)}° — {obs.date} hour {h[ + "utc_hour" + ] || "—"} UTC +
+
+
Temp: {format_number(h["air_temp_f"])}°F
+
Dewpoint: {format_number(h["dew_point_f"])}°F
+
Sky Cover: {format_number(h["skyc_%"])}%
+
+ Wind: + {format_number(wind_speed_from_components(h["uwnd_mps"], h["vwnd_mps"]))} kts +
+
+ Precip: {format_number(h["hourly_precip_in"])}" +
+
+ Soil Temp: {format_number(h["soil4t_f"])}°F +
+
+ <% else %> +
+ <%= case @contact.iemre_status do %> + <% :queued -> %> + + Fetching IEMRE data + + <% :unavailable -> %> + IEMRE data unavailable for this contact. + <% _ -> %> + No IEMRE data available. + <% end %> +
+ <% end %> + """ + end + + attr :surface_observations, :list, default: [] + attr :contact, :map, required: true + attr :queue_counts, :map, default: %{} + + defp surface_observations_panel(assigns) do + ~H""" +

Surface Observations

+ <%= if @surface_observations == [] do %> +
+ <%= if @contact.weather_status == :queued do %> + + + Fetching surface observations from nearby stations {queue_info(@queue_counts, "weather")}... + + <% else %> + No surface observations found nearby. + <% end %> +
+ <% else %> +
+ + + + + + + + + + + + + + + <%= for obs <- @surface_observations do %> + + + + + + + + + + + <% end %> + +
StationTimeTempDewptRH%WindPres (mb)Sky
{obs.station.name || obs.station.station_code}{Calendar.strftime(obs.observed_at, "%H:%M")}{obs.temp_f || "—"}{obs.dewpoint_f || "—"}{format_number(obs.relative_humidity)}{format_wind(obs)}{obs.sea_level_pressure_mb || "—"}{obs.sky_condition || "—"}
+
+ <% end %> + """ + end + + attr :data_sources, :map, required: true + attr :surface_observations, :list, default: [] + attr :soundings, :list, default: [] + attr :solar, :any + + defp data_sources_panel(assigns) do + ~H""" +

Data Sources

+
+
+
+
HRRR Model
+ <%= if @data_sources.hrrr do %> +
{@data_sources.hrrr.profile_count} path profiles
+
{@data_sources.hrrr.levels} pressure levels each
+
Valid: {@data_sources.hrrr.valid_time}
+
Grid: {@data_sources.hrrr.lat}, {@data_sources.hrrr.lon}
<% else %> - No surface observations found nearby. +
Not available
<% end %>
- <% else %> -
- - - - - - - - - - - - - - - <%= for obs <- @surface_observations do %> - - - - - - - - - - - <% end %> - -
StationTimeTempDewptRH%WindPres (mb)Sky
{obs.station.name || obs.station.station_code}{Calendar.strftime(obs.observed_at, "%H:%M")}{obs.temp_f || "—"}{obs.dewpoint_f || "—"}{format_number(obs.relative_humidity)}{format_wind(obs)}{obs.sea_level_pressure_mb || "—"}{obs.sky_condition || "—"}
+
+ +
+
+
Elevation Profile
+ <%= if @data_sources.elevation do %> +
{@data_sources.elevation.samples} samples along path
+
{@data_sources.elevation.dist} path distance
+
Source: {@data_sources.elevation.source}
+
{@data_sources.elevation.duct_count} duct layers detected
+ <% else %> +
Not available
+ <% end %>
- <% end %> +
- <%= if @data_sources do %> -
- -

Data Sources

-
-
-
-
HRRR Model
- <%= if @data_sources.hrrr do %> -
{@data_sources.hrrr.profile_count} path profiles
-
{@data_sources.hrrr.levels} pressure levels each
-
Valid: {@data_sources.hrrr.valid_time}
-
Grid: {@data_sources.hrrr.lat}, {@data_sources.hrrr.lon}
- <% else %> -
Not available
- <% end %> -
-
- -
-
-
Elevation Profile
- <%= if @data_sources.elevation do %> -
{@data_sources.elevation.samples} samples along path
-
{@data_sources.elevation.dist} path distance
-
Source: {@data_sources.elevation.source}
-
{@data_sources.elevation.duct_count} duct layers detected
- <% else %> -
Not available
- <% end %> -
-
- -
-
-
Terrain Analysis
- <%= if @data_sources.terrain do %> -
{@data_sources.terrain.samples} profile samples
-
Verdict: {@data_sources.terrain.verdict}
-
Max elevation: {@data_sources.terrain.max_elev} m
-
Diffraction: {@data_sources.terrain.diffraction} dB
- <% else %> -
Not available
- <% end %> -
-
- -
-
-
Surface Observations
-
{length(@surface_observations)} stations
- <%= if @surface_observations != [] do %> -
- {Calendar.strftime(hd(@surface_observations).observed_at, "%H:%M")} – {Calendar.strftime( - List.last(@surface_observations).observed_at, - "%H:%M" - )} UTC -
- <% end %> -
-
- -
-
-
Soundings (RAOB)
-
{length(@soundings)} profiles
- <%= if @soundings != [] do %> -
- {Enum.map_join(@soundings, ", ", fn s -> - "#{s.station.station_code} (#{s.level_count} levels)" - end)} -
-
{Enum.count(@soundings, & &1.ducting_detected)} with ducting
- <% end %> -
-
- -
-
-
Solar Indices
- <%= if @solar do %> -
SFI: {@solar.sfi || "—"}
-
SSN: {@solar.sunspot_number || "—"}
-
Ap: {@solar.ap_index || "—"}
- <% else %> -
Not available
- <% end %> -
-
+
+
+
Terrain Analysis
+ <%= if @data_sources.terrain do %> +
{@data_sources.terrain.samples} profile samples
+
Verdict: {@data_sources.terrain.verdict}
+
Max elevation: {@data_sources.terrain.max_elev} m
+
Diffraction: {@data_sources.terrain.diffraction} dB
+ <% else %> +
Not available
+ <% end %>
- <% end %> - +
+ +
+
+
Surface Observations
+
{length(@surface_observations)} stations
+ <%= if @surface_observations != [] do %> +
+ {Calendar.strftime(hd(@surface_observations).observed_at, "%H:%M")} – {Calendar.strftime( + List.last(@surface_observations).observed_at, + "%H:%M" + )} UTC +
+ <% end %> +
+
+ +
+
+
Soundings (RAOB)
+
{length(@soundings)} profiles
+ <%= if @soundings != [] do %> +
+ {Enum.map_join(@soundings, ", ", fn s -> + "#{s.station.station_code} (#{s.level_count} levels)" + end)} +
+
{Enum.count(@soundings, & &1.ducting_detected)} with ducting
+ <% end %> +
+
+ +
+
+
Solar Indices
+ <%= if @solar do %> +
SFI: {@solar.sfi || "—"}
+
SSN: {@solar.sunspot_number || "—"}
+
Ap: {@solar.ap_index || "—"}
+ <% else %> +
Not available
+ <% end %> +
+
+
""" end @@ -1774,302 +1944,14 @@ defmodule MicrowavepropWeb.ContactLive.Show do defp format_kp(nil), do: "—" defp format_kp(values) when is_list(values), do: Enum.map_join(values, ", ", &format_number/1) - defp terrain_verdict_class("CLEAR"), do: "badge-success" - defp terrain_verdict_class("FRESNEL_MINOR"), do: "badge-warning" - defp terrain_verdict_class("FRESNEL_PARTIAL"), do: "badge-warning" - defp terrain_verdict_class("BLOCKED"), do: "badge-error" - defp terrain_verdict_class(_), do: "badge-ghost" - - defp compute_elevation_profile(contact, hrrr_path, soundings) do - with %{"lat" => lat1} <- contact.pos1, - lon1 when is_number(lon1) <- contact.pos1["lon"], - %{"lat" => lat2} <- contact.pos2, - lon2 when is_number(lon2) <- contact.pos2["lon"], - {:ok, profile} <- safe_fetch_elevation(lat1, lon1, lat2, lon2) do - freq_ghz = band_to_ghz(contact.band) - dist_km = haversine_km(lat1, lon1, lat2, lon2) - # Default to 10 ft (3.048 m) AGL when the operator didn't record an - # antenna height — better than pretending the antenna is on the dirt. - default_ht_m = 3.048 - ant_ht_a_m = ft_to_m(contact.height1_ft) || default_ht_m - ant_ht_b_m = ft_to_m(contact.height2_ft) || default_ht_m - analysis = TerrainAnalysis.analyse(profile, dist_km, freq_ghz, ant_ht_a: ant_ht_a_m, ant_ht_b: ant_ht_b_m) - - fwd_az = initial_bearing(lat1, lon1, lat2, lon2) - rev_az = initial_bearing(lat2, lon2, lat1, lon1) - - tx_elev = hd(profile).elev + ant_ht_a_m - rx_elev = List.last(profile).elev + ant_ht_b_m - fwd_el = elevation_angle(tx_elev, rx_elev, dist_km * 1000) - rev_el = elevation_angle(rx_elev, tx_elev, dist_km * 1000) - - first_obs = - case Enum.find(analysis.points, & &1.obstructed) do - nil -> nil - p -> p.dist_km - end - - ducts = - hrrr_path - |> extract_ducts(soundings, tx_elev) - |> merge_nearby_ducts() - |> Enum.sort_by(& &1.strength, :desc) - |> Enum.take(3) - |> mark_likely_duct(tx_elev, rx_elev) - - %{ - points: - Enum.map(analysis.points, fn p -> - %{elev: p.elev, beam: p.beam, r1: p.r1, dist_km: p.dist_km} - end), - freq_mhz: freq_ghz * 1000, - dist_km: dist_km, - k_factor: analysis.k_factor, - fwd_az: fwd_az, - rev_az: rev_az, - fwd_el: fwd_el, - rev_el: rev_el, - verdict: analysis.verdict, - first_obstruction_km: first_obs, - ducts: ducts - } - else - _ -> nil - end - end - - # Extract duct layers from best available source across all HRRR path profiles: - # 1. HRRR explicit ducts (M-profile detected) from any path point - # 2. Sounding explicit ducts (high vertical resolution, most reliable) - # 3. Inferred from strongest HRRR refractivity gradient along path - defp extract_ducts(hrrr_path, soundings, surface_elev) do - hrrr_ducts = extract_hrrr_ducts(hrrr_path, surface_elev) - - cond do - hrrr_ducts != [] -> - hrrr_ducts - - (sounding_ducts = extract_sounding_ducts(soundings, surface_elev)) != [] -> - sounding_ducts - - true -> - # Use the profile with the strongest (most negative) gradient - best = - hrrr_path - |> Enum.filter(&is_number(&1.min_refractivity_gradient)) - |> Enum.min_by(& &1.min_refractivity_gradient, fn -> nil end) - - infer_duct_from_gradient(best, surface_elev) - end - end - - defp extract_hrrr_ducts(hrrr_path, surface_elev) when is_list(hrrr_path) do - hrrr_path - |> Enum.flat_map(&duct_characteristics_to_maps(&1.duct_characteristics, surface_elev, "hrrr")) - |> Enum.uniq_by(fn d -> {round(d.base_m_msl), round(d.top_m_msl)} end) - end - - defp duct_characteristics_to_maps(ducts, surface_elev, source) when is_list(ducts) and ducts != [] do - Enum.map(ducts, fn d -> - %{ - base_m_msl: surface_elev + (d["base"] || 0), - top_m_msl: surface_elev + (d["top"] || 0), - strength: d["strength"], - source: source - } - end) - end - - defp duct_characteristics_to_maps(_ducts, _surface_elev, _source), do: [] - - defp extract_sounding_ducts(soundings, surface_elev) when is_list(soundings) do - soundings - |> Enum.filter(& &1.ducting_detected) - |> Enum.flat_map(&duct_characteristics_to_maps(&1.duct_characteristics, surface_elev, "sounding")) - |> Enum.uniq_by(fn d -> {round(d.base_m_msl), round(d.top_m_msl)} end) - end - - defp extract_sounding_ducts(_, _), do: [] - - # HRRR pressure levels are too coarse (~250m) to detect thin ducting layers - # via the M-profile method. When min_refractivity_gradient indicates enhanced - # propagation, infer a duct layer within the boundary layer. - defp infer_duct_from_gradient(nil, _surface_elev), do: [] - - defp infer_duct_from_gradient(hrrr, surface_elev) do - grad = hrrr.min_refractivity_gradient - hpbl = hrrr.hpbl_m - - if is_number(grad) and grad < -100 and is_number(hpbl) and hpbl > 0 do - duct_top = min(hpbl * 0.6, 500) - strength = abs(grad) / 10 - - [ - %{ - base_m_msl: surface_elev, - top_m_msl: surface_elev + duct_top, - strength: Float.round(strength, 1), - source: "inferred" - } - ] - else - [] - end - end - - # Merge duct layers that overlap or are within 100m of each other - defp merge_nearby_ducts(ducts) do - ducts - |> Enum.sort_by(& &1.base_m_msl) - |> Enum.reduce([], fn duct, acc -> - case acc do - [prev | rest] when duct.base_m_msl <= prev.top_m_msl + 100 -> - merged = %{ - prev - | top_m_msl: max(prev.top_m_msl, duct.top_m_msl), - strength: max(prev.strength, duct.strength) - } - - [merged | rest] - - _ -> - [duct | acc] - end - end) - |> Enum.reverse() - end - - # Mark the duct most likely carrying the signal. The signal enters at - # surface level from each end, so the duct whose base is closest to - # the average endpoint elevation is the most probable propagation path. - defp mark_likely_duct([], _tx_elev, _rx_elev), do: [] - - defp mark_likely_duct(ducts, tx_elev, rx_elev) do - avg_elev = (tx_elev + rx_elev) / 2 - - {_, likely_idx} = - ducts - |> Enum.with_index() - |> Enum.min_by(fn {d, _idx} -> - # Distance from average endpoint elevation to the duct layer - # Prefer ducts that the endpoints are actually inside of - if avg_elev >= d.base_m_msl and avg_elev <= d.top_m_msl do - -d.strength - else - min(abs(d.base_m_msl - avg_elev), abs(d.top_m_msl - avg_elev)) - end - end) - - Enum.with_index(ducts, fn d, i -> - Map.put(d, :likely, i == likely_idx) - end) - end - - defp safe_fetch_elevation(lat1, lon1, lat2, lon2) do - ElevationClient.fetch_elevation_profile(lat1, lon1, lat2, lon2, 256) - rescue - e -> - Logger.warning("Elevation profile failed: #{Exception.message(e)}") - {:error, :elevation_unavailable} - end - - defp band_to_ghz(nil), do: 10.0 - - defp band_to_ghz(band) do - band - |> Decimal.to_float() - |> Kernel./(1000) - end - - defp ft_to_m(nil), do: nil - defp ft_to_m(ft) when is_integer(ft), do: ft * 0.3048 + defp terrain_verdict_class(verdict), do: Microwaveprop.Format.terrain_verdict_class(verdict) defp haversine_km(lat1, lon1, lat2, lon2) do - dlat = deg_to_rad(lat2 - lat1) - dlon = deg_to_rad(lon2 - lon1) - rlat1 = deg_to_rad(lat1) - rlat2 = deg_to_rad(lat2) - - a = - :math.sin(dlat / 2) ** 2 + - :math.cos(rlat1) * :math.cos(rlat2) * :math.sin(dlon / 2) ** 2 - - 2 * 6371.0 * :math.asin(:math.sqrt(a)) + Microwaveprop.Geo.haversine_km(lat1, lon1, lat2, lon2) end - defp initial_bearing(lat1, lon1, lat2, lon2) do - rlat1 = deg_to_rad(lat1) - rlat2 = deg_to_rad(lat2) - dlon = deg_to_rad(lon2 - lon1) - - y = :math.sin(dlon) * :math.cos(rlat2) - - x = - :math.cos(rlat1) * :math.cos(rlat2) - - :math.cos(rlat1) * :math.sin(rlat2) * :math.cos(dlon) - - bearing = y |> :math.atan2(x) |> rad_to_deg() - Float.round(rem_float(bearing + 360, 360), 1) - end - - defp elevation_angle(h_tx, h_rx, dist_m) do - k_r = 4 / 3 * @earth_radius_m - angle = :math.atan2(h_rx - h_tx, dist_m) - dist_m / (2 * k_r) - Float.round(rad_to_deg(angle), 2) - end - - defp deg_to_rad(deg), do: deg * :math.pi() / 180 - defp rad_to_deg(rad), do: rad * 180 / :math.pi() defp rem_float(a, b), do: a - Float.floor(a / b) * b - # ── Data sources summary ──────────────────────────────────────── - - defp build_data_sources(hrrr, hrrr_path, terrain, weather, elevation_profile) do - %{ - hrrr: build_hrrr_source(hrrr, hrrr_path), - elevation: build_elevation_source(elevation_profile), - terrain: build_terrain_source(terrain), - obs_count: length(weather.surface_observations), - sounding_count: length(weather.soundings) - } - end - - defp build_hrrr_source(nil, _), do: nil - - defp build_hrrr_source(hrrr, hrrr_path) do - levels = if hrrr.profile, do: length(hrrr.profile), else: 0 - - %{ - profile_count: length(hrrr_path), - levels: levels, - valid_time: Calendar.strftime(hrrr.valid_time, "%Y-%m-%d %H:%M UTC"), - lat: :erlang.float_to_binary(hrrr.lat / 1, decimals: 2), - lon: :erlang.float_to_binary(hrrr.lon / 1, decimals: 2) - } - end - - defp build_elevation_source(nil), do: nil - - defp build_elevation_source(ep) do - %{ - samples: length(ep.points), - dist: format_dist(ep.dist_km), - source: "SRTM 1-arcsecond", - duct_count: length(ep.ducts) - } - end - - defp build_terrain_source(nil), do: nil - - defp build_terrain_source(terrain) do - %{ - samples: terrain.sample_count, - verdict: terrain.verdict, - max_elev: :erlang.float_to_binary((terrain.max_elevation_m || 0) / 1, decimals: 0), - diffraction: :erlang.float_to_binary((terrain.diffraction_db || 0) / 1, decimals: 1) - } - end - # ── Propagation analysis ────────────────────────────────────────── defp build_propagation_analysis(contact, hrrr_path, terrain, elevation_profile, soundings) do @@ -2081,9 +1963,9 @@ defmodule MicrowavepropWeb.ContactLive.Show do {factors, factor_rows, composite} = compute_factors(hrrr_path, contact, band_config) summary = - build_summary(terrain, elevation_profile, hrrr, soundings, dist_km, band_mhz, contact) + PathAnalysis.build_summary(terrain, elevation_profile, hrrr, soundings, dist_km, band_mhz, contact) - details = build_details(hrrr, hrrr_path, soundings, elevation_profile, band_mhz, contact) + details = PathAnalysis.build_details(hrrr, hrrr_path, soundings, elevation_profile, band_mhz, contact) %{ summary: summary, @@ -2137,233 +2019,6 @@ defmodule MicrowavepropWeb.ContactLive.Show do end end - defp build_summary(terrain, elevation_profile, hrrr, soundings, dist_km, band_mhz, contact) do - terrain_status = terrain_summary(terrain, elevation_profile, contact) - mechanism = propagation_mechanism(terrain, elevation_profile, hrrr, soundings, dist_km) - band_note = band_summary(band_mhz, hrrr) - - [terrain_status, mechanism, band_note] - |> Enum.reject(&is_nil/1) - |> Enum.join(" ") - end - - defp terrain_summary(nil, nil, _contact), do: "No terrain data available." - - defp terrain_summary(terrain, elevation_profile, contact) do - verdict = (elevation_profile && elevation_profile.verdict) || (terrain && terrain.verdict) - describe_verdict(verdict, elevation_profile, contact) - end - - defp describe_verdict("CLEAR", _, _), do: "Line of sight is clear between stations." - defp describe_verdict("FRESNEL_MINOR", _, _), do: "Line of sight is clear but with minor Fresnel zone encroachment." - defp describe_verdict("FRESNEL_PARTIAL", _, _), do: "Line of sight has partial Fresnel zone obstruction." - - defp describe_verdict("BLOCKED", elevation_profile, contact) do - obs_dist = elevation_profile && elevation_profile.first_obstruction_km - blocked_message(obs_dist, contact) - end - - defp describe_verdict(_, _, _), do: nil - - defp blocked_message(nil, _contact), do: "Path is terrain-obstructed." - - defp blocked_message(obs_dist, contact) do - origin = contact && contact.station1 - origin_label = if origin, do: origin, else: "station 1" - - "Path is terrain-obstructed at #{:erlang.float_to_binary(obs_dist * 0.621371, decimals: 1)} mi (#{:erlang.float_to_binary(obs_dist, decimals: 1)} km) from #{origin_label}." - end - - defp propagation_mechanism(terrain, elevation_profile, hrrr, soundings, dist_km) do - blocked? = terrain && terrain.verdict == "BLOCKED" - ducting? = has_ducting?(hrrr, soundings) - enhanced_refraction? = enhanced_refraction?(hrrr) - long_path? = dist_km && dist_km > 100 - - ducts = (elevation_profile && elevation_profile.ducts) || [] - likely_duct = Enum.find(ducts, & &1[:likely]) - - props = %{ - blocked?: blocked?, - ducting?: ducting?, - enhanced_refraction?: enhanced_refraction?, - long_path?: long_path?, - likely_duct: likely_duct, - terrain: terrain, - hrrr: hrrr - } - - if blocked? do - blocked_mechanism(props) - else - clear_path_mechanism(props) - end - end - - defp enhanced_refraction?(nil), do: false - - defp enhanced_refraction?(hrrr) do - is_number(hrrr.min_refractivity_gradient) && hrrr.min_refractivity_gradient < -100 - end - - defp blocked_mechanism(%{ducting?: true, likely_duct: duct}) when not is_nil(duct) do - "Signal likely propagated via atmospheric ducting (#{duct_description(duct)}), bending over the terrain obstruction." - end - - defp blocked_mechanism(%{ducting?: true}) do - "Ducting conditions detected — signal likely propagated through a tropospheric duct, bypassing the terrain obstruction." - end - - defp blocked_mechanism(%{enhanced_refraction?: true, hrrr: hrrr}) do - grad = round(hrrr.min_refractivity_gradient) - - "Enhanced refraction (dN/dh = #{grad} N-units/km) likely bent the signal over the obstruction. Conditions are super-refractive but below full ducting threshold." - end - - defp blocked_mechanism(%{terrain: terrain}) do - diff_db = terrain && terrain.diffraction_db - blocked_diffraction_message(diff_db) - end - - defp blocked_diffraction_message(diff_db) when is_number(diff_db) and diff_db > 0 do - "Path is obstructed with #{:erlang.float_to_binary(diff_db, decimals: 1)} dB diffraction loss. Contact may have been enabled by knife-edge diffraction or tropospheric scatter." - end - - defp blocked_diffraction_message(_) do - "Path is obstructed. Contact likely enabled by tropospheric scatter or transient ducting conditions." - end - - defp clear_path_mechanism(%{ducting?: true, long_path?: true, likely_duct: duct}) when not is_nil(duct) do - "Ducting conditions present (#{duct_description(duct)}) — likely extending range beyond normal line of sight." - end - - defp clear_path_mechanism(%{ducting?: true, long_path?: true}) do - "Ducting conditions present — likely contributing to extended range." - end - - defp clear_path_mechanism(%{enhanced_refraction?: true, long_path?: true}) do - "Enhanced refraction present — may have contributed to extended range." - end - - defp clear_path_mechanism(_), do: nil - - defp duct_description(%{source: source, base_m_msl: base, top_m_msl: top, strength: strength}) do - base_ft = round(base * 3.28084) - top_ft = round(top * 3.28084) - - src = - case source do - "sounding" -> "sounding-detected" - "inferred" -> "estimated" - _ -> "detected" - end - - "#{src} layer at #{base_ft}-#{top_ft} ft, #{strength} M-units" - end - - defp duct_description(_), do: "detected layer" - - defp has_ducting?(hrrr, soundings) do - hrrr_ducting = hrrr && hrrr.ducting_detected - sounding_ducting = is_list(soundings) && Enum.any?(soundings, & &1.ducting_detected) - hrrr_ducting || sounding_ducting - end - - defp band_summary(band_mhz, hrrr) when band_mhz <= 12_000 do - if hrrr && is_number(hrrr.pwat_mm) && hrrr.pwat_mm > 20 do - "At 10 GHz, the elevated moisture (PWAT #{:erlang.float_to_binary(hrrr.pwat_mm, decimals: 1)} mm) enhances refractivity and ducting potential." - end - end - - defp band_summary(band_mhz, hrrr) when band_mhz >= 24_000 do - if hrrr && is_number(hrrr.pwat_mm) && hrrr.pwat_mm > 25 do - "At #{round(band_mhz / 1000)} GHz, high moisture (PWAT #{:erlang.float_to_binary(hrrr.pwat_mm, decimals: 1)} mm) causes significant water vapor absorption." - end - end - - defp band_summary(_, _), do: nil - - defp build_details(hrrr, hrrr_path, soundings, elevation_profile, _band_mhz, contact) do - Enum.reject( - [ - antenna_heights_detail(contact), - refractivity_detail(hrrr), - boundary_layer_detail(hrrr), - path_duct_count_detail(hrrr_path), - sounding_ducting_detail(soundings), - duct_layer_detail(elevation_profile) - ], - &is_nil/1 - ) - end - - defp antenna_heights_detail(%{height1_ft: h1, height2_ft: h2} = contact) when is_integer(h1) or is_integer(h2) do - s1 = contact.station1 || "Station 1" - s2 = contact.station2 || "Station 2" - - parts = - Enum.reject( - [if(is_integer(h1), do: "#{s1} @ #{h1} ft AGL"), if(is_integer(h2), do: "#{s2} @ #{h2} ft AGL")], - &is_nil/1 - ) - - "Antenna heights: #{Enum.join(parts, ", ")}." - end - - defp antenna_heights_detail(_), do: nil - - defp path_duct_count_detail(hrrr_path) when is_list(hrrr_path) and hrrr_path != [] do - total = length(hrrr_path) - ducting = Enum.count(hrrr_path, &(&1 && &1.ducting_detected)) - - if ducting > 0 do - "Tropo duct detected at #{ducting}/#{total} HRRR samples along the path." - end - end - - defp path_duct_count_detail(_), do: nil - - defp refractivity_detail(%{min_refractivity_gradient: grad}) when is_number(grad) do - g = round(grad) - label = refractivity_label(g) - "Refractivity gradient: #{g} N-units/km (#{label})." - end - - defp refractivity_detail(_), do: nil - - defp refractivity_label(g) when g < -157, do: "ducting" - defp refractivity_label(g) when g < -100, do: "super-refractive" - defp refractivity_label(g) when g < -79, do: "enhanced" - defp refractivity_label(_), do: "normal" - - defp boundary_layer_detail(%{hpbl_m: hpbl}) when is_number(hpbl) do - h = round(hpbl) - note = if h < 300, do: " — shallow boundary layer favors ducting", else: "" - "Boundary layer depth: #{h} m#{note}." - end - - defp boundary_layer_detail(_), do: nil - - defp sounding_ducting_detail(soundings) when is_list(soundings) do - ducting = Enum.filter(soundings, & &1.ducting_detected) - - if ducting == [] do - nil - else - names = Enum.map_join(ducting, ", ", fn s -> s.station.name || s.station.station_code end) - "Ducting detected by soundings at: #{names}." - end - end - - defp sounding_ducting_detail(_), do: nil - - defp duct_layer_detail(%{ducts: ducts}) when is_list(ducts) do - likely = Enum.find(ducts, & &1[:likely]) - if likely, do: "Most likely propagation duct: #{duct_description(likely)}." - end - - defp duct_layer_detail(_), do: nil - # Factor note helpers defp humidity_note(abs_h, band_config) do h = Float.round(abs_h, 1) @@ -2450,25 +2105,9 @@ defmodule MicrowavepropWeb.ContactLive.Show do "#{Float.round(mb, 1)} mb (#{label})" end - defp propagation_tier_label(score) do - cond do - score >= 80 -> "EXCELLENT" - score >= 65 -> "GOOD" - score >= 50 -> "MARGINAL" - score >= 33 -> "POOR" - true -> "NEGLIGIBLE" - end - end + defp propagation_tier_label(score), do: Microwaveprop.Format.propagation_tier_label(score) - defp propagation_tier_class(score) do - cond do - score >= 80 -> "badge badge-sm badge-success" - score >= 65 -> "badge badge-sm badge-info" - score >= 50 -> "badge badge-sm badge-warning" - score >= 33 -> "badge badge-sm badge-error" - true -> "badge badge-sm badge-ghost" - end - end + defp propagation_tier_class(score), do: Microwaveprop.Format.propagation_tier_badge_class(score) defp path_calc_url(contact) do source = prefer_grid(contact.grid1, contact.station1) diff --git a/lib/microwaveprop_web/live/path_live.ex b/lib/microwaveprop_web/live/path_live.ex index a01ab1cf..6070596b 100644 --- a/lib/microwaveprop_web/live/path_live.ex +++ b/lib/microwaveprop_web/live/path_live.ex @@ -468,23 +468,11 @@ defmodule MicrowavepropWeb.PathLive do # ── Score tier helpers ── - defp tier_label(score) when score >= 80, do: "EXCELLENT" - defp tier_label(score) when score >= 65, do: "GOOD" - defp tier_label(score) when score >= 50, do: "MARGINAL" - defp tier_label(score) when score >= 33, do: "POOR" - defp tier_label(_), do: "NEGLIGIBLE" + defp tier_label(score), do: Microwaveprop.Format.propagation_tier_label(score) - defp tier_color(score) when score >= 80, do: "#059669" - defp tier_color(score) when score >= 65, do: "#0d9488" - defp tier_color(score) when score >= 50, do: "#ca8a04" - defp tier_color(score) when score >= 33, do: "#ea580c" - defp tier_color(_), do: "#dc2626" + defp tier_color(score), do: Microwaveprop.Format.propagation_tier_color(score) - defp terrain_verdict_class("CLEAR"), do: "badge badge-success" - defp terrain_verdict_class("FRESNEL_MINOR"), do: "badge badge-warning" - defp terrain_verdict_class("FRESNEL_PARTIAL"), do: "badge badge-warning" - defp terrain_verdict_class("BLOCKED"), do: "badge badge-error" - defp terrain_verdict_class(_), do: "badge" + defp terrain_verdict_class(verdict), do: Microwaveprop.Format.terrain_verdict_class(verdict) defp format_number(n) when is_float(n), do: :erlang.float_to_binary(n, decimals: 1) defp format_number(n), do: to_string(n) @@ -702,7 +690,7 @@ defmodule MicrowavepropWeb.PathLive do
Terrain
- + {@result.terrain.analysis.verdict}
@@ -1385,11 +1373,7 @@ defmodule MicrowavepropWeb.PathLive do """ end - defp score_bar_color(s) when s >= 80, do: "#059669" - defp score_bar_color(s) when s >= 65, do: "#0d9488" - defp score_bar_color(s) when s >= 50, do: "#ca8a04" - defp score_bar_color(s) when s >= 33, do: "#ea580c" - defp score_bar_color(_), do: "#dc2626" + defp score_bar_color(s), do: Microwaveprop.Format.propagation_tier_color(s) defp factor_name(:humidity), do: "Humidity" defp factor_name(:time_of_day), do: "Time of Day" diff --git a/lib/microwaveprop_web/live/submit_live.ex b/lib/microwaveprop_web/live/submit_live.ex index 94d49ed4..f7a03828 100644 --- a/lib/microwaveprop_web/live/submit_live.ex +++ b/lib/microwaveprop_web/live/submit_live.ex @@ -87,6 +87,8 @@ defmodule MicrowavepropWeb.SubmitLive do {:noreply, socket} end + @max_import_rows 2_000 + # Minimum milliseconds between submissions per LiveView session. # For broader IP-based rate limiting, use a reverse proxy (nginx limit_req, Cloudflare). @submission_cooldown_ms 30_000 @@ -123,29 +125,30 @@ defmodule MicrowavepropWeb.SubmitLive do end def handle_event("upload_csv", params, socket) do - email = - case current_user(socket.assigns) do - %{email: user_email} -> user_email - _ -> params |> Map.get("submitter_email", "") |> String.trim() - end + cond do + recently_submitted?(socket) -> + {:noreply, put_flash(socket, :error, "Please wait before submitting another contact.")} - private = Map.get(params, "private") == "true" + !current_user(socket.assigns) -> + {:noreply, put_flash(socket, :error, "Please sign in to upload files.")} - if email == "" do - {:noreply, put_flash(socket, :error, "Email is required for CSV upload")} - else - uploaded_contents = - consume_uploaded_entries(socket, :csv, fn %{path: path}, _entry -> - {:ok, File.read!(path)} - end) + true -> + user = current_user(socket.assigns) + email = user.email + private = Map.get(params, "private") == "true" - case uploaded_contents do - [content] -> - handle_csv_preview(content, email, private, socket) + uploaded_contents = + consume_uploaded_entries(socket, :csv, fn %{path: path}, _entry -> + {:ok, File.read!(path)} + end) - [] -> - {:noreply, put_flash(socket, :error, "Please select a CSV file")} - end + case uploaded_contents do + [content] -> + handle_csv_preview(content, email, private, socket) + + [] -> + {:noreply, put_flash(socket, :error, "Please select a CSV file")} + end end end @@ -154,29 +157,30 @@ defmodule MicrowavepropWeb.SubmitLive do end def handle_event("upload_adif", params, socket) do - email = - case current_user(socket.assigns) do - %{email: user_email} -> user_email - _ -> params |> Map.get("submitter_email", "") |> String.trim() - end + cond do + recently_submitted?(socket) -> + {:noreply, put_flash(socket, :error, "Please wait before submitting another contact.")} - private = Map.get(params, "private") == "true" + !current_user(socket.assigns) -> + {:noreply, put_flash(socket, :error, "Please sign in to upload files.")} - if email == "" do - {:noreply, put_flash(socket, :error, "Email is required for ADIF upload")} - else - uploaded_contents = - consume_uploaded_entries(socket, :adif, fn %{path: path}, _entry -> - {:ok, File.read!(path)} - end) + true -> + user = current_user(socket.assigns) + email = user.email + private = Map.get(params, "private") == "true" - case uploaded_contents do - [content] -> - handle_adif_preview(content, email, private, socket) + uploaded_contents = + consume_uploaded_entries(socket, :adif, fn %{path: path}, _entry -> + {:ok, File.read!(path)} + end) - [] -> - {:noreply, put_flash(socket, :error, "Please select an ADIF file")} - end + case uploaded_contents do + [content] -> + handle_adif_preview(content, email, private, socket) + + [] -> + {:noreply, put_flash(socket, :error, "Please select an ADIF file")} + end end end @@ -185,19 +189,28 @@ defmodule MicrowavepropWeb.SubmitLive do end def handle_event("confirm_csv", _params, socket) do - case socket.assigns.csv_preview do - %{valid: valid_rows, refinements: refinements} = preview - when valid_rows != [] or refinements != [] -> - private = socket.assigns[:csv_private] || false - {:ok, run_id} = CsvImport.enqueue(preview, socket.assigns.csv_email, private: private) + cond do + recently_submitted?(socket) -> + {:noreply, put_flash(socket, :error, "Please wait before submitting another contact.")} - {:noreply, - socket - |> assign(csv_preview: nil, csv_private: false) - |> push_navigate(to: ~p"/imports/#{run_id}")} + !current_user(socket.assigns) -> + {:noreply, put_flash(socket, :error, "Please sign in to upload files.")} - _ -> - {:noreply, put_flash(socket, :error, "Nothing to import.")} + true -> + case socket.assigns.csv_preview do + %{valid: valid_rows, refinements: refinements} = preview + when valid_rows != [] or refinements != [] -> + private = socket.assigns[:csv_private] || false + {:ok, run_id} = CsvImport.enqueue(preview, socket.assigns.csv_email, private: private) + + {:noreply, + socket + |> assign(csv_preview: nil, csv_private: false) + |> push_navigate(to: ~p"/imports/#{run_id}")} + + _ -> + {:noreply, put_flash(socket, :error, "Nothing to import.")} + end end end @@ -208,23 +221,37 @@ defmodule MicrowavepropWeb.SubmitLive do defp handle_adif_preview(content, email, private, socket) do case AdifImport.preview(content, email) do {:ok, preview} -> - {:noreply, assign(socket, csv_preview: preview, csv_email: email, csv_private: private)} + if preview.total_rows > @max_import_rows do + {:noreply, put_flash(socket, :error, "ADIF file has too many rows (max #{@max_import_rows}).")} + else + {:noreply, assign(socket, csv_preview: preview, csv_email: email, csv_private: private)} + end {:error, :no_records} -> {:noreply, put_flash(socket, :error, "ADIF file contains no records")} + + {:error, :too_many_rows} -> + {:noreply, put_flash(socket, :error, "ADIF file has too many rows (max #{@max_import_rows}).")} end end defp handle_csv_preview(content, email, private, socket) do case CsvImport.preview(content, email) do {:ok, preview} -> - {:noreply, assign(socket, csv_preview: preview, csv_email: email, csv_private: private)} + if preview.total_rows > @max_import_rows do + {:noreply, put_flash(socket, :error, "CSV file has too many rows (max #{@max_import_rows}).")} + else + {:noreply, assign(socket, csv_preview: preview, csv_email: email, csv_private: private)} + end {:error, :empty_csv} -> {:noreply, put_flash(socket, :error, "CSV file is empty")} {:error, :no_data_rows} -> {:noreply, put_flash(socket, :error, "CSV file has no data rows")} + + {:error, :too_many_rows} -> + {:noreply, put_flash(socket, :error, "CSV file has too many rows (max #{@max_import_rows}).")} end end @@ -289,22 +316,24 @@ defmodule MicrowavepropWeb.SubmitLive do > Single Contact - - + <%= if current_user(assigns) do %> + + + <% end %>
<%= case @active_tab do %> @@ -524,19 +553,6 @@ defmodule MicrowavepropWeb.SubmitLive do <%= if @current_user do %> - <% else %> -
- -
<% end %>