graham/prop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rewrite analysis notebook to attach to running node

Browse source 
Uses Repo, Scorer, BandConfig, Weather directly instead of raw SQL.
Scores all historical contacts with the real composite_score/2 and
includes factor heatmaps and weight sensitivity analysis.
This commit is contained in:
Graham McIntire 2026-04-07 08:47:28 -05:00
parent 7f0e1f4182
commit 6252aff728
1 changed files with 331 additions and 263 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

594
notebooks/algorithm_analysis.livemd
View file

@ -1,106 +1,86 @@
# Propagation Algorithm Analysis # Propagation Algorithm Analysis
```elixir ## Setup — Attach to Running Node
Mix.install([
{:postgrex, "~> 0.19"}, Start your dev server with a node name:
{:ecto_sql, "~> 3.12"},
{:kino, "~> 0.14"}, ```
{:kino_vega_lite, "~> 0.1"}, iex --sname prop -S mix phx.server
{:explorer, "~> 0.10"},
{:statistex, "~> 1.0"}
])
``` ```
## Database Connection Then in Livebook, use **Runtime > Configure > Attached node** and connect to `prop@<hostname>`.
```elixir ```elixir
{:ok, conn} = Postgrex.start_link( # Verify we have access to the app
hostname: "localhost", alias Microwaveprop.Repo
database: "microwaveprop_dev", alias Microwaveprop.Radio
username: "postgres", alias Microwaveprop.Radio.Contact
password: "postgres", alias Microwaveprop.Weather
port: 5432 alias Microwaveprop.Weather.HrrrProfile
) alias Microwaveprop.Terrain
``` alias Microwaveprop.Terrain.TerrainAnalysis
alias Microwaveprop.Propagation.Scorer
alias Microwaveprop.Propagation.BandConfig
## Helper Functions import Ecto.Query
```elixir Repo.aggregate(Contact, :count) |> then(&"Connected — #{&1} contacts in DB")
defmodule Q do
@doc "Run a query and return results as a list of maps"
def query!(conn, sql, params \\ []) do
%{columns: cols, rows: rows} = Postgrex.query!(conn, sql, params)
Enum.map(rows, fn row -> Enum.zip(cols, row) |> Map.new() end)
end
@doc "Run a query and return as an Explorer DataFrame"
def df!(conn, sql, params \\ []) do
%{columns: cols, rows: rows} = Postgrex.query!(conn, sql, params)
cols
|> Enum.with_index()
|> Enum.map(fn {col, i} -> {col, Enum.map(rows, &Enum.at(&1, i))} end)
|> Map.new()
|> Explorer.DataFrame.new()
end
end
``` ```
## 1. Contact Overview ## 1. Contact Overview
```elixir ```elixir
Q.query!(conn, """ band_summary =
SELECT COUNT(*) as total, Contact
COUNT(*) FILTER (WHERE distance_km < 3000) as tropo, |> where([c], c.distance_km < 3000)
COUNT(*) FILTER (WHERE distance_km >= 3000) as eme, |> group_by([c], c.band)
COUNT(DISTINCT band) as bands, |> select([c], %{
MIN(qso_timestamp) as earliest, band_mhz: c.band,
MAX(qso_timestamp) as latest count: count(),
FROM contacts avg_dist_km: type(avg(c.distance_km), :float),
""") max_dist_km: type(max(c.distance_km), :float)
|> hd() })
|> Kino.Tree.new() |> order_by([c], c.band)
|> Repo.all()
Kino.DataTable.new(band_summary, name: "Contacts by Band")
``` ```
```elixir ```elixir
# Contacts by band # Contacts by month
Q.df!(conn, """ monthly =
SELECT band::integer as band_mhz, COUNT(*) as count, Contact
ROUND(AVG(distance_km::numeric), 1) as avg_dist_km, |> where([c], c.distance_km < 3000)
ROUND(MAX(distance_km::numeric), 1) as max_dist_km |> group_by([c], fragment("EXTRACT(MONTH FROM ?)", c.qso_timestamp))
FROM contacts |> select([c], %{
WHERE distance_km < 3000 month: fragment("EXTRACT(MONTH FROM ?)::integer", c.qso_timestamp),
GROUP BY band ORDER BY band count: count()
""") })
|> Kino.DataTable.new() |> order_by([c], fragment("1"))
``` |> Repo.all()
```elixir
# Contacts by month — when do contacts happen?
contacts_by_month = Q.df!(conn, """
SELECT EXTRACT(MONTH FROM qso_timestamp)::integer as month,
COUNT(*) as count
FROM contacts WHERE distance_km < 3000
GROUP BY 1 ORDER BY 1
""")
VegaLite.new(width: 600, height: 300, title: "Tropo Contacts by Month") VegaLite.new(width: 600, height: 300, title: "Tropo Contacts by Month")
|> VegaLite.data_from_values(contacts_by_month |> Explorer.DataFrame.to_rows()) |> VegaLite.data_from_values(monthly)
|> VegaLite.mark(:bar) |> VegaLite.mark(:bar)
|> VegaLite.encode_field(:x, "month", type: :ordinal) |> VegaLite.encode_field(:x, "month", type: :ordinal)
|> VegaLite.encode_field(:y, "count", type: :quantitative) |> VegaLite.encode_field(:y, "count", type: :quantitative)
``` ```
```elixir ```elixir
# Contacts by hour of day # Contacts by hour
contacts_by_hour = Q.df!(conn, """ hourly =
SELECT EXTRACT(HOUR FROM qso_timestamp)::integer as utc_hour, Contact
COUNT(*) as count |> where([c], c.distance_km < 3000)
FROM contacts WHERE distance_km < 3000 |> group_by([c], fragment("EXTRACT(HOUR FROM ?)", c.qso_timestamp))
GROUP BY 1 ORDER BY 1 |> select([c], %{
""") utc_hour: fragment("EXTRACT(HOUR FROM ?)::integer", c.qso_timestamp),
count: count()
})
|> order_by([c], fragment("1"))
|> Repo.all()
VegaLite.new(width: 600, height: 300, title: "Tropo Contacts by UTC Hour") VegaLite.new(width: 600, height: 300, title: "Tropo Contacts by UTC Hour")
|> VegaLite.data_from_values(contacts_by_hour |> Explorer.DataFrame.to_rows()) |> VegaLite.data_from_values(hourly)
|> VegaLite.mark(:bar) |> VegaLite.mark(:bar)
|> VegaLite.encode_field(:x, "utc_hour", type: :ordinal) |> VegaLite.encode_field(:x, "utc_hour", type: :ordinal)
|> VegaLite.encode_field(:y, "count", type: :quantitative) |> VegaLite.encode_field(:y, "count", type: :quantitative)
@ -108,56 +88,64 @@ VegaLite.new(width: 600, height: 300, title: "Tropo Contacts by UTC Hour")
## 2. HRRR Conditions at Contact Time ## 2. HRRR Conditions at Contact Time
Join contacts with their nearest HRRR profile to see what atmospheric
conditions were present when contacts actually happened.
```elixir ```elixir
# Contacts with HRRR data — the core analysis dataset # Load tropo contacts with HRRR data via the app's existing lookup
contact_hrrr = Q.df!(conn, """ contacts =
SELECT Contact
c.id, |> where([c], c.distance_km < 3000 and not is_nil(c.pos1))
c.band::integer as band_mhz, |> order_by([c], desc: c.qso_timestamp)
c.distance_km::float as distance_km, |> Repo.all()
c.qso_timestamp,
EXTRACT(MONTH FROM c.qso_timestamp)::integer as month,
EXTRACT(HOUR FROM c.qso_timestamp)::integer as utc_hour,
h.surface_temp as temp_c,
h.surface_dewpoint as dewpoint_c,
(h.surface_temp - h.surface_dewpoint) as td_depression_c,
h.surface_pressure as pressure_mb,
h.surface_refractivity,
h.min_refractivity_gradient,
h.hpbl_m as bl_depth_m,
h.pwat_mm,
h.ducting_detected
FROM contacts c
JOIN LATERAL (
SELECT *
FROM hrrr_profiles hp
WHERE hp.lat BETWEEN ROUND((c.pos1->>'lat')::numeric, 2) - 0.07
AND ROUND((c.pos1->>'lat')::numeric, 2) + 0.07
AND hp.lon BETWEEN ROUND(COALESCE((c.pos1->>'lon')::numeric, (c.pos1->>'lng')::numeric), 2) - 0.07
AND ROUND(COALESCE((c.pos1->>'lon')::numeric, (c.pos1->>'lng')::numeric), 2) + 0.07
AND hp.valid_time BETWEEN c.qso_timestamp - interval '1 hour'
AND c.qso_timestamp + interval '1 hour'
ORDER BY ABS(EXTRACT(EPOCH FROM hp.valid_time - c.qso_timestamp))
LIMIT 1
) h ON true
WHERE c.distance_km < 3000
AND c.pos1 IS NOT NULL
""")
Kino.DataTable.new(contact_hrrr, name: "Contacts + HRRR") contact_hrrr =
contacts
|> Task.async_stream(
fn contact ->
hrrr = Weather.hrrr_for_contact(contact)
if hrrr, do: {contact, hrrr}, else: nil
end,
max_concurrency: 8,
timeout: 10_000
)
|> Enum.flat_map(fn
{:ok, nil} -> []
{:ok, pair} -> [pair]
_ -> []
end)
"#{length(contact_hrrr)} contacts matched with HRRR data (of #{length(contacts)} tropo contacts)"
``` ```
## 3. Factor Distributions — What Does the Atmosphere Look Like During Contacts? ```elixir
# Build analysis rows
analysis_rows =
Enum.map(contact_hrrr, fn {contact, hrrr} ->
%{
id: contact.id,
band_mhz: Decimal.to_integer(contact.band),
distance_km: Decimal.to_float(contact.distance_km),
month: contact.qso_timestamp.month,
utc_hour: contact.qso_timestamp.hour,
temp_c: hrrr.surface_temp,
dewpoint_c: hrrr.surface_dewpoint,
td_depression_c: if(hrrr.surface_temp && hrrr.surface_dewpoint,
do: hrrr.surface_temp - hrrr.surface_dewpoint, else: nil),
pressure_mb: hrrr.surface_pressure,
surface_refractivity: hrrr.surface_refractivity,
min_refractivity_gradient: hrrr.min_refractivity_gradient,
bl_depth_m: hrrr.hpbl_m,
pwat_mm: hrrr.pwat_mm,
ducting_detected: hrrr.ducting_detected
}
end)
Kino.DataTable.new(analysis_rows, name: "Contacts + HRRR")
```
## 3. Factor Distributions
```elixir ```elixir
rows = Explorer.DataFrame.to_rows(contact_hrrr)
# Refractivity gradient distribution
VegaLite.new(width: 600, height: 300, title: "Refractivity Gradient at Contact Time") VegaLite.new(width: 600, height: 300, title: "Refractivity Gradient at Contact Time")
|> VegaLite.data_from_values(rows) |> VegaLite.data_from_values(analysis_rows)
|> VegaLite.mark(:bar) |> VegaLite.mark(:bar)
|> VegaLite.encode_field(:x, "min_refractivity_gradient", |> VegaLite.encode_field(:x, "min_refractivity_gradient",
type: :quantitative, bin: %{step: 20}, title: "Min Refractivity Gradient (N-units/km)" type: :quantitative, bin: %{step: 20}, title: "Min Refractivity Gradient (N-units/km)"
@ -166,9 +154,8 @@ VegaLite.new(width: 600, height: 300, title: "Refractivity Gradient at Contact T
``` ```
```elixir ```elixir
# Td depression distribution
VegaLite.new(width: 600, height: 300, title: "Td Depression at Contact Time") VegaLite.new(width: 600, height: 300, title: "Td Depression at Contact Time")
|> VegaLite.data_from_values(rows) |> VegaLite.data_from_values(analysis_rows)
|> VegaLite.mark(:bar) |> VegaLite.mark(:bar)
|> VegaLite.encode_field(:x, "td_depression_c", |> VegaLite.encode_field(:x, "td_depression_c",
type: :quantitative, bin: %{step: 1}, title: "T - Td (C)" type: :quantitative, bin: %{step: 1}, title: "T - Td (C)"
@ -177,9 +164,8 @@ VegaLite.new(width: 600, height: 300, title: "Td Depression at Contact Time")
``` ```
```elixir ```elixir
# Boundary layer depth
VegaLite.new(width: 600, height: 300, title: "Boundary Layer Depth at Contact Time") VegaLite.new(width: 600, height: 300, title: "Boundary Layer Depth at Contact Time")
|> VegaLite.data_from_values(rows) |> VegaLite.data_from_values(analysis_rows)
|> VegaLite.mark(:bar) |> VegaLite.mark(:bar)
|> VegaLite.encode_field(:x, "bl_depth_m", |> VegaLite.encode_field(:x, "bl_depth_m",
type: :quantitative, bin: %{step: 100}, title: "BL Depth (m)" type: :quantitative, bin: %{step: 100}, title: "BL Depth (m)"
@ -188,9 +174,8 @@ VegaLite.new(width: 600, height: 300, title: "Boundary Layer Depth at Contact Ti
``` ```
```elixir ```elixir
# PWAT distribution
VegaLite.new(width: 600, height: 300, title: "Precipitable Water at Contact Time") VegaLite.new(width: 600, height: 300, title: "Precipitable Water at Contact Time")
|> VegaLite.data_from_values(rows) |> VegaLite.data_from_values(analysis_rows)
|> VegaLite.mark(:bar) |> VegaLite.mark(:bar)
|> VegaLite.encode_field(:x, "pwat_mm", |> VegaLite.encode_field(:x, "pwat_mm",
type: :quantitative, bin: %{step: 3}, title: "PWAT (mm)" type: :quantitative, bin: %{step: 3}, title: "PWAT (mm)"
@ -201,32 +186,18 @@ VegaLite.new(width: 600, height: 300, title: "Precipitable Water at Contact Time
## 4. Ducting Analysis ## 4. Ducting Analysis
```elixir ```elixir
# Ducting rate by month ducting_monthly =
ducting_monthly = Q.df!(conn, """ analysis_rows
SELECT |> Enum.group_by(& &1.month)
EXTRACT(MONTH FROM c.qso_timestamp)::integer as month, |> Enum.map(fn {month, rows} ->
COUNT(*) as total, total = length(rows)
COUNT(*) FILTER (WHERE h.ducting_detected = true) as ducting, ducting = Enum.count(rows, & &1.ducting_detected)
ROUND(100.0 * COUNT(*) FILTER (WHERE h.ducting_detected = true) / COUNT(*), 1) as ducting_pct %{month: month, total: total, ducting: ducting, ducting_pct: Float.round(100.0 * ducting / max(total, 1), 1)}
FROM contacts c end)
JOIN LATERAL ( |> Enum.sort_by(& &1.month)
SELECT ducting_detected
FROM hrrr_profiles hp
WHERE hp.lat BETWEEN ROUND((c.pos1->>'lat')::numeric, 2) - 0.07
AND ROUND((c.pos1->>'lat')::numeric, 2) + 0.07
AND hp.lon BETWEEN ROUND(COALESCE((c.pos1->>'lon')::numeric, (c.pos1->>'lng')::numeric), 2) - 0.07
AND ROUND(COALESCE((c.pos1->>'lon')::numeric, (c.pos1->>'lng')::numeric), 2) + 0.07
AND hp.valid_time BETWEEN c.qso_timestamp - interval '1 hour'
AND c.qso_timestamp + interval '1 hour'
ORDER BY ABS(EXTRACT(EPOCH FROM hp.valid_time - c.qso_timestamp))
LIMIT 1
) h ON true
WHERE c.distance_km < 3000 AND c.pos1 IS NOT NULL
GROUP BY 1 ORDER BY 1
""")
VegaLite.new(width: 600, height: 300, title: "Ducting Detection Rate by Month (at contact time)") VegaLite.new(width: 600, height: 300, title: "Ducting Detection Rate by Month (at contact time)")
|> VegaLite.data_from_values(ducting_monthly |> Explorer.DataFrame.to_rows()) |> VegaLite.data_from_values(ducting_monthly)
|> VegaLite.mark(:bar) |> VegaLite.mark(:bar)
|> VegaLite.encode_field(:x, "month", type: :ordinal) |> VegaLite.encode_field(:x, "month", type: :ordinal)
|> VegaLite.encode_field(:y, "ducting_pct", type: :quantitative, title: "% Ducting") |> VegaLite.encode_field(:y, "ducting_pct", type: :quantitative, title: "% Ducting")
@ -234,71 +205,64 @@ VegaLite.new(width: 600, height: 300, title: "Ducting Detection Rate by Month (a
## 5. Distance vs Atmospheric Conditions ## 5. Distance vs Atmospheric Conditions
Do contacts at longer distances correlate with better atmospheric conditions?
```elixir ```elixir
# Distance vs refractivity gradient (scatter)
VegaLite.new(width: 600, height: 400, title: "Distance vs Refractivity Gradient") VegaLite.new(width: 600, height: 400, title: "Distance vs Refractivity Gradient")
|> VegaLite.data_from_values(rows) |> VegaLite.data_from_values(analysis_rows)
|> VegaLite.mark(:circle, opacity: 0.3, size: 15) |> VegaLite.mark(:circle, opacity: 0.3, size: 15)
|> VegaLite.encode_field(:x, "min_refractivity_gradient", |> VegaLite.encode_field(:x, "min_refractivity_gradient",
type: :quantitative, title: "Min Refractivity Gradient" type: :quantitative, title: "Min Refractivity Gradient"
) )
|> VegaLite.encode_field(:y, "distance_km", |> VegaLite.encode_field(:y, "distance_km", type: :quantitative, title: "Distance (km)")
type: :quantitative, title: "Distance (km)"
)
|> VegaLite.encode_field(:color, "band_mhz", type: :nominal, title: "Band (MHz)") |> VegaLite.encode_field(:color, "band_mhz", type: :nominal, title: "Band (MHz)")
``` ```
```elixir ```elixir
# Distance vs Td depression
VegaLite.new(width: 600, height: 400, title: "Distance vs Td Depression") VegaLite.new(width: 600, height: 400, title: "Distance vs Td Depression")
|> VegaLite.data_from_values(rows) |> VegaLite.data_from_values(analysis_rows)
|> VegaLite.mark(:circle, opacity: 0.3, size: 15) |> VegaLite.mark(:circle, opacity: 0.3, size: 15)
|> VegaLite.encode_field(:x, "td_depression_c", |> VegaLite.encode_field(:x, "td_depression_c", type: :quantitative, title: "T - Td (C)")
type: :quantitative, title: "T - Td (C)" |> VegaLite.encode_field(:y, "distance_km", type: :quantitative, title: "Distance (km)")
)
|> VegaLite.encode_field(:y, "distance_km",
type: :quantitative, title: "Distance (km)"
)
|> VegaLite.encode_field(:color, "band_mhz", type: :nominal, title: "Band (MHz)") |> VegaLite.encode_field(:color, "band_mhz", type: :nominal, title: "Band (MHz)")
``` ```
## 6. Terrain Impact ## 6. Terrain Impact
```elixir ```elixir
terrain_stats = Q.df!(conn, """ terrain_stats =
SELECT Repo.all(
tp.verdict, from tp in "terrain_profiles",
COUNT(*) as count, join: c in Contact, on: c.id == tp.contact_id,
ROUND(AVG(c.distance_km::numeric), 1) as avg_dist_km, where: c.distance_km < 3000,
ROUND(AVG(tp.diffraction_db::numeric), 1) as avg_diffraction_db, group_by: tp.verdict,
ROUND(AVG(tp.min_clearance_m::numeric), 1) as avg_clearance_m select: %{
FROM terrain_profiles tp verdict: tp.verdict,
JOIN contacts c ON c.id = tp.contact_id count: count(),
WHERE c.distance_km < 3000 avg_dist_km: type(avg(c.distance_km), :float),
GROUP BY tp.verdict avg_diffraction_db: type(avg(tp.diffraction_db), :float),
ORDER BY count DESC avg_clearance_m: type(avg(tp.min_clearance_m), :float)
""") },
order_by: [desc: count()]
)
Kino.DataTable.new(terrain_stats, name: "Terrain Verdicts") Kino.DataTable.new(terrain_stats, name: "Terrain Verdicts")
``` ```
```elixir ```elixir
# Diffraction loss vs distance terrain_scatter =
terrain_scatter = Q.df!(conn, """ Repo.all(
SELECT from tp in "terrain_profiles",
c.band::integer as band_mhz, join: c in Contact, on: c.id == tp.contact_id,
c.distance_km::float as distance_km, where: c.distance_km < 3000,
tp.diffraction_db::float as diffraction_db, select: %{
tp.verdict band_mhz: type(c.band, :integer),
FROM terrain_profiles tp distance_km: type(c.distance_km, :float),
JOIN contacts c ON c.id = tp.contact_id diffraction_db: type(tp.diffraction_db, :float),
WHERE c.distance_km < 3000 verdict: tp.verdict
""") }
)
VegaLite.new(width: 600, height: 400, title: "Diffraction Loss vs Distance") VegaLite.new(width: 600, height: 400, title: "Diffraction Loss vs Distance")
|> VegaLite.data_from_values(terrain_scatter |> Explorer.DataFrame.to_rows()) |> VegaLite.data_from_values(terrain_scatter)
|> VegaLite.mark(:circle, opacity: 0.2, size: 10) |> VegaLite.mark(:circle, opacity: 0.2, size: 10)
|> VegaLite.encode_field(:x, "distance_km", type: :quantitative, title: "Distance (km)") |> VegaLite.encode_field(:x, "distance_km", type: :quantitative, title: "Distance (km)")
|> VegaLite.encode_field(:y, "diffraction_db", type: :quantitative, title: "Diffraction Loss (dB)") |> VegaLite.encode_field(:y, "diffraction_db", type: :quantitative, title: "Diffraction Loss (dB)")
@ -307,29 +271,24 @@ VegaLite.new(width: 600, height: 400, title: "Diffraction Loss vs Distance")
## 7. Commercial Link Correlation ## 7. Commercial Link Correlation
Ground truth: do commercial link signal levels correlate with atmospheric conditions?
```elixir ```elixir
commercial = Q.df!(conn, """ commercial =
SELECT Repo.all(
cl.name, cl.frequency_ghz, from cs in "commercial_samples",
cs.sampled_at, join: cl in "commercial_links", on: cl.id == cs.link_id,
EXTRACT(HOUR FROM cs.sampled_at)::integer as utc_hour, where: not is_nil(cs.rx_power_0),
EXTRACT(MONTH FROM cs.sampled_at)::integer as month, select: %{
cs.rx_power_0::float as rx_power_dbm name: cl.name,
FROM commercial_samples cs frequency_ghz: cl.frequency_ghz,
JOIN commercial_links cl ON cl.id = cs.link_id sampled_at: cs.sampled_at,
WHERE cs.rx_power_0 IS NOT NULL utc_hour: fragment("EXTRACT(HOUR FROM ?)::integer", cs.sampled_at),
ORDER BY cs.sampled_at rx_power_dbm: type(cs.rx_power_0, :float)
""") },
order_by: cs.sampled_at
)
Kino.DataTable.new(commercial, name: "Commercial Samples")
```
```elixir
# Signal strength by hour of day per link
VegaLite.new(width: 600, height: 300, title: "Commercial Link Signal by Hour of Day") VegaLite.new(width: 600, height: 300, title: "Commercial Link Signal by Hour of Day")
|> VegaLite.data_from_values(commercial |> Explorer.DataFrame.to_rows()) |> VegaLite.data_from_values(commercial)
|> VegaLite.mark(:line, point: true) |> VegaLite.mark(:line, point: true)
|> VegaLite.encode_field(:x, "utc_hour", type: :ordinal) |> VegaLite.encode_field(:x, "utc_hour", type: :ordinal)
|> VegaLite.encode_field(:y, "rx_power_dbm", |> VegaLite.encode_field(:y, "rx_power_dbm",
@ -338,73 +297,182 @@ VegaLite.new(width: 600, height: 300, title: "Commercial Link Signal by Hour of
|> VegaLite.encode_field(:color, "name", type: :nominal) |> VegaLite.encode_field(:color, "name", type: :nominal)
``` ```
## 8. Scoring the Historical Contacts ## 8. Score Historical Contacts with the Real Scorer
Recompute scores for all contacts using the current algorithm and compare
factor weights. This section is a template — once you import the full dump
and attach to the running node, you can call `Scorer.composite_score/2` directly.
```elixir ```elixir
# For now, compute a simplified composite score from HRRR data # Score every contact using the actual Scorer.composite_score/2
# TODO: attach to running node for full Scorer access scored =
contact_hrrr
|> Task.async_stream(
fn {contact, hrrr} ->
band_mhz = Decimal.to_integer(contact.band)
band_config = BandConfig.for_band(band_mhz)
# Humidity score approximation (beneficial band) temp_c = hrrr.surface_temp
humidity_score = fn temp_c, dewpoint_c -> dewpoint_c = hrrr.surface_dewpoint
if is_nil(temp_c) or is_nil(dewpoint_c), do: 50, temp_f = Scorer.c_to_f(temp_c)
else: min(100, max(0, round(217 * 6.112 * :math.exp(17.67 * dewpoint_c / (dewpoint_c + 243.5)) / (temp_c + 273.15) * 5))) dewpoint_f = Scorer.c_to_f(dewpoint_c)
end
# Td depression score (beneficial band) abs_hum =
td_score = fn temp_c, dewpoint_c -> if temp_c && dewpoint_c,
case temp_c - dewpoint_c do do: Scorer.absolute_humidity(temp_c, dewpoint_c),
d when d <= 1 -> 95 else: 10.0
d when d <= 2 -> 88
d when d <= 3 -> 80
d when d <= 5 -> 65
d when d <= 8 -> 50
d when d <= 12 -> 35
_ -> 20
end
end
# Refractivity score conditions = %{
refrac_score = fn grad -> abs_humidity: abs_hum,
cond do utc_hour: contact.qso_timestamp.hour,
is_nil(grad) -> 50 utc_minute: contact.qso_timestamp.minute,
grad < -200 -> 98 month: contact.qso_timestamp.month,
grad < -150 -> 92 longitude: (contact.pos1["lon"] || contact.pos1["lng"]) |> then(&(&1 || -97.0)),
grad < -100 -> 82 temp_f: temp_f,
grad < -75 -> 68 dewpoint_f: dewpoint_f,
grad < -55 -> 55 min_refractivity_gradient: hrrr.min_refractivity_gradient,
grad < -40 -> 48 bl_depth_m: hrrr.hpbl_m,
true -> 42 sky_cover_pct: nil,
end wind_speed_kts: nil,
end rain_rate_mmhr: 0.0,
pwat_mm: hrrr.pwat_mm,
pressure_mb: hrrr.surface_pressure,
prev_pressure_mb: nil
}
IO.puts("Scoring functions defined. Use with Explorer.DataFrame.mutate/2 or Enum.map/2") %{score: score, factors: factors} = Scorer.composite_score(conditions, band_config)
%{
id: contact.id,
band_mhz: band_mhz,
distance_km: Decimal.to_float(contact.distance_km),
month: contact.qso_timestamp.month,
utc_hour: contact.qso_timestamp.hour,
score: score,
humidity: factors.humidity,
time_of_day: factors.time_of_day,
td_depression: factors.td_depression,
refractivity: factors.refractivity,
season: factors.season,
wind: factors.wind,
rain: factors.rain,
pwat: factors.pwat,
pressure: factors.pressure
}
end,
max_concurrency: System.schedulers_online(),
timeout: 5_000
)
|> Enum.flat_map(fn
{:ok, row} -> [row]
_ -> []
end)
"Scored #{length(scored)} contacts"
```
```elixir
# Score distribution
VegaLite.new(width: 600, height: 300, title: "Composite Score Distribution (at contact time)")
|> VegaLite.data_from_values(scored)
|> VegaLite.mark(:bar)
|> VegaLite.encode_field(:x, "score", type: :quantitative, bin: %{step: 5}, title: "Score")
|> VegaLite.encode(:y, aggregate: :count, type: :quantitative)
```
```elixir
# Score vs distance — does a higher score predict longer contacts?
VegaLite.new(width: 600, height: 400, title: "Score vs Distance Achieved")
|> VegaLite.data_from_values(scored)
|> VegaLite.mark(:circle, opacity: 0.3, size: 15)
|> VegaLite.encode_field(:x, "score", type: :quantitative, title: "Composite Score")
|> VegaLite.encode_field(:y, "distance_km", type: :quantitative, title: "Distance (km)")
|> VegaLite.encode_field(:color, "band_mhz", type: :nominal, title: "Band (MHz)")
```
```elixir
# Score by band — box plot
VegaLite.new(width: 600, height: 300, title: "Score Distribution by Band")
|> VegaLite.data_from_values(scored)
|> VegaLite.mark(:boxplot)
|> VegaLite.encode_field(:x, "band_mhz", type: :nominal, title: "Band (MHz)")
|> VegaLite.encode_field(:y, "score", type: :quantitative, title: "Composite Score")
```
```elixir
# Factor contribution heatmap — which factors drive the score?
factor_avgs =
[:humidity, :time_of_day, :td_depression, :refractivity, :season, :wind, :rain, :pwat, :pressure]
|> Enum.flat_map(fn factor ->
scored
|> Enum.group_by(& &1.band_mhz)
|> Enum.map(fn {band, rows} ->
avg = Enum.map(rows, &Map.get(&1, factor)) |> Enum.sum() |> then(&(&1 / length(rows)))
%{factor: Atom.to_string(factor), band_mhz: band, avg_score: Float.round(avg, 1)}
end)
end)
VegaLite.new(width: 600, height: 300, title: "Average Factor Scores by Band")
|> VegaLite.data_from_values(factor_avgs)
|> VegaLite.mark(:rect)
|> VegaLite.encode_field(:x, "band_mhz", type: :nominal, title: "Band (MHz)")
|> VegaLite.encode_field(:y, "factor", type: :nominal, title: "Factor")
|> VegaLite.encode_field(:color, "avg_score", type: :quantitative,
scale: %{scheme: "redyellowgreen", domain: [0, 100]}, title: "Avg Score"
)
``` ```
## 9. Weight Sensitivity Analysis ## 9. Weight Sensitivity Analysis
Template for testing different weight configurations against historical data. Test different weight configurations against the scored dataset.
```elixir ```elixir
# Define weight sets to test weights = BandConfig.weights()
weight_sets = %{
current: %{humidity: 0.18, time_of_day: 0.10, td_depression: 0.10,
refractivity: 0.08, sky: 0.08, season: 0.08,
wind: 0.05, rain: 0.08, pressure: 0.15, pwat: 0.10},
refractivity_heavy: %{humidity: 0.15, time_of_day: 0.08, td_depression: 0.10, # Recompute composite with custom weights
refractivity: 0.18, sky: 0.06, season: 0.08, rescore = fn rows, custom_weights ->
wind: 0.04, rain: 0.06, pressure: 0.12, pwat: 0.13}, Enum.map(rows, fn row ->
score =
Enum.reduce(custom_weights, 0.0, fn {factor, weight}, acc ->
acc + Map.get(row, factor, 50) * weight
end)
|> round()
humidity_heavy: %{humidity: 0.25, time_of_day: 0.08, td_depression: 0.12, Map.put(row, :custom_score, score)
refractivity: 0.08, sky: 0.06, season: 0.08, end)
wind: 0.04, rain: 0.06, pressure: 0.12, pwat: 0.11} end
# Define alternative weight sets
alternatives = %{
current: weights,
refractivity_heavy: %{
humidity: 0.15, time_of_day: 0.08, td_depression: 0.10,
refractivity: 0.18, sky: 0.06, season: 0.08,
wind: 0.04, rain: 0.06, pressure: 0.12, pwat: 0.13
},
pressure_reduced: %{
humidity: 0.20, time_of_day: 0.12, td_depression: 0.12,
refractivity: 0.10, sky: 0.08, season: 0.08,
wind: 0.05, rain: 0.08, pressure: 0.07, pwat: 0.10
}
} }
# TODO: score all contacts with each weight set, # Compare: correlation of score with distance for each weight set
# then correlate scores with distance achieved and commercial link signal levels comparisons =
IO.puts("Weight sets defined: #{Map.keys(weight_sets) |> Enum.join(", ")}") Enum.map(alternatives, fn {name, w} ->
rescored = rescore.(scored, w)
scores = Enum.map(rescored, & &1.custom_score)
distances = Enum.map(rescored, & &1.distance_km)
n = length(scores)
mean_s = Enum.sum(scores) / n
mean_d = Enum.sum(distances) / n
cov = Enum.zip(scores, distances) |> Enum.map(fn {s, d} -> (s - mean_s) * (d - mean_d) end) |> Enum.sum()
var_s = Enum.map(scores, fn s -> (s - mean_s) ** 2 end) |> Enum.sum()
var_d = Enum.map(distances, fn d -> (d - mean_d) ** 2 end) |> Enum.sum()
r = if var_s > 0 and var_d > 0, do: cov / :math.sqrt(var_s * var_d), else: 0.0
%{weights: name, correlation_r: Float.round(r, 4), mean_score: Float.round(mean_s, 1)}
end)
Kino.DataTable.new(comparisons, name: "Weight Set Comparison (score-distance correlation)")
``` ```