Rewrite analysis notebook to attach to running node
Uses Repo, Scorer, BandConfig, Weather directly instead of raw SQL. Scores all historical contacts with the real composite_score/2 and includes factor heatmaps and weight sensitivity analysis.
This commit is contained in:
parent
7f0e1f4182
commit
6252aff728
1 changed files with 331 additions and 263 deletions
|
|
@ -1,106 +1,86 @@
|
|||
# Propagation Algorithm Analysis
|
||||
|
||||
```elixir
|
||||
Mix.install([
|
||||
{:postgrex, "~> 0.19"},
|
||||
{:ecto_sql, "~> 3.12"},
|
||||
{:kino, "~> 0.14"},
|
||||
{:kino_vega_lite, "~> 0.1"},
|
||||
{:explorer, "~> 0.10"},
|
||||
{:statistex, "~> 1.0"}
|
||||
])
|
||||
## Setup — Attach to Running Node
|
||||
|
||||
Start your dev server with a node name:
|
||||
|
||||
```
|
||||
iex --sname prop -S mix phx.server
|
||||
```
|
||||
|
||||
## Database Connection
|
||||
Then in Livebook, use **Runtime > Configure > Attached node** and connect to `prop@<hostname>`.
|
||||
|
||||
```elixir
|
||||
{:ok, conn} = Postgrex.start_link(
|
||||
hostname: "localhost",
|
||||
database: "microwaveprop_dev",
|
||||
username: "postgres",
|
||||
password: "postgres",
|
||||
port: 5432
|
||||
)
|
||||
```
|
||||
# Verify we have access to the app
|
||||
alias Microwaveprop.Repo
|
||||
alias Microwaveprop.Radio
|
||||
alias Microwaveprop.Radio.Contact
|
||||
alias Microwaveprop.Weather
|
||||
alias Microwaveprop.Weather.HrrrProfile
|
||||
alias Microwaveprop.Terrain
|
||||
alias Microwaveprop.Terrain.TerrainAnalysis
|
||||
alias Microwaveprop.Propagation.Scorer
|
||||
alias Microwaveprop.Propagation.BandConfig
|
||||
|
||||
## Helper Functions
|
||||
import Ecto.Query
|
||||
|
||||
```elixir
|
||||
defmodule Q do
|
||||
@doc "Run a query and return results as a list of maps"
|
||||
def query!(conn, sql, params \\ []) do
|
||||
%{columns: cols, rows: rows} = Postgrex.query!(conn, sql, params)
|
||||
Enum.map(rows, fn row -> Enum.zip(cols, row) |> Map.new() end)
|
||||
end
|
||||
|
||||
@doc "Run a query and return as an Explorer DataFrame"
|
||||
def df!(conn, sql, params \\ []) do
|
||||
%{columns: cols, rows: rows} = Postgrex.query!(conn, sql, params)
|
||||
cols
|
||||
|> Enum.with_index()
|
||||
|> Enum.map(fn {col, i} -> {col, Enum.map(rows, &Enum.at(&1, i))} end)
|
||||
|> Map.new()
|
||||
|> Explorer.DataFrame.new()
|
||||
end
|
||||
end
|
||||
Repo.aggregate(Contact, :count) |> then(&"Connected — #{&1} contacts in DB")
|
||||
```
|
||||
|
||||
## 1. Contact Overview
|
||||
|
||||
```elixir
|
||||
Q.query!(conn, """
|
||||
SELECT COUNT(*) as total,
|
||||
COUNT(*) FILTER (WHERE distance_km < 3000) as tropo,
|
||||
COUNT(*) FILTER (WHERE distance_km >= 3000) as eme,
|
||||
COUNT(DISTINCT band) as bands,
|
||||
MIN(qso_timestamp) as earliest,
|
||||
MAX(qso_timestamp) as latest
|
||||
FROM contacts
|
||||
""")
|
||||
|> hd()
|
||||
|> Kino.Tree.new()
|
||||
band_summary =
|
||||
Contact
|
||||
|> where([c], c.distance_km < 3000)
|
||||
|> group_by([c], c.band)
|
||||
|> select([c], %{
|
||||
band_mhz: c.band,
|
||||
count: count(),
|
||||
avg_dist_km: type(avg(c.distance_km), :float),
|
||||
max_dist_km: type(max(c.distance_km), :float)
|
||||
})
|
||||
|> order_by([c], c.band)
|
||||
|> Repo.all()
|
||||
|
||||
Kino.DataTable.new(band_summary, name: "Contacts by Band")
|
||||
```
|
||||
|
||||
```elixir
|
||||
# Contacts by band
|
||||
Q.df!(conn, """
|
||||
SELECT band::integer as band_mhz, COUNT(*) as count,
|
||||
ROUND(AVG(distance_km::numeric), 1) as avg_dist_km,
|
||||
ROUND(MAX(distance_km::numeric), 1) as max_dist_km
|
||||
FROM contacts
|
||||
WHERE distance_km < 3000
|
||||
GROUP BY band ORDER BY band
|
||||
""")
|
||||
|> Kino.DataTable.new()
|
||||
```
|
||||
|
||||
```elixir
|
||||
# Contacts by month — when do contacts happen?
|
||||
contacts_by_month = Q.df!(conn, """
|
||||
SELECT EXTRACT(MONTH FROM qso_timestamp)::integer as month,
|
||||
COUNT(*) as count
|
||||
FROM contacts WHERE distance_km < 3000
|
||||
GROUP BY 1 ORDER BY 1
|
||||
""")
|
||||
# Contacts by month
|
||||
monthly =
|
||||
Contact
|
||||
|> where([c], c.distance_km < 3000)
|
||||
|> group_by([c], fragment("EXTRACT(MONTH FROM ?)", c.qso_timestamp))
|
||||
|> select([c], %{
|
||||
month: fragment("EXTRACT(MONTH FROM ?)::integer", c.qso_timestamp),
|
||||
count: count()
|
||||
})
|
||||
|> order_by([c], fragment("1"))
|
||||
|> Repo.all()
|
||||
|
||||
VegaLite.new(width: 600, height: 300, title: "Tropo Contacts by Month")
|
||||
|> VegaLite.data_from_values(contacts_by_month |> Explorer.DataFrame.to_rows())
|
||||
|> VegaLite.data_from_values(monthly)
|
||||
|> VegaLite.mark(:bar)
|
||||
|> VegaLite.encode_field(:x, "month", type: :ordinal)
|
||||
|> VegaLite.encode_field(:y, "count", type: :quantitative)
|
||||
```
|
||||
|
||||
```elixir
|
||||
# Contacts by hour of day
|
||||
contacts_by_hour = Q.df!(conn, """
|
||||
SELECT EXTRACT(HOUR FROM qso_timestamp)::integer as utc_hour,
|
||||
COUNT(*) as count
|
||||
FROM contacts WHERE distance_km < 3000
|
||||
GROUP BY 1 ORDER BY 1
|
||||
""")
|
||||
# Contacts by hour
|
||||
hourly =
|
||||
Contact
|
||||
|> where([c], c.distance_km < 3000)
|
||||
|> group_by([c], fragment("EXTRACT(HOUR FROM ?)", c.qso_timestamp))
|
||||
|> select([c], %{
|
||||
utc_hour: fragment("EXTRACT(HOUR FROM ?)::integer", c.qso_timestamp),
|
||||
count: count()
|
||||
})
|
||||
|> order_by([c], fragment("1"))
|
||||
|> Repo.all()
|
||||
|
||||
VegaLite.new(width: 600, height: 300, title: "Tropo Contacts by UTC Hour")
|
||||
|> VegaLite.data_from_values(contacts_by_hour |> Explorer.DataFrame.to_rows())
|
||||
|> VegaLite.data_from_values(hourly)
|
||||
|> VegaLite.mark(:bar)
|
||||
|> VegaLite.encode_field(:x, "utc_hour", type: :ordinal)
|
||||
|> VegaLite.encode_field(:y, "count", type: :quantitative)
|
||||
|
|
@ -108,56 +88,64 @@ VegaLite.new(width: 600, height: 300, title: "Tropo Contacts by UTC Hour")
|
|||
|
||||
## 2. HRRR Conditions at Contact Time
|
||||
|
||||
Join contacts with their nearest HRRR profile to see what atmospheric
|
||||
conditions were present when contacts actually happened.
|
||||
|
||||
```elixir
|
||||
# Contacts with HRRR data — the core analysis dataset
|
||||
contact_hrrr = Q.df!(conn, """
|
||||
SELECT
|
||||
c.id,
|
||||
c.band::integer as band_mhz,
|
||||
c.distance_km::float as distance_km,
|
||||
c.qso_timestamp,
|
||||
EXTRACT(MONTH FROM c.qso_timestamp)::integer as month,
|
||||
EXTRACT(HOUR FROM c.qso_timestamp)::integer as utc_hour,
|
||||
h.surface_temp as temp_c,
|
||||
h.surface_dewpoint as dewpoint_c,
|
||||
(h.surface_temp - h.surface_dewpoint) as td_depression_c,
|
||||
h.surface_pressure as pressure_mb,
|
||||
h.surface_refractivity,
|
||||
h.min_refractivity_gradient,
|
||||
h.hpbl_m as bl_depth_m,
|
||||
h.pwat_mm,
|
||||
h.ducting_detected
|
||||
FROM contacts c
|
||||
JOIN LATERAL (
|
||||
SELECT *
|
||||
FROM hrrr_profiles hp
|
||||
WHERE hp.lat BETWEEN ROUND((c.pos1->>'lat')::numeric, 2) - 0.07
|
||||
AND ROUND((c.pos1->>'lat')::numeric, 2) + 0.07
|
||||
AND hp.lon BETWEEN ROUND(COALESCE((c.pos1->>'lon')::numeric, (c.pos1->>'lng')::numeric), 2) - 0.07
|
||||
AND ROUND(COALESCE((c.pos1->>'lon')::numeric, (c.pos1->>'lng')::numeric), 2) + 0.07
|
||||
AND hp.valid_time BETWEEN c.qso_timestamp - interval '1 hour'
|
||||
AND c.qso_timestamp + interval '1 hour'
|
||||
ORDER BY ABS(EXTRACT(EPOCH FROM hp.valid_time - c.qso_timestamp))
|
||||
LIMIT 1
|
||||
) h ON true
|
||||
WHERE c.distance_km < 3000
|
||||
AND c.pos1 IS NOT NULL
|
||||
""")
|
||||
# Load tropo contacts with HRRR data via the app's existing lookup
|
||||
contacts =
|
||||
Contact
|
||||
|> where([c], c.distance_km < 3000 and not is_nil(c.pos1))
|
||||
|> order_by([c], desc: c.qso_timestamp)
|
||||
|> Repo.all()
|
||||
|
||||
Kino.DataTable.new(contact_hrrr, name: "Contacts + HRRR")
|
||||
contact_hrrr =
|
||||
contacts
|
||||
|> Task.async_stream(
|
||||
fn contact ->
|
||||
hrrr = Weather.hrrr_for_contact(contact)
|
||||
if hrrr, do: {contact, hrrr}, else: nil
|
||||
end,
|
||||
max_concurrency: 8,
|
||||
timeout: 10_000
|
||||
)
|
||||
|> Enum.flat_map(fn
|
||||
{:ok, nil} -> []
|
||||
{:ok, pair} -> [pair]
|
||||
_ -> []
|
||||
end)
|
||||
|
||||
"#{length(contact_hrrr)} contacts matched with HRRR data (of #{length(contacts)} tropo contacts)"
|
||||
```
|
||||
|
||||
## 3. Factor Distributions — What Does the Atmosphere Look Like During Contacts?
|
||||
```elixir
|
||||
# Build analysis rows
|
||||
analysis_rows =
|
||||
Enum.map(contact_hrrr, fn {contact, hrrr} ->
|
||||
%{
|
||||
id: contact.id,
|
||||
band_mhz: Decimal.to_integer(contact.band),
|
||||
distance_km: Decimal.to_float(contact.distance_km),
|
||||
month: contact.qso_timestamp.month,
|
||||
utc_hour: contact.qso_timestamp.hour,
|
||||
temp_c: hrrr.surface_temp,
|
||||
dewpoint_c: hrrr.surface_dewpoint,
|
||||
td_depression_c: if(hrrr.surface_temp && hrrr.surface_dewpoint,
|
||||
do: hrrr.surface_temp - hrrr.surface_dewpoint, else: nil),
|
||||
pressure_mb: hrrr.surface_pressure,
|
||||
surface_refractivity: hrrr.surface_refractivity,
|
||||
min_refractivity_gradient: hrrr.min_refractivity_gradient,
|
||||
bl_depth_m: hrrr.hpbl_m,
|
||||
pwat_mm: hrrr.pwat_mm,
|
||||
ducting_detected: hrrr.ducting_detected
|
||||
}
|
||||
end)
|
||||
|
||||
Kino.DataTable.new(analysis_rows, name: "Contacts + HRRR")
|
||||
```
|
||||
|
||||
## 3. Factor Distributions
|
||||
|
||||
```elixir
|
||||
rows = Explorer.DataFrame.to_rows(contact_hrrr)
|
||||
|
||||
# Refractivity gradient distribution
|
||||
VegaLite.new(width: 600, height: 300, title: "Refractivity Gradient at Contact Time")
|
||||
|> VegaLite.data_from_values(rows)
|
||||
|> VegaLite.data_from_values(analysis_rows)
|
||||
|> VegaLite.mark(:bar)
|
||||
|> VegaLite.encode_field(:x, "min_refractivity_gradient",
|
||||
type: :quantitative, bin: %{step: 20}, title: "Min Refractivity Gradient (N-units/km)"
|
||||
|
|
@ -166,9 +154,8 @@ VegaLite.new(width: 600, height: 300, title: "Refractivity Gradient at Contact T
|
|||
```
|
||||
|
||||
```elixir
|
||||
# Td depression distribution
|
||||
VegaLite.new(width: 600, height: 300, title: "Td Depression at Contact Time")
|
||||
|> VegaLite.data_from_values(rows)
|
||||
|> VegaLite.data_from_values(analysis_rows)
|
||||
|> VegaLite.mark(:bar)
|
||||
|> VegaLite.encode_field(:x, "td_depression_c",
|
||||
type: :quantitative, bin: %{step: 1}, title: "T - Td (C)"
|
||||
|
|
@ -177,9 +164,8 @@ VegaLite.new(width: 600, height: 300, title: "Td Depression at Contact Time")
|
|||
```
|
||||
|
||||
```elixir
|
||||
# Boundary layer depth
|
||||
VegaLite.new(width: 600, height: 300, title: "Boundary Layer Depth at Contact Time")
|
||||
|> VegaLite.data_from_values(rows)
|
||||
|> VegaLite.data_from_values(analysis_rows)
|
||||
|> VegaLite.mark(:bar)
|
||||
|> VegaLite.encode_field(:x, "bl_depth_m",
|
||||
type: :quantitative, bin: %{step: 100}, title: "BL Depth (m)"
|
||||
|
|
@ -188,9 +174,8 @@ VegaLite.new(width: 600, height: 300, title: "Boundary Layer Depth at Contact Ti
|
|||
```
|
||||
|
||||
```elixir
|
||||
# PWAT distribution
|
||||
VegaLite.new(width: 600, height: 300, title: "Precipitable Water at Contact Time")
|
||||
|> VegaLite.data_from_values(rows)
|
||||
|> VegaLite.data_from_values(analysis_rows)
|
||||
|> VegaLite.mark(:bar)
|
||||
|> VegaLite.encode_field(:x, "pwat_mm",
|
||||
type: :quantitative, bin: %{step: 3}, title: "PWAT (mm)"
|
||||
|
|
@ -201,32 +186,18 @@ VegaLite.new(width: 600, height: 300, title: "Precipitable Water at Contact Time
|
|||
## 4. Ducting Analysis
|
||||
|
||||
```elixir
|
||||
# Ducting rate by month
|
||||
ducting_monthly = Q.df!(conn, """
|
||||
SELECT
|
||||
EXTRACT(MONTH FROM c.qso_timestamp)::integer as month,
|
||||
COUNT(*) as total,
|
||||
COUNT(*) FILTER (WHERE h.ducting_detected = true) as ducting,
|
||||
ROUND(100.0 * COUNT(*) FILTER (WHERE h.ducting_detected = true) / COUNT(*), 1) as ducting_pct
|
||||
FROM contacts c
|
||||
JOIN LATERAL (
|
||||
SELECT ducting_detected
|
||||
FROM hrrr_profiles hp
|
||||
WHERE hp.lat BETWEEN ROUND((c.pos1->>'lat')::numeric, 2) - 0.07
|
||||
AND ROUND((c.pos1->>'lat')::numeric, 2) + 0.07
|
||||
AND hp.lon BETWEEN ROUND(COALESCE((c.pos1->>'lon')::numeric, (c.pos1->>'lng')::numeric), 2) - 0.07
|
||||
AND ROUND(COALESCE((c.pos1->>'lon')::numeric, (c.pos1->>'lng')::numeric), 2) + 0.07
|
||||
AND hp.valid_time BETWEEN c.qso_timestamp - interval '1 hour'
|
||||
AND c.qso_timestamp + interval '1 hour'
|
||||
ORDER BY ABS(EXTRACT(EPOCH FROM hp.valid_time - c.qso_timestamp))
|
||||
LIMIT 1
|
||||
) h ON true
|
||||
WHERE c.distance_km < 3000 AND c.pos1 IS NOT NULL
|
||||
GROUP BY 1 ORDER BY 1
|
||||
""")
|
||||
ducting_monthly =
|
||||
analysis_rows
|
||||
|> Enum.group_by(& &1.month)
|
||||
|> Enum.map(fn {month, rows} ->
|
||||
total = length(rows)
|
||||
ducting = Enum.count(rows, & &1.ducting_detected)
|
||||
%{month: month, total: total, ducting: ducting, ducting_pct: Float.round(100.0 * ducting / max(total, 1), 1)}
|
||||
end)
|
||||
|> Enum.sort_by(& &1.month)
|
||||
|
||||
VegaLite.new(width: 600, height: 300, title: "Ducting Detection Rate by Month (at contact time)")
|
||||
|> VegaLite.data_from_values(ducting_monthly |> Explorer.DataFrame.to_rows())
|
||||
|> VegaLite.data_from_values(ducting_monthly)
|
||||
|> VegaLite.mark(:bar)
|
||||
|> VegaLite.encode_field(:x, "month", type: :ordinal)
|
||||
|> VegaLite.encode_field(:y, "ducting_pct", type: :quantitative, title: "% Ducting")
|
||||
|
|
@ -234,71 +205,64 @@ VegaLite.new(width: 600, height: 300, title: "Ducting Detection Rate by Month (a
|
|||
|
||||
## 5. Distance vs Atmospheric Conditions
|
||||
|
||||
Do contacts at longer distances correlate with better atmospheric conditions?
|
||||
|
||||
```elixir
|
||||
# Distance vs refractivity gradient (scatter)
|
||||
VegaLite.new(width: 600, height: 400, title: "Distance vs Refractivity Gradient")
|
||||
|> VegaLite.data_from_values(rows)
|
||||
|> VegaLite.data_from_values(analysis_rows)
|
||||
|> VegaLite.mark(:circle, opacity: 0.3, size: 15)
|
||||
|> VegaLite.encode_field(:x, "min_refractivity_gradient",
|
||||
type: :quantitative, title: "Min Refractivity Gradient"
|
||||
)
|
||||
|> VegaLite.encode_field(:y, "distance_km",
|
||||
type: :quantitative, title: "Distance (km)"
|
||||
)
|
||||
|> VegaLite.encode_field(:y, "distance_km", type: :quantitative, title: "Distance (km)")
|
||||
|> VegaLite.encode_field(:color, "band_mhz", type: :nominal, title: "Band (MHz)")
|
||||
```
|
||||
|
||||
```elixir
|
||||
# Distance vs Td depression
|
||||
VegaLite.new(width: 600, height: 400, title: "Distance vs Td Depression")
|
||||
|> VegaLite.data_from_values(rows)
|
||||
|> VegaLite.data_from_values(analysis_rows)
|
||||
|> VegaLite.mark(:circle, opacity: 0.3, size: 15)
|
||||
|> VegaLite.encode_field(:x, "td_depression_c",
|
||||
type: :quantitative, title: "T - Td (C)"
|
||||
)
|
||||
|> VegaLite.encode_field(:y, "distance_km",
|
||||
type: :quantitative, title: "Distance (km)"
|
||||
)
|
||||
|> VegaLite.encode_field(:x, "td_depression_c", type: :quantitative, title: "T - Td (C)")
|
||||
|> VegaLite.encode_field(:y, "distance_km", type: :quantitative, title: "Distance (km)")
|
||||
|> VegaLite.encode_field(:color, "band_mhz", type: :nominal, title: "Band (MHz)")
|
||||
```
|
||||
|
||||
## 6. Terrain Impact
|
||||
|
||||
```elixir
|
||||
terrain_stats = Q.df!(conn, """
|
||||
SELECT
|
||||
tp.verdict,
|
||||
COUNT(*) as count,
|
||||
ROUND(AVG(c.distance_km::numeric), 1) as avg_dist_km,
|
||||
ROUND(AVG(tp.diffraction_db::numeric), 1) as avg_diffraction_db,
|
||||
ROUND(AVG(tp.min_clearance_m::numeric), 1) as avg_clearance_m
|
||||
FROM terrain_profiles tp
|
||||
JOIN contacts c ON c.id = tp.contact_id
|
||||
WHERE c.distance_km < 3000
|
||||
GROUP BY tp.verdict
|
||||
ORDER BY count DESC
|
||||
""")
|
||||
terrain_stats =
|
||||
Repo.all(
|
||||
from tp in "terrain_profiles",
|
||||
join: c in Contact, on: c.id == tp.contact_id,
|
||||
where: c.distance_km < 3000,
|
||||
group_by: tp.verdict,
|
||||
select: %{
|
||||
verdict: tp.verdict,
|
||||
count: count(),
|
||||
avg_dist_km: type(avg(c.distance_km), :float),
|
||||
avg_diffraction_db: type(avg(tp.diffraction_db), :float),
|
||||
avg_clearance_m: type(avg(tp.min_clearance_m), :float)
|
||||
},
|
||||
order_by: [desc: count()]
|
||||
)
|
||||
|
||||
Kino.DataTable.new(terrain_stats, name: "Terrain Verdicts")
|
||||
```
|
||||
|
||||
```elixir
|
||||
# Diffraction loss vs distance
|
||||
terrain_scatter = Q.df!(conn, """
|
||||
SELECT
|
||||
c.band::integer as band_mhz,
|
||||
c.distance_km::float as distance_km,
|
||||
tp.diffraction_db::float as diffraction_db,
|
||||
tp.verdict
|
||||
FROM terrain_profiles tp
|
||||
JOIN contacts c ON c.id = tp.contact_id
|
||||
WHERE c.distance_km < 3000
|
||||
""")
|
||||
terrain_scatter =
|
||||
Repo.all(
|
||||
from tp in "terrain_profiles",
|
||||
join: c in Contact, on: c.id == tp.contact_id,
|
||||
where: c.distance_km < 3000,
|
||||
select: %{
|
||||
band_mhz: type(c.band, :integer),
|
||||
distance_km: type(c.distance_km, :float),
|
||||
diffraction_db: type(tp.diffraction_db, :float),
|
||||
verdict: tp.verdict
|
||||
}
|
||||
)
|
||||
|
||||
VegaLite.new(width: 600, height: 400, title: "Diffraction Loss vs Distance")
|
||||
|> VegaLite.data_from_values(terrain_scatter |> Explorer.DataFrame.to_rows())
|
||||
|> VegaLite.data_from_values(terrain_scatter)
|
||||
|> VegaLite.mark(:circle, opacity: 0.2, size: 10)
|
||||
|> VegaLite.encode_field(:x, "distance_km", type: :quantitative, title: "Distance (km)")
|
||||
|> VegaLite.encode_field(:y, "diffraction_db", type: :quantitative, title: "Diffraction Loss (dB)")
|
||||
|
|
@ -307,29 +271,24 @@ VegaLite.new(width: 600, height: 400, title: "Diffraction Loss vs Distance")
|
|||
|
||||
## 7. Commercial Link Correlation
|
||||
|
||||
Ground truth: do commercial link signal levels correlate with atmospheric conditions?
|
||||
|
||||
```elixir
|
||||
commercial = Q.df!(conn, """
|
||||
SELECT
|
||||
cl.name, cl.frequency_ghz,
|
||||
cs.sampled_at,
|
||||
EXTRACT(HOUR FROM cs.sampled_at)::integer as utc_hour,
|
||||
EXTRACT(MONTH FROM cs.sampled_at)::integer as month,
|
||||
cs.rx_power_0::float as rx_power_dbm
|
||||
FROM commercial_samples cs
|
||||
JOIN commercial_links cl ON cl.id = cs.link_id
|
||||
WHERE cs.rx_power_0 IS NOT NULL
|
||||
ORDER BY cs.sampled_at
|
||||
""")
|
||||
commercial =
|
||||
Repo.all(
|
||||
from cs in "commercial_samples",
|
||||
join: cl in "commercial_links", on: cl.id == cs.link_id,
|
||||
where: not is_nil(cs.rx_power_0),
|
||||
select: %{
|
||||
name: cl.name,
|
||||
frequency_ghz: cl.frequency_ghz,
|
||||
sampled_at: cs.sampled_at,
|
||||
utc_hour: fragment("EXTRACT(HOUR FROM ?)::integer", cs.sampled_at),
|
||||
rx_power_dbm: type(cs.rx_power_0, :float)
|
||||
},
|
||||
order_by: cs.sampled_at
|
||||
)
|
||||
|
||||
Kino.DataTable.new(commercial, name: "Commercial Samples")
|
||||
```
|
||||
|
||||
```elixir
|
||||
# Signal strength by hour of day per link
|
||||
VegaLite.new(width: 600, height: 300, title: "Commercial Link Signal by Hour of Day")
|
||||
|> VegaLite.data_from_values(commercial |> Explorer.DataFrame.to_rows())
|
||||
|> VegaLite.data_from_values(commercial)
|
||||
|> VegaLite.mark(:line, point: true)
|
||||
|> VegaLite.encode_field(:x, "utc_hour", type: :ordinal)
|
||||
|> VegaLite.encode_field(:y, "rx_power_dbm",
|
||||
|
|
@ -338,73 +297,182 @@ VegaLite.new(width: 600, height: 300, title: "Commercial Link Signal by Hour of
|
|||
|> VegaLite.encode_field(:color, "name", type: :nominal)
|
||||
```
|
||||
|
||||
## 8. Scoring the Historical Contacts
|
||||
|
||||
Recompute scores for all contacts using the current algorithm and compare
|
||||
factor weights. This section is a template — once you import the full dump
|
||||
and attach to the running node, you can call `Scorer.composite_score/2` directly.
|
||||
## 8. Score Historical Contacts with the Real Scorer
|
||||
|
||||
```elixir
|
||||
# For now, compute a simplified composite score from HRRR data
|
||||
# TODO: attach to running node for full Scorer access
|
||||
# Score every contact using the actual Scorer.composite_score/2
|
||||
scored =
|
||||
contact_hrrr
|
||||
|> Task.async_stream(
|
||||
fn {contact, hrrr} ->
|
||||
band_mhz = Decimal.to_integer(contact.band)
|
||||
band_config = BandConfig.for_band(band_mhz)
|
||||
|
||||
# Humidity score approximation (beneficial band)
|
||||
humidity_score = fn temp_c, dewpoint_c ->
|
||||
if is_nil(temp_c) or is_nil(dewpoint_c), do: 50,
|
||||
else: min(100, max(0, round(217 * 6.112 * :math.exp(17.67 * dewpoint_c / (dewpoint_c + 243.5)) / (temp_c + 273.15) * 5)))
|
||||
end
|
||||
temp_c = hrrr.surface_temp
|
||||
dewpoint_c = hrrr.surface_dewpoint
|
||||
temp_f = Scorer.c_to_f(temp_c)
|
||||
dewpoint_f = Scorer.c_to_f(dewpoint_c)
|
||||
|
||||
# Td depression score (beneficial band)
|
||||
td_score = fn temp_c, dewpoint_c ->
|
||||
case temp_c - dewpoint_c do
|
||||
d when d <= 1 -> 95
|
||||
d when d <= 2 -> 88
|
||||
d when d <= 3 -> 80
|
||||
d when d <= 5 -> 65
|
||||
d when d <= 8 -> 50
|
||||
d when d <= 12 -> 35
|
||||
_ -> 20
|
||||
end
|
||||
end
|
||||
abs_hum =
|
||||
if temp_c && dewpoint_c,
|
||||
do: Scorer.absolute_humidity(temp_c, dewpoint_c),
|
||||
else: 10.0
|
||||
|
||||
# Refractivity score
|
||||
refrac_score = fn grad ->
|
||||
cond do
|
||||
is_nil(grad) -> 50
|
||||
grad < -200 -> 98
|
||||
grad < -150 -> 92
|
||||
grad < -100 -> 82
|
||||
grad < -75 -> 68
|
||||
grad < -55 -> 55
|
||||
grad < -40 -> 48
|
||||
true -> 42
|
||||
end
|
||||
end
|
||||
conditions = %{
|
||||
abs_humidity: abs_hum,
|
||||
utc_hour: contact.qso_timestamp.hour,
|
||||
utc_minute: contact.qso_timestamp.minute,
|
||||
month: contact.qso_timestamp.month,
|
||||
longitude: (contact.pos1["lon"] || contact.pos1["lng"]) |> then(&(&1 || -97.0)),
|
||||
temp_f: temp_f,
|
||||
dewpoint_f: dewpoint_f,
|
||||
min_refractivity_gradient: hrrr.min_refractivity_gradient,
|
||||
bl_depth_m: hrrr.hpbl_m,
|
||||
sky_cover_pct: nil,
|
||||
wind_speed_kts: nil,
|
||||
rain_rate_mmhr: 0.0,
|
||||
pwat_mm: hrrr.pwat_mm,
|
||||
pressure_mb: hrrr.surface_pressure,
|
||||
prev_pressure_mb: nil
|
||||
}
|
||||
|
||||
IO.puts("Scoring functions defined. Use with Explorer.DataFrame.mutate/2 or Enum.map/2")
|
||||
%{score: score, factors: factors} = Scorer.composite_score(conditions, band_config)
|
||||
|
||||
%{
|
||||
id: contact.id,
|
||||
band_mhz: band_mhz,
|
||||
distance_km: Decimal.to_float(contact.distance_km),
|
||||
month: contact.qso_timestamp.month,
|
||||
utc_hour: contact.qso_timestamp.hour,
|
||||
score: score,
|
||||
humidity: factors.humidity,
|
||||
time_of_day: factors.time_of_day,
|
||||
td_depression: factors.td_depression,
|
||||
refractivity: factors.refractivity,
|
||||
season: factors.season,
|
||||
wind: factors.wind,
|
||||
rain: factors.rain,
|
||||
pwat: factors.pwat,
|
||||
pressure: factors.pressure
|
||||
}
|
||||
end,
|
||||
max_concurrency: System.schedulers_online(),
|
||||
timeout: 5_000
|
||||
)
|
||||
|> Enum.flat_map(fn
|
||||
{:ok, row} -> [row]
|
||||
_ -> []
|
||||
end)
|
||||
|
||||
"Scored #{length(scored)} contacts"
|
||||
```
|
||||
|
||||
```elixir
|
||||
# Score distribution
|
||||
VegaLite.new(width: 600, height: 300, title: "Composite Score Distribution (at contact time)")
|
||||
|> VegaLite.data_from_values(scored)
|
||||
|> VegaLite.mark(:bar)
|
||||
|> VegaLite.encode_field(:x, "score", type: :quantitative, bin: %{step: 5}, title: "Score")
|
||||
|> VegaLite.encode(:y, aggregate: :count, type: :quantitative)
|
||||
```
|
||||
|
||||
```elixir
|
||||
# Score vs distance — does a higher score predict longer contacts?
|
||||
VegaLite.new(width: 600, height: 400, title: "Score vs Distance Achieved")
|
||||
|> VegaLite.data_from_values(scored)
|
||||
|> VegaLite.mark(:circle, opacity: 0.3, size: 15)
|
||||
|> VegaLite.encode_field(:x, "score", type: :quantitative, title: "Composite Score")
|
||||
|> VegaLite.encode_field(:y, "distance_km", type: :quantitative, title: "Distance (km)")
|
||||
|> VegaLite.encode_field(:color, "band_mhz", type: :nominal, title: "Band (MHz)")
|
||||
```
|
||||
|
||||
```elixir
|
||||
# Score by band — box plot
|
||||
VegaLite.new(width: 600, height: 300, title: "Score Distribution by Band")
|
||||
|> VegaLite.data_from_values(scored)
|
||||
|> VegaLite.mark(:boxplot)
|
||||
|> VegaLite.encode_field(:x, "band_mhz", type: :nominal, title: "Band (MHz)")
|
||||
|> VegaLite.encode_field(:y, "score", type: :quantitative, title: "Composite Score")
|
||||
```
|
||||
|
||||
```elixir
|
||||
# Factor contribution heatmap — which factors drive the score?
|
||||
factor_avgs =
|
||||
[:humidity, :time_of_day, :td_depression, :refractivity, :season, :wind, :rain, :pwat, :pressure]
|
||||
|> Enum.flat_map(fn factor ->
|
||||
scored
|
||||
|> Enum.group_by(& &1.band_mhz)
|
||||
|> Enum.map(fn {band, rows} ->
|
||||
avg = Enum.map(rows, &Map.get(&1, factor)) |> Enum.sum() |> then(&(&1 / length(rows)))
|
||||
%{factor: Atom.to_string(factor), band_mhz: band, avg_score: Float.round(avg, 1)}
|
||||
end)
|
||||
end)
|
||||
|
||||
VegaLite.new(width: 600, height: 300, title: "Average Factor Scores by Band")
|
||||
|> VegaLite.data_from_values(factor_avgs)
|
||||
|> VegaLite.mark(:rect)
|
||||
|> VegaLite.encode_field(:x, "band_mhz", type: :nominal, title: "Band (MHz)")
|
||||
|> VegaLite.encode_field(:y, "factor", type: :nominal, title: "Factor")
|
||||
|> VegaLite.encode_field(:color, "avg_score", type: :quantitative,
|
||||
scale: %{scheme: "redyellowgreen", domain: [0, 100]}, title: "Avg Score"
|
||||
)
|
||||
```
|
||||
|
||||
## 9. Weight Sensitivity Analysis
|
||||
|
||||
Template for testing different weight configurations against historical data.
|
||||
Test different weight configurations against the scored dataset.
|
||||
|
||||
```elixir
|
||||
# Define weight sets to test
|
||||
weight_sets = %{
|
||||
current: %{humidity: 0.18, time_of_day: 0.10, td_depression: 0.10,
|
||||
refractivity: 0.08, sky: 0.08, season: 0.08,
|
||||
wind: 0.05, rain: 0.08, pressure: 0.15, pwat: 0.10},
|
||||
weights = BandConfig.weights()
|
||||
|
||||
refractivity_heavy: %{humidity: 0.15, time_of_day: 0.08, td_depression: 0.10,
|
||||
refractivity: 0.18, sky: 0.06, season: 0.08,
|
||||
wind: 0.04, rain: 0.06, pressure: 0.12, pwat: 0.13},
|
||||
# Recompute composite with custom weights
|
||||
rescore = fn rows, custom_weights ->
|
||||
Enum.map(rows, fn row ->
|
||||
score =
|
||||
Enum.reduce(custom_weights, 0.0, fn {factor, weight}, acc ->
|
||||
acc + Map.get(row, factor, 50) * weight
|
||||
end)
|
||||
|> round()
|
||||
|
||||
humidity_heavy: %{humidity: 0.25, time_of_day: 0.08, td_depression: 0.12,
|
||||
refractivity: 0.08, sky: 0.06, season: 0.08,
|
||||
wind: 0.04, rain: 0.06, pressure: 0.12, pwat: 0.11}
|
||||
Map.put(row, :custom_score, score)
|
||||
end)
|
||||
end
|
||||
|
||||
# Define alternative weight sets
|
||||
alternatives = %{
|
||||
current: weights,
|
||||
|
||||
refractivity_heavy: %{
|
||||
humidity: 0.15, time_of_day: 0.08, td_depression: 0.10,
|
||||
refractivity: 0.18, sky: 0.06, season: 0.08,
|
||||
wind: 0.04, rain: 0.06, pressure: 0.12, pwat: 0.13
|
||||
},
|
||||
|
||||
pressure_reduced: %{
|
||||
humidity: 0.20, time_of_day: 0.12, td_depression: 0.12,
|
||||
refractivity: 0.10, sky: 0.08, season: 0.08,
|
||||
wind: 0.05, rain: 0.08, pressure: 0.07, pwat: 0.10
|
||||
}
|
||||
}
|
||||
|
||||
# TODO: score all contacts with each weight set,
|
||||
# then correlate scores with distance achieved and commercial link signal levels
|
||||
IO.puts("Weight sets defined: #{Map.keys(weight_sets) |> Enum.join(", ")}")
|
||||
# Compare: correlation of score with distance for each weight set
|
||||
comparisons =
|
||||
Enum.map(alternatives, fn {name, w} ->
|
||||
rescored = rescore.(scored, w)
|
||||
scores = Enum.map(rescored, & &1.custom_score)
|
||||
distances = Enum.map(rescored, & &1.distance_km)
|
||||
n = length(scores)
|
||||
|
||||
mean_s = Enum.sum(scores) / n
|
||||
mean_d = Enum.sum(distances) / n
|
||||
|
||||
cov = Enum.zip(scores, distances) |> Enum.map(fn {s, d} -> (s - mean_s) * (d - mean_d) end) |> Enum.sum()
|
||||
var_s = Enum.map(scores, fn s -> (s - mean_s) ** 2 end) |> Enum.sum()
|
||||
var_d = Enum.map(distances, fn d -> (d - mean_d) ** 2 end) |> Enum.sum()
|
||||
r = if var_s > 0 and var_d > 0, do: cov / :math.sqrt(var_s * var_d), else: 0.0
|
||||
|
||||
%{weights: name, correlation_r: Float.round(r, 4), mean_score: Float.round(mean_s, 1)}
|
||||
end)
|
||||
|
||||
Kino.DataTable.new(comparisons, name: "Weight Set Comparison (score-distance correlation)")
|
||||
```
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue