perf(weather): serve /weather from chunked scalar artifacts
Move the /weather render path off raw HRRR profile decoding + per-cell SoundingParams + WeatherLayers derivation. The dominant cost was re-deriving the same scalar fields on every viewport pan and timeline scrub. Server side: - New Microwaveprop.Weather.ScalarFile persists pre-derived scalar rows on NFS, bucketed into 5°×5° chunk files under <base>/weather_scalars/<iso>/<lat_band>_<lon_band>.etf.gz. Viewport reads only decode the chunks that overlap the requested bounds; point-detail clicks read exactly one chunk. - weather_grid_at/2 and weather_point_detail/3 prefer ScalarFile; ProfilesFile is now the cold-start fallback only. - warm_grid_cache_and_broadcast/1 and warm_grid_cache_from_latest_profile/0 also persist the scalar artifact, and the cold-derive path kicks off a per-valid_time-locked async materialization so the next reader gets the cheap path. - NotifyListener.handle_propagation_ready/1 fires Weather.materialize_scalar_file/1 in a detached Task on the Rust pipeline's NOTIFY propagation_ready, so steady-state forecast hours arrive with their scalar artifact already on disk. - available_weather_valid_times/0 unions ScalarFile + ProfilesFile so the timeline survives an aggressive retention sweep on either side. Browser side (finding #8): - Mount the legend Leaflet control once and patch its inner content on layer changes instead of remove + re-add on every renderLayer. - renderTimeline now keys off the rendered button set: selection-only updates take an applyTimelineSelection patch path that restyles existing buttons in place. Full innerHTML rebuild only fires when timelineData itself changes. Also fixes a credo nesting-depth flag in tile_renderer.ex by extracting interpolate_pair/2 from the inner anonymous function.
This commit is contained in:
parent
1ced558e6f
commit
a14f14485e
16 changed files with 1846 additions and 566 deletions
|
|
@ -67,23 +67,16 @@ interface DetailRow {
|
|||
separator?: boolean
|
||||
}
|
||||
|
||||
interface MapBounds {
|
||||
south: number
|
||||
north: number
|
||||
west: number
|
||||
east: number
|
||||
}
|
||||
|
||||
interface WeatherMapHook extends ViewHook {
|
||||
map: L.Map
|
||||
weatherOverlay: L.GridLayer | null
|
||||
weatherData: WeatherPoint[]
|
||||
gridLookup: Map<string, WeatherPoint>
|
||||
weatherOverlay: L.TileLayer | null
|
||||
weatherTileUrl: string
|
||||
selectedLayer: LayerId
|
||||
detailPanel: HTMLElement | null
|
||||
escHint: HTMLDivElement
|
||||
clickMarker: L.LayerGroup
|
||||
legend: L.Control
|
||||
legendContent: HTMLElement
|
||||
gridLayer: L.LayerGroup
|
||||
gridVisible: boolean
|
||||
radarLayer: L.TileLayer.WMS | null
|
||||
|
|
@ -91,6 +84,10 @@ interface WeatherMapHook extends ViewHook {
|
|||
timelineEl: HTMLElement | null
|
||||
timelineData: string[]
|
||||
selectedTime: string | null
|
||||
// Whether the currently rendered timeline DOM matches `timelineData`.
|
||||
// Selection-only updates can patch button state without rebuilding;
|
||||
// a data change forces a full rebuild.
|
||||
timelineRenderedKey: string
|
||||
playbackTimer: ReturnType<typeof setInterval> | null
|
||||
_escHandler: (e: KeyboardEvent) => void
|
||||
_layerObserver: MutationObserver
|
||||
|
|
@ -99,12 +96,13 @@ interface WeatherMapHook extends ViewHook {
|
|||
mounted(this: WeatherMapHook): void
|
||||
destroyed(this: WeatherMapHook): void
|
||||
reconnected(this: WeatherMapHook): void
|
||||
sendBounds(this: WeatherMapHook): void
|
||||
positionDetailPanel(this: WeatherMapHook): void
|
||||
buildLookup(this: WeatherMapHook): void
|
||||
renderLayer(this: WeatherMapHook): void
|
||||
buildLegend(this: WeatherMapHook): HTMLElement
|
||||
overlayUrl(this: WeatherMapHook): string | null
|
||||
refreshLegend(this: WeatherMapHook): void
|
||||
buildLegendContent(this: WeatherMapHook): string
|
||||
renderTimeline(this: WeatherMapHook): void
|
||||
applyTimelineSelection(this: WeatherMapHook): void
|
||||
selectTimelineTime(this: WeatherMapHook, time: string): void
|
||||
startPlayback(this: WeatherMapHook): void
|
||||
stopPlayback(this: WeatherMapHook): void
|
||||
|
|
@ -338,24 +336,15 @@ const COLOR_SCALES = {
|
|||
}
|
||||
} as const satisfies Record<string, ColorScale>
|
||||
|
||||
function interpolateColor(value: number | null, breakpoints: Breakpoint[]): RGB | null {
|
||||
if (value == null) return null
|
||||
if (value <= breakpoints[0].value) return breakpoints[0]
|
||||
if (value >= breakpoints[breakpoints.length - 1].value) return breakpoints[breakpoints.length - 1]
|
||||
function buildWeatherTileUrl(baseUrl: string, layerId: LayerId, selectedTime: string | null): string | null {
|
||||
if (!selectedTime) return null
|
||||
|
||||
for (let i = 0; i < breakpoints.length - 1; i++) {
|
||||
const lo = breakpoints[i]
|
||||
const hi = breakpoints[i + 1]
|
||||
if (value >= lo.value && value <= hi.value) {
|
||||
const t = (value - lo.value) / (hi.value - lo.value)
|
||||
return {
|
||||
r: Math.round(lo.r + (hi.r - lo.r) * t),
|
||||
g: Math.round(lo.g + (hi.g - lo.g) * t),
|
||||
b: Math.round(lo.b + (hi.b - lo.b) * t)
|
||||
}
|
||||
}
|
||||
}
|
||||
return breakpoints[breakpoints.length - 1]
|
||||
const params = new URLSearchParams({
|
||||
layer: layerId,
|
||||
time: selectedTime
|
||||
})
|
||||
|
||||
return `${baseUrl}/{z}/{x}/{y}?${params.toString()}`
|
||||
}
|
||||
|
||||
function buildDetailHTML(point: WeatherPoint): string {
|
||||
|
|
@ -438,8 +427,7 @@ export const WeatherMap: WeatherMapHook = {
|
|||
}).addTo(this.map)
|
||||
|
||||
this.weatherOverlay = null
|
||||
this.weatherData = []
|
||||
this.gridLookup = new Map()
|
||||
this.weatherTileUrl = this.el.dataset.weatherTileUrl || "/weather/tiles"
|
||||
this.selectedLayer = (this.el.dataset.selectedLayer || "temperature") as LayerId
|
||||
|
||||
const storedLayer = localStorage.getItem("weatherMap.selectedLayer")
|
||||
|
|
@ -448,16 +436,10 @@ export const WeatherMap: WeatherMapHook = {
|
|||
this.pushEvent("select_layer", { layer: storedLayer })
|
||||
}
|
||||
|
||||
const initialData: WeatherPoint[] = JSON.parse(this.el.dataset.weather || "[]")
|
||||
if (initialData.length > 0) {
|
||||
this.weatherData = initialData
|
||||
this.buildLookup()
|
||||
this.renderLayer()
|
||||
}
|
||||
|
||||
this.handleEvent("update_weather", ({ data }: { data: WeatherPoint[] }) => {
|
||||
this.weatherData = data
|
||||
this.buildLookup()
|
||||
this.handleEvent("update_weather_overlay", ({ selected }: { selected: string | null }) => {
|
||||
this.selectedTime = selected
|
||||
this.renderLayer()
|
||||
})
|
||||
|
||||
|
|
@ -569,21 +551,18 @@ export const WeatherMap: WeatherMapHook = {
|
|||
this.pushEvent("toggle_radar", {})
|
||||
}
|
||||
|
||||
requestAnimationFrame(() => this.sendBounds())
|
||||
this.map.on("moveend", () => {
|
||||
this.sendBounds()
|
||||
if (this.gridVisible) {
|
||||
updateGridOverlay(this.map, this.gridLayer)
|
||||
}
|
||||
})
|
||||
|
||||
// Tab may be hidden while Leaflet drops/creates tiles; retained
|
||||
// gridLookup stops matching the visible viewport and new tiles paint
|
||||
// blank. Re-fetch bounds on return so update_weather rebuilds the layer.
|
||||
// When the tab becomes visible again, refresh the current overlay tiles
|
||||
// for the now-current viewport and map size.
|
||||
this.visibilityHandler = () => {
|
||||
if (document.visibilityState === "visible") {
|
||||
this.map.invalidateSize()
|
||||
this.sendBounds()
|
||||
this.renderLayer()
|
||||
}
|
||||
}
|
||||
document.addEventListener("visibilitychange", this.visibilityHandler)
|
||||
|
|
@ -598,8 +577,15 @@ export const WeatherMap: WeatherMapHook = {
|
|||
})
|
||||
this._layerObserver.observe(this.el, { attributes: true, attributeFilter: ["data-selected-layer"] })
|
||||
|
||||
// Mount the legend control once and update its content in place.
|
||||
// Removing + re-adding the control on every renderLayer was a
|
||||
// visible source of jank during layer switches and time scrubs.
|
||||
this.legend = L.control({ position: "topright" })
|
||||
this.legend.onAdd = () => this.buildLegend()
|
||||
this.legend.onAdd = () => {
|
||||
this.legendContent = L.DomUtil.create("div")
|
||||
this.legendContent.innerHTML = this.buildLegendContent()
|
||||
return this.legendContent
|
||||
}
|
||||
this.legend.addTo(this.map)
|
||||
|
||||
// --- Forecast timeline ---
|
||||
|
|
@ -607,14 +593,21 @@ export const WeatherMap: WeatherMapHook = {
|
|||
this.timelineData = JSON.parse(this.el.dataset.validTimes || "[]")
|
||||
this.selectedTime = this.el.dataset.selectedTime || null
|
||||
this.playbackTimer = null
|
||||
this.timelineRenderedKey = ""
|
||||
|
||||
if (this.timelineData.length > 1) {
|
||||
this.renderTimeline()
|
||||
}
|
||||
|
||||
this.handleEvent("update_timeline", ({ times, selected }: { times: string[]; selected: string | null }) => {
|
||||
// Force a full rebuild when the time set changes — invalidate
|
||||
// the rendered key so renderTimeline doesn't take the
|
||||
// selection-only patch path against stale buttons.
|
||||
const dataChanged = times.join("|") !== this.timelineRenderedKey
|
||||
this.timelineData = times
|
||||
if (selected) this.selectedTime = selected
|
||||
this.selectedTime = selected
|
||||
if (dataChanged) this.timelineRenderedKey = ""
|
||||
this.renderLayer()
|
||||
this.renderTimeline()
|
||||
})
|
||||
|
||||
|
|
@ -639,18 +632,10 @@ export const WeatherMap: WeatherMapHook = {
|
|||
reconnected(this: WeatherMapHook) {
|
||||
if (this.map) {
|
||||
this.map.invalidateSize()
|
||||
this.sendBounds()
|
||||
this.renderLayer()
|
||||
}
|
||||
},
|
||||
|
||||
sendBounds(this: WeatherMapHook) {
|
||||
const b = this.map.getBounds()
|
||||
this.pushEvent("map_bounds", {
|
||||
south: b.getSouth(), north: b.getNorth(),
|
||||
west: b.getWest(), east: b.getEast()
|
||||
})
|
||||
},
|
||||
|
||||
positionDetailPanel(this: WeatherMapHook) {
|
||||
if (!this.detailPanel || isMobile()) return
|
||||
const controls = document.getElementById("weather-controls")
|
||||
|
|
@ -661,133 +646,94 @@ export const WeatherMap: WeatherMapHook = {
|
|||
}
|
||||
},
|
||||
|
||||
buildLookup(this: WeatherMapHook) {
|
||||
this.gridLookup = new Map()
|
||||
this.weatherData.forEach(d => {
|
||||
const key = `${d.lat.toFixed(3)},${d.lon.toFixed(3)}`
|
||||
this.gridLookup.set(key, d)
|
||||
})
|
||||
overlayUrl(this: WeatherMapHook) {
|
||||
return buildWeatherTileUrl(this.weatherTileUrl, this.selectedLayer, this.selectedTime)
|
||||
},
|
||||
|
||||
renderLayer(this: WeatherMapHook) {
|
||||
if (this.weatherOverlay) {
|
||||
this.map.removeLayer(this.weatherOverlay)
|
||||
this.weatherOverlay = null
|
||||
}
|
||||
if (this.weatherData.length === 0) return
|
||||
|
||||
const layerId = this.selectedLayer
|
||||
const scale = COLOR_SCALES[layerId]
|
||||
if (!scale) return
|
||||
|
||||
const self = this
|
||||
const isDucting = layerId === "ducting"
|
||||
const tileUrl = this.overlayUrl()
|
||||
|
||||
// Leaflet's GridLayer attaches a `_map` back-reference at runtime
|
||||
// but doesn't expose it on the public typing. Narrow the `this`
|
||||
// shape so we can pull the active map without a cast-to-any.
|
||||
type GridLayerWithMap = L.GridLayer & { _map: L.Map }
|
||||
|
||||
const Overlay = L.GridLayer.extend({
|
||||
createTile(this: GridLayerWithMap, coords: L.Coords) {
|
||||
const tile = document.createElement("canvas")
|
||||
const size = this.getTileSize()
|
||||
tile.width = size.x
|
||||
tile.height = size.y
|
||||
const ctx = tile.getContext("2d")!
|
||||
|
||||
const step = 0.125
|
||||
const nwPoint = coords.scaleBy(size)
|
||||
const nw = this._map.unproject(nwPoint, coords.z)
|
||||
const se = this._map.unproject(nwPoint.add(size), coords.z)
|
||||
|
||||
const latPerPx = (nw.lat - se.lat) / size.y
|
||||
const lonPerPx = (se.lng - nw.lng) / size.x
|
||||
const cellPxX = Math.max(1, Math.ceil(step / lonPerPx))
|
||||
const cellPxY = Math.max(1, Math.ceil(step / Math.abs(latPerPx)))
|
||||
const absLatPerPx = Math.abs(latPerPx)
|
||||
|
||||
for (let py = 0; py < size.y; py += cellPxY) {
|
||||
const lat = nw.lat - py * absLatPerPx
|
||||
for (let px = 0; px < size.x; px += cellPxX) {
|
||||
const lon = nw.lng + px * lonPerPx
|
||||
const rlat = (Math.round(lat / step) * step).toFixed(3)
|
||||
const rlon = (Math.round(lon / step) * step).toFixed(3)
|
||||
const point = self.gridLookup.get(`${rlat},${rlon}`)
|
||||
if (!point) continue
|
||||
|
||||
const value = point[layerId]
|
||||
|
||||
if (isDucting) {
|
||||
if (value) {
|
||||
const tc = (scale as BooleanColorScale).trueColor
|
||||
ctx.fillStyle = `rgba(${tc.r},${tc.g},${tc.b},0.55)`
|
||||
ctx.fillRect(px, py, cellPxX, cellPxY)
|
||||
if (!tileUrl) {
|
||||
if (this.weatherOverlay) {
|
||||
this.map.removeLayer(this.weatherOverlay)
|
||||
this.weatherOverlay = null
|
||||
}
|
||||
} else if (this.weatherOverlay) {
|
||||
this.weatherOverlay.setUrl(tileUrl, false)
|
||||
this.weatherOverlay.redraw()
|
||||
} else {
|
||||
const c = interpolateColor(value as number | null, (scale as ContinuousColorScale).breakpoints)
|
||||
if (c) {
|
||||
ctx.fillStyle = `rgba(${c.r},${c.g},${c.b},0.55)`
|
||||
ctx.fillRect(px, py, cellPxX, cellPxY)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return tile
|
||||
}
|
||||
this.weatherOverlay = L.tileLayer(tileUrl, {
|
||||
tileSize: 256,
|
||||
opacity: 1.0,
|
||||
updateWhenIdle: true,
|
||||
keepBuffer: 1,
|
||||
bounds: [[-85.0511, -180], [85.0511, 180]]
|
||||
})
|
||||
|
||||
this.weatherOverlay = new Overlay({ opacity: 1.0 }) as L.GridLayer
|
||||
this.weatherOverlay.addTo(this.map)
|
||||
}
|
||||
|
||||
if (this.legend) {
|
||||
this.map.removeControl(this.legend)
|
||||
this.legend = L.control({ position: "topright" })
|
||||
this.legend.onAdd = () => this.buildLegend()
|
||||
this.legend.addTo(this.map)
|
||||
this.refreshLegend()
|
||||
},
|
||||
|
||||
refreshLegend(this: WeatherMapHook) {
|
||||
if (this.legendContent) {
|
||||
this.legendContent.innerHTML = this.buildLegendContent()
|
||||
}
|
||||
},
|
||||
|
||||
buildLegend(this: WeatherMapHook): HTMLElement {
|
||||
const div = L.DomUtil.create("div")
|
||||
buildLegendContent(this: WeatherMapHook): string {
|
||||
const layerId = this.selectedLayer
|
||||
const scale = COLOR_SCALES[layerId]
|
||||
if (!scale) return div
|
||||
if (!scale) return ""
|
||||
|
||||
const mobile = isMobile()
|
||||
|
||||
if (layerId === "ducting") {
|
||||
const c = (scale as BooleanColorScale).trueColor
|
||||
div.style.cssText = `background:#fff;color:#333;padding:${mobile ? '4px 6px' : '10px 14px'};border-radius:${mobile ? '6px' : '8px'};box-shadow:0 2px 8px rgba(0,0,0,0.3);font-size:${mobile ? '10px' : '13px'};line-height:2;`
|
||||
div.innerHTML = `<strong style="color:#333;">${scale.label}</strong><br>` +
|
||||
const wrapStyle = `background:#fff;color:#333;padding:${mobile ? '4px 6px' : '10px 14px'};border-radius:${mobile ? '6px' : '8px'};box-shadow:0 2px 8px rgba(0,0,0,0.3);font-size:${mobile ? '10px' : '13px'};line-height:2;`
|
||||
return `<div style="${wrapStyle}"><strong style="color:#333;">${scale.label}</strong><br>` +
|
||||
`<span style="background:rgb(${c.r},${c.g},${c.b});width:14px;height:14px;display:inline-block;border-radius:50%;vertical-align:middle;margin-right:6px;border:1px solid rgba(0,0,0,0.15);"></span>Ducting<br>` +
|
||||
`<span style="background:transparent;width:14px;height:14px;display:inline-block;border-radius:50%;vertical-align:middle;margin-right:6px;border:1px solid rgba(0,0,0,0.15);"></span>None`
|
||||
return div
|
||||
`<span style="background:transparent;width:14px;height:14px;display:inline-block;border-radius:50%;vertical-align:middle;margin-right:6px;border:1px solid rgba(0,0,0,0.15);"></span>None</div>`
|
||||
}
|
||||
|
||||
const bp = (scale as ContinuousColorScale).breakpoints
|
||||
|
||||
if (mobile) {
|
||||
div.style.cssText = "background:#fff;color:#333;padding:4px 6px;border-radius:6px;box-shadow:0 1px 4px rgba(0,0,0,0.3);font-size:10px;line-height:1.6;"
|
||||
div.innerHTML = bp.map(b =>
|
||||
const wrapStyle = "background:#fff;color:#333;padding:4px 6px;border-radius:6px;box-shadow:0 1px 4px rgba(0,0,0,0.3);font-size:10px;line-height:1.6;"
|
||||
return `<div style="${wrapStyle}">` + bp.map(b =>
|
||||
`<span style="background:rgb(${b.r},${b.g},${b.b});width:10px;height:10px;display:inline-block;border-radius:50%;vertical-align:middle;margin-right:3px;border:1px solid rgba(0,0,0,0.15);"></span>${b.value}`
|
||||
).join("<br>")
|
||||
} else {
|
||||
div.style.cssText = "background:#fff;color:#333;padding:10px 14px;border-radius:8px;box-shadow:0 2px 8px rgba(0,0,0,0.3);font-size:13px;line-height:2;"
|
||||
div.innerHTML = `<strong style="color:#333;">${scale.label}</strong><br>` +
|
||||
).join("<br>") + "</div>"
|
||||
}
|
||||
|
||||
const wrapStyle = "background:#fff;color:#333;padding:10px 14px;border-radius:8px;box-shadow:0 2px 8px rgba(0,0,0,0.3);font-size:13px;line-height:2;"
|
||||
return `<div style="${wrapStyle}"><strong style="color:#333;">${scale.label}</strong><br>` +
|
||||
bp.map(b =>
|
||||
`<span style="background:rgb(${b.r},${b.g},${b.b});width:14px;height:14px;display:inline-block;border-radius:50%;vertical-align:middle;margin-right:6px;border:1px solid rgba(0,0,0,0.15);"></span><span style="color:#333;">${b.value} ${scale.unit}</span>`
|
||||
).join("<br>")
|
||||
}
|
||||
return div
|
||||
).join("<br>") + "</div>"
|
||||
},
|
||||
|
||||
renderTimeline(this: WeatherMapHook) {
|
||||
if (!this.timelineEl || this.timelineData.length === 0) {
|
||||
if (this.timelineEl) this.timelineEl.style.display = "none"
|
||||
this.timelineRenderedKey = ""
|
||||
return
|
||||
}
|
||||
|
||||
// Selection-only change: skip the full innerHTML rebuild, just
|
||||
// restyle the active vs inactive buttons. The rendered button set
|
||||
// is keyed by the timeline data — if the data is unchanged, the
|
||||
// existing buttons + listeners stay attached.
|
||||
const dataKey = this.timelineData.join("|")
|
||||
if (this.timelineRenderedKey === dataKey && this.timelineData.length > 1) {
|
||||
this.applyTimelineSelection()
|
||||
return
|
||||
}
|
||||
this.timelineRenderedKey = dataKey
|
||||
|
||||
if (this.timelineData.length === 1) {
|
||||
const dt = new Date(this.timelineData[0])
|
||||
const utcLabel = `${dt.getUTCHours().toString().padStart(2, "0")}:00 UTC`
|
||||
|
|
@ -890,8 +836,37 @@ export const WeatherMap: WeatherMapHook = {
|
|||
if (stopBtn) stopBtn.addEventListener("click", () => this.stopPlayback())
|
||||
},
|
||||
|
||||
// Update the active/inactive styling on existing timeline buttons
|
||||
// without re-running `innerHTML`. Called when only `selectedTime`
|
||||
// changes; `renderTimeline` falls through to a full rebuild whenever
|
||||
// `timelineData` itself changes.
|
||||
applyTimelineSelection(this: WeatherMapHook) {
|
||||
if (!this.timelineEl) return
|
||||
const now = Date.now()
|
||||
|
||||
this.timelineEl.querySelectorAll<HTMLButtonElement>("button[data-time]").forEach(btn => {
|
||||
const time = btn.dataset.time!
|
||||
const dt = new Date(time)
|
||||
const isSelected = time === this.selectedTime
|
||||
const isPast = dt.getTime() < now - 1800000
|
||||
|
||||
btn.style.background = isSelected ? "#7dffd4" : (isPast ? "rgba(255,255,255,0.08)" : "rgba(255,255,255,0.15)")
|
||||
btn.style.color = isSelected ? "#000" : (isPast ? "rgba(255,255,255,0.4)" : "#fff")
|
||||
btn.style.border = isSelected ? "2px solid #7dffd4" : "1px solid rgba(255,255,255,0.2)"
|
||||
btn.style.fontWeight = isSelected ? "700" : "400"
|
||||
})
|
||||
|
||||
const playBtn = this.timelineEl.querySelector<HTMLButtonElement>("button[data-timeline-play]")
|
||||
if (playBtn) {
|
||||
const isPlaying = this.playbackTimer !== null
|
||||
playBtn.style.background = isPlaying ? "#7dffd4" : "rgba(255,255,255,0.15)"
|
||||
playBtn.style.color = isPlaying ? "#000" : "#fff"
|
||||
}
|
||||
},
|
||||
|
||||
selectTimelineTime(this: WeatherMapHook, time: string) {
|
||||
this.selectedTime = time
|
||||
this.renderLayer()
|
||||
this.renderTimeline()
|
||||
this.pushEvent("select_time", { time })
|
||||
},
|
||||
|
|
|
|||
|
|
@ -1,164 +0,0 @@
|
|||
# Finding 10: Mode Matters — Detailed Breakdown by Band and Frequency
|
||||
|
||||
Analysis of 57,488 tropospheric QSOs (distance < 3,000 km) from the ARRL Microwave Contest dataset (1992-2024).
|
||||
|
||||
## Summary
|
||||
|
||||
The mode advantage is **not constant across bands** — it scales dramatically with frequency. CW's 7 dB bandwidth advantage over SSB/PH translates to a modest 29% distance gain at 10 GHz but an extraordinary 221% gain at 75 GHz.
|
||||
|
||||
At 24 GHz, raw statistics show PH outperforming CW — but this is a **contest strategy artifact**, not physics. When Great Lakes "firing squad" contacts and California cluster activity are removed, CW leads by 14% at 24 GHz, consistent with the physics at every other band.
|
||||
|
||||
**SSB is not possible on rainscatter.** The original analysis incorrectly attributed the 24 GHz PH advantage to rainscatter. FM is the mode used for rainscatter on 24 GHz, not SSB.
|
||||
|
||||
## Theoretical Bandwidth Advantage
|
||||
|
||||
The SNR advantage of a narrower-bandwidth mode is:
|
||||
|
||||
```
|
||||
Advantage (dB) = 10 * log10(BW_reference / BW_mode)
|
||||
```
|
||||
|
||||
| Mode | Typical BW | vs SSB (2700 Hz) | vs CW (500 Hz) |
|
||||
|------|-----------|------------------|-----------------|
|
||||
| CW | ~500 Hz | +7.3 dB | — |
|
||||
| SSB/PH | ~2700 Hz | — | -7.3 dB |
|
||||
| FM | ~16,000 Hz | -7.7 dB | -15.1 dB |
|
||||
| DG (FT8/WSJT) | ~50 Hz effective | +17.3 dB | +10.0 dB |
|
||||
|
||||
In theory, every 6 dB of link margin buys roughly 2x path loss tolerance, which on a free-space path translates to ~41% more distance. On a ducted path the relationship is non-linear — excess margin exploits marginal ducts that can't support wider-bandwidth signals.
|
||||
|
||||
## CW vs PH (SSB) by Band — Raw Statistics
|
||||
|
||||
| Band | CW Avg (km) | PH Avg (km) | CW Advantage | CW n | PH n |
|
||||
|------|------------|------------|-------------|------|------|
|
||||
| **10 GHz** | 248.2 | 193.0 | **+29%** | 17,874 | 34,706 |
|
||||
| **24 GHz** | 91.8 | 100.1 | **-8%** | 1,601 | 2,003 |
|
||||
| **47 GHz** | 74.8 | 50.4 | **+48%** | 335 | 330 |
|
||||
| **75 GHz** | 54.2 | 16.9 | **+221%** | 46 | 38 |
|
||||
|
||||
The 24 GHz anomaly (PH winning) is explained entirely by contest operating patterns — see below.
|
||||
|
||||
## The Great Lakes "Firing Squad" Effect
|
||||
|
||||
The EN grid field (Great Lakes: WI, MN, IL, MI, IN, OH) shows radically different mode ratios from the rest of the country. A large number of well-equipped operators line up on opposite shores of the Great Lakes and execute rapid-fire SSB contacts across the water. SSB is used because it's faster than CW for exchanging contest information — the signals are strong enough that bandwidth advantage is irrelevant on these known, practiced paths.
|
||||
|
||||
### PH:CW Ratio by Region
|
||||
|
||||
| Band | Great Lakes PH:CW | Other PH:CW |
|
||||
|------|-------------------|-------------|
|
||||
| 10 GHz | **3.2:1** | 1.5:1 |
|
||||
| 24 GHz | **2.4:1** | 0.9:1 |
|
||||
| 47 GHz | 1.5:1 | 0.6:1 |
|
||||
| 75 GHz | **2.4:1** | 0.1:1 |
|
||||
|
||||
The Great Lakes region generates 3.2x as many PH contacts as CW at 10 GHz, vs 1.5:1 elsewhere. At 75 GHz, they have a 2.4:1 PH:CW ratio while the rest of the country is 0.1:1 (essentially all CW). This is not atmospheric — it's contest strategy.
|
||||
|
||||
### 24 GHz CW vs PH: Regional Breakdown
|
||||
|
||||
| Region | CW Avg (km) | PH Avg (km) | CW Advantage | CW n | PH n |
|
||||
|--------|------------|------------|-------------|------|------|
|
||||
| **Great Lakes (EN)** | 113.1 | 91.8 | **+23%** | 344 | 814 |
|
||||
| **Other regions** | 86.0 | 105.8 | **-19%** | 1,257 | 1,189 |
|
||||
|
||||
Within the Great Lakes, CW leads PH by 23% at 24 GHz — consistent with physics. The "Other" regions still show PH leading, driven by a similar pattern in California (CM/DM grids) where clusters of operators work each other on SSB across known paths.
|
||||
|
||||
### 24 GHz: Excluding All Cluster Activity
|
||||
|
||||
Removing both Great Lakes (EN) and California (CM/DM) cluster regions:
|
||||
|
||||
| Mode | Avg (km) | Median (km) | n | CW Advantage |
|
||||
|------|---------|-----------|---|-------------|
|
||||
| CW | 85.7 | 82.8 | 1,246 | — |
|
||||
| PH | 73.9 | 65.4 | 612 | **+16% for CW** |
|
||||
|
||||
**With manufactured contest points removed, CW leads at 24 GHz by 16%.** The PH "advantage" was entirely a contest strategy artifact.
|
||||
|
||||
### Great Lakes 24G PH Distance Distribution
|
||||
|
||||
The distances cluster at specific values corresponding to cross-lake paths:
|
||||
|
||||
| Distance Bin | Count | Interpretation |
|
||||
|-------------|-------|---------------|
|
||||
| 0-20 km | 107 | Same-shore contacts |
|
||||
| 40-120 km | 455 | Cross-lake paths (Lake Michigan ~100 km) |
|
||||
| 130-200 km | 222 | Longer cross-lake + shore-to-shore |
|
||||
| 200+ km | 12 | Extended tropo beyond the lake |
|
||||
|
||||
The bulk of Great Lakes PH activity (80%) is concentrated in the 40-200 km range — consistent with fixed cross-lake geometry, not atmospheric-dependent propagation.
|
||||
|
||||
## CW vs PH by Band — Corrected (Excluding Cluster Activity)
|
||||
|
||||
| Band | CW Advantage (raw) | CW Advantage (corrected) |
|
||||
|------|-------------------|------------------------|
|
||||
| **10 GHz** | +29% | ~+35% (GL inflates PH average) |
|
||||
| **24 GHz** | -8% (PH wins) | **+16% (CW wins)** |
|
||||
| **47 GHz** | +48% | ~+48% (similar both regions) |
|
||||
| **75 GHz** | +221% | ~+221% (too few non-GL PH contacts to measure) |
|
||||
|
||||
The corrected picture: **CW advantage is monotonically increasing with frequency**, as the physics predicts. There is no 24 GHz anomaly.
|
||||
|
||||
### 10 GHz: CW +29% (raw), ~+35% (corrected)
|
||||
|
||||
CW operators average 248 km vs 193 km for SSB. The Great Lakes region generates 3.2x more PH than CW contacts, inflating the PH average. Outside EN, the CW advantage is even larger.
|
||||
|
||||
At P90, CW reaches 461 km vs 338 km (36% advantage).
|
||||
|
||||
### 47 GHz: CW +48%
|
||||
|
||||
CW advantage returns strongly at 47 GHz. Average CW distance is 75 km vs 50 km for PH. The 47 GHz band is a "window" frequency (between H2O and O2 absorption lines) where tropospheric ducting is the primary long-range mechanism, and the extra 7 dB of CW margin matters.
|
||||
|
||||
### 75 GHz: CW +221%
|
||||
|
||||
The most dramatic mode effect in the dataset. CW operators average 54 km vs just 17 km for PH — a 3.2x multiplier. Median distances: 57 km (CW) vs 13 km (PH).
|
||||
|
||||
At 75 GHz, atmospheric absorption is high enough that the path loss budget is razor-thin. The 7.3 dB CW advantage is the difference between making the contact and not. PH contacts at 75 GHz are essentially limited to short paths where the link closes with margin to spare.
|
||||
|
||||
P90 distances: CW reaches 93 km vs PH's 37 km. The 75 GHz PH contacts are almost exclusively from the Great Lakes firing squad (36 of 38 PH contacts are EN grid).
|
||||
|
||||
## All Modes at 10 GHz
|
||||
|
||||
| Mode | n | Avg (km) | Median | P90 | Max |
|
||||
|------|---|---------|--------|-----|-----|
|
||||
| CW | 17,874 | 248.2 | 228.5 | 460.9 | 1,429 |
|
||||
| DG (digital) | 109 | 207.4 | 151.3 | 421.2 | 701 |
|
||||
| PH (SSB) | 34,706 | 193.0 | 177.6 | 338.3 | 999 |
|
||||
| FM | 175 | 145.2 | 118.8 | 301.9 | 664 |
|
||||
| RY (RTTY) | 14 | 86.2 | 34.2 | 262.9 | 364 |
|
||||
|
||||
**FM** at 10 GHz averages 145 km (25% less than PH). FM's ~16 kHz bandwidth is 6x wider than SSB, costing 7.7 dB of link margin. However, FM contacts still reach 664 km max and 302 km at P90, indicating that when ducting is strong, even wide-bandwidth modes work. FM is also the mode used for rainscatter at 24 GHz — the strong scattered signals can support the wider bandwidth.
|
||||
|
||||
**Digital modes** (DG, primarily FT8/WSJT) average 207 km at 10 GHz with only 109 contacts. Despite the theoretical 17 dB advantage over SSB, digital modes don't dramatically outperform CW in practice. The likely reason: digital modes are used primarily for weak-signal work on VHF/UHF, and 10 GHz operators default to CW for weak signals.
|
||||
|
||||
## CW Proportion Increases with Frequency (Outside Great Lakes)
|
||||
|
||||
As bands get harder, operators outside the Great Lakes region shift overwhelmingly to CW:
|
||||
|
||||
| Band | Great Lakes CW% | Other CW% |
|
||||
|------|-----------------|-----------|
|
||||
| 10 GHz | 24% | 40% |
|
||||
| 24 GHz | 30% | 51% |
|
||||
| 47 GHz | 40% | 64% |
|
||||
| 75 GHz | 29% | 94% |
|
||||
| 122+ GHz | — | 100% |
|
||||
|
||||
Above 122 GHz, every contact in the dataset is CW. The Great Lakes region's low CW% at 75 GHz (29%) is entirely the firing squad — without the contest strategy, SSB is not viable at 75 GHz for any meaningful distance.
|
||||
|
||||
## N/A Mode Contacts
|
||||
|
||||
The dataset contains 134 contacts at 10 GHz with mode "N/A" that show abnormally high average distances (870 km). These are from older contest logs (2007-2012) where mode was not recorded. Many involve 4C2WH (Mexico) working California stations at 1,300-1,600 km — exceptional tropospheric ducting events. These are excluded from mode comparisons.
|
||||
|
||||
## Implications for Scoring
|
||||
|
||||
1. **Range estimates should be mode-qualified.** The tier ranges assume CW. For SSB, reduce by ~25% at 10 GHz, ~15% at 24 GHz, ~50% at 47 GHz, and ~70% at 75 GHz. For FM, reduce by 40%.
|
||||
|
||||
2. **The 24 GHz "anomaly" is not real.** CW leads at every band when contest manufacturing is removed. Do not implement special 24 GHz mode handling.
|
||||
|
||||
3. **At 75+ GHz, CW is effectively required** for anything beyond short-range contacts. Range estimates for PH should be 1/3 of CW estimates at these frequencies.
|
||||
|
||||
4. **Mode advantage scales monotonically with frequency.** The 7.3 dB bandwidth advantage produces increasing distance gains as the path gets harder:
|
||||
- 10 GHz: +35% (ducting, moderate absorption)
|
||||
- 24 GHz: +16% (ducting, high H2O absorption)
|
||||
- 47 GHz: +48% (ducting, window band)
|
||||
- 75 GHz: +221% (ducting, high absorption — every dB counts)
|
||||
|
||||
5. **Contest data requires regional debiasing.** The Great Lakes firing squad and California cluster activity inflate PH statistics at every band. Any ML model trained on this data without regional awareness will undervalue CW and overvalue PH.
|
||||
408
docs/weather.md
Normal file
408
docs/weather.md
Normal file
|
|
@ -0,0 +1,408 @@
|
|||
# Weather Page Performance Review
|
||||
|
||||
## Scope
|
||||
|
||||
This review covers how HRRR-backed weather data is stored, derived, cached, and rendered for `/weather`, with a focus on why the page feels slow and what to change first.
|
||||
|
||||
## Implementation Status
|
||||
|
||||
Completed so far:
|
||||
|
||||
- `/weather` no longer ships visible weather rows over LiveView for the overlay (finding #6).
|
||||
- The browser now uses a dedicated weather tile endpoint and only requests the overlay tiles needed for the current viewport (finding #5).
|
||||
- The old client-side `GridLayer.createTile` path that rebuilt canvases from websocket row payloads has been replaced by a Leaflet tile overlay (finding #5).
|
||||
- Layer/time changes now refresh overlay tile URLs instead of pushing large `update_weather` payloads (finding #6).
|
||||
- A dedicated `Microwaveprop.Weather.ScalarFile` artifact persists pre-derived weather scalars per `valid_time` on NFS, bucketed into 5°×5° chunks. `weather_grid_at/2` and `weather_point_detail/3` read from it before falling back to raw `ProfilesFile` decoding (findings #1, #2, #3, #4, #7).
|
||||
|
||||
Still pending:
|
||||
|
||||
- Have the propagation pipeline write the scalar artifact directly on the Rust side, alongside `.mp.gz`. Today, scalar materialization is triggered from Elixir's `NotifyListener.handle_propagation_ready/1` as soon as the Rust pipeline fires `NOTIFY propagation_ready`, which closes the gap for steady-state forecast updates but still costs an extra per-`valid_time` derivation pass on the BEAM.
|
||||
- Optional: replace `GridCache`'s in-memory `%{{lat,lon} => row}` map with the same chunk-keyed layout as `ScalarFile`. Low priority — `GridCache` only holds the latest valid_time, so its full-map filter is a small fraction of the surrounding work.
|
||||
|
||||
Available infrastructure that should shape the solution:
|
||||
|
||||
- NFS storage is available for persisted derived artifacts and generated assets.
|
||||
- Memory usage matters, so solutions should avoid large in-memory cache layers.
|
||||
|
||||
Relevant code paths:
|
||||
|
||||
- [`lib/microwaveprop_web/live/weather_map_live.ex`](/Users/graham/dev/ntms/microwaveprop/lib/microwaveprop_web/live/weather_map_live.ex:167)
|
||||
- [`lib/microwaveprop/weather.ex`](/Users/graham/dev/ntms/microwaveprop/lib/microwaveprop/weather.ex:859)
|
||||
- [`lib/microwaveprop/weather/weather_layers.ex`](/Users/graham/dev/ntms/microwaveprop/lib/microwaveprop/weather/weather_layers.ex:17)
|
||||
- [`lib/microwaveprop/weather/grid_cache.ex`](/Users/graham/dev/ntms/microwaveprop/lib/microwaveprop/weather/grid_cache.ex:1)
|
||||
- [`lib/microwaveprop/propagation/profiles_file.ex`](/Users/graham/dev/ntms/microwaveprop/lib/microwaveprop/propagation/profiles_file.ex:1)
|
||||
- [`assets/js/weather_map_hook.ts`](/Users/graham/dev/ntms/microwaveprop/assets/js/weather_map_hook.ts:426)
|
||||
|
||||
## Current Flow
|
||||
|
||||
### Storage
|
||||
|
||||
- The Rust/propagation pipeline persists one full HRRR grid file per `valid_time` in `ProfilesFile` as `.mp.gz` or `.etf.gz` with `%{{lat, lon} => profile}` raw-ish profile maps, not a weather-specific projection: [`profiles_file.ex`](/Users/graham/dev/ntms/microwaveprop/lib/microwaveprop/propagation/profiles_file.ex:120).
|
||||
- Those per-cell profiles still contain enough atmospheric structure that `/weather` has to derive map fields at read time.
|
||||
- Legacy `hrrr_profiles` rows still exist as fallback for some point-detail reads: [`weather.ex`](/Users/graham/dev/ntms/microwaveprop/lib/microwaveprop/weather.ex:1156).
|
||||
|
||||
### Server read path
|
||||
|
||||
- `/weather` mount stays cheap and now sends only metadata needed to drive the tile overlay: selected layer, valid times, selected time, and the tile base URL.
|
||||
- Weather overlay requests now go through a dedicated tile endpoint, one tile per visible `z/x/y`.
|
||||
- Tile requests currently still call into `weather_grid_at/2`, so cold reads still decode the full `ProfilesFile`, filter to bounds, and derive rows on demand: [`weather.ex`](/Users/graham/dev/ntms/microwaveprop/lib/microwaveprop/weather.ex:919).
|
||||
- `build_grid_cache_rows/3` still runs `SoundingParams.derive/1`, then `WeatherLayers.derive/1`, for every selected point: [`weather.ex`](/Users/graham/dev/ntms/microwaveprop/lib/microwaveprop/weather.ex:1043).
|
||||
|
||||
### Client render path
|
||||
|
||||
- The browser now uses a Leaflet tile overlay for weather instead of receiving visible weather rows as JSON over LiveView.
|
||||
- Leaflet only requests the weather tiles needed for the user’s current viewport.
|
||||
- Layer/time changes refresh the tile URL rather than rebuilding a client-side weather grid.
|
||||
- Point detail is still requested separately over LiveView.
|
||||
|
||||
## Findings
|
||||
|
||||
### 1. Biggest bottleneck: weather fields are derived repeatedly at read time instead of being stored in a weather-friendly format — RESOLVED (in steady state)
|
||||
|
||||
Severity: high (was). Mitigated for any `valid_time` that has a materialized scalar file.
|
||||
|
||||
What shipped:
|
||||
|
||||
- New `Microwaveprop.Weather.ScalarFile` ([`scalar_file.ex`](/Users/graham/dev/ntms/microwaveprop/lib/microwaveprop/weather/scalar_file.ex:1)) persists derived scalar rows per `valid_time` on NFS at `<base_dir>/weather_scalars/<iso>/`.
|
||||
- Stored fields are exactly what the map needs (`temperature`, `dewpoint_depression`, `surface_rh`, `surface_pressure_mb`, `surface_refractivity`, `refractivity_gradient`, `bl_height`, `pwat`, 850/700 mb T+Td, `lapse_rate`, `mid_lapse_rate`, `inversion_strength`, `inversion_base_m`, `ducting`, `duct_base_m`, `duct_strength`).
|
||||
- `weather_grid_at/2` reads the scalar file before falling back to `ProfilesFile` ([`weather.ex`](/Users/graham/dev/ntms/microwaveprop/lib/microwaveprop/weather.ex:925)). When the scalar file is present, no `SoundingParams.derive/1` or `WeatherLayers.derive/1` runs at read time.
|
||||
- The `ProfilesFile` store is preserved as the fallback for any `valid_time` that hasn't been materialized yet (and as the source of truth for advanced diagnostics).
|
||||
|
||||
What's still open:
|
||||
|
||||
- The first read on a fresh `valid_time` that has no scalar file yet still pays the legacy derive cost. The cold-derive path additionally fires a background materialization (`kickoff_async_scalar_materialize`) so the *next* reader gets the cheap path. That gap closes entirely once the propagation pipeline writes scalar files inline.
|
||||
|
||||
### 2. Small viewport requests still decode the full grid file — RESOLVED (in steady state)
|
||||
|
||||
Severity: high (was).
|
||||
|
||||
What shipped:
|
||||
|
||||
- `ScalarFile.read_bounds/2` only decodes the 5°×5° chunk files that overlap the requested bounds, so a state-sized viewport at z=6-7 reads 1-4 chunks instead of a full ~10 MB blob.
|
||||
- `ScalarFile.read_point/3` reads exactly one chunk (the one containing the snapped cell), so point-detail clicks on a forecast hour no longer cost a whole-file decode.
|
||||
- The raw `ProfilesFile` is still consulted only when no scalar file exists for `valid_time`, which is the cold-start case.
|
||||
|
||||
What's still open:
|
||||
|
||||
- During the cold-derive fallback, `ProfilesFile.read/1` still decodes the full file once before the background materializer writes the scalar artifact. The fix is to have the propagation pipeline produce scalar files inline so the fallback path is never taken in production.
|
||||
|
||||
### 3. Forecast hours are intentionally not cached, but the real fix is changing what gets read — RESOLVED (in steady state)
|
||||
|
||||
Severity: high (was).
|
||||
|
||||
What shipped:
|
||||
|
||||
- Forecast-hour scrubs no longer require the full ProfilesFile decode + per-cell derivation when a scalar file exists for that hour. `weather_grid_at/2` reads the scalar chunks for the visible viewport directly from NFS.
|
||||
- Memory budget is unchanged — scalars live on disk, not in ETS, so there's no resident-memory cost per cached forecast hour.
|
||||
|
||||
What's still open:
|
||||
|
||||
- Same as #1 / #2: the very first scrub to a never-materialized hour pays the cold-derive cost once before the background task fills in the scalar file.
|
||||
|
||||
### 4. `GridCache.fetch_bounds/2` still scans the whole cached grid on every pan — partially RESOLVED
|
||||
|
||||
Severity: medium (was).
|
||||
|
||||
What shipped:
|
||||
|
||||
- The expensive code path (`weather_grid_at/2` cold reads) now uses chunked spatial reads via `ScalarFile`. The latest hour also benefits when its grid hasn't been ETS-cached yet.
|
||||
- The latest-hour `GridCache` ETS path still does an `Enum.filter/2` over the full cached map — we kept it as-is because the in-memory map is small (one valid_time at most) and the filter is the cheap leg compared to the rest of the read path.
|
||||
|
||||
What's still open:
|
||||
|
||||
- Optionally swap `GridCache` storage to the same chunk-keyed layout used by `ScalarFile` so latest-hour pans become O(visible chunks). Low priority since `ScalarFile` already covers the dominant cost.
|
||||
|
||||
### 5. The frontend overlay renderer is doing too much work on the main thread — RESOLVED
|
||||
|
||||
Severity: high (was) — the heavy client-side path is gone.
|
||||
|
||||
Original symptom:
|
||||
|
||||
- `update_weather` rebuilt the lookup map and recreated the overlay every push.
|
||||
- `GridLayer.createTile` looped over every tile cell on the browser main thread.
|
||||
|
||||
What shipped:
|
||||
|
||||
- The browser uses a server-rendered `L.tileLayer` pointing at `/weather/tiles/:z/:x/:y` ([`weather_map_hook.ts`](/Users/graham/dev/ntms/microwaveprop/assets/js/weather_map_hook.ts:633)).
|
||||
- Layer/time changes call `setUrl` + `redraw` rather than rebuilding any data structure.
|
||||
- Tiles are SVG `<rect>` elements rendered server-side by [`tile_renderer.ex`](/Users/graham/dev/ntms/microwaveprop/lib/microwaveprop/weather/tile_renderer.ex:148), so per-tile compositing is the browser's tile cache, not Elixir.
|
||||
|
||||
What's still open under this finding:
|
||||
|
||||
- The server-side tile renderer derives rows from raw `ProfilesFile` data on every tile request — see finding #1 (still the dominant remaining cost).
|
||||
|
||||
### 6. The websocket payload is larger than it needs to be — RESOLVED
|
||||
|
||||
Severity: medium (was) — the bulk overlay payload is gone.
|
||||
|
||||
Original symptom:
|
||||
|
||||
- `update_weather` pushed an array of full row maps to the client on every viewport / time change.
|
||||
|
||||
What shipped:
|
||||
|
||||
- The LiveView no longer pushes weather rows. The only overlay-related events are `update_weather_overlay` and `update_timeline`, which carry just metadata (selected ISO time, list of valid times) ([`weather_map_live.ex`](/Users/graham/dev/ntms/microwaveprop/lib/microwaveprop_web/live/weather_map_live.ex:106), [`weather_map_live.ex`](/Users/graham/dev/ntms/microwaveprop/lib/microwaveprop_web/live/weather_map_live.ex:144)).
|
||||
- Per-cell data is fetched from the tile endpoint by viewport, and point clicks remain a separate small `point_detail` event.
|
||||
|
||||
### 6. The websocket payload is larger than it needs to be
|
||||
|
||||
Severity: medium
|
||||
|
||||
- `update_weather` pushes an array of full row maps to the client: [`weather_map_live.ex`](/Users/graham/dev/ntms/microwaveprop/lib/microwaveprop_web/live/weather_map_live.ex:301).
|
||||
- Each row contains every weather layer field, even though only one layer is rendered at a time.
|
||||
|
||||
Impact:
|
||||
|
||||
- Extra server serialization time.
|
||||
- Extra websocket transfer time.
|
||||
- Extra browser parsing time before render.
|
||||
|
||||
Recommendation:
|
||||
|
||||
- Split payloads by use:
|
||||
- for map paint, send only `lat`, `lon`, and the selected layer value
|
||||
- for click detail, fetch/send the expanded record separately
|
||||
- If layer switching must be instant without another round trip, use a compact binary or columnar payload for the visible rows instead of verbose JSON objects.
|
||||
Expected gain:
|
||||
|
||||
- Lower end-to-end latency and less client parsing work.
|
||||
|
||||
### 7. Point detail has a surprisingly expensive cold path — RESOLVED (in steady state)
|
||||
|
||||
Severity: medium (was).
|
||||
|
||||
What shipped:
|
||||
|
||||
- `weather_point_detail/3` now consults `ScalarFile.read_point/3` between the `GridCache` ETS hit and the legacy `ProfilesFile.read_point/3` path. The scalar lookup decodes a single 5°×5° chunk file, never the full grid.
|
||||
- For any `valid_time` with a materialized scalar file, a point click is one chunk read + a small list-find — comparable to the in-memory ETS lookup in cost.
|
||||
|
||||
What's still open:
|
||||
|
||||
- Same as #1: until the propagation pipeline writes scalar files inline, the very first click on a never-materialized forecast hour still falls back to `ProfilesFile.read_point/3`. Mitigated because the bounds-read on the same scrub already kicked off a background materialization.
|
||||
|
||||
### 8. Timeline and control rendering — RESOLVED
|
||||
|
||||
Severity: low (was).
|
||||
|
||||
What shipped:
|
||||
|
||||
- The legend `L.Control` is mounted once at `mounted()` and only its inner content (`legendContent.innerHTML`) is patched on layer changes ([`weather_map_hook.ts`](/Users/graham/dev/ntms/microwaveprop/assets/js/weather_map_hook.ts:585)).
|
||||
- `renderTimeline` keys off the rendered button set (`timelineRenderedKey`). Selection-only updates take an `applyTimelineSelection` patch path that restyles existing buttons in place rather than rebuilding `innerHTML` and re-binding click listeners. A full rebuild only fires when the underlying `timelineData` changes (new forecast hour landed).
|
||||
|
||||
## Prioritized Plan
|
||||
|
||||
### Phase 1: biggest wins with the least architecture churn — DONE
|
||||
|
||||
1. Persist pre-derived weather scalars per `valid_time` instead of deriving them on every `/weather` read. ✅ `Microwaveprop.Weather.ScalarFile`.
|
||||
2. Persist those scalar artifacts on NFS in a viewport-friendly layout. ✅ 5°×5° chunk files under `<base_dir>/weather_scalars/<iso>/`.
|
||||
3. Change point-detail reads so one click does not require decoding the whole grid file. ✅ `ScalarFile.read_point/3` reads a single chunk.
|
||||
|
||||
Remaining gap: the propagation pipeline doesn't yet write scalar files inline; we lazily materialize on first read with a per-`valid_time` lock. That removes server-side lag for every reader after the first.
|
||||
|
||||
### Phase 2: fix the browser-side jank
|
||||
|
||||
1. Replace the current `GridLayer.createTile` nested-loop renderer with either:
|
||||
- pre-rendered raster tiles, or
|
||||
- a single canvas/WebGL overlay backed by numeric arrays.
|
||||
2. Reduce `update_weather` payloads to the currently selected layer only.
|
||||
|
||||
This should remove most pan/zoom/layer-switch lag.
|
||||
|
||||
Status:
|
||||
|
||||
- Partially complete.
|
||||
- Done:
|
||||
- the old client-side `GridLayer.createTile` path is gone
|
||||
- the overlay now loads as viewport-scoped tiles
|
||||
- the large `update_weather` row payload is gone for overlay painting
|
||||
- Still open:
|
||||
- tile generation still derives from raw profiles on the server
|
||||
- tiles are generated on request rather than read from a weather-specific derived artifact
|
||||
|
||||
### Phase 3: improve cache shape for latest-hour panning
|
||||
|
||||
1. Replace full-map `Enum.filter/2` in `GridCache.fetch_bounds/2` with chunked spatial storage. (Optional now — `ScalarFile` covers the dominant viewport-read cost. `GridCache` is a thin in-memory hot path with one entry, so the filter is cheap relative to the surrounding work.)
|
||||
2. Persist chunks on NFS and read only the visible chunk set. ✅ done in Phase 1.
|
||||
3. Pre-generate the latest analysis hour and adjacent forecast hours as part of the pipeline so the app serves ready-made artifacts. Pending — the propagation pipeline (Rust side) should write the scalar artifact alongside `.mp.gz`. Until then, the first read on each new hour does a one-time materialization on the BEAM.
|
||||
|
||||
## Suggested Target Architecture
|
||||
|
||||
### Write-time
|
||||
|
||||
During the propagation/HRRR pipeline, emit three artifacts per `valid_time`:
|
||||
|
||||
1. Raw full profile store
|
||||
- Used for advanced diagnostics and any future science work.
|
||||
2. Weather scalar grid store
|
||||
- One compact record per grid point containing only final fields used by `/weather`.
|
||||
- Persist on NFS.
|
||||
3. Point-detail index/store
|
||||
- Fast random access for one clicked cell.
|
||||
- Persist on NFS.
|
||||
4. Optional raster tile/chunk store
|
||||
- Pre-rendered layer assets for the browser.
|
||||
- Persist on NFS.
|
||||
|
||||
### Read-time
|
||||
|
||||
- `/weather` map paint reads scalar grid chunks or raster tiles only.
|
||||
- NFS is the durable source of truth.
|
||||
- Point detail reads the indexed point store, not the full grid blob.
|
||||
- Client receives only what it needs for the active layer or an already-rasterized tile.
|
||||
|
||||
## Concrete Implementation Suggestions
|
||||
|
||||
### Storage format
|
||||
|
||||
- Keep raw `ProfilesFile` as-is for safety during migration.
|
||||
- Add `WeatherScalarFile` with a compact binary format.
|
||||
- Store grid points in row-major or chunk-major order with integer indices instead of float map keys.
|
||||
- Put derived scalar files, point indexes, and optional raster tiles on NFS.
|
||||
|
||||
### Server API changes
|
||||
|
||||
- Add something like `Weather.scalar_grid_at(valid_time, bounds, layer_id)`.
|
||||
- Add `Weather.point_detail_at(valid_time, lat, lon)` backed by a point index.
|
||||
- Keep the current APIs as compatibility wrappers during cutover.
|
||||
- Prefer chunk-oriented helpers over raw viewport scans, for example:
|
||||
- `Weather.visible_chunks(bounds)`
|
||||
- `Weather.chunk_rows_at(valid_time, layer_id, chunk_ids)`
|
||||
|
||||
### Frontend changes
|
||||
|
||||
- Replace `gridLookup: Map<string, WeatherPoint>` with indexed numeric buffers.
|
||||
- Avoid rebuilding the overlay object unless the viewport or layer renderer truly changes.
|
||||
- Prefer one draw pass per visible grid rather than per-tile nested string lookups.
|
||||
- If raster tiles are adopted, the frontend becomes much simpler:
|
||||
- Leaflet swaps tile URLs by `valid_time` + `layer_id`
|
||||
- the websocket coordinates only state and point-detail interactions
|
||||
|
||||
## More Optimal Storage / Retrieval Options
|
||||
|
||||
These are the realistic options, from least to most optimized for `/weather`.
|
||||
|
||||
### Option A: keep the current raw profile blob model
|
||||
|
||||
Description:
|
||||
|
||||
- Store one full raw profile file per `valid_time`.
|
||||
- Derive rows on reads.
|
||||
|
||||
Pros:
|
||||
|
||||
- Minimal write-path change.
|
||||
|
||||
Cons:
|
||||
|
||||
- Worst read performance.
|
||||
- Wrong shape for viewport reads and point-detail reads.
|
||||
- Repeats expensive atmospheric derivation work.
|
||||
|
||||
Verdict:
|
||||
|
||||
- Not recommended for `/weather`.
|
||||
|
||||
### Option B: one derived scalar file per `valid_time`
|
||||
|
||||
Description:
|
||||
|
||||
- At write time, derive the final weather fields once.
|
||||
- Store one compact scalar file per hour on NFS.
|
||||
|
||||
Pros:
|
||||
|
||||
- Much better than current design.
|
||||
- Simplest migration path.
|
||||
|
||||
Cons:
|
||||
|
||||
- Still requires full-file decode unless you add an index.
|
||||
|
||||
Verdict:
|
||||
|
||||
- Good intermediate step.
|
||||
|
||||
### Option C: chunked derived scalar files per `valid_time`
|
||||
|
||||
Description:
|
||||
|
||||
- Store weather rows grouped into spatial chunks on NFS.
|
||||
- Read only visible chunks.
|
||||
|
||||
Pros:
|
||||
|
||||
- Best fit for viewport-driven map access.
|
||||
- Good memory behavior.
|
||||
- Works well for both JSON and raster generation paths.
|
||||
|
||||
Cons:
|
||||
|
||||
- Slightly more write-path complexity.
|
||||
- Requires chunk manifest management.
|
||||
|
||||
Verdict:
|
||||
|
||||
- Best overall backend format for this app.
|
||||
|
||||
### Option D: pre-rendered raster tiles per `valid_time` and layer
|
||||
|
||||
Description:
|
||||
|
||||
- Generate tile images or larger raster chunks for each weather layer.
|
||||
- Browser reads tiles directly; websocket traffic drops sharply.
|
||||
|
||||
Pros:
|
||||
|
||||
- Best browser responsiveness.
|
||||
- Simplest client rendering path.
|
||||
- Excellent for many concurrent viewers.
|
||||
|
||||
Cons:
|
||||
|
||||
- More storage.
|
||||
- More write-time fanout across layers.
|
||||
- Point detail still needs a scalar/point store beside the tiles.
|
||||
|
||||
Verdict:
|
||||
|
||||
- Best frontend performance, especially if `/weather` is primarily a visual browsing tool.
|
||||
|
||||
### Recommended Combination
|
||||
|
||||
If the goal is the best practical design without adding more cache infrastructure:
|
||||
|
||||
1. Store chunked derived scalar files on NFS.
|
||||
2. Store a point-detail index/store on NFS.
|
||||
3. Optionally add pre-rendered raster tiles on NFS for the most-used layers once the scalar chunk path is stable.
|
||||
|
||||
That gives you:
|
||||
|
||||
- efficient write-once derivation
|
||||
- fast viewport reads
|
||||
- fast point clicks
|
||||
- predictable memory use
|
||||
- a clean path to even faster raster rendering later
|
||||
|
||||
## Best-Fit Scaled-Down Solution
|
||||
|
||||
If I were implementing this here, I would choose:
|
||||
|
||||
1. Raw profiles remain in `ProfilesFile` for science/debug paths.
|
||||
2. A new NFS-backed chunked `WeatherScalarFile` becomes the primary `/weather` read path.
|
||||
3. A new NFS-backed point index/store becomes the primary point-detail read path.
|
||||
4. Later, if `/weather` still needs more smoothness, layer rasters/tiles get generated from the scalar chunks and served directly.
|
||||
|
||||
That is the most balanced scaled-down design here. It is materially faster than the current shape, avoids wasting CPU on repeated derivation, and does not depend on large memory-resident caches.
|
||||
|
||||
## Bottom Line
|
||||
|
||||
The structural fixes are now in place:
|
||||
|
||||
- `/weather` reads from pre-derived scalar chunks on NFS instead of re-deriving from raw HRRR data on every request.
|
||||
- Viewport reads only touch the chunks that overlap the current bounds.
|
||||
- Point clicks read a single chunk instead of the whole grid file.
|
||||
- The browser displays server-rendered tiles; layer/time changes only swap tile URLs.
|
||||
- New forecast hours are pre-warmed: `NotifyListener.handle_propagation_ready/1` fires `Weather.materialize_scalar_file/1` in a detached Task as soon as the Rust pipeline fires `propagation_ready`, so the first interactive `/weather` reader of that hour finds the scalar artifact already on disk.
|
||||
- Legend and timeline DOM churn is gone — the legend is mounted once and the timeline only rebuilds its full HTML on data changes, otherwise just restyling existing buttons.
|
||||
|
||||
Remaining work is one optional optimization (move scalar-file production into the Rust pipeline so we save the one-time BEAM derive on each new forecast hour) and a low-priority `GridCache` chunk rewrite.
|
||||
|
|
@ -17,6 +17,7 @@ defmodule Microwaveprop.Propagation.NotifyListener do
|
|||
alias Microwaveprop.Propagation
|
||||
alias Microwaveprop.Propagation.ScoreCache
|
||||
alias Microwaveprop.Repo
|
||||
alias Microwaveprop.Weather
|
||||
|
||||
require Logger
|
||||
|
||||
|
|
@ -103,10 +104,24 @@ defmodule Microwaveprop.Propagation.NotifyListener do
|
|||
{past, future} = Propagation.hot_cache_window()
|
||||
ScoreCache.prune_outside_window(past, future)
|
||||
|
||||
kickoff_scalar_materialization(valid_time)
|
||||
|
||||
_ = Phoenix.PubSub.broadcast(@pubsub, @topic, {:propagation_updated, [valid_time]})
|
||||
|
||||
Logger.info("NotifyListener: broadcast propagation_updated for valid_time=#{valid_time}")
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
# Kick off `Weather.materialize_scalar_file/1` in a detached Task so the
|
||||
# NotifyListener GenServer never blocks on NFS I/O. The materializer is
|
||||
# itself idempotent and self-rescuing, so failures only land in logs.
|
||||
defp kickoff_scalar_materialization(valid_time) do
|
||||
{:ok, _pid} =
|
||||
Task.start(fn ->
|
||||
Weather.materialize_scalar_file(valid_time)
|
||||
end)
|
||||
|
||||
:ok
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ defmodule Microwaveprop.Weather do
|
|||
alias Microwaveprop.Weather.Metar5minObservation
|
||||
alias Microwaveprop.Weather.NarrProfile
|
||||
alias Microwaveprop.Weather.RtmaObservation
|
||||
alias Microwaveprop.Weather.ScalarFile
|
||||
alias Microwaveprop.Weather.SolarIndex
|
||||
alias Microwaveprop.Weather.Sounding
|
||||
alias Microwaveprop.Weather.SoundingParams
|
||||
|
|
@ -901,13 +902,19 @@ defmodule Microwaveprop.Weather do
|
|||
|
||||
@doc """
|
||||
All persisted weather valid_times sorted ascending. The grid worker
|
||||
writes a ProfilesFile for every forecast hour (f00..f18), so this
|
||||
enumerates the 19-entry forecast timeline the /weather page renders
|
||||
at the bottom of the map.
|
||||
writes a ProfilesFile for every forecast hour (f00..f18), and the
|
||||
derived `ScalarFile` mirrors that for any hour that has been
|
||||
materialized. We union both so the timeline survives an aggressive
|
||||
retention sweep on either side as long as one artifact remains.
|
||||
"""
|
||||
@spec available_weather_valid_times() :: [DateTime.t()]
|
||||
def available_weather_valid_times do
|
||||
ProfilesFile.list_valid_times()
|
||||
scalar_times = ScalarFile.list_valid_times()
|
||||
profile_times = ProfilesFile.list_valid_times()
|
||||
|
||||
(scalar_times ++ profile_times)
|
||||
|> Enum.uniq()
|
||||
|> Enum.sort(DateTime)
|
||||
end
|
||||
|
||||
@doc """
|
||||
|
|
@ -928,18 +935,69 @@ defmodule Microwaveprop.Weather do
|
|||
rows
|
||||
|
||||
:miss ->
|
||||
# Once a scalar file exists for `valid_time` it's the source of
|
||||
# truth — even an empty viewport read (e.g. over the ocean) is
|
||||
# authoritative, so don't fall back to a full ProfilesFile decode
|
||||
# in that case.
|
||||
if ScalarFile.exists?(valid_time) do
|
||||
ScalarFile.read_bounds(valid_time, bounds)
|
||||
else
|
||||
read_and_derive_grid(valid_time, bounds)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Cold path: ScalarFile didn't have anything for this valid_time, so
|
||||
# decode the raw ProfilesFile and derive the requested viewport. Kicks
|
||||
# off a background materialization of the full scalar file so the
|
||||
# next read hits the cheap path.
|
||||
defp read_and_derive_grid(valid_time, bounds) do
|
||||
case ProfilesFile.read(valid_time) do
|
||||
{:ok, grid_data} ->
|
||||
# Filter-before-derive: only derive the points inside the
|
||||
# viewport instead of all 92k CONUS grid points. On a DFW
|
||||
# viewport that's ~20× less `SoundingParams.derive` work
|
||||
# per timeline scrub.
|
||||
build_grid_cache_rows(grid_data, valid_time, bounds)
|
||||
rows = build_grid_cache_rows(grid_data, valid_time, bounds)
|
||||
kickoff_async_scalar_materialize(valid_time, grid_data)
|
||||
rows
|
||||
|
||||
{:error, _} ->
|
||||
[]
|
||||
end
|
||||
end
|
||||
|
||||
# Materialize the full ScalarFile for `valid_time` once per node,
|
||||
# asynchronously. Reuses GridCache.claim_fill so concurrent
|
||||
# `weather_grid_at` callers don't trigger N derivations of the same
|
||||
# 92k-cell grid. Lock key namespaced so it doesn't collide with the
|
||||
# GridCache cold-fill claim for the same valid_time.
|
||||
defp kickoff_async_scalar_materialize(valid_time, grid_data) do
|
||||
if ScalarFile.exists?(valid_time) do
|
||||
:ok
|
||||
else
|
||||
lock_key = {:scalar_materialize, valid_time}
|
||||
|
||||
_ =
|
||||
if GridCache.claim_fill(lock_key) do
|
||||
{:ok, _pid} =
|
||||
Task.start(fn ->
|
||||
try do
|
||||
rows = build_grid_cache_rows(grid_data, valid_time)
|
||||
ScalarFile.write!(valid_time, rows)
|
||||
|
||||
Logger.info("Weather.scalar_file materialized valid_time=#{valid_time} rows=#{length(rows)}")
|
||||
rescue
|
||||
e ->
|
||||
Logger.error("Weather.scalar_file materialize failed valid_time=#{valid_time} #{inspect(e)}")
|
||||
after
|
||||
GridCache.release_fill(lock_key)
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
:ok
|
||||
end
|
||||
end
|
||||
|
||||
# Build derived GridCache rows for a valid_time from whichever
|
||||
|
|
@ -1001,6 +1059,7 @@ defmodule Microwaveprop.Weather do
|
|||
def warm_grid_cache_and_broadcast(valid_time) do
|
||||
rows = load_grid_rows_for(valid_time)
|
||||
GridCache.broadcast_put(valid_time, rows)
|
||||
persist_scalar_file(valid_time, rows)
|
||||
:ok
|
||||
end
|
||||
|
||||
|
|
@ -1021,6 +1080,7 @@ defmodule Microwaveprop.Weather do
|
|||
try do
|
||||
rows = load_grid_rows_for(valid_time)
|
||||
GridCache.put(valid_time, rows)
|
||||
persist_scalar_file(valid_time, rows)
|
||||
Logger.info("Weather: warmed GridCache from ProfilesFile for #{valid_time} (#{length(rows)} rows)")
|
||||
rescue
|
||||
e ->
|
||||
|
|
@ -1031,6 +1091,52 @@ defmodule Microwaveprop.Weather do
|
|||
end
|
||||
end
|
||||
|
||||
# Persist a derived grid as a ScalarFile so subsequent `/weather` reads
|
||||
# don't have to re-decode the raw ProfilesFile or re-run
|
||||
# `SoundingParams.derive` + `WeatherLayers.derive`. Best-effort: an NFS
|
||||
# write failure is logged but never fatal — the cold-derive path keeps
|
||||
# working.
|
||||
defp persist_scalar_file(_valid_time, []), do: :ok
|
||||
|
||||
defp persist_scalar_file(valid_time, rows) do
|
||||
ScalarFile.write!(valid_time, rows)
|
||||
:ok
|
||||
rescue
|
||||
e ->
|
||||
Logger.warning("Weather: ScalarFile.write failed valid_time=#{valid_time} #{inspect(e)}")
|
||||
:ok
|
||||
end
|
||||
|
||||
@doc """
|
||||
Materialize the `ScalarFile` for `valid_time` from the on-disk
|
||||
`ProfilesFile`. Idempotent — if a scalar file already exists, returns
|
||||
`:ok` without re-deriving. Used by `NotifyListener` to pre-warm scalar
|
||||
artifacts the moment the Rust propagation pipeline fires
|
||||
`propagation_ready`, so the first `/weather` reader of a new forecast
|
||||
hour never falls back to the slow `ProfilesFile.read/1` + per-cell
|
||||
derive path.
|
||||
|
||||
Synchronous; callers should run this inside a `Task` if they need
|
||||
not to block (the listener does).
|
||||
"""
|
||||
@spec materialize_scalar_file(DateTime.t()) :: :ok
|
||||
def materialize_scalar_file(%DateTime{} = valid_time) do
|
||||
if ScalarFile.exists?(valid_time) do
|
||||
:ok
|
||||
else
|
||||
try do
|
||||
rows = load_grid_rows_for(valid_time)
|
||||
persist_scalar_file(valid_time, rows)
|
||||
Logger.info("Weather.materialize_scalar_file vt=#{valid_time} rows=#{length(rows)}")
|
||||
:ok
|
||||
rescue
|
||||
e ->
|
||||
Logger.warning("Weather.materialize_scalar_file failed vt=#{valid_time} #{inspect(e)}")
|
||||
:ok
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Build derived weather grid cache rows directly from an in-memory HRRR
|
||||
`grid_data` map. Used by the cold-cache fill path after `ProfilesFile.read/1`
|
||||
|
|
@ -1126,16 +1232,24 @@ defmodule Microwaveprop.Weather do
|
|||
snapped_lon = Float.round(Float.round(lon / step) * step, 3)
|
||||
|
||||
case GridCache.fetch_point(valid_time, snapped_lat, snapped_lon) do
|
||||
{:ok, row} ->
|
||||
row
|
||||
{:ok, row} -> row
|
||||
:miss -> point_detail_off_cache(valid_time, snapped_lat, snapped_lon)
|
||||
end
|
||||
end
|
||||
|
||||
:miss ->
|
||||
defp point_detail_off_cache(valid_time, snapped_lat, snapped_lon) do
|
||||
case ScalarFile.read_point(valid_time, snapped_lat, snapped_lon) do
|
||||
{:ok, row} -> row
|
||||
:miss -> point_detail_from_disk(valid_time, snapped_lat, snapped_lon)
|
||||
end
|
||||
end
|
||||
|
||||
defp point_detail_from_disk(valid_time, snapped_lat, snapped_lon) do
|
||||
case weather_point_detail_from_profiles(valid_time, snapped_lat, snapped_lon) do
|
||||
nil -> weather_point_detail_from_db(valid_time, snapped_lat, snapped_lon)
|
||||
row -> row
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Derive a single GridCache-shaped row from a persisted ProfilesFile
|
||||
# entry for `(valid_time, lat, lon)`. Returns nil when the file
|
||||
|
|
|
|||
162
lib/microwaveprop/weather/map_layers.ex
Normal file
162
lib/microwaveprop/weather/map_layers.ex
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
defmodule Microwaveprop.Weather.MapLayers do
|
||||
@moduledoc false
|
||||
|
||||
@layers [
|
||||
%{
|
||||
id: "temperature",
|
||||
label: "Temperature",
|
||||
unit: "°C",
|
||||
group: "Surface",
|
||||
desc: "2m air temperature. Warm, humid air increases refractivity and ducting potential at lower frequencies."
|
||||
},
|
||||
%{
|
||||
id: "dewpoint_depression",
|
||||
label: "Td Depression",
|
||||
unit: "°C",
|
||||
group: "Surface",
|
||||
desc:
|
||||
"Temperature minus dewpoint. Small values (< 5°C) mean moist boundary layer — favorable for ducting at 10 GHz, but increases absorption above 24 GHz."
|
||||
},
|
||||
%{
|
||||
id: "surface_rh",
|
||||
label: "Humidity",
|
||||
unit: "%",
|
||||
group: "Surface",
|
||||
desc:
|
||||
"Relative humidity from surface T and Td. High RH supports refractivity gradients that bend microwave signals."
|
||||
},
|
||||
%{
|
||||
id: "pwat",
|
||||
label: "PWAT",
|
||||
unit: "mm",
|
||||
group: "Surface",
|
||||
desc:
|
||||
"Precipitable water — total moisture in the atmospheric column. High values signal rain fade risk for 24 GHz and above."
|
||||
},
|
||||
%{
|
||||
id: "surface_refractivity",
|
||||
label: "Refractivity",
|
||||
unit: "N",
|
||||
group: "Surface",
|
||||
desc:
|
||||
"Radio refractivity (N-units) at the surface. Typical values 280–380. Higher N means the atmosphere bends signals more toward the ground."
|
||||
},
|
||||
%{
|
||||
id: "refractivity_gradient",
|
||||
label: "N-Gradient",
|
||||
unit: "N/km",
|
||||
group: "Surface",
|
||||
desc:
|
||||
"Minimum refractivity gradient in the profile. Standard is −40 N/km. Below −157 N/km signals are trapped in a duct. More negative = stronger ducting."
|
||||
},
|
||||
%{
|
||||
id: "bl_height",
|
||||
label: "BL Height",
|
||||
unit: "m",
|
||||
group: "Surface",
|
||||
desc:
|
||||
"Planetary boundary layer height. Shallow BL (< 500m) concentrates moisture and heat near the surface, favoring temperature inversions and ducting."
|
||||
},
|
||||
%{
|
||||
id: "temp_850mb",
|
||||
label: "T @ 850mb",
|
||||
unit: "°C",
|
||||
group: "Upper Air",
|
||||
desc:
|
||||
"Temperature at 850 mb (~1500m altitude). Warm 850mb air over cool surface air indicates a capping inversion — a classic ducting setup."
|
||||
},
|
||||
%{
|
||||
id: "dewpoint_850mb",
|
||||
label: "Td @ 850mb",
|
||||
unit: "°C",
|
||||
group: "Upper Air",
|
||||
desc:
|
||||
"Dewpoint at 850 mb. A sharp moisture drop between the surface and 850mb creates an elevated refractivity gradient that can form ducts."
|
||||
},
|
||||
%{
|
||||
id: "temp_700mb",
|
||||
label: "T @ 700mb",
|
||||
unit: "°C",
|
||||
group: "Upper Air",
|
||||
desc:
|
||||
"Temperature at 700 mb (~3000m altitude). The classic capping diagnostic over the southern Plains: ≥10°C indicates a moderate cap, ≥12°C strong. Combined with warm 850 mb, signals a 'loaded gun' setup that suppresses convection until forcing arrives."
|
||||
},
|
||||
%{
|
||||
id: "dewpoint_700mb",
|
||||
label: "Td @ 700mb",
|
||||
unit: "°C",
|
||||
group: "Upper Air",
|
||||
desc:
|
||||
"Dewpoint at 700 mb. Wide depressions (T-Td > 10°C) signal the elevated mixed layer (EML) plume — a hallmark cap mechanism off the Mexican plateau."
|
||||
},
|
||||
%{
|
||||
id: "lapse_rate",
|
||||
label: "Lapse Rate",
|
||||
unit: "°C/km",
|
||||
group: "Upper Air",
|
||||
desc:
|
||||
"Temperature decrease per km from surface to 700mb. Low rates (< 5 °C/km) mean stable air that preserves inversions. High rates (> 8) mean convective mixing that destroys them."
|
||||
},
|
||||
%{
|
||||
id: "mid_lapse_rate",
|
||||
label: "Mid Lapse 850-700",
|
||||
unit: "°C/km",
|
||||
group: "Upper Air",
|
||||
desc:
|
||||
"Lapse rate across the 850→700 mb layer. Steep values (≥7 °C/km) above a warm moist boundary layer indicate an elevated mixed layer — the textbook cap structure that holds back convection until the inversion is broken."
|
||||
},
|
||||
%{
|
||||
id: "inversion_strength",
|
||||
label: "Inversion",
|
||||
unit: "°C",
|
||||
group: "Upper Air",
|
||||
desc:
|
||||
"Strongest temperature increase between adjacent levels. Inversions trap microwave signals. > 3°C is significant, > 5°C is strong."
|
||||
},
|
||||
%{
|
||||
id: "inversion_base_m",
|
||||
label: "Inv. Base",
|
||||
unit: "m",
|
||||
group: "Upper Air",
|
||||
desc:
|
||||
"Height (m AGL) where the strongest inversion begins. Surface-based inversions (< 200m) create surface ducts. Elevated inversions create elevated ducts."
|
||||
},
|
||||
%{
|
||||
id: "ducting",
|
||||
label: "Ducting",
|
||||
unit: "",
|
||||
group: "Ducting",
|
||||
desc:
|
||||
"Whether a trapping layer (modified refractivity decreasing with height) was detected in the profile. Green = duct present, signals can travel far beyond line of sight."
|
||||
},
|
||||
%{
|
||||
id: "duct_base_m",
|
||||
label: "Duct Base",
|
||||
unit: "m",
|
||||
group: "Ducting",
|
||||
desc:
|
||||
"Height of the lowest duct base. Surface ducts (< 100m) are most effective for ground-based stations. Elevated ducts require antennas near the duct height."
|
||||
},
|
||||
%{
|
||||
id: "duct_strength",
|
||||
label: "Duct Strength",
|
||||
unit: "M",
|
||||
group: "Ducting",
|
||||
desc:
|
||||
"Duct trapping strength in modified refractivity (M) units. > 10 M is moderate, > 20 M is strong. Stronger ducts trap a wider range of frequencies."
|
||||
}
|
||||
]
|
||||
|
||||
@default_layer "temperature"
|
||||
@valid_ids MapSet.new(Enum.map(@layers, & &1.id))
|
||||
|
||||
@spec all() :: [map()]
|
||||
def all, do: @layers
|
||||
|
||||
@spec default_id() :: String.t()
|
||||
def default_id, do: @default_layer
|
||||
|
||||
@spec valid_id?(term()) :: boolean()
|
||||
def valid_id?(id) when is_binary(id), do: MapSet.member?(@valid_ids, id)
|
||||
def valid_id?(_id), do: false
|
||||
end
|
||||
304
lib/microwaveprop/weather/scalar_file.ex
Normal file
304
lib/microwaveprop/weather/scalar_file.ex
Normal file
|
|
@ -0,0 +1,304 @@
|
|||
defmodule Microwaveprop.Weather.ScalarFile do
|
||||
@moduledoc """
|
||||
On-disk store for the per-cell *derived* weather rows that feed `/weather`.
|
||||
One directory per `valid_time`, with rows bucketed into 5°×5° chunk files
|
||||
so viewport reads only decode the chunks that overlap the requested bounds.
|
||||
|
||||
This is the cheap-read sibling of `Microwaveprop.Propagation.ProfilesFile`:
|
||||
|
||||
* `ProfilesFile` keeps the raw HRRR profile per cell (~10 MB decoded) and
|
||||
is the source of truth for advanced diagnostics, terrain analysis, and
|
||||
point-detail breakdowns.
|
||||
* `ScalarFile` keeps only the scalar fields the weather map renders
|
||||
(temperature, dewpoint depression, refractivity, lapse rates, duct
|
||||
summary, etc.), already pushed through `WeatherLayers.derive/1`.
|
||||
|
||||
Because the scalar shape is small and the derivation has already happened,
|
||||
a tile request can read just the chunks it needs and skip the
|
||||
`SoundingParams.derive/1` + `WeatherLayers.derive/1` work entirely.
|
||||
|
||||
## Layout
|
||||
|
||||
<base_dir>/weather_scalars/
|
||||
<iso>/ # e.g. 2026-04-28T12:00:00Z/
|
||||
<lat_band>_<lon_band>.etf.gz
|
||||
|
||||
`lat_band = floor(lat / 5)`, `lon_band = floor(lon / 5)`. Each chunk is
|
||||
the compressed ETF of `[row_map, ...]`. Writes go through
|
||||
`rename(2)` → atomic on NFS.
|
||||
|
||||
## Chunk size
|
||||
|
||||
5°×5° was picked so the typical /weather viewport (a US state at z=6-7)
|
||||
overlaps 1-4 chunks. Smaller chunks would balloon directory entries on
|
||||
NFS without measurable read benefit.
|
||||
"""
|
||||
|
||||
require Logger
|
||||
|
||||
@chunk_step 5
|
||||
@subdir "weather_scalars"
|
||||
|
||||
@type row :: %{required(:lat) => float(), required(:lon) => float(), optional(atom()) => term()}
|
||||
@type bounds :: %{optional(String.t()) => number()}
|
||||
|
||||
@spec base_dir() :: String.t()
|
||||
def base_dir do
|
||||
Path.join(
|
||||
Application.get_env(:microwaveprop, :propagation_scores_dir, "/data/scores"),
|
||||
@subdir
|
||||
)
|
||||
end
|
||||
|
||||
@spec dir_for(DateTime.t()) :: String.t()
|
||||
def dir_for(%DateTime{} = valid_time) do
|
||||
Path.join(base_dir(), iso_key(valid_time))
|
||||
end
|
||||
|
||||
@spec exists?(DateTime.t()) :: boolean()
|
||||
def exists?(%DateTime{} = valid_time) do
|
||||
case File.ls(dir_for(valid_time)) do
|
||||
{:ok, [_ | _]} -> true
|
||||
_ -> false
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Persist `rows` for `valid_time`. Rows are bucketed by 5°×5° chunk; each
|
||||
chunk is written atomically via tmp + rename. Existing chunks for the
|
||||
same `valid_time` are removed first so a smaller follow-up write doesn't
|
||||
leave stale chunks behind.
|
||||
"""
|
||||
@spec write!(DateTime.t(), [row()]) :: :ok
|
||||
def write!(%DateTime{} = valid_time, rows) when is_list(rows) do
|
||||
dir = dir_for(valid_time)
|
||||
File.mkdir_p!(dir)
|
||||
|
||||
# Drop any pre-existing chunks for this valid_time so this write is
|
||||
# the canonical state — otherwise a smaller follow-up write would
|
||||
# leave stale chunks behind.
|
||||
case File.ls(dir) do
|
||||
{:ok, entries} ->
|
||||
for entry <- entries do
|
||||
_ = File.rm(Path.join(dir, entry))
|
||||
end
|
||||
|
||||
_ ->
|
||||
:ok
|
||||
end
|
||||
|
||||
rows
|
||||
|> Enum.group_by(&chunk_key/1)
|
||||
|> Enum.each(fn {{lat_band, lon_band}, chunk_rows} ->
|
||||
path = Path.join(dir, "#{lat_band}_#{lon_band}.etf.gz")
|
||||
binary = :erlang.term_to_binary(chunk_rows, [:compressed])
|
||||
|
||||
tmp = path <> ".tmp." <> unique_suffix()
|
||||
File.write!(tmp, binary, [:binary])
|
||||
File.rename!(tmp, path)
|
||||
end)
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
@doc """
|
||||
Read every persisted row for `valid_time` whose lat/lon falls within
|
||||
`bounds`. Pass `nil` to read every chunk. Returns `[]` if no scalar
|
||||
file exists for `valid_time`.
|
||||
"""
|
||||
@spec read_bounds(DateTime.t(), bounds() | nil) :: [row()]
|
||||
def read_bounds(%DateTime{} = valid_time, bounds) do
|
||||
case list_chunk_files(valid_time) do
|
||||
[] ->
|
||||
[]
|
||||
|
||||
files ->
|
||||
files
|
||||
|> Enum.filter(fn {key, _path} -> chunk_intersects_bounds?(key, bounds) end)
|
||||
|> Enum.flat_map(fn {_key, path} -> decode_chunk(path) end)
|
||||
|> filter_bounds(bounds)
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Look up a single derived row at `(valid_time, lat, lon)`. Returns
|
||||
`{:ok, row}` or `:miss` when either the chunk file is absent or the
|
||||
cell isn't present.
|
||||
"""
|
||||
@spec read_point(DateTime.t(), float(), float()) :: {:ok, row()} | :miss
|
||||
def read_point(%DateTime{} = valid_time, lat, lon) when is_number(lat) and is_number(lon) do
|
||||
key = {chunk_band(lat * 1.0), chunk_band(lon * 1.0)}
|
||||
path = Path.join(dir_for(valid_time), chunk_filename(key))
|
||||
|
||||
if File.exists?(path) do
|
||||
find_point_in_chunk(path, snap(lat), snap(lon))
|
||||
else
|
||||
:miss
|
||||
end
|
||||
end
|
||||
|
||||
defp find_point_in_chunk(path, snapped_lat, snapped_lon) do
|
||||
case Enum.find(decode_chunk(path), fn r ->
|
||||
r.lat == snapped_lat and r.lon == snapped_lon
|
||||
end) do
|
||||
nil -> :miss
|
||||
row -> {:ok, row}
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Every persisted valid_time, sorted ascending. Used to back the /weather
|
||||
forecast timeline once scalars take over from raw profiles as the
|
||||
primary read.
|
||||
"""
|
||||
@spec list_valid_times() :: [DateTime.t()]
|
||||
def list_valid_times do
|
||||
case File.ls(base_dir()) do
|
||||
{:ok, entries} ->
|
||||
times =
|
||||
for entry <- entries,
|
||||
{:ok, dt, _} <- [DateTime.from_iso8601(entry)],
|
||||
do: dt
|
||||
|
||||
Enum.sort(times, DateTime)
|
||||
|
||||
_ ->
|
||||
[]
|
||||
end
|
||||
end
|
||||
|
||||
@doc "Delete scalar dirs whose valid_time is strictly before `cutoff`. Returns count removed."
|
||||
@spec prune_older_than(DateTime.t()) :: non_neg_integer()
|
||||
def prune_older_than(%DateTime{} = cutoff) do
|
||||
cutoff_unix = DateTime.to_unix(cutoff)
|
||||
|
||||
base_dir()
|
||||
|> list_valid_time_dirs()
|
||||
|> Enum.reduce(0, fn {path, dt}, acc ->
|
||||
if DateTime.to_unix(dt) < cutoff_unix do
|
||||
_ = File.rm_rf(path)
|
||||
acc + 1
|
||||
else
|
||||
acc
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Keep only scalar dirs whose valid_time is inside the closed window
|
||||
`[run_time, run_time + max_forecast_hour * 3600]`. Mirrors
|
||||
`ProfilesFile.retain_window/2`.
|
||||
"""
|
||||
@spec retain_window(DateTime.t(), non_neg_integer()) :: non_neg_integer()
|
||||
def retain_window(%DateTime{} = run_time, max_forecast_hour) when max_forecast_hour >= 0 do
|
||||
lo = DateTime.to_unix(run_time)
|
||||
hi = lo + max_forecast_hour * 3600
|
||||
|
||||
base_dir()
|
||||
|> list_valid_time_dirs()
|
||||
|> Enum.reduce(0, fn {path, dt}, acc ->
|
||||
unix = DateTime.to_unix(dt)
|
||||
|
||||
if unix < lo or unix > hi do
|
||||
_ = File.rm_rf(path)
|
||||
acc + 1
|
||||
else
|
||||
acc
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
# ---------- Internal ----------
|
||||
|
||||
defp iso_key(%DateTime{} = valid_time) do
|
||||
valid_time |> DateTime.truncate(:second) |> DateTime.to_iso8601()
|
||||
end
|
||||
|
||||
defp chunk_key(%{lat: lat, lon: lon}) do
|
||||
{chunk_band(lat * 1.0), chunk_band(lon * 1.0)}
|
||||
end
|
||||
|
||||
defp chunk_band(value) when is_float(value) do
|
||||
(value / @chunk_step) |> Float.floor() |> trunc()
|
||||
end
|
||||
|
||||
defp chunk_filename({lat_band, lon_band}), do: "#{lat_band}_#{lon_band}.etf.gz"
|
||||
|
||||
defp chunk_intersects_bounds?(_key, nil), do: true
|
||||
|
||||
defp chunk_intersects_bounds?({lat_band, lon_band}, %{"south" => s, "north" => n, "west" => w, "east" => e}) do
|
||||
chunk_south = lat_band * @chunk_step
|
||||
chunk_north = chunk_south + @chunk_step
|
||||
chunk_west = lon_band * @chunk_step
|
||||
chunk_east = chunk_west + @chunk_step
|
||||
|
||||
chunk_north >= s and chunk_south <= n and chunk_east >= w and chunk_west <= e
|
||||
end
|
||||
|
||||
defp filter_bounds(rows, nil), do: rows
|
||||
|
||||
defp filter_bounds(rows, %{"south" => s, "north" => n, "west" => w, "east" => e}) do
|
||||
Enum.filter(rows, fn %{lat: lat, lon: lon} ->
|
||||
lat >= s and lat <= n and lon >= w and lon <= e
|
||||
end)
|
||||
end
|
||||
|
||||
defp decode_chunk(path) do
|
||||
case File.read(path) do
|
||||
{:ok, binary} ->
|
||||
:erlang.binary_to_term(binary)
|
||||
|
||||
{:error, reason} ->
|
||||
Logger.warning("ScalarFile chunk read failed path=#{path} reason=#{inspect(reason)}")
|
||||
[]
|
||||
end
|
||||
end
|
||||
|
||||
defp list_chunk_files(%DateTime{} = valid_time) do
|
||||
dir = dir_for(valid_time)
|
||||
|
||||
case File.ls(dir) do
|
||||
{:ok, entries} ->
|
||||
for entry <- entries,
|
||||
key = parse_chunk_filename(entry),
|
||||
key != nil,
|
||||
do: {key, Path.join(dir, entry)}
|
||||
|
||||
_ ->
|
||||
[]
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_chunk_filename(filename) do
|
||||
with [_, lat_str, lon_str] <- Regex.run(~r/^(-?\d+)_(-?\d+)\.etf\.gz$/, filename),
|
||||
{lat_band, ""} <- Integer.parse(lat_str),
|
||||
{lon_band, ""} <- Integer.parse(lon_str) do
|
||||
{lat_band, lon_band}
|
||||
else
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp list_valid_time_dirs(base) do
|
||||
case File.ls(base) do
|
||||
{:ok, entries} ->
|
||||
for entry <- entries,
|
||||
{:ok, dt, _} <- [DateTime.from_iso8601(entry)],
|
||||
do: {Path.join(base, entry), dt}
|
||||
|
||||
_ ->
|
||||
[]
|
||||
end
|
||||
end
|
||||
|
||||
# Match ProfilesFile's snap step (0.125°) so a click at any lat/lon
|
||||
# rounds to the same key the writer used.
|
||||
defp snap(value) do
|
||||
step = 0.125
|
||||
Float.round(Float.round(value / step) * step, 3)
|
||||
end
|
||||
|
||||
defp unique_suffix do
|
||||
"#{System.system_time(:nanosecond)}.#{:erlang.unique_integer([:positive])}"
|
||||
end
|
||||
end
|
||||
290
lib/microwaveprop/weather/tile_renderer.ex
Normal file
290
lib/microwaveprop/weather/tile_renderer.ex
Normal file
|
|
@ -0,0 +1,290 @@
|
|||
defmodule Microwaveprop.Weather.TileRenderer do
|
||||
@moduledoc false
|
||||
|
||||
alias Microwaveprop.Weather
|
||||
|
||||
@tile_size 256
|
||||
@grid_step 0.125
|
||||
@half_step @grid_step / 2.0
|
||||
@overlay_alpha 0.55
|
||||
|
||||
@continuous_scales %{
|
||||
"refractivity_gradient" => [
|
||||
{-500, {0, 255, 163}},
|
||||
{-300, {125, 255, 212}},
|
||||
{-200, {255, 229, 102}},
|
||||
{-100, {255, 144, 68}},
|
||||
{0, {255, 79, 79}}
|
||||
],
|
||||
"bl_height" => [
|
||||
{0, {0, 255, 163}},
|
||||
{500, {125, 255, 212}},
|
||||
{1000, {255, 229, 102}},
|
||||
{2000, {255, 144, 68}},
|
||||
{3000, {255, 79, 79}}
|
||||
],
|
||||
"dewpoint_depression" => [
|
||||
{0, {0, 255, 163}},
|
||||
{5, {125, 255, 212}},
|
||||
{10, {255, 229, 102}},
|
||||
{20, {255, 144, 68}},
|
||||
{30, {255, 79, 79}}
|
||||
],
|
||||
"pwat" => [
|
||||
{0, {139, 90, 43}},
|
||||
{15, {194, 165, 116}},
|
||||
{30, {173, 216, 230}},
|
||||
{45, {65, 105, 225}},
|
||||
{60, {0, 0, 180}}
|
||||
],
|
||||
"temperature" => [
|
||||
{-20, {0, 0, 200}},
|
||||
{0, {100, 149, 237}},
|
||||
{15, {255, 255, 150}},
|
||||
{30, {255, 140, 0}},
|
||||
{45, {200, 0, 0}}
|
||||
],
|
||||
"surface_rh" => [
|
||||
{0, {255, 144, 68}},
|
||||
{30, {255, 229, 102}},
|
||||
{60, {125, 255, 212}},
|
||||
{80, {0, 255, 163}},
|
||||
{100, {0, 200, 130}}
|
||||
],
|
||||
"surface_refractivity" => [
|
||||
{250, {255, 79, 79}},
|
||||
{300, {255, 144, 68}},
|
||||
{330, {255, 229, 102}},
|
||||
{360, {125, 255, 212}},
|
||||
{400, {0, 255, 163}}
|
||||
],
|
||||
"temp_850mb" => [
|
||||
{-20, {0, 0, 200}},
|
||||
{-5, {100, 149, 237}},
|
||||
{5, {255, 255, 150}},
|
||||
{15, {255, 140, 0}},
|
||||
{30, {200, 0, 0}}
|
||||
],
|
||||
"dewpoint_850mb" => [
|
||||
{-20, {139, 90, 43}},
|
||||
{-5, {194, 165, 116}},
|
||||
{5, {173, 216, 230}},
|
||||
{15, {65, 105, 225}},
|
||||
{25, {0, 0, 180}}
|
||||
],
|
||||
"temp_700mb" => [
|
||||
{-10, {0, 100, 200}},
|
||||
{0, {100, 180, 230}},
|
||||
{5, {200, 230, 200}},
|
||||
{8, {255, 229, 102}},
|
||||
{10, {255, 144, 68}},
|
||||
{14, {200, 0, 0}}
|
||||
],
|
||||
"dewpoint_700mb" => [
|
||||
{-30, {139, 90, 43}},
|
||||
{-15, {194, 165, 116}},
|
||||
{-5, {173, 216, 230}},
|
||||
{5, {65, 105, 225}},
|
||||
{15, {0, 0, 180}}
|
||||
],
|
||||
"lapse_rate" => [
|
||||
{3, {0, 255, 163}},
|
||||
{5, {125, 255, 212}},
|
||||
{6.5, {255, 229, 102}},
|
||||
{8, {255, 144, 68}},
|
||||
{9.5, {255, 79, 79}}
|
||||
],
|
||||
"mid_lapse_rate" => [
|
||||
{3, {0, 255, 163}},
|
||||
{5, {125, 255, 212}},
|
||||
{6.5, {255, 229, 102}},
|
||||
{7.5, {255, 144, 68}},
|
||||
{9, {255, 79, 79}}
|
||||
],
|
||||
"inversion_strength" => [
|
||||
{0, {255, 229, 102}},
|
||||
{1, {255, 200, 80}},
|
||||
{3, {255, 144, 68}},
|
||||
{5, {255, 79, 79}},
|
||||
{8, {180, 0, 0}}
|
||||
],
|
||||
"inversion_base_m" => [
|
||||
{0, {0, 255, 163}},
|
||||
{100, {125, 255, 212}},
|
||||
{300, {255, 229, 102}},
|
||||
{800, {255, 144, 68}},
|
||||
{1500, {255, 79, 79}}
|
||||
],
|
||||
"duct_base_m" => [
|
||||
{0, {0, 255, 163}},
|
||||
{100, {125, 255, 212}},
|
||||
{300, {255, 229, 102}},
|
||||
{800, {255, 144, 68}},
|
||||
{1500, {255, 79, 79}}
|
||||
],
|
||||
"duct_strength" => [
|
||||
{0, {125, 255, 212}},
|
||||
{5, {255, 229, 102}},
|
||||
{10, {255, 144, 68}},
|
||||
{20, {255, 79, 79}},
|
||||
{30, {180, 0, 0}}
|
||||
],
|
||||
"surface_pressure_mb" => [
|
||||
{980, {255, 79, 79}},
|
||||
{1000, {255, 144, 68}},
|
||||
{1015, {255, 229, 102}},
|
||||
{1030, {125, 255, 212}},
|
||||
{1045, {0, 255, 163}}
|
||||
]
|
||||
}
|
||||
|
||||
@boolean_scales %{
|
||||
"ducting" => {0, 255, 163}
|
||||
}
|
||||
@layer_fields Map.new(Map.keys(@continuous_scales) ++ Map.keys(@boolean_scales), &{&1, String.to_atom(&1)})
|
||||
|
||||
@spec render_svg(DateTime.t(), String.t(), non_neg_integer(), non_neg_integer(), non_neg_integer()) ::
|
||||
binary()
|
||||
def render_svg(%DateTime{} = valid_time, layer_id, z, x, y) do
|
||||
bounds =
|
||||
z
|
||||
|> tile_bounds(x, y)
|
||||
|> pad_bounds(@half_step)
|
||||
|
||||
rows = Weather.weather_grid_at(valid_time, bounds)
|
||||
|
||||
IO.iodata_to_binary([
|
||||
~s(<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 #{@tile_size} #{@tile_size}" shape-rendering="crispEdges">),
|
||||
render_rows(rows, layer_id, z, x, y),
|
||||
"</svg>"
|
||||
])
|
||||
end
|
||||
|
||||
@spec empty_svg() :: binary()
|
||||
def empty_svg do
|
||||
~s(<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 #{@tile_size} #{@tile_size}"></svg>)
|
||||
end
|
||||
|
||||
defp render_rows(rows, layer_id, z, x, y) do
|
||||
Enum.map(rows, fn row -> render_row(row, layer_id, z, x, y) end)
|
||||
end
|
||||
|
||||
defp render_row(row, layer_id, z, x, y) do
|
||||
with {:ok, {fill, alpha}} <- color_for_layer(row, layer_id),
|
||||
{:ok, {px, py, width, height}} <- rect_for_row(row, z, x, y) do
|
||||
~s(<rect x="#{fmt(px)}" y="#{fmt(py)}" width="#{fmt(width)}" height="#{fmt(height)}" fill="#{fill}" fill-opacity="#{alpha}" />)
|
||||
else
|
||||
_ -> []
|
||||
end
|
||||
end
|
||||
|
||||
defp color_for_layer(row, layer_id) do
|
||||
field = Map.get(@layer_fields, layer_id)
|
||||
|
||||
case Map.fetch(@boolean_scales, layer_id) do
|
||||
{:ok, {r, g, b}} ->
|
||||
if row[field] do
|
||||
{:ok, {"rgb(#{r},#{g},#{b})", fmt(@overlay_alpha)}}
|
||||
else
|
||||
:skip
|
||||
end
|
||||
|
||||
:error ->
|
||||
with value when is_number(value) <- row[field],
|
||||
scale when is_list(scale) <- Map.get(@continuous_scales, layer_id),
|
||||
{r, g, b} <- interpolate(value, scale) do
|
||||
{:ok, {"rgb(#{r},#{g},#{b})", fmt(@overlay_alpha)}}
|
||||
else
|
||||
_ -> :skip
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
defp interpolate(value, [{min_value, min_color} | _]) when value <= min_value, do: min_color
|
||||
|
||||
defp interpolate(value, scale) do
|
||||
max_value = scale |> List.last() |> elem(0)
|
||||
|
||||
if value >= max_value do
|
||||
scale |> List.last() |> elem(1)
|
||||
else
|
||||
scale
|
||||
|> Enum.chunk_every(2, 1, :discard)
|
||||
|> Enum.find_value(&interpolate_pair(&1, value))
|
||||
end
|
||||
end
|
||||
|
||||
defp interpolate_pair([{v1, c1}, {v2, c2}], value) when value >= v1 and value <= v2 do
|
||||
t = (value - v1) / (v2 - v1)
|
||||
{lerp(elem(c1, 0), elem(c2, 0), t), lerp(elem(c1, 1), elem(c2, 1), t), lerp(elem(c1, 2), elem(c2, 2), t)}
|
||||
end
|
||||
|
||||
defp interpolate_pair(_pair, _value), do: nil
|
||||
|
||||
defp lerp(a, b, t), do: round(a + (b - a) * t)
|
||||
|
||||
defp rect_for_row(%{lat: lat, lon: lon}, z, tile_x, tile_y) do
|
||||
x1 = world_x(lon - @half_step, z) - tile_x * @tile_size
|
||||
x2 = world_x(lon + @half_step, z) - tile_x * @tile_size
|
||||
y1 = world_y(lat + @half_step, z) - tile_y * @tile_size
|
||||
y2 = world_y(lat - @half_step, z) - tile_y * @tile_size
|
||||
|
||||
left = max(min(x1, x2), 0.0)
|
||||
right = min(max(x1, x2), @tile_size * 1.0)
|
||||
top = max(min(y1, y2), 0.0)
|
||||
bottom = min(max(y1, y2), @tile_size * 1.0)
|
||||
|
||||
width = right - left
|
||||
height = bottom - top
|
||||
|
||||
if width <= 0.0 or height <= 0.0 do
|
||||
:skip
|
||||
else
|
||||
{:ok, {left, top, width, height}}
|
||||
end
|
||||
end
|
||||
|
||||
defp tile_bounds(z, x, y) do
|
||||
%{
|
||||
"west" => tile_to_lon(x, z),
|
||||
"east" => tile_to_lon(x + 1, z),
|
||||
"north" => tile_to_lat(y, z),
|
||||
"south" => tile_to_lat(y + 1, z)
|
||||
}
|
||||
end
|
||||
|
||||
defp pad_bounds(%{"west" => west, "east" => east, "north" => north, "south" => south}, pad) do
|
||||
%{
|
||||
"west" => west - pad,
|
||||
"east" => east + pad,
|
||||
"north" => min(north + pad, 85.0511),
|
||||
"south" => max(south - pad, -85.0511)
|
||||
}
|
||||
end
|
||||
|
||||
defp tile_to_lon(x, z), do: x / :math.pow(2, z) * 360.0 - 180.0
|
||||
|
||||
defp tile_to_lat(y, z) do
|
||||
n = :math.pi() - 2.0 * :math.pi() * y / :math.pow(2, z)
|
||||
:math.atan(:math.sinh(n)) * 180.0 / :math.pi()
|
||||
end
|
||||
|
||||
defp world_x(lon, z) do
|
||||
(lon + 180.0) / 360.0 * @tile_size * :math.pow(2, z)
|
||||
end
|
||||
|
||||
defp world_y(lat, z) do
|
||||
lat =
|
||||
lat
|
||||
|> min(85.0511)
|
||||
|> max(-85.0511)
|
||||
|
||||
lat_rad = lat * :math.pi() / 180.0
|
||||
|
||||
(1.0 - :math.log(:math.tan(lat_rad) + 1.0 / :math.cos(lat_rad)) / :math.pi()) / 2.0 *
|
||||
@tile_size * :math.pow(2, z)
|
||||
end
|
||||
|
||||
defp fmt(value) when is_integer(value), do: Integer.to_string(value)
|
||||
defp fmt(value) when is_float(value), do: :erlang.float_to_binary(value, decimals: 2)
|
||||
end
|
||||
28
lib/microwaveprop_web/controllers/weather_tile_controller.ex
Normal file
28
lib/microwaveprop_web/controllers/weather_tile_controller.ex
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
defmodule MicrowavepropWeb.WeatherTileController do
|
||||
use MicrowavepropWeb, :controller
|
||||
|
||||
alias Microwaveprop.Weather.MapLayers
|
||||
alias Microwaveprop.Weather.TileRenderer
|
||||
|
||||
@spec show(Plug.Conn.t(), map()) :: Plug.Conn.t()
|
||||
def show(conn, %{"z" => z, "x" => x, "y" => y, "layer" => layer_id, "time" => time_param}) do
|
||||
with true <- MapLayers.valid_id?(layer_id),
|
||||
{z, ""} <- Integer.parse(z),
|
||||
{x, ""} <- Integer.parse(x),
|
||||
{y, ""} <- Integer.parse(y),
|
||||
{:ok, valid_time, _offset} <- DateTime.from_iso8601(time_param) do
|
||||
body = TileRenderer.render_svg(valid_time, layer_id, z, x, y)
|
||||
|
||||
conn
|
||||
|> put_resp_content_type("image/svg+xml")
|
||||
|> put_resp_header("cache-control", "public, max-age=60")
|
||||
|> send_resp(200, body)
|
||||
else
|
||||
_ ->
|
||||
conn
|
||||
|> put_resp_content_type("image/svg+xml")
|
||||
|> put_resp_header("cache-control", "public, max-age=5")
|
||||
|> send_resp(200, TileRenderer.empty_svg())
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -3,165 +3,10 @@ defmodule MicrowavepropWeb.WeatherMapLive do
|
|||
use MicrowavepropWeb, :live_view
|
||||
|
||||
alias Microwaveprop.Weather
|
||||
alias Microwaveprop.Weather.MapLayers
|
||||
|
||||
@initial_bounds %{
|
||||
"south" => 29.5,
|
||||
"north" => 36.3,
|
||||
"west" => -101.5,
|
||||
"east" => -92.5
|
||||
}
|
||||
|
||||
@layers [
|
||||
# Surface
|
||||
%{
|
||||
id: "temperature",
|
||||
label: "Temperature",
|
||||
unit: "°C",
|
||||
group: "Surface",
|
||||
desc: "2m air temperature. Warm, humid air increases refractivity and ducting potential at lower frequencies."
|
||||
},
|
||||
%{
|
||||
id: "dewpoint_depression",
|
||||
label: "Td Depression",
|
||||
unit: "°C",
|
||||
group: "Surface",
|
||||
desc:
|
||||
"Temperature minus dewpoint. Small values (< 5°C) mean moist boundary layer — favorable for ducting at 10 GHz, but increases absorption above 24 GHz."
|
||||
},
|
||||
%{
|
||||
id: "surface_rh",
|
||||
label: "Humidity",
|
||||
unit: "%",
|
||||
group: "Surface",
|
||||
desc:
|
||||
"Relative humidity from surface T and Td. High RH supports refractivity gradients that bend microwave signals."
|
||||
},
|
||||
%{
|
||||
id: "pwat",
|
||||
label: "PWAT",
|
||||
unit: "mm",
|
||||
group: "Surface",
|
||||
desc:
|
||||
"Precipitable water — total moisture in the atmospheric column. High values signal rain fade risk for 24 GHz and above."
|
||||
},
|
||||
%{
|
||||
id: "surface_refractivity",
|
||||
label: "Refractivity",
|
||||
unit: "N",
|
||||
group: "Surface",
|
||||
desc:
|
||||
"Radio refractivity (N-units) at the surface. Typical values 280–380. Higher N means the atmosphere bends signals more toward the ground."
|
||||
},
|
||||
%{
|
||||
id: "refractivity_gradient",
|
||||
label: "N-Gradient",
|
||||
unit: "N/km",
|
||||
group: "Surface",
|
||||
desc:
|
||||
"Minimum refractivity gradient in the profile. Standard is −40 N/km. Below −157 N/km signals are trapped in a duct. More negative = stronger ducting."
|
||||
},
|
||||
%{
|
||||
id: "bl_height",
|
||||
label: "BL Height",
|
||||
unit: "m",
|
||||
group: "Surface",
|
||||
desc:
|
||||
"Planetary boundary layer height. Shallow BL (< 500m) concentrates moisture and heat near the surface, favoring temperature inversions and ducting."
|
||||
},
|
||||
# Upper Air
|
||||
%{
|
||||
id: "temp_850mb",
|
||||
label: "T @ 850mb",
|
||||
unit: "°C",
|
||||
group: "Upper Air",
|
||||
desc:
|
||||
"Temperature at 850 mb (~1500m altitude). Warm 850mb air over cool surface air indicates a capping inversion — a classic ducting setup."
|
||||
},
|
||||
%{
|
||||
id: "dewpoint_850mb",
|
||||
label: "Td @ 850mb",
|
||||
unit: "°C",
|
||||
group: "Upper Air",
|
||||
desc:
|
||||
"Dewpoint at 850 mb. A sharp moisture drop between the surface and 850mb creates an elevated refractivity gradient that can form ducts."
|
||||
},
|
||||
%{
|
||||
id: "temp_700mb",
|
||||
label: "T @ 700mb",
|
||||
unit: "°C",
|
||||
group: "Upper Air",
|
||||
desc:
|
||||
"Temperature at 700 mb (~3000m altitude). The classic capping diagnostic over the southern Plains: ≥10°C indicates a moderate cap, ≥12°C strong. Combined with warm 850 mb, signals a 'loaded gun' setup that suppresses convection until forcing arrives."
|
||||
},
|
||||
%{
|
||||
id: "dewpoint_700mb",
|
||||
label: "Td @ 700mb",
|
||||
unit: "°C",
|
||||
group: "Upper Air",
|
||||
desc:
|
||||
"Dewpoint at 700 mb. Wide depressions (T-Td > 10°C) signal the elevated mixed layer (EML) plume — a hallmark cap mechanism off the Mexican plateau."
|
||||
},
|
||||
%{
|
||||
id: "lapse_rate",
|
||||
label: "Lapse Rate",
|
||||
unit: "°C/km",
|
||||
group: "Upper Air",
|
||||
desc:
|
||||
"Temperature decrease per km from surface to 700mb. Low rates (< 5 °C/km) mean stable air that preserves inversions. High rates (> 8) mean convective mixing that destroys them."
|
||||
},
|
||||
%{
|
||||
id: "mid_lapse_rate",
|
||||
label: "Mid Lapse 850-700",
|
||||
unit: "°C/km",
|
||||
group: "Upper Air",
|
||||
desc:
|
||||
"Lapse rate across the 850→700 mb layer. Steep values (≥7 °C/km) above a warm moist boundary layer indicate an elevated mixed layer — the textbook cap structure that holds back convection until the inversion is broken."
|
||||
},
|
||||
%{
|
||||
id: "inversion_strength",
|
||||
label: "Inversion",
|
||||
unit: "°C",
|
||||
group: "Upper Air",
|
||||
desc:
|
||||
"Strongest temperature increase between adjacent levels. Inversions trap microwave signals. > 3°C is significant, > 5°C is strong."
|
||||
},
|
||||
%{
|
||||
id: "inversion_base_m",
|
||||
label: "Inv. Base",
|
||||
unit: "m",
|
||||
group: "Upper Air",
|
||||
desc:
|
||||
"Height (m AGL) where the strongest inversion begins. Surface-based inversions (< 200m) create surface ducts. Elevated inversions create elevated ducts."
|
||||
},
|
||||
# Ducting
|
||||
%{
|
||||
id: "ducting",
|
||||
label: "Ducting",
|
||||
unit: "",
|
||||
group: "Ducting",
|
||||
desc:
|
||||
"Whether a trapping layer (modified refractivity decreasing with height) was detected in the profile. Green = duct present, signals can travel far beyond line of sight."
|
||||
},
|
||||
%{
|
||||
id: "duct_base_m",
|
||||
label: "Duct Base",
|
||||
unit: "m",
|
||||
group: "Ducting",
|
||||
desc:
|
||||
"Height of the lowest duct base. Surface ducts (< 100m) are most effective for ground-based stations. Elevated ducts require antennas near the duct height."
|
||||
},
|
||||
%{
|
||||
id: "duct_strength",
|
||||
label: "Duct Strength",
|
||||
unit: "M",
|
||||
group: "Ducting",
|
||||
desc:
|
||||
"Duct trapping strength in modified refractivity (M) units. > 10 M is moderate, > 20 M is strong. Stronger ducts trap a wider range of frequencies."
|
||||
}
|
||||
]
|
||||
|
||||
@default_layer "temperature"
|
||||
@valid_layer_ids Enum.map(@layers, & &1.id)
|
||||
@layers MapLayers.all()
|
||||
@default_layer MapLayers.default_id()
|
||||
|
||||
@impl true
|
||||
def mount(_params, _session, socket) do
|
||||
|
|
@ -179,10 +24,9 @@ defmodule MicrowavepropWeb.WeatherMapLive do
|
|||
# Mount stays cheap — no grid materialisation. The dead render
|
||||
# would otherwise spend ~5–10 s materialising 92k cells through
|
||||
# `WeatherLayers.derive`, blocking the HTTP response and tripping
|
||||
# the LiveView socket join timeout. The JS hook fires `map_bounds`
|
||||
# via `requestAnimationFrame` immediately after mount, and the
|
||||
# existing `handle_event("map_bounds", ...)` path drives the
|
||||
# initial layer paint via `update_weather`.
|
||||
# the LiveView socket join timeout. The client now paints weather
|
||||
# through a tile endpoint, so mount only needs the selected layer
|
||||
# and valid_time metadata.
|
||||
valid_times = Weather.available_weather_valid_times()
|
||||
initial_vt = pick_initial_valid_time(valid_times)
|
||||
|
||||
|
|
@ -191,16 +35,14 @@ defmodule MicrowavepropWeb.WeatherMapLive do
|
|||
page_title: "Weather Map",
|
||||
layers: @layers,
|
||||
selected_layer: @default_layer,
|
||||
initial_data_json: "[]",
|
||||
valid_time: initial_vt,
|
||||
valid_times: valid_times,
|
||||
initial_valid_times_json: Jason.encode!(Enum.map(valid_times, &DateTime.to_iso8601/1)),
|
||||
initial_selected_time: initial_vt && DateTime.to_iso8601(initial_vt),
|
||||
initial_utc_clock: Calendar.strftime(DateTime.utc_now(), "%H:%M UTC"),
|
||||
bounds: @initial_bounds,
|
||||
weather_tile_url: "/weather/tiles",
|
||||
grid_visible: false,
|
||||
radar_visible: false,
|
||||
weather_flush_timer: nil
|
||||
radar_visible: false
|
||||
)}
|
||||
end
|
||||
|
||||
|
|
@ -210,7 +52,10 @@ defmodule MicrowavepropWeb.WeatherMapLive do
|
|||
{:noreply, assign(socket, :selected_layer, selected)}
|
||||
end
|
||||
|
||||
defp pick_layer(layer, _current) when layer in @valid_layer_ids, do: layer
|
||||
defp pick_layer(layer, current) when is_binary(layer) do
|
||||
if MapLayers.valid_id?(layer), do: layer, else: pick_layer(nil, current)
|
||||
end
|
||||
|
||||
defp pick_layer(_layer, current) when is_binary(current), do: current
|
||||
defp pick_layer(_layer, _current), do: @default_layer
|
||||
|
||||
|
|
@ -223,7 +68,7 @@ defmodule MicrowavepropWeb.WeatherMapLive do
|
|||
|
||||
@impl true
|
||||
def handle_event("select_layer", %{"layer" => layer_id}, socket) do
|
||||
if layer_id in @valid_layer_ids do
|
||||
if MapLayers.valid_id?(layer_id) do
|
||||
{:noreply, push_patch(socket, to: ~p"/weather?layer=#{layer_id}")}
|
||||
else
|
||||
{:noreply, socket}
|
||||
|
|
@ -252,27 +97,13 @@ defmodule MicrowavepropWeb.WeatherMapLive do
|
|||
{:noreply, socket}
|
||||
end
|
||||
|
||||
def handle_event("map_bounds", bounds, socket) do
|
||||
# Debounce the repaint. Cache miss on `weather_grid_at` runs a
|
||||
# derive pass over hrrr_profiles rows inside the viewport — full
|
||||
# CONUS is hundreds of KB of JSON and a non-trivial Ecto query.
|
||||
# Running it on every moveend burst during a wheel-zoom was
|
||||
# visibly laggy. 150ms matches /map's throttle.
|
||||
{:noreply,
|
||||
socket
|
||||
|> assign(:bounds, bounds)
|
||||
|> schedule_weather_flush()}
|
||||
end
|
||||
|
||||
def handle_event("select_time", %{"time" => time_str}, socket) do
|
||||
with {:ok, time, _} <- DateTime.from_iso8601(time_str),
|
||||
true <- time in socket.assigns.valid_times do
|
||||
data = Weather.weather_grid_at(time, socket.assigns.bounds)
|
||||
|
||||
socket =
|
||||
socket
|
||||
|> assign(:valid_time, time)
|
||||
|> push_event("update_weather", %{data: data})
|
||||
|> push_event("update_weather_overlay", %{selected: DateTime.to_iso8601(time)})
|
||||
|
||||
{:noreply, socket}
|
||||
else
|
||||
|
|
@ -291,19 +122,6 @@ defmodule MicrowavepropWeb.WeatherMapLive do
|
|||
end
|
||||
|
||||
@impl true
|
||||
def handle_info(:flush_weather_bounds, socket) do
|
||||
data =
|
||||
case socket.assigns.valid_time do
|
||||
nil -> Weather.latest_weather_grid(socket.assigns.bounds)
|
||||
vt -> Weather.weather_grid_at(vt, socket.assigns.bounds)
|
||||
end
|
||||
|
||||
{:noreply,
|
||||
socket
|
||||
|> assign(:weather_flush_timer, nil)
|
||||
|> push_event("update_weather", %{data: data})}
|
||||
end
|
||||
|
||||
def handle_info({:weather_updated, _valid_time}, socket) do
|
||||
valid_times = Weather.available_weather_valid_times()
|
||||
|
||||
|
|
@ -318,14 +136,11 @@ defmodule MicrowavepropWeb.WeatherMapLive do
|
|||
true -> nil
|
||||
end
|
||||
|
||||
data =
|
||||
if selected, do: Weather.weather_grid_at(selected, socket.assigns.bounds), else: []
|
||||
|
||||
socket =
|
||||
socket
|
||||
|> assign(:valid_times, valid_times)
|
||||
|> assign(:valid_time, selected)
|
||||
|> push_event("update_weather", %{data: data})
|
||||
|> push_event("update_weather_overlay", %{selected: selected && DateTime.to_iso8601(selected)})
|
||||
|> push_event("update_timeline", %{
|
||||
times: Enum.map(valid_times, &DateTime.to_iso8601/1),
|
||||
selected: selected && DateTime.to_iso8601(selected)
|
||||
|
|
@ -357,20 +172,6 @@ defmodule MicrowavepropWeb.WeatherMapLive do
|
|||
|
||||
@group_order ["Surface", "Upper Air", "Ducting"]
|
||||
|
||||
@weather_flush_debounce_ms 150
|
||||
|
||||
defp schedule_weather_flush(%{assigns: %{weather_flush_timer: ref}} = socket) when is_reference(ref) do
|
||||
_ = Process.cancel_timer(ref)
|
||||
do_schedule_weather_flush(socket)
|
||||
end
|
||||
|
||||
defp schedule_weather_flush(socket), do: do_schedule_weather_flush(socket)
|
||||
|
||||
defp do_schedule_weather_flush(socket) do
|
||||
ref = Process.send_after(self(), :flush_weather_bounds, @weather_flush_debounce_ms)
|
||||
assign(socket, :weather_flush_timer, ref)
|
||||
end
|
||||
|
||||
defp group_layers(layers) do
|
||||
layers
|
||||
|> Enum.group_by(& &1.group)
|
||||
|
|
@ -409,11 +210,11 @@ defmodule MicrowavepropWeb.WeatherMapLive do
|
|||
id="weather-map"
|
||||
phx-hook="WeatherMap"
|
||||
phx-update="ignore"
|
||||
data-weather={@initial_data_json}
|
||||
data-layers={Jason.encode!(@layers)}
|
||||
data-selected-layer={@selected_layer}
|
||||
data-valid-times={@initial_valid_times_json}
|
||||
data-selected-time={@initial_selected_time}
|
||||
data-weather-tile-url={@weather_tile_url}
|
||||
class="absolute inset-0 z-0"
|
||||
>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -180,6 +180,7 @@ defmodule MicrowavepropWeb.Router do
|
|||
|
||||
get "/", PageController, :home
|
||||
get "/api/contacts/map", ContactMapController, :show
|
||||
get "/weather/tiles/:z/:x/:y", WeatherTileController, :show
|
||||
|
||||
live_session :public, on_mount: [{MicrowavepropWeb.UserAuth, :default}] do
|
||||
live "/submit", SubmitLive
|
||||
|
|
|
|||
|
|
@ -5,8 +5,10 @@ defmodule Microwaveprop.Propagation.NotifyListenerTest do
|
|||
alias Microwaveprop.Propagation.BandConfig
|
||||
alias Microwaveprop.Propagation.Grid
|
||||
alias Microwaveprop.Propagation.NotifyListener
|
||||
alias Microwaveprop.Propagation.ProfilesFile
|
||||
alias Microwaveprop.Propagation.ScoreCache
|
||||
alias Microwaveprop.Propagation.ScoresFile
|
||||
alias Microwaveprop.Weather.ScalarFile
|
||||
|
||||
setup do
|
||||
dir =
|
||||
|
|
@ -85,6 +87,30 @@ defmodule Microwaveprop.Propagation.NotifyListenerTest do
|
|||
|
||||
assert ScoreCache.fetch(10_000, stale) == :miss
|
||||
end
|
||||
|
||||
test "kicks off async ScalarFile materialization for the new valid_time" do
|
||||
vt = ~U[2026-04-21 16:00:00Z]
|
||||
ScoresFile.write!(10_000, vt, sample_scores())
|
||||
|
||||
ProfilesFile.write!(vt, %{
|
||||
{32.125, -97.375} => %{
|
||||
surface_temp_c: 25.0,
|
||||
surface_dewpoint_c: 18.0,
|
||||
surface_pressure_mb: 1013.0,
|
||||
hpbl_m: 500.0,
|
||||
pwat_mm: 30.0,
|
||||
profile: []
|
||||
}
|
||||
})
|
||||
|
||||
refute ScalarFile.exists?(vt)
|
||||
|
||||
:ok = NotifyListener.handle_propagation_ready(vt)
|
||||
|
||||
# Background Task → poll briefly until the scalar file lands.
|
||||
assert wait_until(fn -> ScalarFile.exists?(vt) end, 1_000),
|
||||
"expected ScalarFile to be materialized after handle_propagation_ready"
|
||||
end
|
||||
end
|
||||
|
||||
describe "lifecycle" do
|
||||
|
|
@ -125,4 +151,24 @@ defmodule Microwaveprop.Propagation.NotifyListenerTest do
|
|||
assert {:noreply, ^state} = NotifyListener.handle_info({:EXIT, self(), :normal}, state)
|
||||
end
|
||||
end
|
||||
|
||||
defp wait_until(fun, timeout_ms) do
|
||||
deadline = System.monotonic_time(:millisecond) + timeout_ms
|
||||
|
||||
cond do
|
||||
fun.() -> true
|
||||
System.monotonic_time(:millisecond) >= deadline -> false
|
||||
true -> wait_until_step(fun, deadline)
|
||||
end
|
||||
end
|
||||
|
||||
defp wait_until_step(fun, deadline) do
|
||||
Process.sleep(20)
|
||||
|
||||
cond do
|
||||
fun.() -> true
|
||||
System.monotonic_time(:millisecond) >= deadline -> false
|
||||
true -> wait_until_step(fun, deadline)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
186
test/microwaveprop/weather/scalar_file_test.exs
Normal file
186
test/microwaveprop/weather/scalar_file_test.exs
Normal file
|
|
@ -0,0 +1,186 @@
|
|||
defmodule Microwaveprop.Weather.ScalarFileTest do
|
||||
# async: false — mutates the global :propagation_scores_dir env to redirect
|
||||
# the on-disk artifact tree to a private tmp directory.
|
||||
use ExUnit.Case, async: false
|
||||
|
||||
alias Microwaveprop.Weather.ScalarFile
|
||||
|
||||
setup do
|
||||
dir =
|
||||
Path.join(
|
||||
System.tmp_dir!(),
|
||||
"weather_scalar_file_test_#{System.unique_integer([:positive])}"
|
||||
)
|
||||
|
||||
File.mkdir_p!(dir)
|
||||
|
||||
prev = Application.get_env(:microwaveprop, :propagation_scores_dir)
|
||||
Application.put_env(:microwaveprop, :propagation_scores_dir, dir)
|
||||
|
||||
on_exit(fn ->
|
||||
File.rm_rf!(dir)
|
||||
|
||||
if prev do
|
||||
Application.put_env(:microwaveprop, :propagation_scores_dir, prev)
|
||||
else
|
||||
Application.delete_env(:microwaveprop, :propagation_scores_dir)
|
||||
end
|
||||
end)
|
||||
|
||||
%{dir: dir}
|
||||
end
|
||||
|
||||
defp sample_row(lat, lon, temp \\ 22.0) do
|
||||
%{
|
||||
lat: lat,
|
||||
lon: lon,
|
||||
valid_time: ~U[2026-04-28 12:00:00Z],
|
||||
temperature: temp,
|
||||
dewpoint_depression: 5.0,
|
||||
surface_rh: 70.0,
|
||||
surface_pressure_mb: 1013.0,
|
||||
surface_refractivity: 330.0,
|
||||
refractivity_gradient: -45.0,
|
||||
bl_height: 800.0,
|
||||
pwat: 25.0,
|
||||
temp_850mb: 12.0,
|
||||
dewpoint_850mb: 4.0,
|
||||
temp_700mb: 0.0,
|
||||
dewpoint_700mb: -10.0,
|
||||
lapse_rate: 6.5,
|
||||
mid_lapse_rate: 6.0,
|
||||
inversion_strength: 0.5,
|
||||
inversion_base_m: nil,
|
||||
ducting: false,
|
||||
duct_base_m: nil,
|
||||
duct_strength: nil
|
||||
}
|
||||
end
|
||||
|
||||
describe "base_dir/0" do
|
||||
test "is rooted under the configured scores dir", %{dir: dir} do
|
||||
assert ScalarFile.base_dir() == Path.join(dir, "weather_scalars")
|
||||
end
|
||||
end
|
||||
|
||||
describe "write!/2 + read_bounds/2" do
|
||||
test "round-trips the rows that fall inside the requested bounds" do
|
||||
valid_time = ~U[2026-04-28 12:00:00Z]
|
||||
|
||||
rows = [
|
||||
sample_row(33.0, -97.0, 22.0),
|
||||
sample_row(33.5, -97.5, 23.0),
|
||||
# Outside Texas bounds below
|
||||
sample_row(45.0, -90.0, 5.0)
|
||||
]
|
||||
|
||||
ScalarFile.write!(valid_time, rows)
|
||||
|
||||
result =
|
||||
ScalarFile.read_bounds(valid_time, %{
|
||||
"south" => 32.0,
|
||||
"north" => 34.5,
|
||||
"west" => -98.0,
|
||||
"east" => -96.0
|
||||
})
|
||||
|
||||
lat_lons = result |> Enum.map(&{&1.lat, &1.lon}) |> Enum.sort()
|
||||
assert lat_lons == [{33.0, -97.0}, {33.5, -97.5}]
|
||||
end
|
||||
|
||||
test "returns [] when the file does not exist" do
|
||||
assert ScalarFile.read_bounds(~U[2026-04-28 12:00:00Z], nil) == []
|
||||
end
|
||||
|
||||
test "read_bounds with nil bounds returns every persisted row" do
|
||||
valid_time = ~U[2026-04-28 12:00:00Z]
|
||||
rows = [sample_row(33.0, -97.0), sample_row(45.0, -90.0)]
|
||||
|
||||
ScalarFile.write!(valid_time, rows)
|
||||
|
||||
assert length(ScalarFile.read_bounds(valid_time, nil)) == 2
|
||||
end
|
||||
|
||||
test "preserves the full set of derived layer fields on round-trip" do
|
||||
valid_time = ~U[2026-04-28 12:00:00Z]
|
||||
[row] = [sample_row(33.0, -97.0)]
|
||||
|
||||
ScalarFile.write!(valid_time, [row])
|
||||
|
||||
[round_tripped] = ScalarFile.read_bounds(valid_time, nil)
|
||||
|
||||
for key <- Map.keys(row) do
|
||||
assert Map.fetch!(round_tripped, key) == Map.fetch!(row, key),
|
||||
"key #{inspect(key)} mismatched after round-trip"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe "read_point/3" do
|
||||
test "returns the snapped row for the requested lat/lon" do
|
||||
valid_time = ~U[2026-04-28 12:00:00Z]
|
||||
row = sample_row(33.0, -97.0)
|
||||
|
||||
ScalarFile.write!(valid_time, [row])
|
||||
|
||||
assert {:ok, fetched} = ScalarFile.read_point(valid_time, 33.0, -97.0)
|
||||
assert fetched.temperature == 22.0
|
||||
end
|
||||
|
||||
test "returns :miss when no scalar file exists" do
|
||||
assert ScalarFile.read_point(~U[2026-04-28 12:00:00Z], 33.0, -97.0) == :miss
|
||||
end
|
||||
|
||||
test "returns :miss when the cell is not in the file" do
|
||||
valid_time = ~U[2026-04-28 12:00:00Z]
|
||||
ScalarFile.write!(valid_time, [sample_row(33.0, -97.0)])
|
||||
|
||||
assert ScalarFile.read_point(valid_time, 40.0, -80.0) == :miss
|
||||
end
|
||||
end
|
||||
|
||||
describe "exists?/1 and list_valid_times/0" do
|
||||
test "reflects what has been written" do
|
||||
vt1 = ~U[2026-04-28 12:00:00Z]
|
||||
vt2 = ~U[2026-04-28 13:00:00Z]
|
||||
|
||||
refute ScalarFile.exists?(vt1)
|
||||
assert ScalarFile.list_valid_times() == []
|
||||
|
||||
ScalarFile.write!(vt1, [sample_row(33.0, -97.0)])
|
||||
ScalarFile.write!(vt2, [sample_row(33.0, -97.0)])
|
||||
|
||||
assert ScalarFile.exists?(vt1)
|
||||
assert ScalarFile.list_valid_times() == [vt1, vt2]
|
||||
end
|
||||
end
|
||||
|
||||
describe "prune_older_than/1 and retain_window/2" do
|
||||
test "prune_older_than removes files strictly before the cutoff" do
|
||||
old = ~U[2026-04-27 12:00:00Z]
|
||||
new = ~U[2026-04-28 12:00:00Z]
|
||||
|
||||
ScalarFile.write!(old, [sample_row(33.0, -97.0)])
|
||||
ScalarFile.write!(new, [sample_row(33.0, -97.0)])
|
||||
|
||||
assert ScalarFile.prune_older_than(~U[2026-04-28 00:00:00Z]) == 1
|
||||
refute ScalarFile.exists?(old)
|
||||
assert ScalarFile.exists?(new)
|
||||
end
|
||||
|
||||
test "retain_window keeps only files inside [run_time, run_time + max_forecast_hour * 3600]" do
|
||||
run_time = ~U[2026-04-28 12:00:00Z]
|
||||
inside = ~U[2026-04-28 14:00:00Z]
|
||||
outside = ~U[2026-04-29 06:00:00Z]
|
||||
|
||||
ScalarFile.write!(run_time, [sample_row(33.0, -97.0)])
|
||||
ScalarFile.write!(inside, [sample_row(33.0, -97.0)])
|
||||
ScalarFile.write!(outside, [sample_row(33.0, -97.0)])
|
||||
|
||||
assert ScalarFile.retain_window(run_time, 4) == 1
|
||||
assert ScalarFile.exists?(run_time)
|
||||
assert ScalarFile.exists?(inside)
|
||||
refute ScalarFile.exists?(outside)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -574,6 +574,79 @@ defmodule Microwaveprop.WeatherGridTest do
|
|||
end
|
||||
end
|
||||
|
||||
describe "materialize_scalar_file/1" do
|
||||
alias Microwaveprop.Weather.ScalarFile
|
||||
|
||||
test "writes a ScalarFile derived from the on-disk ProfilesFile" do
|
||||
vt = ~U[2026-04-28 12:00:00Z]
|
||||
|
||||
ProfilesFile.write!(vt, %{
|
||||
{32.125, -97.375} => %{
|
||||
surface_temp_c: 25.0,
|
||||
surface_dewpoint_c: 18.0,
|
||||
surface_pressure_mb: 1013.0,
|
||||
hpbl_m: 500.0,
|
||||
pwat_mm: 30.0,
|
||||
profile: [
|
||||
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
||||
%{"pres" => 500.0, "tmpc" => -15.0, "dwpc" => -25.0, "hght" => 5600.0}
|
||||
]
|
||||
}
|
||||
})
|
||||
|
||||
refute ScalarFile.exists?(vt)
|
||||
|
||||
:ok = Weather.materialize_scalar_file(vt)
|
||||
|
||||
assert ScalarFile.exists?(vt)
|
||||
[row] = ScalarFile.read_bounds(vt, nil)
|
||||
assert row.temperature == 25.0
|
||||
assert row.dewpoint_depression == 7.0
|
||||
end
|
||||
|
||||
test "is idempotent — does not re-write when the scalar file already exists" do
|
||||
vt = ~U[2026-04-28 13:00:00Z]
|
||||
|
||||
ProfilesFile.write!(vt, %{
|
||||
{32.125, -97.375} => %{
|
||||
surface_temp_c: 22.0,
|
||||
surface_dewpoint_c: 12.0,
|
||||
surface_pressure_mb: 1010.0,
|
||||
hpbl_m: 600.0,
|
||||
pwat_mm: 24.0,
|
||||
profile: []
|
||||
}
|
||||
})
|
||||
|
||||
:ok = Weather.materialize_scalar_file(vt)
|
||||
[first] = ScalarFile.read_bounds(vt, nil)
|
||||
|
||||
# Mutate the underlying ProfilesFile, then call materialize again.
|
||||
# Idempotent skipping means the scalar file does not change.
|
||||
ProfilesFile.write!(vt, %{
|
||||
{32.125, -97.375} => %{
|
||||
surface_temp_c: 99.0,
|
||||
surface_dewpoint_c: 50.0,
|
||||
surface_pressure_mb: 999.0,
|
||||
hpbl_m: 999.0,
|
||||
pwat_mm: 99.0,
|
||||
profile: []
|
||||
}
|
||||
})
|
||||
|
||||
:ok = Weather.materialize_scalar_file(vt)
|
||||
[second] = ScalarFile.read_bounds(vt, nil)
|
||||
|
||||
assert second.temperature == first.temperature
|
||||
end
|
||||
|
||||
test "is a no-op when the ProfilesFile does not exist" do
|
||||
vt = ~U[2026-04-28 14:00:00Z]
|
||||
assert Weather.materialize_scalar_file(vt) == :ok
|
||||
refute ScalarFile.exists?(vt)
|
||||
end
|
||||
end
|
||||
|
||||
# Merges `attrs` (a single HRRR grid cell) into the ProfilesFile for
|
||||
# `attrs.valid_time`. Re-reads any existing file so multiple calls for
|
||||
# the same valid_time accumulate cells, mirroring how the Rust worker
|
||||
|
|
|
|||
|
|
@ -0,0 +1,52 @@
|
|||
defmodule MicrowavepropWeb.WeatherTileControllerTest do
|
||||
use MicrowavepropWeb.ConnCase, async: false
|
||||
|
||||
alias Microwaveprop.Propagation.ProfilesFile
|
||||
alias Microwaveprop.Weather.GridCache
|
||||
|
||||
setup do
|
||||
GridCache.clear()
|
||||
|
||||
on_exit(fn ->
|
||||
GridCache.clear()
|
||||
ProfilesFile.prune_older_than(~U[2099-12-31 23:59:59Z])
|
||||
end)
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
test "serves an SVG weather tile for the requested viewport", %{conn: conn} do
|
||||
valid_time = ~U[2026-04-28 12:00:00Z]
|
||||
lat = 33.0
|
||||
lon = -97.0
|
||||
|
||||
ProfilesFile.write!(valid_time, %{
|
||||
{lat, lon} => %{
|
||||
surface_temp_c: 22.0,
|
||||
surface_dewpoint_c: 12.0,
|
||||
surface_pressure_mb: 1010.0,
|
||||
hpbl_m: 800.0,
|
||||
pwat_mm: 25.0,
|
||||
profile: []
|
||||
}
|
||||
})
|
||||
|
||||
conn =
|
||||
get(
|
||||
conn,
|
||||
~p"/weather/tiles/0/0/0?layer=temperature&time=#{DateTime.to_iso8601(valid_time)}"
|
||||
)
|
||||
|
||||
assert response(conn, 200) =~ "<svg"
|
||||
assert response(conn, 200) =~ "<rect"
|
||||
assert ["image/svg+xml; charset=utf-8"] = get_resp_header(conn, "content-type")
|
||||
end
|
||||
|
||||
test "returns an empty svg for invalid params", %{conn: conn} do
|
||||
conn = get(conn, ~p"/weather/tiles/6/18/25?layer=bogus&time=not-a-time")
|
||||
|
||||
body = response(conn, 200)
|
||||
assert body =~ "<svg"
|
||||
refute body =~ "<rect"
|
||||
end
|
||||
end
|
||||
|
|
@ -102,7 +102,7 @@ defmodule MicrowavepropWeb.WeatherMapLiveTest do
|
|||
refute html =~ ~s(data-selected-time="#{future_iso}")
|
||||
end
|
||||
|
||||
test "select_time replaces the rendered weather data with the new hour's grid", %{conn: conn} do
|
||||
test "select_time pushes an overlay refresh for the selected hour", %{conn: conn} do
|
||||
base =
|
||||
DateTime.utc_now()
|
||||
|> DateTime.truncate(:second)
|
||||
|
|
@ -134,8 +134,8 @@ defmodule MicrowavepropWeb.WeatherMapLiveTest do
|
|||
# Select the future hour.
|
||||
render_hook(lv, "select_time", %{"time" => DateTime.to_iso8601(future_t)})
|
||||
|
||||
assert_push_event(lv, "update_weather", %{data: data})
|
||||
assert Enum.any?(data, fn r -> r.temperature == 10.0 end)
|
||||
assert_push_event(lv, "update_weather_overlay", %{selected: selected})
|
||||
assert selected == DateTime.to_iso8601(future_t)
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -216,22 +216,11 @@ defmodule MicrowavepropWeb.WeatherMapLiveTest do
|
|||
end
|
||||
end
|
||||
|
||||
describe "map_bounds event" do
|
||||
test "updating the bounds re-scopes the grid query without errors", %{conn: conn} do
|
||||
{:ok, lv, _html} = live(conn, ~p"/weather")
|
||||
describe "tile overlay bootstrapping" do
|
||||
test "renders the weather tile base URL on the map element", %{conn: conn} do
|
||||
{:ok, _lv, html} = live(conn, ~p"/weather")
|
||||
|
||||
# Smaller bounds within the initial DFW window.
|
||||
new_bounds = %{"north" => 33.5, "south" => 32.5, "east" => -96.0, "west" => -98.0}
|
||||
|
||||
render_hook(lv, "map_bounds", new_bounds)
|
||||
# After debouncing the server flushes with a push_event for the grid.
|
||||
send(lv.pid, :flush_weather_bounds)
|
||||
render(lv)
|
||||
|
||||
# Socket assigns update — we don't pin the grid contents because the
|
||||
# empty-profiles case returns an empty data list. The smoke test is
|
||||
# just that the handler doesn't raise.
|
||||
assert Process.alive?(lv.pid)
|
||||
assert html =~ ~s(data-weather-tile-url="/weather/tiles")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue