diff --git a/assets/js/weather_map_hook.ts b/assets/js/weather_map_hook.ts index ebbf4225..d394d3cc 100644 --- a/assets/js/weather_map_hook.ts +++ b/assets/js/weather_map_hook.ts @@ -67,39 +67,23 @@ interface DetailRow { separator?: boolean } -interface ViewportCell { - lat: number - lon: number - temperature: number | null - dewpoint_depression: number | null - surface_rh: number | null - surface_pressure_mb: number | null - surface_refractivity: number | null - refractivity_gradient: number | null - bl_height: number | null - pwat: number | null - temp_850mb: number | null - dewpoint_850mb: number | null - temp_700mb: number | null - dewpoint_700mb: number | null - lapse_rate: number | null - mid_lapse_rate: number | null - inversion_strength: number | null - inversion_base_m: number | null - ducting: boolean | null - duct_base_m: number | null - duct_strength: number | null +// Columnar viewport pack: lats/lons shared, one Float32Array per loaded +// layer. NaN = null. Layer switches re-paint from the same pack, no +// server round-trip. +interface CellPack { + cellCount: number + lats: Float32Array + lons: Float32Array + layers: Map } interface WeatherMapHook extends ViewHook { map: L.Map - weatherOverlay: L.SVGOverlay | null + weatherOverlay: WeatherCanvasOverlay | null weatherCellsUrl: string selectedLayer: LayerId - // Cells most recently fetched. Layer switches re-render from this - // without going to the server. - currentCells: ViewportCell[] - cellsRequestSeq: number + pack: CellPack | null + packKey: string | null inflightCellsAbort: AbortController | null moveDebounce: ReturnType | null detailPanel: HTMLElement | null @@ -129,6 +113,7 @@ interface WeatherMapHook extends ViewHook { positionDetailPanel(this: WeatherMapHook): void renderLayer(this: WeatherMapHook): void fetchAndPaintCells(this: WeatherMapHook, opts?: { force?: boolean }): Promise + loadLayers(this: WeatherMapHook, layers: LayerId[], cacheKey: string): Promise paintOverlay(this: WeatherMapHook): void refreshLegend(this: WeatherMapHook): void buildLegendContent(this: WeatherMapHook): string @@ -381,7 +366,49 @@ function lerp(a: number, b: number, t: number): number { return a + (b - a) * t } -function interpolateRgb(value: number, bps: Breakpoint[]): RGB { +// Decode the WCEL binary cell pack (see weather_tile_controller.ex). +function decodeCellPack(buf: ArrayBuffer): { cellCount: number; layerNames: string[]; lats: Float32Array; lons: Float32Array; values: Float32Array[] } { + const view = new DataView(buf) + const magic = String.fromCharCode(view.getUint8(0), view.getUint8(1), view.getUint8(2), view.getUint8(3)) + if (magic !== "WCEL") throw new Error(`bad magic: ${magic}`) + const version = view.getUint8(4) + if (version !== 1) throw new Error(`unsupported version: ${version}`) + const cellCount = view.getUint32(5, true) + const layerCount = view.getUint8(9) + + let off = 10 + const layerNames: string[] = [] + const decoder = new TextDecoder() + for (let i = 0; i < layerCount; i++) { + const len = view.getUint8(off) + off += 1 + layerNames.push(decoder.decode(new Uint8Array(buf, off, len))) + off += len + } + // Pad to 4-byte boundary to align the f32 arrays. + off = Math.ceil(off / 4) * 4 + + const lats = new Float32Array(buf, off, cellCount) + off += cellCount * 4 + const lons = new Float32Array(buf, off, cellCount) + off += cellCount * 4 + + const values: Float32Array[] = [] + for (let i = 0; i < layerCount; i++) { + values.push(new Float32Array(buf, off, cellCount)) + off += cellCount * 4 + } + + return { cellCount, layerNames, lats, lons, values } +} + +interface RgbAndAlpha { + r: number + g: number + b: number +} + +function interpolateRgb(value: number, bps: Breakpoint[]): RgbAndAlpha { if (value <= bps[0].value) return { r: bps[0].r, g: bps[0].g, b: bps[0].b } const last = bps[bps.length - 1] if (value >= last.value) return { r: last.r, g: last.g, b: last.b } @@ -400,22 +427,163 @@ function interpolateRgb(value: number, bps: Breakpoint[]): RGB { return { r: last.r, g: last.g, b: last.b } } -function colorForCell(cell: ViewportCell, layerId: LayerId): string | null { +// Build a 256-bucket lookup table over a continuous breakpoint scale. +// Each bucket stores RGB packed as a CSS color string. Lets the canvas +// paint avoid re-interpolating colors per cell. +function buildContinuousLut(bps: Breakpoint[]): { lo: number; hi: number; palette: string[] } { + const lo = bps[0].value + const hi = bps[bps.length - 1].value + const palette: string[] = new Array(256) + for (let i = 0; i < 256; i++) { + const v = lo + (hi - lo) * (i / 255) + const c = interpolateRgb(v, bps) + palette[i] = `rgb(${c.r},${c.g},${c.b})` + } + return { lo, hi, palette } +} + +interface PaintScale { + kind: "continuous" + lo: number + hi: number + palette: string[] +} + +interface BoolPaintScale { + kind: "bool" + trueColor: string +} + +function buildPaintScale(layerId: LayerId): PaintScale | BoolPaintScale | null { const scale = COLOR_SCALES[layerId] if (!scale) return null - const value = cell[layerId as keyof ViewportCell] - if (scale.breakpoints === null) { - if (value !== true) return null const c = scale.trueColor - return `rgb(${c.r},${c.g},${c.b})` + return { kind: "bool", trueColor: `rgb(${c.r},${c.g},${c.b})` } } - - if (typeof value !== "number" || !Number.isFinite(value)) return null - const c = interpolateRgb(value, scale.breakpoints) - return `rgb(${c.r},${c.g},${c.b})` + return { kind: "continuous", ...buildContinuousLut(scale.breakpoints) } } +// --- Canvas overlay layer --- +// +// A custom Leaflet layer that paints viewport cells onto a single +// HTML canvas. The canvas tracks the map's container; pan repositions +// it, zoom triggers a full repaint. Replaces the old per-tile +// SVG overlay — at low zoom (whole-CONUS, ~80k cells) SVG paint cost +// dominated; canvas is one fillRect per cell with a precomputed LUT. +const WeatherCanvasOverlayProto: L.LayerOptions & { + initialize: (this: WeatherCanvasOverlay, pack: CellPack | null, layerId: LayerId) => void + onAdd: (this: WeatherCanvasOverlay, map: L.Map) => WeatherCanvasOverlay + onRemove: (this: WeatherCanvasOverlay, map: L.Map) => WeatherCanvasOverlay + setPack: (this: WeatherCanvasOverlay, pack: CellPack | null) => void + setLayer: (this: WeatherCanvasOverlay, layerId: LayerId) => void + _reset: (this: WeatherCanvasOverlay) => void + _draw: (this: WeatherCanvasOverlay) => void +} = { + initialize(pack: CellPack | null, layerId: LayerId): void { + this.pack = pack + this.layerId = layerId + }, + onAdd(map: L.Map): WeatherCanvasOverlay { + this._map = map + const canvas = L.DomUtil.create("canvas", "weather-canvas-overlay leaflet-zoom-hide") as HTMLCanvasElement + canvas.style.pointerEvents = "none" + canvas.style.opacity = "0.55" + this._canvas = canvas + map.getPanes().overlayPane.appendChild(canvas) + map.on("moveend", this._reset, this) + map.on("zoomend", this._reset, this) + this._reset() + return this + }, + onRemove(map: L.Map): WeatherCanvasOverlay { + map.off("moveend", this._reset, this) + map.off("zoomend", this._reset, this) + L.DomUtil.remove(this._canvas) + return this + }, + setPack(pack: CellPack | null): void { + this.pack = pack + this._reset() + }, + setLayer(layerId: LayerId): void { + this.layerId = layerId + this._reset() + }, + _reset(): void { + if (!this._map) return + const map = this._map + const size = map.getSize() + const topLeft = map.containerPointToLayerPoint([0, 0]) + L.DomUtil.setPosition(this._canvas, topLeft) + this._canvas.width = size.x + this._canvas.height = size.y + this._draw() + }, + _draw(): void { + const ctx = this._canvas.getContext("2d") + if (!ctx) return + ctx.clearRect(0, 0, this._canvas.width, this._canvas.height) + + if (!this.pack) return + + const values = this.pack.layers.get(this.layerId) + if (!values) return + + const paintScale = buildPaintScale(this.layerId) + if (!paintScale) return + + const map = this._map + const halfStep = 0.0625 + const lats = this.pack.lats + const lons = this.pack.lons + const n = this.pack.cellCount + + // Cache mapping of (lat, lon, halfStep) → container points across + // a single draw. Two adjacent cells share an edge in container + // space, so we project each cell-edge once. + let prevColor = "" + + for (let i = 0; i < n; i++) { + const v = values[i] + if (!Number.isFinite(v)) continue + + let color: string + if (paintScale.kind === "bool") { + if (v < 0.5) continue + color = paintScale.trueColor + } else { + const t = (v - paintScale.lo) / (paintScale.hi - paintScale.lo) + const idx = t <= 0 ? 0 : t >= 1 ? 255 : Math.round(t * 255) + color = paintScale.palette[idx] + } + + const lat = lats[i] + const lon = lons[i] + const ne = map.latLngToContainerPoint([lat + halfStep, lon + halfStep]) + const sw = map.latLngToContainerPoint([lat - halfStep, lon - halfStep]) + + if (color !== prevColor) { + ctx.fillStyle = color + prevColor = color + } + ctx.fillRect(sw.x, ne.y, ne.x - sw.x, sw.y - ne.y) + } + } +} + +interface WeatherCanvasOverlay extends L.Layer { + pack: CellPack | null + layerId: LayerId + _map: L.Map + _canvas: HTMLCanvasElement + setPack(pack: CellPack | null): void + setLayer(layerId: LayerId): void +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const WeatherCanvasOverlayCtor: new (pack: CellPack | null, layerId: LayerId) => WeatherCanvasOverlay = (L.Layer as any).extend(WeatherCanvasOverlayProto) + function buildDetailHTML(point: WeatherPoint): string { if (!point) return "" @@ -498,8 +666,8 @@ export const WeatherMap: WeatherMapHook = { this.weatherOverlay = null this.weatherCellsUrl = "/weather/cells" this.selectedLayer = (this.el.dataset.selectedLayer || "temperature") as LayerId - this.currentCells = [] - this.cellsRequestSeq = 0 + this.pack = null + this.packKey = null this.inflightCellsAbort = null this.moveDebounce = null @@ -734,41 +902,94 @@ export const WeatherMap: WeatherMapHook = { }, renderLayer(this: WeatherMapHook) { - // Kept for back-compat with code paths that still call it. The - // server fetch + paint pipeline lives in fetchAndPaintCells; layer - // switches go through paintOverlay directly. + // Kept for back-compat with code paths that still call it. this.paintOverlay() this.refreshLegend() }, async fetchAndPaintCells(this: WeatherMapHook, opts: { force?: boolean } = {}) { if (!this.selectedTime) { - this.currentCells = [] + this.pack = null + this.packKey = null this.paintOverlay() return } - const seq = ++this.cellsRequestSeq - if (this.inflightCellsAbort) this.inflightCellsAbort.abort() - this.inflightCellsAbort = new AbortController() + const { south, north, west, east } = quantizeBounds(this.map.getBounds()) + const cacheKey = `${this.selectedTime}|${south},${north},${west},${east}` + + // New viewport or time → drop the existing pack. + if (cacheKey !== this.packKey || opts.force) { + if (this.inflightCellsAbort) this.inflightCellsAbort.abort() + this.pack = null + this.packKey = cacheKey + } + + // Step 1: ensure the currently selected layer is loaded, then paint. + if (!this.pack || !this.pack.layers.has(this.selectedLayer)) { + await this.loadLayers([this.selectedLayer], cacheKey) + } + this.paintOverlay() + + // Step 2: in the background, lazy-load every other layer so + // toggling between them later is instant. One request batches them + // all so the server's GridCache lookup runs once. + if (this.pack && this.packKey === cacheKey) { + const remaining = (Object.keys(COLOR_SCALES) as LayerId[]).filter( + (l) => !this.pack!.layers.has(l) + ) + if (remaining.length > 0) { + // Fire-and-forget; loadLayers updates this.pack and triggers + // no repaint (layer switches will pick up the new data). + void this.loadLayers(remaining, cacheKey) + } + } + }, + + async loadLayers(this: WeatherMapHook, layers: LayerId[], cacheKey: string) { + if (!this.selectedTime || layers.length === 0) return + if (cacheKey !== this.packKey) return const { south, north, west, east } = quantizeBounds(this.map.getBounds()) - const url = `${this.weatherCellsUrl}?time=${encodeURIComponent(this.selectedTime)}&south=${south}&north=${north}&west=${west}&east=${east}` + const url = + `${this.weatherCellsUrl}?format=bin&time=${encodeURIComponent(this.selectedTime)}` + + `&south=${south}&north=${north}&west=${west}&east=${east}` + + `&layers=${layers.join(",")}` + + const abort = new AbortController() + if (layers.length === 1 || !this.pack) this.inflightCellsAbort = abort try { - const resp = await fetch(url, { - signal: this.inflightCellsAbort.signal, - cache: opts.force ? "no-cache" : "default" - }) + const resp = await fetch(url, { signal: abort.signal }) if (!resp.ok) return - const data = await resp.json() as { cells: ViewportCell[]; valid_time: string } + const buf = await resp.arrayBuffer() + if (cacheKey !== this.packKey) return - // A newer fetch already started — drop this stale response so we - // don't paint over the in-flight one. - if (seq !== this.cellsRequestSeq) return + const decoded = decodeCellPack(buf) - this.currentCells = data.cells - this.paintOverlay() + if (!this.pack) { + const layerMap = new Map() + decoded.layerNames.forEach((name, i) => layerMap.set(name, decoded.values[i])) + this.pack = { + cellCount: decoded.cellCount, + lats: decoded.lats, + lons: decoded.lons, + layers: layerMap + } + } else if (decoded.cellCount === this.pack.cellCount) { + decoded.layerNames.forEach((name, i) => this.pack!.layers.set(name, decoded.values[i])) + } else { + // Cell count drifted (rare — viewport quantization mismatch). + // Replace the pack rather than mix incompatible arrays. + const layerMap = new Map() + decoded.layerNames.forEach((name, i) => layerMap.set(name, decoded.values[i])) + this.pack = { + cellCount: decoded.cellCount, + lats: decoded.lats, + lons: decoded.lons, + layers: layerMap + } + } } catch (e) { const err = e as Error if (err.name === "AbortError") return @@ -777,9 +998,7 @@ export const WeatherMap: WeatherMapHook = { }, paintOverlay(this: WeatherMapHook) { - const cells = this.currentCells - - if (!cells || cells.length === 0) { + if (!this.pack || this.pack.cellCount === 0) { if (this.weatherOverlay) { this.map.removeLayer(this.weatherOverlay) this.weatherOverlay = null @@ -787,45 +1006,13 @@ export const WeatherMap: WeatherMapHook = { return } - const halfStep = 0.0625 - const layerId = this.selectedLayer - - let south = Infinity - let north = -Infinity - let west = Infinity - let east = -Infinity - for (const c of cells) { - if (c.lat < south) south = c.lat - if (c.lat > north) north = c.lat - if (c.lon < west) west = c.lon - if (c.lon > east) east = c.lon + if (!this.weatherOverlay) { + this.weatherOverlay = new WeatherCanvasOverlayCtor(this.pack, this.selectedLayer) + this.weatherOverlay.addTo(this.map) + } else { + this.weatherOverlay.pack = this.pack + this.weatherOverlay.setLayer(this.selectedLayer) } - south -= halfStep - north += halfStep - west -= halfStep - east += halfStep - - const parts: string[] = [] - for (const cell of cells) { - const color = colorForCell(cell, layerId) - if (!color) continue - const x = (cell.lon - halfStep).toFixed(4) - // SVG y grows downward, so flip latitude. - const y = (-cell.lat - halfStep).toFixed(4) - parts.push(``) - } - - const svgEl = document.createElementNS("http://www.w3.org/2000/svg", "svg") - svgEl.setAttribute("viewBox", `${west} ${-north} ${east - west} ${north - south}`) - svgEl.setAttribute("preserveAspectRatio", "none") - svgEl.setAttribute("shape-rendering", "crispEdges") - svgEl.innerHTML = parts.join("") - - const bounds: L.LatLngBoundsExpression = [[south, west], [north, east]] - - if (this.weatherOverlay) this.map.removeLayer(this.weatherOverlay) - this.weatherOverlay = L.svgOverlay(svgEl, bounds, { opacity: 0.55, interactive: false }) - this.weatherOverlay.addTo(this.map) }, refreshLegend(this: WeatherMapHook) { diff --git a/lib/microwaveprop_web/controllers/weather_tile_controller.ex b/lib/microwaveprop_web/controllers/weather_tile_controller.ex index bd4b0f3e..2b80b3dc 100644 --- a/lib/microwaveprop_web/controllers/weather_tile_controller.ex +++ b/lib/microwaveprop_web/controllers/weather_tile_controller.ex @@ -41,15 +41,25 @@ defmodule MicrowavepropWeb.WeatherTileController do @spec cells(Plug.Conn.t(), map()) :: Plug.Conn.t() def cells(conn, %{"time" => time_param} = params) do with {:ok, valid_time, _offset} <- DateTime.from_iso8601(time_param), - {:ok, bounds} <- parse_bounds(params) do + {:ok, bounds} <- parse_bounds(params), + {:ok, layers} <- parse_layers(params["layers"]) do rows = Weather.weather_grid_at(valid_time, bounds) - conn - |> put_resp_header("cache-control", "public, max-age=60") - |> json(%{ - valid_time: DateTime.to_iso8601(valid_time), - cells: Enum.map(rows, &cell_payload/1) - }) + case params["format"] do + "bin" -> + conn + |> put_resp_header("content-type", "application/octet-stream") + |> put_resp_header("cache-control", "public, max-age=60") + |> send_resp(200, encode_binary(rows, layers)) + + _ -> + conn + |> put_resp_header("cache-control", "public, max-age=60") + |> json(%{ + valid_time: DateTime.to_iso8601(valid_time), + cells: Enum.map(rows, &cell_payload(&1, layers)) + }) + end else _ -> bad_request(conn) end @@ -83,9 +93,91 @@ defmodule MicrowavepropWeb.WeatherTileController do defp parse_float(_), do: :error - defp cell_payload(row) do - Enum.reduce(@cell_fields, %{lat: row.lat, lon: row.lon}, fn field, acc -> + defp cell_payload(row, layers) do + Enum.reduce(layers, %{lat: row.lat, lon: row.lon}, fn field, acc -> Map.put(acc, field, Map.get(row, field)) end) end + + defp parse_layers(nil), do: {:ok, @cell_fields} + + defp parse_layers(value) when is_binary(value) do + fields = + value + |> String.split(",", trim: true) + |> Enum.map(&String.trim/1) + |> Enum.map(&safe_atom/1) + + cond do + Enum.any?(fields, &is_nil/1) -> :error + Enum.empty?(fields) -> {:ok, @cell_fields} + true -> {:ok, fields} + end + end + + defp parse_layers(_), do: :error + + defp safe_atom(name) when is_binary(name) do + if name in Enum.map(@cell_fields, &Atom.to_string/1) do + String.to_existing_atom(name) + end + end + + # Custom binary cell-pack format. Designed for cheap parsing in JS via + # DataView + Float32Array (no library, no JSON parse). The header is + # padded so all f32 arrays land on 4-byte boundaries — DataView reads + # tolerate misalignment but typed-array views over an ArrayBuffer + # don't. + # + # "WCEL" (4 bytes) — magic + # u8 — version (1) + # u32 LE — cell count + # u8 — layer count + # for each layer: + # u8 — name length + # u8 × name_length — name (UTF-8) + # u8 × N — zero padding to next 4-byte boundary + # f32 LE × cell_count — latitudes + # f32 LE × cell_count — longitudes + # for each layer: + # f32 LE × cell_count — values (NaN = null; booleans 0.0/1.0) + defp encode_binary(rows, layers) do + cell_count = length(rows) + layer_names = Enum.map(layers, &Atom.to_string/1) + + header = + <<"WCEL", 1::8, cell_count::little-32, length(layer_names)::8>> <> + Enum.map_join(layer_names, fn n -> <> <> n end) + + pad_size = padding_to_align(byte_size(header), 4) + header_padded = header <> :binary.copy(<<0>>, pad_size) + + lats = rows |> Enum.map(&<>) |> IO.iodata_to_binary() + lons = rows |> Enum.map(&<>) |> IO.iodata_to_binary() + + values = + layers + |> Enum.map(fn field -> + Enum.map(rows, fn row -> + <> + end) + end) + |> IO.iodata_to_binary() + + <> + end + + defp padding_to_align(size, align) do + rem = rem(size, align) + if rem == 0, do: 0, else: align - rem + end + + defp encode_value(nil), do: :nan + defp encode_value(true), do: 1.0 + defp encode_value(false), do: 0.0 + defp encode_value(v) when is_number(v), do: v * 1.0 + defp encode_value(_), do: :nan + + defp float_or_nan(v) when is_number(v), do: v * 1.0 + defp float_or_nan(_), do: :nan end diff --git a/test/microwaveprop_web/controllers/weather_tile_controller_test.exs b/test/microwaveprop_web/controllers/weather_tile_controller_test.exs index 84ddde37..454cf0ea 100644 --- a/test/microwaveprop_web/controllers/weather_tile_controller_test.exs +++ b/test/microwaveprop_web/controllers/weather_tile_controller_test.exs @@ -113,5 +113,90 @@ defmodule MicrowavepropWeb.WeatherTileControllerTest do body = json_response(conn, 200) assert body["cells"] == [] end + + test "with ?layers=foo,bar only includes those fields per cell", %{conn: conn} do + valid_time = ~U[2026-04-28 12:00:00Z] + + ProfilesFile.write!(valid_time, %{ + {33.0, -97.0} => %{ + surface_temp_c: 22.0, + surface_dewpoint_c: 12.0, + surface_pressure_mb: 1010.0, + hpbl_m: 800.0, + pwat_mm: 25.0, + profile: [] + } + }) + + time = DateTime.to_iso8601(valid_time) + + conn = + get( + conn, + ~p"/weather/cells?time=#{time}&south=32.0&north=34.0&west=-98.0&east=-96.0&layers=temperature,pwat" + ) + + body = json_response(conn, 200) + [cell | _] = body["cells"] + assert Map.has_key?(cell, "lat") + assert Map.has_key?(cell, "lon") + assert Map.has_key?(cell, "temperature") + assert Map.has_key?(cell, "pwat") + refute Map.has_key?(cell, "dewpoint_depression") + refute Map.has_key?(cell, "surface_rh") + end + + test "with ?format=bin returns the binary cell pack", %{conn: conn} do + valid_time = ~U[2026-04-28 12:00:00Z] + + ProfilesFile.write!(valid_time, %{ + {33.0, -97.0} => %{ + surface_temp_c: 22.0, + surface_dewpoint_c: 12.0, + surface_pressure_mb: 1010.0, + hpbl_m: 800.0, + pwat_mm: 25.0, + profile: [] + }, + {33.125, -97.0} => %{ + surface_temp_c: 23.0, + surface_dewpoint_c: 13.0, + surface_pressure_mb: 1011.0, + hpbl_m: 700.0, + pwat_mm: 26.0, + profile: [] + } + }) + + time = DateTime.to_iso8601(valid_time) + + conn = + get( + conn, + ~p"/weather/cells?time=#{time}&south=32.0&north=34.0&west=-98.0&east=-96.0&layers=temperature,pwat&format=bin" + ) + + assert ["application/octet-stream"] = get_resp_header(conn, "content-type") + body = response(conn, 200) + + # Magic + version + cell count + layer count + 2 layer names + lats + lons + 2 fields + assert <<"WCEL", 1::8, cell_count::little-32, 2::8, rest::binary>> = body + assert cell_count == 2 + + # Two layer names: "temperature" (11), "pwat" (4) + <<11::8, "temperature", 4::8, "pwat", padded_rest::binary>> = rest + + # The header so far is: 4(magic) + 1 + 4 + 1 + (1+11) + (1+4) = 27 bytes. + # Pad to 4-byte boundary: 1 byte of zero padding → 28 bytes. + <<0::8, lats_and_more::binary>> = padded_rest + + <> = + lats_and_more + + assert_in_delta lat1, 33.0, 0.001 + assert_in_delta lat2, 33.125, 0.001 + + <<_lon1::little-float-32, _lon2::little-float-32, _rest::binary>> = rest_after_lats + end end end