defmodule Mix.Tasks.HrrrClimatology do @shortdoc "Build surface temperature climatology from hrrr_profiles" @moduledoc """ Aggregates `hrrr_profiles.surface_temp_c` by (lat, lon, month, hour) into `hrrr_climatology` for use by the temperature-anomaly feature. With 42M+ grid-point profiles, this runs a single SQL GROUP BY and bulk-inserts the results. Idempotent via ON CONFLICT. mix hrrr_climatology # build from all grid-point profiles mix hrrr_climatology --min-samples 5 # require at least 5 observations per cell """ use Mix.Task alias Microwaveprop.Repo @impl Mix.Task def run(argv) do Mix.Task.run("app.start") {opts, _, _} = OptionParser.parse(argv, switches: [min_samples: :integer]) min_samples = Keyword.get(opts, :min_samples, 3) Mix.shell().info("Building climatology (min_samples=#{min_samples})...") # Single SQL: aggregate and upsert in one shot {count, _} = Repo.query!( """ INSERT INTO hrrr_climatology (id, lat, lon, month, hour, mean_surface_temp_c, stddev_surface_temp_c, sample_count) SELECT gen_random_uuid(), lat, lon, EXTRACT(MONTH FROM valid_time)::int AS month, EXTRACT(HOUR FROM valid_time)::int AS hour, AVG(surface_temp_c), STDDEV_SAMP(surface_temp_c), COUNT(*) FROM hrrr_profiles WHERE surface_temp_c IS NOT NULL AND is_grid_point = true GROUP BY lat, lon, EXTRACT(MONTH FROM valid_time)::int, EXTRACT(HOUR FROM valid_time)::int HAVING COUNT(*) >= $1 ON CONFLICT (lat, lon, month, hour) DO UPDATE SET mean_surface_temp_c = EXCLUDED.mean_surface_temp_c, stddev_surface_temp_c = EXCLUDED.stddev_surface_temp_c, sample_count = EXCLUDED.sample_count """, [min_samples], timeout: 600_000 ) Mix.shell().info("Upserted #{count} climatology records.") end end