Graham McIntire
690a3523e2
First step of the disk-backed scores migration from the DuckDB plan doc. Ended up shipping the raw-binary variant instead of Parquet because the data is disposable after ~2h — the ecosystem benefits of Parquet only pay off for long-lived datasets, and the binary path has zero new dependencies. Microwaveprop.Propagation.ScoresFile writes one file per (band_mhz, valid_time) tuple under the configured scores dir, default /data/scores in prod and priv/dev_scores in dev. Layout is a 33-byte header plus a dense n_rows × n_cols uint8 array (255 is the no-data sentinel). The whole CONUS grid serializes to ~93 KB per band, and writes use the temp-then-rename pattern so NFSv4 concurrent readers never see a partial file. Propagation.replace_scores/2 now materializes the score stream once and dual-writes: Postgres on the primary path, then one ScoresFile per band as a best-effort follow-up (any file error is logged but doesn't fail the DB write, so we can verify the file path in prod before cutting readers over). Propagation.prune_old_scores/0 also clears expired score files so the existing 15-minute prune cron covers both storage layers. Dev configuration points at priv/dev_scores/, added to .gitignore. Test configuration points at a per-run tmp directory.
84 lines
3.3 KiB
Elixir
84 lines
3.3 KiB
Elixir
import Config
|
|
|
|
alias Microwaveprop.Qrz.Client
|
|
|
|
# Only in tests, remove the complexity from the password hashing algorithm
|
|
config :bcrypt_elixir, :log_rounds, 1
|
|
|
|
# Print only warnings and errors during test
|
|
config :logger, level: :warning
|
|
|
|
config :microwaveprop, Client,
|
|
username: "testuser",
|
|
password: "testpass",
|
|
agent: "microwaveprop-test/1.0",
|
|
plug: {Req.Test, Client},
|
|
retry: false
|
|
|
|
config :microwaveprop, Microwaveprop.Geocoder,
|
|
plug: {Req.Test, Microwaveprop.Geocoder},
|
|
retry: false
|
|
|
|
# In test we don't send emails
|
|
config :microwaveprop, Microwaveprop.Mailer, adapter: Swoosh.Adapters.Test
|
|
config :microwaveprop, Microwaveprop.Qrz, cache_ttl_hours: 0
|
|
|
|
# Configure your database
|
|
#
|
|
# The MIX_TEST_PARTITION environment variable can be used
|
|
# to provide built-in test partitioning in CI environment.
|
|
# Run `mix help test` for more information.
|
|
config :microwaveprop, Microwaveprop.Repo,
|
|
username: "postgres",
|
|
password: "postgres",
|
|
hostname: "localhost",
|
|
database: "microwaveprop_test#{System.get_env("MIX_TEST_PARTITION")}",
|
|
pool: Ecto.Adapters.SQL.Sandbox,
|
|
pool_size: System.schedulers_online() * 2,
|
|
ownership_timeout: 60_000,
|
|
timeout: 60_000
|
|
|
|
# We don't run a server during test. If one is required,
|
|
# you can enable the server option below.
|
|
config :microwaveprop, MicrowavepropWeb.Endpoint,
|
|
http: [ip: {127, 0, 0, 1}, port: 4002],
|
|
secret_key_base: "f7KbnwUezGrPtrv/67l6Wi+W9/+wOqwFG5QJ/ZLs24cVM1gg2lBswZwrKuyWsBAZ",
|
|
server: false
|
|
|
|
# Run Oban jobs inline during tests
|
|
config :microwaveprop, Oban, testing: :inline
|
|
|
|
# Score files land under a per-run tmp directory so replace_scores tests
|
|
# don't write into a shared location (and we can assert on the file
|
|
# contents from inside tests).
|
|
config :microwaveprop,
|
|
:propagation_scores_dir,
|
|
Path.join(System.tmp_dir!(), "microwaveprop_test_scores_#{System.unique_integer([:positive])}")
|
|
|
|
config :microwaveprop, cache_contact_count: false
|
|
config :microwaveprop, cache_contact_map: false
|
|
config :microwaveprop, elevation_req_options: [plug: {Req.Test, Microwaveprop.Terrain.ElevationClient}, retry: false]
|
|
config :microwaveprop, era5_req_options: [plug: {Req.Test, Microwaveprop.Weather.Era5Client}, retry: false]
|
|
config :microwaveprop, hrrr_req_options: [plug: {Req.Test, Microwaveprop.Weather.HrrrClient}, retry: false]
|
|
config :microwaveprop, iem_req_options: [plug: {Req.Test, Microwaveprop.Weather.IemClient}, retry: false]
|
|
|
|
# Route HTTP requests through Req.Test stubs
|
|
config :microwaveprop, nexrad_req_options: [plug: {Req.Test, Microwaveprop.Weather.NexradClient}, retry: false]
|
|
config :microwaveprop, solar_req_options: [plug: {Req.Test, Microwaveprop.Weather.SolarClient}]
|
|
config :microwaveprop, srtm_req_options: [plug: {Req.Test, Microwaveprop.Terrain.Srtm}, retry: false]
|
|
config :microwaveprop, start_freshness_monitor: false
|
|
config :microwaveprop, uwyo_req_options: [plug: {Req.Test, Microwaveprop.Weather.UwyoSoundingClient}, retry: false]
|
|
|
|
# Initialize plugs at runtime for faster test compilation
|
|
config :phoenix, :plug_init_mode, :runtime
|
|
|
|
# Sort query params output of verified routes for robust url comparisons
|
|
config :phoenix,
|
|
sort_verified_routes_query_params: true
|
|
|
|
# Enable helpful, but potentially expensive runtime checks
|
|
config :phoenix_live_view,
|
|
enable_expensive_runtime_checks: true
|
|
|
|
# Disable swoosh api client as it is only required for production adapters
|
|
config :swoosh, :api_client, false
|