Graham McIntire
48833f15be
For large CSVs (e.g. the 34k-row ARRL dump), the synchronous commit
path blocked for minutes and a crash mid-commit left no audit trail.
Refactor to run inserts + enrichment enqueues via Oban.
- New import_runs table: stores preview rows + running counters (total,
processed, imported, refined, error) + status + errors map + timing.
- New :contact_import Oban queue (limit 4) + ContactImportWorker.
Args: {import_run_id, offset, limit}. Each chunk (100 rows) inserts
its slice via CsvImport.commit_rows/1, atomically increments counters
in one update_all, flips status, and broadcasts progress on
'csv_import:<id>' PubSub topic.
- CsvImport.commit_rows/1 extracted from commit/1 (back-compat preserved).
New CsvImport.enqueue/2 persists the run + dispatches N chunk jobs.
Also: submit page copy updated from '902 MHz and up' to '50 MHz and up'
matching the band allowlist expansion done earlier.
LiveView UI for /imports/:id is a separate commit.
143 lines
4.9 KiB
Elixir
143 lines
4.9 KiB
Elixir
import Config
|
|
|
|
# Do not include metadata nor timestamps in development logs
|
|
config :logger, :default_formatter, format: "[$level] $message\n"
|
|
|
|
# Configure your database
|
|
config :microwaveprop, Microwaveprop.Repo,
|
|
username: "postgres",
|
|
password: "postgres",
|
|
hostname: "localhost",
|
|
database: "microwaveprop_dev",
|
|
stacktrace: true,
|
|
show_sensitive_data_on_connection_error: true,
|
|
pool_size: 30
|
|
|
|
# For development, we disable any cache and enable
|
|
# debugging and code reloading.
|
|
#
|
|
# The watchers configuration can be used to run external
|
|
# watchers to your application. For example, we can use it
|
|
# to bundle .js and .css sources.
|
|
config :microwaveprop, MicrowavepropWeb.Endpoint,
|
|
# Binding to loopback ipv4 address prevents access from other machines.
|
|
# Change to `ip: {0, 0, 0, 0}` to allow access from other machines.
|
|
http: [ip: {127, 0, 0, 1}],
|
|
check_origin: false,
|
|
code_reloader: true,
|
|
debug_errors: true,
|
|
secret_key_base: "XdcGVxLDrN12g/y157/MN637/UwLJyyPlnYQwXfJJQFN/KKdwnBU84onUFA9hfN7",
|
|
watchers: [
|
|
esbuild: {Esbuild, :install_and_run, [:microwaveprop, ~w(--sourcemap=inline --watch)]},
|
|
tailwind: {Tailwind, :install_and_run, [:microwaveprop, ~w(--watch)]}
|
|
]
|
|
|
|
# ## SSL Support
|
|
#
|
|
# In order to use HTTPS in development, a self-signed
|
|
# certificate can be generated by running the following
|
|
# Mix task:
|
|
#
|
|
# mix phx.gen.cert
|
|
#
|
|
# Run `mix help phx.gen.cert` for more information.
|
|
#
|
|
# The `http:` config above can be replaced with:
|
|
#
|
|
# https: [
|
|
# port: 4001,
|
|
# cipher_suite: :strong,
|
|
# keyfile: "priv/cert/selfsigned_key.pem",
|
|
# certfile: "priv/cert/selfsigned.pem"
|
|
# ],
|
|
#
|
|
# If desired, both `http:` and `https:` keys can be
|
|
# configured to run both http and https servers on
|
|
# different ports.
|
|
|
|
# Reload browser tabs when matching files change.
|
|
config :microwaveprop, MicrowavepropWeb.Endpoint,
|
|
live_reload: [
|
|
web_console_logger: true,
|
|
patterns: [
|
|
# Static assets, except user uploads
|
|
~r"priv/static/(?!uploads/).*\.(js|css|png|jpeg|jpg|gif|svg)$",
|
|
# Gettext translations
|
|
~r"priv/gettext/.*\.po$",
|
|
# Router, Controllers, LiveViews and LiveComponents
|
|
~r"lib/microwaveprop_web/router\.ex$",
|
|
~r"lib/microwaveprop_web/(controllers|live|components)/.*\.(ex|heex)$"
|
|
]
|
|
]
|
|
|
|
config :microwaveprop, Oban,
|
|
queues: [
|
|
# 2 slots so PropagationPruneWorker can run alongside the
|
|
# forecast-hour chain job.
|
|
propagation: 2,
|
|
solar: 1,
|
|
weather: 20,
|
|
enqueue: 1,
|
|
hrrr: 5,
|
|
terrain: 4,
|
|
commercial: 2,
|
|
iemre: 10,
|
|
narr: 6,
|
|
backfill_enqueue: 1,
|
|
exports: 1,
|
|
contact_import: 4
|
|
],
|
|
plugins: [
|
|
{Oban.Plugins.Pruner, max_age: 3600 * 24},
|
|
# See runtime.exs — producer-record-based orphan rescue.
|
|
{Oban.Pro.Plugins.DynamicLifeline, rescue_interval: to_timeout(second: 30)},
|
|
{Oban.Plugins.Cron,
|
|
crontab: [
|
|
{"5 * * * *", Microwaveprop.Workers.PropagationGridWorker},
|
|
{"*/2 * * * *", Microwaveprop.Workers.MrmsFetchWorker},
|
|
# AsosAdjustmentWorker disabled — see runtime.exs for rationale.
|
|
# {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker},
|
|
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
|
|
{"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker}
|
|
]}
|
|
]
|
|
|
|
# Propagation score binary files land in a gitignored tree inside the
|
|
# project root in dev, so iex sessions / PropagationGridWorker runs
|
|
# drop files where the developer can poke at them.
|
|
config :microwaveprop, :propagation_scores_dir, Path.expand("priv/dev_scores", File.cwd!())
|
|
|
|
# Enable dev routes for dashboard and mailbox
|
|
config :microwaveprop, dev_routes: true
|
|
|
|
# HRRR caching handled by nginx proxy on skippy — no local filesystem cache
|
|
|
|
# Proxy HRRR downloads through local caching nginx (set to nil to use S3 directly)
|
|
config :microwaveprop, hrrr_base_url: "http://skippy.w5isp.com:8080"
|
|
|
|
# Load ML model at startup (Nx/Axon/EXLA only available in dev/test)
|
|
config :microwaveprop, load_ml_model: true
|
|
|
|
# Use local SRTM1 tiles for elevation lookups instead of the Open-Meteo API
|
|
config :microwaveprop, srtm_tiles_dir: Path.expand("~/srtm/tiles")
|
|
|
|
# Freshness monitor enabled (hourly PropagationGridWorker runs in dev)
|
|
config :microwaveprop, start_freshness_monitor: true
|
|
|
|
# Initialize plugs at runtime for faster development compilation
|
|
config :phoenix, :plug_init_mode, :runtime
|
|
|
|
# Set a higher stacktrace during development. Avoid configuring such
|
|
# in production as building large stacktraces may be expensive.
|
|
config :phoenix, :stacktrace_depth, 20
|
|
|
|
config :phoenix_live_view,
|
|
# Include debug annotations and locations in rendered markup.
|
|
# Changing this configuration will require mix clean and a full recompile.
|
|
debug_heex_annotations: true,
|
|
debug_attributes: true,
|
|
# Enable helpful, but potentially expensive runtime checks
|
|
enable_expensive_runtime_checks: true
|
|
|
|
# Disable swoosh api client as it is only required for production adapters.
|
|
config :swoosh, :api_client, false
|