From bb86dec559b8b1243adf0343cbdf4bf17cf21090 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Mon, 26 Jan 2026 17:07:42 -0600 Subject: [PATCH] add oban pro --- CLAUDE.md | 90 + config/runtime.exs | 1 + lib/towerops/workers/device_monitor_worker.ex | 9 +- lib/towerops/workers/device_poller_worker.ex | 11 +- lib/towerops/workers/polling_offset.ex | 58 + lib/towerops_web/router.ex | 8 + mix.exs | 4 +- mix.lock | 2 + .../20260126225422_add_oban_pro.exs | 7 + .../20260126225620_add_oban_producers.exs | 5 + .../workers/device_monitor_worker_test.exs | 87 + .../workers/device_poller_worker_test.exs | 88 + test/towerops/workers/polling_offset_test.exs | 135 + vendor/README.md | 127 + vendor/oban_met/.formatter.exs | 4 + vendor/oban_met/.hex | Bin 0 -> 264 bytes vendor/oban_met/CHANGELOG.md | 85 + vendor/oban_met/LICENSE.txt | 202 ++ vendor/oban_met/README.md | 159 + vendor/oban_met/hex_metadata.config | 28 + vendor/oban_met/lib/met.ex | 325 ++ vendor/oban_met/lib/oban/met/application.ex | 83 + vendor/oban_met/lib/oban/met/cronitor.ex | 139 + vendor/oban_met/lib/oban/met/examiner.ex | 180 + vendor/oban_met/lib/oban/met/listener.ex | 149 + vendor/oban_met/lib/oban/met/migration.ex | 103 + vendor/oban_met/lib/oban/met/recorder.ex | 393 +++ vendor/oban_met/lib/oban/met/reporter.ex | 215 ++ vendor/oban_met/lib/oban/met/value.ex | 44 + vendor/oban_met/lib/oban/met/values/gauge.ex | 141 + vendor/oban_met/lib/oban/met/values/sketch.ex | 186 + vendor/oban_met/mix.exs | 131 + vendor/oban_pro/.formatter.exs | 14 + vendor/oban_pro/.hex | Bin 0 -> 264 bytes vendor/oban_pro/hex_metadata.config | 60 + vendor/oban_pro/lib/oban/pro/application.ex | 29 + vendor/oban_pro/lib/oban/pro/batch.ex | 873 +++++ vendor/oban_pro/lib/oban/pro/cloud.ex | 23 + vendor/oban_pro/lib/oban/pro/cron.ex | 125 + vendor/oban_pro/lib/oban/pro/decorator.ex | 587 ++++ vendor/oban_pro/lib/oban/pro/engines/smart.ex | 1589 +++++++++ vendor/oban_pro/lib/oban/pro/exceptions.ex | 12 + vendor/oban_pro/lib/oban/pro/flusher.ex | 24 + vendor/oban_pro/lib/oban/pro/handler.ex | 13 + vendor/oban_pro/lib/oban/pro/limiter.ex | 28 + .../oban_pro/lib/oban/pro/limiters/global.ex | 102 + .../oban_pro/lib/oban/pro/limiters/local.ex | 13 + vendor/oban_pro/lib/oban/pro/limiters/rate.ex | 125 + vendor/oban_pro/lib/oban/pro/migration.ex | 387 +++ .../pro/migrations/dynamic_partitioner.ex | 195 ++ .../lib/oban/pro/migrations/v1_0_0.ex | 97 + .../lib/oban/pro/migrations/v1_4_0.ex | 60 + .../lib/oban/pro/migrations/v1_5_0.ex | 114 + .../lib/oban/pro/migrations/v1_6_0.ex | 109 + vendor/oban_pro/lib/oban/pro/partition.ex | 195 ++ .../lib/oban/pro/plugins/dynamic_cron.ex | 821 +++++ .../lib/oban/pro/plugins/dynamic_lifeline.ex | 404 +++ .../oban/pro/plugins/dynamic_partitioner.ex | 669 ++++ .../oban/pro/plugins/dynamic_prioritizer.ex | 318 ++ .../lib/oban/pro/plugins/dynamic_pruner.ex | 692 ++++ .../lib/oban/pro/plugins/dynamic_queues.ex | 745 ++++ .../lib/oban/pro/plugins/dynamic_scaler.ex | 662 ++++ vendor/oban_pro/lib/oban/pro/producer.ex | 358 ++ vendor/oban_pro/lib/oban/pro/queue.ex | 157 + vendor/oban_pro/lib/oban/pro/refresher.ex | 111 + vendor/oban_pro/lib/oban/pro/relay.ex | 408 +++ vendor/oban_pro/lib/oban/pro/stage.ex | 36 + vendor/oban_pro/lib/oban/pro/stages/chain.ex | 232 ++ .../oban_pro/lib/oban/pro/stages/deadline.ex | 128 + .../oban_pro/lib/oban/pro/stages/encrypted.ex | 83 + vendor/oban_pro/lib/oban/pro/stages/hooks.ex | 147 + .../oban_pro/lib/oban/pro/stages/recorded.ex | 77 + .../lib/oban/pro/stages/structured.ex | 191 ++ vendor/oban_pro/lib/oban/pro/testing.ex | 1334 ++++++++ vendor/oban_pro/lib/oban/pro/unique.ex | 87 + vendor/oban_pro/lib/oban/pro/utils.ex | 159 + vendor/oban_pro/lib/oban/pro/uuidv7.ex | 42 + vendor/oban_pro/lib/oban/pro/validation.ex | 22 + vendor/oban_pro/lib/oban/pro/worker.ex | 1239 +++++++ vendor/oban_pro/lib/oban/pro/workers/batch.ex | 98 + vendor/oban_pro/lib/oban/pro/workers/chain.ex | 170 + vendor/oban_pro/lib/oban/pro/workers/chunk.ex | 735 ++++ .../oban_pro/lib/oban/pro/workers/workflow.ex | 209 ++ vendor/oban_pro/lib/oban/pro/workflow.ex | 3009 +++++++++++++++++ .../oban_pro/lib/oban/pro/workflow/cascade.ex | 87 + vendor/oban_pro/lib/pro.ex | 3 + vendor/oban_pro/mix.exs | 196 ++ vendor/oban_web/.formatter.exs | 14 + vendor/oban_web/.hex | Bin 0 -> 265 bytes vendor/oban_web/CHANGELOG.md | 335 ++ vendor/oban_web/LICENSE.txt | 201 ++ vendor/oban_web/README.md | 133 + vendor/oban_web/hex_metadata.config | 90 + .../lib/mix/tasks/oban_web.install.ex | 134 + vendor/oban_web/lib/oban/web.ex | 53 + vendor/oban_web/lib/oban/web/application.ex | 12 + vendor/oban_web/lib/oban/web/assets.ex | 50 + .../oban_web/lib/oban/web/authentication.ex | 30 + vendor/oban_web/lib/oban/web/cache.ex | 53 + .../oban_web/lib/oban/web/components/core.ex | 244 ++ .../oban_web/lib/oban/web/components/icons.ex | 877 +++++ .../lib/oban/web/components/layouts.ex | 210 ++ .../web/components/layouts/live.html.heex | 31 + .../web/components/layouts/root.html.heex | 42 + .../oban/web/components/sidebar_components.ex | 127 + .../oban_web/lib/oban/web/components/sort.ex | 84 + .../oban_web/lib/oban/web/dashboard_live.ex | 266 ++ vendor/oban_web/lib/oban/web/exceptions.ex | 5 + vendor/oban_web/lib/oban/web/helpers.ex | 302 ++ .../lib/oban/web/helpers/queue_helper.ex | 24 + .../oban/web/live/connectivity_component.ex | 59 + .../lib/oban/web/live/instances_component.ex | 94 + .../lib/oban/web/live/jobs/chart_component.ex | 298 ++ .../oban/web/live/jobs/detail_component.ex | 266 ++ .../oban/web/live/jobs/sidebar_component.ex | 85 + .../lib/oban/web/live/jobs/table_component.ex | 230 ++ .../oban/web/live/jobs/timeline_component.ex | 149 + .../oban/web/live/queues/detail_component.ex | 694 ++++ .../live/queues/detail_instance_component.ex | 129 + .../oban/web/live/queues/sidebar_component.ex | 63 + .../oban/web/live/queues/table_component.ex | 204 ++ .../lib/oban/web/live/refresh_component.ex | 118 + .../lib/oban/web/live/search_component.ex | 330 ++ .../lib/oban/web/live/shortcuts_component.ex | 112 + .../lib/oban/web/live/theme_component.ex | 102 + vendor/oban_web/lib/oban/web/page.ex | 26 + .../oban_web/lib/oban/web/pages/jobs_page.ex | 480 +++ .../lib/oban/web/pages/queues_page.ex | 394 +++ vendor/oban_web/lib/oban/web/plugins/stats.ex | 20 + .../lib/oban/web/queries/job_query.ex | 717 ++++ .../lib/oban/web/queries/queue_query.ex | 195 ++ vendor/oban_web/lib/oban/web/queue.ex | 62 + vendor/oban_web/lib/oban/web/resolver.ex | 487 +++ vendor/oban_web/lib/oban/web/router.ex | 328 ++ vendor/oban_web/lib/oban/web/search.ex | 80 + vendor/oban_web/lib/oban/web/telemetry.ex | 183 + vendor/oban_web/lib/oban/web/timing.ex | 270 ++ vendor/oban_web/mix.exs | 161 + vendor/oban_web/priv/static/app.css | 2 + vendor/oban_web/priv/static/app.js | 21 + 140 files changed, 31273 insertions(+), 5 deletions(-) create mode 100644 lib/towerops/workers/polling_offset.ex create mode 100644 priv/repo/migrations/20260126225422_add_oban_pro.exs create mode 100644 priv/repo/migrations/20260126225620_add_oban_producers.exs create mode 100644 test/towerops/workers/device_monitor_worker_test.exs create mode 100644 test/towerops/workers/device_poller_worker_test.exs create mode 100644 test/towerops/workers/polling_offset_test.exs create mode 100644 vendor/README.md create mode 100644 vendor/oban_met/.formatter.exs create mode 100644 vendor/oban_met/.hex create mode 100644 vendor/oban_met/CHANGELOG.md create mode 100644 vendor/oban_met/LICENSE.txt create mode 100644 vendor/oban_met/README.md create mode 100644 vendor/oban_met/hex_metadata.config create mode 100644 vendor/oban_met/lib/met.ex create mode 100644 vendor/oban_met/lib/oban/met/application.ex create mode 100644 vendor/oban_met/lib/oban/met/cronitor.ex create mode 100644 vendor/oban_met/lib/oban/met/examiner.ex create mode 100644 vendor/oban_met/lib/oban/met/listener.ex create mode 100644 vendor/oban_met/lib/oban/met/migration.ex create mode 100644 vendor/oban_met/lib/oban/met/recorder.ex create mode 100644 vendor/oban_met/lib/oban/met/reporter.ex create mode 100644 vendor/oban_met/lib/oban/met/value.ex create mode 100644 vendor/oban_met/lib/oban/met/values/gauge.ex create mode 100644 vendor/oban_met/lib/oban/met/values/sketch.ex create mode 100644 vendor/oban_met/mix.exs create mode 100644 vendor/oban_pro/.formatter.exs create mode 100644 vendor/oban_pro/.hex create mode 100644 vendor/oban_pro/hex_metadata.config create mode 100644 vendor/oban_pro/lib/oban/pro/application.ex create mode 100644 vendor/oban_pro/lib/oban/pro/batch.ex create mode 100644 vendor/oban_pro/lib/oban/pro/cloud.ex create mode 100644 vendor/oban_pro/lib/oban/pro/cron.ex create mode 100644 vendor/oban_pro/lib/oban/pro/decorator.ex create mode 100644 vendor/oban_pro/lib/oban/pro/engines/smart.ex create mode 100644 vendor/oban_pro/lib/oban/pro/exceptions.ex create mode 100644 vendor/oban_pro/lib/oban/pro/flusher.ex create mode 100644 vendor/oban_pro/lib/oban/pro/handler.ex create mode 100644 vendor/oban_pro/lib/oban/pro/limiter.ex create mode 100644 vendor/oban_pro/lib/oban/pro/limiters/global.ex create mode 100644 vendor/oban_pro/lib/oban/pro/limiters/local.ex create mode 100644 vendor/oban_pro/lib/oban/pro/limiters/rate.ex create mode 100644 vendor/oban_pro/lib/oban/pro/migration.ex create mode 100644 vendor/oban_pro/lib/oban/pro/migrations/dynamic_partitioner.ex create mode 100644 vendor/oban_pro/lib/oban/pro/migrations/v1_0_0.ex create mode 100644 vendor/oban_pro/lib/oban/pro/migrations/v1_4_0.ex create mode 100644 vendor/oban_pro/lib/oban/pro/migrations/v1_5_0.ex create mode 100644 vendor/oban_pro/lib/oban/pro/migrations/v1_6_0.ex create mode 100644 vendor/oban_pro/lib/oban/pro/partition.ex create mode 100644 vendor/oban_pro/lib/oban/pro/plugins/dynamic_cron.ex create mode 100644 vendor/oban_pro/lib/oban/pro/plugins/dynamic_lifeline.ex create mode 100644 vendor/oban_pro/lib/oban/pro/plugins/dynamic_partitioner.ex create mode 100644 vendor/oban_pro/lib/oban/pro/plugins/dynamic_prioritizer.ex create mode 100644 vendor/oban_pro/lib/oban/pro/plugins/dynamic_pruner.ex create mode 100644 vendor/oban_pro/lib/oban/pro/plugins/dynamic_queues.ex create mode 100644 vendor/oban_pro/lib/oban/pro/plugins/dynamic_scaler.ex create mode 100644 vendor/oban_pro/lib/oban/pro/producer.ex create mode 100644 vendor/oban_pro/lib/oban/pro/queue.ex create mode 100644 vendor/oban_pro/lib/oban/pro/refresher.ex create mode 100644 vendor/oban_pro/lib/oban/pro/relay.ex create mode 100644 vendor/oban_pro/lib/oban/pro/stage.ex create mode 100644 vendor/oban_pro/lib/oban/pro/stages/chain.ex create mode 100644 vendor/oban_pro/lib/oban/pro/stages/deadline.ex create mode 100644 vendor/oban_pro/lib/oban/pro/stages/encrypted.ex create mode 100644 vendor/oban_pro/lib/oban/pro/stages/hooks.ex create mode 100644 vendor/oban_pro/lib/oban/pro/stages/recorded.ex create mode 100644 vendor/oban_pro/lib/oban/pro/stages/structured.ex create mode 100644 vendor/oban_pro/lib/oban/pro/testing.ex create mode 100644 vendor/oban_pro/lib/oban/pro/unique.ex create mode 100644 vendor/oban_pro/lib/oban/pro/utils.ex create mode 100644 vendor/oban_pro/lib/oban/pro/uuidv7.ex create mode 100644 vendor/oban_pro/lib/oban/pro/validation.ex create mode 100644 vendor/oban_pro/lib/oban/pro/worker.ex create mode 100644 vendor/oban_pro/lib/oban/pro/workers/batch.ex create mode 100644 vendor/oban_pro/lib/oban/pro/workers/chain.ex create mode 100644 vendor/oban_pro/lib/oban/pro/workers/chunk.ex create mode 100644 vendor/oban_pro/lib/oban/pro/workers/workflow.ex create mode 100644 vendor/oban_pro/lib/oban/pro/workflow.ex create mode 100644 vendor/oban_pro/lib/oban/pro/workflow/cascade.ex create mode 100644 vendor/oban_pro/lib/pro.ex create mode 100644 vendor/oban_pro/mix.exs create mode 100644 vendor/oban_web/.formatter.exs create mode 100644 vendor/oban_web/.hex create mode 100644 vendor/oban_web/CHANGELOG.md create mode 100644 vendor/oban_web/LICENSE.txt create mode 100644 vendor/oban_web/README.md create mode 100644 vendor/oban_web/hex_metadata.config create mode 100644 vendor/oban_web/lib/mix/tasks/oban_web.install.ex create mode 100644 vendor/oban_web/lib/oban/web.ex create mode 100644 vendor/oban_web/lib/oban/web/application.ex create mode 100644 vendor/oban_web/lib/oban/web/assets.ex create mode 100644 vendor/oban_web/lib/oban/web/authentication.ex create mode 100644 vendor/oban_web/lib/oban/web/cache.ex create mode 100644 vendor/oban_web/lib/oban/web/components/core.ex create mode 100644 vendor/oban_web/lib/oban/web/components/icons.ex create mode 100644 vendor/oban_web/lib/oban/web/components/layouts.ex create mode 100644 vendor/oban_web/lib/oban/web/components/layouts/live.html.heex create mode 100644 vendor/oban_web/lib/oban/web/components/layouts/root.html.heex create mode 100644 vendor/oban_web/lib/oban/web/components/sidebar_components.ex create mode 100644 vendor/oban_web/lib/oban/web/components/sort.ex create mode 100644 vendor/oban_web/lib/oban/web/dashboard_live.ex create mode 100644 vendor/oban_web/lib/oban/web/exceptions.ex create mode 100644 vendor/oban_web/lib/oban/web/helpers.ex create mode 100644 vendor/oban_web/lib/oban/web/helpers/queue_helper.ex create mode 100644 vendor/oban_web/lib/oban/web/live/connectivity_component.ex create mode 100644 vendor/oban_web/lib/oban/web/live/instances_component.ex create mode 100644 vendor/oban_web/lib/oban/web/live/jobs/chart_component.ex create mode 100644 vendor/oban_web/lib/oban/web/live/jobs/detail_component.ex create mode 100644 vendor/oban_web/lib/oban/web/live/jobs/sidebar_component.ex create mode 100644 vendor/oban_web/lib/oban/web/live/jobs/table_component.ex create mode 100644 vendor/oban_web/lib/oban/web/live/jobs/timeline_component.ex create mode 100644 vendor/oban_web/lib/oban/web/live/queues/detail_component.ex create mode 100644 vendor/oban_web/lib/oban/web/live/queues/detail_instance_component.ex create mode 100644 vendor/oban_web/lib/oban/web/live/queues/sidebar_component.ex create mode 100644 vendor/oban_web/lib/oban/web/live/queues/table_component.ex create mode 100644 vendor/oban_web/lib/oban/web/live/refresh_component.ex create mode 100644 vendor/oban_web/lib/oban/web/live/search_component.ex create mode 100644 vendor/oban_web/lib/oban/web/live/shortcuts_component.ex create mode 100644 vendor/oban_web/lib/oban/web/live/theme_component.ex create mode 100644 vendor/oban_web/lib/oban/web/page.ex create mode 100644 vendor/oban_web/lib/oban/web/pages/jobs_page.ex create mode 100644 vendor/oban_web/lib/oban/web/pages/queues_page.ex create mode 100644 vendor/oban_web/lib/oban/web/plugins/stats.ex create mode 100644 vendor/oban_web/lib/oban/web/queries/job_query.ex create mode 100644 vendor/oban_web/lib/oban/web/queries/queue_query.ex create mode 100644 vendor/oban_web/lib/oban/web/queue.ex create mode 100644 vendor/oban_web/lib/oban/web/resolver.ex create mode 100644 vendor/oban_web/lib/oban/web/router.ex create mode 100644 vendor/oban_web/lib/oban/web/search.ex create mode 100644 vendor/oban_web/lib/oban/web/telemetry.ex create mode 100644 vendor/oban_web/lib/oban/web/timing.ex create mode 100644 vendor/oban_web/mix.exs create mode 100644 vendor/oban_web/priv/static/app.css create mode 100644 vendor/oban_web/priv/static/app.js diff --git a/CLAUDE.md b/CLAUDE.md index b1facba4..3fa310e6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -227,6 +227,38 @@ To view metrics: 2. Click the "Metrics" tab 3. Scroll to see Oban and Redis sections +### Oban Web Dashboard + +Oban Web provides a dedicated dashboard for job monitoring and management at `/oban` (superuser access required). + +**Access Control**: Only superusers can access the Oban Web dashboard. The route is protected by `:require_superuser` plug. + +**Features**: +- Real-time job queue monitoring +- Job filtering and search +- Job details and execution history +- Queue statistics and insights +- Job cancellation and retry +- Worker and plugin configuration + +**Router Configuration**: +```elixir +# lib/towerops_web/router.ex +scope "/" do + pipe_through [:browser, :require_authenticated_user, :require_superuser] + + oban_dashboard "/oban" +end +``` + +**Vendored Dependencies**: +All Oban packages are vendored in `vendor/` to avoid requiring authentication during CI/CD: +- `oban_pro` (1.6.11) - Commercial Oban Pro features +- `oban_web` (2.11.7) - Web dashboard +- `oban_met` (1.0.5) - Metrics support + +See `vendor/README.md` for update instructions. + ## Project-Specific Constraints Key constraints from AGENTS.md (see that file for complete details): @@ -379,6 +411,64 @@ The application has two separate SNMP collection mechanisms: - `NeighborCleanupWorker` deletes neighbors older than 24 hours (safety net) - This ensures neighbors stay current and topology changes are detected quickly +#### Polling Scheduling Details + +**How Polling Works** (`lib/towerops/workers/device_poller_worker.ex`): + +1. **Job Creation**: + - `DevicePollerWorker.start_polling/1` (lines 66-70) - Creates initial Oban job for a device + - Called from `Devices.create_device/1` (line 275-278) when `snmp_enabled=true` + - Called from `Devices.update_device/2` (line 379-396) when SNMP is enabled + +2. **Self-Scheduling Pattern**: + - After each poll completes, `schedule_next_poll/2` (lines 1066-1070) creates the next job + - Uses `new(schedule_in: interval_seconds)` to schedule the next run + - Interval comes from `device.check_interval_seconds` field (default 300 seconds / 5 minutes) + - Minimum interval enforced by `Application.get_env(:towerops, :snmp_min_poll_interval, 300)` + +3. **Unique Job Constraint**: + - Uses Oban unique constraint with 60-second period (lines 53-63) + - Ensures only one polling job per device exists across the cluster + - Prevents duplicate jobs if multiple pods try to schedule simultaneously + +4. **Current Problem - Synchronized Polling**: + - **All devices start polling at the same time** when first created + - Each device reschedules exactly `check_interval_seconds` after previous poll completes + - Results in "thundering herd" - all devices poll at clock intervals (1:00, 1:05, 1:10, etc.) + - This creates load spikes on the server and Oban queue + +5. **Job Recovery** (`lib/towerops/workers/job_health_check_worker.ex`): + - Runs every 10 minutes via Oban Cron + - Finds devices with `snmp_enabled=true` but no active polling job + - Re-creates missing jobs (lines 63-80) + - Provides resilience after pod failures/crashes + +**Key Configuration**: +- Default interval: 300 seconds (5 minutes) - defined in `lib/towerops/devices/device.ex:31` +- Validation: 1-3600 seconds allowed (line 95) +- Oban queue: `:pollers` with 50 concurrent workers (`config/dev.exs:39`) +- Queue concurrency can handle ~50 devices polling simultaneously + +6. **Load Distribution - Staggered Polling**: + - **Offset Calculation**: Uses deterministic hash-based offsets via `PollingOffset.calculate_offset/2` + - **Formula**: `offset = :erlang.phash2(device_id) |> rem(interval_seconds)` + - **Distribution**: Devices spread evenly across full interval (0 to interval-1 seconds) + - **Example**: With 300s interval and 100 devices: + - Device A → offset 94s → polls at :01:34, :06:34, :11:34 + - Device B → offset 171s → polls at :02:51, :07:51, :12:51 + - Device C → offset 247s → polls at :04:07, :09:07, :14:07 + - **Deterministic**: Same device always gets same offset (stable across restarts) + - **Applied to**: Both `DevicePollerWorker` (SNMP polling) and `DeviceMonitorWorker` (health checks) + - **Implementation**: `lib/towerops/workers/polling_offset.ex` + +**Related Files**: +- Offset calculator: `lib/towerops/workers/polling_offset.ex` +- Poller worker: `lib/towerops/workers/device_poller_worker.ex` +- Monitor worker: `lib/towerops/workers/device_monitor_worker.ex` +- Device context: `lib/towerops/devices.ex` (lines 264-285, 379-416) +- Device schema: `lib/towerops/devices/device.ex` +- Config: `config/dev.exs` (lines 31-61), `config/runtime.exs` (lines 103-131) + ## Kubernetes Deployment ### Prerequisites for Talos Kubernetes Cluster diff --git a/config/runtime.exs b/config/runtime.exs index c188852f..7f766d41 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -101,6 +101,7 @@ if config_env() == :prod do # Configure Oban for distributed job processing config :towerops, Oban, + engine: Oban.Pro.Engines.Smart, repo: Towerops.Repo, queues: [ default: 10, diff --git a/lib/towerops/workers/device_monitor_worker.ex b/lib/towerops/workers/device_monitor_worker.ex index a2bad7f5..cbf3cf05 100644 --- a/lib/towerops/workers/device_monitor_worker.ex +++ b/lib/towerops/workers/device_monitor_worker.ex @@ -17,6 +17,7 @@ defmodule Towerops.Workers.DeviceMonitorWorker do alias Towerops.Alerts alias Towerops.Devices alias Towerops.Monitoring + alias Towerops.Workers.PollingOffset require Logger @@ -57,8 +58,10 @@ defmodule Towerops.Workers.DeviceMonitorWorker do Starts monitoring for a device. """ def start_monitoring(device_id) do + offset = PollingOffset.calculate_offset(device_id, @monitor_interval) + %{device_id: device_id} - |> new() + |> new(schedule_in: offset) |> Oban.insert() end @@ -219,8 +222,10 @@ defmodule Towerops.Workers.DeviceMonitorWorker do end defp schedule_next_check(device_id) do + offset = PollingOffset.calculate_offset(device_id, @monitor_interval) + %{device_id: device_id} - |> new(schedule_in: @monitor_interval) + |> new(schedule_in: @monitor_interval + offset) |> Oban.insert() end end diff --git a/lib/towerops/workers/device_poller_worker.ex b/lib/towerops/workers/device_poller_worker.ex index be8828e9..49adcbb9 100644 --- a/lib/towerops/workers/device_poller_worker.ex +++ b/lib/towerops/workers/device_poller_worker.ex @@ -26,6 +26,7 @@ defmodule Towerops.Workers.DevicePollerWorker do alias Towerops.Snmp.Client alias Towerops.Snmp.MacDiscovery alias Towerops.Snmp.NeighborDiscovery + alias Towerops.Workers.PollingOffset require Logger @@ -64,8 +65,12 @@ defmodule Towerops.Workers.DevicePollerWorker do Starts polling for a device. """ def start_polling(device_id) do + device = Devices.get_device!(device_id) + interval = get_poll_interval(device) + offset = PollingOffset.calculate_offset(device_id, interval) + %{device_id: device_id} - |> new() + |> new(schedule_in: offset) |> Oban.insert() end @@ -1064,8 +1069,10 @@ defmodule Towerops.Workers.DevicePollerWorker do end defp schedule_next_poll(device_id, interval_seconds) do + offset = PollingOffset.calculate_offset(device_id, interval_seconds) + %{device_id: device_id} - |> new(schedule_in: interval_seconds) + |> new(schedule_in: interval_seconds + offset) |> Oban.insert() end diff --git a/lib/towerops/workers/polling_offset.ex b/lib/towerops/workers/polling_offset.ex new file mode 100644 index 00000000..22d7895c --- /dev/null +++ b/lib/towerops/workers/polling_offset.ex @@ -0,0 +1,58 @@ +defmodule Towerops.Workers.PollingOffset do + @moduledoc """ + Calculates deterministic polling offsets to distribute load across time. + + Uses erlang's phash2 to generate stable offsets from device IDs, + ensuring each device polls at a different time within the interval. + + This prevents the "thundering herd" problem where all devices poll + at synchronized clock intervals (e.g., 1:00, 1:05, 1:10), creating + load spikes on the server. + + ## Examples + + # Device with ID "abc-123" and 300-second interval + iex> calculate_offset("abc-123", 300) + 94 # Device polls at :01:34, :06:34, :11:34, etc. + + # Same device always gets same offset + iex> calculate_offset("abc-123", 300) + 94 + + # Different device gets different offset + iex> calculate_offset("def-456", 300) + 171 # Device polls at :02:51, :07:51, :12:51, etc. + """ + + @doc """ + Calculate polling offset for a device. + + Returns an offset in seconds between 0 and interval_seconds-1. + Same device_id always returns same offset for a given interval. + + ## Parameters + + * `device_id` - The device UUID (binary or string format) + * `interval_seconds` - The polling interval in seconds + + ## Returns + + An integer offset in the range [0, interval_seconds) + + ## Examples + + iex> calculate_offset("abc-123", 300) + 94 + + iex> calculate_offset("abc-123", 300) + 94 # Always returns same value for same device + + iex> offset = calculate_offset("some-uuid", 60) + iex> offset >= 0 and offset < 60 + true + """ + @spec calculate_offset(binary(), pos_integer()) :: non_neg_integer() + def calculate_offset(device_id, interval_seconds) do + device_id |> :erlang.phash2() |> rem(interval_seconds) + end +end diff --git a/lib/towerops_web/router.ex b/lib/towerops_web/router.ex index a391c4e0..e723e7d1 100644 --- a/lib/towerops_web/router.ex +++ b/lib/towerops_web/router.ex @@ -4,6 +4,7 @@ defmodule ToweropsWeb.Router do use ToweropsWeb, :router use Honeybadger.Plug + import Oban.Web.Router import Phoenix.LiveDashboard.Router import ToweropsWeb.UserAuth @@ -118,6 +119,13 @@ defmodule ToweropsWeb.Router do ] end + # Enable Oban Web (superuser only) + scope "/" do + pipe_through [:browser, :require_authenticated_user, :require_superuser] + + oban_dashboard("/oban") + end + # Enable Swoosh mailbox preview in development if Application.compile_env(:towerops, :dev_routes) do scope "/dev" do diff --git a/mix.exs b/mix.exs index 20319a05..99d5b2ca 100644 --- a/mix.exs +++ b/mix.exs @@ -67,7 +67,9 @@ defmodule Towerops.MixProject do {:jason, "~> 1.2"}, {:dns_cluster, "~> 0.2.0"}, {:libcluster, "~> 3.4"}, - {:oban, "~> 2.18"}, + {:oban_pro, "~> 1.6", path: "vendor/oban_pro"}, + {:oban_met, "~> 1.0", path: "vendor/oban_met", override: true}, + {:oban_web, "~> 2.11", path: "vendor/oban_web"}, {:bandit, "~> 1.5"}, {:phoenix_pubsub_redis, "~> 3.0"}, {:ecto_psql_extras, "~> 0.6"}, diff --git a/mix.lock b/mix.lock index d8e41840..a6a13ff3 100644 --- a/mix.lock +++ b/mix.lock @@ -38,6 +38,8 @@ "nimble_ownership": {:hex, :nimble_ownership, "1.0.2", "fa8a6f2d8c592ad4d79b2ca617473c6aefd5869abfa02563a77682038bf916cf", [:mix], [], "hexpm", "098af64e1f6f8609c6672127cfe9e9590a5d3fcdd82bc17a377b8692fd81a879"}, "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, "oban": {:hex, :oban, "2.20.3", "e4d27336941955886cc7113420c32c63b70b64f10b27e08e3cf2b001153953cd", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.20", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "075ffbf1279a96bec495bc63d647b08929837d70bcc0427249ffe4d1dddaec33"}, + "oban_met": {:hex, :oban_met, "1.0.5", "bb633ab06448dab2ef9194f6688d33b3d07fc3f2ad793a1a08f4dfbb2cc9fe50", [:mix], [{:oban, "~> 2.19", [hex: :oban, repo: "hexpm", optional: false]}], "hexpm", "64664d50805bbfd3903aeada1f3c39634652a87844797ee400b0bcc95a28f5ea"}, + "oban_web": {:hex, :oban_web, "2.11.7", "a998868bb32b3d3c44f328a8baec820af5c3b46a3a743d89c169e75b0bf7efcc", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:oban, "~> 2.19", [hex: :oban, repo: "hexpm", optional: false]}, {:oban_met, "~> 1.0", [hex: :oban_met, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.7", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}], "hexpm", "f0ef9ec8f97381a0287c60a732736ff77bc351cbbe82eceffb5a635f84788656"}, "phoenix": {:hex, :phoenix, "1.8.3", "49ac5e485083cb1495a905e47eb554277bdd9c65ccb4fc5100306b350151aa95", [:mix], [{:bandit, "~> 1.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "36169f95cc2e155b78be93d9590acc3f462f1e5438db06e6248613f27c80caec"}, "phoenix_ecto": {:hex, :phoenix_ecto, "4.7.0", "75c4b9dfb3efdc42aec2bd5f8bccd978aca0651dbcbc7a3f362ea5d9d43153c6", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "1d75011e4254cb4ddf823e81823a9629559a1be93b4321a6a5f11a5306fbf4cc"}, "phoenix_html": {:hex, :phoenix_html, "4.3.0", "d3577a5df4b6954cd7890c84d955c470b5310bb49647f0a114a6eeecc850f7ad", [:mix], [], "hexpm", "3eaa290a78bab0f075f791a46a981bbe769d94bc776869f4f3063a14f30497ad"}, diff --git a/priv/repo/migrations/20260126225422_add_oban_pro.exs b/priv/repo/migrations/20260126225422_add_oban_pro.exs new file mode 100644 index 00000000..3c1778ad --- /dev/null +++ b/priv/repo/migrations/20260126225422_add_oban_pro.exs @@ -0,0 +1,7 @@ +defmodule Towerops.Repo.Migrations.AddObanPro do + use Ecto.Migration + + def up, do: Oban.Pro.Migration.up() + + def down, do: Oban.Pro.Migration.down() +end diff --git a/priv/repo/migrations/20260126225620_add_oban_producers.exs b/priv/repo/migrations/20260126225620_add_oban_producers.exs new file mode 100644 index 00000000..143965f7 --- /dev/null +++ b/priv/repo/migrations/20260126225620_add_oban_producers.exs @@ -0,0 +1,5 @@ +defmodule Towerops.Repo.Migrations.AddObanProducers do + use Ecto.Migration + + defdelegate change, to: Oban.Pro.Migrations.Producers +end diff --git a/test/towerops/workers/device_monitor_worker_test.exs b/test/towerops/workers/device_monitor_worker_test.exs new file mode 100644 index 00000000..370f563d --- /dev/null +++ b/test/towerops/workers/device_monitor_worker_test.exs @@ -0,0 +1,87 @@ +defmodule Towerops.Workers.DeviceMonitorWorkerTest do + use Towerops.DataCase, async: false + + import Towerops.AccountsFixtures + + alias Towerops.Devices + alias Towerops.Organizations + alias Towerops.Sites + alias Towerops.Workers.DeviceMonitorWorker + alias Towerops.Workers.PollingOffset + + setup do + user = user_fixture() + {:ok, organization} = Organizations.create_organization(%{name: "Test Org"}, user.id) + + {:ok, site} = + Sites.create_site(%{ + name: "Test Site", + organization_id: organization.id + }) + + %{organization: organization, site: site, user: user} + end + + describe "start_monitoring/1" do + test "schedules initial job with offset", %{site: site} do + {:ok, device} = + Devices.create_device(%{ + name: "Router 1", + ip_address: "192.168.1.1", + site_id: site.id, + monitoring_enabled: true + }) + + # Get the expected offset (30 seconds is the monitor interval) + expected_offset = PollingOffset.calculate_offset(device.id, 30) + + # Start monitoring + assert {:ok, job} = DeviceMonitorWorker.start_monitoring(device.id) + + # Verify job is scheduled with offset + assert job.args["device_id"] == device.id + assert job.scheduled_at + + # Calculate the delay (scheduled_at - inserted_at) + delay_seconds = DateTime.diff(job.scheduled_at, job.inserted_at, :second) + assert delay_seconds == expected_offset + end + + test "offset is within monitor interval bounds", %{site: site} do + {:ok, device} = + Devices.create_device(%{ + name: "Router 2", + ip_address: "192.168.1.2", + site_id: site.id, + monitoring_enabled: true + }) + + # Start monitoring + assert {:ok, job} = DeviceMonitorWorker.start_monitoring(device.id) + + # Verify offset is within bounds + delay_seconds = DateTime.diff(job.scheduled_at, job.inserted_at, :second) + assert delay_seconds >= 0 + assert delay_seconds < 30 + end + end + + describe "stop_monitoring/1" do + test "cancels all jobs for device", %{site: site} do + {:ok, device} = + Devices.create_device(%{ + name: "Router 3", + ip_address: "192.168.1.3", + site_id: site.id, + monitoring_enabled: true + }) + + # Start monitoring + assert {:ok, _job} = DeviceMonitorWorker.start_monitoring(device.id) + + # Stop monitoring + assert {:ok, cancelled_jobs} = DeviceMonitorWorker.stop_monitoring(device.id) + refute cancelled_jobs == [] + end + end +end diff --git a/test/towerops/workers/device_poller_worker_test.exs b/test/towerops/workers/device_poller_worker_test.exs new file mode 100644 index 00000000..e30c8375 --- /dev/null +++ b/test/towerops/workers/device_poller_worker_test.exs @@ -0,0 +1,88 @@ +defmodule Towerops.Workers.DevicePollerWorkerTest do + use Towerops.DataCase, async: false + + import Towerops.AccountsFixtures + + alias Towerops.Devices + alias Towerops.Organizations + alias Towerops.Sites + alias Towerops.Workers.DevicePollerWorker + alias Towerops.Workers.PollingOffset + + setup do + user = user_fixture() + {:ok, organization} = Organizations.create_organization(%{name: "Test Org"}, user.id) + + {:ok, site} = + Sites.create_site(%{ + name: "Test Site", + organization_id: organization.id + }) + + %{organization: organization, site: site, user: user} + end + + describe "start_polling/1" do + test "schedules initial job with offset", %{site: site} do + {:ok, device} = + Devices.create_device(%{ + name: "Router 1", + ip_address: "192.168.1.1", + site_id: site.id, + check_interval_seconds: 300 + }) + + # Get the expected offset + expected_offset = PollingOffset.calculate_offset(device.id, 300) + + # Start polling + assert {:ok, job} = DevicePollerWorker.start_polling(device.id) + + # Verify job is scheduled with offset + assert job.args["device_id"] == device.id + assert job.scheduled_at + + # Calculate the delay (scheduled_at - inserted_at) + delay_seconds = DateTime.diff(job.scheduled_at, job.inserted_at, :second) + assert delay_seconds == expected_offset + end + + test "schedules initial job with offset for different interval", %{site: site} do + {:ok, device} = + Devices.create_device(%{ + name: "Router 2", + ip_address: "192.168.1.2", + site_id: site.id, + check_interval_seconds: 600 + }) + + # Get the expected offset + expected_offset = PollingOffset.calculate_offset(device.id, 600) + + # Start polling + assert {:ok, job} = DevicePollerWorker.start_polling(device.id) + + # Verify job is scheduled with offset + delay_seconds = DateTime.diff(job.scheduled_at, job.inserted_at, :second) + assert delay_seconds == expected_offset + end + end + + describe "stop_polling/1" do + test "cancels all jobs for device", %{site: site} do + {:ok, device} = + Devices.create_device(%{ + name: "Router 3", + ip_address: "192.168.1.3", + site_id: site.id + }) + + # Start polling + assert {:ok, _job} = DevicePollerWorker.start_polling(device.id) + + # Stop polling + assert {:ok, cancelled_jobs} = DevicePollerWorker.stop_polling(device.id) + refute cancelled_jobs == [] + end + end +end diff --git a/test/towerops/workers/polling_offset_test.exs b/test/towerops/workers/polling_offset_test.exs new file mode 100644 index 00000000..cce98176 --- /dev/null +++ b/test/towerops/workers/polling_offset_test.exs @@ -0,0 +1,135 @@ +defmodule Towerops.Workers.PollingOffsetTest do + use ExUnit.Case, async: true + + alias Towerops.Workers.PollingOffset + + describe "calculate_offset/2" do + test "returns value within interval bounds" do + device_id = Ecto.UUID.generate() + offset = PollingOffset.calculate_offset(device_id, 300) + + assert offset >= 0 + assert offset < 300 + end + + test "returns value within bounds for different intervals" do + device_id = Ecto.UUID.generate() + + # Test various intervals + for interval <- [60, 120, 300, 600, 3600] do + offset = PollingOffset.calculate_offset(device_id, interval) + assert offset >= 0, "offset should be >= 0 for interval #{interval}" + assert offset < interval, "offset should be < #{interval} for interval #{interval}" + end + end + + test "is deterministic - same device_id always returns same offset" do + device_id = Ecto.UUID.generate() + + offset1 = PollingOffset.calculate_offset(device_id, 300) + offset2 = PollingOffset.calculate_offset(device_id, 300) + offset3 = PollingOffset.calculate_offset(device_id, 300) + + assert offset1 == offset2 + assert offset2 == offset3 + end + + test "different devices get different offsets" do + id1 = Ecto.UUID.generate() + id2 = Ecto.UUID.generate() + id3 = Ecto.UUID.generate() + + offset1 = PollingOffset.calculate_offset(id1, 300) + offset2 = PollingOffset.calculate_offset(id2, 300) + offset3 = PollingOffset.calculate_offset(id3, 300) + + # Very unlikely all three would be equal with good hash distribution + # At least two should be different + unique_offsets = Enum.uniq([offset1, offset2, offset3]) + assert length(unique_offsets) >= 2, "expected at least 2 different offsets from 3 devices" + end + + test "offset changes with different intervals for same device" do + device_id = Ecto.UUID.generate() + + offset_60 = PollingOffset.calculate_offset(device_id, 60) + offset_300 = PollingOffset.calculate_offset(device_id, 300) + offset_600 = PollingOffset.calculate_offset(device_id, 600) + + # Verify all within their respective bounds + assert offset_60 < 60 + assert offset_300 < 300 + assert offset_600 < 600 + + # They're likely different (not guaranteed, but very likely) + # Just verify bounds are correct + end + + test "handles minimum interval (1 second)" do + device_id = Ecto.UUID.generate() + offset = PollingOffset.calculate_offset(device_id, 1) + + assert offset == 0, "with interval of 1, offset must be 0" + end + + test "distributes devices across interval buckets" do + # Generate 100 device IDs + device_ids = for _ <- 1..100, do: Ecto.UUID.generate() + + # Calculate offsets with 300 second interval + offsets = Enum.map(device_ids, &PollingOffset.calculate_offset(&1, 300)) + + # Divide 300 seconds into 10 buckets (0-29, 30-59, 60-89, etc.) + buckets = Enum.group_by(offsets, &div(&1, 30)) + + # With good distribution, each bucket should have roughly 10 devices (100/10) + # Allow for variance - expect 3-20 devices per bucket (hash functions have natural variance) + for {bucket_num, devices_in_bucket} <- buckets do + count = length(devices_in_bucket) + + assert count >= 3, + "bucket #{bucket_num} has only #{count} devices, expected at least 3" + + assert count <= 20, + "bucket #{bucket_num} has #{count} devices, expected at most 20" + end + + # Verify we're actually using multiple buckets (good distribution) + assert map_size(buckets) >= 8, "expected at least 8 of 10 buckets to be used" + end + + test "distributes devices across full interval range" do + # Generate many device IDs + device_ids = for _ <- 1..1000, do: Ecto.UUID.generate() + + # Calculate offsets + offsets = Enum.map(device_ids, &PollingOffset.calculate_offset(&1, 300)) + + # Verify we see offsets across the full range + min_offset = Enum.min(offsets) + max_offset = Enum.max(offsets) + + # With 1000 devices, we should see offsets near both ends + assert min_offset < 30, "expected some offsets near 0, got min #{min_offset}" + assert max_offset > 270, "expected some offsets near 299, got max #{max_offset}" + end + + test "handles binary UUID format" do + # Test with binary UUID (which is what Device.id actually is) + binary_uuid = Ecto.UUID.dump!(Ecto.UUID.generate()) + offset = PollingOffset.calculate_offset(binary_uuid, 300) + + assert offset >= 0 + assert offset < 300 + end + + test "handles string UUID format" do + # Test with string UUID + string_uuid = Ecto.UUID.generate() + offset = PollingOffset.calculate_offset(string_uuid, 300) + + assert offset >= 0 + assert offset < 300 + end + end +end diff --git a/vendor/README.md b/vendor/README.md new file mode 100644 index 00000000..26e2c6a5 --- /dev/null +++ b/vendor/README.md @@ -0,0 +1,127 @@ +# Vendored Dependencies + +This directory contains vendored Elixir dependencies that are included directly in the repository instead of being fetched from external sources during build/CI. + +## Why Vendor? + +These dependencies are vendored to: +- Avoid requiring authentication during CI/CD builds +- Ensure consistent builds without external dependencies +- Enable offline development + +## Vendored Packages + +### Oban Pro + +**Version**: 1.6.11 +**Source**: https://getoban.pro/repo (private) +**License**: Commercial (licensed) + +Advanced Oban features including batching, workflows, dynamic partitioning, and more. + +**Configuration in mix.exs**: +```elixir +{:oban_pro, "~> 1.6", path: "vendor/oban_pro"} +``` + +### Oban Web + +**Version**: 2.11.7 +**Source**: https://hex.pm/packages/oban_web +**License**: Apache-2.0 + +Web dashboard for Oban job monitoring and management. + +**Configuration in mix.exs**: +```elixir +{:oban_web, "~> 2.11", path: "vendor/oban_web"} +``` + +**Router Configuration** (superuser only): +```elixir +# lib/towerops_web/router.ex +scope "/" do + pipe_through [:browser, :require_authenticated_user, :require_superuser] + + oban_dashboard "/oban" +end +``` + +### Oban Met + +**Version**: 1.0.5 +**Source**: https://hex.pm/packages/oban_met +**License**: Apache-2.0 + +Metrics and telemetry support for Oban Web (required dependency). + +**Configuration in mix.exs**: +```elixir +{:oban_met, "~> 1.0", path: "vendor/oban_met", override: true} +``` + +## Updating Vendored Dependencies + +### Step 1: Configure Oban Repository (if needed) + +```bash +mix hex.repo add oban https://getoban.pro/repo \ + --fetch-public-key SHA256:4/OSKi0NRF91QVVXlGAhb/BIMLnK8NHcx/EWs+aIWPc \ + --auth-key +``` + +### Step 2: Update Dependencies + +```bash +# Update version constraints in mix.exs if desired +# Then fetch latest versions +mix deps.unlock oban_pro oban_web oban_met +mix deps.get + +# Verify versions +mix deps | grep oban +``` + +### Step 3: Update Vendored Code + +```bash +# Remove old vendored versions +rm -rf vendor/oban_pro vendor/oban_web vendor/oban_met + +# Copy new versions from deps +cp -r deps/oban_pro vendor/ +cp -r deps/oban_web vendor/ +cp -r deps/oban_met vendor/ + +# Compile and test +mix deps.compile +mix test +``` + +### Step 4: Commit Changes + +```bash +git add vendor/ +git commit -m "Update vendored Oban packages + +- oban_pro: 1.6.x → 1.6.y +- oban_web: 2.11.x → 2.11.y +- oban_met: 1.0.x → 1.0.y +" +``` + +## Verifying Vendored Dependencies + +To verify that vendored dependencies are being used correctly: + +```bash +# Check that path dependencies are recognized +mix deps + +# Compile from scratch +mix deps.clean --all +mix deps.compile + +# Run tests +mix test +``` diff --git a/vendor/oban_met/.formatter.exs b/vendor/oban_met/.formatter.exs new file mode 100644 index 00000000..1b79ee44 --- /dev/null +++ b/vendor/oban_met/.formatter.exs @@ -0,0 +1,4 @@ +[ + import_deps: [:ecto_sql, :oban, :stream_data], + inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] +] diff --git a/vendor/oban_met/.hex b/vendor/oban_met/.hex new file mode 100644 index 0000000000000000000000000000000000000000..45983a567be5d3e6e7ca0b0fb0651bd2007770df GIT binary patch literal 264 zcmZ9EK~lpo5ClyeAg4Sfwbn|qbmJ$fU1^04NKQEp%AJp+aO5^K)zd%IZt814$(?r8 zZC%&B)P12Wxy8f6RL)tp`uZ*FbzaJ2eI8D5GIhVoW%=vve985YsnC1Fz8dpX`#j%2 z>Txbb)^nI7d|q$)fxj_ZL}IX7yGX1g`N8XCt#c8PF~XPvCPi`dNCSaOIwocnf=^ + + Hex Version + + + + Hex Docs + + + + CI Status + + + + Apache 2 License + +

+ + + +Met is a distributed, compacting, multidimensional, telemetry-powered time series datastore for +Oban that requires no configuration. It gathers data for queues, job counts, execution metrics, +active crontabs, historic metrics, and more. + +Met powers the charts and runtime details shown in the [Oban Web][web] dashboard. + +[web]: https://github.com/oban-bg/oban_web + +## Features + +- **🤖 Autonomous** - Supervises a collection of autonomous modules that dynamically start and stop alongside + Oban instances without any code changes. + +- **🎩 Distributed** - Metrics are shared between all connected nodes via pubsub. Leadership is used + to restrict expensive operations, such as performing counts, to a single node. + +- **📼 Recorded** - Telemetry events and scraped data are stored in-memory as time series data. + Values are stored as either gauges or space efficient "sketches". + +- **🪐 Multidimensional** - Metrics are stored with labels such as `node`, `queue`, `worker`, etc. + that can be filtered and grouped dynamically at runtime. + +- **🗜️ Compacting** - Time series values are periodically compacted into larger windows of time to + save space and optimize querying historic data. Compaction periods use safe defaults, but are + configurable. + +- **✏️ Estimating** - In supporting systems (Postgres), count queries use optimized estimates + automatically for tables with a large number of jobs. + +- **🔎 Queryable** - Historic metrics may be filtered and grouped by any label, sliced by + arbitrary time intervals, and numeric values aggregated at dynamic percentiles (e.g. P50, P99) + without pre-computed histogram buckets. + +- **🤝 Handoff** - Ephemeral data storage via data replication with handoff between nodes. All nodes have a + shared view of the cluster's data and new nodes are caught up when they come online. + +## Installation + +Oban Met is included with Oban Web and manual installation is only necessary in hybrid +environments (separate Web and Worker nodes). + +To receive metrics from non-web nodes in a system with separate "web" and "worker" applications +you must explicitly include `oban_met` as a dependency for "workers". + +```elixir +{:oban_met, "~> 1.0"} +``` + +## Usage + +No configuration is necessary and Oban Met will start automatically in a typical application. A +variety of options are provided for more complex or nuanced usage. + +### Auto Start + +Supervised Met instances start automatically along with Oban instances unless Oban is in testing +mode. You can disable auto-starting globally with application configuration: + +```elixir +config :oban_met, auto_start: false +``` + +Then, start instances as a child directly within your Oban app's plugins: + +```elixir +plugins: [ + Oban.Met, + ... +] +``` + +### Customizing Estimates + +Options for internal `Oban.Met` processes can be overridden from the plugin specification. Most +options are internal and not meant to be overridden, but one particularly useful option to tune is +the `estimate_limit`. The `estimate_limit` determines at which point state/queue counts switch +from using an accurate `count(*)` call to a much more efficient, but less accurate, estimate +function. + +The default limit is a conservative 50k, which may be too low for systems with insert spikes. This +declares an override to set the limit to 200k: + +```elixir +{Oban.Met, reporter: [estimate_limit: 200_000]} +``` + +### Explicit Migrations + +Met will create the necessary estimate function automatically when possible. The migration isn't +necessary under normal circumstances, but is provided to avoid permission issues or allow full +control over database changes. + +```bash +mix ecto.gen.migration add_oban_met +``` + +Open the generated migration and delegate the `up/0` and `down/0` functions to +`Oban.Met.Migration`: + +```elixir +defmodule MyApp.Repo.Migrations.AddObanMet do + use Ecto.Migration + + def up, do: Oban.Met.Migration.up() + def down, do: Oban.Met.Migration.down() +end +``` + +Then, after disabling auto-start, configure the reporter not to auto-migrate if you run the +explicit migration: + +```elixir +{Oban.Met, reporter: [auto_migrate: false]} +``` + + + +## Contributing + +To run the test suite you must have PostgreSQL 12+. Once dependencies are installed, setup the +databases and run necessary migrations: + +```bash +mix test.setup +``` + +## Community + +There are a few places to connect and communicate with other Oban users: + +- Ask questions and discuss *#oban* on the [Elixir Forum][forum] +- [Request an invitation][invite] and join the *#oban* channel on Slack +- Learn about bug reports and upcoming features in the [issue tracker][issues] + +[invite]: https://elixir-slack.community/ +[forum]: https://elixirforum.com/ +[issues]: https://github.com/sorentwo/oban/issues diff --git a/vendor/oban_met/hex_metadata.config b/vendor/oban_met/hex_metadata.config new file mode 100644 index 00000000..fb388b4e --- /dev/null +++ b/vendor/oban_met/hex_metadata.config @@ -0,0 +1,28 @@ +{<<"links">>, + [{<<"Website">>,<<"https://oban.pro">>}, + {<<"Changelog">>, + <<"https://github.com/oban-bg/oban_met/blob/main/CHANGELOG.md">>}, + {<<"GitHub">>,<<"https://github.com/oban-bg/oban_met">>}]}. +{<<"name">>,<<"oban_met">>}. +{<<"version">>,<<"1.0.5">>}. +{<<"description">>, + <<"A distributed, compacting, multidimensional, telemetry-powered time series datastore">>}. +{<<"elixir">>,<<"~> 1.15">>}. +{<<"files">>, + [<<"lib">>,<<"lib/oban">>,<<"lib/oban/met">>,<<"lib/oban/met/examiner.ex">>, + <<"lib/oban/met/reporter.ex">>,<<"lib/oban/met/recorder.ex">>, + <<"lib/oban/met/cronitor.ex">>,<<"lib/oban/met/values">>, + <<"lib/oban/met/values/gauge.ex">>,<<"lib/oban/met/values/sketch.ex">>, + <<"lib/oban/met/value.ex">>,<<"lib/oban/met/migration.ex">>, + <<"lib/oban/met/listener.ex">>,<<"lib/oban/met/application.ex">>, + <<"lib/met.ex">>,<<".formatter.exs">>,<<"mix.exs">>,<<"README.md">>, + <<"CHANGELOG.md">>,<<"LICENSE.txt">>]}. +{<<"app">>,<<"oban_met">>}. +{<<"licenses">>,[<<"Apache-2.0">>]}. +{<<"requirements">>, + [[{<<"name">>,<<"oban">>}, + {<<"app">>,<<"oban">>}, + {<<"optional">>,false}, + {<<"requirement">>,<<"~> 2.19">>}, + {<<"repository">>,<<"hexpm">>}]]}. +{<<"build_tools">>,[<<"mix">>]}. diff --git a/vendor/oban_met/lib/met.ex b/vendor/oban_met/lib/met.ex new file mode 100644 index 00000000..bdd56adb --- /dev/null +++ b/vendor/oban_met/lib/met.ex @@ -0,0 +1,325 @@ +defmodule Oban.Met do + @external_resource readme = Path.join([__DIR__, "../README.md"]) + + @moduledoc readme + |> File.read!() + |> String.split("") + |> Enum.fetch!(1) + + use Supervisor + + alias Oban.Met.{Cronitor, Examiner, Listener, Recorder, Reporter, Value} + alias Oban.Registry + + @type counts :: %{optional(String.t()) => non_neg_integer()} + @type sub_counts :: %{optional(String.t()) => non_neg_integer() | counts()} + + @type series :: atom() | String.t() + @type value :: Value.t() + @type label :: String.t() + @type ts :: integer() + + @type filter_value :: label() | [label()] + + @type series_detail :: %{series: series(), labels: [label()], value: module()} + + @type operation :: :max | :sum | {:pct, float()} + + @type latest_opts :: [ + filters: keyword(filter_value()), + group: nil | label(), + lookback: pos_integer() + ] + + @type timeslice_opts :: [ + by: pos_integer(), + filters: keyword(filter_value()), + group: nil | label(), + label: nil | label(), + lookback: pos_integer(), + operation: operation(), + since: pos_integer() + ] + + @doc false + @spec child_spec(Keyword.t()) :: Supervisor.child_spec() + def child_spec(opts) do + conf = Keyword.fetch!(opts, :conf) + name = Registry.via(conf.name, __MODULE__) + + opts + |> super() + |> Map.put(:id, name) + end + + @doc """ + Start a Met supervisor for an Oban instance. + + `Oban.Met` typically starts supervisors automatically when Oban instances initialize. However, + starting a supervisor manually can be used if `auto_start` is disabled. + + ## Options + + These options are required; without them the supervisor won't start: + + * `:conf` — configuration for a running Oban instance, required + + * `:name` — an optional name for the supervisor, defaults to `Oban.Met` + + ## Example + + Start a supervisor for the default Oban instance: + + Oban.Met.start_link(conf: Oban.config()) + + Start a supervisor with a custom name: + + Oban.Met.start_link(conf: Oban.config(), name: MyApp.MetSup) + """ + @spec start_link(Keyword.t()) :: Supervisor.on_start() + def start_link(opts) when is_list(opts) do + conf = Keyword.fetch!(opts, :conf) + name = Registry.via(conf.name, __MODULE__) + + Supervisor.start_link(__MODULE__, opts, name: name) + end + + @doc """ + Retrieve stored producer checks. + + This mimics the output of the legacy `Oban.Web.Plugins.Stats.all_gossip/1` function. + + Checks are queried approximately every second and broadcast to all connected nodes, so each node + is a replica of checks from the entire cluster. Checks are stored for 30 seconds before being + purged. + + ## Output + + Checks are the result of `Oban.check_queue/1`, and the exact contents depends on which + `Oban.Engine` is in use. A `Basic` engine check will look similar to this: + + %{ + uuid: "2dde4c0f-53b8-4f59-9a16-a9487454292d", + limit: 10, + node: "me@local", + paused: false, + queue: "default", + running: [100, 102], + started_at: ~D[2020-10-07 15:31:00], + updated_at: ~D[2020-10-07 15:31:00] + } + + ## Examples + + Get all current checks: + + Oban.Met.checks() + + Get current checks for a non-standard Oban isntance: + + Oban.Met.checks(MyOban) + """ + @spec checks(Oban.name()) :: [map()] + def checks(oban \\ Oban) do + oban + |> Registry.via(Examiner) + |> Examiner.all_checks() + end + + @doc """ + Get a normalized, unified crontab from all connected nodes. + + ## Examples + + Get a merged crontab: + + Oban.Met.crontab() + [ + {"* * * * *", "Worker.A", []}, + {"* * * * *", "Worker.B", [["args", %{"mode" => "foo"}]]} + ] + + Get the crontab for a non-standard Oban instance: + + Oban.Met.crontab(MyOban) + """ + @spec crontab(Oban.name()) :: [{binary(), binary(), map()}] + def crontab(oban \\ Oban) do + oban + |> Registry.via(Cronitor) + |> Cronitor.merged_crontab() + end + + @doc """ + Get all stored, unique values for a particular label. + + ## Examples + + Get all known queues: + + Oban.Met.labels("queue") + ~w(alpha gamma delta) + + Get all known workers: + + Oban.Met.labels("worker") + ~w(MyApp.Worker MyApp.OtherWorker) + """ + @spec labels(Oban.name(), label(), keyword()) :: [label()] + def labels(oban \\ Oban, label, opts \\ []) do + oban + |> Registry.via(Recorder) + |> Recorder.labels(to_string(label), opts) + end + + @doc """ + Get all stored values for a series without any filtering. + """ + @spec lookup(Oban.name(), series()) :: [term()] + def lookup(oban \\ Oban, series) do + oban + |> Registry.via(Recorder) + |> Recorder.lookup(to_string(series)) + end + + @doc """ + Get the latest values for a gauge series, optionally subdivided by a label. + + Unlike queues and workers, states are static and constant, so they'll always show up in the + counts or subdivision maps. + + ## Gauge Series + + Latest counts only apply to `Gauge` series. There are two gauges available (as reported by + `series/1`: + + * `:exec_count` — jobs executing at that moment, including `node`, `queue`, `state`, and + `worker` labels. + + * `:full_count` — jobs in the database, including `queue`, and `state` labels. + + ## Examples + + Get the `:full_count` value without any grouping: + + Oban.Met.latest(:full_count) + %{"all" => 99} + + Group the `:full_count` value by state: + + Oban.Met.latest(:full_count, group: "state") + %{"available" => 9, "completed" => 80, "executing" => 5, ... + + Group results by queue: + + Oban.Met.latest(:exec_count, group: "queue") + %{"alpha" => 9, "gamma" => 3} + + Group results by node: + + Oban.Met.latest(:exec_count, group: "node") + %{"worker.1" => 6, "worker.2" => 5} + + Filter values by node: + + Oban.Met.latest(:exec_count, filters: [node: "worker.1"]) + %{"all" => 6} + + Filter values by queue and state: + + Oban.Met.latest(:exec_count, filters: [node: "worker.1", "worker.2"]) + """ + @spec latest(Oban.name(), series()) :: counts() | sub_counts() + def latest(oban \\ Oban, series, opts \\ []) do + oban + |> Registry.via(Recorder) + |> Recorder.latest(to_string(series), opts) + end + + @doc """ + List all recorded series along with their labels and value type. + + ## Examples + + Oban.Met.series() + [ + %{series: "exec_time", labels: ["state", "queue", "worker"], value: Sketch}, + %{series: "wait_time", labels: ["state", "queue", "worker"], value: Sketch}, + %{series: "exec_count", labels: ["state", "queue", "worker"], value: Gauge}, + %{series: "full_count", labels: ["state", "queue"], value: Gauge} + ] + """ + @spec series(Oban.name()) :: [series_detail()] + def series(oban \\ Oban) do + oban + |> Registry.via(Recorder) + |> Recorder.series() + end + + @doc """ + Summarize a series of data with an aggregate over a configurable window of time. + + ## Examples + + Retreive a 3 second timeslice of the `exec_time` sketch: + + Oban.Met.timeslice(Oban, :exec_time, lookback: 3) + [ + {2, 16771374649.128689, nil}, + {1, 24040058779.3428, nil}, + {0, 22191534459.516357, nil}, + ] + + Group `exec_time` slices by the `queue` label: + + Oban.Met.timeslice(Oban, :exec_time, group: "queue") + [ + {1, 9970235387.031698, "analysis"}, + {0, 11700429279.446463, "analysis"}, + {1, 23097311376.231316, "default"}, + {0, 23097311376.231316, "default"}, + {1, 1520977874.3348415, "events"}, + {0, 2558504265.2738624, "events"}, + ... + """ + @spec timeslice(Oban.name(), series(), timeslice_opts()) :: [{ts(), value(), label()}] + def timeslice(oban \\ Oban, series, opts \\ []) do + oban + |> Registry.via(Recorder) + |> Recorder.timeslice(to_string(series), opts) + end + + # Callbacks + + @impl Supervisor + def init(opts) do + conf = Keyword.fetch!(opts, :conf) + + children = [ + {Cronitor, with_opts(:cronitor, conf, opts)}, + {Examiner, with_opts(:examiner, conf, opts)}, + {Recorder, with_opts(:recorder, conf, opts)}, + {Listener, with_opts(:listener, conf, opts)}, + {Reporter, with_opts(:reporter, conf, opts)}, + {Task, fn -> :telemetry.execute([:oban, :met, :init], %{}, %{pid: self(), conf: conf}) end} + ] + + Supervisor.init(children, strategy: :one_for_one) + end + + defp with_opts(name, conf, opts) do + real_name = + name + |> to_string() + |> String.capitalize() + |> then(&Module.safe_concat([Oban, Met, &1])) + + init_opts = Keyword.get(opts, name, []) + + :oban_met + |> Application.get_env(name, []) + |> Keyword.merge(init_opts) + |> Keyword.put(:conf, conf) + |> Keyword.put(:name, Registry.via(conf.name, real_name)) + end +end diff --git a/vendor/oban_met/lib/oban/met/application.ex b/vendor/oban_met/lib/oban/met/application.ex new file mode 100644 index 00000000..38951188 --- /dev/null +++ b/vendor/oban_met/lib/oban/met/application.ex @@ -0,0 +1,83 @@ +defmodule Oban.Met.Application do + @moduledoc false + + use Application + + alias Oban.{Config, Registry} + + require Logger + + @main_sup Oban.Met.Supervisor + @task_sup Oban.Met.TaskSupervisor + @handler_id :oban_met_handler + + @impl Application + def start(_type, _args) do + :telemetry.attach(@handler_id, [:oban, :supervisor, :init], &__MODULE__.init_metrics/4, []) + + children = [ + {Task.Supervisor, name: @task_sup}, + {Task, &boot_metrics/0} + ] + + Supervisor.start_link(children, strategy: :one_for_one, name: @main_sup) + end + + @impl Application + def prep_stop(state) do + :telemetry.detach(@handler_id) + + state + end + + @doc false + def init_metrics(_event, _measure, %{conf: conf}, _conf) do + start_metrics(conf) + end + + @doc false + def boot_metrics do + guard = {:andalso, {:is_map, :"$1"}, {:==, {:map_get, :__struct__, :"$1"}, Config}} + + [{{:_, :_, :"$1"}, [guard], [:"$1"]}] + |> Registry.select() + |> Enum.each(&start_metrics/1) + end + + defp start_metrics(conf) do + opts = Application.get_all_env(:oban_met) + + if opts[:auto_start] and conf.testing in opts[:auto_testing_modes] do + spec = Oban.Met.child_spec(conf: conf) + + case Supervisor.start_child(@main_sup, spec) do + {:ok, _pid} -> + Task.Supervisor.start_child(@task_sup, fn -> watch_oban(spec, conf) end) + + :ok + + {:error, {:already_started, _pid}} -> + :ok + + {:error, {error, _stack}} -> + Logger.error("Unable to start Oban.Met supervisor: #{inspect(error)}") + end + end + end + + defp watch_oban(%{id: child_id}, conf) do + ref = + conf.name + |> Oban.whereis() + |> Process.monitor() + + receive do + {:DOWN, ^ref, :process, _pid, _reason} -> + Process.demonitor(ref, [:flush]) + + with :ok <- Supervisor.terminate_child(@main_sup, child_id) do + Supervisor.delete_child(@main_sup, child_id) + end + end + end +end diff --git a/vendor/oban_met/lib/oban/met/cronitor.ex b/vendor/oban_met/lib/oban/met/cronitor.ex new file mode 100644 index 00000000..08fe36b8 --- /dev/null +++ b/vendor/oban_met/lib/oban/met/cronitor.ex @@ -0,0 +1,139 @@ +defmodule Oban.Met.Cronitor do + @moduledoc false + + use GenServer + + alias __MODULE__, as: State + alias Oban.Notifier + + require Logger + + defstruct [ + :conf, + :name, + :timer, + crontabs: %{}, + interval: :timer.seconds(15), + ttl: :timer.seconds(60) + ] + + @spec child_spec(Keyword.t()) :: Supervisor.child_spec() + def child_spec(opts) do + name = Keyword.get(opts, :name, __MODULE__) + + opts + |> super() + |> Map.put(:id, name) + end + + @spec start_link(Keyword.t()) :: GenServer.on_start() + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, name: opts[:name]) + end + + def merged_crontab(name) do + GenServer.call(name, :merged_crontab) + end + + # Callbacks + + @impl GenServer + def init(opts) do + Process.flag(:trap_exit, true) + + state = struct!(State, opts) + + Notifier.listen(state.conf.name, [:cronitor]) + + {:ok, state, {:continue, :start}} + end + + @impl GenServer + def handle_continue(:start, %State{} = state) do + handle_info(:share, state) + end + + @impl GenServer + def terminate(_reason, state) do + if is_reference(state.timer), do: Process.cancel_timer(state.timer) + + :ok + end + + @impl GenServer + def handle_call(:share, _from, %State{} = state) do + {:noreply, state} = handle_info(:share, state) + + {:reply, :ok, state} + end + + def handle_call(:merged_crontab, _from, %State{} = state) do + merged_crontab = + for {_key, {_ts, crontab}} <- state.crontabs, entry <- crontab, uniq: true, do: entry + + {:reply, merged_crontab, state} + end + + @impl GenServer + def handle_info(:share, %State{} = state) do + %{name: name, node: node, plugins: plugins} = state.conf + + crontab = + plugins + |> Keyword.get(Oban.Plugins.Cron, []) + |> Keyword.get(:crontab, []) + |> Enum.map(fn + {expr, work} -> {expr, inspect(work), []} + {expr, work, opts} -> {expr, inspect(work), opts} + end) + + payload = %{crontab: crontab, name: inspect(name), node: node} + + Notifier.notify(state.conf, :cronitor, payload) + + state = + state + |> purge_stale() + |> schedule_share() + + {:noreply, state} + end + + def handle_info({:notification, :cronitor, payload}, %State{} = state) do + %{"crontab" => crontab, "name" => name, "node" => node} = payload + + ts = System.system_time(:millisecond) + + crontab = + Enum.map(crontab, fn [expr, work, opts] -> + {expr, work, Map.new(opts, &List.to_tuple/1)} + end) + + crontabs = Map.put(state.crontabs, {node, name}, {ts, crontab}) + + {:noreply, %{state | crontabs: crontabs}} + end + + def handle_info(message, state) do + Logger.warning( + message: "Received unexpected message: #{inspect(message)}", + source: :oban_met, + module: __MODULE__ + ) + + {:noreply, state} + end + + # Helpers + + defp purge_stale(%State{} = state) do + expires = System.system_time(:millisecond) - state.ttl + crontabs = Map.reject(state.crontabs, fn {_key, {ts, _tab}} -> ts < expires end) + + %{state | crontabs: crontabs} + end + + defp schedule_share(%State{} = state) do + %{state | timer: Process.send_after(self(), :share, state.interval)} + end +end diff --git a/vendor/oban_met/lib/oban/met/examiner.ex b/vendor/oban_met/lib/oban/met/examiner.ex new file mode 100644 index 00000000..48777b5c --- /dev/null +++ b/vendor/oban_met/lib/oban/met/examiner.ex @@ -0,0 +1,180 @@ +defmodule Oban.Met.Examiner do + @moduledoc false + + # Examiner uses notifications to periodically exchange queue state information between all + # interested nodes. + + # This module is more of a "producer queue checker", but that name stinks. + + use GenServer + + alias __MODULE__, as: State + alias Oban.Notifier + + require Logger + + @type name_or_table :: :ets.tab() | GenServer.name() + + defstruct [ + :conf, + :name, + :table, + :timer, + interval: 750, + ttl: :timer.seconds(30) + ] + + @spec child_spec(Keyword.t()) :: Supervisor.child_spec() + def child_spec(opts) do + name = Keyword.get(opts, :name, __MODULE__) + + opts + |> super() + |> Map.put(:id, name) + end + + @spec start_link(Keyword.t()) :: GenServer.on_start() + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, name: opts[:name]) + end + + @spec all_checks(name_or_table()) :: [map()] + def all_checks(name_or_table) do + name_or_table + |> table() + |> :ets.select([{{:_, :_, :"$1"}, [], [:"$1"]}]) + end + + @spec store(name_or_table(), map(), timestamp: integer()) :: :ok + def store(name_or_table, check, opts \\ []) when is_map(check) do + %{"node" => node, "name" => name, "queue" => queue} = check + + check = normalize_limit(check) + timestamp = Keyword.get(opts, :timestamp, System.system_time(:millisecond)) + + name_or_table + |> table() + |> :ets.insert({{node, name, queue}, timestamp, check}) + + :ok + end + + @spec purge(name_or_table(), pos_integer()) :: {:ok, non_neg_integer()} + def purge(name_or_table, ttl) when is_integer(ttl) and ttl > 0 do + expires = System.system_time(:millisecond) - ttl + pattern = [{{:_, :"$1", :_}, [{:<, :"$1", expires}], [true]}] + + deleted = + name_or_table + |> table() + |> :ets.select_delete(pattern) + + {:ok, deleted} + end + + # Callbacks + + @impl GenServer + def init(opts) do + Process.flag(:trap_exit, true) + + table = :ets.new(:checks, [:public, read_concurrency: true]) + state = struct!(State, Keyword.put(opts, :table, table)) + + Notifier.listen(state.conf.name, [:gossip]) + + Registry.register(Oban.Registry, state.name, table) + + {:ok, state, {:continue, :start}} + end + + @impl GenServer + def handle_continue(:start, %State{} = state) do + handle_info(:check, state) + end + + @impl GenServer + def terminate(_reason, state) do + if is_reference(state.timer), do: Process.cancel_timer(state.timer) + + :ok + end + + @impl GenServer + def handle_info(:check, %State{} = state) do + match = [{{{state.conf.name, {:producer, :_}}, :"$1", :_}, [], [:"$1"]}] + + checks = + Oban.Registry + |> Registry.select(match) + |> Task.async_stream(&safe_check(&1, state), on_timeout: :kill_task, ordered: false) + |> Enum.reduce([], fn + {:ok, check}, acc when is_map(check) -> [check | acc] + _, acc -> acc + end) + + if Enum.any?(checks), do: Notifier.notify(state.conf, :gossip, %{checks: checks}) + + purge(state.table, state.ttl) + + {:noreply, schedule_check(state)} + end + + def handle_info({:notification, :gossip, %{"checks" => checks}}, %State{} = state) do + Enum.each(checks, &store(state.table, &1)) + + {:noreply, state} + end + + def handle_info({:notification, :gossip, _payload}, state) do + {:noreply, state} + end + + def handle_info(message, state) do + Logger.warning( + message: "Received unexpected message: #{inspect(message)}", + source: :oban_met, + module: __MODULE__ + ) + + {:noreply, state} + end + + # Table + + defp table(tab) when is_reference(tab), do: tab + + defp table(name) do + [{_pid, table}] = Registry.lookup(Oban.Registry, name) + + table + end + + # Scheduling + + defp schedule_check(%State{} = state) do + %{state | timer: Process.send_after(self(), :check, state.interval)} + end + + # Checking + + defp safe_check(pid, state) do + if Process.alive?(pid) do + pid + |> GenServer.call(:check, state.interval) + |> sanitize_name() + end + catch + :exit, _ -> :error + end + + defp sanitize_name(%{name: name} = check) when is_binary(name), do: check + defp sanitize_name(%{name: name} = check), do: %{check | name: inspect(name)} + + defp normalize_limit(check) do + case Map.pop(check, "limit") do + {nil, check} -> check + {lim, check} -> Map.put(check, "local_limit", lim) + end + end +end diff --git a/vendor/oban_met/lib/oban/met/listener.ex b/vendor/oban_met/lib/oban/met/listener.ex new file mode 100644 index 00000000..4d994272 --- /dev/null +++ b/vendor/oban_met/lib/oban/met/listener.ex @@ -0,0 +1,149 @@ +defmodule Oban.Met.Listener do + @moduledoc false + + # Track local telemetry events and periodically relay them to external recorders. + + use GenServer + + alias __MODULE__, as: State + alias Oban.Met.Values.{Gauge, Sketch} + alias Oban.Notifier + + defstruct [ + :conf, + :name, + :table, + :timer, + interval: :timer.seconds(1) + ] + + @spec child_spec(Keyword.t()) :: Supervisor.child_spec() + def child_spec(opts) do + name = Keyword.get(opts, :name, __MODULE__) + + %{super(opts) | id: name} + end + + @spec start_link(Keyword.t()) :: GenServer.on_start() + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, name: opts[:name]) + end + + @doc false + @spec report(GenServer.name()) :: :ok + def report(name) do + GenServer.call(name, :report) + end + + # Callbacks + + @impl GenServer + def init(opts) do + Process.flag(:trap_exit, true) + + table = :ets.new(:reporter, [:duplicate_bag, :public, write_concurrency: true]) + state = struct!(State, Keyword.put(opts, :table, table)) + + :telemetry.attach_many( + handler_id(state), + [[:oban, :job, :stop], [:oban, :job, :exception]], + &__MODULE__.handle_event/4, + {state.conf, table} + ) + + {:ok, schedule_report(state)} + end + + @impl GenServer + def terminate(_reason, %State{timer: timer} = state) do + if is_reference(timer), do: Process.cancel_timer(timer) + + :telemetry.detach(handler_id(state)) + + :ok + end + + @impl GenServer + def handle_info(:report, %State{conf: conf, table: table} = state) do + since = System.monotonic_time() + match = {:_, :"$2", :_} + guard = [{:<, :"$2", since}] + + objects = :ets.select(table, [{match, guard, [:"$_"]}]) + _delete = :ets.select_delete(table, [{match, guard, [true]}]) + + payload = %{ + metrics: objects_to_metrics(objects), + name: inspect(conf.name), + node: conf.node, + time: System.system_time(:second) + } + + Notifier.notify(conf, :metrics, payload) + + {:noreply, schedule_report(state)} + end + + defp objects_to_metrics(objects) do + objects + |> Enum.group_by(&elem(&1, 0), &elem(&1, 2)) + |> Enum.map(fn {{series, state, queue, worker}, values} -> + value = if series == :exec_count, do: Gauge.new(values), else: Sketch.new(values) + + %{series: series, state: state, queue: queue, worker: worker, value: value} + end) + end + + @impl GenServer + def handle_call(:report, _from, %State{} = state) do + handle_info(:report, state) + + {:reply, :ok, state} + end + + # Telemetry Events + + defp handler_id(state) do + "oban-met-recorder-#{inspect(state.name)}" + end + + @doc false + def handle_event([:oban, :job, _], measure, %{conf: conf} = meta, {conf, tab}) do + %{job: %{queue: queue, worker: worker}, state: state} = meta + %{duration: exec_time, queue_time: wait_time} = measure + + time = System.monotonic_time() + trst = trans_state(state) + + # Sketches don't support negative times (caused when the VM wakes from sleep), or zero values + # (infrequently caused by the same situation). + exec_time = exec_time |> abs() |> max(1) + wait_time = wait_time |> abs() |> max(1) + + :ets.insert(tab, [ + {{:exec_time, trst, queue, worker}, time, exec_time}, + {{:wait_time, trst, queue, worker}, time, wait_time}, + {{:exec_count, trst, queue, worker}, time, 1} + ]) + end + + def handle_event(_event, _measure, _meta, _conf), do: :ok + + # Scheduling + + defp schedule_report(state) do + timer = Process.send_after(self(), :report, state.interval) + + %{state | timer: timer} + end + + # For backward compatibility reasons, the telemetry event's state doesn't match the final job + # state. Here we translate the event state to the `t:Oban.Job.state`. + defp trans_state(:cancelled), do: :cancelled + defp trans_state(:discard), do: :discarded + defp trans_state(:exhausted), do: :discarded + defp trans_state(:failure), do: :retryable + defp trans_state(:snoozed), do: :scheduled + defp trans_state(:success), do: :completed + defp trans_state(state), do: state +end diff --git a/vendor/oban_met/lib/oban/met/migration.ex b/vendor/oban_met/lib/oban/met/migration.ex new file mode 100644 index 00000000..d02f3bb6 --- /dev/null +++ b/vendor/oban_met/lib/oban/met/migration.ex @@ -0,0 +1,103 @@ +defmodule Oban.Met.Migration do + @moduledoc """ + Migrations that add estimate functionality. + + > #### Migrations Are Not Required {: .tip} + > + > Met will create the necessary estimate function automatically when possible. This migration + > isn't necessary under normal circumstances, but is provided to avoid permission issues or + > allow full control over database changes. + > + > See the section on [Explicit Migrations](installation.html) for more information. + + ## Usage + + To use migrations in your application you'll need to generate an `Ecto.Migration` that wraps + calls to `Oban.Met.Migration`: + + ```bash + mix ecto.gen.migration add_oban_met + ``` + + Open the generated migration and delegate the `up/0` and `down/0` functions to + `Oban.Met.Migration`: + + ```elixir + defmodule MyApp.Repo.Migrations.AddObanMet do + use Ecto.Migration + + def up, do: Oban.Met.Migration.up() + + def down, do: Oban.Met.Migration.down() + end + ``` + + This will run all of the necessary migrations for your database. + """ + + use Ecto.Migration + + @doc """ + Run the `up` migration. + + ## Example + + Run all migrations up to the current version: + + Oban.Met.Migration.up() + + Run migrations in an alternate prefix: + + Oban.Met.Migration.up(prefix: "payments") + """ + def up(opts \\ []) when is_list(opts) do + opts + |> Keyword.get(:prefix, "public") + |> oban_count_estimate() + |> execute() + end + + @doc """ + Run the `down` migration. + + ## Example + + Run all migrations up to the current version: + + Oban.Met.Migration.down() + + Run migrations in an alternate prefix: + + Oban.Met.Migration.down(prefix: "payments") + """ + def down(opts \\ []) when is_list(opts) do + prefix = Keyword.get(opts, :prefix, "public") + + execute "DROP FUNCTION IF EXISTS #{prefix}.oban_count_estimate(text, text)" + end + + # An `EXPLAIN` can only be executed as the top level of a query, or through an SQL function's + # EXECUTE as we're doing here. A named function helps the performance because it is prepared, + # and we have to support distributed databases that don't allow DO/END functions. + @doc false + def oban_count_estimate(prefix) do + """ + CREATE OR REPLACE FUNCTION #{prefix}.oban_count_estimate(state text, queue text) + RETURNS integer AS $func$ + DECLARE + plan jsonb; + BEGIN + EXECUTE 'EXPLAIN (FORMAT JSON) + SELECT id + FROM #{prefix}.oban_jobs + WHERE state = $1::#{prefix}.oban_job_state + AND queue = $2' + INTO plan + USING state, queue; + RETURN plan->0->'Plan'->'Plan Rows'; + END; + $func$ + LANGUAGE plpgsql + """ + end +end diff --git a/vendor/oban_met/lib/oban/met/recorder.ex b/vendor/oban_met/lib/oban/met/recorder.ex new file mode 100644 index 00000000..77bc0db9 --- /dev/null +++ b/vendor/oban_met/lib/oban/met/recorder.ex @@ -0,0 +1,393 @@ +defmodule Oban.Met.Recorder do + @moduledoc false + + # Aggregate metrics via pubsub for querying and compaction. + + use GenServer + + alias __MODULE__, as: State + alias Oban.Met.{Value, Values.Gauge, Values.Sketch} + alias Oban.{Notifier, Peer} + + @type series :: atom() | String.t() + @type value :: Value.t() + @type label :: String.t() + @type labels :: %{optional(String.t()) => label()} + @type ts :: integer() + @type period :: {pos_integer(), pos_integer()} + + @periods [{1, 300}, {5, 1_200}, {30, 3_600}, {60, 7_200}] + + @default_latest_opts [filters: [], group: nil, lookback: 2] + + @default_timeslice_opts [ + by: 1, + filters: [], + group: nil, + lookback: 60, + operation: :sum + ] + + defstruct [ + :compact_timer, + :conf, + :name, + :table, + compact_periods: @periods, + handoff: :awaiting + ] + + @spec child_spec(keyword()) :: Supervisor.child_spec() + def child_spec(opts) do + name = Keyword.get(opts, :name, __MODULE__) + + %{super(opts) | id: name} + end + + @spec start_link(keyword()) :: GenServer.on_start() + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, name: opts[:name]) + end + + @spec lookup(GenServer.name(), series()) :: [term()] + def lookup(name, series) do + match = {{to_string(series), :_, :_}, :_, :_, :_} + + :ets.select_reverse(table(name), [{match, [], [:"$_"]}]) + end + + @spec labels(GenServer.name(), label(), keyword()) :: [label()] + def labels(name, label, opts \\ []) when is_binary(label) do + opts = Keyword.validate!(opts, [:lookback, :since]) + + stime = Keyword.get(opts, :since, System.system_time(:second)) + lookback = Keyword.get(opts, :lookback, 120) + match = {{:_, :_, :"$2"}, :_, :"$1", :_} + guard = [{:andalso, {:is_map_key, label, :"$1"}, {:>=, :"$2", stime - lookback}}] + value = [{:map_get, label, :"$1"}] + + name + |> table() + |> :ets.select([{match, guard, value}]) + |> :lists.usort() + end + + @spec latest(GenServer.name(), series(), keyword()) :: %{optional(String.t()) => value()} + def latest(name, series, opts \\ []) do + opts = Keyword.validate!(opts, @default_latest_opts) + + group = Keyword.fetch!(opts, :group) + lookback = Keyword.fetch!(opts, :lookback) + filters = Keyword.fetch!(opts, :filters) + + name + |> table() + |> select(series, lookback, filters) + |> Enum.dedup_by(fn {{_, _, _}, _, labels, _} -> labels end) + |> Enum.group_by(fn {{_, _, _}, _, labels, _} -> labels[group] || "all" end) + |> Map.new(fn {group, metrics} -> + total = + metrics + |> Enum.map(&elem(&1, 3)) + |> Enum.reduce(&Value.merge/2) + |> Value.sum() + + {group, total} + end) + end + + @spec series(GenServer.name()) :: [map()] + def series(name) do + match = {{:"$1", :_, :_}, :_, :"$2", :"$3"} + + name + |> table() + |> :ets.select([{match, [], [:"$$"]}]) + |> Enum.group_by(&hd/1) + |> Enum.map(fn {series, [[_series, _labels, %vtype{}] | _] = metrics} -> + labels = + for [_series, labels, _value] <- metrics, + key <- Map.keys(labels), + uniq: true, + do: key + + %{series: series, labels: labels, type: vtype} + end) + |> Enum.sort_by(& &1.series) + end + + @spec timeslice(GenServer.name(), series(), keyword()) :: [{ts(), value(), label()}] + def timeslice(name, series, opts \\ []) do + opts = Keyword.validate!(opts, [:since] ++ @default_timeslice_opts) + + by = Keyword.fetch!(opts, :by) + group = Keyword.fetch!(opts, :group) + lookback = Keyword.fetch!(opts, :lookback) + operation = Keyword.fetch!(opts, :operation) + since = Keyword.get(opts, :since, System.system_time(:second)) + + name + |> table() + |> select(series, lookback, opts[:filters]) + |> Enum.reduce(%{}, &merge_group(&1, &2, group)) + |> Enum.reduce(%{}, &merge_chunk(&1, &2, since, by)) + |> Enum.sort_by(fn {{label, chunk}, _} -> {label, -chunk} end) + |> Enum.map(fn {{label, chunk}, value} -> + value = + case operation do + :sum -> Value.sum(value) + :max -> Value.quantile(value, 1.0) + {:pct, ntile} -> Value.quantile(value, ntile) + end + + {chunk, value, label} + end) + end + + defp merge_group({{_, _, ts}, _, labels, value}, acc, group) do + Map.update(acc, {labels[group], ts}, value, &Value.union(&1, value)) + end + + defp merge_chunk({{label, ts}, value}, acc, since, by) do + chunk = div(since - ts - 1, by) + + Map.update(acc, {label, chunk}, value, &Value.merge(&1, value)) + end + + def compact(name, periods) when is_list(periods) do + GenServer.call(name, {:compact, periods}) + end + + def store(name, series, value, labels, opts \\ []) do + time = Keyword.get(opts, :time, System.system_time(:second)) + + GenServer.call(name, {:store, {series, value, labels, time}}) + end + + # Callbacks + + @impl GenServer + def init(opts) do + table = + :ets.new(:metrics, [ + :compressed, + :ordered_set, + :protected, + read_concurrency: true + ]) + + state = + State + |> struct!(Keyword.put(opts, :table, table)) + |> schedule_compact() + + Registry.register(Oban.Registry, state.name, table) + + {:ok, state, {:continue, :start}} + end + + @impl GenServer + def handle_continue(:start, %State{conf: conf} = state) do + payload = %{ + syn: true, + module: __MODULE__, + name: inspect(conf.name), + node: conf.node + } + + Notifier.notify(conf.name, :handoff, payload) + Notifier.listen(conf.name, [:gossip, :handoff, :metrics]) + + {:noreply, state} + end + + @impl GenServer + def handle_call({:compact, periods}, _from, %State{table: table} = state) do + inner_compact(table, periods) + + {:reply, :ok, state} + end + + def handle_call({:store, params}, _from, %State{table: table} = state) do + {series, value, labels, time} = params + + inner_store(table, series, value, labels, time) + + {:reply, :ok, state} + end + + @impl GenServer + def handle_info({:notification, :handoff, %{"syn" => _}}, state) do + if Peer.leader?(state.conf) do + data = + state.table + |> :ets.tab2list() + |> :erlang.term_to_binary() + |> Base.encode64() + + payload = %{ + ack: true, + module: __MODULE__, + data: data, + node: state.conf.node, + name: inspect(state.conf.name) + } + + Notifier.notify(state.conf, :handoff, payload) + end + + {:noreply, state} + end + + def handle_info({:notification, :handoff, %{"ack" => _, "data" => data}}, %State{} = state) do + if state.handoff == :awaiting and not Peer.leader?(state.conf) do + data + |> Base.decode64!() + |> :erlang.binary_to_term() + |> then(&:ets.insert(state.table, &1)) + end + + {:noreply, %{state | handoff: :complete}} + end + + def handle_info({:notification, :metrics, %{"metrics" => _} = payload}, state) do + %{"metrics" => metrics, "node" => node, "time" => time} = payload + + for %{"series" => series, "value" => value} = metric <- metrics do + labels = + metric + |> Map.drop(~w(series value)) + |> Map.put("node", node) + + inner_store(state.table, series, from_map(value), labels, time) + end + + {:noreply, state} + end + + def handle_info({:notification, _channel, _payload}, state) do + {:noreply, state} + end + + def handle_info(:compact, %State{compact_periods: periods, table: table} = state) do + inner_compact(table, periods) + + :erlang.garbage_collect() + + {:noreply, schedule_compact(state), :hibernate} + end + + defp from_map(%{"size" => _} = value), do: Sketch.from_map(value) + defp from_map(value), do: Gauge.from_map(value) + + # Table + + defp table(name) do + case Registry.lookup(Oban.Registry, name) do + [{_pid, table}] -> + table + + _ -> + raise RuntimeError, "no table registered for #{inspect(name)}" + end + end + + defp inner_compact(table, periods) do + delete_outdated(table, periods) + + Enum.reduce(periods, System.system_time(:second), fn {step, duration}, ts -> + since = ts - duration + match = {{:_, :_, :"$1"}, :"$2", :_, :_} + guard = [{:andalso, {:>=, :"$2", since}, {:"=<", :"$1", ts}}] + + objects = :ets.select(table, [{match, guard, [:"$_"]}]) + _delete = :ets.select_delete(table, [{match, guard, [true]}]) + + objects + |> Enum.chunk_by(fn {{ser, lab, max}, _, _, _} -> {ser, lab, div(ts - max - 1, step)} end) + |> Enum.map(&compact_object/1) + |> then(&:ets.insert(table, &1)) + + since + end) + end + + defp compact_object([{{series, lab_key, _}, _, labels, _} | _] = metrics) do + {min_ts, max_ts} = + metrics + |> Enum.flat_map(fn {{_, _, max_ts}, min_ts, _, _} -> [max_ts, min_ts] end) + |> Enum.min_max() + + value = + metrics + |> Enum.map(&elem(&1, 3)) + |> Enum.reduce(&Value.merge/2) + + {{series, lab_key, max_ts}, min_ts, labels, value} + end + + defp delete_outdated(table, periods) do + systime = System.system_time(:second) + maximum = Enum.reduce(periods, 0, fn {_, duration}, acc -> duration + acc end) + + since = systime - maximum + match = {{:_, :_, :"$1"}, :_, :_, :_} + guard = [{:<, :"$1", since}] + + :ets.select_delete(table, [{match, guard, [true]}]) + end + + defp inner_store(table, series, value, labels, time) do + key = {to_string(series), :erlang.phash2(labels), time} + + value = + case :ets.lookup(table, key) do + [{_key, _time, _labels, old_value}] -> Value.union(old_value, value) + _ -> value + end + + :ets.insert(table, {key, time, labels, value}) + end + + # Scheduling + + defp schedule_compact(state) do + time = Time.utc_now() + + interval = + time + |> Time.add(60) + |> Map.put(:second, 0) + |> Time.diff(time) + |> Integer.mod(86_400) + |> System.convert_time_unit(:second, :millisecond) + + timer = Process.send_after(self(), :compact, interval) + + %{state | compact_timer: timer} + end + + # Fetching & Filtering + + defp select(table, series, since, filters) do + stime = System.system_time(:second) + match = {{to_string(series), :_, :"$2"}, :_, :"$1", :_} + guard = filters_to_guards(filters, {:>=, :"$2", stime - since}) + + :ets.select_reverse(table, [{match, [guard], [:"$_"]}]) + end + + defp filters_to_guards(nil, base), do: base + + defp filters_to_guards(filters, base) do + Enum.reduce(filters, base, fn {field, values}, and_acc -> + and_guard = + values + |> List.wrap() + |> Enum.map(fn value -> {:==, {:map_get, to_string(field), :"$1"}, value} end) + |> Enum.reduce(fn or_guard, or_acc -> {:orelse, or_guard, or_acc} end) + + {:andalso, and_guard, and_acc} + end) + end +end diff --git a/vendor/oban_met/lib/oban/met/reporter.ex b/vendor/oban_met/lib/oban/met/reporter.ex new file mode 100644 index 00000000..5aaa807d --- /dev/null +++ b/vendor/oban_met/lib/oban/met/reporter.ex @@ -0,0 +1,215 @@ +defmodule Oban.Met.Reporter do + @moduledoc false + + # Periodically count and report jobs by state and queue. + # + # Because exact counts are expensive, counts for states with jobs beyond a configurable + # threshold are estimated. This is a tradeoff that aims to preserve system resources at the + # expense of accuracy. + + use GenServer + + import Ecto.Query, only: [from: 2, group_by: 3, select: 3, where: 3] + + alias __MODULE__, as: State + alias Oban.{Job, Notifier, Peer, Repo} + alias Oban.Met.Migration + alias Oban.Met.Values.Gauge + alias Oban.Pro.Engines.Smart + + require Logger + + @empty_states %{ + "available" => [], + "cancelled" => [], + "completed" => [], + "discarded" => [], + "executing" => [], + "retryable" => [], + "scheduled" => [] + } + + defstruct [ + :conf, + :name, + :queue_timer, + :check_timer, + auto_migrate: true, + checks: @empty_states, + check_counter: 0, + check_interval: :timer.seconds(1), + estimate_limit: 50_000, + function_created?: false, + queues: [] + ] + + @spec child_spec(Keyword.t()) :: Supervisor.child_spec() + def child_spec(opts) do + name = Keyword.get(opts, :name, __MODULE__) + + %{super(opts) | id: name} + end + + @spec start_link(Keyword.t()) :: GenServer.on_start() + def start_link(opts) do + conf = Keyword.fetch!(opts, :conf) + + opts = + if conf.repo.__adapter__() == Ecto.Adapters.Postgres do + opts + else + opts + |> Keyword.put(:auto_migrate, false) + |> Keyword.put(:estimate_limit, :infinity) + end + + state = struct!(State, opts) + + GenServer.start_link(__MODULE__, state, name: opts[:name]) + end + + # Callbacks + + @impl GenServer + def init(%State{} = state) do + Process.flag(:trap_exit, true) + + # Used to ensure testing helpers to auto-allow this module for sandbox access. + :telemetry.execute([:oban, :plugin, :init], %{}, %{conf: state.conf, plugin: __MODULE__}) + + {:ok, schedule_checks(state)} + end + + @impl GenServer + def terminate(_reason, %State{} = state) do + if is_reference(state.check_timer), do: Process.cancel_timer(state.check_timer) + + :ok + end + + @impl GenServer + def handle_info(:checkpoint, %State{conf: conf} = state) do + if Peer.leader?(conf.name) do + state = + state + |> create_estimate_function() + |> cache_queues() + + {:ok, checks} = checks(state) + + metrics = + for {_key, counts} <- checks, + count <- counts, + do: Map.update!(count, :value, &Gauge.new/1) + + payload = %{ + metrics: metrics, + name: inspect(conf.name), + node: conf.node, + time: System.system_time(:second) + } + + Notifier.notify(conf, :metrics, payload) + + {:noreply, + schedule_checks(%{state | check_counter: state.check_counter + 1, checks: checks})} + else + {:noreply, schedule_checks(state)} + end + end + + def handle_info(message, state) do + Logger.warning( + message: "Received unexpected message: #{inspect(message)}", + source: :oban_met, + module: __MODULE__ + ) + + {:noreply, state} + end + + # Scheduling + + defp schedule_checks(state) do + timer = Process.send_after(self(), :checkpoint, state.check_interval) + + %{state | check_timer: timer} + end + + # Checking + + defp create_estimate_function(%{auto_migrate: true, function_created?: false} = state) do + %{conf: %{prefix: prefix}} = state + + query = Migration.oban_count_estimate(prefix) + + Repo.query!(state.conf, query, []) + + %{state | function_created?: true} + end + + defp create_estimate_function(state), do: state + + defp cache_queues(state) do + if Integer.mod(state.check_counter, 60) == 0 do + source = if state.conf.engine == Smart, do: "oban_producers", else: "oban_jobs" + query = from(p in source, select: p.queue, distinct: true) + + %{state | queues: Repo.all(state.conf, query)} + else + state + end + end + + defp checks(%{estimate_limit: limit} = state) do + {count_states, guess_states} = + for {state, counts} <- state.checks, reduce: {[], []} do + {count_acc, guess_acc} -> + total = Enum.reduce(counts, 0, &(&2 + &1.value)) + + if total < limit do + {[state | count_acc], guess_acc} + else + {count_acc, [state | guess_acc]} + end + end + + count_query = count_query(count_states) + guess_query = guess_query(guess_states, state.queues, state.conf) + + Repo.transaction(state.conf, fn -> + count_counts = Repo.all(state.conf, count_query) + guess_counts = Repo.all(state.conf, guess_query) + + (count_counts ++ guess_counts) + |> Enum.group_by(& &1.state) + |> Enum.reduce(@empty_states, fn {state, counts}, acc -> + Map.put(acc, state, counts) + end) + end) + end + + defp count_query([]), do: where(Job, [_], false) + + defp count_query(states) when is_list(states) do + Job + |> select([j], %{series: :full_count, state: j.state, queue: j.queue, value: count(j.id)}) + |> where([j], j.state in ^states) + |> group_by([j], [j.state, j.queue]) + end + + defp guess_query([], _queues, _conf), do: where(Job, [_], false) + defp guess_query(_states, [], _conf), do: where(Job, [_], false) + + defp guess_query(states, queues, conf) when is_list(states) and is_list(queues) do + from(p in fragment("json_array_elements_text(?)", ^queues), + cross_join: x in fragment("json_array_elements_text(?)", ^states), + select: %{ + series: :full_count, + state: x.value, + queue: p.value, + value: fragment("?.oban_count_estimate(?, ?)", literal(^conf.prefix), x.value, p.value) + } + ) + end +end diff --git a/vendor/oban_met/lib/oban/met/value.ex b/vendor/oban_met/lib/oban/met/value.ex new file mode 100644 index 00000000..ea886651 --- /dev/null +++ b/vendor/oban_met/lib/oban/met/value.ex @@ -0,0 +1,44 @@ +defprotocol Oban.Met.Value do + @moduledoc """ + Tracked data access functions. + """ + + @doc """ + Add or append a new value to the data type. + """ + def add(struct, value) + + @doc """ + Merge two values into one. + """ + def merge(struct_1, struct_2) + + @doc """ + Compute the quantile for a value. + """ + def quantile(struct, quantile) + + @doc """ + Sum all data points for a value. + """ + def sum(struct) + + @doc """ + Union two values by reducing them into one. + """ + def union(struct_1, struct_2) +end + +for module <- [Oban.Met.Values.Gauge, Oban.Met.Values.Sketch] do + defimpl Oban.Met.Value, for: module do + defdelegate add(struct, value), to: @for + + defdelegate merge(struct_1, struct_2), to: @for + + defdelegate quantile(struct, quantile), to: @for + + defdelegate sum(struct), to: @for + + defdelegate union(struct_1, struct_2), to: @for + end +end diff --git a/vendor/oban_met/lib/oban/met/values/gauge.ex b/vendor/oban_met/lib/oban/met/values/gauge.ex new file mode 100644 index 00000000..fac3b205 --- /dev/null +++ b/vendor/oban_met/lib/oban/met/values/gauge.ex @@ -0,0 +1,141 @@ +defmodule Oban.Met.Values.Gauge do + @moduledoc """ + One or more non-negative integers captured over time. + """ + + alias __MODULE__, as: Gauge + + @encoder if Code.ensure_loaded?(JSON.Encoder), do: JSON.Encoder, else: Jason.Encoder + + @derive @encoder + defstruct data: 0 + + @type t :: %__MODULE__{data: [non_neg_integer()]} + + @doc """ + Initialize a new gauge. + + ## Examples + + iex> Gauge.new(1).data + [1] + + iex> Gauge.new([1, 2]).data + [1, 2] + """ + @spec new(non_neg_integer() | [non_neg_integer()]) :: t() + def new(data) when is_integer(data) and data > 0 do + %Gauge{data: [data]} + end + + def new(data) when is_list(data) do + %Gauge{data: data} + end + + @doc """ + Add to the gauge. + + ## Examples + + iex> 1 + ...> |> Gauge.new() + ...> |> Gauge.add(1) + ...> |> Gauge.add(1) + ...> |> Map.fetch!(:data) + [3] + """ + @spec add(t(), pos_integer()) :: t() + def add(%Gauge{data: [head | tail]}, value) when value > 0 do + %Gauge{data: [head + value | tail]} + end + + @doc """ + Merges two gauges into a one. + + ## Examples + + Merging two gauge retains all values: + + iex> gauge_1 = Gauge.new([1, 2]) + ...> gauge_2 = Gauge.new([2, 3]) + ...> Gauge.merge(gauge_1, gauge_2).data + [1, 2, 2, 3] + """ + @spec merge(t(), t()) :: t() + def merge(%Gauge{data: data_1}, %Gauge{data: data_2}) do + %Gauge{data: data_1 ++ data_2} + end + + @doc """ + Compute the quantile for a gauge. + + ## Examples + + With single values: + + iex> Gauge.quantile(Gauge.new([1, 2, 3]), 1.0) + 3 + + iex> Gauge.quantile(Gauge.new([1, 2, 3]), 0.5) + 2 + """ + @spec quantile(t(), float()) :: non_neg_integer() + def quantile(%Gauge{data: data}, ntile) do + length = length(data) - 1 + + data + |> Enum.sort() + |> Enum.at(floor(length * ntile)) + end + + @doc """ + Compute the sum for a gauge. + + ## Examples + + iex> Gauge.sum(Gauge.new(3)) + 3 + + iex> Gauge.sum(Gauge.new([1, 2, 3])) + 6 + """ + @spec sum(t()) :: non_neg_integer() + def sum(%Gauge{data: data}), do: Enum.sum(data) + + @doc """ + Union two gauges into a single value. + + ## Examples + + Merging two gauge results in a single value: + + iex> gauge_1 = Gauge.new([1, 2]) + ...> gauge_2 = Gauge.new([2, 3]) + ...> Gauge.union(gauge_1, gauge_2).data + [8] + """ + @spec union(t(), t()) :: t() + def union(%Gauge{data: data_1}, %Gauge{data: data_2}) do + data = + (data_1 ++ data_2) + |> Enum.sum() + |> List.wrap() + + %Gauge{data: data} + end + + @doc """ + Initialize a gauge struct from a stringified map, e.g. encoded JSON. + + ## Examples + + iex> Gauge.new(2) + ...> |> Oban.JSON.encode!() + ...> |> Oban.JSON.decode!() + ...> |> Gauge.from_map() + ...> |> Map.fetch!(:data) + [2] + """ + @spec from_map(%{optional(String.t()) => term()}) :: t() + def from_map(%{"data" => data}), do: %Gauge{data: data} +end diff --git a/vendor/oban_met/lib/oban/met/values/sketch.ex b/vendor/oban_met/lib/oban/met/values/sketch.ex new file mode 100644 index 00000000..35f1278d --- /dev/null +++ b/vendor/oban_met/lib/oban/met/values/sketch.ex @@ -0,0 +1,186 @@ +defmodule Oban.Met.Values.Sketch do + @moduledoc """ + A fast and fully mergeable quantile sketch with relative error guarantees. + + Derived from [DogSketch](https://github.com/moosecodebv/dog_sketch), based on DDSketch. This + variant has a hard-coded error rate of 0.02 for the sake of simplicity. + """ + + alias __MODULE__, as: Sketch + + @type t :: %__MODULE__{ + data: %{optional(pos_integer()) => pos_integer()}, + size: non_neg_integer() + } + + @encoder if Code.ensure_loaded?(JSON.Encoder), do: JSON.Encoder, else: Jason.Encoder + + @derive @encoder + defstruct data: %{}, size: 0 + + @error 0.02 + @gamma (1 + @error) / (1 - @error) + @inv_log_gamma 1.0 / :math.log(@gamma) + + @doc """ + Create a new sketch instance with an optional error rate. + + ## Examples + + iex> sketch = Sketch.new() + ...> Sketch.size(sketch) + 0 + + iex> sketch = Sketch.new(1) + ...> Sketch.size(sketch) + 1 + + iex> sketch = Sketch.new([1, 2, 3]) + ...> Sketch.size(sketch) + 3 + """ + @spec new(pos_integer() | [pos_integer()]) :: t() + def new(values \\ []) do + values + |> List.wrap() + |> Enum.reduce(%Sketch{}, &add(&2, &1)) + end + + @doc """ + Insert sample values into a sketch. + + ## Examples + + iex> Sketch.new() + ...> |> Sketch.add(1) + ...> |> Sketch.add(2) + ...> |> Sketch.add(3) + ...> |> Sketch.size() + 3 + """ + @spec add(t(), pos_integer()) :: t() + def add(%Sketch{} = sketch, value) when is_integer(value) and value > 0 do + bin = ceil(:math.log(value) * @inv_log_gamma) + data = Map.update(sketch.data, bin, 1, &(&1 + 1)) + + %{sketch | data: data, size: sketch.size + 1} + end + + @doc """ + Merge two sketch instances. + + ## Examples + + iex> sketch_1 = Sketch.new([1]) + ...> + ...> Sketch.new([2]) + ...> |> Sketch.merge(sketch_1) + ...> |> Sketch.size() + 2 + """ + @spec merge(t(), t()) :: t() + def merge(%Sketch{} = sketch_1, %Sketch{} = sketch_2) do + data = Map.merge(sketch_1.data, sketch_2.data, fn _, val_1, val_2 -> val_1 + val_2 end) + + %{sketch_1 | data: data, size: sketch_1.size + sketch_2.size} + end + + @doc """ + Compute the quantile value for a sketch. + + ## Examples + + Without any values: + + iex> Sketch.quantile(Sketch.new(), 0.5) + nil + + With recorded values: + + iex> Sketch.new() + ...> |> Sketch.add(1) + ...> |> Sketch.add(2) + ...> |> Sketch.add(3) + ...> |> Sketch.quantile(0.5) + ...> |> trunc() + 2 + """ + @spec quantile(t(), float()) :: nil | float() + def quantile(%Sketch{size: 0}, _ntile), do: nil + + def quantile(sketch, quantile) when quantile >= 0 and quantile <= 1 do + size_ntile = sketch.size * quantile + + index = + sketch.data + |> Enum.sort_by(&elem(&1, 0)) + |> Enum.reduce_while(0, fn {key, val}, size -> + if size + val >= size_ntile do + {:halt, key} + else + {:cont, size + val} + end + end) + + 2 * :math.pow(@gamma, index) / (@gamma + 1) + end + + @doc """ + Compute the sum for a sketch. Hardcoded to 0. + + ## Examples + + iex> Sketch.sum(Sketch.new([1, 2, 3, 3])) + 0.0 + """ + @spec sum(t()) :: float() + def sum(%Sketch{data: _data}), do: 0.0 + + @doc """ + Union two sketches into a single value. This is an alias for `merge/2`. + """ + @spec union(t(), t()) :: t() + def union(sketch_1, sketch_2), do: merge(sketch_1, sketch_2) + + @doc """ + Convert a sketch into a list of bins and values. + + ## Examples + + iex> Sketch.new() + ...> |> Sketch.add(1) + ...> |> Sketch.add(2) + ...> |> Sketch.add(3) + ...> |> Sketch.to_list() + ...> |> length() + 3 + """ + @spec to_list(t()) :: [float()] + def to_list(%Sketch{data: data}) do + for {key, val} <- data, do: {2 * :math.pow(@gamma, key) / (@gamma + 1), val} + end + + @doc false + @spec size(t()) :: non_neg_integer() + def size(%Sketch{size: size}), do: size + + @doc """ + Initialize a sketch struct from a stringified map, e.g. encoded JSON. + + ## Examples + + iex> Sketch.new([1, 2]) + ...> |> Oban.JSON.encode!() + ...> |> Oban.JSON.decode!() + ...> |> Sketch.from_map() + ...> |> Sketch.quantile(1.0) + ...> |> floor() + 2 + """ + @spec from_map(%{optional(String.t()) => term()}) :: t() + def from_map(%{"data" => data, "size" => size}) do + data = Map.new(data, fn {key, val} -> {String.to_integer(key), val} end) + + %Sketch{data: data, size: size} + end +end diff --git a/vendor/oban_met/mix.exs b/vendor/oban_met/mix.exs new file mode 100644 index 00000000..f184c9d3 --- /dev/null +++ b/vendor/oban_met/mix.exs @@ -0,0 +1,131 @@ +defmodule Oban.Met.MixProject do + use Mix.Project + + @source_url "https://github.com/oban-bg/oban_met" + @version "1.0.5" + + def project do + [ + app: :oban_met, + version: @version, + elixir: "~> 1.15", + elixirc_paths: elixirc_paths(Mix.env()), + start_permanent: Mix.env() == :prod, + deps: deps(), + docs: docs(), + aliases: aliases(), + package: package(), + + # Description + name: "Oban Met", + description: + "A distributed, compacting, multidimensional, telemetry-powered time series datastore", + + # Dialyzer + dialyzer: [ + plt_add_apps: [:ex_unit], + plt_core_path: "_build/#{Mix.env()}", + flags: [:error_handling, :underspecs] + ] + ] + end + + def application do + [ + extra_applications: [:logger], + mod: {Oban.Met.Application, []}, + env: [auto_start: true, auto_testing_modes: [:disabled]] + ] + end + + def cli do + [ + preferred_envs: [ + "ecto.gen.migration": :test, + "test.ci": :test, + "test.reset": :test, + "test.setup": :test + ] + ] + end + + defp elixirc_paths(:test), do: ["lib", "test/support"] + defp elixirc_paths(_env), do: ["lib"] + + defp package do + [ + maintainers: ["Parker Selbert"], + licenses: ["Apache-2.0"], + files: ~w(lib .formatter.exs mix.exs README* CHANGELOG* LICENSE*), + links: %{ + Website: "https://oban.pro", + Changelog: "#{@source_url}/blob/main/CHANGELOG.md", + GitHub: @source_url + } + ] + end + + defp docs do + [ + main: "Oban.Met", + source_ref: "v#{@version}", + source_url: @source_url, + formatters: ["html"], + api_reference: false, + extra_section: "GUIDES", + extras: extras(), + groups_for_modules: groups_for_modules(), + skip_undefined_reference_warnings_on: ["CHANGELOG.md"] + ] + end + + defp extras do + [ + "CHANGELOG.md": [filename: "changelog", title: "Changelog"] + ] + end + + defp groups_for_modules do + [ + Values: [ + Oban.Met.Value, + Oban.Met.Values.Gauge, + Oban.Met.Values.Sketch + ] + ] + end + + defp deps do + [ + {:oban, "~> 2.19"}, + {:ecto_sqlite3, "~> 0.18", only: [:test, :dev]}, + {:postgrex, "~> 0.20", only: [:test, :dev]}, + {:stream_data, "~> 1.1", only: [:test, :dev]}, + {:benchee, "~> 1.3", only: [:test, :dev], runtime: false}, + {:credo, "~> 1.7", only: [:test, :dev], runtime: false}, + {:dialyxir, "~> 1.3", only: [:test, :dev], runtime: false}, + {:ex_doc, "~> 0.34", only: :dev, runtime: false}, + {:makeup_diff, "~> 0.1", only: :dev, runtime: false} + ] + end + + defp aliases do + [ + release: [ + "cmd git tag v#{@version} -f", + "cmd git push", + "cmd git push --tags", + "hex.publish --yes" + ], + "test.reset": ["ecto.drop --quiet", "test.setup"], + "test.setup": ["ecto.create --quiet", "ecto.migrate --quiet"], + "test.ci": [ + "format --check-formatted", + "deps.unlock --check-unused", + "credo", + "test --raise", + "dialyzer" + ] + ] + end +end diff --git a/vendor/oban_pro/.formatter.exs b/vendor/oban_pro/.formatter.exs new file mode 100644 index 00000000..da6266a2 --- /dev/null +++ b/vendor/oban_pro/.formatter.exs @@ -0,0 +1,14 @@ +locals_without_parens = [ + args_schema: 1, + field: 2, + field: 3, + embeds_one: 2, + embeds_many: 2 +] + +[ + import_deps: [:ecto, :ecto_sql, :oban, :stream_data], + export: [locals_without_parens: locals_without_parens], + locals_without_parens: locals_without_parens, + inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] +] diff --git a/vendor/oban_pro/.hex b/vendor/oban_pro/.hex new file mode 100644 index 0000000000000000000000000000000000000000..fe52fae4c314a4bc163359bbeb257baf1755e8ad GIT binary patch literal 264 zcmZ9EOLD^?5Cs)W<#^?zxEL6L*!d=|0Y;IW05181_qljro87Cfe*L@bmcH~0ziCI^ zqwBhT84^W~bsn;#bh&N0e7<>,[]}. +{<<"name">>,<<"oban_pro">>}. +{<<"version">>,<<"1.6.11">>}. +{<<"description">>,<<"Oban Pro Component">>}. +{<<"elixir">>,<<"~> 1.15">>}. +{<<"files">>, + [<<"lib/pro.ex">>,<<"lib/oban">>,<<"lib/oban/pro">>, + <<"lib/oban/pro/cron.ex">>,<<"lib/oban/pro/decorator.ex">>, + <<"lib/oban/pro/migrations">>,<<"lib/oban/pro/migrations/v1_4_0.ex">>, + <<"lib/oban/pro/migrations/v1_5_0.ex">>, + <<"lib/oban/pro/migrations/v1_6_0.ex">>, + <<"lib/oban/pro/migrations/dynamic_partitioner.ex">>, + <<"lib/oban/pro/migrations/v1_0_0.ex">>,<<"lib/oban/pro/unique.ex">>, + <<"lib/oban/pro/batch.ex">>,<<"lib/oban/pro/producer.ex">>, + <<"lib/oban/pro/worker.ex">>,<<"lib/oban/pro/stages">>, + <<"lib/oban/pro/stages/chain.ex">>,<<"lib/oban/pro/stages/encrypted.ex">>, + <<"lib/oban/pro/stages/recorded.ex">>,<<"lib/oban/pro/stages/hooks.ex">>, + <<"lib/oban/pro/stages/deadline.ex">>, + <<"lib/oban/pro/stages/structured.ex">>,<<"lib/oban/pro/plugins">>, + <<"lib/oban/pro/plugins/dynamic_prioritizer.ex">>, + <<"lib/oban/pro/plugins/dynamic_pruner.ex">>, + <<"lib/oban/pro/plugins/dynamic_scaler.ex">>, + <<"lib/oban/pro/plugins/dynamic_partitioner.ex">>, + <<"lib/oban/pro/plugins/dynamic_lifeline.ex">>, + <<"lib/oban/pro/plugins/dynamic_queues.ex">>, + <<"lib/oban/pro/plugins/dynamic_cron.ex">>,<<"lib/oban/pro/limiters">>, + <<"lib/oban/pro/limiters/local.ex">>,<<"lib/oban/pro/limiters/rate.ex">>, + <<"lib/oban/pro/limiters/global.ex">>,<<"lib/oban/pro/queue.ex">>, + <<"lib/oban/pro/handler.ex">>,<<"lib/oban/pro/partition.ex">>, + <<"lib/oban/pro/cloud.ex">>,<<"lib/oban/pro/uuidv7.ex">>, + <<"lib/oban/pro/stage.ex">>,<<"lib/oban/pro/refresher.ex">>, + <<"lib/oban/pro/workflow.ex">>,<<"lib/oban/pro/testing.ex">>, + <<"lib/oban/pro/migration.ex">>,<<"lib/oban/pro/engines">>, + <<"lib/oban/pro/engines/smart.ex">>,<<"lib/oban/pro/workflow">>, + <<"lib/oban/pro/workflow/cascade.ex">>,<<"lib/oban/pro/limiter.ex">>, + <<"lib/oban/pro/validation.ex">>,<<"lib/oban/pro/flusher.ex">>, + <<"lib/oban/pro/workers">>,<<"lib/oban/pro/workers/chain.ex">>, + <<"lib/oban/pro/workers/batch.ex">>,<<"lib/oban/pro/workers/chunk.ex">>, + <<"lib/oban/pro/workers/workflow.ex">>,<<"lib/oban/pro/application.ex">>, + <<"lib/oban/pro/utils.ex">>,<<"lib/oban/pro/relay.ex">>, + <<"lib/oban/pro/exceptions.ex">>,<<".formatter.exs">>,<<"mix.exs">>]}. +{<<"app">>,<<"oban_pro">>}. +{<<"licenses">>,[<<"Commercial">>]}. +{<<"requirements">>, + [[{<<"name">>,<<"oban">>}, + {<<"app">>,<<"oban">>}, + {<<"optional">>,false}, + {<<"requirement">>,<<"~> 2.19">>}, + {<<"repository">>,<<"hexpm">>}], + [{<<"name">>,<<"ecto_sql">>}, + {<<"app">>,<<"ecto_sql">>}, + {<<"optional">>,false}, + {<<"requirement">>,<<"~> 3.10">>}, + {<<"repository">>,<<"hexpm">>}], + [{<<"name">>,<<"postgrex">>}, + {<<"app">>,<<"postgrex">>}, + {<<"optional">>,true}, + {<<"requirement">>,<<"~> 0.16">>}, + {<<"repository">>,<<"hexpm">>}]]}. +{<<"build_tools">>,[<<"mix">>]}. diff --git a/vendor/oban_pro/lib/oban/pro/application.ex b/vendor/oban_pro/lib/oban/pro/application.ex new file mode 100644 index 00000000..e7b03dad --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/application.ex @@ -0,0 +1,29 @@ +defmodule Oban.Pro.Application do + @moduledoc false + + use Application + + @handlers [ + Oban.Pro.Batch, + Oban.Pro.Engines.Smart, + Oban.Pro.Migration, + Oban.Pro.Partition, + Oban.Pro.Relay, + Oban.Pro.Worker, + Oban.Pro.Workflow + ] + + @impl Application + def start(_type, _args) do + for handler <- @handlers, do: handler.on_start() + + children = [Oban.Pro.Refresher] + + Supervisor.start_link(children, strategy: :one_for_one, name: __MODULE__) + end + + @impl Application + def stop(_state) do + for handler <- @handlers, do: handler.on_stop() + end +end diff --git a/vendor/oban_pro/lib/oban/pro/batch.ex b/vendor/oban_pro/lib/oban/pro/batch.ex new file mode 100644 index 00000000..f63ed592 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/batch.ex @@ -0,0 +1,873 @@ +defmodule Oban.Pro.Batch do + @moduledoc """ + Batches link the execution of many jobs as a group and runs optional callbacks after jobs are + processed. This allows your application to coordinate the execution of any number of jobs in + parallel. + + ## Usage + + Batches are composed of one or more Pro workers that are linked with a shared `batch_id` and + optional callbacks. As a simple example, let's define a worker that delivers daily emails: + + ```elixir + defmodule MyApp.EmailBatch do + use Oban.Pro.Worker, queue: :mailers + + @behaviour Oban.Pro.Batch + + @impl Oban.Pro.Worker + def process(%Job{args: %{"email" => email}}) do + MyApp.Mailer.daily_update(email) + end + + @impl Oban.Pro.Batch + def batch_completed(_job) do + Logger.info("BATCH COMPLETE") + + :ok + end + end + ``` + + Now, create a batch with `new/1` by passing a list of job changesets: + + ```elixir + emails + |> Enum.map(&MyApp.EmailBatch.new(%{email: &1})) + |> Batch.new() + |> Oban.insert_all() + ``` + + After all jobs in the batch are `completed`, the `batch_completed/1` callback will be + triggered. + + ## Handler Callbacks + + After jobs in the batch are processed, a callback job may be inserted. There are four optional + batch handler callbacks that a worker may define: + + | callback | enqueued after | + | --------------------- | -------------------------------------------------------- | + | `c:batch_attempted/1` | _all_ jobs `executed` at least once | + | `c:batch_cancelled/1` | the _first_ job is `cancelled` | + | `c:batch_completed/1` | _all_ jobs in the batch are `completed` | + | `c:batch_discarded/1` | the _first_ job is `discarded` | + | `c:batch_exhausted/1` | _all_ jobs are `completed`, `cancelled`, or `discarded` | + | `c:batch_retryable/1` | the _first_ job is `retryable` | + + Each callback runs in a separate, isolated job, so it may be retried or discarded like any other + job. The callback function receives an `Oban.Job` struct with the `batch_id` in `meta` and + should return `:ok` (or another valid `Worker.result()`). + + Here we'll implement each of the optional handler callbacks and have them print out the batch + status along with the `batch_id`: + + ```elixir + defmodule MyApp.BatchWorker do + use Oban.Pro.Worker + + @behaviour Oban.Pro.Batch + + @impl Oban.Pro.Worker + def process(_job), do: :ok + + @impl Oban.Pro.Batch + def batch_attempted(%Job{meta: %{"batch_id" => batch_id}}) do + IO.inspect({:attempted, batch_id}) + :ok + end + + @impl Oban.Pro.Batch + def batch_cancelled(%Job{meta: %{"batch_id" => batch_id}}) do + IO.inspect({:cancelled, batch_id}) + :ok + end + + @impl Oban.Pro.Batch + def batch_completed(%Job{meta: %{"batch_id" => batch_id}}) do + IO.inspect({:completed, batch_id}) + :ok + end + + @impl Oban.Pro.Batch + def batch_discarded(%Job{meta: %{"batch_id" => batch_id}}) do + IO.inspect({:discarded, batch_id}) + :ok + end + + @impl Oban.Pro.Batch + def batch_exhausted(%Job{meta: %{"batch_id" => batch_id}}) do + IO.inspect({:exhausted, batch_id}) + :ok + end + + @impl Oban.Pro.Batch + def batch_retryable(%Job{meta: %{"batch_id" => batch_id}}) do + IO.inspect({:retryable, batch_id}) + :ok + end + end + ``` + + ## Hybrid Batches + + Batches may also be built from a variety of workers, though you must provide an explicit worker + for callbacks: + + ```elixir + mail_jobs = Enum.map(mail_args, &MyApp.MailWorker.new/1) + push_jobs = Enum.map(push_args, &MyApp.PushWorker.new/1) + + [callback_worker: MyApp.CallbackWorker] + |> Batch.new() + |> Batch.add(mail_jobs) + |> Batch.add(push_jobs) + ``` + + Without an explicit `callback_worker`, any worker in the batch may be used for callbacks. That + makes callback handling unpredictable and unexpected. + + ## Customizing Batches + + Batches accept a variety of options for grouping and callback customization. + + ### Batch ID + + By default a `batch_id` is generated as a time-ordered random [UUIDv7][uuid]. UUIDs are + sufficient to ensure uniqueness between workers and nodes for any time period. However, if you + require control, you can override `batch_id` generation at the worker level or pass a value + directly to the `new/2` function. + + ```elixir + Batch.new(batch_id: "custom-batch-id") + ``` + + ### Batch Name + + Batches accept an optional name to describe the purpose of the batch, beyond the generated id or + individual jobs in it. While the `batch_id` must be unique, the `batch_name` doesn't have to be, + so it can be used as a general purpose label. + + ```elixir + Batch.new(batch_name: "nightly-etl") + ``` + + ### Callback Workers + + For some batches, notably those with heterogeneous jobs, it's necessary to specify a different + worker for callbacks. That is easily accomplished by passing the `:callback_worker` option to + `new/2`: + + ```elixir + Batch.new(callback_worker: MyCallbackWorker) + ``` + + The callback worker **must** be an `Oban.Pro.Worker` that defines one or more of the batch callback + handlers. + + ### Callback Options + + By default, callback jobs have an empty `args` map and inherit other options from batch jobs. + With `callback_opts`, you can set standard job options for batch callback jobs (only `args`, + `max_attempts`, `meta`, `priority`, `queue`, and `tags` are allowed). + + For example, here we're passing a webhook URLs as `args`: + + ```elixir + Batch.new(callback_opts: [args: %{webhook: "https://web.hook"}]) + ``` + + Here, we change the callback queue and knock the priority down: + + ```elixir + Batch.new(callback_opts: [queue: :callbacks, priority: 9]) + ``` + + Be careful to minimize `callback_opts` as they are stored in each batch job's meta. + + ## Fetching Batch Jobs + + To pull more context from batch jobs, it's possible to load all jobs from the batch with + `all_jobs/2` and `stream_jobs/2`. The functions take a single batch job and returns a list or + stream of all non-callback jobs in the batch, which you can then operate on with `Enum` or + `Stream` functions. + + As an example, imagine you have a batch that ran for a few thousand accounts and you'd like to + notify admins that the batch is complete. + + ```elixir + defmodule MyApp.BatchWorker do + use Oban.Pro.Worker + + @behaviour Oban.Pro.Batch + + @impl Oban.Pro.Batch + def batch_completed(%Job{} = job) do + {:ok, account_ids} = + job + |> Oban.Pro.Batch.all_jobs() + |> Enum.map(& &1.args["account_id"]) + + account_ids + |> MyApp.Accounts.all() + |> MyApp.Mailer.notify_admins_about_batch() + end + ``` + + Fetching a thousand jobs may be alright, but for larger batches you don't want to load that much + data into memory. Instead, you can use `stream_jobs` to iterate through them lazily: + + ```elixir + {:ok, account_ids} = + MyApp.Repo.transaction(fn -> + job + |> Batch.stream_jobs() + |> Stream.map(& &1.args["account_id"]) + |> Enum.to_list() + end) + ``` + + Streaming is provided by Ecto's `Repo.stream`, and it must take place within a transaction. + While it may be overkill for small batches, for batches with tens or hundreds of thousands of + jobs, it will prevent massive memory spikes or the database grinding to a halt. + + [uuid]: https://datatracker.ietf.org/doc/html/draft-peabody-dispatch-new-uuid-format-04#section-5.2 + """ + + @behaviour Oban.Pro.Flusher + @behaviour Oban.Pro.Handler + + import Ecto.Query, only: [limit: 2, order_by: 2, select: 3, union_all: 2, where: 3] + + alias Ecto.Changeset + alias Oban.Pro.Engines.Smart + alias Oban.{Job, Repo, Validation, Worker} + alias Oban.Pro.Workflow + + require Logger + + @type changeset :: Job.changeset() + + @type callback_opts :: [ + args: Job.args(), + max_attempts: pos_integer(), + meta: map(), + priority: 0..9, + queue: atom() | String.t(), + tags: Job.tags() + ] + + @type batch_opts :: [ + batch_id: String.t(), + batch_name: String.t(), + callback_opts: callback_opts(), + callback_worker: module() + ] + + @type repo_opts :: [timeout: timeout()] + + @type t :: %__MODULE__{changesets: Enumerable.t(changeset), opts: map()} + + @callbacks_to_functions %{ + "attempted" => :batch_attempted, + "cancelled" => :batch_cancelled, + "completed" => :batch_completed, + "discarded" => :batch_discarded, + "exhausted" => :batch_exhausted, + "retryable" => :batch_retryable + } + + @callbacks_to_deprecated %{ + "attempted" => :handle_attempted, + "cancelled" => :handle_cancelled, + "completed" => :handle_completed, + "discarded" => :handle_discarded, + "exhausted" => :handle_exhausted, + "retryable" => :handle_retryable + } + + @callbacks_to_combined Map.merge( + @callbacks_to_functions, + @callbacks_to_deprecated, + fn _k, new, old -> [new, old] end + ) + + @callbacks_to_states %{ + "attempted" => ~w(scheduled available executing), + "completed" => ~w(scheduled available executing retryable cancelled discarded), + "cancelled" => ~w(cancelled), + "discarded" => ~w(discarded), + "exhausted" => ~w(scheduled retryable available executing), + "retryable" => ~w(retryable) + } + + @all_states Enum.map(Job.states(), &to_string/1) + + @doc """ + Called after all jobs in the batch were attempted at least once. + + If a `batch_attempted/1` function is defined it is executed by an isolated callback job. + + ## Example + + Print when all jobs were attempted: + + def batch_attempted(%Job{meta: %{"batch_id" => batch_id}}) do + IO.inspect(batch_id, label: "Attempted") + end + """ + @callback batch_attempted(job :: Job.t()) :: Worker.result() + + @doc """ + Called after any jobs in the batch have a `cancelled` state. + + If a `batch_cancelled/1` function is defined it is executed by an isolated callback job. + + ## Example + + Print when any jobs are discarded: + + def batch_cancelled(%Job{meta: %{"batch_id" => batch_id}}) do + IO.inspect(batch_id, label: "Cancelled") + end + """ + @callback batch_cancelled(job :: Job.t()) :: Worker.result() + + @doc """ + Called after all jobs in the batch have a `completed` state. + + If a `batch_completed/1` function is defined it is executed by an isolated callback job. + + ## Example + + Print when all jobs are completed: + + def batch_completed(%Job{meta: %{"batch_id" => batch_id}}) do + IO.inspect(batch_id, label: "Completed") + end + """ + @callback batch_completed(job :: Job.t()) :: Worker.result() + + @doc """ + Called after any jobs in the batch have a `discarded` state. + + If a `batch_discarded/1` function is defined it is executed by an isolated callback job. + + ## Example + + Print when any jobs are discarded: + + def batch_discarded(%Job{meta: %{"batch_id" => batch_id}}) do + IO.inspect(batch_id, label: "Discarded") + end + """ + @callback batch_discarded(job :: Job.t()) :: Worker.result() + + @doc """ + Called after all jobs in the batch have either a `cancelled`, `completed` or `discarded` state. + + If a `batch_exhausted/1` function is defined it is executed by an isolated callback job. + + ## Example + + Print when all jobs are completed or discarded: + + def batch_exhausted(%Job{meta: %{"batch_id" => batch_id}}) do + IO.inspect(batch_id, label: "Exhausted") + end + """ + @callback batch_exhausted(job :: Job.t()) :: Worker.result() + + @doc """ + Called after any jobs in the batch have a `retryable` state. + + If a `batch_retryable/1` function is defined it is executed by an isolated callback job. + + ## Example + + Print when any jobs are retryable: + + def batch_retryable(%Job{meta: %{"batch_id" => batch_id}}) do + IO.inspect(batch_id, label: "Retryable") + end + """ + @callback batch_retryable(job :: Job.t()) :: Worker.result() + + @optional_callbacks batch_attempted: 1, + batch_cancelled: 1, + batch_completed: 1, + batch_discarded: 1, + batch_exhausted: 1, + batch_retryable: 1 + + defstruct changesets: [], opts: [] + + defguardp is_list_or_stream(enum) when is_list(enum) or is_struct(enum, Stream) + + # Handler Callbacks + + @impl Oban.Pro.Handler + def on_start do + events = [ + [:oban, :engine, :cancel_all_jobs, :stop] + ] + + :telemetry.attach_many("oban.batch", events, &__MODULE__.handle_event/4, nil) + end + + @impl Oban.Pro.Handler + def on_stop do + :telemetry.detach("oban.batch") + end + + @doc false + def handle_event(_event, _timing, %{conf: conf, jobs: jobs}, _) do + for %{meta: %{"batch_id" => batch_id}} = job <- jobs do + on_flush(job, batch_id, conf) + end + end + + # Constants + + @doc false + def callbacks_to_functions, do: @callbacks_to_functions + + @doc false + def callbacks_to_deprecated, do: @callbacks_to_deprecated + + # Public Functions + + @doc false + def new, do: new([], []) + + @doc false + def new([%Changeset{} | _] = changesets), do: new(changesets, []) + def new(%Stream{} = changesets), do: new(changesets, []) + def new(opts) when is_list(opts), do: new([], opts) + + @doc """ + Build a new batch from a list or stream of job changesets and some options. + + ## Examples + + Build an empty batch without any jobs or options: + + Batch.new() + + Build a batch from a list of changesets: + + 1..3 + |> Enum.map(fn id -> MyWorker.new(%{id: id}) end) + |> Batch.new() + + Build a batch from a stream: + + stream_of_args + |> Stream.map(&MyWorker.new/1) + |> Batch.new() + + Build a batch with callback options: + + Batch.new(list_of_jobs, batch_id: "abc-123", callback_worker: MyApp.CallbackWorker) + """ + @spec new(Enumerable.t(changeset()), batch_opts()) :: t() + def new(changesets, opts) when is_list_or_stream(changesets) and is_list(opts) do + validate!(opts) + + opts = + opts + |> Keyword.put(:batch, true) + |> Keyword.put_new(:callback_opts, []) + |> Keyword.update!(:callback_opts, &Map.new/1) + |> Keyword.put_new_lazy(:batch_id, &Oban.Pro.UUIDv7.generate/0) + |> Map.new() + + changesets = Stream.map(changesets, &put_meta(&1, opts)) + + %__MODULE__{changesets: changesets, opts: opts} + end + + defp put_meta(changeset, opts) do + meta = + changeset + |> Changeset.get_change(:meta, %{}) + |> Map.merge(opts) + + Changeset.put_change(changeset, :meta, meta) + end + + defp validate!(opts) do + Validation.validate_schema!(opts, + batch_id: :string, + batch_name: :string, + callback_opts: :list, + callback_worker: :module + ) + end + + @doc """ + Add one or more jobs to a batch. + + ## Examples + + Add jobs to an existing batche: + + Batch.add(batch, Enum.map(args, &MyWorker.new/1)) + + Add jobs to a batch one at a time: + + Batch.new() + |> Batch.add(MyWorker.new(%{})) + |> Batch.add(MyWorker.new(%{})) + |> Batch.add(MyWorker.new(%{})) + """ + @spec add(t(), Enumerable.t(changeset())) :: t() + def add(batch, %Changeset{} = changeset), do: add(batch, [changeset]) + + def add(%__MODULE__{} = batch, changesets) when is_list_or_stream(changesets) do + changesets = Stream.map(changesets, &put_meta(&1, batch.opts)) + + %{batch | changesets: Stream.concat(batch.changesets, changesets)} + end + + @doc """ + Append to a batch from an existing batch job. + + The `batch_id` and any other batch options are propagated to the newly created batch. + + > #### Appending and Callbacks {: .warning} + > + > Batch callback jobs are _only inserted once_. Appending to a batch where callbacks were already + > triggered, e.g. `batch_completed`, won't re-trigger the callback. + + ## Examples + + Build an empty batch from an existing batch job: + + job + |> Batch.append() + |> Batch.add(MyWorker.new(%{})) + |> Batch.add(MyWorker.new(%{})) + |> Oban.insert_all() + + Build a batch from an existing job with overriden options: + + Batch.append(job, callback_worker: MyApp.OtherWorker) + """ + @spec append(Job.t(), batch_opts()) :: t() + def append(%Job{meta: %{"batch_id" => _} = meta}, opts \\ []) when is_list(opts) do + orig_opts = + for {key, val} <- meta, + key in ~w(batch_id batch_name callback_opts callback_worker), + do: {String.to_existing_atom(key), val} + + orig_opts + |> Keyword.merge(opts) + |> new() + end + + @doc """ + Get all non-callback jobs from a batch. + + ## Examples + + Fetch results from all jobs from within a `batch_completed/1` callback: + + def batch_completed(%Job{} = job) do + results = + job + |> Batch.all_jobs() + |> Enum.map(&fetch_recorded/1) + + ... + end + + Get all batch jobs from anywhere using the `batch_id`: + + Batch.all_jobs("some-uuid-1234-5678") + + Get all jobs from anywhere with a custom Oban instance name: + + Batch.all_jobs(MyApp.Oban, "some-uuid-1234-5678") + """ + @spec all_jobs(Oban.name(), Job.t() | String.t(), [repo_opts()]) :: [Job.t()] + def all_jobs(name \\ Oban, job_or_batch_id, opts \\ []) + + def all_jobs(_name, %Job{conf: conf, meta: %{"batch_id" => batch_id}}, opts) do + Repo.all(conf, batch_query(batch_id), opts) + end + + def all_jobs(name, batch_id, opts) when is_binary(batch_id) do + name + |> Oban.config() + |> Repo.all(batch_query(batch_id), opts) + end + + def all_jobs(%Job{} = job, opts, []), do: all_jobs(Oban, job, opts) + + @doc """ + Cancel all non-callback jobs in a batch. + + Cancelling jobs via this function bulk-updates their state in the database without loading them + into memory or resolving their workers. As a result, **worker `after_process/3` hooks are not + called** for cancelled jobs. If you need to execute cleanup logic or other side effects when jobs + are cancelled, consider using batch callbacks like `batch_cancelled/1` or implement cleanup logic + at the application level. + + ## Examples + + Cancel jobs with the `job` in a `process/1` function: + + def process(job) do + if should_stop_processing?(job.args) do + Batch.cancel_jobs(job) + else + ... + end + end + + Cancel jobs from anywhere using the `batch_id`: + + Batch.cancel_jobs("some-uuid-1234-5678") + + Cancel jobs from anywhere with a custom Oban instance name: + + Batch.cancel_jobs(MyApp.Oban, "some-uuid-1234-5678") + """ + @spec cancel_jobs(Oban.name(), Job.t() | String.t()) :: {:ok, non_neg_integer()} + def cancel_jobs(oban_name \\ Oban, job_or_batch_id) + + def cancel_jobs(_oban_name, %Job{conf: conf, meta: %{"batch_id" => batch_id}}) do + cancel_jobs(conf.name, batch_id) + end + + def cancel_jobs(oban_name, batch_id) when is_binary(batch_id) do + Oban.cancel_all_jobs(oban_name, batch_query(batch_id)) + end + + @doc """ + Create a batch from a workflow. + + All jobs in the workflow are augmented to also be part of a batch. + + ## Examples + + Create a batch from a workflow without any extra options: + + Workflow.new() + |> Workflow.add(:step_1, MyWorker.new(%{id: 123})) + |> Workflow.add(:step_2, MyWorker.new(%{id: 345}), deps: [:step_1]) + |> Workflow.add(:step_3, MyWorker.new(%{id: 456}), deps: [:step_2]) + |> Batch.from_workflow() + |> Oban.insert_all() + + Create a batch with callback options: + + Batch.from_workflow(workflow, callback_opts: [priority: 3], callback_worker: MyApp.Worker) + """ + @spec from_workflow(Workflow.t(), batch_opts()) :: t() + def from_workflow(%Workflow{changesets: changesets}, opts \\ []) do + new(changesets, opts) + end + + @doc """ + Stream all non-callback jobs from a batch. + + ## Examples + + Stream all batch jobs from within a `batch_completed/1` callback: + + def batch_completed(%Job{} = job) do + {:ok, account_ids} = + MyApp.Repo.transaction(fn -> + job + |> Batch.stream_jobs() + |> Enum.map(& &1.args["account_id"]) + end) + + # use_account_ids + end + + Stream all batch jobs from anywhere using the `batch_id`: + + MyApp.Repo.transaction(fn -> + "some-uuid-1234-5678" + |> Batch.stream_jobs() + |> Enum.map(& &1.args["account_id"]) + end) + + Stream all batch jobs using a custom Oban instance name: + + MyApp.Repo.transaction(fn -> + MyApp.Oban + |> Batch.stream_jobs("some-uuid-1234-5678") + |> Enum.map(& &1.args["account_id"]) + end) + """ + @spec stream_jobs(Oban.name(), Job.t() | String.t(), [repo_opts()]) :: Enum.t() + def stream_jobs(name \\ Oban, job_or_batch_id, opts \\ []) + + def stream_jobs(_name, %Job{conf: conf, meta: %{"batch_id" => batch_id}}, opts) do + Repo.stream(conf, batch_query(batch_id), opts) + end + + def stream_jobs(name, batch_id, opts) when is_binary(batch_id) do + name + |> Oban.config() + |> Repo.stream(batch_query(batch_id), opts) + end + + def stream_jobs(%Job{} = job, opts, []), do: stream_jobs(Oban, job, opts) + + defp batch_query(batch_id) do + Job + |> where([j], fragment("? \\? 'batch_id'", j.meta)) + |> where([j], fragment("? ->> 'batch_id'", j.meta) == ^batch_id) + |> where([j], is_nil(fragment("? ->> 'callback'", j.meta))) + |> where([j], j.state in @all_states) + |> order_by(asc: :id) + end + + # Event Handling + + @doc false + @impl Oban.Pro.Flusher + def to_flush_mfa(job, conf) do + case job.meta do + %{"batch_id" => _, "callback" => _} -> :ignore + %{"batch_id" => batch_id} -> {__MODULE__, :on_flush, [job, batch_id, conf]} + _ -> :ignore + end + end + + @doc false + def on_flush(job, batch_id, conf) do + batch_worker = job.meta["batch_callback_worker"] || job.meta["callback_worker"] || job.worker + + with {:ok, worker} <- Worker.from_string(batch_worker), + supported = supported_callbacks(worker), + {:ok, {states, exists}} <- states_for_callbacks(supported, batch_id, conf) do + for callback <- supported, + callback not in exists, + callback_ready?(callback, states) do + insert_callback(callback, worker, job, conf) + end + end + end + + defp supported_callbacks(worker) do + for {name, [new, old]} <- @callbacks_to_combined, + function_exported?(worker, new, 1) or function_exported?(worker, old, 1), + do: name + end + + defp states_for_callbacks([], _batch_id, _conf), do: :ok + + defp states_for_callbacks(callbacks, batch_id, conf) do + state_query = + callbacks + |> Enum.flat_map(&Map.fetch!(@callbacks_to_states, &1)) + |> Enum.uniq() + |> Enum.reduce(:none, fn state, acc -> + query = + Job + |> select([_], [type(^state, :string)]) + |> where([j], fragment("? \\? 'batch_id'", j.meta)) + |> where([j], fragment("? ->> 'batch_id'", j.meta) == ^batch_id) + |> where([j], is_nil(fragment("? ->> 'callback'", j.meta))) + |> where([j], j.state == ^state) + |> limit(1) + + if acc == :none, do: query, else: union_all(acc, ^query) + end) + + exist_query = + Enum.reduce(callbacks, :none, fn callback, acc -> + query = + Job + |> select([_j], [type(^callback, :string)]) + |> where([j], fragment("? \\? 'batch_id'", j.meta)) + |> where([j], fragment("? ->> 'batch_id'", j.meta) == ^batch_id) + |> where([j], fragment("? ->> 'callback'", j.meta) == ^callback) + |> where([j], j.state in @all_states) + + if acc == :none, do: query, else: union_all(acc, ^query) + end) + + Repo.transaction(conf, fn -> + states = conf |> Repo.all(state_query) |> List.flatten() + exists = conf |> Repo.all(exist_query) |> List.flatten() + + {states, exists} + end) + end + + defp callback_ready?(callback, batch_states) do + ready? = + @callbacks_to_states + |> Map.fetch!(callback) + |> Enum.any?(&(&1 in batch_states)) + + # Other callbacks use a negated query to avoid counting `completed` jobs. + if callback in ~w(cancelled discarded retryable) do + ready? + else + not ready? + end + end + + defp insert_callback(callback, worker, job, conf) do + call_opts = + job.meta + |> Map.get("callback_opts", %{}) + |> Map.put_new("args", Map.get(job.meta, "batch_callback_args", %{})) + |> Map.put_new("meta", Map.get(job.meta, "batch_callback_meta", %{})) + |> Map.put_new("queue", Map.get(job.meta, "batch_callback_queue", job.queue)) + + unique = [ + period: :infinity, + fields: [:worker, :queue, :meta], + keys: [:batch_id, :callback], + states: Job.states() + ] + + {args, call_opts} = Map.pop(call_opts, "args") + {meta, call_opts} = Map.pop(call_opts, "meta") + + xtra_meta = + job.meta + |> Map.take(~w(batch_id batch_name)) + |> Map.put("callback", callback) + |> Map.merge(meta) + + opts = + call_opts + |> Keyword.new(fn {key, val} -> {String.to_existing_atom(key), val} end) + |> Keyword.put(:meta, xtra_meta) + |> Keyword.put(:unique, unique) + + changeset = worker.new(args, opts) + + if not changeset.valid? and Keyword.has_key?(changeset.errors, :args) do + changeset + |> structured_error_message() + |> Logger.error() + else + {:ok, Smart.insert_job(conf, changeset, [])} + end + end + + defp structured_error_message(changeset) do + """ + [Oban.Pro.Batch] can't insert batch callback because it has invalid keys: + + #{get_in(changeset.errors, [:args, Access.elem(0)])} + + Use one of the following options to restore batch callbacks: + + * Modify structured `keys` or `required` to allow the missing keys + * Include the required arguments with the `batch_callback_args` option + * Specify a different callback worker with the `batch_callback_worker` option + """ + end +end diff --git a/vendor/oban_pro/lib/oban/pro/cloud.ex b/vendor/oban_pro/lib/oban/pro/cloud.ex new file mode 100644 index 00000000..0d8c2b3b --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/cloud.ex @@ -0,0 +1,23 @@ +defmodule Oban.Pro.Cloud do + @moduledoc """ + A behaviour for interacting with cloud hosting providers. + """ + + @type conf :: term() + @type opts :: keyword() + @type quantity :: non_neg_integer() + + @doc """ + Executed once at runtime to gather, normalize, and transform options. + """ + @callback init(opts()) :: conf() + + @doc """ + Called to horizontally scale a cloud resource up or down. + + Successful scaling requests must return a new `conf` to be used during the next call to + `scale/2`. That allows cloud modules to track responses for additional control and + introspection. + """ + @callback scale(quantity(), conf()) :: {:ok, conf()} | {:error, term()} +end diff --git a/vendor/oban_pro/lib/oban/pro/cron.ex b/vendor/oban_pro/lib/oban/pro/cron.ex new file mode 100644 index 00000000..1f2c686c --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/cron.ex @@ -0,0 +1,125 @@ +defmodule Oban.Pro.Cron do + @moduledoc false + + use Ecto.Schema + + import Ecto.Changeset + + alias Oban.Cron.Expression + alias Oban.Worker + + @primary_key {:name, :string, autogenerate: false} + schema "oban_crons" do + field :expression, :string + field :worker, :string + field :opts, :map + field :paused, :boolean + field :insertions, {:array, :utc_datetime_usec} + field :lock_version, :integer, default: 1 + field :parsed, :any, virtual: true + + timestamps( + inserted_at: :inserted_at, + updated_at: :updated_at, + type: :utc_datetime_usec + ) + end + + @permitted ~w(name expression insertions worker opts paused)a + @requried ~w(name expression worker)a + @allowed_opts ~w(args guaranteed max_attempts meta priority queue tags timezone)a + + @doc false + def allowed_opts, do: @allowed_opts + + @spec changeset({binary(), module()} | {binary(), module(), Keyword.t()}) :: Ecto.Changeset.t() + def changeset({expression, worker}) do + params = %{expression: expression, name: worker, worker: worker, opts: %{}} + + changeset(%__MODULE__{}, params) + end + + def changeset({expression, worker, opts}) do + {name, opts} = Keyword.pop(opts, :name, worker) + {paused, opts} = Keyword.pop(opts, :paused) + {insertions, opts} = Keyword.pop(opts, :insertions) + + params = %{ + expression: expression, + insertions: insertions, + name: name, + opts: Map.new(opts), + paused: paused, + worker: worker + } + + changeset(%__MODULE__{}, params) + end + + @spec changeset({Ecto.Schema.t(), map()}) :: Ecto.Changeset.t() + def changeset(schema, params) when is_map(params) do + params = + params + |> coerce_name(:name) + |> coerce_name(:worker) + |> merge_opts(schema) + + schema + |> cast(params, @permitted) + |> validate_required(@requried) + |> validate_change(:expression, &expression_validator/2) + |> validate_change(:opts, &opts_validator/2) + |> optimistic_lock(:lock_version) + end + + def update_changeset(schema, params) when is_map(params) do + if Map.has_key?(params, :expression) or Map.has_key?(params, :timezone) do + changeset(schema, Map.put(params, :insertions, [])) + else + changeset(schema, params) + end + end + + defp coerce_name(params, key) do + case params do + %{^key => value} when is_atom(value) -> + Map.put(params, key, Worker.to_string(value)) + + _ -> + params + end + end + + defp merge_opts(params, schema) do + case Map.split(params, @allowed_opts) do + {opts, params} when map_size(opts) > 0 -> + opts = Map.new(opts, fn {key, val} -> {to_string(key), val} end) + + Map.put(params, :opts, Map.merge(schema.opts, opts)) + + _ -> + params + end + end + + # Validators + + defp expression_validator(:expression, expression) do + Expression.parse!(expression) + + [] + rescue + ArgumentError -> + [expression: "expected cron expression to be a parsable binary"] + end + + defp opts_validator(:opts, opts) do + string_keys = Enum.map(@allowed_opts, &to_string/1) + + if Enum.all?(opts, fn {key, _} -> to_string(key) in string_keys end) do + [] + else + [opts: "expected cron opts to be one of #{inspect(@allowed_opts)}"] + end + end +end diff --git a/vendor/oban_pro/lib/oban/pro/decorator.ex b/vendor/oban_pro/lib/oban/pro/decorator.ex new file mode 100644 index 00000000..d87ca6d4 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/decorator.ex @@ -0,0 +1,587 @@ +defmodule Oban.Pro.Decorator do + @moduledoc """ + The `Decorator` extension converts functions into Oban jobs with a simple annotation. + + Decorated functions, such as those in contexts or other non-worker modules, can be executed as + fully fledged background jobs with retries, scheduling, and the other guarantees you'd expect + from Oban jobs. + + ## Usage + + To get started, use the `Decorator` module, then annotate functions with the `@job` attribute. + Here's a simple example that uses `@job true` to decorate a function without any options: + + ```elixir + defmodule MyApp.Business do + use Oban.Pro.Decorator + + @job true + def weekly_report(tps_id, opts) do + ... + end + end + ``` + + The `@job` attribute also accepts all standard `Oban.Worker` options, e.g. `max_attempts`, + `priority`, `queue`. This example swaps out `@job true` for a list of options: + + ```elixir + @job [max_attempts: 3, priority: 1, queue: :reports, recorded: true] + def weekly_report(tps_id, opts) do + ... + end + ``` + + Now you can build and insert a job by calling `insert_weekly_report/2`: + + ```elixir + {:ok, job} = MyApp.Business.insert_weekly_report(123, pdf: true, rtf: true) + ``` + + Notice that the second argument is a keyword list of options, which isn't normally allowed in + job args because it's not JSON serializable. + + ## Generated Functions + + Functions decorated with `@job` generate three variants of the original function: + + * `new_*` — builds an `Oban.Job` changeset ready to be inserted for execution + * `insert_*` — builds and inserts an `Oban.Job` using `Oban.insert/2` + * `relay_*` — inserts a job, awaits execution, then returns the job's results + + See the `t:comp_opts/0` typespec for the subset of job options that are supported at compile + time. Additional options are available at runtime, as described in the [Runtime + Options](#module-runtime-options) section below. + + ### Using New + + The `new_` variant will build an `Oban.Job` changeset that's ready for insertion, but not + persisted to the database. This is identical to the output from calling `c:Oban.Worker.new/2` on + a standard worker. + + ```elixir + changeset = Business.new_weekly_report(123) + ``` + + The returned changeset is perfect for bulk inserts via `Oban.insert_all/1`: + + ```elixir + [123, 456, 789] + |> Enum.map(&Business.new_weekly_report/1) + |> Oban.insert_all() + ``` + + It can also be used to compose batches or workflows: + + ```elixir + alias Oban.Pro.Workflow + + Workflow.new() + |> Workflow.add(:rep_1, Business.new_weekly_report(1)) + |> Workflow.add(:rep_2, Business.new_weekly_report(2)) + |> Workflow.add(:rep_3, Business.new_weekly_report(3)) + |> Workflow.add(:fin, Business.new_finish_up(), deps: ~w(rep_1 rep_2 rep_3)a) + |> Oban.insert_all() + ``` + + ### Using Insert + + The `insert_` variant builds a changeset and immediately calls `Oban.insert/3` to enqueue it. + The result is a success tuple containing the job, or a changeset with errors. + + ```elixir + {:ok, job} = Business.insert_weekly_report(123) + ``` + + By default, jobs are inserted using the standard `Oban` instance. For applications that run + multiple Oban instances, or use a non-standard name, you can override the instance with the + `:oban` option: + + ```elixir + {:ok, job} = Business.insert_weekly_report(123, oban: SomeOban) + ``` + + ### Using Relay + + The `relay_` variant builds a changeset, inserts it, then leverages `Oban.Pro.Relay` to await + execution and return a result: + + ```elixir + {:ok, result} = Business.relay_weekly_report(123) + ``` + + The default timeout is a brief 5000ms, which doesn't account for scheduling or queueing time. + Provide an alternate timeout to wait longer: + + ```elixir + case Business.relay_weekly_report(123, timeout: 30_000) do + {:ok, result} -> IO.inspect(result, label: "RESULT") + {:error, reason} -> IO.inspect(reason, label: "ERROR") + end + ``` + + Note that the `timeout` option only controls how long the local process will block while + awaiting a result. The job will keep executing regardless of the timeout period. + + > #### Considerations and Caveats {: .info} + > + > Decorated functions are a convenient way to run functions in the background, and suitable in many + > situations. However, there are circumstances where they're unsuitable and you should exercise + > care: + > + > * Advanced worker functionality such as custom backoffs, hooks, structured args, or + > callbacks requires a dedicated worker module and isn't suitable for decoration. + > + > * Args of any type are safely serialized, but dumping large amounts of data may cause + > performance problems because it must be serialized, stored, and deserialized. + > + > * Changing function signatures while jobs are in-flight can cause jobs to fail, just like + > changing the shape of args passed to a `process/1` callback. + + ## Runtime Options + + Each decorated function has an additional generated clause that accepts job options, e.g. + `new_weekly_report/1` also has a `new_weekly_report/2` variant. + + All compile time options can be overridden at runtime. For example, to override the `queue` and + `max_attempts`: + + ```elixir + Business.insert_weekly_report(123, queue: "default", max_attempts: 10) + ``` + + In addition to compile time options, runtime options accepted by `Oban.Job.new/2` (other than + `worker`) are also allowed. This makes it possible to schedule decorated jobs: + + ```elixir + Business.insert_weekly_report(123, schedule_in: {1, :minute}) + ``` + + See `t:full_opts/0` for the complete typespec of runtime options. + + ## Patterns and Guards + + Each generated variant retains pattern matches and guards from the original function. That + allows expressive, defensive data validation before a job executes. + + For example, use a guard to ensure the `id` argument is an integer: + + ```elixir + @job true + def notify_user(id) when is_integer(id), do: ... + ``` + + Or, pattern match on a map with a `user_id` key and ensure the `id` is an integer: + + ```elixir + @job true + def notify_user(%{user_id: id}) when is_integer(id), do: ... + ``` + + > #### Multiple Function Clauses Not Supported {: .warning} + > + > The `@job` decorator only works with single-clause functions. When decorating a function + > with multiple clauses, only the first clause is captured. Subsequent clauses are ignored for + > decoration, which will cause `FunctionClauseError` when calling the decorated variants. + > + > Instead of using multiple clauses, use a single function clause with pattern matching inside: + > + > ```diff + > @job true + > -def process_data(1, 2), do: :special_case + > -def process_data(a, b), do: {:ok, a + b} + > +def process_data(a, b) do + > + case {a, b} do + > + {1, 2} -> :special_case + > + {a, b} -> {:ok, a + b} + > + end + > +end + > ``` + + ## Complex Types + + Any Elixir term may be passed as an argument, not just JSON encodable values. That enables + passing native data-types such as tuples, keywords, or structs that can't easily be used in + regular jobs. + + ```elixir + @job true + def add_discount(%User{id: _}, amount: 10_000), do: ... + ``` + + > #### Avoid Non-Portable Types {: .warning} + > + > Pass non-portable data types such as `pid`, `reference`, `port` with caution. There's no + > guarantee that a job will run on the same node and those specific values available. + > Furthermore, be careful passing anonymous functions because they are closures over the local + > environment. + + ## Return Values + + Decorated jobs respect the same standard return types as `c:Oban.Pro.Worker.process/1`. That + means you can return an `{:error, reason}` tuple to signify an error, or `{:cancel, reason}` to + quietly cancel a job. However, because decorated functions weren't necessarily designed to be + executed in a job, unexpected returns are considered a success. + + While there's no harm in returning `nil`, a struct, or some other non-standard value, it's best + to return an explicitly support a type such as `:ok` or `{:ok, value}`. + + ```diff + @job true + def update_account(user_id, params) do + user = Repo.get(User, user_id) + + do_update(user, params) + + - user + + {:ok, user} + end + ``` + + ## Unique and Replace + + Unique and replace options are available for decorated jobs. However, they have purposeful + limitations for compatibility with the decorated worker: + + * `unique` — only supports `states`, `period`, and `timestamp` options. The `fields` and `keys` + options aren't supported. + * `replace` — expects the newer, per-state syntax, and it doesn't support replacing the `worker` + or `args`. + + Both options can be defined at compilie in the `@job` annotation: + + ```elixir + @job [unique: [period: :infinity], replace: [scheduled: [:scheduled_at]]] + ``` + + Or as runtime options: + + ```elixir + Business.send_notice(user, schedule_in: 60, replace: [scheduled: [:scheduled_at]]) + ``` + + ## Limiting Decoration + + To avoid generating unnecessary functions you can disable generating `new`, `insert`, or `relay` + functions via passing flags to `use`: + + ```elixir + use Oban.Pro.Decorator, new: false + use Oban.Pro.Decorator, insert: false + use Oban.Pro.Decorator, relay: false + ``` + + Filtering options can be combined to restrict generation to one variant. For example, to only + generate `insert_*` functions: + + ```elixir + use Oban.Pro.Decorator, new: false, relay: false + ``` + + ## Testing Decorated Jobs + + Testing decorated jobs is tricky because they're always enqueued with `Oban.Pro.Decorated` as + the worker. The assert helpers `Oban.Pro.Testing` have a `:decorated` option specifically to + make testing decorated jobs more convenient. + + Pass a captured function with the original arity to the `decorated` option: + + ```elixir + assert_enqueued decorated: &Business.weekly_report/2 + ``` + + Use a list of to assert on args (not a map, as you would for a standard job): + + ```elixir + assert_enqueued args: [123, pdf: true], decorated: &Business.weekly_report/2 + ``` + + ## Accessing the Decorated Job + + The `Decorator` module provides a convenient way to access the currently executing job through + the `current_job/0` function. This can be useful when you need job context information during + execution. + + In any module that uses `Oban.Pro.Decorator`, you can call the `current_job/0` function to + retrieve the current job. This is particularly useful when you need to access job metadata as + part of a workflow. For example: + + ```elixir + defmodule MyApp.Doubler do + use Oban.Pro.Decorator + + alias Oban.Pro.Workflow + + def invoke do + Workflow.new() + |> Workflow.add(:dbl_1, new_double(1)) + |> Workflow.add(:dbl_2, new_double(2)) + |> Workflow.add(:dbl_3, new_double(3)) + |> Workflow.add(:print, new_print(), deps: [:dbl_1, :dbl_2, :dbl_3]) + |> Oban.insert_all() + end + + @job recorded: true + def double(value), do: value * 2 + + @job true + def print do + current_job() + |> Workflow.all_recorded(only_deps: true) + |> IO.inspect() + end + end + ``` + + If `current_job/0` is called outside of a job context (not during job execution), it returns + `nil`. + """ + + use Oban.Pro.Worker + + alias __MODULE__, as: Decorator + alias Oban.Pro.Relay + alias Oban.Validation + + args_schema do + field :mod, :string + field :fun, :string + field :arg, :term + end + + @typedoc """ + Options allowed in `@job` annotations at compile time. + """ + @type comp_opts :: [ + max_attempts: pos_integer(), + oban: GenServer.name(), + priority: 0..9, + queue: atom() | binary(), + recorded: boolean() | keyword(), + replace: [Job.replace_by_state_option()], + tags: Job.tags(), + unique: + true + | [ + period: timeout(), + states: atom() | [Job.unique_state()], + timestamp: :inserted_at | :scheduled_at + ] + ] + + @typedoc """ + Options allowed at runtime when calling generated `new_`, `insert_`, or `relay_` functions. + """ + @type full_opts :: [ + max_attempts: pos_integer(), + meta: map(), + oban: GenServer.name(), + priority: 0..9, + queue: atom() | binary(), + recorded: boolean() | keyword(), + replace: [Job.replace_by_state_option()], + scheduled_at: DateTime.t(), + schedule_in: Job.schedule_in_option(), + tags: Job.tags(), + timeout: timeout(), + unique: + true + | [ + period: Job.unique_period(), + states: Job.unique_state_group() | [Job.unique_state()], + timestamp: Job.unique_timestamp() + ] + ] + + @doc false + defmacro __using__(opts) do + deco_opts = Keyword.merge([new: true, insert: true, relay: true], opts) + + quote bind_quoted: [deco_opts: deco_opts] do + Module.put_attribute(__MODULE__, :oban_deco_opts, deco_opts) + Module.register_attribute(__MODULE__, :job, accumulate: true) + Module.register_attribute(__MODULE__, :oban_decorated, accumulate: true) + + @on_definition Oban.Pro.Decorator + @before_compile Oban.Pro.Decorator + end + end + + @doc false + defmacro __before_compile__(env) do + gen_opts = Module.get_attribute(env.module, :oban_deco_opts) + + for {fun, orig_args, guards, opts} <- Module.get_attribute(env.module, :oban_decorated) do + validate!(opts) + + {args, {_idx, renamed}} = + Macro.prewalk(orig_args, {0, %{}}, fn + {var, _meta, nil}, {idx, acc} -> + ast = {:"arg#{idx}", [], __MODULE__} + + {ast, {idx + 1, Map.put(acc, var, ast)}} + + other, idx_acc -> + {other, idx_acc} + end) + + guards = + case guards do + [] -> + quote(do: [is_boolean(true)]) + + [_ | _] -> + Macro.prewalk(guards, fn + {var, _meta, nil} when is_map_key(renamed, var) -> Map.fetch!(renamed, var) + other -> other + end) + end + + quote generated: true, + bind_quoted: [ + new: :"new_#{fun}", + ins: :"insert_#{fun}", + rel: :"relay_#{fun}", + gen_opts: gen_opts, + fun: fun, + mod: env.module, + opts: opts, + args: Macro.escape(args, unquote: true), + args_opts: Macro.escape(args ++ [Macro.var(:oban_opts, nil)], unquote: true), + guards: Macro.escape(guards, unquote: true) + ] do + def current_job do + with {_module, job, _opts} <- Process.get(:oban_processing), do: job + end + + if gen_opts[:new] do + def unquote(new)(unquote_splicing(args)) when unquote_splicing(guards) do + Decorator.new(unquote(mod), unquote(fun), unquote(args), unquote(opts)) + end + + def unquote(new)(unquote_splicing(args_opts)) when unquote_splicing(guards) do + full_opts = Keyword.merge(unquote(opts), unquote(Macro.var(:oban_opts, nil))) + + Decorator.new(unquote(mod), unquote(fun), unquote(args), full_opts) + end + end + + if gen_opts[:insert] do + def unquote(ins)(unquote_splicing(args)) when unquote_splicing(guards) do + Decorator.insert(unquote(mod), unquote(fun), unquote(args), unquote(opts)) + end + + def unquote(ins)(unquote_splicing(args_opts)) when unquote_splicing(guards) do + full_opts = Keyword.merge(unquote(opts), unquote(Macro.var(:oban_opts, nil))) + + Decorator.insert(unquote(mod), unquote(fun), unquote(args), full_opts) + end + end + + if gen_opts[:relay] do + def unquote(rel)(unquote_splicing(args)) when unquote_splicing(guards) do + Decorator.relay(unquote(mod), unquote(fun), unquote(args), unquote(opts)) + end + + def unquote(rel)(unquote_splicing(args_opts)) when unquote_splicing(guards) do + full_opts = Keyword.merge(unquote(opts), unquote(Macro.var(:oban_opts, nil))) + + Decorator.relay(unquote(mod), unquote(fun), unquote(args), full_opts) + end + end + end + end + end + + @doc false + def __on_definition__(env, :def, fun, args, guards, _body) do + with [opts] <- Module.get_attribute(env.module, :job) do + opts = if is_list(opts), do: opts, else: [] + + Module.put_attribute(env.module, :oban_decorated, {fun, List.wrap(args), guards, opts}) + end + + Module.delete_attribute(env.module, :job) + end + + def __on_definition__(_env, _kind, _fun, _args, _guards, _body), do: :ok + + @doc false + def new(mod, fun, arg, opts) when is_atom(mod) and is_atom(fun) and is_list(arg) do + name = "#{inspect(mod)}.#{fun}/#{length(arg)}" + meta = %{decorated: true, decorated_name: name} + + opts = + opts + |> Keyword.drop([:oban, :worker]) + |> Keyword.update(:meta, meta, &Map.merge(&1, meta)) + + new(%{mod: inspect(mod), fun: to_string(fun), arg: arg}, opts) + end + + @doc false + def insert(mod, fun, arg, opts) do + oban = Keyword.get(opts, :oban, Oban) + + mod + |> new(fun, arg, opts) + |> then(&Oban.insert(oban, &1)) + end + + @doc false + def relay(mod, fun, arg, opts) do + oban = Keyword.get(opts, :oban, Oban) + {timeout, opts} = Keyword.pop(opts, :timeout, 5_000) + + mod + |> new(fun, arg, opts) + |> then(&Relay.async(oban, &1)) + |> Relay.await(timeout) + end + + @impl Oban.Pro.Worker + def process(%{args: %{mod: mod, fun: fun, arg: arg}}) do + mod = + mod + |> String.split(".") + |> Module.safe_concat() + |> Code.ensure_loaded!() + + fun = String.to_existing_atom(fun) + + case apply(mod, fun, arg) do + :ok -> :ok + {:ok, _} = result -> result + {:error, _} = error -> error + {:snooze, _} = snooze -> snooze + {:cancel, _} = cancel -> cancel + other -> {:ok, other} + end + end + + defp validate!(opts) do + Validation.validate_schema!(opts, + max_attempts: :pos_integer, + oban: :any, + priority: {:range, 0..9}, + queue: {:or, [:atom, :string]}, + recorded: {:or, [:boolean, :list]}, + replace: {:custom, &Job.validate_replace/1}, + tags: {:list, :string}, + unique: {:custom, &validate_unique/1} + ) + end + + defp validate_unique(unique) do + cond do + is_list(unique) and Keyword.has_key?(unique, :fields) -> + {:error, "the :fields option isn't allowed for decorated jobs"} + + is_list(unique) and Keyword.has_key?(unique, :keys) -> + {:error, "the :keys option isn't allowed for decorated jobs"} + + true -> + Job.validate_unique(unique) + end + end +end diff --git a/vendor/oban_pro/lib/oban/pro/engines/smart.ex b/vendor/oban_pro/lib/oban/pro/engines/smart.ex new file mode 100644 index 00000000..76c802b4 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/engines/smart.ex @@ -0,0 +1,1589 @@ +defmodule Oban.Pro.Engines.Smart do + @moduledoc """ + The `Smart` engine provides advanced concurrency features, enhanced observability, lighter + weight bulk processing, and provides a foundation for accurate job lifecycle management. As an + `Oban.Engine`, it is responsible for all non-plugin database interaction, from inserting through + executing jobs. + + Major features include: + + * [Global Concurrency](#module-global-concurrency) — limit the number of concurrent jobs that + run across _all_ nodes + * [Rate Limiting](#module-rate-limiting) — limit the number of jobs that execute within a + window of time + * [Queue Partitioning](#module-queue-partitioning) — segment a queue so concurrency or rate + limits apply separately to each partition + * [Async Tracking](#module-async-tracking) — bundle job acks (completed, cancelled, etc.) to + minimize transactions and reduce load on the database + * [Enhanced Unique](#module-enhanced-unique) — enforce job uniqueness with a custom index to + accelerate inserting unique jobs safely between processes and nodes + * [Bulk Inserts](#module-bulk-inserts) — automatically batch inserts to prevent hitting + database limits, reduce data sent over the wire, and respect `unique` options when using + `Oban.insert_all/2` + * [Accurate Snooze](#module-accurate-snooze) — differentiate between attempts with errors and + intentional snoozes + + ## Installation + + See the [Smart Engine](adoption.md#1-smart-engine) section in the [adoption guide](adoption.md) + to get started. + + ## Global Concurrency + + Global concurrency limits the number of concurrent jobs that run across all nodes. + + Typically the global concurrency limit is `local_limit * num_nodes`. For example, with three + nodes and a local limit of 10, you'll have a global limit of 30. If a `global_limit` is present, + and the `local_limit` is omitted, then the `local_limit` falls back to the `global_limit`. + + The only way to guarantee that all connected nodes will run _exactly one job_ concurrently is to + set `global_limit: 1`. + + Here are some examples: + + ```elixir + # Execute 10 jobs concurrently across all nodes, with up to 10 on a single node + my_queue: [global_limit: 10] + + # Execute 10 jobs concurrently, but only 3 jobs on a single node + my_queue: [local_limit: 3, global_limit: 10] + + # Execute at most 1 job concurrently + my_queue: [global_limit: 1] + ``` + + ## Rate Limiting + + Rate limiting controls the number of jobs that execute within a period of time. + + Rate limiting uses counts for the same queue from all other nodes in the cluster (with or + without Distributed Erlang). The limiter uses a sliding window over the configured period to + accurately approximate a limit. + + Every job execution counts toward the rate limit, regardless of whether the job completes, + errors, snoozes, etc. + + Without a modifier, the `rate_limit` period is defined in seconds. However, you can provide a + `:second`, `:minute`, `:hour` or `:day` modifier to use more intuitive values. + + * `period: 30` — 30 seconds + * `period: {1, :minute}` — 60 seconds + * `period: {2, :minutes}` — 120 seconds + * `period: {1, :hour}` — 3,600 seconds + * `period: {1, :day}` —86,400 seconds + + Here are a few examples: + + ```elixir + # Execute at most 60 jobs per minute (1 job per second equivalent) + my_queue: [rate_limit: [allowed: 60, period: {1, :minute}]] + + # Execute at most 10 jobs per 30 seconds + my_queue: [rate_limit: [allowed: 10, period: 30]] + + # Execute at most 10 jobs per minute + my_queue: [rate_limit: [allowed: 10, period: {1, :minute}]] + + # Execute at most 1000 jobs per hour + my_queue: [rate_limit: [allowed: 1000, period: {1, :hour}]] + ``` + + Using larger time periods allows for smoother tracking of rate limits. For example, expressing + "1 job per second" as "60 jobs per minute" provides the same throughput but reduces the + granularity of tracking, resulting in more consistent job execution patterns. + + > #### Understanding Concurrency Limits {: .info} + > + > The local, global, or rate limit with the **lowest value** determines how many jobs are executed + > concurrently. For example, with a `local_limit` of 10 and a `global_limit` of 20, a single node + > will run 10 jobs concurrently. If that same queue had a `rate_limit` that allowed 5 jobs within + > a period, then a single node is limited to 5 jobs. + + ## Queue Partitioning + + In addition to global and rate limits at the queue level, you can partition a queue so that it's + treated as multiple queues where concurrency or rate limits apply separately to each partition. + + Partitions are specified with a list of fields like `:worker`, `:args`, or `:meta`. When + partitioning by `:args`, choosing specific keys is highly recommended to keep partitioning + meaningful. Focused partitioning minimizes the amount of data a queue needs to track and + simplifies job-fetching queries. + + ### Configuring Partitions + + The partition syntax is identical for global and rate limits (note that you can partition by + _global or rate_, but not both.) + + Here are a few examples of viable partitioning schemes: + + ```elixir + # Partition by worker alone + partition: :worker + + # Partition by the `id` and `account_id` from args, ignoring the worker + partition: [args: [:id, :account_id]] + + # Partition by worker and the `account_id` key from args + partition: [:worker, args: :account_id] + ``` + + Remember, take care to minimize partition cardinality by using a few `keys` whenever possible. + Partitioning based on _every permutation_ of your `args` makes concurrency or rate limits hard to + reason about and can negatively impact queue performance. + + ### Global Partitioning + + Global partitioning changes global concurency behavior. Rather than applying a fixed number for + the queue, it applies to every partition within the queue. + + Consider the following example: + + ```elixir + local_limit: 10, global_limit: [allowed: 1, partition: :worker] + ``` + + The queue is configured to run one job per-worker across every node, but only 10 concurrently on a + single node. That is in contrast to the standard behaviour of `global_limit`, which would override + the `local_limit` and only allow 1 concurrent job across every node. + + Alternatively, you could partition by a single key: + + ```elixir + local_limit: 10, global_limit: [allowed: 1, partition: [args: :tenant_id]] + ``` + + That configures the queue to run one job concurrently across the entire cluster per `tenant_id`. + + ### Partitioning with Burst Mode + + Global partitioning includes an advanced feature called "burst mode" for global limits + with partitioning. This feature allows you to maximize throughput by temporarily exceeding + per-partition global limits when there are available resources. + + Each global partition is typically restricted to the configured `allowed` value. However, with + burst mode enabled, the system can intelligently allocate more jobs to each active partition, + potentially exceeding the per-partition limit while still respecting the overall queue + concurrency. + + This is particularly useful when: + + 1. You have many potential partitions but only a few are active at any given time + 2. You want to maximize throughput while maintaining some level of fairness between partitions + 3. You need to ensure your queues aren't sitting idle when capacity is available + + Here's an example of a queue that will 5 jobs from a single partition concurrently under load, + but can burst up to 100 for a single partition when there is available capacity: + + ```elixir + queues: [ + exports: [ + global_limit: [ + allowed: 5, + burst: true, + partition: [args: :tenant_id] + ], + local_limit: 100 + ] + ] + ``` + + Bursting _still respects the overall global concurrency limit_. Using the example above, the + queue can only execute 100 of a given partition's jobs concurrently across all nodes, even if + there are more available resources. + + ### Rate Limit Partitioning + + Rate limit partitions operate similarly to global partitions. Rather than limiting all jobs within + the queue, they limit each partition within the queue. + + For example, to allow one job per-worker, every ten seconds, across every instance of the `alpha` + queue in your cluster: + + ```elixir + local_limit: 10, rate_limit: [allowed: 1, period: 10, partition: :worker] + ``` + + ### Partition Keys Cache + + The Smart engine maintains a cache of available partition keys to optimize performance when + fetching jobs. The cache has a configurable time-to-live (TTL) which controls how long partition + keys are remembered before needing to be refreshed from the database. + + The default TTL is set to `3_000ms`, which is suitable for most applications. You can adjust it + by adding configuration in your application's config: + + ```elixir + # In config/config.exs + config :oban_pro, Oban.Pro.Partition, keys_cache_ttl: 5_000 # 5 seconds + ``` + + A longer TTL reduces database load but might cause the system to be slower to recognize newly + created partition keys. A shorter TTL ensures more up-to-date partition information at the cost + of more frequent database queries. + + The number of partition keys retrieved is based on the queue's `local_limit` multiplied by a + configurable factor (default `3`). This ensures the query scales appropriately with queue + capacity while keeping result sets manageable: + + ```elixir + # In config/config.exs + config :oban_pro, Oban.Pro.Partition, keys_limit_multiplier: 5 + ``` + + Partitions are selected fairly by prioritizing those with the earliest scheduled jobs, ensuring + work is distributed across active partitions rather than favoring any single partition. + + ## Async Tracking + + The results of job execution, e.g. `completed`, `cancelled`, etc., are bundled together into a + single transaction to minimize load on an app's Ecto pool and the database. + + Bundling updates and reporting them asynchronously dramatically reduces the number of + transactions per second. However, async bundling introduces a slight lag (up to 5ms) between job + execution finishing and recording the outcome in the database. + + Async tracking can be disabled for specific queues with the `ack_async` option: + + ```elixir + queues: [ + standard: 30, + critical: [ack_async: false, local_limit: 10] + ] + ``` + + ## Enhanced Unique + + The `Smart` engine uses an alternative mechanism for unique jobs that's designed for speed, + correctness, scalability, and simplicity. Uniqueness is enforced through a unique index that + makes insertion entirely safe between processes and nodes, without the use of advisory locks or + multiple queries. + + Unlike standard uniqueness, which is only checked as jobs are inserted, the index-backed version + applies for the job's entire lifetime. That prevents race conditions where a job changes states + and inadvertently causes a conflict. + + When conflicts are detected, the conflicted job, i.e. the one already in the database, is + annotated with `uniq_conflict: true`. + + > #### Safe Hash for Uniqueness {: .info} + > + > To avoid potential hash collisions when using unique jobs with sub-fields in `args`, enable + > "safe" hashing: + > + > ```elixir + > config :oban_pro, Oban.Pro.Utils, safe_hash: true + > ``` + > + > This applies to `uniq_key`, `chain_key`, and `partition_key` values stored in job meta. + > Note that generated values will not match previous values for configurations using + > sub-fields in `args`. + + > #### Period-based Uniqueness Uses Buckets {: .info} + > + > To leverage unique indexes, period-based uniqueness snaps timestamps to fixed time buckets + > rather than using a sliding window. The current time is rounded down to the nearest multiple + > of the period, and uniqueness is enforced within that bucket. + > + > For example, with a period of 15 minutes (`unique: [period: {15, :minutes}]`) and a job + > inserted at 10:21:00, uniqueness applies from 10:15:00 to 10:30:00. A duplicate job + > inserted at 10:28:00 would be rejected, but one inserted at 10:31:00 would be allowed + > because it falls into the next bucket. + + ## Bulk Inserts + + Where the `Basic` engine requires you to insert unique jobs individually, the `Smart` engine adds + unique job support to `Oban.insert_all/2`. No additional configuration is necessary—simply use + `insert_all` instead for unique jobs. + + ```elixir + Oban.insert_all(lots_of_unique_jobs) + ``` + + Bulk insert also features automatic batching to support inserting an arbitrary number of jobs + without hitting database limits (PostgreSQL's binary protocol has a limit of 65,535 parameters + that may be sent in a single call. That presents an upper limit on the number of rows that may be + inserted at one time.) + + ```elixir + list_of_args + |> Enum.map(&MyApp.MyWorker.new/1) + |> Oban.insert_all() + ``` + + The default batch size for unique jobs is `250`, and `1_000` for non-unique jobs. Regardless, you + can override with `batch_size`: + + ```elixir + Oban.insert_all(lots_of_jobs, batch_size: 1500) + ``` + + It's also possible to set a custom timeout for batch inserts: + + ```elixir + Oban.insert_all(lots_of_jobs, timeout: :infinity) + ``` + + ## Accurate Snooze + + Unlike the `Basic` engine which increments `attempts` and `max_attempts`, the Smart engine rolls + back the `attempt` on snooze. This approach preserves the original `max_attempts` and records a + `snoozed` count in `meta`. As a result, it's simple to differentiate between "real" attempts and + snoozes, and backoff calculation remains accurate regardless of snoozing. + + The following `process/1` function demonstrates checking a job's `meta` for a `snoozed` count: + + ```elixir + def process(job) do + case job.meta do + %{"orig_scheduled_at" => unix_microseconds, "snoozed" => snoozed} -> + IO.inspect({snoozed, unix_microseconds}, label: "Times snoozed since") + + _ -> + # This job has never snoozed before + end + end + ``` + """ + + @behaviour Oban.Engine + @behaviour Oban.Pro.Handler + + import Ecto.Query + import DateTime, only: [utc_now: 0] + + alias Ecto.{Changeset, Multi} + alias Oban.{Backoff, Config, Engine, Job, Repo} + alias Oban.Pro.Limiters.{Global, Local, Rate} + alias Oban.Pro.{Flusher, Handler, Partition, Producer, Unique, Utils} + alias Oban.Pro.Stages.Chain + + require Logger + + @type partition :: + :worker + | {:args, atom()} + | [:worker | {:args, atom()}] + | [fields: [:worker | :args], keys: [atom()]] + + @type period :: pos_integer() | {pos_integer(), unit()} + + @type global_limit :: pos_integer() | [allowed: pos_integer(), partition: partition()] + + @type local_limit :: pos_integer() + + @type rate_limit :: [allowed: pos_integer(), period: period(), partition: partition()] + + @type unit :: :second | :seconds | :minute | :minutes | :hour | :hours | :day | :days + + # Module Attributes + + @ack_tabs Map.new(0..7, &{&1, :"pro_ack_tab_#{&1}"}) + + @drain_uuid "00000000-0000-0000-0000-000000000000" + + @registry Oban.Registry + + @virtual_opts ~w(ack_async queue refresh_interval xact_delay xact_retry xact_timeout updated_at)a + + @smart_opts Application.compile_env(:oban_pro, __MODULE__, []) + + @xact_expected_delay Access.get(@smart_opts, :xact_expected_delay, 20) + @base_batch_size Access.get(@smart_opts, :base_batch_size, 1_000) + + # Macros + + defguardp is_global(producer) when is_map(producer.meta) and is_map(producer.meta.global_limit) + + defmacrop merge_jsonb(column, map) do + quote do + fragment("? || ?", unquote(column), unquote(map)) + end + end + + defmacrop xact_lock(pref_key, lock_key) do + quote do + fragment("pg_try_advisory_xact_lock(?::int, ?::int)", unquote(pref_key), unquote(lock_key)) + end + end + + defmacrop state_in_string(column, string, prefix) do + quote do + fragment( + "? = ANY(regexp_split_to_array(?, ',')::?.oban_job_state[])", + unquote(column), + unquote(string), + unquote(prefix) + ) + end + end + + # Handler + + @impl Handler + def on_start do + for {_idx, name} <- @ack_tabs do + :ets.new(name, [:public, :named_table, read_concurrency: true, write_concurrency: true]) + end + end + + @impl Handler + def on_stop, do: :ok + + # Engine + + @impl Engine + def init(%Config{} = conf, [_ | _] = opts) do + {validate?, opts} = Keyword.pop(opts, :validate, false) + + {prod_opts, meta_opts} = + opts + |> Keyword.put_new(:ack_async, conf.testing == :disabled) + |> Keyword.split(@virtual_opts) + + changeset = + prod_opts + |> Keyword.put(:name, conf.name) + |> Keyword.put(:node, conf.node) + |> Keyword.put(:meta, meta_opts) + |> Keyword.put_new(:queue, :default) + |> Keyword.put_new(:started_at, utc_now()) + |> Keyword.put_new(:updated_at, utc_now()) + |> Producer.new() + + case Changeset.apply_action(changeset, :insert) do + {:ok, producer} -> + if validate? do + {:ok, producer} + else + ack_tab = + producer.queue + |> :erlang.phash2(map_size(@ack_tabs)) + |> then(&Map.fetch!(@ack_tabs, &1)) + + virt_opts = + prod_opts + |> Keyword.take(~w(ack_async refresh_interval xact_delay xact_retry)a) + |> Map.new() + + fun = + fn -> + conf + |> Repo.insert!(changeset) + |> Map.put(:ack_tab, ack_tab) + |> Map.merge(virt_opts) + |> put_producer(conf) + end + + transaction(conf, fun, virt_opts) + end + + {:error, changeset} -> + {:error, Utils.to_exception(changeset)} + end + end + + @impl Engine + def refresh(_conf, %Producer{} = producer) do + # The `Refresher` module handles updating the database rows independently of this callback + # being triggered. When the queue's producer is stuck in a transaction retry loop it isn't + # able to handle new messages, including periodic refresh. That would allow the producer + # record to become outdated, at which point it is subject to erroneous cleanup. + %{producer | updated_at: utc_now()} + end + + @impl Engine + def shutdown(%Config{} = conf, %Producer{} = producer) do + monitor_unregister(conf, producer) + + put_meta(conf, producer, :shutdown, true) + catch + _kind, _reason -> + producer + |> Producer.update_meta(:paused, true) + |> Changeset.apply_action!(:update) + end + + defp monitor_unregister(conf, producer) do + parent = self() + + Task.start(fn -> + ref = Process.monitor(parent) + + receive do + {:DOWN, ^ref, :process, _pid, _reason} -> + Process.demonitor(ref, [:flush]) + + Registry.delete_meta(@registry, reg_key(conf, producer)) + end + end) + end + + @impl Engine + def put_meta(%Config{} = conf, %Producer{} = producer, :flush, _any) do + producer = run_acks(conf, producer) + + if is_global(producer) do + Producer + |> where(uuid: ^producer.uuid) + |> then(&Repo.update_all(conf, &1, set: [meta: producer.meta, updated_at: utc_now()])) + end + + producer + end + + def put_meta(%Config{} = conf, %Producer{} = producer, :paused, true) do + changeset = + conf + |> run_acks(producer) + |> Producer.update_meta(:paused, true) + + conf + |> Repo.update!(changeset) + |> put_producer(conf) + end + + def put_meta(%Config{} = conf, %Producer{} = producer, :shutdown, true) do + changeset = + conf + |> run_acks(producer) + |> Producer.update_meta(%{paused: true, shutdown_started_at: utc_now()}) + + conf + |> Repo.update!(changeset) + |> put_producer(conf) + end + + def put_meta(%Config{} = conf, %Producer{} = producer, key, value) do + changeset = Producer.update_meta(producer, key, value) + + conf + |> Repo.update!(changeset) + |> put_producer(conf) + end + + @impl Engine + def check_meta(_conf, %Producer{} = producer, running) do + jids = for {_, {_, exec}} <- running, do: exec.job.id + + meta = + producer.meta + |> Map.from_struct() + |> flatten_windows() + + producer + |> Map.take(~w(name node queue uuid started_at updated_at)a) + |> Map.put(:running, jids) + |> Map.merge(meta) + end + + defp flatten_windows(%{rate_limit: %{windows: windows}} = meta) do + {pacc, cacc} = + Enum.reduce(windows, {0, 0}, fn {_key, map}, {pacc, cacc} -> + %{"prev_count" => prev, "curr_count" => curr} = map + + {pacc + prev, cacc + curr} + end) + + put_in(meta.rate_limit.windows, [%{"curr_count" => cacc, "prev_count" => pacc}]) + end + + defp flatten_windows(meta), do: meta + + @impl Engine + def fetch_jobs(_conf, %{meta: %{paused: true}} = prod, _running) do + {:ok, {prod, []}} + end + + def fetch_jobs(_conf, %{meta: %{local_limit: limit}} = prod, running) + when map_size(running) >= limit do + {:ok, {prod, []}} + end + + def fetch_jobs(%Config{} = conf, %Producer{} = producer, running) do + acks = get_acks(producer) + + multi = + Multi.new() + |> Multi.put(:acks, acks) + |> Multi.put(:conf, conf) + |> Multi.put(:running, running) + |> Multi.put(:producer, track_acks(acks, producer)) + |> Multi.run(:all_producers, &all_producers/2) + |> Multi.run(:ack_ids, &ack_jobs/2) + |> Multi.run(:afh_ids, &run_flush_handlers/2) + |> Multi.run(:local_demand, &Local.check/2) + |> Multi.run(:global_demand, &Global.check/2) + |> Multi.run(:rate_demand, &Rate.check/2) + |> Multi.run(:jobs, &fetch_jobs/2) + |> Multi.run(:tracked, &track_jobs(&1, &2, running)) + + case transaction(conf, multi, producer) do + {:ok, %{ack_ids: ack_ids, jobs: jobs, tracked: producer}} -> + del_acks(ack_ids, producer) + + {:ok, {producer, jobs}} + + {:error, _op, %{postgres: %{code: :unique_violation, detail: detail}}, changes} -> + clear_uniq_violation(changes.conf, detail) + + fetch_jobs(conf, producer, running) + + {:error, _operation, :xact_lock, _changes} -> + jittery_sleep() + + fetch_jobs(conf, producer, running) + + {:error, _operation, error, _changes} -> + raise error + end + catch + :error, %Postgrex.Error{postgres: %{code: :unique_violation, detail: detail}} -> + clear_uniq_violation(conf, detail, Enum.map(Job.states(), &to_string/1)) + + fetch_jobs(conf, producer, running) + end + + @doc false + def fetch_for_drain(conf, %{queue: queue, with_limit: limit} = opts) do + subset_query = + if queue == :__all__ do + where(Job, state: "available") + else + where(Job, state: "available", queue: ^to_string(queue)) + end + + subset_query = + case opts do + %{batch_ids: ids} -> where(subset_query, [j], j.meta["batch_id"] in ^ids) + %{workflow_ids: ids} -> where(subset_query, [j], j.meta["workflow_id"] in ^ids) + _ -> subset_query + end + + subset_query = + subset_query + |> order_by(asc: :priority, asc: :scheduled_at, asc: :id) + |> lock("FOR UPDATE SKIP LOCKED") + |> limit(^limit) + + query = + Job + |> with_cte("subset", as: ^subset_query) + |> join(:inner, [j], x in fragment(~s("subset")), on: true) + |> where([j, x], j.id == x.id) + |> select([j, _], j) + + updates = [ + set: [state: "executing", attempted_at: utc_now(), attempted_by: [conf.node, @drain_uuid]], + inc: [attempt: 1] + ] + + {_count, jobs} = Repo.update_all(conf, query, updates, []) + + jobs + end + + @impl Engine + def stage_jobs(%Config{} = conf, queryable, opts) do + limit = Keyword.fetch!(opts, :limit) + + subquery = + queryable + |> select([:id, :state]) + |> where([j], j.state in ~w(scheduled retryable)) + |> where([j], not is_nil(j.queue)) + |> where([j], j.scheduled_at <= ^DateTime.utc_now()) + |> limit(^limit) + + query = + Job + |> join(:inner, [j], x in subquery(subquery), on: j.id == x.id) + |> select([j, x], %{id: j.id, queue: j.queue, state: x.state}) + + {_count, staged} = Repo.update_all(conf, query, set: [state: "available"]) + + {:ok, staged} + catch + :error, %Postgrex.Error{postgres: %{code: :unique_violation, detail: detail}} -> + clear_uniq_violation(conf, detail, ["scheduled", "retryable"]) + + stage_jobs(conf, queryable, opts) + end + + @impl Engine + def check_available(%Config{} = conf) do + exists = + Job + |> where([j], j.state == "available" and parent_as(:producers).queue == j.queue) + |> select(1) + + query = + from(p in Producer, as: :producers) + |> where([p], exists(subquery(exists))) + |> select([p], p.queue) + + {:ok, Repo.all(conf, query)} + end + + @impl Engine + def complete_job(%Config{} = conf, %Job{} = job) do + set = [state: "completed", completed_at: utc_now()] + + case Process.get(:oban_recorded) do + nil -> put_ack(conf, %{job | state: "completed"}, set) + map -> put_ack(conf, %{job | state: "completed"}, set ++ [meta: map]) + end + + :ok + end + + @impl Engine + def discard_job(%Config{} = conf, %Job{} = job) do + put_ack(conf, %{job | state: "discarded"}, + state: "discarded", + discarded_at: utc_now(), + error: Job.format_attempt(job) + ) + + :ok + end + + @impl Engine + def error_job(%Config{} = conf, %Job{} = job, seconds) do + orig_at = Map.get(job.meta, "orig_scheduled_at", to_unix(job.scheduled_at)) + + set = + if job.attempt >= job.max_attempts do + [state: "discarded", discarded_at: utc_now()] + else + [state: "retryable", scheduled_at: seconds_from_now(seconds)] + end + + put_ack( + conf, + %{job | state: set[:state]}, + set ++ [error: Job.format_attempt(job), meta: %{orig_scheduled_at: orig_at}] + ) + + :ok + end + + @impl Engine + def snooze_job(%Config{} = conf, %Job{} = job, seconds) do + snoozed = Map.get(job.meta, "snoozed", 0) + orig_at = Map.get(job.meta, "orig_scheduled_at", to_unix(job.scheduled_at)) + + put_ack(conf, %{job | state: "scheduled"}, + attempt_change: -1, + state: "scheduled", + scheduled_at: seconds_from_now(seconds), + meta: %{orig_scheduled_at: orig_at, snoozed: snoozed + 1} + ) + + :ok + end + + @impl Engine + def cancel_job(%Config{} = conf, %Job{unsaved_error: %{}} = job) do + put_ack(conf, %{job | state: "cancelled"}, + state: "cancelled", + cancelled_at: utc_now(), + error: Job.format_attempt(job) + ) + end + + def cancel_job(%Config{} = conf, %Job{} = job) do + cancel_all_jobs(conf, where(Job, id: ^job.id)) + + :ok + end + + @impl Engine + def cancel_all_jobs(%Config{} = conf, queryable) do + subquery = where(queryable, [j], j.state not in ["cancelled", "completed", "discarded"]) + + query = + Job + |> join(:inner, [j], x in subquery(subquery), on: j.id == x.id) + |> update(set: [state: "cancelled", cancelled_at: ^utc_now()]) + |> select([_, x], map(x, [:id, :args, :attempted_by, :meta, :queue, :state, :worker])) + + {:ok, %{jobs: {_count, jobs}}} = + Multi.new() + |> Multi.put(:conf, conf) + |> Multi.update_all(:jobs, query, [], Repo.default_options(conf)) + |> Multi.run(:ack, &track_cancelled_jobs/2) + |> then(&Repo.transaction(conf, &1)) + + {:ok, jobs} + end + + @doc false + # This is used to unblock a uniqueness violation by clearing out the uniq_bmp field. The + # generated `uniq_key` column uses the `uniq_key` from meta, but is only present when the job's + # state maps to a value in `uniq_bmp`. + # + # The `uniq_key` column isn't present in `Oban.Job`, so we have to target it with a fragment. + # The row _without_ a `uniq_key` is the conflicting job, so we specifically clear that one's + # uniqueness. + def clear_uniq_violation(%Config{} = conf, detail, states \\ ["executing"]) do + with %{"key" => key} <- Regex.named_captures(~r/\(uniq_key\)=\((?.+)\)/, detail) do + query = + Job + |> where([j], j.state in ^states) + |> where([_], fragment("uniq_key IS NULL")) + |> where([j], fragment("? @> ?", j.meta, ^%{uniq_key: key})) + |> update([j], set: [meta: merge_jsonb(j.meta, ^%{uniq_bmp: []})]) + + Repo.update_all(conf, query, []) + + :telemetry.execute( + [:oban, :engine, :uniq_violation_repaired], + %{count: 1}, + %{conf: conf, uniq_key: key, states: states} + ) + + log_key = {__MODULE__, :uniq_violation_logged, key} + + unless :persistent_term.get(log_key, false) do + :persistent_term.put(log_key, true) + + Logger.info(fn -> + """ + [Oban.Pro.Engines.Smart] Unique constraint violation repaired. + + This may indicate a unique job misconfiguration where jobs are being inserted + with conflicting unique keys across different states. The engine repaired the + violation for uniq_key: #{key} + + Check your unique job configuration to ensure the :states option matches the + expected job lifecycle for this worker. + """ + end) + end + end + end + + @impl Engine + def insert_job(conf, changeset, opts) do + if changeset.valid? do + case insert_all_jobs(conf, [changeset], opts) do + [job] -> + {:ok, job} + + _ -> + {:error, changeset} + end + else + {:error, changeset} + end + end + + @impl Engine + def insert_all_jobs(conf, changesets, opts) do + fun = fn -> insert_all_batches(conf, changesets, opts) end + + {:ok, jobs} = Repo.transaction(conf, fun, opts) + + jobs + end + + defp insert_all_batches(conf, changesets, opts) do + batch_size = Keyword.get(opts, :batch_size, @base_batch_size) + + changesets + |> Stream.map(&Unique.with_uniq_meta/1) + |> Stream.map(&Partition.with_partition_meta(&1, conf)) + |> Stream.uniq_by(&Unique.get_key(&1, System.unique_integer())) + |> Stream.chunk_every(batch_size) + |> Enum.flat_map(fn changesets -> + {uniq_map, link_map, replace?} = + Enum.reduce(changesets, {%{}, %{}, false}, fn changeset, {uniq_map, link_map, replace?} -> + uniq_map = + case Unique.get_key(changeset) do + uniq_key when is_binary(uniq_key) -> + Map.put(uniq_map, uniq_key, changeset) + + _ -> + uniq_map + end + + link_map = + case Chain.get_key(changeset) do + chain_id when is_binary(chain_id) -> + Map.update(link_map, chain_id, [changeset], &[changeset | &1]) + + _ -> + link_map + end + + {uniq_map, link_map, replace? or Unique.replace?(changeset)} + end) + + {:ok, %{all_jobs: jobs}} = + Multi.new() + |> Multi.put(:conf, conf) + |> Multi.put(:opts, opts) + |> Multi.put(:changesets, changesets) + |> Multi.put(:uniq_map, uniq_map) + |> Multi.put(:link_map, link_map) + |> Multi.put(:replacements?, replace?) + |> Multi.run(:uniq_index?, &uniq_index?/2) + |> Multi.run(:chain_changesets, &prepare_chains/2) + |> Multi.run(:dupe_map, &find_dupes/2) + |> Multi.run(:new_jobs, &insert_entries/2) + |> Multi.run(:rep_jobs, &apply_replacements/2) + |> Multi.run(:all_jobs, &apply_conflicts/2) + |> then(&Repo.transaction(conf, &1, opts)) + + jobs + end) + end + + defp uniq_index?(_repo, %{conf: conf}) do + %{name: name, prefix: prefix} = conf + + Utils.persistent_cache({__MODULE__, :uniq_index?, name}, fn -> + query = + from("columns") + |> put_query_prefix("information_schema") + |> where(table_schema: ^prefix, table_name: "oban_jobs", column_name: "uniq_key") + |> select(true) + + {:ok, Repo.one(conf, query) == true} + end) + end + + defp prepare_chains(_repo, %{link_map: link_map, conf: conf}) when map_size(link_map) > 0 do + query = + from( + f in fragment("json_array_elements_text(?)", ^Map.keys(link_map)), + as: :list, + inner_lateral_join: + j in subquery( + Job + |> select([:state]) + |> where([j], j.state != "completed") + |> where([j], fragment("? \\? 'chain_id'", j.meta)) + |> where([j], fragment("?->>'chain_id'", j.meta) == parent_as(:list).value) + |> where([j], fragment("?->>'on_hold'", j.meta) in ~w(true false)) + |> order_by(desc: :id) + |> lock("FOR UPDATE") + |> limit(1) + ), + on: true, + select: {f.value, j.state} + ) + + found_map = + conf + |> Repo.all(query) + |> Map.new() + + linked = + Enum.flat_map(link_map, fn {chain_id, changesets} -> + changesets + |> Enum.reverse() + |> Enum.with_index() + |> Enum.map(fn {changeset, index} -> + state = Map.get(found_map, chain_id) + meta = Changeset.get_field(changeset, :meta) + + cond do + index == 0 and is_nil(state) -> changeset + index == 0 and Chain.continuable?(state, meta) -> changeset + true -> Chain.to_postponed(changeset) + end + end) + end) + + {:ok, linked} + end + + defp prepare_chains(_repo, _changes), do: {:ok, []} + + @impl Engine + def retry_job(%Config{} = conf, %Job{id: id}) do + retry_all_jobs(conf, where(Job, [j], j.id == ^id)) + + :ok + end + + @impl Engine + def retry_all_jobs(%Config{} = conf, queryable) do + subquery = + queryable + |> where([j], j.state not in ["available", "executing"]) + |> where([j], not fragment("? @> ?", j.meta, ^%{on_hold: true})) + + query = + Job + |> join(:inner, [j], x in subquery(subquery), on: j.id == x.id) + |> select([_, x], map(x, [:id, :queue, :state])) + |> update([j], + set: [ + state: "available", + max_attempts: fragment("GREATEST(?, ? + 1)", j.max_attempts, j.attempt), + scheduled_at: ^utc_now(), + completed_at: nil, + cancelled_at: nil, + discarded_at: nil + ] + ) + + {_, jobs} = Repo.update_all(conf, query, []) + + {:ok, jobs} + end + + @impl Engine + def update_job(%Config{} = conf, %Job{id: id}, changes) when is_map(changes) do + updater = fn -> + query = + Job + |> where([j], j.id == ^id) + |> lock("FOR UPDATE SKIP LOCKED") + + case Repo.one(conf, query) do + nil -> + {:error, :locked_or_not_found} + + job when is_map_key(job.meta, "structured") and is_map_key(changes, :args) -> + {:ok, worker} = Oban.Worker.from_string(job.worker) + + changeset = worker.new(changes.args) + + if changeset.valid? do + job + |> Job.update(changes) + |> then(&Repo.update(conf, &1)) + else + {:error, changeset} + end + + job -> + job + |> Job.update(changes) + |> then(&Repo.update(conf, &1)) + end + end + + case Repo.transaction(conf, updater) do + {:ok, result} -> result + {:error, reason} -> {:error, reason} + end + end + + # Producer Fetching Helpers + + defp get_producer(conf, job_or_producer) do + case Registry.meta(@registry, reg_key(conf, job_or_producer)) do + {:ok, producer} -> producer + :error -> %Producer{ack_async: false} + end + end + + defp put_producer(producer, conf) do + Registry.put_meta(@registry, reg_key(conf, producer), producer) + + producer + end + + defp reg_key(%{name: name}, %{queue: queue}), do: {name, {:producer, queue}} + + # Acking Helpers + + defp put_ack(conf, job, updates) do + producer = get_producer(conf, job) + global_key = Partition.get_key(job, "*") + flush_handlers = Flusher.get_flush_handlers(job, conf) + + ack_entry = {{:ack, producer.name, job.queue, job.id}, global_key, flush_handlers, updates} + + if producer.ack_tab, do: :ets.insert(producer.ack_tab, ack_entry) + + cond do + Process.get(:oban_draining) -> + Process.delete(:oban_recorded) + + {:ok, jids} = ack_jobs([ack_entry], conf) + + run_flush_handlers([ack_entry]) + + if producer.ack_tab, do: del_acks(jids, producer) + + not producer.ack_async or producer.meta.paused -> + pid = Oban.Registry.whereis(conf.name, {:producer, producer.queue}) + + if is_pid(pid) and Process.alive?(pid) do + GenServer.call(pid, {:put_meta, :flush, true}) + end + + true -> + :ok + end + end + + defp get_acks(%{ack_tab: tab, name: name, queue: queue}) do + :ets.select(tab, [{{{:ack, name, queue, :_}, :_, :_, :_}, [], [:"$_"]}]) + end + + defp del_acks(ids, %{ack_tab: tab, name: name, queue: queue}) do + Enum.each(ids, &:ets.delete(tab, {:ack, name, queue, &1})) + end + + defp run_acks(conf, producer) do + acks = get_acks(producer) + + {:ok, jids} = ack_jobs(acks, conf) + + run_flush_handlers(acks) + + del_acks(jids, producer) + + track_acks(acks, producer) + end + + defp run_flush_handlers(_repo, %{acks: acks}) do + {:ok, run_flush_handlers(acks)} + end + + defp run_flush_handlers(acks) do + mfas = for {_, _, all, _} <- acks, mfa <- all, is_tuple(mfa), uniq: true, do: mfa + + Enum.each(mfas, fn {mod, fun, arg} -> apply(mod, fun, arg) end) + end + + # Fetch Helpers + + defp all_producers(_repo, %{conf: conf, producer: producer}) do + %{ack_tab: tab, meta: meta, queue: queue} = producer + + needs_lock? = is_map(meta.global_limit) or is_map(meta.rate_limit) or any_flush_handlers?(tab) + query = where(Producer, queue: ^queue) + + cond do + not needs_lock? -> + {:ok, []} + + not has_advisory_locks?(conf) -> + {:ok, Repo.all(conf, lock(query, "FOR UPDATE NOWAIT"))} + + take_advisory_lock?(conf, queue) -> + {:ok, Repo.all(conf, query)} + + true -> + {:error, :xact_lock} + end + end + + defp any_flush_handlers?(tab) do + match = {:_, :_, :"$1", :_} + guard = [{:"/=", :"$1", []}] + + :ets.select_count(tab, [{match, guard, [true]}]) > 0 + end + + defp has_advisory_locks?(conf) do + Utils.persistent_cache({__MODULE__, :has_advisory_locks?}, fn -> + query = + from("pg_proc") + |> put_query_prefix("pg_catalog") + |> where(proname: "pg_try_advisory_xact_lock", pronargs: 2) + |> select(true) + + Repo.one(conf, query) == true + end) + end + + defp take_advisory_lock?(conf, queue) do + pre = :erlang.phash2(conf.prefix) + key = :erlang.phash2(queue) + + query = from(f in xact_lock(^pre, ^key), select: f.f0) + + Repo.one(conf, query) + end + + defp fetch_jobs(_repo, %{conf: conf, producer: producer} = changes) do + subset_query = fetch_subquery(changes) + + query = + Job + |> with_cte("subset", as: ^subset_query) + |> join(:inner, [j], x in fragment(~s("subset")), on: true) + |> where([j, x], j.id == x.id and j.state == "available" and j.attempt < j.max_attempts) + |> select([j, _], j) + + updates = [ + set: [state: "executing", attempted_at: utc_now(), attempted_by: [conf.node, producer.uuid]], + inc: [attempt: 1] + ] + + {_count, jobs} = Repo.update_all(conf, query, updates) + + {:ok, jobs} + end + + defp fetch_subquery(%{local_demand: local, producer: producer} = changes) do + case changes do + %{global_demand: nil, rate_demand: nil} -> + fetch_subquery(producer, local) + + %{global_demand: global, rate_demand: nil} when is_integer(global) -> + fetch_subquery(producer, min(local, global)) + + %{global_demand: nil, rate_demand: rated} when is_integer(rated) -> + fetch_subquery(producer, min(local, rated)) + + %{global_demand: global, rate_demand: %{} = demands} -> + fetch_subquery(producer, demands, min(local, global || local)) + + %{global_demand: %{} = demands, rate_demand: rated} -> + fetch_subquery(producer, demands, min(local, rated || local)) + + %{global_demand: global, rate_demand: rated} -> + limit = rated |> min(local) |> min(global) + + fetch_subquery(producer, limit) + end + end + + defp fetch_subquery(producer, limit) do + Job + |> select([:id]) + |> where(state: "available", queue: ^producer.queue) + |> order_by([:priority, :scheduled_at, :id]) + |> limit(^max(limit, 0)) + |> lock("FOR UPDATE SKIP LOCKED") + end + + defp fetch_subquery(producer, demands, limit) do + {keys, lims} = Enum.unzip(demands) + + part_query = + from( + p in fragment("SELECT unnest(?::text[]) as key, unnest(?::int[]) AS lim", ^keys, ^lims), + as: :part, + inner_lateral_join: + j in subquery( + Job + |> select([:id, :priority, :scheduled_at]) + |> where([j], j.state == "available" and j.queue == ^producer.queue) + |> where([j], fragment("partition_key") == parent_as(:part).key) + |> order_by([:priority, :scheduled_at, :id]) + |> limit(parent_as(:part).lim) + ), + on: true, + select: %{id: j.id, priority: j.priority, scheduled_at: j.scheduled_at} + ) + + from(j in subquery(part_query), + select: j.id, + limit: ^limit, + order_by: [:priority, :scheduled_at, :id] + ) + end + + # Tracking Helpers + + @ack_fields ~w(state cancelled_at completed_at discarded_at scheduled_at attempt_change error meta)a + + defp ack_jobs(_repo, %{acks: acks, conf: conf}), do: ack_jobs(acks, conf) + defp ack_jobs([], _conf), do: {:ok, []} + + defp ack_jobs(acks, conf) do + [ids | params] = + acks + |> Enum.map(fn {{:ack, _, _, id}, _, _, set} -> [id | Enum.map(@ack_fields, &set[&1])] end) + |> Enum.zip_with(&Function.identity/1) + + case Repo.query(conf, ack_query(conf), [ids | params]) do + {:ok, %{rows: rows}} -> {:ok, List.flatten(rows)} + error -> error + end + end + + defp ack_query(conf) do + Utils.persistent_cache({__MODULE__, :ack_query, conf.prefix}, fn -> + """ + WITH params AS ( + SELECT unnest($1::bigint[]) AS id, + unnest($2::"#{conf.prefix}".oban_job_state[]) AS state, + unnest($3::timestamp[]) AS cancelled_at, + unnest($4::timestamp[]) AS completed_at, + unnest($5::timestamp[]) AS discarded_at, + unnest($6::timestamp[]) AS scheduled_at, + unnest($7::integer[]) AS attempt_change, + unnest($8::jsonb[]) AS error, + unnest($9::jsonb[]) AS meta + ), + locked AS ( + SELECT oj.id + FROM "#{conf.prefix}"."oban_jobs" oj + INNER JOIN params tmp ON oj.id = tmp.id + FOR UPDATE OF oj + ) + UPDATE "#{conf.prefix}"."oban_jobs" oj + SET state = tmp.state, + cancelled_at = COALESCE(tmp.cancelled_at, oj.cancelled_at), + completed_at = COALESCE(tmp.completed_at, oj.completed_at), + discarded_at = COALESCE(tmp.discarded_at, oj.discarded_at), + scheduled_at = COALESCE(tmp.scheduled_at, oj.scheduled_at), + attempt = COALESCE(tmp.attempt_change + oj.attempt, oj.attempt), + errors = CASE WHEN tmp.error IS NULL THEN oj.errors ELSE oj.errors || tmp.error END, + meta = CASE WHEN tmp.meta IS NULL THEN oj.meta ELSE oj.meta || tmp.meta END + FROM params tmp + INNER JOIN locked l ON tmp.id = l.id + WHERE oj.id = tmp.id + RETURNING oj.id + """ + end) + end + + defp track_jobs(_repo, %{conf: conf, jobs: jobs, producer: producer}, running) do + old_jobs = for {_ref, {_pid, %{job: job}}} <- running, do: job + all_jobs = jobs ++ old_jobs + + meta = + producer.meta + |> Global.track(all_jobs) + |> Local.track(jobs) + |> Rate.track(jobs) + + if meta == producer.meta do + {:ok, producer} + else + now = utc_now() + query = where(Producer, uuid: ^producer.uuid) + + case Repo.update_all(conf, query, set: [meta: meta, updated_at: now]) do + {1, _} -> + {:ok, %{producer | meta: meta, updated_at: now}} + + # In this case the producer was erroneously deleted, possibly due to a connection error, + # downtime, or in development after waking from sleep. + {0, _} -> + %{producer | meta: meta, updated_at: now} + |> Changeset.change() + |> then(&Repo.insert(conf, &1)) + end + end + end + + defp track_acks(acks, producer) when is_global(producer) do + keys = Enum.map(acks, &elem(&1, 1)) + meta = Global.prepare_tracked(producer.meta, keys) + + %{producer | meta: meta} + end + + defp track_acks(_acks, producer), do: producer + + defp track_cancelled_jobs(_repo, %{conf: conf, jobs: {_count, jobs}}) do + jobs + |> Enum.filter(&(&1.state == "executing")) + |> Enum.group_by(& &1.attempted_by) + |> Enum.each(fn {[_node, uuid | _], jobs} -> + query = + Producer + |> where([p], p.uuid == ^uuid) + |> where([p], fragment("?->'global_limit' \\? 'tracked'", p.meta)) + + with %Producer{meta: meta} = producer <- Repo.one(conf, query) do + keys = Enum.map(jobs, &Partition.get_key(&1, "*")) + meta = Global.prepare_tracked(meta, keys) + + %{producer | meta: meta, updated_at: utc_now()} + |> Changeset.change() + |> then(&Repo.update(conf, &1)) + end + end) + + {:ok, nil} + end + + # Insert Helpers + + defp find_dupes(_repo, %{conf: conf, uniq_index?: false, uniq_map: uniq_map}) + when map_size(uniq_map) > 0 do + {uniq_keys, uniq_states} = + Enum.reduce(uniq_map, {[], []}, fn {key, changeset}, {key_acc, sta_acc} -> + states = changeset |> Unique.get_states() |> Enum.join(",") + + {[key | key_acc], [states | sta_acc]} + end) + + query = + from( + t in fragment( + "SELECT unnest(?::text[]) AS uk, unnest(?::text[]) AS us", + ^uniq_keys, + ^uniq_states + ), + join: j in Job, + on: + fragment("? \\? 'uniq_key'", j.meta) and + fragment("?->>'uniq_key'", j.meta) == t.uk and + state_in_string(j.state, t.us, literal(^conf.prefix)), + select: {t.uk, j.id, j.state} + ) + + dupes = + conf + |> Repo.all(query) + |> Map.new(fn {key, id, state} -> {key, {id, state, Map.get(uniq_map, key)}} end) + + {:ok, dupes} + end + + defp find_dupes(_repo, _changes), do: {:ok, %{}} + + defp insert_entries(_repo, %{changesets: []}), do: {:ok, []} + + defp insert_entries(_repo, changes) do + %{conf: conf, changesets: changesets, chain_changesets: chains} = changes + %{dupe_map: dupe_map, opts: opts} = changes + + changesets = + if chains == [] do + changesets + else + Enum.reject(changesets, &Chain.chain?/1) ++ chains + end + + {entries, placeholders} = + for changeset <- changesets, + not is_map_key(dupe_map, Unique.get_key(changeset)), + reduce: {[], nil} do + {entries, acc} -> + map = Job.to_map(changeset) + + if is_nil(acc) do + {[map | entries], map} + else + {[map | entries], Map.filter(acc, fn {key, val} -> map[key] == val end)} + end + end + + # Ensure placeholders aren't nil before use in is_map_key. + placeholders = placeholders || %{} + + entries = + for entry <- Enum.reverse(entries) do + Map.new(entry, fn {key, val} -> + if is_map_key(placeholders, key) do + {key, {:placeholder, key}} + else + {key, val} + end + end) + end + + opts = + opts + |> Keyword.merge(placeholders: placeholders, returning: true) + |> Keyword.merge(conflict_opts(changes)) + + {_count, jobs} = Repo.insert_all(conf, Job, entries, opts) + + {:ok, jobs} + end + + defp conflict_opts(%{uniq_map: uniq_map, uniq_index?: uniq_index?}) do + if uniq_map == %{} or not uniq_index? do + [] + else + [ + conflict_target: {:unsafe_fragment, "(uniq_key) WHERE uniq_key IS NOT NULL"}, + on_conflict: update(Job, [j], set: [meta: merge_jsonb(j.meta, ^%{uniq_conflict: true})]) + ] + end + end + + defp apply_replacements(_repo, %{replacements?: false}), do: {:ok, []} + + defp apply_replacements(_repo, %{conf: conf, opts: opts} = changes) do + updates = + for {_key, {id, state, %{changes: %{replace: replace} = changes}}} <- dupe_map(changes), + reduce: %{} do + acc -> + state = String.to_existing_atom(state) + rep_keys = Keyword.get(replace, state, []) + + changes + |> Map.take(rep_keys) + |> Enum.reduce(acc, fn {key, val}, sub_acc -> + Map.update(sub_acc, {key, val}, [id], &[id | &1]) + end) + end + + Enum.each(updates, fn {val, ids} -> + Repo.update_all(conf, where(Job, [j], j.id in ^ids), [set: [val]], opts) + end) + + {:ok, []} + end + + defp dupe_map(%{dupe_map: dupe_map, uniq_index?: false}), do: dupe_map + + defp dupe_map(%{new_jobs: new_jobs, uniq_map: uniq_map}) do + Enum.reduce(new_jobs, %{}, fn job, acc -> + case job do + %{meta: %{"uniq_conflict" => true, "uniq_key" => uniq_key}} -> + Map.put(acc, uniq_key, {job.id, job.state, Map.get(uniq_map, uniq_key)}) + + _ -> + acc + end + end) + end + + defp apply_conflicts(_repo, %{uniq_index?: false, dupe_map: dupe_map, new_jobs: new_jobs}) do + old_jobs = + dupe_map + |> Map.values() + |> Enum.map(fn {_id, _state, changeset} -> + changeset + |> Changeset.apply_action!(:insert) + |> Map.replace!(:conflict?, true) + end) + + {:ok, old_jobs ++ new_jobs} + end + + defp apply_conflicts(_repo, %{new_jobs: new_jobs}) do + new_jobs = + Enum.map(new_jobs, fn + %{meta: %{"uniq_conflict" => true}} = job -> %{job | conflict?: true} + job -> job + end) + + {:ok, new_jobs} + end + + # Time Helpers + + defp seconds_from_now(seconds), do: DateTime.add(utc_now(), seconds, :second) + + defp to_unix(datetime), do: DateTime.to_unix(datetime, :microsecond) + + # Xact Helpers + + defp jittery_sleep do + @xact_expected_delay + |> Backoff.jitter() + |> Process.sleep() + end + + defp transaction(conf, fun_or_multi, prod_opts) do + opts = [ + delay: Map.get(prod_opts, :xact_delay, 1000), + retry: Map.get(prod_opts, :xact_retry, 5), + timeout: Map.get(prod_opts, :xact_timeout, :timer.seconds(30)), + expected_delay: @xact_expected_delay, + expected_retry: 500 + ] + + Repo.transaction(conf, fun_or_multi, opts) + end +end diff --git a/vendor/oban_pro/lib/oban/pro/exceptions.ex b/vendor/oban_pro/lib/oban/pro/exceptions.ex new file mode 100644 index 00000000..bed23340 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/exceptions.ex @@ -0,0 +1,12 @@ +defmodule Oban.Pro.WorkflowError do + @moduledoc false + + defexception [:message, :reason] + + @impl Exception + def exception(reason) do + message = "upstream job was #{reason}, workflow can't complete" + + %__MODULE__{message: message, reason: reason} + end +end diff --git a/vendor/oban_pro/lib/oban/pro/flusher.ex b/vendor/oban_pro/lib/oban/pro/flusher.ex new file mode 100644 index 00000000..e6b5de42 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/flusher.ex @@ -0,0 +1,24 @@ +defmodule Oban.Pro.Flusher do + @moduledoc false + + alias Oban.{Config, Job} + + @doc """ + Construct an MFA for flushing accumulated operations after the Smart engine transaction commits. + + By convention, the function should be named `:on_flush`. + """ + @callback to_flush_mfa(Job.t(), Config.t()) :: :ignore | mfa() + + @doc """ + Extract flush handlers from all registered flushing modules. + + The flush order is important. Workflow flushing must complete before Batch flushing to ensure + batch wrapped workflow handlers are triggered. + """ + def get_flush_handlers(job, conf) do + [Oban.Pro.Workflow, Oban.Pro.Batch, Oban.Pro.Stages.Chain] + |> Enum.map(fn handler -> handler.to_flush_mfa(job, conf) end) + |> Enum.reject(&(&1 == :ignore)) + end +end diff --git a/vendor/oban_pro/lib/oban/pro/handler.ex b/vendor/oban_pro/lib/oban/pro/handler.ex new file mode 100644 index 00000000..38751182 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/handler.ex @@ -0,0 +1,13 @@ +defmodule Oban.Pro.Handler do + @moduledoc false + + @doc """ + Attach handler hooks. + """ + @callback on_start() :: any() + + @doc """ + Teardown handler hooks. + """ + @callback on_stop() :: any() +end diff --git a/vendor/oban_pro/lib/oban/pro/limiter.ex b/vendor/oban_pro/lib/oban/pro/limiter.ex new file mode 100644 index 00000000..35e20e30 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/limiter.ex @@ -0,0 +1,28 @@ +defmodule Oban.Pro.Limiter do + @moduledoc false + + alias Oban.Pro.Producer + alias Oban.{Config, Job} + + @type changes :: %{ + :conf => Config.t(), + :prod => Producer.t(), + :running => map(), + optional(atom()) => any() + } + + @type demand :: non_neg_integer() + + @type limit :: nil | demand | [{demand, term(), term()}] + @type repo :: Ecto.Repo.t() + + @doc """ + Calculate the current demand based on capacity and usage. + """ + @callback check(repo(), changes()) :: {:ok, limit()} + + @doc """ + Record demand usage. + """ + @callback track(Producer.meta(), [Job.t()]) :: Producer.meta() +end diff --git a/vendor/oban_pro/lib/oban/pro/limiters/global.ex b/vendor/oban_pro/lib/oban/pro/limiters/global.ex new file mode 100644 index 00000000..43fb7e10 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/limiters/global.ex @@ -0,0 +1,102 @@ +defmodule Oban.Pro.Limiters.Global do + @moduledoc false + + @behaviour Oban.Pro.Limiter + + alias Oban.Pro.Partition + + @impl Oban.Pro.Limiter + def check(_repo, %{producer: producer} = changes) when is_map(producer.meta.global_limit) do + tracked = + [producer | changes.all_producers] + |> Enum.uniq_by(& &1.uuid) + |> Enum.map(& &1.meta.global_limit) + |> Enum.filter(&is_map/1) + |> Enum.reduce(%{}, &merge/2) + + allowed = producer.meta.global_limit.allowed + tracked = update_legacy_tracked(tracked) + + if is_nil(producer.meta.global_limit.partition) do + {:ok, combined_demands(allowed, tracked)} + else + {:ok, separate_demands(changes.conf, allowed, producer, tracked)} + end + end + + def check(_repo, _changes), do: {:ok, nil} + + defp merge(global_limit, acc) do + Map.merge(global_limit.tracked, acc, fn _, v1, v2 -> v1 + v2 end) + end + + defp combined_demands(allowed, tracked) do + total = tracked |> Map.values() |> Enum.sum() + + max(allowed - total, 0) + end + + defp separate_demands(conf, allowed, producer, tracked) do + untracked = + for key <- Partition.available_keys(conf, producer.queue, producer.meta.local_limit), + not is_map_key(tracked, key), + into: %{}, + do: {key, allowed} + + allowed = + if producer.meta.global_limit.burst do + size = + tracked + |> Map.merge(untracked) + |> Map.delete("none") + |> map_size() + |> max(1) + + producer.meta.local_limit + |> div(size) + |> max(allowed) + else + allowed + end + + Enum.reduce(tracked, untracked, fn {key, total}, acc -> + case max(allowed - total, 0) do + demand when demand > 0 -> Map.put(acc, key, demand) + _ -> acc + end + end) + end + + # Tracking must be updated after acking and before fetching in order to have accurate limits for + # the current fetch transaction. + def prepare_tracked(%{global_limit: global} = meta, keys) do + tracked = + Enum.reduce(keys, global.tracked, fn key, acc -> + if is_map_key(acc, key) do + Map.update!(acc, key, &(&1 - 1)) + else + acc + end + end) + + put_in(meta.global_limit.tracked, tracked) + end + + @impl Oban.Pro.Limiter + def track(%{global_limit: global} = meta, jobs) when is_map(global) do + tracked = Enum.frequencies_by(jobs, &Partition.get_key(&1, "*")) + + put_in(meta.global_limit.tracked, tracked) + end + + def track(meta, _jobs), do: meta + + defp update_legacy_tracked(%{"count" => count}), do: %{"none" => count} + + defp update_legacy_tracked(tracked) do + Map.new(tracked, fn + {key, %{"count" => count}} -> {key, count} + tuple -> tuple + end) + end +end diff --git a/vendor/oban_pro/lib/oban/pro/limiters/local.ex b/vendor/oban_pro/lib/oban/pro/limiters/local.ex new file mode 100644 index 00000000..a9b498d0 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/limiters/local.ex @@ -0,0 +1,13 @@ +defmodule Oban.Pro.Limiters.Local do + @moduledoc false + + @behaviour Oban.Pro.Limiter + + @impl Oban.Pro.Limiter + def check(_repo, %{producer: producer, running: running}) do + {:ok, producer.meta.local_limit - map_size(running)} + end + + @impl Oban.Pro.Limiter + def track(meta, _jobs), do: meta +end diff --git a/vendor/oban_pro/lib/oban/pro/limiters/rate.ex b/vendor/oban_pro/lib/oban/pro/limiters/rate.ex new file mode 100644 index 00000000..ab6c132f --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/limiters/rate.ex @@ -0,0 +1,125 @@ +defmodule Oban.Pro.Limiters.Rate do + @moduledoc false + + @behaviour Oban.Pro.Limiter + + alias Oban.Pro.Partition + + @impl Oban.Pro.Limiter + def check(_repo, %{producer: producer} = changes) when is_map(producer.meta.rate_limit) do + %{allowed: allowed, period: period, window_time: prev_time} = producer.meta.rate_limit + + curr_time = unix_now() + weight = div(max(period - curr_time - prev_time, 0), period) + + windows = + [producer | changes.all_producers] + |> Enum.uniq_by(& &1.uuid) + |> Enum.map(& &1.meta.rate_limit) + |> Enum.filter(&(is_map(&1) and &1.window_time >= curr_time - &1.period)) + |> Enum.reduce(%{}, &merge/2) + + windows = update_legacy_windows(windows) + + if is_nil(producer.meta.rate_limit.partition) do + {:ok, combined_demands(weight, allowed, windows)} + else + {:ok, separate_demands(changes.conf, weight, allowed, producer, windows)} + end + end + + def check(_repo, _changes), do: {:ok, nil} + + defp merge(rate_limit, acc) do + Map.merge(rate_limit.windows, acc, fn _key, v1, v2 -> + %{"curr_count" => c1, "prev_count" => p1} = v1 + %{"curr_count" => c2, "prev_count" => p2} = v2 + %{"curr_count" => c1 + c2, "prev_count" => p1 + p2} + end) + end + + defp update_legacy_windows(%{"args" => _} = window), do: %{"none" => window} + defp update_legacy_windows(windows), do: windows + + defp combined_demands(weight, allowed, windows) do + total = + Enum.reduce(windows, 0, fn {_, val}, acc -> + acc + (val["prev_count"] * weight + val["curr_count"]) + end) + + max(allowed - total, 0) + end + + defp separate_demands(conf, weight, allowed, producer, windows) do + untracked = + for key <- Partition.available_keys(conf, producer.queue, producer.meta.local_limit), + not is_map_key(windows, key), + into: %{}, + do: {key, allowed} + + Enum.reduce(windows, untracked, fn {key, val}, acc -> + %{"curr_count" => curr, "prev_count" => prev} = val + + case max(allowed - (prev * weight + curr), 0) do + demand when demand > 0 -> Map.put(acc, key, demand) + _ -> acc + end + end) + end + + @impl Oban.Pro.Limiter + def track(%{rate_limit: rate_limit} = meta, jobs) when is_map(rate_limit) do + new_mapping = + Enum.reduce(jobs, %{}, fn job, acc -> + key = Partition.get_key(job, "*") + map = %{"curr_count" => 1, "prev_count" => 0} + + Map.update(acc, key, map, &%{&1 | "curr_count" => &1["curr_count"] + 1}) + end) + + new_windows = + for {key, val} <- new_mapping, + not is_map_key(rate_limit.windows, key), + into: %{}, + do: {key, val} + + prev_unix = rate_limit.window_time + unix_diff = unix_now() - prev_unix + + {mode, next_time} = + cond do + unix_diff > rate_limit.period * 2 -> {:dump_prev, unix_now()} + unix_diff > rate_limit.period -> {:swap_prev, unix_now()} + true -> {:bump_curr, prev_unix} + end + + all_windows = + for {key, window} <- rate_limit.windows, + not match?(%{"curr_count" => 0, "prev_count" => 0}, window), + into: new_windows, + do: {key, put_next_count(key, window, mode, new_mapping)} + + %{meta | rate_limit: %{rate_limit | windows: all_windows, window_time: next_time}} + end + + def track(meta, _jobs), do: meta + + defp put_next_count(key, window, mode, mapping) do + %{"curr_count" => curr_count, "prev_count" => prev_count} = window + + next_count = get_in(mapping, [key, "curr_count"]) || 0 + + {curr_count, prev_count} = + case mode do + :dump_prev -> {next_count, 0} + :swap_prev -> {next_count, curr_count} + :bump_curr -> {curr_count + next_count, prev_count} + end + + %{window | "curr_count" => curr_count, "prev_count" => prev_count} + end + + defp unix_now do + DateTime.to_unix(DateTime.utc_now(), :second) + end +end diff --git a/vendor/oban_pro/lib/oban/pro/migration.ex b/vendor/oban_pro/lib/oban/pro/migration.ex new file mode 100644 index 00000000..a61d9e79 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/migration.ex @@ -0,0 +1,387 @@ +defmodule Oban.Pro.Migration do + @moduledoc """ + Migrations create and modify the database tables and indexes Oban Pro needs to function. + + If you use prefixes, or have multiple instances with different prefixes, you can specify the + prefix and create multiple tables in one migration: + + ## Usage + + To use migrations in your application you'll need to generate an `Ecto.Migration` that wraps + calls to `Oban.Pro.Migration`: + + ```bash + mix ecto.gen.migration add_oban_pro + ``` + + Open the generated migration and delegate the `up/0` and `down/0` functions to + `Oban.Pro.Migration`: + + ```elixir + defmodule MyApp.Repo.Migrations.AddObanPro do + use Ecto.Migration + + def up, do: Oban.Pro.Migration.up() + + def down, do: Oban.Pro.Migration.down() + end + ``` + + This will run all of the necessary migrations for your database. + + Now, run the migration to create the table: + + ```bash + mix ecto.migrate + ``` + + ## Concurrent Migrations + + For systems with many retained jobs, you may want to create indexes concurrently to avoid + blocking writes to your tables during migration. The `only` option allows splitting migrations + into separate steps for schema changes and index creation: + + ```bash + mix ecto.gen.migration add_oban_pro_schemas + mix ecto.gen.migration add_oban_pro_indexes + ``` + + The `schemas` migration will add all necessary tables and columns within a DDL transaction: + + ```elixir + defmodule MyApp.Repo.Migrations.AddObanProSchemas do + use Ecto.Migration + + def up, do: Oban.Pro.Migration.up(only: :schemas) + def down, do: Oban.Pro.Migration.down(only: :schemas) + end + ``` + + Then the `indexes` migration will add indexes concurrently, outside of a transaction: + + ```elixir + defmodule MyApp.Repo.Migrations.AddObanProIndexes do + use Ecto.Migration + + # Not needed with use advisory locks, i.e. `migration_lock: :pg_advisory_lock` + @disable_migration_lock true + @disable_ddl_transaction true + + def up, do: Oban.Pro.Migration.up(only: :indexes) + def down, do: Oban.Pro.Migration.down(only: :indexes) + end + ``` + + #### When to Use Split Migrations + + Use the default, combined approach when: + + 1. Setting up a development or test environment + 2. Working with small tables where index creation is quick + 3. Simplicity is preferred over non-blocking operations + + Choose split migrations when: + + 1. Operating on a database where blocking writes could cause downtime + 2. Working with large tables where index creation would block for an extended period + + ## Upgrading + + Like `Oban`, migrations are versioned and you may need to run the migration again when new + `oban_pro` versions are released. To do this, generate a new migration: + + ```bash + mix ecto.gen.migration upgrade_oban_pro + ``` + + Open the generated migration in your editor and call the `up` and `down` functions on + `Oban.Pro.Migration`, with an optional version number: + + ```elixir + defmodule MyApp.Repo.Migrations.UpgradeObanPro do + use Ecto.Migration + + def up, do: Oban.Pro.Migration.up(version: "1.6.0") + + def down, do: Oban.Pro.Migration.down(version: "1.6.0") + end + ``` + + > #### Differences from Oban : .tip + > + > Unlike Oban, the migration versions correspond directly to the `oban_pro` package version. + > In addition, running `down/0` will only step back one version, not all the way down. + + ## Isolation with Prefixes + + Oban supports namespacing through PostgreSQL schemas, therefore so does Oban Pro. Specify a + prefix within your migration: + + ```elixir + defmodule MyApp.Repo.Migrations.AddPrefixedObanPro do + use Ecto.Migration + + def up, do: Oban.Pro.Migration.up(prefix: "private") + + def down, do: Oban.Pro.Migration.down(prefix: "private") + end + ``` + """ + + @behaviour Oban.Pro.Handler + + use Ecto.Migration + + require Logger + + @versions ~w(1.0.0 1.4.0 1.5.0 1.6.0) + + @impl Oban.Pro.Handler + def on_start do + event = [:oban, :supervisor, :init] + + :telemetry.attach("oban.migration", event, &__MODULE__.handle_event/4, nil) + end + + @impl Oban.Pro.Handler + def on_stop do + :telemetry.detach("oban.migration") + end + + @doc false + def handle_event(_event, _timing, %{conf: conf}, _) do + prefix = conf.prefix + + case :persistent_term.get(__MODULE__, %{}) do + %{^prefix => _} -> + :ok + + versions -> + :persistent_term.put(__MODULE__, Map.put(versions, prefix, true)) + + last = List.last(@versions) + + with_dynamic_repo(conf, fn repo -> + with :off <- Keyword.get(repo.config(), :pool, :off), + curr <- migrated_version(%{prefix: prefix, repo: repo}) do + cond do + is_nil(curr) -> + Logger.warning( + "Oban Pro for prefix #{inspect(prefix)} isn't migrated. Run migrations to create " <> + "the tables and indexes necessary for Oban Pro to function." + ) + + not String.contains?(curr, last) -> + Logger.warning( + "Oban Pro for prefix #{inspect(prefix)} is migrated to #{inspect(curr)}, but the " <> + "latest is #{inspect(last)}. Run migrations to upgrade to the latest version." + ) + + true -> + :ok + end + end + end) + end + end + + defp with_dynamic_repo(%{get_dynamic_repo: nil, repo: repo}, fun) do + fun.(repo) + end + + defp with_dynamic_repo(%{get_dynamic_repo: dyn_fun, repo: repo}, fun) do + prev_repo = repo.get_dynamic_repo() + + dyna_repo = + case dyn_fun do + {mod, fun, arg} -> apply(mod, fun, arg) + fun when is_function(fun, 0) -> fun.() + end + + try do + repo.put_dynamic_repo(dyna_repo) + + fun.(repo) + after + repo.put_dynamic_repo(prev_repo) + end + + fun.(repo) + end + + @doc """ + Run the `up` changes for all migrations between the initial version and the current version. + + ## Example + + Run all migrations up to the current version: + + Oban.Pro.Migration.up() + + Run migrations up to a specified version: + + Oban.Pro.Migration.up(version: "1.4.0") + + Run migrations in an alternate prefix: + + Oban.Pro.Migration.up(prefix: "payments") + """ + def up(opts \\ []) when is_list(opts) do + opts = with_defaults(opts) + curr = migrated_version(opts) + indx = migrated_version_index(curr, opts) + + cond do + is_nil(indx) -> inner_change(0..opts.next, :up, curr, opts) + indx < opts.next -> inner_change((indx + 1)..opts.next, :up, curr, opts) + true -> :ok + end + end + + @doc """ + Run the `down` changes for migrations between the current version and the previous version. + + This behaviour differs from `Oban.Migration` by only stepping down **one version** per + invocation. You must explicitly specify version `1.0.0` to rollback all the way and drop initial + tables. + + ## Example + + Run migrations down from the current version to the previous: + + Oban.Pro.Migration.down() + + Run migrations down to and including a specified version: + + Oban.Pro.Migration.down(version: "1.4.0") + + Run all down migrations back to the original: + + Oban.Pro.Migration.down(version: "1.0.0") + + Run migrations in an alternate prefix: + + Oban.Pro.Migration.down(prefix: "payments") + """ + def down(opts \\ []) when is_list(opts) do + opts = with_defaults(opts) + curr = migrated_version(opts) + indx = migrated_version_index(curr, opts) + + if is_integer(indx) and indx >= opts.next do + inner_change(indx..opts.next//-1, :down, curr, opts) + end + end + + @doc false + def migrated_version(opts) when is_map(opts) do + repo = Map.get_lazy(opts, :repo, &repo/0) + opts = with_defaults(opts) + + query = """ + SELECT pg_catalog.obj_description(pg_class.oid, 'pg_class') + FROM pg_class + LEFT JOIN pg_namespace ON pg_namespace.oid = pg_class.relnamespace + WHERE pg_class.relname = 'oban_producers' + AND pg_namespace.nspname = '#{opts.escaped}' + """ + + case repo.query(query, [], log: false) do + {:ok, %{rows: [[version]]}} when is_binary(version) -> version + _ -> nil + end + end + + defp migrated_version_index(version, opts) do + [schemas_version, indexes_version] = normalize_recorded(version) + + case opts.only do + [:schemas] -> find_version_index(schemas_version) + [:indexes] -> find_version_index(indexes_version) + _ -> find_version_index(schemas_version) + end + end + + defp inner_change(_curr_index..next_index//_ = range, direction, current, opts) do + for index <- range do + version = Enum.at(@versions, index) + migration = Module.concat(Oban.Pro.Migrations, "V#{String.replace(version, ".", "")}") + + {do_ddl, do_idx} = + if direction == :up do + {:up_ddl, :up_idx} + else + {:down_ddl, :down_idx} + end + + if :schemas in opts.only, do: apply(migration, do_ddl, [opts]) + if :indexes in opts.only, do: apply(migration, do_idx, [opts]) + + flush() + end + + case direction do + :up -> record_version(next_index, current, opts) + :down -> record_version(next_index - 1, current, opts) + end + end + + defp record_version(-1, _recorded, _opts), do: :ok + + defp record_version(index, recorded, opts) do + [recorded_schemas, recorded_indexes] = normalize_recorded(recorded) + + prefix = Enum.at(@versions, index) + + versions = + case opts.only do + [:indexes] -> [recorded_schemas, "#{prefix}-indexes"] + [:schemas] -> ["#{prefix}-schemas", recorded_indexes] + _ -> ["#{prefix}-schemas", "#{prefix}-indexes"] + end + + versions = versions |> Enum.filter(& &1) |> Enum.join(",") + + execute "COMMENT ON TABLE #{opts.quoted}.oban_producers IS '#{versions}'" + end + + defp with_defaults(opts) do + prefix = Access.get(opts, :prefix, "public") + unlogged = Access.get(opts, :unlogged, true) + version = Access.get(opts, :version, List.last(@versions)) + + only = + opts + |> Access.get(:only, [:schemas, :indexes]) + |> List.wrap() + + %{ + concurrently: only == [:indexes], + escaped: String.replace(prefix, "'", "\\'"), + next: find_version_index(version), + only: only, + prefix: prefix, + quoted: inspect(prefix), + unlogged: unlogged, + version: version + } + end + + defp find_version_index(nil), do: nil + + defp find_version_index(version) do + version = String.replace(version, ["-schemas", "-indexes"], "") + + Enum.find_index(@versions, &(&1 == version)) + end + + defp normalize_recorded(nil), do: [nil, nil] + + defp normalize_recorded(version) do + case String.split(version, ",", parts: 2) do + [<<_prefix::binary-size(5), "-schemas">> = schemas] -> [schemas, nil] + [version] -> ["#{version}-schemas", "#{version}-indexes"] + recorded -> recorded + end + end +end diff --git a/vendor/oban_pro/lib/oban/pro/migrations/dynamic_partitioner.ex b/vendor/oban_pro/lib/oban/pro/migrations/dynamic_partitioner.ex new file mode 100644 index 00000000..2f01335d --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/migrations/dynamic_partitioner.ex @@ -0,0 +1,195 @@ +defmodule Oban.Pro.Migrations.DynamicPartitioner do + @moduledoc false + + use Ecto.Migration + + alias Oban.Config + alias Oban.Pro.Plugins.DynamicPartitioner + + def change(opts \\ []) do + case direction() do + :up -> up(opts) + :down -> down(opts) + end + end + + def up(opts \\ []) do + prefix = Keyword.get(opts, :prefix, "public") + quoted = inspect(prefix) + + if Keyword.get(opts, :create_schema, prefix != "public") do + execute "CREATE SCHEMA IF NOT EXISTS #{quoted}", "" + end + + execute """ + ALTER TABLE IF EXISTS #{quoted}.oban_jobs RENAME TO oban_jobs_old + """ + + execute """ + ALTER INDEX IF EXISTS #{quoted}.oban_jobs_pkey RENAME TO oban_jobs_pkey_old + """ + + execute """ + ALTER INDEX IF EXISTS #{quoted}.oban_jobs_args_index RENAME TO oban_jobs_args_index_old + """ + + execute """ + ALTER INDEX IF EXISTS #{quoted}.oban_jobs_meta_index RENAME TO oban_jobs_meta_index_old + """ + + execute """ + DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type + WHERE typname = 'oban_job_state' + AND typnamespace = '#{quoted}'::regnamespace::oid) THEN + CREATE TYPE #{quoted}.oban_job_state AS ENUM ( + 'available', + 'scheduled', + 'executing', + 'retryable', + 'completed', + 'discarded', + 'cancelled' + ); + END IF; + END$$; + """ + + create table(:oban_jobs, + primary_key: false, + prefix: prefix, + options: "PARTITION BY LIST (state)" + ) do + add :id, :bigserial + add :state, :"#{quoted}.oban_job_state", null: false, default: "available" + add :queue, :text, null: false, default: "default" + add :worker, :text, null: false + add :attempt, :smallint, null: false, default: 0 + add :max_attempts, :smallint, null: false, default: 20 + add :priority, :smallint, null: false, default: 0 + + add :args, :map, null: false + add :meta, :map, null: false + + add :attempted_by, {:array, :text} + add :errors, {:array, :map}, null: false, default: [] + add :tags, {:array, :text}, null: false, default: [] + + add :inserted_at, :utc_datetime_usec, + null: false, + default: fragment("timezone('UTC', now())") + + add :scheduled_at, :utc_datetime_usec, + null: false, + default: fragment("timezone('UTC', now())") + + add :attempted_at, :utc_datetime_usec + add :cancelled_at, :utc_datetime_usec + add :completed_at, :utc_datetime_usec + add :discarded_at, :utc_datetime_usec + end + + create_if_not_exists index(:oban_jobs, [:id], prefix: prefix) + create_if_not_exists index(:oban_jobs, [:args], using: :gin, prefix: prefix) + create_if_not_exists index(:oban_jobs, [:meta], using: :gin, prefix: prefix) + + execute "COMMENT ON TABLE #{quoted}.oban_jobs IS '∞'", "" + + create_if_not_exists table(:oban_peers, primary_key: false, prefix: prefix) do + add :name, :text, null: false, primary_key: true + add :node, :text, null: false + add :started_at, :utc_datetime_usec, null: false + add :expires_at, :utc_datetime_usec, null: false + end + + execute "ALTER TABLE #{quoted}.oban_peers SET UNLOGGED", "" + + # Collocate available and executing to prevent concurrency anomalies such as: + # > ERROR: tuple to be locked was already moved to another partition due to concurrent update + execute """ + CREATE TABLE #{quoted}.oban_jobs_incomplete PARTITION OF #{quoted}.oban_jobs + FOR VALUES IN ('available', 'executing', 'scheduled', 'retryable') + """ + + create_if_not_exists index( + :oban_jobs_incomplete, + [:state, :queue, :priority, :scheduled_at, :id], + prefix: prefix + ) + + date_partition? = fn -> + if Code.ensure_loaded?(Mix), do: Mix.env() in ~w(dev prod)a, else: true + end + + {part_states, date_states} = + if Keyword.get_lazy(opts, :date_partition?, date_partition?) do + {[], ~w(completed cancelled discarded)} + else + {~w(completed cancelled discarded), []} + end + + for state <- part_states do + execute """ + CREATE TABLE #{quoted}.oban_jobs_#{state} PARTITION OF #{quoted}.oban_jobs + FOR VALUES IN ('#{state}') + """ + + create_if_not_exists index("oban_jobs_#{state}", [:queue, :scheduled_at, :id], + prefix: prefix + ) + end + + for state <- date_states do + execute """ + CREATE TABLE #{quoted}.oban_jobs_#{state} PARTITION OF #{quoted}.oban_jobs + FOR VALUES IN ('#{state}') + PARTITION BY RANGE (#{state}_at) + """ + + date = Date.utc_today() + next = Date.add(date, 1) + safe = Calendar.strftime(date, "%Y%m%d") + + execute """ + CREATE TABLE IF NOT EXISTS #{quoted}.oban_jobs_#{state}_#{safe} + PARTITION OF #{quoted}.oban_jobs_#{state} + FOR VALUES FROM ('#{date} 00:00:00') TO ('#{next} 00:00:00') + """ + end + end + + def down(opts \\ []) do + prefix = Keyword.get(opts, :prefix, "public") + quoted = inspect(prefix) + + drop_if_exists table(:oban_jobs, prefix: prefix) + + execute """ + ALTER TABLE IF EXISTS #{quoted}.oban_jobs_old RENAME TO oban_jobs + """ + + execute """ + ALTER INDEX IF EXISTS #{quoted}.oban_jobs_pkey_old RENAME TO oban_jobs_pkey + """ + + execute """ + ALTER INDEX IF EXISTS #{quoted}.oban_jobs_args_index_old RENAME TO oban_jobs_args_index + """ + + execute """ + ALTER INDEX IF EXISTS #{quoted}.oban_jobs_meta_index_old RENAME TO oban_jobs_meta_index + """ + end + + def backfill(opts \\ []) do + if direction() == :up do + # Ensure the oban_jobs_old table exists if this is ran in a single migration + flush() + + DynamicPartitioner.backfill_jobs(conf(), opts) + end + end + + defp conf, do: Config.new(repo: repo()) +end diff --git a/vendor/oban_pro/lib/oban/pro/migrations/v1_0_0.ex b/vendor/oban_pro/lib/oban/pro/migrations/v1_0_0.ex new file mode 100644 index 00000000..46537ca4 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/migrations/v1_0_0.ex @@ -0,0 +1,97 @@ +defmodule Oban.Pro.Migrations.DynamicCron do + @moduledoc false + + use Ecto.Migration + + def change(_opts \\ []), do: :ok + + def up(_opts \\ []), do: :ok + + def down(_opts \\ []), do: :ok +end + +defmodule Oban.Pro.Migrations.DynamicQueues do + @moduledoc false + + use Ecto.Migration + + def change(_opts \\ []), do: :ok + + def up(_opts \\ []), do: :ok + + def down(_opts \\ []), do: :ok +end + +defmodule Oban.Pro.Migrations.Producers do + @moduledoc false + + use Ecto.Migration + + def change(_opts \\ []), do: :ok + + def up(_opts \\ []), do: :ok + + def down(_opts \\ []), do: :ok +end + +defmodule Oban.Pro.Migrations.V100 do + @moduledoc false + + use Ecto.Migration + + defmacrop utc_now do + quote do + fragment("timezone('utc', now())") + end + end + + def up_ddl(opts) do + create_if_not_exists table(:oban_producers, primary_key: false, prefix: opts.prefix) do + add :uuid, :uuid, primary_key: true, null: false + add :name, :text, null: false + add :node, :text, null: false + add :queue, :text, null: false + add :meta, :map, null: false, default: %{} + + add :started_at, :utc_datetime_usec, null: false, default: utc_now() + add :updated_at, :utc_datetime_usec, null: false, default: utc_now() + end + + if opts.unlogged do + execute "ALTER TABLE #{opts.quoted}.oban_producers SET UNLOGGED" + end + + create_if_not_exists table(:oban_crons, primary_key: false, prefix: opts.prefix) do + add :name, :text, primary_key: true, null: false + add :expression, :text, null: false + add :worker, :text, null: false + add :opts, :map, null: false + add :insertions, {:array, :utc_datetime_usec}, default: [], null: false + add :paused, :boolean, null: false, default: false + add :lock_version, :integer, default: 1 + + add :inserted_at, :utc_datetime_usec, null: false, default: utc_now() + add :updated_at, :utc_datetime_usec, null: false, default: utc_now() + end + + create_if_not_exists table(:oban_queues, primary_key: false, prefix: opts.prefix) do + add :name, :text, primary_key: true, null: false + add :opts, :map, null: false, default: %{} + add :only, :map, null: false, default: %{} + add :lock_version, :integer, default: 1 + + add :inserted_at, :utc_datetime_usec, null: false, default: utc_now() + add :updated_at, :utc_datetime_usec, null: false, default: utc_now() + end + end + + def up_idx(_opts), do: :ok + + def down_ddl(opts) do + drop_if_exists table(:oban_queues, prefix: opts.prefix) + drop_if_exists table(:oban_crons, prefix: opts.prefix) + drop_if_exists table(:oban_producers, prefix: opts.prefix) + end + + def down_idx(_opts), do: :ok +end diff --git a/vendor/oban_pro/lib/oban/pro/migrations/v1_4_0.ex b/vendor/oban_pro/lib/oban/pro/migrations/v1_4_0.ex new file mode 100644 index 00000000..b0a30ac7 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/migrations/v1_4_0.ex @@ -0,0 +1,60 @@ +defmodule Oban.Pro.Migrations.Batch do + @moduledoc false + + use Ecto.Migration + + def change(_opts \\ []), do: :ok + + def up(_opts \\ []), do: :ok + + def down(_opts \\ []), do: :ok +end + +defmodule Oban.Pro.Migrations.Workflow do + @moduledoc false + + use Ecto.Migration + + def change(_opts \\ []), do: :ok + + def up(_opts \\ []), do: :ok + + def down(_opts \\ []), do: :ok +end + +defmodule Oban.Pro.Migrations.V140 do + @moduledoc false + + use Ecto.Migration + + def up_ddl(opts) do + alter table(:oban_crons, prefix: opts.prefix) do + add_if_not_exists :insertions, {:array, :utc_datetime_usec}, default: [] + end + end + + def up_idx(opts) do + create_if_not_exists index( + :oban_jobs, + [:state, "(meta->>'workflow_id')", "(meta->>'name')"], + concurrently: opts.concurrently, + name: :oban_jobs_state_meta_workflow_index, + prefix: opts.prefix, + where: "meta ? 'workflow_id'" + ) + end + + def down_ddl(opts) do + alter table(:oban_crons, prefix: opts.prefix) do + remove_if_exists :insertions, {:array, :utc_datetime_usec} + end + end + + def down_idx(opts) do + drop_if_exists index(:oban_jobs, [], + name: :oban_jobs_state_meta_workflow_index, + concurrently: opts.concurrently, + prefix: opts.prefix + ) + end +end diff --git a/vendor/oban_pro/lib/oban/pro/migrations/v1_5_0.ex b/vendor/oban_pro/lib/oban/pro/migrations/v1_5_0.ex new file mode 100644 index 00000000..af7ac3e8 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/migrations/v1_5_0.ex @@ -0,0 +1,114 @@ +defmodule Oban.Pro.Migrations.V150 do + @moduledoc false + + use Ecto.Migration + + def up_ddl(opts) do + if not partitioned?(opts) do + execute """ + CREATE OR REPLACE FUNCTION #{opts.quoted}.oban_state_to_bit(state #{opts.quoted}.oban_job_state) + RETURNS jsonb AS $$ + SELECT CASE + WHEN state = 'scheduled' THEN '0'::jsonb + WHEN state = 'available' THEN '1'::jsonb + WHEN state = 'executing' THEN '2'::jsonb + WHEN state = 'retryable' THEN '3'::jsonb + WHEN state = 'completed' THEN '4'::jsonb + WHEN state = 'cancelled' THEN '5'::jsonb + WHEN state = 'discarded' THEN '6'::jsonb + END; + $$ LANGUAGE SQL IMMUTABLE STRICT + """ + + alter table(:oban_jobs, prefix: opts.prefix) do + add_if_not_exists :uniq_key, :text, + generated: """ + ALWAYS AS (CASE WHEN meta->'uniq_bmp' @> #{opts.quoted}.oban_state_to_bit(state) THEN meta->>'uniq_key' END) STORED + """ + end + end + end + + def up_idx(opts) do + if partitioned?(opts) do + create_if_not_exists index( + :oban_jobs, + ["(meta->>'uniq_key')"], + concurrently: opts.concurrently, + name: :oban_jobs_unique_index, + prefix: opts.prefix, + where: "meta ? 'uniq_key'" + ) + else + create_if_not_exists index( + :oban_jobs, + [:uniq_key], + concurrently: opts.concurrently, + name: :oban_jobs_unique_index, + prefix: opts.prefix, + unique: true, + where: "uniq_key IS NOT NULL" + ) + end + + create_if_not_exists index( + :oban_jobs, + [:state, "(meta->>'batch_id')", "(meta->>'callback')"], + concurrently: opts.concurrently, + name: :oban_jobs_batch_index, + prefix: opts.prefix, + where: "meta ? 'batch_id'" + ) + + create_if_not_exists index( + :oban_jobs, + [:state, "(meta->>'chain_id')", "(meta->>'on_hold')"], + concurrently: opts.concurrently, + name: :oban_jobs_chain_index, + prefix: opts.prefix, + where: "meta ? 'chain_id'" + ) + end + + def down_ddl(opts) do + alter table(:oban_jobs, prefix: opts.prefix) do + remove_if_exists :uniq_key, :text + end + + execute "DROP FUNCTION IF EXISTS #{opts.quoted}.oban_state_to_bit" + end + + def down_idx(opts) do + drop_if_exists index(:oban_jobs, [], + name: :oban_jobs_batch_index, + concurrently: opts.concurrently, + prefix: opts.prefix + ) + + drop_if_exists index(:oban_jobs, [], + name: :oban_jobs_chain_index, + concurrently: opts.concurrently, + prefix: opts.prefix + ) + + drop_if_exists index(:oban_jobs, [], + name: :oban_jobs_unique_index, + concurrently: opts.concurrently, + prefix: opts.prefix + ) + end + + defp partitioned?(opts) do + query = """ + SELECT EXISTS ( + SELECT 1 + FROM pg_tables + WHERE schemaname = '#{opts.prefix}' AND tablename = 'oban_jobs_completed' + ) + """ + + {:ok, %{rows: [[partitioned?]]}} = repo().query(query, [], log: false) + + partitioned? + end +end diff --git a/vendor/oban_pro/lib/oban/pro/migrations/v1_6_0.ex b/vendor/oban_pro/lib/oban/pro/migrations/v1_6_0.ex new file mode 100644 index 00000000..add75afa --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/migrations/v1_6_0.ex @@ -0,0 +1,109 @@ +defmodule Oban.Pro.Migrations.V160 do + @moduledoc false + + use Ecto.Migration + + def up_ddl(opts) do + # Partitioning + + alter table(:oban_jobs, prefix: opts.prefix) do + add_if_not_exists :partition_key, :text, + generated: """ + ALWAYS AS (CASE WHEN meta ? 'partition_key' THEN meta->>'partition_key' END) STORED + """ + end + + # Dynamic Queues + + alter table(:oban_queues, prefix: opts.prefix) do + add_if_not_exists :hash, :text + end + end + + def up_idx(opts) do + # Partitioning + + create_if_not_exists index( + :oban_jobs, + [:partition_key, :queue, :priority, :scheduled_at, :id], + concurrently: opts.concurrently, + name: :oban_jobs_partition_index, + prefix: opts.prefix, + where: "state = 'available' AND partition_key IS NOT NULL" + ) + + # Workflows + + drop_if_exists index(:oban_jobs, [], + concurrently: opts.concurrently, + name: :oban_jobs_state_meta_workflow_index, + prefix: opts.prefix + ) + + create_if_not_exists index( + :oban_jobs, + [ + "(meta->>'workflow_id')", + :state, + "(meta->>'on_hold')", + "(meta->>'name')" + ], + concurrently: opts.concurrently, + name: :oban_jobs_workflow_index, + prefix: opts.prefix, + where: "meta ? 'workflow_id'" + ) + + create_if_not_exists index( + :oban_jobs, + [ + "(meta->>'sup_workflow_id')", + :state, + "(meta->>'on_hold')" + ], + concurrently: opts.concurrently, + name: :oban_jobs_sup_workflow_index, + prefix: opts.prefix, + where: "meta ? 'sup_workflow_id'" + ) + end + + def down_ddl(opts) do + alter table(:oban_queues, prefix: opts.prefix) do + remove_if_exists :hash, :text + end + + alter table(:oban_jobs, prefix: opts.prefix) do + remove_if_exists :partition_key, :text + end + end + + def down_idx(opts) do + drop_if_exists index(:oban_jobs, [], + name: :oban_jobs_partition_index, + concurrently: opts.concurrently, + prefix: opts.prefix + ) + + drop_if_exists index(:oban_jobs, [], + name: :oban_jobs_workflow_index, + concurrently: opts.concurrently, + prefix: opts.prefix + ) + + drop_if_exists index(:oban_jobs, [], + name: :oban_jobs_sup_workflow_index, + concurrently: opts.concurrently, + prefix: opts.prefix + ) + + create_if_not_exists index( + :oban_jobs, + [:state, "(meta->>'workflow_id')", "(meta->>'name')"], + concurrently: opts.concurrently, + name: :oban_jobs_state_meta_workflow_index, + prefix: opts.prefix, + where: "meta ? 'workflow_id'" + ) + end +end diff --git a/vendor/oban_pro/lib/oban/pro/partition.ex b/vendor/oban_pro/lib/oban/pro/partition.ex new file mode 100644 index 00000000..402df2b6 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/partition.ex @@ -0,0 +1,195 @@ +defmodule Oban.Pro.Partition do + @moduledoc false + + @behaviour Oban.Pro.Handler + + import Ecto.Query + + alias Ecto.Changeset + alias Oban.Pro.{Producer, Queue, Utils} + alias Oban.{Config, Job, Repo} + + @keys_cache_ttl Application.compile_env(:oban_pro, [__MODULE__, :keys_cache_ttl], 3_000) + @keys_limit_mlt Application.compile_env(:oban_pro, [__MODULE__, :keys_limit_multiplier], 3) + + @conf_tab Module.concat(__MODULE__, Conf) + @keys_tab Module.concat(__MODULE__, Keys) + + @impl Oban.Pro.Handler + def on_start do + :ets.new(@conf_tab, [:set, :named_table, :public, read_concurrency: true]) + :ets.new(@keys_tab, [:set, :named_table, :public, read_concurrency: true]) + + # Clear the cache with a timeout, as not all queues are guaranteed to be running on a given + # node and the cache won't clear naturally. + :timer.apply_interval(@keys_cache_ttl, __MODULE__, :__clear_cache__, [@keys_cache_ttl]) + + :telemetry.attach( + "oban.pro.partition", + [:oban, :engine, :put_meta, :stop], + &__MODULE__.handle_event/4, + nil + ) + end + + @impl Oban.Pro.Handler + def on_stop do + :telemetry.detach("oban.pro.partition") + end + + def __clear_cache__(ttl) do + now = System.monotonic_time(:millisecond) + + :ets.select_delete(@keys_tab, [ + {{:_, :_, :"$3"}, [{:>=, {:-, now, :"$3"}, ttl}], [true]} + ]) + end + + def handle_event(_event, _measure, %{conf: conf, key: key, meta: meta}, nil) do + # Bust the partition cache on any change to a partitionable limit. This is rare enough an + # event that it's better to be safe. + if key in ~w(global_limit rate_limit)a do + :ets.delete(@conf_tab, {conf.name, meta.queue}) + end + end + + @spec get_key(Job.t() | Job.changeset(), any()) :: any() + def get_key(changeset, default \\ nil) + + def get_key(%{changes: %{meta: %{partition_key: key}}}, _default), do: key + def get_key(%{meta: %{"partition_key" => key}}, _default), do: key + def get_key(_changeset, default), do: default + + @spec with_partition_meta(Job.changeset(), Config.t()) :: Job.changeset() + def with_partition_meta(changeset, conf) do + queue = Changeset.get_field(changeset, :queue, "default") + + case config_for_queue(conf, queue) do + :missing -> + changeset + + partition -> + part_meta = %{partition: true, partition_key: gen_key(changeset, partition)} + + meta = + changeset + |> Changeset.get_change(:meta, %{}) + |> Map.merge(part_meta) + + Changeset.put_change(changeset, :meta, meta) + end + end + + @spec gen_key(Job.changeset(), %{fields: [atom()], keys: [atom()]}) :: String.t() + def gen_key(changeset, %{fields: fields, keys: keys}) do + fields + |> Enum.sort() + |> Enum.map(fn + :args -> Utils.take_keys(changeset, :args, keys) + :meta -> Utils.take_keys(changeset, :meta, keys) + :worker -> Changeset.get_field(changeset, :worker) + end) + |> Utils.hash64() + end + + @spec available_keys(Config.t(), String.t(), pos_integer()) :: [String.t()] + def available_keys(conf, queue, limit) do + timed_cache(conf.name, queue, fn -> + query_limit = @keys_limit_mlt * limit + + subquery = + Job + |> where([j], j.state == "available" and j.queue == ^queue) + |> where([j], not is_nil(fragment("partition_key"))) + |> group_by([j], fragment("partition_key")) + |> order_by([j], asc: min(j.priority), asc: min(j.scheduled_at)) + |> select([j], %{partition_key: fragment("partition_key")}) + |> limit(^query_limit) + + partitioned_keys = + subquery + |> subquery() + |> select([s], s.partition_key) + |> then(&Repo.all(conf, &1)) + + ["none" | partitioned_keys] + end) + end + + defp timed_cache(name, queue, fun) do + cache_key = {name, queue} + now = System.monotonic_time(:millisecond) + + case :ets.lookup(@keys_tab, cache_key) do + [{^cache_key, keys, timestamp}] when now - timestamp < @keys_cache_ttl -> + keys + + _ -> + keys = fun.() + :ets.insert(@keys_tab, {cache_key, keys, now}) + keys + end + end + + @doc """ + We can't guarantee that the queue is running locally, running on a current node, or running at + all. This will check the following locations: + + - From a locally running producer + - From another node's producer in the database + - From a DynamicQueues entry in the database + """ + @spec config_for_queue(Config.t(), String.t()) :: :missing | %{fields: list(), keys: list()} + def config_for_queue(conf, queue) do + # Partitioning isn't used for draining and there's no point in hitting the database just to + # find out there's no config available. + if Process.get(:oban_draining) do + :missing + else + case :ets.lookup(@conf_tab, {conf.name, queue}) do + [{_key, value}] -> + value + + [] -> + value = + with :missing <- reg_config(conf, queue), + :missing <- pro_config(conf, queue), + do: dyn_config(conf, queue) + + :ets.insert(@conf_tab, {{conf.name, queue}, value}) + + value + end + end + end + + defp reg_config(conf, queue) do + case Registry.meta(Oban.Registry, {conf.name, {:producer, queue}}) do + {:ok, %{meta: %{global_limit: %{partition: %{} = partition}}}} -> partition + {:ok, %{meta: %{rate_limit: %{partition: %{} = partition}}}} -> partition + _other -> :missing + end + end + + defp pro_config(conf, queue) do + query = + Producer + |> where(queue: ^queue) + |> order_by(desc: :started_at) + |> limit(1) + + case Repo.one(conf, query) do + %{meta: %{global_limit: %{partition: %{} = partition}}} -> partition + %{meta: %{rate_limit: %{partition: %{} = partition}}} -> partition + _other -> :missing + end + end + + defp dyn_config(conf, queue) do + case Repo.get_by(conf, Queue, name: queue) do + %{opts: %{global_limit: %{partition: %{} = partition}}} -> partition + %{opts: %{rate_limit: %{partition: %{} = partition}}} -> partition + _other -> :missing + end + end +end diff --git a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_cron.ex b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_cron.ex new file mode 100644 index 00000000..60c11005 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_cron.ex @@ -0,0 +1,821 @@ +defmodule Oban.Pro.Plugins.DynamicCron do + @moduledoc """ + Enhanced cron scheduling that's configurable at runtime across your entire cluster. + + `DynamicCron` can configure cron scheduling before boot or during runtime, globally, with + scheduling guarantees and per-entry timezone overrides. It's an ideal solution for applications + that can't miss a cron job or must dynamically start and manage scheduled jobs at runtime. + + ## Using and Configuring + + To begin using `DynamicCron`, add the module to your list of Oban plugins in `config.exs`: + + ```elixir + config :my_app, Oban, + plugins: [Oban.Pro.Plugins.DynamicCron] + ... + ``` + + By itself, without providing a crontab or dynamically inserting cron entries, the plugin doesn't + have anything to schedule. To get scheduling started, provide a list of `{cron, worker}` or + `{cron, worker, options}` tuples to the plugin. The syntax is identical to Oban's built in + `:crontab` option, which means you can _probably_ copy an existing standard `:crontab` list into + the plugin's `:crontab`. + + ```elixir + plugins: [{ + Oban.Pro.Plugins.DynamicCron, + crontab: [ + {"* * * * *", MyApp.MinuteJob}, + {"0 * * * *", MyApp.HourlyJob, queue: :scheduled}, + {"0 0 * * *", MyApp.DailyJob, max_attempts: 1}, + {"0 12 * * MON", MyApp.MondayWorker, tags: ["scheduled"]}, + {"@daily", MyApp.AnotherDailyWorker} + ] + }] + ``` + + > #### Beware of Conflicts {: .tip} + > + > While the syntax is syntactically identical to Oban's basic `:crontab`, you can't copy an + > existing `:crontab` if you have multiple entries using the same worker. `DynamicCron` + > requires unique names for all entries, which defaults to the worker's module name. Multiple + > entries with the same worker will cause naming conflicts unless you provide explicit `:name` + > overrides for each duplicate worker. + + Now, when dynamic cron initializes, it will persist those cron entries to the database and start + scheduling them according to their CRON expression. The plugin's `crontab` format is nearly + identical to Oban's standard crontab, with a few important enhancements we'll look at soon. + + Each of the crontab entries are persisted to the database and referenced globally, by all the + other connected Oban instances. That allows us to insert, update, or delete cron entries at any + time. In fact, changing the schedule or options of an entry in the crontab provided to the + plugin will automatically update the persisted entry. To demonstrate, let's modify the + `MinuteJob` we specified so that it runs every other minute in the `:scheduled` queue: + + ```elixir + crontab: [ + {"*/2 * * * *", MyApp.MinuteJob, queue: :scheduled}, + ... + ] + ``` + + Now it isn't really a "minute job" any more, and the name is no longer suitable. However, we + didn't provide a name for the entry and it's using the module name instead. To provide more + flexibility we can add a `:name` overrride, then we can update the worker's name as well: + + ```elixir + crontab: [ + {"*/2 * * * *", MyApp.FrequentJob, name: "frequent", queue: :scheduled}, + ... + ] + ``` + + All entries are referenced by name, which defaults to the worker's name and must be unique. You + may define the same worker multiple times _as long as_ you provide a name override: + + ```elixir + crontab: [ + {"*/3 * * * *", MyApp.BasicJob, name: "client-1", args: %{client_id: 1}}, + {"*/3 * * * *", MyApp.BasicJob, name: "client-2", args: %{client_id: 2}}, + ... + ] + ``` + + To temporarily disable scheduling jobs you can set the `paused` flag: + + + ```elixir + crontab: [ + {"* * * * *", MyApp.BasicJob, paused: true}, + ... + ] + ``` + + To resume the job you must supply `paused: false` (or use `update/2` to resume it manually), + simply removing the `paused` option will have no effect. + + ```elixir + crontab: [ + {"* * * * *", MyApp.BasicJob, paused: false}, + ... + ] + ``` + + It is also possible to delete a persisted entry during initialization by passing the `:delete` + option: + + ```elixir + crontab: [ + {"* * * * *", MyApp.MinuteJob, delete: true}, + ... + ] + ``` + + One or more entries can be deleted this way. Deleting entries is idempotent, nothing will happen + if no matching entry can be found. + + ## Automatic Synchronization + + Synchronizing persisted entries manually requires two deploys: one to flag it with `deleted: + true` and another to clean up the entry entirely. That extra step isn't ideal for applications + that don't insert or delete jobs at runtime. + + To delete entries that are no longer listed in the crontab automatically set the `sync_mode` + option to `:automatic`: + + + ```elixir + [ + sync_mode: :automatic, + crontab: [ + {"0 * * * *", MyApp.BasicJob}, + {"0 0 * * *", MyApp.OtherJob} + ] + ] + ``` + + To remove unwanted entries, simply delete them from the crontab: + + ```diff + crontab: [ + {"0 * * * *", MyApp.BasicJob}, + - {"0 0 * * *", MyApp.OtherJob} + ] + ``` + + With `:automatic` sync, the entry for `MyApp.OtherJob` will be deleted on the next deployment. + + ## Scheduling Guarantees + + Depending on an application's restart timing or as the result of unexpected downtime, a job's + scheduled insert period may be missed. To compensate, enable `guaranteed` mode for the entire + crontab or on an individual bases. + + Here's an example of enabling guaranteed insertion for the entire crontab: + + ```elixir + [ + guaranteed: true, + crontab: [ + {"0 * * * *", MyApp.HourlyJob}, + {"0 0 * * *", MyApp.DailyJob}, + {"0 0 1 * *", MyApp.MonthlyJob}, + {"0 0 1 */2 *", MyApp.BiMonthlyJob} + ] + ] + ``` + + Guaranteed mode may be enabled or disabled for individual jobs instead if it's a poor fit for + the entire crontab: + + ```elixir + [ + crontab: [ + {"@daily", MyApp.DailyJob, guaranteed: true} + {"@monthly", MyApp.MonthlyJob, guaranteed: false} + ] + ] + ``` + + Guaranteed insertion is enforced by cron name and it's durable across option changes. Changing + an entry's `expression` or `timezone` will clear any prior insertions and reset the guarantee. + + ## Overriding the Timezone + + Without any configuration the default timezone is `Etc/UTC`. You can override that for all cron + entries by passing a `timezone` option to the plugin: + + ```elixir + plugins: [{ + Oban.Pro.Plugins.DynamicCron, + timezone: "America/Chicago", + # ... + ``` + + You can also override the timezone for individual entries by passing it as an option to the + `crontab` list or to `DynamicCron.insert/1`: + + ```elixir + DynamicCron.insert([ + {"0 0 * * *", MyApp.Pinger, name: "oslo", timezone: "Europe/Oslo"}, + {"0 0 * * *", MyApp.Pinger, name: "chicago", timezone: "America/Chicago"}, + {"0 0 * * *", MyApp.Pinger, name: "zagreb", timezone: "Europe/Zagreb"} + ]) + ``` + + ## Runtime Updates + + Dynamic cron entries are persisted to the database, making it easy to manipulate them through + typical CRUD operations. The `DynamicCron` plugin provides convenience functions to simplify + working those operations. + + The `insert/1` function takes a list of one or more tuples with the same `{expression, worker}` + or `{expression, worker, options}` format as the plugin's `crontab` option: + + ```elixir + DynamicCron.insert([ + {"0 0 * * *", MyApp.GenericWorker}, + {"* * * * *", MyApp.ClientWorker, name: "client-1", args: %{client_id: 1}}, + {"* * * * *", MyApp.ClientWorker, name: "client-2", args: %{client_id: 2}}, + {"* * * * *", MyApp.ClientWorker, name: "client-3", args: %{client_id: 3}} + ]) + ``` + + Be aware that `insert/1` acts like an "upsert", making it possible to modify existing entries if + the worker or name matches. + + ## Isolation and Namespacing + + All `DynamicCron` functions have an alternate clause that accepts an Oban instance name as the + first argument. This is in line with base `Oban` functions such as `Oban.insert/2`, which allow + you to seamlessly work with multiple Oban instances and across multiple database prefixes. For + example, you can use `all/1` to list all cron entries for the instance named `ObanPrivate`: + + ```elixir + entries = DynamicCron.all(ObanPrivate) + ``` + + Likewise, to insert a new entry using the configuration associated with the `ObanPrivate` + instance: + + ```elixir + {:ok, _} = DynamicCron.insert(ObanPrivate, [{"* * * * *", PrivateWorker}]) + ``` + + ## Instrumenting with Telemetry + + The `DynamicCron` plugin adds the following metadata to the `[:oban, :plugin, :stop]` event: + + * `:jobs` - a list of jobs that were inserted into the database + """ + + @behaviour Oban.Plugin + + use GenServer + + import Ecto.Query, only: [from: 2, order_by: 2, select: 3, where: 2, where: 3] + + alias Oban.Cron.Expression + alias Oban.Plugins.Cron, as: CronPlugin + alias Oban.Pro.Cron, as: CronEntry + alias Oban.{Job, Peer, Repo, Validation, Worker} + + require Logger + + @type cron_expr :: String.t() + @type cron_name :: String.t() | atom() + @type cron_opt :: [ + args: Job.args(), + expression: cron_expr(), + guaranteed: boolean(), + max_attempts: pos_integer(), + paused: boolean(), + priority: 0..9, + meta: map(), + name: cron_name(), + queue: atom() | String.t(), + tags: Job.tags(), + timezone: String.t() + ] + @type cron_input :: {cron_expr(), module()} | {cron_expr(), module(), cron_opt()} + + @type sync_mode :: :manual | :automatic + + @type option :: [ + conf: Oban.Config.t(), + guaranteed: boolean(), + crontab: [cron_input()], + name: Oban.name(), + sync_mode: sync_mode(), + timezone: Calendar.time_zone(), + timeout: timeout() + ] + + defstruct [ + :conf, + :timer, + crontab: [], + guaranteed: false, + insert_history: 360, + rebooted: false, + sync_mode: :manual, + timezone: "Etc/UTC" + ] + + defguardp is_name(name) when is_binary(name) or (is_atom(name) and not is_nil(name)) + + # Trim the array using an array_agg for CRDB and PG < 14 compatibility. When we require PG 14+ + # we can use `trim_array`, or if CRDB allows slice operators we can use `[1:?]` instead. + defmacrop push_trim(insertions, datetime, length) do + quote do + fragment( + """ + array_prepend( + ?, + (select array_agg(val) from (select ?[idx] from generate_series(1, ? - 1) as idx) AS g(val) where val is not null) + ) + """, + unquote(datetime), + unquote(insertions), + unquote(length) + ) + end + end + + # Extract the first and second without using a slice for CRDB compatibility. + defmacrop slice_insertions(insertions) do + quote do + type( + fragment("ARRAY[?[1],?[2]]", unquote(insertions), unquote(insertions)), + {:array, :utc_datetime_usec} + ) + end + end + + defmacrop maybe_clear_insertions(cron, expression, timezone) do + quote do + fragment( + "CASE WHEN ? != ? OR COALESCE(?->>'timezone', 'NA') != COALESCE(?, 'NA') THEN '{}' ELSE ? END", + field(unquote(cron), :expression), + unquote(expression), + field(unquote(cron), :opts), + unquote(timezone), + field(unquote(cron), :insertions) + ) + end + end + + @doc false + def child_spec(args), do: super(args) + + @impl Oban.Plugin + @spec start_link([option()]) :: GenServer.on_start() + def start_link(opts) do + {name, opts} = + opts + |> Keyword.update(:crontab, [], &normalize_crontab/1) + |> Keyword.delete(:timeout) + |> Keyword.pop(:name) + + GenServer.start_link(__MODULE__, struct!(__MODULE__, opts), name: name) + end + + @impl Oban.Plugin + def validate(opts) do + Validation.validate_schema(opts, + conf: :any, + crontab: {:custom, &validate_crontab(&1, opts)}, + guaranteed: :boolean, + insert_history: :pos_integer, + name: :any, + sync_mode: {:enum, [:manual, :automatic]}, + timezone: :timezone, + timeout: :timeout + ) + end + + @impl Oban.Plugin + def format_logger_output(_conf, %{jobs: jobs}), do: %{jobs: Enum.map(jobs, & &1.id)} + + @doc """ + Used to retrieve all persisted cron entries. + + The `all/0` function is provided as a convenience to inspect persisted entries. + + ## Examples + + Return a list of cron schemas with raw attributes: + + entries = DynamicCron.all() + """ + @spec all(term()) :: [Ecto.Schema.t()] + def all(oban_name \\ Oban) do + oban_name + |> Oban.config() + |> all_entries() + end + + @doc """ + Insert cron entries into the database to start scheduling new jobs. + + Be aware that `insert/1` acts like an "upsert", making it possible to modify existing entries if + the worker or name matches. Still, it is better to use `update/2` to make targeted updates. + + ## Examples + + Insert a list of tuples with the same `{expression, worker}` or `{expression, worker, options}` + format as the plugin's `crontab` option. + + DynamicCron.insert([ + {"0 0 * * *", MyApp.GenericWorker}, + {"* * * * *", MyApp.ClientWorker, name: "client-1", args: %{client_id: 1}}, + {"* * * * *", MyApp.ClientWorker, name: "client-2", args: %{client_id: 2}}, + {"* * * * *", MyApp.ClientWorker, name: "client-3", args: %{client_id: 3}} + ]) + """ + @spec insert(term(), [cron_input()]) :: {:ok, [Ecto.Schema.t()]} | {:error, Ecto.Changeset.t()} + def insert(oban_name \\ Oban, [_ | _] = crontab) do + conf = Oban.config(oban_name) + + Repo.transaction(conf, fn -> insert_entries(conf, crontab) end) + rescue + error in [Ecto.InvalidChangesetError] -> + {:error, error.changeset} + end + + @doc """ + Update a single cron entry, as identified by worker or name. + + Any option available when specifying an entry in the `crontab` list or when calling `insert/2` + can be updated—that includes the cron `expression` and the `worker`. + + ## Examples + + The following call demonstrates updating every possible option: + + {:ok, _} = + DynamicCron.update( + "cron-1", + expression: "1 * * * *", + max_attempts: 10, + name: "special-cron", + paused: false, + priority: 0, + queue: "dedicated", + tags: ["client", "scheduled"], + timezone: "Europe/Amsterdam", + worker: Other.Worker, + ) + + Naturally, individual options may be updated instead. For example, set `paused: true` to pause + an entry: + + {:ok, _} = DynamicCron.update(MyApp.ClientWorker, paused: true) + + Since `update/2` operates on a single entry at a time, it is possible to rename an entry without + doing a `delete`/`insert` dance: + + {:ok, _} = DynamicCron.update(MyApp.ClientWorker, name: "client-worker") + + Or, update an entry with a custom entry name already set: + + {:ok, _} = DynamicCron.update("cron-1", name: "special-cron") + """ + @spec update(term(), cron_name(), cron_opt()) :: + {:ok, Ecto.Schema.t()} | {:error, Ecto.Changeset.t() | String.t()} + def update(oban_name \\ Oban, name, opts) when is_name(name) do + oban_name + |> Oban.config() + |> update_entry(Worker.to_string(name), opts) + end + + @doc """ + Delete individual entries, by worker or name. + + Use `delete/1` to remove entries at runtime, rather than hard-coding the `:delete` flag into the + `crontab` list at compile time. + + ## Examples + + With the worker as the entry name: + + {:ok, _} = DynamicCron.delete(Worker) + + With a custom name: + + {:ok, _} = DynamicCron.delete("cron-1") + """ + @spec delete(term(), cron_name()) :: + {:ok, Ecto.Schema.t()} + | {:error, Ecto.Changeset.t() | String.t()} + def delete(oban_name \\ Oban, name) when is_name(name) do + oban_name + |> Oban.config() + |> delete_cron(Worker.to_string(name)) + end + + # Callbacks + + @impl GenServer + def init(state) do + :telemetry.execute([:oban, :plugin, :init], %{}, %{conf: state.conf, plugin: __MODULE__}) + + {:ok, schedule_evaluate(state)} + end + + @impl GenServer + def handle_call(:evaluate, _from, state) do + manage_entries(state) + reload_and_insert(state) + + {:reply, :ok, %{state | rebooted: true}} + end + + @impl GenServer + def handle_info(:evaluate, state) do + if Peer.leader?(state.conf) do + manage_entries(state) + reload_and_insert(state) + + {:noreply, schedule_evaluate(%{state | rebooted: true})} + else + {:noreply, schedule_evaluate(state)} + end + end + + def handle_info(message, state) do + Logger.warning( + message: "Received unexpected message: #{inspect(message)}", + source: :oban_pro, + module: __MODULE__ + ) + + {:noreply, state} + end + + # Validations + + defp normalize_crontab(crontab) do + Enum.map(crontab, fn + {expr, worker} -> {expr, worker, []} + tuple -> tuple + end) + end + + defp validate_crontab(crontab, opts) when is_list(crontab) and is_list(opts) do + automatic? = opts[:sync_mode] == :automatic + + Validation.validate(:crontab, crontab, &validate_crontab(&1, automatic?)) + end + + defp validate_crontab({expression, worker, opts}, automatic?) do + with {:ok, _} <- CronPlugin.parse(expression) do + cond do + not Code.ensure_loaded?(worker) -> + {:error, "#{inspect(worker)} not found or can't be loaded"} + + not function_exported?(worker, :perform, 1) -> + {:error, "#{inspect(worker)} does not implement `perform/1` callback"} + + not Keyword.keyword?(opts) -> + {:error, "options must be a keyword list, got: #{inspect(opts)}"} + + not valid_entry?(opts, automatic?) -> + {:error, + "expected cron options to be one of #{inspect(CronEntry.allowed_opts())}, got: #{inspect(opts)}"} + + not valid_job?(worker, opts) -> + {:error, "expected valid job options, got: #{inspect(opts)}"} + + true -> + :ok + end + end + end + + defp validate_crontab({expression, worker}, automatic?) do + validate_crontab({expression, worker, []}, automatic?) + end + + defp validate_crontab(invalid, _automatic?) do + {:error, + "expected crontab entry to be an {expression, worker} or " <> + "{expression, worker, options} tuple, got: #{inspect(invalid)}"} + end + + defp valid_entry?(opts, automatic?) do + string_keys = Enum.map(CronEntry.allowed_opts(), &to_string/1) + + ignore_keys = + if automatic? do + ~w(guaranteed insertions name paused)a + else + ~w(delete guaranteed insertions name paused)a + end + + opts + |> Keyword.drop(ignore_keys) + |> Enum.all?(fn {key, _} -> to_string(key) in string_keys end) + end + + defp valid_job?(worker, opts) do + args = Keyword.get(opts, :args, %{}) + opts = Keyword.drop(opts, ~w(args delete guaranteed insertions name paused timezone)a) + + worker.new(args, opts).valid? + end + + # Scheduling + + defp schedule_evaluate(state) do + timer = Process.send_after(self(), :evaluate, CronPlugin.interval_to_next_minute()) + + %{state | timer: timer} + end + + # Updating + + defp reload_and_insert(state) do + meta = %{conf: state.conf, plugin: __MODULE__} + + :telemetry.span([:oban, :plugin], meta, fn -> + {:ok, jobs} = + state + |> reload_entries() + |> insert_scheduled_jobs(state) + + {:ok, Map.put(meta, :jobs, jobs)} + end) + end + + defp reload_entries(state) do + query = + CronEntry + |> where(paused: false) + |> order_by(asc: :inserted_at) + |> select([c], %{c | insertions: slice_insertions(c.insertions)}) + + state.conf + |> Repo.all(query) + |> Enum.reduce([], fn entry, acc -> + with {:ok, parsed} <- Expression.parse(entry.expression), + {:ok, worker} <- Worker.from_string(entry.worker) do + opts = Keyword.new(entry.opts, fn {key, val} -> {String.to_existing_atom(key), val} end) + insertions = Enum.reject(entry.insertions, &is_nil/1) + entry = %{entry | insertions: insertions, opts: opts, parsed: parsed, worker: worker} + + [entry | acc] + else + _ -> acc + end + end) + end + + # Inserting + + defp insert_scheduled_jobs(entries, state) do + entries = Enum.filter(entries, &(now?(&1, state) or missed?(&1, state))) + + jobs = + Enum.map(entries, fn entry -> + {args, opts} = Keyword.pop(entry.opts, :args, %{}) + {tz, opts} = Keyword.pop(opts, :timezone, state.timezone) + + meta = %{cron: true, cron_expr: entry.expression, cron_name: entry.name, cron_tz: tz} + + opts = + entry.worker.__opts__() + |> Worker.merge_opts(opts) + |> Keyword.drop([:guaranteed, :timezone]) + |> Keyword.update(:meta, meta, &Map.merge(&1, meta)) + + entry.worker.new(args, opts) + end) + + Repo.transaction(state.conf, fn -> + now = DateTime.utc_now() + + # Using `from` to avoid a conflict with the local update/3 + query = + from c in CronEntry, + where: c.name in ^Enum.map(entries, & &1.name), + update: [set: [insertions: push_trim(c.insertions, ^now, ^state.insert_history)]] + + Repo.update_all(state.conf, query, []) + Oban.insert_all(state.conf.name, jobs) + end) + end + + defp now?(%{parsed: %{reboot?: true}}, %{rebooted: true}), do: false + + defp now?(entry, state) do + datetime = + entry.opts + |> Keyword.get(:timezone, state.timezone) + |> DateTime.now!() + + Expression.now?(entry.parsed, datetime) + end + + defp missed?(entry, state) do + guaranteed? = Keyword.get(entry.opts, :guaranteed, state.guaranteed) + + cond do + [] == entry.insertions -> + false + + not guaranteed? -> + false + + entry.parsed.reboot? -> + false + + true -> + timezone = Keyword.get(entry.opts, :timezone, state.timezone) + + last_inserted_at = List.first(entry.insertions) + + expected_last_at = + entry.parsed + |> Expression.last_at(timezone) + |> DateTime.shift_zone!("Etc/UTC") + + # Compare in UTC and ensure it has been at least 2 minutes, which safely handles DST where + # the same clock time can occur within 1 hour. + DateTime.compare(expected_last_at, last_inserted_at) == :gt and + DateTime.diff(expected_last_at, last_inserted_at, :minute) >= 2 + end + end + + # Entry Management + + defp manage_entries(%{rebooted: true}), do: :ok + + defp manage_entries(%{sync_mode: :automatic} = state) do + Repo.transaction(state.conf, fn -> + delete_missing(state.conf, state.crontab) + insert_entries(state.conf, state.crontab) + end) + end + + defp manage_entries(state) do + Repo.transaction(state.conf, fn -> + {deletes, inserts} = + Enum.split_with(state.crontab, fn {_expr, _work, opts} -> + Keyword.get(opts, :delete) + end) + + delete_entries(state.conf, deletes) + insert_entries(state.conf, inserts) + end) + end + + defp insert_entries(conf, crontab) do + Enum.map(crontab, fn tuple -> + changeset = CronEntry.changeset(tuple) + + replace = + for {key, val} <- changeset.changes, + key in ~w(expression worker opts paused)a, + val != %{}, + do: {key, val} + + expression = changeset.changes.expression + timezone = changeset.changes.opts[:timezone] + + repo_opts = [ + conflict_target: :name, + on_conflict: + CronEntry + |> Ecto.Query.update([_], set: ^replace) + |> Ecto.Query.update([c], + set: [insertions: maybe_clear_insertions(c, ^expression, ^timezone)] + ) + ] + + Repo.insert!(conf, changeset, repo_opts) + end) + end + + defp delete_entries(conf, crontab) do + Repo.delete_all(conf, where(CronEntry, [e], e.name in ^crontab_names(crontab))) + end + + defp delete_missing(conf, crontab) do + Repo.delete_all(conf, where(CronEntry, [e], e.name not in ^crontab_names(crontab))) + end + + defp crontab_names(crontab) do + Enum.map(crontab, fn {_expr, worker, opts} -> + opts + |> Keyword.get(:name, worker) + |> Worker.to_string() + end) + end + + # Queries + + defp all_entries(conf) do + Repo.all(conf, order_by(CronEntry, asc: :inserted_at)) + end + + defp update_entry(conf, name, params) do + with {:ok, entry} <- fetch_entry(conf, name) do + changeset = CronEntry.update_changeset(entry, Map.new(params)) + + Repo.update(conf, changeset) + end + end + + defp delete_cron(conf, name) do + with {:ok, entry} <- fetch_entry(conf, name), do: Repo.delete(conf, entry) + end + + defp fetch_entry(conf, name) do + case Repo.one(conf, where(CronEntry, name: ^name)) do + nil -> {:error, "no cron entry named #{inspect(name)} could be found"} + entry -> {:ok, entry} + end + end +end diff --git a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_lifeline.ex b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_lifeline.ex new file mode 100644 index 00000000..8a78b3d3 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_lifeline.ex @@ -0,0 +1,404 @@ +defmodule Oban.Pro.Plugins.DynamicLifeline do + @moduledoc """ + The `DynamicLifeline` plugin uses producer records to periodically rescue orphaned jobs, i.e. + jobs that are stuck in the `executing` state because the node was shut down before the job could + finish. In addition, it performs the following maintenance tasks: + + * Discard jobs left `available` with an attempt equal to max attempts + * Repair stuck workflows with deleted dependencies or missed scheduling events + * Repair stuck chains with deleted dependencies or missed scheduling events + * Repair jobs in partitioned queues that are missing a partition key + + Without `DynamicLifeline` you'll need to manually rescue stuck jobs or perform maintenance. + + ## Using the Plugin + + To use the `DynamicLifeline` plugin, add the module to your list of Oban plugins in + `config.exs`: + + ```elixir + config :my_app, Oban, + plugins: [Oban.Pro.Plugins.DynamicLifeline] + ... + ``` + + There isn't any configuration necessary. By default, the plugin rescues orphaned jobs every 1 + minute. If necessary, you can override the rescue interval: + + ```elixir + plugins: [{Oban.Pro.Plugins.DynamicLifeline, rescue_interval: :timer.minutes(5)}] + ``` + + If your system is under high load or produces a multitude of orphans you may wish to increase + the query timeout beyond the `45s` default: + + ```elixir + plugins: [{Oban.Pro.Plugins.DynamicLifeline, timeout: :timer.minutes(1)}] + ``` + + Note that rescuing orphans relies on producer records as used by the `Smart` engine. + + ## Repairing Workflows, Chains, and Partitions + + The plugin automatically repairs stuck workflows, chains, and partitioned jobs during each + rescue cycle. These repair operations ensure: + + * **Workflow repair** — Jobs held waiting for dependencies that were deleted or missed a + scheduling event are released. This handles edge cases where workflow jobs get stuck due to + incomplete dependency resolution. + + * **Chain repair** — Similar to workflows, chain jobs waiting on deleted or stuck predecessors + are released to continue processing. + + * **Partition repair** — Jobs in partitioned queues that are missing a `partition_key` in their + metadata (e.g., jobs scheduled before a queue was partitioned) have their partition key + computed and set automatically. + + By default, each repair operation processes up to 1000 jobs per cycle. You can adjust this + limit with the `repair_limit` option: + + ```elixir + plugins: [{Oban.Pro.Plugins.DynamicLifeline, repair_limit: 500}] + ``` + + ## Identifying Rescued Jobs + + Rescued jobs can be identified by a `rescued` value in `meta`. Each rescue increments the + `rescued` count by one. + + ## Rescuing Exhausted Jobs + + When a job's `attempt` matches its `max_attempts` its retries are considered "exhausted". + Normally, the `DynamicLifeline` plugin transitions exhausted jobs to the `discarded` state and + they won't be retried again. It does this for a couple of reasons: + + 1. To ensure at-most-once semantics. Suppose a long-running job interacted with a non-idempotent + service and was shut down while waiting for a reply; you may not want that job to retry. + + 2. To prevent infinitely crashing BEAM nodes. Poorly behaving jobs may crash the node (through + NIFs, memory exhaustion, etc.) We don't want to repeatedly rescue and rerun a job that + repeatedly crashes the entire node. + + Discarding exhausted jobs may not always be desired. Use the `retry_exhausted` option if you'd + prefer to retry exhausted jobs when they are rescued, rather than discarding them: + + ```elixir + plugins: [{Oban.Pro.Plugins.DynamicLifeline, retry_exhausted: true}] + ``` + + During rescues, with `retry_exhausted: true`, a job's `max_attempts` is incremented and it is + moved back to the `available` state. + + ## Instrumenting with Telemetry + + The `DynamicLifeline` plugin adds the following metadata to the `[:oban, :plugin, :stop]` event: + + * `:rescued_jobs` — a list of jobs transitioned back to `available` + + * `:discarded_jobs` — a list of jobs transitioned to `discarded` + + _Note: jobs only include `id`, `queue`, and `state` fields._ + """ + + @behaviour Oban.Plugin + + use GenServer + + import Ecto.Query + + alias Oban.Pro.Engines.Smart + alias Oban.Pro.Stages.Chain + alias Oban.Pro.{Partition, Producer, Workflow} + alias Oban.{Job, Peer, Repo, Validation} + + require Logger + + @type option :: + {:conf, Oban.Config.t()} + | {:name, Oban.name()} + | {:repair_limit, pos_integer()} + | {:retry_exhausted, boolean()} + | {:rescue_interval, timeout()} + | {:timeout, timeout()} + + defstruct [ + :conf, + :rescue_timer, + repair_limit: 1000, + retry_exhausted: false, + rescue_interval: :timer.minutes(1), + timeout: :timer.seconds(45) + ] + + defmacrop attempted_join(attempted_by, uuid) do + quote do + fragment( + "array_length(?, 1) <> 1 and ? = uuid (?[2])", + unquote(attempted_by), + unquote(uuid), + unquote(attempted_by) + ) + end + end + + defmacrop coalesce_partition(meta) do + quote do + fragment( + "COALESCE(?->'global_limit'->'partition', ?->'rate_limit'->'partition')", + unquote(meta), + unquote(meta) + ) + end + end + + defmacrop track_rescued(meta) do + quote do + fragment( + """ + jsonb_set(?, '{rescued}', (COALESCE(?->>'rescued', '0')::int + 1)::text::jsonb) + """, + unquote(meta), + unquote(meta) + ) + end + end + + @doc false + def child_spec(args), do: super(args) + + @impl Oban.Plugin + @spec start_link([option()]) :: GenServer.on_start() + def start_link(opts) do + {name, opts} = Keyword.pop(opts, :name) + + case opts[:conf] do + %{engine: Smart} -> + GenServer.start_link(__MODULE__, struct!(__MODULE__, opts), name: name) + + %{engine: engine} -> + raise RuntimeError, """ + DynamicLifeline requires the Smart engine to run correctly, but you're using: + + engine: #{inspect(engine)} + + You can either switch to use the Smart or remove DynamicLifeline from your plugins. + """ + end + end + + @impl Oban.Plugin + def validate(opts) do + Validation.validate_schema(opts, + conf: :any, + name: :any, + repair_limit: :pos_integer, + rescue_interval: :pos_integer, + rescue_after: :pos_integer, + retry_exhausted: :boolean, + timeout: :timeout + ) + end + + @impl Oban.Plugin + def format_logger_output(_conf, meta) do + for {key, val} <- meta, + key in ~w(discarded_jobs rescued_jobs)a, + into: %{}, + do: {key, Enum.map(val, & &1.id)} + end + + @impl GenServer + def init(state) do + Process.flag(:trap_exit, true) + + :telemetry.execute([:oban, :plugin, :init], %{}, %{conf: state.conf, plugin: __MODULE__}) + + {:ok, schedule_rescue(state)} + end + + @impl GenServer + def terminate(_reason, state) do + if is_reference(state.rescue_timer), do: Process.cancel_timer(state.rescue_timer) + + :ok + end + + @impl GenServer + def handle_info(:rescue, state) do + if Peer.leader?(state.conf) do + meta = %{conf: state.conf, plugin: __MODULE__} + + :telemetry.span([:oban, :plugin], meta, fn -> + orphan_meta = rescue_orphaned(state) + + :ok = repair_workflows(state) + :ok = repair_chains(state) + :ok = repair_partitions(state) + + {:ok, Map.merge(meta, orphan_meta)} + end) + end + + {:noreply, schedule_rescue(state)} + end + + def handle_info(message, state) do + Logger.warning( + message: "Received unexpected message: #{inspect(message)}", + source: :oban_pro, + module: __MODULE__ + ) + + {:noreply, state} + end + + # Scheduling + + defp schedule_rescue(state) do + timer = Process.send_after(self(), :rescue, state.rescue_interval) + + %{state | rescue_timer: timer} + end + + # Orphan Rescuing + + defp rescue_orphaned(state) do + {res_count, res_jobs} = transition_available(orphan_query(), state) + {dis_count, dis_jobs} = transition_discarded(orphan_query(), state) + {exh_count, exh_jobs} = transition_discarded(stuck_query(), state) + + if state.retry_exhausted do + %{ + discarded_count: 0, + discarded_jobs: [], + rescued_count: res_count + dis_count + exh_count, + rescued_jobs: res_jobs ++ dis_jobs ++ exh_jobs + } + else + %{ + discarded_count: dis_count + exh_count, + discarded_jobs: dis_jobs ++ exh_jobs, + rescued_count: res_count, + rescued_jobs: res_jobs + } + end + end + + defp orphan_query do + subquery = + Job + |> where([j], not is_nil(j.queue) and j.state == "executing") + |> join(:left, [j], p in Producer, on: attempted_join(j.attempted_by, p.uuid)) + |> where([_, p], is_nil(p.uuid)) + + Job + |> join(:inner, [j], x in subquery(subquery), on: j.id == x.id) + |> select([_, x], map(x, [:id, :meta, :queue, :state])) + end + + # We don't know how it's possible for a job to be available with attempts >= max_attempts, but + # we can clean up to prevent clogging a queue at least. + defp stuck_query do + Job + |> where([j], j.state == "available") + |> select([j], map(j, [:id, :meta, :queue, :state])) + end + + defp transition_available(query, state) do + query = + query + |> where([j], j.attempt < j.max_attempts) + |> update([j], set: [meta: track_rescued(j.meta), state: "available"]) + + Repo.update_all(state.conf, query, [], timeout: state.timeout) + catch + :error, %Postgrex.Error{postgres: %{code: :unique_violation, detail: detail}} -> + Smart.clear_uniq_violation(state.conf, detail) + + {0, []} + end + + defp transition_discarded(query, %{retry_exhausted: true} = state) do + query = + query + |> where([j], j.attempt >= j.max_attempts) + |> update([j], + inc: [max_attempts: 1], + set: [meta: track_rescued(j.meta), state: "available"] + ) + + Repo.update_all(state.conf, query, [], timeout: state.timeout) + catch + :error, %Postgrex.Error{postgres: %{code: :unique_violation, detail: detail}} -> + Smart.clear_uniq_violation(state.conf, detail) + + {0, []} + end + + defp transition_discarded(query, state) do + Repo.update_all( + state.conf, + where(query, [j], j.attempt >= j.max_attempts), + [set: [state: "discarded", discarded_at: DateTime.utc_now()]], + timeout: state.timeout + ) + end + + # Composition Repair + + defp repair_workflows(state) do + Workflow.rescue_workflows(state.conf, timeout: state.timeout, limit: state.repair_limit) + end + + defp repair_chains(state) do + Chain.rescue_chains(state.conf, timeout: state.timeout, limit: state.repair_limit) + end + + # Partition Repair + + defp repair_partitions(state) do + for {queue, partition} <- partitioned_queues(state) do + query = + Job + |> where([j], j.queue == ^queue) + |> where([j], j.state in ~w(available scheduled retryable)) + |> where([j], fragment("NOT ? \\? 'partition_key'", j.meta)) + |> limit(^state.repair_limit) + + state.conf + |> Repo.all(query, timeout: state.timeout) + |> Enum.group_by(fn job -> gen_partition_key(job, partition) end) + |> Enum.each(fn {key, grouped_jobs} -> + ids = Enum.map(grouped_jobs, & &1.id) + + Job + |> where([j], j.id in ^ids) + |> update([j], set: [meta: fragment("jsonb_set(?, '{partition_key}', ?)", j.meta, ^key)]) + |> then(&Repo.update_all(state.conf, &1, [], timeout: state.timeout)) + end) + end + + :ok + end + + defp gen_partition_key(job, partition) do + job + |> Ecto.Changeset.change() + |> Partition.gen_key(partition) + end + + defp partitioned_queues(state) do + normalize_partition = fn partition -> + fields = partition |> Map.get("fields", []) |> Enum.map(&String.to_existing_atom/1) + + %{fields: fields, keys: Map.get(partition, "keys", [])} + end + + Producer + |> where([p], fragment("jsonb_typeof(?) = 'object'", coalesce_partition(p.meta))) + |> select([p], {p.queue, coalesce_partition(p.meta)}) + |> distinct(true) + |> then(&Repo.all(state.conf, &1, timeout: state.timeout)) + |> Enum.map(fn {queue, partition} -> {queue, normalize_partition.(partition)} end) + end +end diff --git a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_partitioner.ex b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_partitioner.ex new file mode 100644 index 00000000..0c725f03 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_partitioner.ex @@ -0,0 +1,669 @@ +defmodule Oban.Pro.Plugins.DynamicPartitioner do + @moduledoc """ + The `DynamicPartitioner` plugin manages a partitioned `oban_jobs` table for optimized query + performance, minimal database bloat, and efficiently pruned historic jobs. + + > #### When to use Partitioned Tables {: .warning} + > + > Partitioning has performance tradeoffs because it doesn't support unique indexes and some + > queries need to check each partition. It is ideally **suited for high throughput** + > applications that run **tens of millions of jobs a day**. Be absolutely sure your application + > will benefit from partitioning. + + Partitioning is only officially supported on Postgres 11 and higher. While older versions of + Postgres support partitioning, they have prohibitive technical limitations and your experience + may vary. + + ## Installation + + Before running the `DynamicPartitioner` plugin, you must run a migration to create a partitioned + `oban_jobs` table to your database. + + > #### Table Name Conflicts {: .info} + > + > Existing `oban_jobs` tables can't be converted to a partitioned table in place and require a + > transition stage. The migration will automatically handle table conflicts by renaming the + > existing table to `oban_jobs_old`. However, if the partitioned table is added to a different + > prefix without a conflict, then the original table is left untouched and both tables are named + > `oban_jobs`. + > + > See the [Backfilling and Migrating](##module-backfilling-and-migrating) section for strategies + > before running migrations. + + ```bash + mix ecto.gen.migration add_partitioned_oban_jobs + ``` + + Open the generated migration in your editor and delegate to the dynamic partitions migration: + + ```elixir + defmodule MyApp.Repo.Migrations.AddPartitionedObanJobs do + use Ecto.Migration + + defdelegate up, to: Oban.Pro.Migrations.DynamicPartitioner + defdelegate down, to: Oban.Pro.Migrations.DynamicPartitioner + end + ``` + + As with the standard `oban_jobs` table, you can optionally provide a `prefix` to "namespace" the table + within your database. Here we specify a `"partitioned"` prefix: + + ```elixir + defmodule MyApp.Repo.Migrations.AddPartitionedObanJobs do + use Ecto.Migration + + def up do + Oban.Pro.Migrations.DynamicPartitioner.up(prefix: "partitioned") + end + + def down do + Oban.Pro.Migrations.DynamicPartitioner.down(prefix: "partitioned") + end + end + ``` + + Run the migration to create the new table: + + ```bash + mix ecto.migrate + ``` + + The new table is partitioned for optimal inserts, ready for you to backfill any existing jobs + and configure retention periods. + + ### Date Partitioning in Test Environments + + To prevent testing errors after migration, the `completed`, `cancelled`, and `discarded` states + are sub-partitioned by date only in `:dev` and `:prod` environments. + + You can explicitly enable date partitioning in other production-like environments with the + `date_partition?` flag: + + ```elixir + Oban.Pro.Migrations.DynamicPartitioner.change(date_partition?: true) + ``` + + ## Backfilling and Migrating + + Below are several recommended strategies for backfilling original jobs into the new partitioned + table. They're listed in order of complexity, beginning with the least invasive approach. + + ### Strategy 1: Don't Backfill at All + + The most performant strategy is to not backfill jobs at all. It's perfectly acceptable to leave + old jobs untouched if you **don't need them for uniqueness checks** or other observability + concerns. Though, in an active system, you'll still want to finish processing jobs in the + original table. + + During the transition period, until all of the original jobs are processed, you'll run two + separate, entirely isolated, Oban instances: + + 1. **Original** — configured with the original prefix, `public` if you never changed it. This + instance will run all of the original queues, without any plugins and it won't insert new + jobs. + + 2. **Partitioned** — configured with the new prefix for the partitioned table, all of your + original queues, plugins, and any other options. + + Here's an example of that configuration: + + ```elixir + queues = [ + default: 10, + other_queue: 10, + and_another: 10 + ] + + config :my_app, Oban.Original, queues: queues + + config :my_app, Oban, + prefix: "partitioned", + queues: queues, + plugins: [ + ... + ``` + + Now, start both Oban instances within your application's supervisor: + + ```diff + children = [ + MyApp.Repo, + {Oban, Application.fetch_env!(:my_app, Oban)}, + + {Oban, Application.fetch_env!(:my_app, Oban.Original)}, + ... + ] + ``` + + New jobs will be inserted into the `partitioned` table while existing jobs keep processing + through the `Oban.Original` instance. Once the all original jobs have executed you're free to + remove the extra instance and [drop the original table](#module-cleaning-up). + + ### Strategy 2: Backfill Old Jobs + + This strategy moves old jobs automatically as part of a migration. The `backfill/1` migration + helper, which delegates to `backfill_jobs/2`, helps move jobs in batches for each state. + + By default, backfilling includes _all states_ and is intended for smaller tables, i.e. 50k-100k + total jobs. + + Backfill by creating an additional migration with ddl transaction disabled: + + ```elixir + defmodule MyApp.Repo.Migrations.BackfillPartitionedObanJobs do + use Ecto.Migration + + @disable_ddl_transaction true + + def change do + Oban.Pro.Migrations.DynamicPartitioner.backfill() + end + end + ``` + + Like all other migrations, `backfill/1` accepts options to control the table's prefix. You can + specify both old and new prefixes to handle situations where the partitioned table lives in a + different prefix: + + ```elixir + def change do + Oban.Pro.Migrations.DynamicPartitioner.backfill(new_prefix: "private", old_prefix: "public") + end + ``` + + For larger tables, or applications that are sensitive to longer migrations, you can split + backfilling between migrations and prioritize in-flight jobs. + + Use the `states` option to restrict backfilling to actively `executing` jobs: + + ```elixir + def change do + Oban.Pro.Migrations.DynamicPartitioner.backfill(states: ~w(executing)) + end + ``` + + For the remaining jobs, you can either use a secondary migration or manually call + `backfill_jobs/1` from your application code. + + See `backfill_jobs/1` for the full range of backfill options including changing the batch size + and automatically sleeping between batches. + + ### Cleaning Up + + After backfilling is complete you can drop the original `oban_jobs` table. Be _very_ careful to + ensure you're dropping the old job table! If the new and old prefix was the same, which it is by + default, then the table has `_old` appended. + + ```elixir + defmodule MyApp.Repo.Migrations.DropStandardObanJobs do + use Ecto.Migration + + def change do + drop_if_exists table(:oban_jobs_old) + end + end + ``` + + ## Using and Configuring + + After running the migration to partition tables, enable the plugin to manage sub-partitions: + + ```elixir + config :my_app, Oban, + plugins: [Oban.Pro.Plugins.DynamicPartitioner] + ... + ``` + + The plugin will preemptively create sub-partitions for finished job states (`completed`, + `cancelled`, `discarded`) as well as prune partitions older than the retention period. By + default, older jobs are retained for 3 days. + + You can override the retention period for states individually. For example, to retain `completed` + jobs for 2 days, `cancelled` for 7, and `discarded` for 30: + + ```elixir + plugins: [{ + Oban.Pro.Plugins.DynamicPartitioner, + retention: [completed: 2, cancelled: 7, discarded: 30] + }] + ``` + + Pruning sub-partitions is an extremely fast operation akin to dropping a table. As a result, + there is zero lingering bloat. It's not advised that you use the `DynamicPruner`, unless + you're pruning a subset jobs aggressively after a few minutes, hours, etc. + + ```diff + plugins: [ + Oban.Pro.Plugins.DynamicPartitioner, + - Oban.Plugins.Pruner, + - Oban.Pro.Plugins.DynamicPruner + ] + ``` + + `DynamicPartitioner` will warn you if the standard `Pruner` is enabled at the same time. + + ### Tuning Partition Management + + The partitioner attempts once an hour to pre-create partitions two days in advance. That + schedule and buffer should be suitable for most applications. However, you can increase the + buffer period and set an alternate schedule if necessary. + + For example, to increase the buffer to 3 days and run at 05:00 in the Europe/Paris timezone: + + ```elixir + plugins: [{ + Oban.Pro.Plugins.DynamicPartitioner, + buffer: 3, + schedule: "0 5 * * *", + timezone: "Europe/Paris" + }] + ``` + + ## Instrumenting with Telemetry + + The `DynamicPartitioner` plugin adds the following metadata to the `[:oban, :plugin, :stop]` event: + + * `:created_count` — the number of partitions created + * `:deleted_count` - the number of partitions deleted + """ + + @behaviour Oban.Plugin + + use GenServer + + alias __MODULE__, as: State + alias Oban.Cron.Expression + alias Oban.Plugins.{Cron, Pruner} + alias Oban.{Config, Peer, Repo, Validation} + + require Logger + + @type retention :: [ + completed: pos_integer(), + cancelled: pos_integer(), + discarded: pos_integer() + ] + + @type option :: + {:conf, Oban.Config.t()} + | {:name, GenServer.name()} + | {:retention, retention()} + | {:schedule, String.t()} + | {:timeout, timeout()} + | {:timezone, String.t()} + + @retention [completed: 3, cancelled: 3, discarded: 3] + @states Keyword.keys(@retention) + + defstruct [ + :conf, + :name, + :retention, + :schedule, + :timer, + buffer: 2, + timeout: :timer.minutes(1), + timezone: "Etc/UTC" + ] + + @doc false + def child_spec(args), do: super(args) + + @impl Oban.Plugin + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, name: opts[:name]) + end + + @impl Oban.Plugin + def validate(opts) do + Validation.validate_schema(opts, + buffer: :pos_integer, + conf: {:custom, &validate_conf/1}, + name: :any, + retention: {:custom, &validate_retention/1}, + schedule: :schedule, + timeout: :timeout, + timezone: :timezone + ) + end + + @impl Oban.Plugin + def format_logger_output(_conf, meta) do + Map.take(meta, ~w(created_count deleted_count)a) + end + + @sub_states ~w(cancelled completed discarded) + + @defaults [ + new_prefix: "public", + old_prefix: "public", + states: ~w(executing available retryable scheduled cancelled discarded completed), + batch_size: 5_000, + batch_sleep: 0 + ] + + @doc false + def backfill_jobs, do: backfill_jobs(Oban, []) + + @doc """ + Backfill jobs from a standard table into a newly partitioned table. + + Backfilling is flexible enough to run against one or more job states, with arbitrary batch + sizes, and without transactional blocks. That allows repeated backfill runs in the face of + restarts or database errors. + + Sub-partitions by date are created for final states (`completed`, `cancelled`, `discarded`) + automatically before jobs are moved. + + ## Options + + * `:new_prefix` — The prefix where the new partitioned `oban_jobs` table resides. Defaults to + `public`. + + * `:old_prefix` — The prefix where the standard `oban_jobs_old` table resides. Defaults to + `public`. + + * `:batch_size` — The number of jobs to move (delete/insert) in a single query. Defaults to a + conservative 5,000 jobs per batch. + + * `:batch_sleep` — The amount of time to sleep between backfill batches in order to minimize + load on the database. Defaults to 0, no downtime between batches. + + * `:states` — A list of job states to backfill jobs from. Defaults to all states. + + ## Examples + + Backfill old jobs across all states in the default `public` prefix: + + DynamicPartitioner.backfill_jobs() + + Restrict backfilling to incomplete job states: + + DynamicPartitioner.backfill_jobs(states: ~w(executing available scheduled retryable)) + + Backfill to and from an alternate prefix: + + DynamicPartitioner.backfill_jobs(old_prefix: "private", new_prefix: "private") + + Backfill using larger batches with half a second between queries: + + DynamicPartitioner.backfill_jobs(batch_size: 20_000, batch_sleep: 500) + """ + @spec backfill_jobs(name_or_conf :: Oban.name() | Config.t(), opts :: Keyword.t()) :: :ok + def backfill_jobs(conf_or_name, opts \\ []) + + def backfill_jobs(%Config{} = conf, opts) when is_list(opts) do + opts = + opts + |> Keyword.validate!(@defaults) + |> Keyword.update!(:states, fn states -> Enum.map(states, &to_string/1) end) + |> Map.new() + + old_table = + if opts.old_prefix == opts.new_prefix do + "oban_jobs_old" + else + "oban_jobs" + end + + conf = %{conf | prefix: opts.new_prefix} + opts = Map.put(opts, :old_table, old_table) + + for state <- opts.states do + backfill_partitions(conf, state, opts) + backfill_jobs(conf, state, opts) + end + + :ok + end + + def backfill_jobs(oban_name, opts) do + oban_name + |> Oban.config() + |> backfill_jobs(opts) + end + + defp backfill_partitions(conf, state, opts) when state in @sub_states do + query = """ + SELECT DISTINCT date_trunc('day', #{state}_at)::date + FROM #{opts.old_prefix}.#{opts.old_table} + WHERE state = '#{state}' + """ + + {:ok, %{rows: rows}} = Repo.query(conf, query) + + rows + |> List.flatten() + |> Enum.reject(&is_nil/1) + |> Enum.each(&create_sub_partition(conf, state, &1)) + end + + defp backfill_partitions(_conf, _state, _opts), do: :ok + + defp backfill_jobs(conf, state, opts) do + query = """ + WITH old_jobs AS ( + DELETE FROM #{opts.old_prefix}.#{opts.old_table} + WHERE id IN ( + SELECT id + FROM #{opts.old_prefix}.#{opts.old_table} + WHERE state = '#{state}' + LIMIT #{opts.batch_size} + ) + RETURNING * + ), new_jobs AS ( + INSERT INTO #{opts.new_prefix}.oban_jobs ( + id, + state, + queue, + worker, + attempt, + max_attempts, + priority, + args, + meta, + attempted_by, + errors, + tags, + inserted_at, + scheduled_at, + attempted_at, + cancelled_at, + completed_at, + discarded_at + ) SELECT + id, + state, + queue, + worker, + attempt, + max_attempts, + priority, + args, + meta, + attempted_by, + errors, + tags, + inserted_at, + scheduled_at, + attempted_at, + cancelled_at, + completed_at, + discarded_at + FROM old_jobs + RETURNING 1 + ) + + SELECT count(*) from new_jobs + """ + + {:ok, %{rows: [[count]]}} = Repo.query(conf, query) + + if count > 0 do + Process.sleep(opts.batch_sleep) + + backfill_jobs(conf, state, opts) + end + end + + @impl GenServer + def init(opts) do + opts = + opts + |> Keyword.update(:retention, @retention, &Keyword.merge(@retention, &1)) + |> Keyword.update(:schedule, Expression.parse!("@hourly"), &Expression.parse!/1) + + state = + State + |> struct!(opts) + |> schedule_manage() + + :telemetry.execute([:oban, :plugin, :init], %{}, %{conf: state.conf, plugin: __MODULE__}) + + {:ok, state, {:continue, :start}} + end + + @impl GenServer + def handle_continue(:start, state) do + manage_tables(state) + + {:noreply, state} + end + + @impl GenServer + def handle_info(:manage, state) do + datetime = DateTime.now!(state.timezone) + + if Peer.leader?(state.conf) and Expression.now?(state.schedule, datetime) do + manage_tables(state) + end + + {:noreply, schedule_manage(state)} + end + + def handle_info(message, state) do + Logger.warning( + message: "Received unexpected message: #{inspect(message)}", + source: :oban, + module: __MODULE__ + ) + + {:noreply, state} + end + + defp manage_tables(%{conf: conf} = state) do + meta = %{conf: conf, plugin: __MODULE__} + + :telemetry.span([:oban, :plugin], meta, fn -> + xact_opts = [retry: 1, timeout: state.timeout] + + {:ok, created} = Repo.transaction(conf, fn -> create_sub_partitions(state) end, xact_opts) + {:ok, deleted} = Repo.transaction(conf, fn -> delete_sub_partitions(state) end, xact_opts) + + counts = %{created_count: length(created), deleted_count: length(deleted)} + + {:ok, Map.merge(meta, counts)} + end) + catch + kind, value -> + Logger.error(fn -> + "[#{__MODULE__}] management error: " <> Exception.format(kind, value, __STACKTRACE__) + end) + end + + defp create_sub_partitions(%{buffer: buffer, conf: conf}) do + for state <- @states, days <- 0..buffer do + date = Date.add(Date.utc_today(), days) + + create_sub_partition(conf, state, date) + end + end + + defp create_sub_partition(conf, state, date) do + quoted = inspect(conf.prefix) + next = Date.add(date, 1) + safe = Calendar.strftime(date, "%Y%m%d") + + command = """ + CREATE TABLE IF NOT EXISTS #{quoted}.oban_jobs_#{state}_#{safe} + PARTITION OF #{quoted}.oban_jobs_#{state} + FOR VALUES FROM ('#{date} 00:00:00') TO ('#{next} 00:00:00') + """ + + query!(conf, command) + end + + defp delete_sub_partitions(%{conf: conf, retention: retention}) do + rules = + Map.new(retention, fn {state, days} -> + stamp = + Date.utc_today() + |> Date.add(-days) + |> Calendar.strftime("%Y%m%d") + |> String.to_integer() + + {to_string(state), stamp} + end) + + query = """ + SELECT table_name + FROM information_schema.tables + WHERE table_schema = '#{conf.prefix}' + AND table_name ~ 'oban_jobs_(cancelled|completed|discarded)_.+' + """ + + for [table] <- query!(conf, query), + <<"oban_jobs_", state::binary-size(9), "_", date::binary-size(8)>> = table, + String.to_integer(date) <= Map.fetch!(rules, state) do + query!(conf, "DROP TABLE IF EXISTS #{inspect(conf.prefix)}.#{table}") + end + end + + defp query!(conf, command) do + case Repo.query(conf, command) do + {:ok, %{rows: rows}} -> rows + {:error, error} -> raise(error) + end + end + + # Scheduling + + defp schedule_manage(state) do + timer = Process.send_after(self(), :manage, Cron.interval_to_next_minute()) + + %{state | timer: timer} + end + + # Validation + + defp validate_conf(conf) do + cond do + conf.engine != Oban.Pro.Engines.Smart -> + {:error, "requires the Smart engine to run, got: #{inspect(conf.engine)}"} + + Keyword.has_key?(conf.plugins, Pruner) -> + {:error, "pruning isn't compatibile with partitioning, found: #{Pruner}"} + + true -> + :ok + end + end + + defp validate_retention(retention) do + keys = Keyword.keys(retention) + vals = Keyword.values(retention) + + cond do + not Keyword.keyword?(retention) -> + {:error, "expected :retention to be a keyword list, got: #{inspect(retention)}"} + + Enum.any?(keys, &(&1 not in @states)) -> + {:error, "expected :retention keys to overlap #{inspect(@states)}, got: #{inspect(keys)}"} + + Enum.any?(vals, &((is_integer(&1) and &1 <= 0) or not is_integer(&1))) -> + {:error, "expected :retention values to be positive integers, got: #{inspect(vals)}"} + + true -> + :ok + end + end +end diff --git a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_prioritizer.ex b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_prioritizer.ex new file mode 100644 index 00000000..fff57c0e --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_prioritizer.ex @@ -0,0 +1,318 @@ +defmodule Oban.Pro.Plugins.DynamicPrioritizer do + @moduledoc """ + The DynamicPrioritizer plugin automatically adjusts job's priorities to ensure all jobs are + eventually processed. + + Using mixed priorities in a queue causes certain jobs to execute before others. For example, a + queue that processes jobs from various customers may prioritize customers that are in a higher + tier or plan. All high priority (`0`) jobs are guaranteed to run before any with lower priority + (`1..9`), which is wonderful for the higher tier customers but can lead to resource starvation. + When there is a constant flow of high priority jobs the lower priority jobs will never get the + chance to run. + + ## Using the Plugin + + To use the `DynamicPrioritizer` plugin add the module to your list of Oban plugins in + `config.exs`: + + ```elixir + config :my_app, Oban, + plugins: [Oban.Pro.Plugins.DynamicPrioritizer] + ... + ``` + + Without any additional options the plugin will automatically increase the priority of any jobs + that are `available` for 5 minutes or more. To reprioritize after less time waiting you can + configure the `:after` time: + + ```elixir + plugins: [{Oban.Pro.Plugins.DynamicPrioritizer, after: :timer.minutes(2)}] + ``` + + Now lower job priorities are bumped after 2 minutes of waiting, and every minute after that. To + lower the reprioritization frequency you can change the `:interval` along with the `:after` + time: + + ```elixir + plugins: [{ + Oban.Pro.Plugins.DynamicPrioritizer, + after: :timer.minutes(10), + interval: :timer.minutes(5) + }] + ``` + + Here we've specified that job priority will increase every 5 minutes after the first 10 minutes + of waiting. + + It's also possible to limit the maximum priority jobs may transition to. Typically, jobs will + converge on `0` priority, as that's the highest priority. Set the `max_priority` option to + change to a lower priority: + + ```elixir + plugins: [{Oban.Pro.Plugins.DynamicPrioritizer, max_priority: 2}] + ``` + + ## Providing Overrides + + The `after` option applies to jobs for any workers across all queues. The `DynamicPrioritizer` + plugin allows you to specify per-queue and per-worker overrides that fine tune reprioritization. + + Let's configure reprioritization for the `analysis` queue so that it nudges jobs after only 1 + minute: + + ```elixir + plugins: [{ + Oban.Pro.Plugins.DynamicPrioritizer, + queue_overrides: [analysis: :timer.minutes(1)] + }] + ``` + + We can also effectively disable reprioritization for all other queues by setting the period to + `:infinity`: + + ```elixir + plugins: [{ + Oban.Pro.Plugins.DynamicPrioritizer, + after: :infinity, + queue_overrides: [analysis: :timer.minutes(1)] + }] + ``` + + If per-queue overrides aren't enough we can override on a per-worker basis instead: + + ```elixir + plugins: [{ + Oban.Pro.Plugins.DynamicPrioritizer, + interval: :timer.seconds(15), + limit: 20_000, + worker_overrides: [ + "MyApp.HighSLAWorker": :timer.seconds(30), + "MyApp.LowSLAWorker": :timer.minutes(10) + ] + }] + ``` + + Naturally you can mix and match overrides to finely control reprioritization: + + ```elixir + plugins: [{ + Oban.Pro.Plugins.DynamicPrioritizer, + interval: :timer.minutes(2), + after: :timer.minutes(5), + queue_overrides: [media: :timer.minutes(10)], + worker_overrides: ["MyApp.HighSLAWorker": :timer.seconds(30)] + }] + ``` + + ## Instrumenting with Telemetry + + The `DynamicPrioritizer` plugin adds the following metadata to the `[:oban, :plugin, + :stop]` event: + + * `:reprioritized_count` — the number of jobs reprioritized + """ + + @behaviour Oban.Plugin + + use GenServer + + import Ecto.Query, only: [join: 5, limit: 2, order_by: 2, where: 3] + + alias Oban.{Job, Peer, Repo, Validation} + + require Logger + + @type option :: [ + after: timeout(), + interval: timeout(), + limit: pos_integer(), + max_priority: 0..9, + queue_overrides: [{atom() | String.t(), timeout()}], + worker_overrides: [{atom() | String.t(), timeout()}] + ] + + defstruct [ + :conf, + :timer, + after: :timer.minutes(5), + interval: :timer.seconds(60), + limit: 10_000, + max_priority: 0, + queue_overrides: [], + worker_overrides: [] + ] + + @doc false + def child_spec(args), do: super(args) + + # Callbacks + + @impl Oban.Plugin + def start_link(opts) do + {name, opts} = Keyword.pop(opts, :name) + + GenServer.start_link(__MODULE__, struct!(__MODULE__, opts), name: name) + end + + @impl Oban.Plugin + def validate(opts) do + Validation.validate_schema(opts, + conf: :any, + name: :any, + after: :timeout, + interval: :pos_integer, + limit: :pos_integer, + max_priority: {:range, 0..9}, + queue_overrides: {:custom, &validate_overrides(:queues, &1)}, + worker_overrides: {:custom, &validate_overrides(:workers, &1)} + ) + end + + @impl Oban.Plugin + def format_logger_output(_conf, meta) do + Map.take(meta, ~w(reprioritized_count)a) + end + + @impl GenServer + def init(state) do + Process.flag(:trap_exit, true) + + :telemetry.execute([:oban, :plugin, :init], %{}, %{conf: state.conf, plugin: __MODULE__}) + + {:ok, schedule_reprioritization(state)} + end + + @impl GenServer + def terminate(_reason, state) do + if is_reference(state.timer), do: Process.cancel_timer(state.timer) + + :ok + end + + @impl GenServer + def handle_info(:reprioritize, state) do + meta = %{conf: state.conf, plugin: __MODULE__} + + :telemetry.span([:oban, :plugin], meta, fn -> + case reprioritize_starved_jobs(state) do + {:ok, count} when is_integer(count) -> + {:ok, Map.put(meta, :reprioritized_count, count)} + + error -> + {:error, Map.put(meta, :error, error)} + end + end) + + {:noreply, schedule_reprioritization(state)} + end + + def handle_info(message, state) do + Logger.warning( + message: "Received unexpected message: #{inspect(message)}", + source: :oban_pro, + module: __MODULE__ + ) + + {:noreply, state} + end + + # Validation + + defp validate_overrides(parent_key, overrides) do + Validation.validate(parent_key, overrides, &validate_override/1) + end + + defp validate_override({key, value}) when is_atom(key) do + Validation.validate_timeout(key, value) + end + + defp validate_override(option) do + {:error, "expected override option to be a tuple, got: #{inspect(option)}"} + end + + # Scheduling + + defp schedule_reprioritization(state) do + %{state | timer: Process.send_after(self(), :reprioritize, state.interval)} + end + + # Queries + + defp reprioritize_starved_jobs(state) do + if Peer.leader?(state.conf) do + Repo.transaction(state.conf, fn -> + queue_counts = + for {queue, period} <- state.queue_overrides do + state + |> base_query() + |> query_for_queues(:any, [to_string(queue)]) + |> query_for_period(period) + |> update_all(state) + end + + worker_counts = + for {worker, period} <- state.worker_overrides do + state + |> base_query() + |> query_for_workers(:any, [to_string(worker)]) + |> query_for_period(period) + |> update_all(state) + end + + default_count = + state + |> base_query() + |> query_for_queues(:not, string_keys(state.queue_overrides)) + |> query_for_workers(:not, string_keys(state.worker_overrides)) + |> query_for_period(state.after) + |> update_all(state) + + [queue_counts, worker_counts] + |> List.flatten() + |> Enum.reduce(default_count, &(&1 + &2)) + end) + else + {:ok, 0} + end + end + + defp string_keys(keyword) do + for {key, _} <- keyword, do: to_string(key) + end + + defp base_query(%{limit: limit, max_priority: max_priority}) do + Job + |> where([j], j.state == "available") + |> where([j], j.priority > ^max_priority) + |> order_by(asc: :id) + |> limit(^limit) + end + + defp query_for_period(query, :infinity) do + where(query, [j], true == false) + end + + defp query_for_period(query, period) do + where(query, [j], j.scheduled_at <= ^to_timestamp(period)) + end + + defp query_for_queues(query, :not, []), do: query + defp query_for_queues(query, :not, queues), do: where(query, [j], j.queue not in ^queues) + defp query_for_queues(query, :any, queues), do: where(query, [j], j.queue in ^queues) + + defp query_for_workers(query, :not, []), do: query + defp query_for_workers(query, :not, workers), do: where(query, [j], j.worker not in ^workers) + defp query_for_workers(query, :any, workers), do: where(query, [j], j.worker in ^workers) + + defp update_all(subquery, state) do + query = join(Job, :inner, [j], x in subquery(subquery), on: j.id == x.id) + + state.conf + |> Repo.update_all(query, inc: [priority: -1]) + |> elem(0) + end + + defp to_timestamp(milliseconds) do + DateTime.add(DateTime.utc_now(), -milliseconds, :millisecond) + end +end diff --git a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_pruner.ex b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_pruner.ex new file mode 100644 index 00000000..36991ada --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_pruner.ex @@ -0,0 +1,692 @@ +defmodule Oban.Pro.Plugins.DynamicPruner do + @moduledoc """ + DynamicPruner enhances the default Pruner plugin's behaviour by allowing you to customize how + jobs are retained in the jobs table. Where the Pruner operates on a fixed schedule and treats + all jobs the same, with the DynamicPruner, you can use a flexible CRON schedule and provide + custom rules for specific queues, workers, and job states. + + ## Using the Plugin + + To start using the `DynamicPruner` add the module to your list of Oban plugins in `config.exs`: + + ```elixir + config :my_app, Oban, + plugins: [Oban.Pro.Plugins.DynamicPruner] + ... + ``` + + Without any additional options the pruner operates in maximum length mode (`max_len`) and + retains a conservative 1,000 `completed`, `cancelled`, or `discarded` jobs. To increase the + number of jobs retained you can provide your own `mode` configuration: + + ```elixir + plugins: [{Oban.Pro.Plugins.DynamicPruner, mode: {:max_len, 50_000}}] + ``` + + Now the pruner will retain the most recent 50,000 jobs instead. + + A fixed limit on the number of jobs isn't always ideal. Often you want to retain jobs based on + their age instead. For example, if you your application needs to ensure that a duplicate job + hasn't been enqueued within the past 24 hours you need to retain jobs for at least 24 hours; a + fixed limit simply won't work. For that we can use maximum age (`max_age`) mode instead: + + ```elixir + plugins: [{Oban.Pro.Plugins.DynamicPruner, mode: {:max_age, 60 * 60 * 48}}] + ``` + + Here we've specified `max_age` using seconds, where `60 * 60 * 48` is the number of seconds in + two days. + + Calculating the number of seconds in a period isn't especially readable, particularly when you + have numerous `max_age` declarations in overrides (see below). For clarity you can specify the + age's time unit as `:second`, `:minute`, `:hour`, `:day` or `:month`. Here is the same 48 hour + configuration from above, but specified in terms of days: + + ```elixir + plugins: [{Oban.Pro.Plugins.DynamicPruner, mode: {:max_age, {2, :days}}}] + ``` + + Now you can tell exactly how long jobs should be retained, without reverse calculating how many + seconds an expression represents. + + ## Providing Overrides + + The `mode` option is indiscriminate when determining which jobs to prune. It pays no attention + to which queue they are in, what worker the job is for, or which state they landed in. The + `DynamicPruner` allows you to specify per-queue, per-worker and per-state overrides that fine + tune pruning. + + We'll start with a simple example of limiting the total number of retained jobs in the `events` + queue: + + ```elixir + plugins: [{ + Oban.Pro.Plugins.DynamicPruner, + mode: {:max_age, {7, :days}}, + queue_overrides: [events: {:max_len, 1_000}] + }] + ``` + + With this configuration most jobs will be retained for seven days, but we'll only keep the + latest 1,000 jobs in the events queue. We can extend this further and override all of our queues + (and omit the default mode entirely): + + ```elixir + plugins: [{ + Oban.Pro.Plugins.DynamicPruner, + queue_overrides: [ + default: {:max_age, {6, :hours}}, + analysis: {:max_age, {1, :day}}, + events: {:max_age, {10, :minutes}}, + mailers: {:max_age, {2, :weeks}}, + media: {:max_age, {2, :months}} + ] + }] + ``` + + When pruning by queue isn't granular enough you can provide overrides by worker instead: + + ```elixir + plugins: [{ + Oban.Pro.Plugins.DynamicPruner, + worker_overrides: [ + "MyApp.BusyWorker": {:max_age, {1, :day}}, + "MyApp.SecretWorker": {:max_age, {1, :second}}, + "MyApp.HistoricWorker": {:max_age, {1, :month}} + ] + }] + ``` + + You can also override by state, which allows you to keep discarded jobs for inspection while + quickly purging cancelled or successfully completed jobs: + + ```elixir + plugins: [{ + Oban.Pro.Plugins.DynamicPruner, + state_overrides: [ + cancelled: {:max_age, {1, :hour}}, + completed: {:max_age, {1, :day}}, + discarded: {:max_age, {1, :month}} + ] + }] + ``` + + Naturally you can mix and match overrides to finely control job retention: + + ```elixir + plugins: [{ + Oban.Pro.Plugins.DynamicPruner, + mode: {:max_age, {7, :days}}, + queue_overrides: [events: {:max_age, {10, :minutes}}], + state_overrides: [discarded: {:max_age, {2, :days}}], + worker_overrides: ["MyApp.SecretWorker": {:max_age, {1, :second}}] + }] + ``` + + ### Override Precedence + + Overrides are applied sequentially, in this order: + + 1. Queue + 2. State + 3. Worker + 4. Default + + Using the example above, jobs in the `events` queue are deleted first, followed by jobs in the + `discarded` state, then the `MyApp.SecretWorker` worker, and finally any other jobs older than 7 + days that **weren't covered by any overrides**. + + ### Preventing Timeouts with Overrides + + Worker override queries aren't able to use any of Oban's standard indexes. If you're processing + a high volume of jobs, pruning with worker overrides may be extremely slow due to sequential + scans. To prevent timeouts, and speed up pruning altogether, you should add a compound index to + the `oban_jobs` table: + + ```elixir + create_if_not_exists index(:oban_jobs, [:worker, :state, :id], concurrently: true) + ``` + + ## Retaining Jobs Forever + + To retain jobs in a queue, state, or for a particular worker forever (without using something + like `{:max_age, {999, :years}}` use `:infinity` as the length or duration: + + ```elixir + plugins: [{ + Oban.Pro.Plugins.DynamicPruner, + mode: {:max_age, {7, :days}}, + state_overrides: [ + cancelled: {:max_len, :infinity}, + discarded: {:max_age, :infinity} + ] + }] + ``` + + ## Keeping Up With Inserts + + With the default settings the `DynamicPruner` will only delete 10,000 jobs each time it prunes. + The limit exists to prevent connection timeouts and excessive table locks. A busy system can + easily insert more than 10,000 jobs per minute during standard operation. If you find that jobs + are accumulating despite active pruning you can override the `limit`. + + Here we set the delete limit to 25,000 and give it 60 seconds to complete: + + ```elixir + plugins: [{ + Oban.Pro.Plugins.DynamicPruner, + mode: {:max_len, 100_000}, + limit: 25_000, + timeout: :timer.seconds(60) + }] + ``` + + Deleting in PostgreSQL is _very_ fast, and the 10k default is rather conservative. Feel free to + increase the limit to a number that your system can handle. + + ## Setting a Schedule + + By default, pruning happens at the top of every minute based on the CRON schedule `* * * * *`. + You're free to set any CRON schedule you prefer for greater control over when to prune. For + example, to prune once an hour instead: + + ```elixir + plugins: [{Oban.Pro.Plugins.DynamicPruner, schedule: "0 * * * *"}] + ``` + + Or, to prune once a day at midnight in your local timezone: + + ```elixir + plugins: [{ + Oban.Pro.Plugins.DynamicPruner, + limit: 100_000, + schedule: "0 0 * * *", + timezone: "America/Chicago", + timeout: :timer.minutes(1) + }] + ``` + + Pruning less frequently can reduce load on your system, particularly if you're using + multiple [overrides](#providing-overrides). However, be sure to set a higher `limit` and + `timeout` (as shown above) to compensate for more accumulated jobs. + + ## Executing a Callback Before Delete + + Sometimes jobs are a historic record of activity and it's desirable to operate on them _before_ + they're deleted. For example, you may want copy some jobs into cold storage prior to completion. + + To accomplish this, specify a callback to execute before proceeding with the deletion.: + + ```elixir + defmodule DeleteHandler do + def call(job_ids) do + # Use the ids at this point, from within a transaction + end + end + + plugins: [{ + Oban.Pro.Plugins.DynamicPruner, + mode: {:max_age, {7, :days}}, + before_delete: {DeleteHandler, :call, []} + }] + ``` + + The callback receives a list of the ids for the jobs that are about to be deleted. The callback + runs within the same transaction that's used for deletion, and you should keep it quick or move + heavy processing to an async process. Note that because it runs in the same transaction as + deletion, the jobs _won't be available after the callback exits_. + + To pass in extra arguments as "configuration" you can provide args to the callback MFA: + + ```elixir + defmodule DeleteHandler do + import Ecto.Query + + def call(job_ids, storage_name) do + jobs = MyApp.Repo.all(where(Oban.Job, [j], j.id in ^job_ids)) + + Storage.call(storage_name, jobs) + end + end + + before_delete: {DeleteHandler, :call, [ColdStorage]} + ``` + + ## Implementation Notes + + Some additional notes about pruning in general and nuances of the `DynamicPruner` plugin: + + * Pruning is best-effort and performed out-of-band. This means that all limits are soft; jobs + beyond a specified age may not be pruned immediately after jobs complete. + + * Pruning is only applied to jobs that are `completed`, `cancelled` or `discarded` (has reached + the maximum number of retries or has been manually killed). It'll never delete a new, + scheduled, or retryable job. + + * Jobs that are part of an active workflow are retained regardless of their state. A workflow + is considered active if it contains any jobs in `scheduled`, `retryable`, `available`, or + `executing` states. + + * Only a single node will prune at any given time, which prevents potential deadlocks between + transactions. + + ## Instrumenting with Telemetry + + The `DynamicPruner` plugin adds the following metadata to the `[:oban, :plugin, :stop]` event: + + * `:pruned_jobs` - the jobs that were deleted from the database + + _Note: jobs only include `id`, `queue`, and `state` fields._ + """ + + @behaviour Oban.Plugin + + use GenServer + + import Ecto.Query + + alias Oban.Cron.Expression + alias Oban.Plugins.Cron + alias Oban.{Job, Peer, Repo, Validation} + + require Logger + + @type time_unit :: + :second + | :seconds + | :minute + | :minutes + | :hour + | :hours + | :day + | :days + | :week + | :weeks + | :month + | :month + + @type max_age :: pos_integer() | {pos_integer(), time_unit()} + + @type max_len :: pos_integer() + + @type mode :: {:max_age, max_age()} | {:max_len, max_len()} + + @type state_override :: {:completed | :cancelled | :discarded, mode()} + + @type queue_override :: {atom(), mode()} + + @type worker_override :: {module(), mode()} + + @type override :: state_override() | queue_override() | worker_override() + + @type option :: + {:conf, Oban.Config.t()} + | {:schedule, String.t()} + | {:name, Oban.name()} + + @modes [:max_age, :max_len] + @states [:completed, :cancelled, :discarded] + @time_units [ + :second, + :seconds, + :minute, + :minutes, + :hour, + :hours, + :day, + :days, + :week, + :weeks, + :month, + :months + ] + + defstruct [ + :before_delete, + :conf, + :schedule, + :timer, + limit: 10_000, + mode: {:max_len, 1_000}, + queue_overrides: [], + state_overrides: [], + timeout: :timer.seconds(60), + timezone: "Etc/UTC", + worker_overrides: [] + ] + + defmacrop same_workflow(meta_a, meta_b) do + quote do + fragment( + """ + ? \\? 'workflow_id' AND ?->> 'workflow_id' = ? ->> 'workflow_id' + """, + unquote(meta_a), + unquote(meta_a), + unquote(meta_b) + ) + end + end + + defmacrop same_sub_workflow(meta_a, meta_b) do + quote do + fragment( + """ + ? \\? 'sup_workflow_id' AND ?->> 'sup_workflow_id' = ? ->> 'workflow_id' + """, + unquote(meta_a), + unquote(meta_a), + unquote(meta_b) + ) + end + end + + defmacrop same_sup_workflow(meta_a, meta_b) do + quote do + fragment( + """ + ? \\? 'workflow_id' AND ?->> 'workflow_id' = ? ->> 'sup_workflow_id' + """, + unquote(meta_a), + unquote(meta_a), + unquote(meta_b) + ) + end + end + + @doc false + def child_spec(args), do: super(args) + + @impl Oban.Plugin + def start_link(opts) do + {name, opts} = Keyword.pop(opts, :name) + + opts = + opts + |> Keyword.put_new(:schedule, "* * * * *") + |> Keyword.update!(:schedule, &Expression.parse!/1) + + GenServer.start_link(__MODULE__, struct!(__MODULE__, opts), name: name) + end + + @impl Oban.Plugin + def validate(opts) do + opts + |> Keyword.delete(:by_state_timestamp) + |> Validation.validate_schema( + before_delete: {:custom, &validate_before_delete/1}, + conf: :any, + limit: :pos_integer, + mode: {:custom, &validate_mode/1}, + name: :any, + queue_overrides: {:custom, &validate_overrides(:queue_overrides, &1)}, + state_overrides: {:custom, &validate_overrides(:state_overrides, &1)}, + schedule: :schedule, + timeout: :timeout, + timezone: :timezone, + worker_overrides: {:custom, &validate_overrides(:worker_overrides, &1)} + ) + end + + @impl Oban.Plugin + def format_logger_output(_conf, meta) do + Map.take(meta, ~w(pruned_count)a) + end + + @impl GenServer + def init(state) do + Process.flag(:trap_exit, true) + + :telemetry.execute([:oban, :plugin, :init], %{}, %{conf: state.conf, plugin: __MODULE__}) + + {:ok, schedule_prune(state)} + end + + @impl GenServer + def terminate(_reason, state) do + if is_reference(state.timer), do: Process.cancel_timer(state.timer) + + :ok + end + + @impl GenServer + def handle_info(:prune, state) do + meta = %{conf: state.conf, plugin: __MODULE__} + + :telemetry.span([:oban, :plugin], meta, fn -> + with {:ok, extra} <- prune_jobs(state), do: {:ok, Map.merge(meta, extra)} + end) + + {:noreply, schedule_prune(state)} + end + + def handle_info(message, state) do + Logger.warning( + message: "Received unexpected message: #{inspect(message)}", + source: :oban_pro, + module: __MODULE__ + ) + + {:noreply, state} + end + + # Scheduling + + defp schedule_prune(state) do + timer = Process.send_after(self(), :prune, Cron.interval_to_next_minute()) + + %{state | timer: timer} + end + + # Query + + defp prune_jobs(state) do + {:ok, datetime} = DateTime.now(state.timezone) + + if Peer.leader?(state.conf) and Expression.now?(state.schedule, datetime) do + string_queues = string_keys(state.queue_overrides) + string_states = string_keys(state.state_overrides) + string_workers = string_keys(state.worker_overrides) + + queue_pruned = + for {queue, mode} <- state.queue_overrides do + Job + |> query_for_queues(:any, [to_string(queue)]) + |> delete_for_mode(mode, state) + end + + state_pruned = + for {queue_state, mode} <- state.state_overrides do + Job + |> query_for_states(:any, [to_string(queue_state)]) + |> delete_for_mode(mode, state) + end + + worker_pruned = + for {worker, mode} <- state.worker_overrides do + Job + |> query_for_workers(:any, [to_string(worker)]) + |> delete_for_mode(mode, state) + end + + default_pruned = + Job + |> query_for_queues(:not, string_queues) + |> query_for_states(:not, string_states) + |> query_for_workers(:not, string_workers) + |> delete_for_mode(state.mode, state) + + pruned = List.flatten([queue_pruned, state_pruned, worker_pruned, default_pruned]) + + {:ok, %{pruned_count: length(pruned), pruned_jobs: pruned}} + else + {:ok, %{pruned_count: 0, pruned_jobs: []}} + end + end + + defp string_keys(keyword) do + for {key, _} <- keyword, do: to_string(key) + end + + defp query_for_queues(query, :not, []), do: query + defp query_for_queues(query, :not, queues), do: where(query, [j], j.queue not in ^queues) + defp query_for_queues(query, :any, queues), do: where(query, [j], j.queue in ^queues) + + defp query_for_states(query, :not, []), do: query + defp query_for_states(query, :not, states), do: where(query, [j], j.state not in ^states) + defp query_for_states(query, :any, states), do: where(query, [j], j.state in ^states) + + defp query_for_workers(query, :not, []), do: query + defp query_for_workers(query, :not, workers), do: where(query, [j], j.worker not in ^workers) + defp query_for_workers(query, :any, workers), do: where(query, [j], j.worker in ^workers) + + defp delete_for_mode(_query, {_mode, :infinity}, _state), do: [] + + defp delete_for_mode(query, {:max_age, age}, state) do + time = to_timestamp(age) + + query + |> select([j], %{id: j.id, queue: j.queue, state: j.state, rn: 100_000_000}) + |> where( + [j], + (j.state == "completed" and j.scheduled_at < ^time) or + (j.state == "cancelled" and j.cancelled_at < ^time) or + (j.state == "discarded" and j.discarded_at < ^time) + ) + |> delete_all(state.limit + 1, state) + end + + defp delete_for_mode(query, {:max_len, len}, state) do + query + |> where([j], j.state in ~w(cancelled completed discarded)) + |> select([j], %{ + id: j.id, + queue: j.queue, + state: j.state, + rn: fragment("row_number() over (order by id desc)") + }) + |> delete_all(len, state) + end + + defp delete_all(query, offset, state) do + workflow_subquery = + Job + |> where([j], j.state in ~w(scheduled retryable available executing)) + |> where([j], fragment("? \\? 'workflow_id'", parent_as(:jobs).meta)) + |> where( + [j], + same_workflow(j.meta, parent_as(:jobs).meta) or + same_sub_workflow(j.meta, parent_as(:jobs).meta) or + same_sup_workflow(j.meta, parent_as(:jobs).meta) + ) + |> select(1) + + subquery = + query + |> from(as: :jobs) + |> where([j], not exists(workflow_subquery)) + |> order_by(asc: :id) + |> limit(^state.limit) + + query = + Job + |> join(:inner, [j], x in subquery(subquery), on: j.id == x.id and x.rn > ^offset) + |> select([_, x], map(x, [:id, :queue, :state])) + + fun = fn -> + apply_before_delete(query, state) + + Repo.delete_all(state.conf, query, timeout: state.timeout) + end + + {:ok, {_count, deleted}} = Repo.transaction(state.conf, fun, timeout: state.timeout) + + deleted + end + + defp apply_before_delete(query, state) do + with {mod, fun, args} <- state.before_delete do + ids = + state.conf + |> Repo.all(query) + |> Enum.map(& &1.id) + + apply(mod, fun, [ids | args]) + end + + :ok + end + + # Timestamp + + defp to_timestamp(seconds) when is_integer(seconds) do + DateTime.add(DateTime.utc_now(), -seconds, :second) + end + + defp to_timestamp({seconds, :second}), do: to_timestamp(seconds) + defp to_timestamp({seconds, :seconds}), do: to_timestamp(seconds) + defp to_timestamp({minutes, :minute}), do: to_timestamp(minutes * 60) + defp to_timestamp({minutes, :minutes}), do: to_timestamp({minutes, :minute}) + defp to_timestamp({hours, :hour}), do: to_timestamp(hours * 60 * 60) + defp to_timestamp({hours, :hours}), do: to_timestamp({hours, :hour}) + defp to_timestamp({days, :day}), do: to_timestamp(days * 24 * 60 * 60) + defp to_timestamp({days, :days}), do: to_timestamp({days, :day}) + defp to_timestamp({weeks, :week}), do: to_timestamp(weeks * 7 * 24 * 60 * 60) + defp to_timestamp({weeks, :weeks}), do: to_timestamp({weeks, :week}) + defp to_timestamp({months, :month}), do: to_timestamp(months * 30 * 24 * 60 * 60) + defp to_timestamp({months, :months}), do: to_timestamp({months, :month}) + + # Validations + + defp validate_before_delete({mod, fun, args}) + when is_atom(mod) and is_atom(fun) and is_list(args) do + cond do + not Code.ensure_loaded?(mod) -> + {:error, "module #{inspect(mod)} can't be loaded"} + + not function_exported?(mod, fun, length(args) + 1) -> + {:error, "no function #{inspect(fun)} with arity #{length(args) + 1}"} + + true -> + :ok + end + end + + defp validate_before_delete(mfa) do + {:error, "expected :before_delete to be a {module, fun, args} tuple, got: #{inspect(mfa)}"} + end + + defp validate_mode({mode, :infinity}) when mode in @modes, do: :ok + defp validate_mode({mode, int}) when is_integer(int) and mode in @modes, do: :ok + + defp validate_mode({mode, {value, time_unit}}) + when is_integer(value) and mode in @modes and time_unit in @time_units, + do: :ok + + defp validate_mode(mode) do + {:error, "expected :mode to be {:max_len, length} or {:max_age, age}, got: #{inspect(mode)}"} + end + + defp validate_overrides(:state_overrides = parent_key, overrides) do + Validation.validate(parent_key, overrides, fn + {state, mode} when state in @states -> + validate_mode(mode) + + {state, _mode} -> + {:error, "expected state to be included in #{inspect(@states)}, got: #{inspect(state)}"} + + override -> + {:error, "expected #{inspect(parent_key)} to contain a tuple, got: #{inspect(override)}"} + end) + end + + defp validate_overrides(parent_key, overrides) do + Validation.validate(parent_key, overrides, fn + {key, mode} when is_atom(key) -> + validate_mode(mode) + + override -> + {:error, "expected #{inspect(parent_key)} to contain a tuple, got: #{inspect(override)}"} + end) + end +end diff --git a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_queues.ex b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_queues.ex new file mode 100644 index 00000000..77b34209 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_queues.ex @@ -0,0 +1,745 @@ +defmodule Oban.Pro.Plugins.DynamicQueues do + @moduledoc """ + The `DynamicQueue` plugin extends Oban's basic queue management by persisting queue changes + across restarts, globally, across all connected nodes. It also boasts a declarative syntax for + specifying which nodes a queue will run on. + + `DynamicQueues` are ideal for applications that dynamically start, stop, or modify queues at + runtime. + + ## Using and Configuring + + To begin using `DynamicQueues`, add the module to your list of Oban plugins in `config.exs`: + + ```elixir + config :my_app, Oban, + plugins: [Oban.Pro.Plugins.DynamicQueues] + ... + ``` + + Without providing a list of queues the plugin doesn't have anything to run. The syntax for + specifying dynamic queues is identical to Oban's built-in `:queues` option, which means you can + copy them over: + + ```diff + - queues: [ + - default: 20, + - mailers: [global_limit: 20], + - events: [local_limit: 30, rate_limit: [allowed: 100, period: 60]] + - ] + + plugins: [{ + + Oban.Pro.Plugins.DynamicQueues, + + queues: [ + + default: 20, + + mailers: [global_limit: 20], + + events: [local_limit: 30, rate_limit: [allowed: 100, period: 60]] + + ] + + }] + ``` + + Now, when `DynamicQueues` initializes, it will persist all of the queues to the database and + start supervising them. The `queues` syntax is _nearly_ identical to Oban's standard queues, + with an important enhancement we'll look at shortly. + + Each of the persisted queues are referenced globally, by all other connected Oban instances that + are running the `DynamicQueues` plugin. Changing the queue's name, pausing, scaling, or changing + any other options will automatically update queues across all nodes—and persist across restarts. + + Persisted queues are referenced by name, so you can tweak a queue's options by changing the + definition within your config. For example, to bump the mailer's global limit up to `30`: + + queues: [ + mailers: [global_limit: 30], + ... + ] + + That isn't especially interesting—after all, that's exactly how regular queues work! Dynamic + queues start to shine when you insert, update, or delete them _dynamically_, either through Oban + Web or your own application code. + + ### Synchronizing Queues + + The `:sync_mode` option controls how DynamicQueues handles queue synchronization during startup. + There are two available modes: + + * `:manual` - Only deletes queues explicitly marked with `delete: true` and inserts/updates + the remaining queues. This is the default. + * `:automatic` - Automatically deletes any queues that aren't defined in the configuration and + inserts/updates the remaining queues. + + Here's how to configure each mode: + + ```elixir + # Manual mode (default) + config :my_app, Oban, plugins: [{DynamicQueues, sync_mode: :manual, queues: [...]}] + + # Automatic mode + config :my_app, Oban, plugins: [{DynamicQueues, sync_mode: :automatic, queues: [...]}] + ``` + + In manual mode, you must explicitly mark queues for deletion: + + ```elixir + queues: [ + old_queue: [delete: true], + default: 20, + mailers: [limit: 10] + ] + ``` + + In automatic mode, any queue that exists in the database but isn't defined in the configuration + will be automatically deleted during startup. This is useful when you want to ensure your + runtime queue configuration exactly matches what's defined in your application config. + + > #### Choosing a Sync Mode {: .info} + > + > Use `:manual` mode when you want fine-grained control over queue deletion and want to preserve + > dynamically created queues across restarts. Use `:automatic` mode when you want to ensure your + > queue configuration is the single source of truth and automatically clean up old queues. + > + > In either mode, changes to queues are persisted across restarts **until the configuration is + > changed**. For example, if you change the `global_limit` of a queue through the Web dashboard, + > that change will persist across restarts until you change the `global_limit` in your config. + + ### Limiting Where Queues Run + + Dynamic queues can be configured to run on a subset of available nodes. This is especially + useful when wish to restrict resource-intensive queues to only dedicated nodes. Restriction is + configured through the `only` option, which you use like this: + + queues: [ + basic: [local_limit: 10, only: {:node, :=~, "web|worker"}], + audio: [local_limit: 5, only: {:node, "worker.1"}], + video: [local_limit: 5, only: {:node, "worker.2"}], + learn: [local_limit: 5, only: {:sys_env, "EXLA", "CUDA"}], + store: [local_limit: 1, only: {:sys_env, "WAREHOUSE", true}] + ] + + In this example we've defined five queues, with the following restrictions: + + * `basic` — will run on a node named `web` or `worker` + * `audio` — will only run on a node named `worker.1` + * `video` — will only run on a node named `worker.2` + * `learn` — will run wherever `EXLA=CUDA` is an environment variable + * `store` — will run wherever `WAREHOUSE=true` is an environment variable + + Here are the various match modes, operators, and allowed patterns: + + #### Modes + + * `:node` — matches the node name as set in your Oban config. By default, `node` + is the node's id in a cluster, the hostname outside a cluster, or a `DYNO` + variable on Heroku. + * `:sys_env` — matches a single system environment variable as retrieved by + `System.get_env/1` + + #### Operators + + * `:==` — compares the pattern and runtime value for _equality_ as strings. This + is the default operator if nothing is specified. + * `:!=` — compares the pattern and runtime value for _inequality_ as strings + * `:=~` — treats the pattern as a regex and matches it against a runtime value + + #### Patterns + + * `boolean` — either `true` or `false`, which is stringified before comparison + * `string` — either a literal pattern or a regular expression, depending on the + supplied operator + + ## Runtime Updates + + Dynamic queues are persisted to the database, making it easy to manipulate them directly through + CRUD operations, or indirectly with Oban's queue operations, i.e. `pause_queue/2`, + `scale_queue/2`. + + Explicit queue options will be overwritten on restart, while omitted fields are retained. For + example, consider the following dynamic queue entry: + + queues: [ + default: [limit: 10], + ... + ] + + Scaling that `default` queue up or down at runtime _wouldn't_ persist across a restart, because + the definition will overwrite the `limit`. However, pausing the queue or changing the + `global_limit` _would_ persist because they aren't included in the queue definition. + + # pause, global_limit, etc. will persist, but limit won't + default: [limit: 10] + + # pause will persist, but limit and global_limit won't + default: [limit: 10, global_limit: 20] + + # neither limits nor pausing will persist + default: [limit: 10, global_limit: 20, paused: true] + + See function documentation for `all/0`, `insert/1`, `update/2`, and `delete/1` for more + information about runtime updates. + + ## Enabling Polling Mode + + In environments with restricted connectivity (where PubSub doesn't work) you can still use + DynamicQueues at runtime through polling mode. The polling interval is entirely up to you, as + it's disabled by default. + + config :my_app, Oban, + plugins: [{Oban.Pro.Plugins.DynamicQueues, interval: :timer.minutes(1)}] + + With the interval above each DynamicQueues instance will wake up every minute, check the + database for changes, and start new queues. + + ## Isolation and Namespacing + + All DynamicQueues functions have an alternate clause that accepts an Oban instance name for the + first argument. This matches base `Oban` functions such as `Oban.pause_queue/2`, which allow you + to seamlessly work with multiple Oban instances and across multiple database prefixes. For + example, you can use `all/1` to list all queues for the instance named `ObanPrivate`: + + queues = DynamicQueues.all(ObanPrivate) + + Likewise, to insert a new queue using the configuration associated with the `ObanPrivate` + instance: + + DynamicQueues.insert(ObanPrivate, private: limit: 10) + """ + + @behaviour Oban.Plugin + + use GenServer + + import Ecto.Query + + alias Ecto.Changeset + alias Oban.Pro.Engines.Smart + alias Oban.Pro.{Queue, Utils} + alias Oban.{Config, Midwife, Notifier, Peer, Registry, Repo, Validation} + + require Logger + + @type oban_name :: term() + @type operator :: :== | :!= | :=~ + @type partition :: Smart.partition() + @type pattern :: boolean() | String.t() + @type period :: Smart.period() + @type sync_mode :: :manual | :automatic + @type sys_key :: String.t() + + @type queue_name :: atom() | binary() + @type queue_input :: [{queue_name(), pos_integer() | queue_opts() | [queue_opts()]}] + @type queue_opts :: + {:local_limit, pos_integer()} + | {:global_limit, Smart.global_limit()} + | {:only, only()} + | {:paused, boolean()} + | {:rate_limit, Smart.rate_limit()} + + @type only :: + {:node, pattern()} + | {:node, operator(), pattern()} + | {:sys_env, sys_key(), pattern()} + | {:sys_env, sys_key(), operator(), pattern()} + + defstruct [ + :conf, + :foreman_ref, + :timer, + interval: :infinity, + queues: [], + restart_attempts: 50, + restart_delay: 10, + sync_mode: :manual + ] + + defguardp is_name(name) when is_binary(name) or (is_atom(name) and not is_nil(name)) + + @impl Oban.Plugin + def start_link(opts) do + {name, opts} = Keyword.pop(opts, :name) + + GenServer.start_link(__MODULE__, struct!(__MODULE__, opts), name: name) + end + + @impl Oban.Plugin + def validate(opts) do + Validation.validate_schema(opts, + conf: {:custom, &validate_engine/1}, + name: :any, + interval: :timeout, + queues: {:custom, &validate_queues/1}, + sync_mode: {:enum, [:manual, :automatic]} + ) + end + + @doc """ + Retrieve all persisted queues. + + While it's possible to modify queue's returned from `all/0`, it is recommended that you use + `update/2` to ensure options are cast and validated correctly. + + ## Examples + + Retrieve a list of all queue schemas with persisted attributes: + + DynamicQueues.all() + """ + @spec all(oban_name()) :: [Ecto.Schema.t()] + def all(oban_name \\ Oban) do + oban_name + |> Oban.config() + |> all_queues() + end + + @doc """ + Retrieve a single persisted queue. + + ## Examples + + Retrieve the default queue's schema with persisted attributes: + + DynamicQueues.get(:default) + """ + @spec get(oban_name(), queue_name()) :: nil | Ecto.Schema.t() + def get(oban_name \\ Oban, queue_name) do + oban_name + |> Oban.config() + |> Repo.get_by(Queue, name: to_string(queue_name)) + end + + @doc """ + Persist a list of queue inputs, exactly like the `:queues` option passed as configuration. + + Note that `insert/1` acts like an upsert, making it possible to modify queues if the name + matches. Still, it is better to use `update/2` to make targeted updates. + + ## Examples + + Insert a variety of queues with standard and advanced options: + + DynamicQueues.insert( + basic: 10, + audio: [global_limit: 10], + video: [global_limit: 10], + learn: [local_limit: 5, only: {:node, :=~, "learn"}] + ) + """ + @spec insert(oban_name(), queue_input()) :: + {:ok, [Ecto.Schema.t()]} | {:error, Ecto.Changeset.t()} + def insert(oban_name \\ Oban, [_ | _] = entries) do + conf = Oban.config(oban_name) + + case insert_queues(conf, entries) do + {:ok, changes} -> + inserted = Map.values(changes) + + Enum.each(inserted, ¬ify(conf, :dyn_start, &1.name)) + + {:ok, inserted} + + {:error, _name, changeset, _changes} -> + {:error, changeset} + end + end + + @doc """ + Modify a single queue's options. + + Every option available when inserting queues can be updated. + + ## Examples + + The following call demonstrates updating every possible option: + + DynamicQueues.update( + :video, + local_limit: 5, + global_limit: 20, + rate_limit: [allowed: 10, period: 30, partition: :worker]], + only: {:node, :=~, "media"}, + paused: false + ) + + Updating a single option won't remove other persisted options. If you'd like to + clear an uption you must set them to `nil`: + + DynamicQueues.update(:video, global_limit: nil) + + Since `update/2` operates on a single queue, it is possible to rename a queue + without doing a `delete`/`insert` dance: + + DynamicQueues.update(:video, name: :media) + """ + @spec update(oban_name(), queue_name(), [queue_opts()]) :: + {:ok, Ecto.Schema.t()} | {:error, Ecto.Changeset.t()} + def update(oban_name \\ Oban, name, opts) when is_name(name) do + conf = Oban.config(oban_name) + + with {:ok, queue} <- fetch_queue(conf, to_string(name)), + {:ok, queue} <- update_queue(conf, queue, opts) do + if queue.name == to_string(name) do + opts = + opts + |> Map.new() + |> Map.delete(:only) + |> Map.merge(%{action: :scale, ident: :any, queue: name, update: false}) + + # Manually send the notification because Oban.scale_queue won't allow unknown fields + Notifier.notify(conf.name, :signal, opts) + else + notify(conf, :dyn_start, queue.name) + notify(conf, :dyn_stop, name) + end + + {:ok, queue} + end + end + + defp update_queue(conf, queue, opts) do + params = + case Keyword.pop(opts, :name) do + {nil, opts} -> %{opts: Map.new(opts)} + {name, []} -> %{name: name} + {name, opts} -> %{name: name, opts: Map.new(opts)} + end + + changeset = Queue.changeset(queue, params) + + Repo.update(conf, changeset) + rescue + Ecto.StaleEntryError -> + update_queue(conf, Repo.reload!(conf, queue), opts) + end + + @doc """ + Delete a queue by name at runtime, rather than using the `:delete` option into the `queues` list + in your configuration. + + ## Examples + + Delete ethe "audio" queue: + + {:ok, _} = DynamicQueues.delete(:audio) + """ + @spec delete(oban_name(), queue_name()) :: {:ok, Ecto.Schema.t()} | {:error, Ecto.Changeset.t()} + def delete(oban_name \\ Oban, name) when is_name(name) do + conf = Oban.config(oban_name) + + with {:ok, queue} <- fetch_queue(conf, to_string(name)), + {:ok, queue} <- Repo.delete(conf, queue) do + notify(conf, :dyn_stop, queue.name) + + {:ok, queue} + end + end + + # Callbacks + + @impl GenServer + def init(state) do + :telemetry.execute([:oban, :plugin, :init], %{}, %{conf: state.conf, plugin: __MODULE__}) + + ref = + state.conf.name + |> Registry.whereis(Foreman) + |> Process.monitor() + + {:ok, %{state | foreman_ref: ref}, {:continue, :start}} + end + + @impl GenServer + def handle_continue(:start, state) do + :ok = Notifier.listen(state.conf.name, [:signal]) + + case manage_queues(state) do + {:ok, _inserted} -> + state.conf + |> all_queues() + |> Enum.reject(&queue_running?(state.conf, &1)) + |> Enum.each(&start_queue(state.conf, &1)) + + {:noreply, schedule_refresh(state)} + + {:error, _name, changeset, _changes} -> + {:stop, %ArgumentError{message: changeset.errors}, state} + end + end + + @impl GenServer + def handle_call(:flush, _from, state) do + {:reply, :ok, state} + end + + @impl GenServer + def handle_info({:DOWN, ref, :process, _pid, _reason}, %{foreman_ref: ref} = state) do + state = restart_queues(state, state.restart_attempts) + + {:noreply, state} + end + + def handle_info({:notification, :signal, payload}, state) do + case payload do + %{"action" => "scale", "ident" => "any", "queue" => queue_name} -> + scale_queue(state.conf, queue_name, payload) + + %{"action" => "pause", "ident" => "any", "queue" => queue_name} -> + scale_queue(state.conf, queue_name, %{"paused" => true}) + + %{"action" => "resume", "ident" => "any", "queue" => queue_name} -> + scale_queue(state.conf, queue_name, %{"paused" => false}) + + %{"action" => "dyn_start", "queue" => queue_name} -> + start_queue(state.conf, queue_name) + + %{"action" => "dyn_stop", "queue" => queue_name} -> + stop_queue(state.conf, queue_name) + + _ -> + :ignore + end + + {:noreply, state} + end + + def handle_info(:refresh, state) do + for queue <- all_queues(state.conf), not queue_running?(state.conf, queue) do + start_queue(state.conf, queue) + end + + {:noreply, schedule_refresh(state)} + end + + def handle_info(message, state) do + Logger.warning( + message: "Received unexpected message: #{inspect(message)}", + source: :oban_pro, + module: __MODULE__ + ) + + {:noreply, state} + end + + # Validations + + defp validate_engine(%Config{engine: engine}) do + if engine == Smart do + :ok + else + {:error, + """ + DynamicQueues requires the Smart engine to run correctly, but you're using: + + engine: #{inspect(engine)} + + You can either use the Smart engine or remove DynamicQueue from your plugins. + """} + end + end + + defp validate_queues(queues) do + Validation.validate(:queues, queues, &validate_queue/1) + end + + defp validate_queue({_name, delete: true}), do: :ok + + defp validate_queue({name, _} = tuple) when is_atom(name) do + changeset = Queue.changeset(tuple) + + if changeset.valid? do + :ok + else + exception = Utils.to_exception(changeset) + + {:error, "expected #{inspect(name)} to be a valid queue, but: #{exception.message}"} + end + end + + defp validate_queue(queue) do + {:error, "expected queue to be a keyword pair, got: #{inspect(queue)}"} + end + + # Queries + + defp all_queues(conf) do + Repo.all(conf, order_by(Queue, asc: :inserted_at)) + end + + defp manage_queues(%{sync_mode: :automatic} = state) do + Repo.transaction(state.conf, fn -> + delete_missing(state.conf, state.queues) + insert_queues(state.conf, state.queues) + end) + end + + defp manage_queues(state) do + Repo.transaction(state.conf, fn -> + {deletes, inserts} = + Enum.split_with(state.queues, fn + {_name, opts} when is_list(opts) -> Keyword.get(opts, :delete) + {_name, _lim} -> false + end) + + delete_queues(state.conf, deletes) + insert_queues(state.conf, inserts) + end) + end + + defp delete_missing(conf, queues) do + Repo.delete_all(conf, where(Queue, [q], q.name not in ^queue_names(queues))) + end + + defp delete_queues(conf, queues) do + Repo.delete_all(conf, where(Queue, [q], q.name in ^queue_names(queues))) + end + + defp queue_names(queues), do: Enum.map(queues, &to_string(elem(&1, 0))) + + defp insert_queues(conf, queues) do + changesets = + Map.new(queues, fn {name, opts} -> + changeset = Queue.changeset({name, opts}) + + {changeset.changes.name, {changeset, opts}} + end) + + Repo.transaction(conf, fn -> + existing = existing_map(conf, Map.keys(changesets)) + + Enum.reduce(changesets, %{}, fn {name, {changeset, opts}}, acc -> + cond do + is_map_key(existing, name) and Map.get(existing, name).hash == changeset.changes.hash -> + acc + + is_map_key(existing, name) -> + params = Queue.normalize({name, opts}) + + changeset = + existing + |> Map.get(name) + |> Queue.changeset(params) + + Map.put(acc, name, Repo.update!(conf, changeset)) + + true -> + Map.put(acc, name, Repo.insert!(conf, changeset)) + end + end) + end) + end + + defp existing_map(conf, names) do + query = where(Queue, [q], q.name in ^names) + + conf + |> Repo.all(query) + |> Map.new(&{&1.name, &1}) + end + + defp fetch_queue(conf, name) do + case Repo.one(conf, where(Queue, name: ^name)) do + nil -> {:error, "no queue named #{inspect(name)} could be found"} + queue -> {:ok, queue} + end + end + + # Scheduling + + defp schedule_refresh(%{interval: :infinity} = state), do: state + + defp schedule_refresh(state) do + %{state | timer: Process.send_after(self(), :refresh, state.interval)} + end + + # Notification + + defp notify(conf, action, queue_name) do + Notifier.notify(conf.name, :signal, %{action: action, ident: :any, queue: queue_name}) + end + + # Supervision + + defp restart_queues(state, 0) do + state + end + + defp restart_queues(state, attempt) do + case Registry.whereis(state.conf.name, Foreman) do + pid when is_pid(pid) -> + ref = Process.monitor(pid) + + state.conf + |> all_queues() + |> Enum.each(&start_queue(state.conf, &1)) + + %{state | foreman_ref: ref} + + nil -> + Process.sleep(state.restart_delay) + + restart_queues(state, attempt - 1) + end + end + + defp queue_running?(conf, queue) do + conf.name + |> Registry.whereis({:supervisor, queue.name}) + |> is_pid() + end + + defp scale_queue(_conf, _, %{"update" => false}), do: :ok + + defp scale_queue(conf, "*", %{"paused" => paused}) do + if Peer.leader?(conf) do + query = + from q in Queue, + update: [set: [opts: fragment("jsonb_set(?, '{paused}', ?)", q.opts, ^paused)]] + + Repo.update_all(conf, query, []) + end + end + + defp scale_queue(conf, queue_name, payload) do + with true <- Peer.leader?(conf), + {:ok, queue} <- fetch_queue(conf, queue_name) do + new_opts = Map.drop(payload, ["action", "ident", "queue"]) + + changeset = + queue + |> Queue.changeset(%{opts: new_opts}) + |> Changeset.delete_change(:hash) + + {:ok, _} = Repo.update(conf, changeset) + end + end + + defp start_queue(conf, queue_name) when is_binary(queue_name) do + with {:ok, queue} <- fetch_queue(conf, queue_name), do: start_queue(conf, queue) + end + + defp start_queue(conf, queue) do + if run_locally?(conf, queue.only) do + Midwife.start_queue(conf, Queue.to_keyword_opts(queue)) + else + :ignore + end + end + + defp stop_queue(conf, queue_name) when is_binary(queue_name) do + Midwife.stop_queue(conf, queue_name) + end + + defp run_locally?(%{node: node}, %{mode: :node, op: op, value: value}) do + compare(node, op, value) + end + + defp run_locally?(_conf, %{mode: :sys_env, op: op, key: key, value: value}) do + key + |> System.get_env() + |> to_string() + |> compare(op, value) + end + + defp run_locally?(_conf, _only), do: true + + defp compare(value_a, :==, value_b), do: value_a == value_b + defp compare(value_a, :!=, value_b), do: value_a != value_b + defp compare(value_a, :=~, value_b), do: value_a =~ Regex.compile!(value_b, "i") +end diff --git a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_scaler.ex b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_scaler.ex new file mode 100644 index 00000000..6f727d0a --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_scaler.ex @@ -0,0 +1,662 @@ +defmodule Oban.Pro.Plugins.DynamicScaler do + @moduledoc """ + The `DynamicScaler` examines queue throughput and issues commands to horizontally scale + cloud infrastructure to optimize processing. With auto-scaling you can spin up additional nodes + during high traffic events, and pare down to a single node during a lull. Beyond optimizing + throughput, scaling may save money in environments with little to no usage at off-peak times, + e.g. staging. + + Horizontal scaling is applied at the _node_ level, not the _queue_ level, so you can distribute + processing over more phyiscal hardware. + + * Predictive Scaling — The optimal scale is calculated by predicting the future size of a queue + based on recent trends. Multiple samples are then used to prevent overreacting to changes in + queue depth or throughput. Your provide an acceptible `range` of nodes and auto-scaling takes + care of the rest. + + * Multi-Cloud — Cloud integration is provided by a simple, flexible, behaviour that you + implement for your specific environment and configure for each scaler. + + * Queue Filtering — By default, all queues are considered for scale calculations. However, you + can restrict calculations to one or more ciritical queues. + + * Multiple Scalers — Some systems may restrict work to specific node types, e.g. generating + exports or processing videos. Other hybrid systems may straddle multiple clouds. In either + case, you can configure multiple independent scalers driven by distinct queues. + + * Non Linear — An optional `step` parameter allows you to conservatively scale up or down one + node at a time, or optimize for responsiveness and jump from the min to the max in a single + scaling period. + + * Prevent Thrashing — A `cooldown` period skips scaling when there was recent scale activity to + prevent unnecessarily scaling nodes up or down. Nodes may take several minutes to start within + an environment, so the default `cooldown` period is 2 minutes. + + ## Using and Configuring + + > #### Clouds {: .info} + > + > There are ample hosting platforms, aka "clouds", out there and we can't support them all! + > Before you can begin dynamic scaling you'll need to implement a `Cloud` module for your + > environment. Don't worry, we have [copy-and-paste examples](#module-cloud-modules) for some + > popular platforms and a guide to walk through [implementations for your + > environment](#module-writing-cloud-modules). + + With a `cloud` module in hand you're ready to add the `DynamicScaler` plugin to your Oban + config: + + config :my_app, Oban, + plugins: [ + {Oban.Pro.Plugins.DynamicScaler, ...} + ] + + Then, add a scaler with a `range` to define the minimum and maximum nodes, and your `cloud` + strategy: + + {DynamicScaler, scalers: [range: 1..5, cloud: MyApp.Cloud]} + + Now, every minute, `DynamicScaler` will calculate the optimal number of nodes for your queue's + throughput and issue scaling commands accordingly. + + ### Configuring Scalers + + Scalers have options beyond `:cloud` and `:range` for more advanced systems or to constrain + resource usage. Here's a breakdown of all options, followed by specific examples of each. + + * `:cloud` — A `module` or `{module, options}` tuple that interacts with the external cloud + during scale events. Required. + + * `:range` — The range of compute units to scale between. For example, `1..3` declares a minimum + of 1 node and a maximum of 3. The minimum must be 0 or more, and the maximum must be 1 or at + least match the minimum. Required. + + * `:cooldown` — The minimum time between scaling events. Defaults to 120 seconds. + + * `:lookback` — The historic time to check queues. Defaults to 60 seconds. + + * `:queues` — Either `:all` or a list of queues to consider when measuring throughput and + backlog. + + * `:step` — Either `:none` or the maximum nodes to scale up or down at once. Defaults to + `:none`. + + ### Scaler Examples + + Filter throughput queries to the `:media` queue: + + scalers: [queues: :media, range: 1..3, cloud: MyApp.Cloud] + + Filter throughput queries to both `:audio` and `:video` queues: + + scalers: [queues: [:audio, :video], range: 1..3, cloud: MyApp.Cloud] + + Configure scalers driven by different queues (note, queues _may not_ overlap): + + scalers: [ + [queues: :audio, range: 0..2, cloud: {MyApp.Cloud, asg: "my-audio-asg"}], + [queues: :video, range: 0..5, cloud: {MyApp.Cloud, asg: "my-video-asg"}] + ] + + Limit scaling to one node up or down at a time: + + scalers: [range: 1..3, step: 1, cloud: MyApp.Cloud] + + Wait at least 5 minutes (300 seconds) between scaling events: + + scalers: [range: 1..3, cloud: MyApp.Cloud, cooldown: 300] + + Increase the period used to calculate historic throughput to 90 seconds: + + scalers: [range: 1..3, cloud: MyApp.Cloud, lookback: 90] + + > ### Scaling Down to Zero Nodes {: .warning} + > + > It's possible to scale down to zero nodes in staging environments or production applications + > with periods of downtime. However, it is **only viable for multi-node setups** with dedicated + > worker nodes and another instance type that isn't controlled by `DynamicScaler`. Without a + > separate "web" node, or something that is always running, you run the risk of scaling down + > without the ability to scale back up. + + ## Cloud Modules + + There are a lot of hosting platforms, aka "clouds" out there and we can't support them all! Even + with optional dependencies, it would be a mess of libraries that may not agree with your + application decisions. Instead, the `Oban.Pro.Cloud` behaviour defines two simple callbacks, and + integrating with platforms typically takes a single HTTP query or library call. + + The following links contain gists of full implementations for popular cloud platforms. Feel free + to copy-and-paste to use them as-is or as the basis for your own cloud modules. + + * [EC2](https://gist.github.com/sorentwo/ba4a07d3a011d212c19a5bb775a6c536) + * [Fly](https://gist.github.com/sorentwo/d6be222091db7ba3c5b50d8bcabca252) + * [GCP](https://gist.github.com/sorentwo/a54a1e2b37123fd627cca16f07aed951) + * [Gigalixir](https://gist.github.com/sorentwo/90b4a427819756f7ceb72254ad53d9bb) + * [Heroku](https://gist.github.com/sorentwo/54d99fb2ac05cb63ea1e30aa1935b6fc) + * [Kubernetes](https://gist.github.com/sorentwo/f5ba9048e1e91456d37a8c7d4b8e4d58) + + Let us know if an integration for your platform is missing (which is rather likely) and you'd + like assistance. Otherwise, follow the guide below to write your own integration! + + ### Writing Cloud Modules + + Cloud callback modules must define an `init/1` function to prepare configuration at runtime, and + a `scale/2` callback called with the desired number of nodes and the prepared configuration. + + The following example demonstrates a complete callback module for scaling EC2 Auto Scaling + Groups on AWS using the [SetDesiredCapacity][sdc] action. It assumes you're using the + [ex_aws][exa] package with the proper credentials. + + defmodule MyApp.ASG do + @behaviour Oban.Pro.Cloud + + @impl Oban.Pro.Cloud + def init(opts), do: Map.new(opts) + + @impl Oban.Pro.Cloud + def scale(desired, conf) do + params = %{ + "Action" => "SetDesiredCapacity", + "AutoScalingGroupName" => conf.asg, + "DesiredCapacity" => desired, + "Version" => "2011-01-01" + } + + query = %ExAws.Operation.Query{ + path: "", + params: params, + service: :autoscaling, + action: :set_desired_capacity + } + + with {:ok, _} <- ExAws.request(query), do: {:ok, conf} + end + end + + You'd then use your cloud module as a scaler option: + + {DynamicScaler, scalers: [range: 1..3, cloud: {MyApp.ASG, asg: "my-asg-name"}]} + + Clouds can also pull from the application or system environment to build configuration. If your + module pulls from the environment exclusively, then you can pass the module name rather than a + tuple: + + {DynamicScaler, scalers: [range: 1..3, cloud: MyApp.ASG]} + + [sdc]: https://docs.aws.amazon.com/autoscaling/ec2/APIReference/API_SetDesiredCapacity.html + [exa]: https://github.com/ex-aws/ex_aws + + ### Optimizing Throughput Queries + + While the scaler's throughput queries are optimized for a standard load, high throughput queues, + or systems that retain a large volume of jobs, may benefit from an additional index that aids + calculating throughput. Use the following migration to add an index if you find that scaling + queries are too slow or timing out: + + @disable_ddl_transaction true + @disable_migration_lock true + + def change do + create_if_not_exists index( + :oban_jobs, + [:state, :queue, :attempted_at, :attempted_by], + concurrently: true, + where: "attempted_at IS NOT NULL", + prefix: "public" + ) + end + + Alternatively, you can change the timeout used for scaler inspection queries: + + {DynamicScaler, timeout: :timer.seconds(15), scalers: ...} + + ## Instrumenting with Telemetry + + The `DynamicScaler` plugin adds the following metadata to the `[:oban, :plugin, :stop]` event: + + * `:scaler` - details of the active scaler config with recent scaling values + * `:skipped` - the reason scaling is skipped, either `:recently_scaled` or `:already_scaled` + * `:error` — the reason returned from `scale/2` when scaling fails + + When multiple `scalers` are configured one event is emitted for _each_ scaler. + """ + + @behaviour Oban.Plugin + + use GenServer + + import DateTime, only: [utc_now: 0] + import Ecto.Query + + alias __MODULE__, as: State + alias Oban.Plugins.Cron + alias Oban.{Job, Notifier, Peer, Repo, Validation} + + require Logger + + defmodule Scaler do + @moduledoc false + + @enforce_keys [:cloud, :range] + defstruct [ + :cloud, + :last_rate, + :last_scaled_at, + :last_scaled_to, + :last_size, + :range, + cooldown: 120, + lookback: 60, + queues: :all, + step: :none + ] + + def new(opts) do + opts + |> Keyword.update(:cloud, nil, &init_cloud/1) + |> Keyword.update(:queues, :all, &normalize_queues/1) + |> then(&struct!(__MODULE__, &1)) + end + + defp init_cloud(cmod) when is_atom(cmod), do: {cmod, cmod.init([])} + defp init_cloud({cmod, copt}), do: {cmod, cmod.init(copt)} + + defp normalize_queues(:all), do: :all + + defp normalize_queues(queues) do + queues + |> List.wrap() + |> Enum.map(&to_string/1) + end + end + + defstruct [:conf, :name, :timer, scalers: [], timeout: :timer.seconds(15)] + + defmacrop attempted_by_node(column, position) do + quote do + fragment("?[?]", unquote(column), unquote(position)) + end + end + + @doc false + def child_spec(args), do: super(args) + + @impl Oban.Plugin + def start_link(opts) do + opts = + Keyword.update!(opts, :scalers, fn scalers -> + scalers + |> wrap_keyword() + |> Enum.map(&Scaler.new/1) + end) + + state = struct!(State, opts) + + GenServer.start_link(__MODULE__, state, name: state.name) + end + + defp wrap_keyword([elem | _] = list) when is_tuple(elem), do: [list] + defp wrap_keyword(list), do: list + + @impl Oban.Plugin + def validate(opts) do + Validation.validate_schema(opts, + conf: :ok, + name: :ok, + scalers: {:custom, &validate_scalers(wrap_keyword(&1))}, + timeout: :timeout + ) + end + + @impl Oban.Plugin + def format_logger_output(_conf, %{skipped: reason}), do: %{skipped: reason} + + def format_logger_output(_conf, %{error: reason}), do: %{error: inspect(reason)} + + def format_logger_output(_conf, %{scaler: scaler}) do + cloud = + case scaler.cloud do + {module, _opts} -> module + module -> module + end + + Map.from_struct(%{scaler | cloud: inspect(cloud), range: inspect(scaler.range)}) + end + + @doc false + def optimal_scale(size, rate, min..max//_) do + cond do + rate == 0 and size > 0 and min == 0 -> + 1 + + size == 0 and rate > 0 and min == 0 -> + 1 + + rate == 0 -> + min + + true -> + optimal = size / rate + + optimal + |> ceil() + |> max(min) + |> min(max) + end + end + + @doc false + def clamped_scale(size, _last, _range, :none), do: size + def clamped_scale(size, size, _range, _step), do: size + + def clamped_scale(size, last, _min..max//_, step) when size > last do + (last + step) + |> min(size) + |> min(max) + end + + def clamped_scale(size, last, min.._max//_, step) when size < last do + (last - step) + |> max(size) + |> max(min) + end + + # Callbacks + + @impl GenServer + def init(state) do + :telemetry.execute([:oban, :plugin, :init], %{}, %{conf: state.conf, plugin: __MODULE__}) + + {:ok, state, {:continue, :start}} + end + + @impl GenServer + def handle_continue(:start, state) do + :ok = Notifier.listen(state.conf.name, [:scaler]) + + {:noreply, schedule_scaling(state)} + end + + @impl GenServer + def handle_call(:check_scale, _from, %State{} = state) do + {:reply, :ok, check_and_scale(state)} + end + + @impl GenServer + def handle_info(:check_scale, %State{} = state) do + state = + if Peer.leader?(state.conf) do + check_and_scale(state) + else + state + end + + {:noreply, schedule_scaling(state)} + end + + def handle_info({:notification, :scaler, payload}, state) do + {queues, payload} = Map.pop!(payload, "queues") + + if Peer.leader?(state.conf) do + {:noreply, state} + else + scalers = + Enum.map(state.scalers, fn scaler -> + if queues == to_scaler_id(scaler.queues) do + Enum.reduce(payload, scaler, fn + {"last_rate", val}, acc -> + %{acc | last_rate: val} + + {"last_size", val}, acc -> + %{acc | last_size: val} + + {"last_scaled_to", val}, acc -> + %{acc | last_scaled_to: val} + + {"last_scaled_at", nil}, acc -> + %{acc | last_scaled_at: nil} + + {"last_scaled_at", val}, acc -> + {:ok, at, _offset} = DateTime.from_iso8601(val) + + %{acc | last_scaled_at: at} + end) + else + scaler + end + end) + + {:noreply, %{state | scalers: scalers}} + end + end + + def handle_info(message, state) do + Logger.warning( + message: "Received unexpected message: #{inspect(message)}", + source: :oban_pro, + module: __MODULE__ + ) + + {:noreply, state} + end + + defp to_scaler_id(:all), do: "all" + defp to_scaler_id(queues) when is_list(queues), do: Enum.map(queues, &to_string/1) + + # Validations + + defp validate_scalers([]), do: :ok + + defp validate_scalers([head | tail]) do + with :ok <- Validation.validate(:scalers, head, &validate_scaler/1) do + validate_scalers(tail) + end + end + + defp validate_scalers(scalers) do + {:error, "expected :scaler to be a list of keywords, got: #{inspect(scalers)}"} + end + + defp validate_scaler({:cloud, cloud}) do + case cloud do + {cmod, copt} when is_atom(cmod) and is_list(copt) -> + :ok + + cmod when is_atom(cmod) -> + :ok + + _ -> + {:error, + "expected :cloud to be a module or {module, options} tuple, got: #{inspect(cloud)}"} + end + end + + defp validate_scaler({:cooldown, cooldown}) do + Validation.validate_integer(:cooldown, cooldown, min: 0) + end + + defp validate_scaler({:lookback, cooldown}) do + Validation.validate_integer(:lookback, cooldown) + end + + defp validate_scaler({:queues, queues}) do + case queues do + :all -> + :ok + + queue when is_atom(queue) and not is_nil(queue) -> + :ok + + [queue | _] when is_atom(queue) or is_binary(queue) -> + :ok + + _ -> + {:error, "expected :queues to be :all or a list of queue names, got: #{inspect(queues)}"} + end + end + + defp validate_scaler({:range, range}) do + case range do + min..max//_ when min <= max and max > 0 -> + :ok + + _min.._max//_ -> + {:error, "expected :range to have a min less than max, got: #{inspect(range)}"} + + _ -> + {:error, "exepcted :range to be a range of integers, got: #{inspect(range)}"} + end + end + + defp validate_scaler({:step, :none}), do: :ok + + defp validate_scaler({:step, limit}) do + Validation.validate_integer(:step, limit) + end + + # Necessary to seed testing, not documented + defp validate_scaler({:last_rate, _}), do: :ok + defp validate_scaler({:last_scaled_at, _}), do: :ok + defp validate_scaler({:last_scaled_to, _}), do: :ok + defp validate_scaler({:last_size, _}), do: :ok + + defp validate_scaler(option), do: {:unknown, option, Scaler} + + # Scheduling + + defp schedule_scaling(state) do + timer = Process.send_after(self(), :check_scale, Cron.interval_to_next_minute()) + + %{state | timer: timer} + end + + # Scaling + + defp check_and_scale(%State{scalers: scalers} = state) do + scalers = + for scaler <- scalers do + meta = %{conf: state.conf, plugin: __MODULE__, scaler: scaler} + + :telemetry.span([:oban, :plugin], meta, fn -> + prev_size = scaler.last_size + + scaler = + scaler + |> record_size(state) + |> record_rate(state) + + case apply_scale(scaler, prev_size) do + {:ok, scaler} -> + notify_scalers(scaler, state) + + {scaler, Map.put(meta, :scaler, scaler)} + + {:skipped, reason} -> + notify_scalers(scaler, state) + + {scaler, Map.put(meta, :skipped, reason)} + + {:error, reason} -> + {scaler, Map.put(meta, :error, reason)} + end + end) + end + + %{state | scalers: scalers} + end + + defp notify_scalers(scaler, %{conf: conf}) do + payload = Map.take(scaler, ~w(last_rate last_scaled_at last_scaled_to last_size queues)a) + + Notifier.notify(conf, :scaler, payload) + end + + defp record_size(scaler, state) do + next = DateTime.add(utc_now(), scaler.lookback, :second) + + query = + scaler.queues + |> base_query() + |> where([j], j.state in ~w(available scheduled retryable) and j.scheduled_at <= ^next) + |> select(count()) + + %{scaler | last_size: Repo.one(state.conf, query, timeout: state.timeout)} + end + + defp record_rate(scaler, state) do + since = DateTime.add(utc_now(), -scaler.lookback, :second) + + query = + scaler.queues + |> base_query() + |> where([j], j.state in ~w(executing retryable completed cancelled discarded)) + |> where([j], j.attempted_at > ^since) + |> group_by([j], attempted_by_node(j.attempted_by, 1)) + |> select([j], {attempted_by_node(j.attempted_by, 1), count()}) + + case Repo.all(state.conf, query, timeout: state.timeout) do + [] -> + %{scaler | last_rate: 0, last_scaled_to: scaler.last_scaled_to || 0} + + node_counts -> + size = length(node_counts) + + rate = + node_counts + |> Enum.map(&elem(&1, 1)) + |> Enum.sum() + |> div(size) + + %{scaler | last_rate: rate, last_scaled_to: scaler.last_scaled_to || size} + end + end + + defp base_query(queues) do + if queues == :all do + where(Job, [j], not is_nil(j.queue)) + else + where(Job, [j], j.queue in ^queues) + end + end + + defp apply_scale(scaler, prev_size) do + scale_to = + prev_size + |> next_size(scaler.last_size) + |> optimal_scale(scaler.last_rate, scaler.range) + |> clamped_scale(scaler.last_scaled_to, scaler.range, scaler.step) + + cond do + recently_scaled?(scaler) -> + {:skipped, :recently_scaled} + + already_scaled?(scale_to, scaler) -> + {:skipped, :already_scaled} + + true -> + {cloud_mod, cloud_opt} = scaler.cloud + + case cloud_mod.scale(scale_to, cloud_opt) do + {:ok, cloud_opt} -> + cloud = {cloud_mod, cloud_opt} + + {:ok, %{scaler | cloud: cloud, last_scaled_at: utc_now(), last_scaled_to: scale_to}} + + {:error, reason} -> + {:error, reason} + end + end + end + + defp next_size(nil, size), do: size + defp next_size(prev, curr), do: max(curr + (curr - prev), 0) + + defp recently_scaled?(%{last_scaled_at: nil}), do: false + + defp recently_scaled?(%{cooldown: cooldown, last_scaled_at: scaled_at}) do + scaling_allowed_at = DateTime.add(scaled_at, cooldown) + + DateTime.compare(utc_now(), scaling_allowed_at) != :gt + end + + defp already_scaled?(scale_to, %{last_scaled_to: to}), do: scale_to == to +end diff --git a/vendor/oban_pro/lib/oban/pro/producer.ex b/vendor/oban_pro/lib/oban/pro/producer.ex new file mode 100644 index 00000000..d7f9b7ca --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/producer.ex @@ -0,0 +1,358 @@ +require Protocol + +defmodule Oban.Pro.Producer do + @moduledoc false + + use Ecto.Schema + + import Ecto.Changeset + + alias Ecto.Changeset + alias Oban.Pro.Utils + + defmodule ListOrMap do + @moduledoc false + + # This is a custom type to allow a graceful transition from the legacy rate limit windows + # structure to the new, flattened map. + + @behaviour Ecto.Type + + @impl true + def type, do: :any + + @impl true + def cast(data) when is_map(data), do: {:ok, data} + def cast([data | _]), do: {:ok, data} + def cast(_data), do: :error + + @impl true + def load(data) when is_map(data), do: {:ok, data} + def load([data | _]), do: {:ok, data} + def load([]), do: {:ok, %{}} + def load(_data), do: :error + + @impl true + def dump(data) when is_map(data), do: {:ok, data} + def dump([data | _]), do: {:ok, data} + def dump([]), do: {:ok, %{}} + def dump(_data), do: :error + + @impl true + def equal?(a, b), do: a == b + + @impl true + def embed_as(_data), do: :dump + end + + @type t :: %__MODULE__{ + uuid: Ecto.UUID.t(), + name: binary(), + node: binary(), + queue: binary(), + meta: map(), + started_at: DateTime.t(), + updated_at: DateTime.t() + } + + @type meta :: %__MODULE__.Meta{local_limit: pos_integer(), paused: boolean()} + + @primary_key {:uuid, :binary_id, autogenerate: false} + schema "oban_producers" do + field :name, :string + field :node, :string + field :queue, :string + field :started_at, :utc_datetime_usec + field :updated_at, :utc_datetime_usec + + field :ack_async, :boolean, virtual: true, default: true + field :ack_tab, :any, virtual: true + field :refresh_interval, :integer, virtual: true, default: :timer.seconds(30) + field :xact_delay, :integer, default: :timer.seconds(1), virtual: true + field :xact_retry, :integer, default: 5, virtual: true + field :xact_timeout, :integer, default: :timer.seconds(30), virtual: true + + embeds_one :meta, Meta, on_replace: :update, primary_key: false do + @moduledoc false + + field :local_limit, :integer, default: 10 + field :paused, :boolean, default: false + field :shutdown_started_at, :utc_datetime_usec + + embeds_one :global_limit, GlobalLimit, on_replace: :update, primary_key: false do + @moduledoc false + + field :allowed, :integer + field :burst, :boolean, default: false + field :tracked, ListOrMap, default: %{} + + embeds_one :partition, Partition, on_replace: :update, primary_key: false do + @moduledoc false + + field :fields, {:array, Ecto.Enum}, values: [:args, :meta, :worker] + field :keys, {:array, :string} + end + end + + embeds_one :rate_limit, RateLimit, on_replace: :update, primary_key: false do + @moduledoc false + + field :allowed, :integer + field :period, :integer + field :window_time, :integer, skip_default_validation: true + field :windows, ListOrMap, default: %{} + + embeds_one :partition, Partition, on_replace: :update, primary_key: false do + @moduledoc false + + field :fields, {:array, Ecto.Enum}, values: [:args, :meta, :worker] + field :keys, {:array, :string} + end + end + end + end + + @spec new(map() | Keyword.t()) :: Changeset.t(t()) + def new(params) when is_list(params) or is_map(params) do + params = + params + |> Map.new() + |> Map.put_new_lazy(:uuid, &Oban.Pro.UUIDv7.generate/0) + |> Map.update!(:name, &to_clean_string/1) + |> Map.update!(:node, &to_clean_string/1) + |> Map.update!(:queue, &to_clean_string/1) + + %__MODULE__{} + |> cast(params, ~w(name node queue refresh_interval started_at updated_at uuid)a) + |> cast_embed(:meta, required: true, with: &meta_changeset/2) + |> validate_required(~w(name node queue)a) + |> validate_length(:name, min: 1) + |> validate_length(:node, min: 1) + |> validate_length(:queue, min: 1) + |> Utils.enforce_keys(params, __MODULE__) + end + + @spec update_meta(t(), atom(), term()) :: Changeset.t() + def update_meta(schema, key, value) do + update_meta(schema, %{key => value}) + end + + @spec update_meta(t(), map()) :: Changeset.t() + def update_meta(schema, changes) do + meta = + schema.meta + |> meta_changeset(changes) + |> apply_changes() + |> Ecto.embedded_dump(:json) + + schema + |> cast(%{meta: meta}, []) + |> cast_embed(:meta, with: &meta_changeset/2) + end + + @allowed ~w(local_limit paused shutdown_started_at)a + + @doc false + def meta_changeset(schema, params, allowed \\ @allowed) do + params = + params + |> Map.new() + |> Map.delete(:limit) + |> Map.put_new_lazy(:local_limit, fn -> default_local_limit(params, schema) end) + + params = + if Map.get(params, :global_limit) do + Map.update!(params, :global_limit, &cast_global_limit/1) + else + params + end + + params = + if Map.get(params, :rate_limit) do + Map.update!(params, :rate_limit, &cast_rate_limit/1) + else + params + end + + schema + |> cast(params, allowed) + |> cast_embed(:global_limit, with: &global_changeset/2) + |> cast_embed(:rate_limit, with: &rate_changeset/2) + |> validate_required(~w(local_limit)a) + |> validate_number(:local_limit, greater_than: 0) + |> validate_single_partitioner() + |> Utils.enforce_keys(params, schema.__struct__) + end + + @doc false + @spec default_local_limit(map(), Ecto.Schema.t()) :: non_neg_integer() + def default_local_limit(params, schema) do + cond do + is_integer(params[:limit]) -> + params[:limit] + + is_integer(params[:global_limit]) -> + params[:global_limit] + + is_integer(get_in(params, [:global_limit, :allowed])) -> + get_in(params, [:global_limit, :allowed]) + + true -> + schema.local_limit + end + end + + @doc false + @spec validate_single_partitioner(Changeset.t()) :: Changeset.t() + def validate_single_partitioner(changeset) do + case {get_field(changeset, :global_limit), get_field(changeset, :rate_limit)} do + {%{partition: %{}}, %{partition: %{}}} -> + add_error(changeset, :global_limit, "only one limiter may have partitioning") + + _ -> + changeset + end + end + + @doc false + @spec global_changeset(Ecto.Schema.t(), integer() | map() | Keyword.t()) :: Changeset.t() + def global_changeset(schema, params) do + params = Map.update(params, :partition, nil, &normalize_partition/1) + + schema + |> cast(params, ~w(allowed burst)a) + |> cast_embed(:partition, with: &partition_changeset/2) + |> validate_number(:allowed, greater_than: 0) + |> Utils.enforce_keys(params, schema.__struct__) + end + + @doc false + @spec rate_changeset(Ecto.Schema.t(), map() | Keyword.t()) :: Changeset.t() + def rate_changeset(schema, params) do + params = + params + |> Map.update(:period, nil, &Utils.cast_period/1) + |> Map.update(:partition, nil, &normalize_partition/1) + |> Map.put_new_lazy(:window_time, fn -> DateTime.to_unix(DateTime.utc_now(), :second) end) + + schema + |> cast(params, ~w(allowed period window_time)a) + |> cast_embed(:partition, with: &partition_changeset/2) + |> validate_required(~w(allowed period)a) + |> validate_number(:allowed, greater_than: 0) + |> validate_number(:period, greater_than: 0) + |> Utils.enforce_keys(params, schema.__struct__) + end + + @doc false + @spec partition_changeset(Ecto.Schema.t(), map() | Keyword.t()) :: Changeset.t() + def partition_changeset(schema, params) do + params = + params + |> Map.new() + |> Map.update(:keys, [], &for(key <- &1, do: to_string(key))) + + schema + |> cast(params, ~w(fields keys)a) + |> validate_required(~w(fields)a) + |> Utils.enforce_keys(params, schema.__struct__) + end + + # Global Limit Helpers + + @doc false + @spec cast_global_limit(pos_integer() | list() | map()) :: map() + def cast_global_limit(limit) when is_integer(limit) do + %{allowed: limit} + end + + def cast_global_limit(opts), do: cast_rate_limit(opts) + + # Rate Limit Helpers + + @doc false + @spec cast_rate_limit(list() | map()) :: map() + def cast_rate_limit([[key | _] | _] = opts) when is_binary(key) do + opts + |> Enum.map(&List.to_tuple/1) + |> Map.new() + |> cast_rate_limit() + end + + def cast_rate_limit(opts) + when is_map_key(opts, "allowed") or + is_map_key(opts, "burst") or + is_map_key(opts, "period") or + is_map_key(opts, "partition") do + Map.new(opts, fn + {"allowed", allowed} -> + {:allowed, allowed} + + {"burst", allowed} -> + {:burst, allowed} + + {"period", [time, unit]} -> + {:period, {time, unit}} + + {"period", period} -> + {:period, period} + + {"partition", [["fields", fields]]} -> + {:partition, fields: fields} + + {"partition", [["fields", fields], ["keys", keys]]} -> + {:partition, fields: fields, keys: keys} + + {"partition", field} when is_binary(field) -> + {:partition, fields: [field]} + end) + end + + def cast_rate_limit(opts), do: opts + + def normalize_partition(partition) do + cond do + is_nil(partition) -> + nil + + is_map(partition) -> + partition + + Keyword.keyword?(partition) and Keyword.has_key?(partition, :fields) -> + partition + + true -> + partition + |> List.wrap() + |> Enum.reduce([fields: [], keys: []], fn + {field, keys}, opts when field in ~w(args meta)a -> + opts + |> Keyword.put(:keys, List.wrap(keys)) + |> Keyword.update!(:fields, &[field | &1]) + + field, opts -> + Keyword.update!(opts, :fields, &[field | &1]) + end) + end + end + + # Helpers + + defp to_clean_string(value) when is_reference(value) do + inspect(value) + end + + defp to_clean_string(value) do + value + |> to_string() + |> String.trim_leading("Elixir.") + end +end + +encoder = if Code.ensure_loaded?(JSON.Encoder), do: JSON.Encoder, else: Jason.Encoder + +Protocol.derive(encoder, Oban.Pro.Producer, except: [:__meta__]) +Protocol.derive(encoder, Oban.Pro.Producer.Meta.GlobalLimit) +Protocol.derive(encoder, Oban.Pro.Producer.Meta.GlobalLimit.Partition) +Protocol.derive(encoder, Oban.Pro.Producer.Meta.RateLimit) +Protocol.derive(encoder, Oban.Pro.Producer.Meta.RateLimit.Partition) diff --git a/vendor/oban_pro/lib/oban/pro/queue.ex b/vendor/oban_pro/lib/oban/pro/queue.ex new file mode 100644 index 00000000..21c976f8 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/queue.ex @@ -0,0 +1,157 @@ +defmodule Oban.Pro.Queue do + @moduledoc false + + use Ecto.Schema + + import Ecto.Changeset + + alias Oban.Pro.{Producer, Utils} + + @allowed ~w(ack_async local_limit paused refresh_interval)a + + @primary_key {:name, :string, autogenerate: false} + schema "oban_queues" do + field :lock_version, :integer, default: 1 + field :hash, :string + + embeds_one :only, Only, on_replace: :update, primary_key: false do + @moduledoc false + + field :mode, Ecto.Enum, values: [:node, :sys_env] + field :op, Ecto.Enum, values: [:==, :!=, :=~] + field :key, :string + field :value, :string + end + + embeds_one :opts, Opts, on_replace: :update, primary_key: false do + @moduledoc false + + field :ack_async, :boolean + field :local_limit, :integer + field :paused, :boolean + field :refresh_interval, :integer + field :xact_delay, :integer + field :xact_retry, :integer + + embeds_one :global_limit, GlobalLimit, on_replace: :update, primary_key: false do + @moduledoc false + + field :allowed, :integer + field :burst, :boolean, default: false + + embeds_one :partition, Partition, on_replace: :update, primary_key: false do + @moduledoc false + + field :fields, {:array, Ecto.Enum}, values: [:args, :meta, :worker] + field :keys, {:array, :string} + end + end + + embeds_one :rate_limit, RateLimit, on_replace: :update, primary_key: false do + @moduledoc false + + field :allowed, :integer + field :period, :integer + field :window_time, :integer, virtual: true, skip_default_validation: true + + embeds_one :partition, Partition, on_replace: :update, primary_key: false do + @moduledoc false + + field :fields, {:array, Ecto.Enum}, values: [:args, :meta, :worker] + field :keys, {:array, :string} + end + end + end + + timestamps(inserted_at: :inserted_at, updated_at: :updated_at, type: :utc_datetime_usec) + end + + def normalize(params) do + case params do + [_ | _] -> Map.new(params) + {name, [_ | _] = opts} -> %{name: name, opts: Map.new(opts)} + {name, limit} -> %{name: name, opts: %{local_limit: limit}} + end + end + + def changeset(params) do + changeset(%__MODULE__{}, normalize(params)) + end + + def changeset(schema, params) do + params = + params + |> Map.replace_lazy(:name, fn _ -> to_string(params[:name]) end) + |> extract_only() + + schema + |> cast(params, [:name]) + |> cast_embed(:only, with: &only_changeset/2) + |> cast_embed(:opts, required: true, with: &opts_changeset/2) + |> validate_required([:name]) + |> validate_length(:name, min: 1) + |> optimistic_lock(:lock_version) + |> inject_hash(params) + |> Utils.enforce_keys(params, __MODULE__) + end + + defp extract_only(params) do + case params do + %{opts: %{only: _}} -> + {only, params} = pop_in(params, [:opts, :only]) + + Map.put(params, :only, cast_only(only)) + + _ -> + params + end + end + + defp only_changeset(schema, params) do + schema + |> cast(params, ~w(mode op key value)a) + |> validate_required(~w(mode op value)a) + |> Utils.enforce_keys(params, __MODULE__.Only) + end + + defp opts_changeset(schema, params) do + params = Map.new(params, fn {key, val} -> {Utils.maybe_to_atom(key), val} end) + + Producer.meta_changeset(schema, params, @allowed) + end + + defp inject_hash(changeset, params) do + hash = + params + |> Map.take(~w(only opts)a) + |> Enum.map_join(&:erlang.phash2/1) + |> Utils.hash64() + + put_change(changeset, :hash, hash) + end + + @spec to_keyword_opts(%{opts: map()} | map()) :: Keyword.t() + def to_keyword_opts(%__MODULE__{name: queue, opts: opts}) do + opts + |> Ecto.embedded_dump(:json) + |> Map.put(:queue, queue) + |> to_keyword_opts() + end + + def to_keyword_opts(opts) do + for {key, val} <- opts, not is_nil(val), do: {Utils.maybe_to_atom(key), val} + end + + # Helpers + + defp cast_only({:node, value}), do: cast_only({:node, :==, value}) + defp cast_only({:node, op, value}), do: %{mode: :node, op: op, value: to_string(value)} + + defp cast_only({:sys_env, key, value}), do: cast_only({:sys_env, key, :==, value}) + + defp cast_only({:sys_env, key, op, value}) do + %{mode: :sys_env, op: op, key: key, value: to_string(value)} + end + + defp cast_only(other), do: other +end diff --git a/vendor/oban_pro/lib/oban/pro/refresher.ex b/vendor/oban_pro/lib/oban/pro/refresher.ex new file mode 100644 index 00000000..d8125143 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/refresher.ex @@ -0,0 +1,111 @@ +defmodule Oban.Pro.Refresher do + @moduledoc false + + # Centralized refreshing and cleaning of producer records across all Oban instances. + # + # The Refresher runs as a single process supervised by the Oban.Pro application. It refreshes + # producer records for all actively running queues on the current node. + + use GenServer + + import Ecto.Query, only: [where: 3] + + alias __MODULE__, as: State + alias Oban.Pro.Producer + alias Oban.{Peer, Repo} + + defstruct [ + :timer, + interval: :timer.seconds(15), + producers: %{}, + producer_ttl: :timer.minutes(1) + ] + + @doc false + def start_link(opts \\ []) do + state = struct!(State, opts) + + GenServer.start_link(__MODULE__, state, name: __MODULE__) + end + + # GenServer Callbacks + + @impl GenServer + def init(state) do + Process.flag(:trap_exit, true) + + {:ok, schedule_refresh(state)} + end + + @impl GenServer + def terminate(_reason, state) do + if is_reference(state.timer), do: Process.cancel_timer(state.timer) + + :ok + end + + @impl GenServer + def handle_call(:refresh, _from, state) do + {:noreply, state} = handle_info(:refresh, state) + + {:reply, :ok, state} + end + + @impl GenServer + def handle_info(:refresh, state) do + refresh_producers(state) + cleanup_producers(state) + + {:noreply, schedule_refresh(state)} + end + + # Helpers + + defp schedule_refresh(state) do + timer = Process.send_after(self(), :refresh, state.interval) + + %{state | timer: timer} + end + + defp refresh_producers(_state) do + match = [{{{:"$1", {:producer, :"$2"}}, :_, :_}, [], [[:"$1", :"$2"]]}] + + Oban.Registry + |> Registry.select(match) + |> Enum.group_by(&hd/1, &Enum.at(&1, 1)) + |> Enum.each(fn {oban_name, queues} -> + with {_pid, %{testing: :disabled} = conf} <- Oban.Registry.lookup(oban_name) do + now = DateTime.utc_now() + ids = Enum.flat_map(queues, &lookup_uuid(&1, oban_name)) + + refresh_query = where(Producer, [p], p.uuid in ^ids) + + Repo.update_all(conf, refresh_query, set: [updated_at: now]) + end + end) + end + + defp cleanup_producers(state) do + match = [{{:"$1", :_, :"$2"}, [{:not, {:is_tuple, :"$1"}}], [:"$2"]}] + + Oban.Registry + |> Registry.select(match) + |> Enum.filter(&Peer.leader?/1) + |> Enum.each(fn conf -> + now = DateTime.utc_now() + outdated_at = DateTime.add(now, -state.producer_ttl, :millisecond) + cleanup_query = where(Producer, [p], p.updated_at <= ^outdated_at) + + Repo.delete_all(conf, cleanup_query) + end) + end + + defp lookup_uuid(queue, oban_name) do + prod_name = {oban_name, {:producer, queue}} + + case Registry.meta(Oban.Registry, prod_name) do + {:ok, %{uuid: uuid}} -> [uuid] + _error -> [] + end + end +end diff --git a/vendor/oban_pro/lib/oban/pro/relay.ex b/vendor/oban_pro/lib/oban/pro/relay.ex new file mode 100644 index 00000000..21a2f574 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/relay.ex @@ -0,0 +1,408 @@ +defmodule Oban.Pro.Relay do + @moduledoc """ + The `Relay` extension lets you insert and await the results of jobs locally or remotely, across + any number of nodes, so you can seamlessly distribute jobs and await the results synchronously. + + Think of `Relay` as persistent, distributed tasks. + + `Relay` uses PubSub for to transmit results. That means it will work without Erlang distribution + or clustering, but it does require Oban to have functional PubSub. It doesn't matter which node + executes a job, the result will still be broadcast back. + + Results are encoded using `term_to_binary/2` and decoded using `binary_to_term/2` using the + `:safe` option to prevent the creation of new atoms or function references. If you're returning + results with atoms you _must be sure_ that those atoms are defined locally, where the `await/2` + or `await_many/2` function is called. + + ## Usage + + Use `async/1` to insert a job for asynchronous execution: + + ```elixir + relay = + %{id: 123} + |> MyApp.Worker.new() + |> Oban.Pro.Relay.async() + ``` + + After inserting a job and constructing a relay, use `await/1` to await the job's execution and + return the result: + + ```elixir + {:ok, result} = + %{id: 123} + |> MyApp.Worker.new() + |> Oban.Pro.Relay.async() + |> Oban.Pro.Relay.await() + ``` + + By default, `await/1` will timeout after 5 seconds and return an `{:error, :timeout}` tuple. The job + itself may continue to run, only the local process stops waiting on it. Pass an explicit timeout + to wait longer: + + ```elixir + {:ok, result} = Oban.Pro.Relay.await(relay, :timer.seconds(30)) + ``` + + Any successful result such as `:ok`, `{:ok, value}`, or `{:cancel, reason}` is passed back as + the await result. When the executed job returns an `{:error, reason}` tuple, raises an + exception, or crashes, the result comes through as an error tuple. + + Use `await_many/1` when you need the results of multiple async relays: + + ```elixir + relayed = + 1..3 + |> Enum.map(&DoubleWorker.new(%{int: &1})) + |> Enum.map(&Oban.Pro.Relay.async/1) + |> Oban.Pro.Relay.await_many() + + #=> [{:ok, 2}, {:ok, 4}, {:ok, 6}] + ``` + + ## Testing with Relay + + Calls to `Relay.await/2` will block until the job runs. That will cause tests to hang when + testing in `:manual` mode until jobs are drained. Unit tests that wrap `Relay` processing can + switch to `:inline` mode so that jobs run immediately. + + For example, assuming `Oban` is configured to run in `:manual` testing mode: + + ```elixir + defmodule MyWorker do + use Oban.Pro.Worker + + alias Oban.Pro.Relay + + @impl Oban.Pro.Worker + def process(%{args: %{"sub" => sub}}) do + results = + sub + |> Enum.map(&MyOtherWorker.new(%{id: &1}) + |> Enum.map(&Relay.async/1) + |> Relay.await_many() + + {:ok, results} + end + end + + test "running multiple sub workers from perform_job/2" do + Oban.Testing.with_testing_mode(:inline, fn -> + assert {:ok, _} = perform_job(MyWorker, %{subs: [1, 2, 3]}) + end) + end + ``` + + ## Usage with Chunks + + `Relay` is intended for use with a single job and isn't suited to awaiting results from + `Oban.Pro.Workers.Chunk` jobs. That's because only one of the chunk's jobs (the leader) will + relay results back. Awaiting any other job in the chunk will time out without returning a proper + result. + """ + + @behaviour Oban.Pro.Handler + + alias Ecto.{Changeset, UUID} + alias Oban.Pro.{Unique, Utils} + alias Oban.{Job, Notifier} + + require Logger + + @type t :: %__MODULE__{job: Job.t(), pid: pid(), ref: UUID.t()} + + @type await_result :: + :ok + | {:ok, term()} + | {:cancel, term()} + | {:discard, term()} + | {:error, :result_too_large | :timeout | Exception.t()} + | {:snooze, integer()} + + defstruct [:job, :name, :pid, :ref] + + @postgres_max_bytes 8000 + + @doc """ + Insert a job for asynchronous execution. + + The returned map contains the caller's pid and a unique ref that is used to await the results. + + ## Examples + + The single arity version takes a job changeset and inserts it: + + relay = + %{id: 123} + |> MyApp.Worker.new() + |> Oban.Pro.Relay.async() + + When the Oban instance has a custom name, or an app has multiple Oban + instances, you can use the two arity version to select an instance: + + changeset = MyApp.Worker.new(%{id: 123}) + Oban.Pro.Relay.async(MyOban, changeset) + """ + @spec async(Oban.name(), Job.changeset()) :: t() | {:error, Job.changeset()} + def async(name \\ Oban, %Changeset{} = changeset) do + pid = self() + ref = if Unique.unique?(changeset), do: Unique.gen_key(changeset), else: UUID.generate() + + meta = + changeset + |> Changeset.get_change(:meta, %{}) + |> Map.put_new(:await_ref, ref) + + changeset = Changeset.put_change(changeset, :meta, meta) + + :ok = Notifier.listen(name, :relay) + + with {:ok, job} <- Oban.insert(name, changeset) do + %__MODULE__{job: job, name: name, pid: pid, ref: ref} + end + end + + @doc """ + Await a relay's execution and return the result. + + Any successful result such as `:ok`, `{:ok, value}`, or `{:cancel, reason}` is passed back as + the await result. When the executed job returns an `{:error, reason}` tuple, raises an + exception, or crashes, the result comes back as an error tuple with the exception. + + By default, `await/1` will timeout after 5 seconds and return `{:error, :timeout}`. The job + itself may continue to run, only the local process stops waiting on it. + + > #### Result Size Limits {: .warning} + > + > When using the default `Oban.Notifiers.Postgres` notifier for PubSub, any value larger than 8kb + > (compressed) can't be broadcast due to a Postgres `NOTIFY` limitation. Instead, awaiting will + > return an `{:error, :result_too_large}` tuple. The `Oban.Notifiers.PG` notifier doesn't have any + > such size limitation, but it requires Distributed Erlang. + + ## Options + + * `:timeout` - the maximum time in milliseconds to wait for the result. Defaults to `5_000`. + * `:with_retries` - when `true`, wait through all retry attempts until the job reaches a final + state (`completed`, `cancelled`, `discarded`), or a timeout is reached. Defaults to `false`. + + ## Examples + + Await a job: + + {:ok, result} = + %{id: 123} + |> MyApp.Worker.new() + |> Oban.Pro.Relay.async() + |> Oban.Pro.Relay.await() + + Increase the wait time with a timeout value: + + %{id: 456} + |> MyApp.Worker.new() + |> Oban.Pro.Relay.async() + |> Oban.Pro.Relay.await(timeout: :timer.seconds(30)) + + Wait through all retry attempts: + + %{id: 789} + |> MyApp.Worker.new() + |> Oban.Pro.Relay.async() + |> Oban.Pro.Relay.await(with_retries: true, timeout: :timer.minutes(1)) + """ + @spec await(t(), timeout() | keyword()) :: await_result() + def await(relay, opts \\ []) + + def await(relay, timeout) when timeout == :infinity or is_integer(timeout) do + await(relay, timeout: timeout) + end + + def await(%{name: name, pid: pid, ref: ref}, opts) when is_list(opts) do + check_ownership!(pid) + + timeout = Keyword.get(opts, :timeout, 5_000) + retries = Keyword.get(opts, :with_retries, false) + + try do + await(ref, timeout, retries) + after + Notifier.unlisten(name, :relay) + end + end + + defp await(ref, timeout, with_retries) do + receive do + {:notification, :relay, %{"ref" => ^ref, "result" => result} = payload} -> + # For backward compati. Existing nodes won't return these values in the payload + state = Map.get(payload, "state", "success") + attempt = Map.get(payload, "attempt", 1) + max_attempts = Map.get(payload, "max_attempts", 20) + + decoded = Utils.decode64(result) + + if with_retries and state == "failure" and attempt < max_attempts do + await(ref, timeout, with_retries) + else + decoded + end + after + timeout -> {:error, :timeout} + end + end + + @doc """ + Await replies from multiple relays and return the results. + + It returns a list of the results in the same order as the relays supplied as the first argument. + + Unlike `Task.await_many` or `Task.yield_many`, `await_many/2` may return partial results when + the timeout is reached. When a job hasn't finished executing the value will be `{:error, + :timeout}` + + ## Examples + + Await multiple jobs without any errors or timeouts: + + relayed = + 1..3 + |> Enum.map(&DoubleWorker.new(%{int: &1})) + |> Enum.map(&Oban.Pro.Relay.async(&1)) + |> Oban.Pro.Relay.await_many() + + #=> [{:ok, 2}, {:ok, 4}, {:ok, 6}] + + Await multiple jobs with an error timeout: + + relayed = + [1, 2, 300_000_000] + |> Enum.map(&SlowWorker.new(%{int: &1})) + |> Enum.map(&Oban.Pro.Relay.async(&1)) + |> Oban.Pro.Relay.await_many(100) + + #=> [{:ok, 2}, {:ok, 4}, {:error, :timeout}] + """ + @spec await_many([t()], timeout()) :: [await_result()] + def await_many([%__MODULE__{} | _] = relays, timeout \\ 5_000) do + awaited = + for %{pid: pid, ref: ref} <- relays, into: %{} do + check_ownership!(pid) + + {ref, {:error, :timeout}} + end + + error_ref = make_ref() + timer_ref = maybe_send_after(error_ref, timeout) + + try do + await_many(relays, awaited, %{}, error_ref) + after + if is_reference(timer_ref), do: Process.cancel_timer(timer_ref) + + receive do: (^error_ref -> :ok), after: (0 -> :ok) + end + after + relays + |> Enum.map(& &1.name) + |> Enum.uniq() + |> Enum.each(&Notifier.unlisten(&1, :relay)) + end + + defp await_many(relays, awaited, replied, _error_ref) when map_size(awaited) == 0 do + for %{ref: ref} <- relays, do: Map.fetch!(replied, ref) + end + + defp await_many(relays, awaited, replied, error_ref) do + receive do + ^error_ref -> + for %{ref: ref} <- relays, do: replied[ref] || awaited[ref] + + {:notification, :relay, %{"ref" => ref, "result" => res}} -> + if Map.has_key?(awaited, ref) do + await_many( + relays, + Map.delete(awaited, ref), + Map.put(replied, ref, Utils.decode64(res)), + error_ref + ) + else + await_many(relays, awaited, replied, error_ref) + end + end + end + + @doc false + def handle_event([:oban, :job, _], _, %{conf: conf, job: job, state: state} = meta, _) do + with %{"await_ref" => aref} <- job.meta, + oban_pid when is_pid(oban_pid) <- Oban.whereis(conf.name) do + payload = %{ + "ref" => aref, + "result" => extract_result(conf, meta), + "state" => state, + "attempt" => job.attempt, + "max_attempts" => job.max_attempts + } + + Notifier.notify(conf, :relay, payload) + end + catch + kind, value -> + Logger.error(fn -> + "[Oban.Pro.Relay] handler error: " <> Exception.format(kind, value) + end) + + :ok + end + + @doc false + @impl Oban.Pro.Handler + def on_start do + events = [ + [:oban, :job, :stop], + [:oban, :job, :exception] + ] + + :telemetry.attach_many("oban.pro.relay", events, &__MODULE__.handle_event/4, nil) + end + + @doc false + @impl Oban.Pro.Handler + def on_stop do + :telemetry.detach("oban.pro.relay") + end + + # Messaging + + defp check_ownership!(pid) do + unless pid == self() do + raise ArgumentError, + "relay must be awaited by #{inspect(pid)}, was awaited by #{inspect(self())}" + end + end + + defp maybe_send_after(_error_ref, :infinity), do: nil + + defp maybe_send_after(error_ref, timeout) do + Process.send_after(self(), error_ref, timeout) + end + + # Result Handling + + defp extract_result(conf, %{state: state, result: result}) + when state in [:cancelled, :snoozed, :success] do + encoded = Utils.encode64(result) + + if match?({Oban.Notifiers.Postgres, _}, conf.notifier) and + byte_size(encoded) > @postgres_max_bytes do + Utils.encode64({:error, :result_too_large}) + else + encoded + end + end + + defp extract_result(_conf, %{state: :discard, job: job}) do + Utils.encode64({:discard, job.unsaved_error.reason}) + end + + defp extract_result(_conf, %{state: :failure, error: error}) do + Utils.encode64({:error, error}) + end +end diff --git a/vendor/oban_pro/lib/oban/pro/stage.ex b/vendor/oban_pro/lib/oban/pro/stage.ex new file mode 100644 index 00000000..8403c634 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/stage.ex @@ -0,0 +1,36 @@ +defmodule Oban.Pro.Stage do + @moduledoc false + + alias Oban.Job + + @type args :: Job.args() + @type changeset :: Job.changeset() + @type conf :: term() + @type job :: Job.t() + @type opts :: Keyword.t() + @type reason :: term() + @type result :: term() + + @doc """ + Initialize configuration passed to various before callbacks. + """ + @callback init(module(), opts()) :: {:ok, conf()} | {:error, reason()} + + @doc """ + Pre-process the args and/or opts for a job before calling `new/2`. + """ + @callback before_new(args(), opts(), conf()) :: {:ok, args(), opts()} | {:error, reason()} + + @doc """ + Pre-process a job before calling `process/1`. + """ + @callback before_process(job(), conf()) :: + {:ok, job()} | {:cancel, reason()} | {:error, reason()} | {:snooze, integer()} + + @doc """ + Post-process a job before returning from the wrapping `perform/1` + """ + @callback after_process(result(), job(), conf()) :: :ok + + @optional_callbacks before_new: 3, before_process: 2, after_process: 3 +end diff --git a/vendor/oban_pro/lib/oban/pro/stages/chain.ex b/vendor/oban_pro/lib/oban/pro/stages/chain.ex new file mode 100644 index 00000000..68af9fba --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/stages/chain.ex @@ -0,0 +1,232 @@ +defmodule Oban.Pro.Stages.Chain do + @moduledoc false + + @behaviour Oban.Pro.Flusher + @behaviour Oban.Pro.Stage + + import Ecto.Query + + alias Ecto.Changeset + alias Oban.{Job, Repo, Validation} + alias Oban.Pro.Utils + + @hold_date ~U[3000-01-01 00:00:00.000000Z] + + defstruct [:by, :worker, on_cancelled: :ignore, on_discarded: :ignore] + + defmacrop drop_hold(meta) do + quote do + fragment(~s(? || '{"on_hold":false, "uniq_bmp":[]}'), unquote(meta)) + end + end + + defmacrop continue_state(prefix, meta) do + quote do + fragment( + """ + (CASE WHEN ? \\? 'orig_scheduled_at' THEN 'scheduled' ELSE 'available' END)::?.oban_job_state + """, + unquote(meta), + unquote(prefix) + ) + end + end + + defmacrop continue_at(meta, now) do + quote do + fragment( + """ + CASE + WHEN ? \\? 'orig_scheduled_at' THEN timezone('utc', to_timestamp(div((?->'orig_scheduled_at')::bigint, 1000000)::float)) + ELSE ? + END + """, + unquote(meta), + unquote(meta), + unquote(now) + ) + end + end + + defmacrop same_chain(meta_a, meta_b) do + quote do + fragment("?->>'chain_id' = ?->>'chain_id'", unquote(meta_a), unquote(meta_b)) + end + end + + @spec chain?(Job.changeset()) :: boolean() + def chain?(changeset), do: match?(%{changes: %{meta: %{"chain_id" => _}}}, changeset) + + @spec get_key(Job.changeset()) :: nil | String.t() + def get_key(%{changes: %{meta: %{"chain_id" => chain_id}}}), do: chain_id + def get_key(_changeset), do: nil + + @spec continuable?(String.t(), map()) :: boolean() + def continuable?(state, meta) do + state == "completed" or + (state == "cancelled" and meta["on_cancelled"] != "hold") or + (state == "discarded" and meta["on_discarded"] != "hold") + end + + @spec to_postponed(Job.changeset()) :: Job.changeset() + def to_postponed(changeset) do + meta = + if Changeset.get_field(changeset, :state) == "scheduled" do + orig_at = + changeset + |> Changeset.get_change(:scheduled_at) + |> DateTime.to_unix(:microsecond) + + %{"on_hold" => true, "orig_scheduled_at" => orig_at} + else + %{"on_hold" => true} + end + + changeset + |> Changeset.update_change(:meta, &Map.merge(&1, meta)) + |> Changeset.put_change(:state, "scheduled") + |> Changeset.put_change(:scheduled_at, @hold_date) + end + + @doc false + def rescue_chains(conf, opts \\ []) do + prev_query = + Job + |> where([j], j.state in ~w(available scheduled retryable executing)) + |> where([j], fragment("? \\? 'chain_id'", j.meta)) + |> where([j], same_chain(j.meta, parent_as(:jobs).meta)) + |> where([j], fragment("?->>'on_hold'", j.meta) in ~w(true false)) + |> where([j], j.id < parent_as(:jobs).id) + + subquery = + from(j in Job, as: :jobs) + |> where([j], j.state == "scheduled") + |> where([j], fragment("? \\? 'chain_id'", j.meta)) + |> where([j], fragment("?->>'on_hold' = 'true'", j.meta)) + |> where([j], not exists(prev_query)) + |> order_by(asc: :id) + |> limit(1) + + query = + Job + |> join(:inner, [j], x in subquery(subquery), on: j.id == x.id) + |> update([j], + set: [ + meta: drop_hold(j.meta), + state: continue_state(literal(^conf.prefix), j.meta), + scheduled_at: continue_at(j.meta, ^DateTime.utc_now()) + ] + ) + + Repo.update_all(conf, query, [], opts) + + :ok + end + + @doc false + def to_chain_id(chain_by, worker, args, opts) do + chain_data = + Enum.map([:queue | chain_by], fn + :queue -> opts |> Keyword.get(:queue, "default") |> to_string() + :worker -> inspect(worker) + [:args, keys] -> take_keys(args, keys) + [:meta, keys] -> opts |> Keyword.get(:meta, %{}) |> take_keys(keys) + end) + + Utils.hash64(chain_data) + end + + # Stage + + @impl Oban.Pro.Stage + def init(worker, opts) do + with :ok <- validate(opts) do + opts = + opts + |> Keyword.put_new(:by, :worker) + |> Keyword.update!(:by, &Utils.normalize_by/1) + |> Keyword.put(:worker, worker) + + {:ok, struct!(__MODULE__, opts)} + end + end + + @impl Oban.Pro.Stage + def before_new(args, opts, conf) do + conf = merge_conf_opts(conf, opts) + + meta = %{ + "chain" => true, + "chain_id" => to_chain_id(conf.by, conf.worker, args, opts), + "on_cancelled" => to_string(conf.on_cancelled), + "on_discarded" => to_string(conf.on_discarded), + "on_hold" => false + } + + opts = Keyword.update(opts, :meta, meta, &Map.merge(&1, meta)) + + {:ok, args, opts} + end + + # Flushing + + @impl Oban.Pro.Flusher + def to_flush_mfa(job, conf) do + case job do + %{meta: %{"chain_id" => _}} -> {__MODULE__, :on_flush, [job, conf]} + _ -> :ignore + end + end + + def on_flush(%{state: state, meta: %{"chain_id" => chain_id} = meta}, conf) do + if continuable?(state, meta) do + subquery = + Job + |> select([j], j.id) + |> where([j], j.state == "scheduled") + |> where([j], fragment("? \\? 'chain_id'", j.meta)) + |> where([j], fragment("?->>'chain_id'", j.meta) == ^chain_id) + |> where([j], fragment("?->>'on_hold'", j.meta) == "true") + |> order_by(asc: :id) + |> limit(1) + |> lock("FOR UPDATE") + + query = + Job + |> join(:inner, [j], x in subquery(subquery), on: j.id == x.id) + |> update([j], + set: [ + meta: drop_hold(j.meta), + state: continue_state(literal(^conf.prefix), j.meta), + scheduled_at: continue_at(j.meta, ^DateTime.utc_now()) + ] + ) + + Repo.update_all(conf, query, []) + end + end + + # Helpers + + defp merge_conf_opts(conf, opts) do + case Keyword.fetch(opts, :chain) do + {:ok, chain} -> + Enum.reduce(chain, conf, fn {key, val}, acc -> Map.replace!(acc, key, val) end) + + :error -> + conf + end + end + + defp take_keys(map, keys) do + Enum.map(keys, fn key -> inspect(map[key] || map[to_string(key)]) end) + end + + defp validate(opts) do + Validation.validate_schema(opts, + by: {:custom, &Oban.Pro.Validation.validate_by/1}, + on_cancelled: {:enum, [:hold, :ignore]}, + on_discarded: {:enum, [:hold, :ignore]} + ) + end +end diff --git a/vendor/oban_pro/lib/oban/pro/stages/deadline.ex b/vendor/oban_pro/lib/oban/pro/stages/deadline.ex new file mode 100644 index 00000000..7a026206 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/stages/deadline.ex @@ -0,0 +1,128 @@ +defmodule Oban.Pro.Stages.Deadline do + @moduledoc false + + @behaviour Oban.Pro.Stage + + alias Oban.Pro.Utils + alias Oban.{Registry, Validation} + + @time_units ~w( + second + seconds + minute + minutes + hour + hours + day + days + week + weeks + )a + + defstruct [:in, force: false] + + @impl Oban.Pro.Stage + def init(worker \\ nil, period) + + def init(worker, period) when is_integer(period) or is_tuple(period) do + init(worker, in: period) + end + + def init(_worker, opts) when is_list(opts) do + with :ok <- validate(opts) do + conf = + opts + |> Keyword.update!(:in, &Utils.cast_period/1) + |> then(&struct(__MODULE__, &1)) + + {:ok, conf} + end + end + + @impl Oban.Pro.Stage + def before_new(args, opts, conf) do + {deadline, opts} = Keyword.pop(opts, :deadline, conf.in) + + seconds = Utils.cast_period(deadline) + now = DateTime.utc_now() + + offset = + cond do + schedule_in = opts[:schedule_in] -> Utils.cast_period(schedule_in) + seconds + scheduled_at = opts[:scheduled_at] -> DateTime.diff(scheduled_at, now) + seconds + true -> seconds + end + + at = + now + |> DateTime.add(offset) + |> DateTime.to_unix() + + meta = %{"deadline_at" => at} + opts = Keyword.update(opts, :meta, meta, &Map.merge(&1, meta)) + + {:ok, args, opts} + end + + @impl Oban.Pro.Stage + def before_process(%{meta: %{"deadline_at" => at}} = job, conf) do + now = DateTime.utc_now() + dat = DateTime.from_unix!(at) + + if :gt == DateTime.compare(now, dat) do + {:cancel, "deadline at #{inspect(dat)} exired"} + else + if conf.force, do: force_timeout(now, dat, job) + + {:ok, job} + end + end + + def before_process(job, conf) do + opts = [scheduled_at: job.scheduled_at, meta: job.meta] + + {:ok, _args, opts} = before_new(job.args, opts, conf) + + meta = Keyword.fetch!(opts, :meta) + + before_process(%{job | meta: meta}, conf) + end + + defp force_timeout(now, dat, job) do + parent = self() + + Task.start(fn -> + ref = Process.monitor(parent) + + Process.send_after(self(), :cancel, DateTime.diff(dat, now, :millisecond)) + + receive do + {:DOWN, ^ref, :process, _pid, _reason} -> + Process.demonitor(ref, [:flush]) + + :cancel -> + with pid when is_pid(pid) <- Registry.whereis(job.conf.name, {:producer, job.queue}) do + payload = %{"action" => "pkill", "job_id" => job.id} + + send(pid, {:notification, :signal, payload}) + end + end + end) + end + + # Validation + + defp validate(opts) do + Validation.validate_schema(opts, + in: {:custom, &validate_period/1}, + force: :boolean + ) + end + + defp validate_period(period) when is_integer(period), do: :ok + defp validate_period({value, units}) when is_integer(value) and units in @time_units, do: :ok + + defp validate_period(period) do + {:error, "expected deadline to be an integer in seconds or a tuple, got: #{inspect(period)}"} + end +end diff --git a/vendor/oban_pro/lib/oban/pro/stages/encrypted.ex b/vendor/oban_pro/lib/oban/pro/stages/encrypted.ex new file mode 100644 index 00000000..32672dd4 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/stages/encrypted.ex @@ -0,0 +1,83 @@ +defmodule Oban.Pro.Stages.Encrypted do + @moduledoc false + + @behaviour Oban.Pro.Stage + + defstruct [:key, cipher: :aes_256_ctr, iv_bytes: 16] + + @impl Oban.Pro.Stage + def init(_worker \\ nil, opts) do + case Keyword.fetch(opts, :key) do + {:ok, key} when is_binary(key) -> + {:ok, %__MODULE__{key: Base.decode64!(key)}} + + {:ok, {_mod, _fun, _arg} = key} -> + {:ok, %__MODULE__{key: key}} + + _ -> + {:error, "expected :key to be a binary or mfa, got: #{inspect(opts)}"} + end + end + + @impl Oban.Pro.Stage + def before_new(args, opts, conf) do + iv = :crypto.strong_rand_bytes(conf.iv_bytes) + + meta = %{"encrypted" => true, "iv" => Base.encode64(iv)} + + opts = + opts + |> Keyword.put_new(:meta, %{}) + |> Keyword.update!(:meta, &Map.merge(&1, meta)) + + {:ok, %{"data" => encode(args, iv, conf)}, opts} + end + + @impl Oban.Pro.Stage + def before_process(%{args: args, meta: meta} = job, conf) do + with %{"encrypted" => true, "iv" => iv} <- meta, + %{"data" => data} <- args do + {:ok, %{job | args: decode(data, iv, conf)}} + else + _ -> + {:ok, job} + end + end + + # Helpers + + defp resolve_key({mod, fun, arg}) do + mod + |> apply(fun, arg) + |> Base.decode64!() + end + + defp resolve_key(key), do: key + + defp encode(data, iv, conf) do + key = resolve_key(conf.key) + json = Oban.JSON.encode!(data) + + conf.cipher + |> :crypto.crypto_one_time(key, iv, json, true) + |> Base.encode64() + rescue + error -> reraise error, prune_args_from_stacktrace(__STACKTRACE__) + end + + defp decode(data, iv, conf) do + key = resolve_key(conf.key) + + conf.cipher + |> :crypto.crypto_one_time(key, Base.decode64!(iv), Base.decode64!(data), false) + |> Oban.JSON.decode!() + rescue + error -> reraise error, prune_args_from_stacktrace(__STACKTRACE__) + end + + defp prune_args_from_stacktrace([{mod, fun, [_ | _] = args, info} | rest]) do + [{mod, fun, length(args), info} | rest] + end + + defp prune_args_from_stacktrace(stacktrace), do: stacktrace +end diff --git a/vendor/oban_pro/lib/oban/pro/stages/hooks.ex b/vendor/oban_pro/lib/oban/pro/stages/hooks.ex new file mode 100644 index 00000000..18490ae4 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/stages/hooks.ex @@ -0,0 +1,147 @@ +defmodule Oban.Pro.Stages.Hooks do + @moduledoc false + + @behaviour Oban.Pro.Stage + + alias Oban.Pro.Utils + + require Logger + + defstruct modules: [] + + @impl Oban.Pro.Stage + def init(worker, modules) do + {:ok, struct!(__MODULE__, modules: [worker | modules])} + end + + @impl Oban.Pro.Stage + def before_new(args, opts, %{modules: modules}) do + (modules ++ get_hooks()) + |> Enum.filter(&hook_exported?(&1, :before_new, 2)) + |> Enum.reduce_while({:ok, args, opts}, fn module, {:ok, args, opts} -> + case module.before_new(args, opts) do + {:ok, _, _} = ok -> {:cont, ok} + other -> {:halt, other} + end + end) + end + + @impl Oban.Pro.Stage + def before_process(job, %{modules: modules}) do + (modules ++ get_hooks()) + |> Enum.filter(&hook_exported?(&1, :before_process, 1)) + |> Enum.reduce_while({:ok, job}, fn module, {:ok, job} -> + case module.before_process(job) do + {:ok, _} = ok -> {:cont, ok} + other -> {:halt, other} + end + end) + end + + def attach_hook(module) do + with :ok <- Utils.validate_opts([module], &validate_module/1) do + modules = + get_hooks() + |> Enum.concat([module]) + |> Enum.uniq() + + :persistent_term.put(:oban_pro_hooks, modules) + end + end + + def detach_hook(module) do + case get_hooks() do + [_ | _] = modules -> + :persistent_term.put(:oban_pro_hooks, modules -- [module]) + + _ -> + :ok + end + end + + def handle_event(_event, _timing, meta, _conf) do + %{job: %{unsaved_error: unsaved_error}, state: state} = meta + + with {worker, job, opts} <- resolve_processing(meta.job) do + hook_state = exec_to_hook_state(state) + module_hooks = Keyword.get(opts, :hooks, []) + global_hooks = get_hooks() + + job = %{job | unsaved_error: unsaved_error} + result = Map.get(meta, :result, nil) + + ([worker | module_hooks] ++ global_hooks) + |> Enum.uniq() + |> Enum.each(fn module -> + cond do + hook_exported?(module, :after_process, 3) -> + module.after_process(hook_state, job, result) + + hook_exported?(module, :after_process, 2) -> + module.after_process(hook_state, job) + + true -> + :ok + end + end) + end + catch + kind, value -> + Logger.error(fn -> + "[Oban.Pro.Worker] hook error: " <> Exception.format(kind, value, __STACKTRACE__) + end) + after + Process.delete(:oban_processing) + end + + defp resolve_processing(job) do + # The event may be triggered by a producer after the process was killed or experienced a + # linked crash. In that event, the processing data isn't in the dictionary and we need to + # resolve and apply before hooks to the job again. + with nil <- Process.get(:oban_processing), + {:ok, worker} <- Oban.Worker.from_string(job.worker), + opts = worker.__opts__(), + {:ok, job} <- Oban.Pro.Worker.before_process(worker, job, opts) do + processing = {worker, job, opts} + Process.put(:oban_processing, processing) + processing + end + end + + defp get_hooks, do: :persistent_term.get(:oban_pro_hooks, []) + + defp exec_to_hook_state(:cancelled), do: :cancel + defp exec_to_hook_state(:success), do: :complete + defp exec_to_hook_state(:discard), do: :discard + defp exec_to_hook_state(:failure), do: :error + defp exec_to_hook_state(:snoozed), do: :snooze + + defp hook_exported?(mod, fun, arity) do + Code.ensure_loaded?(mod) and function_exported?(mod, fun, arity) + end + + # Validation + + defp validate_module(module) do + cond do + not Code.ensure_loaded?(module) -> + {:error, "unable to load callback module #{inspect(module)}"} + + function_exported?(module, :before_new, 2) -> + :ok + + function_exported?(module, :after_process, 2) -> + :ok + + function_exported?(module, :after_process, 3) -> + :ok + + function_exported?(module, :before_process, 1) -> + :ok + + true -> + {:error, + "#{inspect(module)} doesn't implement after_process/3, before_new/2, before_process/1"} + end + end +end diff --git a/vendor/oban_pro/lib/oban/pro/stages/recorded.ex b/vendor/oban_pro/lib/oban/pro/stages/recorded.ex new file mode 100644 index 00000000..42215c3c --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/stages/recorded.ex @@ -0,0 +1,77 @@ +defmodule Oban.Pro.Stages.Recorded do + @moduledoc false + + @behaviour Oban.Pro.Stage + + alias Oban.Pro.Utils + alias Oban.Validation + + defstruct to: :return, limit: 64_000_000, safe_decode: false + + @impl Oban.Pro.Stage + def init(_worker, true) do + {:ok, %__MODULE__{}} + end + + def init(_worker, opts) do + with :ok <- validate(opts) do + {:ok, struct(__MODULE__, opts)} + end + end + + @impl Oban.Pro.Stage + def before_new(args, opts, _conf) do + meta = %{"recorded" => true} + opts = Keyword.update(opts, :meta, meta, &Map.merge(&1, meta)) + + {:ok, args, opts} + end + + @impl Oban.Pro.Stage + def after_process(result, %{meta: %{"recorded" => true}} = job, conf) do + with {:ok, return} <- result do + encoded = Utils.encode64(return) + + cond do + job.conf.engine != Oban.Pro.Engines.Smart -> + {:error, "recording requires the Smart engine"} + + byte_size(encoded) > conf.limit -> + {:error, "return is #{byte_size(encoded)} bytes, larger than the limit: #{conf.limit}"} + + true -> + Process.put(:oban_recorded, %{to_string(conf.to) => encoded}) + + :ok + end + end + end + + def after_process(_result, _job, _conf), do: :ok + + @spec fetch_recorded(Oban.Job.t(), conf :: map()) :: {:ok, term()} | {:error, term()} + def fetch_recorded(job, conf) do + to = to_string(conf.to) + + case job.meta do + %{"recorded" => true, ^to => value} -> + {:ok, Utils.decode64(value, decode_opts(job.meta))} + + _ -> + {:error, :missing} + end + end + + # Helpers + + defp validate(opts) do + Validation.validate_schema(opts, + to: {:or, [:atom, :string]}, + limit: :pos_integer, + safe_decode: :boolean + ) + end + + defp decode_opts(%{"safe_decode" => true}), do: [:safe] + defp decode_opts(_meta), do: [] +end diff --git a/vendor/oban_pro/lib/oban/pro/stages/structured.ex b/vendor/oban_pro/lib/oban/pro/stages/structured.ex new file mode 100644 index 00000000..99c93782 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/stages/structured.ex @@ -0,0 +1,191 @@ +defmodule Oban.Pro.Stages.Structured do + @moduledoc false + + @behaviour Oban.Pro.Stage + + alias Ecto.Changeset + alias Oban.Pro.Utils + + defstruct [:worker] + + defmodule Term do + @moduledoc false + + use Ecto.Type + + alias Oban.Pro.Utils + + @impl Ecto.Type + def type, do: :string + + @impl Ecto.Type + def cast("term-" <> data), do: {:ok, Utils.decode64(data)} + def cast(term), do: {:ok, "term-" <> Utils.encode64(term)} + + @impl Ecto.Type + def dump(_term), do: {:ok, :noop} + + @impl Ecto.Type + def load(_data), do: {:ok, :noop} + end + + @impl Oban.Pro.Stage + def init(worker, _opts) do + if function_exported?(worker, :__args_schema__, 0) do + {:ok, %__MODULE__{worker: worker}} + else + {:ok, :ignore} + end + end + + @impl Oban.Pro.Stage + def before_new(args, opts, :ignore), do: {:ok, args, opts} + + def before_new(args, opts, conf) do + {:ok, changes} = gather_changes(conf.worker, args) + + meta = %{structured: true} + opts = Keyword.update(opts, :meta, meta, &Map.merge(&1, meta)) + + {:ok, dump_changes(changes), opts} + catch + message -> {:error, message} + end + + @impl Oban.Pro.Stage + def before_process(job, :ignore), do: {:ok, job} + + def before_process(%{args: args} = job, conf) do + {:ok, changes} = gather_changes(conf.worker, args) + + {:ok, %{job | args: changes}} + catch + message -> {:error, message} + end + + defp gather_changes(module, args) do + fields = module.__args_schema__() + struct = struct(module, []) + + {fields, required, defaults, merge} = split_fields(fields, args) + + keys = Map.keys(fields) + + args = + if Enum.any?(Map.keys(args), &is_binary/1) do + defaults + |> Map.merge(args) + |> Map.new(fn {key, val} -> {to_string(key), val} end) + else + Map.merge(defaults, args) + end + + changeset = + {struct, fields} + |> Changeset.cast(args, keys) + |> Changeset.validate_required(required) + |> validate_allowed(args, keys) + + if changeset.valid? do + changeset + |> Changeset.apply_changes() + |> Map.merge(merge) + |> then(&{:ok, &1}) + else + changeset + |> Utils.to_translated_errors() + |> Enum.map_join(", ", fn {field, error} -> "#{inspect(field)} #{error}" end) + |> throw() + end + end + + defp split_fields(fields, args) do + Enum.reduce(fields, {%{}, [], %{}, %{}}, fn {name, opts}, {keep, required, defaults, merge} -> + required = if opts[:required], do: [name | required], else: required + + defaults = + case Keyword.fetch(opts, :default) do + {:ok, default} -> Map.put(defaults, name, default) + :error -> defaults + end + + case Map.new(opts) do + %{cardinality: :one, module: module, required: embed_required?, type: :embed} -> + data = args[name] || args[to_string(name)] + keep = Map.put(keep, name, :any) + + if is_nil(data) and not embed_required? do + {keep, required, defaults, merge} + else + {:ok, embed} = gather_changes(module, data || %{}) + + {keep, required, defaults, Map.put(merge, name, embed)} + end + + %{cardinality: :many, module: module, type: :embed} -> + embed_list = + for sub_arg <- args[name] || args[to_string(name)] || [] do + {:ok, embed} = gather_changes(module, sub_arg) + + embed + end + + {Map.put(keep, name, :any), required, defaults, Map.put(merge, name, embed_list)} + + %{type: :enum} -> + enum = Ecto.ParameterizedType.init(Ecto.Enum, values: opts[:values]) + + {Map.put(keep, name, enum), required, defaults, merge} + + %{type: :term} -> + {Map.put(keep, name, Term), required, defaults, merge} + + %{type: :uuid} -> + {Map.put(keep, name, :binary_id), required, defaults, merge} + + %{type: {:array, :enum}} -> + enum = Ecto.ParameterizedType.init(Ecto.Enum, values: opts[:values]) + + {Map.put(keep, name, {:array, enum}), required, defaults, merge} + + %{type: {:array, :uuid}} -> + {Map.put(keep, name, {:array, :binary_id}), required, defaults, merge} + + %{type: type} -> + {Map.put(keep, name, type), required, defaults, merge} + end + end) + end + + defp validate_allowed(changeset, args, keys) do + keys = MapSet.new(keys, &to_string/1) + + Enum.reduce(args, changeset, fn {key, _val}, changeset -> + if to_string(key) in keys do + changeset + else + Changeset.add_error(changeset, key, "is an unexpected key") + end + end) + end + + defp dump_changes(struct) when is_struct(struct, Date), do: struct + defp dump_changes(struct) when is_struct(struct, DateTime), do: struct + defp dump_changes(struct) when is_struct(struct, Decimal), do: struct + defp dump_changes(struct) when is_struct(struct, NaiveDateTime), do: struct + defp dump_changes(struct) when is_struct(struct, Time), do: struct + + defp dump_changes(struct) when is_struct(struct) do + for {key, val} <- Map.from_struct(struct), not is_nil(val), into: %{} do + {key, dump_changes(val)} + end + end + + defp dump_changes(map) when is_map(map) do + Map.new(map, fn {key, val} -> {key, dump_changes(val)} end) + end + + defp dump_changes(list) when is_list(list), do: Enum.map(list, &dump_changes/1) + + defp dump_changes(term), do: term +end diff --git a/vendor/oban_pro/lib/oban/pro/testing.ex b/vendor/oban_pro/lib/oban/pro/testing.ex new file mode 100644 index 00000000..baadf578 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/testing.ex @@ -0,0 +1,1334 @@ +defmodule Oban.Pro.Testing do + @moduledoc """ + Advanced helpers for testing supervised Oban instances, workers, and making assertions about + enqueued jobs. + + The `Oban.Pro.Testing` module is a drop-in replacement for `Oban.Testing`, with additional + functions tailored toward integration testing and Pro modules. + + ## Usage in Tests + + The most convenient way to use `Oban.Pro.Testing` is to `use` the module: + + use Oban.Pro.Testing, repo: MyApp.Repo + + Other repo-specific configuration options can also be used: + + use Oban.Pro.Testing, repo: MyApp.Repo, prefix: "private", log: :debug + + If you already have `use Oban.Testing` in your tests or test cases, simply replace it with `use + Oban.Pro.Testing`. + + ## Naming Convention + + The testing helpers in this module adhere to the following naming convention: + + * `perform_*` — executes jobs locally, without touching the database, for unit testing. + + * `drain_*` — execute jobs inline, for integration testing. + + * `run_*` — insert jobs into the database and execute them inline, for integration testing. + + ## Shared Repo Options + + The `use` macro accepts all of these repo-specific configuration options, and they may be passed + to all database functions (`run_`, `drain_`, etc.) + + * `:log` — a usable log level or `false` to disable logging. See `t:Logger.level/0` for valid + options. + + * `:prefix` — an optional database prefix. Defaults to `public`. + + * `:repo` — the name of an Ecto repo, which should be running in sandbox mode. + """ + + import Ecto.Query + import ExUnit.Assertions, only: [assert: 2, flunk: 1] + import ExUnit.Callbacks, only: [on_exit: 2, start_supervised!: 1] + + alias Ecto.Adapters.SQL.Sandbox + alias Ecto.Changeset + alias Oban.Pro.Engines.Smart + alias Oban.Pro.Workers.Chunk + alias Oban.Pro.{Batch, Utils, Workflow} + alias Oban.Queue.Executor + alias Oban.{Config, Job, Repo, Worker} + + @type repo_option :: {:log, false | Logger.level()} | {:prefix, String.t()} | {:repo, module()} + + @type drain_option :: + repo_option() + | {:queue, atom()} + | {:with_limit, pos_integer()} + | {:with_recursion, boolean()} + | {:with_safety, boolean()} + | {:with_scheduled, boolean()} + | {:with_summary, boolean()} + + @type drain_summary :: %{ + cancelled: non_neg_integer(), + completed: non_neg_integer(), + discarded: non_neg_integer(), + exhausted: non_neg_integer(), + retryable: non_neg_integer(), + scheduled: non_neg_integer() + } + + @type drain_result :: drain_summary() | [Job.t()] + + @type perform_option :: Job.option() | repo_option() + + @typedoc """ + Batch callback identifiers, correlating to a `handle_` callback function. + """ + @type callback :: :attempted | :completed | :discarded | :exhausted + + @doc false + defmacro __using__(repo_opts) do + if not Keyword.has_key?(repo_opts, :repo) do + raise ArgumentError, "testing requires a :repo option to be set" + end + + quote do + def all_enqueued(opts \\ []) do + unquote(repo_opts) + |> Keyword.merge(opts) + |> Oban.Pro.Testing.all_enqueued() + end + + def assert_enqueue(opts \\ [], fun) do + unquote(repo_opts) + |> Keyword.merge(opts) + |> Oban.Pro.Testing.assert_enqueue(fun) + end + + def assert_enqueued(opts \\ [], timeout \\ :none) do + unquote(repo_opts) + |> Keyword.merge(opts) + |> Oban.Pro.Testing.assert_enqueued(timeout) + end + + def refute_enqueue(opts \\ [], fun) do + unquote(repo_opts) + |> Keyword.merge(opts) + |> Oban.Pro.Testing.refute_enqueue(fun) + end + + def refute_enqueued(opts \\ [], timeout \\ :none) do + unquote(repo_opts) + |> Keyword.merge(opts) + |> Oban.Pro.Testing.refute_enqueued(timeout) + end + + def build_job(worker, args, opts \\ []) do + Oban.Pro.Testing.build_job(worker, args, Keyword.merge(unquote(repo_opts), opts)) + end + + def drain_jobs(opts \\ []) do + unquote(repo_opts) + |> Keyword.merge(opts) + |> Oban.Pro.Testing.drain_jobs() + end + + def perform_job(job) when is_struct(job) do + Oban.Pro.Testing.perform_job(job, unquote(repo_opts)) + end + + def perform_job(job, opts) when is_struct(job) and is_list(opts) do + Oban.Pro.Testing.perform_job(job, Keyword.merge(unquote(repo_opts), opts)) + end + + def perform_job(worker, args, opts \\ []) do + Oban.Pro.Testing.perform_job(worker, args, Keyword.merge(unquote(repo_opts), opts)) + end + + def perform_callback(worker, callback, args, opts \\ []) do + opts = Keyword.merge(unquote(repo_opts), opts) + + Oban.Pro.Testing.perform_callback(worker, callback, args, opts) + end + + def perform_chunk(worker, args, opts \\ []) do + Oban.Pro.Testing.perform_chunk(worker, args, Keyword.merge(unquote(repo_opts), opts)) + end + + def run_batch(batch, opts \\ []) do + Oban.Pro.Testing.run_batch(batch, Keyword.merge(unquote(repo_opts), opts)) + end + + def run_chain([_ | _] = chain, opts \\ []) do + Oban.Pro.Testing.run_chain(chain, Keyword.merge(unquote(repo_opts), opts)) + end + + def run_chunk([_ | _] = chunk, opts \\ []) do + Oban.Pro.Testing.run_chunk(chunk, Keyword.merge(unquote(repo_opts), opts)) + end + + def run_jobs([_ | _] = changesets, opts \\ []) do + Oban.Pro.Testing.run_jobs(changesets, Keyword.merge(unquote(repo_opts), opts)) + end + + def run_workflow(%_{} = workflow, opts \\ []) do + Oban.Pro.Testing.run_workflow(workflow, Keyword.merge(unquote(repo_opts), opts)) + end + + def start_supervised_oban!(opts \\ []) do + unquote(repo_opts) + |> Keyword.merge(opts) + |> Oban.Pro.Testing.start_supervised_oban!() + end + end + end + + @conf_keys [] + |> Config.new() + |> Map.from_struct() + |> Map.keys() + + @callbacks ~w(attempted cancelled completed discarded exhausted retryable)a + + @default_supervised_opts [ + engine: Oban.Pro.Engines.Smart, + notifier: Oban.Notifiers.Isolated, + peer: Oban.Peers.Isolated, + stage_interval: :infinity, + shutdown_grace_period: 250 + ] + + @empty_summary %{ + cancelled: 0, + completed: 0, + discarded: 0, + exhausted: 0, + retryable: 0, + scheduled: 0 + } + + @doc """ + Construct a job from a worker, args, and options. + + This is a wrapper around `Oban.Testing.build_job/3` that ensures compatibility with Pro worker + features by using the Smart engine. + + The helper makes the following assertions: + + * That the worker implements the `Oban.Worker` behaviour + * That the options provided build a valid job + + This helper is used to build jobs for execution by `perform_job/2`. + + ## Options + + See [shared options](#module-shared-repo-options) for additional repo-specific options. + + ## Examples + + Build a job without args: + + job = build_job(MyWorker, %{}) + + Build a job with stringified args: + + assert %{args: %{"id" => 1}} = build_job(MyWorker, %{id: 1}) + + Build a job with custom options: + + assert %{attempt: 5, priority: 9} = build_job(MyWorker, %{}, attempt: 5, priority: 9) + """ + @doc since: "1.6.8" + @spec build_job(Worker.t(), term(), [Job.option() | repo_option()]) :: Job.t() + def build_job(worker, args, opts) when is_atom(worker) and is_list(opts) do + {_conf_opts, job_opts} = Keyword.split(opts, @conf_keys) + + Oban.Testing.build_job(worker, args, job_opts) + end + + @doc """ + Retrieve all currently enqueued jobs matching a set of criteria. + + This is a wrapper around `Oban.Testing.all_enqueued/1`, see `Oban.Testing` for more details. + + ## Options + + See [shared options](#module-shared-repo-options) for additional repo-specific options. + """ + @doc since: "0.11.0" + @spec all_enqueued(keyword()) :: [Job.t()] + def all_enqueued(opts) do + {repo, opts} = Keyword.pop!(opts, :repo) + + Oban.Testing.all_enqueued(repo, opts) + end + + @doc """ + Assert that one or more jobs were enqueued during a function call. + + Any pre-existing jobs are ignored for the assertion. If the assertion passes then the function's + return value is passed back. + + ## Options + + See `assert_enqueued/2` for standard options, and [shared options](#module-shared-repo-options) + for additional repo-specific options. + + ## Examples + + Assert that a `MyApp.Worker` job was added to the `default` queue: + + assert_enqueue([queue: :default, worker: MyApp.Worker], fn -> + MyApp.do_some_business() + end) + + Make an assertion about the return value: + + result = assert_enqueue([worker: MyApp.Worker], &MyApp.more_business/0) + + assert {:ok, _} = result + """ + @doc since: "1.1.0" + @spec assert_enqueue(keyword(), (-> return)) :: return when return: any() + def assert_enqueue(opts, fun) when is_function(fun, 0) do + opts = expand_decorated(opts) + + {enqueued, returned} = diff_enqueued(opts, fun) + + if Enum.any?(enqueued) do + returned + else + flunk(""" + Expected a job matching: + + #{inspect_opts(opts)} + + to be enqueued. + """) + end + end + + @doc """ + Assert that a job with particular criteria is enqueued. + + This is a wrapper around `Oban.Testing.assert_enqueued/2` with additions to support jobs built + with `Oban.Pro.Decorator` decorated functions. + + Only values for the provided fields are checked. For example, an assertion made on `worker: + "MyWorker"` will match _any_ jobs for that worker, regardless of every other field. + + ## Options + + The following options are supported in addition to all `t:Oban.Job` fields: + + * `:decorated` — a function capture for the decorated function, e.g. `&MyApp.Foo.bar/2`. By + default, only the module and function name are matched. To match on `args`, a _full list_ of + arguments must be provided rather than the standard map. + + See [shared options](#module-shared-repo-options) for additional repo-specific options. + + ## Examples + + Assert that a job is enqueued for a certain worker and args: + + assert_enqueued worker: MyWorker, args: %{id: 1} + + Assert that a job is enqueued for a particular queue and priority: + + assert_enqueued queue: :business, priority: 3 + + Assert that a job's args deeply match: + + assert_enqueued args: %{config: %{enabled: true}} + + Use the `:_` wildcard to assert that a job's meta has a key with any value: + + assert_enqueued meta: %{batch_id: :_} + + Assert on a decorated function with any args: + + assert_enqueued decorated: &MyApp.notify/2 + + Assert on exact args used to enqueue a decorated function: + + assert_enqueued args: ["example", 123], decorated: &MyApp.notify/2 + """ + @doc since: "0.11.0" + @spec assert_enqueued(keyword(), timeout() | :none) :: true + def assert_enqueued(opts, timeout \\ :none) do + opts = expand_decorated(opts) + + if timeout == :none do + Oban.Testing.assert_enqueued(opts) + else + Oban.Testing.assert_enqueued(opts, timeout) + end + end + + @doc """ + Refute that any jobs were enqueued during a function call. + + Any pre-existing jobs are ignored for the refutation. If the refutation passes then the + function's return value is passed back. + + ## Options + + See `refute_enqueued/2` for standard options, and [shared options](#module-shared-repo-options) + for additional repo-specific options. + + ## Examples + + Refute that a `MyApp.Worker` job was added to the `default` queue: + + refute_enqueue([queue: :default, worker: MyApp.Worker], fn -> + MyApp.do_some_business() + end) + + Make a refutation about the return value: + + result = refute_enqueue([worker: MyApp.Worker], &MyApp.more_business/0) + + assert {:ok, _} = result + """ + @doc since: "1.1.0" + @spec refute_enqueue(keyword(), (-> return)) :: return when return: any() + def refute_enqueue(opts, fun) when is_function(fun, 0) do + opts = expand_decorated(opts) + + {enqueued, returned} = diff_enqueued(opts, fun) + + if Enum.empty?(enqueued) do + returned + else + flunk(""" + Expected no jobs matching: + + #{inspect_opts(opts)} + + to be enqueued. + """) + end + end + + @doc """ + Refute that a job with particular criteria is enqueued. + + This is a wrapper around `Oban.Testing.refute_enqueued/2` with additions to support jobs built + with `Oban.Pro.Decorator` decorated functions. + + ## Options + + The following options are supported in addition to all `t:Oban.Job` fields: + + * `:decorated` — a function capture for the decorated function, e.g. `&MyApp.Foo.bar/2`. By + default, only the module and function name are matched. To match on `args`, a _full list_ of + arguments must be provided rather than the standard map. + + See [shared options](#module-shared-repo-options) for additional repo-specific options. + + ## Examples + + Refute that a job is enqueued for a certain worker: + + refute_enqueued worker: MyWorker + + Refute that a job is enqueued for a certain worker and args: + + refute_enqueued worker: MyWorker, args: %{id: 1} + + Refute that a job's nested args match: + + refute_enqueued args: %{config: %{enabled: false}} + + Use the `:_` wildcard to refute that a job's meta has a key with any value: + + refute_enqueued meta: %{batch_id: :_} + + Refute a decorated function was enqueued with any args: + + refute_enqueued decorated: &MyApp.notify/2 + + Refute that exact args were used to enqueue a decorated function: + + refute_enqueued args: ["example", 123], decorated: &MyApp.notify/2 + """ + @doc since: "0.11.0" + @spec refute_enqueued(keyword(), timeout() | :none) :: false + def refute_enqueued(opts, timeout \\ :none) do + opts = expand_decorated(opts) + + if timeout == :none do + Oban.Testing.refute_enqueued(opts) + else + Oban.Testing.refute_enqueued(opts, timeout) + end + end + + @doc """ + Synchronously execute jobs in one or all queues, from within the current process. + + Jobs that are enqueued by a process when `Ecto` is in sandbox mode are only visible to that + process. Calling `drain_jobs/1` allows you to control when the jobs are executed and to wait + synchronously for all jobs to complete. + + This function provides several distinct advantages over the standard `Oban.drain_queue/2`: + + * It can drain jobs across one or all queues simultaneously + * It can return the drained jobs rather than a count summary + * It optimizes the defaults for testing batches, workflows, etc. + * It always uses the `Smart` engine to guarantee that Pro worker features work as expected + + ## Options + + * `:queue` - an atom specifying the queue to drain, or `:all` to drain jobs across all queues at + once. Defaults to `:all` when no queue is provided. + + * `:with_limit` — the maximum number of jobs to fetch for draining at once. The limit only + impacts how many jobs are fetched at once, _not_ concurrency. When recursion is enabled this + is how many jobs are processed per-iteration, and it defaults to `1`. Otherwise, there isn't a + limit and all available jobs are fetched. + + * `:with_recursion` — whether to draining jobs recursively, or all in a single pass. Either way, + jobs are processed sequentially, one at a time. Recursion is required when jobs insert other + jobs (e.g. batches), or depend on the execution of other jobs (e.g. workflows). Defaults to + `true`. + + * `:with_safety` — whether to silently catch errors when draining. When `false`, raised + exceptions or unhandled exits are reraised (unhandled exits are wrapped in `Oban.CrashError`). + Defaults to `false`. + + * `:with_scheduled` — whether to include scheduled or retryable jobs when draining. In recursive + mode, which is the default, this will include snoozed jobs, and may lead to an infinite loop if + the job snoozes repeatedly. Defaults to `true`. + + * `:with_summary` — whether to summarize execution results with a map of counts by state, or + return a list of each job that was drained. Defaults to `true`, which returns a summary map. + + See [shared options](#module-shared-repo-options) for additional repo-specific options. + + ## Examples + + Drain all available jobs across all queues: + + assert %{completed: 3} = drain_jobs(queue: :all) + + Drain and return all executed jobs without a count summary: + + assert [%Oban.Job{}, %Oban.Job{}] = drain_jobs(with_summary: false) + + Drain including a job that you expect to raise: + + assert_raise RuntimeError, fn -> drain_jobs(queue: :risky) end + + Drain without recursion to identify snoozed jobs: + + assert %{scheduled: 3} = drain_jobs(with_recursion: false) + + Drain without staging scheduled jobs: + + assert %{completed: 1, scheduled: 0} = drain_jobs(with_scheduled: false) + + Drain within a custom prefix: + + assert %{completed: 3} = drain_jobs(queue: :default, prefix: "private") + + Drain using a specific repo (necessary when calling this function directly): + + assert %{completed: 3} = drain_jobs(queue: :default, repo: MyApp.Repo) + """ + @doc since: "0.11.0" + @spec drain_jobs([drain_option()]) :: drain_result() + def drain_jobs(opts) when is_list(opts) do + conf = init_conf(opts) + opts = Keyword.drop(opts, @conf_keys) + + with_limit = if Keyword.get(opts, :with_recursion, true), do: 1, else: 999_999 + + opts = + opts + |> Map.new() + |> Map.put_new(:queue, :all) + |> Map.update!(:queue, &if(&1 == :all, do: :__all__, else: &1)) + |> Map.put_new(:with_limit, with_limit) + |> Map.put_new(:with_recursion, true) + |> Map.put_new(:with_safety, false) + |> Map.put_new(:with_scheduled, true) + |> Map.put_new(:with_summary, true) + + Process.put(:oban_draining, true) + + drain(conf, [], opts) + after + Process.delete(:oban_draining) + end + + @doc """ + Execute a job using the given config options. + + This is a wrapper around `Oban.Testing.perform_job/2` that ensures the Smart engine is used for + compatibility with Pro worker features. + + See `perform_job/3` for more details and examples. + + ## Options + + See [shared options](#module-shared-repo-options) for additional repo-specific options. + + ## Examples + + Execute a job without any options: + + job = build_job(MyWorker, %{id: 1}) + assert :ok = perform_job(job) + + Execute a job with a custom prefix and repo: + + assert :ok = perform_job(job, prefix: "private", repo: MyApp.Repo) + """ + @doc since: "1.6.8" + @spec perform_job(Job.t(), [repo_option()]) :: Worker.result() + def perform_job(%Job{} = job, opts) when is_list(opts) do + opts = Keyword.put_new(opts, :engine, Smart) + + Oban.Testing.perform_job(job, opts) + end + + @doc """ + Construct a job and execute it with a worker module. + + This is a wrapper around `Oban.Testing.perform_job/3` that ensures the Smart engine is used for + compatibility with Pro worker features. + + ## Options + + See [shared options](#module-shared-repo-options) for additional repo-specific options. + + ## Examples + + Successfully execute a job with some string arguments: + + assert :ok = perform_job(MyWorker, %{"id" => 1}) + + Successfully execute a job and assert that it returns an error tuple: + + assert {:error, _} = perform_job(MyWorker, %{"bad" => "arg"}) + + Execute a job with the args keys automatically stringified: + + assert :ok = perform_job(MyWorker, %{id: 1}) + """ + @doc since: "0.11.0" + @spec perform_job(Worker.t(), term(), [perform_option()]) :: Worker.result() + def perform_job(worker, args, opts) do + opts = Keyword.put_new(opts, :engine, Smart) + + Oban.Testing.perform_job(worker, args, opts) + end + + @doc """ + Change the testing mode within the context of a function. + + This is a direct delegation to `Oban.Testing.with_testing_mode/2`. Only `:manual` and `:inline` + mode are supported, as `:disabled` implies that supervised queues and plugins are running and + this function won't start any processes. + + ## Examples + + Switch to `:manual` mode when an Oban instance is configured for `:inline` testing: + + with_testing_mode(:manual, fn -> + Oban.insert(MyWorker.new(%{id: 123})) + + assert_enqueued worker: MyWorker, args: %{id: 123} + end) + + Visa-versa, switch to `:inline` mode: + + with_testing_mode(:inline, fn -> + {:ok, %Job{state: "completed"}} = Oban.insert(MyWorker.new(%{id: 123})) + end) + """ + @doc since: "1.6.8" + @spec with_testing_mode(:inline | :manual, (-> any())) :: any() + defdelegate with_testing_mode(mode, fun), to: Oban.Testing + + @doc """ + Construct and execute a job with a batch `handle_*` callback. + + This helper verifies that the batch worker exports the requested callback handler, + along with the standard assertions made by `perform_job/3`. + + ## Examples + + Execute the `handle_attempted` callback without any args: + + assert :ok = perform_callback(MyBatch, :attempted, %{}) + + Execute the `handle_exhausted` callback with args: + + assert :ok = perform_callback(MyBatch, :exhausted, %{for_account_id: 123}) + """ + @doc since: "0.11.0" + @spec perform_callback(Worker.t(), callback(), term(), [perform_option()]) :: Worker.result() + def perform_callback(worker, callback, args, opts) when is_list(opts) do + assert_valid_callback(worker, callback) + + opts = + opts + |> Keyword.put_new(:meta, %{}) + |> Keyword.update!(:meta, &Map.put_new(&1, "callback", to_string(callback))) + |> Keyword.update!(:meta, &Map.put_new(&1, "batch_id", Oban.Pro.UUIDv7.generate())) + + perform_job(worker, args, opts) + end + + @doc """ + Construct a list of jobs and process them with a Chunk worker. + + Like `perform_job/3`, this helper reduces boilerplate when constructing jobs and checks for + common pitfalls. Unlike `perform_job/3`, this helper calls the chunk's `process/1` function + directly and it **won't trigger telemetry events**. + + ## Examples + + Successfully process a chunk of jobs: + + assert :ok = perform_chunk(MyChunk, [%{id: 1}, %{id: 2}]) + + Process a chunk of jobs with job options: + + assert :ok = perform_chunk(MyChunk, [%{id: 1}, %{id: 2}], attempt: 5, priority: 3) + """ + @doc since: "0.11.0" + @spec perform_chunk(Worker.t(), [term()], [perform_option()]) :: Chunk.result() + def perform_chunk(worker, args_list, opts) when is_list(args_list) and is_list(opts) do + {conf_opts, opts} = Keyword.split(opts, @conf_keys) + + conf = Config.new(conf_opts) + + opts = + opts + |> Keyword.merge(worker.__opts__()) + |> Keyword.put_new(:attempt, 1) + + chunk = + Enum.reduce_while(args_list, [], fn args, acc -> + now = DateTime.utc_now() + + job = + args + |> worker.new(opts) + |> Changeset.update_change(:args, &json_encode_decode/1) + |> Changeset.put_change(:attempted_at, now) + |> Changeset.put_change(:scheduled_at, now) + |> Changeset.apply_action!(:insert) + |> Map.replace!(:conf, conf) + + case Oban.Pro.Worker.before_process(worker, job, opts) do + {:ok, job} -> + {:cont, [job | acc]} + + {:error, error} -> + {:halt, {:error, error}} + end + end) + + if is_list(chunk) do + chunk + |> Enum.reverse() + |> worker.process() + |> tap(&assert_valid_chunk_result/1) + else + chunk + end + end + + @doc """ + Insert and execute a complete batch of jobs, along with callbacks, within the test process. + + ## Options + + Accepts all options for `drain_jobs/1`, including the repo-specific options listed in + [shared-options](#module-shared-repo-options). + + ## Examples + + Run a batch: + + ids + |> Enum.map(&MyBatch.new(%{id: &1})) + |> MyBatch.new_batch() + |> run_batch() + + Run a batch with a specific repo (necessary when calling this function directly): + + run_batch(my_batch, repo: MyApp.Repo, prefix: "private") + """ + @doc since: "0.11.0" + @spec run_batch(Batch.t(), [drain_option()]) :: drain_result() + def run_batch(batch, opts) when is_list(opts) do + opts = Keyword.put(opts, :batch_ids, [batch.opts.batch_id]) + + batch.changesets + |> Enum.to_list() + |> run_jobs(opts) + end + + @doc """ + Insert and execute chained jobs within the test process. + + ## Options + + Accepts all options for `drain_jobs/1`, including the repo-specific options listed in + [shared-options](#module-shared-repo-options). + + ## Examples + + Run all jobs in a chain: + + 1..10 + |> Enum.map(&MyChain.new(%{id: &1})) + |> run_chain() + """ + @doc since: "1.1.0" + @doc deprecated: "Use run_jobs/2 instead" + def run_chain(chain, opts) when is_list(opts) do + chain_keys = ~w(hold_snooze on_cancelled on_discarded wait_retry wait_sleep wait_snooze)a + chain_opts = opts |> Keyword.take(chain_keys) |> Map.new() + + chain + |> Enum.map(fn changeset -> + Changeset.update_change(changeset, :meta, &Map.merge(&1, chain_opts)) + end) + |> run_jobs(opts) + end + + @doc """ + Insert and execute chunked jobs within the test process. + + This helper overrides the chunk's `timeout` to force immediate processing of jobs up to the + chunk size. + + ## Options + + Accepts all options for `drain_jobs/1`, including the repo-specific options listed in + [shared-options](#module-shared-repo-options). + + ## Examples + + Run jobs in chunks: + + 1..50 + |> Enum.map(&MyChunk.new(%{id: &1})) + |> run_chunk() + + Run a chunk with an overriden size: + + 1..10 + |> Enum.map(&MyChunk.new(%{id: &1})) + |> run_chunk(size: 10) + + Run chunks with an explicit repo (necessary when calling this function directly): + + run_chunk(changesets, repo: MyApp.Repo, prefix: "private") + + Run chunks only for a specific queue: + + run_chunk(changesets, queue: "default") + """ + @doc since: "0.11.0" + @spec run_chunk([Job.changeset()], [drain_option()]) :: drain_result() + def run_chunk([_ | _] = chunk, opts) when is_list(opts) do + chunk_opts = + opts + |> Keyword.take(~w(by leading size)a) + |> Map.new(fn + {:by, by} -> {:chunk_by, Utils.normalize_by(by)} + {:leading, leading} -> {:chunk_leading, leading} + {:size, size} -> {:chunk_size, size} + end) + + merge = fn meta -> Map.merge(meta, chunk_opts) end + + chunk + |> Enum.map(fn change -> Changeset.update_change(change, :meta, merge) end) + |> run_jobs(Keyword.put_new(opts, :with_limit, 1)) + end + + @doc """ + Insert and execute jobs synchronously, within the test process. + + This is the basis of all other `run_*` helpers. + + ## Options + + Accepts all options for `drain_jobs/1`, including the repo-specific options listed in + [shared-options](#module-shared-repo-options). + + ## Examples + + Run a list of jobs: + + ids + |> Enum.map(&MyWorker.new(%{id: &1})) + |> run_jobs() + + Run jobs with an explicit repo (necessary when calling this function directly): + + run_jobs(changesets, repo: MyApp.Repo) + """ + @doc since: "0.11.0" + @spec run_jobs([Job.changeset()], [drain_option()]) :: drain_result() + def run_jobs([_ | _] = changesets, opts) when is_list(changesets) and is_list(opts) do + opts + |> init_conf() + |> Smart.insert_all_jobs(changesets, []) + + opts + |> Keyword.put_new(:queue, :all) + |> drain_jobs() + end + + @doc """ + Insert and execute a workflow synchronously, within the test process. + + This helper augments the workflow with options optimized for testing, but it will still respect + all standard workflow options. + + ## Options + + Accepts all options for `drain_jobs/1`, including the repo-specific options listed in + [shared-options](#module-shared-repo-options). + + ## Examples + + Run a basic workflow: + + MyFlow.new_workflow() + |> MyFlow.add(:a, MyFlow.new(%{id: 1})) + |> MyFlow.add(:b, MyFlow.new(%{id: 2}), deps: [:a]) + |> MyFlow.add(:c, MyFlow.new(%{id: 3}), deps: [:b]) + |> run_workflow() + + Run a workflow and match on returned jobs, but be careful that the execution order may differ + from the insertion order: + + workflow = + MyFlow.new_workflow() + |> MyFlow.add(:a, MyFlow.new(%{id: 1})) + |> MyFlow.add(:b, MyFlow.new(%{id: 2}), deps: [:a]) + |> MyFlow.add(:c, MyFlow.new(%{id: 3}), deps: [:b]) + + [_job_a, _job_b, _job_c] = run_workflow(workflow, with_summary: false) + + Run a workflow with increased parallelism to prevent slow or deadlocked workflows: + + run_workflow(my_expansive_workflow, with_limit: 5) + + Run a workflow with an explicit repo (necessary when calling this function directly): + + run_workflow(workflow, repo: MyApp.Repo) + """ + @doc since: "0.11.0" + @spec run_workflow(Workflow.t(), Workflow.new_opts() | drain_option()) :: drain_result() + def run_workflow(%Workflow{} = workflow, opts \\ []) when is_list(opts) do + conf = init_conf(opts) + + Smart.insert_all_jobs(conf, workflow.changesets, []) + + workflow_ids = + workflow.changesets + |> Enum.map(& &1.changes.meta.workflow_id) + |> Enum.uniq() + + opts + |> Keyword.put(:workflow_ids, workflow_ids) + |> drain_jobs() + + # The workflow facilitator won't run "held" jobs, so we have to reload everything that was + # inserted to get all jobs and their real fields. Additionally, the workflow ids may be added + # while the workflow is executing so they must be refreshed afterwards. + workflow_ids = all_workflow_ids(conf, workflow_ids) + + jobs = + Job + |> where([j], j.meta["workflow_id"] in ^workflow_ids) + |> then(&Repo.all(conf, &1)) + + if Keyword.get(opts, :with_summary, true) do + Enum.reduce(jobs, @empty_summary, fn job, acc -> + state = + if job.state == "discarded" and job.attempt >= job.max_attempts do + :exhausted + else + String.to_existing_atom(job.state) + end + + Map.update(acc, state, 1, &(&1 + 1)) + end) + else + jobs + end + end + + @doc """ + Start a supervised Oban instance under the test supervisor. + + All valid Oban options are accepted. The supervised instance is registered with a unique + reference, rather than the default `Oban`. That prevents any conflict with Oban instances + started by your Application, or with other tests running asynchronously. + + Furthermore, this helper automatically adds sandbox allowances for any plugins or queue + producers, allowing tests to run async. + + After the test finishes the test process will wait for the Oban instance to shut down cleanly. + + ## Running Jobs + + By default, the supervised instance won't process any jobs because the `stage_interval` is set + to `:infinity`. Set the `stage_interval` to a low value to process jobs normally, without manual + draining. + + ## Options + + Any option accepted by `Oban.start_link/1` is acceptable, including the repo-specific options + listed in [shared options](#module-shared-repo-options). + + ## Examples + + Start a basic supervised instance without any queues or plugins: + + name = start_supervised_oban!(repo: MyApp.Repo) + Oban.insert(name, MyWorker.new()) + + Start the supervisor with a single queue and polling every 10ms: + + start_supervised_oban!(repo: MyApp.Repo, stage_interval: 10, queues: [alpha: 10]) + """ + @doc since: "0.11.0" + @spec start_supervised_oban!([Oban.option()]) :: Registry.key() + def start_supervised_oban!(opts) when is_list(opts) do + opts = conf_opts(opts) + name = Keyword.fetch!(opts, :name) + repo = Keyword.fetch!(opts, :repo) + + attach_auto_allow(repo, name) + + start_supervised!({Oban, opts}) + + name + |> Oban.config() + |> ensure_queues_started() + + name + end + + defp json_encode_decode(map) do + map + |> Oban.JSON.encode!() + |> Oban.JSON.decode!() + end + + # Enqueued Helpers + + defp diff_enqueued(opts, fun) do + existing = all_enqueued(opts) + returned = fun.() + inserted = all_enqueued(opts) + + existing_ids = Enum.map(existing, & &1.id) + difference = Enum.reject(inserted, &(&1.id in existing_ids)) + + {difference, returned} + end + + defp inspect_opts(opts) do + opts + |> Keyword.drop([:log, :prefix, :repo]) + |> inspect(charlists: :as_lists, pretty: true) + end + + defp expand_decorated(opts) do + case Keyword.fetch(opts, :decorated) do + {:ok, fun} when is_function(fun) -> + info = Function.info(fun) + args = %{mod: inspect(info[:module]), fun: info[:name], arg: args_to_term(info, opts)} + + opts + |> Keyword.delete(:decorated) + |> Keyword.merge(args: args, worker: Oban.Pro.Decorator) + + {:ok, term} -> + raise ArgumentError, + "expected :decorated option to be a function capture, got: #{inspect(term)}" + + :error -> + opts + end + end + + defp args_to_term(info, opts) do + arity = info[:arity] + + case Keyword.fetch(opts, :args) do + {:ok, list} when is_list(list) and length(list) == arity -> + "term-" <> Utils.encode64(list) + + {:ok, list} when is_list(list) -> + raise ArgumentError, "expected :args list to have arity #{arity}, got: #{length(list)}" + + {:ok, term} -> + raise ArgumentError, "expected :args option to be a list, got: #{inspect(term)}" + + :error -> + :_ + end + end + + # Callback Helpers + + defp assert_valid_callback(worker, callback) do + assert is_atom(callback) and callback in @callbacks, """ + Expected callback to be included in #{inspect(@callbacks)}, got: #{inspect(callback)} + """ + + assert Code.ensure_loaded?(worker), """ + Expected worker to be an existing module, got: #{inspect(worker)} + """ + + new_fun = Map.fetch!(Batch.callbacks_to_functions(), to_string(callback)) + old_fun = Map.fetch!(Batch.callbacks_to_deprecated(), to_string(callback)) + + assert function_exported?(worker, new_fun, 1) or function_exported?(worker, old_fun, 1), """ + Expected #{inspect(new_fun)} callback to be implemented + """ + end + + # Startup Helpers + + defp attach_auto_allow(repo, name) do + telemetry_name = "oban-auto-allow-#{inspect(name)}" + + :telemetry.attach_many( + telemetry_name, + [[:oban, :engine, :init, :start], [:oban, :plugin, :init]], + &__MODULE__.auto_allow/4, + {name, repo, self()} + ) + + on_exit(name, fn -> :telemetry.detach(telemetry_name) end) + end + + @doc false + def auto_allow(_event, _measure, %{conf: conf}, {name, repo, test_pid}) do + if conf.name == name, do: Sandbox.allow(repo, test_pid, self()) + end + + defp ensure_queues_started(%{name: name, queues: queues}) do + with [_ | _] <- queues do + queues + |> Keyword.keys() + |> Enum.each(&Oban.check_queue(name, queue: &1)) + end + end + + # Draining Helpers + + defp conf_opts(opts) do + conf_opts = Keyword.take(opts, @conf_keys) + + @default_supervised_opts + |> Keyword.merge(conf_opts) + |> Keyword.put_new(:name, make_ref()) + end + + defp init_conf(opts) do + opts + |> conf_opts() + |> Config.new() + |> tap(&Registry.register(Oban.Registry, &1.name, &1)) + end + + defp drain(conf, acc, opts) do + if opts.with_scheduled, do: stage_scheduled(conf) + + case Smart.fetch_for_drain(conf, opts) do + [_ | _] = jobs -> + executed = + jobs + |> streamline_pro_workers() + |> Enum.map(fn job -> + conf + |> Executor.new(job, safe: opts.with_safety) + |> Executor.call() + end) + + if opts.with_recursion do + drain(conf, acc ++ executed, with_workflow_ids(conf, opts)) + else + complete_drain(conf, executed, opts) + end + + [] -> + case opts do + %{workflow_ids: [_ | _], flushed: true} -> + complete_drain(conf, acc, opts) + + %{workflow_ids: [_ | _] = workflow_ids} -> + # Final flush to release held jobs whose deps became satisfied during processing. + Enum.each(workflow_ids, &Workflow.on_flush(&1, conf)) + + drain(conf, acc, Map.put(opts, :flushed, true)) + + _ -> + complete_drain(conf, acc, opts) + end + end + end + + defp stage_scheduled(conf) do + query = + Job + |> where([j], j.state in ["scheduled", "retryable"]) + |> where([j], not fragment("? @> ?", j.meta, ^%{on_hold: true})) + + Repo.update_all(conf, query, set: [state: "available"]) + end + + defp streamline_pro_workers(jobs) do + Enum.map(jobs, fn %{meta: meta} = job -> + case meta do + %{"chain_key" => _} -> + meta = + meta + |> Map.put_new("wait_retry", 1) + |> Map.put_new("wait_sleep", 1) + |> Map.put_new("wait_snooze", 1) + + %{job | meta: meta} + + %{"chunk_size" => _} -> + meta = + meta + |> Map.put("chunk_sleep", 1) + |> Map.put("chunk_timeout", 0) + + %{job | meta: meta} + + _ -> + job + end + end) + end + + defp with_workflow_ids(conf, opts) do + case opts do + %{workflow_ids: [_ | _]} -> + Map.put(opts, :workflow_ids, all_workflow_ids(conf, opts.workflow_ids)) + + _ -> + opts + end + end + + defp all_workflow_ids(conf, [_ | _] = workflow_ids) do + all_workflow_ids(conf, workflow_ids, MapSet.new(workflow_ids)) + end + + defp all_workflow_ids(_conf, ids), do: ids + + defp all_workflow_ids(conf, search_ids, seen) do + graft_query = + Job + |> select([j], j.meta["graft_id"]) + |> where([j], j.meta["workflow_id"] in ^search_ids) + |> where([j], not is_nil(j.meta["graft_id"])) + |> distinct(true) + + graft_ids = Repo.all(conf, graft_query) + all_search_ids = search_ids ++ graft_ids + + query = + Job + |> select([j], j.meta["workflow_id"]) + |> where([j], j.meta["sup_workflow_id"] in ^all_search_ids) + |> distinct(true) + + new_ids = + conf + |> Repo.all(query) + |> Enum.reject(&MapSet.member?(seen, &1)) + + case new_ids do + [] -> MapSet.to_list(seen) + ids -> all_workflow_ids(conf, ids, MapSet.union(seen, MapSet.new(ids))) + end + end + + defp complete_drain(_conf, executed, %{with_summary: true}) do + Enum.reduce(executed, @empty_summary, fn exec, acc -> + state = + case exec.state do + :cancelled -> :cancelled + :discard -> :discarded + :exhausted -> :exhausted + :failure -> :retryable + :snoozed -> :scheduled + :success -> :completed + end + + Map.update(acc, state, 1, &(&1 + 1)) + end) + end + + defp complete_drain(conf, executed, _opts) do + ids = + executed + |> Enum.map(& &1.job.id) + |> Enum.uniq() + + jobs = + conf + |> Repo.all(where(Job, [j], j.id in ^ids)) + |> Map.new(fn job -> {job.id, job} end) + + Enum.map(ids, &Map.fetch!(jobs, &1)) + end + + # Chunk Helpers + + @ops ~w(cancel discard error)a + + defp assert_valid_chunk_result(result) do + valid? = + case result do + :ok -> + true + + {:ok, _value} -> + true + + {ops, _reason, [_ | _]} when ops in @ops -> + true + + [{_op, {_reason, _jobs}} | _] = list -> + Keyword.keyword?(list) and + Enum.all?(list, &match?({_key, {_reason, [_ | _]}}, &1)) + + _ -> + false + end + + assert valid?, """ + Expected result to be one of + + - `:ok` + - `{:ok, value}` + - `{:cancel, reason, jobs}` + - `{:discard, reason, jobs}` + - `{:error, reason, jobs}` + - `[cancel: {reason, jobs}, discard: {reason, jobs}, error: {reason, jobs}]` + + Instead received: + + #{inspect(result, pretty: true)} + """ + end +end diff --git a/vendor/oban_pro/lib/oban/pro/unique.ex b/vendor/oban_pro/lib/oban/pro/unique.ex new file mode 100644 index 00000000..cc142430 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/unique.ex @@ -0,0 +1,87 @@ +defmodule Oban.Pro.Unique do + @moduledoc false + + alias Ecto.Changeset + alias Oban.Job + alias Oban.Pro.Utils + + @states_to_ints %{ + "scheduled" => 0, + "available" => 1, + "executing" => 2, + "retryable" => 3, + "completed" => 4, + "cancelled" => 5, + "discarded" => 6 + } + + @spec replace?(Job.changeset()) :: boolean() + def replace?(changeset), do: match?(%{changes: %{replace: _}}, changeset) + + @spec unique?(Job.changeset()) :: boolean() + def unique?(changeset), do: match?(%{changes: %{unique: %{}}}, changeset) + + @spec get_key(Job.changeset(), any()) :: any() + def get_key(changeset, default \\ nil) + + def get_key(%{changes: %{meta: %{uniq_key: uniq_key}}}, _default), do: uniq_key + def get_key(_changeset, default), do: default + + @spec get_states(Job.changeset()) :: [atom()] + def get_states(%{changes: %{unique: %{states: states}}}), do: states + + # For backward compatibility, `unique` works with non-pro workers, so it can't be injected into + # the meta through a stage. + @spec with_uniq_meta(Job.changeset()) :: Job.changeset() + def with_uniq_meta(%{changes: %{unique: %{period: _}}} = changeset) do + uniq_meta = %{uniq: true, uniq_bmp: gen_bmp(changeset), uniq_key: gen_key(changeset)} + + meta = + changeset + |> Changeset.get_change(:meta, %{}) + |> Map.merge(uniq_meta) + + Changeset.put_change(changeset, :meta, meta) + end + + def with_uniq_meta(changeset), do: changeset + + @spec gen_bmp(Job.changeset()) :: [integer()] + def gen_bmp(%{changes: %{unique: %{states: states}}}) do + for state <- states, do: Map.fetch!(@states_to_ints, to_string(state)) + end + + @spec gen_key(Job.changeset()) :: String.t() + def gen_key(%{changes: %{unique: unique}} = changeset) do + %{fields: fields, keys: keys, period: period, timestamp: timestamp} = unique + + data = + fields + |> Enum.sort() + |> Enum.map(fn + :args -> Utils.take_keys(changeset, :args, keys) + :meta -> Utils.take_keys(changeset, :meta, keys) + field -> Changeset.get_field(changeset, field) + end) + + data = + if period == :infinity do + data + else + [truncate(period, Changeset.get_field(changeset, timestamp)) | data] + end + + Utils.hash64(data) + end + + defp truncate(period, timestamp) do + now = timestamp || DateTime.utc_now() + + now + |> DateTime.to_unix(:second) + |> rem(period) + |> then(&DateTime.add(now, -&1)) + |> DateTime.truncate(:second) + |> DateTime.to_iso8601() + end +end diff --git a/vendor/oban_pro/lib/oban/pro/utils.ex b/vendor/oban_pro/lib/oban/pro/utils.ex new file mode 100644 index 00000000..92e28feb --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/utils.ex @@ -0,0 +1,159 @@ +defmodule Oban.Pro.Utils do + @moduledoc false + + alias Ecto.Changeset + + @spec encode64(term(), [:compressed | :deterministic]) :: String.t() + def encode64(term, opts \\ [:compressed]) do + term + |> :erlang.term_to_binary(opts) + |> Base.encode64(padding: false) + end + + @spec decode64(binary(), [:safe | :used]) :: term() + def decode64(bin, opts \\ [:safe]) do + bin + |> Base.decode64!(padding: false) + |> :erlang.binary_to_term(opts) + end + + @spec hash64(iodata()) :: String.t() + def hash64(iodata) when is_binary(iodata) or is_list(iodata) do + :blake2s + |> :crypto.hash(iodata) + |> Base.encode64(padding: false) + end + + @spec persistent_cache(tuple(), fun()) :: any() + def persistent_cache(key, fun) when is_function(fun, 0) do + case :persistent_term.get(key, nil) do + nil -> tap(fun.(), &:persistent_term.put(key, &1)) + val -> val + end + end + + @spec validate_opts(list(), fun()) :: :ok | {:error, term()} + def validate_opts(opts, validator) do + Enum.reduce_while(opts, :ok, fn opt, acc -> + case validator.(opt) do + {:error, _reason} = error -> {:halt, error} + _ -> {:cont, acc} + end + end) + end + + @spec cast_period(pos_integer() | {atom(), pos_integer()}) :: pos_integer() + def cast_period({value, unit}) do + unit = to_string(unit) + + cond do + unit in ~w(second seconds) -> value + unit in ~w(minute minutes) -> value * 60 + unit in ~w(hour hours) -> value * 60 * 60 + unit in ~w(day days) -> value * 24 * 60 * 60 + true -> unit + end + end + + def cast_period(period), do: period + + @spec enforce_keys(Changeset.t(), map(), module()) :: Changeset.t() + def enforce_keys(changeset, params, schema) do + fields = + [:fields, :virtual_fields] + |> Enum.flat_map(&schema.__schema__/1) + |> Enum.map(&to_string/1) + + Enum.reduce(params, changeset, fn {key, _val}, acc -> + if to_string(key) in fields do + acc + else + Changeset.add_error(acc, :base, "unknown field #{key} provided") + end + end) + end + + @spec take_keys(Changeset.t(), atom(), list()) :: [any()] + def take_keys(changeset, field, keys) do + changeset + |> Changeset.get_field(field) + |> take_keys(keys) + end + + @spec take_keys(map(), [atom()]) :: [any()] + def take_keys(map, _keys) when map_size(map) == 0, do: [] + + def take_keys(map, []) when is_map(map), do: take_keys(map, Map.keys(map)) + + def take_keys(map, keys) when is_map(map) and is_list(keys) do + map = Map.new(map, fn {key, val} -> {to_string(key), val} end) + + keys + |> Enum.map(&to_string/1) + |> Enum.sort() + |> Enum.map(fn key -> + val = map |> Map.get(key) |> to_hash_val() + + [key, val] + end) + end + + if Application.compile_env(:oban_pro, [__MODULE__, :safe_hash], false) do + defp to_hash_val(val) when is_binary(val), do: val + defp to_hash_val(val) when is_number(val), do: to_string(val) + defp to_hash_val(val) when is_atom(val), do: to_string(val) + defp to_hash_val(val), do: inspect(val) + else + defp to_hash_val(val), do: val |> :erlang.phash2() |> Integer.to_string() + end + + def normalize_by(by) do + by + |> List.wrap() + |> Enum.map(fn + {key, val} -> [key, val |> List.wrap() |> Enum.sort()] + field -> field + end) + end + + @spec to_exception(Changeset.t()) :: Exception.t() + def to_exception(changeset) do + changeset + |> to_translated_errors() + |> Enum.reverse() + |> Enum.map(fn {field, message} -> ArgumentError.exception("#{field} #{message}") end) + |> List.first() + end + + @spec to_translated_errors(Changeset.t()) :: Keyword.t() + def to_translated_errors(changeset) do + changeset + |> Changeset.traverse_errors(&translate_errors/1) + |> Enum.map(&extract_errors/1) + |> List.flatten() + end + + ## Conversions + + @spec maybe_to_atom(atom() | String.t()) :: atom() + def maybe_to_atom(key) when is_binary(key), do: String.to_existing_atom(key) + def maybe_to_atom(key), do: key + + # Helpers + + defp translate_errors({msg, opt}) do + Regex.replace(~r"%{(\w+)}", msg, fn _, key -> + opt + |> Keyword.get(String.to_existing_atom(key), key) + |> to_string() + end) + end + + defp extract_errors({_key, val}) when is_map(val) do + val + |> Map.to_list() + |> Enum.map(&extract_errors/1) + end + + defp extract_errors({key, [message | _]}), do: {key, message} +end diff --git a/vendor/oban_pro/lib/oban/pro/uuidv7.ex b/vendor/oban_pro/lib/oban/pro/uuidv7.ex new file mode 100644 index 00000000..0b406266 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/uuidv7.ex @@ -0,0 +1,42 @@ +defmodule Oban.Pro.UUIDv7 do + @moduledoc false + + # Derived from [bitwalker/uniq](https://github.com/bitwalker/uniq) + + defmacrop biguint(n), do: quote(do: big - unsigned - integer - size(unquote(n))) + + @spec generate() :: <<_::288>> + def generate do + time = System.system_time(:millisecond) + + <> = :crypto.strong_rand_bytes(10) + + <> = + <>, rand_b::62>> + + <> + end + + @compile {:inline, e: 1} + + defp e(0), do: ?0 + defp e(1), do: ?1 + defp e(2), do: ?2 + defp e(3), do: ?3 + defp e(4), do: ?4 + defp e(5), do: ?5 + defp e(6), do: ?6 + defp e(7), do: ?7 + defp e(8), do: ?8 + defp e(9), do: ?9 + defp e(10), do: ?a + defp e(11), do: ?b + defp e(12), do: ?c + defp e(13), do: ?d + defp e(14), do: ?e + defp e(15), do: ?f +end diff --git a/vendor/oban_pro/lib/oban/pro/validation.ex b/vendor/oban_pro/lib/oban/pro/validation.ex new file mode 100644 index 00000000..3b84119d --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/validation.ex @@ -0,0 +1,22 @@ +defmodule Oban.Pro.Validation do + @moduledoc false + + # Shared validations for use with `Oban.Validation.validate/2,3` + + @doc """ + Validate `by:` options for partitioning workers. + """ + def validate_by(:worker), do: :ok + def validate_by([{:args, key}]) when is_atom(key), do: :ok + def validate_by([{:meta, key}]) when is_atom(key), do: :ok + def validate_by([{:args, [key | _]}]) when is_atom(key), do: :ok + def validate_by([{:meta, [key | _]}]) when is_atom(key), do: :ok + def validate_by([:worker, {:args, key}]) when is_atom(key), do: :ok + def validate_by([:worker, {:meta, key}]) when is_atom(key), do: :ok + def validate_by([:worker, {:args, [key | _]}]) when is_atom(key), do: :ok + def validate_by([:worker, {:meta, [key | _]}]) when is_atom(key), do: :ok + + def validate_by(fields) do + {:error, "expected :by to be :worker or an :args/:meta tuple, got: #{inspect(fields)}"} + end +end diff --git a/vendor/oban_pro/lib/oban/pro/worker.ex b/vendor/oban_pro/lib/oban/pro/worker.ex new file mode 100644 index 00000000..9e93cf94 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/worker.ex @@ -0,0 +1,1239 @@ +defmodule Oban.Pro.Worker do + @moduledoc """ + `Oban.Pro.Worker` is a replacement for `Oban.Worker` with expanded capabilities. + + Major features: + + * [🏗️ Structured Jobs](#module-structured-jobs) — types for job args with validation, casting, + defaults, enums and more. + * [📼 Recorded Jobs](#module-recorded-jobs) — store a job's return output for retrieval later + by other jobs, in a console, or in the Web dashboard. + * [🔗 Chained Jobs](#module-chained-jobs) — link jobs together to ensure they run in a strict + sequential order regardless of scheduling or retries. + * [🔐 Encrypted Jobs](#module-encrypted-jobs) — store job args with encryption at rest so sensitive + data can't be seen from the database or Web dashboard. + * [🪦 Job Deadlines](#module-job-deadlines) — preemptively cancel jobs that shouldn't run after a + period of time has elapsed. + * [🧑‍🤝‍🧑 Worker Aliases](#module-worker-aliases) — aliasing allows jobs enqueued with the original + worker name to continue without exceptions using the new worker code. + * [🪝 Worker Hooks](#module-worker-hooks) — callbacks triggered around job execution, defined as + callback functions on the worker, or in a separate module for reuse. + + ## Usage + + Using `Oban.Pro.Worker` is identical to using `Oban.Worker`, with a few additional options. All + of the basic options such as `queue`, `priority`, and `unique` are still available along with + more advanced options. + + To create a basic Pro worker point `use` at `Oban.Pro.Worker` and define a `process/1` callback: + + ```elixir + def MyApp.Worker do + use Oban.Pro.Worker + + @impl Oban.Pro.Worker + def process(%Job{} = job) do + # Do stuff with the job + end + end + ``` + + If you have existing workers that you'd like to convert you only need to change the `use` + definition and replace `perform/1` with `process/1`. + + Without any of the advanced Pro features there isn't any difference between the basic and pro + workers. + + ## Structured Jobs + + Structured workers help you catch invalid data within your jobs by validating args on insert and + casting args before execution. They also automatically generate structs for compile-time checks + and friendly dot access. + + #### Defining a Structured Worker + + Structured workers use `args_schema/1` to define which fields are allowed, required, and their + expected types. Another benefit, aside from validation, is that args passed to `process/1` are + converted into a struct named after the worker module. Here's an example that demonstrates + defining a worker with several field types and embedded data structures: + + ```elixir + defmodule MyApp.StructuredWorker do + use Oban.Pro.Worker + + args_schema do + field :id, :id, required: true + field :name, :string, required: true + field :mode, :enum, values: ~w(enabled disabled paused)a, default: :enabled + field :xtra, :term + + embeds_one :data, required: true do + field :office_id, :uuid, required: true + field :has_notes, :boolean, default: false + field :addons, {:array, :string} + end + + embeds_many :addresses do + field :street, :string + field :city, :string + field :country, :string + end + end + + @impl Oban.Pro.Worker + def process(%Job{args: %__MODULE__{id: id, name: name, mode: mode, data: data}}) do + %{office_id: office_id, notes: notes} = data + + # Use the matched, cast values + end + end + ``` + + The example's schema declares five top level keys, `:id`, `:name`, `:mode`, `:data`, and + `:addresses`. Of those, only `:id`, `:name`, and the `:office_id` subkey are marked required. + The `:mode` field is an enum that validates values and casts to an atom. The embedded `:data` + field declares a nested map with its own type coercion and validation, including a custom + `Ecto.UUID` type. Finally, `:addresses` specifies an embedded list of maps. + + Job args are validated on `new/1` and errors bubble up to prevent insertion: + + ```elixir + StructuredWorker.new(%{id: "not-an-id", mode: "unknown"}).valid? + # => false (invalid id, invalid mode, missing name) + + StructuredWorker.new(%{id: "123", mode: "enabled"}).valid? + # => false (missing name) + + StructuredWorker.new(%{id: "123", name: "NewBiz", mode: "enabled"}).valid? + # => true + ``` + + This shows how args, stored as JSON, are cast before passing to `process/1`: + + ```elixir + # {"id":123,"name":"NewBiz","mode":"enabled","data":{"parent_id":456}} + + %MyApp.StructuredWorker{ + id: 123, + name: "NewBiz", + mode: :enabled, + data: %{parent_id:456} + } + ``` + + #### Structured Types and Casting + + Type casting and validation are handled by changesets. All types supported in Ecto schemas are + allowed, e.g. `:id`, `:integer`, `:string`, `:float`, or `:map`. See the [Ecto documentation for + a complete list of Ecto types][ecto] and their Elixir counterparts. + + [ecto]: https://hexdocs.pm/ecto/3.9.4/Ecto.Schema.html#module-types-and-casting + + #### Structured Options + + Structured fields support a few common options. + + * `:default` — Sets the default value for a field. The default value is calculated at + compilation time, so don't use expressions like `DateTime.utc_now/0` as they'd be the same for + all records. + + * `:required` — Validates that a value is provided for the field. Values that are `nil` or an + empty string aren't considered valid. + + #### Structured Extensions + + Structured workers support some convenient extensions beyond Ecto's standard type casting. + + * `:enum` — Maps atoms to strings based on a list of predefiend atoms passed like `values: + ~w(foo bar baz)a`. Both validates that values are included in the list and casts them to an + atom. + + * `:term` — Safely encodes any Elixir term as a string for storage, then decodes it back to the + original term on load. This is similar to `:any`, but works with terms like tuples or pids + that can't usually be serialied. For safety, terms are encoded with the `:safe` option to + prevent decoding data that may be used to attack the runtime. + + * `:uuid` — an intention revealing alias for `binary_id` + + * `{:array, *}` — A list of one or more values of any type, including `:enum` and `:uuid` + + * `embeds_one/2,3` — Declares a nested map with an explicit set of fields + + * `embeds_many/2,3` — Delcares a list of nested maps with an explicit set of fields + + #### Defining Typespecs for Structured Workers + + Typespecs aren't generated automatically. If desired, you must to define a sctuctured worker's + type manually: + + ```elixir + defmodule MyApp.StructuredWorker do + use Oban.Pro.Worker + + @type t :: %__MODULE__{id: integer(), active: boolean()} + ``` + + ## Recorded Jobs + + Sometimes the output of a job is just as important as any side effects. When that's the case, + you can use the `recorded` option to stash a job's output back into the job itself. Results are + compressed and safely encoded for retrieval later, either manually, in a batch callback, or a in + downstream workflow job. + + #### Defining a Recorded Worker + + ```elixir + defmodule MyApp.RecordedWorker do + use Oban.Pro.Worker, recorded: true + + @impl true + def process(%Job{args: args}) do + # Do your typical work here. + end + end + ``` + + If your process function returns an `{:ok, value}` tuple, it is recorded. Any other value, i.e. + an plain `:ok`, error, or snooze, is ignored. + + The example above uses `recorded: true` to opt into recording with the defaults. That means an + output `limit` of 64mb after compression and encoding—anything larger than the configured limit + will return an error tuple. If you expect larger results (and you want them stored in the + database) you can override the limit. For example, to set the limit to 64mb instead: + + ```elixir + use Oban.Pro.Worker, recorded: [limit: 128_000_000] + ``` + + ### Retrieving Results + + The `fetch_recorded/1` function is your ticket to extracting recorded results. If a job has ran + and recorded a value, it will return an `{:ok, result}` tuple: + + ```elixir + job = MyApp.Repo.get(Oban.Job, job_id) + + case MyApp.RecordedWorker.fetch_recorded(job) do + {:ok, result} -> + # Use the result + + {:error, :missing} -> + # Nothing recorded yet + end + ``` + + ## Chained Jobs + + Chains link jobs together to ensure they run in a strict sequential order. Downstream jobs in the + chain won't execute until the upstream job is `completed`, `cancelled`, or `discarded`. Behaviour + in the event of cancellation or discards is customizable to allow for uninterrupted processing or + holding for outside intervention + + Jobs in a chain only run after the previous job completes successfully, regardless of + scheduling, snoozing, or retries. + + #### Defining a Chain + + Chains are appropriate in situations where jobs are used to synchronize internal state with + outside state via events. For example, imagine a system that relies on webhooks from a payment + processor to track account balance: + + ```elixir + defmodule MyApp.WebhookWorker do + use Oban.Pro.Worker, queue: :webhooks, chain: [by: :worker] + + @impl true + def process(%Job{args: %{"account_id" => account_id, "event" => event}}) do + account_id + |> MyApp.Account.find() + |> MyApp.Account.handle_webhook_event(event) + end + end + ``` + + Now imagine that it's essential that jobs for an account are processed in order, while jobs from + separate accounts can run concurrently. Modify the `:by` option to partition by worker and + `:account_id`: + + ```elixir + defmodule MyApp.WebhookWorker do + use Oban.Pro.Worker, queue: :webhooks, chain: [by: [args: :account_id]] + + ... + ``` + + Webhooks for each account are guaranteed to run in order regardless of queue concurrency or + errors. The following section shows more partitioning examples. + + #### Chain Partitioning + + By default, chains are sequenced by `worker`, which means any job with the same worker forms a + chain. This approach may not always be suitable. For instance, you may want to link workers + based on a field like `:account_id` instead of just the worker. In such cases, you can use the + [`:by`](#t:chain_by/0) option to customize how chains are partitioned. + + Note that chaining always includes the `queue` field for performance and correctness. This means + that specifying `by: :worker` is equivalent to `by: [:queue, :worker]`, and `by: [args: :account_id]` + is equivalent to `by: [:queue, args: :account_id]`. Jobs in different queues will never be part + of the same chain, ensuring better performance and preventing unexpected dependencies across queues. + + Here are a few examples of using `:by` to achieve fine-grained control over chain partitioning: + + ```elixir + # Chain by :worker + use Oban.Pro.Worker, chain: [by: :worker] + + # Chain by a single args key without considering the worker + use Oban.Pro.Worker, chain: [by: [args: :account_id]] + + # Chain by multiple args keys without considering the worker + use Oban.Pro.Worker, chain: [by: [args: [:account_id, :cohort]]] + + # Chain by worker and a single args key + use Oban.Pro.Worker, chain: [by: [:worker, args: :account_id]] + + # Chain by worker and multiple args key + use Oban.Pro.Worker, chain: [by: [:worker, args: [:account_id, :cohort]]] + + # Chain by a single meta key + use Oban.Pro.Worker, chain: [by: [meta: :source_id]] + ``` + + #### Handling Cancelled/Discarded + + The way a chain behaves when jobs are cancelled or discarded is customizable with the + `:on_discarded` and `:on_cancelled` options. + + There are three strategies for handling upstream discards and cancellations: + + * `:ignore` — keep processing jobs in the chain as if upstream `cancelled` or `discarded` jobs + completed successfully. This is the default behaviour. + + * `:hold` — stop processing any jobs in the chain until the `cancelled` or `discarded` job is + `completed` or eventually deleted. + + Here's an example of a chain that holds on `discarded` or `cancelled`: + + ```elixir + use Oban.Pro.Worker, chain: [on_discarded: :hold, on_cancelled: :hold] + ``` + + > #### Chains and Workflows {: .warning} + > + > Using chained jobs from a workflow isn't allowed. Chains and workflows share an "on hold" + > scheduling mechanism which prevents predictable ordering. + + ## Encrypted Jobs + + Some applications have strong regulations around the storage of personal information. For + example, medical records, financial details, social security numbers, or other data that should + never leak. The `encrypted` option lets you store all job data at rest with encryption so + sensitive data can't be seen. + + #### Defining an Encrypted Worker + + Encryption is handled transparently as jobs are inserted and executed. All you need to do is + flag the worker as encrypted and configure it to fetch a secret key: + + ```elixir + defmodule MyApp.SensitiveWorker do + use Oban.Pro.Worker, encrypted: [key: {module, fun, args}] + + @impl true + def process(%Job{args: args}) do + # Args are decrypted, use them as you normally would + end + end + ``` + + Now job args are encrypted before insertion into the database and decrypted when the job runs. + + #### Generating Encryption Keys + + Encryption requires a 32 byte, Base 64 encoded key. You can generate one with the `:crypto` and + `Base` modules: + + ```elixir + key = 32 |> :crypto.strong_rand_bytes() |> Base.encode64() + ``` + + The result will look something like this `"w7xGJClzEh1pbWuq6zsZfKfwdINu2VIkgCe3IO0hpsA="`. + + While it's possible to use the generated key in your worker directly, that defeats the purpose + of encrypting sensitive data because anybody with access to the codebase can read the encryption + key. That's why it is _highly_ recommended that you use an MFA to retrieve the key dynamically + at runtime. For example, here's how you could pull the key from the Application environment: + + ```elixir + use Oban.Pro.Worker, encrypted: [key: {Application, :fetch_key!, [:enc_key]}] + ``` + + #### Encryption Implementation Details + + * Erlang's `crypto` module is used with the `aes_256_ctr` cipher for encryption. + + * Encoding and decoding stacktraces are pruned to prevent leaking the private key or + initialization vector. + + * Only `args` are encrypted, `meta` is kept as plaintext. You can use that to your advantage for + uniqueness, but be careful not to put anything sensitive in `meta`. + + * Error messages and stacktraces aren't encrypted and are stored as plaintext. Be careful not to + expose sensitive data when raising errors. + + * Args are encrypted at rest _as well as_ in Oban Web. You won't be able to view or search + encrypted args in the Web dashboard. + + * Uniqueness works for encrypted jobs, but not for arguments because the same args are encrypted + differently every time. Favor `meta` over `args` to enforce uniqueness for encrypted jobs. + + ## Job Deadlines + + Jobs that shouldn't run after some period of time can be marked with a `deadline`. After the + deadline has passed the job will be pre-emptively cancelled on its next run, or optionally + _during_ its next run if desired. + + ```elixir + defmodule DeadlinedWorker do + use Oban.Pro.Worker, deadline: {1, :hour} + + @impl true + def process(%Job{args: args}) do + # If this doesn't run within an hour, it's cancelled + end + end + ``` + + Deadlines may be set at runtime as well, provided the worker was _already_ configured with a + `deadline` option. + + ```elixir + # Specify the deadline in seconds + DeadlinedWorker.new(args, deadline: 60) + + # Specify the deadline in minutes, using the tuple syntax + DeadlinedWorker.new(args, deadline: {30, :minutes}) + + # Specify the deadline in days + DeadlinedWorker.new(args, deadline: {1, :day}) + ``` + + In either case, the deadline is always relative and computed at runtime. That also allows the + deadline to consider scheduling—a job scheduled to run 1 hour from now with a 1 hour deadline + will expire 2 hours in the future. Note that deadlines only account for scheduling on insert, + not on retry or after a snooze. + + ### Applying Deadlines to Running Jobs + + A job that has already started may be cancelled at runtime if it won't complete before the + deadline. Consider a slower job that takes 5 minutes to execute and has a deadline 10 minutes in + the future. If the job starts 9 minutes from now it wouldn't finish until 4 minutes past the + deadline. + + By setting the `force` flag, the job will cancel itself if it exceeds the deadline: + + ```elixir + defmodule FirmDeadlinedWorker do + use Oban.Pro.Worker, deadline: [in: {10, :minutes}, force: true] + + ... + ``` + + ## Worker Aliases + + Worker aliases solve a perennial production issue—how to rename workers without breaking existing jobs. + Aliasing allows jobs enqueued with the original worker name to continue executing without + exceptions using the new worker code. + + As an example, imagine that a `UserPurge` worker must expand to purge more than user data. The + `UserPurge` name is no longer accurate, and you want to rename it to `DataPurge`. To rename the + worker and add an alias for the original name: + + ```diff + -defmodule MyApp.UserPurge do + +defmodule MyApp.DataPurge do + - use Oban.Pro.Worker + + use Oban.Pro.Worker, aliases: [MyApp.UserPurge] + + @impl Oban.Pro.Worker + def process(job) do + # purge data, not just users + end + end + ``` + + Now any existing `MyApp.UserPurge` jobs can safely run, and they'll delegate to the new + `DataPurge` code. + + Workers may also have multiple aliases to account for multiple renames or merging workers: + + ```elixir + use Oban.Pro.Worker, aliases: [MyApp.WorkerA, MyApp.WorkerB] + ``` + + ## Worker Hooks + + Worker hooks are called before and after a job finishes executing. They can be defined as + callback functions on the worker, or in a separate module for reuse across workers. + + Hooks are called synchronously, from within the job's process. + + ### Defining Hooks + + There are three mechanisms for defining and attaching `c:before_new/2`, `c:before_process/1`, + and `c:after_process/3` hooks: + + 1. **Implicitly**—hooks are defined directly on the worker and they only run for that worker + 2. **Explicitly**—hooks are listed when defining a worker and they run anywhere they are listed + 3. **Globally**—hooks are executed for all Pro workers + + It's possible to combine each type of hook on a single worker. When multiple hooks are stacked + they're executed in the order: implicit, explicit, and then global. + + ### After Process + + After hooks _do not modify_ the job or execution results. Think of them as a convenient + alternative to globally attached telemetry handlers. They are purely for side-effects such as + cleanup, logging, recording metrics, broadcasting notifications, updating other records, error + notifications, etc. + + Any exceptions or crashes are caught and logged, they won't cause the job to fail or the queue + to crash. + + An `c:after_process/3` hook is called with the job and an execution state corresponding to the + result from `process/1`: + + * `complete`—when `process/1` returns `:ok` or `{:ok, result}` + * `cancel`—when `process/1` returns `{:cancel, reason}` + * `discard`—when a job errors and exhausts retries, or returns `{:discard, reason}` + * `error`—when a job crashes, raises an exception, or returns `{:error, value}` + * `snooze`—when a job returns `{:snooze, seconds}` + + > #### When After Process Hooks Are Called {: .info} + > + > The `c:after_process/3` hook is **only** called for jobs that are actively executing, i.e., + > jobs that go through the worker's `c:process/1` function. This includes jobs that return + > `{:cancel, reason}` from within `c:process/1`. + > + > The hook is **not** called when jobs are cancelled at the database level via + > `Oban.cancel_job/1`, `Oban.cancel_all_jobs/1`, `Oban.Pro.Workflow.cancel_jobs/1,2,3`, or + > `Oban.Pro.Batch.cancel_jobs/1,2`. Those functions bulk-update job states without loading + > jobs into memory, resolving their workers, or executing any callbacks. + + First, here's how to define a single implicit local hook on the worker using + `c:after_process/3`: + + ```elixir + defmodule MyApp.HookWorker do + use Oban.Pro.Worker + + @impl Oban.Pro.Worker + def process(_job) do + # ... + end + + @impl Oban.Pro.Worker + def after_process(state, %Job{} = job, _result) do + MyApp.Notifier.broadcast("oban-jobs", {state, %{id: job.id}}) + end + end + ``` + + Any module that exports `c:after_process/3` can be used as a hook. For example, here we'll + define a shared error notification hook: + + ```elixir + defmodule MyApp.ErrorHook do + def after_process(state, job, _result) when state in [:discard, :error] do + error = job.unsaved_error + extra = Map.take(job, [:attempt, :id, :args, :max_attempts, :meta, :queue, :worker]) + tags = %{oban_worker: job.worker, oban_queue: job.queue, oban_state: job.state} + + Sentry.capture_exception(error.reason, stacktrace: error.stacktrace, tags: tags, extra: extra) + end + + def after_process(_state, _job, _result), do: :ok + end + + defmodule MyApp.HookWorker do + use Oban.Pro.Worker, hooks: [MyApp.ErrorHook] + + @impl Oban.Pro.Worker + def process(_job) do + # ... + end + end + ``` + + The same module can be attached globally, for all `Oban.Pro.Worker` modules, using + `attach_hook/1`: + + ```elixir + :ok = Oban.Pro.Worker.attach_hook(MyApp.ErrorHook) + ``` + + Attaching hooks in your application's `start/2` function is an easy way to ensure hooks are + registered before your application starts processing jobs. + + ```elixir + def start(_type, _args) do + :ok = Oban.Pro.Worker.attach_hook(MyApp.ErrorHook) + + children = [ + ... + ``` + + ### Before New + + The `c:before_new/2` hook can augment or override the `args` and `opts` used to construct new + jobs. It's akin to overriding a worker's `new/2` callback, but the logic can be shared between + workers or operate on a global scale. + + Returning `{:ok, args, opts}` will continue building a job changeset using the provided `args` + and `opts`, while an `{:error, reason}` tuple will mark the changeset as invalid with a custom + error. + + The hook is most useful for augmenting jobs from a central point or injecting custom meta, i.e. + to add span ids for tracing. Here we define a global hook that injects a `span_id` into the + meta: + + ```elixir + defmodule MyApp.SpanHook do + def before_new(args, opts) do + opts = + opts + |> Keyword.put_new(:meta, %{}) + |> put_in([:meta, :span_id], MyApp.Telemetry.gen_span_id()) + + {:ok, args, opts} + end + end + ``` + + ### Before Process + + Before process hooks _may modify_ the job, or prevent calling `c:process/1` entirely depending + on the return value. Returning an `{:ok, job}` tuple will continue processing the job, while + `:error` or `:cancel` tuples will short circuit and record a standard error or cancel the job, + respectively. + + The `c:before_process/1` callback is executed within the job's process, after other internal + processing such as encryption and args structuring are applied. + + It's most useful in situations where the worker needs to perform generic logic such as adding + logger metadata, setting up a dynamic repo, etc. For example, let's define a global hook to set + `user_id` in the logger metadata for every job that contains that field: + + ```elixir + defmodule MyApp.UserMetadataHook do + def before_process(job) do + with %{"user_id" => user_id} <- job.args do + Logger.metadata(user_id: user_id) + end + + {:ok, job} + end + end + ``` + + Then attach the hook globally, as demonstrated earlier: + + ```elixir + :ok = Oban.Pro.Worker.attach_hook(MyApp.UserMetadataHook) + ``` + """ + + @behaviour Oban.Pro.Handler + + alias Oban.{Job, Validation, Worker} + alias Oban.Pro.Batch + alias Oban.Pro.Stages.{Chain, Deadline, Encrypted, Hooks, Recorded, Structured} + + @typedoc """ + Options to enable and configure `alias` mode. + """ + @type aliases :: [module()] + + @typedoc """ + Options to enable and configure `chain` mode. + """ + @type chain :: [ + by: chain_by(), + on_cancelled: :ignore | :hold, + on_discarded: :ignore | :hold + ] + + @typedoc """ + Configuration that controls how chains are linked together. + """ + @type chain_by :: + :worker + | {:args, atom() | [atom()]} + | {:meta, atom() | [atom()]} + | [:worker | {:args, atom() | [atom()]} | {:meta, atom() | [atom()]}] + + @typedoc """ + Options to enable and configure `deadline` mode. + """ + @type deadline :: [in: timeout(), force: boolean()] + + @typedoc """ + Options to enable and configure `encrypted` mode. + """ + @type encrypted :: [key: mfa()] + + @typedoc """ + Options to enable and configure `recorded` mode. + """ + @type recorded :: true | [to: atom(), limit: pos_integer(), safe_decode: boolean()] + + @typedoc """ + All possible states for the `c:after_process/3` hook. + """ + @type hook_state :: :cancel | :complete | :discard | :error | :snooze + + @type args :: Job.args() + @type opts :: [Job.option()] + + @doc """ + Called when executing a job. + + The `process/1` callback behaves identically to `c:Oban.Worker.perform/1`, except that it may + have pre-processing and post-processing applied. + """ + @callback process(job :: Job.t() | [Job.t()]) :: Worker.result() + + @doc """ + Called before `new/2` when constructing a job changeset. + + The `args` and `opts` returned in a `{:ok, args, opts}` tuple will be used to construct the job + changeset. Returning an `{:error, reason}` tuple will add the reason as a custom error for the + changeset and make it invalid. + + This callback is intended for global or explicit hooks. For workers that override the `new/2` + callback, the `c:before_new/2` hook will _only_ be called if `super(args, opts)` is used and + only after `new/2`, not before. + """ + @callback before_new(args :: Job.args(), opts :: Keyword.t()) :: + {:ok, Job.args(), Keyword.t()} | {:error, any()} + + @doc """ + Called before `process/1` when executing a job. + + The callback is executed within the job's process, after other internal processing such as + encryption and args structuring are applied. + + Returning an `{:ok, job}` tuple will continue processing the job, while `:error` or `:cancel` + tuples will short circuit and record a standard error or cancel the job, respectively. + """ + @callback before_process(job :: Job.t()) :: + {:ok, Job.t()} | {:error, reason :: term()} | {:cancel, reason :: term()} + + @doc """ + Called after a job finishes processing regardless of status (complete, failure, etc). The + `result` is available for any job that returns a value, not just recorded jobs. + + Since crashes and exceptions don't return anything, `result` will always by `nil`. + + See the shared "Worker Hooks" section for more details. + """ + @callback after_process(hook_state(), job :: Job.t(), result :: nil | Worker.result()) :: :ok + + @doc deprecated: "Use after_process/3 instead" + @callback after_process(hook_state(), job :: Job.t()) :: :ok + + @doc """ + Extract the results of a previously executed job. + + If a job has ran and recorded a value, it will return an `{:ok, result}` tuple. Otherwise, + you'll get `{:error, :missing}`. + """ + @callback fetch_recorded(job :: Job.t()) :: {:ok, term()} | {:error, :missing} + + @optional_callbacks after_process: 2, + after_process: 3, + before_new: 2, + before_process: 1, + fetch_recorded: 1 + + @stages [ + encrypted: Encrypted, + recorded: Recorded, + structured: Structured, + chain: Chain, + deadline: Deadline, + hooks: Hooks + ] + + @doc false + defmacro __using__(opts) do + opts = Macro.expand_literals(opts, %{__CALLER__ | function: {:__using__, 1}}) + + stage_keys = Keyword.keys(@stages) + + {stage_opts, stand_opts} = + [hooks: [], structured: []] + |> Keyword.merge(opts) + |> Keyword.drop(~w(aliases)a) + |> Keyword.split(stage_keys) + + quote do + @behaviour Oban.Worker + @behaviour Oban.Pro.Worker + + @after_compile {Oban.Pro.Worker, :__define_aliases__} + @after_compile {Oban.Pro.Worker, :__validate_opts__} + @after_compile {Oban.Pro.Worker, :__validate_stages__} + + import Oban.Pro.Worker, + only: [ + args_schema: 1, + field: 2, + field: 3, + embeds_one: 2, + embeds_one: 3, + embeds_many: 2, + embeds_many: 3 + ] + + alias Oban.{Job, Worker} + + @stand_opts Keyword.put(unquote(stand_opts), :worker, inspect(__MODULE__)) + @stage_opts unquote(stage_opts) + @opts @stand_opts ++ @stage_opts + @worker_aliases Keyword.get(unquote(opts), :aliases, []) + + @doc false + def __stage_opts__, do: @stage_opts + + @doc false + def __stand_opts__, do: @stand_opts + + @doc false + def __opts__, do: @opts + + @impl Worker + def new(args, opts \\ []) when is_map(args) and is_list(opts) do + {stage_opts, stand_opts} = + __opts__() + |> Worker.merge_opts(opts) + |> Keyword.split(unquote(stage_keys)) + + stage_opts = Keyword.merge(__stage_opts__(), stage_opts) + + case Oban.Pro.Worker.before_new(__MODULE__, args, stand_opts, stage_opts) do + {:ok, args, opts} -> + Job.new(args, opts) + + {:error, message} -> + args + |> Job.new(stand_opts) + |> Ecto.Changeset.add_error(:args, message) + end + end + + @impl Worker + def backoff(%Job{} = job) do + Worker.backoff(job) + end + + @impl Worker + def timeout(%Job{} = job) do + Worker.timeout(job) + end + + @impl Worker + def perform(%Job{} = job) do + Oban.Pro.Worker.process(__MODULE__, job, __opts__()) + end + + @impl Oban.Pro.Worker + def fetch_recorded(%Job{} = job) do + Oban.Pro.Worker.fetch_recorded(job) + end + + defoverridable backoff: 1, new: 2, perform: 1, timeout: 1 + end + end + + # Compile Callbacks + + @doc false + def __validate_opts__(%{module: module}, _bytecode) do + module + |> Module.get_attribute(:stand_opts) + |> __validate_opts__() + end + + def __validate_opts__(opts) when is_list(opts) do + Validation.validate_schema(opts, + max_attempts: :pos_integer, + priority: {:range, 0..9}, + queue: {:or, [:atom, :string]}, + replace: {:custom, &Job.validate_replace/1}, + tags: {:list, :string}, + unique: {:custom, &Job.validate_unique/1}, + worker: :string + ) + end + + @doc false + def __validate_stages__(%{module: module}, _bytecode) do + module + |> Module.get_attribute(:stage_opts) + |> Enum.each(fn {name, opts} -> + @stages + |> Keyword.fetch!(name) + |> init_stage!(module, opts) + end) + end + + @doc false + def __define_aliases__(env, _bytecode) do + with [_ | _] = aliases <- Module.get_attribute(env.module, :worker_aliases) do + contents = + quote bind_quoted: [worker: env.module] do + @behaviour Oban.Worker + + defdelegate __opts__, to: worker + + defdelegate __stage_opts__, to: worker + + defdelegate __stand_opts__, to: worker + + @impl Oban.Worker + def new(_args, _opts \\ []) do + raise RuntimeError, "unable to call new/1,2 on a worker alias" + end + + @impl Oban.Worker + defdelegate perform(job), to: worker + + @impl Oban.Worker + defdelegate backoff(job), to: worker + + @impl Oban.Worker + defdelegate timeout(job), to: worker + end + + for alias <- aliases do + Module.create(alias, contents, Macro.Env.location(env)) + end + end + end + + # Schema + + @doc """ + Define an args schema struct with field definitions and optional embedded structs. + + The schema is used to validate args before insertion or execution. See the [Structured + Workers](#module-structured-jobs) section for more details. + + ## Example + + Define an args schema for a worker: + + defmodule MyApp.Worker do + use Oban.Pro.Worker + + args_schema do + field :id, :id, required: true + field :name, :string, required: true + field :mode, :enum, values: ~w(on off paused)a + field :safe, :boolean, default: false + + embeds_one :address, required: true do + field :street, :string + field :number, :integer + field :city, :string + end + end + + ... + """ + @doc since: "0.14.0" + defmacro args_schema(do: block) do + quote do + Module.register_attribute(__MODULE__, :oban_fields, accumulate: true) + + try do + import Oban.Pro.Worker, + only: [ + field: 2, + field: 3, + embeds_one: 2, + embeds_one: 3, + embeds_many: 2, + embeds_many: 3 + ] + + unquote(block) + after + :ok + end + + defstruct Enum.map(@oban_fields, &elem(&1, 0)) + + def __args_schema__, do: Enum.reverse(@oban_fields) + end + end + + @doc false + defmacro field(name, type \\ :string, opts \\ []) do + check_type!(name, type) + check_opts!(name, type, opts) + + opts = Keyword.put(opts, :type, type) + + quote do + Module.put_attribute(__MODULE__, :oban_fields, {unquote(name), unquote(opts)}) + end + end + + @doc false + defmacro embeds_one(name, do: block) do + quote do + Oban.Pro.Worker.__embed__(__ENV__, :one, unquote(name), [], unquote(Macro.escape(block))) + end + end + + @doc false + defmacro embeds_one(name, opts, do: block) do + quote do + Oban.Pro.Worker.__embed__( + __ENV__, + :one, + unquote(name), + unquote(opts), + unquote(Macro.escape(block)) + ) + end + end + + @doc false + defmacro embeds_many(name, do: block) do + quote do + Oban.Pro.Worker.__embed__(__ENV__, :many, unquote(name), [], unquote(Macro.escape(block))) + end + end + + @doc false + defmacro embeds_many(name, opts, do: block) do + quote do + Oban.Pro.Worker.__embed__( + __ENV__, + :many, + unquote(name), + unquote(opts), + unquote(Macro.escape(block)) + ) + end + end + + @doc false + def __embed__(env, cardinality, name, opts, inner_block) do + block = + quote do + import Oban.Pro.Worker, only: [args_schema: 1] + + args_schema do + unquote(inner_block) + end + end + + module = + name + |> to_string() + |> Macro.camelize() + |> then(&Module.concat(env.module, &1)) + + required = Keyword.get(opts, :required, false) + opts = [cardinality: cardinality, module: module, required: required, type: :embed] + + Module.create(module, block, env) + Module.put_attribute(env.module, :oban_fields, {name, opts}) + end + + defp check_type!(name, {:array, type}), do: check_type!(name, type) + + defp check_type!(_name, type) when type in ~w(enum term uuid)a, do: :ok + + defp check_type!(name, type) do + unless Ecto.Type.base?(type) do + raise ArgumentError, "invalid type #{inspect(type)} for field #{inspect(name)}" + end + end + + defp check_opts!(name, type, opts) do + allowed = + if type in [:enum, {:array, :enum}] do + [:default, :required, :values] + else + [:default, :required] + end + + with {key, _} <- Enum.find(opts, fn {key, _} -> key not in allowed end) do + raise ArgumentError, "invalid option #{inspect(key)} for field #{inspect(name)}" + end + end + + # Global Hooks + + @doc """ + Register a worker hook to be ran after any Pro worker executes. + + The module must define a function that matches the hook. For example, a module that handles + an `:on_complete` hook must define an `on_complete/1` function. + + ## Example + + Attach a hook handler globally: + + defmodule MyApp.Hook do + def after_process(_state, %Oban.Job{} = job) do + # Do something with the job + + :ok + end + end + + :ok = Oban.Pro.Worker.attach_hook(MyApp.Hook) + """ + @doc since: "0.12.0" + @spec attach_hook(module()) :: :ok | {:error, term()} + defdelegate attach_hook(module), to: Hooks + + @doc """ + Unregister a worker hook. + + ## Example + + Detach a previously registered global hook: + + :ok = Oban.Pro.Worker.detach_hook(MyApp.Hook) + """ + @doc since: "0.12.0" + @spec detach_hook(module()) :: :ok + defdelegate detach_hook(module), to: Hooks + + # Stages + + @doc false + def init_stage!(stage_mod, worker, opts) do + case stage_mod.init(worker, opts) do + {:ok, conf} -> {stage_mod, conf} + {:error, reason} -> raise ArgumentError, "#{stage_mod}: " <> reason + end + end + + @doc false + def before_new(worker, args, opts, stage_opts) do + @stages + |> Enum.reverse() + |> Enum.reduce_while({:ok, args, opts}, fn {key, mod}, {:ok, args, opts} -> + case Keyword.fetch(stage_opts, key) do + {:ok, stage_opts} -> + {:ok, conf} = mod.init(worker, stage_opts) + + case mod.before_new(args, opts, conf) do + {:ok, _, _} = ok -> {:cont, ok} + other -> {:halt, other} + end + + :error -> + {:cont, {:ok, args, opts}} + end + end) + end + + @doc false + def process(worker, job, opts) do + with {:ok, job} <- before_process(worker, job, opts) do + Process.put(:oban_processing, {worker, job, opts}) + + worker + |> handle_or_process(job) + |> after_process(worker, job, opts) + end + end + + @doc false + def before_process(worker, job, opts) do + @stages + |> Keyword.take(~w(encrypted structured deadline hooks)a) + |> Enum.reduce_while({:ok, job}, fn {key, mod}, {:ok, job} -> + case Keyword.fetch(opts, key) do + {:ok, stage_opts} -> + {:ok, conf} = mod.init(worker, stage_opts) + + case mod.before_process(job, conf) do + {:ok, _} = ok -> {:cont, ok} + other -> {:halt, other} + end + + :error -> + {:cont, {:ok, job}} + end + end) + end + + defp handle_or_process(module, %{meta: %{"batch_id" => _, "callback" => callback}} = job) do + new_callback = Map.fetch!(Batch.callbacks_to_functions(), callback) + old_callback = Map.fetch!(Batch.callbacks_to_deprecated(), callback) + + cond do + function_exported?(module, new_callback, 1) -> + apply(module, new_callback, [job]) + + function_exported?(module, old_callback, 1) -> + apply(module, old_callback, [job]) + + true -> + :ok + end + end + + defp handle_or_process(module, job), do: module.process(job) + + @doc false + def after_process(result, worker, job, opts) do + @stages + |> Keyword.take(~w(recorded)a) + |> Enum.reduce_while(result, fn {key, mod}, result -> + stage_opts = Keyword.get(opts, key, []) + + {:ok, conf} = mod.init(worker, stage_opts) + + case mod.after_process(result, job, conf) do + :ok -> {:cont, result} + other -> {:halt, other} + end + end) + end + + @doc false + def fetch_recorded(job) do + with {:ok, worker} <- Worker.from_string(job.worker), + opts = Keyword.get(worker.__stage_opts__(), :recorded, []), + {:ok, conf} <- Recorded.init(worker, opts) do + Recorded.fetch_recorded(job, conf) + end + end + + # Telemetry + + @doc false + @impl Oban.Pro.Handler + def on_start do + events = [[:oban, :job, :stop], [:oban, :job, :exception]] + + :telemetry.attach_many("oban.pro.worker", events, &Hooks.handle_event/4, nil) + end + + @doc false + @impl Oban.Pro.Handler + def on_stop do + :telemetry.detach("oban.pro.worker") + end +end diff --git a/vendor/oban_pro/lib/oban/pro/workers/batch.ex b/vendor/oban_pro/lib/oban/pro/workers/batch.ex new file mode 100644 index 00000000..ccf34606 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/workers/batch.ex @@ -0,0 +1,98 @@ +defmodule Oban.Pro.Workers.Batch do + @moduledoc """ + A dedicated worker for grouping the execution of jobs. + + This worker is deprecated in favor of `Oban.Pro.Batch`, which is more flexible and supports more + options. + """ + + @moduledoc deprecated: "Use Oban.Pro.Batch instead" + + alias Oban.{Job, Worker} + + @callback new_batch([Job.t() | Job.args()], [keyword()]) :: Oban.Pro.Batch.t() + + @callback gen_id() :: String.t() + + @callback handle_attempted(Job.t()) :: Worker.result() + + @callback handle_cancelled(Job.t()) :: Worker.result() + + @callback handle_completed(Job.t()) :: Worker.result() + + @callback handle_discarded(Job.t()) :: Worker.result() + + @callback handle_exhausted(Job.t()) :: Worker.result() + + @callback handle_retryable(Job.t()) :: Worker.result() + + @callback all_batch_jobs(Job.t(), [keyword()]) :: Enum.t() + + @callback stream_batch_jobs(Job.t(), [keyword()]) :: Enum.t() + + @optional_callbacks handle_attempted: 1, + handle_cancelled: 1, + handle_completed: 1, + handle_discarded: 1, + handle_exhausted: 1, + handle_retryable: 1 + + @doc false + defmacro __using__(opts) do + quote do + use Oban.Pro.Worker, unquote(opts) + + alias Oban.Pro.Workers.Batch + + @behaviour Oban.Pro.Workers.Batch + + @batch_opts ~w( + batch_id + batch_name + batch_callback_args + batch_callback_meta + batch_callback_queue + batch_callback_worker + callback_opts + callback_worker + )a + + @impl Batch + def new_batch([_ | _] = args_or_changesets, opts \\ []) when is_list(opts) do + {batch_opts, job_opts} = Keyword.split(opts, @batch_opts) + + batch_opts = + Enum.reduce(batch_opts, [callback_opts: []], fn {key, val}, acc -> + case key do + :batch_callback_args -> put_in(acc, [:callback_opts, :args], val) + :batch_callback_meta -> put_in(acc, [:callback_opts, :meta], val) + :batch_callback_queue -> put_in(acc, [:callback_opts, :queue], val) + :batch_callback_worker -> Keyword.put(acc, :callback_worker, val) + _ -> Keyword.put(acc, key, val) + end + end) + + case args_or_changesets do + [changeset | _] = changesets when is_struct(changeset) -> + Oban.Pro.Batch.new(changesets, batch_opts) + + args -> + args + |> Enum.map(&new(&1, job_opts)) + |> Oban.Pro.Batch.new(batch_opts) + end + end + + @impl Batch + def gen_id, do: Oban.Pro.UUIDv7.generate() + + @impl Batch + def all_batch_jobs(job, opts \\ []), do: Oban.Pro.Batch.all_jobs(Oban, job, opts) + + @impl Batch + def stream_batch_jobs(job, opts \\ []), do: Oban.Pro.Batch.stream_jobs(Oban, job, opts) + + defoverridable Oban.Pro.Workers.Batch + end + end +end diff --git a/vendor/oban_pro/lib/oban/pro/workers/chain.ex b/vendor/oban_pro/lib/oban/pro/workers/chain.ex new file mode 100644 index 00000000..61bd5ecf --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/workers/chain.ex @@ -0,0 +1,170 @@ +defmodule Oban.Pro.Workers.Chain do + @moduledoc """ + A dedicated worker for linking jobs to run in strict sequential order. + + This worker is deprecated in favor of `Oban.Pro.Worker`, which is composable and allows better + parallelism. + """ + + @moduledoc deprecated: "Use Oban.Pro.Worker instead" + + import Ecto.Query + + alias Oban.{Job, Repo, Validation} + alias Oban.Pro.Utils + + @default_meta %{ + on_discarded: "ignore", + on_cancelled: "ignore", + chain_key: nil, + hold_snooze: 60, + wait_retry: 10, + wait_sleep: 1000, + wait_snooze: 5 + } + + @opts_keys ~w(by hold_snooze on_cancelled on_discarded wait_retry wait_sleep wait_snooze)a + + @doc false + defmacro __using__(opts) do + {chain_opts, other_opts} = Keyword.split(opts, @opts_keys) + + quote do + Validation.validate!(unquote(chain_opts), &Oban.Pro.Workers.Chain.validate/1) + + use Oban.Pro.Worker, unquote(other_opts) + + @base_meta unquote(chain_opts) + |> Keyword.delete(:by) + |> Map.new() + + @impl Oban.Worker + def new(args, opts) when is_map(args) and is_list(opts) do + meta = + opts + |> Keyword.get(:meta, %{}) + |> Map.merge(@base_meta) + |> Map.put_new(:chain_by, Keyword.fetch!(unquote(chain_opts), :by)) + |> Map.update!(:chain_by, &Utils.normalize_by/1) + + chain_key = Oban.Pro.Workers.Chain.to_chain_key(__MODULE__, args, meta) + chain_id = Oban.Pro.Stages.Chain.to_chain_id(meta.chain_by, __MODULE__, args, opts) + + # Include the `chain_id` for compatibility with chains in v1.5 + meta = + meta + |> Map.put(:chain_key, chain_key) + |> Map.put(:chain_id, chain_id) + + super(args, Keyword.put(opts, :meta, meta)) + end + + @impl Oban.Worker + def perform(%Job{} = job) do + opts = __opts__() + + with {:ok, job} <- Oban.Pro.Worker.before_process(__MODULE__, job, opts) do + job + |> Oban.Pro.Workers.Chain.maybe_process(__MODULE__) + |> Oban.Pro.Worker.after_process(__MODULE__, job, opts) + end + end + end + end + + @doc false + def validate(opts) do + Validation.validate_schema(opts, + by: {:custom, &Oban.Pro.Validation.validate_by/1}, + hold_snooze: :non_neg_integer, + on_cancelled: {:custom, &validate_on_action(:on_cancelled, &1)}, + on_discarded: {:custom, &validate_on_action(:on_discarded, &1)}, + wait_retry: :non_neg_integer, + wait_sleep: :timeout, + wait_snooze: :non_neg_integer + ) + end + + defp validate_on_action(_key, :halt), do: :ok + defp validate_on_action(_key, :hold), do: :ok + defp validate_on_action(_key, :ignore), do: :ok + + defp validate_on_action(key, action) do + {:error, "expected #{key} to be :halt, :hold, or :ignore, got: #{inspect(action)}"} + end + + @doc false + def to_chain_key(worker, args, meta) do + meta + |> Map.fetch!(:chain_by) + |> List.wrap() + |> Enum.reduce(%{}, fn + :worker, acc -> Map.put(acc, :worker, worker) + [:args, keys], acc -> Map.put(acc, :args, take_keys(args, keys)) + [:meta, keys], acc -> Map.put(acc, :meta, take_keys(meta, keys)) + end) + |> :erlang.phash2() + end + + defp take_keys(map, keys) do + keys + |> Enum.sort() + |> Enum.map(fn key -> map[key] || map[to_string(key)] end) + end + + @doc false + def maybe_process(job, worker, wait_count \\ 0) + + # It's running in :inline testing mode and the job doesn't have an id. + def maybe_process(%Job{id: nil} = job, worker, _wait_count) do + worker.process(job) + end + + def maybe_process(%Job{conf: conf, id: id, meta: meta} = job, worker, wait_count) do + meta = meta_with_defaults(meta) + + query = + Job + |> select([j], {j.state, j.scheduled_at}) + |> where([j], fragment("? @> ?", j.meta, ^%{chain_key: meta.chain_key})) + |> where([j], j.state != "completed") + |> where([j], j.id < ^id) + |> order_by(desc: :id) + |> limit(1) + + case Repo.one(conf, query) do + {"executing", _at} -> + if wait_count >= meta.wait_retry do + {:snooze, meta.wait_snooze} + else + Process.sleep(meta.wait_sleep) + + maybe_process(job, worker, wait_count + 1) + end + + {state, _at} when state in ~w(cancelled discarded) -> + key = if state == "cancelled", do: :on_cancelled, else: :on_discarded + + case Map.fetch!(meta, key) do + "halt" -> {:cancel, "chain halted by upstream job"} + "hold" -> {:snooze, meta.hold_snooze} + "ignore" -> worker.process(job) + end + + {_state, scheduled_at} -> + seconds = + scheduled_at + |> DateTime.diff(DateTime.utc_now()) + |> max(meta.wait_snooze) + + {:snooze, seconds} + + nil -> + worker.process(job) + end + end + + defp meta_with_defaults(meta) do + Map.new(@default_meta, fn {key, val} -> {key, meta[to_string(key)] || val} end) + end +end diff --git a/vendor/oban_pro/lib/oban/pro/workers/chunk.ex b/vendor/oban_pro/lib/oban/pro/workers/chunk.ex new file mode 100644 index 00000000..773d096d --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/workers/chunk.ex @@ -0,0 +1,735 @@ +defmodule Oban.Pro.Workers.Chunk do + @moduledoc """ + Chunk workers execute jobs together in groups based on a size or a timeout option. e.g. when + 1000 jobs are available or after 10 minutes have ellapsed. + + Multiple chunks can run in parallel within a single queue, and each chunk may be composed of + many thousands of jobs. Aside from improved throughput for a single queue, chunks are ideal as + the initial stage of data-ingestion and data-processing pipelines. + + ## Usage + + Let's define a worker that sends SMS messages in chunks, rather than individually: + + ```elixir + defmodule MyApp.ChunkWorker do + use Oban.Pro.Workers.Chunk, queue: :messages, size: 100, timeout: 1000 + + @impl true + def process([_ | _] = jobs) do + jobs + |> Enum.map(& &1.args) + |> MyApp.Messaging.send_batch() + + :ok + end + end + ``` + + Notice that we declared a `size` and a `timeout` along with a `queue` for the worker. If `size` + or `timeout` are omitted they fall back to their defaults: 100 `size` and 1000ms respectively. + + To process _larger_ batches _less_ frequently, we can increase both values: + + ```elixir + use Oban.Pro.Workers.Chunk, size: 500, timeout: :timer.seconds(5) + ``` + + Now chunks will run with up to 500 jobs or every 5 seconds, whichever comes first. + + Unlike other Pro workers, a `Chunk` worker's `process/1` receives a list of jobs rather than a + single job struct. Fittingly, the [expected return values](#t:result/0) are different as well. + + ## Chunk Partitioning + + By default, chunks are divided into groups based on the `queue` and `worker`. This means that + each chunk consists of workers belonging to the same queue, regardless of their `args` or + `meta`. However, this approach may not always be suitable. For instance, you may want to group + workers based on a specific argument such as `:account_id` instead of just the worker. In such + cases, you can use the [`:by`](#t:chunk_by/0) option to customize how chunks are partitioned. + + Here are a few examples of the `:by` option that you can use to achieve fine-grained control over + chunk partitioning: + + ```elixir + # Explicitly chunk by :worker + use Oban.Pro.Workers.Chunk, by: :worker + + # Chunk by a single args key without considering the worker + use Oban.Pro.Workers.Chunk, by: [args: :account_id] + + # Chunk by multiple args keys without considering the worker + use Oban.Pro.Workers.Chunk, by: [args: [:account_id, :cohort]] + + # Chunk by worker and a single args key + use Oban.Pro.Workers.Chunk, by: [:worker, args: :account_id] + + # Chunk by a single meta key + use Oban.Pro.Workers.Chunk, by: [meta: :source_id] + ``` + + When partitioning chunks of jobs, it's important to note that using only `:args` or `:meta` + without `:worker` may result in heterogeneous chunks of jobs from different workers. + Nevertheless, regardless of the partitioning method chunks always consist of jobs from the same + queue. + + Here's a simple example of partitioning by `worker` and an `author_id` field to batch analysis + of recent messages per author: + + ```elixir + defmodule MyApp.LLMAnalysis do + use Oban.Pro.Workers.Chunk, by: [:worker, args: :author_id], size: 50, timeout: 30_000 + + @impl true + def process([%{args: %{"author_id" => author_id}} | _] = jobs) do + messages = + jobs + |> Enum.map(& &1.args["message_id"]) + |> MyApp.Messages.all() + + {:ok, sentiment} = MyApp.GPT.gauge_sentiment(messages) + + MyApp.Messages.record_sentiment(author_id) + end + end + ``` + + ## Chunk Results and Error Handling + + `Chunk` worker's result type is tailored to handling multiple jobs at once. For reference, here + are the types and callback definition for `process/1`: + + The result types allow you to succeed an entire chunk, or selectively fail parts of it. Here are + each of the possible results: + + * `:ok` — The chunk succeeded and all jobs can be marked complete + + * `{:ok, result}` — Like `:ok`, but with a result for testing. + + * `{:error, reason, jobs}` — One or more jobs in the chunk failed and may be retried, any + unlisted jobs are a success. + + * `{:cancel, reason, jobs}` — One or more jobs in the chunk should be cancelled, any unlisted + jobs are a success. + + * `[error: {reason, jobs}, cancel: {reason, jobs}]` — Retry some jobs and cancel some other + jobs, leaving any jobs not in either list a success. + + To illustrate using chunk results let's expand on the message processing example from earlier. + We'll extend it to complete the whole batch when all messages are delivered or cancel + undeliverable messages: + + ```elixir + def process([_ | _] = jobs) do + notifications = + jobs + |> Enum.map(& &1.args) + |> MyApp.Messaging.send_batch() + + bad_token = fn %{response: response} -> response == :bad_token end + + if Enum.any?(notifications, bad_token) do + cancels = + notifications + |> Enum.zip(jobs) + |> Enum.filter(fn {notification, _job} -> bad_token.(notification) end) + |> Enum.map(&elem(&1, 1)) + + {:cancel, :bad_token, cancels} + else + {:ok, notifications} + end + end + ``` + + In the event of an ephemeral crash, like a network issue, the entire batch may be retried if + there are any remaining attempts. + + Cancelling any jobs in a chunk will cancel the _entire_ chunk, including the leader job. + + ## Chunk Organization + + Chunks are ran by a leader job (which has nothing to do with peer leadership). When the leader + executes it determines whether a complete chunk is available or if enough time has elapsed to + run anyhow. If neither case applies, then the leader will delay until the timeout elapsed and + execute with as many jobs as it can find. + + Completion queries run every `1000ms` by default. You can use the `:sleep` option to control how + long the leader delays between queries to check for complete chunks: + + ```elixir + use Oban.Pro.Workers.Chunk, size: 50, sleep: 2_000, timeout: 10_000 + ``` + + ## Run Chunks Immediately + + A chunk won't run until either the full number of jobs are available (`size`), or the total time + has elapsed (`timeout`). In situations where there is a long gap between incoming jobs it may be + desirable to run the chunk as soon as the chunk runs. This can be done with the `leading` + option. For example, to chunk all jobs that arrive within a 5 minute window, but run immediately + if more than 5 minutes have elapsed: + + ```elixir + use Oban.Pro.Workers.Chunk, size: 100, leading: true, timeout: :timer.minutes(5) + ``` + + ## Optimizing Chunks + + Queue's with high concurrency and low throughput may have multiple chunk leaders running + simultaneously rather than getting bundled into a single chunk. The solution is to reduce the + queue’s global concurrency to a smaller number so that new chunk leader jobs don’t start and the + existing chunk can run a bigger batch. + + ```elixir + queues: [ + chunked_queue: [global_limit: 1] + ] + ``` + + A low global limit will reduce overall throughput. Partitioning the queue with the same options + as the chunk can preserve throughput by allowing one concurrent leader per chunk. For example, + if you were chunking by `:user_id` in args, you'd partition the queue with the same option: + + ```elixir + queues: [ + chunked_queue: [local_limit: 20, global_limit: [allowed: 1, partition: [args: :user_id]]] + ] + ``` + """ + + import Ecto.Query + import DateTime, only: [utc_now: 0] + + alias Ecto.Multi + alias Oban.{CrashError, Job, Notifier, PerformError, Repo, Validation} + alias Oban.Pro.Utils + + @type key_or_keys :: atom() | [atom()] + + @type chunk_by :: + :worker + | {:args, key_or_keys()} + | {:meta, key_or_keys()} + | [:worker | {:args, key_or_keys()} | {:meta, key_or_keys()}] + + @type sub_result :: {reason :: term(), [Job.t()]} + + @type result :: + :ok + | {:ok, term()} + | {:cancel | :discard | :error, reason :: term(), [Job.t()]} + | [cancel: sub_result(), discard: sub_result(), error: sub_result()] + + @type options :: [ + by: chunk_by(), + leading: boolean(), + size: pos_integer(), + sleep: pos_integer(), + timeout: pos_integer() + ] + + @doc false + defmacro __using__(opts) do + {chunk_opts, other_opts} = Keyword.split(opts, ~w(by leading size sleep timeout)a) + + quote do + Validation.validate!(unquote(chunk_opts), &Oban.Pro.Workers.Chunk.validate/1) + + use Oban.Pro.Worker, unquote(other_opts) + + alias Oban.Pro.Workers.Chunk + + @doc false + def default_meta do + %{ + chunk_by: Keyword.get(unquote(chunk_opts), :by, [:worker]), + chunk_leading: Keyword.get(unquote(chunk_opts), :leading, false), + chunk_size: Keyword.get(unquote(chunk_opts), :size, 100), + chunk_sleep: Keyword.get(unquote(chunk_opts), :sleep, 1000), + chunk_timeout: Keyword.get(unquote(chunk_opts), :timeout, 1000) + } + end + + @impl Oban.Worker + def new(args, opts) when is_map(args) and is_list(opts) do + opts = + opts + |> Keyword.update(:meta, default_meta(), &Map.merge(default_meta(), &1)) + |> update_in([:meta, :chunk_by], &Utils.normalize_by/1) + + super(args, opts) + end + + @impl Oban.Worker + def perform(%Job{} = job) do + opts = __opts__() + + with {:ok, job} <- Oban.Pro.Worker.before_process(__MODULE__, job, opts) do + Process.put(:oban_processing, {__MODULE__, job, opts}) + + job + |> Chunk.maybe_process(__MODULE__) + |> Oban.Pro.Worker.after_process(__MODULE__, job, opts) + end + end + end + end + + @doc false + def validate(opts) do + Validation.validate_schema(opts, + by: {:custom, &Oban.Pro.Validation.validate_by/1}, + leading: :boolean, + size: :pos_integer, + sleep: :timeout, + timeout: :timeout + ) + end + + # Public Interface + + @doc false + def maybe_process(%Job{conf: conf} = job, worker) do + job = with_defaults(job, worker) + + cond do + full_timeout?(conf, job) -> + fetch_and_process(worker, job) + + full_chunk?(conf, job) -> + fetch_and_process(worker, job) + + true -> + job.meta + |> Map.get("chunk_timeout", 1000) + |> sleep_and_process(worker, job) + end + end + + defp with_defaults(%{meta: %{"chunk_size" => _}} = job, _worker), do: job + + defp with_defaults(job, worker) do + chunk_meta = + Map.new(worker.default_meta(), fn + {:chunk_by, val} -> {"chunk_by", Enum.map(val, &to_string/1)} + {key, val} -> {to_string(key), val} + end) + + %{job | meta: Map.merge(chunk_meta, job.meta)} + end + + # Check Helpers + + defp full_chunk?(conf, %{meta: %{"chunk_size" => size}} = job) do + limit = size - 1 + + query = + Job + |> select([j], j.id) + |> where([j], j.state == "available") + |> chunk_where(job) + |> limit(^limit) + + Repo.one(conf, from(oj in subquery(query), select: count(oj.id) >= ^limit)) + end + + defp full_timeout?(_, %{meta: %{"chunk_leading" => false}}), do: false + + defp full_timeout?(conf, %{meta: %{"chunk_timeout" => timeout}} = job) do + query = + Job + |> where([j], j.id != ^job.id) + |> where([j], j.state in ~w(completed cancelled discarded)) + |> chunk_where(job) + |> order_by(desc: :id) + |> limit(1) + |> select( + [j], + type( + fragment( + "greatest(?, ?, ?)", + j.completed_at, + j.discarded_at, + j.cancelled_at + ), + :utc_datetime_usec + ) + ) + + last_ts = Repo.one(conf, query) + + last_ts && + utc_now() + |> DateTime.add(-timeout, :millisecond) + |> DateTime.after?(last_ts) + end + + # Processing Helpers + + defp fetch_and_process(worker, %{conf: conf, meta: %{"chunk_size" => size}} = job) do + {:ok, chunk} = fetch_chunk(conf, job, size - 1) + {chunk, _errored} = prepare_chunk(chunk) + + guard_cancel(conf, worker, job, chunk) + guard_timeout(conf, worker.timeout(job), worker, job, chunk) + + try do + case worker.process([job | chunk]) do + :ok -> + ack_completed(conf, chunk, :ok) + + {:ok, result} -> + ack_completed(conf, chunk, {:ok, result}) + + {:cancel, reason, cancelled} -> + ack_cancelled(conf, worker, job, chunk, cancelled, reason) + + {:discard, reason, discarded} -> + ack_discarded(conf, worker, job, chunk, discarded, reason) + + {:error, reason, errored} -> + ack_errored(conf, worker, job, chunk, errored, reason) + + [_ | _] = multiple -> + ack_multiple(conf, worker, job, chunk, multiple) + end + rescue + error -> + ack_raised(conf, worker, chunk, job, error, __STACKTRACE__) + + reraise error, __STACKTRACE__ + catch + kind, reason -> + error = CrashError.exception({kind, reason, __STACKTRACE__}) + + ack_raised(conf, worker, chunk, job, error, __STACKTRACE__) + + reraise error, __STACKTRACE__ + end + end + + # This replicates the query used in Smart.fetch_jobs/3, without the meta tracking or any other + # complications. Any modifications to the original query must be replicated here. Another + defp fetch_chunk(conf, job, limit) do + subset_query = + Job + |> select([:id]) + |> where(state: "available") + |> chunk_where(job) + |> order_by(asc: :priority, asc: :scheduled_at, asc: :id) + |> limit(^limit) + |> lock("FOR UPDATE SKIP LOCKED") + + query = + Job + |> with_cte("subset", as: ^subset_query) + |> join(:inner, [j], x in fragment(~s("subset")), on: true) + |> where([j, x], j.id == x.id) + |> select([j, _], j) + + attempted_by = job.attempted_by ++ ["chunk-#{job.id}"] + + updates = [ + set: [state: "executing", attempted_at: utc_now(), attempted_by: attempted_by], + inc: [attempt: 1] + ] + + Repo.transaction(conf, fn -> + {_count, chunk} = Repo.update_all(conf, query, updates) + + chunk + end) + end + + defp prepare_chunk(chunk) do + Enum.reduce(chunk, {[], []}, fn job, {acc, err} -> + with {:ok, worker} <- Oban.Worker.from_string(job.worker), + {:ok, job} <- Oban.Pro.Worker.before_process(worker, job, worker.__opts__()) do + {[job | acc], err} + else + {:error, reason} -> + {acc, [{job, reason} | err]} + end + end) + end + + # Only the leader job is registered as "running" by the queue producer. We listen for pkill + # messages for _other_ jobs in the chunk and apply that to all jobs. Without this, cancelling + # doesn't apply to the leader and chunk jobs are orphaned. + defp guard_cancel(conf, worker, job, chunk) do + # No supervision tree is running when draining, attempting to notify will raise an exception. + if Process.get(:oban_draining) do + :ok + else + parent = self() + + Task.start(fn -> + ref = Process.monitor(parent) + :ok = Notifier.listen(conf.name, [:signal]) + + await_cancel(conf, ref, worker, job, chunk) + end) + end + end + + defp await_cancel(conf, ref, worker, job, chunk) do + receive do + {:notification, :signal, %{"action" => "pkill", "job_id" => kill_id}} -> + if Enum.any?(chunk, &(&1.id == kill_id)) do + reason = PerformError.exception({job.worker, {:cancel, :shutdown}}) + + ack_cancelled(conf, worker, job, chunk, chunk, reason) + + Notifier.notify(conf.name, :signal, %{action: "pkill", job_id: job.id}) + else + await_cancel(conf, ref, worker, job, chunk) + end + + {:DOWN, ^ref, :process, _pid, _reason} -> + :ok + end + end + + # The executor uses :timer.exit_after/2 to kill jobs that exceed the timeout. The queue's + # producer then catches the DOWN message and uses that to record a job error. The producer + # isn't aware of the job's chunk, so we monitor the parent process and ack the chunk jobs. + defp guard_timeout(_conf, :infinity, _worker, _job, _chunk), do: :ok + + defp guard_timeout(conf, timeout, worker, job, chunk) do + parent = self() + + Task.start(fn -> + ref = Process.monitor(parent) + + receive do + {:DOWN, ^ref, :process, _pid, %Oban.TimeoutError{} = reason} -> + ack_errored(conf, worker, job, chunk, chunk, reason) + + {:DOWN, ^ref, :process, _pid, _reason} -> + :ok + after + timeout + 100 -> :ok + end + end) + end + + defp sleep_and_process(total, worker, %Job{conf: conf, meta: meta} = job) do + sleep = Map.get(meta, "chunk_sleep", 1000) + + Process.sleep(sleep) + + if total - sleep < 0 or full_chunk?(conf, job) do + fetch_and_process(worker, job) + else + sleep_and_process(total - sleep, worker, job) + end + end + + # Chunk Helpers + + defp chunk_where(query, job) do + query = where(query, queue: ^job.queue) + + job.meta + |> Map.get("chunk_by") + |> Enum.reduce(query, fn + "worker", acc -> + where(acc, worker: ^job.worker) + + ["args", keys], acc -> + where(acc, [j], fragment("? @> ?", j.args, ^take_keys(job.args, keys))) + + ["meta", keys], acc -> + where(acc, [j], fragment("? @> ?", j.meta, ^take_keys(job.meta, keys))) + end) + end + + defp take_keys(args, keys) when is_struct(args) do + Map.take(args, Enum.map(keys, &String.to_existing_atom/1)) + end + + defp take_keys(map, keys), do: Map.take(map, keys) + + # Ack Helpers + + defp ack_completed(conf, jobs, result) do + Repo.update_all(conf, ids_query(jobs), com_ups()) + + result + end + + defp ack_raised(conf, worker, chunk, job, error, stacktrace) do + {dis, ers} = Enum.split_with(chunk, &exhausted?/1) + + opts = Repo.default_options(conf) + + multi = + Multi.new() + |> Multi.update_all(:err, ids_query(ers), err_ups(worker, job, error, stacktrace), opts) + |> Multi.update_all(:dis, ids_query(dis), dis_ups(worker, job, error, stacktrace), opts) + + Repo.transaction(conf, multi) + + :ok + end + + defp ack_errored(conf, worker, job, chunk, errored, reason) do + {ers, oks, set} = split_with_set(errored, chunk) + {dis, ers} = Enum.split_with(ers, &exhausted?/1) + + opts = Repo.default_options(conf) + + multi = + Multi.new() + |> Multi.update_all(:com, ids_query(oks), com_ups(), opts) + |> Multi.update_all(:err, ids_query(ers), err_ups(worker, job, {:error, reason}), opts) + |> Multi.update_all(:dis, ids_query(dis), dis_ups(worker, job, {:error, reason}), opts) + + with {:ok, _result} <- Repo.transaction(conf, multi) do + if MapSet.member?(set, job.id), do: {:error, reason}, else: :ok + end + end + + defp ack_cancelled(conf, worker, job, chunk, cancelled, reason) do + {can, oks, set} = split_with_set(cancelled, chunk) + + opts = Repo.default_options(conf) + + multi = + Multi.new() + |> Multi.update_all(:com, ids_query(oks), com_ups(), opts) + |> Multi.update_all(:can, ids_query(can), can_ups(worker, job, {:cancel, reason}), opts) + + with {:ok, _result} <- Repo.transaction(conf, multi) do + if MapSet.member?(set, job.id), do: {:cancel, reason}, else: :ok + end + end + + defp ack_discarded(conf, worker, job, chunk, discarded, reason) do + {dis, oks, set} = split_with_set(discarded, chunk) + + opts = Repo.default_options(conf) + + multi = + Multi.new() + |> Multi.update_all(:com, ids_query(oks), com_ups(), opts) + |> Multi.update_all(:dis, ids_query(dis), dis_ups(worker, job, {:discard, reason}), opts) + + with {:ok, _result} <- Repo.transaction(conf, multi) do + if MapSet.member?(set, job.id), do: {:discard, reason}, else: :ok + end + end + + defp ack_multiple(conf, worker, host, chunk, multiple) do + {oks, result} = split_multiple_result(host, chunk, multiple) + + opts = Repo.default_options(conf) + + group = fn {oper, reason, %{attempt: attempt}} -> {oper, reason, attempt} end + multi = Multi.update_all(Multi.new(), :com, ids_query(oks), com_ups(), opts) + + multi = + for({oper, {reas, jobs}} <- multiple, job <- jobs, job.id != host.id, do: {oper, reas, job}) + |> Enum.map(&maybe_discard/1) + |> Enum.group_by(group, &elem(&1, 2)) + |> Enum.with_index() + |> Enum.reduce(multi, fn {{{oper, reason, _}, jobs}, index}, acc -> + update = + case oper do + :cancel -> can_ups(worker, hd(jobs), {:cancel, reason}) + :discard -> dis_ups(worker, hd(jobs), {:discard, reason}) + :error -> err_ups(worker, hd(jobs), {:error, reason}) + end + + Multi.update_all(acc, {oper, index}, ids_query(jobs), update, opts) + end) + + with {:ok, _} <- Repo.transaction(conf, multi), do: result + end + + # Single Helpers + + defp ids_query(jobs) do + where(Job, [j], j.id in ^Enum.map(jobs, & &1.id)) + end + + defp split_with_set(sub_jobs, all_jobs) do + set = MapSet.new(sub_jobs, & &1.id) + + all_jobs + |> Enum.split_with(&MapSet.member?(set, &1.id)) + |> Tuple.insert_at(2, set) + end + + defp exhausted?(%Job{attempt: attempt, max_attempts: max}), do: attempt >= max + + # Multiple Helpers + + defp split_multiple_result(%{id: jid}, chunk, multiple) do + set = for {_, {_, jobs}} <- multiple, %{id: id} <- jobs, into: MapSet.new(), do: id + + result = + if MapSet.member?(set, jid) do + {oper, {reason, _}} = + Enum.find(multiple, fn {_, {_, jobs}} -> jid in Enum.map(jobs, & &1.id) end) + + {oper, reason} + else + :ok + end + + {Enum.reject(chunk, &MapSet.member?(set, &1.id)), result} + end + + defp maybe_discard({:error, reason, job}) when job.attempt >= job.max_attempts do + {:discard, reason, job} + end + + defp maybe_discard(tuple), do: tuple + + # Update Helpers + + defp com_ups do + [set: [state: "completed", completed_at: utc_now()]] + end + + defp can_ups(worker, job, reason) do + error = format_error(job, worker, reason, []) + + Keyword.new() + |> Keyword.put(:set, state: "cancelled", cancelled_at: utc_now()) + |> Keyword.put(:push, errors: error) + end + + defp err_ups(worker, job, error, stacktrace \\ []) do + error = format_error(job, worker, error, stacktrace) + + Keyword.new() + |> Keyword.put(:set, state: "retryable", scheduled_at: backoff_at(worker, job)) + |> Keyword.put(:push, errors: error) + end + + defp dis_ups(worker, job, error, stacktrace \\ []) do + error = format_error(job, worker, error, stacktrace) + + Keyword.new() + |> Keyword.put(:set, state: "discarded", discarded_at: utc_now()) + |> Keyword.put(:push, errors: error) + end + + defp backoff_at(worker, job) do + DateTime.add(utc_now(), worker.backoff(job), :second) + end + + defp format_error(job, worker, reason, stacktrace) when is_tuple(reason) do + format_error(job, worker, PerformError.exception({worker, reason}), stacktrace) + end + + defp format_error(job, _worker, error, stacktrace) do + {blamed, stacktrace} = Exception.blame(:error, error, stacktrace) + + formatted = Exception.format(:error, blamed, stacktrace) + + %{attempt: job.attempt, at: utc_now(), error: formatted} + end +end diff --git a/vendor/oban_pro/lib/oban/pro/workers/workflow.ex b/vendor/oban_pro/lib/oban/pro/workers/workflow.ex new file mode 100644 index 00000000..b94947ee --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/workers/workflow.ex @@ -0,0 +1,209 @@ +defmodule Oban.Pro.Workers.Workflow do + @moduledoc """ + A dedicated worker for linking the execution of jobs. + + This worker is deprecated in favor of `Oban.Pro.Workflow`, which is more flexible and supports + more options. + """ + + @moduledoc deprecated: "Use Oban.Pro.Workflow instead" + + import Ecto.Query, only: [group_by: 3, select: 3, where: 3] + + alias Oban.{Job, Repo} + alias Oban.Pro.Workflow + + @callback new_workflow(Workflow.new_opts()) :: Workflow.t() + + @callback add(Workflow.t(), Workflow.name(), Job.changeset(), Workflow.add_opts()) :: + Workflow.t() + + @callback after_cancelled(Workflow.cancel_reason(), Job.t()) :: :ok + + @callback all_workflow_jobs(Job.t(), Workflow.fetch_opts()) :: [Job.t()] + + @callback append_workflow(Job.t() | [Job.t()], Workflow.append_opts()) :: Workflow.t() + + @callback gen_id() :: String.t() + + @callback stream_workflow_jobs(Job.t(), Workflow.fetch_opts()) :: Enum.t() + + @callback to_dot(Workflow.t()) :: String.t() + + @optional_callbacks after_cancelled: 2 + + defmacro __using__(opts) do + quote do + use Oban.Pro.Worker, unquote(opts) + + @behaviour Oban.Pro.Workers.Workflow + + @impl Oban.Pro.Workers.Workflow + def new_workflow(opts \\ []) when is_list(opts) do + opts + |> Keyword.put_new(:workflow_id, gen_id()) + |> Oban.Pro.Workflow.new() + end + + @impl Oban.Pro.Workers.Workflow + def append_workflow(jobs, opts \\ []) do + Oban.Pro.Workflow.append(jobs, opts) + end + + @impl Oban.Pro.Workers.Workflow + def add(workflow, name, changeset, opts \\ []) do + Oban.Pro.Workflow.add(workflow, name, changeset, opts) + end + + @impl Oban.Pro.Workers.Workflow + def gen_id, do: Oban.Pro.UUIDv7.generate() + + @impl Oban.Pro.Workers.Workflow + def to_dot(workflow) do + Oban.Pro.Workflow.to_dot(workflow) + end + + @impl Oban.Pro.Workers.Workflow + def stream_workflow_jobs(%Job{} = job, opts \\ []) do + Oban.Pro.Workflow.stream_jobs(Oban, job, opts) + end + + @impl Oban.Pro.Workers.Workflow + def all_workflow_jobs(%Job{} = job, opts \\ []) do + Oban.Pro.Workflow.all_jobs(Oban, job, opts) + end + + @impl Oban.Worker + def perform(%Job{} = job) do + opts = __opts__() + + with {:ok, job} <- Oban.Pro.Worker.before_process(__MODULE__, job, opts) do + job + |> Oban.Pro.Workers.Workflow.maybe_process(__MODULE__) + |> Oban.Pro.Worker.after_process(__MODULE__, job, opts) + end + end + + defoverridable Oban.Pro.Workers.Workflow + end + end + + @doc false + defdelegate add(workflow, name, changeset, opts \\ []), to: Workflow + + @doc false + def all_jobs(job, opts \\ []), do: Workflow.all_jobs(Oban, job, opts) + + @doc false + defdelegate append(job, opts \\ []), to: Workflow + + @doc false + def gen_id, do: Oban.Pro.UUIDv7.generate() + + @doc false + defdelegate new(opts \\ []), to: Workflow + + @doc false + def stream_jobs(job, opts \\ []), do: Workflow.stream_jobs(Oban, job, opts) + + @doc false + defdelegate to_dot(workflow), to: Workflow + + @doc false + def maybe_process(%{meta: meta} = job, module) do + if is_map_key(meta, "workflow_id") and not is_map_key(meta, "on_hold") do + legacy_process(job, module) + else + module.process(job) + end + end + + @legacy_meta %{ + ignore_cancelled: false, + ignore_deleted: false, + ignore_discarded: false, + waiting_delay: :timer.seconds(1), + waiting_limit: 10, + waiting_snooze: 5 + } + + # This is necessary for backward compatibility + defp legacy_process(job, module, waiting_count \\ 0) do + meta = for {key, val} <- @legacy_meta, into: %{}, do: {key, job.meta[to_string(key)] || val} + + case legacy_check_deps(job) do + :completed -> + module.process(job) + + :available -> + {:snooze, meta.waiting_snooze} + + :executing -> + if waiting_count >= meta.waiting_limit do + {:snooze, meta.waiting_snooze} + else + Process.sleep(meta.waiting_delay) + + legacy_process(job, module, waiting_count + 1) + end + + {:scheduled, scheduled_at} -> + seconds = + scheduled_at + |> DateTime.diff(DateTime.utc_now()) + |> max(meta.waiting_snooze) + + {:snooze, seconds} + + :cancelled -> + if meta.ignore_cancelled do + module.process(job) + else + {:cancel, "upstream deps cancelled, workflow will never complete"} + end + + :discarded -> + if meta.ignore_discarded do + module.process(job) + else + {:cancel, "upstream deps discarded, workflow will never complete"} + end + + :deleted -> + if meta.ignore_deleted do + module.process(job) + else + {:cancel, "upstream deps deleted, workflow will never complete"} + end + end + end + + defp legacy_check_deps(%{conf: conf, meta: %{"deps" => [_ | _]}} = job) do + %{"deps" => deps, "workflow_id" => workflow_id} = job.meta + + deps_count = length(deps) + + query = + Job + |> where([j], fragment("? @> ?", j.meta, ^%{workflow_id: workflow_id})) + |> where([j], fragment("?->>'name'", j.meta) in ^deps) + |> group_by([j], j.state) + |> select([j], {j.state, count(j.id), max(j.scheduled_at)}) + + conf + |> Repo.all(query) + |> Map.new(fn {state, count, sc_at} -> {state, {count, sc_at}} end) + |> case do + %{"completed" => {^deps_count, _}} -> :completed + %{"scheduled" => {_, scheduled_at}} -> {:scheduled, scheduled_at} + %{"retryable" => {_, scheduled_at}} -> {:scheduled, scheduled_at} + %{"executing" => _} -> :executing + %{"available" => _} -> :available + %{"cancelled" => _} -> :cancelled + %{"discarded" => _} -> :discarded + %{} -> :deleted + end + end + + defp legacy_check_deps(_job), do: :completed +end diff --git a/vendor/oban_pro/lib/oban/pro/workflow.ex b/vendor/oban_pro/lib/oban/pro/workflow.ex new file mode 100644 index 00000000..3f203f5a --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/workflow.ex @@ -0,0 +1,3009 @@ +defmodule Oban.Pro.Workflow do + @moduledoc ~S""" + Workflows compose jobs with flexible dependency relationships, enabling sequential execution, + fan-out parallelization, and fan-in convergence patterns. By declaratively defining jobs and + their connections, you can build complex processing pipelines that are both fault-tolerant and + horizontally scalable across all nodes. + + ## Basic Usage + + Workflows are composed of regular jobs or decorated functions linked together through + declarative dependencies. A workflow consists of these key elements: + + - **Jobs** - Individual units of work identified by unique names + - **Dependencies** - Relationships specifying execution order + + Let's create a workflow using a decorated `echo/1` function that demonstrates how jobs execute + in order: + + ```elixir + defmodule EchoWorkflow do + use Oban.Pro.Decorator + + alias Oban.Pro.Workflow + + def insert do + Workflow.new() + |> Workflow.add(:a, new_echo(1)) + |> Workflow.add(:b, new_echo(2), deps: :a) + |> Workflow.add(:c, new_echo(3), deps: :b) + |> Workflow.add(:d, new_echo(4), deps: :b) + |> Workflow.add(:e, new_echo(5), deps: [:c, :d]) + |> Oban.insert_all() + end + + @job true + def echo(value), do: IO.inspect(value) + end + ``` + + The workflow is initialized with `Workflow.new/1`, jobs are added with `Workflow.add/3`, + connections between jobs are added with the `deps` option, and finally the workflow is inserted + with `Oban.insert_all/1`. + + When executed, this workflow will print each job's `value` in the prescribed order. Steps `:c` + and `:d` both depend on `:b`, so they may execute in parallel. Visually, the workflow looks like + this: + + ![Simple Diagram](assets/simple-workflow.svg) + + ## Common Patterns + + Workflows support several execution patterns. These patterns form the building blocks of + workflow design, allowing you to create both simple linear processes and complex dependency + graphs. Each pattern serves a specific purpose in orchestrating job execution: + + > #### Example Workers {: .info} + > + > The following examples all use a generic `Worker` module to help focus on declaring + > dependencies. The patterns apply to any type of worker module or decorated function. + + ### Sequential Execution + + Jobs run one after another, waiting for `scheduled` or `retryable` delays: + + ```elixir + Workflow.new() + |> Workflow.add(:first, new_job()) + |> Workflow.add(:second, Worker.new(%{}), deps: :first) + |> Workflow.add(:third, Worker.new(%{}), deps: :second) + ``` + + ### Fan-Out (1-to-many) + + One job triggers multiple parallel jobs: + + ```elixir + Workflow.new() + |> Workflow.add(:parent, Worker.new(%{})) + |> Workflow.add(:child_1, Worker.new(%{}), deps: :parent) + |> Workflow.add(:child_2, Worker.new(%{}), deps: :parent) + |> Workflow.add(:child_3, Worker.new(%{}), deps: :parent) + ``` + + ### Fan-In (many-to-1) + + Multiple parallel jobs converge to a single job: + + ```elixir + Workflow.new() + |> Workflow.add(:step_1, Worker.new(%{})) + |> Workflow.add(:step_2, Worker.new(%{})) + |> Workflow.add(:step_3, Worker.new(%{})) + |> Workflow.add(:final, Worker.new(%{}), deps: [:step_1, :step_2, :step_3]) + ``` + + ### Diamond Pattern + + Combines fan-out and fan-in: + + ```elixir + Workflow.new() + |> Workflow.add(:start, Worker.new(%{})) + |> Workflow.add(:left, Worker.new(%{}), deps: :start) + |> Workflow.add(:right, Worker.new(%{}), deps: :start) + |> Workflow.add(:end, Worker.new(%{}), deps: [:left, :right]) + ``` + + These patterns can be combined to create complex workflows tailored to your specific business + processes. + + ## Advanced Patterns + + Once you're comfortable with basic workflows, you can leverage more advanced patterns to handle + complex scenarios. These patterns enable shared state, modular composition, simplified + dependencies, and dynamic generation of workflows. + + ### Sub-Workflows + + Workflows can be nested hierarchically using sub workflows. This allows you to compose complex + workflows from simpler ones, making it easier to organize and reuse workflow patterns. + + The `add_workflow/4` function lets you add an entire workflow as a dependency of another + workflow. Like `add/4`, it accepts a name and optional dependencies, but instead of a job + changeset, it takes another workflow. + + Here's an example of creating a primary workflow and adding sub workflows with dependencies: + + ```elixir + alias MyApp.{WorkerA, WorkerB, WorkerC} + alias Oban.Pro.Workflow + + extr_flow = + Workflow.new(workflow_name: "extract") + |> Workflow.add(:extract, WorkerA.new(%{source: "database"})) + |> Workflow.add(:transform, WorkerB.new(%{type: "normalize"}), deps: :extract) + + note_flow = + Workflow.new(workflow_name: "notify") + |> Workflow.add(:prepare, WorkerA.new(%{template: "report"})) + |> Workflow.add(:send, WorkerB.new(%{method: "email"}), deps: :prepare) + + # Create the main workflow and add sub workflows to it + Workflow.new() + |> Workflow.add(:setup, WorkerC.new(%{mode: "initialize"})) + |> Workflow.add_workflow(:extract, extr_flow, deps: :setup) + |> Workflow.add_workflow(:notify, note_flow, deps: :extract) + |> Workflow.add(:finalize, WorkerC.new(%{mode: "cleanup"}), deps: :notify) + |> Oban.insert_all() + ``` + + In this example, the main workflow has a single job named `:setup`, followed by two sub + workflows, and ends with a `:finalize` job. The dependencies ensure proper execution order: + `setup -> extraction -> notification -> finalize`. + + Sub-workflows are a powerful pattern for: + + - Organizing large job dependencies into logical units + - Enabling reuse across applications + - Simplifying the maintenance of complex job graphs + + ### Sharing Results + + Directed dependencies between jobs, paired with the `recorded` option, allow downstream jobs to + fetch the output of upstream jobs. This is particularly useful for multi-step processes where + each step builds on previous results. + + Consider this workflow that simulates a multi-step API interaction: + + ```elixir + defmodule MyApp.WorkerA do + use Oban.Pro.Worker, recorded: true + + @impl true + def process(%Job{args: %{"api_key" => api_key}}) do + token = + api_key + |> String.graphemes() + |> Enum.shuffle() + |> to_string() + + {:ok, token} # This return value will be recorded + end + end + ``` + + The second worker fetches the `token` from the first job by calling `get_recorded/2` with the name + `:a`, which we'll set while building the workflow later. + + ```elixir + defmodule MyApp.WorkerB do + use Oban.Pro.Worker, recorded: true + + @impl true + def process(%Job{args: %{"url" => url}} = job) do + token = Oban.Pro.Workflow.get_recorded(job, :a) + + {:ok, {token, url}} + end + end + ``` + + Then the final worker uses `all_recorded/3` with the `only_deps` option to fetch the results + from all upstream jobs, then it prints out everything that was fetched. + + ```elixir + defmodule MyApp.WorkerC do + use Oban.Pro.Worker + + @impl true + def process(job) do + job + |> Oban.Pro.Workflow.all_recorded(only_deps: true) + |> IO.inspect() + + :ok + end + end + ``` + + Now compose the workers together: + + ```elixir + alias MyApp.{WorkerA, WorkerB, WorkerC} + + Workflow.new() + |> Workflow.add(:a, WorkerA.new(%{api_key: "23kl239bjljlk309af"})) + |> Workflow.add(:b, WorkerB.new(%{url: "elixir-lang.org"}), deps: [:a]) + |> Workflow.add(:c, WorkerB.new(%{url: "www.erlang.org"}), deps: [:a]) + |> Workflow.add(:d, WorkerB.new(%{url: "oban.pro"}), deps: [:a]) + |> Workflow.add(:e, WorkerC.new(%{}), deps: [:b, :c, :d]) + |> Oban.insert_all() + ``` + + When the workflow runs, the final step prints something like: + + ```elixir + %{ + "b" => {"93l2jlj3kl90baf2k3", "elixir-lang.org"}, + "c" => {"93l2jlj3kl90baf2k3", "www.erlang.org"}, + "d" => {"93l2jlj3kl90baf2k3", "oban.pro"} + } + ``` + + Sharing results between jobs is a powerful building block for processing pipelines. While the + approach above works well, cascading functions provide an even more elegant way to share and + transform data between workflow steps. + + ### Cascading Functions + + Cascade mode allows you to build workflows using function captures that automatically receive + context and previous step results. Each function receives a map containing the workflow context + and the results of its dependencies. Results from each step are recorded and made available to + subsequent steps. + + Here's an ETL (Extract, Transform, Load) pipeline using cascading functions: + + ```elixir + defmodule MyApp.ETL do + def insert(source, date) do + Workflow.new() + |> Workflow.put_context(%{source: source, date: date}) + |> Workflow.add_cascade(:extract, &extract/1) + |> Workflow.add_cascade(:transform, &transform/1, deps: :extract) + |> Workflow.add_cascade(:load, &load/1, deps: :transform) + |> Workflow.add_cascade(:notify, ¬ify/1, deps: [:transform, :load]) + |> Oban.insert_all() + end + + def extract(%{source: source, date: date}) do + if source =~ ~r/[a-z0-9]+/ do + %{records: [1, 2, 3], extracted_at: DateTime.utc_now()} + else + {:cancel, "unprocessable source"} + end + end + + def transform(%{extract: %{records: records}}) do + transformed = Enum.map(records, & &1 * 2) + %{records: transformed, count: length(transformed)} + end + + def load(%{transform: %{records: records}}) do + %{loaded: true, count: length(records)} + end + + def notify(%{transform: transform, load: load}) do + IO.puts("Processed #{transform.count} records, loaded #{load.count}") + end + end + ``` + + When this workflow runs, each function is automatically called with a context map containing the + shared context (source and date), and results from dependencies (accessible by their step + names). + + There's no need to manually fetch recorded results as with standard workflow jobs! For standard + workflows that need to fetch results from dependencies, including grafted sub-workflows, see + `all_recorded/3`. + + Cascading functions are especially valuable for: + + - Data pipelines where each step needs the output of previous steps + - Validating aggregate context before proceeding + - Minimizing worker and function definition boilerplate + + ### Cascading Sub-Workflow + + Cascades can be added as sub-workflows that efficiently fan out workloads across multiple items. + This works similar to `add_many/4`, but for cascading functions. + + To create a fan-out cascade, provide a tuple containing an enumerable (like a list, map, or + range) and a function capture with arity 2. The function will be called once for each item in + the enumerable, with the first argument receiving the current item and the second argument + receiving the cumulative context. + + ```elixir + defmodule MyApp.BatchETL do + def insert(sources, date) do + Workflow.new() + |> Workflow.put_context(%{date: date}) + |> Workflow.add_cascade(:sources, {sources, &extract_source/2}) + |> Workflow.add_cascade(:transform, &transform_all/1, deps: :sources) + |> Workflow.add_cascade(:load, &load_data/1, deps: :transform) + |> Workflow.add_cascade(:notify, &send_notification/1, deps: [:transform, :load]) + |> Oban.insert_all() + end + + # Each `source` from the list is passed as the first argument + def extract_source(source, context) do + %{source: source, records: [1, 2, 3], extracted_at: DateTime.utc_now()} + end + + def transform_all(%{sources: sources}) do + total_records = Enum.sum_by(sources, fn {_key, result} -> length(result.records) end) + + transformed = + sources + |> Enum.flat_map(fn {_key, %{records: records}} -> records end) + |> Enum.map(&(&1 * 2)) + + %{records: transformed, count: total_records} + end + + def load_data(%{transform: %{records: records}}) do + %{loaded: true, count: length(records)} + end + + def send_notification(%{transform: transform, load: load, date: date}) do + IO.puts("On #{date}, processed #{transform.count} records, loaded #{load.count}") + end + end + ``` + + This example processes multiple data sources in parallel, then consolidates the results in + subsequent steps. The `extract_source/2` function is automatically called for each source in the + provided list, with the results collected into a map. + + Cascading sub-workflows are useful for: + + - Processing collections of items in parallel with shared context + - Building data pipelines that operate on multiple sources simultaneously + - Implementing fan-out/fan-in patterns with minimal boilerplate + - Maintaining clean separation between item processing and result aggregation + + ### Dynamic Workflows + + Many workflows aren't static—the number of jobs and their interdependencies aren't known + beforehand. You can generate workflows dynamically based on runtime conditions. + + The following worker creates a workflow that fans-out and back in twice, using a variable number + of dependencies: + + ```elixir + defmodule MyApp.Dynamic do + use Oban.Pro.Worker + + alias Oban.Pro.Workflow + + @impl true + def process(%{meta: %{"name" => name}}) do + IO.puts(name) + end + + def insert_workflow(count) when is_integer(count) do + range = Range.new(0, count) + a_deps = Enum.map(range, &"a_#{&1}") + b_deps = Enum.map(range, &"b_#{&1}") + + Workflow.new() + |> Workflow.add(:a, new(%{}), []) + |> fan_out(:a, range) + |> Workflow.add(:b, new(%{}), deps: a_deps) + |> fan_out(:b, range) + |> Workflow.add(:c, new(%{}), deps: b_deps) + |> Oban.insert_all() + end + + defp fan_out(workflow, base, range) do + Enum.reduce(range, workflow, fn key, acc -> + Workflow.add(acc, "#{base}_#{key}", new(%{}), deps: [base]) + end) + end + end + ``` + + This approach is useful for: + + - Processing variable-sized collections + - Creating workflows based on database queries + - Building complex workflows from configuration + - Implementing multi-tenant workflows with different requirements + + ### Grafting Sub-Workflows + + Grafting allows sub-workflows to be attached after a workflow has started. With grafting, you + define placeholders that are expanded into full sub-workflows later when data becomes available. + + Graft jobs serve as placeholders in a workflow. When a graft job executes, it _must_ build + and attach a new sub-workflow at that point. Any downstream jobs that depend on the grafter + will wait for the entire grafted sub-workflow to complete. + + Here's an example workflow that takes an `account_id` then grafts on a sub-workflow to contact + all users in the account: + + ```elixir + defmodule MyApp.AccountNotifier do + alias Oban.Pro.Workflow + + def insert(account_id) do + Workflow.new() + |> Workflow.put_context(%{account_id: account_id}) + |> Workflow.add_graft(:users, &graft_users/1) + |> Workflow.add_cascade(:notify, &send_summary/1, deps: :users) + |> Oban.insert_all() + end + + def graft_users(%{account_id: account_id}) do + user_ids = MyApp.Account.get_user_ids!(account_id) + + {user_ids, &send_notice/2} + |> Workflow.apply_graft() + |> Oban.insert_all() + end + + def send_notice(user_id, context) do + # Notify users + end + + def send_summary(%{account_id: account_id, users: user_ids}) do + # Deliver summary + end + end + ``` + + The summary job waits for the grafting job and the grafted workflow to complete before + executing. + + This pattern is particularly useful for: + + - Building sub-workflows dynamically based on data discovered during execution + - Conditionally executing different workflow branches + - Managing resource-intensive sub-workflows + + ### Appending Jobs + + Sometimes all jobs aren't known when the workflow is created. In that case, you can add more + jobs with optional dependency checking using `append/2`. An appended workflow starts with one or + more jobs, which reuses the original `workflow_id`, and optionally builds a set of dependencies + to check against. + + In this example we disable deps checking with `check_deps: false`: + + ```elixir + def process(job) do + jobs = + job + |> Workflow.append(check_deps: false) + |> Workflow.add(:d, WorkerD.new(%{}), deps: [:a]) + |> Workflow.add(:e, WorkerE.new(%{}), deps: [:b]) + |> Workflow.add(:f, WorkerF.new(%{}), deps: [:c]) + |> Oban.insert_all() + + {:ok, jobs} + end + ``` + + The new jobs specify deps on preexisting jobs named `:a`, `:b`, and `:c`, but there isn't any + guarantee those jobs actually exist. That could lead to an incomplete workflow where the new + jobs may never complete. + + To be safe and check jobs while appending we'll fetch all of the previous jobs with `all_jobs/3` + and feed them in: + + ```elixir + def process(job) do + jobs = Workflow.all_jobs(job) + + jobs + |> Workflow.append() + |> Workflow.add(:d, WorkerD.new(%{}), deps: :a) + |> Workflow.add(:e, WorkerE.new(%{}), deps: :b) + |> Workflow.add(:f, WorkerF.new(%{}), deps: :c) + |> Oban.insert_all() + + :ok + end + ``` + + Now there isn't any risk of an incomplete workflow from missing dependencies, at the expense of + loading some extraneous jobs. + + Appending jobs is particularly useful for: + + - Extending workflows based on user interaction or external events + - Adding additional reporting or cleanup steps after seeing initial results + - Breaking large workflows into stages that are built incrementally + + ## Customization + + Workflows provide several customization options to control their behavior, from identification + to error handling. This section covers how to tailor workflows to your specific requirements. + + #### Workflow ID + + Every workflow needs a unique identifier. By default, `workflow_id` is a time-ordered, random + [UUIDv7][uuid], which ensures uniqueness for any period. For more control, you can provide a + custom ID: + + ```elixir + Workflow.new(workflow_id: "custom-but-still-unique-id") + ``` + + [uuid]: https://datatracker.ietf.org/doc/html/draft-peabody-dispatch-new-uuid-format-04#section-5.2 + + #### Workflow Name + + Additionally, workflows accept an optional name that describes their purpose: + + ```elixir + Workflow.new(workflow_name: "nightly-etl") + ``` + + While the `workflow_id` must be unique, the `workflow_name` doesn't have to be, so it can serve + as a general-purpose label for categorizing or grouping workflows. + + #### Context Key Format + + By default, cascade functions receive context with atom keys for top level dependencies. + However, _nested_ dependencies will have string keys. To prevent this inconsistency you can opt + in to atomizing string keys to atoms with the `atom_keys` option: + + * `atom_keys` — Atomize top level keys for cascade context. Defaults to `true`. + + Apply this option when creating a workflow: + + ```elixir + Workflow.new(atom_keys: true) + |> Workflow.add_cascade(:fetch, &MyApp.fetch/1) + |> Workflow.add_cascade(:process, &MyApp.process/1, deps: :fetch) + ``` + + With `atom_keys: false`, the `process/1` function receives `%{"fetch" => result}` instead of + `%{fetch: result}`. + + #### Dependency Handling + + By default, workflows use conservative dependency handling - if an upstream job is cancelled, + discarded, or deleted, dependent jobs are automatically cancelled. You can customize this + behavior with these options: + + * `ignore_cancelled` — Treat `cancelled` dependencies as completed rather than cancelling + downstream jobs. Defaults to `false`. + + * `ignore_discarded` — Treat `discarded` dependencies as completed rather than cancelling + downstream jobs. Defaults to `false`. + + * `ignore_deleted` — Treat `deleted` (typically pruned) dependencies as completed rather than + cancelling downstream jobs. Defaults to `false`. + + Apply these options to the entire workflow: + + ```elixir + Workflow.new(ignore_cancelled: true, ignore_deleted: true, ignore_discarded: true) + ``` + + Or configure individual jobs within a workflow: + + ```elixir + Workflow.new() + |> Workflow.add(:a, MyWorkflow.new(%{})) + |> Workflow.add(:b, MyWorkflow.new(%{}), deps: :a, ignore_cancelled: true) + |> Workflow.add(:c, MyWorkflow.new(%{}), deps: :b, ignore_discarded: true) + |> Workflow.add(:d, MyWorkflow.new(%{}), deps: :c, ignore_deleted: true) + ``` + + > #### Preventing Stuck Workflows {: .tip} + > + > Be sure that you're running the `DynamicLifeline` to rescue stuck workflows when upstream + > dependencies are deleted unexpectedly. + > + > ```elixir + > config :my_app, Oban, + > plugins: [Oban.Pro.Plugins.DynamicLifeline], + > ... + > ``` + + #### Cancellation Callbacks + + Workflow jobs are automatically `cancelled` when their upstream dependencies are `cancelled`, + `discarded`, or `deleted` (unless specifically configured with the `ignore_*` options mentioned + above). Since these workflow jobs are cancelled before execution, standard + `c:Oban.Pro.Worker.after_process/3` hooks won't be called. + + Instead, you can implement the optional `c:after_cancelled/2` callback specifically for + workflows: + + ```elixir + defmodule MyApp.Workflow do + use Oban.Pro.Worker + + @behaviour Oban.Pro.Workflow + + require Logger + + @impl Oban.Pro.Workflow + def after_cancelled(reason, job) do + Logger.warn("Workflow job #{job.id} cancelled because a dependency was #{reason}") + + # Perform any cleanup required + notify_monitoring_system(job, reason) + + :ok + end + end + ``` + + The `reason` parameter will be one of: + + - `:cancelled` - A dependency was cancelled + - `:discarded` - A dependency was discarded + - `:deleted` - A dependency was deleted + + #### Handling Failures with Status Checking + + For situations where you can't use the `after_cancelled` callback, or when you have many + different workers in a workflow, you can combine the `ignore_cancelled` option with status + checking in a downstream job. + + This pattern allows a final "cleanup" job to examine the state of all upstream jobs and take + appropriate action: + + ```elixir + Workflow.new(ignore_cancelled: true) + |> Workflow.add(:step_1, StepOne.new(%{})) + |> Workflow.add(:step_2, StepTwo.new(%{}), deps: :step_1) + |> Workflow.add(:step_3, StepThree.new(%{}), deps: :step_2) + |> Workflow.add(:report, Reporter.new(%{}), deps: ~w(step_1 step_2 step_3)) + ``` + + Then, in your final worker, examine the state of the upstream jobs using `status/1`: + + ```elixir + defmodule MyApp.Reporter do + use Oban.Pro.Worker + + @impl true + def process(job) do + status = Workflow.status(job) + + if status.counts.cancelled > 0 do + # Some results were cancelled + else + :ok + end + end + end + ``` + + This approach offers several advantages over hooks: + + - It works with any type of worker, including decorated functions and cascades + - You can centralize error handling in a single place + - The workflow continues to the reporting step even if some steps fail + + ## Introspection + + Workflow jobs are tied together through `meta` attributes. You can retrieve these jobs using + several functions, each with different optimization characteristics. + + ### Fetching Jobs + + There are several functions for retrieving jobs, `get_job/2` and `all_jobs/3`, which are + optimized for different use cases: + + ```elixir + # Get a single specific job by name + job_a = Workflow.get_job(job, :a) + + # Get all jobs in a workflow + all_jobs = Workflow.all_jobs(job) + + # Get only the dependencies of the current job + deps = Workflow.all_jobs(job, only_deps: true) + + # Get specific named jobs + [job_a, job_b] = Workflow.all_jobs(job, names: [:a, :b]) + + # Access from outside a worker using workflow_id + jobs = Workflow.all_jobs("some-uuid-1234-5678") + ``` + + ### Streaming Jobs + + For large workflows where loading all jobs at once would consume too much memory, use + `stream_jobs/3`: + + ```elixir + def process(job) do + {:ok, workflow_jobs} = + MyApp.Repo.transaction(fn -> + job + |> Oban.Pro.Workflow.stream_jobs() + |> Stream.filter(& &1.state == "completed") + |> Enum.to_list() + end) + + process_completed_jobs(workflow_jobs) + + :ok + end + ``` + + > #### Transaction Required {: .info} + > + > Streaming is provided by Ecto's `Repo.stream`, and must take place within a transaction. + > This approach lets you control the number of jobs loaded from the database, minimizing memory + > usage for large workflows. + + ### Retrieving Results + + To fetch recorded output from workflow jobs, without loading the full job first, use + `get_recorded/2` and `all_recorded/3`: + + ```elixir + # Get results from a specific job + result_a = Workflow.get_recorded(job, :a) + + # Get results from all jobs + results = Workflow.all_recorded(job) + + # Get only results from dependencies + dep_results = Workflow.all_recorded(job, only_deps: true) + ``` + + ### Checking Status + + To get detailed status information about a workflow, including the overall workflow state, job + counts, time measurement, and sub-workflow details, use `status/1`: + + ```elixir + status = Workflow.status(job) + ``` + + This outputs a map with details, somewhat like this: + + ```elixir + %{ + id: "some-uuid-1234-5678", + name: "my-workflow", + total: 3, + state: :completed, + duration: 72, + counts: %{ + cancelled: 0, + available: 0, + completed: 3, + scheduled: 0, + discarded: 0, + executing: 0, + retryable: 0 + }, + started_at: ~U[2025-04-10 16:34:34], + stopped_at: ~U[2025-04-10 16:34:34], + subs: %{} + } + ``` + + ## Visualization + + Workflows are a type of a [Directed Acyclic Graph][dag], which means we can represent them + visually as nodes (jobs) connected by edges (dependencies). By converting workflows into + standard graph description languages, we can create clear visual representations of job + execution patterns. + + ### Converting to Graph Formats + + Along with the general purpose `to_graph/1` that outputs a `digraph`, there are several built-in + visualization options: + + * `to_dot/1` - Generates [GraphViz][gv] DOT format + * `to_mermaid/1` - Produces [Mermaid][mermaid] flowchart markup + + For example, let's generate a mermaid flowchart from our account workflow: + + ```elixir + Workflow.to_mermaid(archive_account_workflow(123)) + ``` + + This produces the following mermaid output, where each vertex represents a job's name and the + label shows the worker: + + ```text + flowchart TD + backup_1[MyApp.BackupPost] --> delete[MyApp.DeleteAccount] + backup_2[MyApp.BackupPost] --> delete[MyApp.DeleteAccount] + backup_3[MyApp.BackupPost] --> delete[MyApp.DeleteAccount] + receipt[MyApp.FinalReceipt] --> backup_1[MyApp.BackupPost] + receipt[MyApp.FinalReceipt] --> backup_2[MyApp.BackupPost] + receipt[MyApp.FinalReceipt] --> backup_3[MyApp.BackupPost] + receipt[MyApp.FinalReceipt] --> email_1[MyApp.EmailSubscriber] + receipt[MyApp.FinalReceipt] --> email_2[MyApp.EmailSubscriber] + email_1[MyApp.EmailSubscriber] --> delete[MyApp.DeleteAccount] + email_2[MyApp.EmailSubscriber] --> delete[MyApp.DeleteAccount] + ``` + + ### Rendering Visualizations + + You can render these visualizations using the command line with a `mermaid` binary, directly in + [LiveBook][live] using [Kino][kino], or in documentation sites. + + For LiveBook, you can pipe a workflow through `to_mermaid/1` and then into `Kino.Mermaid`: + + ```elixir + workflow + |> Workflow.to_mermaid() + |> Kino.Mermaid.new() + ``` + + This generates an SVG diagram like this: + + ![Mermaid Diagram](assets/mermaid-workflow.svg) + + The visualization clearly shows how our workflow starts with a single `receipt` job, fans-out to + multiple `email` and `backup` jobs, and finally fans-in to the `delete` job—making complex + dependency relationships immediately apparent. + + [dag]: https://en.wikipedia.org/wiki/Directed_acyclic_graph + [mermaid]: https://mermaid.js.org/ + [gv]: https://graphviz.org + [live]: https://livebook.dev/ + [kino]: https://hexdocs.pm/kino/Kino.Mermaid.html + """ + + @behaviour Oban.Pro.Flusher + @behaviour Oban.Pro.Handler + + import Ecto.Query + + alias Ecto.Changeset + alias Oban.Pro.Stages.Chain + alias Oban.Pro.Utils + alias Oban.Pro.Workflow.Cascade + alias Oban.{Config, Job, Repo, Validation} + + require Logger + + # Types + + @type add_opt :: + {:deps, name() | [name()]} + | {:ignore_cancelled, boolean()} + | {:ignore_deleted, boolean()} + | {:ignore_discarded, boolean()} + + @type add_opts :: [add_opt()] + + @type add_cascade_opts :: [Job.option() | add_opt()] + + @type add_workflow_opts :: [deps: name() | [name()]] + + @type append_opts :: [ + check_deps: boolean(), + ignore_cancelled: boolean(), + ignore_deleted: boolean(), + ignore_discarded: boolean(), + workflow_id: String.t(), + workflow_name: String.t() + ] + + @type cancel_opts :: [names: [name()]] + + @type cancel_reason :: :deleted | :discarded | :cancelled + + @type cascade_capture :: (map() -> any()) | {Enum.t(), (any(), map() -> any())} + + @type fetch_opts :: [ + log: Logger.level(), + names: [name()], + only_deps: boolean(), + timeout: timeout(), + with_subs: boolean() + ] + + @type job_or_wid :: Job.t() | workflow_id() + + @type name :: atom() | String.t() + + @type new_opts :: [ + ignore_cancelled: boolean(), + ignore_deleted: boolean(), + ignore_discarded: boolean(), + workflow_id: String.t(), + workflow_name: String.t() + ] + + @type status :: %{ + id: workflow_id(), + name: String.t() | nil, + total: non_neg_integer(), + state: :executing | :completed | :cancelled | :discarded | :unknown, + counts: %{atom() => non_neg_integer()}, + duration: non_neg_integer() | nil, + started_at: DateTime.t() | nil, + stopped_at: DateTime.t() | nil, + subs: %{String.t() => map()} + } + + @type t :: %__MODULE__{ + id: workflow_id(), + changesets: [Job.changeset()], + check_deps: boolean(), + names: MapSet.t(), + opts: map() + } + + @type workflow_id :: String.t() + + # Callbacks + + @doc """ + Called after a workflow job is cancelled due to upstream jobs being `cancelled`, `deleted`, or + `discarded`. + + This callback is _only_ called when a job is cancelled because of an upstream dependency. It is + _never_ called after normal job execution. For that, use `c:Oban.Pro.Worker.after_process/3`. + """ + @callback after_cancelled(cancel_reason(), job :: Job.t()) :: :ok + + @optional_callbacks after_cancelled: 2 + + # Constants + + @all_states Enum.map(Job.states(), &to_string/1) + @hold_date ~U[3000-01-01 00:00:00.000000Z] + @incomplete_set MapSet.new(~w(available executing retryable scheduled)) + @chan_keys ~w(max_attempts meta priority queue schedule_in scheduled_at tags)a + @work_keys ~w(ignore_cancelled ignore_deleted ignore_discarded workflow_id workflow_name)a + + # Macros + + defstruct [ + :id, + changesets: [], + check_deps: true, + eliminations: %{}, + grafts: %{}, + names: MapSet.new(), + opts: %{}, + subs: %{} + ] + + defguardp is_job_or_wid(job_or_wid) when is_struct(job_or_wid, Job) or is_binary(job_or_wid) + + defguardp is_name(name) when (is_atom(name) and not is_nil(name)) or is_binary(name) + + # This is a replacement for `push`, which uses `array_append` and isn't compatible with jsonb + # arrays. The `||` operator works with both arrays and jsonb. + defmacrop concat_errors(column, error) do + quote do + fragment("? || ?", unquote(column), unquote(error)) + end + end + + defmacrop drop_hold(meta) do + quote do + fragment(~s(? || '{"on_hold":false}'), unquote(meta)) + end + end + + defmacrop in_workflow(meta, workflow_id) do + quote do + fragment( + "? \\? 'workflow_id' AND ?->>'workflow_id' = ?", + unquote(meta), + unquote(meta), + unquote(workflow_id) + ) + end + end + + defmacrop in_workflow_or_sup(meta, workflow_id) do + quote do + fragment( + "? \\? 'workflow_id' AND (?->>'workflow_id' = ? OR ?->>'sup_workflow_id' = ?)", + unquote(meta), + unquote(meta), + unquote(workflow_id), + unquote(meta), + unquote(workflow_id) + ) + end + end + + defmacrop in_held_workflow(meta, workflow_id) do + quote do + fragment( + "? \\? 'workflow_id' AND ?->>'workflow_id' = ? AND ?->>'on_hold' = 'true'", + unquote(meta), + unquote(meta), + unquote(workflow_id), + unquote(meta) + ) + end + end + + defmacrop in_held_sup_workflow(meta, workflow_id) do + quote do + fragment( + "? \\? 'sup_workflow_id' AND ?->>'sup_workflow_id' = ? AND ?->>'on_hold' = 'true'", + unquote(meta), + unquote(meta), + unquote(workflow_id), + unquote(meta) + ) + end + end + + defguardp is_legacy_scheduled(meta) + when is_map_key(meta, "orig_scheduled_at") and + :erlang.map_get("orig_scheduled_at", meta) < 9_999_999_999 + + # Telemetry + + @impl Oban.Pro.Handler + def on_start do + events = [ + [:oban, :engine, :cancel_all_jobs, :stop], + [:oban, :plugin, :stop] + ] + + :telemetry.attach_many("oban.workflow", events, &__MODULE__.handle_event/4, nil) + end + + @impl Oban.Pro.Handler + def on_stop do + :telemetry.detach("oban.workflow") + end + + @doc false + def handle_event([:oban, :plugin, _], _time, %{conf: conf, discarded_jobs: jobs}, nil) do + for %{meta: %{"workflow_id" => workflow_id}} <- jobs do + on_flush(workflow_id, conf) + end + end + + def handle_event([:oban, :engine, _, _], _time, %{conf: conf, jobs: jobs}, nil) do + for %{meta: %{"workflow_id" => workflow_id}} <- jobs do + on_flush(workflow_id, conf) + end + end + + def handle_event(_event, _time, _meta, _conf), do: :ok + + # Handling + + @doc false + @impl Oban.Pro.Flusher + def to_flush_mfa(job, conf) do + case job.meta do + %{"ancestor_ids" => ancestors, "workflow_id" => workflow_id} -> + {__MODULE__, :on_flush, [Enum.uniq([workflow_id | ancestors]), conf]} + + %{"sup_workflow_id" => sup_workflow_id, "workflow_id" => workflow_id} -> + {__MODULE__, :on_flush, [[sup_workflow_id, workflow_id], conf]} + + %{"on_hold" => _, "workflow_id" => workflow_id} -> + {__MODULE__, :on_flush, [workflow_id, conf]} + + _ -> + :ignore + end + end + + @doc false + def on_flush(workflow_id, conf) when is_binary(workflow_id) do + # The internal, external, and wildcard deps must be queried independently in order for the + # planner to utilize indexes properly. + lateral_query = + Job + |> select([j], j.state) + |> where( + [j], + # This uses a hardcoded reference to the `sf0` alias for the `parent_as(:elem)` source. + # This is to avoid using `parent_as(:elem).value`, which is valid syntax but + # incompatible with how CRDB handles jsonb references. Ecto requires a field reference + # on any source currently, and this is a workaround. + fragment("? \\? 'workflow_id'", j.meta) and + fragment( + """ + ( + (jsonb_typeof(sf0) = 'string' AND ?->>'workflow_id' = ?->>'workflow_id' AND ?->>'name' = sf0 #>>'{}') OR + (jsonb_typeof(sf0) = 'array' AND sf0->>1 <> '*' AND ?->>'workflow_id' = sf0->>0 AND ?->>'name' = sf0->>1) OR + (jsonb_typeof(sf0) = 'array' AND sf0->>1 = '*' AND ?->>'workflow_id' = sf0->>0) OR + (jsonb_typeof(sf0) = 'array' AND sf0->>1 = '*' AND ? \\? 'sup_workflow_id' AND ?->>'sup_workflow_id' = sf0->>0) + ) + """, + j.meta, + parent_as(:jobs).meta, + j.meta, + j.meta, + j.meta, + j.meta, + j.meta, + j.meta + ) + ) + + deps_query = + from v in fragment("jsonb_array_elements(?->'deps')", parent_as(:jobs).meta), + as: :elem, + inner_lateral_join: j in subquery(lateral_query), + on: true, + select: fragment("coalesce(array_agg(?), '{}')", j.state) + + # A complete set of conditions must be used for held workflows and sub workflows. The planner + # won't use the correct indexes when an "OR" is used. + query = + from(j in Job, as: :jobs) + |> select([j], {j, subquery(deps_query)}) + |> lock("FOR UPDATE SKIP LOCKED") + |> where( + [j], + (j.state == "scheduled" and in_held_workflow(j.meta, ^workflow_id)) or + (j.state == "scheduled" and in_held_sup_workflow(j.meta, ^workflow_id)) + ) + + conf + |> Repo.all(query) + |> Enum.map(&to_stage_operation/1) + |> Enum.group_by(&elem(&1, 0), &elem(&1, 1)) + |> Enum.map(&apply_operations(&1, conf)) + |> on_flush(conf) + end + + def on_flush(workflow_ids, conf) when is_list(workflow_ids) do + workflow_ids + |> List.flatten() + |> Enum.each(&on_flush(&1, conf)) + end + + @doc """ + Instantiate a new workflow struct with a unique workflow id. + + ## Examples + + Create a standard workflow without any options: + + Workflow.new() + + Create a workflow with a custom name: + + Workflow.new(workflow_name: "logistics") + + Create a workflow with a static id and some options: + + Workflow.new(workflow_id: "workflow-id", ignore_cancelled: true, ignore_discarded: true) + """ + @spec new(opts :: new_opts()) :: t() + def new(opts \\ []) do + validate!(opts) + + opts = + opts + |> Keyword.put(:workflow, true) + |> Keyword.put_new_lazy(:workflow_id, &Oban.Pro.UUIDv7.generate/0) + |> Map.new() + + %__MODULE__{id: opts.workflow_id, opts: opts} + end + + defp validate!(opts) do + Validation.validate_schema!(opts, + atom_keys: :boolean, + ignore_cancelled: :boolean, + ignore_deleted: :boolean, + ignore_discarded: :boolean, + workflow_id: :string, + workflow_name: :string + ) + end + + @doc """ + Add a named job to the workflow along with optional dependencies. + + ## Examples + + Add jobs to a workflow with dependencies: + + Workflow.new() + |> Workflow.add(:a, WorkerA.new(%{id: id})) + |> Workflow.add(:b, WorkerB.new(%{id: id}), deps: :a) + |> Workflow.add(:c, WorkerC.new(%{id: id}), deps: :a) + |> Workflow.add(:d, WorkerC.new(%{id: id}), deps: [:b, :c]) + """ + @doc since: "1.5.0" + @spec add(t(), name(), Job.changeset(), add_opts()) :: t() + def add(%__MODULE__{} = workflow, name, changeset, opts \\ []) when is_name(name) do + {deps, opts} = Keyword.pop(opts, :deps, []) + + deps = expand_deps(deps, workflow) + name = to_string(name) + + prevent_chain!(changeset, name) + prevent_dupe!(workflow, name) + confirm_deps!(workflow, deps) + + meta = + changeset + |> Changeset.get_change(:meta, %{}) + |> Map.put(:deps, deps) + |> Map.put(:name, name) + |> Map.put(:on_hold, Enum.any?(deps)) + |> Map.put(:workflow_id, workflow.id) + |> Map.merge(workflow.opts) + |> Map.merge(Map.new(opts)) + + meta = + if Changeset.get_field(changeset, :state) == "scheduled" do + orig_at = + changeset + |> Changeset.get_change(:scheduled_at) + |> DateTime.to_unix(:microsecond) + + Map.put(meta, :orig_scheduled_at, orig_at) + else + meta + end + + changeset = + if Enum.any?(deps) do + changeset + |> Changeset.put_change(:meta, meta) + |> Changeset.put_change(:state, "scheduled") + |> Changeset.put_change(:scheduled_at, @hold_date) + else + Changeset.put_change(changeset, :meta, meta) + end + + changesets = workflow.changesets ++ [changeset] + + %{workflow | changesets: changesets, names: MapSet.put(workflow.names, name)} + end + + defp expand_deps(deps, workflow, mode \\ :main) do + deps + |> List.wrap() + |> Enum.reduce([], fn name, acc -> + name = to_string(name) + + cond do + is_map_key(workflow.grafts, name) -> + grafter_name = "grafter_#{name}" + + grafter_dep = + if is_map_key(workflow.subs, grafter_name) do + [Map.get(workflow.subs, grafter_name), "*"] + else + grafter_name + end + + [grafter_dep | [[Map.get(workflow.subs, name), "*"] | acc]] + + is_map_key(workflow.eliminations, name) -> + Map.get(workflow.eliminations, name) ++ acc + + is_map_key(workflow.subs, name) -> + [[Map.get(workflow.subs, name), "*"] | acc] + + mode == :sub -> + [[workflow.id, name] | acc] + + true -> + [name | acc] + end + end) + |> Enum.reverse() + end + + @doc """ + Add a cascading function to the workflow with optional dependencies. + + Cascading functions receive a context map containing the workflow's shared context and any + results from their upstream dependencies. Results of the function are automatically recorded, + unless they're one of the following standard control tuples: + + - `{:cancel, reason}` + - `{:error, reason}` + - `{:snooze, seconds}` + + > #### Function Captures Only {: .warning} + > + > Cascading functions _must_ be captures like `&MyApp.foo/1` for external functions or `&foo/1` + > for local functions. Anonymous functions won't work. The function always receives a single + > map parameter containing all context and previous results. + + ## Cascading Sub-Workflow + + Cascades can be added as sub-workflows that efficiently fan out workloads across multiple items. + This works similar to `add_many/4`, but for cascading functions. + + To create a fan-out cascade, provide a tuple containing an enumerable (like a list, map, or + range) and a function capture with arity 2. The function will be called once for each item in + the enumerable, with the first argument receiving the current item and the second argument + receiving the cumulative context. + + ```elixir + user_ids = [101, 202, 303] + + Workflow.new() + |> Workflow.put_context(%{operation: "sync"}) + |> Workflow.add_cascade(:users, {user_ids, &MyApp.sync_user/2}) + |> Workflow.add_cascade(:notify, &MyApp.send_summary/1, deps: :users) + + def sync_user(user_id, context) do + # ... + end + ``` + + ## Options + + A subset of standard job options can be passed directly to customize the cascade jobs. The + available options are: + + * `:max_attempts` + * `:meta` + * `:priority` + * `:queue` + * `:schedule_in` + * `:scheduled_at` + * `:tags` + + Pass the additional options alongside `deps` or other `add/4` workflow options. + + ## Examples + + Add a cascading function to a workflow: + + Workflow.new() + |> Workflow.put_context(%{user_id: 123}) + |> Workflow.add_cascade(:activate, &MyApp.activate/1) + |> Workflow.add_cascade(:notify, &MyApp.notify/1, deps: :activate) + |> Oban.insert_all() + + Customize the cascade job: + + Workflow.new() + |> Workflow.add_cascade(:activate, &MyApp.activate/1, queue: :activation) + |> Workflow.add_cascade(:retrofit, &MyApp.retrofit/1, max_attempts: 3) + + Add cascade steps from a range: + + Workflow.new() + |> Workflow.put_context(%{user_id: 123}) + |> Workflow.add_cascade(:activations, {1..5, &MyApp.activate/2}) + + Add cascade steps from a map, using the map keys as custom names: + + accounts = %{admin: 1, user: 2, guest: 3} + + Workflow.new() + |> Workflow.put_context(%{action: "sync"}) + |> Workflow.add_cascade(:accounts, {accounts, &MyApp.sync_account/2}) + """ + @doc since: "1.6.0" + @spec add_cascade(t(), name(), cascade_capture(), add_cascade_opts()) :: t() + def add_cascade(workflow, name, fun, opts \\ []) + + def add_cascade(%__MODULE__{} = workflow, name, fun, opts) when is_function(fun) do + {changeset, opts} = build_cascade(fun, opts) + + add(workflow, name, changeset, opts) + end + + def add_cascade(%__MODULE__{} = workflow, name, {items, fun}, opts) + when is_name(name) and is_function(fun, 2) do + info = Function.info(fun) + + cond do + info[:arity] != 2 -> + raise ArgumentError, "context function must have arity of 2" + + info[:module] == :erl_eval -> + raise ArgumentError, "context function must be a capture and not anonymous" + + true -> + argf = &%{arg: &1, mod: inspect(info[:module]), fun: to_string(info[:name])} + decn = "#{inspect(info[:module])}.#{info[:name]}/#{info[:arity]}" + meta = %{decorated: true, decorated_name: decn} + + {chan_opts, opts} = Keyword.split(opts, @chan_keys) + {work_opts, opts} = Keyword.split(opts, @work_keys) + + chan_opts = Keyword.update(chan_opts, :meta, meta, &Map.merge(&1, meta)) + + sub = + items + |> with_names() + |> Enum.reduce(new(work_opts), fn {key, item}, acc -> + add(acc, to_string(key), Cascade.new(argf.(item), chan_opts)) + end) + + add_workflow(workflow, name, sub, opts) + end + end + + @doc """ + Adds a grafting point to the workflow where sub-workflows can be attached later. + + A grafter serves as a placeholder in the workflow topology that downstream jobs can + depend upon. Later, another job can dynamically generate a sub-workflow and attach it + at this grafting point using `graft/3`. + + The grafter can be defined using either a job changeset or a cascading function. + + ## Examples + + Add a grafting point with a job changeset: + + Workflow.new() + |> Workflow.add_graft(:users, MyWorker.new(%{})) + |> Workflow.add(:notify, WorkerC.new(%{}), deps: :users) + + Add a grafting point with a cascading function: + + Workflow.new() + |> Workflow.put_context(%{account_id: account_id}) + |> Workflow.add_graft(:users, &build_users_workflow/1) + |> Workflow.add_cascade(:notify, &send_summary/1, deps: :users) + + Add multiple grafting points with a fan-out tuple: + + Workflow.new() + |> Workflow.add_graft(:accounts, {account_ids, &sync_account/2}) + |> Workflow.add(:finalize, FinalizerWorker.new(%{}), deps: :accounts) + + Each function in a fan-out graft receives the item and context, and should call + `apply_graft/1` to dynamically create a sub-workflow: + + def sync_account(account_id, _context) do + {user_ids(account_id), &sync_user/2} + |> Workflow.apply_graft() + |> Oban.insert_all() + + :ok + end + """ + @doc since: "1.6.0" + @spec add_graft(t(), name(), Job.changeset() | cascade_capture(), add_opts()) :: t() + def add_graft(workflow, name, changeset_or_fun, opts \\ []) + + def add_graft(%__MODULE__{} = workflow, name, fun, opts) when is_function(fun) do + {changeset, opts} = build_cascade(fun, opts) + + add_graft(workflow, name, changeset, opts) + end + + def add_graft(%__MODULE__{} = workflow, name, {_items, fun} = cascade, opts) + when is_name(name) and is_function(fun, 2) do + name = to_string(name) + uuid = "grafted_#{Oban.Pro.UUIDv7.generate()}" + grafter_name = "grafter_#{name}" + + graft_meta = %{graft_id: uuid, graft_name: name} + opts = Keyword.update(opts, :meta, graft_meta, &Map.merge(&1, graft_meta)) + + workflow = %{ + workflow + | grafts: Map.put(workflow.grafts, name, grafter_name), + subs: Map.put(workflow.subs, name, uuid) + } + + add_cascade(workflow, grafter_name, cascade, opts) + end + + def add_graft(%__MODULE__{} = workflow, name, changeset, opts) when is_name(name) do + name = to_string(name) + uuid = "grafted_#{Oban.Pro.UUIDv7.generate()}" + grafter_name = "grafter_#{name}" + + meta = + changeset + |> Changeset.get_change(:meta, %{}) + |> Map.put(:graft_id, uuid) + |> Map.put(:graft_name, name) + + changeset = Changeset.put_change(changeset, :meta, meta) + + workflow = + workflow + |> Map.update!(:grafts, &Map.put(&1, name, grafter_name)) + |> Map.update!(:subs, &Map.put(&1, name, uuid)) + + add(workflow, grafter_name, changeset, opts) + end + + defp build_cascade(fun, opts) do + info = Function.info(fun) + + cond do + info[:arity] != 1 -> + raise ArgumentError, "context function must have arity 1" + + info[:module] == :erl_eval -> + raise ArgumentError, "context function must be a capture and not anonymous" + + true -> + args = %{mod: inspect(info[:module]), fun: to_string(info[:name])} + decn = "#{inspect(info[:module])}.#{info[:name]}/#{info[:arity]}" + meta = %{decorated: true, decorated_name: decn} + + {chan_opts, opts} = + opts + |> Keyword.update(:meta, meta, &Map.merge(&1, meta)) + |> Keyword.split(@chan_keys) + + {Cascade.new(args, chan_opts), opts} + end + end + + @doc """ + Add multiple named jobs as a sub workflow along with optional dependencies. + + ## Empty Sub-Workflows + + When an empty collection of jobs is provided, the sub-workflow step is skipped entirely. + Any dependencies on this step are automatically transferred to downstream steps to maintain + the correct execution order. + + Empty sub-workflows have no job results to retrieve or cascade outputs to reference. Consider + this when creating dependencies on sub-workflows that might be empty at runtime. + + ## Examples + + Add multiple jobs as a sub workflow with a list, where the keys will be the job's position, + starting at 0: + + Workflow.add_many(workflow, :sub_1, [WorkerA.new(%{}), WorkerB.new(%{})]) + + Add multiple jobs as a sub workflow with a map, where the keys are the job names: + + Workflow.add_many(workflow, :sub_1, %{a: WorkerA.new(%{}), b: WorkerB.new(%{})}) + + Add multiple jobs as sub workflows and add a dependency on them: + + Workflow.new() + |> Workflow.add_many(:a, changesets_a) + |> Workflow.add_many(:b, changesets_b) + |> Workflow.add(:c, WorkerC.new(%{}), deps: [:a, :b]) + + Workflow options pay be provided to name or otherwise customize the sub-workflow: + + Workflow.add_many(workflow, :sub_1, changesets, ignore_cancelled: true) + """ + @doc since: "1.6.0" + @spec add_many(t(), name(), Enum.t(), new_opts() | add_opts()) :: t() + def add_many(%__MODULE__{} = workflow, name, changesets, opts \\ []) when is_name(name) do + {work_opts, opts} = Keyword.split(opts, @work_keys) + + sub = + changesets + |> with_names() + |> Enum.reduce(new(work_opts), fn {key, chg}, acc -> add(acc, to_string(key), chg) end) + + add_workflow(workflow, name, sub, opts) + end + + defp with_names(items) when is_list(items), do: Enum.with_index(items, &{&2, &1}) + defp with_names(items) when is_struct(items), do: items |> Enum.to_list() |> with_names() + defp with_names(items) when is_map(items), do: items + + @doc """ + Add a sub workflow with a name and optional dependencies to another workflow. + + This function lets you compose workflows by adding one workflow as a dependency within another + workflow. Workflows may depend on jobs or other workflows, and jobs may depend on entire + workflows. If a job depends on a workflow, it won't execute until every job in the workflow is + in a final state (completed, cancelled, or discarded). + + ## Examples + + Add a sub workflow to a workflow with dependencies: + + sub_1 = + Workflow.new() + |> Workflow.add(:a, WorkerB.new(%{id: id})) + |> Workflow.add(:b, WorkerC.new(%{id: id})) + + sub_2 = + Workflow.new() + |> Workflow.add(:a, WorkerB.new(%{id: id})) + |> Workflow.add(:b, WorkerC.new(%{id: id})) + + Workflow.new() + |> Workflow.add(:a, WorkerA.new(%{id: id})) + |> Workflow.add_workflow(:b, sub_1, deps: :a) + |> Workflow.add_workflow(:c, sub_2, deps: :b) + |> Workflow.add(:d, WorkerD.new(%{id: id}), deps: [:b, :c]) + """ + @doc since: "1.6.0" + @spec add_workflow(t(), name(), t(), add_workflow_opts()) :: t() + def add_workflow(%_{} = workflow, name, %_{} = sub_workflow, opts \\ []) when is_name(name) do + {deps, opts} = Keyword.pop(opts, :deps, []) + + deps = expand_deps(deps, workflow, :sub) + name = to_string(name) + + prevent_dupe!(workflow, name) + confirm_deps!(workflow, deps) + + sub_changesets = + Enum.map(sub_workflow.changesets, fn changeset -> + meta_deps = get_in(changeset.changes, [:meta, :deps]) + + if deps != [] and meta_deps == [] do + opts + |> Map.new() + |> Map.put(:deps, deps) + |> Map.put(:sup_workflow_id, workflow.id) + |> Map.put(:sub_name, name) + |> Map.put(:workflow_id, sub_workflow.id) + |> put_hold(changeset) + else + # Include workflow's ancestors (from append) in the chain + ancestor_ids = Map.get(workflow.opts, :ancestor_ids, []) + + meta = + if get_in(changeset.changes, [:meta, :sup_workflow_id]) do + existing = get_in(changeset.changes, [:meta, :ancestor_ids]) || [] + + %{ + sup_workflow_id: workflow.id, + ancestor_ids: existing ++ [workflow.id] ++ ancestor_ids + } + else + %{ + sub_name: name, + sup_workflow_id: workflow.id, + workflow_id: sub_workflow.id, + ancestor_ids: [workflow.id] ++ ancestor_ids + } + end + + Changeset.update_change(changeset, :meta, &Map.merge(&1, meta)) + end + end) + + subs = Map.put(workflow.subs, name, sub_workflow.id) + + case sub_changesets do + [_ | _] -> + changesets = workflow.changesets ++ sub_changesets + + %{workflow | changesets: changesets, subs: subs} + + [] -> + eliminations = Map.put(workflow.eliminations, name, deps) + + %{workflow | eliminations: eliminations, subs: subs} + end + end + + defp put_hold(meta, changeset) do + meta = + if Changeset.get_field(changeset, :state) == "scheduled" do + orig_at = + changeset + |> Changeset.get_change(:scheduled_at) + |> DateTime.to_unix(:microsecond) + + meta + |> Map.put(:on_hold, true) + |> Map.put(:orig_scheduled_at, orig_at) + else + Map.put(meta, :on_hold, true) + end + + changeset + |> Changeset.update_change(:meta, &Map.merge(&1, meta)) + |> Changeset.put_change(:state, "scheduled") + |> Changeset.put_change(:scheduled_at, @hold_date) + end + + @doc """ + Adds a context value to the workflow that can be accessed by all jobs. + + This is a convenience function that creates a recorded job named `:context` containing + any value that needs to be shared across multiple workflow jobs. + + ## Examples + + Add a context value to a workflow: + + Workflow.new() + |> Workflow.put_context(%{user_id: 123, action: "process"}) + |> Workflow.add(:job_a, WorkerA.new(%{})) + |> Workflow.add(:job_b, WorkerB.new(%{})) + |> Oban.insert_all() + + Later, retrieve the context in a worker: + + def process(job) do + context = Workflow.get_context(job) + # Use context map... + end + """ + @doc since: "1.6.0" + @spec put_context(t(), map()) :: t() + def put_context(%__MODULE__{} = workflow, value) when is_map(value) do + opts = [ + meta: %{context: true, recorded: true, return: Utils.encode64(value)}, + worker: Oban.Pro.Workers.Context + ] + + changeset = + %{} + |> Job.new(opts) + |> Changeset.put_change(:state, "completed") + |> Changeset.put_change(:completed_at, DateTime.utc_now()) + + add(workflow, :context, changeset) + end + + @doc """ + Gets a workflow's context value. + + This is a convenience function that internally calls `get_recorded/2` with the `:context` name. + + ## Examples + + Retrieve the context within a worker: + + def process(job) do + context = Workflow.get_context(job) + end + + Retrieve the context with a workflow id: + + Workflow.get_context("some-uuid-1234-5678") + + Retrieve the context using a custom Oban instance name: + + Workflow.get_context(MyApp.Oban, "some-uuid-1234-5678") + """ + @doc since: "1.6.0" + @spec get_context(Oban.name(), job_or_wid()) :: nil | map() + def get_context(oban_name \\ Oban, job_or_wid) do + get_recorded(oban_name, job_or_wid, :context) + end + + @doc """ + Instantiate a new workflow from one or more existing workflow jobs. + + ## Examples + + Append to a workflow seeded with all other jobs in the workflow: + + jobs + |> Workflow.append() + |> Workflow.add(:d, WorkerD.new(%{}), deps: [:a]) + |> Workflow.add(:e, WorkerE.new(%{}), deps: [:b]) + |> Oban.insert_all() + + Append to a workflow from a single job and bypass checking deps: + + job + |> Workflow.append(check_deps: false) + |> Workflow.add(:d, WorkerD.new(%{}), deps: [:a]) + |> Workflow.add(:e, WorkerE.new(%{}), deps: [:b]) + |> Oban.insert_all() + """ + @doc since: "1.5.0" + @spec append(Job.t() | [Job.t()], append_opts()) :: t() + def append(jobs, opts \\ []) + + def append([%Job{meta: %{"workflow_id" => _} = meta} | _] = jobs, opts) do + {check, opts} = Keyword.pop(opts, :check_deps, true) + + workflow_opts = + meta + |> Map.take(~w(workflow_id workflow_name)) + |> Keyword.new(fn {key, val} -> {String.to_existing_atom(key), val} end) + + # Preserve ancestor chain for nested sub-workflows + ancestors = + case meta do + %{"ancestor_ids" => ids} -> ids + %{"sup_workflow_id" => sup} -> [sup] + _ -> [] + end + + workflow = + opts + |> Keyword.merge(workflow_opts) + |> new() + + opts = Map.put(workflow.opts, :ancestor_ids, ancestors) + + %{workflow | check_deps: check, names: MapSet.new(jobs, & &1.meta["name"]), opts: opts} + end + + def append(job, opts), do: append([job], opts) + + @doc """ + Attaches a sub-workflow to a previously defined grafting point. + + Grafting points are defined using `add_graft/4` and serve as placeholders where sub-workflows can be + attached dynamically during execution. This function must be called from within a worker's + `process/1` callback, and only from a job that was defined using `add_graft/4`. + + ## Examples + + Graft a sub-workflow using a cascade: + + def process(job) do + user_ids = active_user_ids(job.args["account_id"]) + + {user_ids, &build_user_workflow/2} + |> Workflow.apply_graft() + |> Oban.insert_all() + + :ok + end + + Graft a sub-workflow using a workflow: + + def process(job) do + workflow = + job.args["account_id"] + |> active_user_ids() + |> Enum.reduce(Workflow.new(), fn user_id, workflow -> + Workflow.add(workflow, user_id, UserWorker.new(%{user_id: user_id})) + end) + + workflow + |> Workflow.apply_graft() + |> Oban.insert_all() + + :ok + end + + Add workflow-specific options when grafting: + + {user_ids, &build_user_workflow/2} + |> Workflow.apply_graft(workflow_name: "my-workflow") + |> Oban.insert_all() + """ + @doc since: "1.6.0" + @spec apply_graft(t() | cascade_capture(), new_opts() | add_cascade_opts()) :: t() + def apply_graft(cascade_or_workflow, opts \\ []) + + def apply_graft({_enum, fun} = cascade, opts) when is_function(fun) do + {job, graft_id, graft_name} = extract_graft!() + + job + |> append(check_deps: false) + |> add_cascade(graft_name, cascade, Keyword.put(opts, :workflow_id, graft_id)) + end + + def apply_graft(%__MODULE__{} = workflow, opts) do + {job, graft_id, graft_name} = extract_graft!() + + root_id = + case job.meta["workflow_id"] do + "grafted_" <> _ = parent_graft_id -> parent_graft_id + _ -> graft_id + end + + sub_names = expand_sub_names(workflow) + + changesets = + Enum.map(workflow.changesets, fn changeset -> + changeset = Changeset.update_change(changeset, :meta, &Map.put(&1, :workflow_id, root_id)) + + # Convert internal wildcard deps to explicit name deps so internal dependencies resolve + # correctly. + deps = get_in(changeset.changes, [:meta, :deps]) || [] + + updated_deps = + Enum.flat_map(deps, fn + [sub_id, "*"] -> + case Map.get(sub_names, sub_id) do + nil -> [[sub_id, "*"]] + names -> names + end + + other -> + [other] + end) + + if updated_deps != deps do + Changeset.update_change(changeset, :meta, &Map.put(&1, :deps, updated_deps)) + else + changeset + end + end) + + workflow = %{workflow | id: root_id, changesets: changesets} + + job + |> append(check_deps: false) + |> add_workflow(graft_name, workflow, opts) + end + + defp expand_sub_names(workflow) do + Enum.reduce(workflow.changesets, %{}, fn changeset, acc -> + meta = changeset.changes[:meta] || %{} + name = meta[:name] + + sub_id = if meta[:sup_workflow_id], do: meta[:workflow_id] + + if sub_id && name do + Map.update(acc, sub_id, [name], &[name | &1]) + else + acc + end + end) + end + + defp extract_graft! do + case Process.get(:oban_processing) do + {_module, %{meta: %{"graft_id" => _}} = job, _opts} -> + %{"graft_id" => graft_id, "graft_name" => graft_name} = job.meta + + {job, graft_id, graft_name} + + {_module, _job, _opts} -> + raise "current job isnt't marked for grafting with add_graft/4" + + _ -> + raise "current job isn't available, graft/1 must be called from process/1" + end + end + + @doc false + def all_jobs(job_or_wid), do: all_jobs(Oban, job_or_wid, []) + + @doc false + def all_jobs(job_or_wid, opts) when is_job_or_wid(job_or_wid) and is_list(opts) do + all_jobs(Oban, job_or_wid, opts) + end + + def all_jobs(oban_name, job_or_wid) when is_job_or_wid(job_or_wid) do + all_jobs(oban_name, job_or_wid, []) + end + + @doc """ + Get all jobs for a workflow, optionally filtered by upstream deps. + + ## Examples + + Retrieve all workflow jobs within a `process/1` function: + + def process(job) do + job + |> Workflow.all_jobs() + |> do_things_with_jobs() + + :ok + end + + Retrieve all of the current job's deps: + + jobs = Workflow.all_jobs(job, only_deps: true) + + Retrieve an explicit list of dependencies: + + [job_a, job_b] = Workflow.all_jobs(job, names: [:a, :b]) + + Include sub workflow jobs: + + [sub_a, sub_b] = Workflow.all_jobs(job, names: [:sub], with_subs: true) + + Retrieve all jobs using a `workflow_id`: + + jobs = Workflow.all_jobs("some-uuid-1234-5678") + + Retrieve only some named jobs using a `workflow_id`: + + [job_a, job_b] = Workflow.all_jobs("some-uuid-1234-5678", deps: [:a, :b]) + + Retrieve all jobs using a `workflow_id` and a custom Oban instance name: + + jobs = Workflow.all_jobs(MyApp.Oban, "some-uuid-1234-5678") + """ + @doc since: "1.5.0" + @spec all_jobs(Oban.name(), job_or_wid(), fetch_opts()) :: [Job.t()] + def all_jobs(oban_name, job_or_wid, opts) + + def all_jobs(_oban_name, %Job{conf: conf, meta: meta}, opts) do + %{"deps" => deps, "workflow_id" => workflow_id} = meta + + {query_opts, opts} = Keyword.split(opts, ~w(names only_deps with_subs)a) + + Repo.all(conf, workflow_query(workflow_id, deps, query_opts), opts) + end + + def all_jobs(oban_name, workflow_id, opts) when is_binary(workflow_id) do + if Keyword.has_key?(opts, :only_deps) do + raise RuntimeError, ":only_deps isn't a valid option with a workflow_id" + end + + {query_opts, opts} = Keyword.split(opts, ~w(names with_subs)a) + + oban_name + |> Oban.config() + |> Repo.all(workflow_query(workflow_id, [], query_opts), opts) + end + + @doc false + def all_recorded(job_or_wid), do: all_recorded(Oban, job_or_wid, []) + + @doc false + def all_recorded(job_or_wid, opts) when is_job_or_wid(job_or_wid) and is_list(opts) do + all_recorded(Oban, job_or_wid, opts) + end + + def all_recorded(oban_name, job_or_wid) when is_job_or_wid(job_or_wid) do + all_recorded(oban_name, job_or_wid, []) + end + + @doc """ + Get all recordings for workflow jobs, optionally filtered by name or relationship. + + The result is a map of the job's name and recorded output. Unrecorded jobs, or those without a + recording will be present in the map with a a `nil` value. + + ## Examples + + Retrieve results for all workflow jobs within a `process/1` function: + + def process(job) do + %{"a" => job_a_return, "b" => job_b_return} = Workflow.all_recorded(job) + + :ok + end + + Retrieve recordings for all of the current job's deps: + + recorded = Workflow.all_recorded(job, only_deps: true) + + Retrieve recordings for a list of dependencies: + + %{"a" => _, "b" => _} = Workflow.all_recorded(job, names: [:a, :b]) + + Retrieve recordings for some named jobs using the `workflow_id`: + + %{"a" => _, "b" => _} = Workflow.all_recorded("some-uuid-1234-5678", names: [:a, :b]) + + Retrieve recordings including results from grafted sub-workflows: + + recorded = Workflow.all_recorded(job, with_subs: true) + + Retrieve recordings for a specific grafted sub-workflow by name: + + %{"users" => user_results} = Workflow.all_recorded(job, names: [:users], with_subs: true) + """ + @doc since: "1.5.0" + @spec all_recorded(Oban.name(), job_or_wid(), fetch_opts()) :: map() + def all_recorded(oban_name, job_or_wid, opts) + + def all_recorded(_oban_name, %Job{conf: conf, meta: meta}, opts) do + %{"deps" => deps, "workflow_id" => workflow_id} = meta + + all_recorded(conf, workflow_id, deps, opts) + end + + def all_recorded(%Config{} = conf, workflow_id, opts) when is_binary(workflow_id) do + if Keyword.has_key?(opts, :only_deps) do + raise RuntimeError, ":only_deps isn't a valid option with a workflow_id" + end + + all_recorded(conf, workflow_id, [], opts) + end + + def all_recorded(oban_name, workflow_id, opts) when is_binary(workflow_id) do + oban_name + |> Oban.config() + |> all_recorded(workflow_id, [], opts) + end + + defp all_recorded(conf, workflow_id, deps, opts) when is_binary(workflow_id) do + {query_opts, opts} = Keyword.split(opts, ~w(names only_deps with_subs)a) + + with_subs = Keyword.get(query_opts, :with_subs, false) + + query = + workflow_id + |> workflow_query(deps, query_opts) + |> select([j], j.meta) + + conf + |> Repo.all(query, opts) + |> Enum.reject(&Map.get(&1, "context")) + |> Enum.reduce(%{}, fn meta, acc -> + case meta do + %{"name" => name, "sub_name" => work_name} when with_subs -> + acc + |> Map.put_new(work_name, %{}) + |> put_in([work_name, name], extract_recorded(meta)) + + %{"name" => name} -> + Map.put(acc, name, extract_recorded(meta)) + end + end) + end + + @doc false + def cancel_jobs(job_or_wid), do: cancel_jobs(Oban, job_or_wid, []) + + @doc false + def cancel_jobs(job_or_wid, opts) when is_job_or_wid(job_or_wid) and is_list(opts) do + cancel_jobs(Oban, job_or_wid, opts) + end + + def cancel_jobs(oban_name, job_or_wid) when is_job_or_wid(job_or_wid) do + cancel_jobs(oban_name, job_or_wid, []) + end + + @doc """ + Cancel one or more workflow jobs. + + This uses `Oban.cancel_all_jobs/2` internally, and adheres to the same cancellation rules. + Namely, it won't touch jobs in a `completed`, `cancelled`, or `discarded` state. + + Cancelling jobs via this function bulk-updates their state in the database without loading them + into memory or resolving their workers. As a result, **worker `after_process/3` hooks are not + called** for cancelled jobs. If you need to execute cleanup logic or other side effects when jobs + are cancelled, consider using workflow callbacks like `after_cancelled/2` or implement cleanup + logic at the application level. + + ## Examples + + Cancel jobs with a workflow `job` from a `process/1` function: + + def process(job) do + if should_stop_processing?(job.args) do + Workflow.cancel_jobs(job) + else + ... + end + end + + Cancel specific workflow jobs: + + Workflow.cancel_jobs("some-uuid-1234-5678", names: [:a, :b]) + + Cancel all jobs in a workflow by `workflow_id`: + + Workflow.cancel_jobs("some-uuid-1234-5678") + + Cancel jobs from anywhere with a custom Oban instance name: + + Workflow.cancel_jobs(MyApp.Oban, "some-uuid-1234-5678") + """ + @doc since: "1.5.0" + @spec cancel_jobs(Oban.name(), job_or_wid(), cancel_opts()) :: {:ok, integer()} + def cancel_jobs(oban_name, job_or_wid, opts) + + def cancel_jobs(_oban_name, %Job{conf: conf, meta: %{"workflow_id" => wid}}, opts) do + cancel_jobs(conf.name, wid, opts) + end + + def cancel_jobs(oban_name, wid, opts) when is_binary(wid) do + query_opts = + opts + |> Keyword.take([:names, :with_subs]) + |> Keyword.put_new(:with_subs, true) + + Oban.cancel_all_jobs(oban_name, workflow_query(wid, [], query_opts)) + end + + @doc """ + Retry all jobs in a workflow. + + By default, this function will retry all jobs that aren't `executing` or `available`. Jobs with + dependencies will be placed on hold to run again once their dependencies are complete. + + ## Examples + + Retry only failed jobs in a workflow from within a process function: + + def process(job) do + Workflow.retry_jobs(job) + end + + Retry a workflow by ID: + + Workflow.retry_jobs("some-uuid-1234-5678") + + Retry a workflow with a custom Oban instance: + + Workflow.retry_jobs(MyApp.Oban, "some-uuid-1234-5678") + """ + @doc since: "1.6.0" + @spec retry_jobs(Oban.name(), job_or_wid(), keyword()) :: {:ok, integer()} + def retry_jobs(oban_name \\ Oban, job_or_wid, opts \\ []) + + def retry_jobs(_oban_name, %Job{conf: conf, meta: %{"workflow_id" => wid}}, opts) do + retry_jobs(conf.name, wid, opts) + end + + def retry_jobs(oban_name, wid, opts) when is_binary(wid) do + conf = Oban.config(oban_name) + + Repo.transaction(conf, fn -> + jobs = all_jobs(oban_name, wid, opts) + + {with_deps, wout_deps} = + jobs + |> Enum.reject(& &1.meta["context"]) + |> Enum.split_with(fn %{meta: meta} -> Enum.any?(meta["deps"]) end) + + base = where(Job, [j], j.state not in ~w(available executing)) + + if Enum.any?(wout_deps) do + query = + base + |> where([j], j.id in ^Enum.map(wout_deps, & &1.id)) + |> update([j], + set: [ + state: "available", + max_attempts: fragment("GREATEST(?, ? + 1)", j.max_attempts, j.attempt), + scheduled_at: ^DateTime.utc_now(), + completed_at: nil, + cancelled_at: nil, + discarded_at: nil + ] + ) + + Repo.update_all(conf, query, []) + end + + if Enum.any?(with_deps) do + query = + base + |> where([j], j.id in ^Enum.map(with_deps, & &1.id)) + |> update([j], + set: [ + state: "scheduled", + max_attempts: fragment("GREATEST(?, ? + 1)", j.max_attempts, j.attempt), + meta: fragment("(? || ?) - 'orig_scheduled_at'", j.meta, ^%{on_hold: true}), + scheduled_at: ^DateTime.utc_now(), + completed_at: nil, + cancelled_at: nil, + discarded_at: nil + ] + ) + + Repo.update_all(conf, query, []) + end + + length(jobs) + end) + end + + @doc """ + Get a single workflow job by name. + + ## Examples + + Get the workflow job named `:step_1` from within `process/`: + + def process(job) do + case Workflow.get_job(job, :step_1) do + nil -> ... + dep_job -> use_the_job(dep_job) + end + end + + Get a named workflow job with the `workflow_id`: + + Workflow.get_job("some-uuid-1234-5678", :step_2) + + Get a named workflow job with the `workflow_id` and a custom Oban instance name: + + Workflow.get_job(MyApp.Oban, "some-uuid-1234-5678", :step_2) + """ + @doc since: "1.5.0" + @spec get_job(Oban.name(), job_or_wid(), name()) :: nil | Job.t() + def get_job(oban_name \\ Oban, job_or_wid, deps_name) + + def get_job(_oban_name, %Job{conf: conf, meta: %{"workflow_id" => wid}}, deps_name) do + get_job(conf.name, wid, deps_name) + end + + def get_job(oban_name, wid, deps_name) when is_binary(wid) do + case all_jobs(oban_name, wid, names: [deps_name]) do + [job | _] -> job + [] -> nil + end + end + + @doc """ + Fetch the recorded output from a workflow job. + + This function provides an optimized way to get recorded output with a single query. + + A `nil` value is returned when a dependency is incomplete, missing, unrecorded, or lacks + recorded output. + + ## Examples + + Get the recorded output from `:step_1` within `process/`: + + def process(job) do + case Workflow.get_recorded(job, :step_1) do + nil -> ... + val -> use_recorded_output(val) + end + end + + Get recorded output using the workflow id: + + Workflow.get_recorded("some-uuid-1234-5678", :step_2) + + Get recorded output using the `workflow_id` and a custom Oban instance name: + + Workflow.get_recorded(MyApp.Oban, "some-uuid-1234-5678", :step_2) + """ + @doc since: "1.5.0" + @spec get_recorded(Oban.name(), job_or_wid(), name()) :: any() + def get_recorded(oban_name \\ Oban, job_or_wid, deps_name) + + def get_recorded(_oban_name, %Job{conf: conf, meta: %{"workflow_id" => wid}}, deps_name) do + get_recorded(conf, wid, deps_name) + end + + def get_recorded(%Config{} = conf, wid, deps_name) when is_binary(wid) do + query = + wid + |> workflow_query([], names: [deps_name]) + |> select([j], j.meta) + + conf + |> Repo.one(query) + |> extract_recorded() + end + + def get_recorded(oban_name, wid, deps_name) when is_binary(wid) do + oban_name + |> Oban.config() + |> get_recorded(wid, deps_name) + end + + @doc false + def stream_jobs(job_or_wid), do: stream_jobs(Oban, job_or_wid, []) + + @doc false + def stream_jobs(job_or_wid, opts) when is_job_or_wid(job_or_wid) and is_list(opts) do + stream_jobs(Oban, job_or_wid, opts) + end + + def stream_jobs(oban_name, job_or_wid) when is_job_or_wid(job_or_wid) do + stream_jobs(oban_name, job_or_wid, []) + end + + @doc """ + Stream all jobs for a workflow. + + This function behaves identically to `all_jobs/3`, except it streams jobs lazily from within a + `Repo` transaction. + + ## Examples + + Stream with filtering to only preserve `completed` jobs: + + def process(job) do + {:ok, workflow_jobs} = + MyApp.Repo.transaction(fn -> + job + |> Workflow.stream_jobs() + |> Stream.filter(& &1.state == "completed") + |> Enum.to_list() + end) + + do_things_with_jobs(workflow_jobs) + + :ok + end + + Stream workflow jobs from anywhere using the `workflow_id`: + + MyApp.Repo.transaction(fn -> + "some-uuid-1234-5678" + |> Workflow.stream_jobs() + |> Enum.map(& &1.args["account_id"]) + end) + + Stream workflow jobs using a custom Oban instance name: + + MyApp.Repo.transaction(fn -> + MyApp.Oban + |> Workflow.stream_jobs("some-uuid-1234-5678") + |> Enum.map(& &1.args["account_id"]) + end) + """ + @doc since: "1.5.0" + @spec stream_jobs(Oban.name(), Job.t() | String.t(), fetch_opts()) :: Enum.t() + def stream_jobs(oban_name, job_or_wid, opts) + + def stream_jobs(_oban_name, %Job{conf: conf, meta: meta}, opts) do + %{"deps" => deps, "workflow_id" => wid} = meta + + {query_opts, opts} = Keyword.split(opts, [:names, :only_deps]) + + Repo.stream(conf, workflow_query(wid, deps, query_opts), opts) + end + + def stream_jobs(oban_name, wid, opts) when is_binary(wid) do + if Keyword.has_key?(opts, :only_deps) do + raise RuntimeError, ":only_deps isn't a valid option with a workflow_id" + end + + {query_opts, opts} = Keyword.split(opts, [:names]) + + oban_name + |> Oban.config() + |> Repo.stream(workflow_query(wid, [], query_opts), opts) + end + + @doc """ + Get detailed status information about a workflow. + + Returns a map with the following fields: + + - `:id` - The workflow's unique id + - `:name` - The name of the workflow, if it has one + - `:total` - Total number of jobs in the workflow + - `:state` - Overall status of the workflow (e.g. executing, completed, cancelled, discarded) + - `:counts` - Map of job state counts (e.g. `%{completed: 10, available: 2}`) + - `:duration` - Elapsed time of the workflow in milliseconds (nil if not yet ran) + - `:started_at` - When the first job in the workflow was inserted + - `:stopped_at` - When the last job in the workflow finished + - `:subs` - A map of status output for sub-worfklows where the key is the sub name and the value + is a `status/1` output + + An example of the full output: + + ```elixir + %{ + id: "some-uuid-1234-5678", + name: "my-workflow", + total: 3, + state: :completed, + duration: 72, + counts: %{ + cancelled: 0, + available: 0, + completed: 3, + scheduled: 0, + discarded: 0, + executing: 0, + retryable: 0 + }, + started_at: ~U[2025-04-10 16:34:34.114730Z], + stopped_at: ~U[2025-04-10 16:34:34.186935Z], + subs: %{} + } + ``` + + ## Examples + + Get a workflow's status using a job: + + status = Workflow.status(job) + + Get a workflow's status using the workflow_id: + + status = Workflow.status("some-uuid-1234-5678") + + Get a workflow's status using a custom Oban instance name: + + status = Workflow.status(MyApp.Oban, "some-uuid-1234-5678") + """ + @doc since: "1.6.0" + @spec status(Oban.name(), job_or_wid()) :: status() + def status(oban_name \\ Oban, job_or_wid) + + def status(_oban_name, %Job{conf: conf, meta: %{"workflow_id" => wid}}) do + status(conf, wid) + end + + def status(%Config{} = conf, workflow_id) when is_binary(workflow_id) do + query = + Job + |> where([j], in_workflow(j.meta, ^workflow_id)) + |> group_by([j], [j.meta["workflow_id"], j.meta["workflow_name"]]) + |> select([j], %{ + id: j.meta["workflow_id"], + name: j.meta["workflow_name"], + ignore_cancelled: fragment("array_agg(?)", j.meta["ignore_cancelled"]), + ignore_discarded: fragment("array_agg(?)", j.meta["ignore_discarded"]), + states: fragment("array_agg(?)", j.state), + min_scheduled: min(j.inserted_at), + max_completed: max(j.completed_at), + max_cancelled: max(j.cancelled_at), + max_discarded: max(j.discarded_at) + }) + + case Repo.one(conf, query) do + nil -> + %{ + id: workflow_id, + name: nil, + total: 0, + state: :unknown, + counts: Map.new(Job.states(), &{&1, 0}), + duration: nil, + started_at: nil, + stopped_at: nil, + subs: %{} + } + + result -> + expand_status(conf, workflow_id, result) + end + end + + def status(oban_name, workflow_id) when is_binary(workflow_id) do + oban_name + |> Oban.config() + |> status(workflow_id) + end + + defp expand_status(conf, workflow_id, result) do + total = length(result.states) + empty = Map.new(Job.states(), &{&1, 0}) + + counts = + [result.states, result.ignore_cancelled, result.ignore_discarded] + |> Enum.zip() + |> Enum.reduce(empty, fn tuple, acc -> + state = + case tuple do + {"cancelled", true, _} -> :completed + {"discarded", _, true} -> :completed + {state, _ican, _idisc} -> String.to_existing_atom(state) + end + + Map.update(acc, state, 1, &(&1 + 1)) + end) + + state = + cond do + counts.completed + counts.cancelled + counts.discarded < total -> :executing + counts.cancelled > 0 -> :cancelled + counts.discarded > 0 -> :discarded + true -> :completed + end + + stopped_at = + [result.max_completed, result.max_cancelled, result.max_discarded] + |> Enum.reject(&is_nil/1) + |> Enum.max(fn -> nil end) + + duration = + if stopped_at do + DateTime.diff(stopped_at, result.min_scheduled, :millisecond) + else + nil + end + + subs = + Job + |> where([j], fragment("? \\? 'sup_workflow_id'", j.meta)) + |> where([j], fragment("?->>'sup_workflow_id' = ?", j.meta, ^workflow_id)) + |> select([j], %{wid: j.meta["workflow_id"], name: j.meta["sub_name"]}) + |> distinct(true) + |> then(&Repo.all(conf, &1)) + |> Map.new(fn %{wid: wid, name: name} -> {name, status(conf, wid)} end) + + %{ + id: result.id, + name: result.name, + total: total, + state: state, + counts: counts, + duration: duration, + started_at: result.min_scheduled, + stopped_at: stopped_at, + subs: subs + } + end + + @doc """ + Converts the given workflow to a `:digraph`. + + The resulting digraph can be explored, evaluated, and then be converted to a number of other + formats. + + ## Examples + + Generate a digraph from a workflow: + + graph = Workflow.to_graph(workflow) + + Check the path between jobs in a workflow: + + workflow + |> Workflow.to_graph() + |> :digraph.get_path("step_1", "step_5") + """ + @doc since: "1.5.0" + @spec to_graph(flow :: t()) :: :digraph.graph() + def to_graph(%__MODULE__{changesets: changesets, subs: subs}) do + graph = :digraph.new([:acyclic]) + + changesets + |> Enum.group_by(& &1.changes.meta[:sub_name]) + |> Enum.each(fn + {nil, changesets} -> + add_graph_nodes(graph, changesets) + + {sub, changesets} -> + sub_graph = + [:acyclic] + |> :digraph.new() + |> add_graph_nodes(changesets) + + :digraph.add_vertex(graph, sub, sub_graph) + end) + + for %{changes: %{meta: meta}} <- changesets, dep <- meta.deps do + # This would also create edges in the top graph for values that shouldn't be there. + case dep do + [wid, "*"] -> + sub_name = Enum.find_value(subs, &if(elem(&1, 1) == wid, do: elem(&1, 0))) + dep_name = meta[:sub_name] || meta[:name] + + :digraph.add_edge(graph, sub_name, dep_name) + + [_id, dep] -> + :digraph.add_edge(graph, dep, meta.sub_name) + + dep -> + :digraph.add_edge(graph, dep, meta.name) + end + end + + graph + end + + defp add_graph_nodes(graph, changesets) do + for %{changes: %{meta: meta, worker: worker}} <- changesets do + label = Map.get(meta, :decorated_name, worker) + + :digraph.add_vertex(graph, meta.name, label) + end + + graph + end + + @doc """ + Converts the given workflow to a [graphviz](https://graphviz.org/) `dot` digraph. + + ## Examples + + Generate a dot digraph: + + Workflow.to_dot(workflow) + """ + @doc since: "1.5.0" + @spec to_dot(flow :: t()) :: String.t() + def to_dot(%__MODULE__{} = workflow) do + graph = to_graph(workflow) + nodes = :digraph.vertices(graph) + edges = :digraph.edges(graph) + + nodes_dot = visit_nodes(nodes, graph, :dot) + edges_dot = visit_edges(edges, graph, :dot) + + """ + digraph { + node [shape=box]; + #{nodes_dot} + #{edges_dot} + } + """ + end + + @doc """ + Generate a [Mermaid](https://mermaid.js.org/) flowchart in top-down orientation from a workflow. + + ## Examples + + Generate a flowchart: + + Workflow.to_mermaid(workflow) + """ + @doc since: "1.5.0" + @spec to_mermaid(flow :: t()) :: String.t() + def to_mermaid(%__MODULE__{} = workflow) do + graph = to_graph(workflow) + nodes = :digraph.vertices(graph) + edges = :digraph.edges(graph) + + nodes_mermaid = visit_nodes(nodes, graph, :mermaid) + edges_mermaid = visit_edges(edges, graph, :mermaid) + + """ + flowchart TD + #{nodes_mermaid} + #{edges_mermaid} + """ + end + + @doc false + def rescue_workflows(conf, opts \\ []) do + {limit, opts} = Keyword.pop(opts, :limit, 1_000) + + query = + Job + |> where([j], j.state == "scheduled") + |> where([j], fragment("? \\? 'workflow_id'", j.meta)) + |> where([j], fragment("?->>'on_hold' = 'true'", j.meta)) + |> select([j], fragment("?->>'workflow_id'", j.meta)) + |> distinct(true) + |> limit(^limit) + + conf + |> Repo.all(query, opts) + |> on_flush(conf) + end + + # Construction Helpers + + defp prevent_chain!(changeset, name) do + if Chain.chain?(changeset) do + raise "#{inspect(name)} is a chain, which isn't allowed in a workflow" + end + end + + defp prevent_dupe!(%{names: names, subs: subs}, name) do + if MapSet.member?(names, name) or is_map_key(subs, name) do + raise "#{inspect(name)} is already a member of the workflow" + end + end + + defp confirm_deps!(%{check_deps: true, names: names, grafts: grafts}, [_ | _] = deps) do + Enum.each(deps, fn + [_wd, "*"] -> + :ok + + [_wd, name] -> + if not MapSet.member?(names, name) do + raise "deps #{inspect(name)} is not a member of the workflow" + end + + name when is_binary(name) -> + if not (MapSet.member?(names, name) or is_map_key(grafts, name)) do + raise "deps #{inspect(name)} is not a member of the workflow" + end + end) + end + + defp confirm_deps!(_workflow, _deps), do: :ok + + # Visualization Helpers + + defp visit_nodes(nodes, graph, mode, prefix \\ nil) do + Enum.map_join(nodes, "\n", fn node -> + {_node, label} = :digraph.vertex(graph, node) + + if is_tuple(label) do + sub_nodes = visit_nodes(:digraph.vertices(label), label, mode, "#{node}_") + sub_edges = visit_edges(:digraph.edges(label), label, mode) + + format_subg(mode, node, sub_nodes, sub_edges) + else + format_node(mode, "#{prefix}#{node}", label) + end + end) + end + + defp visit_edges(edges, graph, mode) do + edges + |> Enum.map(fn edge -> + {_, in_vert, out_vert, _} = :digraph.edge(graph, edge) + + {in_vert, out_vert} + end) + |> Enum.uniq() + |> Enum.map_join("\n ", &format_edge(mode, &1)) + end + + defp format_node(:dot, node, label) do + ~s|"#{node}" [label="#{node} (#{label})"];| + end + + defp format_node(:mermaid, node, label) do + ~s|#{node}["#{node} (#{label})"]| + end + + defp format_edge(:dot, {in_vert, out_vert}) do + ~s|"#{in_vert}" -> "#{out_vert}";| + end + + defp format_edge(:mermaid, {in_vert, out_vert}) do + "#{in_vert} --> #{out_vert}" + end + + defp format_subg(:dot, node, sub_nodes, sub_edges) do + """ + subgraph "cluster_#{node}" { + label="#{node}"; + #{sub_nodes} + #{sub_edges} + } + """ + end + + defp format_subg(:mermaid, node, sub_nodes, sub_edges) do + """ + subgraph #{node} + #{sub_nodes} + #{sub_edges} + end + """ + end + + # Query Helpers + + defp workflow_query(wid, deps, opts) do + query = + Job + |> where([j], j.state in @all_states) + |> order_by(asc: :id) + + query = + if opts[:with_subs] do + where(query, [j], in_workflow_or_sup(j.meta, ^wid)) + else + where(query, [j], in_workflow(j.meta, ^wid)) + end + + cond do + opts[:with_subs] && is_list(opts[:names]) -> + names = Enum.map(opts[:names], &to_string/1) + + where( + query, + [j], + fragment("coalesce(?->>'sub_name', ?->>'name')", j.meta, j.meta) in ^names + ) + + is_list(opts[:names]) -> + names = Enum.map(opts[:names], &to_string/1) + + where(query, [j], j.meta["name"] in ^names) + + opts[:with_subs] && opts[:only_deps] -> + where( + query, + [j], + fragment("coalesce(?->>'sub_name', ?->>'name')", j.meta, j.meta) in ^deps + ) + + opts[:only_deps] -> + where(query, [j], j.meta["name"] in ^deps) + + true -> + query + end + end + + defp to_stage_operation({%{meta: meta} = job, deps_states}) do + # A grafted sub-workflow won't have jobs until the grafter runs. Ignore the "deleted" check + # for the sub-workflow and rely entirely on the grafter job itself. + deps = Enum.reject(meta["deps"], &match?(["grafted_" <> _, "*"], &1)) + + deps_set = + if length(deps_states) < length(deps) do + MapSet.new(["deleted" | deps_states]) + else + MapSet.new(deps_states) + end + + op = + cond do + MapSet.member?(deps_set, "deleted") and meta["ignore_deleted"] != true -> + {:cancelled, :deleted} + + MapSet.member?(deps_set, "cancelled") and meta["ignore_cancelled"] != true -> + {:cancelled, :cancelled} + + MapSet.member?(deps_set, "discarded") and meta["ignore_discarded"] != true -> + {:cancelled, :discarded} + + not MapSet.disjoint?(deps_set, @incomplete_set) -> + :ignore + + meta["context"] == true -> + :ignore + + is_legacy_scheduled(meta) -> + {:scheduled, DateTime.from_unix!(meta["orig_scheduled_at"])} + + is_map_key(meta, "orig_scheduled_at") -> + {:scheduled, DateTime.from_unix!(meta["orig_scheduled_at"], :microsecond)} + + true -> + :available + end + + {op, job} + end + + defp apply_operations({:ignore, _}, _), do: [] + + defp apply_operations({op, jobs}, conf) do + now = DateTime.utc_now() + ids = Enum.map(jobs, & &1.id) + + base = + Job + |> where([j], j.id in ^ids and j.state == "scheduled") + |> update([j], set: [meta: drop_hold(j.meta)]) + + query = + case op do + :available -> + base + |> select([j], nil) + |> update([j], set: [scheduled_at: ^now, state: "available"]) + + {:scheduled, at} -> + base + |> select([j], nil) + |> update([j], set: [scheduled_at: ^at, state: "scheduled"]) + + {:cancelled, reason} -> + error = %{ + kind: :error, + reason: Oban.Pro.WorkflowError.exception(reason), + stacktrace: [] + } + + formatted = Job.format_attempt(%Job{attempt: 1, unsaved_error: error}) + + base + |> select([j], j.meta["workflow_id"]) + |> update([j], + set: [ + cancelled_at: ^now, + state: "cancelled", + errors: concat_errors(j.errors, ^[formatted]) + ] + ) + end + + {_count, workflow_ids} = Repo.update_all(conf, query, set: []) + + with {:cancelled, reason} <- op, do: Enum.each(jobs, &apply_callback(reason, &1, conf)) + + workflow_ids + |> Enum.reject(&is_nil/1) + |> Enum.uniq() + end + + defp apply_callback(reason, job, conf) do + with {:ok, worker} <- Oban.Worker.from_string(job.worker), + true <- function_exported?(worker, :after_cancelled, 2), + {:ok, job} <- Oban.Pro.Worker.before_process(worker, job, worker.__opts__()) do + worker.after_cancelled(reason, %{job | conf: conf}) + end + catch + kind, value -> + Logger.error(fn -> + "[Oban.Pro.Workflow] callback error: " <> Exception.format(kind, value, __STACKTRACE__) + end) + + :ok + end + + # Recorded Helpers + + defp extract_recorded(meta) do + case meta do + %{"recorded" => true, "return" => value, "safe_decode" => true} -> + Utils.decode64(value, [:safe]) + + %{"recorded" => true, "return" => value} -> + Utils.decode64(value, []) + + _ -> + nil + end + end +end diff --git a/vendor/oban_pro/lib/oban/pro/workflow/cascade.ex b/vendor/oban_pro/lib/oban/pro/workflow/cascade.ex new file mode 100644 index 00000000..997614ba --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/workflow/cascade.ex @@ -0,0 +1,87 @@ +defmodule Oban.Pro.Workflow.Cascade do + @moduledoc false + + use Oban.Pro.Worker, recorded: [limit: 256_000] + + alias Oban.Pro.Workflow + + args_schema do + field :mod, :string + field :fun, :string + field :arg, :term, default: :none + end + + @impl Oban.Pro.Worker + def process(%{args: %{arg: arg, mod: mod, fun: fun}, conf: conf, meta: meta}) do + mod = + mod + |> String.split(".") + |> Module.safe_concat() + + fun = String.to_existing_atom(fun) + + full = Map.merge(all_context(conf, meta), all_recorded(conf, meta)) + args = if arg == :none, do: [full], else: [arg, full] + + case apply(mod, fun, args) do + {:ok, _} = result -> result + {:error, _} = error -> error + {:snooze, _} = snooze -> snooze + {:cancel, _} = cancel -> cancel + other -> {:ok, other} + end + end + + defp all_context(conf, meta) do + sup = Map.get(meta, "sup_workflow_id") + wid = Map.get(meta, "workflow_id") + ancestors = Map.get(meta, "ancestor_ids", []) + + (Enum.reverse(ancestors) ++ [sup, wid]) + |> Enum.filter(& &1) + |> Enum.uniq() + |> Enum.reduce(%{}, fn id, acc -> + Map.merge(acc, Workflow.get_context(conf, id) || %{}) + end) + end + + defp all_recorded(_conf, %{"deps" => []}), do: %{} + + defp all_recorded(conf, %{"deps" => deps, "workflow_id" => wid} = meta) do + atomize? = Map.get(meta, "atom_keys", true) + + grouper = fn + [sup, name], acc -> Map.update(acc, sup, [name], &[name | &1]) + name, acc -> Map.update(acc, wid, [name], &[name | &1]) + end + + deps + |> Enum.reject(&match?("grafter_" <> _, &1)) + |> Enum.reduce(%{}, grouper) + |> Enum.reduce(%{}, fn {dep_wid, names}, acc -> + filter = if names == ["*"], do: [], else: [names: names] + filter = if dep_wid != wid, do: Keyword.put(filter, :with_subs, true), else: filter + + recorded = + conf + |> Workflow.all_recorded(dep_wid, filter) + |> then(fn map -> if atomize?, do: atomize_map(map), else: map end) + + Map.merge(acc, recorded) + end) + end + + defp atomize_map(map) when is_map(map) and not is_struct(map) do + Map.new(map, fn {key, val} -> {atomize_key(key), val} end) + end + + defp atomize_map(val), do: val + + defp atomize_key(key) when is_binary(key) do + String.to_existing_atom(key) + rescue + ArgumentError -> key + end + + defp atomize_key(key), do: key +end diff --git a/vendor/oban_pro/lib/pro.ex b/vendor/oban_pro/lib/pro.ex new file mode 100644 index 00000000..7e0248e0 --- /dev/null +++ b/vendor/oban_pro/lib/pro.ex @@ -0,0 +1,3 @@ +defmodule Oban.Pro do + @moduledoc false +end diff --git a/vendor/oban_pro/mix.exs b/vendor/oban_pro/mix.exs new file mode 100644 index 00000000..a150006d --- /dev/null +++ b/vendor/oban_pro/mix.exs @@ -0,0 +1,196 @@ +defmodule Oban.Pro.MixProject do + use Mix.Project + + @version "1.6.11" + + def project do + [ + app: :oban_pro, + version: @version, + elixir: "~> 1.15", + start_permanent: Mix.env() == :prod, + elixirc_paths: elixirc_paths(Mix.env()), + prune_code_paths: false, + deps: deps(), + docs: docs(), + aliases: aliases(), + package: package(), + name: "Oban Pro", + description: "Oban Pro Component", + xref: [exclude: [Oban.JSON, Oban.Validation]], + + # Dialyzer + dialyzer: [ + plt_add_apps: [:ex_unit, :mix, :postgrex], + plt_core_path: "_build/#{Mix.env()}", + flags: [:error_handling, :missing_return, :no_opaque, :underspecs] + ] + ] + end + + def application do + [ + extra_applications: [:logger], + mod: {Oban.Pro.Application, []} + ] + end + + def cli do + [ + preferred_envs: [ + "test.ci": :test, + "test.reset": :test, + "test.rollback": :test, + "test.setup": :test + ] + ] + end + + def package do + [ + organization: "oban", + files: ~w(lib/pro.ex lib/oban .formatter.exs mix.exs), + licenses: ["Commercial"], + links: [] + ] + end + + defp elixirc_paths(:test), do: ["lib", "test/support"] + defp elixirc_paths(_env), do: ["lib"] + + defp deps do + [ + {:oban, "~> 2.19"}, + {:ecto_sql, "~> 3.10"}, + {:postgrex, "~> 0.16", optional: true}, + + # Test and Dev + {:stream_data, "~> 1.1", only: [:test, :dev]}, + {:tz, "~> 0.26", only: [:test, :dev]}, + {:benchee, "~> 1.0", only: [:test, :dev], runtime: false}, + {:credo, "~> 1.7.7-rc.0", only: [:test, :dev], runtime: false}, + {:dialyxir, "~> 1.0", only: [:test, :dev], runtime: false}, + + # Dev + {:ex_doc, "~> 0.21", only: :dev, runtime: false}, + {:makeup_diff, "~> 0.1", only: :dev, runtime: false}, + {:lys_publish, "~> 0.1", only: :dev, path: "../lys_publish", optional: true, runtime: false} + ] + end + + defp aliases do + [ + release: [ + "cmd git tag v#{@version}", + "cmd git push", + "cmd git push --tags", + "hex.publish package --yes", + "lys.publish" + ], + "test.reset": ["ecto.drop -r Oban.Pro.Repo --quiet", "test.setup"], + "test.rollback": ["ecto.rollback -r Oban.Pro.Repo --all --quiet"], + "test.setup": [ + "ecto.create -r Oban.Pro.Repo --quiet", + "ecto.migrate -r Oban.Pro.Repo --quiet" + ], + "test.ci": [ + "format --check-formatted", + "deps.unlock --check-unused", + "credo --strict", + "test --raise", + "dialyzer" + ] + ] + end + + defp docs do + [ + main: "overview", + source_ref: "v#{@version}", + formatters: ["html"], + api_reference: false, + extra_section: "GUIDES", + extras: extras(), + groups_for_extras: groups_for_extras(), + groups_for_modules: groups_for_modules(), + homepage_url: "/", + logo: "assets/oban-pro-logo.svg", + assets: %{"assets" => "assets"}, + nest_modules_by_prefix: nest_modules_by_prefix(), + skip_undefined_reference_warnings_on: ["CHANGELOG.md"], + before_closing_body_tag: fn _ -> + """ + + """ + end + ] + end + + defp extras do + [ + "guides/introduction/overview.md", + "guides/introduction/installation.md", + "guides/introduction/adoption.md", + "guides/introduction/composition.md", + "guides/testing/testing.md", + "guides/testing/testing_workers.md", + "guides/deployment/ci_cd.md", + "guides/deployment/docker.md", + "guides/deployment/fly.md", + "guides/deployment/gigalixir.md", + "guides/deployment/heroku.md", + "guides/deployment/dependabot.md", + "guides/upgrading/v1.0.md", + "guides/upgrading/v1.4.md", + "guides/upgrading/v1.5.md", + "guides/upgrading/v1.6.md", + "CHANGELOG.md": [filename: "changelog", title: "Changelog"] + ] + end + + defp groups_for_extras do + [ + Introduction: ~r/guides\/introduction\/.?/, + Testing: ~r/guides\/testing\/.?/, + Deployment: ~r/guides\/deployment\/.?/, + Upgrading: ~r/guides\/upgrading\/.?/ + ] + end + + defp groups_for_modules do + [ + Extensions: [ + Oban.Pro.Decorator, + Oban.Pro.Relay, + Oban.Pro.Testing, + Oban.Pro.Worker + ], + Composition: [ + Oban.Pro.Batch, + Oban.Pro.Workflow + ], + Plugins: [ + Oban.Pro.Plugins.DynamicCron, + Oban.Pro.Plugins.DynamicLifeline, + Oban.Pro.Plugins.DynamicPartitioner, + Oban.Pro.Plugins.DynamicPrioritizer, + Oban.Pro.Plugins.DynamicPruner, + Oban.Pro.Plugins.DynamicQueues, + Oban.Pro.Plugins.DynamicScaler + ], + Scaling: [ + Oban.Pro.Cloud + ], + Workers: [ + Oban.Pro.Workers.Batch, + Oban.Pro.Workers.Chain, + Oban.Pro.Workers.Chunk, + Oban.Pro.Workers.Workflow + ] + ] + end + + defp nest_modules_by_prefix do + [Oban.Pro, Oban.Pro.Plugins, Oban.Pro.Workers] + end +end diff --git a/vendor/oban_web/.formatter.exs b/vendor/oban_web/.formatter.exs new file mode 100644 index 00000000..48f82b0c --- /dev/null +++ b/vendor/oban_web/.formatter.exs @@ -0,0 +1,14 @@ +# Used by "mix format" +[ + import_deps: [ + :ecto, + :ecto_sql, + :phoenix + ], + inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs,heex}"], + export: [ + locals_without_parens: [oban_dashboard: 1, oban_dashboard: 2] + ], + locals_without_parens: [oban_dashboard: 1, oban_dashboard: 2], + plugins: [Phoenix.LiveView.HTMLFormatter] +] diff --git a/vendor/oban_web/.hex b/vendor/oban_web/.hex new file mode 100644 index 0000000000000000000000000000000000000000..29d0182002a68554e4c27baf208f5c123229b760 GIT binary patch literal 265 zcmZ9GOK!t33`E^HO|!~T0$a3xyWAuo8M1a;$$|4x_Pu!JZFhr*d4u^KH)C7cQC_&g z=H4{Tu52?VNj2_g!h9}5<>M*KZJP5+cjnmlM_;y!T;~Ot+Z)+ zeU-zMvn*#Gg}-l)wAN1;V+bw~fg!j64}*dU7zl-f=};{L4GsbQ0GANCcTv2hj?{}X vm&5$H{~y&(G*(}uhtOkZf@jyE2jju1dQS|tXCetkSX9ZP1FK>1^^A3Yit0?S literal 0 HcmV?d00001 diff --git a/vendor/oban_web/CHANGELOG.md b/vendor/oban_web/CHANGELOG.md new file mode 100644 index 00000000..4127634f --- /dev/null +++ b/vendor/oban_web/CHANGELOG.md @@ -0,0 +1,335 @@ +# Changelog for Oban Web v2.11 + +All notable changes to `Oban.Web` are documented here. + +## 🥂 Web is Free and Open Source + +Oban Web is now [fully open source][fos] and free (as in champagne 😉)! From now on, Oban Web will +be [published to Hex][hex] and available for use in your Oban powered applications. See the +updated, much slimmer, [installation guide][ins] to get started! + +Web v2.11 is licensed under Apache 2.0, just like Oban and Elixir itself. Previous versions are +commercially licensed, therefore private, and won't be published to Hex. + +Special thanks to all of our customers that supported the project for the past 5 years and made +open source Oban, and now Web, possible 💛. + +[fos]: https://github.com/oban-bg/oban_web +[hex]: https://hex.pm/packages/oban_web +[ins]: installation.md + +## 🐬🪶 MySQL and SQLite + +All official Oban and Oban Pro engines are fully supported. Listing, filtering, ordering, and +searching through jobs works for Postgres, MySQL, and SQLite. That includes the particularly +gnarly issue of dynamically generating and manipulating JSON for filter auto-suggestions! Nested +args queries, such as `args.address.city:Edinburgh` work equally well with each engine. + +## 🎛️ Instance Select + +The dashboard will now support switching between Oban instances. A new instance select menu in the +header allows switching between running Oban instances at runtime. There's no need to mount +additional dashboards in your application's router to handle multiple instances. The most recently +selected instance persists between mounts. + +```diff +- oban_dashboard "/oban_a", oban_name: Oban.A +- oban_dashboard "/oban_b", oban_name: Oban.B +- oban_dashboard "/oban_c", oban_name: Oban.C ++ oban_dashboard "/oban" +``` + +This also eliminates the need for additional router configuration, as the dashboard will select +the first running Oban instance it finds (with a preference for the default `Oban`). + +```diff +- oban_dashboard "/oban", oban_name: MyOban ++ oban_dashboard "/oban" +``` + +## ☯️ Unified Tables + +The queue and jobs tables are fully rebuilt with shared, reusable components and matching +functionality. This makes interacting with jobs clearer while queues gain some much requested +functionality: + +- Sidebar - a new queue sidebar shows status counts and enables filtering by statuses such as + `paused` or `terminating`. + +- Filtering - queues are auto-complete filterable just like jobs, making it possible to find + queues running on a particular node or narrow down by status. + +- Shared Sorting - queue sorting now behaves identically to jobs, through a shared dropdown. + +- Uniform Navigation - click on any part of the queue row to navigate to details. + +- Condensed Rows - simplify the queue page by removing nested row components. Extra queue details + are in the sub-queue page. + +## 🕯️ Operate on Full Selection + +Apply bulk actions to all selected jobs, not just those visible on the current page. + +This expands the select functionality to extend beyond the current page and include all filtered +jobs, up to a configurable limit. The limit defaults to 1000 and may be overridden with a [resolver +callback][rsc]. + +[rsc]: Oban.Web.Resolver.html#c:bulk_action_limit/1 + +## v2.11.7 - 2026-01-22 + +### Enhancements + +- [Router] Add configurable `:logo_path` option for dashboard + + Allow users to customize where the Oban logo links to by passing a `:logo_path` option to the + `oban_dashboard/1` router helper. When provided, the logo will link to the specified path + instead of the default jobs page. + +- [Query] Help query planner with value rather than subquery + + Since the condition uses a result of an inner query, database can't reliably estimate that it's + worth to use an index. It "thinks" that many rows will be returned in result and decides to use + a sequential scan. + + However, when a specific value is provided, database can use the collected statistics to come up + with a better plan and use the index. + + +### Bug Fixes + +- [Jobs] Escape regex match in search highlighting to prevent exceptions + + Unescaped values would crash the LiveView process rather than gracefully returning no matches. + +- [Dashboard] Refactor inline styles to include CSP nonces + + Avoid CSP warnings for inline styles used to control the sidebar. + +- [Dashboard] Respect custom variant for applying dark mode + + Additional configuration is required to manually apply dark mode after upgrading to Tailwind 4. + +- [Jobs] Correct detail background color for dark modes + +## v2.11.6 - 2025-10-24 + +### Bug Fixes + +- [Dashboard] Extract current prefix for correct asset routing + + The prefix was mistakenly hard coded to `/oban`, which broke the dashboard for any non-standard + paths. Now the value is extracted from the live view session. + +## v2.11.5 - 2025-10-23 + +### Enhancements + +- [Dashboard] Extract CSS and JS assets into dedicated plug module + + The changes move CSS/JS assets from layouts into a dedicated plug module, enabling cached and + immutable asset serving. + +- [Dashboard] Upgrade to Tailwind v4 with updated styles + +- [Dashboard] Add sidebar resizing with local persistence + + The sidebar can now be resized within a small range of sizes to compensate for long queue names + or larger screens. The adjusted size is persisted as a setting within local storage, preserving + it across reloads. + +- [Dashboard] Overhaul navigation for state retention between pages + + Page navigation uses patch rather than navigate to prevent re-mounting the dashboard. This was + essential to retain sidebar changes during page transitions. + +- [Telemetry] Add `queues` to metadata of `*_queues` telemetry events + + Telemetry events for `pause_queues`, `resume_queues`, and `stop_queues` now include the + selected queues. + +### Bug Fixes + +- [Router] Ensure the resolver module is loaded + + Optional resolver modules are now loaded before checking whether they export a function. This + ensures functions are correctly identified, as they aren't automatically loaded locally by + Elixir (unless eager module loading is turned on). + +- [Jobs] Display correct completed_at timestamp in details + + Completed jobs would show a relative time value that matched how long the job took rather than + when it completed. + +- [Dashboard] Use history navigation for detail back buttons + + Clicking the back arrow in job and queue detail pages removed any applied params, as it dropped + any non-id params on navigation. The new approach uses a history hook that leverages native + window history instead. + +## v2.11.4 - 2025-07-31 + +### Enhancements + +- [Connectivity] Consider metric checks for disconnected status. + + The connectivity status is now determined by `Met` output as well as pubsub connectivity. + This should make it easier to identify metric issues on solo nodes, e.g. in dev or a + staging environment. + +### Bug Fixes + +- [Dashboard] Read phoenix js assets at compile time. + + Stop bundling phoenix and liveview assets. Instead, read them at compile time and concatenate + with app js. + +- [Search] Trim strings when splitting to parse integers. + + This prevents "not a textual representation of an integer" errors when splitting on a comma + with an empty string. + +- [Search] Move all regexes out of module attributes. + + Regexes aren't allowed in module attributes as of OTP 28. This moves them inline rather than + hoisted at the top of the module. + +- [Sidebar] Fix column header mismatch in sidebar. + + The headers and values in the sidebar were misaligned and showed the wrong values. + +## v2.11.3 - 2025-04-21 + +### Bug Fixes + +- [Installer] Prevent compilation errors when igniter isn't installed. + +## v2.11.2 - 2025-04-21 + +### Enhancements + +- [Installer] Add igniter powered `oban_web` installer. + + It's now possible to install oban_web with a single igniter command: + + ```bash + mix igniter.install oban_web + ``` + + Or install `oban` and `oban_web` at the same time: + + ```bash + mix igniter.install oban,oban_web + ``` + +- [Resolver] Pattern match on `arg` rather than checking for `decorated` annotation. + + Matching on term encoded `arg` is more accurate than checking for decorated metadata. This makes + the default `format_job_args` compatible with workflow cascade jobs that don't have any `arg` + set. + +- [Page] Upgrade bundled assets to use the Phoenix LiveView JavaScript version to v1.10 + +- [Chart] Replace inline styles with tailwind classes to avoid inline style CSP warnings. + +## v2.11.1 — 2025-02-06 + +### Enhancements + +- [Layout] Display Oban.Met version in layout footer + + The Met version is highly relevant to how Web behaves. This also refactors the version display + for reuse with consistent conditionals. + +- [Layout] Only show Pro version number when available. + + The version footer shows that Pro isn't available rather than showing a `v` followed by a blank + space. + +- [Jobs] Auto-complete worker priorities from 0 to 9 + + Priority completion only matched values from 0..3, but the full range is 0..9. + +### Bug Fixes + +- [Jobs] Preserve quotes in `args` and `meta` searches. + + Parsing would strip quotes from args and meta queries. This prevented quoted numeric vaalues to + be treated as integers. Now quotes are preserved for `args` and `meta`, just as they are for + other qualifiers. + +- [Queue Details] Eliminate duplicate id warnings for inputs in queue details. + + The latest live_view alerts when nodes in a view have conflicting ids, which caught a number of + instances on the queue details page. + +- [Queues] Fix duplicate ids for nodes and queues in sidebar. + +- [Search] Correct assigns typo in search component handler. + + The key is `assigns`, not `asigns`. + +- [Resolver] Fix resolver access typespec. + + Access options must be a keyword list with boolean values, not just a list of option atoms. + +## v2.11.0 — 2025-01-16 + +### Enhancements + +- [Dashboard] Load using non-default Oban instance without any config. + + The dashboard now loads the first running non-default Oban instance, even without anything + configured. + +- [Jobs] Decode decorated `arg` when formatting job args. + + The decorated `arg` are term encoded and inscrutiable in the standard display. Now the value is + decoded and display as the original value. + +- [Jobs] Prevent decoding executable functions when displaying `recorded` output. + + By default, recorded content uses the `:safe` flag, which now prevents both atom creation and + executable content. This is an extra security precaution to prevent any possible exploits + through Web. + +- [Queues] Add sidebar similar to the jobs table, including filters to make bulk operations on + queues possible. + +- [Queues] Add search bar with filtering functionality to query queues. + +- [Queues] Add multi-select mode to allow operating on multiple queues at once. + +- [Queues] Expose a "Stop Queues" operation along with corresponding access controls to prevent + unintended shutdown. + +- [Resolver] Add `resolve_instances/1` callback to restrict selectable Oban instances. + + Not all instances should be available to all users. This adds the `resolve_instances/1` callback + to allow scoping instances by user: + + ```elixir + def resolve_instances(%{admin?: true}), do: :all + def resolve_instances(_user), do: [Oban] + ``` + +- [Resolver] Add `bulk_action_limit/1` to restrict the number of jobs operated on at once. + + Bulk operations now effect all filtered jobs, not just those visible on the current page. As + there may be millions of jobs that match a filter, some limiting factor is necessary. The + default of 1000 may be overridden: + + ```elixir + def bulk_action_limit(:completed), do: 10_000 + def bulk_action_limit(_state), do: :infinity + ``` + +### Bug Fixes + +- [Page] Address deprecations from recent upgrades to Elixir 1.17+, Phoenix 1.5+, and Phoenix + LiveView 1.0+ + +- [Jobs] Always treat args wrapped in quotes as a string when filtering. + + Manually wrapping values in quotes enforces searching by string, rather than as a parsed + integer. As values are cast to JSON the type matters, and `123 != "123"`. diff --git a/vendor/oban_web/LICENSE.txt b/vendor/oban_web/LICENSE.txt new file mode 100644 index 00000000..227bf3fc --- /dev/null +++ b/vendor/oban_web/LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2025 The Oban Team + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/oban_web/README.md b/vendor/oban_web/README.md new file mode 100644 index 00000000..71029477 --- /dev/null +++ b/vendor/oban_web/README.md @@ -0,0 +1,133 @@ +# Oban Web + +

+ + Hex Version + + + + Hex Docs + + + + CI Status + + + + Apache 2 License + +

+ + + +Oban Web is a view of [Oban's][oba] inner workings that you host directly within your application. +Powered by [Oban Metrics][met] and [Phoenix Live View][liv], it is distributed, lightweight, and +fully realtime. + +[oba]: https://github.com/oban-bg/oban +[met]: https://github.com/oban-bg/oban_met +[liv]: https://github.com/phoenixframework/phoenix_live_view + +

+ + + + + +

+ +## Features + +- **🐦‍🔥 Embedded LiveView** - Mount the dashboard directly in your application without any + external dependencies. + +- **📊 Realtime Charts** - Powered by a custom, distributed time-series data store that's compacted + for hours of efficient storage and filterable by node, queue, state, and worker. + +- **🛸 Live Updates** - Monitor background job activity across all queues and nodes in real + time, with customizable refresh rates and automatic pausing on blur. + +- **🔍 Powerful Filtering** - Intelligently filter jobs by worker, queue, args, tags and more with + auto-completed suggestions. + +- **🔬 Detailed Inspection** - View job details including when, where and how it was ran (or how + it failed to run). + +- **🔄 Batch Actions** - Cancel, delete and retry selected jobs or all jobs matching the current + filters. + +- **🎛️ Queue Controls** - Scale, pause, resume, and stop queues across all running nodes. Queues + running with [Oban Pro](https://oban.pro) can also edit global limits, rate limiting, and + partitioning. + +- **♊ Multiple Dashboards** - Switch between all running Oban instance from a single mount point, + or restrict access to some dashboards with exclusion controls. + +- **🔒 Access Control** - Allow admins to control queues and interract with jobs while restricting + other users to read-only use of the dashboard. + +- **🎬 Action Logging** - Use telemetry events to instrument and report all of a user's dashboard + activity. A telemetry-powered logger is provided for easy reporting. + +## Installation + +See the [installation guide](https://hexdocs.pm/oban_web/installation.html) for details on +installing and configuring Oban Web for your application. + + + +## Contributing + +To run the Oban Web test suite you must have PostgreSQL 12+ and MySQL 8+ running. Once dependencies +are installed, setup the databases and run necessary migrations: + +```bash +mix test.setup +``` + +### Development Server + +For development, a single file server that generates a wide variety of fake jobs is built in: + +```bash +iex -S mix dev +``` + +### Testing with Oban Pro + +Oban Pro is an optional dependency for development and testing. The test suite will automatically +skip Pro-dependent tests (tagged with `@tag :pro`) when Oban Pro is not available. + +To run tests with Pro features, you'll need access to a valid [Oban Pro](https://oban.pro) +license. First, authorize with the Oban repository: + +```bash +mix hex.repo add oban https://repo.oban.pro \ + --fetch-public-key SHA256:4/OSKi0NRF91QVVXlGAhb/BIMLnK8NHcx/EWs+aIWPc \ + --auth-key YOUR_AUTH_KEY +``` + +Then fetch dependencies and run the full test suite: + +```bash +mix deps.get +mix test +``` + +To explicitly run tests without Oban Pro: + +```bash +mix test --exclude pro +``` + +## Community + +There are a few places to connect and communicate with other Oban users: + +- Ask questions and discuss *#oban* on the [Elixir Forum][forum] +- [Request an invitation][invite] and join the *#oban* channel on Slack +- Learn about bug reports and upcoming features in the [issue tracker][issues] + +[invite]: https://elixir-slack.community/ +[forum]: https://elixirforum.com/ +[issues]: https://github.com/oban-bg/oban_web/issues diff --git a/vendor/oban_web/hex_metadata.config b/vendor/oban_web/hex_metadata.config new file mode 100644 index 00000000..66026a7b --- /dev/null +++ b/vendor/oban_web/hex_metadata.config @@ -0,0 +1,90 @@ +{<<"links">>, + [{<<"Website">>,<<"https://oban.pro">>}, + {<<"Changelog">>, + <<"https://github.com/oban-bg/oban_web/blob/main/CHANGELOG.md">>}, + {<<"GitHub">>,<<"https://github.com/oban-bg/oban_web">>}]}. +{<<"name">>,<<"oban_web">>}. +{<<"version">>,<<"2.11.7">>}. +{<<"description">>,<<"Dashboard for the Oban job processing framework">>}. +{<<"elixir">>,<<"~> 1.15">>}. +{<<"files">>, + [<<"lib">>,<<"lib/mix">>,<<"lib/mix/tasks">>, + <<"lib/mix/tasks/oban_web.install.ex">>,<<"lib/oban">>,<<"lib/oban/web">>, + <<"lib/oban/web/dashboard_live.ex">>,<<"lib/oban/web/page.ex">>, + <<"lib/oban/web/telemetry.ex">>,<<"lib/oban/web/cache.ex">>, + <<"lib/oban/web/plugins">>,<<"lib/oban/web/plugins/stats.ex">>, + <<"lib/oban/web/queue.ex">>,<<"lib/oban/web/queries">>, + <<"lib/oban/web/queries/job_query.ex">>, + <<"lib/oban/web/queries/queue_query.ex">>,<<"lib/oban/web/router.ex">>, + <<"lib/oban/web/components">>,<<"lib/oban/web/components/sort.ex">>, + <<"lib/oban/web/components/icons.ex">>, + <<"lib/oban/web/components/sidebar_components.ex">>, + <<"lib/oban/web/components/core.ex">>, + <<"lib/oban/web/components/layouts.ex">>, + <<"lib/oban/web/components/layouts">>, + <<"lib/oban/web/components/layouts/live.html.heex">>, + <<"lib/oban/web/components/layouts/root.html.heex">>, + <<"lib/oban/web/live">>,<<"lib/oban/web/live/search_component.ex">>, + <<"lib/oban/web/live/shortcuts_component.ex">>, + <<"lib/oban/web/live/theme_component.ex">>,<<"lib/oban/web/live/queues">>, + <<"lib/oban/web/live/queues/table_component.ex">>, + <<"lib/oban/web/live/queues/sidebar_component.ex">>, + <<"lib/oban/web/live/queues/detail_component.ex">>, + <<"lib/oban/web/live/queues/detail_instance_component.ex">>, + <<"lib/oban/web/live/jobs">>, + <<"lib/oban/web/live/jobs/table_component.ex">>, + <<"lib/oban/web/live/jobs/sidebar_component.ex">>, + <<"lib/oban/web/live/jobs/detail_component.ex">>, + <<"lib/oban/web/live/jobs/timeline_component.ex">>, + <<"lib/oban/web/live/jobs/chart_component.ex">>, + <<"lib/oban/web/live/connectivity_component.ex">>, + <<"lib/oban/web/live/instances_component.ex">>, + <<"lib/oban/web/live/refresh_component.ex">>,<<"lib/oban/web/helpers.ex">>, + <<"lib/oban/web/application.ex">>,<<"lib/oban/web/timing.ex">>, + <<"lib/oban/web/pages">>,<<"lib/oban/web/pages/jobs_page.ex">>, + <<"lib/oban/web/pages/queues_page.ex">>,<<"lib/oban/web/resolver.ex">>, + <<"lib/oban/web/authentication.ex">>,<<"lib/oban/web/helpers">>, + <<"lib/oban/web/helpers/queue_helper.ex">>,<<"lib/oban/web/assets.ex">>, + <<"lib/oban/web/search.ex">>,<<"lib/oban/web/exceptions.ex">>, + <<"lib/oban/web.ex">>,<<"priv/static">>,<<"priv/static/app.css">>, + <<"priv/static/app.js">>,<<".formatter.exs">>,<<"mix.exs">>,<<"README.md">>, + <<"CHANGELOG.md">>,<<"LICENSE.txt">>]}. +{<<"app">>,<<"oban_web">>}. +{<<"licenses">>,[<<"Apache-2.0">>]}. +{<<"requirements">>, + [[{<<"name">>,<<"jason">>}, + {<<"app">>,<<"jason">>}, + {<<"optional">>,false}, + {<<"requirement">>,<<"~> 1.2">>}, + {<<"repository">>,<<"hexpm">>}], + [{<<"name">>,<<"phoenix">>}, + {<<"app">>,<<"phoenix">>}, + {<<"optional">>,false}, + {<<"requirement">>,<<"~> 1.7">>}, + {<<"repository">>,<<"hexpm">>}], + [{<<"name">>,<<"phoenix_html">>}, + {<<"app">>,<<"phoenix_html">>}, + {<<"optional">>,false}, + {<<"requirement">>,<<"~> 3.3 or ~> 4.0">>}, + {<<"repository">>,<<"hexpm">>}], + [{<<"name">>,<<"phoenix_pubsub">>}, + {<<"app">>,<<"phoenix_pubsub">>}, + {<<"optional">>,false}, + {<<"requirement">>,<<"~> 2.1">>}, + {<<"repository">>,<<"hexpm">>}], + [{<<"name">>,<<"phoenix_live_view">>}, + {<<"app">>,<<"phoenix_live_view">>}, + {<<"optional">>,false}, + {<<"requirement">>,<<"~> 1.0">>}, + {<<"repository">>,<<"hexpm">>}], + [{<<"name">>,<<"oban">>}, + {<<"app">>,<<"oban">>}, + {<<"optional">>,false}, + {<<"requirement">>,<<"~> 2.19">>}, + {<<"repository">>,<<"hexpm">>}], + [{<<"name">>,<<"oban_met">>}, + {<<"app">>,<<"oban_met">>}, + {<<"optional">>,false}, + {<<"requirement">>,<<"~> 1.0">>}, + {<<"repository">>,<<"hexpm">>}]]}. +{<<"build_tools">>,[<<"mix">>]}. diff --git a/vendor/oban_web/lib/mix/tasks/oban_web.install.ex b/vendor/oban_web/lib/mix/tasks/oban_web.install.ex new file mode 100644 index 00000000..1e675a9f --- /dev/null +++ b/vendor/oban_web/lib/mix/tasks/oban_web.install.ex @@ -0,0 +1,134 @@ +defmodule Mix.Tasks.ObanWeb.Install.Docs do + @moduledoc false + + def short_doc do + "Installs Oban Web into your Phoenix application" + end + + def example do + "mix oban_web.install" + end + + def long_doc do + """ + #{short_doc()} + + This task configures your Phoenix application to use the Oban Web dashboard: + + * Adds the required `Oban.Web.Router` import + * Sets up the dashboard route at "/oban" within the :dev_routes conditional + + ## Example + + ```bash + #{example()} + ``` + """ + end +end + +if Code.ensure_loaded?(Igniter) do + defmodule Mix.Tasks.ObanWeb.Install do + @shortdoc "#{__MODULE__.Docs.short_doc()}" + + @moduledoc __MODULE__.Docs.long_doc() + use Igniter.Mix.Task + + @impl Igniter.Mix.Task + def info(_argv, _composing_task) do + %Igniter.Mix.Task.Info{ + group: :oban, + installs: [{:oban, "~> 2.0"}], + example: __MODULE__.Docs.example() + } + end + + @impl Igniter.Mix.Task + def igniter(igniter) do + case Igniter.Libs.Phoenix.select_router(igniter) do + {igniter, nil} -> + Igniter.add_warning(igniter, """ + No Phoenix router found, Phoenix Liveview is needed for Oban Web + """) + + {igniter, router} -> + update_router(igniter, router) + end + end + + defp update_router(igniter, router) do + zipper = &do_update_router(igniter, &1) + + case Igniter.Project.Module.find_and_update_module(igniter, router, zipper) do + {:ok, igniter} -> + igniter + + {:error, igniter} -> + Igniter.add_warning(igniter, """ + Something went wrong, please check the Oban Web install docs for manual setup instructions + """) + end + end + + defp do_update_router(igniter, zipper) do + web_module = Igniter.Libs.Phoenix.web_module(igniter) + app_name = Igniter.Project.Application.app_name(igniter) + + with {:ok, zipper} <- add_import(zipper, web_module) do + add_route(zipper, app_name) + end + end + + defp add_import(zipper, web_module) do + with {:ok, zipper} <- Igniter.Code.Module.move_to_use(zipper, web_module) do + {:ok, Igniter.Code.Common.add_code(zipper, "\nimport Oban.Web.Router")} + end + end + + defp add_route(zipper, app_name) do + matcher = &dev_routes?(&1, app_name) + + with {:ok, zipper} <- Igniter.Code.Function.move_to_function_call(zipper, :if, 2, matcher), + {:ok, zipper} <- Igniter.Code.Common.move_to_do_block(zipper) do + {:ok, + Igniter.Code.Common.add_code(zipper, """ + scope "/" do + pipe_through :browser + + oban_dashboard "/oban" + end + """)} + end + end + + defp dev_routes?(zipper, app_name) do + case Igniter.Code.Function.move_to_nth_argument(zipper, 0) do + {:ok, zipper} -> + Igniter.Code.Function.function_call?(zipper, {Application, :compile_env}, 2) and + Igniter.Code.Function.argument_equals?(zipper, 0, app_name) and + Igniter.Code.Function.argument_equals?(zipper, 1, :dev_routes) + + _ -> + false + end + end + end +else + defmodule Mix.Tasks.ObanWeb.Install do + @shortdoc "#{__MODULE__.Docs.short_doc()} | Install `igniter` to use" + + @moduledoc __MODULE__.Docs.long_doc() + + use Mix.Task + + def run(_argv) do + Mix.shell().error(""" + The task 'ObanWeb.Task.Install' requires igniter. Please install igniter and try again. + + For more information, see: https://hexdocs.pm/igniter/readme.html#installation + """) + + exit({:shutdown, 1}) + end + end +end diff --git a/vendor/oban_web/lib/oban/web.ex b/vendor/oban_web/lib/oban/web.ex new file mode 100644 index 00000000..7708aca3 --- /dev/null +++ b/vendor/oban_web/lib/oban/web.ex @@ -0,0 +1,53 @@ +defmodule Oban.Web do + @moduledoc false + + alias Oban.Web.Layouts + + def html do + quote do + @moduledoc false + + import Phoenix.Controller, only: [get_csrf_token: 0, view_module: 1, view_template: 1] + + unquote(html_helpers()) + end + end + + def live_view do + quote do + @moduledoc false + + use Phoenix.LiveView, layout: {Layouts, :live} + + unquote(html_helpers()) + end + end + + def live_component do + quote do + @moduledoc false + + use Phoenix.LiveComponent + + unquote(html_helpers()) + end + end + + defp html_helpers do + quote do + use Phoenix.Component + + import Oban.Web.Helpers + import Phoenix.HTML + import Phoenix.LiveView.Helpers + + alias Oban.Web.Components.{Core, Icons} + alias Phoenix.LiveView.JS + end + end + + @doc false + defmacro __using__(which) when is_atom(which) do + apply(__MODULE__, which, []) + end +end diff --git a/vendor/oban_web/lib/oban/web/application.ex b/vendor/oban_web/lib/oban/web/application.ex new file mode 100644 index 00000000..a947e5be --- /dev/null +++ b/vendor/oban_web/lib/oban/web/application.ex @@ -0,0 +1,12 @@ +defmodule Oban.Web.Application do + @moduledoc false + + use Application + + @impl Application + def start(_type, _args) do + children = [Oban.Web.Cache] + + Supervisor.start_link(children, strategy: :one_for_one, name: __MODULE__) + end +end diff --git a/vendor/oban_web/lib/oban/web/assets.ex b/vendor/oban_web/lib/oban/web/assets.ex new file mode 100644 index 00000000..2c0dbbf8 --- /dev/null +++ b/vendor/oban_web/lib/oban/web/assets.ex @@ -0,0 +1,50 @@ +defmodule Oban.Web.Assets do + @moduledoc false + + @behaviour Plug + + import Plug.Conn + + phoenix_js_paths = + for app <- ~w(phoenix phoenix_html phoenix_live_view)a do + path = Application.app_dir(app, ["priv", "static", "#{app}.js"]) + Module.put_attribute(__MODULE__, :external_resource, path) + path + end + + @static_path Application.app_dir(:oban_web, ["priv", "static"]) + + @external_resource css_path = Path.join(@static_path, "app.css") + @external_resource js_path = Path.join(@static_path, "app.js") + + @css File.read!(css_path) + + @js """ + #{for path <- phoenix_js_paths, do: path |> File.read!() |> String.replace("//# sourceMappingURL=", "// ")} + #{File.read!(js_path)} + """ + + @impl Plug + def init(asset), do: asset + + @impl Plug + def call(conn, asset) do + {contents, content_type} = contents_and_type(asset) + + conn + |> put_resp_header("content-type", content_type) + |> put_resp_header("cache-control", "public, max-age=31536000, immutable") + |> put_private(:plug_skip_csrf_protection, true) + |> send_resp(200, contents) + |> halt() + end + + defp contents_and_type(:css), do: {@css, "text/css"} + defp contents_and_type(:js), do: {@js, "text/javascript"} + + for {key, val} <- [css: @css, js: @js] do + md5 = Base.encode16(:crypto.hash(:md5, val), case: :lower) + + def current_hash(unquote(key)), do: unquote(md5) + end +end diff --git a/vendor/oban_web/lib/oban/web/authentication.ex b/vendor/oban_web/lib/oban/web/authentication.ex new file mode 100644 index 00000000..6a361290 --- /dev/null +++ b/vendor/oban_web/lib/oban/web/authentication.ex @@ -0,0 +1,30 @@ +defmodule Oban.Web.Authentication do + @moduledoc false + + import Phoenix.Component + import Phoenix.LiveView + + alias Oban.Web.{Resolver, Telemetry} + + def on_mount(:default, params, session, socket) do + %{"oban" => oban, "resolver" => resolver, "user" => user} = session + + conf = if Oban.whereis(oban), do: Oban.config(oban), else: nil + socket = assign(socket, conf: conf, user: user) + + Telemetry.action(:mount, socket, [params: params], fn -> + case Resolver.call_with_fallback(resolver, :resolve_access, [user]) do + {:forbidden, path} -> + socket = + socket + |> put_flash(:error, "Access forbidden") + |> redirect(to: path) + + {:halt, socket} + + _ -> + {:cont, socket} + end + end) + end +end diff --git a/vendor/oban_web/lib/oban/web/cache.ex b/vendor/oban_web/lib/oban/web/cache.ex new file mode 100644 index 00000000..e46d0cbc --- /dev/null +++ b/vendor/oban_web/lib/oban/web/cache.ex @@ -0,0 +1,53 @@ +defmodule Oban.Web.Cache do + @moduledoc false + + use GenServer + + defstruct [:name, purge_interval: :timer.seconds(300)] + + def start_link(opts) do + opts = Keyword.put_new(opts, :name, __MODULE__) + + GenServer.start_link(__MODULE__, opts, name: opts[:name]) + end + + def fetch(name \\ __MODULE__, key, fun) do + if cache_enabled?() do + case :ets.lookup(name, key) do + [{^key, val}] -> + val + + [] -> + tap(fun.(), &:ets.insert(name, {key, &1})) + end + else + fun.() + end + end + + @impl GenServer + def init(opts) do + state = struct!(__MODULE__, opts) + + :ets.new(state.name, [:set, :public, :compressed, :named_table]) + + {:ok, schedule_purge(state)} + end + + @impl GenServer + def handle_info(:purge, state) do + :ets.delete_all_objects(state.name) + + {:noreply, schedule_purge(state)} + end + + defp cache_enabled? do + Process.get(:cache_enabled, Application.get_env(:oban_web, :cache)) + end + + defp schedule_purge(state) do + Process.send_after(self(), :purge, state.purge_interval) + + state + end +end diff --git a/vendor/oban_web/lib/oban/web/components/core.ex b/vendor/oban_web/lib/oban/web/components/core.ex new file mode 100644 index 00000000..109a2e94 --- /dev/null +++ b/vendor/oban_web/lib/oban/web/components/core.ex @@ -0,0 +1,244 @@ +defmodule Oban.Web.Components.Core do + use Oban.Web, :html + + attr :click, :string, required: true + attr :danger, :boolean, default: false + attr :disabled, :boolean, default: false + attr :label, :string, required: true + attr :target, :any + slot :icon + slot :title + + def action_button(assigns) do + class = + cond do + assigns.disabled -> + "text-gray-400" + + assigns.danger -> + "text-red-500 group-hover:text-red-600 hover:bg-gray-100 dark:hover:bg-gray-950" + + true -> + "text-gray-500 group-hover:text-gray-600 hover:bg-gray-100 dark:hover:bg-gray-950" + end + + assigns = assign(assigns, :class, class) + + ~H""" + + """ + end + + @doc """ + A numerical input with increment and decrement buttons. + """ + def number_input(assigns) do + ~H""" +
+ <%= if @label do %> + + <% end %> + +
+ + +
+ + + +
+
+
+ """ + end + + slot :inner_block, required: true + attr :name, :string, required: true + attr :options, :list, required: true + attr :selected, :any, required: true + attr :title, :string, required: true + attr :disabled, :boolean, default: false + attr :target, :any, default: "myself" + + def dropdown_button(assigns) do + ~H""" +
+ + + +
+ """ + end + + attr :name, :any, required: true + attr :value, :any, required: true + attr :selected, :any, required: true + attr :target, :any + + defp dropdown_option(assigns) do + class = + if assigns.selected == assigns.value do + "text-blue-500 dark:text-blue-400" + else + "text-gray-500 dark:text-gray-400 " + end + + assigns = assign(assigns, :class, class) + + ~H""" +
  • + <%= if to_string(@value) == to_string(@selected) do %> + + <% else %> + + <% end %> + + + {@value |> to_string() |> String.replace("_", " ")} + +
    • + """ + end + + attr :checked, :boolean, default: false + attr :click, :string, required: true + attr :myself, :any, required: true + attr :value, :string, required: true + + def row_checkbox(assigns) do + style = + if assigns.checked do + "border-blue-500 bg-blue-500" + else + "border-gray-400 dark:border-gray-600 group-hover:bg-gray-400 dark:group-hover:bg-gray-600" + end + + assigns = assign(assigns, :style, style) + + ~H""" + + """ + end + + attr :checked, :atom, required: true + attr :click, :string, required: true + attr :myself, :any, required: true + + def all_checkbox(assigns) do + style = + if assigns.checked in [:all, :some] do + "border-blue-500 bg-blue-500" + else + "border-gray-400 dark:border-gray-600 group-hover:bg-gray-400 dark:group-hover:bg-gray-600" + end + + assigns = assign(assigns, :style, style) + + ~H""" + + """ + end +end diff --git a/vendor/oban_web/lib/oban/web/components/icons.ex b/vendor/oban_web/lib/oban/web/components/icons.ex new file mode 100644 index 00000000..24d04234 --- /dev/null +++ b/vendor/oban_web/lib/oban/web/components/icons.ex @@ -0,0 +1,877 @@ +defmodule Oban.Web.Components.Icons do + use Oban.Web, :html + + # Helpers + + attr :rest, :global, + default: %{ + "aria-hidden": "true", + class: "w-4 h-4", + fill: "currentColor", + viewBox: "0 0 16 16" + } + + slot :inner_block, required: true + + defp svg_mini(assigns) do + ~H""" + + {render_slot(@inner_block)} + + """ + end + + attr :rest, :global, + default: %{ + "stroke-width": "1.5", + class: "w-6 h-6", + fill: "none", + stroke: "currentColor", + viewBox: "0 0 24 24" + } + + slot :inner_block, required: true + + defp svg_outline(assigns) do + ~H""" + + {render_slot(@inner_block)} + + """ + end + + attr :rest, :global, + default: %{ + "aria-hidden": "true", + class: "w-6 h-6", + fill: "currentColor", + viewBox: "0 0 24 24" + } + + slot :inner_block, required: true + + defp svg_solid(assigns) do + ~H""" + + {render_slot(@inner_block)} + + """ + end + + # Icons + + attr :rest, :global + + def adjustments_horizontal(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def adjustments_vertical(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def arrow_left(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def arrow_path(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def arrow_path_rounded(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def arrow_right_circle(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def arrow_top_right_on_square(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def arrow_trending_down(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def calendar_days(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def crossbones_circle(assigns) do + ~H""" + <.svg_solid {@rest}> + + + """ + end + + attr :rest, :global + + def bars_arrow_down(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def bars_arrow_up(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def bolt_circle(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def bolt_slash(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def camera(assigns) do + ~H""" + <.svg_outline {@rest}> + + + + """ + end + + attr :rest, :global + + def chart_bar_square(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def check(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def indeterminate(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def check_circle(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def chevron_down(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def chevron_right(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def chevron_up(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def clock(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def cog(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def command_line(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def computer_desktop(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def exclamation_circle(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def globe(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def info_circle(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def hashtag(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def life_buoy(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def lock_closed(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def map_pin(assigns) do + ~H""" + <.svg_outline {@rest}> + + + + """ + end + + attr :rest, :global + + def magnifying_glass(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def minus_circle(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def moon(assigns) do + ~H""" + <.svg_outline {@rest}> + + + + """ + end + + attr :rest, :global + + def no_symbol(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def pause_circle(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def pause_circle_solid(assigns) do + ~H""" + <.svg_solid {@rest}> + + + """ + end + + attr :rest, :global + + def percent_square(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def play_circle(assigns) do + ~H""" + <.svg_outline {@rest}> + + + + """ + end + + attr :rest, :global + + def play_circle_solid(assigns) do + ~H""" + <.svg_solid {@rest}> + + + """ + end + + attr :rest, :global + + def play_pause_circle(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def plus_circle(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def power(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def soren_logo(assigns) do + ~H""" + + + + + + """ + end + + attr :rest, :global + + def square_stack(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def square_2x2(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def rectangle_group(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def spinner(assigns) do + ~H""" + + + + + """ + end + + attr :rest, :global + + def table_cells(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def sparkles(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def sun(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def trash(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def user_circle(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def x_circle(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end + + attr :rest, :global + + def x_mark(assigns) do + ~H""" + <.svg_outline {@rest}> + + + """ + end +end diff --git a/vendor/oban_web/lib/oban/web/components/layouts.ex b/vendor/oban_web/lib/oban/web/components/layouts.ex new file mode 100644 index 00000000..29f8bbe7 --- /dev/null +++ b/vendor/oban_web/lib/oban/web/components/layouts.ex @@ -0,0 +1,210 @@ +defmodule Oban.Web.Layouts do + use Oban.Web, :html + + embed_templates "layouts/*" + + defp asset_path(conn, asset) when asset in [:css, :js] do + hash = Oban.Web.Assets.current_hash(asset) + + {_dash, _routing, meta} = conn.private.phoenix_live_view + + prefix = get_in(meta, [:extra, :session, Access.elem(2), Access.at(0)]) + + Phoenix.VerifiedRoutes.unverified_path( + conn, + conn.private.phoenix_router, + "#{prefix}/#{asset}-#{hash}" + ) + end + + attr :path, :string, default: nil + + def logo(assigns) do + ~H""" + + + + + + + + + """ + end + + def notify(assigns) do + ~H""" +
    +
    +
    +
    +
    +
    + +
    +
    +

    + {Phoenix.Flash.get(assigns.flash, :info)} +

    +
    +
    + +
    +
    +
    +
    +
    +
    + """ + end + + def nav(assigns) do + ~H""" + + """ + end + + def footer(assigns) do + assigns = + assign(assigns, + oss_version: Application.spec(:oban, :vsn), + met_version: Application.spec(:oban_met, :vsn), + web_version: Application.spec(:oban_web, :vsn), + pro_version: Application.spec(:oban_pro, :vsn) + ) + + ~H""" +
    + <.version name="Oban" version={@oss_version} /> + <.version name="Oban.Web" version={@web_version} /> + <.version name="Oban.Met" version={@met_version} /> + <.version name="Oban.Pro" version={@pro_version} /> + + + + + Made by Oban Pro + + +
    + """ + end + + attr :name, :string + attr :version, :string + + defp version(assigns) do + ~H""" + + {@name} {if @version, do: "v#{@version}", else: "–"} + + """ + end + + defp link_class(curr, page) do + base = "font-semibold text-sm rounded-md px-3 py-2" + + if curr == page do + base <> " text-gray-950 dark:text-gray-100 bg-gray-50 dark:bg-gray-800" + else + base <> " text-gray-600 dark:text-gray-300 hover:text-gray-900 dark:hover:text-gray-100" + end + end +end diff --git a/vendor/oban_web/lib/oban/web/components/layouts/live.html.heex b/vendor/oban_web/lib/oban/web/components/layouts/live.html.heex new file mode 100644 index 00000000..c030e93b --- /dev/null +++ b/vendor/oban_web/lib/oban/web/components/layouts/live.html.heex @@ -0,0 +1,31 @@ + + + +
    + <%= if Phoenix.Flash.get(@flash, :info) do %> + <.notify flash={@flash} /> + <% end %> + +
    +
    + <.logo path={@logo_path} /> +
    + + <.nav socket={@socket} page={@page.name} /> + +
    + <.live_component module={Oban.Web.ShortcutsComponent} id="shortcuts" /> + <.live_component module={Oban.Web.ConnectivityComponent} id="connectivity" conf={@conf} /> + <.live_component module={Oban.Web.ThemeComponent} id="theme" theme={@theme} /> + <.live_component module={Oban.Web.RefreshComponent} id="refresh" refresh={@refresh} /> + <.live_component + module={Oban.Web.InstancesComponent} + id="instances" + conf={@conf} + resolver={@resolver} + user={@user} + /> +
    +
    + {@inner_content} <.footer /> +
    diff --git a/vendor/oban_web/lib/oban/web/components/layouts/root.html.heex b/vendor/oban_web/lib/oban/web/components/layouts/root.html.heex new file mode 100644 index 00000000..082e62f4 --- /dev/null +++ b/vendor/oban_web/lib/oban/web/components/layouts/root.html.heex @@ -0,0 +1,42 @@ + + + + + + + + + {assigns[:page_title] || "Jobs • Oban"} + + + + + + + + + + + + {@inner_content} + + diff --git a/vendor/oban_web/lib/oban/web/components/sidebar_components.ex b/vendor/oban_web/lib/oban/web/components/sidebar_components.ex new file mode 100644 index 00000000..808d8ab8 --- /dev/null +++ b/vendor/oban_web/lib/oban/web/components/sidebar_components.ex @@ -0,0 +1,127 @@ +defmodule Oban.Web.SidebarComponents do + use Oban.Web, :html + + attr :csp_nonces, :map + attr :width, :integer, default: 320 + slot :inner_block + + def sidebar(assigns) do + ~H""" + + + """ + end + + attr :name, :string, required: true + attr :headers, :list, required: true + slot :inner_block + + def section(assigns) do + ~H""" +
    +
    + + +

    {String.capitalize(@name)}

    + +
    + {header} +
    +
    + +
    {render_slot(@inner_block)}
    +
    + """ + end + + attr :name, :string, required: true + attr :values, :list, required: true + attr :patch, :any, required: true + attr :active, :boolean, default: false + attr :exclusive, :boolean, default: false + slot :statuses + + def filter_row(assigns) do + class = + cond do + assigns.exclusive and assigns.active -> + "rounded-md bg-white hover:bg-white dark:bg-gray-800 dark:hover:bg-gray-800" + + assigns.exclusive -> + "rounded-md hover:bg-gray-100 dark:hover:bg-gray-800" + + assigns.active -> + "border-violet-500 hover:border-violet-500 focus:border-violet-500" + + true -> + "hover:border-violet-400 focus:border-violet-200" + end + + assigns = assign(assigns, :class, class) + + ~H""" + <.link + class={["flex justify-between pr-2 py-2 ml-2 my-0.5 border-l-4 border-transparent", @class]} + id={"filter-#{@name}"} + patch={@patch} + replace={true} + > + + {String.downcase(@name)} + + +
    +
    + {render_slot(@statuses)} +
    + + + {integer_to_estimate(value)} + +
    + + """ + end + + defp toggle(prefix) do + %JS{} + |> JS.toggle(in: "fade-in-scale", out: "fade-out-scale", to: "##{prefix}-rows") + |> JS.add_class("rotate-90", to: "##{prefix}-toggle svg:not(.rotate-90)") + |> JS.remove_class("rotate-90", to: "##{prefix}-toggle svg.rotate-90") + end +end diff --git a/vendor/oban_web/lib/oban/web/components/sort.ex b/vendor/oban_web/lib/oban/web/components/sort.ex new file mode 100644 index 00000000..7c87cd5f --- /dev/null +++ b/vendor/oban_web/lib/oban/web/components/sort.ex @@ -0,0 +1,84 @@ +defmodule Oban.Web.SortComponent do + use Oban.Web, :html + + attr :id, :string, default: "job-sort" + attr :by, :list, required: true + attr :page, :atom, default: :jobs + attr :params, :map, required: true + + def select(assigns) do + ~H""" +
    + + + +
    + """ + end + + attr :link, :any, required: true + attr :selected, :string, required: true + attr :value, :string, required: true + + defp option(assigns) do + ~H""" + <.link + class="block w-full flex items-center py-1 px-2 cursor-pointer select-none space-x-2 hover:bg-gray-50 hover:dark:bg-gray-600/30" + id={"sort-#{@value}"} + patch={@link} + phx-click-away={JS.hide(to: "#sort-menu")} + phx-click={JS.hide(to: "#sort-menu")} + role="option" + > + <%= if @value == @selected do %> + + <% else %> + + <% end %> + + {String.replace(@value, "_", " ")} + + + """ + end +end diff --git a/vendor/oban_web/lib/oban/web/dashboard_live.ex b/vendor/oban_web/lib/oban/web/dashboard_live.ex new file mode 100644 index 00000000..7c6cdf9b --- /dev/null +++ b/vendor/oban_web/lib/oban/web/dashboard_live.ex @@ -0,0 +1,266 @@ +defmodule Oban.Web.DashboardLive do + use Oban.Web, :live_view + + alias Oban.Web.{JobsPage, QueuesPage} + + @impl Phoenix.LiveView + def mount(params, session, socket) do + %{"prefix" => prefix, "resolver" => resolver} = session + %{"live_path" => live_path, "live_transport" => live_transport} = session + %{"user" => user, "access" => access, "csp_nonces" => csp_nonces} = session + %{"refresh" => refresh, "logo_path" => logo_path} = session + + refresh = restore_state(socket, "refresh", refresh) + theme = restore_state(socket, "theme", "system") + sidebar_width = restore_state(socket, "sidebar_width", 320) + oban = current_oban_instance(session, socket) + + conf = await_init([oban], :supervisor) + _met = await_init([oban, Oban.Met], :met) + page = resolve_page(params) + + Process.put(:routing, {socket, prefix}) + + socket = + socket + |> assign(conf: conf, params: params, page: page, init_state: init_state(socket)) + |> assign(live_path: live_path, live_transport: live_transport, logo_path: logo_path) + |> assign(access: access, csp_nonces: csp_nonces, resolver: resolver, user: user) + |> assign(original_refresh: nil, refresh: refresh, timer: nil, theme: theme) + |> assign(sidebar_width: sidebar_width) + |> init_schedule_refresh() + |> page.comp.handle_mount() + + {:ok, socket} + end + + defp current_oban_instance(session, socket) do + stashed = restore_state(socket, "instance") + default = List.first(oban_instances()) + + case stashed || session["oban"] || default || Oban do + name when is_binary(name) -> name |> String.split(".") |> Module.safe_concat() + name -> name + end + end + + defp init_state(socket) do + case get_connect_params(socket) do + %{"init_state" => state} -> state + _ -> %{} + end + end + + defp restore_state(socket, key, default \\ nil) do + socket + |> init_state() + |> Map.get("oban:" <> key, default) + end + + @impl Phoenix.LiveView + def render(assigns) do + assigns = + assigns + |> Map.put(:id, "page") + |> Map.drop(~w(flash live_path live_transport refresh socket timer)a) + + ~H""" + <.live_component id="page" module={@page.comp} {assigns} /> + """ + end + + @impl Phoenix.LiveView + def terminate(_reason, %{assigns: %{timer: timer}}) do + if is_reference(timer), do: Process.cancel_timer(timer) + + :ok + end + + def terminate(_reason, _socket), do: :ok + + @impl Phoenix.LiveView + def handle_params(params, uri, socket) do + page = resolve_page(params) + + if page == socket.assigns.page do + page.comp.handle_params(params, uri, socket) + else + socket = + socket + |> assign(params: params, page: page) + |> page.comp.handle_mount() + + page.comp.handle_params(params, uri, socket) + end + end + + @impl Phoenix.LiveView + def handle_info(:clear_flash, socket) do + {:noreply, clear_flash(socket)} + end + + def handle_info(:pause_refresh, socket) do + socket = + if socket.assigns.refresh > 0 do + cancel_timer(socket) + + assign(socket, refresh: -1, original_refresh: socket.assigns.refresh, timer: nil) + else + socket + end + + {:noreply, socket} + end + + def handle_info(:resume_refresh, socket) do + if original = socket.assigns.original_refresh do + handle_info({:update_refresh, original}, socket) + else + {:noreply, socket} + end + end + + def handle_info({:select_instance, name}, socket) do + name = name |> String.split(".") |> Module.safe_concat() + conf = Oban.config(name) + name = inspect(name) + page = socket.assigns.page + + socket = + socket + |> assign(oban: name, conf: conf) + |> push_event("select-instance", %{name: name}) + |> put_flash_with_clear(:info, "Switched instance to #{name}") + |> page.comp.handle_refresh() + + {:noreply, socket} + end + + def handle_info(:toggle_refresh, socket) do + %{original_refresh: original, refresh: refresh} = socket.assigns + + cond do + refresh > 0 -> + socket = + socket + |> assign(refresh: -1, original_refresh: refresh) + |> push_event("update-refresh", %{refresh: -1}) + + {:noreply, socket} + + is_integer(original) -> + handle_info({:update_refresh, original}, socket) + + true -> + handle_info({:update_refresh, 1}, socket) + end + end + + def handle_info({:update_refresh, refresh}, socket) do + socket = + socket + |> assign(refresh: refresh, original_refresh: nil) + |> schedule_refresh() + + {:noreply, push_event(socket, "update-refresh", %{refresh: refresh})} + end + + def handle_info({:update_theme, theme}, socket) do + {:noreply, + socket + |> assign(theme: theme) + |> push_event("update-theme", %{theme: theme})} + end + + def handle_info(:refresh, socket) do + socket = + socket + |> socket.assigns.page.comp.handle_refresh() + |> schedule_refresh() + + {:noreply, socket} + end + + def handle_info(message, socket) do + socket.assigns.page.comp.handle_info(message, socket) + end + + @impl Phoenix.LiveView + def handle_event("sidebar_resize", %{"width" => width}, socket) do + {:noreply, assign(socket, sidebar_width: width)} + end + + def handle_event(event, params, socket) do + socket.assigns.page.comp.handle_event(event, params, socket) + end + + ## Mount Helpers + + defp await_init([oban_name | _] = args, proc, timeout \\ 15_000) do + case apply(Oban.Registry, :whereis, args) do + nil -> + ref = make_ref() + + :telemetry.attach( + ref, + [:oban, proc, :init], + &__MODULE__.relay_init/4, + {args, self()} + ) + + receive do + {^args, %{name: ^oban_name} = conf} -> + :telemetry.detach(ref) + + # Sleep briefly to prevent race conditions between init and child process + # initialization. + Process.sleep(5) + + conf + after + timeout -> + raise RuntimeError, "no config registered for #{inspect(args)} instance" + end + + pid when is_pid(pid) -> + Oban.config(oban_name) + end + end + + @doc false + def relay_init(_event, _timing, %{conf: conf}, {args, pid}) do + send(pid, {args, conf}) + end + + ## Render Helpers + + defp resolve_page(%{"page" => "jobs"}), do: %{name: :jobs, comp: JobsPage} + defp resolve_page(%{"page" => "queues"}), do: %{name: :queues, comp: QueuesPage} + defp resolve_page(_params), do: %{name: :jobs, comp: JobsPage} + + ## Refresh Helpers + + defp init_schedule_refresh(socket) do + if connected?(socket) do + schedule_refresh(socket) + else + assign(socket, timer: nil) + end + end + + defp schedule_refresh(socket) do + cancel_timer(socket) + + if socket.assigns.refresh > 0 do + interval = :timer.seconds(socket.assigns.refresh) - 50 + + assign(socket, timer: Process.send_after(self(), :refresh, interval)) + else + assign(socket, timer: nil) + end + end + + defp cancel_timer(socket) do + if is_reference(socket.assigns.timer), do: Process.cancel_timer(socket.assigns.timer) + end +end diff --git a/vendor/oban_web/lib/oban/web/exceptions.ex b/vendor/oban_web/lib/oban/web/exceptions.ex new file mode 100644 index 00000000..2d0a5d3a --- /dev/null +++ b/vendor/oban_web/lib/oban/web/exceptions.ex @@ -0,0 +1,5 @@ +defmodule Oban.Web.AccessError do + @moduledoc false + + defexception [:message, :reason] +end diff --git a/vendor/oban_web/lib/oban/web/helpers.ex b/vendor/oban_web/lib/oban/web/helpers.ex new file mode 100644 index 00000000..566381be --- /dev/null +++ b/vendor/oban_web/lib/oban/web/helpers.ex @@ -0,0 +1,302 @@ +defmodule Oban.Web.Helpers do + @moduledoc false + + alias Oban.{Job, Registry} + alias Oban.Web.AccessError + alias Phoenix.{LiveView, VerifiedRoutes} + + # Instance Helpers + + def oban_instances do + # Only the top level instance is registered as an atom, all other keys are tuples + pattern = [{{:"$1", :_, :_}, [{:is_atom, :"$1"}], [:"$1"]}] + + pattern + |> Registry.select() + |> Enum.sort_by(&if &1 == Oban, do: 0, else: &1) + end + + # Routing Helpers + + @doc """ + Construct a path to a dashboard page with optional params. + + Routing is based on a socket and prefix tuple stored in the process dictionary. Proper routing + can be disabled for testing by setting the value to `:nowhere`. + """ + def oban_path(route, params \\ %{}) + + def oban_path(route, params) when is_list(route) do + route + |> Enum.join("/") + |> oban_path(params) + end + + def oban_path(route, params) do + params = + params + |> Enum.sort() + |> encode_params() + + case Process.get(:routing) do + {socket, prefix} -> + VerifiedRoutes.unverified_path(socket, socket.router, "#{prefix}/#{route}", params) + + :nowhere -> + "/" + + nil -> + raise RuntimeError, "nothing stored in the :routing key" + end + end + + @doc """ + Toggle filterable params into a patch path. + """ + def patch_params(params, page, key, value) when is_map(params) and is_atom(page) do + value = to_string(value) + param_value = params[key] + + params = + cond do + value == param_value or [value] == param_value -> + Map.delete(params, key) + + is_list(param_value) and value in param_value -> + Map.put(params, key, List.delete(param_value, value)) + + is_list(param_value) -> + Map.put(params, key, [value | param_value]) + + true -> + Map.put(params, key, value) + end + + oban_path(page, params) + end + + @doc """ + Prepare parsed params for URI encoding. + """ + def encode_params(params) do + for {key, val} <- params, val != nil, val != "" do + case val do + [path, frag] when is_list(path) -> + {key, Enum.join(path, ",") <> "++" <> frag} + + [_ | _] -> + {key, Enum.join(val, ",")} + + _ -> + {key, val} + end + end + end + + @doc """ + Restore params from URI encoding. + """ + def decode_params(params) do + Map.new(params, fn + {"limit", val} -> + {:limit, String.to_integer(val)} + + {key, val} when key in ~w(args meta) -> + val = + val + |> String.split("++") + |> List.update_at(0, &String.split(&1, ",")) + + {String.to_existing_atom(key), val} + + {key, val} when key in ~w(ids modes nodes priorities queues stats tags workers) -> + {String.to_existing_atom(key), String.split(val, ",")} + + {key, val} -> + {String.to_existing_atom(key), val} + end) + end + + @doc """ + """ + def active_filter?(params, :state, value) do + params[:state] == value or (is_nil(params[:state]) and value == "executing") + end + + def active_filter?(params, key, value) do + params + |> Map.get(key, []) + |> List.wrap() + |> Enum.member?(to_string(value)) + end + + @doc """ + Put a flash message that will clear automatically after a timeout. + """ + def put_flash_with_clear(socket, mode, message, timing \\ 5_000) do + Process.send_after(self(), :clear_flash, timing) + + LiveView.put_flash(socket, mode, message) + end + + @doc """ + Construct a map without any default values included. + """ + def without_defaults(%_params{}, _defaults), do: %{} + + def without_defaults(params, defaults) do + params + |> Enum.reject(fn {key, val} -> val == defaults[key] end) + |> Map.new() + end + + # Title Helpers + + def page_title(%Job{id: id, worker: worker}), do: page_title("#{worker} (#{id})") + def page_title(prefix), do: "#{prefix} • Oban" + + # Authorization Helpers + + @doc """ + Check an action against the current access controls. + """ + def can?(_action, :all), do: true + def can?(_action, :read_only), do: false + def can?(action, [_ | _] = opts), do: Keyword.get(opts, action, false) + + @doc """ + Enforce access by raising an error if access isn't allowed. + """ + def enforce_access!(action, opts) do + unless can?(action, opts), do: raise(AccessError) + + :ok + end + + # Formatting Helpers + + @doc """ + Delimit large integers with a comma separator. + """ + @spec integer_to_delimited(integer()) :: String.t() + def integer_to_delimited(integer) when is_integer(integer) do + integer + |> Integer.to_charlist() + |> Enum.reverse() + |> Enum.chunk_every(3, 3, []) + |> Enum.join(",") + |> String.reverse() + end + + @doc """ + Truncate strings beyond a fixed limit and append an ellipsis. + """ + @spec truncate(String.t(), Range.t()) :: String.t() + def truncate(string, range \\ 0..90) do + if String.length(string) > Enum.max(range) do + String.slice(string, range) <> "…" + else + string + end + end + + @doc """ + Round numbers to human readable values with a scale suffix. + """ + def integer_to_estimate(nil), do: "0" + + def integer_to_estimate(number) when number < 1000, do: to_string(number) + + def integer_to_estimate(number) when number < 10_000 do + power = 3 + mult = Integer.pow(10, power) + base = floor(number / mult) + part = round(rem(number, mult) / Integer.pow(10, power - 1)) + + case part do + 0 -> "#{base}k" + 10 -> "#{base + 1}k" + _ -> "#{base}.#{part}k" + end + end + + def integer_to_estimate(number) do + {power, suffix} = + cond do + number < 1_000_000 -> {3, "k"} + number < 1_000_000_000 -> {6, "m"} + true -> {9, "b"} + end + + mult = Integer.pow(10, power) + base = round(number / mult) + + "#{base}#{suffix}" + end + + @doc """ + Extract the name of the node that attempted a job. + """ + def attempted_by(%Job{attempted_by: [node | _]}), do: node + def attempted_by(%Job{}), do: "Not Attempted" + + @doc """ + Format job tags using a delimiter. + """ + def formatted_tags(%Job{tags: []}), do: "..." + def formatted_tags(%Job{tags: tags}), do: Enum.join(tags, ", ") + + @doc """ + A normalized, globally unique combination of instance and node names. + """ + def node_name(%{"node" => node, "name" => name}), do: node_name(node, name) + + def node_name(node, name) do + [node, name] + |> Enum.join("/") + |> String.trim_leading("Elixir.") + |> String.downcase() + end + + # State Helpers + + @doc """ + Whether the job can be cancelled in its current state. + """ + @spec cancelable?(Job.t()) :: boolean() + def cancelable?(%Job{state: state}) do + state in ~w(inserted scheduled available executing retryable) + end + + @doc """ + Whether the job can be ran immediately in its current state. + """ + @spec runnable?(Job.t()) :: boolean() + def runnable?(%Job{state: state}) do + state in ~w(inserted scheduled) + end + + @doc """ + Whether the job can be retried in its current state. + """ + @spec retryable?(Job.t()) :: boolean() + def retryable?(%Job{state: state}) do + state in ~w(completed retryable discarded cancelled) + end + + @doc """ + Whether the job can be deleted in its current state. + """ + @spec deletable?(Job.t()) :: boolean() + def deletable?(%Job{state: state}), do: state != "executing" + + @doc """ + Whether the job was left in an executing state when the node or producer running it shut down. + """ + def orphaned?(%Job{} = job, %MapSet{} = producers) do + case {job.state, job.attempted_by} do + {"executing", [_node, uuid | _]} -> not MapSet.member?(producers, uuid) + _ -> false + end + end +end diff --git a/vendor/oban_web/lib/oban/web/helpers/queue_helper.ex b/vendor/oban_web/lib/oban/web/helpers/queue_helper.ex new file mode 100644 index 00000000..457c0a86 --- /dev/null +++ b/vendor/oban_web/lib/oban/web/helpers/queue_helper.ex @@ -0,0 +1,24 @@ +defmodule Oban.Web.Helpers.QueueHelper do + @moduledoc false + + alias Oban.Web.{Queue, Timing} + + def executing_count(checks) do + checks + |> List.wrap() + |> Enum.map(&length(&1["running"])) + |> Enum.sum() + end + + def started_at(%{checks: _} = queue) do + queue + |> Queue.started_at() + |> Timing.to_words() + end + + def started_at(checks) do + %Queue{checks: checks} + |> Queue.started_at() + |> Timing.to_words() + end +end diff --git a/vendor/oban_web/lib/oban/web/live/connectivity_component.ex b/vendor/oban_web/lib/oban/web/live/connectivity_component.ex new file mode 100644 index 00000000..37b23ad8 --- /dev/null +++ b/vendor/oban_web/lib/oban/web/live/connectivity_component.ex @@ -0,0 +1,59 @@ +defmodule Oban.Web.ConnectivityComponent do + use Oban.Web, :live_component + + alias Oban.{Met, Notifier} + + @refresh :timer.seconds(15) + + @impl Phoenix.LiveComponent + def update(assigns, socket) do + checks = Met.checks(assigns.conf.name) + + status = + assigns.conf.name + |> Notifier.status() + |> determine_status(checks) + + title = + case status do + :isolated -> "Node is isolated: Updates are disabled" + :solitary -> "Node is solitary: Not connected to any cluster" + :disconnected -> "Node is disconnected: No metrics, queues, or nodes detected" + _ -> "" + end + + socket = + socket + |> assign(conf: assigns.conf, id: assigns.id) + |> assign(status: status, title: title) + + if connected?(socket) do + send_update_after(__MODULE__, %{socket.assigns | status: :reset}, @refresh) + end + + {:ok, socket} + end + + defp determine_status(pubsub, checks) do + cond do + Enum.empty?(checks) -> :disconnected + pubsub == :isolated -> :isolated + pubsub == :solitary -> :solitary + true -> :connected + end + end + + @impl Phoenix.LiveComponent + def render(assigns) do + ~H""" +
    + + + +
    + """ + end +end diff --git a/vendor/oban_web/lib/oban/web/live/instances_component.ex b/vendor/oban_web/lib/oban/web/live/instances_component.ex new file mode 100644 index 00000000..7ad03813 --- /dev/null +++ b/vendor/oban_web/lib/oban/web/live/instances_component.ex @@ -0,0 +1,94 @@ +defmodule Oban.Web.InstancesComponent do + use Oban.Web, :live_component + + alias Oban.Web.Resolver + + @impl Phoenix.LiveComponent + def update(assigns, socket) do + instances = available_instances(assigns.resolver, assigns.user) + active = inspect(assigns.conf.name) + + socket = + socket + |> assign(conf: assigns.conf, id: assigns.id) + |> assign(resolver: assigns.resolver, user: assigns.user) + |> assign(active: active, instances: instances) + + {:ok, socket} + end + + defp available_instances(resolver, user) do + instances = oban_instances() + + available = + case Resolver.call_with_fallback(resolver, :resolve_instances, [user]) do + :all -> instances + list -> Enum.filter(instances, &(&1 in list)) + end + + available + |> Enum.map(&inspect/1) + |> Enum.sort() + end + + @impl Phoenix.LiveComponent + def render(assigns) do + ~H""" +
    + + + +
    + """ + end + + @impl Phoenix.LiveComponent + def handle_event("select-instance", %{"name" => name}, socket) do + %{resolver: resolver, user: user} = socket.assigns + + allowed = Resolver.call_with_fallback(resolver, :resolve_instances, [user]) + + if allowed == :all or Enum.member?(allowed, name) do + send(self(), {:select_instance, name}) + end + + {:noreply, socket} + end +end diff --git a/vendor/oban_web/lib/oban/web/live/jobs/chart_component.ex b/vendor/oban_web/lib/oban/web/live/jobs/chart_component.ex new file mode 100644 index 00000000..90181df5 --- /dev/null +++ b/vendor/oban_web/lib/oban/web/live/jobs/chart_component.ex @@ -0,0 +1,298 @@ +defmodule Oban.Web.Jobs.ChartComponent do + use Oban.Web, :live_component + + alias Oban.Met + alias Oban.Web.Components.Core + alias Oban.Web.Timing + + @impl Phoenix.LiveComponent + def mount(socket) do + {:ok, assign(socket, last_os_time: 0, max_cols: 100, max_data: 7)} + end + + @impl Phoenix.LiveComponent + def update(assigns, socket) do + socket = + socket + |> assign(conf: assigns.conf, params: assigns.params) + |> assign_new(:group, fn -> init_lazy(:group, assigns, hd(groups())) end) + |> assign_new(:ntile, fn -> init_lazy(:ntile, assigns, "sum") end) + |> assign_new(:period, fn -> init_lazy(:period, assigns, hd(periods())) end) + |> assign_new(:series, fn -> init_lazy(:series, assigns, hd(series())) end) + |> assign_new(:visible, fn -> init_lazy(:visible, assigns, true) end) + + socket = + if socket.assigns.visible do + step = period_to_step(socket.assigns.period) + os_time = Timing.snap(assigns.os_time, step) + points = points(os_time, socket.assigns) + update = %{group: socket.assigns.group, points: points, series: socket.assigns.series} + + socket + |> assign(last_os_time: os_time) + |> push_event("chart-change", update) + else + socket + end + + {:ok, socket} + end + + defp init_lazy(key, %{init_state: init_state}, default) do + Map.get(init_state, "oban:chart-#{key}", default) + end + + @impl Phoenix.LiveComponent + def render(assigns) do + ~H""" +
    +
    +
    + + +

    + {metric_label(@series)} +

    + + + ({@period} by {String.capitalize(@group)}) + + + params_to_filters() |> Enum.any?()} + id="chart-filtered-alert" + class="w-3 h-3 ml-1 bg-violet-300 rounded-full" + data-title={"Filtered by #{params_to_filters_list(@params)}"} + phx-hook="Tippy" + > + +
    + +
    + + + + + + + + + + + + + + + +
    +
    + +
    + +
    +
    + """ + end + + # Data + + defp points(os_time, assigns) do + step = period_to_step(assigns.period) + cols = assigns.max_cols + sy_time = Timing.snap(System.system_time(:second), step) + + opts = [ + by: step, + filters: params_to_filters(assigns.params), + group: assigns.group, + lookback: cols * step, + operation: ntile_to_operation(assigns.ntile), + since: sy_time + ] + + assigns.conf.name + |> Met.timeslice(String.to_existing_atom(assigns.series), opts) + |> Enum.group_by(&elem(&1, 2), &Tuple.delete_at(&1, 2)) + |> top_n(assigns.max_data) + |> Map.new(fn {label, slices} -> {label, interpolate(slices, cols, step, os_time)} end) + end + + defp top_n(points, limit) do + points + |> Enum.sort_by(fn {_key, data} -> Enum.reduce(data, 0, &(elem(&1, 1) + &2)) end, :desc) + |> Enum.take(limit) + end + + defp interpolate(slices, cols, step, time) do + lookup = Map.new(slices) + + for index <- 0..(cols - 1) do + x = time - step * index + y = Map.get(lookup, index, nil) + + %{x: to_string(x), y: y} + end + end + + # Events + + @impl Phoenix.LiveComponent + def handle_event("select-group", %{"choice" => group}, socket) do + {:noreply, push_change(socket, group: group)} + end + + def handle_event("select-ntile", %{"choice" => ntile}, socket) do + {:noreply, push_change(socket, ntile: ntile)} + end + + def handle_event("select-period", %{"choice" => period}, socket) do + {:noreply, push_change(socket, period: period)} + end + + def handle_event("select-series", %{"choice" => series}, socket) do + assigns = + cond do + series == "full_count" and socket.assigns.group in ~w(node worker) -> + [ntile: "max", group: "state", series: series] + + series == "full_count" -> + [ntile: "max", series: series] + + series in ~w(exec_time wait_time) -> + [ntile: "p95", series: series] + + true -> + [ntile: "sum", series: series] + end + + {:noreply, push_change(socket, assigns)} + end + + def handle_event("toggle-visible", _params, socket) do + socket = + if socket.assigns.visible do + push_change(socket, visible: false) + else + push_change(socket, visible: true) + end + + {:noreply, socket} + end + + defp push_change(socket, change) do + os_time = socket.assigns.last_os_time + socket = assign(socket, change) + points = points(os_time, Map.put(socket.assigns, :last_os_time, 0)) + + update = %{ + group: socket.assigns.group, + ntile: socket.assigns.ntile, + period: socket.assigns.period, + points: points, + series: socket.assigns.series, + visible: socket.assigns.visible + } + + push_event(socket, "chart-change", update) + end + + # Lookups + + def groups, do: ~w(state queue node worker) + def ntiles, do: ~w(max p99 p95 p75 p50) + def periods, do: ~w(1s 5s 10s 30s 1m 2m) + def series, do: ~w(exec_count full_count exec_time wait_time) + + defp groups_for_series("full_count"), do: ~w(state queue) + defp groups_for_series(_series), do: groups() + + defp metric_label("exec_count"), do: "Executed Count" + defp metric_label("full_count"), do: "Full Count" + defp metric_label("exec_time"), do: "Execution Time" + defp metric_label("wait_time"), do: "Queue Time" + + defp ntile_to_operation("sum"), do: :sum + defp ntile_to_operation("max"), do: :max + defp ntile_to_operation("p99"), do: {:pct, 0.99} + defp ntile_to_operation("p95"), do: {:pct, 0.95} + defp ntile_to_operation("p75"), do: {:pct, 0.75} + defp ntile_to_operation("p50"), do: {:pct, 0.50} + + defp period_to_step("1s"), do: 1 + defp period_to_step("5s"), do: 5 + defp period_to_step("10s"), do: 10 + defp period_to_step("30s"), do: 30 + defp period_to_step("1m"), do: 60 + defp period_to_step("2m"), do: 120 + + @filterable_params ~w(nodes queues workers)a + + defp params_to_filters(params) do + for {key, vals} <- params, key in @filterable_params do + case key do + :nodes -> {:node, vals} + :queues -> {:queue, vals} + :workers -> {:worker, vals} + end + end + end + + defp params_to_filters_list(params) do + params + |> Map.take(@filterable_params) + |> Map.keys() + |> Enum.sort() + |> Enum.join(", ") + end + + # JS Commands + + defp toggle_chart(target) do + %JS{} + |> JS.toggle(in: "fade-in-scale", out: "fade-out-scale", to: "#chart") + |> JS.add_class("rotate-90", to: "#chart-toggle svg:not(.rotate-90)") + |> JS.remove_class("rotate-90", to: "#chart-toggle svg.rotate-90") + |> JS.push("toggle-visible", target: target) + end +end diff --git a/vendor/oban_web/lib/oban/web/live/jobs/detail_component.ex b/vendor/oban_web/lib/oban/web/live/jobs/detail_component.ex new file mode 100644 index 00000000..924f3d22 --- /dev/null +++ b/vendor/oban_web/lib/oban/web/live/jobs/detail_component.ex @@ -0,0 +1,266 @@ +defmodule Oban.Web.Jobs.DetailComponent do + use Oban.Web, :live_component + + alias Oban.Web.Jobs.TimelineComponent + alias Oban.Web.{Resolver, Timing} + + @impl Phoenix.LiveComponent + def render(assigns) do + ~H""" +
    +
    + + +
    + <%= if can?(:cancel_jobs, @access) and cancelable?(@job) do %> + + <% end %> + + <%= if can?(:retry_jobs, @access) and runnable?(@job) do %> + + <% end %> + + <%= if can?(:retry_jobs, @access) and retryable?(@job) do %> + + <% end %> + + <%= if can?(:delete_jobs, @access) and deletable?(@job) do %> + + <% end %> +
    +
    + +
    +
    +
    + {@job.id} + + {@job.worker} + +
    + +
    + <%= if @job.meta["recorded"] do %> + + + + <% end %> + + <%= if @job.meta["encrypted"] do %> + + + + <% end %> + + <%= if @job.meta["structured"] do %> + + + + <% end %> +
    +
    + +
    +
    + + Queue + + {@job.queue} +
    +
    + + Attempt + + {@job.attempt} of {@job.max_attempts} +
    +
    + + Priority + + {@job.priority} +
    +
    + + Tags + + {formatted_tags(@job)} +
    +
    + + Node + + {attempted_by(@job)} +
    +
    + + Queue Time + + {Timing.queue_time(@job)} +
    +
    + + Run Time + + {Timing.run_time(@job)} +
    +
    +
    + +
    + + + + + +
    + +
    +

    + + Args +

    + 
    {format_args(@job, @resolver)}
    +
    + +
    +

    + + Meta +

    + 
    {format_meta(@job, @resolver)}
    +
    + + <%= if @job.meta["recorded"] do %> +
    +

    + + Recorded Output +

    + 
    <%= format_recorded(@job, @resolver) %>
    +
    + <% end %> + +
    +

    + + Errors +

    + + <%= if Enum.any?(@job.errors) do %> + <%= for %{"at" => at, "attempt" => attempt, "error" => error} <- Enum.reverse(@job.errors) do %> +
    +

    +
    + Attempt {attempt}—{Timing.iso8601_to_words(at)} +
    +
    + +
    +

    + 
    <%= error %>
    +
    + <% end %> + <% else %> + 
    No Errors
    + <% end %> +
    +
    + """ + end + + # Handlers + + @impl Phoenix.LiveComponent + def handle_event("cancel", _params, socket) do + if can?(:cancel_jobs, socket.assigns.access) do + send(self(), {:cancel_job, socket.assigns.job}) + end + + {:noreply, socket} + end + + def handle_event("delete", _params, socket) do + if can?(:delete_jobs, socket.assigns.access) do + send(self(), {:delete_job, socket.assigns.job}) + end + + {:noreply, socket} + end + + def handle_event("retry", _params, socket) do + if can?(:retry_jobs, socket.assigns.access) do + send(self(), {:retry_job, socket.assigns.job}) + end + + {:noreply, socket} + end + + # Helpers + + defp format_args(job, resolver) do + Resolver.call_with_fallback(resolver, :format_job_args, [job]) + end + + defp format_meta(job, resolver) do + Resolver.call_with_fallback(resolver, :format_job_meta, [job]) + end + + defp format_recorded(%{meta: meta} = job, resolver) do + case meta do + %{"recorded" => true, "return" => value} -> + Resolver.call_with_fallback(resolver, :format_recorded, [value, job]) + + _ -> + "No Recording Yet" + end + end +end diff --git a/vendor/oban_web/lib/oban/web/live/jobs/sidebar_component.ex b/vendor/oban_web/lib/oban/web/live/jobs/sidebar_component.ex new file mode 100644 index 00000000..2b791f94 --- /dev/null +++ b/vendor/oban_web/lib/oban/web/live/jobs/sidebar_component.ex @@ -0,0 +1,85 @@ +defmodule Oban.Web.Jobs.SidebarComponent do + use Oban.Web, :html + + alias Oban.Web.Queue + alias Oban.Web.SidebarComponents + + attr :nodes, :list + attr :params, :map + attr :queues, :list + attr :states, :list + attr :csp_nonces, :map + attr :width, :integer, default: 320 + + def sidebar(assigns) do + ~H""" + + + + + + + + + + + + <:statuses> + + + + + + + + + """ + end +end diff --git a/vendor/oban_web/lib/oban/web/live/jobs/table_component.ex b/vendor/oban_web/lib/oban/web/live/jobs/table_component.ex new file mode 100644 index 00000000..d6f323cd --- /dev/null +++ b/vendor/oban_web/lib/oban/web/live/jobs/table_component.ex @@ -0,0 +1,230 @@ +defmodule Oban.Web.Jobs.TableComponent do + use Oban.Web, :live_component + + alias Oban.Web.Resolver + + @inc_limit 20 + @max_limit 200 + @min_limit 20 + + @impl Phoenix.LiveComponent + def update(assigns, socket) do + producers = + assigns.conf.name + |> Oban.Met.checks() + |> Enum.map(& &1["uuid"]) + |> MapSet.new() + + socket = + socket + |> assign(jobs: assigns.jobs, params: assigns.params) + |> assign(query_limit: query_limit(assigns.resolver, assigns.params)) + |> assign(producers: producers, resolver: assigns.resolver, selected: assigns.selected) + |> assign(show_less?: assigns.params.limit > @min_limit) + |> assign(show_more?: assigns.params.limit < @max_limit) + + {:ok, socket} + end + + attr :label, :string, required: true + attr :class, :string, default: "" + + defp header(assigns) do + ~H""" + + {@label} + + """ + end + + @impl Phoenix.LiveComponent + def render(assigns) do + ~H""" +
    +
      + <.header label="details" class="ml-12" /> + <.header label="queue" class="ml-auto text-right" /> + <.header label="time" class="w-20 pr-3 text-right" /> +
    + +
    +
    + No jobs match the current set of filters. +
    +

    + Filtering limited to latest {integer_to_delimited(@query_limit)} jobs. See filtering docs. +

    +
    + +
      + <.job_row + :for={job <- @jobs} + job={job} + myself={@myself} + producers={@producers} + resolver={@resolver} + selected={@selected} + /> +
    + +
    + <.load_button label="Show Less" click="load-less" active={@show_less?} myself={@myself} /> + <.load_button label="Show More" click="load-more" active={@show_more?} myself={@myself} /> +
    +
    + """ + end + + defp job_row(assigns) do + ~H""" +
  • + + + <.link patch={oban_path([:jobs, @job.id])} class="flex flex-grow items-center"> +
    + + {@job.worker} + + + + {@job.attempt} ⁄ {@job.max_attempts} + + + + {format_args(@job, @resolver)} + +
    + +
    + + + + + + {@job.queue} + +
    + +
    + 00:00 +
    + +
    • + """ + end + + defp load_button(assigns) do + ~H""" + + """ + end + + @impl Phoenix.LiveComponent + def handle_event("toggle-select", %{"id" => id}, socket) do + send(self(), {:toggle_select, String.to_integer(id)}) + + {:noreply, socket} + end + + def handle_event("load-less", _params, socket) do + if socket.assigns.show_less? do + send(self(), {:params, :limit, -@inc_limit}) + end + + {:noreply, socket} + end + + def handle_event("load-more", _params, socket) do + if socket.assigns.show_more? do + send(self(), {:params, :limit, @inc_limit}) + end + + {:noreply, socket} + end + + # Resolver Helpers + + defp query_limit(resolver, params) do + state = String.to_existing_atom(params.state) + + Resolver.call_with_fallback(resolver, :jobs_query_limit, [state]) + end + + defp format_args(job, resolver) do + resolver + |> Resolver.call_with_fallback(:format_job_args, [job]) + |> truncate(0..98) + end + + # Time Helpers + + defp timestamp(job) do + datetime = + case job.state do + "available" -> job.scheduled_at + "cancelled" -> job.cancelled_at + "completed" -> job.completed_at + "discarded" -> job.discarded_at + "executing" -> job.attempted_at + "retryable" -> job.scheduled_at + "scheduled" -> job.scheduled_at + end + + DateTime.to_unix(datetime, :millisecond) + end + + defp relative_mode(job) do + if job.state == "executing", do: "duration", else: "words" + end + + # Class Helpers + + defp hidden_class(%{hidden?: true}), do: "opacity-25 pointer-events-none" + defp hidden_class(_job), do: "" + + defp loader_class(true) do + """ + text-gray-700 dark:text-gray-300 cursor-pointer transition ease-in-out duration-200 border-b + border-gray-200 dark:border-gray-800 hover:border-gray-400 + """ + end + + defp loader_class(_), do: "text-gray-400 dark:text-gray-600 cursor-not-allowed" +end diff --git a/vendor/oban_web/lib/oban/web/live/jobs/timeline_component.ex b/vendor/oban_web/lib/oban/web/live/jobs/timeline_component.ex new file mode 100644 index 00000000..731b2a1e --- /dev/null +++ b/vendor/oban_web/lib/oban/web/live/jobs/timeline_component.ex @@ -0,0 +1,149 @@ +defmodule Oban.Web.Jobs.TimelineComponent do + @moduledoc false + + use Phoenix.Component + + alias Oban.Web.Components.Icons + alias Oban.Web.Timing + + @empty_time "—" + + @state_to_timestamp %{ + "cancelled" => :cancelled_at, + "discarded" => :discarded_at, + "executing" => :attempted_at, + "inserted" => :inserted_at, + "scheduled" => :scheduled_at + } + + def render(assigns) do + ~H""" +
    + + <%= if timeline_icon(@state, @job) == :checkmark do %> + + <% end %> + <%= if timeline_icon(@state, @job) == :spinner do %> + + + + <% end %> + + + {timestamp_name(@state, @job)} + + + {timeline_time(@state, @job, @os_time)} + +
    + """ + end + + # Helpers + + defp timestamp_title(state, job) do + case state do + "inserted" -> "Inserted At: #{truncate_sec(job.inserted_at)}" + "scheduled" -> "Scheduled At: #{truncate_sec(job.scheduled_at)}" + "executing" -> "Attempted At: #{truncate_sec(job.attempted_at)}" + "completed" -> "Completed At: #{truncate_sec(job.completed_at)}" + "cancelled" -> "Cancelled At: #{truncate_sec(job.cancelled_at)}" + "discarded" -> "Discarded At: #{truncate_sec(job.discarded_at)}" + end + end + + defp timeline_class(state, job) do + case absolute_state(state, job) do + :finished -> "bg-green-500 text-white" + :retrying -> "bg-orange-400 text-white" + :started -> "bg-yellow-400 text-white" + :unstarted -> "bg-gray-100 text-white dark:bg-black dark:bg-opacity-25" + end + end + + defp timeline_icon(state, job) do + case absolute_state(state, job) do + :finished -> :checkmark + :retrying -> :spinner + :started -> :spinner + :unstarted -> nil + end + end + + defp timestamp_name(state, job) do + case {state, job.state} do + {"retryable", "retryable"} -> "Retryable" + {"retryable", _} -> "Scheduled" + {"executing", "executing"} -> "Executing" + {"executing", _} -> "Completed" + _ -> String.capitalize(state) + end + end + + defp timeline_time(state, job, os_time) do + for_state = Map.get(@state_to_timestamp, state) + timestamp = Map.get(job, for_state) + now = DateTime.from_unix!(os_time) + + case {state, job.state, timestamp} do + {_, _, nil} -> + @empty_time + + {"executing", "executing", at} -> + at + |> DateTime.diff(now) + |> Timing.to_duration() + + {"executing", "completed", _at} -> + words = + job.completed_at + |> DateTime.diff(now) + |> Timing.to_words() + + duration = + job.attempted_at + |> DateTime.diff(job.completed_at) + |> Timing.to_duration() + + "#{words} (#{duration})" + + {"executing", _, _} -> + @empty_time + + {_, _, at} -> + at + |> DateTime.diff(now) + |> Timing.to_words() + end + end + + defp absolute_state(state, job) do + for_state = Map.get(@state_to_timestamp, state) + timestamp = Map.get(job, for_state) + + absolute_state(state, job.state, timestamp) + end + + defp absolute_state("executing", "completed", _), do: :finished + defp absolute_state("executing", "executing", _), do: :started + defp absolute_state("executing", _, _), do: :unstarted + defp absolute_state("cancelled", "cancelled", _), do: :finished + defp absolute_state("cancelled", "retryable", _), do: :unstarted + defp absolute_state("discarded", "discarded", _), do: :finished + defp absolute_state("discarded", "retryable", _), do: :unstarted + defp absolute_state("scheduled", "retryable", _), do: :retrying + defp absolute_state(state, state, _), do: :started + defp absolute_state(_, _, at) when not is_nil(at), do: :finished + defp absolute_state(_, _, _), do: :unstarted + + defp truncate_sec(nil), do: @empty_time + defp truncate_sec(datetime), do: NaiveDateTime.truncate(datetime, :second) +end diff --git a/vendor/oban_web/lib/oban/web/live/queues/detail_component.ex b/vendor/oban_web/lib/oban/web/live/queues/detail_component.ex new file mode 100644 index 00000000..dd961b46 --- /dev/null +++ b/vendor/oban_web/lib/oban/web/live/queues/detail_component.ex @@ -0,0 +1,694 @@ +defmodule Oban.Web.Queues.DetailComponent do + use Oban.Web, :live_component + + import Oban.Web.Helpers.QueueHelper + import Phoenix.HTML.Form, only: [options_for_select: 2] + + alias Oban.Config + alias Oban.Met + alias Oban.Web.Components.Core + alias Oban.Web.Queues.DetailInsanceComponent + + @impl Phoenix.LiveComponent + def update(%{local_limit: new_limit}, socket) do + %{checks: checks, inputs: inputs} = socket.assigns + + local_limit = + cond do + match?([_], checks) -> + new_limit + + match?([_ | _], checks) -> + max(local_limit(checks), new_limit) + + true -> + inputs.local_limit + end + + inputs = %{inputs | local_limit: local_limit} + + {:ok, assign(socket, inputs: inputs)} + end + + def update(assigns, socket) do + checks = Enum.filter(assigns.checks, &(&1["queue"] == assigns.queue)) + + counts = + Met.latest(assigns.conf.name, :full_count, group: "state", filters: [queue: assigns.queue]) + + socket = + socket + |> assign(access: assigns.access, conf: assigns.conf, queue: assigns.queue) + |> assign(counts: counts, checks: checks) + |> assign_new(:inputs, fn -> + %{ + local_limit: local_limit(checks), + global_allowed: global_allowed(checks), + global_partition_fields: partition_value(checks, "global_limit", "fields"), + global_partition_keys: partition_value(checks, "global_limit", "keys"), + rate_allowed: rate_allowed(checks), + rate_period: rate_period(checks), + rate_partition_fields: partition_value(checks, "rate_limit", "fields"), + rate_partition_keys: partition_value(checks, "rate_limit", "keys") + } + end) + + {:ok, socket} + end + + @impl Phoenix.LiveComponent + def render(assigns) do + ~H""" +
    +
    + +
    + + + + + + + + + + + + + + + + + + + + + <%= for state <- ~w(available scheduled retryable cancelled discarded completed) do %> + + <% end %> + + +
    StartedExecutingAvailableScheduledRetryableCancelledDiscardedCompleted
    +
    + + {started_at(@checks)} +
    +     		+
    + + {executing_count(@checks)} +
    +     		+
    + + {integer_to_estimate(@counts[state])} +
    +
    + +
    +
    + +

    Configuration

    +
    + +
    +
    +

    + + Local +

    + + + + <.submit_input + locked={not can?(:scale_queues, @access)} + disabled={ + @inputs.local_limit == local_limit(@checks) or not can?(:scale_queues, @access) + } + label="Scale" + /> + + +
    + <.pro_blocker :if={missing_pro?(@conf)} /> + +
    +
    +

    + + Global +

    + + <.toggle_button + disabled={not can?(:scale_queues, @access)} + enabled={not is_nil(@inputs.global_allowed)} + feature="global" + myself={@myself} + /> +
    + +
    + +
    + +
    +
    + + +
    + +
    + + + +
    +
    + + <.submit_input + locked={not can?(:scale_queues, @access)} + disabled={global_unchanged?(@checks, @inputs) or not can?(:scale_queues, @access)} + label="Apply" + /> +
    +
    + +
    + <.pro_blocker :if={missing_pro?(@conf)} /> + +
    +
    +

    + + Rate Limit +

    + + <.toggle_button + disabled={not can?(:scale_queues, @access)} + enabled={not is_nil(@inputs.rate_allowed)} + feature="rate-limit" + myself={@myself} + /> +
    + +
    +
    + +
    + +
    + +
    +
    + +
    +
    + + +
    + +
    + + + +
    +
    + + <.submit_input + locked={not can?(:scale_queues, @access)} + disabled={rate_unchanged?(@checks, @inputs) or not can?(:scale_queues, @access)} + label="Apply" + /> +
    +
    +
    +
    + +
    +
    + +

    Instances

    +
    + + + + + + + + + + + + + + <%= for checks <- @checks do %> + <.live_component + access={@access} + checks={checks} + id={node_name(checks)} + module={DetailInsanceComponent} + /> + <% end %> + +
    + Node/Name + + Executing + + Started + + Pause + + Scale +
    +
    +
    + """ + end + + # Handlers + + @integer_inputs ~w(local_limit global_allowed rate_allowed rate_period) + + @impl Phoenix.LiveComponent + def handle_event("form-change", %{"_target" => ["rate_partition_fields"]} = params, socket) do + fields = params["rate_partition_fields"] + inputs = %{socket.assigns.inputs | rate_partition_fields: fields} + + {:noreply, assign(socket, inputs: inputs)} + end + + def handle_event("form-change", %{"_target" => ["global_partition_fields"]} = params, socket) do + fields = params["global_partition_fields"] + inputs = %{socket.assigns.inputs | global_partition_fields: fields} + + {:noreply, assign(socket, inputs: inputs)} + end + + def handle_event("form-change", params, socket) do + inputs = + for {key, val} <- params, key in @integer_inputs, reduce: socket.assigns.inputs do + acc -> + case Integer.parse(val) do + {int, _} when int > 0 -> + %{acc | String.to_existing_atom(key) => int} + + _ -> + acc + end + end + + {:noreply, assign(socket, inputs: inputs)} + end + + def handle_event("local-submit", params, socket) do + enforce_access!(:scale_queues, socket.assigns.access) + + limit = String.to_integer(params["local_limit"]) + + send(self(), {:scale_queue, socket.assigns.queue, limit: limit}) + + inputs = %{socket.assigns.inputs | local_limit: limit} + + {:noreply, assign(socket, inputs: inputs)} + end + + def handle_event("global-update", params, socket) do + enforce_access!(:scale_queues, socket.assigns.access) + + inputs = + if is_nil(params["global_allowed"]) do + send(self(), {:scale_queue, socket.assigns.queue, global_limit: nil}) + + socket.assigns.inputs + |> Map.replace!(:global_allowed, nil) + |> Map.replace!(:global_partition_fields, nil) + |> Map.replace!(:global_partition_keys, nil) + else + allowed = String.to_integer(params["global_allowed"]) + fields = maybe_split(params["global_partition_fields"]) + keys = maybe_split(params["global_partition_keys"]) + + global_limit = + case fields do + [] -> + %{allowed: allowed} + + ["worker"] -> + %{allowed: allowed, partition: [fields: fields]} + + _ -> + %{allowed: allowed, partition: [fields: fields, keys: keys]} + end + + send(self(), {:scale_queue, socket.assigns.queue, global_limit: global_limit}) + + socket.assigns.inputs + |> Map.replace!(:global_allowed, allowed) + |> Map.replace!(:global_partition_fields, Enum.join(fields, ",")) + |> Map.replace!(:global_partition_keys, Enum.join(keys, ",")) + end + + {:noreply, assign(socket, inputs: inputs)} + end + + def handle_event("rate-limit-update", params, socket) do + enforce_access!(:scale_queues, socket.assigns.access) + + inputs = + if is_nil(params["rate_allowed"]) do + send(self(), {:scale_queue, socket.assigns.queue, rate_limit: nil}) + + socket.assigns.inputs + |> Map.replace!(:rate_allowed, nil) + |> Map.replace!(:rate_period, nil) + |> Map.replace!(:rate_partition_fields, nil) + |> Map.replace!(:rate_partition_keys, nil) + else + allowed = String.to_integer(params["rate_allowed"]) + period = String.to_integer(params["rate_period"]) + fields = maybe_split(params["rate_partition_fields"]) + keys = maybe_split(params["rate_partition_keys"]) + + rate_limit = + case fields do + [] -> + %{allowed: allowed, period: period} + + ["worker"] -> + %{allowed: allowed, period: period, partition: [fields: fields]} + + _ -> + %{allowed: allowed, period: period, partition: [fields: fields, keys: keys]} + end + + send(self(), {:scale_queue, socket.assigns.queue, rate_limit: rate_limit}) + + socket.assigns.inputs + |> Map.replace!(:rate_allowed, allowed) + |> Map.replace!(:rate_period, period) + |> Map.replace!(:rate_partition_fields, Enum.join(fields, ",")) + |> Map.replace!(:rate_partition_keys, Enum.join(keys, ",")) + end + + {:noreply, assign(socket, inputs: inputs)} + end + + def handle_event("increment", %{"field" => field}, socket) do + {:noreply, change_input(socket, field, 1)} + end + + def handle_event("decrement", %{"field" => field}, socket) do + {:noreply, change_input(socket, field, -1)} + end + + def handle_event("toggle-feature", %{"feature" => "global"}, socket) do + inputs = + Map.update!(socket.assigns.inputs, :global_allowed, fn value -> + if is_nil(value), do: socket.assigns.inputs.local_limit, else: nil + end) + + {:noreply, assign(socket, inputs: inputs)} + end + + def handle_event("toggle-feature", %{"feature" => "rate-limit"}, socket) do + inputs = + if is_nil(socket.assigns.inputs.rate_allowed) do + socket.assigns.inputs + |> Map.put(:rate_allowed, socket.assigns.inputs.local_limit) + |> Map.put(:rate_period, 60) + else + socket.assigns.inputs + |> Map.put(:rate_allowed, nil) + |> Map.put(:rate_period, nil) + |> Map.put(:rate_partition_fields, nil) + |> Map.put(:rate_partition_keys, nil) + end + + {:noreply, assign(socket, inputs: inputs)} + end + + # Components + + defp toggle_button(assigns) do + ~H""" + + """ + end + + defp submit_input(assigns) do + ~H""" +
    + <%= if @locked do %> + + <% end %> + + +
    + """ + end + + defp pro_blocker(assigns) do + ~H""" + + Requires + + Oban Pro + + + """ + end + + # Helpers + + defp local_limit([]), do: 0 + + defp local_limit(checks) do + checks + |> Enum.map(& &1["local_limit"]) + |> Enum.max() + end + + defp global_allowed(checks) do + checks + |> Enum.map(& &1["global_limit"]) + |> Enum.filter(&is_map/1) + |> Enum.find_value(& &1["allowed"]) + end + + defp rate_allowed(checks) do + checks + |> Enum.map(& &1["rate_limit"]) + |> Enum.filter(&is_map/1) + |> Enum.find_value(& &1["allowed"]) + end + + defp rate_period(checks) do + checks + |> Enum.map(& &1["rate_limit"]) + |> Enum.filter(&is_map/1) + |> Enum.find_value(& &1["period"]) + end + + defp partition_value(checks, parent, key) do + checks + |> Enum.map(& &1[parent]) + |> Enum.filter(&is_map/1) + |> List.first() + |> case do + %{"partition" => %{^key => [_ | _] = value}} -> Enum.join(value, ",") + _ -> nil + end + end + + defp global_unchanged?(checks, inputs) do + inputs.global_allowed == global_allowed(checks) and + inputs.global_partition_fields == partition_value(checks, "global_limit", "fields") and + inputs.global_partition_keys == partition_value(checks, "global_limit", "keys") + end + + defp rate_unchanged?(checks, inputs) do + inputs.rate_allowed == rate_allowed(checks) and + inputs.rate_period == rate_period(checks) and + inputs.rate_partition_fields == partition_value(checks, "rate_limit", "fields") and + inputs.rate_partition_keys == partition_value(checks, "rate_limit", "keys") + end + + defp change_input(socket, field, change) do + field = String.to_existing_atom(field) + + inputs = + Map.update!(socket.assigns.inputs, field, fn value -> + if is_integer(value) and value + change > 0 do + value + change + else + value + end + end) + + assign(socket, inputs: inputs) + end + + defp maybe_split(""), do: [] + defp maybe_split(nil), do: [] + defp maybe_split(value) when is_binary(value), do: String.split(value, ",") + + defp partition_options do + [Off: nil, Worker: "worker", Args: "args", "Worker + Args": "args,worker"] + end + + defp keyless_partition?(inputs), do: inputs not in ["args", "args,worker"] + + # Pro Helpers + + defp missing_pro?(%Config{engine: engine}) do + engine in [Oban.Queue.BasicEngine, Oban.Engines.Basic] + end +end diff --git a/vendor/oban_web/lib/oban/web/live/queues/detail_instance_component.ex b/vendor/oban_web/lib/oban/web/live/queues/detail_instance_component.ex new file mode 100644 index 00000000..03a75fa9 --- /dev/null +++ b/vendor/oban_web/lib/oban/web/live/queues/detail_instance_component.ex @@ -0,0 +1,129 @@ +defmodule Oban.Web.Queues.DetailInsanceComponent do + use Oban.Web, :live_component + + import Oban.Web.Helpers.QueueHelper + + alias Oban.Web.Components.Core + + @impl Phoenix.LiveComponent + def update(%{local_limit: local_limit}, socket) do + {:ok, assign(socket, local_limit: local_limit)} + end + + def update(assigns, socket) do + socket = + socket + |> assign(access: assigns.access, checks: assigns.checks) + |> assign_new(:queue, fn -> assigns.checks["queue"] end) + |> assign_new(:paused, fn -> assigns.checks["paused"] end) + |> assign_new(:local_limit, fn -> assigns.checks["local_limit"] end) + + {:ok, socket} + end + + @impl Phoenix.LiveComponent + def render(assigns) do + ~H""" + + {node_name(@checks)} + {executing_count(@checks)} + {started_at(@checks)} + + <.pause_button + disabled={not can?(:pause_queues, @access)} + node={node_name(@checks)} + paused={@paused} + queue={@queue} + target={@myself} + /> + + +
    + + + + + + + + + """ + end + + defp pause_button(assigns) do + assigns = + assigns + |> assign_new(:id, fn -> "#{assigns.queue}-toggle-pause" end) + |> assign_new(:title, fn -> if assigns.paused, do: "Resume queue", else: "Pause queue" end) + + ~H""" + + """ + end + + @impl Phoenix.LiveComponent + def handle_event("toggle-pause", _params, socket) do + enforce_access!(:pause_queues, socket.assigns.access) + + checks = socket.assigns.checks + action = if socket.assigns.paused, do: :resume_queue, else: :pause_queue + + send(self(), {action, checks["queue"], checks["name"], checks["node"]}) + + {:noreply, assign(socket, paused: not socket.assigns.paused)} + end + + def handle_event("update", %{"local_limit" => limit}, socket) do + enforce_access!(:scale_queues, socket.assigns.access) + + limit = String.to_integer(limit) + checks = socket.assigns.checks + + send(self(), {:scale_queue, checks["queue"], checks["name"], checks["node"], limit}) + + {:noreply, assign(socket, local_limit: limit)} + end + + def handle_event("increment", _params, socket) do + {:noreply, assign(socket, local_limit: socket.assigns.local_limit + 1)} + end + + def handle_event("decrement", _params, socket) do + if socket.assigns.local_limit > 1 do + {:noreply, assign(socket, local_limit: socket.assigns.local_limit - 1)} + else + {:noreply, socket} + end + end +end diff --git a/vendor/oban_web/lib/oban/web/live/queues/sidebar_component.ex b/vendor/oban_web/lib/oban/web/live/queues/sidebar_component.ex new file mode 100644 index 00000000..f62fde10 --- /dev/null +++ b/vendor/oban_web/lib/oban/web/live/queues/sidebar_component.ex @@ -0,0 +1,63 @@ +defmodule Oban.Web.Queues.SidebarComponent do + use Oban.Web, :html + + alias Oban.Web.Queue + alias Oban.Web.SidebarComponents + + attr :queues, :list + attr :params, :map + attr :csp_nonces, :map + attr :width, :integer, default: 320 + + def sidebar(assigns) do + ~H""" + + + + + + + + + + + + + + + + """ + end + + defp nodes(queues) do + queues + |> Enum.flat_map(& &1.checks) + |> Enum.reduce(%{}, fn %{"node" => node}, acc -> Map.update(acc, node, 1, &(&1 + 1)) end) + end +end diff --git a/vendor/oban_web/lib/oban/web/live/queues/table_component.ex b/vendor/oban_web/lib/oban/web/live/queues/table_component.ex new file mode 100644 index 00000000..2a4ed415 --- /dev/null +++ b/vendor/oban_web/lib/oban/web/live/queues/table_component.ex @@ -0,0 +1,204 @@ +defmodule Oban.Web.Queues.TableComponent do + use Oban.Web, :live_component + + import Oban.Web.Helpers.QueueHelper + + alias Oban.Web.{Queue, Timing} + + @impl Phoenix.LiveComponent + def render(assigns) do + ~H""" +
    +
      + <.queue_header label="name" class="ml-12 w-1/3 text-left" /> +
      + <.queue_header label="nodes" class="w-16 text-right" /> + <.queue_header label="exec" class="w-16 text-right" /> + <.queue_header label="avail" class="w-16 text-right" /> + <.queue_header label="local" class="w-16 text-right" /> + <.queue_header label="global" class="w-16 text-right" /> + <.queue_header label="rate limit" class="w-32 text-right" /> + <.queue_header label="started" class="w-28 text-right" /> + <.queue_header label="status" class="w-20 pr-3 text-right" /> +
      +
    + +
    + No queues running match the current set of filters. +
    + +
      + <.queue_row + :for={queue <- @queues} + access={@access} + myself={@myself} + queue={queue} + selected={MapSet.member?(@selected, queue.name)} + /> +
    +
    + """ + end + + # Components + + attr :label, :string, required: true + attr :class, :string, default: "" + + defp queue_header(assigns) do + ~H""" + + {@label} + + """ + end + + attr :access, :map, required: true + attr :myself, :any, required: true + attr :queue, :string, required: true + attr :selected, :boolean, default: false + + defp queue_row(assigns) do + ~H""" +
  • + + + <.link patch={oban_path([:queues, @queue.name])} class="py-5 flex flex-grow items-center"> +
    + {@queue.name} +
    + +
    + + {length(@queue.checks)} + + + + {@queue.counts.executing} + + + + {integer_to_estimate(@queue.counts.available)} + + + + {local_limit(@queue)} + + + + {global_limit(@queue)} + + + + {rate_limit(@queue)} + + + + {started_at(@queue)} + + +
    + + + +
    +
    + +
    • + """ + end + + # Handlers + + @impl Phoenix.LiveComponent + def handle_event("toggle-select", %{"id" => queue}, socket) do + send(self(), {:toggle_select, queue}) + + {:noreply, socket} + end + + # Helpers + + defp local_limit(queue) do + queue.checks + |> Enum.map(& &1["local_limit"]) + |> Enum.min_max() + |> case do + {min, min} -> min + {min, max} -> "#{min}..#{max}" + end + end + + defp global_limit(queue) do + Queue.global_limit(queue) || "-" + end + + defp rate_limit(queue) do + queue.checks + |> Enum.map(& &1["rate_limit"]) + |> Enum.reject(&is_nil/1) + |> case do + [head_limit | _rest] = rate_limits -> + %{"allowed" => allowed, "period" => period, "window_time" => time} = head_limit + + {prev_total, curr_total} = + rate_limits + |> Enum.flat_map(& &1["windows"]) + |> Enum.reduce({0, 0}, fn %{"prev_count" => pcnt, "curr_count" => ccnt}, {pacc, cacc} -> + {pacc + pcnt, cacc + ccnt} + end) + + unix_now = DateTime.to_unix(DateTime.utc_now(), :second) + ellapsed = unix_now - time_to_unix(time) + weight = div(max(period - ellapsed, 0), period) + remaining = prev_total * weight + curr_total + + period_in_words = Timing.to_words(period, relative: false) + + "#{remaining}/#{allowed} per #{period_in_words}" + + [] -> + "-" + end + end + + defp time_to_unix(unix) when is_integer(unix), do: unix + + defp time_to_unix(time) do + Date.utc_today() + |> DateTime.new!(Time.from_iso8601!(time)) + |> DateTime.to_unix(:second) + end +end diff --git a/vendor/oban_web/lib/oban/web/live/refresh_component.ex b/vendor/oban_web/lib/oban/web/live/refresh_component.ex new file mode 100644 index 00000000..4cd0f558 --- /dev/null +++ b/vendor/oban_web/lib/oban/web/live/refresh_component.ex @@ -0,0 +1,118 @@ +defmodule Oban.Web.RefreshComponent do + use Oban.Web, :live_component + + @options [ + {01, "1s"}, + {02, "2s"}, + {05, "5s"}, + {15, "15s"}, + {60, "1m"}, + {-1, "Off"} + ] + + @impl Phoenix.LiveComponent + def mount(socket) do + {:ok, assign(socket, :options, @options)} + end + + @impl Phoenix.LiveComponent + def render(assigns) do + ~H""" +
    + + + +
    + """ + end + + attr :refresh, :integer, required: true + attr :value, :integer, required: true + attr :display, :string, required: true + + defp option(assigns) do + class = + if assigns.refresh == assigns.value do + "text-blue-500 dark:text-blue-400" + else + "text-gray-500 dark:text-gray-400 " + end + + assigns = assign(assigns, :class, class) + + ~H""" +
  • + <%= if @value == @refresh do %> + + <% else %> + + <% end %> + + {@display} +
    • + """ + end + + # Handlers + + @impl Phoenix.LiveComponent + def handle_event("pause-refresh", _params, socket) do + send(self(), :pause_refresh) + + {:noreply, socket} + end + + def handle_event("resume-refresh", _params, socket) do + send(self(), :resume_refresh) + + {:noreply, socket} + end + + def handle_event("select-refresh", %{"interval" => interval}, socket) do + value = if is_binary(interval), do: String.to_integer(interval), else: interval + + send(self(), {:update_refresh, value}) + + {:noreply, socket} + end + + def handle_event("toggle-refresh", _params, socket) do + send(self(), :toggle_refresh) + + {:noreply, socket} + end +end diff --git a/vendor/oban_web/lib/oban/web/live/search_component.ex b/vendor/oban_web/lib/oban/web/live/search_component.ex new file mode 100644 index 00000000..2408eeb4 --- /dev/null +++ b/vendor/oban_web/lib/oban/web/live/search_component.ex @@ -0,0 +1,330 @@ +defmodule Oban.Web.SearchComponent do + use Oban.Web, :live_component + + @spinner_timeout 100 + + @impl Phoenix.LiveComponent + def mount(socket) do + {:ok, assign(socket, buffer: "", loading: false)} + end + + @impl Phoenix.LiveComponent + def update(assigns, socket) do + suggestions = + Map.get_lazy(assigns, :suggestions, fn -> + assigns.queryable.suggest(socket.assigns.buffer, assigns.conf, resolver: assigns.resolver) + end) + + socket = + socket + |> assign(assigns) + |> assign(suggestions: suggestions) + + {:ok, socket} + end + + @impl Phoenix.LiveComponent + def render(assigns) do + ~H""" + + """ + end + + attr :param, :any, required: true + attr :terms, :any, required: true + + defp filter(assigns) do + ~H""" +
    + + {format_filter(@param, @terms)} + + + +
    + """ + end + + defp format_filter(param, [path, term]) when is_list(path) do + "#{param}.#{Enum.join(path, ".")}:#{term}" + end + + defp format_filter(param, term) when is_list(term) do + "#{param}:#{Enum.join(term, ",")}" + end + + defp format_filter(param, term) do + "#{param}:#{term}" + end + + attr :buff, :string + attr :name, :string + attr :desc, :string + attr :exmp, :string + + defp option(assigns) do + ~H""" + + """ + end + + defp clearable?(buffer, params, queryable) do + String.length(buffer) > 0 or map_size(filterable(params, queryable)) > 0 + end + + defp filterable(params, queryable), do: Map.take(params, queryable.filterable()) + + defp highlight(value, substr) do + match = + substr + |> String.split([":", "."], trim: true) + |> List.last() + + if is_binary(value) and is_binary(match) do + pattern = Regex.compile!("(#{Regex.escape(match)})", "i") + + value + |> String.replace(pattern, "\\1") + |> raw() + else + value + end + end + + def show_focus do + %JS{} + |> JS.remove_class("ring-gray-300 dark:ring-gray-700", to: "#search-wrapper") + |> JS.add_class("shadow-blue-100 dark:shadow-blue-950", to: "#search-wrapper") + |> JS.add_class("ring-blue-500 dark:ring-blue-700", to: "#search-wrapper") + |> JS.add_class("bg-blue-100/30 dark:bg-blue-900/30", to: "#search-wrapper") + |> JS.show(to: "#search-suggest") + end + + # Closing the suggest menu is done with push events to allow clicking on a suggestion to fire + # before the menu closes. + def hide_focus do + %JS{} + |> JS.add_class("ring-gray-300 dark:ring-gray-700", to: "#search-wrapper") + |> JS.remove_class("shadow-blue-100 dark:shadow-blue-950", to: "#search-wrapper") + |> JS.remove_class("ring-blue-500 dark:ring-blue-700", to: "#search-wrapper") + |> JS.remove_class("bg-blue-100/30 dark:bg-blue-900/30", to: "#search-wrapper") + end + + # Events + + @impl Phoenix.LiveComponent + def handle_event("change", %{"terms" => buffer}, socket) do + socket = + socket + |> assign(buffer: buffer) + |> async_suggest(buffer) + + {:noreply, socket} + end + + def handle_event("clear", _params, socket) do + suggestions = socket.assigns.queryable.suggest("", socket.assigns.conf) + + {:noreply, + socket + |> assign(buffer: "", loading: false, suggestions: suggestions) + |> push_patch(to: oban_path(socket.assigns.page))} + end + + def handle_event("append", %{"choice" => choice}, socket) do + buffer = socket.assigns.queryable.append(socket.assigns.buffer, choice) + + socket + |> assign(buffer: buffer) + |> handle_submit() + end + + def handle_event("complete", _params, socket) do + buffer = socket.assigns.queryable.complete(socket.assigns.buffer, socket.assigns.conf) + + socket = + socket + |> assign(buffer: buffer) + |> push_event("completed", %{buffer: buffer}) + |> async_suggest(buffer) + + {:noreply, socket} + end + + def handle_event("search", _, socket) do + handle_submit(socket) + end + + def handle_event("remove-filter", %{"param" => param, "terms" => _}, socket) do + params = Map.delete(socket.assigns.params, String.to_existing_atom(param)) + + {:noreply, push_patch(socket, to: oban_path(socket.assigns.page, params))} + end + + defp handle_submit(socket) do + %{buffer: buffer, queryable: queryable} = socket.assigns + + if String.ends_with?(buffer, ":") or String.ends_with?(buffer, ".") do + {:noreply, async_suggest(socket, buffer)} + else + parsed = queryable.parse(buffer) + params = Map.merge(socket.assigns.params, parsed, fn _key, old, new -> old ++ new end) + suggestions = queryable.suggest("", socket.assigns.conf) + + {:noreply, + socket + |> assign(buffer: "", loading: false, suggestions: suggestions) + |> push_patch(to: oban_path(socket.assigns.page, params))} + end + end + + def handle_info({:DOWN, _ref, :process, _pid, _reason}, socket) do + {:noreply, socket} + end + + def handle_info({ref, suggestions}, socket) when is_reference(ref) do + {:noreply, assign(socket, loading: false, suggestions: suggestions)} + end + + defp async_suggest(socket, buffer) do + self = self() + + fun = fn -> + suggestions = + socket.assigns.queryable.suggest(buffer, socket.assigns.conf, + resolver: socket.assigns.resolver + ) + + assigns = + socket.assigns + |> Map.take(~w(id conf params queryable resolver)a) + |> Map.put(:loading, false) + |> Map.put(:suggestions, suggestions) + + send_update(self, __MODULE__, assigns) + end + + fun + |> Task.async() + |> Task.yield(@spinner_timeout) + |> case do + nil -> assign(socket, loading: true) + _ok -> socket + end + end +end diff --git a/vendor/oban_web/lib/oban/web/live/shortcuts_component.ex b/vendor/oban_web/lib/oban/web/live/shortcuts_component.ex new file mode 100644 index 00000000..4da84984 --- /dev/null +++ b/vendor/oban_web/lib/oban/web/live/shortcuts_component.ex @@ -0,0 +1,112 @@ +defmodule Oban.Web.ShortcutsComponent do + use Oban.Web, :live_component + + @impl Phoenix.LiveComponent + def render(assigns) do + ~H""" +