From 3c988a5ff3ee0d26db2ee41f2145995a4db7e162 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Thu, 30 Apr 2026 09:25:11 -0500 Subject: [PATCH] =?UTF-8?q?deps:=20oban=5Fpro=201.6.14=20(vendored)=20?= =?UTF-8?q?=E2=86=92=201.7.0=20(hex)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Oban Pro 1.7.0 ships database-backed workflow tracking, replaces the generated `uniq_key` / `partition_key` columns with expression indexes, and adds new partial indexes for query performance. Switches oban_pro from the vendor/ tree to the licensed `oban` hex repo (auth already configured via `mix hex.repo`). vendor/oban_pro deleted — the vendoring was a 1.6.x stopgap, 1.7 is the first version we land directly from hex. oban_met (1.1.0) and oban_web (2.12.1) stay vendored — they're not published to the licensed `oban` repo (verified by 404 on `mix hex.package fetch`). Migration 20260430142341 runs `Oban.Pro.Migration.up(version: "1.7.0")` which creates the workflow tables and the new indexes. Verified on dev + test DBs. Production is well below the size threshold the v1.7 guide flags for split-migration handling, so the inline default suffices. No code changes for the breaking-change list — the codebase only references Oban.Pro.Engines.Smart and Oban.Pro.Plugins.DynamicLifeline, both unchanged in 1.7. We don't use Workflow.after_cancelled/2 or Oban.Pro.Workers.Chunk. Test suite stays at 3132/3132. --- mix.exs | 5 +- mix.lock | 1 + .../20260430142341_oban_pro_v1_7.exs | 26 + vendor/oban_pro/.formatter.exs | 14 - vendor/oban_pro/.hex | Bin 264 -> 0 bytes vendor/oban_pro/hex_metadata.config | 62 - vendor/oban_pro/lib/oban/pro/application.ex | 29 - vendor/oban_pro/lib/oban/pro/batch.ex | 873 ----- vendor/oban_pro/lib/oban/pro/cloud.ex | 23 - vendor/oban_pro/lib/oban/pro/cron.ex | 125 - vendor/oban_pro/lib/oban/pro/decorator.ex | 587 ---- vendor/oban_pro/lib/oban/pro/diagnostics.ex | 135 - vendor/oban_pro/lib/oban/pro/engines/smart.ex | 1628 --------- vendor/oban_pro/lib/oban/pro/exceptions.ex | 12 - vendor/oban_pro/lib/oban/pro/flusher.ex | 24 - vendor/oban_pro/lib/oban/pro/handler.ex | 13 - vendor/oban_pro/lib/oban/pro/limiter.ex | 28 - .../oban_pro/lib/oban/pro/limiters/global.ex | 102 - .../oban_pro/lib/oban/pro/limiters/local.ex | 13 - vendor/oban_pro/lib/oban/pro/limiters/rate.ex | 125 - vendor/oban_pro/lib/oban/pro/migration.ex | 387 --- .../pro/migrations/dynamic_partitioner.ex | 195 -- .../lib/oban/pro/migrations/v1_0_0.ex | 97 - .../lib/oban/pro/migrations/v1_4_0.ex | 60 - .../lib/oban/pro/migrations/v1_5_0.ex | 114 - .../lib/oban/pro/migrations/v1_6_0.ex | 109 - vendor/oban_pro/lib/oban/pro/partition.ex | 195 -- .../lib/oban/pro/plugins/dynamic_cron.ex | 793 ----- .../lib/oban/pro/plugins/dynamic_lifeline.ex | 404 --- .../oban/pro/plugins/dynamic_partitioner.ex | 669 ---- .../oban/pro/plugins/dynamic_prioritizer.ex | 318 -- .../lib/oban/pro/plugins/dynamic_pruner.ex | 692 ---- .../lib/oban/pro/plugins/dynamic_queues.ex | 745 ---- .../lib/oban/pro/plugins/dynamic_scaler.ex | 662 ---- vendor/oban_pro/lib/oban/pro/producer.ex | 358 -- vendor/oban_pro/lib/oban/pro/queue.ex | 157 - vendor/oban_pro/lib/oban/pro/refresher.ex | 111 - vendor/oban_pro/lib/oban/pro/relay.ex | 408 --- vendor/oban_pro/lib/oban/pro/stage.ex | 36 - vendor/oban_pro/lib/oban/pro/stages/chain.ex | 232 -- .../oban_pro/lib/oban/pro/stages/deadline.ex | 128 - .../oban_pro/lib/oban/pro/stages/encrypted.ex | 83 - .../oban_pro/lib/oban/pro/stages/executing.ex | 17 - vendor/oban_pro/lib/oban/pro/stages/hooks.ex | 147 - .../oban_pro/lib/oban/pro/stages/recorded.ex | 77 - .../lib/oban/pro/stages/structured.ex | 191 -- vendor/oban_pro/lib/oban/pro/testing.ex | 1334 -------- vendor/oban_pro/lib/oban/pro/unique.ex | 88 - vendor/oban_pro/lib/oban/pro/utils.ex | 159 - vendor/oban_pro/lib/oban/pro/uuidv7.ex | 42 - vendor/oban_pro/lib/oban/pro/validation.ex | 22 - vendor/oban_pro/lib/oban/pro/worker.ex | 1254 ------- vendor/oban_pro/lib/oban/pro/workers/batch.ex | 98 - vendor/oban_pro/lib/oban/pro/workers/chain.ex | 170 - vendor/oban_pro/lib/oban/pro/workers/chunk.ex | 752 ---- .../oban_pro/lib/oban/pro/workers/workflow.ex | 209 -- vendor/oban_pro/lib/oban/pro/workflow.ex | 3011 ----------------- .../oban_pro/lib/oban/pro/workflow/cascade.ex | 87 - vendor/oban_pro/lib/pro.ex | 3 - vendor/oban_pro/mix.exs | 203 -- 60 files changed, 31 insertions(+), 18611 deletions(-) create mode 100644 priv/repo/migrations/20260430142341_oban_pro_v1_7.exs delete mode 100644 vendor/oban_pro/.formatter.exs delete mode 100644 vendor/oban_pro/.hex delete mode 100644 vendor/oban_pro/hex_metadata.config delete mode 100644 vendor/oban_pro/lib/oban/pro/application.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/batch.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/cloud.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/cron.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/decorator.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/diagnostics.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/engines/smart.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/exceptions.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/flusher.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/handler.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/limiter.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/limiters/global.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/limiters/local.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/limiters/rate.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/migration.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/migrations/dynamic_partitioner.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/migrations/v1_0_0.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/migrations/v1_4_0.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/migrations/v1_5_0.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/migrations/v1_6_0.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/partition.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/plugins/dynamic_cron.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/plugins/dynamic_lifeline.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/plugins/dynamic_partitioner.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/plugins/dynamic_prioritizer.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/plugins/dynamic_pruner.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/plugins/dynamic_queues.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/plugins/dynamic_scaler.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/producer.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/queue.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/refresher.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/relay.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/stage.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/stages/chain.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/stages/deadline.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/stages/encrypted.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/stages/executing.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/stages/hooks.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/stages/recorded.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/stages/structured.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/testing.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/unique.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/utils.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/uuidv7.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/validation.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/worker.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/workers/batch.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/workers/chain.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/workers/chunk.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/workers/workflow.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/workflow.ex delete mode 100644 vendor/oban_pro/lib/oban/pro/workflow/cascade.ex delete mode 100644 vendor/oban_pro/lib/pro.ex delete mode 100644 vendor/oban_pro/mix.exs diff --git a/mix.exs b/mix.exs index 5e2278dc..cad61366 100644 --- a/mix.exs +++ b/mix.exs @@ -85,7 +85,10 @@ defmodule Microwaveprop.MixProject do {:styler, "~> 1.11", only: [:dev, :test], runtime: false}, {:oban, "~> 2.21"}, {:oban_met, "~> 1.0", path: "vendor/oban_met", override: true}, - {:oban_pro, "~> 1.6", path: "vendor/oban_pro"}, + # Oban Pro 1.7+ pulled from the licensed `oban` hex repo + # (configured via `mix hex.repo`). Pre-1.7 was vendored; + # vendor/oban_pro was removed when this bump landed. + {:oban_pro, "~> 1.7.0", repo: "oban"}, {:oban_web, "~> 2.12", path: "vendor/oban_web", override: true}, {:credo, "~> 1.7", only: [:dev, :test], runtime: false}, {:nx, "~> 0.9", only: [:dev, :test]}, diff --git a/mix.lock b/mix.lock index ae07c74d..498ad3ad 100644 --- a/mix.lock +++ b/mix.lock @@ -42,6 +42,7 @@ "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, "nx": {:hex, :nx, "0.11.0", "d37723dbd6cfa274a5def6d6664f5680c32e2eb8a1ce25ec6d91751967fa0abf", [:mix], [{:complex, "~> 0.6", [hex: :complex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "36157b21239aeb251d6cbac23eb0eb3495a5e1e0cbc2e6df16afd2ede1575205"}, "oban": {:hex, :oban, "2.22.0", "d50fb8be22a58ca8839dbc69610e271773d851b7deae01acd8b62e7197bcd318", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.20", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6675164c11eadd82044732dd7b6e448aa6d6f429a41a1e7d6dbf5fd358f52f26"}, + "oban_pro": {:hex, :oban_pro, "1.7.0", "e501c3957540dbd4b2d69c9045f04f4bd451d4f54d7cf185bf22c8c8cc56323c", [:mix], [{:ecto_sql, "~> 3.12", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:oban, "~> 2.21", [hex: :oban, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.22", [hex: :postgrex, repo: "hexpm", optional: true]}], "oban", "9e02b1abe356ee2c382d4a3606a59386158560c6ceab9aa95019d39363cde46a"}, "octo_fetch": {:hex, :octo_fetch, "0.5.0", "f50701568b9fc752656367f82cc134d5fbefff37c5a0e8ddfcceb02ceee3f5fc", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "6226cc3c14ca948ee9f25fb0446322e5c288e215da9beba7899b6b5f4cd3ccb0"}, "owl": {:hex, :owl, "0.13.0", "26010e066d5992774268f3163506972ddac0a7e77bfe57fa42a250f24d6b876e", [:mix], [{:ucwidth, "~> 0.2", [hex: :ucwidth, repo: "hexpm", optional: true]}], "hexpm", "59bf9d11ce37a4db98f57cb68fbfd61593bf419ec4ed302852b6683d3d2f7475"}, "peep": {:hex, :peep, "3.5.0", "9f6ead7b0f2c684494200c8fc02e7e62e8c459afe861b29bd859e4c96f402ed8", [:mix], [{:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:plug, "~> 1.16", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry_metrics, "~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "5a73a99c6e60062415efeb7e536a663387146463a3d3df1417da31fd665ac210"}, diff --git a/priv/repo/migrations/20260430142341_oban_pro_v1_7.exs b/priv/repo/migrations/20260430142341_oban_pro_v1_7.exs new file mode 100644 index 00000000..61b1100d --- /dev/null +++ b/priv/repo/migrations/20260430142341_oban_pro_v1_7.exs @@ -0,0 +1,26 @@ +defmodule Microwaveprop.Repo.Migrations.ObanProV17 do + @moduledoc """ + Oban Pro 1.7 schema migration. Per the v1.7 upgrade guide: + + * Adds the database-backed workflow tracking tables. + * Replaces the legacy generated `uniq_key` / `partition_key` + columns with expression indexes (faster + reclaims storage). + * Adds new partial indexes for improved query performance. + * Renames the legacy workflow indexes with an `_old` suffix — + a follow-up migration can drop them once we're confident the + new indexes are healthy. + + The 1.7 release docs note that for very large `oban_jobs` tables, + the schema and index work should be split into separate migrations + with DDL transactions disabled for the index half. Our prod + `oban_jobs` table holds at most a few hours of jobs (Pruner runs + daily, max_age 24h) so the inline default is fine — we're nowhere + near the size where Postgres would lock-storm on a regular index + CREATE. + """ + use Ecto.Migration + + def up, do: Oban.Pro.Migration.up(version: "1.7.0") + + def down, do: Oban.Pro.Migration.down(version: "1.7.0") +end diff --git a/vendor/oban_pro/.formatter.exs b/vendor/oban_pro/.formatter.exs deleted file mode 100644 index da6266a2..00000000 --- a/vendor/oban_pro/.formatter.exs +++ /dev/null @@ -1,14 +0,0 @@ -locals_without_parens = [ - args_schema: 1, - field: 2, - field: 3, - embeds_one: 2, - embeds_many: 2 -] - -[ - import_deps: [:ecto, :ecto_sql, :oban, :stream_data], - export: [locals_without_parens: locals_without_parens], - locals_without_parens: locals_without_parens, - inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] -] diff --git a/vendor/oban_pro/.hex b/vendor/oban_pro/.hex deleted file mode 100644 index ae7476956f352fe7b7fbbda2416b01bfe99780b9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 264 zcmZ9EL2|+{5Ck3UBAj>%B_ZufzU3#>YPE_@BAGG}anHviUvr!3>Y3kZH}$ojrg05Zt<`LDwo^3)b}4*@AFbNy4NQ>X;X()Zp#LSO{<4Yg+811-OyX@^ZdHj z<6MfY7oLQ_?$5m09|mFo?a)IS)fgOkA0lfz`h;hz6Fbk=j3SICokakn6C^-L2DRgQ tT%M2rr{IilLvN$ADiDEcJ|{M|Va7>,[]}. -{<<"name">>,<<"oban_pro">>}. -{<<"version">>,<<"1.6.14">>}. -{<<"description">>,<<"Oban Pro Component">>}. -{<<"elixir">>,<<"~> 1.15">>}. -{<<"files">>, - [<<"lib/pro.ex">>,<<"lib/oban">>,<<"lib/oban/pro">>, - <<"lib/oban/pro/diagnostics.ex">>,<<"lib/oban/pro/cron.ex">>, - <<"lib/oban/pro/decorator.ex">>,<<"lib/oban/pro/migrations">>, - <<"lib/oban/pro/migrations/v1_4_0.ex">>, - <<"lib/oban/pro/migrations/v1_5_0.ex">>, - <<"lib/oban/pro/migrations/v1_6_0.ex">>, - <<"lib/oban/pro/migrations/dynamic_partitioner.ex">>, - <<"lib/oban/pro/migrations/v1_0_0.ex">>,<<"lib/oban/pro/unique.ex">>, - <<"lib/oban/pro/batch.ex">>,<<"lib/oban/pro/producer.ex">>, - <<"lib/oban/pro/worker.ex">>,<<"lib/oban/pro/stages">>, - <<"lib/oban/pro/stages/chain.ex">>,<<"lib/oban/pro/stages/encrypted.ex">>, - <<"lib/oban/pro/stages/recorded.ex">>,<<"lib/oban/pro/stages/hooks.ex">>, - <<"lib/oban/pro/stages/deadline.ex">>, - <<"lib/oban/pro/stages/executing.ex">>, - <<"lib/oban/pro/stages/structured.ex">>,<<"lib/oban/pro/plugins">>, - <<"lib/oban/pro/plugins/dynamic_prioritizer.ex">>, - <<"lib/oban/pro/plugins/dynamic_pruner.ex">>, - <<"lib/oban/pro/plugins/dynamic_scaler.ex">>, - <<"lib/oban/pro/plugins/dynamic_partitioner.ex">>, - <<"lib/oban/pro/plugins/dynamic_lifeline.ex">>, - <<"lib/oban/pro/plugins/dynamic_queues.ex">>, - <<"lib/oban/pro/plugins/dynamic_cron.ex">>,<<"lib/oban/pro/limiters">>, - <<"lib/oban/pro/limiters/local.ex">>,<<"lib/oban/pro/limiters/rate.ex">>, - <<"lib/oban/pro/limiters/global.ex">>,<<"lib/oban/pro/queue.ex">>, - <<"lib/oban/pro/handler.ex">>,<<"lib/oban/pro/partition.ex">>, - <<"lib/oban/pro/cloud.ex">>,<<"lib/oban/pro/uuidv7.ex">>, - <<"lib/oban/pro/stage.ex">>,<<"lib/oban/pro/refresher.ex">>, - <<"lib/oban/pro/workflow.ex">>,<<"lib/oban/pro/testing.ex">>, - <<"lib/oban/pro/migration.ex">>,<<"lib/oban/pro/engines">>, - <<"lib/oban/pro/engines/smart.ex">>,<<"lib/oban/pro/workflow">>, - <<"lib/oban/pro/workflow/cascade.ex">>,<<"lib/oban/pro/limiter.ex">>, - <<"lib/oban/pro/validation.ex">>,<<"lib/oban/pro/flusher.ex">>, - <<"lib/oban/pro/workers">>,<<"lib/oban/pro/workers/chain.ex">>, - <<"lib/oban/pro/workers/batch.ex">>,<<"lib/oban/pro/workers/chunk.ex">>, - <<"lib/oban/pro/workers/workflow.ex">>,<<"lib/oban/pro/application.ex">>, - <<"lib/oban/pro/utils.ex">>,<<"lib/oban/pro/relay.ex">>, - <<"lib/oban/pro/exceptions.ex">>,<<".formatter.exs">>,<<"mix.exs">>]}. -{<<"app">>,<<"oban_pro">>}. -{<<"licenses">>,[<<"Commercial">>]}. -{<<"requirements">>, - [[{<<"name">>,<<"oban">>}, - {<<"app">>,<<"oban">>}, - {<<"optional">>,false}, - {<<"requirement">>,<<"~> 2.19">>}, - {<<"repository">>,<<"hexpm">>}], - [{<<"name">>,<<"ecto_sql">>}, - {<<"app">>,<<"ecto_sql">>}, - {<<"optional">>,false}, - {<<"requirement">>,<<"~> 3.10">>}, - {<<"repository">>,<<"hexpm">>}], - [{<<"name">>,<<"postgrex">>}, - {<<"app">>,<<"postgrex">>}, - {<<"optional">>,true}, - {<<"requirement">>,<<"~> 0.16">>}, - {<<"repository">>,<<"hexpm">>}]]}. -{<<"build_tools">>,[<<"mix">>]}. diff --git a/vendor/oban_pro/lib/oban/pro/application.ex b/vendor/oban_pro/lib/oban/pro/application.ex deleted file mode 100644 index 86d96fa1..00000000 --- a/vendor/oban_pro/lib/oban/pro/application.ex +++ /dev/null @@ -1,29 +0,0 @@ -defmodule Oban.Pro.Application do - @moduledoc false - - use Application - - @handlers [ - Oban.Pro.Batch, - Oban.Pro.Engines.Smart, - Oban.Pro.Migration, - Oban.Pro.Partition, - Oban.Pro.Relay, - Oban.Pro.Worker, - Oban.Pro.Workflow - ] - - @impl Application - def start(_type, _args) do - for handler <- @handlers, do: handler.on_start() - - children = [Oban.Pro.Diagnostics, Oban.Pro.Refresher] - - Supervisor.start_link(children, strategy: :one_for_one, name: __MODULE__) - end - - @impl Application - def stop(_state) do - for handler <- @handlers, do: handler.on_stop() - end -end diff --git a/vendor/oban_pro/lib/oban/pro/batch.ex b/vendor/oban_pro/lib/oban/pro/batch.ex deleted file mode 100644 index f63ed592..00000000 --- a/vendor/oban_pro/lib/oban/pro/batch.ex +++ /dev/null @@ -1,873 +0,0 @@ -defmodule Oban.Pro.Batch do - @moduledoc """ - Batches link the execution of many jobs as a group and runs optional callbacks after jobs are - processed. This allows your application to coordinate the execution of any number of jobs in - parallel. - - ## Usage - - Batches are composed of one or more Pro workers that are linked with a shared `batch_id` and - optional callbacks. As a simple example, let's define a worker that delivers daily emails: - - ```elixir - defmodule MyApp.EmailBatch do - use Oban.Pro.Worker, queue: :mailers - - @behaviour Oban.Pro.Batch - - @impl Oban.Pro.Worker - def process(%Job{args: %{"email" => email}}) do - MyApp.Mailer.daily_update(email) - end - - @impl Oban.Pro.Batch - def batch_completed(_job) do - Logger.info("BATCH COMPLETE") - - :ok - end - end - ``` - - Now, create a batch with `new/1` by passing a list of job changesets: - - ```elixir - emails - |> Enum.map(&MyApp.EmailBatch.new(%{email: &1})) - |> Batch.new() - |> Oban.insert_all() - ``` - - After all jobs in the batch are `completed`, the `batch_completed/1` callback will be - triggered. - - ## Handler Callbacks - - After jobs in the batch are processed, a callback job may be inserted. There are four optional - batch handler callbacks that a worker may define: - - | callback | enqueued after | - | --------------------- | -------------------------------------------------------- | - | `c:batch_attempted/1` | _all_ jobs `executed` at least once | - | `c:batch_cancelled/1` | the _first_ job is `cancelled` | - | `c:batch_completed/1` | _all_ jobs in the batch are `completed` | - | `c:batch_discarded/1` | the _first_ job is `discarded` | - | `c:batch_exhausted/1` | _all_ jobs are `completed`, `cancelled`, or `discarded` | - | `c:batch_retryable/1` | the _first_ job is `retryable` | - - Each callback runs in a separate, isolated job, so it may be retried or discarded like any other - job. The callback function receives an `Oban.Job` struct with the `batch_id` in `meta` and - should return `:ok` (or another valid `Worker.result()`). - - Here we'll implement each of the optional handler callbacks and have them print out the batch - status along with the `batch_id`: - - ```elixir - defmodule MyApp.BatchWorker do - use Oban.Pro.Worker - - @behaviour Oban.Pro.Batch - - @impl Oban.Pro.Worker - def process(_job), do: :ok - - @impl Oban.Pro.Batch - def batch_attempted(%Job{meta: %{"batch_id" => batch_id}}) do - IO.inspect({:attempted, batch_id}) - :ok - end - - @impl Oban.Pro.Batch - def batch_cancelled(%Job{meta: %{"batch_id" => batch_id}}) do - IO.inspect({:cancelled, batch_id}) - :ok - end - - @impl Oban.Pro.Batch - def batch_completed(%Job{meta: %{"batch_id" => batch_id}}) do - IO.inspect({:completed, batch_id}) - :ok - end - - @impl Oban.Pro.Batch - def batch_discarded(%Job{meta: %{"batch_id" => batch_id}}) do - IO.inspect({:discarded, batch_id}) - :ok - end - - @impl Oban.Pro.Batch - def batch_exhausted(%Job{meta: %{"batch_id" => batch_id}}) do - IO.inspect({:exhausted, batch_id}) - :ok - end - - @impl Oban.Pro.Batch - def batch_retryable(%Job{meta: %{"batch_id" => batch_id}}) do - IO.inspect({:retryable, batch_id}) - :ok - end - end - ``` - - ## Hybrid Batches - - Batches may also be built from a variety of workers, though you must provide an explicit worker - for callbacks: - - ```elixir - mail_jobs = Enum.map(mail_args, &MyApp.MailWorker.new/1) - push_jobs = Enum.map(push_args, &MyApp.PushWorker.new/1) - - [callback_worker: MyApp.CallbackWorker] - |> Batch.new() - |> Batch.add(mail_jobs) - |> Batch.add(push_jobs) - ``` - - Without an explicit `callback_worker`, any worker in the batch may be used for callbacks. That - makes callback handling unpredictable and unexpected. - - ## Customizing Batches - - Batches accept a variety of options for grouping and callback customization. - - ### Batch ID - - By default a `batch_id` is generated as a time-ordered random [UUIDv7][uuid]. UUIDs are - sufficient to ensure uniqueness between workers and nodes for any time period. However, if you - require control, you can override `batch_id` generation at the worker level or pass a value - directly to the `new/2` function. - - ```elixir - Batch.new(batch_id: "custom-batch-id") - ``` - - ### Batch Name - - Batches accept an optional name to describe the purpose of the batch, beyond the generated id or - individual jobs in it. While the `batch_id` must be unique, the `batch_name` doesn't have to be, - so it can be used as a general purpose label. - - ```elixir - Batch.new(batch_name: "nightly-etl") - ``` - - ### Callback Workers - - For some batches, notably those with heterogeneous jobs, it's necessary to specify a different - worker for callbacks. That is easily accomplished by passing the `:callback_worker` option to - `new/2`: - - ```elixir - Batch.new(callback_worker: MyCallbackWorker) - ``` - - The callback worker **must** be an `Oban.Pro.Worker` that defines one or more of the batch callback - handlers. - - ### Callback Options - - By default, callback jobs have an empty `args` map and inherit other options from batch jobs. - With `callback_opts`, you can set standard job options for batch callback jobs (only `args`, - `max_attempts`, `meta`, `priority`, `queue`, and `tags` are allowed). - - For example, here we're passing a webhook URLs as `args`: - - ```elixir - Batch.new(callback_opts: [args: %{webhook: "https://web.hook"}]) - ``` - - Here, we change the callback queue and knock the priority down: - - ```elixir - Batch.new(callback_opts: [queue: :callbacks, priority: 9]) - ``` - - Be careful to minimize `callback_opts` as they are stored in each batch job's meta. - - ## Fetching Batch Jobs - - To pull more context from batch jobs, it's possible to load all jobs from the batch with - `all_jobs/2` and `stream_jobs/2`. The functions take a single batch job and returns a list or - stream of all non-callback jobs in the batch, which you can then operate on with `Enum` or - `Stream` functions. - - As an example, imagine you have a batch that ran for a few thousand accounts and you'd like to - notify admins that the batch is complete. - - ```elixir - defmodule MyApp.BatchWorker do - use Oban.Pro.Worker - - @behaviour Oban.Pro.Batch - - @impl Oban.Pro.Batch - def batch_completed(%Job{} = job) do - {:ok, account_ids} = - job - |> Oban.Pro.Batch.all_jobs() - |> Enum.map(& &1.args["account_id"]) - - account_ids - |> MyApp.Accounts.all() - |> MyApp.Mailer.notify_admins_about_batch() - end - ``` - - Fetching a thousand jobs may be alright, but for larger batches you don't want to load that much - data into memory. Instead, you can use `stream_jobs` to iterate through them lazily: - - ```elixir - {:ok, account_ids} = - MyApp.Repo.transaction(fn -> - job - |> Batch.stream_jobs() - |> Stream.map(& &1.args["account_id"]) - |> Enum.to_list() - end) - ``` - - Streaming is provided by Ecto's `Repo.stream`, and it must take place within a transaction. - While it may be overkill for small batches, for batches with tens or hundreds of thousands of - jobs, it will prevent massive memory spikes or the database grinding to a halt. - - [uuid]: https://datatracker.ietf.org/doc/html/draft-peabody-dispatch-new-uuid-format-04#section-5.2 - """ - - @behaviour Oban.Pro.Flusher - @behaviour Oban.Pro.Handler - - import Ecto.Query, only: [limit: 2, order_by: 2, select: 3, union_all: 2, where: 3] - - alias Ecto.Changeset - alias Oban.Pro.Engines.Smart - alias Oban.{Job, Repo, Validation, Worker} - alias Oban.Pro.Workflow - - require Logger - - @type changeset :: Job.changeset() - - @type callback_opts :: [ - args: Job.args(), - max_attempts: pos_integer(), - meta: map(), - priority: 0..9, - queue: atom() | String.t(), - tags: Job.tags() - ] - - @type batch_opts :: [ - batch_id: String.t(), - batch_name: String.t(), - callback_opts: callback_opts(), - callback_worker: module() - ] - - @type repo_opts :: [timeout: timeout()] - - @type t :: %__MODULE__{changesets: Enumerable.t(changeset), opts: map()} - - @callbacks_to_functions %{ - "attempted" => :batch_attempted, - "cancelled" => :batch_cancelled, - "completed" => :batch_completed, - "discarded" => :batch_discarded, - "exhausted" => :batch_exhausted, - "retryable" => :batch_retryable - } - - @callbacks_to_deprecated %{ - "attempted" => :handle_attempted, - "cancelled" => :handle_cancelled, - "completed" => :handle_completed, - "discarded" => :handle_discarded, - "exhausted" => :handle_exhausted, - "retryable" => :handle_retryable - } - - @callbacks_to_combined Map.merge( - @callbacks_to_functions, - @callbacks_to_deprecated, - fn _k, new, old -> [new, old] end - ) - - @callbacks_to_states %{ - "attempted" => ~w(scheduled available executing), - "completed" => ~w(scheduled available executing retryable cancelled discarded), - "cancelled" => ~w(cancelled), - "discarded" => ~w(discarded), - "exhausted" => ~w(scheduled retryable available executing), - "retryable" => ~w(retryable) - } - - @all_states Enum.map(Job.states(), &to_string/1) - - @doc """ - Called after all jobs in the batch were attempted at least once. - - If a `batch_attempted/1` function is defined it is executed by an isolated callback job. - - ## Example - - Print when all jobs were attempted: - - def batch_attempted(%Job{meta: %{"batch_id" => batch_id}}) do - IO.inspect(batch_id, label: "Attempted") - end - """ - @callback batch_attempted(job :: Job.t()) :: Worker.result() - - @doc """ - Called after any jobs in the batch have a `cancelled` state. - - If a `batch_cancelled/1` function is defined it is executed by an isolated callback job. - - ## Example - - Print when any jobs are discarded: - - def batch_cancelled(%Job{meta: %{"batch_id" => batch_id}}) do - IO.inspect(batch_id, label: "Cancelled") - end - """ - @callback batch_cancelled(job :: Job.t()) :: Worker.result() - - @doc """ - Called after all jobs in the batch have a `completed` state. - - If a `batch_completed/1` function is defined it is executed by an isolated callback job. - - ## Example - - Print when all jobs are completed: - - def batch_completed(%Job{meta: %{"batch_id" => batch_id}}) do - IO.inspect(batch_id, label: "Completed") - end - """ - @callback batch_completed(job :: Job.t()) :: Worker.result() - - @doc """ - Called after any jobs in the batch have a `discarded` state. - - If a `batch_discarded/1` function is defined it is executed by an isolated callback job. - - ## Example - - Print when any jobs are discarded: - - def batch_discarded(%Job{meta: %{"batch_id" => batch_id}}) do - IO.inspect(batch_id, label: "Discarded") - end - """ - @callback batch_discarded(job :: Job.t()) :: Worker.result() - - @doc """ - Called after all jobs in the batch have either a `cancelled`, `completed` or `discarded` state. - - If a `batch_exhausted/1` function is defined it is executed by an isolated callback job. - - ## Example - - Print when all jobs are completed or discarded: - - def batch_exhausted(%Job{meta: %{"batch_id" => batch_id}}) do - IO.inspect(batch_id, label: "Exhausted") - end - """ - @callback batch_exhausted(job :: Job.t()) :: Worker.result() - - @doc """ - Called after any jobs in the batch have a `retryable` state. - - If a `batch_retryable/1` function is defined it is executed by an isolated callback job. - - ## Example - - Print when any jobs are retryable: - - def batch_retryable(%Job{meta: %{"batch_id" => batch_id}}) do - IO.inspect(batch_id, label: "Retryable") - end - """ - @callback batch_retryable(job :: Job.t()) :: Worker.result() - - @optional_callbacks batch_attempted: 1, - batch_cancelled: 1, - batch_completed: 1, - batch_discarded: 1, - batch_exhausted: 1, - batch_retryable: 1 - - defstruct changesets: [], opts: [] - - defguardp is_list_or_stream(enum) when is_list(enum) or is_struct(enum, Stream) - - # Handler Callbacks - - @impl Oban.Pro.Handler - def on_start do - events = [ - [:oban, :engine, :cancel_all_jobs, :stop] - ] - - :telemetry.attach_many("oban.batch", events, &__MODULE__.handle_event/4, nil) - end - - @impl Oban.Pro.Handler - def on_stop do - :telemetry.detach("oban.batch") - end - - @doc false - def handle_event(_event, _timing, %{conf: conf, jobs: jobs}, _) do - for %{meta: %{"batch_id" => batch_id}} = job <- jobs do - on_flush(job, batch_id, conf) - end - end - - # Constants - - @doc false - def callbacks_to_functions, do: @callbacks_to_functions - - @doc false - def callbacks_to_deprecated, do: @callbacks_to_deprecated - - # Public Functions - - @doc false - def new, do: new([], []) - - @doc false - def new([%Changeset{} | _] = changesets), do: new(changesets, []) - def new(%Stream{} = changesets), do: new(changesets, []) - def new(opts) when is_list(opts), do: new([], opts) - - @doc """ - Build a new batch from a list or stream of job changesets and some options. - - ## Examples - - Build an empty batch without any jobs or options: - - Batch.new() - - Build a batch from a list of changesets: - - 1..3 - |> Enum.map(fn id -> MyWorker.new(%{id: id}) end) - |> Batch.new() - - Build a batch from a stream: - - stream_of_args - |> Stream.map(&MyWorker.new/1) - |> Batch.new() - - Build a batch with callback options: - - Batch.new(list_of_jobs, batch_id: "abc-123", callback_worker: MyApp.CallbackWorker) - """ - @spec new(Enumerable.t(changeset()), batch_opts()) :: t() - def new(changesets, opts) when is_list_or_stream(changesets) and is_list(opts) do - validate!(opts) - - opts = - opts - |> Keyword.put(:batch, true) - |> Keyword.put_new(:callback_opts, []) - |> Keyword.update!(:callback_opts, &Map.new/1) - |> Keyword.put_new_lazy(:batch_id, &Oban.Pro.UUIDv7.generate/0) - |> Map.new() - - changesets = Stream.map(changesets, &put_meta(&1, opts)) - - %__MODULE__{changesets: changesets, opts: opts} - end - - defp put_meta(changeset, opts) do - meta = - changeset - |> Changeset.get_change(:meta, %{}) - |> Map.merge(opts) - - Changeset.put_change(changeset, :meta, meta) - end - - defp validate!(opts) do - Validation.validate_schema!(opts, - batch_id: :string, - batch_name: :string, - callback_opts: :list, - callback_worker: :module - ) - end - - @doc """ - Add one or more jobs to a batch. - - ## Examples - - Add jobs to an existing batche: - - Batch.add(batch, Enum.map(args, &MyWorker.new/1)) - - Add jobs to a batch one at a time: - - Batch.new() - |> Batch.add(MyWorker.new(%{})) - |> Batch.add(MyWorker.new(%{})) - |> Batch.add(MyWorker.new(%{})) - """ - @spec add(t(), Enumerable.t(changeset())) :: t() - def add(batch, %Changeset{} = changeset), do: add(batch, [changeset]) - - def add(%__MODULE__{} = batch, changesets) when is_list_or_stream(changesets) do - changesets = Stream.map(changesets, &put_meta(&1, batch.opts)) - - %{batch | changesets: Stream.concat(batch.changesets, changesets)} - end - - @doc """ - Append to a batch from an existing batch job. - - The `batch_id` and any other batch options are propagated to the newly created batch. - - > #### Appending and Callbacks {: .warning} - > - > Batch callback jobs are _only inserted once_. Appending to a batch where callbacks were already - > triggered, e.g. `batch_completed`, won't re-trigger the callback. - - ## Examples - - Build an empty batch from an existing batch job: - - job - |> Batch.append() - |> Batch.add(MyWorker.new(%{})) - |> Batch.add(MyWorker.new(%{})) - |> Oban.insert_all() - - Build a batch from an existing job with overriden options: - - Batch.append(job, callback_worker: MyApp.OtherWorker) - """ - @spec append(Job.t(), batch_opts()) :: t() - def append(%Job{meta: %{"batch_id" => _} = meta}, opts \\ []) when is_list(opts) do - orig_opts = - for {key, val} <- meta, - key in ~w(batch_id batch_name callback_opts callback_worker), - do: {String.to_existing_atom(key), val} - - orig_opts - |> Keyword.merge(opts) - |> new() - end - - @doc """ - Get all non-callback jobs from a batch. - - ## Examples - - Fetch results from all jobs from within a `batch_completed/1` callback: - - def batch_completed(%Job{} = job) do - results = - job - |> Batch.all_jobs() - |> Enum.map(&fetch_recorded/1) - - ... - end - - Get all batch jobs from anywhere using the `batch_id`: - - Batch.all_jobs("some-uuid-1234-5678") - - Get all jobs from anywhere with a custom Oban instance name: - - Batch.all_jobs(MyApp.Oban, "some-uuid-1234-5678") - """ - @spec all_jobs(Oban.name(), Job.t() | String.t(), [repo_opts()]) :: [Job.t()] - def all_jobs(name \\ Oban, job_or_batch_id, opts \\ []) - - def all_jobs(_name, %Job{conf: conf, meta: %{"batch_id" => batch_id}}, opts) do - Repo.all(conf, batch_query(batch_id), opts) - end - - def all_jobs(name, batch_id, opts) when is_binary(batch_id) do - name - |> Oban.config() - |> Repo.all(batch_query(batch_id), opts) - end - - def all_jobs(%Job{} = job, opts, []), do: all_jobs(Oban, job, opts) - - @doc """ - Cancel all non-callback jobs in a batch. - - Cancelling jobs via this function bulk-updates their state in the database without loading them - into memory or resolving their workers. As a result, **worker `after_process/3` hooks are not - called** for cancelled jobs. If you need to execute cleanup logic or other side effects when jobs - are cancelled, consider using batch callbacks like `batch_cancelled/1` or implement cleanup logic - at the application level. - - ## Examples - - Cancel jobs with the `job` in a `process/1` function: - - def process(job) do - if should_stop_processing?(job.args) do - Batch.cancel_jobs(job) - else - ... - end - end - - Cancel jobs from anywhere using the `batch_id`: - - Batch.cancel_jobs("some-uuid-1234-5678") - - Cancel jobs from anywhere with a custom Oban instance name: - - Batch.cancel_jobs(MyApp.Oban, "some-uuid-1234-5678") - """ - @spec cancel_jobs(Oban.name(), Job.t() | String.t()) :: {:ok, non_neg_integer()} - def cancel_jobs(oban_name \\ Oban, job_or_batch_id) - - def cancel_jobs(_oban_name, %Job{conf: conf, meta: %{"batch_id" => batch_id}}) do - cancel_jobs(conf.name, batch_id) - end - - def cancel_jobs(oban_name, batch_id) when is_binary(batch_id) do - Oban.cancel_all_jobs(oban_name, batch_query(batch_id)) - end - - @doc """ - Create a batch from a workflow. - - All jobs in the workflow are augmented to also be part of a batch. - - ## Examples - - Create a batch from a workflow without any extra options: - - Workflow.new() - |> Workflow.add(:step_1, MyWorker.new(%{id: 123})) - |> Workflow.add(:step_2, MyWorker.new(%{id: 345}), deps: [:step_1]) - |> Workflow.add(:step_3, MyWorker.new(%{id: 456}), deps: [:step_2]) - |> Batch.from_workflow() - |> Oban.insert_all() - - Create a batch with callback options: - - Batch.from_workflow(workflow, callback_opts: [priority: 3], callback_worker: MyApp.Worker) - """ - @spec from_workflow(Workflow.t(), batch_opts()) :: t() - def from_workflow(%Workflow{changesets: changesets}, opts \\ []) do - new(changesets, opts) - end - - @doc """ - Stream all non-callback jobs from a batch. - - ## Examples - - Stream all batch jobs from within a `batch_completed/1` callback: - - def batch_completed(%Job{} = job) do - {:ok, account_ids} = - MyApp.Repo.transaction(fn -> - job - |> Batch.stream_jobs() - |> Enum.map(& &1.args["account_id"]) - end) - - # use_account_ids - end - - Stream all batch jobs from anywhere using the `batch_id`: - - MyApp.Repo.transaction(fn -> - "some-uuid-1234-5678" - |> Batch.stream_jobs() - |> Enum.map(& &1.args["account_id"]) - end) - - Stream all batch jobs using a custom Oban instance name: - - MyApp.Repo.transaction(fn -> - MyApp.Oban - |> Batch.stream_jobs("some-uuid-1234-5678") - |> Enum.map(& &1.args["account_id"]) - end) - """ - @spec stream_jobs(Oban.name(), Job.t() | String.t(), [repo_opts()]) :: Enum.t() - def stream_jobs(name \\ Oban, job_or_batch_id, opts \\ []) - - def stream_jobs(_name, %Job{conf: conf, meta: %{"batch_id" => batch_id}}, opts) do - Repo.stream(conf, batch_query(batch_id), opts) - end - - def stream_jobs(name, batch_id, opts) when is_binary(batch_id) do - name - |> Oban.config() - |> Repo.stream(batch_query(batch_id), opts) - end - - def stream_jobs(%Job{} = job, opts, []), do: stream_jobs(Oban, job, opts) - - defp batch_query(batch_id) do - Job - |> where([j], fragment("? \\? 'batch_id'", j.meta)) - |> where([j], fragment("? ->> 'batch_id'", j.meta) == ^batch_id) - |> where([j], is_nil(fragment("? ->> 'callback'", j.meta))) - |> where([j], j.state in @all_states) - |> order_by(asc: :id) - end - - # Event Handling - - @doc false - @impl Oban.Pro.Flusher - def to_flush_mfa(job, conf) do - case job.meta do - %{"batch_id" => _, "callback" => _} -> :ignore - %{"batch_id" => batch_id} -> {__MODULE__, :on_flush, [job, batch_id, conf]} - _ -> :ignore - end - end - - @doc false - def on_flush(job, batch_id, conf) do - batch_worker = job.meta["batch_callback_worker"] || job.meta["callback_worker"] || job.worker - - with {:ok, worker} <- Worker.from_string(batch_worker), - supported = supported_callbacks(worker), - {:ok, {states, exists}} <- states_for_callbacks(supported, batch_id, conf) do - for callback <- supported, - callback not in exists, - callback_ready?(callback, states) do - insert_callback(callback, worker, job, conf) - end - end - end - - defp supported_callbacks(worker) do - for {name, [new, old]} <- @callbacks_to_combined, - function_exported?(worker, new, 1) or function_exported?(worker, old, 1), - do: name - end - - defp states_for_callbacks([], _batch_id, _conf), do: :ok - - defp states_for_callbacks(callbacks, batch_id, conf) do - state_query = - callbacks - |> Enum.flat_map(&Map.fetch!(@callbacks_to_states, &1)) - |> Enum.uniq() - |> Enum.reduce(:none, fn state, acc -> - query = - Job - |> select([_], [type(^state, :string)]) - |> where([j], fragment("? \\? 'batch_id'", j.meta)) - |> where([j], fragment("? ->> 'batch_id'", j.meta) == ^batch_id) - |> where([j], is_nil(fragment("? ->> 'callback'", j.meta))) - |> where([j], j.state == ^state) - |> limit(1) - - if acc == :none, do: query, else: union_all(acc, ^query) - end) - - exist_query = - Enum.reduce(callbacks, :none, fn callback, acc -> - query = - Job - |> select([_j], [type(^callback, :string)]) - |> where([j], fragment("? \\? 'batch_id'", j.meta)) - |> where([j], fragment("? ->> 'batch_id'", j.meta) == ^batch_id) - |> where([j], fragment("? ->> 'callback'", j.meta) == ^callback) - |> where([j], j.state in @all_states) - - if acc == :none, do: query, else: union_all(acc, ^query) - end) - - Repo.transaction(conf, fn -> - states = conf |> Repo.all(state_query) |> List.flatten() - exists = conf |> Repo.all(exist_query) |> List.flatten() - - {states, exists} - end) - end - - defp callback_ready?(callback, batch_states) do - ready? = - @callbacks_to_states - |> Map.fetch!(callback) - |> Enum.any?(&(&1 in batch_states)) - - # Other callbacks use a negated query to avoid counting `completed` jobs. - if callback in ~w(cancelled discarded retryable) do - ready? - else - not ready? - end - end - - defp insert_callback(callback, worker, job, conf) do - call_opts = - job.meta - |> Map.get("callback_opts", %{}) - |> Map.put_new("args", Map.get(job.meta, "batch_callback_args", %{})) - |> Map.put_new("meta", Map.get(job.meta, "batch_callback_meta", %{})) - |> Map.put_new("queue", Map.get(job.meta, "batch_callback_queue", job.queue)) - - unique = [ - period: :infinity, - fields: [:worker, :queue, :meta], - keys: [:batch_id, :callback], - states: Job.states() - ] - - {args, call_opts} = Map.pop(call_opts, "args") - {meta, call_opts} = Map.pop(call_opts, "meta") - - xtra_meta = - job.meta - |> Map.take(~w(batch_id batch_name)) - |> Map.put("callback", callback) - |> Map.merge(meta) - - opts = - call_opts - |> Keyword.new(fn {key, val} -> {String.to_existing_atom(key), val} end) - |> Keyword.put(:meta, xtra_meta) - |> Keyword.put(:unique, unique) - - changeset = worker.new(args, opts) - - if not changeset.valid? and Keyword.has_key?(changeset.errors, :args) do - changeset - |> structured_error_message() - |> Logger.error() - else - {:ok, Smart.insert_job(conf, changeset, [])} - end - end - - defp structured_error_message(changeset) do - """ - [Oban.Pro.Batch] can't insert batch callback because it has invalid keys: - - #{get_in(changeset.errors, [:args, Access.elem(0)])} - - Use one of the following options to restore batch callbacks: - - * Modify structured `keys` or `required` to allow the missing keys - * Include the required arguments with the `batch_callback_args` option - * Specify a different callback worker with the `batch_callback_worker` option - """ - end -end diff --git a/vendor/oban_pro/lib/oban/pro/cloud.ex b/vendor/oban_pro/lib/oban/pro/cloud.ex deleted file mode 100644 index 0d8c2b3b..00000000 --- a/vendor/oban_pro/lib/oban/pro/cloud.ex +++ /dev/null @@ -1,23 +0,0 @@ -defmodule Oban.Pro.Cloud do - @moduledoc """ - A behaviour for interacting with cloud hosting providers. - """ - - @type conf :: term() - @type opts :: keyword() - @type quantity :: non_neg_integer() - - @doc """ - Executed once at runtime to gather, normalize, and transform options. - """ - @callback init(opts()) :: conf() - - @doc """ - Called to horizontally scale a cloud resource up or down. - - Successful scaling requests must return a new `conf` to be used during the next call to - `scale/2`. That allows cloud modules to track responses for additional control and - introspection. - """ - @callback scale(quantity(), conf()) :: {:ok, conf()} | {:error, term()} -end diff --git a/vendor/oban_pro/lib/oban/pro/cron.ex b/vendor/oban_pro/lib/oban/pro/cron.ex deleted file mode 100644 index 1f2c686c..00000000 --- a/vendor/oban_pro/lib/oban/pro/cron.ex +++ /dev/null @@ -1,125 +0,0 @@ -defmodule Oban.Pro.Cron do - @moduledoc false - - use Ecto.Schema - - import Ecto.Changeset - - alias Oban.Cron.Expression - alias Oban.Worker - - @primary_key {:name, :string, autogenerate: false} - schema "oban_crons" do - field :expression, :string - field :worker, :string - field :opts, :map - field :paused, :boolean - field :insertions, {:array, :utc_datetime_usec} - field :lock_version, :integer, default: 1 - field :parsed, :any, virtual: true - - timestamps( - inserted_at: :inserted_at, - updated_at: :updated_at, - type: :utc_datetime_usec - ) - end - - @permitted ~w(name expression insertions worker opts paused)a - @requried ~w(name expression worker)a - @allowed_opts ~w(args guaranteed max_attempts meta priority queue tags timezone)a - - @doc false - def allowed_opts, do: @allowed_opts - - @spec changeset({binary(), module()} | {binary(), module(), Keyword.t()}) :: Ecto.Changeset.t() - def changeset({expression, worker}) do - params = %{expression: expression, name: worker, worker: worker, opts: %{}} - - changeset(%__MODULE__{}, params) - end - - def changeset({expression, worker, opts}) do - {name, opts} = Keyword.pop(opts, :name, worker) - {paused, opts} = Keyword.pop(opts, :paused) - {insertions, opts} = Keyword.pop(opts, :insertions) - - params = %{ - expression: expression, - insertions: insertions, - name: name, - opts: Map.new(opts), - paused: paused, - worker: worker - } - - changeset(%__MODULE__{}, params) - end - - @spec changeset({Ecto.Schema.t(), map()}) :: Ecto.Changeset.t() - def changeset(schema, params) when is_map(params) do - params = - params - |> coerce_name(:name) - |> coerce_name(:worker) - |> merge_opts(schema) - - schema - |> cast(params, @permitted) - |> validate_required(@requried) - |> validate_change(:expression, &expression_validator/2) - |> validate_change(:opts, &opts_validator/2) - |> optimistic_lock(:lock_version) - end - - def update_changeset(schema, params) when is_map(params) do - if Map.has_key?(params, :expression) or Map.has_key?(params, :timezone) do - changeset(schema, Map.put(params, :insertions, [])) - else - changeset(schema, params) - end - end - - defp coerce_name(params, key) do - case params do - %{^key => value} when is_atom(value) -> - Map.put(params, key, Worker.to_string(value)) - - _ -> - params - end - end - - defp merge_opts(params, schema) do - case Map.split(params, @allowed_opts) do - {opts, params} when map_size(opts) > 0 -> - opts = Map.new(opts, fn {key, val} -> {to_string(key), val} end) - - Map.put(params, :opts, Map.merge(schema.opts, opts)) - - _ -> - params - end - end - - # Validators - - defp expression_validator(:expression, expression) do - Expression.parse!(expression) - - [] - rescue - ArgumentError -> - [expression: "expected cron expression to be a parsable binary"] - end - - defp opts_validator(:opts, opts) do - string_keys = Enum.map(@allowed_opts, &to_string/1) - - if Enum.all?(opts, fn {key, _} -> to_string(key) in string_keys end) do - [] - else - [opts: "expected cron opts to be one of #{inspect(@allowed_opts)}"] - end - end -end diff --git a/vendor/oban_pro/lib/oban/pro/decorator.ex b/vendor/oban_pro/lib/oban/pro/decorator.ex deleted file mode 100644 index d87ca6d4..00000000 --- a/vendor/oban_pro/lib/oban/pro/decorator.ex +++ /dev/null @@ -1,587 +0,0 @@ -defmodule Oban.Pro.Decorator do - @moduledoc """ - The `Decorator` extension converts functions into Oban jobs with a simple annotation. - - Decorated functions, such as those in contexts or other non-worker modules, can be executed as - fully fledged background jobs with retries, scheduling, and the other guarantees you'd expect - from Oban jobs. - - ## Usage - - To get started, use the `Decorator` module, then annotate functions with the `@job` attribute. - Here's a simple example that uses `@job true` to decorate a function without any options: - - ```elixir - defmodule MyApp.Business do - use Oban.Pro.Decorator - - @job true - def weekly_report(tps_id, opts) do - ... - end - end - ``` - - The `@job` attribute also accepts all standard `Oban.Worker` options, e.g. `max_attempts`, - `priority`, `queue`. This example swaps out `@job true` for a list of options: - - ```elixir - @job [max_attempts: 3, priority: 1, queue: :reports, recorded: true] - def weekly_report(tps_id, opts) do - ... - end - ``` - - Now you can build and insert a job by calling `insert_weekly_report/2`: - - ```elixir - {:ok, job} = MyApp.Business.insert_weekly_report(123, pdf: true, rtf: true) - ``` - - Notice that the second argument is a keyword list of options, which isn't normally allowed in - job args because it's not JSON serializable. - - ## Generated Functions - - Functions decorated with `@job` generate three variants of the original function: - - * `new_*` — builds an `Oban.Job` changeset ready to be inserted for execution - * `insert_*` — builds and inserts an `Oban.Job` using `Oban.insert/2` - * `relay_*` — inserts a job, awaits execution, then returns the job's results - - See the `t:comp_opts/0` typespec for the subset of job options that are supported at compile - time. Additional options are available at runtime, as described in the [Runtime - Options](#module-runtime-options) section below. - - ### Using New - - The `new_` variant will build an `Oban.Job` changeset that's ready for insertion, but not - persisted to the database. This is identical to the output from calling `c:Oban.Worker.new/2` on - a standard worker. - - ```elixir - changeset = Business.new_weekly_report(123) - ``` - - The returned changeset is perfect for bulk inserts via `Oban.insert_all/1`: - - ```elixir - [123, 456, 789] - |> Enum.map(&Business.new_weekly_report/1) - |> Oban.insert_all() - ``` - - It can also be used to compose batches or workflows: - - ```elixir - alias Oban.Pro.Workflow - - Workflow.new() - |> Workflow.add(:rep_1, Business.new_weekly_report(1)) - |> Workflow.add(:rep_2, Business.new_weekly_report(2)) - |> Workflow.add(:rep_3, Business.new_weekly_report(3)) - |> Workflow.add(:fin, Business.new_finish_up(), deps: ~w(rep_1 rep_2 rep_3)a) - |> Oban.insert_all() - ``` - - ### Using Insert - - The `insert_` variant builds a changeset and immediately calls `Oban.insert/3` to enqueue it. - The result is a success tuple containing the job, or a changeset with errors. - - ```elixir - {:ok, job} = Business.insert_weekly_report(123) - ``` - - By default, jobs are inserted using the standard `Oban` instance. For applications that run - multiple Oban instances, or use a non-standard name, you can override the instance with the - `:oban` option: - - ```elixir - {:ok, job} = Business.insert_weekly_report(123, oban: SomeOban) - ``` - - ### Using Relay - - The `relay_` variant builds a changeset, inserts it, then leverages `Oban.Pro.Relay` to await - execution and return a result: - - ```elixir - {:ok, result} = Business.relay_weekly_report(123) - ``` - - The default timeout is a brief 5000ms, which doesn't account for scheduling or queueing time. - Provide an alternate timeout to wait longer: - - ```elixir - case Business.relay_weekly_report(123, timeout: 30_000) do - {:ok, result} -> IO.inspect(result, label: "RESULT") - {:error, reason} -> IO.inspect(reason, label: "ERROR") - end - ``` - - Note that the `timeout` option only controls how long the local process will block while - awaiting a result. The job will keep executing regardless of the timeout period. - - > #### Considerations and Caveats {: .info} - > - > Decorated functions are a convenient way to run functions in the background, and suitable in many - > situations. However, there are circumstances where they're unsuitable and you should exercise - > care: - > - > * Advanced worker functionality such as custom backoffs, hooks, structured args, or - > callbacks requires a dedicated worker module and isn't suitable for decoration. - > - > * Args of any type are safely serialized, but dumping large amounts of data may cause - > performance problems because it must be serialized, stored, and deserialized. - > - > * Changing function signatures while jobs are in-flight can cause jobs to fail, just like - > changing the shape of args passed to a `process/1` callback. - - ## Runtime Options - - Each decorated function has an additional generated clause that accepts job options, e.g. - `new_weekly_report/1` also has a `new_weekly_report/2` variant. - - All compile time options can be overridden at runtime. For example, to override the `queue` and - `max_attempts`: - - ```elixir - Business.insert_weekly_report(123, queue: "default", max_attempts: 10) - ``` - - In addition to compile time options, runtime options accepted by `Oban.Job.new/2` (other than - `worker`) are also allowed. This makes it possible to schedule decorated jobs: - - ```elixir - Business.insert_weekly_report(123, schedule_in: {1, :minute}) - ``` - - See `t:full_opts/0` for the complete typespec of runtime options. - - ## Patterns and Guards - - Each generated variant retains pattern matches and guards from the original function. That - allows expressive, defensive data validation before a job executes. - - For example, use a guard to ensure the `id` argument is an integer: - - ```elixir - @job true - def notify_user(id) when is_integer(id), do: ... - ``` - - Or, pattern match on a map with a `user_id` key and ensure the `id` is an integer: - - ```elixir - @job true - def notify_user(%{user_id: id}) when is_integer(id), do: ... - ``` - - > #### Multiple Function Clauses Not Supported {: .warning} - > - > The `@job` decorator only works with single-clause functions. When decorating a function - > with multiple clauses, only the first clause is captured. Subsequent clauses are ignored for - > decoration, which will cause `FunctionClauseError` when calling the decorated variants. - > - > Instead of using multiple clauses, use a single function clause with pattern matching inside: - > - > ```diff - > @job true - > -def process_data(1, 2), do: :special_case - > -def process_data(a, b), do: {:ok, a + b} - > +def process_data(a, b) do - > + case {a, b} do - > + {1, 2} -> :special_case - > + {a, b} -> {:ok, a + b} - > + end - > +end - > ``` - - ## Complex Types - - Any Elixir term may be passed as an argument, not just JSON encodable values. That enables - passing native data-types such as tuples, keywords, or structs that can't easily be used in - regular jobs. - - ```elixir - @job true - def add_discount(%User{id: _}, amount: 10_000), do: ... - ``` - - > #### Avoid Non-Portable Types {: .warning} - > - > Pass non-portable data types such as `pid`, `reference`, `port` with caution. There's no - > guarantee that a job will run on the same node and those specific values available. - > Furthermore, be careful passing anonymous functions because they are closures over the local - > environment. - - ## Return Values - - Decorated jobs respect the same standard return types as `c:Oban.Pro.Worker.process/1`. That - means you can return an `{:error, reason}` tuple to signify an error, or `{:cancel, reason}` to - quietly cancel a job. However, because decorated functions weren't necessarily designed to be - executed in a job, unexpected returns are considered a success. - - While there's no harm in returning `nil`, a struct, or some other non-standard value, it's best - to return an explicitly support a type such as `:ok` or `{:ok, value}`. - - ```diff - @job true - def update_account(user_id, params) do - user = Repo.get(User, user_id) - - do_update(user, params) - - - user - + {:ok, user} - end - ``` - - ## Unique and Replace - - Unique and replace options are available for decorated jobs. However, they have purposeful - limitations for compatibility with the decorated worker: - - * `unique` — only supports `states`, `period`, and `timestamp` options. The `fields` and `keys` - options aren't supported. - * `replace` — expects the newer, per-state syntax, and it doesn't support replacing the `worker` - or `args`. - - Both options can be defined at compilie in the `@job` annotation: - - ```elixir - @job [unique: [period: :infinity], replace: [scheduled: [:scheduled_at]]] - ``` - - Or as runtime options: - - ```elixir - Business.send_notice(user, schedule_in: 60, replace: [scheduled: [:scheduled_at]]) - ``` - - ## Limiting Decoration - - To avoid generating unnecessary functions you can disable generating `new`, `insert`, or `relay` - functions via passing flags to `use`: - - ```elixir - use Oban.Pro.Decorator, new: false - use Oban.Pro.Decorator, insert: false - use Oban.Pro.Decorator, relay: false - ``` - - Filtering options can be combined to restrict generation to one variant. For example, to only - generate `insert_*` functions: - - ```elixir - use Oban.Pro.Decorator, new: false, relay: false - ``` - - ## Testing Decorated Jobs - - Testing decorated jobs is tricky because they're always enqueued with `Oban.Pro.Decorated` as - the worker. The assert helpers `Oban.Pro.Testing` have a `:decorated` option specifically to - make testing decorated jobs more convenient. - - Pass a captured function with the original arity to the `decorated` option: - - ```elixir - assert_enqueued decorated: &Business.weekly_report/2 - ``` - - Use a list of to assert on args (not a map, as you would for a standard job): - - ```elixir - assert_enqueued args: [123, pdf: true], decorated: &Business.weekly_report/2 - ``` - - ## Accessing the Decorated Job - - The `Decorator` module provides a convenient way to access the currently executing job through - the `current_job/0` function. This can be useful when you need job context information during - execution. - - In any module that uses `Oban.Pro.Decorator`, you can call the `current_job/0` function to - retrieve the current job. This is particularly useful when you need to access job metadata as - part of a workflow. For example: - - ```elixir - defmodule MyApp.Doubler do - use Oban.Pro.Decorator - - alias Oban.Pro.Workflow - - def invoke do - Workflow.new() - |> Workflow.add(:dbl_1, new_double(1)) - |> Workflow.add(:dbl_2, new_double(2)) - |> Workflow.add(:dbl_3, new_double(3)) - |> Workflow.add(:print, new_print(), deps: [:dbl_1, :dbl_2, :dbl_3]) - |> Oban.insert_all() - end - - @job recorded: true - def double(value), do: value * 2 - - @job true - def print do - current_job() - |> Workflow.all_recorded(only_deps: true) - |> IO.inspect() - end - end - ``` - - If `current_job/0` is called outside of a job context (not during job execution), it returns - `nil`. - """ - - use Oban.Pro.Worker - - alias __MODULE__, as: Decorator - alias Oban.Pro.Relay - alias Oban.Validation - - args_schema do - field :mod, :string - field :fun, :string - field :arg, :term - end - - @typedoc """ - Options allowed in `@job` annotations at compile time. - """ - @type comp_opts :: [ - max_attempts: pos_integer(), - oban: GenServer.name(), - priority: 0..9, - queue: atom() | binary(), - recorded: boolean() | keyword(), - replace: [Job.replace_by_state_option()], - tags: Job.tags(), - unique: - true - | [ - period: timeout(), - states: atom() | [Job.unique_state()], - timestamp: :inserted_at | :scheduled_at - ] - ] - - @typedoc """ - Options allowed at runtime when calling generated `new_`, `insert_`, or `relay_` functions. - """ - @type full_opts :: [ - max_attempts: pos_integer(), - meta: map(), - oban: GenServer.name(), - priority: 0..9, - queue: atom() | binary(), - recorded: boolean() | keyword(), - replace: [Job.replace_by_state_option()], - scheduled_at: DateTime.t(), - schedule_in: Job.schedule_in_option(), - tags: Job.tags(), - timeout: timeout(), - unique: - true - | [ - period: Job.unique_period(), - states: Job.unique_state_group() | [Job.unique_state()], - timestamp: Job.unique_timestamp() - ] - ] - - @doc false - defmacro __using__(opts) do - deco_opts = Keyword.merge([new: true, insert: true, relay: true], opts) - - quote bind_quoted: [deco_opts: deco_opts] do - Module.put_attribute(__MODULE__, :oban_deco_opts, deco_opts) - Module.register_attribute(__MODULE__, :job, accumulate: true) - Module.register_attribute(__MODULE__, :oban_decorated, accumulate: true) - - @on_definition Oban.Pro.Decorator - @before_compile Oban.Pro.Decorator - end - end - - @doc false - defmacro __before_compile__(env) do - gen_opts = Module.get_attribute(env.module, :oban_deco_opts) - - for {fun, orig_args, guards, opts} <- Module.get_attribute(env.module, :oban_decorated) do - validate!(opts) - - {args, {_idx, renamed}} = - Macro.prewalk(orig_args, {0, %{}}, fn - {var, _meta, nil}, {idx, acc} -> - ast = {:"arg#{idx}", [], __MODULE__} - - {ast, {idx + 1, Map.put(acc, var, ast)}} - - other, idx_acc -> - {other, idx_acc} - end) - - guards = - case guards do - [] -> - quote(do: [is_boolean(true)]) - - [_ | _] -> - Macro.prewalk(guards, fn - {var, _meta, nil} when is_map_key(renamed, var) -> Map.fetch!(renamed, var) - other -> other - end) - end - - quote generated: true, - bind_quoted: [ - new: :"new_#{fun}", - ins: :"insert_#{fun}", - rel: :"relay_#{fun}", - gen_opts: gen_opts, - fun: fun, - mod: env.module, - opts: opts, - args: Macro.escape(args, unquote: true), - args_opts: Macro.escape(args ++ [Macro.var(:oban_opts, nil)], unquote: true), - guards: Macro.escape(guards, unquote: true) - ] do - def current_job do - with {_module, job, _opts} <- Process.get(:oban_processing), do: job - end - - if gen_opts[:new] do - def unquote(new)(unquote_splicing(args)) when unquote_splicing(guards) do - Decorator.new(unquote(mod), unquote(fun), unquote(args), unquote(opts)) - end - - def unquote(new)(unquote_splicing(args_opts)) when unquote_splicing(guards) do - full_opts = Keyword.merge(unquote(opts), unquote(Macro.var(:oban_opts, nil))) - - Decorator.new(unquote(mod), unquote(fun), unquote(args), full_opts) - end - end - - if gen_opts[:insert] do - def unquote(ins)(unquote_splicing(args)) when unquote_splicing(guards) do - Decorator.insert(unquote(mod), unquote(fun), unquote(args), unquote(opts)) - end - - def unquote(ins)(unquote_splicing(args_opts)) when unquote_splicing(guards) do - full_opts = Keyword.merge(unquote(opts), unquote(Macro.var(:oban_opts, nil))) - - Decorator.insert(unquote(mod), unquote(fun), unquote(args), full_opts) - end - end - - if gen_opts[:relay] do - def unquote(rel)(unquote_splicing(args)) when unquote_splicing(guards) do - Decorator.relay(unquote(mod), unquote(fun), unquote(args), unquote(opts)) - end - - def unquote(rel)(unquote_splicing(args_opts)) when unquote_splicing(guards) do - full_opts = Keyword.merge(unquote(opts), unquote(Macro.var(:oban_opts, nil))) - - Decorator.relay(unquote(mod), unquote(fun), unquote(args), full_opts) - end - end - end - end - end - - @doc false - def __on_definition__(env, :def, fun, args, guards, _body) do - with [opts] <- Module.get_attribute(env.module, :job) do - opts = if is_list(opts), do: opts, else: [] - - Module.put_attribute(env.module, :oban_decorated, {fun, List.wrap(args), guards, opts}) - end - - Module.delete_attribute(env.module, :job) - end - - def __on_definition__(_env, _kind, _fun, _args, _guards, _body), do: :ok - - @doc false - def new(mod, fun, arg, opts) when is_atom(mod) and is_atom(fun) and is_list(arg) do - name = "#{inspect(mod)}.#{fun}/#{length(arg)}" - meta = %{decorated: true, decorated_name: name} - - opts = - opts - |> Keyword.drop([:oban, :worker]) - |> Keyword.update(:meta, meta, &Map.merge(&1, meta)) - - new(%{mod: inspect(mod), fun: to_string(fun), arg: arg}, opts) - end - - @doc false - def insert(mod, fun, arg, opts) do - oban = Keyword.get(opts, :oban, Oban) - - mod - |> new(fun, arg, opts) - |> then(&Oban.insert(oban, &1)) - end - - @doc false - def relay(mod, fun, arg, opts) do - oban = Keyword.get(opts, :oban, Oban) - {timeout, opts} = Keyword.pop(opts, :timeout, 5_000) - - mod - |> new(fun, arg, opts) - |> then(&Relay.async(oban, &1)) - |> Relay.await(timeout) - end - - @impl Oban.Pro.Worker - def process(%{args: %{mod: mod, fun: fun, arg: arg}}) do - mod = - mod - |> String.split(".") - |> Module.safe_concat() - |> Code.ensure_loaded!() - - fun = String.to_existing_atom(fun) - - case apply(mod, fun, arg) do - :ok -> :ok - {:ok, _} = result -> result - {:error, _} = error -> error - {:snooze, _} = snooze -> snooze - {:cancel, _} = cancel -> cancel - other -> {:ok, other} - end - end - - defp validate!(opts) do - Validation.validate_schema!(opts, - max_attempts: :pos_integer, - oban: :any, - priority: {:range, 0..9}, - queue: {:or, [:atom, :string]}, - recorded: {:or, [:boolean, :list]}, - replace: {:custom, &Job.validate_replace/1}, - tags: {:list, :string}, - unique: {:custom, &validate_unique/1} - ) - end - - defp validate_unique(unique) do - cond do - is_list(unique) and Keyword.has_key?(unique, :fields) -> - {:error, "the :fields option isn't allowed for decorated jobs"} - - is_list(unique) and Keyword.has_key?(unique, :keys) -> - {:error, "the :keys option isn't allowed for decorated jobs"} - - true -> - Job.validate_unique(unique) - end - end -end diff --git a/vendor/oban_pro/lib/oban/pro/diagnostics.ex b/vendor/oban_pro/lib/oban/pro/diagnostics.ex deleted file mode 100644 index 936cedf6..00000000 --- a/vendor/oban_pro/lib/oban/pro/diagnostics.ex +++ /dev/null @@ -1,135 +0,0 @@ -defmodule Oban.Pro.Diagnostics do - @moduledoc false - - use GenServer - - alias __MODULE__, as: State - alias Oban.Notifier - - defstruct [:timer, interval: :timer.seconds(5), subscribed: MapSet.new()] - - @info_keys ~w( - current_stacktrace - heap_size - memory - message_queue_len - reductions - stack_size - status - )a - - @doc false - def start_link(opts \\ []) do - state = struct!(State, opts) - - GenServer.start_link(__MODULE__, state, name: __MODULE__) - end - - # Callbacks - - @impl GenServer - def init(state) do - Process.flag(:trap_exit, true) - - {:ok, schedule_discover(state)} - end - - @impl GenServer - def terminate(_reason, state) do - if is_reference(state.timer), do: Process.cancel_timer(state.timer) - - :ok - end - - @impl GenServer - def handle_call(:discover, _from, state) do - {:reply, :ok, discover_and_subscribe(state)} - end - - @impl GenServer - def handle_info(:discover, state) do - {:noreply, discover_and_subscribe(state)} - end - - def handle_info({:notification, :diagnostics, %{"job_id" => job_id}}, state) do - finder = fn conf -> - case Registry.lookup(Oban.Registry, {conf.name, {:executing, job_id}}) do - [{pid, _}] -> {conf, pid} - _ -> nil - end - end - - case Enum.find_value(list_oban_instances(), finder) do - {conf, pid} -> - payload = %{"info" => process_info(pid), "job_id" => job_id, "node" => inspect(conf.node)} - - Notifier.notify(conf, :diagnostics_reply, payload) - - nil -> - :ok - end - - {:noreply, state} - end - - def handle_info(_message, state) do - {:noreply, state} - end - - # Helpers - - defp discover_and_subscribe(state) do - state = - Enum.reduce(list_oban_instances(), state, fn conf, acc -> - if MapSet.member?(acc.subscribed, conf.name) do - acc - else - :ok = Notifier.listen(conf.name, [:diagnostics]) - - %{acc | subscribed: MapSet.put(acc.subscribed, conf.name)} - end - end) - - state - |> cancel_timer() - |> schedule_discover() - end - - defp list_oban_instances do - match = [{{{:"$1", Oban.Notifier}, :_, :_}, [], [:"$1"]}] - - Oban.Registry - |> Registry.select(match) - |> Enum.map(&Oban.config/1) - end - - defp process_info(pid) do - case Process.info(pid, @info_keys) do - nil -> - %{} - - info -> - info - |> Keyword.update!(:current_stacktrace, &format_stacktrace/1) - |> Keyword.update!(:status, &to_string/1) - |> Map.new(fn {key, val} -> {to_string(key), val} end) - end - end - - defp format_stacktrace(nil), do: "" - defp format_stacktrace(stacktrace), do: Exception.format_stacktrace(stacktrace) - - defp cancel_timer(%{timer: timer} = state) when is_reference(timer) do - Process.cancel_timer(timer) - - %{state | timer: nil} - end - - defp cancel_timer(state), do: state - - defp schedule_discover(state) do - timer = Process.send_after(self(), :discover, state.interval) - - %{state | timer: timer} - end -end diff --git a/vendor/oban_pro/lib/oban/pro/engines/smart.ex b/vendor/oban_pro/lib/oban/pro/engines/smart.ex deleted file mode 100644 index c16377c3..00000000 --- a/vendor/oban_pro/lib/oban/pro/engines/smart.ex +++ /dev/null @@ -1,1628 +0,0 @@ -defmodule Oban.Pro.Engines.Smart do - @moduledoc """ - The `Smart` engine provides advanced concurrency features, enhanced observability, lighter - weight bulk processing, and provides a foundation for accurate job lifecycle management. As an - `Oban.Engine`, it is responsible for all non-plugin database interaction, from inserting through - executing jobs. - - Major features include: - - * [Global Concurrency](#module-global-concurrency) — limit the number of concurrent jobs that - run across _all_ nodes - * [Rate Limiting](#module-rate-limiting) — limit the number of jobs that execute within a - window of time - * [Queue Partitioning](#module-queue-partitioning) — segment a queue so concurrency or rate - limits apply separately to each partition - * [Async Tracking](#module-async-tracking) — bundle job acks (completed, cancelled, etc.) to - minimize transactions and reduce load on the database - * [Enhanced Unique](#module-enhanced-unique) — enforce job uniqueness with a custom index to - accelerate inserting unique jobs safely between processes and nodes - * [Bulk Inserts](#module-bulk-inserts) — automatically batch inserts to prevent hitting - database limits, reduce data sent over the wire, and respect `unique` options when using - `Oban.insert_all/2` - * [Accurate Snooze](#module-accurate-snooze) — differentiate between attempts with errors and - intentional snoozes - - ## Installation - - See the [Smart Engine](adoption.md#1-smart-engine) section in the [adoption guide](adoption.md) - to get started. - - ## Global Concurrency - - Global concurrency limits the number of concurrent jobs that run across all nodes. - - Typically the global concurrency limit is `local_limit * num_nodes`. For example, with three - nodes and a local limit of 10, you'll have a global limit of 30. If a `global_limit` is present, - and the `local_limit` is omitted, then the `local_limit` falls back to the `global_limit`. - - The only way to guarantee that all connected nodes will run _exactly one job_ concurrently is to - set `global_limit: 1`. - - Here are some examples: - - ```elixir - # Execute 10 jobs concurrently across all nodes, with up to 10 on a single node - my_queue: [global_limit: 10] - - # Execute 10 jobs concurrently, but only 3 jobs on a single node - my_queue: [local_limit: 3, global_limit: 10] - - # Execute at most 1 job concurrently - my_queue: [global_limit: 1] - ``` - - ## Rate Limiting - - Rate limiting controls the number of jobs that execute within a period of time. - - Rate limiting uses counts for the same queue from all other nodes in the cluster (with or - without Distributed Erlang). The limiter uses a sliding window over the configured period to - accurately approximate a limit. - - Every job execution counts toward the rate limit, regardless of whether the job completes, - errors, snoozes, etc. - - Without a modifier, the `rate_limit` period is defined in seconds. However, you can provide a - `:second`, `:minute`, `:hour` or `:day` modifier to use more intuitive values. - - * `period: 30` — 30 seconds - * `period: {1, :minute}` — 60 seconds - * `period: {2, :minutes}` — 120 seconds - * `period: {1, :hour}` — 3,600 seconds - * `period: {1, :day}` —86,400 seconds - - Here are a few examples: - - ```elixir - # Execute at most 60 jobs per minute (1 job per second equivalent) - my_queue: [rate_limit: [allowed: 60, period: {1, :minute}]] - - # Execute at most 10 jobs per 30 seconds - my_queue: [rate_limit: [allowed: 10, period: 30]] - - # Execute at most 10 jobs per minute - my_queue: [rate_limit: [allowed: 10, period: {1, :minute}]] - - # Execute at most 1000 jobs per hour - my_queue: [rate_limit: [allowed: 1000, period: {1, :hour}]] - ``` - - Using larger time periods allows for smoother tracking of rate limits. For example, expressing - "1 job per second" as "60 jobs per minute" provides the same throughput but reduces the - granularity of tracking, resulting in more consistent job execution patterns. - - > #### Understanding Concurrency Limits {: .info} - > - > The local, global, or rate limit with the **lowest value** determines how many jobs are executed - > concurrently. For example, with a `local_limit` of 10 and a `global_limit` of 20, a single node - > will run 10 jobs concurrently. If that same queue had a `rate_limit` that allowed 5 jobs within - > a period, then a single node is limited to 5 jobs. - - ## Queue Partitioning - - In addition to global and rate limits at the queue level, you can partition a queue so that it's - treated as multiple queues where concurrency or rate limits apply separately to each partition. - - Partitions are specified with a list of fields like `:worker`, `:args`, or `:meta`. When - partitioning by `:args`, choosing specific keys is highly recommended to keep partitioning - meaningful. Focused partitioning minimizes the amount of data a queue needs to track and - simplifies job-fetching queries. - - ### Configuring Partitions - - The partition syntax is identical for global and rate limits (note that you can partition by - _global or rate_, but not both.) - - Here are a few examples of viable partitioning schemes: - - ```elixir - # Partition by worker alone - partition: :worker - - # Partition by the `id` and `account_id` from args, ignoring the worker - partition: [args: [:id, :account_id]] - - # Partition by worker and the `account_id` key from args - partition: [:worker, args: :account_id] - ``` - - Remember, take care to minimize partition cardinality by using a few `keys` whenever possible. - Partitioning based on _every permutation_ of your `args` makes concurrency or rate limits hard to - reason about and can negatively impact queue performance. - - ### Global Partitioning - - Global partitioning changes global concurency behavior. Rather than applying a fixed number for - the queue, it applies to every partition within the queue. - - Consider the following example: - - ```elixir - local_limit: 10, global_limit: [allowed: 1, partition: :worker] - ``` - - The queue is configured to run one job per-worker across every node, but only 10 concurrently on a - single node. That is in contrast to the standard behaviour of `global_limit`, which would override - the `local_limit` and only allow 1 concurrent job across every node. - - Alternatively, you could partition by a single key: - - ```elixir - local_limit: 10, global_limit: [allowed: 1, partition: [args: :tenant_id]] - ``` - - That configures the queue to run one job concurrently across the entire cluster per `tenant_id`. - - ### Partitioning with Burst Mode - - Global partitioning includes an advanced feature called "burst mode" for global limits - with partitioning. This feature allows you to maximize throughput by temporarily exceeding - per-partition global limits when there are available resources. - - Each global partition is typically restricted to the configured `allowed` value. However, with - burst mode enabled, the system can intelligently allocate more jobs to each active partition, - potentially exceeding the per-partition limit while still respecting the overall queue - concurrency. - - This is particularly useful when: - - 1. You have many potential partitions but only a few are active at any given time - 2. You want to maximize throughput while maintaining some level of fairness between partitions - 3. You need to ensure your queues aren't sitting idle when capacity is available - - Here's an example of a queue that will 5 jobs from a single partition concurrently under load, - but can burst up to 100 for a single partition when there is available capacity: - - ```elixir - queues: [ - exports: [ - global_limit: [ - allowed: 5, - burst: true, - partition: [args: :tenant_id] - ], - local_limit: 100 - ] - ] - ``` - - Bursting _still respects the overall global concurrency limit_. Using the example above, the - queue can only execute 100 of a given partition's jobs concurrently across all nodes, even if - there are more available resources. - - ### Rate Limit Partitioning - - Rate limit partitions operate similarly to global partitions. Rather than limiting all jobs within - the queue, they limit each partition within the queue. - - For example, to allow one job per-worker, every ten seconds, across every instance of the `alpha` - queue in your cluster: - - ```elixir - local_limit: 10, rate_limit: [allowed: 1, period: 10, partition: :worker] - ``` - - ### Partition Keys Cache - - The Smart engine maintains a cache of available partition keys to optimize performance when - fetching jobs. The cache has a configurable time-to-live (TTL) which controls how long partition - keys are remembered before needing to be refreshed from the database. - - The default TTL is set to `3_000ms`, which is suitable for most applications. You can adjust it - by adding configuration in your application's config: - - ```elixir - # In config/config.exs - config :oban_pro, Oban.Pro.Partition, keys_cache_ttl: 5_000 # 5 seconds - ``` - - A longer TTL reduces database load but might cause the system to be slower to recognize newly - created partition keys. A shorter TTL ensures more up-to-date partition information at the cost - of more frequent database queries. - - The number of partition keys retrieved is based on the queue's `local_limit` multiplied by a - configurable factor (default `3`). This ensures the query scales appropriately with queue - capacity while keeping result sets manageable: - - ```elixir - # In config/config.exs - config :oban_pro, Oban.Pro.Partition, keys_limit_multiplier: 5 - ``` - - Partitions are selected fairly by prioritizing those with the earliest scheduled jobs, ensuring - work is distributed across active partitions rather than favoring any single partition. - - ## Async Tracking - - The results of job execution, e.g. `completed`, `cancelled`, etc., are bundled together into a - single transaction to minimize load on an app's Ecto pool and the database. - - Bundling updates and reporting them asynchronously dramatically reduces the number of - transactions per second. However, async bundling introduces a slight lag (up to 5ms) between job - execution finishing and recording the outcome in the database. - - Async tracking can be disabled for specific queues with the `ack_async` option: - - ```elixir - queues: [ - standard: 30, - critical: [ack_async: false, local_limit: 10] - ] - ``` - - ## Enhanced Unique - - The `Smart` engine uses an alternative mechanism for unique jobs that's designed for speed, - correctness, scalability, and simplicity. Uniqueness is enforced through a unique index that - makes insertion entirely safe between processes and nodes, without the use of advisory locks or - multiple queries. - - Unlike standard uniqueness, which is only checked as jobs are inserted, the index-backed version - applies for the job's entire lifetime. That prevents race conditions where a job changes states - and inadvertently causes a conflict. - - When conflicts are detected, the conflicted job, i.e. the one already in the database, is - annotated with `uniq_conflict: true`. - - > #### Safe Hash for Uniqueness {: .info} - > - > To avoid potential hash collisions when using unique jobs with sub-fields in `args`, enable - > "safe" hashing: - > - > ```elixir - > config :oban_pro, Oban.Pro.Utils, safe_hash: true - > ``` - > - > This applies to `uniq_key`, `chain_key`, and `partition_key` values stored in job meta. - > Note that generated values will not match previous values for configurations using - > sub-fields in `args`. - - > #### Period-based Uniqueness Uses Buckets {: .info} - > - > To leverage unique indexes, period-based uniqueness snaps timestamps to fixed time buckets - > rather than using a sliding window. The current time is rounded down to the nearest multiple - > of the period, and uniqueness is enforced within that bucket. - > - > For example, with a period of 15 minutes (`unique: [period: {15, :minutes}]`) and a job - > inserted at 10:21:00, uniqueness applies from 10:15:00 to 10:30:00. A duplicate job - > inserted at 10:28:00 would be rejected, but one inserted at 10:31:00 would be allowed - > because it falls into the next bucket. - - ## Bulk Inserts - - Where the `Basic` engine requires you to insert unique jobs individually, the `Smart` engine adds - unique job support to `Oban.insert_all/2`. No additional configuration is necessary—simply use - `insert_all` instead for unique jobs. - - ```elixir - Oban.insert_all(lots_of_unique_jobs) - ``` - - Bulk insert also features automatic batching to support inserting an arbitrary number of jobs - without hitting database limits (PostgreSQL's binary protocol has a limit of 65,535 parameters - that may be sent in a single call. That presents an upper limit on the number of rows that may be - inserted at one time.) - - ```elixir - list_of_args - |> Enum.map(&MyApp.MyWorker.new/1) - |> Oban.insert_all() - ``` - - The default batch size for unique jobs is `250`, and `1_000` for non-unique jobs. Regardless, you - can override with `batch_size`: - - ```elixir - Oban.insert_all(lots_of_jobs, batch_size: 1500) - ``` - - It's also possible to set a custom timeout for batch inserts: - - ```elixir - Oban.insert_all(lots_of_jobs, timeout: :infinity) - ``` - - ## Accurate Snooze - - Unlike the `Basic` engine which increments `attempts` and `max_attempts`, the Smart engine rolls - back the `attempt` on snooze. This approach preserves the original `max_attempts` and records a - `snoozed` count in `meta`. As a result, it's simple to differentiate between "real" attempts and - snoozes, and backoff calculation remains accurate regardless of snoozing. - - The following `process/1` function demonstrates checking a job's `meta` for a `snoozed` count: - - ```elixir - def process(job) do - case job.meta do - %{"orig_scheduled_at" => unix_microseconds, "snoozed" => snoozed} -> - IO.inspect({snoozed, unix_microseconds}, label: "Times snoozed since") - - _ -> - # This job has never snoozed before - end - end - ``` - """ - - @behaviour Oban.Engine - @behaviour Oban.Pro.Handler - - import Ecto.Query - import DateTime, only: [utc_now: 0] - - alias Ecto.{Changeset, Multi} - alias Oban.{Backoff, Config, Engine, Job, Repo} - alias Oban.Pro.Limiters.{Global, Local, Rate} - alias Oban.Pro.{Flusher, Handler, Partition, Producer, Unique, Utils} - alias Oban.Pro.Stages.Chain - - require Logger - - @type partition :: - :worker - | {:args, atom()} - | {:meta, atom()} - | [:worker | {:args, atom()} | {:meta, atom()}] - | [fields: [:worker | :args | :meta], keys: [atom()]] - - @type period :: pos_integer() | {pos_integer(), unit()} - - @type global_limit :: pos_integer() | [allowed: pos_integer(), partition: partition()] - - @type local_limit :: pos_integer() - - @type rate_limit :: [allowed: pos_integer(), period: period(), partition: partition()] - - @type unit :: :second | :seconds | :minute | :minutes | :hour | :hours | :day | :days - - # Module Attributes - - @ack_tabs Map.new(0..7, &{&1, :"pro_ack_tab_#{&1}"}) - - @drain_uuid "00000000-0000-0000-0000-000000000000" - - @registry Oban.Registry - - @virtual_opts ~w(ack_async queue refresh_interval xact_delay xact_retry xact_timeout updated_at)a - - @smart_opts Application.compile_env(:oban_pro, __MODULE__, []) - - @xact_expected_delay Access.get(@smart_opts, :xact_expected_delay, 20) - @base_batch_size Access.get(@smart_opts, :base_batch_size, 1_000) - @max_uniq_retries 10 - - # Macros - - defguardp is_global(producer) when is_map(producer.meta) and is_map(producer.meta.global_limit) - - defmacrop merge_jsonb(column, map) do - quote do - fragment("? || ?", unquote(column), unquote(map)) - end - end - - defmacrop xact_lock(pref_key, lock_key) do - quote do - fragment("pg_try_advisory_xact_lock(?::int, ?::int)", unquote(pref_key), unquote(lock_key)) - end - end - - defmacrop state_in_string(column, string, prefix) do - quote do - fragment( - "? = ANY(regexp_split_to_array(?, ',')::?.oban_job_state[])", - unquote(column), - unquote(string), - unquote(prefix) - ) - end - end - - # Handler - - @impl Handler - def on_start do - for {_idx, name} <- @ack_tabs do - :ets.new(name, [:public, :named_table, read_concurrency: true, write_concurrency: true]) - end - end - - @impl Handler - def on_stop, do: :ok - - # Engine - - @impl Engine - def init(%Config{} = conf, [_ | _] = opts) do - {validate?, opts} = Keyword.pop(opts, :validate, false) - - {prod_opts, meta_opts} = - opts - |> Keyword.put_new(:ack_async, conf.testing == :disabled) - |> Keyword.split(@virtual_opts) - - changeset = - prod_opts - |> Keyword.put(:name, conf.name) - |> Keyword.put(:node, conf.node) - |> Keyword.put(:meta, meta_opts) - |> Keyword.put_new(:queue, :default) - |> Keyword.put_new(:started_at, utc_now()) - |> Keyword.put_new(:updated_at, utc_now()) - |> Producer.new() - - case Changeset.apply_action(changeset, :insert) do - {:ok, producer} -> - if validate? do - {:ok, producer} - else - ack_tab = - producer.queue - |> :erlang.phash2(map_size(@ack_tabs)) - |> then(&Map.fetch!(@ack_tabs, &1)) - - virt_opts = - prod_opts - |> Keyword.take(~w(ack_async refresh_interval xact_delay xact_retry)a) - |> Map.new() - - fun = - fn -> - conf - |> Repo.insert!(changeset) - |> Map.put(:ack_tab, ack_tab) - |> Map.merge(virt_opts) - |> put_producer(conf) - end - - transaction(conf, fun, virt_opts) - end - - {:error, changeset} -> - {:error, Utils.to_exception(changeset)} - end - end - - @impl Engine - def refresh(_conf, %Producer{} = producer) do - # The `Refresher` module handles updating the database rows independently of this callback - # being triggered. When the queue's producer is stuck in a transaction retry loop it isn't - # able to handle new messages, including periodic refresh. That would allow the producer - # record to become outdated, at which point it is subject to erroneous cleanup. - %{producer | updated_at: utc_now()} - end - - @impl Engine - def shutdown(%Config{} = conf, %Producer{} = producer) do - monitor_unregister(conf, producer) - - put_meta(conf, producer, :shutdown, true) - catch - _kind, _reason -> - producer - |> Producer.update_meta(:paused, true) - |> Changeset.apply_action!(:update) - end - - defp monitor_unregister(conf, producer) do - parent = self() - - Task.start(fn -> - ref = Process.monitor(parent) - - receive do - {:DOWN, ^ref, :process, _pid, _reason} -> - Process.demonitor(ref, [:flush]) - - Registry.delete_meta(@registry, reg_key(conf, producer)) - end - end) - end - - @impl Engine - def put_meta(%Config{} = conf, %Producer{} = producer, :flush, _any) do - producer = run_acks(conf, producer) - - if is_global(producer) do - Producer - |> where(uuid: ^producer.uuid) - |> then(&Repo.update_all(conf, &1, set: [meta: producer.meta, updated_at: utc_now()])) - end - - producer - end - - def put_meta(%Config{} = conf, %Producer{} = producer, :paused, true) do - changeset = - conf - |> run_acks(producer) - |> Producer.update_meta(:paused, true) - - conf - |> Repo.update!(changeset) - |> put_producer(conf) - end - - def put_meta(%Config{} = conf, %Producer{} = producer, :shutdown, true) do - changeset = - conf - |> run_acks(producer) - |> Producer.update_meta(%{paused: true, shutdown_started_at: utc_now()}) - - conf - |> Repo.update!(changeset) - |> put_producer(conf) - end - - def put_meta(%Config{} = conf, %Producer{} = producer, key, value) do - changeset = Producer.update_meta(producer, key, value) - - conf - |> Repo.update!(changeset) - |> put_producer(conf) - end - - @impl Engine - def check_meta(_conf, %Producer{} = producer, running) do - jids = for {_, {_, exec}} <- running, do: exec.job.id - - meta = - producer.meta - |> Map.from_struct() - |> flatten_windows() - - producer - |> Map.take(~w(name node queue uuid started_at updated_at)a) - |> Map.put(:running, jids) - |> Map.merge(meta) - end - - defp flatten_windows(%{rate_limit: %{windows: windows}} = meta) do - {pacc, cacc} = - Enum.reduce(windows, {0, 0}, fn {_key, map}, {pacc, cacc} -> - %{"prev_count" => prev, "curr_count" => curr} = map - - {pacc + prev, cacc + curr} - end) - - put_in(meta.rate_limit.windows, [%{"curr_count" => cacc, "prev_count" => pacc}]) - end - - defp flatten_windows(meta), do: meta - - @impl Engine - def fetch_jobs(_conf, %{meta: %{paused: true}} = prod, _running) do - {:ok, {prod, []}} - end - - def fetch_jobs(_conf, %{meta: %{local_limit: limit}} = prod, running) - when map_size(running) >= limit do - {:ok, {prod, []}} - end - - def fetch_jobs(%Config{} = conf, %Producer{} = producer, running) do - acks = get_acks(producer) - - multi = - Multi.new() - |> Multi.put(:acks, acks) - |> Multi.put(:conf, conf) - |> Multi.put(:running, running) - |> Multi.put(:producer, track_acks(acks, producer)) - |> Multi.run(:all_producers, &all_producers/2) - |> Multi.run(:ack_ids, &ack_jobs/2) - |> Multi.run(:afh_ids, &run_flush_handlers/2) - |> Multi.run(:local_demand, &Local.check/2) - |> Multi.run(:global_demand, &Global.check/2) - |> Multi.run(:rate_demand, &Rate.check/2) - |> Multi.run(:jobs, &fetch_jobs/2) - |> Multi.run(:tracked, &track_jobs(&1, &2, running)) - - case transaction(conf, multi, producer) do - {:ok, %{ack_ids: ack_ids, jobs: jobs, tracked: producer}} -> - del_acks(ack_ids, producer) - - {:ok, {producer, jobs}} - - {:error, _op, %{postgres: %{code: :unique_violation, detail: detail}}, changes} -> - clear_uniq_violation(changes.conf, detail) - - retry_unique_violation(:fetch_jobs, [conf, producer, running]) - - {:error, _operation, :xact_lock, _changes} -> - jittery_sleep() - - fetch_jobs(conf, producer, running) - - {:error, _operation, error, _changes} -> - raise error - end - catch - :error, %Postgrex.Error{postgres: %{code: :unique_violation, detail: detail}} -> - clear_uniq_violation(conf, detail, Enum.map(Job.states(), &to_string/1)) - - retry_unique_violation(:fetch_jobs, [conf, producer, running]) - end - - @doc false - def fetch_for_drain(conf, %{queue: queue, with_limit: limit} = opts) do - subset_query = - if queue == :__all__ do - where(Job, state: "available") - else - where(Job, state: "available", queue: ^to_string(queue)) - end - - subset_query = - case opts do - %{batch_ids: ids} -> where(subset_query, [j], j.meta["batch_id"] in ^ids) - %{workflow_ids: ids} -> where(subset_query, [j], j.meta["workflow_id"] in ^ids) - _ -> subset_query - end - - subset_query = - subset_query - |> order_by(asc: :priority, asc: :scheduled_at, asc: :id) - |> lock("FOR UPDATE SKIP LOCKED") - |> limit(^limit) - - query = - Job - |> with_cte("subset", as: ^subset_query) - |> join(:inner, [j], x in fragment(~s("subset")), on: true) - |> where([j, x], j.id == x.id) - |> select([j, _], j) - - updates = [ - set: [state: "executing", attempted_at: utc_now(), attempted_by: [conf.node, @drain_uuid]], - inc: [attempt: 1] - ] - - {_count, jobs} = Repo.update_all(conf, query, updates, []) - - jobs - end - - @impl Engine - def stage_jobs(%Config{} = conf, queryable, opts) do - limit = Keyword.fetch!(opts, :limit) - - subquery = - queryable - |> select([:id, :state]) - |> where([j], j.state in ~w(scheduled retryable)) - |> where([j], not is_nil(j.queue)) - |> where([j], j.scheduled_at <= ^DateTime.utc_now()) - |> limit(^limit) - - query = - Job - |> join(:inner, [j], x in subquery(subquery), on: j.id == x.id) - |> select([j, x], %{id: j.id, queue: j.queue, state: x.state}) - - {_count, staged} = Repo.update_all(conf, query, set: [state: "available"]) - - {:ok, staged} - catch - :error, %Postgrex.Error{postgres: %{code: :unique_violation, detail: detail}} -> - clear_uniq_violation(conf, detail, ["scheduled", "retryable"]) - - retry_unique_violation(:stage_jobs, [conf, queryable, opts]) - end - - defp retry_unique_violation(fun, args) do - attempt = Process.get(fun, 0) - - if attempt >= @max_uniq_retries do - Process.delete(fun) - - Logger.error(fn -> - "[Oban.Pro.Engines.Smart] Unique violation retry limit (#{@max_uniq_retries}) exceeded " <> - "in #{fun}. This may indicate a persistent unique constraint conflict that cannot " <> - "be automatically resolved." - end) - - uniq_violation_failure(fun, args) - else - Process.put(fun, attempt + 1) - - apply(__MODULE__, fun, args) - end - end - - defp uniq_violation_failure(:fetch_jobs, [_conf, producer, _running]) do - {:ok, {producer, []}} - end - - defp uniq_violation_failure(:stage_jobs, _args) do - {:ok, []} - end - - @impl Engine - def check_available(%Config{} = conf) do - exists = - Job - |> where([j], j.state == "available" and parent_as(:producers).queue == j.queue) - |> select(1) - - query = - from(p in Producer, as: :producers) - |> where([p], exists(subquery(exists))) - |> select([p], p.queue) - - {:ok, Repo.all(conf, query)} - end - - @impl Engine - def complete_job(%Config{} = conf, %Job{} = job) do - set = [state: "completed", completed_at: utc_now()] - - case Process.get(:oban_recorded) do - nil -> put_ack(conf, %{job | state: "completed"}, set) - map -> put_ack(conf, %{job | state: "completed"}, set ++ [meta: map]) - end - - :ok - end - - @impl Engine - def discard_job(%Config{} = conf, %Job{} = job) do - put_ack(conf, %{job | state: "discarded"}, - state: "discarded", - discarded_at: utc_now(), - error: Job.format_attempt(job) - ) - - :ok - end - - @impl Engine - def error_job(%Config{} = conf, %Job{} = job, seconds) do - orig_at = Map.get(job.meta, "orig_scheduled_at", to_unix(job.scheduled_at)) - - set = - if job.attempt >= job.max_attempts do - [state: "discarded", discarded_at: utc_now()] - else - [state: "retryable", scheduled_at: seconds_from_now(seconds)] - end - - put_ack( - conf, - %{job | state: set[:state]}, - set ++ [error: Job.format_attempt(job), meta: %{orig_scheduled_at: orig_at}] - ) - - :ok - end - - @impl Engine - def snooze_job(%Config{} = conf, %Job{} = job, seconds) do - snoozed = Map.get(job.meta, "snoozed", 0) - orig_at = Map.get(job.meta, "orig_scheduled_at", to_unix(job.scheduled_at)) - - put_ack(conf, %{job | state: "scheduled"}, - attempt_change: -1, - state: "scheduled", - scheduled_at: seconds_from_now(seconds), - meta: %{orig_scheduled_at: orig_at, snoozed: snoozed + 1} - ) - - :ok - end - - @impl Engine - def cancel_job(%Config{} = conf, %Job{unsaved_error: %{}} = job) do - put_ack(conf, %{job | state: "cancelled"}, - state: "cancelled", - cancelled_at: utc_now(), - error: Job.format_attempt(job) - ) - end - - def cancel_job(%Config{} = conf, %Job{} = job) do - cancel_all_jobs(conf, where(Job, id: ^job.id)) - - :ok - end - - @impl Engine - def cancel_all_jobs(%Config{} = conf, queryable) do - subquery = where(queryable, [j], j.state not in ["cancelled", "completed", "discarded"]) - - query = - Job - |> join(:inner, [j], x in subquery(subquery), on: j.id == x.id) - |> update(set: [state: "cancelled", cancelled_at: ^utc_now()]) - |> select([_, x], map(x, [:id, :args, :attempted_by, :meta, :queue, :state, :worker])) - - {:ok, %{jobs: {_count, jobs}}} = - Multi.new() - |> Multi.put(:conf, conf) - |> Multi.update_all(:jobs, query, [], Repo.default_options(conf)) - |> Multi.run(:ack, &track_cancelled_jobs/2) - |> then(&Repo.transaction(conf, &1)) - - {:ok, jobs} - end - - @doc false - # This is used to unblock a uniqueness violation by clearing out the uniq_bmp field. The - # generated `uniq_key` column uses the `uniq_key` from meta, but is only present when the job's - # state maps to a value in `uniq_bmp`. - # - # We find conflicting jobs by matching the unique key in meta and filtering by the expected - # states. Clearing their `uniq_bmp` disables the unique constraint, allowing the transition. - def clear_uniq_violation(%Config{} = conf, detail, states \\ ["executing"]) do - with %{"key" => key} <- Regex.named_captures(~r/\(uniq_key\)=\((?.+)\)/, detail) do - query = - Job - |> where([j], j.state in ^states) - |> where([j], fragment("? @> ?", j.meta, ^%{uniq_key: key})) - |> update([j], set: [meta: merge_jsonb(j.meta, ^%{uniq_bmp: []})]) - - Repo.update_all(conf, query, []) - - :telemetry.execute( - [:oban, :engine, :uniq_violation_repaired], - %{count: 1}, - %{conf: conf, uniq_key: key, states: states} - ) - - log_key = {__MODULE__, :uniq_violation_logged, key} - - unless :persistent_term.get(log_key, false) do - :persistent_term.put(log_key, true) - - Logger.info(fn -> - """ - [Oban.Pro.Engines.Smart] Unique constraint violation repaired. - - This may indicate a unique job misconfiguration where jobs are being inserted - with conflicting unique keys across different states. The engine repaired the - violation for uniq_key: #{key} - - Check your unique job configuration to ensure the :states option matches the - expected job lifecycle for this worker. - """ - end) - end - end - end - - @impl Engine - def insert_job(conf, changeset, opts) do - if changeset.valid? do - case insert_all_jobs(conf, [changeset], opts) do - [job] -> - {:ok, job} - - _ -> - {:error, changeset} - end - else - {:error, changeset} - end - end - - @impl Engine - def insert_all_jobs(conf, changesets, opts) do - fun = fn -> insert_all_batches(conf, changesets, opts) end - - {:ok, jobs} = Repo.transaction(conf, fun, opts) - - jobs - end - - defp insert_all_batches(conf, changesets, opts) do - batch_size = Keyword.get(opts, :batch_size, @base_batch_size) - - changesets - |> Stream.map(&Unique.with_uniq_meta/1) - |> Stream.map(&Partition.with_partition_meta(&1, conf)) - |> Stream.uniq_by(&Unique.get_key(&1, System.unique_integer())) - |> Stream.chunk_every(batch_size) - |> Enum.flat_map(fn changesets -> - {uniq_map, link_map, replace?} = - Enum.reduce(changesets, {%{}, %{}, false}, fn changeset, {uniq_map, link_map, replace?} -> - uniq_map = - case Unique.get_key(changeset) do - uniq_key when is_binary(uniq_key) -> - Map.put(uniq_map, uniq_key, changeset) - - _ -> - uniq_map - end - - link_map = - case Chain.get_key(changeset) do - chain_id when is_binary(chain_id) -> - Map.update(link_map, chain_id, [changeset], &[changeset | &1]) - - _ -> - link_map - end - - {uniq_map, link_map, replace? or Unique.replace?(changeset)} - end) - - {:ok, %{all_jobs: jobs}} = - Multi.new() - |> Multi.put(:conf, conf) - |> Multi.put(:opts, opts) - |> Multi.put(:changesets, changesets) - |> Multi.put(:uniq_map, uniq_map) - |> Multi.put(:link_map, link_map) - |> Multi.put(:replacements?, replace?) - |> Multi.run(:uniq_index?, &uniq_index?/2) - |> Multi.run(:chain_changesets, &prepare_chains/2) - |> Multi.run(:dupe_map, &find_dupes/2) - |> Multi.run(:new_jobs, &insert_entries/2) - |> Multi.run(:rep_jobs, &apply_replacements/2) - |> Multi.run(:all_jobs, &apply_conflicts/2) - |> then(&Repo.transaction(conf, &1, opts)) - - jobs - end) - end - - defp uniq_index?(_repo, %{conf: conf}) do - %{name: name, prefix: prefix} = conf - - Utils.persistent_cache({__MODULE__, :uniq_index?, name}, fn -> - query = - from("columns") - |> put_query_prefix("information_schema") - |> where(table_schema: ^prefix, table_name: "oban_jobs", column_name: "uniq_key") - |> select(true) - - {:ok, Repo.one(conf, query) == true} - end) - end - - defp prepare_chains(_repo, %{link_map: link_map, conf: conf}) when map_size(link_map) > 0 do - chain_ids = Map.keys(link_map) - - if has_advisory_locks?(conf) do - lock_keys = - chain_ids - |> Enum.map(&:erlang.phash2({conf.prefix, &1})) - |> Enum.sort() - - Repo.query!(conf, "SELECT pg_advisory_xact_lock(unnest($1::bigint[]))", [lock_keys]) - end - - query = - from( - f in fragment("json_array_elements_text(?)", ^chain_ids), - as: :list, - inner_lateral_join: - j in subquery( - Job - |> select([:state]) - |> where([j], j.state != "completed") - |> where([j], fragment("? \\? 'chain_id'", j.meta)) - |> where([j], fragment("?->>'chain_id'", j.meta) == parent_as(:list).value) - |> where([j], fragment("?->>'on_hold'", j.meta) in ~w(true false)) - |> order_by(desc: :id) - |> lock("FOR UPDATE") - |> limit(1) - ), - on: true, - select: {f.value, j.state} - ) - - found_map = - conf - |> Repo.all(query) - |> Map.new() - - linked = - Enum.flat_map(link_map, fn {chain_id, changesets} -> - changesets - |> Enum.reverse() - |> Enum.with_index() - |> Enum.map(fn {changeset, index} -> - state = Map.get(found_map, chain_id) - meta = Changeset.get_field(changeset, :meta) - - cond do - index == 0 and is_nil(state) -> changeset - index == 0 and Chain.continuable?(state, meta) -> changeset - true -> Chain.to_postponed(changeset) - end - end) - end) - - {:ok, linked} - end - - defp prepare_chains(_repo, _changes), do: {:ok, []} - - @impl Engine - def retry_job(%Config{} = conf, %Job{id: id}) do - retry_all_jobs(conf, where(Job, [j], j.id == ^id)) - - :ok - end - - @impl Engine - def retry_all_jobs(%Config{} = conf, queryable) do - subquery = - queryable - |> where([j], j.state not in ["available", "executing"]) - |> where([j], not fragment("? @> ?", j.meta, ^%{on_hold: true})) - - query = - Job - |> join(:inner, [j], x in subquery(subquery), on: j.id == x.id) - |> select([_, x], map(x, [:id, :queue, :state])) - |> update([j], - set: [ - state: "available", - max_attempts: fragment("GREATEST(?, ? + 1)", j.max_attempts, j.attempt), - scheduled_at: ^utc_now(), - completed_at: nil, - cancelled_at: nil, - discarded_at: nil - ] - ) - - {_, jobs} = Repo.update_all(conf, query, []) - - {:ok, jobs} - end - - @impl Engine - def update_job(%Config{} = conf, %Job{id: id}, changes) when is_map(changes) do - updater = fn -> - query = - Job - |> where([j], j.id == ^id) - |> lock("FOR UPDATE SKIP LOCKED") - - case Repo.one(conf, query) do - nil -> - {:error, :locked_or_not_found} - - job when is_map_key(job.meta, "structured") and is_map_key(changes, :args) -> - {:ok, worker} = Oban.Worker.from_string(job.worker) - - changeset = worker.new(changes.args) - - if changeset.valid? do - job - |> Job.update(changes) - |> then(&Repo.update(conf, &1)) - else - {:error, changeset} - end - - job -> - job - |> Job.update(changes) - |> then(&Repo.update(conf, &1)) - end - end - - case Repo.transaction(conf, updater) do - {:ok, result} -> result - {:error, reason} -> {:error, reason} - end - end - - # Producer Fetching Helpers - - defp get_producer(conf, job_or_producer) do - case Registry.meta(@registry, reg_key(conf, job_or_producer)) do - {:ok, producer} -> producer - :error -> %Producer{ack_async: false} - end - end - - defp put_producer(producer, conf) do - Registry.put_meta(@registry, reg_key(conf, producer), producer) - - producer - end - - defp reg_key(%{name: name}, %{queue: queue}), do: {name, {:producer, queue}} - - # Acking Helpers - - defp put_ack(conf, job, updates) do - producer = get_producer(conf, job) - global_key = Partition.get_key(job, "*") - flush_handlers = Flusher.get_flush_handlers(job, conf) - - ack_entry = {{:ack, producer.name, job.queue, job.id}, global_key, flush_handlers, updates} - - if producer.ack_tab, do: :ets.insert(producer.ack_tab, ack_entry) - - cond do - Process.get(:oban_draining) -> - Process.delete(:oban_recorded) - - {:ok, jids} = ack_jobs([ack_entry], conf) - - run_flush_handlers([ack_entry]) - - if producer.ack_tab, do: del_acks(jids, producer) - - not producer.ack_async or producer.meta.paused -> - pid = Oban.Registry.whereis(conf.name, {:producer, producer.queue}) - - if is_pid(pid) and Process.alive?(pid) do - GenServer.call(pid, {:put_meta, :flush, true}) - end - - true -> - :ok - end - end - - defp get_acks(%{ack_tab: tab, name: name, queue: queue}) do - :ets.select(tab, [{{{:ack, name, queue, :_}, :_, :_, :_}, [], [:"$_"]}]) - end - - defp del_acks(ids, %{ack_tab: tab, name: name, queue: queue}) do - Enum.each(ids, &:ets.delete(tab, {:ack, name, queue, &1})) - end - - defp run_acks(conf, producer) do - acks = get_acks(producer) - - {:ok, jids} = ack_jobs(acks, conf) - - run_flush_handlers(acks) - - del_acks(jids, producer) - - track_acks(acks, producer) - end - - defp run_flush_handlers(_repo, %{acks: acks}) do - {:ok, run_flush_handlers(acks)} - end - - defp run_flush_handlers(acks) do - mfas = for {_, _, all, _} <- acks, mfa <- all, is_tuple(mfa), uniq: true, do: mfa - - Enum.each(mfas, fn {mod, fun, arg} -> apply(mod, fun, arg) end) - end - - # Fetch Helpers - - defp all_producers(_repo, %{conf: conf, producer: producer}) do - %{ack_tab: tab, meta: meta, queue: queue} = producer - - needs_lock? = is_map(meta.global_limit) or is_map(meta.rate_limit) or any_flush_handlers?(tab) - query = where(Producer, queue: ^queue) - - cond do - not needs_lock? -> - {:ok, []} - - not has_advisory_locks?(conf) -> - {:ok, Repo.all(conf, lock(query, "FOR UPDATE NOWAIT"))} - - take_advisory_lock?(conf, queue) -> - {:ok, Repo.all(conf, query)} - - true -> - {:error, :xact_lock} - end - end - - defp any_flush_handlers?(tab) do - match = {:_, :_, :"$1", :_} - guard = [{:"/=", :"$1", []}] - - :ets.select_count(tab, [{match, guard, [true]}]) > 0 - end - - defp has_advisory_locks?(conf) do - Utils.persistent_cache({__MODULE__, :has_advisory_locks?}, fn -> - query = - from("pg_proc") - |> put_query_prefix("pg_catalog") - |> where(proname: "pg_try_advisory_xact_lock", pronargs: 2) - |> select(true) - - Repo.one(conf, query) == true - end) - end - - defp take_advisory_lock?(conf, queue) do - pre = :erlang.phash2(conf.prefix) - key = :erlang.phash2(queue) - - query = from(f in xact_lock(^pre, ^key), select: f.f0) - - Repo.one(conf, query) - end - - defp fetch_jobs(_repo, %{conf: conf, producer: producer} = changes) do - subset_query = fetch_subquery(changes) - - query = - Job - |> with_cte("subset", as: ^subset_query) - |> join(:inner, [j], x in fragment(~s("subset")), on: true) - |> where([j, x], j.id == x.id and j.state == "available" and j.attempt < j.max_attempts) - |> select([j, _], j) - - updates = [ - set: [state: "executing", attempted_at: utc_now(), attempted_by: [conf.node, producer.uuid]], - inc: [attempt: 1] - ] - - {_count, jobs} = Repo.update_all(conf, query, updates) - - {:ok, jobs} - end - - defp fetch_subquery(%{local_demand: local, producer: producer} = changes) do - case changes do - %{global_demand: nil, rate_demand: nil} -> - fetch_subquery(producer, local) - - %{global_demand: global, rate_demand: nil} when is_integer(global) -> - fetch_subquery(producer, min(local, global)) - - %{global_demand: nil, rate_demand: rated} when is_integer(rated) -> - fetch_subquery(producer, min(local, rated)) - - %{global_demand: global, rate_demand: %{} = demands} -> - fetch_subquery(producer, demands, min(local, global || local)) - - %{global_demand: %{} = demands, rate_demand: rated} -> - fetch_subquery(producer, demands, min(local, rated || local)) - - %{global_demand: global, rate_demand: rated} -> - limit = rated |> min(local) |> min(global) - - fetch_subquery(producer, limit) - end - end - - defp fetch_subquery(producer, limit) do - Job - |> select([:id]) - |> where(state: "available", queue: ^producer.queue) - |> order_by([:priority, :scheduled_at, :id]) - |> limit(^max(limit, 0)) - |> lock("FOR UPDATE SKIP LOCKED") - end - - defp fetch_subquery(producer, demands, limit) do - {keys, lims} = Enum.unzip(demands) - - part_query = - from( - p in fragment("SELECT unnest(?::text[]) as key, unnest(?::int[]) AS lim", ^keys, ^lims), - as: :part, - inner_lateral_join: - j in subquery( - Job - |> select([:id, :priority, :scheduled_at]) - |> where([j], j.state == "available" and j.queue == ^producer.queue) - |> where([j], fragment("partition_key") == parent_as(:part).key) - |> order_by([:priority, :scheduled_at, :id]) - |> limit(parent_as(:part).lim) - ), - on: true, - select: %{id: j.id, priority: j.priority, scheduled_at: j.scheduled_at} - ) - - from(j in subquery(part_query), - select: j.id, - limit: ^limit, - order_by: [:priority, :scheduled_at, :id] - ) - end - - # Tracking Helpers - - @ack_fields ~w(state cancelled_at completed_at discarded_at scheduled_at attempt_change error meta)a - - defp ack_jobs(_repo, %{acks: acks, conf: conf}), do: ack_jobs(acks, conf) - defp ack_jobs([], _conf), do: {:ok, []} - - defp ack_jobs(acks, conf) do - [ids | params] = - acks - |> Enum.map(fn {{:ack, _, _, id}, _, _, set} -> [id | Enum.map(@ack_fields, &set[&1])] end) - |> Enum.zip_with(&Function.identity/1) - - case Repo.query(conf, ack_query(conf), [ids | params]) do - {:ok, %{rows: rows}} -> {:ok, List.flatten(rows)} - error -> error - end - end - - defp ack_query(conf) do - Utils.persistent_cache({__MODULE__, :ack_query, conf.prefix}, fn -> - """ - WITH params AS ( - SELECT unnest($1::bigint[]) AS id, - unnest($2::"#{conf.prefix}".oban_job_state[]) AS state, - unnest($3::timestamp[]) AS cancelled_at, - unnest($4::timestamp[]) AS completed_at, - unnest($5::timestamp[]) AS discarded_at, - unnest($6::timestamp[]) AS scheduled_at, - unnest($7::integer[]) AS attempt_change, - unnest($8::jsonb[]) AS error, - unnest($9::jsonb[]) AS meta - ), - locked AS ( - SELECT oj.id - FROM "#{conf.prefix}"."oban_jobs" oj - INNER JOIN params tmp ON oj.id = tmp.id - FOR UPDATE OF oj - ) - UPDATE "#{conf.prefix}"."oban_jobs" oj - SET state = tmp.state, - cancelled_at = COALESCE(tmp.cancelled_at, oj.cancelled_at), - completed_at = COALESCE(tmp.completed_at, oj.completed_at), - discarded_at = COALESCE(tmp.discarded_at, oj.discarded_at), - scheduled_at = COALESCE(tmp.scheduled_at, oj.scheduled_at), - attempt = COALESCE(tmp.attempt_change + oj.attempt, oj.attempt), - errors = CASE WHEN tmp.error IS NULL THEN oj.errors ELSE oj.errors || tmp.error END, - meta = CASE WHEN tmp.meta IS NULL THEN oj.meta ELSE oj.meta || tmp.meta END - FROM params tmp - INNER JOIN locked l ON tmp.id = l.id - WHERE oj.id = tmp.id - RETURNING oj.id - """ - end) - end - - defp track_jobs(_repo, %{conf: conf, jobs: jobs, producer: producer}, running) do - old_jobs = for {_ref, {_pid, %{job: job}}} <- running, do: job - all_jobs = jobs ++ old_jobs - - meta = - producer.meta - |> Global.track(all_jobs) - |> Local.track(jobs) - |> Rate.track(jobs) - - if meta == producer.meta do - {:ok, producer} - else - now = utc_now() - query = where(Producer, uuid: ^producer.uuid) - - case Repo.update_all(conf, query, set: [meta: meta, updated_at: now]) do - {1, _} -> - {:ok, %{producer | meta: meta, updated_at: now}} - - # In this case the producer was erroneously deleted, possibly due to a connection error, - # downtime, or in development after waking from sleep. - {0, _} -> - %{producer | meta: meta, updated_at: now} - |> Changeset.change() - |> then(&Repo.insert(conf, &1)) - end - end - end - - defp track_acks(acks, producer) when is_global(producer) do - keys = Enum.map(acks, &elem(&1, 1)) - meta = Global.prepare_tracked(producer.meta, keys) - - %{producer | meta: meta} - end - - defp track_acks(_acks, producer), do: producer - - defp track_cancelled_jobs(_repo, %{conf: conf, jobs: {_count, jobs}}) do - jobs - |> Enum.filter(&(&1.state == "executing")) - |> Enum.group_by(& &1.attempted_by) - |> Enum.each(fn {[_node, uuid | _], jobs} -> - query = - Producer - |> where([p], p.uuid == ^uuid) - |> where([p], fragment("?->'global_limit' \\? 'tracked'", p.meta)) - - with %Producer{meta: meta} = producer <- Repo.one(conf, query) do - keys = Enum.map(jobs, &Partition.get_key(&1, "*")) - meta = Global.prepare_tracked(meta, keys) - - %{producer | meta: meta, updated_at: utc_now()} - |> Changeset.change() - |> then(&Repo.update(conf, &1)) - end - end) - - {:ok, nil} - end - - # Insert Helpers - - defp find_dupes(_repo, %{conf: conf, uniq_index?: false, uniq_map: uniq_map}) - when map_size(uniq_map) > 0 do - {uniq_keys, uniq_states} = - Enum.reduce(uniq_map, {[], []}, fn {key, changeset}, {key_acc, sta_acc} -> - states = changeset |> Unique.get_states() |> Enum.join(",") - - {[key | key_acc], [states | sta_acc]} - end) - - query = - from( - t in fragment( - "SELECT unnest(?::text[]) AS uk, unnest(?::text[]) AS us", - ^uniq_keys, - ^uniq_states - ), - join: j in Job, - on: - fragment("? \\? 'uniq_key'", j.meta) and - fragment("?->>'uniq_key'", j.meta) == t.uk and - state_in_string(j.state, t.us, literal(^conf.prefix)), - select: {t.uk, j.id, j.state} - ) - - dupes = - conf - |> Repo.all(query) - |> Map.new(fn {key, id, state} -> {key, {id, state, Map.get(uniq_map, key)}} end) - - {:ok, dupes} - end - - defp find_dupes(_repo, _changes), do: {:ok, %{}} - - defp insert_entries(_repo, %{changesets: []}), do: {:ok, []} - - defp insert_entries(_repo, changes) do - %{conf: conf, changesets: changesets, chain_changesets: chains} = changes - %{dupe_map: dupe_map, opts: opts} = changes - - changesets = - if chains == [] do - changesets - else - Enum.reject(changesets, &Chain.chain?/1) ++ chains - end - - {entries, placeholders} = - for changeset <- changesets, - not is_map_key(dupe_map, Unique.get_key(changeset)), - reduce: {[], nil} do - {entries, acc} -> - map = Job.to_map(changeset) - - if is_nil(acc) do - {[map | entries], map} - else - {[map | entries], Map.filter(acc, fn {key, val} -> map[key] == val end)} - end - end - - # Ensure placeholders aren't nil before use in is_map_key. - placeholders = placeholders || %{} - - entries = - for entry <- Enum.reverse(entries) do - Map.new(entry, fn {key, val} -> - if is_map_key(placeholders, key) do - {key, {:placeholder, key}} - else - {key, val} - end - end) - end - - opts = - opts - |> Keyword.merge(placeholders: placeholders, returning: true) - |> Keyword.merge(conflict_opts(changes)) - - {_count, jobs} = Repo.insert_all(conf, Job, entries, opts) - - {:ok, jobs} - end - - defp conflict_opts(%{uniq_map: uniq_map, uniq_index?: uniq_index?}) do - if uniq_map == %{} or not uniq_index? do - [] - else - [ - conflict_target: {:unsafe_fragment, "(uniq_key) WHERE uniq_key IS NOT NULL"}, - on_conflict: update(Job, [j], set: [meta: merge_jsonb(j.meta, ^%{uniq_conflict: true})]) - ] - end - end - - defp apply_replacements(_repo, %{replacements?: false}), do: {:ok, []} - - defp apply_replacements(_repo, %{conf: conf, opts: opts} = changes) do - updates = - for {_key, {id, state, %{changes: %{replace: replace} = changes}}} <- dupe_map(changes), - reduce: %{} do - acc -> - state = String.to_existing_atom(state) - rep_keys = Keyword.get(replace, state, []) - - changes - |> Map.take(rep_keys) - |> Enum.reduce(acc, fn {key, val}, sub_acc -> - Map.update(sub_acc, {key, val}, [id], &[id | &1]) - end) - end - - Enum.each(updates, fn {val, ids} -> - Repo.update_all(conf, where(Job, [j], j.id in ^ids), [set: [val]], opts) - end) - - {:ok, []} - end - - defp dupe_map(%{dupe_map: dupe_map, uniq_index?: false}), do: dupe_map - - defp dupe_map(%{new_jobs: new_jobs, uniq_map: uniq_map}) do - Enum.reduce(new_jobs, %{}, fn job, acc -> - case job do - %{meta: %{"uniq_conflict" => true, "uniq_key" => uniq_key}} -> - Map.put(acc, uniq_key, {job.id, job.state, Map.get(uniq_map, uniq_key)}) - - _ -> - acc - end - end) - end - - defp apply_conflicts(_repo, %{uniq_index?: false, dupe_map: dupe_map, new_jobs: new_jobs}) do - old_jobs = - dupe_map - |> Map.values() - |> Enum.map(fn {_id, _state, changeset} -> - changeset - |> Changeset.apply_action!(:insert) - |> Map.replace!(:conflict?, true) - end) - - {:ok, old_jobs ++ new_jobs} - end - - defp apply_conflicts(_repo, %{new_jobs: new_jobs}) do - new_jobs = - Enum.map(new_jobs, fn - %{meta: %{"uniq_conflict" => true}} = job -> %{job | conflict?: true} - job -> job - end) - - {:ok, new_jobs} - end - - # Time Helpers - - defp seconds_from_now(seconds), do: DateTime.add(utc_now(), seconds, :second) - - defp to_unix(datetime), do: DateTime.to_unix(datetime, :microsecond) - - # Xact Helpers - - defp jittery_sleep do - @xact_expected_delay - |> Backoff.jitter() - |> Process.sleep() - end - - defp transaction(conf, fun_or_multi, prod_opts) do - opts = [ - delay: Map.get(prod_opts, :xact_delay, 1000), - retry: Map.get(prod_opts, :xact_retry, 5), - timeout: Map.get(prod_opts, :xact_timeout, :timer.seconds(30)), - expected_delay: @xact_expected_delay, - expected_retry: 500 - ] - - Repo.transaction(conf, fun_or_multi, opts) - end -end diff --git a/vendor/oban_pro/lib/oban/pro/exceptions.ex b/vendor/oban_pro/lib/oban/pro/exceptions.ex deleted file mode 100644 index bed23340..00000000 --- a/vendor/oban_pro/lib/oban/pro/exceptions.ex +++ /dev/null @@ -1,12 +0,0 @@ -defmodule Oban.Pro.WorkflowError do - @moduledoc false - - defexception [:message, :reason] - - @impl Exception - def exception(reason) do - message = "upstream job was #{reason}, workflow can't complete" - - %__MODULE__{message: message, reason: reason} - end -end diff --git a/vendor/oban_pro/lib/oban/pro/flusher.ex b/vendor/oban_pro/lib/oban/pro/flusher.ex deleted file mode 100644 index e6b5de42..00000000 --- a/vendor/oban_pro/lib/oban/pro/flusher.ex +++ /dev/null @@ -1,24 +0,0 @@ -defmodule Oban.Pro.Flusher do - @moduledoc false - - alias Oban.{Config, Job} - - @doc """ - Construct an MFA for flushing accumulated operations after the Smart engine transaction commits. - - By convention, the function should be named `:on_flush`. - """ - @callback to_flush_mfa(Job.t(), Config.t()) :: :ignore | mfa() - - @doc """ - Extract flush handlers from all registered flushing modules. - - The flush order is important. Workflow flushing must complete before Batch flushing to ensure - batch wrapped workflow handlers are triggered. - """ - def get_flush_handlers(job, conf) do - [Oban.Pro.Workflow, Oban.Pro.Batch, Oban.Pro.Stages.Chain] - |> Enum.map(fn handler -> handler.to_flush_mfa(job, conf) end) - |> Enum.reject(&(&1 == :ignore)) - end -end diff --git a/vendor/oban_pro/lib/oban/pro/handler.ex b/vendor/oban_pro/lib/oban/pro/handler.ex deleted file mode 100644 index 38751182..00000000 --- a/vendor/oban_pro/lib/oban/pro/handler.ex +++ /dev/null @@ -1,13 +0,0 @@ -defmodule Oban.Pro.Handler do - @moduledoc false - - @doc """ - Attach handler hooks. - """ - @callback on_start() :: any() - - @doc """ - Teardown handler hooks. - """ - @callback on_stop() :: any() -end diff --git a/vendor/oban_pro/lib/oban/pro/limiter.ex b/vendor/oban_pro/lib/oban/pro/limiter.ex deleted file mode 100644 index 35e20e30..00000000 --- a/vendor/oban_pro/lib/oban/pro/limiter.ex +++ /dev/null @@ -1,28 +0,0 @@ -defmodule Oban.Pro.Limiter do - @moduledoc false - - alias Oban.Pro.Producer - alias Oban.{Config, Job} - - @type changes :: %{ - :conf => Config.t(), - :prod => Producer.t(), - :running => map(), - optional(atom()) => any() - } - - @type demand :: non_neg_integer() - - @type limit :: nil | demand | [{demand, term(), term()}] - @type repo :: Ecto.Repo.t() - - @doc """ - Calculate the current demand based on capacity and usage. - """ - @callback check(repo(), changes()) :: {:ok, limit()} - - @doc """ - Record demand usage. - """ - @callback track(Producer.meta(), [Job.t()]) :: Producer.meta() -end diff --git a/vendor/oban_pro/lib/oban/pro/limiters/global.ex b/vendor/oban_pro/lib/oban/pro/limiters/global.ex deleted file mode 100644 index 43fb7e10..00000000 --- a/vendor/oban_pro/lib/oban/pro/limiters/global.ex +++ /dev/null @@ -1,102 +0,0 @@ -defmodule Oban.Pro.Limiters.Global do - @moduledoc false - - @behaviour Oban.Pro.Limiter - - alias Oban.Pro.Partition - - @impl Oban.Pro.Limiter - def check(_repo, %{producer: producer} = changes) when is_map(producer.meta.global_limit) do - tracked = - [producer | changes.all_producers] - |> Enum.uniq_by(& &1.uuid) - |> Enum.map(& &1.meta.global_limit) - |> Enum.filter(&is_map/1) - |> Enum.reduce(%{}, &merge/2) - - allowed = producer.meta.global_limit.allowed - tracked = update_legacy_tracked(tracked) - - if is_nil(producer.meta.global_limit.partition) do - {:ok, combined_demands(allowed, tracked)} - else - {:ok, separate_demands(changes.conf, allowed, producer, tracked)} - end - end - - def check(_repo, _changes), do: {:ok, nil} - - defp merge(global_limit, acc) do - Map.merge(global_limit.tracked, acc, fn _, v1, v2 -> v1 + v2 end) - end - - defp combined_demands(allowed, tracked) do - total = tracked |> Map.values() |> Enum.sum() - - max(allowed - total, 0) - end - - defp separate_demands(conf, allowed, producer, tracked) do - untracked = - for key <- Partition.available_keys(conf, producer.queue, producer.meta.local_limit), - not is_map_key(tracked, key), - into: %{}, - do: {key, allowed} - - allowed = - if producer.meta.global_limit.burst do - size = - tracked - |> Map.merge(untracked) - |> Map.delete("none") - |> map_size() - |> max(1) - - producer.meta.local_limit - |> div(size) - |> max(allowed) - else - allowed - end - - Enum.reduce(tracked, untracked, fn {key, total}, acc -> - case max(allowed - total, 0) do - demand when demand > 0 -> Map.put(acc, key, demand) - _ -> acc - end - end) - end - - # Tracking must be updated after acking and before fetching in order to have accurate limits for - # the current fetch transaction. - def prepare_tracked(%{global_limit: global} = meta, keys) do - tracked = - Enum.reduce(keys, global.tracked, fn key, acc -> - if is_map_key(acc, key) do - Map.update!(acc, key, &(&1 - 1)) - else - acc - end - end) - - put_in(meta.global_limit.tracked, tracked) - end - - @impl Oban.Pro.Limiter - def track(%{global_limit: global} = meta, jobs) when is_map(global) do - tracked = Enum.frequencies_by(jobs, &Partition.get_key(&1, "*")) - - put_in(meta.global_limit.tracked, tracked) - end - - def track(meta, _jobs), do: meta - - defp update_legacy_tracked(%{"count" => count}), do: %{"none" => count} - - defp update_legacy_tracked(tracked) do - Map.new(tracked, fn - {key, %{"count" => count}} -> {key, count} - tuple -> tuple - end) - end -end diff --git a/vendor/oban_pro/lib/oban/pro/limiters/local.ex b/vendor/oban_pro/lib/oban/pro/limiters/local.ex deleted file mode 100644 index a9b498d0..00000000 --- a/vendor/oban_pro/lib/oban/pro/limiters/local.ex +++ /dev/null @@ -1,13 +0,0 @@ -defmodule Oban.Pro.Limiters.Local do - @moduledoc false - - @behaviour Oban.Pro.Limiter - - @impl Oban.Pro.Limiter - def check(_repo, %{producer: producer, running: running}) do - {:ok, producer.meta.local_limit - map_size(running)} - end - - @impl Oban.Pro.Limiter - def track(meta, _jobs), do: meta -end diff --git a/vendor/oban_pro/lib/oban/pro/limiters/rate.ex b/vendor/oban_pro/lib/oban/pro/limiters/rate.ex deleted file mode 100644 index ab6c132f..00000000 --- a/vendor/oban_pro/lib/oban/pro/limiters/rate.ex +++ /dev/null @@ -1,125 +0,0 @@ -defmodule Oban.Pro.Limiters.Rate do - @moduledoc false - - @behaviour Oban.Pro.Limiter - - alias Oban.Pro.Partition - - @impl Oban.Pro.Limiter - def check(_repo, %{producer: producer} = changes) when is_map(producer.meta.rate_limit) do - %{allowed: allowed, period: period, window_time: prev_time} = producer.meta.rate_limit - - curr_time = unix_now() - weight = div(max(period - curr_time - prev_time, 0), period) - - windows = - [producer | changes.all_producers] - |> Enum.uniq_by(& &1.uuid) - |> Enum.map(& &1.meta.rate_limit) - |> Enum.filter(&(is_map(&1) and &1.window_time >= curr_time - &1.period)) - |> Enum.reduce(%{}, &merge/2) - - windows = update_legacy_windows(windows) - - if is_nil(producer.meta.rate_limit.partition) do - {:ok, combined_demands(weight, allowed, windows)} - else - {:ok, separate_demands(changes.conf, weight, allowed, producer, windows)} - end - end - - def check(_repo, _changes), do: {:ok, nil} - - defp merge(rate_limit, acc) do - Map.merge(rate_limit.windows, acc, fn _key, v1, v2 -> - %{"curr_count" => c1, "prev_count" => p1} = v1 - %{"curr_count" => c2, "prev_count" => p2} = v2 - %{"curr_count" => c1 + c2, "prev_count" => p1 + p2} - end) - end - - defp update_legacy_windows(%{"args" => _} = window), do: %{"none" => window} - defp update_legacy_windows(windows), do: windows - - defp combined_demands(weight, allowed, windows) do - total = - Enum.reduce(windows, 0, fn {_, val}, acc -> - acc + (val["prev_count"] * weight + val["curr_count"]) - end) - - max(allowed - total, 0) - end - - defp separate_demands(conf, weight, allowed, producer, windows) do - untracked = - for key <- Partition.available_keys(conf, producer.queue, producer.meta.local_limit), - not is_map_key(windows, key), - into: %{}, - do: {key, allowed} - - Enum.reduce(windows, untracked, fn {key, val}, acc -> - %{"curr_count" => curr, "prev_count" => prev} = val - - case max(allowed - (prev * weight + curr), 0) do - demand when demand > 0 -> Map.put(acc, key, demand) - _ -> acc - end - end) - end - - @impl Oban.Pro.Limiter - def track(%{rate_limit: rate_limit} = meta, jobs) when is_map(rate_limit) do - new_mapping = - Enum.reduce(jobs, %{}, fn job, acc -> - key = Partition.get_key(job, "*") - map = %{"curr_count" => 1, "prev_count" => 0} - - Map.update(acc, key, map, &%{&1 | "curr_count" => &1["curr_count"] + 1}) - end) - - new_windows = - for {key, val} <- new_mapping, - not is_map_key(rate_limit.windows, key), - into: %{}, - do: {key, val} - - prev_unix = rate_limit.window_time - unix_diff = unix_now() - prev_unix - - {mode, next_time} = - cond do - unix_diff > rate_limit.period * 2 -> {:dump_prev, unix_now()} - unix_diff > rate_limit.period -> {:swap_prev, unix_now()} - true -> {:bump_curr, prev_unix} - end - - all_windows = - for {key, window} <- rate_limit.windows, - not match?(%{"curr_count" => 0, "prev_count" => 0}, window), - into: new_windows, - do: {key, put_next_count(key, window, mode, new_mapping)} - - %{meta | rate_limit: %{rate_limit | windows: all_windows, window_time: next_time}} - end - - def track(meta, _jobs), do: meta - - defp put_next_count(key, window, mode, mapping) do - %{"curr_count" => curr_count, "prev_count" => prev_count} = window - - next_count = get_in(mapping, [key, "curr_count"]) || 0 - - {curr_count, prev_count} = - case mode do - :dump_prev -> {next_count, 0} - :swap_prev -> {next_count, curr_count} - :bump_curr -> {curr_count + next_count, prev_count} - end - - %{window | "curr_count" => curr_count, "prev_count" => prev_count} - end - - defp unix_now do - DateTime.to_unix(DateTime.utc_now(), :second) - end -end diff --git a/vendor/oban_pro/lib/oban/pro/migration.ex b/vendor/oban_pro/lib/oban/pro/migration.ex deleted file mode 100644 index a61d9e79..00000000 --- a/vendor/oban_pro/lib/oban/pro/migration.ex +++ /dev/null @@ -1,387 +0,0 @@ -defmodule Oban.Pro.Migration do - @moduledoc """ - Migrations create and modify the database tables and indexes Oban Pro needs to function. - - If you use prefixes, or have multiple instances with different prefixes, you can specify the - prefix and create multiple tables in one migration: - - ## Usage - - To use migrations in your application you'll need to generate an `Ecto.Migration` that wraps - calls to `Oban.Pro.Migration`: - - ```bash - mix ecto.gen.migration add_oban_pro - ``` - - Open the generated migration and delegate the `up/0` and `down/0` functions to - `Oban.Pro.Migration`: - - ```elixir - defmodule MyApp.Repo.Migrations.AddObanPro do - use Ecto.Migration - - def up, do: Oban.Pro.Migration.up() - - def down, do: Oban.Pro.Migration.down() - end - ``` - - This will run all of the necessary migrations for your database. - - Now, run the migration to create the table: - - ```bash - mix ecto.migrate - ``` - - ## Concurrent Migrations - - For systems with many retained jobs, you may want to create indexes concurrently to avoid - blocking writes to your tables during migration. The `only` option allows splitting migrations - into separate steps for schema changes and index creation: - - ```bash - mix ecto.gen.migration add_oban_pro_schemas - mix ecto.gen.migration add_oban_pro_indexes - ``` - - The `schemas` migration will add all necessary tables and columns within a DDL transaction: - - ```elixir - defmodule MyApp.Repo.Migrations.AddObanProSchemas do - use Ecto.Migration - - def up, do: Oban.Pro.Migration.up(only: :schemas) - def down, do: Oban.Pro.Migration.down(only: :schemas) - end - ``` - - Then the `indexes` migration will add indexes concurrently, outside of a transaction: - - ```elixir - defmodule MyApp.Repo.Migrations.AddObanProIndexes do - use Ecto.Migration - - # Not needed with use advisory locks, i.e. `migration_lock: :pg_advisory_lock` - @disable_migration_lock true - @disable_ddl_transaction true - - def up, do: Oban.Pro.Migration.up(only: :indexes) - def down, do: Oban.Pro.Migration.down(only: :indexes) - end - ``` - - #### When to Use Split Migrations - - Use the default, combined approach when: - - 1. Setting up a development or test environment - 2. Working with small tables where index creation is quick - 3. Simplicity is preferred over non-blocking operations - - Choose split migrations when: - - 1. Operating on a database where blocking writes could cause downtime - 2. Working with large tables where index creation would block for an extended period - - ## Upgrading - - Like `Oban`, migrations are versioned and you may need to run the migration again when new - `oban_pro` versions are released. To do this, generate a new migration: - - ```bash - mix ecto.gen.migration upgrade_oban_pro - ``` - - Open the generated migration in your editor and call the `up` and `down` functions on - `Oban.Pro.Migration`, with an optional version number: - - ```elixir - defmodule MyApp.Repo.Migrations.UpgradeObanPro do - use Ecto.Migration - - def up, do: Oban.Pro.Migration.up(version: "1.6.0") - - def down, do: Oban.Pro.Migration.down(version: "1.6.0") - end - ``` - - > #### Differences from Oban : .tip - > - > Unlike Oban, the migration versions correspond directly to the `oban_pro` package version. - > In addition, running `down/0` will only step back one version, not all the way down. - - ## Isolation with Prefixes - - Oban supports namespacing through PostgreSQL schemas, therefore so does Oban Pro. Specify a - prefix within your migration: - - ```elixir - defmodule MyApp.Repo.Migrations.AddPrefixedObanPro do - use Ecto.Migration - - def up, do: Oban.Pro.Migration.up(prefix: "private") - - def down, do: Oban.Pro.Migration.down(prefix: "private") - end - ``` - """ - - @behaviour Oban.Pro.Handler - - use Ecto.Migration - - require Logger - - @versions ~w(1.0.0 1.4.0 1.5.0 1.6.0) - - @impl Oban.Pro.Handler - def on_start do - event = [:oban, :supervisor, :init] - - :telemetry.attach("oban.migration", event, &__MODULE__.handle_event/4, nil) - end - - @impl Oban.Pro.Handler - def on_stop do - :telemetry.detach("oban.migration") - end - - @doc false - def handle_event(_event, _timing, %{conf: conf}, _) do - prefix = conf.prefix - - case :persistent_term.get(__MODULE__, %{}) do - %{^prefix => _} -> - :ok - - versions -> - :persistent_term.put(__MODULE__, Map.put(versions, prefix, true)) - - last = List.last(@versions) - - with_dynamic_repo(conf, fn repo -> - with :off <- Keyword.get(repo.config(), :pool, :off), - curr <- migrated_version(%{prefix: prefix, repo: repo}) do - cond do - is_nil(curr) -> - Logger.warning( - "Oban Pro for prefix #{inspect(prefix)} isn't migrated. Run migrations to create " <> - "the tables and indexes necessary for Oban Pro to function." - ) - - not String.contains?(curr, last) -> - Logger.warning( - "Oban Pro for prefix #{inspect(prefix)} is migrated to #{inspect(curr)}, but the " <> - "latest is #{inspect(last)}. Run migrations to upgrade to the latest version." - ) - - true -> - :ok - end - end - end) - end - end - - defp with_dynamic_repo(%{get_dynamic_repo: nil, repo: repo}, fun) do - fun.(repo) - end - - defp with_dynamic_repo(%{get_dynamic_repo: dyn_fun, repo: repo}, fun) do - prev_repo = repo.get_dynamic_repo() - - dyna_repo = - case dyn_fun do - {mod, fun, arg} -> apply(mod, fun, arg) - fun when is_function(fun, 0) -> fun.() - end - - try do - repo.put_dynamic_repo(dyna_repo) - - fun.(repo) - after - repo.put_dynamic_repo(prev_repo) - end - - fun.(repo) - end - - @doc """ - Run the `up` changes for all migrations between the initial version and the current version. - - ## Example - - Run all migrations up to the current version: - - Oban.Pro.Migration.up() - - Run migrations up to a specified version: - - Oban.Pro.Migration.up(version: "1.4.0") - - Run migrations in an alternate prefix: - - Oban.Pro.Migration.up(prefix: "payments") - """ - def up(opts \\ []) when is_list(opts) do - opts = with_defaults(opts) - curr = migrated_version(opts) - indx = migrated_version_index(curr, opts) - - cond do - is_nil(indx) -> inner_change(0..opts.next, :up, curr, opts) - indx < opts.next -> inner_change((indx + 1)..opts.next, :up, curr, opts) - true -> :ok - end - end - - @doc """ - Run the `down` changes for migrations between the current version and the previous version. - - This behaviour differs from `Oban.Migration` by only stepping down **one version** per - invocation. You must explicitly specify version `1.0.0` to rollback all the way and drop initial - tables. - - ## Example - - Run migrations down from the current version to the previous: - - Oban.Pro.Migration.down() - - Run migrations down to and including a specified version: - - Oban.Pro.Migration.down(version: "1.4.0") - - Run all down migrations back to the original: - - Oban.Pro.Migration.down(version: "1.0.0") - - Run migrations in an alternate prefix: - - Oban.Pro.Migration.down(prefix: "payments") - """ - def down(opts \\ []) when is_list(opts) do - opts = with_defaults(opts) - curr = migrated_version(opts) - indx = migrated_version_index(curr, opts) - - if is_integer(indx) and indx >= opts.next do - inner_change(indx..opts.next//-1, :down, curr, opts) - end - end - - @doc false - def migrated_version(opts) when is_map(opts) do - repo = Map.get_lazy(opts, :repo, &repo/0) - opts = with_defaults(opts) - - query = """ - SELECT pg_catalog.obj_description(pg_class.oid, 'pg_class') - FROM pg_class - LEFT JOIN pg_namespace ON pg_namespace.oid = pg_class.relnamespace - WHERE pg_class.relname = 'oban_producers' - AND pg_namespace.nspname = '#{opts.escaped}' - """ - - case repo.query(query, [], log: false) do - {:ok, %{rows: [[version]]}} when is_binary(version) -> version - _ -> nil - end - end - - defp migrated_version_index(version, opts) do - [schemas_version, indexes_version] = normalize_recorded(version) - - case opts.only do - [:schemas] -> find_version_index(schemas_version) - [:indexes] -> find_version_index(indexes_version) - _ -> find_version_index(schemas_version) - end - end - - defp inner_change(_curr_index..next_index//_ = range, direction, current, opts) do - for index <- range do - version = Enum.at(@versions, index) - migration = Module.concat(Oban.Pro.Migrations, "V#{String.replace(version, ".", "")}") - - {do_ddl, do_idx} = - if direction == :up do - {:up_ddl, :up_idx} - else - {:down_ddl, :down_idx} - end - - if :schemas in opts.only, do: apply(migration, do_ddl, [opts]) - if :indexes in opts.only, do: apply(migration, do_idx, [opts]) - - flush() - end - - case direction do - :up -> record_version(next_index, current, opts) - :down -> record_version(next_index - 1, current, opts) - end - end - - defp record_version(-1, _recorded, _opts), do: :ok - - defp record_version(index, recorded, opts) do - [recorded_schemas, recorded_indexes] = normalize_recorded(recorded) - - prefix = Enum.at(@versions, index) - - versions = - case opts.only do - [:indexes] -> [recorded_schemas, "#{prefix}-indexes"] - [:schemas] -> ["#{prefix}-schemas", recorded_indexes] - _ -> ["#{prefix}-schemas", "#{prefix}-indexes"] - end - - versions = versions |> Enum.filter(& &1) |> Enum.join(",") - - execute "COMMENT ON TABLE #{opts.quoted}.oban_producers IS '#{versions}'" - end - - defp with_defaults(opts) do - prefix = Access.get(opts, :prefix, "public") - unlogged = Access.get(opts, :unlogged, true) - version = Access.get(opts, :version, List.last(@versions)) - - only = - opts - |> Access.get(:only, [:schemas, :indexes]) - |> List.wrap() - - %{ - concurrently: only == [:indexes], - escaped: String.replace(prefix, "'", "\\'"), - next: find_version_index(version), - only: only, - prefix: prefix, - quoted: inspect(prefix), - unlogged: unlogged, - version: version - } - end - - defp find_version_index(nil), do: nil - - defp find_version_index(version) do - version = String.replace(version, ["-schemas", "-indexes"], "") - - Enum.find_index(@versions, &(&1 == version)) - end - - defp normalize_recorded(nil), do: [nil, nil] - - defp normalize_recorded(version) do - case String.split(version, ",", parts: 2) do - [<<_prefix::binary-size(5), "-schemas">> = schemas] -> [schemas, nil] - [version] -> ["#{version}-schemas", "#{version}-indexes"] - recorded -> recorded - end - end -end diff --git a/vendor/oban_pro/lib/oban/pro/migrations/dynamic_partitioner.ex b/vendor/oban_pro/lib/oban/pro/migrations/dynamic_partitioner.ex deleted file mode 100644 index 2f01335d..00000000 --- a/vendor/oban_pro/lib/oban/pro/migrations/dynamic_partitioner.ex +++ /dev/null @@ -1,195 +0,0 @@ -defmodule Oban.Pro.Migrations.DynamicPartitioner do - @moduledoc false - - use Ecto.Migration - - alias Oban.Config - alias Oban.Pro.Plugins.DynamicPartitioner - - def change(opts \\ []) do - case direction() do - :up -> up(opts) - :down -> down(opts) - end - end - - def up(opts \\ []) do - prefix = Keyword.get(opts, :prefix, "public") - quoted = inspect(prefix) - - if Keyword.get(opts, :create_schema, prefix != "public") do - execute "CREATE SCHEMA IF NOT EXISTS #{quoted}", "" - end - - execute """ - ALTER TABLE IF EXISTS #{quoted}.oban_jobs RENAME TO oban_jobs_old - """ - - execute """ - ALTER INDEX IF EXISTS #{quoted}.oban_jobs_pkey RENAME TO oban_jobs_pkey_old - """ - - execute """ - ALTER INDEX IF EXISTS #{quoted}.oban_jobs_args_index RENAME TO oban_jobs_args_index_old - """ - - execute """ - ALTER INDEX IF EXISTS #{quoted}.oban_jobs_meta_index RENAME TO oban_jobs_meta_index_old - """ - - execute """ - DO $$ - BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_type - WHERE typname = 'oban_job_state' - AND typnamespace = '#{quoted}'::regnamespace::oid) THEN - CREATE TYPE #{quoted}.oban_job_state AS ENUM ( - 'available', - 'scheduled', - 'executing', - 'retryable', - 'completed', - 'discarded', - 'cancelled' - ); - END IF; - END$$; - """ - - create table(:oban_jobs, - primary_key: false, - prefix: prefix, - options: "PARTITION BY LIST (state)" - ) do - add :id, :bigserial - add :state, :"#{quoted}.oban_job_state", null: false, default: "available" - add :queue, :text, null: false, default: "default" - add :worker, :text, null: false - add :attempt, :smallint, null: false, default: 0 - add :max_attempts, :smallint, null: false, default: 20 - add :priority, :smallint, null: false, default: 0 - - add :args, :map, null: false - add :meta, :map, null: false - - add :attempted_by, {:array, :text} - add :errors, {:array, :map}, null: false, default: [] - add :tags, {:array, :text}, null: false, default: [] - - add :inserted_at, :utc_datetime_usec, - null: false, - default: fragment("timezone('UTC', now())") - - add :scheduled_at, :utc_datetime_usec, - null: false, - default: fragment("timezone('UTC', now())") - - add :attempted_at, :utc_datetime_usec - add :cancelled_at, :utc_datetime_usec - add :completed_at, :utc_datetime_usec - add :discarded_at, :utc_datetime_usec - end - - create_if_not_exists index(:oban_jobs, [:id], prefix: prefix) - create_if_not_exists index(:oban_jobs, [:args], using: :gin, prefix: prefix) - create_if_not_exists index(:oban_jobs, [:meta], using: :gin, prefix: prefix) - - execute "COMMENT ON TABLE #{quoted}.oban_jobs IS '∞'", "" - - create_if_not_exists table(:oban_peers, primary_key: false, prefix: prefix) do - add :name, :text, null: false, primary_key: true - add :node, :text, null: false - add :started_at, :utc_datetime_usec, null: false - add :expires_at, :utc_datetime_usec, null: false - end - - execute "ALTER TABLE #{quoted}.oban_peers SET UNLOGGED", "" - - # Collocate available and executing to prevent concurrency anomalies such as: - # > ERROR: tuple to be locked was already moved to another partition due to concurrent update - execute """ - CREATE TABLE #{quoted}.oban_jobs_incomplete PARTITION OF #{quoted}.oban_jobs - FOR VALUES IN ('available', 'executing', 'scheduled', 'retryable') - """ - - create_if_not_exists index( - :oban_jobs_incomplete, - [:state, :queue, :priority, :scheduled_at, :id], - prefix: prefix - ) - - date_partition? = fn -> - if Code.ensure_loaded?(Mix), do: Mix.env() in ~w(dev prod)a, else: true - end - - {part_states, date_states} = - if Keyword.get_lazy(opts, :date_partition?, date_partition?) do - {[], ~w(completed cancelled discarded)} - else - {~w(completed cancelled discarded), []} - end - - for state <- part_states do - execute """ - CREATE TABLE #{quoted}.oban_jobs_#{state} PARTITION OF #{quoted}.oban_jobs - FOR VALUES IN ('#{state}') - """ - - create_if_not_exists index("oban_jobs_#{state}", [:queue, :scheduled_at, :id], - prefix: prefix - ) - end - - for state <- date_states do - execute """ - CREATE TABLE #{quoted}.oban_jobs_#{state} PARTITION OF #{quoted}.oban_jobs - FOR VALUES IN ('#{state}') - PARTITION BY RANGE (#{state}_at) - """ - - date = Date.utc_today() - next = Date.add(date, 1) - safe = Calendar.strftime(date, "%Y%m%d") - - execute """ - CREATE TABLE IF NOT EXISTS #{quoted}.oban_jobs_#{state}_#{safe} - PARTITION OF #{quoted}.oban_jobs_#{state} - FOR VALUES FROM ('#{date} 00:00:00') TO ('#{next} 00:00:00') - """ - end - end - - def down(opts \\ []) do - prefix = Keyword.get(opts, :prefix, "public") - quoted = inspect(prefix) - - drop_if_exists table(:oban_jobs, prefix: prefix) - - execute """ - ALTER TABLE IF EXISTS #{quoted}.oban_jobs_old RENAME TO oban_jobs - """ - - execute """ - ALTER INDEX IF EXISTS #{quoted}.oban_jobs_pkey_old RENAME TO oban_jobs_pkey - """ - - execute """ - ALTER INDEX IF EXISTS #{quoted}.oban_jobs_args_index_old RENAME TO oban_jobs_args_index - """ - - execute """ - ALTER INDEX IF EXISTS #{quoted}.oban_jobs_meta_index_old RENAME TO oban_jobs_meta_index - """ - end - - def backfill(opts \\ []) do - if direction() == :up do - # Ensure the oban_jobs_old table exists if this is ran in a single migration - flush() - - DynamicPartitioner.backfill_jobs(conf(), opts) - end - end - - defp conf, do: Config.new(repo: repo()) -end diff --git a/vendor/oban_pro/lib/oban/pro/migrations/v1_0_0.ex b/vendor/oban_pro/lib/oban/pro/migrations/v1_0_0.ex deleted file mode 100644 index 46537ca4..00000000 --- a/vendor/oban_pro/lib/oban/pro/migrations/v1_0_0.ex +++ /dev/null @@ -1,97 +0,0 @@ -defmodule Oban.Pro.Migrations.DynamicCron do - @moduledoc false - - use Ecto.Migration - - def change(_opts \\ []), do: :ok - - def up(_opts \\ []), do: :ok - - def down(_opts \\ []), do: :ok -end - -defmodule Oban.Pro.Migrations.DynamicQueues do - @moduledoc false - - use Ecto.Migration - - def change(_opts \\ []), do: :ok - - def up(_opts \\ []), do: :ok - - def down(_opts \\ []), do: :ok -end - -defmodule Oban.Pro.Migrations.Producers do - @moduledoc false - - use Ecto.Migration - - def change(_opts \\ []), do: :ok - - def up(_opts \\ []), do: :ok - - def down(_opts \\ []), do: :ok -end - -defmodule Oban.Pro.Migrations.V100 do - @moduledoc false - - use Ecto.Migration - - defmacrop utc_now do - quote do - fragment("timezone('utc', now())") - end - end - - def up_ddl(opts) do - create_if_not_exists table(:oban_producers, primary_key: false, prefix: opts.prefix) do - add :uuid, :uuid, primary_key: true, null: false - add :name, :text, null: false - add :node, :text, null: false - add :queue, :text, null: false - add :meta, :map, null: false, default: %{} - - add :started_at, :utc_datetime_usec, null: false, default: utc_now() - add :updated_at, :utc_datetime_usec, null: false, default: utc_now() - end - - if opts.unlogged do - execute "ALTER TABLE #{opts.quoted}.oban_producers SET UNLOGGED" - end - - create_if_not_exists table(:oban_crons, primary_key: false, prefix: opts.prefix) do - add :name, :text, primary_key: true, null: false - add :expression, :text, null: false - add :worker, :text, null: false - add :opts, :map, null: false - add :insertions, {:array, :utc_datetime_usec}, default: [], null: false - add :paused, :boolean, null: false, default: false - add :lock_version, :integer, default: 1 - - add :inserted_at, :utc_datetime_usec, null: false, default: utc_now() - add :updated_at, :utc_datetime_usec, null: false, default: utc_now() - end - - create_if_not_exists table(:oban_queues, primary_key: false, prefix: opts.prefix) do - add :name, :text, primary_key: true, null: false - add :opts, :map, null: false, default: %{} - add :only, :map, null: false, default: %{} - add :lock_version, :integer, default: 1 - - add :inserted_at, :utc_datetime_usec, null: false, default: utc_now() - add :updated_at, :utc_datetime_usec, null: false, default: utc_now() - end - end - - def up_idx(_opts), do: :ok - - def down_ddl(opts) do - drop_if_exists table(:oban_queues, prefix: opts.prefix) - drop_if_exists table(:oban_crons, prefix: opts.prefix) - drop_if_exists table(:oban_producers, prefix: opts.prefix) - end - - def down_idx(_opts), do: :ok -end diff --git a/vendor/oban_pro/lib/oban/pro/migrations/v1_4_0.ex b/vendor/oban_pro/lib/oban/pro/migrations/v1_4_0.ex deleted file mode 100644 index b0a30ac7..00000000 --- a/vendor/oban_pro/lib/oban/pro/migrations/v1_4_0.ex +++ /dev/null @@ -1,60 +0,0 @@ -defmodule Oban.Pro.Migrations.Batch do - @moduledoc false - - use Ecto.Migration - - def change(_opts \\ []), do: :ok - - def up(_opts \\ []), do: :ok - - def down(_opts \\ []), do: :ok -end - -defmodule Oban.Pro.Migrations.Workflow do - @moduledoc false - - use Ecto.Migration - - def change(_opts \\ []), do: :ok - - def up(_opts \\ []), do: :ok - - def down(_opts \\ []), do: :ok -end - -defmodule Oban.Pro.Migrations.V140 do - @moduledoc false - - use Ecto.Migration - - def up_ddl(opts) do - alter table(:oban_crons, prefix: opts.prefix) do - add_if_not_exists :insertions, {:array, :utc_datetime_usec}, default: [] - end - end - - def up_idx(opts) do - create_if_not_exists index( - :oban_jobs, - [:state, "(meta->>'workflow_id')", "(meta->>'name')"], - concurrently: opts.concurrently, - name: :oban_jobs_state_meta_workflow_index, - prefix: opts.prefix, - where: "meta ? 'workflow_id'" - ) - end - - def down_ddl(opts) do - alter table(:oban_crons, prefix: opts.prefix) do - remove_if_exists :insertions, {:array, :utc_datetime_usec} - end - end - - def down_idx(opts) do - drop_if_exists index(:oban_jobs, [], - name: :oban_jobs_state_meta_workflow_index, - concurrently: opts.concurrently, - prefix: opts.prefix - ) - end -end diff --git a/vendor/oban_pro/lib/oban/pro/migrations/v1_5_0.ex b/vendor/oban_pro/lib/oban/pro/migrations/v1_5_0.ex deleted file mode 100644 index af7ac3e8..00000000 --- a/vendor/oban_pro/lib/oban/pro/migrations/v1_5_0.ex +++ /dev/null @@ -1,114 +0,0 @@ -defmodule Oban.Pro.Migrations.V150 do - @moduledoc false - - use Ecto.Migration - - def up_ddl(opts) do - if not partitioned?(opts) do - execute """ - CREATE OR REPLACE FUNCTION #{opts.quoted}.oban_state_to_bit(state #{opts.quoted}.oban_job_state) - RETURNS jsonb AS $$ - SELECT CASE - WHEN state = 'scheduled' THEN '0'::jsonb - WHEN state = 'available' THEN '1'::jsonb - WHEN state = 'executing' THEN '2'::jsonb - WHEN state = 'retryable' THEN '3'::jsonb - WHEN state = 'completed' THEN '4'::jsonb - WHEN state = 'cancelled' THEN '5'::jsonb - WHEN state = 'discarded' THEN '6'::jsonb - END; - $$ LANGUAGE SQL IMMUTABLE STRICT - """ - - alter table(:oban_jobs, prefix: opts.prefix) do - add_if_not_exists :uniq_key, :text, - generated: """ - ALWAYS AS (CASE WHEN meta->'uniq_bmp' @> #{opts.quoted}.oban_state_to_bit(state) THEN meta->>'uniq_key' END) STORED - """ - end - end - end - - def up_idx(opts) do - if partitioned?(opts) do - create_if_not_exists index( - :oban_jobs, - ["(meta->>'uniq_key')"], - concurrently: opts.concurrently, - name: :oban_jobs_unique_index, - prefix: opts.prefix, - where: "meta ? 'uniq_key'" - ) - else - create_if_not_exists index( - :oban_jobs, - [:uniq_key], - concurrently: opts.concurrently, - name: :oban_jobs_unique_index, - prefix: opts.prefix, - unique: true, - where: "uniq_key IS NOT NULL" - ) - end - - create_if_not_exists index( - :oban_jobs, - [:state, "(meta->>'batch_id')", "(meta->>'callback')"], - concurrently: opts.concurrently, - name: :oban_jobs_batch_index, - prefix: opts.prefix, - where: "meta ? 'batch_id'" - ) - - create_if_not_exists index( - :oban_jobs, - [:state, "(meta->>'chain_id')", "(meta->>'on_hold')"], - concurrently: opts.concurrently, - name: :oban_jobs_chain_index, - prefix: opts.prefix, - where: "meta ? 'chain_id'" - ) - end - - def down_ddl(opts) do - alter table(:oban_jobs, prefix: opts.prefix) do - remove_if_exists :uniq_key, :text - end - - execute "DROP FUNCTION IF EXISTS #{opts.quoted}.oban_state_to_bit" - end - - def down_idx(opts) do - drop_if_exists index(:oban_jobs, [], - name: :oban_jobs_batch_index, - concurrently: opts.concurrently, - prefix: opts.prefix - ) - - drop_if_exists index(:oban_jobs, [], - name: :oban_jobs_chain_index, - concurrently: opts.concurrently, - prefix: opts.prefix - ) - - drop_if_exists index(:oban_jobs, [], - name: :oban_jobs_unique_index, - concurrently: opts.concurrently, - prefix: opts.prefix - ) - end - - defp partitioned?(opts) do - query = """ - SELECT EXISTS ( - SELECT 1 - FROM pg_tables - WHERE schemaname = '#{opts.prefix}' AND tablename = 'oban_jobs_completed' - ) - """ - - {:ok, %{rows: [[partitioned?]]}} = repo().query(query, [], log: false) - - partitioned? - end -end diff --git a/vendor/oban_pro/lib/oban/pro/migrations/v1_6_0.ex b/vendor/oban_pro/lib/oban/pro/migrations/v1_6_0.ex deleted file mode 100644 index add75afa..00000000 --- a/vendor/oban_pro/lib/oban/pro/migrations/v1_6_0.ex +++ /dev/null @@ -1,109 +0,0 @@ -defmodule Oban.Pro.Migrations.V160 do - @moduledoc false - - use Ecto.Migration - - def up_ddl(opts) do - # Partitioning - - alter table(:oban_jobs, prefix: opts.prefix) do - add_if_not_exists :partition_key, :text, - generated: """ - ALWAYS AS (CASE WHEN meta ? 'partition_key' THEN meta->>'partition_key' END) STORED - """ - end - - # Dynamic Queues - - alter table(:oban_queues, prefix: opts.prefix) do - add_if_not_exists :hash, :text - end - end - - def up_idx(opts) do - # Partitioning - - create_if_not_exists index( - :oban_jobs, - [:partition_key, :queue, :priority, :scheduled_at, :id], - concurrently: opts.concurrently, - name: :oban_jobs_partition_index, - prefix: opts.prefix, - where: "state = 'available' AND partition_key IS NOT NULL" - ) - - # Workflows - - drop_if_exists index(:oban_jobs, [], - concurrently: opts.concurrently, - name: :oban_jobs_state_meta_workflow_index, - prefix: opts.prefix - ) - - create_if_not_exists index( - :oban_jobs, - [ - "(meta->>'workflow_id')", - :state, - "(meta->>'on_hold')", - "(meta->>'name')" - ], - concurrently: opts.concurrently, - name: :oban_jobs_workflow_index, - prefix: opts.prefix, - where: "meta ? 'workflow_id'" - ) - - create_if_not_exists index( - :oban_jobs, - [ - "(meta->>'sup_workflow_id')", - :state, - "(meta->>'on_hold')" - ], - concurrently: opts.concurrently, - name: :oban_jobs_sup_workflow_index, - prefix: opts.prefix, - where: "meta ? 'sup_workflow_id'" - ) - end - - def down_ddl(opts) do - alter table(:oban_queues, prefix: opts.prefix) do - remove_if_exists :hash, :text - end - - alter table(:oban_jobs, prefix: opts.prefix) do - remove_if_exists :partition_key, :text - end - end - - def down_idx(opts) do - drop_if_exists index(:oban_jobs, [], - name: :oban_jobs_partition_index, - concurrently: opts.concurrently, - prefix: opts.prefix - ) - - drop_if_exists index(:oban_jobs, [], - name: :oban_jobs_workflow_index, - concurrently: opts.concurrently, - prefix: opts.prefix - ) - - drop_if_exists index(:oban_jobs, [], - name: :oban_jobs_sup_workflow_index, - concurrently: opts.concurrently, - prefix: opts.prefix - ) - - create_if_not_exists index( - :oban_jobs, - [:state, "(meta->>'workflow_id')", "(meta->>'name')"], - concurrently: opts.concurrently, - name: :oban_jobs_state_meta_workflow_index, - prefix: opts.prefix, - where: "meta ? 'workflow_id'" - ) - end -end diff --git a/vendor/oban_pro/lib/oban/pro/partition.ex b/vendor/oban_pro/lib/oban/pro/partition.ex deleted file mode 100644 index 402df2b6..00000000 --- a/vendor/oban_pro/lib/oban/pro/partition.ex +++ /dev/null @@ -1,195 +0,0 @@ -defmodule Oban.Pro.Partition do - @moduledoc false - - @behaviour Oban.Pro.Handler - - import Ecto.Query - - alias Ecto.Changeset - alias Oban.Pro.{Producer, Queue, Utils} - alias Oban.{Config, Job, Repo} - - @keys_cache_ttl Application.compile_env(:oban_pro, [__MODULE__, :keys_cache_ttl], 3_000) - @keys_limit_mlt Application.compile_env(:oban_pro, [__MODULE__, :keys_limit_multiplier], 3) - - @conf_tab Module.concat(__MODULE__, Conf) - @keys_tab Module.concat(__MODULE__, Keys) - - @impl Oban.Pro.Handler - def on_start do - :ets.new(@conf_tab, [:set, :named_table, :public, read_concurrency: true]) - :ets.new(@keys_tab, [:set, :named_table, :public, read_concurrency: true]) - - # Clear the cache with a timeout, as not all queues are guaranteed to be running on a given - # node and the cache won't clear naturally. - :timer.apply_interval(@keys_cache_ttl, __MODULE__, :__clear_cache__, [@keys_cache_ttl]) - - :telemetry.attach( - "oban.pro.partition", - [:oban, :engine, :put_meta, :stop], - &__MODULE__.handle_event/4, - nil - ) - end - - @impl Oban.Pro.Handler - def on_stop do - :telemetry.detach("oban.pro.partition") - end - - def __clear_cache__(ttl) do - now = System.monotonic_time(:millisecond) - - :ets.select_delete(@keys_tab, [ - {{:_, :_, :"$3"}, [{:>=, {:-, now, :"$3"}, ttl}], [true]} - ]) - end - - def handle_event(_event, _measure, %{conf: conf, key: key, meta: meta}, nil) do - # Bust the partition cache on any change to a partitionable limit. This is rare enough an - # event that it's better to be safe. - if key in ~w(global_limit rate_limit)a do - :ets.delete(@conf_tab, {conf.name, meta.queue}) - end - end - - @spec get_key(Job.t() | Job.changeset(), any()) :: any() - def get_key(changeset, default \\ nil) - - def get_key(%{changes: %{meta: %{partition_key: key}}}, _default), do: key - def get_key(%{meta: %{"partition_key" => key}}, _default), do: key - def get_key(_changeset, default), do: default - - @spec with_partition_meta(Job.changeset(), Config.t()) :: Job.changeset() - def with_partition_meta(changeset, conf) do - queue = Changeset.get_field(changeset, :queue, "default") - - case config_for_queue(conf, queue) do - :missing -> - changeset - - partition -> - part_meta = %{partition: true, partition_key: gen_key(changeset, partition)} - - meta = - changeset - |> Changeset.get_change(:meta, %{}) - |> Map.merge(part_meta) - - Changeset.put_change(changeset, :meta, meta) - end - end - - @spec gen_key(Job.changeset(), %{fields: [atom()], keys: [atom()]}) :: String.t() - def gen_key(changeset, %{fields: fields, keys: keys}) do - fields - |> Enum.sort() - |> Enum.map(fn - :args -> Utils.take_keys(changeset, :args, keys) - :meta -> Utils.take_keys(changeset, :meta, keys) - :worker -> Changeset.get_field(changeset, :worker) - end) - |> Utils.hash64() - end - - @spec available_keys(Config.t(), String.t(), pos_integer()) :: [String.t()] - def available_keys(conf, queue, limit) do - timed_cache(conf.name, queue, fn -> - query_limit = @keys_limit_mlt * limit - - subquery = - Job - |> where([j], j.state == "available" and j.queue == ^queue) - |> where([j], not is_nil(fragment("partition_key"))) - |> group_by([j], fragment("partition_key")) - |> order_by([j], asc: min(j.priority), asc: min(j.scheduled_at)) - |> select([j], %{partition_key: fragment("partition_key")}) - |> limit(^query_limit) - - partitioned_keys = - subquery - |> subquery() - |> select([s], s.partition_key) - |> then(&Repo.all(conf, &1)) - - ["none" | partitioned_keys] - end) - end - - defp timed_cache(name, queue, fun) do - cache_key = {name, queue} - now = System.monotonic_time(:millisecond) - - case :ets.lookup(@keys_tab, cache_key) do - [{^cache_key, keys, timestamp}] when now - timestamp < @keys_cache_ttl -> - keys - - _ -> - keys = fun.() - :ets.insert(@keys_tab, {cache_key, keys, now}) - keys - end - end - - @doc """ - We can't guarantee that the queue is running locally, running on a current node, or running at - all. This will check the following locations: - - - From a locally running producer - - From another node's producer in the database - - From a DynamicQueues entry in the database - """ - @spec config_for_queue(Config.t(), String.t()) :: :missing | %{fields: list(), keys: list()} - def config_for_queue(conf, queue) do - # Partitioning isn't used for draining and there's no point in hitting the database just to - # find out there's no config available. - if Process.get(:oban_draining) do - :missing - else - case :ets.lookup(@conf_tab, {conf.name, queue}) do - [{_key, value}] -> - value - - [] -> - value = - with :missing <- reg_config(conf, queue), - :missing <- pro_config(conf, queue), - do: dyn_config(conf, queue) - - :ets.insert(@conf_tab, {{conf.name, queue}, value}) - - value - end - end - end - - defp reg_config(conf, queue) do - case Registry.meta(Oban.Registry, {conf.name, {:producer, queue}}) do - {:ok, %{meta: %{global_limit: %{partition: %{} = partition}}}} -> partition - {:ok, %{meta: %{rate_limit: %{partition: %{} = partition}}}} -> partition - _other -> :missing - end - end - - defp pro_config(conf, queue) do - query = - Producer - |> where(queue: ^queue) - |> order_by(desc: :started_at) - |> limit(1) - - case Repo.one(conf, query) do - %{meta: %{global_limit: %{partition: %{} = partition}}} -> partition - %{meta: %{rate_limit: %{partition: %{} = partition}}} -> partition - _other -> :missing - end - end - - defp dyn_config(conf, queue) do - case Repo.get_by(conf, Queue, name: queue) do - %{opts: %{global_limit: %{partition: %{} = partition}}} -> partition - %{opts: %{rate_limit: %{partition: %{} = partition}}} -> partition - _other -> :missing - end - end -end diff --git a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_cron.ex b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_cron.ex deleted file mode 100644 index 3a13d796..00000000 --- a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_cron.ex +++ /dev/null @@ -1,793 +0,0 @@ -defmodule Oban.Pro.Plugins.DynamicCron do - @moduledoc """ - Enhanced cron scheduling that's configurable at runtime across your entire cluster. - - `DynamicCron` can configure cron scheduling before boot or during runtime, globally, with - scheduling guarantees and per-entry timezone overrides. It's an ideal solution for applications - that can't miss a cron job or must dynamically start and manage scheduled jobs at runtime. - - ## Using and Configuring - - To begin using `DynamicCron`, add the module to your list of Oban plugins in `config.exs`: - - ```elixir - config :my_app, Oban, - plugins: [Oban.Pro.Plugins.DynamicCron] - ... - ``` - - By itself, without providing a crontab or dynamically inserting cron entries, the plugin doesn't - have anything to schedule. To get scheduling started, provide a list of `{cron, worker}` or - `{cron, worker, options}` tuples to the plugin. The syntax is identical to Oban's built in - `:crontab` option, which means you can _probably_ copy an existing standard `:crontab` list into - the plugin's `:crontab`. - - ```elixir - plugins: [{ - Oban.Pro.Plugins.DynamicCron, - crontab: [ - {"* * * * *", MyApp.MinuteJob}, - {"0 * * * *", MyApp.HourlyJob, queue: :scheduled}, - {"0 0 * * *", MyApp.DailyJob, max_attempts: 1}, - {"0 12 * * MON", MyApp.MondayWorker, tags: ["scheduled"]}, - {"@daily", MyApp.AnotherDailyWorker} - ] - }] - ``` - - > #### Beware of Conflicts {: .tip} - > - > While the syntax is syntactically identical to Oban's basic `:crontab`, you can't copy an - > existing `:crontab` if you have multiple entries using the same worker. `DynamicCron` - > requires unique names for all entries, which defaults to the worker's module name. Multiple - > entries with the same worker will cause naming conflicts unless you provide explicit `:name` - > overrides for each duplicate worker. - - Now, when dynamic cron initializes, it will persist those cron entries to the database and start - scheduling them according to their CRON expression. The plugin's `crontab` format is nearly - identical to Oban's standard crontab, with a few important enhancements we'll look at soon. - - Each of the crontab entries are persisted to the database and referenced globally, by all the - other connected Oban instances. That allows us to insert, update, or delete cron entries at any - time. In fact, changing the schedule or options of an entry in the crontab provided to the - plugin will automatically update the persisted entry. To demonstrate, let's modify the - `MinuteJob` we specified so that it runs every other minute in the `:scheduled` queue: - - ```elixir - crontab: [ - {"*/2 * * * *", MyApp.MinuteJob, queue: :scheduled}, - ... - ] - ``` - - Now it isn't really a "minute job" any more, and the name is no longer suitable. However, we - didn't provide a name for the entry and it's using the module name instead. To provide more - flexibility we can add a `:name` overrride, then we can update the worker's name as well: - - ```elixir - crontab: [ - {"*/2 * * * *", MyApp.FrequentJob, name: "frequent", queue: :scheduled}, - ... - ] - ``` - - All entries are referenced by name, which defaults to the worker's name and must be unique. You - may define the same worker multiple times _as long as_ you provide a name override: - - ```elixir - crontab: [ - {"*/3 * * * *", MyApp.BasicJob, name: "client-1", args: %{client_id: 1}}, - {"*/3 * * * *", MyApp.BasicJob, name: "client-2", args: %{client_id: 2}}, - ... - ] - ``` - - To temporarily disable scheduling jobs you can set the `paused` flag: - - - ```elixir - crontab: [ - {"* * * * *", MyApp.BasicJob, paused: true}, - ... - ] - ``` - - To resume the job you must supply `paused: false` (or use `update/2` to resume it manually), - simply removing the `paused` option will have no effect. - - ```elixir - crontab: [ - {"* * * * *", MyApp.BasicJob, paused: false}, - ... - ] - ``` - - It is also possible to delete a persisted entry during initialization by passing the `:delete` - option: - - ```elixir - crontab: [ - {"* * * * *", MyApp.MinuteJob, delete: true}, - ... - ] - ``` - - One or more entries can be deleted this way. Deleting entries is idempotent, nothing will happen - if no matching entry can be found. - - ## Automatic Synchronization - - Synchronizing persisted entries manually requires two deploys: one to flag it with `deleted: - true` and another to clean up the entry entirely. That extra step isn't ideal for applications - that don't insert or delete jobs at runtime. - - To delete entries that are no longer listed in the crontab automatically set the `sync_mode` - option to `:automatic`: - - - ```elixir - [ - sync_mode: :automatic, - crontab: [ - {"0 * * * *", MyApp.BasicJob}, - {"0 0 * * *", MyApp.OtherJob} - ] - ] - ``` - - To remove unwanted entries, simply delete them from the crontab: - - ```diff - crontab: [ - {"0 * * * *", MyApp.BasicJob}, - - {"0 0 * * *", MyApp.OtherJob} - ] - ``` - - With `:automatic` sync, the entry for `MyApp.OtherJob` will be deleted on the next deployment. - - ## Scheduling Guarantees - - Depending on an application's restart timing or as the result of unexpected downtime, a job's - scheduled insert period may be missed. To compensate, enable `guaranteed` mode for the entire - crontab or on an individual bases. - - Here's an example of enabling guaranteed insertion for the entire crontab: - - ```elixir - [ - guaranteed: true, - crontab: [ - {"0 * * * *", MyApp.HourlyJob}, - {"0 0 * * *", MyApp.DailyJob}, - {"0 0 1 * *", MyApp.MonthlyJob}, - {"0 0 1 */2 *", MyApp.BiMonthlyJob} - ] - ] - ``` - - Guaranteed mode may be enabled or disabled for individual jobs instead if it's a poor fit for - the entire crontab: - - ```elixir - [ - crontab: [ - {"@daily", MyApp.DailyJob, guaranteed: true} - {"@monthly", MyApp.MonthlyJob, guaranteed: false} - ] - ] - ``` - - Guaranteed insertion is enforced by cron name and it's durable across option changes. Changing - an entry's `expression` or `timezone` will clear any prior insertions and reset the guarantee. - - ## Overriding the Timezone - - Without any configuration the default timezone is `Etc/UTC`. You can override that for all cron - entries by passing a `timezone` option to the plugin: - - ```elixir - plugins: [{ - Oban.Pro.Plugins.DynamicCron, - timezone: "America/Chicago", - # ... - ``` - - You can also override the timezone for individual entries by passing it as an option to the - `crontab` list or to `DynamicCron.insert/1`: - - ```elixir - DynamicCron.insert([ - {"0 0 * * *", MyApp.Pinger, name: "oslo", timezone: "Europe/Oslo"}, - {"0 0 * * *", MyApp.Pinger, name: "chicago", timezone: "America/Chicago"}, - {"0 0 * * *", MyApp.Pinger, name: "zagreb", timezone: "Europe/Zagreb"} - ]) - ``` - - ## Runtime Updates - - Dynamic cron entries are persisted to the database, making it easy to manipulate them through - typical CRUD operations. The `DynamicCron` plugin provides convenience functions to simplify - working those operations. - - The `insert/1` function takes a list of one or more tuples with the same `{expression, worker}` - or `{expression, worker, options}` format as the plugin's `crontab` option: - - ```elixir - DynamicCron.insert([ - {"0 0 * * *", MyApp.GenericWorker}, - {"* * * * *", MyApp.ClientWorker, name: "client-1", args: %{client_id: 1}}, - {"* * * * *", MyApp.ClientWorker, name: "client-2", args: %{client_id: 2}}, - {"* * * * *", MyApp.ClientWorker, name: "client-3", args: %{client_id: 3}} - ]) - ``` - - Be aware that `insert/1` acts like an "upsert", making it possible to modify existing entries if - the worker or name matches. - - ## Isolation and Namespacing - - All `DynamicCron` functions have an alternate clause that accepts an Oban instance name as the - first argument. This is in line with base `Oban` functions such as `Oban.insert/2`, which allow - you to seamlessly work with multiple Oban instances and across multiple database prefixes. For - example, you can use `all/1` to list all cron entries for the instance named `ObanPrivate`: - - ```elixir - entries = DynamicCron.all(ObanPrivate) - ``` - - Likewise, to insert a new entry using the configuration associated with the `ObanPrivate` - instance: - - ```elixir - {:ok, _} = DynamicCron.insert(ObanPrivate, [{"* * * * *", PrivateWorker}]) - ``` - - ## Instrumenting with Telemetry - - The `DynamicCron` plugin adds the following metadata to the `[:oban, :plugin, :stop]` event: - - * `:jobs` - a list of jobs that were inserted into the database - """ - - @behaviour Oban.Plugin - - use GenServer - - import Ecto.Query, only: [order_by: 2, where: 2, where: 3] - - alias Oban.Cron.Expression - alias Oban.Plugins.Cron, as: CronPlugin - alias Oban.Pro.Cron, as: CronEntry - alias Oban.{Job, Peer, Repo, Validation, Worker} - - require Logger - - @type cron_expr :: String.t() - @type cron_name :: String.t() | atom() - @type cron_opt :: [ - args: Job.args(), - expression: cron_expr(), - guaranteed: boolean(), - max_attempts: pos_integer(), - paused: boolean(), - priority: 0..9, - meta: map(), - name: cron_name(), - queue: atom() | String.t(), - tags: Job.tags(), - timezone: String.t() - ] - @type cron_input :: {cron_expr(), module()} | {cron_expr(), module(), cron_opt()} - - @type sync_mode :: :manual | :automatic - - @type option :: [ - conf: Oban.Config.t(), - guaranteed: boolean(), - crontab: [cron_input()], - name: Oban.name(), - sync_mode: sync_mode(), - timezone: Calendar.time_zone(), - timeout: timeout() - ] - - defstruct [ - :conf, - :timer, - crontab: [], - guaranteed: false, - rebooted: false, - sync_mode: :manual, - timezone: "Etc/UTC" - ] - - defguardp is_name(name) when is_binary(name) or (is_atom(name) and not is_nil(name)) - - defmacrop maybe_clear_insertions(cron, expression, timezone) do - quote do - fragment( - "CASE WHEN ? != ? OR COALESCE(?->>'timezone', 'NA') != COALESCE(?, 'NA') THEN '{}' ELSE ? END", - field(unquote(cron), :expression), - unquote(expression), - field(unquote(cron), :opts), - unquote(timezone), - field(unquote(cron), :insertions) - ) - end - end - - @doc false - def child_spec(args), do: super(args) - - @impl Oban.Plugin - @spec start_link([option()]) :: GenServer.on_start() - def start_link(opts) do - {name, opts} = - opts - |> Keyword.update(:crontab, [], &normalize_crontab/1) - |> Keyword.delete(:insert_history) - |> Keyword.delete(:timeout) - |> Keyword.pop(:name) - - GenServer.start_link(__MODULE__, struct!(__MODULE__, opts), name: name) - end - - @impl Oban.Plugin - def validate(opts) do - Validation.validate_schema(opts, - conf: :any, - crontab: {:custom, &validate_crontab(&1, opts)}, - guaranteed: :boolean, - insert_history: :pos_integer, - name: :any, - sync_mode: {:enum, [:manual, :automatic]}, - timezone: :timezone, - timeout: :timeout - ) - end - - @impl Oban.Plugin - def format_logger_output(_conf, %{jobs: jobs}), do: %{jobs: Enum.map(jobs, & &1.id)} - - @doc """ - Used to retrieve all persisted cron entries. - - The `all/0` function is provided as a convenience to inspect persisted entries. - - ## Examples - - Return a list of cron schemas with raw attributes: - - entries = DynamicCron.all() - """ - @spec all(term()) :: [Ecto.Schema.t()] - def all(oban_name \\ Oban) do - oban_name - |> Oban.config() - |> all_entries() - end - - @doc """ - Insert cron entries into the database to start scheduling new jobs. - - Be aware that `insert/1` acts like an "upsert", making it possible to modify existing entries if - the worker or name matches. Still, it is better to use `update/2` to make targeted updates. - - ## Examples - - Insert a list of tuples with the same `{expression, worker}` or `{expression, worker, options}` - format as the plugin's `crontab` option. - - DynamicCron.insert([ - {"0 0 * * *", MyApp.GenericWorker}, - {"* * * * *", MyApp.ClientWorker, name: "client-1", args: %{client_id: 1}}, - {"* * * * *", MyApp.ClientWorker, name: "client-2", args: %{client_id: 2}}, - {"* * * * *", MyApp.ClientWorker, name: "client-3", args: %{client_id: 3}} - ]) - """ - @spec insert(term(), [cron_input()]) :: {:ok, [Ecto.Schema.t()]} | {:error, Ecto.Changeset.t()} - def insert(oban_name \\ Oban, [_ | _] = crontab) do - conf = Oban.config(oban_name) - - Repo.transaction(conf, fn -> insert_entries(conf, crontab) end) - rescue - error in [Ecto.InvalidChangesetError] -> - {:error, error.changeset} - end - - @doc """ - Update a single cron entry, as identified by worker or name. - - Any option available when specifying an entry in the `crontab` list or when calling `insert/2` - can be updated—that includes the cron `expression` and the `worker`. - - ## Examples - - The following call demonstrates updating every possible option: - - {:ok, _} = - DynamicCron.update( - "cron-1", - expression: "1 * * * *", - max_attempts: 10, - name: "special-cron", - paused: false, - priority: 0, - queue: "dedicated", - tags: ["client", "scheduled"], - timezone: "Europe/Amsterdam", - worker: Other.Worker, - ) - - Naturally, individual options may be updated instead. For example, set `paused: true` to pause - an entry: - - {:ok, _} = DynamicCron.update(MyApp.ClientWorker, paused: true) - - Since `update/2` operates on a single entry at a time, it is possible to rename an entry without - doing a `delete`/`insert` dance: - - {:ok, _} = DynamicCron.update(MyApp.ClientWorker, name: "client-worker") - - Or, update an entry with a custom entry name already set: - - {:ok, _} = DynamicCron.update("cron-1", name: "special-cron") - """ - @spec update(term(), cron_name(), cron_opt()) :: - {:ok, Ecto.Schema.t()} | {:error, Ecto.Changeset.t() | String.t()} - def update(oban_name \\ Oban, name, opts) when is_name(name) do - oban_name - |> Oban.config() - |> update_entry(Worker.to_string(name), opts) - end - - @doc """ - Delete individual entries, by worker or name. - - Use `delete/1` to remove entries at runtime, rather than hard-coding the `:delete` flag into the - `crontab` list at compile time. - - ## Examples - - With the worker as the entry name: - - {:ok, _} = DynamicCron.delete(Worker) - - With a custom name: - - {:ok, _} = DynamicCron.delete("cron-1") - """ - @spec delete(term(), cron_name()) :: - {:ok, Ecto.Schema.t()} - | {:error, Ecto.Changeset.t() | String.t()} - def delete(oban_name \\ Oban, name) when is_name(name) do - oban_name - |> Oban.config() - |> delete_cron(Worker.to_string(name)) - end - - # Callbacks - - @impl GenServer - def init(state) do - :telemetry.execute([:oban, :plugin, :init], %{}, %{conf: state.conf, plugin: __MODULE__}) - - {:ok, schedule_evaluate(state)} - end - - @impl GenServer - def handle_call(:evaluate, _from, state) do - manage_entries(state) - reload_and_insert(state) - - {:reply, :ok, %{state | rebooted: true}} - end - - @impl GenServer - def handle_info(:evaluate, state) do - if Peer.leader?(state.conf) do - manage_entries(state) - reload_and_insert(state) - - {:noreply, schedule_evaluate(%{state | rebooted: true})} - else - {:noreply, schedule_evaluate(state)} - end - end - - def handle_info(message, state) do - Logger.warning( - message: "Received unexpected message: #{inspect(message)}", - source: :oban_pro, - module: __MODULE__ - ) - - {:noreply, state} - end - - # Validations - - defp normalize_crontab(crontab) do - Enum.map(crontab, fn - {expr, worker} -> {expr, worker, []} - tuple -> tuple - end) - end - - defp validate_crontab(crontab, opts) when is_list(crontab) and is_list(opts) do - automatic? = opts[:sync_mode] == :automatic - - Validation.validate(:crontab, crontab, &validate_crontab(&1, automatic?)) - end - - defp validate_crontab({expression, worker, opts}, automatic?) do - with {:ok, _} <- CronPlugin.parse(expression) do - delete? = Keyword.get(opts, :delete, false) - - cond do - not is_atom(worker) -> - {:error, "expected worker to be an atom, got: #{inspect(worker)}"} - - not delete? and not Code.ensure_loaded?(worker) -> - {:error, "#{inspect(worker)} not found or can't be loaded"} - - not delete? and not function_exported?(worker, :perform, 1) -> - {:error, "#{inspect(worker)} does not implement `perform/1` callback"} - - not Keyword.keyword?(opts) -> - {:error, "options must be a keyword list, got: #{inspect(opts)}"} - - not valid_entry?(opts, automatic?) -> - {:error, - "expected cron options to be one of #{inspect(CronEntry.allowed_opts())}, got: #{inspect(opts)}"} - - not delete? and not valid_job?(worker, opts) -> - {:error, "expected valid job options, got: #{inspect(opts)}"} - - true -> - :ok - end - end - end - - defp validate_crontab({expression, worker}, automatic?) do - validate_crontab({expression, worker, []}, automatic?) - end - - defp validate_crontab(invalid, _automatic?) do - {:error, - "expected crontab entry to be an {expression, worker} or " <> - "{expression, worker, options} tuple, got: #{inspect(invalid)}"} - end - - defp valid_entry?(opts, automatic?) do - string_keys = Enum.map(CronEntry.allowed_opts(), &to_string/1) - - ignore_keys = - if automatic? do - ~w(guaranteed insertions name paused)a - else - ~w(delete guaranteed insertions name paused)a - end - - opts - |> Keyword.drop(ignore_keys) - |> Enum.all?(fn {key, _} -> to_string(key) in string_keys end) - end - - defp valid_job?(worker, opts) do - args = Keyword.get(opts, :args, %{}) - opts = Keyword.drop(opts, ~w(args delete guaranteed insertions name paused timezone)a) - - worker.new(args, opts).valid? - end - - # Scheduling - - defp schedule_evaluate(state) do - timer = Process.send_after(self(), :evaluate, CronPlugin.interval_to_next_minute()) - - %{state | timer: timer} - end - - # Updating - - defp reload_and_insert(state) do - meta = %{conf: state.conf, plugin: __MODULE__} - - :telemetry.span([:oban, :plugin], meta, fn -> - {:ok, jobs} = - state - |> reload_entries() - |> insert_scheduled_jobs(state) - - {:ok, Map.put(meta, :jobs, jobs)} - end) - end - - defp reload_entries(state) do - query = - CronEntry - |> where(paused: false) - |> order_by(asc: :inserted_at) - - state.conf - |> Repo.all(query) - |> Enum.reduce([], fn entry, acc -> - with {:ok, parsed} <- Expression.parse(entry.expression), - {:ok, worker} <- Worker.from_string(entry.worker) do - opts = Keyword.new(entry.opts, fn {key, val} -> {String.to_existing_atom(key), val} end) - insertions = Enum.reject(entry.insertions, &is_nil/1) - - entry = %{entry | insertions: insertions, opts: opts, parsed: parsed, worker: worker} - - [entry | acc] - else - _ -> acc - end - end) - end - - # Inserting - - defp insert_scheduled_jobs(entries, state) do - entries = Enum.filter(entries, &(now?(&1, state) or missed?(&1, state))) - - jobs = - Enum.map(entries, fn entry -> - {args, opts} = Keyword.pop(entry.opts, :args, %{}) - {tz, opts} = Keyword.pop(opts, :timezone, state.timezone) - - meta = %{cron: true, cron_expr: entry.expression, cron_name: entry.name, cron_tz: tz} - - opts = - entry.worker.__opts__() - |> Worker.merge_opts(opts) - |> Keyword.drop([:guaranteed, :timezone]) - |> Keyword.update(:meta, meta, &Map.merge(&1, meta)) - - entry.worker.new(args, opts) - end) - - Repo.transaction(state.conf, fn -> - now = DateTime.utc_now() - query = where(CronEntry, [c], c.name in ^Enum.map(entries, & &1.name)) - - Repo.update_all(state.conf, query, set: [insertions: [now]]) - Oban.insert_all(state.conf.name, jobs) - end) - end - - defp now?(%{parsed: %{reboot?: true}}, %{rebooted: true}), do: false - - defp now?(entry, state) do - datetime = - entry.opts - |> Keyword.get(:timezone, state.timezone) - |> DateTime.now!() - - Expression.now?(entry.parsed, datetime) - end - - defp missed?(entry, state) do - guaranteed? = Keyword.get(entry.opts, :guaranteed, state.guaranteed) - - cond do - [] == entry.insertions -> - false - - not guaranteed? -> - false - - entry.parsed.reboot? -> - false - - true -> - timezone = Keyword.get(entry.opts, :timezone, state.timezone) - - last_inserted_at = List.first(entry.insertions) - - expected_last_at = - entry.parsed - |> Expression.last_at(timezone) - |> DateTime.shift_zone!("Etc/UTC") - - # Compare in UTC and ensure it has been at least 2 minutes, which safely handles DST where - # the same clock time can occur within 1 hour. - DateTime.compare(expected_last_at, last_inserted_at) == :gt and - DateTime.diff(expected_last_at, last_inserted_at, :minute) >= 2 - end - end - - # Entry Management - - defp manage_entries(%{rebooted: true}), do: :ok - - defp manage_entries(%{sync_mode: :automatic} = state) do - Repo.transaction(state.conf, fn -> - delete_missing(state.conf, state.crontab) - insert_entries(state.conf, state.crontab) - end) - end - - defp manage_entries(state) do - Repo.transaction(state.conf, fn -> - {deletes, inserts} = - Enum.split_with(state.crontab, fn {_expr, _work, opts} -> - Keyword.get(opts, :delete) - end) - - delete_entries(state.conf, deletes) - insert_entries(state.conf, inserts) - end) - end - - defp insert_entries(conf, crontab) do - Enum.map(crontab, fn tuple -> - changeset = CronEntry.changeset(tuple) - - replace = - for {key, val} <- changeset.changes, - key in ~w(expression worker opts paused)a, - val != %{}, - do: {key, val} - - expression = changeset.changes.expression - timezone = changeset.changes.opts[:timezone] - - repo_opts = [ - conflict_target: :name, - on_conflict: - CronEntry - |> Ecto.Query.update([_], set: ^replace) - |> Ecto.Query.update([c], - set: [insertions: maybe_clear_insertions(c, ^expression, ^timezone)] - ) - ] - - Repo.insert!(conf, changeset, repo_opts) - end) - end - - defp delete_entries(conf, crontab) do - Repo.delete_all(conf, where(CronEntry, [e], e.name in ^crontab_names(crontab))) - end - - defp delete_missing(conf, crontab) do - Repo.delete_all(conf, where(CronEntry, [e], e.name not in ^crontab_names(crontab))) - end - - defp crontab_names(crontab) do - Enum.map(crontab, fn {_expr, worker, opts} -> - opts - |> Keyword.get(:name, worker) - |> Worker.to_string() - end) - end - - # Queries - - defp all_entries(conf) do - Repo.all(conf, order_by(CronEntry, asc: :inserted_at)) - end - - defp update_entry(conf, name, params) do - with {:ok, entry} <- fetch_entry(conf, name) do - changeset = CronEntry.update_changeset(entry, Map.new(params)) - - Repo.update(conf, changeset) - end - end - - defp delete_cron(conf, name) do - with {:ok, entry} <- fetch_entry(conf, name), do: Repo.delete(conf, entry) - end - - defp fetch_entry(conf, name) do - case Repo.one(conf, where(CronEntry, name: ^name)) do - nil -> {:error, "no cron entry named #{inspect(name)} could be found"} - entry -> {:ok, entry} - end - end -end diff --git a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_lifeline.ex b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_lifeline.ex deleted file mode 100644 index 8a78b3d3..00000000 --- a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_lifeline.ex +++ /dev/null @@ -1,404 +0,0 @@ -defmodule Oban.Pro.Plugins.DynamicLifeline do - @moduledoc """ - The `DynamicLifeline` plugin uses producer records to periodically rescue orphaned jobs, i.e. - jobs that are stuck in the `executing` state because the node was shut down before the job could - finish. In addition, it performs the following maintenance tasks: - - * Discard jobs left `available` with an attempt equal to max attempts - * Repair stuck workflows with deleted dependencies or missed scheduling events - * Repair stuck chains with deleted dependencies or missed scheduling events - * Repair jobs in partitioned queues that are missing a partition key - - Without `DynamicLifeline` you'll need to manually rescue stuck jobs or perform maintenance. - - ## Using the Plugin - - To use the `DynamicLifeline` plugin, add the module to your list of Oban plugins in - `config.exs`: - - ```elixir - config :my_app, Oban, - plugins: [Oban.Pro.Plugins.DynamicLifeline] - ... - ``` - - There isn't any configuration necessary. By default, the plugin rescues orphaned jobs every 1 - minute. If necessary, you can override the rescue interval: - - ```elixir - plugins: [{Oban.Pro.Plugins.DynamicLifeline, rescue_interval: :timer.minutes(5)}] - ``` - - If your system is under high load or produces a multitude of orphans you may wish to increase - the query timeout beyond the `45s` default: - - ```elixir - plugins: [{Oban.Pro.Plugins.DynamicLifeline, timeout: :timer.minutes(1)}] - ``` - - Note that rescuing orphans relies on producer records as used by the `Smart` engine. - - ## Repairing Workflows, Chains, and Partitions - - The plugin automatically repairs stuck workflows, chains, and partitioned jobs during each - rescue cycle. These repair operations ensure: - - * **Workflow repair** — Jobs held waiting for dependencies that were deleted or missed a - scheduling event are released. This handles edge cases where workflow jobs get stuck due to - incomplete dependency resolution. - - * **Chain repair** — Similar to workflows, chain jobs waiting on deleted or stuck predecessors - are released to continue processing. - - * **Partition repair** — Jobs in partitioned queues that are missing a `partition_key` in their - metadata (e.g., jobs scheduled before a queue was partitioned) have their partition key - computed and set automatically. - - By default, each repair operation processes up to 1000 jobs per cycle. You can adjust this - limit with the `repair_limit` option: - - ```elixir - plugins: [{Oban.Pro.Plugins.DynamicLifeline, repair_limit: 500}] - ``` - - ## Identifying Rescued Jobs - - Rescued jobs can be identified by a `rescued` value in `meta`. Each rescue increments the - `rescued` count by one. - - ## Rescuing Exhausted Jobs - - When a job's `attempt` matches its `max_attempts` its retries are considered "exhausted". - Normally, the `DynamicLifeline` plugin transitions exhausted jobs to the `discarded` state and - they won't be retried again. It does this for a couple of reasons: - - 1. To ensure at-most-once semantics. Suppose a long-running job interacted with a non-idempotent - service and was shut down while waiting for a reply; you may not want that job to retry. - - 2. To prevent infinitely crashing BEAM nodes. Poorly behaving jobs may crash the node (through - NIFs, memory exhaustion, etc.) We don't want to repeatedly rescue and rerun a job that - repeatedly crashes the entire node. - - Discarding exhausted jobs may not always be desired. Use the `retry_exhausted` option if you'd - prefer to retry exhausted jobs when they are rescued, rather than discarding them: - - ```elixir - plugins: [{Oban.Pro.Plugins.DynamicLifeline, retry_exhausted: true}] - ``` - - During rescues, with `retry_exhausted: true`, a job's `max_attempts` is incremented and it is - moved back to the `available` state. - - ## Instrumenting with Telemetry - - The `DynamicLifeline` plugin adds the following metadata to the `[:oban, :plugin, :stop]` event: - - * `:rescued_jobs` — a list of jobs transitioned back to `available` - - * `:discarded_jobs` — a list of jobs transitioned to `discarded` - - _Note: jobs only include `id`, `queue`, and `state` fields._ - """ - - @behaviour Oban.Plugin - - use GenServer - - import Ecto.Query - - alias Oban.Pro.Engines.Smart - alias Oban.Pro.Stages.Chain - alias Oban.Pro.{Partition, Producer, Workflow} - alias Oban.{Job, Peer, Repo, Validation} - - require Logger - - @type option :: - {:conf, Oban.Config.t()} - | {:name, Oban.name()} - | {:repair_limit, pos_integer()} - | {:retry_exhausted, boolean()} - | {:rescue_interval, timeout()} - | {:timeout, timeout()} - - defstruct [ - :conf, - :rescue_timer, - repair_limit: 1000, - retry_exhausted: false, - rescue_interval: :timer.minutes(1), - timeout: :timer.seconds(45) - ] - - defmacrop attempted_join(attempted_by, uuid) do - quote do - fragment( - "array_length(?, 1) <> 1 and ? = uuid (?[2])", - unquote(attempted_by), - unquote(uuid), - unquote(attempted_by) - ) - end - end - - defmacrop coalesce_partition(meta) do - quote do - fragment( - "COALESCE(?->'global_limit'->'partition', ?->'rate_limit'->'partition')", - unquote(meta), - unquote(meta) - ) - end - end - - defmacrop track_rescued(meta) do - quote do - fragment( - """ - jsonb_set(?, '{rescued}', (COALESCE(?->>'rescued', '0')::int + 1)::text::jsonb) - """, - unquote(meta), - unquote(meta) - ) - end - end - - @doc false - def child_spec(args), do: super(args) - - @impl Oban.Plugin - @spec start_link([option()]) :: GenServer.on_start() - def start_link(opts) do - {name, opts} = Keyword.pop(opts, :name) - - case opts[:conf] do - %{engine: Smart} -> - GenServer.start_link(__MODULE__, struct!(__MODULE__, opts), name: name) - - %{engine: engine} -> - raise RuntimeError, """ - DynamicLifeline requires the Smart engine to run correctly, but you're using: - - engine: #{inspect(engine)} - - You can either switch to use the Smart or remove DynamicLifeline from your plugins. - """ - end - end - - @impl Oban.Plugin - def validate(opts) do - Validation.validate_schema(opts, - conf: :any, - name: :any, - repair_limit: :pos_integer, - rescue_interval: :pos_integer, - rescue_after: :pos_integer, - retry_exhausted: :boolean, - timeout: :timeout - ) - end - - @impl Oban.Plugin - def format_logger_output(_conf, meta) do - for {key, val} <- meta, - key in ~w(discarded_jobs rescued_jobs)a, - into: %{}, - do: {key, Enum.map(val, & &1.id)} - end - - @impl GenServer - def init(state) do - Process.flag(:trap_exit, true) - - :telemetry.execute([:oban, :plugin, :init], %{}, %{conf: state.conf, plugin: __MODULE__}) - - {:ok, schedule_rescue(state)} - end - - @impl GenServer - def terminate(_reason, state) do - if is_reference(state.rescue_timer), do: Process.cancel_timer(state.rescue_timer) - - :ok - end - - @impl GenServer - def handle_info(:rescue, state) do - if Peer.leader?(state.conf) do - meta = %{conf: state.conf, plugin: __MODULE__} - - :telemetry.span([:oban, :plugin], meta, fn -> - orphan_meta = rescue_orphaned(state) - - :ok = repair_workflows(state) - :ok = repair_chains(state) - :ok = repair_partitions(state) - - {:ok, Map.merge(meta, orphan_meta)} - end) - end - - {:noreply, schedule_rescue(state)} - end - - def handle_info(message, state) do - Logger.warning( - message: "Received unexpected message: #{inspect(message)}", - source: :oban_pro, - module: __MODULE__ - ) - - {:noreply, state} - end - - # Scheduling - - defp schedule_rescue(state) do - timer = Process.send_after(self(), :rescue, state.rescue_interval) - - %{state | rescue_timer: timer} - end - - # Orphan Rescuing - - defp rescue_orphaned(state) do - {res_count, res_jobs} = transition_available(orphan_query(), state) - {dis_count, dis_jobs} = transition_discarded(orphan_query(), state) - {exh_count, exh_jobs} = transition_discarded(stuck_query(), state) - - if state.retry_exhausted do - %{ - discarded_count: 0, - discarded_jobs: [], - rescued_count: res_count + dis_count + exh_count, - rescued_jobs: res_jobs ++ dis_jobs ++ exh_jobs - } - else - %{ - discarded_count: dis_count + exh_count, - discarded_jobs: dis_jobs ++ exh_jobs, - rescued_count: res_count, - rescued_jobs: res_jobs - } - end - end - - defp orphan_query do - subquery = - Job - |> where([j], not is_nil(j.queue) and j.state == "executing") - |> join(:left, [j], p in Producer, on: attempted_join(j.attempted_by, p.uuid)) - |> where([_, p], is_nil(p.uuid)) - - Job - |> join(:inner, [j], x in subquery(subquery), on: j.id == x.id) - |> select([_, x], map(x, [:id, :meta, :queue, :state])) - end - - # We don't know how it's possible for a job to be available with attempts >= max_attempts, but - # we can clean up to prevent clogging a queue at least. - defp stuck_query do - Job - |> where([j], j.state == "available") - |> select([j], map(j, [:id, :meta, :queue, :state])) - end - - defp transition_available(query, state) do - query = - query - |> where([j], j.attempt < j.max_attempts) - |> update([j], set: [meta: track_rescued(j.meta), state: "available"]) - - Repo.update_all(state.conf, query, [], timeout: state.timeout) - catch - :error, %Postgrex.Error{postgres: %{code: :unique_violation, detail: detail}} -> - Smart.clear_uniq_violation(state.conf, detail) - - {0, []} - end - - defp transition_discarded(query, %{retry_exhausted: true} = state) do - query = - query - |> where([j], j.attempt >= j.max_attempts) - |> update([j], - inc: [max_attempts: 1], - set: [meta: track_rescued(j.meta), state: "available"] - ) - - Repo.update_all(state.conf, query, [], timeout: state.timeout) - catch - :error, %Postgrex.Error{postgres: %{code: :unique_violation, detail: detail}} -> - Smart.clear_uniq_violation(state.conf, detail) - - {0, []} - end - - defp transition_discarded(query, state) do - Repo.update_all( - state.conf, - where(query, [j], j.attempt >= j.max_attempts), - [set: [state: "discarded", discarded_at: DateTime.utc_now()]], - timeout: state.timeout - ) - end - - # Composition Repair - - defp repair_workflows(state) do - Workflow.rescue_workflows(state.conf, timeout: state.timeout, limit: state.repair_limit) - end - - defp repair_chains(state) do - Chain.rescue_chains(state.conf, timeout: state.timeout, limit: state.repair_limit) - end - - # Partition Repair - - defp repair_partitions(state) do - for {queue, partition} <- partitioned_queues(state) do - query = - Job - |> where([j], j.queue == ^queue) - |> where([j], j.state in ~w(available scheduled retryable)) - |> where([j], fragment("NOT ? \\? 'partition_key'", j.meta)) - |> limit(^state.repair_limit) - - state.conf - |> Repo.all(query, timeout: state.timeout) - |> Enum.group_by(fn job -> gen_partition_key(job, partition) end) - |> Enum.each(fn {key, grouped_jobs} -> - ids = Enum.map(grouped_jobs, & &1.id) - - Job - |> where([j], j.id in ^ids) - |> update([j], set: [meta: fragment("jsonb_set(?, '{partition_key}', ?)", j.meta, ^key)]) - |> then(&Repo.update_all(state.conf, &1, [], timeout: state.timeout)) - end) - end - - :ok - end - - defp gen_partition_key(job, partition) do - job - |> Ecto.Changeset.change() - |> Partition.gen_key(partition) - end - - defp partitioned_queues(state) do - normalize_partition = fn partition -> - fields = partition |> Map.get("fields", []) |> Enum.map(&String.to_existing_atom/1) - - %{fields: fields, keys: Map.get(partition, "keys", [])} - end - - Producer - |> where([p], fragment("jsonb_typeof(?) = 'object'", coalesce_partition(p.meta))) - |> select([p], {p.queue, coalesce_partition(p.meta)}) - |> distinct(true) - |> then(&Repo.all(state.conf, &1, timeout: state.timeout)) - |> Enum.map(fn {queue, partition} -> {queue, normalize_partition.(partition)} end) - end -end diff --git a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_partitioner.ex b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_partitioner.ex deleted file mode 100644 index 0c725f03..00000000 --- a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_partitioner.ex +++ /dev/null @@ -1,669 +0,0 @@ -defmodule Oban.Pro.Plugins.DynamicPartitioner do - @moduledoc """ - The `DynamicPartitioner` plugin manages a partitioned `oban_jobs` table for optimized query - performance, minimal database bloat, and efficiently pruned historic jobs. - - > #### When to use Partitioned Tables {: .warning} - > - > Partitioning has performance tradeoffs because it doesn't support unique indexes and some - > queries need to check each partition. It is ideally **suited for high throughput** - > applications that run **tens of millions of jobs a day**. Be absolutely sure your application - > will benefit from partitioning. - - Partitioning is only officially supported on Postgres 11 and higher. While older versions of - Postgres support partitioning, they have prohibitive technical limitations and your experience - may vary. - - ## Installation - - Before running the `DynamicPartitioner` plugin, you must run a migration to create a partitioned - `oban_jobs` table to your database. - - > #### Table Name Conflicts {: .info} - > - > Existing `oban_jobs` tables can't be converted to a partitioned table in place and require a - > transition stage. The migration will automatically handle table conflicts by renaming the - > existing table to `oban_jobs_old`. However, if the partitioned table is added to a different - > prefix without a conflict, then the original table is left untouched and both tables are named - > `oban_jobs`. - > - > See the [Backfilling and Migrating](##module-backfilling-and-migrating) section for strategies - > before running migrations. - - ```bash - mix ecto.gen.migration add_partitioned_oban_jobs - ``` - - Open the generated migration in your editor and delegate to the dynamic partitions migration: - - ```elixir - defmodule MyApp.Repo.Migrations.AddPartitionedObanJobs do - use Ecto.Migration - - defdelegate up, to: Oban.Pro.Migrations.DynamicPartitioner - defdelegate down, to: Oban.Pro.Migrations.DynamicPartitioner - end - ``` - - As with the standard `oban_jobs` table, you can optionally provide a `prefix` to "namespace" the table - within your database. Here we specify a `"partitioned"` prefix: - - ```elixir - defmodule MyApp.Repo.Migrations.AddPartitionedObanJobs do - use Ecto.Migration - - def up do - Oban.Pro.Migrations.DynamicPartitioner.up(prefix: "partitioned") - end - - def down do - Oban.Pro.Migrations.DynamicPartitioner.down(prefix: "partitioned") - end - end - ``` - - Run the migration to create the new table: - - ```bash - mix ecto.migrate - ``` - - The new table is partitioned for optimal inserts, ready for you to backfill any existing jobs - and configure retention periods. - - ### Date Partitioning in Test Environments - - To prevent testing errors after migration, the `completed`, `cancelled`, and `discarded` states - are sub-partitioned by date only in `:dev` and `:prod` environments. - - You can explicitly enable date partitioning in other production-like environments with the - `date_partition?` flag: - - ```elixir - Oban.Pro.Migrations.DynamicPartitioner.change(date_partition?: true) - ``` - - ## Backfilling and Migrating - - Below are several recommended strategies for backfilling original jobs into the new partitioned - table. They're listed in order of complexity, beginning with the least invasive approach. - - ### Strategy 1: Don't Backfill at All - - The most performant strategy is to not backfill jobs at all. It's perfectly acceptable to leave - old jobs untouched if you **don't need them for uniqueness checks** or other observability - concerns. Though, in an active system, you'll still want to finish processing jobs in the - original table. - - During the transition period, until all of the original jobs are processed, you'll run two - separate, entirely isolated, Oban instances: - - 1. **Original** — configured with the original prefix, `public` if you never changed it. This - instance will run all of the original queues, without any plugins and it won't insert new - jobs. - - 2. **Partitioned** — configured with the new prefix for the partitioned table, all of your - original queues, plugins, and any other options. - - Here's an example of that configuration: - - ```elixir - queues = [ - default: 10, - other_queue: 10, - and_another: 10 - ] - - config :my_app, Oban.Original, queues: queues - - config :my_app, Oban, - prefix: "partitioned", - queues: queues, - plugins: [ - ... - ``` - - Now, start both Oban instances within your application's supervisor: - - ```diff - children = [ - MyApp.Repo, - {Oban, Application.fetch_env!(:my_app, Oban)}, - + {Oban, Application.fetch_env!(:my_app, Oban.Original)}, - ... - ] - ``` - - New jobs will be inserted into the `partitioned` table while existing jobs keep processing - through the `Oban.Original` instance. Once the all original jobs have executed you're free to - remove the extra instance and [drop the original table](#module-cleaning-up). - - ### Strategy 2: Backfill Old Jobs - - This strategy moves old jobs automatically as part of a migration. The `backfill/1` migration - helper, which delegates to `backfill_jobs/2`, helps move jobs in batches for each state. - - By default, backfilling includes _all states_ and is intended for smaller tables, i.e. 50k-100k - total jobs. - - Backfill by creating an additional migration with ddl transaction disabled: - - ```elixir - defmodule MyApp.Repo.Migrations.BackfillPartitionedObanJobs do - use Ecto.Migration - - @disable_ddl_transaction true - - def change do - Oban.Pro.Migrations.DynamicPartitioner.backfill() - end - end - ``` - - Like all other migrations, `backfill/1` accepts options to control the table's prefix. You can - specify both old and new prefixes to handle situations where the partitioned table lives in a - different prefix: - - ```elixir - def change do - Oban.Pro.Migrations.DynamicPartitioner.backfill(new_prefix: "private", old_prefix: "public") - end - ``` - - For larger tables, or applications that are sensitive to longer migrations, you can split - backfilling between migrations and prioritize in-flight jobs. - - Use the `states` option to restrict backfilling to actively `executing` jobs: - - ```elixir - def change do - Oban.Pro.Migrations.DynamicPartitioner.backfill(states: ~w(executing)) - end - ``` - - For the remaining jobs, you can either use a secondary migration or manually call - `backfill_jobs/1` from your application code. - - See `backfill_jobs/1` for the full range of backfill options including changing the batch size - and automatically sleeping between batches. - - ### Cleaning Up - - After backfilling is complete you can drop the original `oban_jobs` table. Be _very_ careful to - ensure you're dropping the old job table! If the new and old prefix was the same, which it is by - default, then the table has `_old` appended. - - ```elixir - defmodule MyApp.Repo.Migrations.DropStandardObanJobs do - use Ecto.Migration - - def change do - drop_if_exists table(:oban_jobs_old) - end - end - ``` - - ## Using and Configuring - - After running the migration to partition tables, enable the plugin to manage sub-partitions: - - ```elixir - config :my_app, Oban, - plugins: [Oban.Pro.Plugins.DynamicPartitioner] - ... - ``` - - The plugin will preemptively create sub-partitions for finished job states (`completed`, - `cancelled`, `discarded`) as well as prune partitions older than the retention period. By - default, older jobs are retained for 3 days. - - You can override the retention period for states individually. For example, to retain `completed` - jobs for 2 days, `cancelled` for 7, and `discarded` for 30: - - ```elixir - plugins: [{ - Oban.Pro.Plugins.DynamicPartitioner, - retention: [completed: 2, cancelled: 7, discarded: 30] - }] - ``` - - Pruning sub-partitions is an extremely fast operation akin to dropping a table. As a result, - there is zero lingering bloat. It's not advised that you use the `DynamicPruner`, unless - you're pruning a subset jobs aggressively after a few minutes, hours, etc. - - ```diff - plugins: [ - Oban.Pro.Plugins.DynamicPartitioner, - - Oban.Plugins.Pruner, - - Oban.Pro.Plugins.DynamicPruner - ] - ``` - - `DynamicPartitioner` will warn you if the standard `Pruner` is enabled at the same time. - - ### Tuning Partition Management - - The partitioner attempts once an hour to pre-create partitions two days in advance. That - schedule and buffer should be suitable for most applications. However, you can increase the - buffer period and set an alternate schedule if necessary. - - For example, to increase the buffer to 3 days and run at 05:00 in the Europe/Paris timezone: - - ```elixir - plugins: [{ - Oban.Pro.Plugins.DynamicPartitioner, - buffer: 3, - schedule: "0 5 * * *", - timezone: "Europe/Paris" - }] - ``` - - ## Instrumenting with Telemetry - - The `DynamicPartitioner` plugin adds the following metadata to the `[:oban, :plugin, :stop]` event: - - * `:created_count` — the number of partitions created - * `:deleted_count` - the number of partitions deleted - """ - - @behaviour Oban.Plugin - - use GenServer - - alias __MODULE__, as: State - alias Oban.Cron.Expression - alias Oban.Plugins.{Cron, Pruner} - alias Oban.{Config, Peer, Repo, Validation} - - require Logger - - @type retention :: [ - completed: pos_integer(), - cancelled: pos_integer(), - discarded: pos_integer() - ] - - @type option :: - {:conf, Oban.Config.t()} - | {:name, GenServer.name()} - | {:retention, retention()} - | {:schedule, String.t()} - | {:timeout, timeout()} - | {:timezone, String.t()} - - @retention [completed: 3, cancelled: 3, discarded: 3] - @states Keyword.keys(@retention) - - defstruct [ - :conf, - :name, - :retention, - :schedule, - :timer, - buffer: 2, - timeout: :timer.minutes(1), - timezone: "Etc/UTC" - ] - - @doc false - def child_spec(args), do: super(args) - - @impl Oban.Plugin - def start_link(opts) do - GenServer.start_link(__MODULE__, opts, name: opts[:name]) - end - - @impl Oban.Plugin - def validate(opts) do - Validation.validate_schema(opts, - buffer: :pos_integer, - conf: {:custom, &validate_conf/1}, - name: :any, - retention: {:custom, &validate_retention/1}, - schedule: :schedule, - timeout: :timeout, - timezone: :timezone - ) - end - - @impl Oban.Plugin - def format_logger_output(_conf, meta) do - Map.take(meta, ~w(created_count deleted_count)a) - end - - @sub_states ~w(cancelled completed discarded) - - @defaults [ - new_prefix: "public", - old_prefix: "public", - states: ~w(executing available retryable scheduled cancelled discarded completed), - batch_size: 5_000, - batch_sleep: 0 - ] - - @doc false - def backfill_jobs, do: backfill_jobs(Oban, []) - - @doc """ - Backfill jobs from a standard table into a newly partitioned table. - - Backfilling is flexible enough to run against one or more job states, with arbitrary batch - sizes, and without transactional blocks. That allows repeated backfill runs in the face of - restarts or database errors. - - Sub-partitions by date are created for final states (`completed`, `cancelled`, `discarded`) - automatically before jobs are moved. - - ## Options - - * `:new_prefix` — The prefix where the new partitioned `oban_jobs` table resides. Defaults to - `public`. - - * `:old_prefix` — The prefix where the standard `oban_jobs_old` table resides. Defaults to - `public`. - - * `:batch_size` — The number of jobs to move (delete/insert) in a single query. Defaults to a - conservative 5,000 jobs per batch. - - * `:batch_sleep` — The amount of time to sleep between backfill batches in order to minimize - load on the database. Defaults to 0, no downtime between batches. - - * `:states` — A list of job states to backfill jobs from. Defaults to all states. - - ## Examples - - Backfill old jobs across all states in the default `public` prefix: - - DynamicPartitioner.backfill_jobs() - - Restrict backfilling to incomplete job states: - - DynamicPartitioner.backfill_jobs(states: ~w(executing available scheduled retryable)) - - Backfill to and from an alternate prefix: - - DynamicPartitioner.backfill_jobs(old_prefix: "private", new_prefix: "private") - - Backfill using larger batches with half a second between queries: - - DynamicPartitioner.backfill_jobs(batch_size: 20_000, batch_sleep: 500) - """ - @spec backfill_jobs(name_or_conf :: Oban.name() | Config.t(), opts :: Keyword.t()) :: :ok - def backfill_jobs(conf_or_name, opts \\ []) - - def backfill_jobs(%Config{} = conf, opts) when is_list(opts) do - opts = - opts - |> Keyword.validate!(@defaults) - |> Keyword.update!(:states, fn states -> Enum.map(states, &to_string/1) end) - |> Map.new() - - old_table = - if opts.old_prefix == opts.new_prefix do - "oban_jobs_old" - else - "oban_jobs" - end - - conf = %{conf | prefix: opts.new_prefix} - opts = Map.put(opts, :old_table, old_table) - - for state <- opts.states do - backfill_partitions(conf, state, opts) - backfill_jobs(conf, state, opts) - end - - :ok - end - - def backfill_jobs(oban_name, opts) do - oban_name - |> Oban.config() - |> backfill_jobs(opts) - end - - defp backfill_partitions(conf, state, opts) when state in @sub_states do - query = """ - SELECT DISTINCT date_trunc('day', #{state}_at)::date - FROM #{opts.old_prefix}.#{opts.old_table} - WHERE state = '#{state}' - """ - - {:ok, %{rows: rows}} = Repo.query(conf, query) - - rows - |> List.flatten() - |> Enum.reject(&is_nil/1) - |> Enum.each(&create_sub_partition(conf, state, &1)) - end - - defp backfill_partitions(_conf, _state, _opts), do: :ok - - defp backfill_jobs(conf, state, opts) do - query = """ - WITH old_jobs AS ( - DELETE FROM #{opts.old_prefix}.#{opts.old_table} - WHERE id IN ( - SELECT id - FROM #{opts.old_prefix}.#{opts.old_table} - WHERE state = '#{state}' - LIMIT #{opts.batch_size} - ) - RETURNING * - ), new_jobs AS ( - INSERT INTO #{opts.new_prefix}.oban_jobs ( - id, - state, - queue, - worker, - attempt, - max_attempts, - priority, - args, - meta, - attempted_by, - errors, - tags, - inserted_at, - scheduled_at, - attempted_at, - cancelled_at, - completed_at, - discarded_at - ) SELECT - id, - state, - queue, - worker, - attempt, - max_attempts, - priority, - args, - meta, - attempted_by, - errors, - tags, - inserted_at, - scheduled_at, - attempted_at, - cancelled_at, - completed_at, - discarded_at - FROM old_jobs - RETURNING 1 - ) - - SELECT count(*) from new_jobs - """ - - {:ok, %{rows: [[count]]}} = Repo.query(conf, query) - - if count > 0 do - Process.sleep(opts.batch_sleep) - - backfill_jobs(conf, state, opts) - end - end - - @impl GenServer - def init(opts) do - opts = - opts - |> Keyword.update(:retention, @retention, &Keyword.merge(@retention, &1)) - |> Keyword.update(:schedule, Expression.parse!("@hourly"), &Expression.parse!/1) - - state = - State - |> struct!(opts) - |> schedule_manage() - - :telemetry.execute([:oban, :plugin, :init], %{}, %{conf: state.conf, plugin: __MODULE__}) - - {:ok, state, {:continue, :start}} - end - - @impl GenServer - def handle_continue(:start, state) do - manage_tables(state) - - {:noreply, state} - end - - @impl GenServer - def handle_info(:manage, state) do - datetime = DateTime.now!(state.timezone) - - if Peer.leader?(state.conf) and Expression.now?(state.schedule, datetime) do - manage_tables(state) - end - - {:noreply, schedule_manage(state)} - end - - def handle_info(message, state) do - Logger.warning( - message: "Received unexpected message: #{inspect(message)}", - source: :oban, - module: __MODULE__ - ) - - {:noreply, state} - end - - defp manage_tables(%{conf: conf} = state) do - meta = %{conf: conf, plugin: __MODULE__} - - :telemetry.span([:oban, :plugin], meta, fn -> - xact_opts = [retry: 1, timeout: state.timeout] - - {:ok, created} = Repo.transaction(conf, fn -> create_sub_partitions(state) end, xact_opts) - {:ok, deleted} = Repo.transaction(conf, fn -> delete_sub_partitions(state) end, xact_opts) - - counts = %{created_count: length(created), deleted_count: length(deleted)} - - {:ok, Map.merge(meta, counts)} - end) - catch - kind, value -> - Logger.error(fn -> - "[#{__MODULE__}] management error: " <> Exception.format(kind, value, __STACKTRACE__) - end) - end - - defp create_sub_partitions(%{buffer: buffer, conf: conf}) do - for state <- @states, days <- 0..buffer do - date = Date.add(Date.utc_today(), days) - - create_sub_partition(conf, state, date) - end - end - - defp create_sub_partition(conf, state, date) do - quoted = inspect(conf.prefix) - next = Date.add(date, 1) - safe = Calendar.strftime(date, "%Y%m%d") - - command = """ - CREATE TABLE IF NOT EXISTS #{quoted}.oban_jobs_#{state}_#{safe} - PARTITION OF #{quoted}.oban_jobs_#{state} - FOR VALUES FROM ('#{date} 00:00:00') TO ('#{next} 00:00:00') - """ - - query!(conf, command) - end - - defp delete_sub_partitions(%{conf: conf, retention: retention}) do - rules = - Map.new(retention, fn {state, days} -> - stamp = - Date.utc_today() - |> Date.add(-days) - |> Calendar.strftime("%Y%m%d") - |> String.to_integer() - - {to_string(state), stamp} - end) - - query = """ - SELECT table_name - FROM information_schema.tables - WHERE table_schema = '#{conf.prefix}' - AND table_name ~ 'oban_jobs_(cancelled|completed|discarded)_.+' - """ - - for [table] <- query!(conf, query), - <<"oban_jobs_", state::binary-size(9), "_", date::binary-size(8)>> = table, - String.to_integer(date) <= Map.fetch!(rules, state) do - query!(conf, "DROP TABLE IF EXISTS #{inspect(conf.prefix)}.#{table}") - end - end - - defp query!(conf, command) do - case Repo.query(conf, command) do - {:ok, %{rows: rows}} -> rows - {:error, error} -> raise(error) - end - end - - # Scheduling - - defp schedule_manage(state) do - timer = Process.send_after(self(), :manage, Cron.interval_to_next_minute()) - - %{state | timer: timer} - end - - # Validation - - defp validate_conf(conf) do - cond do - conf.engine != Oban.Pro.Engines.Smart -> - {:error, "requires the Smart engine to run, got: #{inspect(conf.engine)}"} - - Keyword.has_key?(conf.plugins, Pruner) -> - {:error, "pruning isn't compatibile with partitioning, found: #{Pruner}"} - - true -> - :ok - end - end - - defp validate_retention(retention) do - keys = Keyword.keys(retention) - vals = Keyword.values(retention) - - cond do - not Keyword.keyword?(retention) -> - {:error, "expected :retention to be a keyword list, got: #{inspect(retention)}"} - - Enum.any?(keys, &(&1 not in @states)) -> - {:error, "expected :retention keys to overlap #{inspect(@states)}, got: #{inspect(keys)}"} - - Enum.any?(vals, &((is_integer(&1) and &1 <= 0) or not is_integer(&1))) -> - {:error, "expected :retention values to be positive integers, got: #{inspect(vals)}"} - - true -> - :ok - end - end -end diff --git a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_prioritizer.ex b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_prioritizer.ex deleted file mode 100644 index fff57c0e..00000000 --- a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_prioritizer.ex +++ /dev/null @@ -1,318 +0,0 @@ -defmodule Oban.Pro.Plugins.DynamicPrioritizer do - @moduledoc """ - The DynamicPrioritizer plugin automatically adjusts job's priorities to ensure all jobs are - eventually processed. - - Using mixed priorities in a queue causes certain jobs to execute before others. For example, a - queue that processes jobs from various customers may prioritize customers that are in a higher - tier or plan. All high priority (`0`) jobs are guaranteed to run before any with lower priority - (`1..9`), which is wonderful for the higher tier customers but can lead to resource starvation. - When there is a constant flow of high priority jobs the lower priority jobs will never get the - chance to run. - - ## Using the Plugin - - To use the `DynamicPrioritizer` plugin add the module to your list of Oban plugins in - `config.exs`: - - ```elixir - config :my_app, Oban, - plugins: [Oban.Pro.Plugins.DynamicPrioritizer] - ... - ``` - - Without any additional options the plugin will automatically increase the priority of any jobs - that are `available` for 5 minutes or more. To reprioritize after less time waiting you can - configure the `:after` time: - - ```elixir - plugins: [{Oban.Pro.Plugins.DynamicPrioritizer, after: :timer.minutes(2)}] - ``` - - Now lower job priorities are bumped after 2 minutes of waiting, and every minute after that. To - lower the reprioritization frequency you can change the `:interval` along with the `:after` - time: - - ```elixir - plugins: [{ - Oban.Pro.Plugins.DynamicPrioritizer, - after: :timer.minutes(10), - interval: :timer.minutes(5) - }] - ``` - - Here we've specified that job priority will increase every 5 minutes after the first 10 minutes - of waiting. - - It's also possible to limit the maximum priority jobs may transition to. Typically, jobs will - converge on `0` priority, as that's the highest priority. Set the `max_priority` option to - change to a lower priority: - - ```elixir - plugins: [{Oban.Pro.Plugins.DynamicPrioritizer, max_priority: 2}] - ``` - - ## Providing Overrides - - The `after` option applies to jobs for any workers across all queues. The `DynamicPrioritizer` - plugin allows you to specify per-queue and per-worker overrides that fine tune reprioritization. - - Let's configure reprioritization for the `analysis` queue so that it nudges jobs after only 1 - minute: - - ```elixir - plugins: [{ - Oban.Pro.Plugins.DynamicPrioritizer, - queue_overrides: [analysis: :timer.minutes(1)] - }] - ``` - - We can also effectively disable reprioritization for all other queues by setting the period to - `:infinity`: - - ```elixir - plugins: [{ - Oban.Pro.Plugins.DynamicPrioritizer, - after: :infinity, - queue_overrides: [analysis: :timer.minutes(1)] - }] - ``` - - If per-queue overrides aren't enough we can override on a per-worker basis instead: - - ```elixir - plugins: [{ - Oban.Pro.Plugins.DynamicPrioritizer, - interval: :timer.seconds(15), - limit: 20_000, - worker_overrides: [ - "MyApp.HighSLAWorker": :timer.seconds(30), - "MyApp.LowSLAWorker": :timer.minutes(10) - ] - }] - ``` - - Naturally you can mix and match overrides to finely control reprioritization: - - ```elixir - plugins: [{ - Oban.Pro.Plugins.DynamicPrioritizer, - interval: :timer.minutes(2), - after: :timer.minutes(5), - queue_overrides: [media: :timer.minutes(10)], - worker_overrides: ["MyApp.HighSLAWorker": :timer.seconds(30)] - }] - ``` - - ## Instrumenting with Telemetry - - The `DynamicPrioritizer` plugin adds the following metadata to the `[:oban, :plugin, - :stop]` event: - - * `:reprioritized_count` — the number of jobs reprioritized - """ - - @behaviour Oban.Plugin - - use GenServer - - import Ecto.Query, only: [join: 5, limit: 2, order_by: 2, where: 3] - - alias Oban.{Job, Peer, Repo, Validation} - - require Logger - - @type option :: [ - after: timeout(), - interval: timeout(), - limit: pos_integer(), - max_priority: 0..9, - queue_overrides: [{atom() | String.t(), timeout()}], - worker_overrides: [{atom() | String.t(), timeout()}] - ] - - defstruct [ - :conf, - :timer, - after: :timer.minutes(5), - interval: :timer.seconds(60), - limit: 10_000, - max_priority: 0, - queue_overrides: [], - worker_overrides: [] - ] - - @doc false - def child_spec(args), do: super(args) - - # Callbacks - - @impl Oban.Plugin - def start_link(opts) do - {name, opts} = Keyword.pop(opts, :name) - - GenServer.start_link(__MODULE__, struct!(__MODULE__, opts), name: name) - end - - @impl Oban.Plugin - def validate(opts) do - Validation.validate_schema(opts, - conf: :any, - name: :any, - after: :timeout, - interval: :pos_integer, - limit: :pos_integer, - max_priority: {:range, 0..9}, - queue_overrides: {:custom, &validate_overrides(:queues, &1)}, - worker_overrides: {:custom, &validate_overrides(:workers, &1)} - ) - end - - @impl Oban.Plugin - def format_logger_output(_conf, meta) do - Map.take(meta, ~w(reprioritized_count)a) - end - - @impl GenServer - def init(state) do - Process.flag(:trap_exit, true) - - :telemetry.execute([:oban, :plugin, :init], %{}, %{conf: state.conf, plugin: __MODULE__}) - - {:ok, schedule_reprioritization(state)} - end - - @impl GenServer - def terminate(_reason, state) do - if is_reference(state.timer), do: Process.cancel_timer(state.timer) - - :ok - end - - @impl GenServer - def handle_info(:reprioritize, state) do - meta = %{conf: state.conf, plugin: __MODULE__} - - :telemetry.span([:oban, :plugin], meta, fn -> - case reprioritize_starved_jobs(state) do - {:ok, count} when is_integer(count) -> - {:ok, Map.put(meta, :reprioritized_count, count)} - - error -> - {:error, Map.put(meta, :error, error)} - end - end) - - {:noreply, schedule_reprioritization(state)} - end - - def handle_info(message, state) do - Logger.warning( - message: "Received unexpected message: #{inspect(message)}", - source: :oban_pro, - module: __MODULE__ - ) - - {:noreply, state} - end - - # Validation - - defp validate_overrides(parent_key, overrides) do - Validation.validate(parent_key, overrides, &validate_override/1) - end - - defp validate_override({key, value}) when is_atom(key) do - Validation.validate_timeout(key, value) - end - - defp validate_override(option) do - {:error, "expected override option to be a tuple, got: #{inspect(option)}"} - end - - # Scheduling - - defp schedule_reprioritization(state) do - %{state | timer: Process.send_after(self(), :reprioritize, state.interval)} - end - - # Queries - - defp reprioritize_starved_jobs(state) do - if Peer.leader?(state.conf) do - Repo.transaction(state.conf, fn -> - queue_counts = - for {queue, period} <- state.queue_overrides do - state - |> base_query() - |> query_for_queues(:any, [to_string(queue)]) - |> query_for_period(period) - |> update_all(state) - end - - worker_counts = - for {worker, period} <- state.worker_overrides do - state - |> base_query() - |> query_for_workers(:any, [to_string(worker)]) - |> query_for_period(period) - |> update_all(state) - end - - default_count = - state - |> base_query() - |> query_for_queues(:not, string_keys(state.queue_overrides)) - |> query_for_workers(:not, string_keys(state.worker_overrides)) - |> query_for_period(state.after) - |> update_all(state) - - [queue_counts, worker_counts] - |> List.flatten() - |> Enum.reduce(default_count, &(&1 + &2)) - end) - else - {:ok, 0} - end - end - - defp string_keys(keyword) do - for {key, _} <- keyword, do: to_string(key) - end - - defp base_query(%{limit: limit, max_priority: max_priority}) do - Job - |> where([j], j.state == "available") - |> where([j], j.priority > ^max_priority) - |> order_by(asc: :id) - |> limit(^limit) - end - - defp query_for_period(query, :infinity) do - where(query, [j], true == false) - end - - defp query_for_period(query, period) do - where(query, [j], j.scheduled_at <= ^to_timestamp(period)) - end - - defp query_for_queues(query, :not, []), do: query - defp query_for_queues(query, :not, queues), do: where(query, [j], j.queue not in ^queues) - defp query_for_queues(query, :any, queues), do: where(query, [j], j.queue in ^queues) - - defp query_for_workers(query, :not, []), do: query - defp query_for_workers(query, :not, workers), do: where(query, [j], j.worker not in ^workers) - defp query_for_workers(query, :any, workers), do: where(query, [j], j.worker in ^workers) - - defp update_all(subquery, state) do - query = join(Job, :inner, [j], x in subquery(subquery), on: j.id == x.id) - - state.conf - |> Repo.update_all(query, inc: [priority: -1]) - |> elem(0) - end - - defp to_timestamp(milliseconds) do - DateTime.add(DateTime.utc_now(), -milliseconds, :millisecond) - end -end diff --git a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_pruner.ex b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_pruner.ex deleted file mode 100644 index 36991ada..00000000 --- a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_pruner.ex +++ /dev/null @@ -1,692 +0,0 @@ -defmodule Oban.Pro.Plugins.DynamicPruner do - @moduledoc """ - DynamicPruner enhances the default Pruner plugin's behaviour by allowing you to customize how - jobs are retained in the jobs table. Where the Pruner operates on a fixed schedule and treats - all jobs the same, with the DynamicPruner, you can use a flexible CRON schedule and provide - custom rules for specific queues, workers, and job states. - - ## Using the Plugin - - To start using the `DynamicPruner` add the module to your list of Oban plugins in `config.exs`: - - ```elixir - config :my_app, Oban, - plugins: [Oban.Pro.Plugins.DynamicPruner] - ... - ``` - - Without any additional options the pruner operates in maximum length mode (`max_len`) and - retains a conservative 1,000 `completed`, `cancelled`, or `discarded` jobs. To increase the - number of jobs retained you can provide your own `mode` configuration: - - ```elixir - plugins: [{Oban.Pro.Plugins.DynamicPruner, mode: {:max_len, 50_000}}] - ``` - - Now the pruner will retain the most recent 50,000 jobs instead. - - A fixed limit on the number of jobs isn't always ideal. Often you want to retain jobs based on - their age instead. For example, if you your application needs to ensure that a duplicate job - hasn't been enqueued within the past 24 hours you need to retain jobs for at least 24 hours; a - fixed limit simply won't work. For that we can use maximum age (`max_age`) mode instead: - - ```elixir - plugins: [{Oban.Pro.Plugins.DynamicPruner, mode: {:max_age, 60 * 60 * 48}}] - ``` - - Here we've specified `max_age` using seconds, where `60 * 60 * 48` is the number of seconds in - two days. - - Calculating the number of seconds in a period isn't especially readable, particularly when you - have numerous `max_age` declarations in overrides (see below). For clarity you can specify the - age's time unit as `:second`, `:minute`, `:hour`, `:day` or `:month`. Here is the same 48 hour - configuration from above, but specified in terms of days: - - ```elixir - plugins: [{Oban.Pro.Plugins.DynamicPruner, mode: {:max_age, {2, :days}}}] - ``` - - Now you can tell exactly how long jobs should be retained, without reverse calculating how many - seconds an expression represents. - - ## Providing Overrides - - The `mode` option is indiscriminate when determining which jobs to prune. It pays no attention - to which queue they are in, what worker the job is for, or which state they landed in. The - `DynamicPruner` allows you to specify per-queue, per-worker and per-state overrides that fine - tune pruning. - - We'll start with a simple example of limiting the total number of retained jobs in the `events` - queue: - - ```elixir - plugins: [{ - Oban.Pro.Plugins.DynamicPruner, - mode: {:max_age, {7, :days}}, - queue_overrides: [events: {:max_len, 1_000}] - }] - ``` - - With this configuration most jobs will be retained for seven days, but we'll only keep the - latest 1,000 jobs in the events queue. We can extend this further and override all of our queues - (and omit the default mode entirely): - - ```elixir - plugins: [{ - Oban.Pro.Plugins.DynamicPruner, - queue_overrides: [ - default: {:max_age, {6, :hours}}, - analysis: {:max_age, {1, :day}}, - events: {:max_age, {10, :minutes}}, - mailers: {:max_age, {2, :weeks}}, - media: {:max_age, {2, :months}} - ] - }] - ``` - - When pruning by queue isn't granular enough you can provide overrides by worker instead: - - ```elixir - plugins: [{ - Oban.Pro.Plugins.DynamicPruner, - worker_overrides: [ - "MyApp.BusyWorker": {:max_age, {1, :day}}, - "MyApp.SecretWorker": {:max_age, {1, :second}}, - "MyApp.HistoricWorker": {:max_age, {1, :month}} - ] - }] - ``` - - You can also override by state, which allows you to keep discarded jobs for inspection while - quickly purging cancelled or successfully completed jobs: - - ```elixir - plugins: [{ - Oban.Pro.Plugins.DynamicPruner, - state_overrides: [ - cancelled: {:max_age, {1, :hour}}, - completed: {:max_age, {1, :day}}, - discarded: {:max_age, {1, :month}} - ] - }] - ``` - - Naturally you can mix and match overrides to finely control job retention: - - ```elixir - plugins: [{ - Oban.Pro.Plugins.DynamicPruner, - mode: {:max_age, {7, :days}}, - queue_overrides: [events: {:max_age, {10, :minutes}}], - state_overrides: [discarded: {:max_age, {2, :days}}], - worker_overrides: ["MyApp.SecretWorker": {:max_age, {1, :second}}] - }] - ``` - - ### Override Precedence - - Overrides are applied sequentially, in this order: - - 1. Queue - 2. State - 3. Worker - 4. Default - - Using the example above, jobs in the `events` queue are deleted first, followed by jobs in the - `discarded` state, then the `MyApp.SecretWorker` worker, and finally any other jobs older than 7 - days that **weren't covered by any overrides**. - - ### Preventing Timeouts with Overrides - - Worker override queries aren't able to use any of Oban's standard indexes. If you're processing - a high volume of jobs, pruning with worker overrides may be extremely slow due to sequential - scans. To prevent timeouts, and speed up pruning altogether, you should add a compound index to - the `oban_jobs` table: - - ```elixir - create_if_not_exists index(:oban_jobs, [:worker, :state, :id], concurrently: true) - ``` - - ## Retaining Jobs Forever - - To retain jobs in a queue, state, or for a particular worker forever (without using something - like `{:max_age, {999, :years}}` use `:infinity` as the length or duration: - - ```elixir - plugins: [{ - Oban.Pro.Plugins.DynamicPruner, - mode: {:max_age, {7, :days}}, - state_overrides: [ - cancelled: {:max_len, :infinity}, - discarded: {:max_age, :infinity} - ] - }] - ``` - - ## Keeping Up With Inserts - - With the default settings the `DynamicPruner` will only delete 10,000 jobs each time it prunes. - The limit exists to prevent connection timeouts and excessive table locks. A busy system can - easily insert more than 10,000 jobs per minute during standard operation. If you find that jobs - are accumulating despite active pruning you can override the `limit`. - - Here we set the delete limit to 25,000 and give it 60 seconds to complete: - - ```elixir - plugins: [{ - Oban.Pro.Plugins.DynamicPruner, - mode: {:max_len, 100_000}, - limit: 25_000, - timeout: :timer.seconds(60) - }] - ``` - - Deleting in PostgreSQL is _very_ fast, and the 10k default is rather conservative. Feel free to - increase the limit to a number that your system can handle. - - ## Setting a Schedule - - By default, pruning happens at the top of every minute based on the CRON schedule `* * * * *`. - You're free to set any CRON schedule you prefer for greater control over when to prune. For - example, to prune once an hour instead: - - ```elixir - plugins: [{Oban.Pro.Plugins.DynamicPruner, schedule: "0 * * * *"}] - ``` - - Or, to prune once a day at midnight in your local timezone: - - ```elixir - plugins: [{ - Oban.Pro.Plugins.DynamicPruner, - limit: 100_000, - schedule: "0 0 * * *", - timezone: "America/Chicago", - timeout: :timer.minutes(1) - }] - ``` - - Pruning less frequently can reduce load on your system, particularly if you're using - multiple [overrides](#providing-overrides). However, be sure to set a higher `limit` and - `timeout` (as shown above) to compensate for more accumulated jobs. - - ## Executing a Callback Before Delete - - Sometimes jobs are a historic record of activity and it's desirable to operate on them _before_ - they're deleted. For example, you may want copy some jobs into cold storage prior to completion. - - To accomplish this, specify a callback to execute before proceeding with the deletion.: - - ```elixir - defmodule DeleteHandler do - def call(job_ids) do - # Use the ids at this point, from within a transaction - end - end - - plugins: [{ - Oban.Pro.Plugins.DynamicPruner, - mode: {:max_age, {7, :days}}, - before_delete: {DeleteHandler, :call, []} - }] - ``` - - The callback receives a list of the ids for the jobs that are about to be deleted. The callback - runs within the same transaction that's used for deletion, and you should keep it quick or move - heavy processing to an async process. Note that because it runs in the same transaction as - deletion, the jobs _won't be available after the callback exits_. - - To pass in extra arguments as "configuration" you can provide args to the callback MFA: - - ```elixir - defmodule DeleteHandler do - import Ecto.Query - - def call(job_ids, storage_name) do - jobs = MyApp.Repo.all(where(Oban.Job, [j], j.id in ^job_ids)) - - Storage.call(storage_name, jobs) - end - end - - before_delete: {DeleteHandler, :call, [ColdStorage]} - ``` - - ## Implementation Notes - - Some additional notes about pruning in general and nuances of the `DynamicPruner` plugin: - - * Pruning is best-effort and performed out-of-band. This means that all limits are soft; jobs - beyond a specified age may not be pruned immediately after jobs complete. - - * Pruning is only applied to jobs that are `completed`, `cancelled` or `discarded` (has reached - the maximum number of retries or has been manually killed). It'll never delete a new, - scheduled, or retryable job. - - * Jobs that are part of an active workflow are retained regardless of their state. A workflow - is considered active if it contains any jobs in `scheduled`, `retryable`, `available`, or - `executing` states. - - * Only a single node will prune at any given time, which prevents potential deadlocks between - transactions. - - ## Instrumenting with Telemetry - - The `DynamicPruner` plugin adds the following metadata to the `[:oban, :plugin, :stop]` event: - - * `:pruned_jobs` - the jobs that were deleted from the database - - _Note: jobs only include `id`, `queue`, and `state` fields._ - """ - - @behaviour Oban.Plugin - - use GenServer - - import Ecto.Query - - alias Oban.Cron.Expression - alias Oban.Plugins.Cron - alias Oban.{Job, Peer, Repo, Validation} - - require Logger - - @type time_unit :: - :second - | :seconds - | :minute - | :minutes - | :hour - | :hours - | :day - | :days - | :week - | :weeks - | :month - | :month - - @type max_age :: pos_integer() | {pos_integer(), time_unit()} - - @type max_len :: pos_integer() - - @type mode :: {:max_age, max_age()} | {:max_len, max_len()} - - @type state_override :: {:completed | :cancelled | :discarded, mode()} - - @type queue_override :: {atom(), mode()} - - @type worker_override :: {module(), mode()} - - @type override :: state_override() | queue_override() | worker_override() - - @type option :: - {:conf, Oban.Config.t()} - | {:schedule, String.t()} - | {:name, Oban.name()} - - @modes [:max_age, :max_len] - @states [:completed, :cancelled, :discarded] - @time_units [ - :second, - :seconds, - :minute, - :minutes, - :hour, - :hours, - :day, - :days, - :week, - :weeks, - :month, - :months - ] - - defstruct [ - :before_delete, - :conf, - :schedule, - :timer, - limit: 10_000, - mode: {:max_len, 1_000}, - queue_overrides: [], - state_overrides: [], - timeout: :timer.seconds(60), - timezone: "Etc/UTC", - worker_overrides: [] - ] - - defmacrop same_workflow(meta_a, meta_b) do - quote do - fragment( - """ - ? \\? 'workflow_id' AND ?->> 'workflow_id' = ? ->> 'workflow_id' - """, - unquote(meta_a), - unquote(meta_a), - unquote(meta_b) - ) - end - end - - defmacrop same_sub_workflow(meta_a, meta_b) do - quote do - fragment( - """ - ? \\? 'sup_workflow_id' AND ?->> 'sup_workflow_id' = ? ->> 'workflow_id' - """, - unquote(meta_a), - unquote(meta_a), - unquote(meta_b) - ) - end - end - - defmacrop same_sup_workflow(meta_a, meta_b) do - quote do - fragment( - """ - ? \\? 'workflow_id' AND ?->> 'workflow_id' = ? ->> 'sup_workflow_id' - """, - unquote(meta_a), - unquote(meta_a), - unquote(meta_b) - ) - end - end - - @doc false - def child_spec(args), do: super(args) - - @impl Oban.Plugin - def start_link(opts) do - {name, opts} = Keyword.pop(opts, :name) - - opts = - opts - |> Keyword.put_new(:schedule, "* * * * *") - |> Keyword.update!(:schedule, &Expression.parse!/1) - - GenServer.start_link(__MODULE__, struct!(__MODULE__, opts), name: name) - end - - @impl Oban.Plugin - def validate(opts) do - opts - |> Keyword.delete(:by_state_timestamp) - |> Validation.validate_schema( - before_delete: {:custom, &validate_before_delete/1}, - conf: :any, - limit: :pos_integer, - mode: {:custom, &validate_mode/1}, - name: :any, - queue_overrides: {:custom, &validate_overrides(:queue_overrides, &1)}, - state_overrides: {:custom, &validate_overrides(:state_overrides, &1)}, - schedule: :schedule, - timeout: :timeout, - timezone: :timezone, - worker_overrides: {:custom, &validate_overrides(:worker_overrides, &1)} - ) - end - - @impl Oban.Plugin - def format_logger_output(_conf, meta) do - Map.take(meta, ~w(pruned_count)a) - end - - @impl GenServer - def init(state) do - Process.flag(:trap_exit, true) - - :telemetry.execute([:oban, :plugin, :init], %{}, %{conf: state.conf, plugin: __MODULE__}) - - {:ok, schedule_prune(state)} - end - - @impl GenServer - def terminate(_reason, state) do - if is_reference(state.timer), do: Process.cancel_timer(state.timer) - - :ok - end - - @impl GenServer - def handle_info(:prune, state) do - meta = %{conf: state.conf, plugin: __MODULE__} - - :telemetry.span([:oban, :plugin], meta, fn -> - with {:ok, extra} <- prune_jobs(state), do: {:ok, Map.merge(meta, extra)} - end) - - {:noreply, schedule_prune(state)} - end - - def handle_info(message, state) do - Logger.warning( - message: "Received unexpected message: #{inspect(message)}", - source: :oban_pro, - module: __MODULE__ - ) - - {:noreply, state} - end - - # Scheduling - - defp schedule_prune(state) do - timer = Process.send_after(self(), :prune, Cron.interval_to_next_minute()) - - %{state | timer: timer} - end - - # Query - - defp prune_jobs(state) do - {:ok, datetime} = DateTime.now(state.timezone) - - if Peer.leader?(state.conf) and Expression.now?(state.schedule, datetime) do - string_queues = string_keys(state.queue_overrides) - string_states = string_keys(state.state_overrides) - string_workers = string_keys(state.worker_overrides) - - queue_pruned = - for {queue, mode} <- state.queue_overrides do - Job - |> query_for_queues(:any, [to_string(queue)]) - |> delete_for_mode(mode, state) - end - - state_pruned = - for {queue_state, mode} <- state.state_overrides do - Job - |> query_for_states(:any, [to_string(queue_state)]) - |> delete_for_mode(mode, state) - end - - worker_pruned = - for {worker, mode} <- state.worker_overrides do - Job - |> query_for_workers(:any, [to_string(worker)]) - |> delete_for_mode(mode, state) - end - - default_pruned = - Job - |> query_for_queues(:not, string_queues) - |> query_for_states(:not, string_states) - |> query_for_workers(:not, string_workers) - |> delete_for_mode(state.mode, state) - - pruned = List.flatten([queue_pruned, state_pruned, worker_pruned, default_pruned]) - - {:ok, %{pruned_count: length(pruned), pruned_jobs: pruned}} - else - {:ok, %{pruned_count: 0, pruned_jobs: []}} - end - end - - defp string_keys(keyword) do - for {key, _} <- keyword, do: to_string(key) - end - - defp query_for_queues(query, :not, []), do: query - defp query_for_queues(query, :not, queues), do: where(query, [j], j.queue not in ^queues) - defp query_for_queues(query, :any, queues), do: where(query, [j], j.queue in ^queues) - - defp query_for_states(query, :not, []), do: query - defp query_for_states(query, :not, states), do: where(query, [j], j.state not in ^states) - defp query_for_states(query, :any, states), do: where(query, [j], j.state in ^states) - - defp query_for_workers(query, :not, []), do: query - defp query_for_workers(query, :not, workers), do: where(query, [j], j.worker not in ^workers) - defp query_for_workers(query, :any, workers), do: where(query, [j], j.worker in ^workers) - - defp delete_for_mode(_query, {_mode, :infinity}, _state), do: [] - - defp delete_for_mode(query, {:max_age, age}, state) do - time = to_timestamp(age) - - query - |> select([j], %{id: j.id, queue: j.queue, state: j.state, rn: 100_000_000}) - |> where( - [j], - (j.state == "completed" and j.scheduled_at < ^time) or - (j.state == "cancelled" and j.cancelled_at < ^time) or - (j.state == "discarded" and j.discarded_at < ^time) - ) - |> delete_all(state.limit + 1, state) - end - - defp delete_for_mode(query, {:max_len, len}, state) do - query - |> where([j], j.state in ~w(cancelled completed discarded)) - |> select([j], %{ - id: j.id, - queue: j.queue, - state: j.state, - rn: fragment("row_number() over (order by id desc)") - }) - |> delete_all(len, state) - end - - defp delete_all(query, offset, state) do - workflow_subquery = - Job - |> where([j], j.state in ~w(scheduled retryable available executing)) - |> where([j], fragment("? \\? 'workflow_id'", parent_as(:jobs).meta)) - |> where( - [j], - same_workflow(j.meta, parent_as(:jobs).meta) or - same_sub_workflow(j.meta, parent_as(:jobs).meta) or - same_sup_workflow(j.meta, parent_as(:jobs).meta) - ) - |> select(1) - - subquery = - query - |> from(as: :jobs) - |> where([j], not exists(workflow_subquery)) - |> order_by(asc: :id) - |> limit(^state.limit) - - query = - Job - |> join(:inner, [j], x in subquery(subquery), on: j.id == x.id and x.rn > ^offset) - |> select([_, x], map(x, [:id, :queue, :state])) - - fun = fn -> - apply_before_delete(query, state) - - Repo.delete_all(state.conf, query, timeout: state.timeout) - end - - {:ok, {_count, deleted}} = Repo.transaction(state.conf, fun, timeout: state.timeout) - - deleted - end - - defp apply_before_delete(query, state) do - with {mod, fun, args} <- state.before_delete do - ids = - state.conf - |> Repo.all(query) - |> Enum.map(& &1.id) - - apply(mod, fun, [ids | args]) - end - - :ok - end - - # Timestamp - - defp to_timestamp(seconds) when is_integer(seconds) do - DateTime.add(DateTime.utc_now(), -seconds, :second) - end - - defp to_timestamp({seconds, :second}), do: to_timestamp(seconds) - defp to_timestamp({seconds, :seconds}), do: to_timestamp(seconds) - defp to_timestamp({minutes, :minute}), do: to_timestamp(minutes * 60) - defp to_timestamp({minutes, :minutes}), do: to_timestamp({minutes, :minute}) - defp to_timestamp({hours, :hour}), do: to_timestamp(hours * 60 * 60) - defp to_timestamp({hours, :hours}), do: to_timestamp({hours, :hour}) - defp to_timestamp({days, :day}), do: to_timestamp(days * 24 * 60 * 60) - defp to_timestamp({days, :days}), do: to_timestamp({days, :day}) - defp to_timestamp({weeks, :week}), do: to_timestamp(weeks * 7 * 24 * 60 * 60) - defp to_timestamp({weeks, :weeks}), do: to_timestamp({weeks, :week}) - defp to_timestamp({months, :month}), do: to_timestamp(months * 30 * 24 * 60 * 60) - defp to_timestamp({months, :months}), do: to_timestamp({months, :month}) - - # Validations - - defp validate_before_delete({mod, fun, args}) - when is_atom(mod) and is_atom(fun) and is_list(args) do - cond do - not Code.ensure_loaded?(mod) -> - {:error, "module #{inspect(mod)} can't be loaded"} - - not function_exported?(mod, fun, length(args) + 1) -> - {:error, "no function #{inspect(fun)} with arity #{length(args) + 1}"} - - true -> - :ok - end - end - - defp validate_before_delete(mfa) do - {:error, "expected :before_delete to be a {module, fun, args} tuple, got: #{inspect(mfa)}"} - end - - defp validate_mode({mode, :infinity}) when mode in @modes, do: :ok - defp validate_mode({mode, int}) when is_integer(int) and mode in @modes, do: :ok - - defp validate_mode({mode, {value, time_unit}}) - when is_integer(value) and mode in @modes and time_unit in @time_units, - do: :ok - - defp validate_mode(mode) do - {:error, "expected :mode to be {:max_len, length} or {:max_age, age}, got: #{inspect(mode)}"} - end - - defp validate_overrides(:state_overrides = parent_key, overrides) do - Validation.validate(parent_key, overrides, fn - {state, mode} when state in @states -> - validate_mode(mode) - - {state, _mode} -> - {:error, "expected state to be included in #{inspect(@states)}, got: #{inspect(state)}"} - - override -> - {:error, "expected #{inspect(parent_key)} to contain a tuple, got: #{inspect(override)}"} - end) - end - - defp validate_overrides(parent_key, overrides) do - Validation.validate(parent_key, overrides, fn - {key, mode} when is_atom(key) -> - validate_mode(mode) - - override -> - {:error, "expected #{inspect(parent_key)} to contain a tuple, got: #{inspect(override)}"} - end) - end -end diff --git a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_queues.ex b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_queues.ex deleted file mode 100644 index 77b34209..00000000 --- a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_queues.ex +++ /dev/null @@ -1,745 +0,0 @@ -defmodule Oban.Pro.Plugins.DynamicQueues do - @moduledoc """ - The `DynamicQueue` plugin extends Oban's basic queue management by persisting queue changes - across restarts, globally, across all connected nodes. It also boasts a declarative syntax for - specifying which nodes a queue will run on. - - `DynamicQueues` are ideal for applications that dynamically start, stop, or modify queues at - runtime. - - ## Using and Configuring - - To begin using `DynamicQueues`, add the module to your list of Oban plugins in `config.exs`: - - ```elixir - config :my_app, Oban, - plugins: [Oban.Pro.Plugins.DynamicQueues] - ... - ``` - - Without providing a list of queues the plugin doesn't have anything to run. The syntax for - specifying dynamic queues is identical to Oban's built-in `:queues` option, which means you can - copy them over: - - ```diff - - queues: [ - - default: 20, - - mailers: [global_limit: 20], - - events: [local_limit: 30, rate_limit: [allowed: 100, period: 60]] - - ] - + plugins: [{ - + Oban.Pro.Plugins.DynamicQueues, - + queues: [ - + default: 20, - + mailers: [global_limit: 20], - + events: [local_limit: 30, rate_limit: [allowed: 100, period: 60]] - + ] - + }] - ``` - - Now, when `DynamicQueues` initializes, it will persist all of the queues to the database and - start supervising them. The `queues` syntax is _nearly_ identical to Oban's standard queues, - with an important enhancement we'll look at shortly. - - Each of the persisted queues are referenced globally, by all other connected Oban instances that - are running the `DynamicQueues` plugin. Changing the queue's name, pausing, scaling, or changing - any other options will automatically update queues across all nodes—and persist across restarts. - - Persisted queues are referenced by name, so you can tweak a queue's options by changing the - definition within your config. For example, to bump the mailer's global limit up to `30`: - - queues: [ - mailers: [global_limit: 30], - ... - ] - - That isn't especially interesting—after all, that's exactly how regular queues work! Dynamic - queues start to shine when you insert, update, or delete them _dynamically_, either through Oban - Web or your own application code. - - ### Synchronizing Queues - - The `:sync_mode` option controls how DynamicQueues handles queue synchronization during startup. - There are two available modes: - - * `:manual` - Only deletes queues explicitly marked with `delete: true` and inserts/updates - the remaining queues. This is the default. - * `:automatic` - Automatically deletes any queues that aren't defined in the configuration and - inserts/updates the remaining queues. - - Here's how to configure each mode: - - ```elixir - # Manual mode (default) - config :my_app, Oban, plugins: [{DynamicQueues, sync_mode: :manual, queues: [...]}] - - # Automatic mode - config :my_app, Oban, plugins: [{DynamicQueues, sync_mode: :automatic, queues: [...]}] - ``` - - In manual mode, you must explicitly mark queues for deletion: - - ```elixir - queues: [ - old_queue: [delete: true], - default: 20, - mailers: [limit: 10] - ] - ``` - - In automatic mode, any queue that exists in the database but isn't defined in the configuration - will be automatically deleted during startup. This is useful when you want to ensure your - runtime queue configuration exactly matches what's defined in your application config. - - > #### Choosing a Sync Mode {: .info} - > - > Use `:manual` mode when you want fine-grained control over queue deletion and want to preserve - > dynamically created queues across restarts. Use `:automatic` mode when you want to ensure your - > queue configuration is the single source of truth and automatically clean up old queues. - > - > In either mode, changes to queues are persisted across restarts **until the configuration is - > changed**. For example, if you change the `global_limit` of a queue through the Web dashboard, - > that change will persist across restarts until you change the `global_limit` in your config. - - ### Limiting Where Queues Run - - Dynamic queues can be configured to run on a subset of available nodes. This is especially - useful when wish to restrict resource-intensive queues to only dedicated nodes. Restriction is - configured through the `only` option, which you use like this: - - queues: [ - basic: [local_limit: 10, only: {:node, :=~, "web|worker"}], - audio: [local_limit: 5, only: {:node, "worker.1"}], - video: [local_limit: 5, only: {:node, "worker.2"}], - learn: [local_limit: 5, only: {:sys_env, "EXLA", "CUDA"}], - store: [local_limit: 1, only: {:sys_env, "WAREHOUSE", true}] - ] - - In this example we've defined five queues, with the following restrictions: - - * `basic` — will run on a node named `web` or `worker` - * `audio` — will only run on a node named `worker.1` - * `video` — will only run on a node named `worker.2` - * `learn` — will run wherever `EXLA=CUDA` is an environment variable - * `store` — will run wherever `WAREHOUSE=true` is an environment variable - - Here are the various match modes, operators, and allowed patterns: - - #### Modes - - * `:node` — matches the node name as set in your Oban config. By default, `node` - is the node's id in a cluster, the hostname outside a cluster, or a `DYNO` - variable on Heroku. - * `:sys_env` — matches a single system environment variable as retrieved by - `System.get_env/1` - - #### Operators - - * `:==` — compares the pattern and runtime value for _equality_ as strings. This - is the default operator if nothing is specified. - * `:!=` — compares the pattern and runtime value for _inequality_ as strings - * `:=~` — treats the pattern as a regex and matches it against a runtime value - - #### Patterns - - * `boolean` — either `true` or `false`, which is stringified before comparison - * `string` — either a literal pattern or a regular expression, depending on the - supplied operator - - ## Runtime Updates - - Dynamic queues are persisted to the database, making it easy to manipulate them directly through - CRUD operations, or indirectly with Oban's queue operations, i.e. `pause_queue/2`, - `scale_queue/2`. - - Explicit queue options will be overwritten on restart, while omitted fields are retained. For - example, consider the following dynamic queue entry: - - queues: [ - default: [limit: 10], - ... - ] - - Scaling that `default` queue up or down at runtime _wouldn't_ persist across a restart, because - the definition will overwrite the `limit`. However, pausing the queue or changing the - `global_limit` _would_ persist because they aren't included in the queue definition. - - # pause, global_limit, etc. will persist, but limit won't - default: [limit: 10] - - # pause will persist, but limit and global_limit won't - default: [limit: 10, global_limit: 20] - - # neither limits nor pausing will persist - default: [limit: 10, global_limit: 20, paused: true] - - See function documentation for `all/0`, `insert/1`, `update/2`, and `delete/1` for more - information about runtime updates. - - ## Enabling Polling Mode - - In environments with restricted connectivity (where PubSub doesn't work) you can still use - DynamicQueues at runtime through polling mode. The polling interval is entirely up to you, as - it's disabled by default. - - config :my_app, Oban, - plugins: [{Oban.Pro.Plugins.DynamicQueues, interval: :timer.minutes(1)}] - - With the interval above each DynamicQueues instance will wake up every minute, check the - database for changes, and start new queues. - - ## Isolation and Namespacing - - All DynamicQueues functions have an alternate clause that accepts an Oban instance name for the - first argument. This matches base `Oban` functions such as `Oban.pause_queue/2`, which allow you - to seamlessly work with multiple Oban instances and across multiple database prefixes. For - example, you can use `all/1` to list all queues for the instance named `ObanPrivate`: - - queues = DynamicQueues.all(ObanPrivate) - - Likewise, to insert a new queue using the configuration associated with the `ObanPrivate` - instance: - - DynamicQueues.insert(ObanPrivate, private: limit: 10) - """ - - @behaviour Oban.Plugin - - use GenServer - - import Ecto.Query - - alias Ecto.Changeset - alias Oban.Pro.Engines.Smart - alias Oban.Pro.{Queue, Utils} - alias Oban.{Config, Midwife, Notifier, Peer, Registry, Repo, Validation} - - require Logger - - @type oban_name :: term() - @type operator :: :== | :!= | :=~ - @type partition :: Smart.partition() - @type pattern :: boolean() | String.t() - @type period :: Smart.period() - @type sync_mode :: :manual | :automatic - @type sys_key :: String.t() - - @type queue_name :: atom() | binary() - @type queue_input :: [{queue_name(), pos_integer() | queue_opts() | [queue_opts()]}] - @type queue_opts :: - {:local_limit, pos_integer()} - | {:global_limit, Smart.global_limit()} - | {:only, only()} - | {:paused, boolean()} - | {:rate_limit, Smart.rate_limit()} - - @type only :: - {:node, pattern()} - | {:node, operator(), pattern()} - | {:sys_env, sys_key(), pattern()} - | {:sys_env, sys_key(), operator(), pattern()} - - defstruct [ - :conf, - :foreman_ref, - :timer, - interval: :infinity, - queues: [], - restart_attempts: 50, - restart_delay: 10, - sync_mode: :manual - ] - - defguardp is_name(name) when is_binary(name) or (is_atom(name) and not is_nil(name)) - - @impl Oban.Plugin - def start_link(opts) do - {name, opts} = Keyword.pop(opts, :name) - - GenServer.start_link(__MODULE__, struct!(__MODULE__, opts), name: name) - end - - @impl Oban.Plugin - def validate(opts) do - Validation.validate_schema(opts, - conf: {:custom, &validate_engine/1}, - name: :any, - interval: :timeout, - queues: {:custom, &validate_queues/1}, - sync_mode: {:enum, [:manual, :automatic]} - ) - end - - @doc """ - Retrieve all persisted queues. - - While it's possible to modify queue's returned from `all/0`, it is recommended that you use - `update/2` to ensure options are cast and validated correctly. - - ## Examples - - Retrieve a list of all queue schemas with persisted attributes: - - DynamicQueues.all() - """ - @spec all(oban_name()) :: [Ecto.Schema.t()] - def all(oban_name \\ Oban) do - oban_name - |> Oban.config() - |> all_queues() - end - - @doc """ - Retrieve a single persisted queue. - - ## Examples - - Retrieve the default queue's schema with persisted attributes: - - DynamicQueues.get(:default) - """ - @spec get(oban_name(), queue_name()) :: nil | Ecto.Schema.t() - def get(oban_name \\ Oban, queue_name) do - oban_name - |> Oban.config() - |> Repo.get_by(Queue, name: to_string(queue_name)) - end - - @doc """ - Persist a list of queue inputs, exactly like the `:queues` option passed as configuration. - - Note that `insert/1` acts like an upsert, making it possible to modify queues if the name - matches. Still, it is better to use `update/2` to make targeted updates. - - ## Examples - - Insert a variety of queues with standard and advanced options: - - DynamicQueues.insert( - basic: 10, - audio: [global_limit: 10], - video: [global_limit: 10], - learn: [local_limit: 5, only: {:node, :=~, "learn"}] - ) - """ - @spec insert(oban_name(), queue_input()) :: - {:ok, [Ecto.Schema.t()]} | {:error, Ecto.Changeset.t()} - def insert(oban_name \\ Oban, [_ | _] = entries) do - conf = Oban.config(oban_name) - - case insert_queues(conf, entries) do - {:ok, changes} -> - inserted = Map.values(changes) - - Enum.each(inserted, ¬ify(conf, :dyn_start, &1.name)) - - {:ok, inserted} - - {:error, _name, changeset, _changes} -> - {:error, changeset} - end - end - - @doc """ - Modify a single queue's options. - - Every option available when inserting queues can be updated. - - ## Examples - - The following call demonstrates updating every possible option: - - DynamicQueues.update( - :video, - local_limit: 5, - global_limit: 20, - rate_limit: [allowed: 10, period: 30, partition: :worker]], - only: {:node, :=~, "media"}, - paused: false - ) - - Updating a single option won't remove other persisted options. If you'd like to - clear an uption you must set them to `nil`: - - DynamicQueues.update(:video, global_limit: nil) - - Since `update/2` operates on a single queue, it is possible to rename a queue - without doing a `delete`/`insert` dance: - - DynamicQueues.update(:video, name: :media) - """ - @spec update(oban_name(), queue_name(), [queue_opts()]) :: - {:ok, Ecto.Schema.t()} | {:error, Ecto.Changeset.t()} - def update(oban_name \\ Oban, name, opts) when is_name(name) do - conf = Oban.config(oban_name) - - with {:ok, queue} <- fetch_queue(conf, to_string(name)), - {:ok, queue} <- update_queue(conf, queue, opts) do - if queue.name == to_string(name) do - opts = - opts - |> Map.new() - |> Map.delete(:only) - |> Map.merge(%{action: :scale, ident: :any, queue: name, update: false}) - - # Manually send the notification because Oban.scale_queue won't allow unknown fields - Notifier.notify(conf.name, :signal, opts) - else - notify(conf, :dyn_start, queue.name) - notify(conf, :dyn_stop, name) - end - - {:ok, queue} - end - end - - defp update_queue(conf, queue, opts) do - params = - case Keyword.pop(opts, :name) do - {nil, opts} -> %{opts: Map.new(opts)} - {name, []} -> %{name: name} - {name, opts} -> %{name: name, opts: Map.new(opts)} - end - - changeset = Queue.changeset(queue, params) - - Repo.update(conf, changeset) - rescue - Ecto.StaleEntryError -> - update_queue(conf, Repo.reload!(conf, queue), opts) - end - - @doc """ - Delete a queue by name at runtime, rather than using the `:delete` option into the `queues` list - in your configuration. - - ## Examples - - Delete ethe "audio" queue: - - {:ok, _} = DynamicQueues.delete(:audio) - """ - @spec delete(oban_name(), queue_name()) :: {:ok, Ecto.Schema.t()} | {:error, Ecto.Changeset.t()} - def delete(oban_name \\ Oban, name) when is_name(name) do - conf = Oban.config(oban_name) - - with {:ok, queue} <- fetch_queue(conf, to_string(name)), - {:ok, queue} <- Repo.delete(conf, queue) do - notify(conf, :dyn_stop, queue.name) - - {:ok, queue} - end - end - - # Callbacks - - @impl GenServer - def init(state) do - :telemetry.execute([:oban, :plugin, :init], %{}, %{conf: state.conf, plugin: __MODULE__}) - - ref = - state.conf.name - |> Registry.whereis(Foreman) - |> Process.monitor() - - {:ok, %{state | foreman_ref: ref}, {:continue, :start}} - end - - @impl GenServer - def handle_continue(:start, state) do - :ok = Notifier.listen(state.conf.name, [:signal]) - - case manage_queues(state) do - {:ok, _inserted} -> - state.conf - |> all_queues() - |> Enum.reject(&queue_running?(state.conf, &1)) - |> Enum.each(&start_queue(state.conf, &1)) - - {:noreply, schedule_refresh(state)} - - {:error, _name, changeset, _changes} -> - {:stop, %ArgumentError{message: changeset.errors}, state} - end - end - - @impl GenServer - def handle_call(:flush, _from, state) do - {:reply, :ok, state} - end - - @impl GenServer - def handle_info({:DOWN, ref, :process, _pid, _reason}, %{foreman_ref: ref} = state) do - state = restart_queues(state, state.restart_attempts) - - {:noreply, state} - end - - def handle_info({:notification, :signal, payload}, state) do - case payload do - %{"action" => "scale", "ident" => "any", "queue" => queue_name} -> - scale_queue(state.conf, queue_name, payload) - - %{"action" => "pause", "ident" => "any", "queue" => queue_name} -> - scale_queue(state.conf, queue_name, %{"paused" => true}) - - %{"action" => "resume", "ident" => "any", "queue" => queue_name} -> - scale_queue(state.conf, queue_name, %{"paused" => false}) - - %{"action" => "dyn_start", "queue" => queue_name} -> - start_queue(state.conf, queue_name) - - %{"action" => "dyn_stop", "queue" => queue_name} -> - stop_queue(state.conf, queue_name) - - _ -> - :ignore - end - - {:noreply, state} - end - - def handle_info(:refresh, state) do - for queue <- all_queues(state.conf), not queue_running?(state.conf, queue) do - start_queue(state.conf, queue) - end - - {:noreply, schedule_refresh(state)} - end - - def handle_info(message, state) do - Logger.warning( - message: "Received unexpected message: #{inspect(message)}", - source: :oban_pro, - module: __MODULE__ - ) - - {:noreply, state} - end - - # Validations - - defp validate_engine(%Config{engine: engine}) do - if engine == Smart do - :ok - else - {:error, - """ - DynamicQueues requires the Smart engine to run correctly, but you're using: - - engine: #{inspect(engine)} - - You can either use the Smart engine or remove DynamicQueue from your plugins. - """} - end - end - - defp validate_queues(queues) do - Validation.validate(:queues, queues, &validate_queue/1) - end - - defp validate_queue({_name, delete: true}), do: :ok - - defp validate_queue({name, _} = tuple) when is_atom(name) do - changeset = Queue.changeset(tuple) - - if changeset.valid? do - :ok - else - exception = Utils.to_exception(changeset) - - {:error, "expected #{inspect(name)} to be a valid queue, but: #{exception.message}"} - end - end - - defp validate_queue(queue) do - {:error, "expected queue to be a keyword pair, got: #{inspect(queue)}"} - end - - # Queries - - defp all_queues(conf) do - Repo.all(conf, order_by(Queue, asc: :inserted_at)) - end - - defp manage_queues(%{sync_mode: :automatic} = state) do - Repo.transaction(state.conf, fn -> - delete_missing(state.conf, state.queues) - insert_queues(state.conf, state.queues) - end) - end - - defp manage_queues(state) do - Repo.transaction(state.conf, fn -> - {deletes, inserts} = - Enum.split_with(state.queues, fn - {_name, opts} when is_list(opts) -> Keyword.get(opts, :delete) - {_name, _lim} -> false - end) - - delete_queues(state.conf, deletes) - insert_queues(state.conf, inserts) - end) - end - - defp delete_missing(conf, queues) do - Repo.delete_all(conf, where(Queue, [q], q.name not in ^queue_names(queues))) - end - - defp delete_queues(conf, queues) do - Repo.delete_all(conf, where(Queue, [q], q.name in ^queue_names(queues))) - end - - defp queue_names(queues), do: Enum.map(queues, &to_string(elem(&1, 0))) - - defp insert_queues(conf, queues) do - changesets = - Map.new(queues, fn {name, opts} -> - changeset = Queue.changeset({name, opts}) - - {changeset.changes.name, {changeset, opts}} - end) - - Repo.transaction(conf, fn -> - existing = existing_map(conf, Map.keys(changesets)) - - Enum.reduce(changesets, %{}, fn {name, {changeset, opts}}, acc -> - cond do - is_map_key(existing, name) and Map.get(existing, name).hash == changeset.changes.hash -> - acc - - is_map_key(existing, name) -> - params = Queue.normalize({name, opts}) - - changeset = - existing - |> Map.get(name) - |> Queue.changeset(params) - - Map.put(acc, name, Repo.update!(conf, changeset)) - - true -> - Map.put(acc, name, Repo.insert!(conf, changeset)) - end - end) - end) - end - - defp existing_map(conf, names) do - query = where(Queue, [q], q.name in ^names) - - conf - |> Repo.all(query) - |> Map.new(&{&1.name, &1}) - end - - defp fetch_queue(conf, name) do - case Repo.one(conf, where(Queue, name: ^name)) do - nil -> {:error, "no queue named #{inspect(name)} could be found"} - queue -> {:ok, queue} - end - end - - # Scheduling - - defp schedule_refresh(%{interval: :infinity} = state), do: state - - defp schedule_refresh(state) do - %{state | timer: Process.send_after(self(), :refresh, state.interval)} - end - - # Notification - - defp notify(conf, action, queue_name) do - Notifier.notify(conf.name, :signal, %{action: action, ident: :any, queue: queue_name}) - end - - # Supervision - - defp restart_queues(state, 0) do - state - end - - defp restart_queues(state, attempt) do - case Registry.whereis(state.conf.name, Foreman) do - pid when is_pid(pid) -> - ref = Process.monitor(pid) - - state.conf - |> all_queues() - |> Enum.each(&start_queue(state.conf, &1)) - - %{state | foreman_ref: ref} - - nil -> - Process.sleep(state.restart_delay) - - restart_queues(state, attempt - 1) - end - end - - defp queue_running?(conf, queue) do - conf.name - |> Registry.whereis({:supervisor, queue.name}) - |> is_pid() - end - - defp scale_queue(_conf, _, %{"update" => false}), do: :ok - - defp scale_queue(conf, "*", %{"paused" => paused}) do - if Peer.leader?(conf) do - query = - from q in Queue, - update: [set: [opts: fragment("jsonb_set(?, '{paused}', ?)", q.opts, ^paused)]] - - Repo.update_all(conf, query, []) - end - end - - defp scale_queue(conf, queue_name, payload) do - with true <- Peer.leader?(conf), - {:ok, queue} <- fetch_queue(conf, queue_name) do - new_opts = Map.drop(payload, ["action", "ident", "queue"]) - - changeset = - queue - |> Queue.changeset(%{opts: new_opts}) - |> Changeset.delete_change(:hash) - - {:ok, _} = Repo.update(conf, changeset) - end - end - - defp start_queue(conf, queue_name) when is_binary(queue_name) do - with {:ok, queue} <- fetch_queue(conf, queue_name), do: start_queue(conf, queue) - end - - defp start_queue(conf, queue) do - if run_locally?(conf, queue.only) do - Midwife.start_queue(conf, Queue.to_keyword_opts(queue)) - else - :ignore - end - end - - defp stop_queue(conf, queue_name) when is_binary(queue_name) do - Midwife.stop_queue(conf, queue_name) - end - - defp run_locally?(%{node: node}, %{mode: :node, op: op, value: value}) do - compare(node, op, value) - end - - defp run_locally?(_conf, %{mode: :sys_env, op: op, key: key, value: value}) do - key - |> System.get_env() - |> to_string() - |> compare(op, value) - end - - defp run_locally?(_conf, _only), do: true - - defp compare(value_a, :==, value_b), do: value_a == value_b - defp compare(value_a, :!=, value_b), do: value_a != value_b - defp compare(value_a, :=~, value_b), do: value_a =~ Regex.compile!(value_b, "i") -end diff --git a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_scaler.ex b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_scaler.ex deleted file mode 100644 index 6f727d0a..00000000 --- a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_scaler.ex +++ /dev/null @@ -1,662 +0,0 @@ -defmodule Oban.Pro.Plugins.DynamicScaler do - @moduledoc """ - The `DynamicScaler` examines queue throughput and issues commands to horizontally scale - cloud infrastructure to optimize processing. With auto-scaling you can spin up additional nodes - during high traffic events, and pare down to a single node during a lull. Beyond optimizing - throughput, scaling may save money in environments with little to no usage at off-peak times, - e.g. staging. - - Horizontal scaling is applied at the _node_ level, not the _queue_ level, so you can distribute - processing over more phyiscal hardware. - - * Predictive Scaling — The optimal scale is calculated by predicting the future size of a queue - based on recent trends. Multiple samples are then used to prevent overreacting to changes in - queue depth or throughput. Your provide an acceptible `range` of nodes and auto-scaling takes - care of the rest. - - * Multi-Cloud — Cloud integration is provided by a simple, flexible, behaviour that you - implement for your specific environment and configure for each scaler. - - * Queue Filtering — By default, all queues are considered for scale calculations. However, you - can restrict calculations to one or more ciritical queues. - - * Multiple Scalers — Some systems may restrict work to specific node types, e.g. generating - exports or processing videos. Other hybrid systems may straddle multiple clouds. In either - case, you can configure multiple independent scalers driven by distinct queues. - - * Non Linear — An optional `step` parameter allows you to conservatively scale up or down one - node at a time, or optimize for responsiveness and jump from the min to the max in a single - scaling period. - - * Prevent Thrashing — A `cooldown` period skips scaling when there was recent scale activity to - prevent unnecessarily scaling nodes up or down. Nodes may take several minutes to start within - an environment, so the default `cooldown` period is 2 minutes. - - ## Using and Configuring - - > #### Clouds {: .info} - > - > There are ample hosting platforms, aka "clouds", out there and we can't support them all! - > Before you can begin dynamic scaling you'll need to implement a `Cloud` module for your - > environment. Don't worry, we have [copy-and-paste examples](#module-cloud-modules) for some - > popular platforms and a guide to walk through [implementations for your - > environment](#module-writing-cloud-modules). - - With a `cloud` module in hand you're ready to add the `DynamicScaler` plugin to your Oban - config: - - config :my_app, Oban, - plugins: [ - {Oban.Pro.Plugins.DynamicScaler, ...} - ] - - Then, add a scaler with a `range` to define the minimum and maximum nodes, and your `cloud` - strategy: - - {DynamicScaler, scalers: [range: 1..5, cloud: MyApp.Cloud]} - - Now, every minute, `DynamicScaler` will calculate the optimal number of nodes for your queue's - throughput and issue scaling commands accordingly. - - ### Configuring Scalers - - Scalers have options beyond `:cloud` and `:range` for more advanced systems or to constrain - resource usage. Here's a breakdown of all options, followed by specific examples of each. - - * `:cloud` — A `module` or `{module, options}` tuple that interacts with the external cloud - during scale events. Required. - - * `:range` — The range of compute units to scale between. For example, `1..3` declares a minimum - of 1 node and a maximum of 3. The minimum must be 0 or more, and the maximum must be 1 or at - least match the minimum. Required. - - * `:cooldown` — The minimum time between scaling events. Defaults to 120 seconds. - - * `:lookback` — The historic time to check queues. Defaults to 60 seconds. - - * `:queues` — Either `:all` or a list of queues to consider when measuring throughput and - backlog. - - * `:step` — Either `:none` or the maximum nodes to scale up or down at once. Defaults to - `:none`. - - ### Scaler Examples - - Filter throughput queries to the `:media` queue: - - scalers: [queues: :media, range: 1..3, cloud: MyApp.Cloud] - - Filter throughput queries to both `:audio` and `:video` queues: - - scalers: [queues: [:audio, :video], range: 1..3, cloud: MyApp.Cloud] - - Configure scalers driven by different queues (note, queues _may not_ overlap): - - scalers: [ - [queues: :audio, range: 0..2, cloud: {MyApp.Cloud, asg: "my-audio-asg"}], - [queues: :video, range: 0..5, cloud: {MyApp.Cloud, asg: "my-video-asg"}] - ] - - Limit scaling to one node up or down at a time: - - scalers: [range: 1..3, step: 1, cloud: MyApp.Cloud] - - Wait at least 5 minutes (300 seconds) between scaling events: - - scalers: [range: 1..3, cloud: MyApp.Cloud, cooldown: 300] - - Increase the period used to calculate historic throughput to 90 seconds: - - scalers: [range: 1..3, cloud: MyApp.Cloud, lookback: 90] - - > ### Scaling Down to Zero Nodes {: .warning} - > - > It's possible to scale down to zero nodes in staging environments or production applications - > with periods of downtime. However, it is **only viable for multi-node setups** with dedicated - > worker nodes and another instance type that isn't controlled by `DynamicScaler`. Without a - > separate "web" node, or something that is always running, you run the risk of scaling down - > without the ability to scale back up. - - ## Cloud Modules - - There are a lot of hosting platforms, aka "clouds" out there and we can't support them all! Even - with optional dependencies, it would be a mess of libraries that may not agree with your - application decisions. Instead, the `Oban.Pro.Cloud` behaviour defines two simple callbacks, and - integrating with platforms typically takes a single HTTP query or library call. - - The following links contain gists of full implementations for popular cloud platforms. Feel free - to copy-and-paste to use them as-is or as the basis for your own cloud modules. - - * [EC2](https://gist.github.com/sorentwo/ba4a07d3a011d212c19a5bb775a6c536) - * [Fly](https://gist.github.com/sorentwo/d6be222091db7ba3c5b50d8bcabca252) - * [GCP](https://gist.github.com/sorentwo/a54a1e2b37123fd627cca16f07aed951) - * [Gigalixir](https://gist.github.com/sorentwo/90b4a427819756f7ceb72254ad53d9bb) - * [Heroku](https://gist.github.com/sorentwo/54d99fb2ac05cb63ea1e30aa1935b6fc) - * [Kubernetes](https://gist.github.com/sorentwo/f5ba9048e1e91456d37a8c7d4b8e4d58) - - Let us know if an integration for your platform is missing (which is rather likely) and you'd - like assistance. Otherwise, follow the guide below to write your own integration! - - ### Writing Cloud Modules - - Cloud callback modules must define an `init/1` function to prepare configuration at runtime, and - a `scale/2` callback called with the desired number of nodes and the prepared configuration. - - The following example demonstrates a complete callback module for scaling EC2 Auto Scaling - Groups on AWS using the [SetDesiredCapacity][sdc] action. It assumes you're using the - [ex_aws][exa] package with the proper credentials. - - defmodule MyApp.ASG do - @behaviour Oban.Pro.Cloud - - @impl Oban.Pro.Cloud - def init(opts), do: Map.new(opts) - - @impl Oban.Pro.Cloud - def scale(desired, conf) do - params = %{ - "Action" => "SetDesiredCapacity", - "AutoScalingGroupName" => conf.asg, - "DesiredCapacity" => desired, - "Version" => "2011-01-01" - } - - query = %ExAws.Operation.Query{ - path: "", - params: params, - service: :autoscaling, - action: :set_desired_capacity - } - - with {:ok, _} <- ExAws.request(query), do: {:ok, conf} - end - end - - You'd then use your cloud module as a scaler option: - - {DynamicScaler, scalers: [range: 1..3, cloud: {MyApp.ASG, asg: "my-asg-name"}]} - - Clouds can also pull from the application or system environment to build configuration. If your - module pulls from the environment exclusively, then you can pass the module name rather than a - tuple: - - {DynamicScaler, scalers: [range: 1..3, cloud: MyApp.ASG]} - - [sdc]: https://docs.aws.amazon.com/autoscaling/ec2/APIReference/API_SetDesiredCapacity.html - [exa]: https://github.com/ex-aws/ex_aws - - ### Optimizing Throughput Queries - - While the scaler's throughput queries are optimized for a standard load, high throughput queues, - or systems that retain a large volume of jobs, may benefit from an additional index that aids - calculating throughput. Use the following migration to add an index if you find that scaling - queries are too slow or timing out: - - @disable_ddl_transaction true - @disable_migration_lock true - - def change do - create_if_not_exists index( - :oban_jobs, - [:state, :queue, :attempted_at, :attempted_by], - concurrently: true, - where: "attempted_at IS NOT NULL", - prefix: "public" - ) - end - - Alternatively, you can change the timeout used for scaler inspection queries: - - {DynamicScaler, timeout: :timer.seconds(15), scalers: ...} - - ## Instrumenting with Telemetry - - The `DynamicScaler` plugin adds the following metadata to the `[:oban, :plugin, :stop]` event: - - * `:scaler` - details of the active scaler config with recent scaling values - * `:skipped` - the reason scaling is skipped, either `:recently_scaled` or `:already_scaled` - * `:error` — the reason returned from `scale/2` when scaling fails - - When multiple `scalers` are configured one event is emitted for _each_ scaler. - """ - - @behaviour Oban.Plugin - - use GenServer - - import DateTime, only: [utc_now: 0] - import Ecto.Query - - alias __MODULE__, as: State - alias Oban.Plugins.Cron - alias Oban.{Job, Notifier, Peer, Repo, Validation} - - require Logger - - defmodule Scaler do - @moduledoc false - - @enforce_keys [:cloud, :range] - defstruct [ - :cloud, - :last_rate, - :last_scaled_at, - :last_scaled_to, - :last_size, - :range, - cooldown: 120, - lookback: 60, - queues: :all, - step: :none - ] - - def new(opts) do - opts - |> Keyword.update(:cloud, nil, &init_cloud/1) - |> Keyword.update(:queues, :all, &normalize_queues/1) - |> then(&struct!(__MODULE__, &1)) - end - - defp init_cloud(cmod) when is_atom(cmod), do: {cmod, cmod.init([])} - defp init_cloud({cmod, copt}), do: {cmod, cmod.init(copt)} - - defp normalize_queues(:all), do: :all - - defp normalize_queues(queues) do - queues - |> List.wrap() - |> Enum.map(&to_string/1) - end - end - - defstruct [:conf, :name, :timer, scalers: [], timeout: :timer.seconds(15)] - - defmacrop attempted_by_node(column, position) do - quote do - fragment("?[?]", unquote(column), unquote(position)) - end - end - - @doc false - def child_spec(args), do: super(args) - - @impl Oban.Plugin - def start_link(opts) do - opts = - Keyword.update!(opts, :scalers, fn scalers -> - scalers - |> wrap_keyword() - |> Enum.map(&Scaler.new/1) - end) - - state = struct!(State, opts) - - GenServer.start_link(__MODULE__, state, name: state.name) - end - - defp wrap_keyword([elem | _] = list) when is_tuple(elem), do: [list] - defp wrap_keyword(list), do: list - - @impl Oban.Plugin - def validate(opts) do - Validation.validate_schema(opts, - conf: :ok, - name: :ok, - scalers: {:custom, &validate_scalers(wrap_keyword(&1))}, - timeout: :timeout - ) - end - - @impl Oban.Plugin - def format_logger_output(_conf, %{skipped: reason}), do: %{skipped: reason} - - def format_logger_output(_conf, %{error: reason}), do: %{error: inspect(reason)} - - def format_logger_output(_conf, %{scaler: scaler}) do - cloud = - case scaler.cloud do - {module, _opts} -> module - module -> module - end - - Map.from_struct(%{scaler | cloud: inspect(cloud), range: inspect(scaler.range)}) - end - - @doc false - def optimal_scale(size, rate, min..max//_) do - cond do - rate == 0 and size > 0 and min == 0 -> - 1 - - size == 0 and rate > 0 and min == 0 -> - 1 - - rate == 0 -> - min - - true -> - optimal = size / rate - - optimal - |> ceil() - |> max(min) - |> min(max) - end - end - - @doc false - def clamped_scale(size, _last, _range, :none), do: size - def clamped_scale(size, size, _range, _step), do: size - - def clamped_scale(size, last, _min..max//_, step) when size > last do - (last + step) - |> min(size) - |> min(max) - end - - def clamped_scale(size, last, min.._max//_, step) when size < last do - (last - step) - |> max(size) - |> max(min) - end - - # Callbacks - - @impl GenServer - def init(state) do - :telemetry.execute([:oban, :plugin, :init], %{}, %{conf: state.conf, plugin: __MODULE__}) - - {:ok, state, {:continue, :start}} - end - - @impl GenServer - def handle_continue(:start, state) do - :ok = Notifier.listen(state.conf.name, [:scaler]) - - {:noreply, schedule_scaling(state)} - end - - @impl GenServer - def handle_call(:check_scale, _from, %State{} = state) do - {:reply, :ok, check_and_scale(state)} - end - - @impl GenServer - def handle_info(:check_scale, %State{} = state) do - state = - if Peer.leader?(state.conf) do - check_and_scale(state) - else - state - end - - {:noreply, schedule_scaling(state)} - end - - def handle_info({:notification, :scaler, payload}, state) do - {queues, payload} = Map.pop!(payload, "queues") - - if Peer.leader?(state.conf) do - {:noreply, state} - else - scalers = - Enum.map(state.scalers, fn scaler -> - if queues == to_scaler_id(scaler.queues) do - Enum.reduce(payload, scaler, fn - {"last_rate", val}, acc -> - %{acc | last_rate: val} - - {"last_size", val}, acc -> - %{acc | last_size: val} - - {"last_scaled_to", val}, acc -> - %{acc | last_scaled_to: val} - - {"last_scaled_at", nil}, acc -> - %{acc | last_scaled_at: nil} - - {"last_scaled_at", val}, acc -> - {:ok, at, _offset} = DateTime.from_iso8601(val) - - %{acc | last_scaled_at: at} - end) - else - scaler - end - end) - - {:noreply, %{state | scalers: scalers}} - end - end - - def handle_info(message, state) do - Logger.warning( - message: "Received unexpected message: #{inspect(message)}", - source: :oban_pro, - module: __MODULE__ - ) - - {:noreply, state} - end - - defp to_scaler_id(:all), do: "all" - defp to_scaler_id(queues) when is_list(queues), do: Enum.map(queues, &to_string/1) - - # Validations - - defp validate_scalers([]), do: :ok - - defp validate_scalers([head | tail]) do - with :ok <- Validation.validate(:scalers, head, &validate_scaler/1) do - validate_scalers(tail) - end - end - - defp validate_scalers(scalers) do - {:error, "expected :scaler to be a list of keywords, got: #{inspect(scalers)}"} - end - - defp validate_scaler({:cloud, cloud}) do - case cloud do - {cmod, copt} when is_atom(cmod) and is_list(copt) -> - :ok - - cmod when is_atom(cmod) -> - :ok - - _ -> - {:error, - "expected :cloud to be a module or {module, options} tuple, got: #{inspect(cloud)}"} - end - end - - defp validate_scaler({:cooldown, cooldown}) do - Validation.validate_integer(:cooldown, cooldown, min: 0) - end - - defp validate_scaler({:lookback, cooldown}) do - Validation.validate_integer(:lookback, cooldown) - end - - defp validate_scaler({:queues, queues}) do - case queues do - :all -> - :ok - - queue when is_atom(queue) and not is_nil(queue) -> - :ok - - [queue | _] when is_atom(queue) or is_binary(queue) -> - :ok - - _ -> - {:error, "expected :queues to be :all or a list of queue names, got: #{inspect(queues)}"} - end - end - - defp validate_scaler({:range, range}) do - case range do - min..max//_ when min <= max and max > 0 -> - :ok - - _min.._max//_ -> - {:error, "expected :range to have a min less than max, got: #{inspect(range)}"} - - _ -> - {:error, "exepcted :range to be a range of integers, got: #{inspect(range)}"} - end - end - - defp validate_scaler({:step, :none}), do: :ok - - defp validate_scaler({:step, limit}) do - Validation.validate_integer(:step, limit) - end - - # Necessary to seed testing, not documented - defp validate_scaler({:last_rate, _}), do: :ok - defp validate_scaler({:last_scaled_at, _}), do: :ok - defp validate_scaler({:last_scaled_to, _}), do: :ok - defp validate_scaler({:last_size, _}), do: :ok - - defp validate_scaler(option), do: {:unknown, option, Scaler} - - # Scheduling - - defp schedule_scaling(state) do - timer = Process.send_after(self(), :check_scale, Cron.interval_to_next_minute()) - - %{state | timer: timer} - end - - # Scaling - - defp check_and_scale(%State{scalers: scalers} = state) do - scalers = - for scaler <- scalers do - meta = %{conf: state.conf, plugin: __MODULE__, scaler: scaler} - - :telemetry.span([:oban, :plugin], meta, fn -> - prev_size = scaler.last_size - - scaler = - scaler - |> record_size(state) - |> record_rate(state) - - case apply_scale(scaler, prev_size) do - {:ok, scaler} -> - notify_scalers(scaler, state) - - {scaler, Map.put(meta, :scaler, scaler)} - - {:skipped, reason} -> - notify_scalers(scaler, state) - - {scaler, Map.put(meta, :skipped, reason)} - - {:error, reason} -> - {scaler, Map.put(meta, :error, reason)} - end - end) - end - - %{state | scalers: scalers} - end - - defp notify_scalers(scaler, %{conf: conf}) do - payload = Map.take(scaler, ~w(last_rate last_scaled_at last_scaled_to last_size queues)a) - - Notifier.notify(conf, :scaler, payload) - end - - defp record_size(scaler, state) do - next = DateTime.add(utc_now(), scaler.lookback, :second) - - query = - scaler.queues - |> base_query() - |> where([j], j.state in ~w(available scheduled retryable) and j.scheduled_at <= ^next) - |> select(count()) - - %{scaler | last_size: Repo.one(state.conf, query, timeout: state.timeout)} - end - - defp record_rate(scaler, state) do - since = DateTime.add(utc_now(), -scaler.lookback, :second) - - query = - scaler.queues - |> base_query() - |> where([j], j.state in ~w(executing retryable completed cancelled discarded)) - |> where([j], j.attempted_at > ^since) - |> group_by([j], attempted_by_node(j.attempted_by, 1)) - |> select([j], {attempted_by_node(j.attempted_by, 1), count()}) - - case Repo.all(state.conf, query, timeout: state.timeout) do - [] -> - %{scaler | last_rate: 0, last_scaled_to: scaler.last_scaled_to || 0} - - node_counts -> - size = length(node_counts) - - rate = - node_counts - |> Enum.map(&elem(&1, 1)) - |> Enum.sum() - |> div(size) - - %{scaler | last_rate: rate, last_scaled_to: scaler.last_scaled_to || size} - end - end - - defp base_query(queues) do - if queues == :all do - where(Job, [j], not is_nil(j.queue)) - else - where(Job, [j], j.queue in ^queues) - end - end - - defp apply_scale(scaler, prev_size) do - scale_to = - prev_size - |> next_size(scaler.last_size) - |> optimal_scale(scaler.last_rate, scaler.range) - |> clamped_scale(scaler.last_scaled_to, scaler.range, scaler.step) - - cond do - recently_scaled?(scaler) -> - {:skipped, :recently_scaled} - - already_scaled?(scale_to, scaler) -> - {:skipped, :already_scaled} - - true -> - {cloud_mod, cloud_opt} = scaler.cloud - - case cloud_mod.scale(scale_to, cloud_opt) do - {:ok, cloud_opt} -> - cloud = {cloud_mod, cloud_opt} - - {:ok, %{scaler | cloud: cloud, last_scaled_at: utc_now(), last_scaled_to: scale_to}} - - {:error, reason} -> - {:error, reason} - end - end - end - - defp next_size(nil, size), do: size - defp next_size(prev, curr), do: max(curr + (curr - prev), 0) - - defp recently_scaled?(%{last_scaled_at: nil}), do: false - - defp recently_scaled?(%{cooldown: cooldown, last_scaled_at: scaled_at}) do - scaling_allowed_at = DateTime.add(scaled_at, cooldown) - - DateTime.compare(utc_now(), scaling_allowed_at) != :gt - end - - defp already_scaled?(scale_to, %{last_scaled_to: to}), do: scale_to == to -end diff --git a/vendor/oban_pro/lib/oban/pro/producer.ex b/vendor/oban_pro/lib/oban/pro/producer.ex deleted file mode 100644 index d7f9b7ca..00000000 --- a/vendor/oban_pro/lib/oban/pro/producer.ex +++ /dev/null @@ -1,358 +0,0 @@ -require Protocol - -defmodule Oban.Pro.Producer do - @moduledoc false - - use Ecto.Schema - - import Ecto.Changeset - - alias Ecto.Changeset - alias Oban.Pro.Utils - - defmodule ListOrMap do - @moduledoc false - - # This is a custom type to allow a graceful transition from the legacy rate limit windows - # structure to the new, flattened map. - - @behaviour Ecto.Type - - @impl true - def type, do: :any - - @impl true - def cast(data) when is_map(data), do: {:ok, data} - def cast([data | _]), do: {:ok, data} - def cast(_data), do: :error - - @impl true - def load(data) when is_map(data), do: {:ok, data} - def load([data | _]), do: {:ok, data} - def load([]), do: {:ok, %{}} - def load(_data), do: :error - - @impl true - def dump(data) when is_map(data), do: {:ok, data} - def dump([data | _]), do: {:ok, data} - def dump([]), do: {:ok, %{}} - def dump(_data), do: :error - - @impl true - def equal?(a, b), do: a == b - - @impl true - def embed_as(_data), do: :dump - end - - @type t :: %__MODULE__{ - uuid: Ecto.UUID.t(), - name: binary(), - node: binary(), - queue: binary(), - meta: map(), - started_at: DateTime.t(), - updated_at: DateTime.t() - } - - @type meta :: %__MODULE__.Meta{local_limit: pos_integer(), paused: boolean()} - - @primary_key {:uuid, :binary_id, autogenerate: false} - schema "oban_producers" do - field :name, :string - field :node, :string - field :queue, :string - field :started_at, :utc_datetime_usec - field :updated_at, :utc_datetime_usec - - field :ack_async, :boolean, virtual: true, default: true - field :ack_tab, :any, virtual: true - field :refresh_interval, :integer, virtual: true, default: :timer.seconds(30) - field :xact_delay, :integer, default: :timer.seconds(1), virtual: true - field :xact_retry, :integer, default: 5, virtual: true - field :xact_timeout, :integer, default: :timer.seconds(30), virtual: true - - embeds_one :meta, Meta, on_replace: :update, primary_key: false do - @moduledoc false - - field :local_limit, :integer, default: 10 - field :paused, :boolean, default: false - field :shutdown_started_at, :utc_datetime_usec - - embeds_one :global_limit, GlobalLimit, on_replace: :update, primary_key: false do - @moduledoc false - - field :allowed, :integer - field :burst, :boolean, default: false - field :tracked, ListOrMap, default: %{} - - embeds_one :partition, Partition, on_replace: :update, primary_key: false do - @moduledoc false - - field :fields, {:array, Ecto.Enum}, values: [:args, :meta, :worker] - field :keys, {:array, :string} - end - end - - embeds_one :rate_limit, RateLimit, on_replace: :update, primary_key: false do - @moduledoc false - - field :allowed, :integer - field :period, :integer - field :window_time, :integer, skip_default_validation: true - field :windows, ListOrMap, default: %{} - - embeds_one :partition, Partition, on_replace: :update, primary_key: false do - @moduledoc false - - field :fields, {:array, Ecto.Enum}, values: [:args, :meta, :worker] - field :keys, {:array, :string} - end - end - end - end - - @spec new(map() | Keyword.t()) :: Changeset.t(t()) - def new(params) when is_list(params) or is_map(params) do - params = - params - |> Map.new() - |> Map.put_new_lazy(:uuid, &Oban.Pro.UUIDv7.generate/0) - |> Map.update!(:name, &to_clean_string/1) - |> Map.update!(:node, &to_clean_string/1) - |> Map.update!(:queue, &to_clean_string/1) - - %__MODULE__{} - |> cast(params, ~w(name node queue refresh_interval started_at updated_at uuid)a) - |> cast_embed(:meta, required: true, with: &meta_changeset/2) - |> validate_required(~w(name node queue)a) - |> validate_length(:name, min: 1) - |> validate_length(:node, min: 1) - |> validate_length(:queue, min: 1) - |> Utils.enforce_keys(params, __MODULE__) - end - - @spec update_meta(t(), atom(), term()) :: Changeset.t() - def update_meta(schema, key, value) do - update_meta(schema, %{key => value}) - end - - @spec update_meta(t(), map()) :: Changeset.t() - def update_meta(schema, changes) do - meta = - schema.meta - |> meta_changeset(changes) - |> apply_changes() - |> Ecto.embedded_dump(:json) - - schema - |> cast(%{meta: meta}, []) - |> cast_embed(:meta, with: &meta_changeset/2) - end - - @allowed ~w(local_limit paused shutdown_started_at)a - - @doc false - def meta_changeset(schema, params, allowed \\ @allowed) do - params = - params - |> Map.new() - |> Map.delete(:limit) - |> Map.put_new_lazy(:local_limit, fn -> default_local_limit(params, schema) end) - - params = - if Map.get(params, :global_limit) do - Map.update!(params, :global_limit, &cast_global_limit/1) - else - params - end - - params = - if Map.get(params, :rate_limit) do - Map.update!(params, :rate_limit, &cast_rate_limit/1) - else - params - end - - schema - |> cast(params, allowed) - |> cast_embed(:global_limit, with: &global_changeset/2) - |> cast_embed(:rate_limit, with: &rate_changeset/2) - |> validate_required(~w(local_limit)a) - |> validate_number(:local_limit, greater_than: 0) - |> validate_single_partitioner() - |> Utils.enforce_keys(params, schema.__struct__) - end - - @doc false - @spec default_local_limit(map(), Ecto.Schema.t()) :: non_neg_integer() - def default_local_limit(params, schema) do - cond do - is_integer(params[:limit]) -> - params[:limit] - - is_integer(params[:global_limit]) -> - params[:global_limit] - - is_integer(get_in(params, [:global_limit, :allowed])) -> - get_in(params, [:global_limit, :allowed]) - - true -> - schema.local_limit - end - end - - @doc false - @spec validate_single_partitioner(Changeset.t()) :: Changeset.t() - def validate_single_partitioner(changeset) do - case {get_field(changeset, :global_limit), get_field(changeset, :rate_limit)} do - {%{partition: %{}}, %{partition: %{}}} -> - add_error(changeset, :global_limit, "only one limiter may have partitioning") - - _ -> - changeset - end - end - - @doc false - @spec global_changeset(Ecto.Schema.t(), integer() | map() | Keyword.t()) :: Changeset.t() - def global_changeset(schema, params) do - params = Map.update(params, :partition, nil, &normalize_partition/1) - - schema - |> cast(params, ~w(allowed burst)a) - |> cast_embed(:partition, with: &partition_changeset/2) - |> validate_number(:allowed, greater_than: 0) - |> Utils.enforce_keys(params, schema.__struct__) - end - - @doc false - @spec rate_changeset(Ecto.Schema.t(), map() | Keyword.t()) :: Changeset.t() - def rate_changeset(schema, params) do - params = - params - |> Map.update(:period, nil, &Utils.cast_period/1) - |> Map.update(:partition, nil, &normalize_partition/1) - |> Map.put_new_lazy(:window_time, fn -> DateTime.to_unix(DateTime.utc_now(), :second) end) - - schema - |> cast(params, ~w(allowed period window_time)a) - |> cast_embed(:partition, with: &partition_changeset/2) - |> validate_required(~w(allowed period)a) - |> validate_number(:allowed, greater_than: 0) - |> validate_number(:period, greater_than: 0) - |> Utils.enforce_keys(params, schema.__struct__) - end - - @doc false - @spec partition_changeset(Ecto.Schema.t(), map() | Keyword.t()) :: Changeset.t() - def partition_changeset(schema, params) do - params = - params - |> Map.new() - |> Map.update(:keys, [], &for(key <- &1, do: to_string(key))) - - schema - |> cast(params, ~w(fields keys)a) - |> validate_required(~w(fields)a) - |> Utils.enforce_keys(params, schema.__struct__) - end - - # Global Limit Helpers - - @doc false - @spec cast_global_limit(pos_integer() | list() | map()) :: map() - def cast_global_limit(limit) when is_integer(limit) do - %{allowed: limit} - end - - def cast_global_limit(opts), do: cast_rate_limit(opts) - - # Rate Limit Helpers - - @doc false - @spec cast_rate_limit(list() | map()) :: map() - def cast_rate_limit([[key | _] | _] = opts) when is_binary(key) do - opts - |> Enum.map(&List.to_tuple/1) - |> Map.new() - |> cast_rate_limit() - end - - def cast_rate_limit(opts) - when is_map_key(opts, "allowed") or - is_map_key(opts, "burst") or - is_map_key(opts, "period") or - is_map_key(opts, "partition") do - Map.new(opts, fn - {"allowed", allowed} -> - {:allowed, allowed} - - {"burst", allowed} -> - {:burst, allowed} - - {"period", [time, unit]} -> - {:period, {time, unit}} - - {"period", period} -> - {:period, period} - - {"partition", [["fields", fields]]} -> - {:partition, fields: fields} - - {"partition", [["fields", fields], ["keys", keys]]} -> - {:partition, fields: fields, keys: keys} - - {"partition", field} when is_binary(field) -> - {:partition, fields: [field]} - end) - end - - def cast_rate_limit(opts), do: opts - - def normalize_partition(partition) do - cond do - is_nil(partition) -> - nil - - is_map(partition) -> - partition - - Keyword.keyword?(partition) and Keyword.has_key?(partition, :fields) -> - partition - - true -> - partition - |> List.wrap() - |> Enum.reduce([fields: [], keys: []], fn - {field, keys}, opts when field in ~w(args meta)a -> - opts - |> Keyword.put(:keys, List.wrap(keys)) - |> Keyword.update!(:fields, &[field | &1]) - - field, opts -> - Keyword.update!(opts, :fields, &[field | &1]) - end) - end - end - - # Helpers - - defp to_clean_string(value) when is_reference(value) do - inspect(value) - end - - defp to_clean_string(value) do - value - |> to_string() - |> String.trim_leading("Elixir.") - end -end - -encoder = if Code.ensure_loaded?(JSON.Encoder), do: JSON.Encoder, else: Jason.Encoder - -Protocol.derive(encoder, Oban.Pro.Producer, except: [:__meta__]) -Protocol.derive(encoder, Oban.Pro.Producer.Meta.GlobalLimit) -Protocol.derive(encoder, Oban.Pro.Producer.Meta.GlobalLimit.Partition) -Protocol.derive(encoder, Oban.Pro.Producer.Meta.RateLimit) -Protocol.derive(encoder, Oban.Pro.Producer.Meta.RateLimit.Partition) diff --git a/vendor/oban_pro/lib/oban/pro/queue.ex b/vendor/oban_pro/lib/oban/pro/queue.ex deleted file mode 100644 index 21c976f8..00000000 --- a/vendor/oban_pro/lib/oban/pro/queue.ex +++ /dev/null @@ -1,157 +0,0 @@ -defmodule Oban.Pro.Queue do - @moduledoc false - - use Ecto.Schema - - import Ecto.Changeset - - alias Oban.Pro.{Producer, Utils} - - @allowed ~w(ack_async local_limit paused refresh_interval)a - - @primary_key {:name, :string, autogenerate: false} - schema "oban_queues" do - field :lock_version, :integer, default: 1 - field :hash, :string - - embeds_one :only, Only, on_replace: :update, primary_key: false do - @moduledoc false - - field :mode, Ecto.Enum, values: [:node, :sys_env] - field :op, Ecto.Enum, values: [:==, :!=, :=~] - field :key, :string - field :value, :string - end - - embeds_one :opts, Opts, on_replace: :update, primary_key: false do - @moduledoc false - - field :ack_async, :boolean - field :local_limit, :integer - field :paused, :boolean - field :refresh_interval, :integer - field :xact_delay, :integer - field :xact_retry, :integer - - embeds_one :global_limit, GlobalLimit, on_replace: :update, primary_key: false do - @moduledoc false - - field :allowed, :integer - field :burst, :boolean, default: false - - embeds_one :partition, Partition, on_replace: :update, primary_key: false do - @moduledoc false - - field :fields, {:array, Ecto.Enum}, values: [:args, :meta, :worker] - field :keys, {:array, :string} - end - end - - embeds_one :rate_limit, RateLimit, on_replace: :update, primary_key: false do - @moduledoc false - - field :allowed, :integer - field :period, :integer - field :window_time, :integer, virtual: true, skip_default_validation: true - - embeds_one :partition, Partition, on_replace: :update, primary_key: false do - @moduledoc false - - field :fields, {:array, Ecto.Enum}, values: [:args, :meta, :worker] - field :keys, {:array, :string} - end - end - end - - timestamps(inserted_at: :inserted_at, updated_at: :updated_at, type: :utc_datetime_usec) - end - - def normalize(params) do - case params do - [_ | _] -> Map.new(params) - {name, [_ | _] = opts} -> %{name: name, opts: Map.new(opts)} - {name, limit} -> %{name: name, opts: %{local_limit: limit}} - end - end - - def changeset(params) do - changeset(%__MODULE__{}, normalize(params)) - end - - def changeset(schema, params) do - params = - params - |> Map.replace_lazy(:name, fn _ -> to_string(params[:name]) end) - |> extract_only() - - schema - |> cast(params, [:name]) - |> cast_embed(:only, with: &only_changeset/2) - |> cast_embed(:opts, required: true, with: &opts_changeset/2) - |> validate_required([:name]) - |> validate_length(:name, min: 1) - |> optimistic_lock(:lock_version) - |> inject_hash(params) - |> Utils.enforce_keys(params, __MODULE__) - end - - defp extract_only(params) do - case params do - %{opts: %{only: _}} -> - {only, params} = pop_in(params, [:opts, :only]) - - Map.put(params, :only, cast_only(only)) - - _ -> - params - end - end - - defp only_changeset(schema, params) do - schema - |> cast(params, ~w(mode op key value)a) - |> validate_required(~w(mode op value)a) - |> Utils.enforce_keys(params, __MODULE__.Only) - end - - defp opts_changeset(schema, params) do - params = Map.new(params, fn {key, val} -> {Utils.maybe_to_atom(key), val} end) - - Producer.meta_changeset(schema, params, @allowed) - end - - defp inject_hash(changeset, params) do - hash = - params - |> Map.take(~w(only opts)a) - |> Enum.map_join(&:erlang.phash2/1) - |> Utils.hash64() - - put_change(changeset, :hash, hash) - end - - @spec to_keyword_opts(%{opts: map()} | map()) :: Keyword.t() - def to_keyword_opts(%__MODULE__{name: queue, opts: opts}) do - opts - |> Ecto.embedded_dump(:json) - |> Map.put(:queue, queue) - |> to_keyword_opts() - end - - def to_keyword_opts(opts) do - for {key, val} <- opts, not is_nil(val), do: {Utils.maybe_to_atom(key), val} - end - - # Helpers - - defp cast_only({:node, value}), do: cast_only({:node, :==, value}) - defp cast_only({:node, op, value}), do: %{mode: :node, op: op, value: to_string(value)} - - defp cast_only({:sys_env, key, value}), do: cast_only({:sys_env, key, :==, value}) - - defp cast_only({:sys_env, key, op, value}) do - %{mode: :sys_env, op: op, key: key, value: to_string(value)} - end - - defp cast_only(other), do: other -end diff --git a/vendor/oban_pro/lib/oban/pro/refresher.ex b/vendor/oban_pro/lib/oban/pro/refresher.ex deleted file mode 100644 index d8125143..00000000 --- a/vendor/oban_pro/lib/oban/pro/refresher.ex +++ /dev/null @@ -1,111 +0,0 @@ -defmodule Oban.Pro.Refresher do - @moduledoc false - - # Centralized refreshing and cleaning of producer records across all Oban instances. - # - # The Refresher runs as a single process supervised by the Oban.Pro application. It refreshes - # producer records for all actively running queues on the current node. - - use GenServer - - import Ecto.Query, only: [where: 3] - - alias __MODULE__, as: State - alias Oban.Pro.Producer - alias Oban.{Peer, Repo} - - defstruct [ - :timer, - interval: :timer.seconds(15), - producers: %{}, - producer_ttl: :timer.minutes(1) - ] - - @doc false - def start_link(opts \\ []) do - state = struct!(State, opts) - - GenServer.start_link(__MODULE__, state, name: __MODULE__) - end - - # GenServer Callbacks - - @impl GenServer - def init(state) do - Process.flag(:trap_exit, true) - - {:ok, schedule_refresh(state)} - end - - @impl GenServer - def terminate(_reason, state) do - if is_reference(state.timer), do: Process.cancel_timer(state.timer) - - :ok - end - - @impl GenServer - def handle_call(:refresh, _from, state) do - {:noreply, state} = handle_info(:refresh, state) - - {:reply, :ok, state} - end - - @impl GenServer - def handle_info(:refresh, state) do - refresh_producers(state) - cleanup_producers(state) - - {:noreply, schedule_refresh(state)} - end - - # Helpers - - defp schedule_refresh(state) do - timer = Process.send_after(self(), :refresh, state.interval) - - %{state | timer: timer} - end - - defp refresh_producers(_state) do - match = [{{{:"$1", {:producer, :"$2"}}, :_, :_}, [], [[:"$1", :"$2"]]}] - - Oban.Registry - |> Registry.select(match) - |> Enum.group_by(&hd/1, &Enum.at(&1, 1)) - |> Enum.each(fn {oban_name, queues} -> - with {_pid, %{testing: :disabled} = conf} <- Oban.Registry.lookup(oban_name) do - now = DateTime.utc_now() - ids = Enum.flat_map(queues, &lookup_uuid(&1, oban_name)) - - refresh_query = where(Producer, [p], p.uuid in ^ids) - - Repo.update_all(conf, refresh_query, set: [updated_at: now]) - end - end) - end - - defp cleanup_producers(state) do - match = [{{:"$1", :_, :"$2"}, [{:not, {:is_tuple, :"$1"}}], [:"$2"]}] - - Oban.Registry - |> Registry.select(match) - |> Enum.filter(&Peer.leader?/1) - |> Enum.each(fn conf -> - now = DateTime.utc_now() - outdated_at = DateTime.add(now, -state.producer_ttl, :millisecond) - cleanup_query = where(Producer, [p], p.updated_at <= ^outdated_at) - - Repo.delete_all(conf, cleanup_query) - end) - end - - defp lookup_uuid(queue, oban_name) do - prod_name = {oban_name, {:producer, queue}} - - case Registry.meta(Oban.Registry, prod_name) do - {:ok, %{uuid: uuid}} -> [uuid] - _error -> [] - end - end -end diff --git a/vendor/oban_pro/lib/oban/pro/relay.ex b/vendor/oban_pro/lib/oban/pro/relay.ex deleted file mode 100644 index 21a2f574..00000000 --- a/vendor/oban_pro/lib/oban/pro/relay.ex +++ /dev/null @@ -1,408 +0,0 @@ -defmodule Oban.Pro.Relay do - @moduledoc """ - The `Relay` extension lets you insert and await the results of jobs locally or remotely, across - any number of nodes, so you can seamlessly distribute jobs and await the results synchronously. - - Think of `Relay` as persistent, distributed tasks. - - `Relay` uses PubSub for to transmit results. That means it will work without Erlang distribution - or clustering, but it does require Oban to have functional PubSub. It doesn't matter which node - executes a job, the result will still be broadcast back. - - Results are encoded using `term_to_binary/2` and decoded using `binary_to_term/2` using the - `:safe` option to prevent the creation of new atoms or function references. If you're returning - results with atoms you _must be sure_ that those atoms are defined locally, where the `await/2` - or `await_many/2` function is called. - - ## Usage - - Use `async/1` to insert a job for asynchronous execution: - - ```elixir - relay = - %{id: 123} - |> MyApp.Worker.new() - |> Oban.Pro.Relay.async() - ``` - - After inserting a job and constructing a relay, use `await/1` to await the job's execution and - return the result: - - ```elixir - {:ok, result} = - %{id: 123} - |> MyApp.Worker.new() - |> Oban.Pro.Relay.async() - |> Oban.Pro.Relay.await() - ``` - - By default, `await/1` will timeout after 5 seconds and return an `{:error, :timeout}` tuple. The job - itself may continue to run, only the local process stops waiting on it. Pass an explicit timeout - to wait longer: - - ```elixir - {:ok, result} = Oban.Pro.Relay.await(relay, :timer.seconds(30)) - ``` - - Any successful result such as `:ok`, `{:ok, value}`, or `{:cancel, reason}` is passed back as - the await result. When the executed job returns an `{:error, reason}` tuple, raises an - exception, or crashes, the result comes through as an error tuple. - - Use `await_many/1` when you need the results of multiple async relays: - - ```elixir - relayed = - 1..3 - |> Enum.map(&DoubleWorker.new(%{int: &1})) - |> Enum.map(&Oban.Pro.Relay.async/1) - |> Oban.Pro.Relay.await_many() - - #=> [{:ok, 2}, {:ok, 4}, {:ok, 6}] - ``` - - ## Testing with Relay - - Calls to `Relay.await/2` will block until the job runs. That will cause tests to hang when - testing in `:manual` mode until jobs are drained. Unit tests that wrap `Relay` processing can - switch to `:inline` mode so that jobs run immediately. - - For example, assuming `Oban` is configured to run in `:manual` testing mode: - - ```elixir - defmodule MyWorker do - use Oban.Pro.Worker - - alias Oban.Pro.Relay - - @impl Oban.Pro.Worker - def process(%{args: %{"sub" => sub}}) do - results = - sub - |> Enum.map(&MyOtherWorker.new(%{id: &1}) - |> Enum.map(&Relay.async/1) - |> Relay.await_many() - - {:ok, results} - end - end - - test "running multiple sub workers from perform_job/2" do - Oban.Testing.with_testing_mode(:inline, fn -> - assert {:ok, _} = perform_job(MyWorker, %{subs: [1, 2, 3]}) - end) - end - ``` - - ## Usage with Chunks - - `Relay` is intended for use with a single job and isn't suited to awaiting results from - `Oban.Pro.Workers.Chunk` jobs. That's because only one of the chunk's jobs (the leader) will - relay results back. Awaiting any other job in the chunk will time out without returning a proper - result. - """ - - @behaviour Oban.Pro.Handler - - alias Ecto.{Changeset, UUID} - alias Oban.Pro.{Unique, Utils} - alias Oban.{Job, Notifier} - - require Logger - - @type t :: %__MODULE__{job: Job.t(), pid: pid(), ref: UUID.t()} - - @type await_result :: - :ok - | {:ok, term()} - | {:cancel, term()} - | {:discard, term()} - | {:error, :result_too_large | :timeout | Exception.t()} - | {:snooze, integer()} - - defstruct [:job, :name, :pid, :ref] - - @postgres_max_bytes 8000 - - @doc """ - Insert a job for asynchronous execution. - - The returned map contains the caller's pid and a unique ref that is used to await the results. - - ## Examples - - The single arity version takes a job changeset and inserts it: - - relay = - %{id: 123} - |> MyApp.Worker.new() - |> Oban.Pro.Relay.async() - - When the Oban instance has a custom name, or an app has multiple Oban - instances, you can use the two arity version to select an instance: - - changeset = MyApp.Worker.new(%{id: 123}) - Oban.Pro.Relay.async(MyOban, changeset) - """ - @spec async(Oban.name(), Job.changeset()) :: t() | {:error, Job.changeset()} - def async(name \\ Oban, %Changeset{} = changeset) do - pid = self() - ref = if Unique.unique?(changeset), do: Unique.gen_key(changeset), else: UUID.generate() - - meta = - changeset - |> Changeset.get_change(:meta, %{}) - |> Map.put_new(:await_ref, ref) - - changeset = Changeset.put_change(changeset, :meta, meta) - - :ok = Notifier.listen(name, :relay) - - with {:ok, job} <- Oban.insert(name, changeset) do - %__MODULE__{job: job, name: name, pid: pid, ref: ref} - end - end - - @doc """ - Await a relay's execution and return the result. - - Any successful result such as `:ok`, `{:ok, value}`, or `{:cancel, reason}` is passed back as - the await result. When the executed job returns an `{:error, reason}` tuple, raises an - exception, or crashes, the result comes back as an error tuple with the exception. - - By default, `await/1` will timeout after 5 seconds and return `{:error, :timeout}`. The job - itself may continue to run, only the local process stops waiting on it. - - > #### Result Size Limits {: .warning} - > - > When using the default `Oban.Notifiers.Postgres` notifier for PubSub, any value larger than 8kb - > (compressed) can't be broadcast due to a Postgres `NOTIFY` limitation. Instead, awaiting will - > return an `{:error, :result_too_large}` tuple. The `Oban.Notifiers.PG` notifier doesn't have any - > such size limitation, but it requires Distributed Erlang. - - ## Options - - * `:timeout` - the maximum time in milliseconds to wait for the result. Defaults to `5_000`. - * `:with_retries` - when `true`, wait through all retry attempts until the job reaches a final - state (`completed`, `cancelled`, `discarded`), or a timeout is reached. Defaults to `false`. - - ## Examples - - Await a job: - - {:ok, result} = - %{id: 123} - |> MyApp.Worker.new() - |> Oban.Pro.Relay.async() - |> Oban.Pro.Relay.await() - - Increase the wait time with a timeout value: - - %{id: 456} - |> MyApp.Worker.new() - |> Oban.Pro.Relay.async() - |> Oban.Pro.Relay.await(timeout: :timer.seconds(30)) - - Wait through all retry attempts: - - %{id: 789} - |> MyApp.Worker.new() - |> Oban.Pro.Relay.async() - |> Oban.Pro.Relay.await(with_retries: true, timeout: :timer.minutes(1)) - """ - @spec await(t(), timeout() | keyword()) :: await_result() - def await(relay, opts \\ []) - - def await(relay, timeout) when timeout == :infinity or is_integer(timeout) do - await(relay, timeout: timeout) - end - - def await(%{name: name, pid: pid, ref: ref}, opts) when is_list(opts) do - check_ownership!(pid) - - timeout = Keyword.get(opts, :timeout, 5_000) - retries = Keyword.get(opts, :with_retries, false) - - try do - await(ref, timeout, retries) - after - Notifier.unlisten(name, :relay) - end - end - - defp await(ref, timeout, with_retries) do - receive do - {:notification, :relay, %{"ref" => ^ref, "result" => result} = payload} -> - # For backward compati. Existing nodes won't return these values in the payload - state = Map.get(payload, "state", "success") - attempt = Map.get(payload, "attempt", 1) - max_attempts = Map.get(payload, "max_attempts", 20) - - decoded = Utils.decode64(result) - - if with_retries and state == "failure" and attempt < max_attempts do - await(ref, timeout, with_retries) - else - decoded - end - after - timeout -> {:error, :timeout} - end - end - - @doc """ - Await replies from multiple relays and return the results. - - It returns a list of the results in the same order as the relays supplied as the first argument. - - Unlike `Task.await_many` or `Task.yield_many`, `await_many/2` may return partial results when - the timeout is reached. When a job hasn't finished executing the value will be `{:error, - :timeout}` - - ## Examples - - Await multiple jobs without any errors or timeouts: - - relayed = - 1..3 - |> Enum.map(&DoubleWorker.new(%{int: &1})) - |> Enum.map(&Oban.Pro.Relay.async(&1)) - |> Oban.Pro.Relay.await_many() - - #=> [{:ok, 2}, {:ok, 4}, {:ok, 6}] - - Await multiple jobs with an error timeout: - - relayed = - [1, 2, 300_000_000] - |> Enum.map(&SlowWorker.new(%{int: &1})) - |> Enum.map(&Oban.Pro.Relay.async(&1)) - |> Oban.Pro.Relay.await_many(100) - - #=> [{:ok, 2}, {:ok, 4}, {:error, :timeout}] - """ - @spec await_many([t()], timeout()) :: [await_result()] - def await_many([%__MODULE__{} | _] = relays, timeout \\ 5_000) do - awaited = - for %{pid: pid, ref: ref} <- relays, into: %{} do - check_ownership!(pid) - - {ref, {:error, :timeout}} - end - - error_ref = make_ref() - timer_ref = maybe_send_after(error_ref, timeout) - - try do - await_many(relays, awaited, %{}, error_ref) - after - if is_reference(timer_ref), do: Process.cancel_timer(timer_ref) - - receive do: (^error_ref -> :ok), after: (0 -> :ok) - end - after - relays - |> Enum.map(& &1.name) - |> Enum.uniq() - |> Enum.each(&Notifier.unlisten(&1, :relay)) - end - - defp await_many(relays, awaited, replied, _error_ref) when map_size(awaited) == 0 do - for %{ref: ref} <- relays, do: Map.fetch!(replied, ref) - end - - defp await_many(relays, awaited, replied, error_ref) do - receive do - ^error_ref -> - for %{ref: ref} <- relays, do: replied[ref] || awaited[ref] - - {:notification, :relay, %{"ref" => ref, "result" => res}} -> - if Map.has_key?(awaited, ref) do - await_many( - relays, - Map.delete(awaited, ref), - Map.put(replied, ref, Utils.decode64(res)), - error_ref - ) - else - await_many(relays, awaited, replied, error_ref) - end - end - end - - @doc false - def handle_event([:oban, :job, _], _, %{conf: conf, job: job, state: state} = meta, _) do - with %{"await_ref" => aref} <- job.meta, - oban_pid when is_pid(oban_pid) <- Oban.whereis(conf.name) do - payload = %{ - "ref" => aref, - "result" => extract_result(conf, meta), - "state" => state, - "attempt" => job.attempt, - "max_attempts" => job.max_attempts - } - - Notifier.notify(conf, :relay, payload) - end - catch - kind, value -> - Logger.error(fn -> - "[Oban.Pro.Relay] handler error: " <> Exception.format(kind, value) - end) - - :ok - end - - @doc false - @impl Oban.Pro.Handler - def on_start do - events = [ - [:oban, :job, :stop], - [:oban, :job, :exception] - ] - - :telemetry.attach_many("oban.pro.relay", events, &__MODULE__.handle_event/4, nil) - end - - @doc false - @impl Oban.Pro.Handler - def on_stop do - :telemetry.detach("oban.pro.relay") - end - - # Messaging - - defp check_ownership!(pid) do - unless pid == self() do - raise ArgumentError, - "relay must be awaited by #{inspect(pid)}, was awaited by #{inspect(self())}" - end - end - - defp maybe_send_after(_error_ref, :infinity), do: nil - - defp maybe_send_after(error_ref, timeout) do - Process.send_after(self(), error_ref, timeout) - end - - # Result Handling - - defp extract_result(conf, %{state: state, result: result}) - when state in [:cancelled, :snoozed, :success] do - encoded = Utils.encode64(result) - - if match?({Oban.Notifiers.Postgres, _}, conf.notifier) and - byte_size(encoded) > @postgres_max_bytes do - Utils.encode64({:error, :result_too_large}) - else - encoded - end - end - - defp extract_result(_conf, %{state: :discard, job: job}) do - Utils.encode64({:discard, job.unsaved_error.reason}) - end - - defp extract_result(_conf, %{state: :failure, error: error}) do - Utils.encode64({:error, error}) - end -end diff --git a/vendor/oban_pro/lib/oban/pro/stage.ex b/vendor/oban_pro/lib/oban/pro/stage.ex deleted file mode 100644 index 8403c634..00000000 --- a/vendor/oban_pro/lib/oban/pro/stage.ex +++ /dev/null @@ -1,36 +0,0 @@ -defmodule Oban.Pro.Stage do - @moduledoc false - - alias Oban.Job - - @type args :: Job.args() - @type changeset :: Job.changeset() - @type conf :: term() - @type job :: Job.t() - @type opts :: Keyword.t() - @type reason :: term() - @type result :: term() - - @doc """ - Initialize configuration passed to various before callbacks. - """ - @callback init(module(), opts()) :: {:ok, conf()} | {:error, reason()} - - @doc """ - Pre-process the args and/or opts for a job before calling `new/2`. - """ - @callback before_new(args(), opts(), conf()) :: {:ok, args(), opts()} | {:error, reason()} - - @doc """ - Pre-process a job before calling `process/1`. - """ - @callback before_process(job(), conf()) :: - {:ok, job()} | {:cancel, reason()} | {:error, reason()} | {:snooze, integer()} - - @doc """ - Post-process a job before returning from the wrapping `perform/1` - """ - @callback after_process(result(), job(), conf()) :: :ok - - @optional_callbacks before_new: 3, before_process: 2, after_process: 3 -end diff --git a/vendor/oban_pro/lib/oban/pro/stages/chain.ex b/vendor/oban_pro/lib/oban/pro/stages/chain.ex deleted file mode 100644 index 68af9fba..00000000 --- a/vendor/oban_pro/lib/oban/pro/stages/chain.ex +++ /dev/null @@ -1,232 +0,0 @@ -defmodule Oban.Pro.Stages.Chain do - @moduledoc false - - @behaviour Oban.Pro.Flusher - @behaviour Oban.Pro.Stage - - import Ecto.Query - - alias Ecto.Changeset - alias Oban.{Job, Repo, Validation} - alias Oban.Pro.Utils - - @hold_date ~U[3000-01-01 00:00:00.000000Z] - - defstruct [:by, :worker, on_cancelled: :ignore, on_discarded: :ignore] - - defmacrop drop_hold(meta) do - quote do - fragment(~s(? || '{"on_hold":false, "uniq_bmp":[]}'), unquote(meta)) - end - end - - defmacrop continue_state(prefix, meta) do - quote do - fragment( - """ - (CASE WHEN ? \\? 'orig_scheduled_at' THEN 'scheduled' ELSE 'available' END)::?.oban_job_state - """, - unquote(meta), - unquote(prefix) - ) - end - end - - defmacrop continue_at(meta, now) do - quote do - fragment( - """ - CASE - WHEN ? \\? 'orig_scheduled_at' THEN timezone('utc', to_timestamp(div((?->'orig_scheduled_at')::bigint, 1000000)::float)) - ELSE ? - END - """, - unquote(meta), - unquote(meta), - unquote(now) - ) - end - end - - defmacrop same_chain(meta_a, meta_b) do - quote do - fragment("?->>'chain_id' = ?->>'chain_id'", unquote(meta_a), unquote(meta_b)) - end - end - - @spec chain?(Job.changeset()) :: boolean() - def chain?(changeset), do: match?(%{changes: %{meta: %{"chain_id" => _}}}, changeset) - - @spec get_key(Job.changeset()) :: nil | String.t() - def get_key(%{changes: %{meta: %{"chain_id" => chain_id}}}), do: chain_id - def get_key(_changeset), do: nil - - @spec continuable?(String.t(), map()) :: boolean() - def continuable?(state, meta) do - state == "completed" or - (state == "cancelled" and meta["on_cancelled"] != "hold") or - (state == "discarded" and meta["on_discarded"] != "hold") - end - - @spec to_postponed(Job.changeset()) :: Job.changeset() - def to_postponed(changeset) do - meta = - if Changeset.get_field(changeset, :state) == "scheduled" do - orig_at = - changeset - |> Changeset.get_change(:scheduled_at) - |> DateTime.to_unix(:microsecond) - - %{"on_hold" => true, "orig_scheduled_at" => orig_at} - else - %{"on_hold" => true} - end - - changeset - |> Changeset.update_change(:meta, &Map.merge(&1, meta)) - |> Changeset.put_change(:state, "scheduled") - |> Changeset.put_change(:scheduled_at, @hold_date) - end - - @doc false - def rescue_chains(conf, opts \\ []) do - prev_query = - Job - |> where([j], j.state in ~w(available scheduled retryable executing)) - |> where([j], fragment("? \\? 'chain_id'", j.meta)) - |> where([j], same_chain(j.meta, parent_as(:jobs).meta)) - |> where([j], fragment("?->>'on_hold'", j.meta) in ~w(true false)) - |> where([j], j.id < parent_as(:jobs).id) - - subquery = - from(j in Job, as: :jobs) - |> where([j], j.state == "scheduled") - |> where([j], fragment("? \\? 'chain_id'", j.meta)) - |> where([j], fragment("?->>'on_hold' = 'true'", j.meta)) - |> where([j], not exists(prev_query)) - |> order_by(asc: :id) - |> limit(1) - - query = - Job - |> join(:inner, [j], x in subquery(subquery), on: j.id == x.id) - |> update([j], - set: [ - meta: drop_hold(j.meta), - state: continue_state(literal(^conf.prefix), j.meta), - scheduled_at: continue_at(j.meta, ^DateTime.utc_now()) - ] - ) - - Repo.update_all(conf, query, [], opts) - - :ok - end - - @doc false - def to_chain_id(chain_by, worker, args, opts) do - chain_data = - Enum.map([:queue | chain_by], fn - :queue -> opts |> Keyword.get(:queue, "default") |> to_string() - :worker -> inspect(worker) - [:args, keys] -> take_keys(args, keys) - [:meta, keys] -> opts |> Keyword.get(:meta, %{}) |> take_keys(keys) - end) - - Utils.hash64(chain_data) - end - - # Stage - - @impl Oban.Pro.Stage - def init(worker, opts) do - with :ok <- validate(opts) do - opts = - opts - |> Keyword.put_new(:by, :worker) - |> Keyword.update!(:by, &Utils.normalize_by/1) - |> Keyword.put(:worker, worker) - - {:ok, struct!(__MODULE__, opts)} - end - end - - @impl Oban.Pro.Stage - def before_new(args, opts, conf) do - conf = merge_conf_opts(conf, opts) - - meta = %{ - "chain" => true, - "chain_id" => to_chain_id(conf.by, conf.worker, args, opts), - "on_cancelled" => to_string(conf.on_cancelled), - "on_discarded" => to_string(conf.on_discarded), - "on_hold" => false - } - - opts = Keyword.update(opts, :meta, meta, &Map.merge(&1, meta)) - - {:ok, args, opts} - end - - # Flushing - - @impl Oban.Pro.Flusher - def to_flush_mfa(job, conf) do - case job do - %{meta: %{"chain_id" => _}} -> {__MODULE__, :on_flush, [job, conf]} - _ -> :ignore - end - end - - def on_flush(%{state: state, meta: %{"chain_id" => chain_id} = meta}, conf) do - if continuable?(state, meta) do - subquery = - Job - |> select([j], j.id) - |> where([j], j.state == "scheduled") - |> where([j], fragment("? \\? 'chain_id'", j.meta)) - |> where([j], fragment("?->>'chain_id'", j.meta) == ^chain_id) - |> where([j], fragment("?->>'on_hold'", j.meta) == "true") - |> order_by(asc: :id) - |> limit(1) - |> lock("FOR UPDATE") - - query = - Job - |> join(:inner, [j], x in subquery(subquery), on: j.id == x.id) - |> update([j], - set: [ - meta: drop_hold(j.meta), - state: continue_state(literal(^conf.prefix), j.meta), - scheduled_at: continue_at(j.meta, ^DateTime.utc_now()) - ] - ) - - Repo.update_all(conf, query, []) - end - end - - # Helpers - - defp merge_conf_opts(conf, opts) do - case Keyword.fetch(opts, :chain) do - {:ok, chain} -> - Enum.reduce(chain, conf, fn {key, val}, acc -> Map.replace!(acc, key, val) end) - - :error -> - conf - end - end - - defp take_keys(map, keys) do - Enum.map(keys, fn key -> inspect(map[key] || map[to_string(key)]) end) - end - - defp validate(opts) do - Validation.validate_schema(opts, - by: {:custom, &Oban.Pro.Validation.validate_by/1}, - on_cancelled: {:enum, [:hold, :ignore]}, - on_discarded: {:enum, [:hold, :ignore]} - ) - end -end diff --git a/vendor/oban_pro/lib/oban/pro/stages/deadline.ex b/vendor/oban_pro/lib/oban/pro/stages/deadline.ex deleted file mode 100644 index 7a026206..00000000 --- a/vendor/oban_pro/lib/oban/pro/stages/deadline.ex +++ /dev/null @@ -1,128 +0,0 @@ -defmodule Oban.Pro.Stages.Deadline do - @moduledoc false - - @behaviour Oban.Pro.Stage - - alias Oban.Pro.Utils - alias Oban.{Registry, Validation} - - @time_units ~w( - second - seconds - minute - minutes - hour - hours - day - days - week - weeks - )a - - defstruct [:in, force: false] - - @impl Oban.Pro.Stage - def init(worker \\ nil, period) - - def init(worker, period) when is_integer(period) or is_tuple(period) do - init(worker, in: period) - end - - def init(_worker, opts) when is_list(opts) do - with :ok <- validate(opts) do - conf = - opts - |> Keyword.update!(:in, &Utils.cast_period/1) - |> then(&struct(__MODULE__, &1)) - - {:ok, conf} - end - end - - @impl Oban.Pro.Stage - def before_new(args, opts, conf) do - {deadline, opts} = Keyword.pop(opts, :deadline, conf.in) - - seconds = Utils.cast_period(deadline) - now = DateTime.utc_now() - - offset = - cond do - schedule_in = opts[:schedule_in] -> Utils.cast_period(schedule_in) + seconds - scheduled_at = opts[:scheduled_at] -> DateTime.diff(scheduled_at, now) + seconds - true -> seconds - end - - at = - now - |> DateTime.add(offset) - |> DateTime.to_unix() - - meta = %{"deadline_at" => at} - opts = Keyword.update(opts, :meta, meta, &Map.merge(&1, meta)) - - {:ok, args, opts} - end - - @impl Oban.Pro.Stage - def before_process(%{meta: %{"deadline_at" => at}} = job, conf) do - now = DateTime.utc_now() - dat = DateTime.from_unix!(at) - - if :gt == DateTime.compare(now, dat) do - {:cancel, "deadline at #{inspect(dat)} exired"} - else - if conf.force, do: force_timeout(now, dat, job) - - {:ok, job} - end - end - - def before_process(job, conf) do - opts = [scheduled_at: job.scheduled_at, meta: job.meta] - - {:ok, _args, opts} = before_new(job.args, opts, conf) - - meta = Keyword.fetch!(opts, :meta) - - before_process(%{job | meta: meta}, conf) - end - - defp force_timeout(now, dat, job) do - parent = self() - - Task.start(fn -> - ref = Process.monitor(parent) - - Process.send_after(self(), :cancel, DateTime.diff(dat, now, :millisecond)) - - receive do - {:DOWN, ^ref, :process, _pid, _reason} -> - Process.demonitor(ref, [:flush]) - - :cancel -> - with pid when is_pid(pid) <- Registry.whereis(job.conf.name, {:producer, job.queue}) do - payload = %{"action" => "pkill", "job_id" => job.id} - - send(pid, {:notification, :signal, payload}) - end - end - end) - end - - # Validation - - defp validate(opts) do - Validation.validate_schema(opts, - in: {:custom, &validate_period/1}, - force: :boolean - ) - end - - defp validate_period(period) when is_integer(period), do: :ok - defp validate_period({value, units}) when is_integer(value) and units in @time_units, do: :ok - - defp validate_period(period) do - {:error, "expected deadline to be an integer in seconds or a tuple, got: #{inspect(period)}"} - end -end diff --git a/vendor/oban_pro/lib/oban/pro/stages/encrypted.ex b/vendor/oban_pro/lib/oban/pro/stages/encrypted.ex deleted file mode 100644 index 32672dd4..00000000 --- a/vendor/oban_pro/lib/oban/pro/stages/encrypted.ex +++ /dev/null @@ -1,83 +0,0 @@ -defmodule Oban.Pro.Stages.Encrypted do - @moduledoc false - - @behaviour Oban.Pro.Stage - - defstruct [:key, cipher: :aes_256_ctr, iv_bytes: 16] - - @impl Oban.Pro.Stage - def init(_worker \\ nil, opts) do - case Keyword.fetch(opts, :key) do - {:ok, key} when is_binary(key) -> - {:ok, %__MODULE__{key: Base.decode64!(key)}} - - {:ok, {_mod, _fun, _arg} = key} -> - {:ok, %__MODULE__{key: key}} - - _ -> - {:error, "expected :key to be a binary or mfa, got: #{inspect(opts)}"} - end - end - - @impl Oban.Pro.Stage - def before_new(args, opts, conf) do - iv = :crypto.strong_rand_bytes(conf.iv_bytes) - - meta = %{"encrypted" => true, "iv" => Base.encode64(iv)} - - opts = - opts - |> Keyword.put_new(:meta, %{}) - |> Keyword.update!(:meta, &Map.merge(&1, meta)) - - {:ok, %{"data" => encode(args, iv, conf)}, opts} - end - - @impl Oban.Pro.Stage - def before_process(%{args: args, meta: meta} = job, conf) do - with %{"encrypted" => true, "iv" => iv} <- meta, - %{"data" => data} <- args do - {:ok, %{job | args: decode(data, iv, conf)}} - else - _ -> - {:ok, job} - end - end - - # Helpers - - defp resolve_key({mod, fun, arg}) do - mod - |> apply(fun, arg) - |> Base.decode64!() - end - - defp resolve_key(key), do: key - - defp encode(data, iv, conf) do - key = resolve_key(conf.key) - json = Oban.JSON.encode!(data) - - conf.cipher - |> :crypto.crypto_one_time(key, iv, json, true) - |> Base.encode64() - rescue - error -> reraise error, prune_args_from_stacktrace(__STACKTRACE__) - end - - defp decode(data, iv, conf) do - key = resolve_key(conf.key) - - conf.cipher - |> :crypto.crypto_one_time(key, Base.decode64!(iv), Base.decode64!(data), false) - |> Oban.JSON.decode!() - rescue - error -> reraise error, prune_args_from_stacktrace(__STACKTRACE__) - end - - defp prune_args_from_stacktrace([{mod, fun, [_ | _] = args, info} | rest]) do - [{mod, fun, length(args), info} | rest] - end - - defp prune_args_from_stacktrace(stacktrace), do: stacktrace -end diff --git a/vendor/oban_pro/lib/oban/pro/stages/executing.ex b/vendor/oban_pro/lib/oban/pro/stages/executing.ex deleted file mode 100644 index 8373154d..00000000 --- a/vendor/oban_pro/lib/oban/pro/stages/executing.ex +++ /dev/null @@ -1,17 +0,0 @@ -defmodule Oban.Pro.Stages.Executing do - @moduledoc false - - @behaviour Oban.Pro.Stage - - @impl Oban.Pro.Stage - def init(_worker, _opts), do: {:ok, :ignore} - - @impl Oban.Pro.Stage - def before_process(%{conf: conf} = job, _conf) when is_struct(conf) do - Registry.register(Oban.Registry, {conf.name, {:executing, job.id}}, nil) - - {:ok, job} - end - - def before_process(job, _conf), do: {:ok, job} -end diff --git a/vendor/oban_pro/lib/oban/pro/stages/hooks.ex b/vendor/oban_pro/lib/oban/pro/stages/hooks.ex deleted file mode 100644 index 18490ae4..00000000 --- a/vendor/oban_pro/lib/oban/pro/stages/hooks.ex +++ /dev/null @@ -1,147 +0,0 @@ -defmodule Oban.Pro.Stages.Hooks do - @moduledoc false - - @behaviour Oban.Pro.Stage - - alias Oban.Pro.Utils - - require Logger - - defstruct modules: [] - - @impl Oban.Pro.Stage - def init(worker, modules) do - {:ok, struct!(__MODULE__, modules: [worker | modules])} - end - - @impl Oban.Pro.Stage - def before_new(args, opts, %{modules: modules}) do - (modules ++ get_hooks()) - |> Enum.filter(&hook_exported?(&1, :before_new, 2)) - |> Enum.reduce_while({:ok, args, opts}, fn module, {:ok, args, opts} -> - case module.before_new(args, opts) do - {:ok, _, _} = ok -> {:cont, ok} - other -> {:halt, other} - end - end) - end - - @impl Oban.Pro.Stage - def before_process(job, %{modules: modules}) do - (modules ++ get_hooks()) - |> Enum.filter(&hook_exported?(&1, :before_process, 1)) - |> Enum.reduce_while({:ok, job}, fn module, {:ok, job} -> - case module.before_process(job) do - {:ok, _} = ok -> {:cont, ok} - other -> {:halt, other} - end - end) - end - - def attach_hook(module) do - with :ok <- Utils.validate_opts([module], &validate_module/1) do - modules = - get_hooks() - |> Enum.concat([module]) - |> Enum.uniq() - - :persistent_term.put(:oban_pro_hooks, modules) - end - end - - def detach_hook(module) do - case get_hooks() do - [_ | _] = modules -> - :persistent_term.put(:oban_pro_hooks, modules -- [module]) - - _ -> - :ok - end - end - - def handle_event(_event, _timing, meta, _conf) do - %{job: %{unsaved_error: unsaved_error}, state: state} = meta - - with {worker, job, opts} <- resolve_processing(meta.job) do - hook_state = exec_to_hook_state(state) - module_hooks = Keyword.get(opts, :hooks, []) - global_hooks = get_hooks() - - job = %{job | unsaved_error: unsaved_error} - result = Map.get(meta, :result, nil) - - ([worker | module_hooks] ++ global_hooks) - |> Enum.uniq() - |> Enum.each(fn module -> - cond do - hook_exported?(module, :after_process, 3) -> - module.after_process(hook_state, job, result) - - hook_exported?(module, :after_process, 2) -> - module.after_process(hook_state, job) - - true -> - :ok - end - end) - end - catch - kind, value -> - Logger.error(fn -> - "[Oban.Pro.Worker] hook error: " <> Exception.format(kind, value, __STACKTRACE__) - end) - after - Process.delete(:oban_processing) - end - - defp resolve_processing(job) do - # The event may be triggered by a producer after the process was killed or experienced a - # linked crash. In that event, the processing data isn't in the dictionary and we need to - # resolve and apply before hooks to the job again. - with nil <- Process.get(:oban_processing), - {:ok, worker} <- Oban.Worker.from_string(job.worker), - opts = worker.__opts__(), - {:ok, job} <- Oban.Pro.Worker.before_process(worker, job, opts) do - processing = {worker, job, opts} - Process.put(:oban_processing, processing) - processing - end - end - - defp get_hooks, do: :persistent_term.get(:oban_pro_hooks, []) - - defp exec_to_hook_state(:cancelled), do: :cancel - defp exec_to_hook_state(:success), do: :complete - defp exec_to_hook_state(:discard), do: :discard - defp exec_to_hook_state(:failure), do: :error - defp exec_to_hook_state(:snoozed), do: :snooze - - defp hook_exported?(mod, fun, arity) do - Code.ensure_loaded?(mod) and function_exported?(mod, fun, arity) - end - - # Validation - - defp validate_module(module) do - cond do - not Code.ensure_loaded?(module) -> - {:error, "unable to load callback module #{inspect(module)}"} - - function_exported?(module, :before_new, 2) -> - :ok - - function_exported?(module, :after_process, 2) -> - :ok - - function_exported?(module, :after_process, 3) -> - :ok - - function_exported?(module, :before_process, 1) -> - :ok - - true -> - {:error, - "#{inspect(module)} doesn't implement after_process/3, before_new/2, before_process/1"} - end - end -end diff --git a/vendor/oban_pro/lib/oban/pro/stages/recorded.ex b/vendor/oban_pro/lib/oban/pro/stages/recorded.ex deleted file mode 100644 index 42215c3c..00000000 --- a/vendor/oban_pro/lib/oban/pro/stages/recorded.ex +++ /dev/null @@ -1,77 +0,0 @@ -defmodule Oban.Pro.Stages.Recorded do - @moduledoc false - - @behaviour Oban.Pro.Stage - - alias Oban.Pro.Utils - alias Oban.Validation - - defstruct to: :return, limit: 64_000_000, safe_decode: false - - @impl Oban.Pro.Stage - def init(_worker, true) do - {:ok, %__MODULE__{}} - end - - def init(_worker, opts) do - with :ok <- validate(opts) do - {:ok, struct(__MODULE__, opts)} - end - end - - @impl Oban.Pro.Stage - def before_new(args, opts, _conf) do - meta = %{"recorded" => true} - opts = Keyword.update(opts, :meta, meta, &Map.merge(&1, meta)) - - {:ok, args, opts} - end - - @impl Oban.Pro.Stage - def after_process(result, %{meta: %{"recorded" => true}} = job, conf) do - with {:ok, return} <- result do - encoded = Utils.encode64(return) - - cond do - job.conf.engine != Oban.Pro.Engines.Smart -> - {:error, "recording requires the Smart engine"} - - byte_size(encoded) > conf.limit -> - {:error, "return is #{byte_size(encoded)} bytes, larger than the limit: #{conf.limit}"} - - true -> - Process.put(:oban_recorded, %{to_string(conf.to) => encoded}) - - :ok - end - end - end - - def after_process(_result, _job, _conf), do: :ok - - @spec fetch_recorded(Oban.Job.t(), conf :: map()) :: {:ok, term()} | {:error, term()} - def fetch_recorded(job, conf) do - to = to_string(conf.to) - - case job.meta do - %{"recorded" => true, ^to => value} -> - {:ok, Utils.decode64(value, decode_opts(job.meta))} - - _ -> - {:error, :missing} - end - end - - # Helpers - - defp validate(opts) do - Validation.validate_schema(opts, - to: {:or, [:atom, :string]}, - limit: :pos_integer, - safe_decode: :boolean - ) - end - - defp decode_opts(%{"safe_decode" => true}), do: [:safe] - defp decode_opts(_meta), do: [] -end diff --git a/vendor/oban_pro/lib/oban/pro/stages/structured.ex b/vendor/oban_pro/lib/oban/pro/stages/structured.ex deleted file mode 100644 index 99c93782..00000000 --- a/vendor/oban_pro/lib/oban/pro/stages/structured.ex +++ /dev/null @@ -1,191 +0,0 @@ -defmodule Oban.Pro.Stages.Structured do - @moduledoc false - - @behaviour Oban.Pro.Stage - - alias Ecto.Changeset - alias Oban.Pro.Utils - - defstruct [:worker] - - defmodule Term do - @moduledoc false - - use Ecto.Type - - alias Oban.Pro.Utils - - @impl Ecto.Type - def type, do: :string - - @impl Ecto.Type - def cast("term-" <> data), do: {:ok, Utils.decode64(data)} - def cast(term), do: {:ok, "term-" <> Utils.encode64(term)} - - @impl Ecto.Type - def dump(_term), do: {:ok, :noop} - - @impl Ecto.Type - def load(_data), do: {:ok, :noop} - end - - @impl Oban.Pro.Stage - def init(worker, _opts) do - if function_exported?(worker, :__args_schema__, 0) do - {:ok, %__MODULE__{worker: worker}} - else - {:ok, :ignore} - end - end - - @impl Oban.Pro.Stage - def before_new(args, opts, :ignore), do: {:ok, args, opts} - - def before_new(args, opts, conf) do - {:ok, changes} = gather_changes(conf.worker, args) - - meta = %{structured: true} - opts = Keyword.update(opts, :meta, meta, &Map.merge(&1, meta)) - - {:ok, dump_changes(changes), opts} - catch - message -> {:error, message} - end - - @impl Oban.Pro.Stage - def before_process(job, :ignore), do: {:ok, job} - - def before_process(%{args: args} = job, conf) do - {:ok, changes} = gather_changes(conf.worker, args) - - {:ok, %{job | args: changes}} - catch - message -> {:error, message} - end - - defp gather_changes(module, args) do - fields = module.__args_schema__() - struct = struct(module, []) - - {fields, required, defaults, merge} = split_fields(fields, args) - - keys = Map.keys(fields) - - args = - if Enum.any?(Map.keys(args), &is_binary/1) do - defaults - |> Map.merge(args) - |> Map.new(fn {key, val} -> {to_string(key), val} end) - else - Map.merge(defaults, args) - end - - changeset = - {struct, fields} - |> Changeset.cast(args, keys) - |> Changeset.validate_required(required) - |> validate_allowed(args, keys) - - if changeset.valid? do - changeset - |> Changeset.apply_changes() - |> Map.merge(merge) - |> then(&{:ok, &1}) - else - changeset - |> Utils.to_translated_errors() - |> Enum.map_join(", ", fn {field, error} -> "#{inspect(field)} #{error}" end) - |> throw() - end - end - - defp split_fields(fields, args) do - Enum.reduce(fields, {%{}, [], %{}, %{}}, fn {name, opts}, {keep, required, defaults, merge} -> - required = if opts[:required], do: [name | required], else: required - - defaults = - case Keyword.fetch(opts, :default) do - {:ok, default} -> Map.put(defaults, name, default) - :error -> defaults - end - - case Map.new(opts) do - %{cardinality: :one, module: module, required: embed_required?, type: :embed} -> - data = args[name] || args[to_string(name)] - keep = Map.put(keep, name, :any) - - if is_nil(data) and not embed_required? do - {keep, required, defaults, merge} - else - {:ok, embed} = gather_changes(module, data || %{}) - - {keep, required, defaults, Map.put(merge, name, embed)} - end - - %{cardinality: :many, module: module, type: :embed} -> - embed_list = - for sub_arg <- args[name] || args[to_string(name)] || [] do - {:ok, embed} = gather_changes(module, sub_arg) - - embed - end - - {Map.put(keep, name, :any), required, defaults, Map.put(merge, name, embed_list)} - - %{type: :enum} -> - enum = Ecto.ParameterizedType.init(Ecto.Enum, values: opts[:values]) - - {Map.put(keep, name, enum), required, defaults, merge} - - %{type: :term} -> - {Map.put(keep, name, Term), required, defaults, merge} - - %{type: :uuid} -> - {Map.put(keep, name, :binary_id), required, defaults, merge} - - %{type: {:array, :enum}} -> - enum = Ecto.ParameterizedType.init(Ecto.Enum, values: opts[:values]) - - {Map.put(keep, name, {:array, enum}), required, defaults, merge} - - %{type: {:array, :uuid}} -> - {Map.put(keep, name, {:array, :binary_id}), required, defaults, merge} - - %{type: type} -> - {Map.put(keep, name, type), required, defaults, merge} - end - end) - end - - defp validate_allowed(changeset, args, keys) do - keys = MapSet.new(keys, &to_string/1) - - Enum.reduce(args, changeset, fn {key, _val}, changeset -> - if to_string(key) in keys do - changeset - else - Changeset.add_error(changeset, key, "is an unexpected key") - end - end) - end - - defp dump_changes(struct) when is_struct(struct, Date), do: struct - defp dump_changes(struct) when is_struct(struct, DateTime), do: struct - defp dump_changes(struct) when is_struct(struct, Decimal), do: struct - defp dump_changes(struct) when is_struct(struct, NaiveDateTime), do: struct - defp dump_changes(struct) when is_struct(struct, Time), do: struct - - defp dump_changes(struct) when is_struct(struct) do - for {key, val} <- Map.from_struct(struct), not is_nil(val), into: %{} do - {key, dump_changes(val)} - end - end - - defp dump_changes(map) when is_map(map) do - Map.new(map, fn {key, val} -> {key, dump_changes(val)} end) - end - - defp dump_changes(list) when is_list(list), do: Enum.map(list, &dump_changes/1) - - defp dump_changes(term), do: term -end diff --git a/vendor/oban_pro/lib/oban/pro/testing.ex b/vendor/oban_pro/lib/oban/pro/testing.ex deleted file mode 100644 index baadf578..00000000 --- a/vendor/oban_pro/lib/oban/pro/testing.ex +++ /dev/null @@ -1,1334 +0,0 @@ -defmodule Oban.Pro.Testing do - @moduledoc """ - Advanced helpers for testing supervised Oban instances, workers, and making assertions about - enqueued jobs. - - The `Oban.Pro.Testing` module is a drop-in replacement for `Oban.Testing`, with additional - functions tailored toward integration testing and Pro modules. - - ## Usage in Tests - - The most convenient way to use `Oban.Pro.Testing` is to `use` the module: - - use Oban.Pro.Testing, repo: MyApp.Repo - - Other repo-specific configuration options can also be used: - - use Oban.Pro.Testing, repo: MyApp.Repo, prefix: "private", log: :debug - - If you already have `use Oban.Testing` in your tests or test cases, simply replace it with `use - Oban.Pro.Testing`. - - ## Naming Convention - - The testing helpers in this module adhere to the following naming convention: - - * `perform_*` — executes jobs locally, without touching the database, for unit testing. - - * `drain_*` — execute jobs inline, for integration testing. - - * `run_*` — insert jobs into the database and execute them inline, for integration testing. - - ## Shared Repo Options - - The `use` macro accepts all of these repo-specific configuration options, and they may be passed - to all database functions (`run_`, `drain_`, etc.) - - * `:log` — a usable log level or `false` to disable logging. See `t:Logger.level/0` for valid - options. - - * `:prefix` — an optional database prefix. Defaults to `public`. - - * `:repo` — the name of an Ecto repo, which should be running in sandbox mode. - """ - - import Ecto.Query - import ExUnit.Assertions, only: [assert: 2, flunk: 1] - import ExUnit.Callbacks, only: [on_exit: 2, start_supervised!: 1] - - alias Ecto.Adapters.SQL.Sandbox - alias Ecto.Changeset - alias Oban.Pro.Engines.Smart - alias Oban.Pro.Workers.Chunk - alias Oban.Pro.{Batch, Utils, Workflow} - alias Oban.Queue.Executor - alias Oban.{Config, Job, Repo, Worker} - - @type repo_option :: {:log, false | Logger.level()} | {:prefix, String.t()} | {:repo, module()} - - @type drain_option :: - repo_option() - | {:queue, atom()} - | {:with_limit, pos_integer()} - | {:with_recursion, boolean()} - | {:with_safety, boolean()} - | {:with_scheduled, boolean()} - | {:with_summary, boolean()} - - @type drain_summary :: %{ - cancelled: non_neg_integer(), - completed: non_neg_integer(), - discarded: non_neg_integer(), - exhausted: non_neg_integer(), - retryable: non_neg_integer(), - scheduled: non_neg_integer() - } - - @type drain_result :: drain_summary() | [Job.t()] - - @type perform_option :: Job.option() | repo_option() - - @typedoc """ - Batch callback identifiers, correlating to a `handle_` callback function. - """ - @type callback :: :attempted | :completed | :discarded | :exhausted - - @doc false - defmacro __using__(repo_opts) do - if not Keyword.has_key?(repo_opts, :repo) do - raise ArgumentError, "testing requires a :repo option to be set" - end - - quote do - def all_enqueued(opts \\ []) do - unquote(repo_opts) - |> Keyword.merge(opts) - |> Oban.Pro.Testing.all_enqueued() - end - - def assert_enqueue(opts \\ [], fun) do - unquote(repo_opts) - |> Keyword.merge(opts) - |> Oban.Pro.Testing.assert_enqueue(fun) - end - - def assert_enqueued(opts \\ [], timeout \\ :none) do - unquote(repo_opts) - |> Keyword.merge(opts) - |> Oban.Pro.Testing.assert_enqueued(timeout) - end - - def refute_enqueue(opts \\ [], fun) do - unquote(repo_opts) - |> Keyword.merge(opts) - |> Oban.Pro.Testing.refute_enqueue(fun) - end - - def refute_enqueued(opts \\ [], timeout \\ :none) do - unquote(repo_opts) - |> Keyword.merge(opts) - |> Oban.Pro.Testing.refute_enqueued(timeout) - end - - def build_job(worker, args, opts \\ []) do - Oban.Pro.Testing.build_job(worker, args, Keyword.merge(unquote(repo_opts), opts)) - end - - def drain_jobs(opts \\ []) do - unquote(repo_opts) - |> Keyword.merge(opts) - |> Oban.Pro.Testing.drain_jobs() - end - - def perform_job(job) when is_struct(job) do - Oban.Pro.Testing.perform_job(job, unquote(repo_opts)) - end - - def perform_job(job, opts) when is_struct(job) and is_list(opts) do - Oban.Pro.Testing.perform_job(job, Keyword.merge(unquote(repo_opts), opts)) - end - - def perform_job(worker, args, opts \\ []) do - Oban.Pro.Testing.perform_job(worker, args, Keyword.merge(unquote(repo_opts), opts)) - end - - def perform_callback(worker, callback, args, opts \\ []) do - opts = Keyword.merge(unquote(repo_opts), opts) - - Oban.Pro.Testing.perform_callback(worker, callback, args, opts) - end - - def perform_chunk(worker, args, opts \\ []) do - Oban.Pro.Testing.perform_chunk(worker, args, Keyword.merge(unquote(repo_opts), opts)) - end - - def run_batch(batch, opts \\ []) do - Oban.Pro.Testing.run_batch(batch, Keyword.merge(unquote(repo_opts), opts)) - end - - def run_chain([_ | _] = chain, opts \\ []) do - Oban.Pro.Testing.run_chain(chain, Keyword.merge(unquote(repo_opts), opts)) - end - - def run_chunk([_ | _] = chunk, opts \\ []) do - Oban.Pro.Testing.run_chunk(chunk, Keyword.merge(unquote(repo_opts), opts)) - end - - def run_jobs([_ | _] = changesets, opts \\ []) do - Oban.Pro.Testing.run_jobs(changesets, Keyword.merge(unquote(repo_opts), opts)) - end - - def run_workflow(%_{} = workflow, opts \\ []) do - Oban.Pro.Testing.run_workflow(workflow, Keyword.merge(unquote(repo_opts), opts)) - end - - def start_supervised_oban!(opts \\ []) do - unquote(repo_opts) - |> Keyword.merge(opts) - |> Oban.Pro.Testing.start_supervised_oban!() - end - end - end - - @conf_keys [] - |> Config.new() - |> Map.from_struct() - |> Map.keys() - - @callbacks ~w(attempted cancelled completed discarded exhausted retryable)a - - @default_supervised_opts [ - engine: Oban.Pro.Engines.Smart, - notifier: Oban.Notifiers.Isolated, - peer: Oban.Peers.Isolated, - stage_interval: :infinity, - shutdown_grace_period: 250 - ] - - @empty_summary %{ - cancelled: 0, - completed: 0, - discarded: 0, - exhausted: 0, - retryable: 0, - scheduled: 0 - } - - @doc """ - Construct a job from a worker, args, and options. - - This is a wrapper around `Oban.Testing.build_job/3` that ensures compatibility with Pro worker - features by using the Smart engine. - - The helper makes the following assertions: - - * That the worker implements the `Oban.Worker` behaviour - * That the options provided build a valid job - - This helper is used to build jobs for execution by `perform_job/2`. - - ## Options - - See [shared options](#module-shared-repo-options) for additional repo-specific options. - - ## Examples - - Build a job without args: - - job = build_job(MyWorker, %{}) - - Build a job with stringified args: - - assert %{args: %{"id" => 1}} = build_job(MyWorker, %{id: 1}) - - Build a job with custom options: - - assert %{attempt: 5, priority: 9} = build_job(MyWorker, %{}, attempt: 5, priority: 9) - """ - @doc since: "1.6.8" - @spec build_job(Worker.t(), term(), [Job.option() | repo_option()]) :: Job.t() - def build_job(worker, args, opts) when is_atom(worker) and is_list(opts) do - {_conf_opts, job_opts} = Keyword.split(opts, @conf_keys) - - Oban.Testing.build_job(worker, args, job_opts) - end - - @doc """ - Retrieve all currently enqueued jobs matching a set of criteria. - - This is a wrapper around `Oban.Testing.all_enqueued/1`, see `Oban.Testing` for more details. - - ## Options - - See [shared options](#module-shared-repo-options) for additional repo-specific options. - """ - @doc since: "0.11.0" - @spec all_enqueued(keyword()) :: [Job.t()] - def all_enqueued(opts) do - {repo, opts} = Keyword.pop!(opts, :repo) - - Oban.Testing.all_enqueued(repo, opts) - end - - @doc """ - Assert that one or more jobs were enqueued during a function call. - - Any pre-existing jobs are ignored for the assertion. If the assertion passes then the function's - return value is passed back. - - ## Options - - See `assert_enqueued/2` for standard options, and [shared options](#module-shared-repo-options) - for additional repo-specific options. - - ## Examples - - Assert that a `MyApp.Worker` job was added to the `default` queue: - - assert_enqueue([queue: :default, worker: MyApp.Worker], fn -> - MyApp.do_some_business() - end) - - Make an assertion about the return value: - - result = assert_enqueue([worker: MyApp.Worker], &MyApp.more_business/0) - - assert {:ok, _} = result - """ - @doc since: "1.1.0" - @spec assert_enqueue(keyword(), (-> return)) :: return when return: any() - def assert_enqueue(opts, fun) when is_function(fun, 0) do - opts = expand_decorated(opts) - - {enqueued, returned} = diff_enqueued(opts, fun) - - if Enum.any?(enqueued) do - returned - else - flunk(""" - Expected a job matching: - - #{inspect_opts(opts)} - - to be enqueued. - """) - end - end - - @doc """ - Assert that a job with particular criteria is enqueued. - - This is a wrapper around `Oban.Testing.assert_enqueued/2` with additions to support jobs built - with `Oban.Pro.Decorator` decorated functions. - - Only values for the provided fields are checked. For example, an assertion made on `worker: - "MyWorker"` will match _any_ jobs for that worker, regardless of every other field. - - ## Options - - The following options are supported in addition to all `t:Oban.Job` fields: - - * `:decorated` — a function capture for the decorated function, e.g. `&MyApp.Foo.bar/2`. By - default, only the module and function name are matched. To match on `args`, a _full list_ of - arguments must be provided rather than the standard map. - - See [shared options](#module-shared-repo-options) for additional repo-specific options. - - ## Examples - - Assert that a job is enqueued for a certain worker and args: - - assert_enqueued worker: MyWorker, args: %{id: 1} - - Assert that a job is enqueued for a particular queue and priority: - - assert_enqueued queue: :business, priority: 3 - - Assert that a job's args deeply match: - - assert_enqueued args: %{config: %{enabled: true}} - - Use the `:_` wildcard to assert that a job's meta has a key with any value: - - assert_enqueued meta: %{batch_id: :_} - - Assert on a decorated function with any args: - - assert_enqueued decorated: &MyApp.notify/2 - - Assert on exact args used to enqueue a decorated function: - - assert_enqueued args: ["example", 123], decorated: &MyApp.notify/2 - """ - @doc since: "0.11.0" - @spec assert_enqueued(keyword(), timeout() | :none) :: true - def assert_enqueued(opts, timeout \\ :none) do - opts = expand_decorated(opts) - - if timeout == :none do - Oban.Testing.assert_enqueued(opts) - else - Oban.Testing.assert_enqueued(opts, timeout) - end - end - - @doc """ - Refute that any jobs were enqueued during a function call. - - Any pre-existing jobs are ignored for the refutation. If the refutation passes then the - function's return value is passed back. - - ## Options - - See `refute_enqueued/2` for standard options, and [shared options](#module-shared-repo-options) - for additional repo-specific options. - - ## Examples - - Refute that a `MyApp.Worker` job was added to the `default` queue: - - refute_enqueue([queue: :default, worker: MyApp.Worker], fn -> - MyApp.do_some_business() - end) - - Make a refutation about the return value: - - result = refute_enqueue([worker: MyApp.Worker], &MyApp.more_business/0) - - assert {:ok, _} = result - """ - @doc since: "1.1.0" - @spec refute_enqueue(keyword(), (-> return)) :: return when return: any() - def refute_enqueue(opts, fun) when is_function(fun, 0) do - opts = expand_decorated(opts) - - {enqueued, returned} = diff_enqueued(opts, fun) - - if Enum.empty?(enqueued) do - returned - else - flunk(""" - Expected no jobs matching: - - #{inspect_opts(opts)} - - to be enqueued. - """) - end - end - - @doc """ - Refute that a job with particular criteria is enqueued. - - This is a wrapper around `Oban.Testing.refute_enqueued/2` with additions to support jobs built - with `Oban.Pro.Decorator` decorated functions. - - ## Options - - The following options are supported in addition to all `t:Oban.Job` fields: - - * `:decorated` — a function capture for the decorated function, e.g. `&MyApp.Foo.bar/2`. By - default, only the module and function name are matched. To match on `args`, a _full list_ of - arguments must be provided rather than the standard map. - - See [shared options](#module-shared-repo-options) for additional repo-specific options. - - ## Examples - - Refute that a job is enqueued for a certain worker: - - refute_enqueued worker: MyWorker - - Refute that a job is enqueued for a certain worker and args: - - refute_enqueued worker: MyWorker, args: %{id: 1} - - Refute that a job's nested args match: - - refute_enqueued args: %{config: %{enabled: false}} - - Use the `:_` wildcard to refute that a job's meta has a key with any value: - - refute_enqueued meta: %{batch_id: :_} - - Refute a decorated function was enqueued with any args: - - refute_enqueued decorated: &MyApp.notify/2 - - Refute that exact args were used to enqueue a decorated function: - - refute_enqueued args: ["example", 123], decorated: &MyApp.notify/2 - """ - @doc since: "0.11.0" - @spec refute_enqueued(keyword(), timeout() | :none) :: false - def refute_enqueued(opts, timeout \\ :none) do - opts = expand_decorated(opts) - - if timeout == :none do - Oban.Testing.refute_enqueued(opts) - else - Oban.Testing.refute_enqueued(opts, timeout) - end - end - - @doc """ - Synchronously execute jobs in one or all queues, from within the current process. - - Jobs that are enqueued by a process when `Ecto` is in sandbox mode are only visible to that - process. Calling `drain_jobs/1` allows you to control when the jobs are executed and to wait - synchronously for all jobs to complete. - - This function provides several distinct advantages over the standard `Oban.drain_queue/2`: - - * It can drain jobs across one or all queues simultaneously - * It can return the drained jobs rather than a count summary - * It optimizes the defaults for testing batches, workflows, etc. - * It always uses the `Smart` engine to guarantee that Pro worker features work as expected - - ## Options - - * `:queue` - an atom specifying the queue to drain, or `:all` to drain jobs across all queues at - once. Defaults to `:all` when no queue is provided. - - * `:with_limit` — the maximum number of jobs to fetch for draining at once. The limit only - impacts how many jobs are fetched at once, _not_ concurrency. When recursion is enabled this - is how many jobs are processed per-iteration, and it defaults to `1`. Otherwise, there isn't a - limit and all available jobs are fetched. - - * `:with_recursion` — whether to draining jobs recursively, or all in a single pass. Either way, - jobs are processed sequentially, one at a time. Recursion is required when jobs insert other - jobs (e.g. batches), or depend on the execution of other jobs (e.g. workflows). Defaults to - `true`. - - * `:with_safety` — whether to silently catch errors when draining. When `false`, raised - exceptions or unhandled exits are reraised (unhandled exits are wrapped in `Oban.CrashError`). - Defaults to `false`. - - * `:with_scheduled` — whether to include scheduled or retryable jobs when draining. In recursive - mode, which is the default, this will include snoozed jobs, and may lead to an infinite loop if - the job snoozes repeatedly. Defaults to `true`. - - * `:with_summary` — whether to summarize execution results with a map of counts by state, or - return a list of each job that was drained. Defaults to `true`, which returns a summary map. - - See [shared options](#module-shared-repo-options) for additional repo-specific options. - - ## Examples - - Drain all available jobs across all queues: - - assert %{completed: 3} = drain_jobs(queue: :all) - - Drain and return all executed jobs without a count summary: - - assert [%Oban.Job{}, %Oban.Job{}] = drain_jobs(with_summary: false) - - Drain including a job that you expect to raise: - - assert_raise RuntimeError, fn -> drain_jobs(queue: :risky) end - - Drain without recursion to identify snoozed jobs: - - assert %{scheduled: 3} = drain_jobs(with_recursion: false) - - Drain without staging scheduled jobs: - - assert %{completed: 1, scheduled: 0} = drain_jobs(with_scheduled: false) - - Drain within a custom prefix: - - assert %{completed: 3} = drain_jobs(queue: :default, prefix: "private") - - Drain using a specific repo (necessary when calling this function directly): - - assert %{completed: 3} = drain_jobs(queue: :default, repo: MyApp.Repo) - """ - @doc since: "0.11.0" - @spec drain_jobs([drain_option()]) :: drain_result() - def drain_jobs(opts) when is_list(opts) do - conf = init_conf(opts) - opts = Keyword.drop(opts, @conf_keys) - - with_limit = if Keyword.get(opts, :with_recursion, true), do: 1, else: 999_999 - - opts = - opts - |> Map.new() - |> Map.put_new(:queue, :all) - |> Map.update!(:queue, &if(&1 == :all, do: :__all__, else: &1)) - |> Map.put_new(:with_limit, with_limit) - |> Map.put_new(:with_recursion, true) - |> Map.put_new(:with_safety, false) - |> Map.put_new(:with_scheduled, true) - |> Map.put_new(:with_summary, true) - - Process.put(:oban_draining, true) - - drain(conf, [], opts) - after - Process.delete(:oban_draining) - end - - @doc """ - Execute a job using the given config options. - - This is a wrapper around `Oban.Testing.perform_job/2` that ensures the Smart engine is used for - compatibility with Pro worker features. - - See `perform_job/3` for more details and examples. - - ## Options - - See [shared options](#module-shared-repo-options) for additional repo-specific options. - - ## Examples - - Execute a job without any options: - - job = build_job(MyWorker, %{id: 1}) - assert :ok = perform_job(job) - - Execute a job with a custom prefix and repo: - - assert :ok = perform_job(job, prefix: "private", repo: MyApp.Repo) - """ - @doc since: "1.6.8" - @spec perform_job(Job.t(), [repo_option()]) :: Worker.result() - def perform_job(%Job{} = job, opts) when is_list(opts) do - opts = Keyword.put_new(opts, :engine, Smart) - - Oban.Testing.perform_job(job, opts) - end - - @doc """ - Construct a job and execute it with a worker module. - - This is a wrapper around `Oban.Testing.perform_job/3` that ensures the Smart engine is used for - compatibility with Pro worker features. - - ## Options - - See [shared options](#module-shared-repo-options) for additional repo-specific options. - - ## Examples - - Successfully execute a job with some string arguments: - - assert :ok = perform_job(MyWorker, %{"id" => 1}) - - Successfully execute a job and assert that it returns an error tuple: - - assert {:error, _} = perform_job(MyWorker, %{"bad" => "arg"}) - - Execute a job with the args keys automatically stringified: - - assert :ok = perform_job(MyWorker, %{id: 1}) - """ - @doc since: "0.11.0" - @spec perform_job(Worker.t(), term(), [perform_option()]) :: Worker.result() - def perform_job(worker, args, opts) do - opts = Keyword.put_new(opts, :engine, Smart) - - Oban.Testing.perform_job(worker, args, opts) - end - - @doc """ - Change the testing mode within the context of a function. - - This is a direct delegation to `Oban.Testing.with_testing_mode/2`. Only `:manual` and `:inline` - mode are supported, as `:disabled` implies that supervised queues and plugins are running and - this function won't start any processes. - - ## Examples - - Switch to `:manual` mode when an Oban instance is configured for `:inline` testing: - - with_testing_mode(:manual, fn -> - Oban.insert(MyWorker.new(%{id: 123})) - - assert_enqueued worker: MyWorker, args: %{id: 123} - end) - - Visa-versa, switch to `:inline` mode: - - with_testing_mode(:inline, fn -> - {:ok, %Job{state: "completed"}} = Oban.insert(MyWorker.new(%{id: 123})) - end) - """ - @doc since: "1.6.8" - @spec with_testing_mode(:inline | :manual, (-> any())) :: any() - defdelegate with_testing_mode(mode, fun), to: Oban.Testing - - @doc """ - Construct and execute a job with a batch `handle_*` callback. - - This helper verifies that the batch worker exports the requested callback handler, - along with the standard assertions made by `perform_job/3`. - - ## Examples - - Execute the `handle_attempted` callback without any args: - - assert :ok = perform_callback(MyBatch, :attempted, %{}) - - Execute the `handle_exhausted` callback with args: - - assert :ok = perform_callback(MyBatch, :exhausted, %{for_account_id: 123}) - """ - @doc since: "0.11.0" - @spec perform_callback(Worker.t(), callback(), term(), [perform_option()]) :: Worker.result() - def perform_callback(worker, callback, args, opts) when is_list(opts) do - assert_valid_callback(worker, callback) - - opts = - opts - |> Keyword.put_new(:meta, %{}) - |> Keyword.update!(:meta, &Map.put_new(&1, "callback", to_string(callback))) - |> Keyword.update!(:meta, &Map.put_new(&1, "batch_id", Oban.Pro.UUIDv7.generate())) - - perform_job(worker, args, opts) - end - - @doc """ - Construct a list of jobs and process them with a Chunk worker. - - Like `perform_job/3`, this helper reduces boilerplate when constructing jobs and checks for - common pitfalls. Unlike `perform_job/3`, this helper calls the chunk's `process/1` function - directly and it **won't trigger telemetry events**. - - ## Examples - - Successfully process a chunk of jobs: - - assert :ok = perform_chunk(MyChunk, [%{id: 1}, %{id: 2}]) - - Process a chunk of jobs with job options: - - assert :ok = perform_chunk(MyChunk, [%{id: 1}, %{id: 2}], attempt: 5, priority: 3) - """ - @doc since: "0.11.0" - @spec perform_chunk(Worker.t(), [term()], [perform_option()]) :: Chunk.result() - def perform_chunk(worker, args_list, opts) when is_list(args_list) and is_list(opts) do - {conf_opts, opts} = Keyword.split(opts, @conf_keys) - - conf = Config.new(conf_opts) - - opts = - opts - |> Keyword.merge(worker.__opts__()) - |> Keyword.put_new(:attempt, 1) - - chunk = - Enum.reduce_while(args_list, [], fn args, acc -> - now = DateTime.utc_now() - - job = - args - |> worker.new(opts) - |> Changeset.update_change(:args, &json_encode_decode/1) - |> Changeset.put_change(:attempted_at, now) - |> Changeset.put_change(:scheduled_at, now) - |> Changeset.apply_action!(:insert) - |> Map.replace!(:conf, conf) - - case Oban.Pro.Worker.before_process(worker, job, opts) do - {:ok, job} -> - {:cont, [job | acc]} - - {:error, error} -> - {:halt, {:error, error}} - end - end) - - if is_list(chunk) do - chunk - |> Enum.reverse() - |> worker.process() - |> tap(&assert_valid_chunk_result/1) - else - chunk - end - end - - @doc """ - Insert and execute a complete batch of jobs, along with callbacks, within the test process. - - ## Options - - Accepts all options for `drain_jobs/1`, including the repo-specific options listed in - [shared-options](#module-shared-repo-options). - - ## Examples - - Run a batch: - - ids - |> Enum.map(&MyBatch.new(%{id: &1})) - |> MyBatch.new_batch() - |> run_batch() - - Run a batch with a specific repo (necessary when calling this function directly): - - run_batch(my_batch, repo: MyApp.Repo, prefix: "private") - """ - @doc since: "0.11.0" - @spec run_batch(Batch.t(), [drain_option()]) :: drain_result() - def run_batch(batch, opts) when is_list(opts) do - opts = Keyword.put(opts, :batch_ids, [batch.opts.batch_id]) - - batch.changesets - |> Enum.to_list() - |> run_jobs(opts) - end - - @doc """ - Insert and execute chained jobs within the test process. - - ## Options - - Accepts all options for `drain_jobs/1`, including the repo-specific options listed in - [shared-options](#module-shared-repo-options). - - ## Examples - - Run all jobs in a chain: - - 1..10 - |> Enum.map(&MyChain.new(%{id: &1})) - |> run_chain() - """ - @doc since: "1.1.0" - @doc deprecated: "Use run_jobs/2 instead" - def run_chain(chain, opts) when is_list(opts) do - chain_keys = ~w(hold_snooze on_cancelled on_discarded wait_retry wait_sleep wait_snooze)a - chain_opts = opts |> Keyword.take(chain_keys) |> Map.new() - - chain - |> Enum.map(fn changeset -> - Changeset.update_change(changeset, :meta, &Map.merge(&1, chain_opts)) - end) - |> run_jobs(opts) - end - - @doc """ - Insert and execute chunked jobs within the test process. - - This helper overrides the chunk's `timeout` to force immediate processing of jobs up to the - chunk size. - - ## Options - - Accepts all options for `drain_jobs/1`, including the repo-specific options listed in - [shared-options](#module-shared-repo-options). - - ## Examples - - Run jobs in chunks: - - 1..50 - |> Enum.map(&MyChunk.new(%{id: &1})) - |> run_chunk() - - Run a chunk with an overriden size: - - 1..10 - |> Enum.map(&MyChunk.new(%{id: &1})) - |> run_chunk(size: 10) - - Run chunks with an explicit repo (necessary when calling this function directly): - - run_chunk(changesets, repo: MyApp.Repo, prefix: "private") - - Run chunks only for a specific queue: - - run_chunk(changesets, queue: "default") - """ - @doc since: "0.11.0" - @spec run_chunk([Job.changeset()], [drain_option()]) :: drain_result() - def run_chunk([_ | _] = chunk, opts) when is_list(opts) do - chunk_opts = - opts - |> Keyword.take(~w(by leading size)a) - |> Map.new(fn - {:by, by} -> {:chunk_by, Utils.normalize_by(by)} - {:leading, leading} -> {:chunk_leading, leading} - {:size, size} -> {:chunk_size, size} - end) - - merge = fn meta -> Map.merge(meta, chunk_opts) end - - chunk - |> Enum.map(fn change -> Changeset.update_change(change, :meta, merge) end) - |> run_jobs(Keyword.put_new(opts, :with_limit, 1)) - end - - @doc """ - Insert and execute jobs synchronously, within the test process. - - This is the basis of all other `run_*` helpers. - - ## Options - - Accepts all options for `drain_jobs/1`, including the repo-specific options listed in - [shared-options](#module-shared-repo-options). - - ## Examples - - Run a list of jobs: - - ids - |> Enum.map(&MyWorker.new(%{id: &1})) - |> run_jobs() - - Run jobs with an explicit repo (necessary when calling this function directly): - - run_jobs(changesets, repo: MyApp.Repo) - """ - @doc since: "0.11.0" - @spec run_jobs([Job.changeset()], [drain_option()]) :: drain_result() - def run_jobs([_ | _] = changesets, opts) when is_list(changesets) and is_list(opts) do - opts - |> init_conf() - |> Smart.insert_all_jobs(changesets, []) - - opts - |> Keyword.put_new(:queue, :all) - |> drain_jobs() - end - - @doc """ - Insert and execute a workflow synchronously, within the test process. - - This helper augments the workflow with options optimized for testing, but it will still respect - all standard workflow options. - - ## Options - - Accepts all options for `drain_jobs/1`, including the repo-specific options listed in - [shared-options](#module-shared-repo-options). - - ## Examples - - Run a basic workflow: - - MyFlow.new_workflow() - |> MyFlow.add(:a, MyFlow.new(%{id: 1})) - |> MyFlow.add(:b, MyFlow.new(%{id: 2}), deps: [:a]) - |> MyFlow.add(:c, MyFlow.new(%{id: 3}), deps: [:b]) - |> run_workflow() - - Run a workflow and match on returned jobs, but be careful that the execution order may differ - from the insertion order: - - workflow = - MyFlow.new_workflow() - |> MyFlow.add(:a, MyFlow.new(%{id: 1})) - |> MyFlow.add(:b, MyFlow.new(%{id: 2}), deps: [:a]) - |> MyFlow.add(:c, MyFlow.new(%{id: 3}), deps: [:b]) - - [_job_a, _job_b, _job_c] = run_workflow(workflow, with_summary: false) - - Run a workflow with increased parallelism to prevent slow or deadlocked workflows: - - run_workflow(my_expansive_workflow, with_limit: 5) - - Run a workflow with an explicit repo (necessary when calling this function directly): - - run_workflow(workflow, repo: MyApp.Repo) - """ - @doc since: "0.11.0" - @spec run_workflow(Workflow.t(), Workflow.new_opts() | drain_option()) :: drain_result() - def run_workflow(%Workflow{} = workflow, opts \\ []) when is_list(opts) do - conf = init_conf(opts) - - Smart.insert_all_jobs(conf, workflow.changesets, []) - - workflow_ids = - workflow.changesets - |> Enum.map(& &1.changes.meta.workflow_id) - |> Enum.uniq() - - opts - |> Keyword.put(:workflow_ids, workflow_ids) - |> drain_jobs() - - # The workflow facilitator won't run "held" jobs, so we have to reload everything that was - # inserted to get all jobs and their real fields. Additionally, the workflow ids may be added - # while the workflow is executing so they must be refreshed afterwards. - workflow_ids = all_workflow_ids(conf, workflow_ids) - - jobs = - Job - |> where([j], j.meta["workflow_id"] in ^workflow_ids) - |> then(&Repo.all(conf, &1)) - - if Keyword.get(opts, :with_summary, true) do - Enum.reduce(jobs, @empty_summary, fn job, acc -> - state = - if job.state == "discarded" and job.attempt >= job.max_attempts do - :exhausted - else - String.to_existing_atom(job.state) - end - - Map.update(acc, state, 1, &(&1 + 1)) - end) - else - jobs - end - end - - @doc """ - Start a supervised Oban instance under the test supervisor. - - All valid Oban options are accepted. The supervised instance is registered with a unique - reference, rather than the default `Oban`. That prevents any conflict with Oban instances - started by your Application, or with other tests running asynchronously. - - Furthermore, this helper automatically adds sandbox allowances for any plugins or queue - producers, allowing tests to run async. - - After the test finishes the test process will wait for the Oban instance to shut down cleanly. - - ## Running Jobs - - By default, the supervised instance won't process any jobs because the `stage_interval` is set - to `:infinity`. Set the `stage_interval` to a low value to process jobs normally, without manual - draining. - - ## Options - - Any option accepted by `Oban.start_link/1` is acceptable, including the repo-specific options - listed in [shared options](#module-shared-repo-options). - - ## Examples - - Start a basic supervised instance without any queues or plugins: - - name = start_supervised_oban!(repo: MyApp.Repo) - Oban.insert(name, MyWorker.new()) - - Start the supervisor with a single queue and polling every 10ms: - - start_supervised_oban!(repo: MyApp.Repo, stage_interval: 10, queues: [alpha: 10]) - """ - @doc since: "0.11.0" - @spec start_supervised_oban!([Oban.option()]) :: Registry.key() - def start_supervised_oban!(opts) when is_list(opts) do - opts = conf_opts(opts) - name = Keyword.fetch!(opts, :name) - repo = Keyword.fetch!(opts, :repo) - - attach_auto_allow(repo, name) - - start_supervised!({Oban, opts}) - - name - |> Oban.config() - |> ensure_queues_started() - - name - end - - defp json_encode_decode(map) do - map - |> Oban.JSON.encode!() - |> Oban.JSON.decode!() - end - - # Enqueued Helpers - - defp diff_enqueued(opts, fun) do - existing = all_enqueued(opts) - returned = fun.() - inserted = all_enqueued(opts) - - existing_ids = Enum.map(existing, & &1.id) - difference = Enum.reject(inserted, &(&1.id in existing_ids)) - - {difference, returned} - end - - defp inspect_opts(opts) do - opts - |> Keyword.drop([:log, :prefix, :repo]) - |> inspect(charlists: :as_lists, pretty: true) - end - - defp expand_decorated(opts) do - case Keyword.fetch(opts, :decorated) do - {:ok, fun} when is_function(fun) -> - info = Function.info(fun) - args = %{mod: inspect(info[:module]), fun: info[:name], arg: args_to_term(info, opts)} - - opts - |> Keyword.delete(:decorated) - |> Keyword.merge(args: args, worker: Oban.Pro.Decorator) - - {:ok, term} -> - raise ArgumentError, - "expected :decorated option to be a function capture, got: #{inspect(term)}" - - :error -> - opts - end - end - - defp args_to_term(info, opts) do - arity = info[:arity] - - case Keyword.fetch(opts, :args) do - {:ok, list} when is_list(list) and length(list) == arity -> - "term-" <> Utils.encode64(list) - - {:ok, list} when is_list(list) -> - raise ArgumentError, "expected :args list to have arity #{arity}, got: #{length(list)}" - - {:ok, term} -> - raise ArgumentError, "expected :args option to be a list, got: #{inspect(term)}" - - :error -> - :_ - end - end - - # Callback Helpers - - defp assert_valid_callback(worker, callback) do - assert is_atom(callback) and callback in @callbacks, """ - Expected callback to be included in #{inspect(@callbacks)}, got: #{inspect(callback)} - """ - - assert Code.ensure_loaded?(worker), """ - Expected worker to be an existing module, got: #{inspect(worker)} - """ - - new_fun = Map.fetch!(Batch.callbacks_to_functions(), to_string(callback)) - old_fun = Map.fetch!(Batch.callbacks_to_deprecated(), to_string(callback)) - - assert function_exported?(worker, new_fun, 1) or function_exported?(worker, old_fun, 1), """ - Expected #{inspect(new_fun)} callback to be implemented - """ - end - - # Startup Helpers - - defp attach_auto_allow(repo, name) do - telemetry_name = "oban-auto-allow-#{inspect(name)}" - - :telemetry.attach_many( - telemetry_name, - [[:oban, :engine, :init, :start], [:oban, :plugin, :init]], - &__MODULE__.auto_allow/4, - {name, repo, self()} - ) - - on_exit(name, fn -> :telemetry.detach(telemetry_name) end) - end - - @doc false - def auto_allow(_event, _measure, %{conf: conf}, {name, repo, test_pid}) do - if conf.name == name, do: Sandbox.allow(repo, test_pid, self()) - end - - defp ensure_queues_started(%{name: name, queues: queues}) do - with [_ | _] <- queues do - queues - |> Keyword.keys() - |> Enum.each(&Oban.check_queue(name, queue: &1)) - end - end - - # Draining Helpers - - defp conf_opts(opts) do - conf_opts = Keyword.take(opts, @conf_keys) - - @default_supervised_opts - |> Keyword.merge(conf_opts) - |> Keyword.put_new(:name, make_ref()) - end - - defp init_conf(opts) do - opts - |> conf_opts() - |> Config.new() - |> tap(&Registry.register(Oban.Registry, &1.name, &1)) - end - - defp drain(conf, acc, opts) do - if opts.with_scheduled, do: stage_scheduled(conf) - - case Smart.fetch_for_drain(conf, opts) do - [_ | _] = jobs -> - executed = - jobs - |> streamline_pro_workers() - |> Enum.map(fn job -> - conf - |> Executor.new(job, safe: opts.with_safety) - |> Executor.call() - end) - - if opts.with_recursion do - drain(conf, acc ++ executed, with_workflow_ids(conf, opts)) - else - complete_drain(conf, executed, opts) - end - - [] -> - case opts do - %{workflow_ids: [_ | _], flushed: true} -> - complete_drain(conf, acc, opts) - - %{workflow_ids: [_ | _] = workflow_ids} -> - # Final flush to release held jobs whose deps became satisfied during processing. - Enum.each(workflow_ids, &Workflow.on_flush(&1, conf)) - - drain(conf, acc, Map.put(opts, :flushed, true)) - - _ -> - complete_drain(conf, acc, opts) - end - end - end - - defp stage_scheduled(conf) do - query = - Job - |> where([j], j.state in ["scheduled", "retryable"]) - |> where([j], not fragment("? @> ?", j.meta, ^%{on_hold: true})) - - Repo.update_all(conf, query, set: [state: "available"]) - end - - defp streamline_pro_workers(jobs) do - Enum.map(jobs, fn %{meta: meta} = job -> - case meta do - %{"chain_key" => _} -> - meta = - meta - |> Map.put_new("wait_retry", 1) - |> Map.put_new("wait_sleep", 1) - |> Map.put_new("wait_snooze", 1) - - %{job | meta: meta} - - %{"chunk_size" => _} -> - meta = - meta - |> Map.put("chunk_sleep", 1) - |> Map.put("chunk_timeout", 0) - - %{job | meta: meta} - - _ -> - job - end - end) - end - - defp with_workflow_ids(conf, opts) do - case opts do - %{workflow_ids: [_ | _]} -> - Map.put(opts, :workflow_ids, all_workflow_ids(conf, opts.workflow_ids)) - - _ -> - opts - end - end - - defp all_workflow_ids(conf, [_ | _] = workflow_ids) do - all_workflow_ids(conf, workflow_ids, MapSet.new(workflow_ids)) - end - - defp all_workflow_ids(_conf, ids), do: ids - - defp all_workflow_ids(conf, search_ids, seen) do - graft_query = - Job - |> select([j], j.meta["graft_id"]) - |> where([j], j.meta["workflow_id"] in ^search_ids) - |> where([j], not is_nil(j.meta["graft_id"])) - |> distinct(true) - - graft_ids = Repo.all(conf, graft_query) - all_search_ids = search_ids ++ graft_ids - - query = - Job - |> select([j], j.meta["workflow_id"]) - |> where([j], j.meta["sup_workflow_id"] in ^all_search_ids) - |> distinct(true) - - new_ids = - conf - |> Repo.all(query) - |> Enum.reject(&MapSet.member?(seen, &1)) - - case new_ids do - [] -> MapSet.to_list(seen) - ids -> all_workflow_ids(conf, ids, MapSet.union(seen, MapSet.new(ids))) - end - end - - defp complete_drain(_conf, executed, %{with_summary: true}) do - Enum.reduce(executed, @empty_summary, fn exec, acc -> - state = - case exec.state do - :cancelled -> :cancelled - :discard -> :discarded - :exhausted -> :exhausted - :failure -> :retryable - :snoozed -> :scheduled - :success -> :completed - end - - Map.update(acc, state, 1, &(&1 + 1)) - end) - end - - defp complete_drain(conf, executed, _opts) do - ids = - executed - |> Enum.map(& &1.job.id) - |> Enum.uniq() - - jobs = - conf - |> Repo.all(where(Job, [j], j.id in ^ids)) - |> Map.new(fn job -> {job.id, job} end) - - Enum.map(ids, &Map.fetch!(jobs, &1)) - end - - # Chunk Helpers - - @ops ~w(cancel discard error)a - - defp assert_valid_chunk_result(result) do - valid? = - case result do - :ok -> - true - - {:ok, _value} -> - true - - {ops, _reason, [_ | _]} when ops in @ops -> - true - - [{_op, {_reason, _jobs}} | _] = list -> - Keyword.keyword?(list) and - Enum.all?(list, &match?({_key, {_reason, [_ | _]}}, &1)) - - _ -> - false - end - - assert valid?, """ - Expected result to be one of - - - `:ok` - - `{:ok, value}` - - `{:cancel, reason, jobs}` - - `{:discard, reason, jobs}` - - `{:error, reason, jobs}` - - `[cancel: {reason, jobs}, discard: {reason, jobs}, error: {reason, jobs}]` - - Instead received: - - #{inspect(result, pretty: true)} - """ - end -end diff --git a/vendor/oban_pro/lib/oban/pro/unique.ex b/vendor/oban_pro/lib/oban/pro/unique.ex deleted file mode 100644 index 6c32453d..00000000 --- a/vendor/oban_pro/lib/oban/pro/unique.ex +++ /dev/null @@ -1,88 +0,0 @@ -defmodule Oban.Pro.Unique do - @moduledoc false - - alias Ecto.Changeset - alias Oban.Job - alias Oban.Pro.Utils - - @states_to_ints %{ - "scheduled" => 0, - "available" => 1, - "executing" => 2, - "retryable" => 3, - "completed" => 4, - "cancelled" => 5, - "discarded" => 6, - "suspended" => 7 - } - - @spec replace?(Job.changeset()) :: boolean() - def replace?(changeset), do: match?(%{changes: %{replace: _}}, changeset) - - @spec unique?(Job.changeset()) :: boolean() - def unique?(changeset), do: match?(%{changes: %{unique: %{}}}, changeset) - - @spec get_key(Job.changeset(), any()) :: any() - def get_key(changeset, default \\ nil) - - def get_key(%{changes: %{meta: %{uniq_key: uniq_key}}}, _default), do: uniq_key - def get_key(_changeset, default), do: default - - @spec get_states(Job.changeset()) :: [atom()] - def get_states(%{changes: %{unique: %{states: states}}}), do: states - - # For backward compatibility, `unique` works with non-pro workers, so it can't be injected into - # the meta through a stage. - @spec with_uniq_meta(Job.changeset()) :: Job.changeset() - def with_uniq_meta(%{changes: %{unique: %{period: _}}} = changeset) do - uniq_meta = %{uniq: true, uniq_bmp: gen_bmp(changeset), uniq_key: gen_key(changeset)} - - meta = - changeset - |> Changeset.get_change(:meta, %{}) - |> Map.merge(uniq_meta) - - Changeset.put_change(changeset, :meta, meta) - end - - def with_uniq_meta(changeset), do: changeset - - @spec gen_bmp(Job.changeset()) :: [integer()] - def gen_bmp(%{changes: %{unique: %{states: states}}}) do - for state <- states, do: Map.fetch!(@states_to_ints, to_string(state)) - end - - @spec gen_key(Job.changeset()) :: String.t() - def gen_key(%{changes: %{unique: unique}} = changeset) do - %{fields: fields, keys: keys, period: period, timestamp: timestamp} = unique - - data = - fields - |> Enum.sort() - |> Enum.map(fn - :args -> Utils.take_keys(changeset, :args, keys) - :meta -> Utils.take_keys(changeset, :meta, keys) - field -> Changeset.get_field(changeset, field) - end) - - data = - if period == :infinity do - data - else - [truncate(period, Changeset.get_field(changeset, timestamp)) | data] - end - - Utils.hash64(data) - end - - defp truncate(period, timestamp) do - now = timestamp || DateTime.utc_now() - - now - |> DateTime.to_unix(:second) - |> rem(period) - |> then(&DateTime.add(now, -&1)) - |> DateTime.truncate(:second) - |> DateTime.to_iso8601() - end -end diff --git a/vendor/oban_pro/lib/oban/pro/utils.ex b/vendor/oban_pro/lib/oban/pro/utils.ex deleted file mode 100644 index 92e28feb..00000000 --- a/vendor/oban_pro/lib/oban/pro/utils.ex +++ /dev/null @@ -1,159 +0,0 @@ -defmodule Oban.Pro.Utils do - @moduledoc false - - alias Ecto.Changeset - - @spec encode64(term(), [:compressed | :deterministic]) :: String.t() - def encode64(term, opts \\ [:compressed]) do - term - |> :erlang.term_to_binary(opts) - |> Base.encode64(padding: false) - end - - @spec decode64(binary(), [:safe | :used]) :: term() - def decode64(bin, opts \\ [:safe]) do - bin - |> Base.decode64!(padding: false) - |> :erlang.binary_to_term(opts) - end - - @spec hash64(iodata()) :: String.t() - def hash64(iodata) when is_binary(iodata) or is_list(iodata) do - :blake2s - |> :crypto.hash(iodata) - |> Base.encode64(padding: false) - end - - @spec persistent_cache(tuple(), fun()) :: any() - def persistent_cache(key, fun) when is_function(fun, 0) do - case :persistent_term.get(key, nil) do - nil -> tap(fun.(), &:persistent_term.put(key, &1)) - val -> val - end - end - - @spec validate_opts(list(), fun()) :: :ok | {:error, term()} - def validate_opts(opts, validator) do - Enum.reduce_while(opts, :ok, fn opt, acc -> - case validator.(opt) do - {:error, _reason} = error -> {:halt, error} - _ -> {:cont, acc} - end - end) - end - - @spec cast_period(pos_integer() | {atom(), pos_integer()}) :: pos_integer() - def cast_period({value, unit}) do - unit = to_string(unit) - - cond do - unit in ~w(second seconds) -> value - unit in ~w(minute minutes) -> value * 60 - unit in ~w(hour hours) -> value * 60 * 60 - unit in ~w(day days) -> value * 24 * 60 * 60 - true -> unit - end - end - - def cast_period(period), do: period - - @spec enforce_keys(Changeset.t(), map(), module()) :: Changeset.t() - def enforce_keys(changeset, params, schema) do - fields = - [:fields, :virtual_fields] - |> Enum.flat_map(&schema.__schema__/1) - |> Enum.map(&to_string/1) - - Enum.reduce(params, changeset, fn {key, _val}, acc -> - if to_string(key) in fields do - acc - else - Changeset.add_error(acc, :base, "unknown field #{key} provided") - end - end) - end - - @spec take_keys(Changeset.t(), atom(), list()) :: [any()] - def take_keys(changeset, field, keys) do - changeset - |> Changeset.get_field(field) - |> take_keys(keys) - end - - @spec take_keys(map(), [atom()]) :: [any()] - def take_keys(map, _keys) when map_size(map) == 0, do: [] - - def take_keys(map, []) when is_map(map), do: take_keys(map, Map.keys(map)) - - def take_keys(map, keys) when is_map(map) and is_list(keys) do - map = Map.new(map, fn {key, val} -> {to_string(key), val} end) - - keys - |> Enum.map(&to_string/1) - |> Enum.sort() - |> Enum.map(fn key -> - val = map |> Map.get(key) |> to_hash_val() - - [key, val] - end) - end - - if Application.compile_env(:oban_pro, [__MODULE__, :safe_hash], false) do - defp to_hash_val(val) when is_binary(val), do: val - defp to_hash_val(val) when is_number(val), do: to_string(val) - defp to_hash_val(val) when is_atom(val), do: to_string(val) - defp to_hash_val(val), do: inspect(val) - else - defp to_hash_val(val), do: val |> :erlang.phash2() |> Integer.to_string() - end - - def normalize_by(by) do - by - |> List.wrap() - |> Enum.map(fn - {key, val} -> [key, val |> List.wrap() |> Enum.sort()] - field -> field - end) - end - - @spec to_exception(Changeset.t()) :: Exception.t() - def to_exception(changeset) do - changeset - |> to_translated_errors() - |> Enum.reverse() - |> Enum.map(fn {field, message} -> ArgumentError.exception("#{field} #{message}") end) - |> List.first() - end - - @spec to_translated_errors(Changeset.t()) :: Keyword.t() - def to_translated_errors(changeset) do - changeset - |> Changeset.traverse_errors(&translate_errors/1) - |> Enum.map(&extract_errors/1) - |> List.flatten() - end - - ## Conversions - - @spec maybe_to_atom(atom() | String.t()) :: atom() - def maybe_to_atom(key) when is_binary(key), do: String.to_existing_atom(key) - def maybe_to_atom(key), do: key - - # Helpers - - defp translate_errors({msg, opt}) do - Regex.replace(~r"%{(\w+)}", msg, fn _, key -> - opt - |> Keyword.get(String.to_existing_atom(key), key) - |> to_string() - end) - end - - defp extract_errors({_key, val}) when is_map(val) do - val - |> Map.to_list() - |> Enum.map(&extract_errors/1) - end - - defp extract_errors({key, [message | _]}), do: {key, message} -end diff --git a/vendor/oban_pro/lib/oban/pro/uuidv7.ex b/vendor/oban_pro/lib/oban/pro/uuidv7.ex deleted file mode 100644 index 0b406266..00000000 --- a/vendor/oban_pro/lib/oban/pro/uuidv7.ex +++ /dev/null @@ -1,42 +0,0 @@ -defmodule Oban.Pro.UUIDv7 do - @moduledoc false - - # Derived from [bitwalker/uniq](https://github.com/bitwalker/uniq) - - defmacrop biguint(n), do: quote(do: big - unsigned - integer - size(unquote(n))) - - @spec generate() :: <<_::288>> - def generate do - time = System.system_time(:millisecond) - - <> = :crypto.strong_rand_bytes(10) - - <> = - <>, rand_b::62>> - - <> - end - - @compile {:inline, e: 1} - - defp e(0), do: ?0 - defp e(1), do: ?1 - defp e(2), do: ?2 - defp e(3), do: ?3 - defp e(4), do: ?4 - defp e(5), do: ?5 - defp e(6), do: ?6 - defp e(7), do: ?7 - defp e(8), do: ?8 - defp e(9), do: ?9 - defp e(10), do: ?a - defp e(11), do: ?b - defp e(12), do: ?c - defp e(13), do: ?d - defp e(14), do: ?e - defp e(15), do: ?f -end diff --git a/vendor/oban_pro/lib/oban/pro/validation.ex b/vendor/oban_pro/lib/oban/pro/validation.ex deleted file mode 100644 index 3b84119d..00000000 --- a/vendor/oban_pro/lib/oban/pro/validation.ex +++ /dev/null @@ -1,22 +0,0 @@ -defmodule Oban.Pro.Validation do - @moduledoc false - - # Shared validations for use with `Oban.Validation.validate/2,3` - - @doc """ - Validate `by:` options for partitioning workers. - """ - def validate_by(:worker), do: :ok - def validate_by([{:args, key}]) when is_atom(key), do: :ok - def validate_by([{:meta, key}]) when is_atom(key), do: :ok - def validate_by([{:args, [key | _]}]) when is_atom(key), do: :ok - def validate_by([{:meta, [key | _]}]) when is_atom(key), do: :ok - def validate_by([:worker, {:args, key}]) when is_atom(key), do: :ok - def validate_by([:worker, {:meta, key}]) when is_atom(key), do: :ok - def validate_by([:worker, {:args, [key | _]}]) when is_atom(key), do: :ok - def validate_by([:worker, {:meta, [key | _]}]) when is_atom(key), do: :ok - - def validate_by(fields) do - {:error, "expected :by to be :worker or an :args/:meta tuple, got: #{inspect(fields)}"} - end -end diff --git a/vendor/oban_pro/lib/oban/pro/worker.ex b/vendor/oban_pro/lib/oban/pro/worker.ex deleted file mode 100644 index 2ae9c4c6..00000000 --- a/vendor/oban_pro/lib/oban/pro/worker.ex +++ /dev/null @@ -1,1254 +0,0 @@ -defmodule Oban.Pro.Worker do - @moduledoc """ - `Oban.Pro.Worker` is a replacement for `Oban.Worker` with expanded capabilities. - - Major features: - - * [🏗️ Structured Jobs](#module-structured-jobs) — types for job args with validation, casting, - defaults, enums and more. - * [📼 Recorded Jobs](#module-recorded-jobs) — store a job's return output for retrieval later - by other jobs, in a console, or in the Web dashboard. - * [🔗 Chained Jobs](#module-chained-jobs) — link jobs together to ensure they run in a strict - sequential order regardless of scheduling or retries. - * [🔐 Encrypted Jobs](#module-encrypted-jobs) — store job args with encryption at rest so sensitive - data can't be seen from the database or Web dashboard. - * [🪦 Job Deadlines](#module-job-deadlines) — preemptively cancel jobs that shouldn't run after a - period of time has elapsed. - * [🧑‍🤝‍🧑 Worker Aliases](#module-worker-aliases) — aliasing allows jobs enqueued with the original - worker name to continue without exceptions using the new worker code. - * [🪝 Worker Hooks](#module-worker-hooks) — callbacks triggered around job execution, defined as - callback functions on the worker, or in a separate module for reuse. - - ## Usage - - Using `Oban.Pro.Worker` is identical to using `Oban.Worker`, with a few additional options. All - of the basic options such as `queue`, `priority`, and `unique` are still available along with - more advanced options. - - To create a basic Pro worker point `use` at `Oban.Pro.Worker` and define a `process/1` callback: - - ```elixir - def MyApp.Worker do - use Oban.Pro.Worker - - @impl Oban.Pro.Worker - def process(%Job{} = job) do - # Do stuff with the job - end - end - ``` - - If you have existing workers that you'd like to convert you only need to change the `use` - definition and replace `perform/1` with `process/1`. - - Without any of the advanced Pro features there isn't any difference between the basic and pro - workers. - - ## Structured Jobs - - Structured workers help you catch invalid data within your jobs by validating args on insert and - casting args before execution. They also automatically generate structs for compile-time checks - and friendly dot access. - - #### Defining a Structured Worker - - Structured workers use `args_schema/1` to define which fields are allowed, required, and their - expected types. Another benefit, aside from validation, is that args passed to `process/1` are - converted into a struct named after the worker module. Here's an example that demonstrates - defining a worker with several field types and embedded data structures: - - ```elixir - defmodule MyApp.StructuredWorker do - use Oban.Pro.Worker - - args_schema do - field :id, :id, required: true - field :name, :string, required: true - field :mode, :enum, values: ~w(enabled disabled paused)a, default: :enabled - field :xtra, :term - - embeds_one :data, required: true do - field :office_id, :uuid, required: true - field :has_notes, :boolean, default: false - field :addons, {:array, :string} - end - - embeds_many :addresses do - field :street, :string - field :city, :string - field :country, :string - end - end - - @impl Oban.Pro.Worker - def process(%Job{args: %__MODULE__{id: id, name: name, mode: mode, data: data}}) do - %{office_id: office_id, notes: notes} = data - - # Use the matched, cast values - end - end - ``` - - The example's schema declares five top level keys, `:id`, `:name`, `:mode`, `:data`, and - `:addresses`. Of those, only `:id`, `:name`, and the `:office_id` subkey are marked required. - The `:mode` field is an enum that validates values and casts to an atom. The embedded `:data` - field declares a nested map with its own type coercion and validation, including a custom - `Ecto.UUID` type. Finally, `:addresses` specifies an embedded list of maps. - - Job args are validated on `new/1` and errors bubble up to prevent insertion: - - ```elixir - StructuredWorker.new(%{id: "not-an-id", mode: "unknown"}).valid? - # => false (invalid id, invalid mode, missing name) - - StructuredWorker.new(%{id: "123", mode: "enabled"}).valid? - # => false (missing name) - - StructuredWorker.new(%{id: "123", name: "NewBiz", mode: "enabled"}).valid? - # => true - ``` - - This shows how args, stored as JSON, are cast before passing to `process/1`: - - ```elixir - # {"id":123,"name":"NewBiz","mode":"enabled","data":{"parent_id":456}} - - %MyApp.StructuredWorker{ - id: 123, - name: "NewBiz", - mode: :enabled, - data: %{parent_id:456} - } - ``` - - #### Structured Types and Casting - - Type casting and validation are handled by changesets. All types supported in Ecto schemas are - allowed, e.g. `:id`, `:integer`, `:string`, `:float`, or `:map`. See the [Ecto documentation for - a complete list of Ecto types][ecto] and their Elixir counterparts. - - [ecto]: https://hexdocs.pm/ecto/3.9.4/Ecto.Schema.html#module-types-and-casting - - #### Structured Options - - Structured fields support a few common options. - - * `:default` — Sets the default value for a field. The default value is calculated at - compilation time, so don't use expressions like `DateTime.utc_now/0` as they'd be the same for - all records. - - * `:required` — Validates that a value is provided for the field. Values that are `nil` or an - empty string aren't considered valid. - - #### Structured Extensions - - Structured workers support some convenient extensions beyond Ecto's standard type casting. - - * `:enum` — Maps atoms to strings based on a list of predefiend atoms passed like `values: - ~w(foo bar baz)a`. Both validates that values are included in the list and casts them to an - atom. - - * `:term` — Safely encodes any Elixir term as a string for storage, then decodes it back to the - original term on load. This is similar to `:any`, but works with terms like tuples or pids - that can't usually be serialied. For safety, terms are encoded with the `:safe` option to - prevent decoding data that may be used to attack the runtime. - - * `:uuid` — an intention revealing alias for `binary_id` - - * `{:array, *}` — A list of one or more values of any type, including `:enum` and `:uuid` - - * `embeds_one/2,3` — Declares a nested map with an explicit set of fields - - * `embeds_many/2,3` — Delcares a list of nested maps with an explicit set of fields - - #### Defining Typespecs for Structured Workers - - Typespecs aren't generated automatically. If desired, you must to define a sctuctured worker's - type manually: - - ```elixir - defmodule MyApp.StructuredWorker do - use Oban.Pro.Worker - - @type t :: %__MODULE__{id: integer(), active: boolean()} - ``` - - ## Recorded Jobs - - Sometimes the output of a job is just as important as any side effects. When that's the case, - you can use the `recorded` option to stash a job's output back into the job itself. Results are - compressed and safely encoded for retrieval later, either manually, in a batch callback, or a in - downstream workflow job. - - #### Defining a Recorded Worker - - ```elixir - defmodule MyApp.RecordedWorker do - use Oban.Pro.Worker, recorded: true - - @impl true - def process(%Job{args: args}) do - # Do your typical work here. - end - end - ``` - - If your process function returns an `{:ok, value}` tuple, it is recorded. Any other value, i.e. - an plain `:ok`, error, or snooze, is ignored. - - The example above uses `recorded: true` to opt into recording with the defaults. That means an - output `limit` of 64mb after compression and encoding—anything larger than the configured limit - will return an error tuple. If you expect larger results (and you want them stored in the - database) you can override the limit. For example, to set the limit to 64mb instead: - - ```elixir - use Oban.Pro.Worker, recorded: [limit: 128_000_000] - ``` - - ### Retrieving Results - - The `fetch_recorded/1` function is your ticket to extracting recorded results. If a job has ran - and recorded a value, it will return an `{:ok, result}` tuple: - - ```elixir - job = MyApp.Repo.get(Oban.Job, job_id) - - case MyApp.RecordedWorker.fetch_recorded(job) do - {:ok, result} -> - # Use the result - - {:error, :missing} -> - # Nothing recorded yet - end - ``` - - ## Chained Jobs - - Chains link jobs together to ensure they run in a strict sequential order. Downstream jobs in the - chain won't execute until the upstream job is `completed`, `cancelled`, or `discarded`. Behaviour - in the event of cancellation or discards is customizable to allow for uninterrupted processing or - holding for outside intervention - - Jobs in a chain only run after the previous job completes successfully, regardless of - scheduling, snoozing, or retries. - - #### Defining a Chain - - Chains are appropriate in situations where jobs are used to synchronize internal state with - outside state via events. For example, imagine a system that relies on webhooks from a payment - processor to track account balance: - - ```elixir - defmodule MyApp.WebhookWorker do - use Oban.Pro.Worker, queue: :webhooks, chain: [by: :worker] - - @impl true - def process(%Job{args: %{"account_id" => account_id, "event" => event}}) do - account_id - |> MyApp.Account.find() - |> MyApp.Account.handle_webhook_event(event) - end - end - ``` - - Now imagine that it's essential that jobs for an account are processed in order, while jobs from - separate accounts can run concurrently. Modify the `:by` option to partition by worker and - `:account_id`: - - ```elixir - defmodule MyApp.WebhookWorker do - use Oban.Pro.Worker, queue: :webhooks, chain: [by: [args: :account_id]] - - ... - ``` - - Webhooks for each account are guaranteed to run in order regardless of queue concurrency or - errors. The following section shows more partitioning examples. - - #### Chain Partitioning - - By default, chains are sequenced by `worker`, which means any job with the same worker forms a - chain. This approach may not always be suitable. For instance, you may want to link workers - based on a field like `:account_id` instead of just the worker. In such cases, you can use the - [`:by`](#t:chain_by/0) option to customize how chains are partitioned. - - Note that chaining always includes the `queue` field for performance and correctness. This means - that specifying `by: :worker` is equivalent to `by: [:queue, :worker]`, and `by: [args: :account_id]` - is equivalent to `by: [:queue, args: :account_id]`. Jobs in different queues will never be part - of the same chain, ensuring better performance and preventing unexpected dependencies across queues. - - Here are a few examples of using `:by` to achieve fine-grained control over chain partitioning: - - ```elixir - # Chain by :worker - use Oban.Pro.Worker, chain: [by: :worker] - - # Chain by a single args key without considering the worker - use Oban.Pro.Worker, chain: [by: [args: :account_id]] - - # Chain by multiple args keys without considering the worker - use Oban.Pro.Worker, chain: [by: [args: [:account_id, :cohort]]] - - # Chain by worker and a single args key - use Oban.Pro.Worker, chain: [by: [:worker, args: :account_id]] - - # Chain by worker and multiple args key - use Oban.Pro.Worker, chain: [by: [:worker, args: [:account_id, :cohort]]] - - # Chain by a single meta key - use Oban.Pro.Worker, chain: [by: [meta: :source_id]] - ``` - - #### Handling Cancelled/Discarded - - The way a chain behaves when jobs are cancelled or discarded is customizable with the - `:on_discarded` and `:on_cancelled` options. - - There are three strategies for handling upstream discards and cancellations: - - * `:ignore` — keep processing jobs in the chain as if upstream `cancelled` or `discarded` jobs - completed successfully. This is the default behaviour. - - * `:hold` — stop processing any jobs in the chain until the `cancelled` or `discarded` job is - `completed` or eventually deleted. - - Here's an example of a chain that holds on `discarded` or `cancelled`: - - ```elixir - use Oban.Pro.Worker, chain: [on_discarded: :hold, on_cancelled: :hold] - ``` - - > #### Chains and Workflows {: .warning} - > - > Using chained jobs from a workflow isn't allowed. Chains and workflows share an "on hold" - > scheduling mechanism which prevents predictable ordering. - - ## Encrypted Jobs - - Some applications have strong regulations around the storage of personal information. For - example, medical records, financial details, social security numbers, or other data that should - never leak. The `encrypted` option lets you store all job data at rest with encryption so - sensitive data can't be seen. - - #### Defining an Encrypted Worker - - Encryption is handled transparently as jobs are inserted and executed. All you need to do is - flag the worker as encrypted and configure it to fetch a secret key: - - ```elixir - defmodule MyApp.SensitiveWorker do - use Oban.Pro.Worker, encrypted: [key: {module, fun, args}] - - @impl true - def process(%Job{args: args}) do - # Args are decrypted, use them as you normally would - end - end - ``` - - Now job args are encrypted before insertion into the database and decrypted when the job runs. - - #### Generating Encryption Keys - - Encryption requires a 32 byte, Base 64 encoded key. You can generate one with the `:crypto` and - `Base` modules: - - ```elixir - key = 32 |> :crypto.strong_rand_bytes() |> Base.encode64() - ``` - - The result will look something like this `"w7xGJClzEh1pbWuq6zsZfKfwdINu2VIkgCe3IO0hpsA="`. - - While it's possible to use the generated key in your worker directly, that defeats the purpose - of encrypting sensitive data because anybody with access to the codebase can read the encryption - key. That's why it is _highly_ recommended that you use an MFA to retrieve the key dynamically - at runtime. For example, here's how you could pull the key from the Application environment: - - ```elixir - use Oban.Pro.Worker, encrypted: [key: {Application, :fetch_key!, [:enc_key]}] - ``` - - #### Encryption Implementation Details - - * Erlang's `crypto` module is used with the `aes_256_ctr` cipher for encryption. - - * Encoding and decoding stacktraces are pruned to prevent leaking the private key or - initialization vector. - - * Only `args` are encrypted, `meta` is kept as plaintext. You can use that to your advantage for - uniqueness, but be careful not to put anything sensitive in `meta`. - - * Error messages and stacktraces aren't encrypted and are stored as plaintext. Be careful not to - expose sensitive data when raising errors. - - * Args are encrypted at rest _as well as_ in Oban Web. You won't be able to view or search - encrypted args in the Web dashboard. - - * Uniqueness works for encrypted jobs, but not for arguments because the same args are encrypted - differently every time. Favor `meta` over `args` to enforce uniqueness for encrypted jobs. - - ## Job Deadlines - - Jobs that shouldn't run after some period of time can be marked with a `deadline`. After the - deadline has passed the job will be pre-emptively cancelled on its next run, or optionally - _during_ its next run if desired. - - ```elixir - defmodule DeadlinedWorker do - use Oban.Pro.Worker, deadline: {1, :hour} - - @impl true - def process(%Job{args: args}) do - # If this doesn't run within an hour, it's cancelled - end - end - ``` - - Deadlines may be set at runtime as well, provided the worker was _already_ configured with a - `deadline` option. - - ```elixir - # Specify the deadline in seconds - DeadlinedWorker.new(args, deadline: 60) - - # Specify the deadline in minutes, using the tuple syntax - DeadlinedWorker.new(args, deadline: {30, :minutes}) - - # Specify the deadline in days - DeadlinedWorker.new(args, deadline: {1, :day}) - ``` - - In either case, the deadline is always relative and computed at runtime. That also allows the - deadline to consider scheduling—a job scheduled to run 1 hour from now with a 1 hour deadline - will expire 2 hours in the future. Note that deadlines only account for scheduling on insert, - not on retry or after a snooze. - - ### Applying Deadlines to Running Jobs - - A job that has already started may be cancelled at runtime if it won't complete before the - deadline. Consider a slower job that takes 5 minutes to execute and has a deadline 10 minutes in - the future. If the job starts 9 minutes from now it wouldn't finish until 4 minutes past the - deadline. - - By setting the `force` flag, the job will cancel itself if it exceeds the deadline: - - ```elixir - defmodule FirmDeadlinedWorker do - use Oban.Pro.Worker, deadline: [in: {10, :minutes}, force: true] - - ... - ``` - - ## Worker Aliases - - Worker aliases solve a perennial production issue—how to rename workers without breaking existing jobs. - Aliasing allows jobs enqueued with the original worker name to continue executing without - exceptions using the new worker code. - - As an example, imagine that a `UserPurge` worker must expand to purge more than user data. The - `UserPurge` name is no longer accurate, and you want to rename it to `DataPurge`. To rename the - worker and add an alias for the original name: - - ```diff - -defmodule MyApp.UserPurge do - +defmodule MyApp.DataPurge do - - use Oban.Pro.Worker - + use Oban.Pro.Worker, aliases: [MyApp.UserPurge] - - @impl Oban.Pro.Worker - def process(job) do - # purge data, not just users - end - end - ``` - - Now any existing `MyApp.UserPurge` jobs can safely run, and they'll delegate to the new - `DataPurge` code. - - Workers may also have multiple aliases to account for multiple renames or merging workers: - - ```elixir - use Oban.Pro.Worker, aliases: [MyApp.WorkerA, MyApp.WorkerB] - ``` - - ## Worker Hooks - - Worker hooks are called before and after a job finishes executing. They can be defined as - callback functions on the worker, or in a separate module for reuse across workers. - - Hooks are called synchronously, from within the job's process. - - ### Defining Hooks - - There are three mechanisms for defining and attaching `c:before_new/2`, `c:before_process/1`, - and `c:after_process/3` hooks: - - 1. **Implicitly**—hooks are defined directly on the worker and they only run for that worker - 2. **Explicitly**—hooks are listed when defining a worker and they run anywhere they are listed - 3. **Globally**—hooks are executed for all Pro workers - - It's possible to combine each type of hook on a single worker. When multiple hooks are stacked - they're executed in the order: implicit, explicit, and then global. - - ### After Process - - After hooks _do not modify_ the job or execution results. Think of them as a convenient - alternative to globally attached telemetry handlers. They are purely for side-effects such as - cleanup, logging, recording metrics, broadcasting notifications, updating other records, error - notifications, etc. - - Any exceptions or crashes are caught and logged, they won't cause the job to fail or the queue - to crash. - - An `c:after_process/3` hook is called with the job and an execution state corresponding to the - result from `process/1`: - - * `complete`—when `process/1` returns `:ok` or `{:ok, result}` - * `cancel`—when `process/1` returns `{:cancel, reason}` - * `discard`—when a job errors and exhausts retries, or returns `{:discard, reason}` - * `error`—when a job crashes, raises an exception, or returns `{:error, value}` - * `snooze`—when a job returns `{:snooze, seconds}` - - > #### When After Process Hooks Are Called {: .info} - > - > The `c:after_process/3` hook is **only** called for jobs that are actively executing, i.e., - > jobs that go through the worker's `c:process/1` function. This includes jobs that return - > `{:cancel, reason}` from within `c:process/1`. - > - > The hook is **not** called when jobs are cancelled at the database level via - > `Oban.cancel_job/1`, `Oban.cancel_all_jobs/1`, `Oban.Pro.Workflow.cancel_jobs/1,2,3`, or - > `Oban.Pro.Batch.cancel_jobs/1,2`. Those functions bulk-update job states without loading - > jobs into memory, resolving their workers, or executing any callbacks. - - First, here's how to define a single implicit local hook on the worker using - `c:after_process/3`: - - ```elixir - defmodule MyApp.HookWorker do - use Oban.Pro.Worker - - @impl Oban.Pro.Worker - def process(_job) do - # ... - end - - @impl Oban.Pro.Worker - def after_process(state, %Job{} = job, _result) do - MyApp.Notifier.broadcast("oban-jobs", {state, %{id: job.id}}) - end - end - ``` - - Any module that exports `c:after_process/3` can be used as a hook. For example, here we'll - define a shared error notification hook: - - ```elixir - defmodule MyApp.ErrorHook do - def after_process(state, job, _result) when state in [:discard, :error] do - error = job.unsaved_error - extra = Map.take(job, [:attempt, :id, :args, :max_attempts, :meta, :queue, :worker]) - tags = %{oban_worker: job.worker, oban_queue: job.queue, oban_state: job.state} - - Sentry.capture_exception(error.reason, stacktrace: error.stacktrace, tags: tags, extra: extra) - end - - def after_process(_state, _job, _result), do: :ok - end - - defmodule MyApp.HookWorker do - use Oban.Pro.Worker, hooks: [MyApp.ErrorHook] - - @impl Oban.Pro.Worker - def process(_job) do - # ... - end - end - ``` - - The same module can be attached globally, for all `Oban.Pro.Worker` modules, using - `attach_hook/1`: - - ```elixir - :ok = Oban.Pro.Worker.attach_hook(MyApp.ErrorHook) - ``` - - Attaching hooks in your application's `start/2` function is an easy way to ensure hooks are - registered before your application starts processing jobs. - - ```elixir - def start(_type, _args) do - :ok = Oban.Pro.Worker.attach_hook(MyApp.ErrorHook) - - children = [ - ... - ``` - - ### Before New - - The `c:before_new/2` hook can augment or override the `args` and `opts` used to construct new - jobs. It's akin to overriding a worker's `new/2` callback, but the logic can be shared between - workers or operate on a global scale. - - Returning `{:ok, args, opts}` will continue building a job changeset using the provided `args` - and `opts`, while an `{:error, reason}` tuple will mark the changeset as invalid with a custom - error. - - The hook is most useful for augmenting jobs from a central point or injecting custom meta, i.e. - to add span ids for tracing. Here we define a global hook that injects a `span_id` into the - meta: - - ```elixir - defmodule MyApp.SpanHook do - def before_new(args, opts) do - opts = - opts - |> Keyword.put_new(:meta, %{}) - |> put_in([:meta, :span_id], MyApp.Telemetry.gen_span_id()) - - {:ok, args, opts} - end - end - ``` - - ### Before Process - - Before process hooks _may modify_ the job, or prevent calling `c:process/1` entirely depending - on the return value. Returning an `{:ok, job}` tuple will continue processing the job, while - `:error` or `:cancel` tuples will short circuit and record a standard error or cancel the job, - respectively. - - The `c:before_process/1` callback is executed within the job's process, after other internal - processing such as encryption and args structuring are applied. - - It's most useful in situations where the worker needs to perform generic logic such as adding - logger metadata, setting up a dynamic repo, etc. For example, let's define a global hook to set - `user_id` in the logger metadata for every job that contains that field: - - ```elixir - defmodule MyApp.UserMetadataHook do - def before_process(job) do - with %{"user_id" => user_id} <- job.args do - Logger.metadata(user_id: user_id) - end - - {:ok, job} - end - end - ``` - - Then attach the hook globally, as demonstrated earlier: - - ```elixir - :ok = Oban.Pro.Worker.attach_hook(MyApp.UserMetadataHook) - ``` - """ - - @behaviour Oban.Pro.Handler - - alias Oban.{Job, Validation, Worker} - alias Oban.Pro.Batch - alias Oban.Pro.Stages.{Chain, Deadline, Encrypted, Executing, Hooks, Recorded, Structured} - - @typedoc """ - Options to enable and configure `alias` mode. - """ - @type aliases :: [module()] - - @typedoc """ - Options to enable and configure `chain` mode. - """ - @type chain :: [ - by: chain_by(), - on_cancelled: :ignore | :hold, - on_discarded: :ignore | :hold - ] - - @typedoc """ - Configuration that controls how chains are linked together. - """ - @type chain_by :: - :worker - | {:args, atom() | [atom()]} - | {:meta, atom() | [atom()]} - | [:worker | {:args, atom() | [atom()]} | {:meta, atom() | [atom()]}] - - @typedoc """ - Options to enable and configure `deadline` mode. - """ - @type deadline :: [in: timeout(), force: boolean()] - - @typedoc """ - Options to enable and configure `encrypted` mode. - """ - @type encrypted :: [key: mfa()] - - @typedoc """ - Options to enable and configure `recorded` mode. - """ - @type recorded :: true | [to: atom(), limit: pos_integer(), safe_decode: boolean()] - - @typedoc """ - All possible states for the `c:after_process/3` hook. - """ - @type hook_state :: :cancel | :complete | :discard | :error | :snooze - - @type args :: Job.args() - @type opts :: [Job.option()] - - @doc """ - Called when executing a job. - - The `process/1` callback behaves identically to `c:Oban.Worker.perform/1`, except that it may - have pre-processing and post-processing applied. - """ - @callback process(job :: Job.t() | [Job.t()]) :: Worker.result() - - @doc """ - Called before `new/2` when constructing a job changeset. - - The `args` and `opts` returned in a `{:ok, args, opts}` tuple will be used to construct the job - changeset. Returning an `{:error, reason}` tuple will add the reason as a custom error for the - changeset and make it invalid. - - This callback is intended for global or explicit hooks. For workers that override the `new/2` - callback, the `c:before_new/2` hook will _only_ be called if `super(args, opts)` is used and - only after `new/2`, not before. - """ - @callback before_new(args :: Job.args(), opts :: Keyword.t()) :: - {:ok, Job.args(), Keyword.t()} | {:error, any()} - - @doc """ - Called before `process/1` when executing a job. - - The callback is executed within the job's process, after other internal processing such as - encryption and args structuring are applied. - - Returning an `{:ok, job}` tuple will continue processing the job, while `:error` or `:cancel` - tuples will short circuit and record a standard error or cancel the job, respectively. - """ - @callback before_process(job :: Job.t()) :: - {:ok, Job.t()} | {:error, reason :: term()} | {:cancel, reason :: term()} - - @doc """ - Called after a job finishes processing regardless of status (complete, failure, etc). The - `result` is available for any job that returns a value, not just recorded jobs. - - Since crashes and exceptions don't return anything, `result` will always by `nil`. - - See the shared "Worker Hooks" section for more details. - """ - @callback after_process(hook_state(), job :: Job.t(), result :: nil | Worker.result()) :: :ok - - @doc deprecated: "Use after_process/3 instead" - @callback after_process(hook_state(), job :: Job.t()) :: :ok - - @doc """ - Extract the results of a previously executed job. - - If a job has ran and recorded a value, it will return an `{:ok, result}` tuple. Otherwise, - you'll get `{:error, :missing}`. - """ - @callback fetch_recorded(job :: Job.t()) :: {:ok, term()} | {:error, :missing} - - @optional_callbacks after_process: 2, - after_process: 3, - before_new: 2, - before_process: 1, - fetch_recorded: 1 - - @stages [ - executing: Executing, - encrypted: Encrypted, - recorded: Recorded, - structured: Structured, - chain: Chain, - deadline: Deadline, - hooks: Hooks - ] - - @before_new @stages |> Keyword.drop(~w(executing)a) |> Enum.reverse() - - @before_process Keyword.take(@stages, ~w(executing encrypted structured deadline hooks)a) - - @after_process Keyword.take(@stages, ~w(recorded)a) - - # Oban v2.21 declares __opts__ as a callback, but we support v2.19+ - defp maybe_impl_opts do - Code.ensure_loaded?(Oban.Worker) - - if {:__opts__, 0} in Oban.Worker.behaviour_info(:callbacks) do - quote do: @impl(Oban.Worker) - end - end - - @doc false - defmacro __using__(opts) do - opts = Macro.expand_literals(opts, %{__CALLER__ | function: {:__using__, 1}}) - - stage_keys = Keyword.keys(@stages) - - {stage_opts, stand_opts} = - [executing: [], hooks: [], structured: []] - |> Keyword.merge(opts) - |> Keyword.drop(~w(aliases)a) - |> Keyword.split(stage_keys) - - quote do - @behaviour Oban.Worker - @behaviour Oban.Pro.Worker - - @after_compile {Oban.Pro.Worker, :__define_aliases__} - @after_compile {Oban.Pro.Worker, :__validate_opts__} - @after_compile {Oban.Pro.Worker, :__validate_stages__} - - import Oban.Pro.Worker, - only: [ - args_schema: 1, - field: 2, - field: 3, - embeds_one: 2, - embeds_one: 3, - embeds_many: 2, - embeds_many: 3 - ] - - alias Oban.{Job, Worker} - - @stand_opts Keyword.put(unquote(stand_opts), :worker, inspect(__MODULE__)) - @stage_opts unquote(stage_opts) - @opts @stand_opts ++ @stage_opts - @worker_aliases Keyword.get(unquote(opts), :aliases, []) - - @doc false - def __stage_opts__, do: @stage_opts - - @doc false - def __stand_opts__, do: @stand_opts - - @doc false - unquote(maybe_impl_opts()) - def __opts__, do: @opts - - @impl Worker - def new(args, opts \\ []) when is_map(args) and is_list(opts) do - {stage_opts, stand_opts} = - __opts__() - |> Worker.merge_opts(opts) - |> Keyword.split(unquote(stage_keys)) - - stage_opts = Keyword.merge(__stage_opts__(), stage_opts) - - case Oban.Pro.Worker.before_new(__MODULE__, args, stand_opts, stage_opts) do - {:ok, args, opts} -> - Job.new(args, opts) - - {:error, message} -> - args - |> Job.new(stand_opts) - |> Ecto.Changeset.add_error(:args, message) - end - end - - @impl Worker - def backoff(%Job{} = job) do - Worker.backoff(job) - end - - @impl Worker - def timeout(%Job{} = job) do - Worker.timeout(job) - end - - @impl Worker - def perform(%Job{} = job) do - Oban.Pro.Worker.process(__MODULE__, job, __opts__()) - end - - @impl Oban.Pro.Worker - def fetch_recorded(%Job{} = job) do - Oban.Pro.Worker.fetch_recorded(job) - end - - defoverridable backoff: 1, new: 2, perform: 1, timeout: 1 - end - end - - # Compile Callbacks - - @doc false - def __validate_opts__(%{module: module}, _bytecode) do - module - |> Module.get_attribute(:stand_opts) - |> __validate_opts__() - end - - def __validate_opts__(opts) when is_list(opts) do - Validation.validate_schema(opts, - max_attempts: :pos_integer, - priority: {:range, 0..9}, - queue: {:or, [:atom, :string]}, - replace: {:custom, &Job.validate_replace/1}, - tags: {:list, :string}, - unique: {:custom, &Job.validate_unique/1}, - worker: :string - ) - end - - @doc false - def __validate_stages__(%{module: module}, _bytecode) do - module - |> Module.get_attribute(:stage_opts) - |> Enum.each(fn {name, opts} -> - @stages - |> Keyword.fetch!(name) - |> init_stage!(module, opts) - end) - end - - @doc false - def __define_aliases__(env, _bytecode) do - with [_ | _] = aliases <- Module.get_attribute(env.module, :worker_aliases) do - worker = env.module - impl_opts = maybe_impl_opts() - - contents = - quote do - @behaviour Oban.Worker - - unquote(impl_opts) - defdelegate __opts__, to: unquote(worker) - - defdelegate __stage_opts__, to: unquote(worker) - - defdelegate __stand_opts__, to: unquote(worker) - - @impl Oban.Worker - def new(_args, _opts \\ []) do - raise RuntimeError, "unable to call new/1,2 on a worker alias" - end - - @impl Oban.Worker - defdelegate perform(job), to: unquote(worker) - - @impl Oban.Worker - defdelegate backoff(job), to: unquote(worker) - - @impl Oban.Worker - defdelegate timeout(job), to: unquote(worker) - end - - for alias <- aliases do - Module.create(alias, contents, Macro.Env.location(env)) - end - end - end - - # Schema - - @doc """ - Define an args schema struct with field definitions and optional embedded structs. - - The schema is used to validate args before insertion or execution. See the [Structured - Workers](#module-structured-jobs) section for more details. - - ## Example - - Define an args schema for a worker: - - defmodule MyApp.Worker do - use Oban.Pro.Worker - - args_schema do - field :id, :id, required: true - field :name, :string, required: true - field :mode, :enum, values: ~w(on off paused)a - field :safe, :boolean, default: false - - embeds_one :address, required: true do - field :street, :string - field :number, :integer - field :city, :string - end - end - - ... - """ - @doc since: "0.14.0" - defmacro args_schema(do: block) do - quote do - Module.register_attribute(__MODULE__, :oban_fields, accumulate: true) - - try do - import Oban.Pro.Worker, - only: [ - field: 2, - field: 3, - embeds_one: 2, - embeds_one: 3, - embeds_many: 2, - embeds_many: 3 - ] - - unquote(block) - after - :ok - end - - defstruct Enum.map(@oban_fields, &elem(&1, 0)) - - def __args_schema__, do: Enum.reverse(@oban_fields) - end - end - - @doc false - defmacro field(name, type \\ :string, opts \\ []) do - check_type!(name, type) - check_opts!(name, type, opts) - - opts = Keyword.put(opts, :type, type) - - quote do - Module.put_attribute(__MODULE__, :oban_fields, {unquote(name), unquote(opts)}) - end - end - - @doc false - defmacro embeds_one(name, do: block) do - quote do - Oban.Pro.Worker.__embed__(__ENV__, :one, unquote(name), [], unquote(Macro.escape(block))) - end - end - - @doc false - defmacro embeds_one(name, opts, do: block) do - quote do - Oban.Pro.Worker.__embed__( - __ENV__, - :one, - unquote(name), - unquote(opts), - unquote(Macro.escape(block)) - ) - end - end - - @doc false - defmacro embeds_many(name, do: block) do - quote do - Oban.Pro.Worker.__embed__(__ENV__, :many, unquote(name), [], unquote(Macro.escape(block))) - end - end - - @doc false - defmacro embeds_many(name, opts, do: block) do - quote do - Oban.Pro.Worker.__embed__( - __ENV__, - :many, - unquote(name), - unquote(opts), - unquote(Macro.escape(block)) - ) - end - end - - @doc false - def __embed__(env, cardinality, name, opts, inner_block) do - block = - quote do - import Oban.Pro.Worker, only: [args_schema: 1] - - args_schema do - unquote(inner_block) - end - end - - module = - name - |> to_string() - |> Macro.camelize() - |> then(&Module.concat(env.module, &1)) - - required = Keyword.get(opts, :required, false) - opts = [cardinality: cardinality, module: module, required: required, type: :embed] - - Module.create(module, block, env) - Module.put_attribute(env.module, :oban_fields, {name, opts}) - end - - defp check_type!(name, {:array, type}), do: check_type!(name, type) - - defp check_type!(_name, type) when type in ~w(enum term uuid)a, do: :ok - - defp check_type!(name, type) do - unless Ecto.Type.base?(type) do - raise ArgumentError, "invalid type #{inspect(type)} for field #{inspect(name)}" - end - end - - defp check_opts!(name, type, opts) do - allowed = - if type in [:enum, {:array, :enum}] do - [:default, :required, :values] - else - [:default, :required] - end - - with {key, _} <- Enum.find(opts, fn {key, _} -> key not in allowed end) do - raise ArgumentError, "invalid option #{inspect(key)} for field #{inspect(name)}" - end - end - - # Global Hooks - - @doc """ - Register a worker hook to be ran after any Pro worker executes. - - The module must define a function that matches the hook. For example, a module that handles - an `:on_complete` hook must define an `on_complete/1` function. - - ## Example - - Attach a hook handler globally: - - defmodule MyApp.Hook do - def after_process(_state, %Oban.Job{} = job) do - # Do something with the job - - :ok - end - end - - :ok = Oban.Pro.Worker.attach_hook(MyApp.Hook) - """ - @doc since: "0.12.0" - @spec attach_hook(module()) :: :ok | {:error, term()} - defdelegate attach_hook(module), to: Hooks - - @doc """ - Unregister a worker hook. - - ## Example - - Detach a previously registered global hook: - - :ok = Oban.Pro.Worker.detach_hook(MyApp.Hook) - """ - @doc since: "0.12.0" - @spec detach_hook(module()) :: :ok - defdelegate detach_hook(module), to: Hooks - - # Stages - - @doc false - def init_stage!(stage_mod, worker, opts) do - case stage_mod.init(worker, opts) do - {:ok, conf} -> {stage_mod, conf} - {:error, reason} -> raise ArgumentError, "#{stage_mod}: " <> reason - end - end - - @doc false - def before_new(worker, args, opts, stage_opts) do - Enum.reduce_while(@before_new, {:ok, args, opts}, fn {key, mod}, {:ok, args, opts} -> - case Keyword.fetch(stage_opts, key) do - {:ok, stage_opts} -> - {:ok, conf} = mod.init(worker, stage_opts) - - case mod.before_new(args, opts, conf) do - {:ok, _, _} = ok -> {:cont, ok} - other -> {:halt, other} - end - - :error -> - {:cont, {:ok, args, opts}} - end - end) - end - - @doc false - def process(worker, job, opts) do - with {:ok, job} <- before_process(worker, job, opts) do - Process.put(:oban_processing, {worker, job, opts}) - - worker - |> handle_or_process(job) - |> after_process(worker, job, opts) - end - end - - @doc false - def before_process(worker, job, opts) do - Enum.reduce_while(@before_process, {:ok, job}, fn {key, mod}, {:ok, job} -> - case Keyword.fetch(opts, key) do - {:ok, stage_opts} -> - {:ok, conf} = mod.init(worker, stage_opts) - - case mod.before_process(job, conf) do - {:ok, _} = ok -> {:cont, ok} - other -> {:halt, other} - end - - :error -> - {:cont, {:ok, job}} - end - end) - end - - defp handle_or_process(module, %{meta: %{"batch_id" => _, "callback" => callback}} = job) do - new_callback = Map.fetch!(Batch.callbacks_to_functions(), callback) - old_callback = Map.fetch!(Batch.callbacks_to_deprecated(), callback) - - cond do - function_exported?(module, new_callback, 1) -> - apply(module, new_callback, [job]) - - function_exported?(module, old_callback, 1) -> - apply(module, old_callback, [job]) - - true -> - :ok - end - end - - defp handle_or_process(module, job), do: module.process(job) - - @doc false - def after_process(result, worker, job, opts) do - Enum.reduce_while(@after_process, result, fn {key, mod}, result -> - stage_opts = Keyword.get(opts, key, []) - - {:ok, conf} = mod.init(worker, stage_opts) - - case mod.after_process(result, job, conf) do - :ok -> {:cont, result} - other -> {:halt, other} - end - end) - end - - @doc false - def fetch_recorded(job) do - with {:ok, worker} <- Worker.from_string(job.worker), - opts = Keyword.get(worker.__stage_opts__(), :recorded, []), - {:ok, conf} <- Recorded.init(worker, opts) do - Recorded.fetch_recorded(job, conf) - end - end - - # Telemetry - - @doc false - @impl Oban.Pro.Handler - def on_start do - events = [[:oban, :job, :stop], [:oban, :job, :exception]] - - :telemetry.attach_many("oban.pro.worker", events, &Hooks.handle_event/4, nil) - end - - @doc false - @impl Oban.Pro.Handler - def on_stop do - :telemetry.detach("oban.pro.worker") - end -end diff --git a/vendor/oban_pro/lib/oban/pro/workers/batch.ex b/vendor/oban_pro/lib/oban/pro/workers/batch.ex deleted file mode 100644 index ccf34606..00000000 --- a/vendor/oban_pro/lib/oban/pro/workers/batch.ex +++ /dev/null @@ -1,98 +0,0 @@ -defmodule Oban.Pro.Workers.Batch do - @moduledoc """ - A dedicated worker for grouping the execution of jobs. - - This worker is deprecated in favor of `Oban.Pro.Batch`, which is more flexible and supports more - options. - """ - - @moduledoc deprecated: "Use Oban.Pro.Batch instead" - - alias Oban.{Job, Worker} - - @callback new_batch([Job.t() | Job.args()], [keyword()]) :: Oban.Pro.Batch.t() - - @callback gen_id() :: String.t() - - @callback handle_attempted(Job.t()) :: Worker.result() - - @callback handle_cancelled(Job.t()) :: Worker.result() - - @callback handle_completed(Job.t()) :: Worker.result() - - @callback handle_discarded(Job.t()) :: Worker.result() - - @callback handle_exhausted(Job.t()) :: Worker.result() - - @callback handle_retryable(Job.t()) :: Worker.result() - - @callback all_batch_jobs(Job.t(), [keyword()]) :: Enum.t() - - @callback stream_batch_jobs(Job.t(), [keyword()]) :: Enum.t() - - @optional_callbacks handle_attempted: 1, - handle_cancelled: 1, - handle_completed: 1, - handle_discarded: 1, - handle_exhausted: 1, - handle_retryable: 1 - - @doc false - defmacro __using__(opts) do - quote do - use Oban.Pro.Worker, unquote(opts) - - alias Oban.Pro.Workers.Batch - - @behaviour Oban.Pro.Workers.Batch - - @batch_opts ~w( - batch_id - batch_name - batch_callback_args - batch_callback_meta - batch_callback_queue - batch_callback_worker - callback_opts - callback_worker - )a - - @impl Batch - def new_batch([_ | _] = args_or_changesets, opts \\ []) when is_list(opts) do - {batch_opts, job_opts} = Keyword.split(opts, @batch_opts) - - batch_opts = - Enum.reduce(batch_opts, [callback_opts: []], fn {key, val}, acc -> - case key do - :batch_callback_args -> put_in(acc, [:callback_opts, :args], val) - :batch_callback_meta -> put_in(acc, [:callback_opts, :meta], val) - :batch_callback_queue -> put_in(acc, [:callback_opts, :queue], val) - :batch_callback_worker -> Keyword.put(acc, :callback_worker, val) - _ -> Keyword.put(acc, key, val) - end - end) - - case args_or_changesets do - [changeset | _] = changesets when is_struct(changeset) -> - Oban.Pro.Batch.new(changesets, batch_opts) - - args -> - args - |> Enum.map(&new(&1, job_opts)) - |> Oban.Pro.Batch.new(batch_opts) - end - end - - @impl Batch - def gen_id, do: Oban.Pro.UUIDv7.generate() - - @impl Batch - def all_batch_jobs(job, opts \\ []), do: Oban.Pro.Batch.all_jobs(Oban, job, opts) - - @impl Batch - def stream_batch_jobs(job, opts \\ []), do: Oban.Pro.Batch.stream_jobs(Oban, job, opts) - - defoverridable Oban.Pro.Workers.Batch - end - end -end diff --git a/vendor/oban_pro/lib/oban/pro/workers/chain.ex b/vendor/oban_pro/lib/oban/pro/workers/chain.ex deleted file mode 100644 index 61bd5ecf..00000000 --- a/vendor/oban_pro/lib/oban/pro/workers/chain.ex +++ /dev/null @@ -1,170 +0,0 @@ -defmodule Oban.Pro.Workers.Chain do - @moduledoc """ - A dedicated worker for linking jobs to run in strict sequential order. - - This worker is deprecated in favor of `Oban.Pro.Worker`, which is composable and allows better - parallelism. - """ - - @moduledoc deprecated: "Use Oban.Pro.Worker instead" - - import Ecto.Query - - alias Oban.{Job, Repo, Validation} - alias Oban.Pro.Utils - - @default_meta %{ - on_discarded: "ignore", - on_cancelled: "ignore", - chain_key: nil, - hold_snooze: 60, - wait_retry: 10, - wait_sleep: 1000, - wait_snooze: 5 - } - - @opts_keys ~w(by hold_snooze on_cancelled on_discarded wait_retry wait_sleep wait_snooze)a - - @doc false - defmacro __using__(opts) do - {chain_opts, other_opts} = Keyword.split(opts, @opts_keys) - - quote do - Validation.validate!(unquote(chain_opts), &Oban.Pro.Workers.Chain.validate/1) - - use Oban.Pro.Worker, unquote(other_opts) - - @base_meta unquote(chain_opts) - |> Keyword.delete(:by) - |> Map.new() - - @impl Oban.Worker - def new(args, opts) when is_map(args) and is_list(opts) do - meta = - opts - |> Keyword.get(:meta, %{}) - |> Map.merge(@base_meta) - |> Map.put_new(:chain_by, Keyword.fetch!(unquote(chain_opts), :by)) - |> Map.update!(:chain_by, &Utils.normalize_by/1) - - chain_key = Oban.Pro.Workers.Chain.to_chain_key(__MODULE__, args, meta) - chain_id = Oban.Pro.Stages.Chain.to_chain_id(meta.chain_by, __MODULE__, args, opts) - - # Include the `chain_id` for compatibility with chains in v1.5 - meta = - meta - |> Map.put(:chain_key, chain_key) - |> Map.put(:chain_id, chain_id) - - super(args, Keyword.put(opts, :meta, meta)) - end - - @impl Oban.Worker - def perform(%Job{} = job) do - opts = __opts__() - - with {:ok, job} <- Oban.Pro.Worker.before_process(__MODULE__, job, opts) do - job - |> Oban.Pro.Workers.Chain.maybe_process(__MODULE__) - |> Oban.Pro.Worker.after_process(__MODULE__, job, opts) - end - end - end - end - - @doc false - def validate(opts) do - Validation.validate_schema(opts, - by: {:custom, &Oban.Pro.Validation.validate_by/1}, - hold_snooze: :non_neg_integer, - on_cancelled: {:custom, &validate_on_action(:on_cancelled, &1)}, - on_discarded: {:custom, &validate_on_action(:on_discarded, &1)}, - wait_retry: :non_neg_integer, - wait_sleep: :timeout, - wait_snooze: :non_neg_integer - ) - end - - defp validate_on_action(_key, :halt), do: :ok - defp validate_on_action(_key, :hold), do: :ok - defp validate_on_action(_key, :ignore), do: :ok - - defp validate_on_action(key, action) do - {:error, "expected #{key} to be :halt, :hold, or :ignore, got: #{inspect(action)}"} - end - - @doc false - def to_chain_key(worker, args, meta) do - meta - |> Map.fetch!(:chain_by) - |> List.wrap() - |> Enum.reduce(%{}, fn - :worker, acc -> Map.put(acc, :worker, worker) - [:args, keys], acc -> Map.put(acc, :args, take_keys(args, keys)) - [:meta, keys], acc -> Map.put(acc, :meta, take_keys(meta, keys)) - end) - |> :erlang.phash2() - end - - defp take_keys(map, keys) do - keys - |> Enum.sort() - |> Enum.map(fn key -> map[key] || map[to_string(key)] end) - end - - @doc false - def maybe_process(job, worker, wait_count \\ 0) - - # It's running in :inline testing mode and the job doesn't have an id. - def maybe_process(%Job{id: nil} = job, worker, _wait_count) do - worker.process(job) - end - - def maybe_process(%Job{conf: conf, id: id, meta: meta} = job, worker, wait_count) do - meta = meta_with_defaults(meta) - - query = - Job - |> select([j], {j.state, j.scheduled_at}) - |> where([j], fragment("? @> ?", j.meta, ^%{chain_key: meta.chain_key})) - |> where([j], j.state != "completed") - |> where([j], j.id < ^id) - |> order_by(desc: :id) - |> limit(1) - - case Repo.one(conf, query) do - {"executing", _at} -> - if wait_count >= meta.wait_retry do - {:snooze, meta.wait_snooze} - else - Process.sleep(meta.wait_sleep) - - maybe_process(job, worker, wait_count + 1) - end - - {state, _at} when state in ~w(cancelled discarded) -> - key = if state == "cancelled", do: :on_cancelled, else: :on_discarded - - case Map.fetch!(meta, key) do - "halt" -> {:cancel, "chain halted by upstream job"} - "hold" -> {:snooze, meta.hold_snooze} - "ignore" -> worker.process(job) - end - - {_state, scheduled_at} -> - seconds = - scheduled_at - |> DateTime.diff(DateTime.utc_now()) - |> max(meta.wait_snooze) - - {:snooze, seconds} - - nil -> - worker.process(job) - end - end - - defp meta_with_defaults(meta) do - Map.new(@default_meta, fn {key, val} -> {key, meta[to_string(key)] || val} end) - end -end diff --git a/vendor/oban_pro/lib/oban/pro/workers/chunk.ex b/vendor/oban_pro/lib/oban/pro/workers/chunk.ex deleted file mode 100644 index b5b723f8..00000000 --- a/vendor/oban_pro/lib/oban/pro/workers/chunk.ex +++ /dev/null @@ -1,752 +0,0 @@ -defmodule Oban.Pro.Workers.Chunk do - @moduledoc """ - Chunk workers execute jobs together in groups based on a size or a timeout option. e.g. when - 1000 jobs are available or after 10 minutes have ellapsed. - - Multiple chunks can run in parallel within a single queue, and each chunk may be composed of - many thousands of jobs. Aside from improved throughput for a single queue, chunks are ideal as - the initial stage of data-ingestion and data-processing pipelines. - - ## Usage - - Let's define a worker that sends SMS messages in chunks, rather than individually: - - ```elixir - defmodule MyApp.ChunkWorker do - use Oban.Pro.Workers.Chunk, queue: :messages, size: 100, timeout: 1000 - - @impl true - def process([_ | _] = jobs) do - jobs - |> Enum.map(& &1.args) - |> MyApp.Messaging.send_batch() - - :ok - end - end - ``` - - Notice that we declared a `size` and a `timeout` along with a `queue` for the worker. If `size` - or `timeout` are omitted they fall back to their defaults: 100 `size` and 1000ms respectively. - - To process _larger_ batches _less_ frequently, we can increase both values: - - ```elixir - use Oban.Pro.Workers.Chunk, size: 500, timeout: :timer.seconds(5) - ``` - - Now chunks will run with up to 500 jobs or every 5 seconds, whichever comes first. - - Unlike other Pro workers, a `Chunk` worker's `process/1` receives a list of jobs rather than a - single job struct. Fittingly, the [expected return values](#t:result/0) are different as well. - - ## Chunk Partitioning - - By default, chunks are divided into groups based on the `queue` and `worker`. This means that - each chunk consists of workers belonging to the same queue, regardless of their `args` or - `meta`. However, this approach may not always be suitable. For instance, you may want to group - workers based on a specific argument such as `:account_id` instead of just the worker. In such - cases, you can use the [`:by`](#t:chunk_by/0) option to customize how chunks are partitioned. - - Here are a few examples of the `:by` option that you can use to achieve fine-grained control over - chunk partitioning: - - ```elixir - # Explicitly chunk by :worker - use Oban.Pro.Workers.Chunk, by: :worker - - # Chunk by a single args key without considering the worker - use Oban.Pro.Workers.Chunk, by: [args: :account_id] - - # Chunk by multiple args keys without considering the worker - use Oban.Pro.Workers.Chunk, by: [args: [:account_id, :cohort]] - - # Chunk by worker and a single args key - use Oban.Pro.Workers.Chunk, by: [:worker, args: :account_id] - - # Chunk by a single meta key - use Oban.Pro.Workers.Chunk, by: [meta: :source_id] - ``` - - When partitioning chunks of jobs, it's important to note that using only `:args` or `:meta` - without `:worker` may result in heterogeneous chunks of jobs from different workers. - Nevertheless, regardless of the partitioning method chunks always consist of jobs from the same - queue. - - Here's a simple example of partitioning by `worker` and an `author_id` field to batch analysis - of recent messages per author: - - ```elixir - defmodule MyApp.LLMAnalysis do - use Oban.Pro.Workers.Chunk, by: [:worker, args: :author_id], size: 50, timeout: 30_000 - - @impl true - def process([%{args: %{"author_id" => author_id}} | _] = jobs) do - messages = - jobs - |> Enum.map(& &1.args["message_id"]) - |> MyApp.Messages.all() - - {:ok, sentiment} = MyApp.GPT.gauge_sentiment(messages) - - MyApp.Messages.record_sentiment(author_id) - end - end - ``` - - ## Chunk Results and Error Handling - - `Chunk` worker's result type is tailored to handling multiple jobs at once. For reference, here - are the types and callback definition for `process/1`: - - The result types allow you to succeed an entire chunk, or selectively fail parts of it. Here are - each of the possible results: - - * `:ok` — The chunk succeeded and all jobs can be marked complete - - * `{:ok, result}` — Like `:ok`, but with a result for testing. - - * `{:error, reason, jobs}` — One or more jobs in the chunk failed and may be retried, any - unlisted jobs are a success. - - * `{:cancel, reason, jobs}` — One or more jobs in the chunk should be cancelled, any unlisted - jobs are a success. - - * `[error: {reason, jobs}, cancel: {reason, jobs}]` — Retry some jobs and cancel some other - jobs, leaving any jobs not in either list a success. - - To illustrate using chunk results let's expand on the message processing example from earlier. - We'll extend it to complete the whole batch when all messages are delivered or cancel - undeliverable messages: - - ```elixir - def process([_ | _] = jobs) do - notifications = - jobs - |> Enum.map(& &1.args) - |> MyApp.Messaging.send_batch() - - bad_token = fn %{response: response} -> response == :bad_token end - - if Enum.any?(notifications, bad_token) do - cancels = - notifications - |> Enum.zip(jobs) - |> Enum.filter(fn {notification, _job} -> bad_token.(notification) end) - |> Enum.map(&elem(&1, 1)) - - {:cancel, :bad_token, cancels} - else - {:ok, notifications} - end - end - ``` - - In the event of an ephemeral crash, like a network issue, the entire batch may be retried if - there are any remaining attempts. - - Cancelling any jobs in a chunk will cancel the _entire_ chunk, including the leader job. - - ## Chunk Organization - - Chunks are ran by a leader job (which has nothing to do with peer leadership). When the leader - executes it determines whether a complete chunk is available or if enough time has elapsed to - run anyhow. If neither case applies, then the leader will delay until the timeout elapsed and - execute with as many jobs as it can find. - - Completion queries run every `1000ms` by default. You can use the `:sleep` option to control how - long the leader delays between queries to check for complete chunks: - - ```elixir - use Oban.Pro.Workers.Chunk, size: 50, sleep: 2_000, timeout: 10_000 - ``` - - ## Run Chunks Immediately - - A chunk won't run until either the full number of jobs are available (`size`), or the total time - has elapsed (`timeout`). In situations where there is a long gap between incoming jobs it may be - desirable to run the chunk as soon as the chunk runs. This can be done with the `leading` - option. For example, to chunk all jobs that arrive within a 5 minute window, but run immediately - if more than 5 minutes have elapsed: - - ```elixir - use Oban.Pro.Workers.Chunk, size: 100, leading: true, timeout: :timer.minutes(5) - ``` - - ## Optimizing Chunks - - Queue's with high concurrency and low throughput may have multiple chunk leaders running - simultaneously rather than getting bundled into a single chunk. The solution is to reduce the - queue’s global concurrency to a smaller number so that new chunk leader jobs don’t start and the - existing chunk can run a bigger batch. - - ```elixir - queues: [ - chunked_queue: [global_limit: 1] - ] - ``` - - A low global limit will reduce overall throughput. Partitioning the queue with the same options - as the chunk can preserve throughput by allowing one concurrent leader per chunk. For example, - if you were chunking by `:user_id` in args, you'd partition the queue with the same option: - - ```elixir - queues: [ - chunked_queue: [local_limit: 20, global_limit: [allowed: 1, partition: [args: :user_id]]] - ] - ``` - """ - - import Ecto.Query - import DateTime, only: [utc_now: 0] - - alias Ecto.Multi - alias Oban.{CrashError, Job, Notifier, PerformError, Repo, Validation} - alias Oban.Pro.Utils - - @type key_or_keys :: atom() | [atom()] - - @type chunk_by :: - :worker - | {:args, key_or_keys()} - | {:meta, key_or_keys()} - | [:worker | {:args, key_or_keys()} | {:meta, key_or_keys()}] - - @type sub_result :: {reason :: term(), [Job.t()]} - - @type result :: - :ok - | {:ok, term()} - | {:cancel | :discard | :error, reason :: term(), [Job.t()]} - | [cancel: sub_result(), discard: sub_result(), error: sub_result()] - - @type options :: [ - by: chunk_by(), - leading: boolean(), - size: pos_integer(), - sleep: pos_integer(), - timeout: pos_integer() - ] - - @doc false - defmacro __using__(opts) do - {chunk_opts, other_opts} = Keyword.split(opts, ~w(by leading size sleep timeout)a) - - quote do - Validation.validate!(unquote(chunk_opts), &Oban.Pro.Workers.Chunk.validate/1) - - use Oban.Pro.Worker, unquote(other_opts) - - alias Oban.Pro.Workers.Chunk - - @doc false - def default_meta do - %{ - chunk: true, - chunk_by: Keyword.get(unquote(chunk_opts), :by, [:worker]), - chunk_leading: Keyword.get(unquote(chunk_opts), :leading, false), - chunk_size: Keyword.get(unquote(chunk_opts), :size, 100), - chunk_sleep: Keyword.get(unquote(chunk_opts), :sleep, 1000), - chunk_timeout: Keyword.get(unquote(chunk_opts), :timeout, 1000) - } - end - - @impl Oban.Worker - def new(args, opts) when is_map(args) and is_list(opts) do - meta = - default_meta() - |> Map.merge(Keyword.get(opts, :meta, %{})) - |> Map.update!(:chunk_by, &Utils.normalize_by/1) - - chunk_id = Chunk.to_chunk_id(meta.chunk_by, __MODULE__, args, opts) - meta = Map.put(meta, :chunk_id, chunk_id) - - super(args, Keyword.put(opts, :meta, meta)) - end - - @impl Oban.Worker - def perform(%Job{} = job) do - opts = __opts__() - - with {:ok, job} <- Oban.Pro.Worker.before_process(__MODULE__, job, opts) do - Process.put(:oban_processing, {__MODULE__, job, opts}) - - job - |> Chunk.maybe_process(__MODULE__) - |> Oban.Pro.Worker.after_process(__MODULE__, job, opts) - end - end - end - end - - @doc false - def validate(opts) do - Validation.validate_schema(opts, - by: {:custom, &Oban.Pro.Validation.validate_by/1}, - leading: :boolean, - size: :pos_integer, - sleep: :timeout, - timeout: :timeout - ) - end - - @doc false - def to_chunk_id(chunk_by, worker, args, opts) do - chunk_data = - Enum.map([:queue | chunk_by], fn - :queue -> opts |> Keyword.get(:queue, "default") |> to_string() - :worker -> inspect(worker) - [:args, keys] -> Utils.take_keys(args, keys) - [:meta, keys] -> opts |> Keyword.get(:meta, %{}) |> Utils.take_keys(keys) - end) - - Utils.hash64(chunk_data) - end - - # Public Interface - - @doc false - def maybe_process(%Job{conf: conf} = job, worker) do - job = with_defaults(job, worker) - - cond do - full_timeout?(conf, job) -> - fetch_and_process(worker, job) - - full_chunk?(conf, job) -> - fetch_and_process(worker, job) - - true -> - job.meta - |> Map.get("chunk_timeout", 1000) - |> sleep_and_process(worker, job) - end - end - - defp with_defaults(%{meta: %{"chunk_size" => _}} = job, _worker), do: job - - defp with_defaults(job, worker) do - chunk_meta = - Map.new(worker.default_meta(), fn - {:chunk_by, val} -> {"chunk_by", Enum.map(val, &to_string/1)} - {key, val} -> {to_string(key), val} - end) - - %{job | meta: Map.merge(chunk_meta, job.meta)} - end - - # Check Helpers - - defp full_chunk?(conf, %{meta: %{"chunk_size" => size}} = job) do - limit = size - 1 - - query = - Job - |> select([j], j.id) - |> where([j], j.state == "available") - |> chunk_where(job) - |> limit(^limit) - - Repo.one(conf, from(oj in subquery(query), select: count(oj.id) >= ^limit)) - end - - defp full_timeout?(_, %{meta: %{"chunk_leading" => false}}), do: false - - defp full_timeout?(conf, %{meta: %{"chunk_timeout" => timeout}} = job) do - query = - Job - |> where([j], j.id != ^job.id) - |> where([j], j.state in ~w(completed cancelled discarded)) - |> chunk_where(job) - |> order_by(desc: :id) - |> limit(1) - |> select( - [j], - type( - fragment( - "greatest(?, ?, ?)", - j.completed_at, - j.discarded_at, - j.cancelled_at - ), - :utc_datetime_usec - ) - ) - - last_ts = Repo.one(conf, query) - - last_ts && - utc_now() - |> DateTime.add(-timeout, :millisecond) - |> DateTime.after?(last_ts) - end - - # Processing Helpers - - defp fetch_and_process(worker, %{conf: conf, meta: %{"chunk_size" => size}} = job) do - {:ok, chunk} = fetch_chunk(conf, job, size - 1) - {chunk, _errored} = prepare_chunk(chunk) - - guard_cancel(conf, worker, job, chunk) - guard_timeout(conf, worker.timeout(job), worker, job, chunk) - - try do - case worker.process([job | chunk]) do - :ok -> - ack_completed(conf, chunk, :ok) - - {:ok, result} -> - ack_completed(conf, chunk, {:ok, result}) - - {:cancel, reason, cancelled} -> - ack_cancelled(conf, worker, job, chunk, cancelled, reason) - - {:discard, reason, discarded} -> - ack_discarded(conf, worker, job, chunk, discarded, reason) - - {:error, reason, errored} -> - ack_errored(conf, worker, job, chunk, errored, reason) - - [_ | _] = multiple -> - ack_multiple(conf, worker, job, chunk, multiple) - end - rescue - error -> - ack_raised(conf, worker, chunk, job, error, __STACKTRACE__) - - reraise error, __STACKTRACE__ - catch - kind, reason -> - error = CrashError.exception({kind, reason, __STACKTRACE__}) - - ack_raised(conf, worker, chunk, job, error, __STACKTRACE__) - - reraise error, __STACKTRACE__ - end - end - - # This replicates the query used in Smart.fetch_jobs/3, without the meta tracking or any other - # complications. Any modifications to the original query must be replicated here. Another - defp fetch_chunk(conf, job, limit) do - subset_query = - Job - |> select([:id]) - |> where(state: "available") - |> chunk_where(job) - |> order_by(asc: :priority, asc: :scheduled_at, asc: :id) - |> limit(^limit) - |> lock("FOR UPDATE SKIP LOCKED") - - query = - Job - |> with_cte("subset", as: ^subset_query) - |> join(:inner, [j], x in fragment(~s("subset")), on: true) - |> where([j, x], j.id == x.id) - |> select([j, _], j) - - attempted_by = job.attempted_by ++ ["chunk-#{job.id}"] - - updates = [ - set: [state: "executing", attempted_at: utc_now(), attempted_by: attempted_by], - inc: [attempt: 1] - ] - - Repo.transaction(conf, fn -> - {_count, chunk} = Repo.update_all(conf, query, updates) - - chunk - end) - end - - defp prepare_chunk(chunk) do - Enum.reduce(chunk, {[], []}, fn job, {acc, err} -> - with {:ok, worker} <- Oban.Worker.from_string(job.worker), - {:ok, job} <- Oban.Pro.Worker.before_process(worker, job, worker.__opts__()) do - {[job | acc], err} - else - {:error, reason} -> - {acc, [{job, reason} | err]} - end - end) - end - - # Only the leader job is registered as "running" by the queue producer. We listen for pkill - # messages for _other_ jobs in the chunk and apply that to all jobs. Without this, cancelling - # doesn't apply to the leader and chunk jobs are orphaned. - defp guard_cancel(conf, worker, job, chunk) do - # No supervision tree is running when draining, attempting to notify will raise an exception. - if Process.get(:oban_draining) do - :ok - else - parent = self() - - Task.start(fn -> - ref = Process.monitor(parent) - :ok = Notifier.listen(conf.name, [:signal]) - - await_cancel(conf, ref, worker, job, chunk) - end) - end - end - - defp await_cancel(conf, ref, worker, job, chunk) do - receive do - {:notification, :signal, %{"action" => "pkill", "job_id" => kill_id}} -> - if Enum.any?(chunk, &(&1.id == kill_id)) do - reason = PerformError.exception({job.worker, {:cancel, :shutdown}}) - - ack_cancelled(conf, worker, job, chunk, chunk, reason) - - Notifier.notify(conf.name, :signal, %{action: "pkill", job_id: job.id}) - else - await_cancel(conf, ref, worker, job, chunk) - end - - {:DOWN, ^ref, :process, _pid, _reason} -> - :ok - end - end - - # The executor uses :timer.exit_after/2 to kill jobs that exceed the timeout. The queue's - # producer then catches the DOWN message and uses that to record a job error. The producer - # isn't aware of the job's chunk, so we monitor the parent process and ack the chunk jobs. - defp guard_timeout(_conf, :infinity, _worker, _job, _chunk), do: :ok - - defp guard_timeout(conf, timeout, worker, job, chunk) do - parent = self() - - Task.start(fn -> - ref = Process.monitor(parent) - - receive do - {:DOWN, ^ref, :process, _pid, %Oban.TimeoutError{} = reason} -> - ack_errored(conf, worker, job, chunk, chunk, reason) - - {:DOWN, ^ref, :process, _pid, _reason} -> - :ok - after - timeout + 100 -> :ok - end - end) - end - - defp sleep_and_process(total, worker, %Job{conf: conf, meta: meta} = job) do - sleep = Map.get(meta, "chunk_sleep", 1000) - - Process.sleep(sleep) - - if total - sleep < 0 or full_chunk?(conf, job) do - fetch_and_process(worker, job) - else - sleep_and_process(total - sleep, worker, job) - end - end - - # Chunk Helpers - - defp chunk_where(query, job) do - query = where(query, queue: ^job.queue) - - job.meta - |> Map.get("chunk_by") - |> Enum.reduce(query, fn - "worker", acc -> - where(acc, worker: ^job.worker) - - ["args", keys], acc -> - where(acc, [j], fragment("? @> ?", j.args, ^take_keys(job.args, keys))) - - ["meta", keys], acc -> - where(acc, [j], fragment("? @> ?", j.meta, ^take_keys(job.meta, keys))) - end) - end - - defp take_keys(args, keys) when is_struct(args) do - Map.take(args, Enum.map(keys, &String.to_existing_atom/1)) - end - - defp take_keys(map, keys), do: Map.take(map, keys) - - # Ack Helpers - - defp ack_completed(conf, jobs, result) do - Repo.update_all(conf, ids_query(jobs), com_ups()) - - result - end - - defp ack_raised(conf, worker, chunk, job, error, stacktrace) do - {dis, ers} = Enum.split_with(chunk, &exhausted?/1) - - opts = Repo.default_options(conf) - - multi = - Multi.new() - |> Multi.update_all(:err, ids_query(ers), err_ups(worker, job, error, stacktrace), opts) - |> Multi.update_all(:dis, ids_query(dis), dis_ups(worker, job, error, stacktrace), opts) - - Repo.transaction(conf, multi) - - :ok - end - - defp ack_errored(conf, worker, job, chunk, errored, reason) do - {ers, oks, set} = split_with_set(errored, chunk) - {dis, ers} = Enum.split_with(ers, &exhausted?/1) - - opts = Repo.default_options(conf) - - multi = - Multi.new() - |> Multi.update_all(:com, ids_query(oks), com_ups(), opts) - |> Multi.update_all(:err, ids_query(ers), err_ups(worker, job, {:error, reason}), opts) - |> Multi.update_all(:dis, ids_query(dis), dis_ups(worker, job, {:error, reason}), opts) - - with {:ok, _result} <- Repo.transaction(conf, multi) do - if MapSet.member?(set, job.id), do: {:error, reason}, else: :ok - end - end - - defp ack_cancelled(conf, worker, job, chunk, cancelled, reason) do - {can, oks, set} = split_with_set(cancelled, chunk) - - opts = Repo.default_options(conf) - - multi = - Multi.new() - |> Multi.update_all(:com, ids_query(oks), com_ups(), opts) - |> Multi.update_all(:can, ids_query(can), can_ups(worker, job, {:cancel, reason}), opts) - - with {:ok, _result} <- Repo.transaction(conf, multi) do - if MapSet.member?(set, job.id), do: {:cancel, reason}, else: :ok - end - end - - defp ack_discarded(conf, worker, job, chunk, discarded, reason) do - {dis, oks, set} = split_with_set(discarded, chunk) - - opts = Repo.default_options(conf) - - multi = - Multi.new() - |> Multi.update_all(:com, ids_query(oks), com_ups(), opts) - |> Multi.update_all(:dis, ids_query(dis), dis_ups(worker, job, {:discard, reason}), opts) - - with {:ok, _result} <- Repo.transaction(conf, multi) do - if MapSet.member?(set, job.id), do: {:discard, reason}, else: :ok - end - end - - defp ack_multiple(conf, worker, host, chunk, multiple) do - {oks, result} = split_multiple_result(host, chunk, multiple) - - opts = Repo.default_options(conf) - - group = fn {oper, reason, %{attempt: attempt}} -> {oper, reason, attempt} end - multi = Multi.update_all(Multi.new(), :com, ids_query(oks), com_ups(), opts) - - multi = - for({oper, {reas, jobs}} <- multiple, job <- jobs, job.id != host.id, do: {oper, reas, job}) - |> Enum.map(&maybe_discard/1) - |> Enum.group_by(group, &elem(&1, 2)) - |> Enum.with_index() - |> Enum.reduce(multi, fn {{{oper, reason, _}, jobs}, index}, acc -> - update = - case oper do - :cancel -> can_ups(worker, hd(jobs), {:cancel, reason}) - :discard -> dis_ups(worker, hd(jobs), {:discard, reason}) - :error -> err_ups(worker, hd(jobs), {:error, reason}) - end - - Multi.update_all(acc, {oper, index}, ids_query(jobs), update, opts) - end) - - with {:ok, _} <- Repo.transaction(conf, multi), do: result - end - - # Single Helpers - - defp ids_query(jobs) do - where(Job, [j], j.id in ^Enum.map(jobs, & &1.id)) - end - - defp split_with_set(sub_jobs, all_jobs) do - set = MapSet.new(sub_jobs, & &1.id) - - all_jobs - |> Enum.split_with(&MapSet.member?(set, &1.id)) - |> Tuple.insert_at(2, set) - end - - defp exhausted?(%Job{attempt: attempt, max_attempts: max}), do: attempt >= max - - # Multiple Helpers - - defp split_multiple_result(%{id: jid}, chunk, multiple) do - set = for {_, {_, jobs}} <- multiple, %{id: id} <- jobs, into: MapSet.new(), do: id - - result = - if MapSet.member?(set, jid) do - {oper, {reason, _}} = - Enum.find(multiple, fn {_, {_, jobs}} -> jid in Enum.map(jobs, & &1.id) end) - - {oper, reason} - else - :ok - end - - {Enum.reject(chunk, &MapSet.member?(set, &1.id)), result} - end - - defp maybe_discard({:error, reason, job}) when job.attempt >= job.max_attempts do - {:discard, reason, job} - end - - defp maybe_discard(tuple), do: tuple - - # Update Helpers - - defp com_ups do - [set: [state: "completed", completed_at: utc_now()]] - end - - defp can_ups(worker, job, reason) do - error = format_error(job, worker, reason, []) - - Keyword.new() - |> Keyword.put(:set, state: "cancelled", cancelled_at: utc_now()) - |> Keyword.put(:push, errors: error) - end - - defp err_ups(worker, job, error, stacktrace \\ []) do - error = format_error(job, worker, error, stacktrace) - - Keyword.new() - |> Keyword.put(:set, state: "retryable", scheduled_at: backoff_at(worker, job)) - |> Keyword.put(:push, errors: error) - end - - defp dis_ups(worker, job, error, stacktrace \\ []) do - error = format_error(job, worker, error, stacktrace) - - Keyword.new() - |> Keyword.put(:set, state: "discarded", discarded_at: utc_now()) - |> Keyword.put(:push, errors: error) - end - - defp backoff_at(worker, job) do - DateTime.add(utc_now(), worker.backoff(job), :second) - end - - defp format_error(job, worker, reason, stacktrace) when is_tuple(reason) do - format_error(job, worker, PerformError.exception({worker, reason}), stacktrace) - end - - defp format_error(job, _worker, error, stacktrace) do - {blamed, stacktrace} = Exception.blame(:error, error, stacktrace) - - formatted = Exception.format(:error, blamed, stacktrace) - - %{attempt: job.attempt, at: utc_now(), error: formatted} - end -end diff --git a/vendor/oban_pro/lib/oban/pro/workers/workflow.ex b/vendor/oban_pro/lib/oban/pro/workers/workflow.ex deleted file mode 100644 index b94947ee..00000000 --- a/vendor/oban_pro/lib/oban/pro/workers/workflow.ex +++ /dev/null @@ -1,209 +0,0 @@ -defmodule Oban.Pro.Workers.Workflow do - @moduledoc """ - A dedicated worker for linking the execution of jobs. - - This worker is deprecated in favor of `Oban.Pro.Workflow`, which is more flexible and supports - more options. - """ - - @moduledoc deprecated: "Use Oban.Pro.Workflow instead" - - import Ecto.Query, only: [group_by: 3, select: 3, where: 3] - - alias Oban.{Job, Repo} - alias Oban.Pro.Workflow - - @callback new_workflow(Workflow.new_opts()) :: Workflow.t() - - @callback add(Workflow.t(), Workflow.name(), Job.changeset(), Workflow.add_opts()) :: - Workflow.t() - - @callback after_cancelled(Workflow.cancel_reason(), Job.t()) :: :ok - - @callback all_workflow_jobs(Job.t(), Workflow.fetch_opts()) :: [Job.t()] - - @callback append_workflow(Job.t() | [Job.t()], Workflow.append_opts()) :: Workflow.t() - - @callback gen_id() :: String.t() - - @callback stream_workflow_jobs(Job.t(), Workflow.fetch_opts()) :: Enum.t() - - @callback to_dot(Workflow.t()) :: String.t() - - @optional_callbacks after_cancelled: 2 - - defmacro __using__(opts) do - quote do - use Oban.Pro.Worker, unquote(opts) - - @behaviour Oban.Pro.Workers.Workflow - - @impl Oban.Pro.Workers.Workflow - def new_workflow(opts \\ []) when is_list(opts) do - opts - |> Keyword.put_new(:workflow_id, gen_id()) - |> Oban.Pro.Workflow.new() - end - - @impl Oban.Pro.Workers.Workflow - def append_workflow(jobs, opts \\ []) do - Oban.Pro.Workflow.append(jobs, opts) - end - - @impl Oban.Pro.Workers.Workflow - def add(workflow, name, changeset, opts \\ []) do - Oban.Pro.Workflow.add(workflow, name, changeset, opts) - end - - @impl Oban.Pro.Workers.Workflow - def gen_id, do: Oban.Pro.UUIDv7.generate() - - @impl Oban.Pro.Workers.Workflow - def to_dot(workflow) do - Oban.Pro.Workflow.to_dot(workflow) - end - - @impl Oban.Pro.Workers.Workflow - def stream_workflow_jobs(%Job{} = job, opts \\ []) do - Oban.Pro.Workflow.stream_jobs(Oban, job, opts) - end - - @impl Oban.Pro.Workers.Workflow - def all_workflow_jobs(%Job{} = job, opts \\ []) do - Oban.Pro.Workflow.all_jobs(Oban, job, opts) - end - - @impl Oban.Worker - def perform(%Job{} = job) do - opts = __opts__() - - with {:ok, job} <- Oban.Pro.Worker.before_process(__MODULE__, job, opts) do - job - |> Oban.Pro.Workers.Workflow.maybe_process(__MODULE__) - |> Oban.Pro.Worker.after_process(__MODULE__, job, opts) - end - end - - defoverridable Oban.Pro.Workers.Workflow - end - end - - @doc false - defdelegate add(workflow, name, changeset, opts \\ []), to: Workflow - - @doc false - def all_jobs(job, opts \\ []), do: Workflow.all_jobs(Oban, job, opts) - - @doc false - defdelegate append(job, opts \\ []), to: Workflow - - @doc false - def gen_id, do: Oban.Pro.UUIDv7.generate() - - @doc false - defdelegate new(opts \\ []), to: Workflow - - @doc false - def stream_jobs(job, opts \\ []), do: Workflow.stream_jobs(Oban, job, opts) - - @doc false - defdelegate to_dot(workflow), to: Workflow - - @doc false - def maybe_process(%{meta: meta} = job, module) do - if is_map_key(meta, "workflow_id") and not is_map_key(meta, "on_hold") do - legacy_process(job, module) - else - module.process(job) - end - end - - @legacy_meta %{ - ignore_cancelled: false, - ignore_deleted: false, - ignore_discarded: false, - waiting_delay: :timer.seconds(1), - waiting_limit: 10, - waiting_snooze: 5 - } - - # This is necessary for backward compatibility - defp legacy_process(job, module, waiting_count \\ 0) do - meta = for {key, val} <- @legacy_meta, into: %{}, do: {key, job.meta[to_string(key)] || val} - - case legacy_check_deps(job) do - :completed -> - module.process(job) - - :available -> - {:snooze, meta.waiting_snooze} - - :executing -> - if waiting_count >= meta.waiting_limit do - {:snooze, meta.waiting_snooze} - else - Process.sleep(meta.waiting_delay) - - legacy_process(job, module, waiting_count + 1) - end - - {:scheduled, scheduled_at} -> - seconds = - scheduled_at - |> DateTime.diff(DateTime.utc_now()) - |> max(meta.waiting_snooze) - - {:snooze, seconds} - - :cancelled -> - if meta.ignore_cancelled do - module.process(job) - else - {:cancel, "upstream deps cancelled, workflow will never complete"} - end - - :discarded -> - if meta.ignore_discarded do - module.process(job) - else - {:cancel, "upstream deps discarded, workflow will never complete"} - end - - :deleted -> - if meta.ignore_deleted do - module.process(job) - else - {:cancel, "upstream deps deleted, workflow will never complete"} - end - end - end - - defp legacy_check_deps(%{conf: conf, meta: %{"deps" => [_ | _]}} = job) do - %{"deps" => deps, "workflow_id" => workflow_id} = job.meta - - deps_count = length(deps) - - query = - Job - |> where([j], fragment("? @> ?", j.meta, ^%{workflow_id: workflow_id})) - |> where([j], fragment("?->>'name'", j.meta) in ^deps) - |> group_by([j], j.state) - |> select([j], {j.state, count(j.id), max(j.scheduled_at)}) - - conf - |> Repo.all(query) - |> Map.new(fn {state, count, sc_at} -> {state, {count, sc_at}} end) - |> case do - %{"completed" => {^deps_count, _}} -> :completed - %{"scheduled" => {_, scheduled_at}} -> {:scheduled, scheduled_at} - %{"retryable" => {_, scheduled_at}} -> {:scheduled, scheduled_at} - %{"executing" => _} -> :executing - %{"available" => _} -> :available - %{"cancelled" => _} -> :cancelled - %{"discarded" => _} -> :discarded - %{} -> :deleted - end - end - - defp legacy_check_deps(_job), do: :completed -end diff --git a/vendor/oban_pro/lib/oban/pro/workflow.ex b/vendor/oban_pro/lib/oban/pro/workflow.ex deleted file mode 100644 index 78f28120..00000000 --- a/vendor/oban_pro/lib/oban/pro/workflow.ex +++ /dev/null @@ -1,3011 +0,0 @@ -defmodule Oban.Pro.Workflow do - @moduledoc ~S""" - Workflows compose jobs with flexible dependency relationships, enabling sequential execution, - fan-out parallelization, and fan-in convergence patterns. By declaratively defining jobs and - their connections, you can build complex processing pipelines that are both fault-tolerant and - horizontally scalable across all nodes. - - ## Basic Usage - - Workflows are composed of regular jobs or decorated functions linked together through - declarative dependencies. A workflow consists of these key elements: - - - **Jobs** - Individual units of work identified by unique names - - **Dependencies** - Relationships specifying execution order - - Let's create a workflow using a decorated `echo/1` function that demonstrates how jobs execute - in order: - - ```elixir - defmodule EchoWorkflow do - use Oban.Pro.Decorator - - alias Oban.Pro.Workflow - - def insert do - Workflow.new() - |> Workflow.add(:a, new_echo(1)) - |> Workflow.add(:b, new_echo(2), deps: :a) - |> Workflow.add(:c, new_echo(3), deps: :b) - |> Workflow.add(:d, new_echo(4), deps: :b) - |> Workflow.add(:e, new_echo(5), deps: [:c, :d]) - |> Oban.insert_all() - end - - @job true - def echo(value), do: IO.inspect(value) - end - ``` - - The workflow is initialized with `Workflow.new/1`, jobs are added with `Workflow.add/3`, - connections between jobs are added with the `deps` option, and finally the workflow is inserted - with `Oban.insert_all/1`. - - When executed, this workflow will print each job's `value` in the prescribed order. Steps `:c` - and `:d` both depend on `:b`, so they may execute in parallel. Visually, the workflow looks like - this: - - ![Simple Diagram](assets/simple-workflow.svg) - - ## Common Patterns - - Workflows support several execution patterns. These patterns form the building blocks of - workflow design, allowing you to create both simple linear processes and complex dependency - graphs. Each pattern serves a specific purpose in orchestrating job execution: - - > #### Example Workers {: .info} - > - > The following examples all use a generic `Worker` module to help focus on declaring - > dependencies. The patterns apply to any type of worker module or decorated function. - - ### Sequential Execution - - Jobs run one after another, waiting for `scheduled` or `retryable` delays: - - ```elixir - Workflow.new() - |> Workflow.add(:first, new_job()) - |> Workflow.add(:second, Worker.new(%{}), deps: :first) - |> Workflow.add(:third, Worker.new(%{}), deps: :second) - ``` - - ### Fan-Out (1-to-many) - - One job triggers multiple parallel jobs: - - ```elixir - Workflow.new() - |> Workflow.add(:parent, Worker.new(%{})) - |> Workflow.add(:child_1, Worker.new(%{}), deps: :parent) - |> Workflow.add(:child_2, Worker.new(%{}), deps: :parent) - |> Workflow.add(:child_3, Worker.new(%{}), deps: :parent) - ``` - - ### Fan-In (many-to-1) - - Multiple parallel jobs converge to a single job: - - ```elixir - Workflow.new() - |> Workflow.add(:step_1, Worker.new(%{})) - |> Workflow.add(:step_2, Worker.new(%{})) - |> Workflow.add(:step_3, Worker.new(%{})) - |> Workflow.add(:final, Worker.new(%{}), deps: [:step_1, :step_2, :step_3]) - ``` - - ### Diamond Pattern - - Combines fan-out and fan-in: - - ```elixir - Workflow.new() - |> Workflow.add(:start, Worker.new(%{})) - |> Workflow.add(:left, Worker.new(%{}), deps: :start) - |> Workflow.add(:right, Worker.new(%{}), deps: :start) - |> Workflow.add(:end, Worker.new(%{}), deps: [:left, :right]) - ``` - - These patterns can be combined to create complex workflows tailored to your specific business - processes. - - ## Advanced Patterns - - Once you're comfortable with basic workflows, you can leverage more advanced patterns to handle - complex scenarios. These patterns enable shared state, modular composition, simplified - dependencies, and dynamic generation of workflows. - - ### Sub-Workflows - - Workflows can be nested hierarchically using sub workflows. This allows you to compose complex - workflows from simpler ones, making it easier to organize and reuse workflow patterns. - - The `add_workflow/4` function lets you add an entire workflow as a dependency of another - workflow. Like `add/4`, it accepts a name and optional dependencies, but instead of a job - changeset, it takes another workflow. - - Here's an example of creating a primary workflow and adding sub workflows with dependencies: - - ```elixir - alias MyApp.{WorkerA, WorkerB, WorkerC} - alias Oban.Pro.Workflow - - extr_flow = - Workflow.new(workflow_name: "extract") - |> Workflow.add(:extract, WorkerA.new(%{source: "database"})) - |> Workflow.add(:transform, WorkerB.new(%{type: "normalize"}), deps: :extract) - - note_flow = - Workflow.new(workflow_name: "notify") - |> Workflow.add(:prepare, WorkerA.new(%{template: "report"})) - |> Workflow.add(:send, WorkerB.new(%{method: "email"}), deps: :prepare) - - # Create the main workflow and add sub workflows to it - Workflow.new() - |> Workflow.add(:setup, WorkerC.new(%{mode: "initialize"})) - |> Workflow.add_workflow(:extract, extr_flow, deps: :setup) - |> Workflow.add_workflow(:notify, note_flow, deps: :extract) - |> Workflow.add(:finalize, WorkerC.new(%{mode: "cleanup"}), deps: :notify) - |> Oban.insert_all() - ``` - - In this example, the main workflow has a single job named `:setup`, followed by two sub - workflows, and ends with a `:finalize` job. The dependencies ensure proper execution order: - `setup -> extraction -> notification -> finalize`. - - Sub-workflows are a powerful pattern for: - - - Organizing large job dependencies into logical units - - Enabling reuse across applications - - Simplifying the maintenance of complex job graphs - - ### Sharing Results - - Directed dependencies between jobs, paired with the `recorded` option, allow downstream jobs to - fetch the output of upstream jobs. This is particularly useful for multi-step processes where - each step builds on previous results. - - Consider this workflow that simulates a multi-step API interaction: - - ```elixir - defmodule MyApp.WorkerA do - use Oban.Pro.Worker, recorded: true - - @impl true - def process(%Job{args: %{"api_key" => api_key}}) do - token = - api_key - |> String.graphemes() - |> Enum.shuffle() - |> to_string() - - {:ok, token} # This return value will be recorded - end - end - ``` - - The second worker fetches the `token` from the first job by calling `get_recorded/2` with the name - `:a`, which we'll set while building the workflow later. - - ```elixir - defmodule MyApp.WorkerB do - use Oban.Pro.Worker, recorded: true - - @impl true - def process(%Job{args: %{"url" => url}} = job) do - token = Oban.Pro.Workflow.get_recorded(job, :a) - - {:ok, {token, url}} - end - end - ``` - - Then the final worker uses `all_recorded/3` with the `only_deps` option to fetch the results - from all upstream jobs, then it prints out everything that was fetched. - - ```elixir - defmodule MyApp.WorkerC do - use Oban.Pro.Worker - - @impl true - def process(job) do - job - |> Oban.Pro.Workflow.all_recorded(only_deps: true) - |> IO.inspect() - - :ok - end - end - ``` - - Now compose the workers together: - - ```elixir - alias MyApp.{WorkerA, WorkerB, WorkerC} - - Workflow.new() - |> Workflow.add(:a, WorkerA.new(%{api_key: "23kl239bjljlk309af"})) - |> Workflow.add(:b, WorkerB.new(%{url: "elixir-lang.org"}), deps: [:a]) - |> Workflow.add(:c, WorkerB.new(%{url: "www.erlang.org"}), deps: [:a]) - |> Workflow.add(:d, WorkerB.new(%{url: "oban.pro"}), deps: [:a]) - |> Workflow.add(:e, WorkerC.new(%{}), deps: [:b, :c, :d]) - |> Oban.insert_all() - ``` - - When the workflow runs, the final step prints something like: - - ```elixir - %{ - "b" => {"93l2jlj3kl90baf2k3", "elixir-lang.org"}, - "c" => {"93l2jlj3kl90baf2k3", "www.erlang.org"}, - "d" => {"93l2jlj3kl90baf2k3", "oban.pro"} - } - ``` - - Sharing results between jobs is a powerful building block for processing pipelines. While the - approach above works well, cascading functions provide an even more elegant way to share and - transform data between workflow steps. - - ### Cascading Functions - - Cascade mode allows you to build workflows using function captures that automatically receive - context and previous step results. Each function receives a map containing the workflow context - and the results of its dependencies. Results from each step are recorded and made available to - subsequent steps. - - Here's an ETL (Extract, Transform, Load) pipeline using cascading functions: - - ```elixir - defmodule MyApp.ETL do - def insert(source, date) do - Workflow.new() - |> Workflow.put_context(%{source: source, date: date}) - |> Workflow.add_cascade(:extract, &extract/1) - |> Workflow.add_cascade(:transform, &transform/1, deps: :extract) - |> Workflow.add_cascade(:load, &load/1, deps: :transform) - |> Workflow.add_cascade(:notify, ¬ify/1, deps: [:transform, :load]) - |> Oban.insert_all() - end - - def extract(%{source: source, date: date}) do - if source =~ ~r/[a-z0-9]+/ do - %{records: [1, 2, 3], extracted_at: DateTime.utc_now()} - else - {:cancel, "unprocessable source"} - end - end - - def transform(%{extract: %{records: records}}) do - transformed = Enum.map(records, & &1 * 2) - %{records: transformed, count: length(transformed)} - end - - def load(%{transform: %{records: records}}) do - %{loaded: true, count: length(records)} - end - - def notify(%{transform: transform, load: load}) do - IO.puts("Processed #{transform.count} records, loaded #{load.count}") - end - end - ``` - - When this workflow runs, each function is automatically called with a context map containing the - shared context (source and date), and results from dependencies (accessible by their step - names). - - There's no need to manually fetch recorded results as with standard workflow jobs! For standard - workflows that need to fetch results from dependencies, including grafted sub-workflows, see - `all_recorded/3`. - - Cascading functions are especially valuable for: - - - Data pipelines where each step needs the output of previous steps - - Validating aggregate context before proceeding - - Minimizing worker and function definition boilerplate - - ### Cascading Sub-Workflow - - Cascades can be added as sub-workflows that efficiently fan out workloads across multiple items. - This works similar to `add_many/4`, but for cascading functions. - - To create a fan-out cascade, provide a tuple containing an enumerable (like a list, map, or - range) and a function capture with arity 2. The function will be called once for each item in - the enumerable, with the first argument receiving the current item and the second argument - receiving the cumulative context. - - ```elixir - defmodule MyApp.BatchETL do - def insert(sources, date) do - Workflow.new() - |> Workflow.put_context(%{date: date}) - |> Workflow.add_cascade(:sources, {sources, &extract_source/2}) - |> Workflow.add_cascade(:transform, &transform_all/1, deps: :sources) - |> Workflow.add_cascade(:load, &load_data/1, deps: :transform) - |> Workflow.add_cascade(:notify, &send_notification/1, deps: [:transform, :load]) - |> Oban.insert_all() - end - - # Each `source` from the list is passed as the first argument - def extract_source(source, context) do - %{source: source, records: [1, 2, 3], extracted_at: DateTime.utc_now()} - end - - def transform_all(%{sources: sources}) do - total_records = Enum.sum_by(sources, fn {_key, result} -> length(result.records) end) - - transformed = - sources - |> Enum.flat_map(fn {_key, %{records: records}} -> records end) - |> Enum.map(&(&1 * 2)) - - %{records: transformed, count: total_records} - end - - def load_data(%{transform: %{records: records}}) do - %{loaded: true, count: length(records)} - end - - def send_notification(%{transform: transform, load: load, date: date}) do - IO.puts("On #{date}, processed #{transform.count} records, loaded #{load.count}") - end - end - ``` - - This example processes multiple data sources in parallel, then consolidates the results in - subsequent steps. The `extract_source/2` function is automatically called for each source in the - provided list, with the results collected into a map. - - Cascading sub-workflows are useful for: - - - Processing collections of items in parallel with shared context - - Building data pipelines that operate on multiple sources simultaneously - - Implementing fan-out/fan-in patterns with minimal boilerplate - - Maintaining clean separation between item processing and result aggregation - - ### Dynamic Workflows - - Many workflows aren't static—the number of jobs and their interdependencies aren't known - beforehand. You can generate workflows dynamically based on runtime conditions. - - The following worker creates a workflow that fans-out and back in twice, using a variable number - of dependencies: - - ```elixir - defmodule MyApp.Dynamic do - use Oban.Pro.Worker - - alias Oban.Pro.Workflow - - @impl true - def process(%{meta: %{"name" => name}}) do - IO.puts(name) - end - - def insert_workflow(count) when is_integer(count) do - range = Range.new(0, count) - a_deps = Enum.map(range, &"a_#{&1}") - b_deps = Enum.map(range, &"b_#{&1}") - - Workflow.new() - |> Workflow.add(:a, new(%{}), []) - |> fan_out(:a, range) - |> Workflow.add(:b, new(%{}), deps: a_deps) - |> fan_out(:b, range) - |> Workflow.add(:c, new(%{}), deps: b_deps) - |> Oban.insert_all() - end - - defp fan_out(workflow, base, range) do - Enum.reduce(range, workflow, fn key, acc -> - Workflow.add(acc, "#{base}_#{key}", new(%{}), deps: [base]) - end) - end - end - ``` - - This approach is useful for: - - - Processing variable-sized collections - - Creating workflows based on database queries - - Building complex workflows from configuration - - Implementing multi-tenant workflows with different requirements - - ### Grafting Sub-Workflows - - Grafting allows sub-workflows to be attached after a workflow has started. With grafting, you - define placeholders that are expanded into full sub-workflows later when data becomes available. - - Graft jobs serve as placeholders in a workflow. When a graft job executes, it _must_ build - and attach a new sub-workflow at that point. Any downstream jobs that depend on the grafter - will wait for the entire grafted sub-workflow to complete. - - Here's an example workflow that takes an `account_id` then grafts on a sub-workflow to contact - all users in the account: - - ```elixir - defmodule MyApp.AccountNotifier do - alias Oban.Pro.Workflow - - def insert(account_id) do - Workflow.new() - |> Workflow.put_context(%{account_id: account_id}) - |> Workflow.add_graft(:users, &graft_users/1) - |> Workflow.add_cascade(:notify, &send_summary/1, deps: :users) - |> Oban.insert_all() - end - - def graft_users(%{account_id: account_id}) do - user_ids = MyApp.Account.get_user_ids!(account_id) - - {user_ids, &send_notice/2} - |> Workflow.apply_graft() - |> Oban.insert_all() - end - - def send_notice(user_id, context) do - # Notify users - end - - def send_summary(%{account_id: account_id, users: user_ids}) do - # Deliver summary - end - end - ``` - - The summary job waits for the grafting job and the grafted workflow to complete before - executing. - - This pattern is particularly useful for: - - - Building sub-workflows dynamically based on data discovered during execution - - Conditionally executing different workflow branches - - Managing resource-intensive sub-workflows - - ### Appending Jobs - - Sometimes all jobs aren't known when the workflow is created. In that case, you can add more - jobs with optional dependency checking using `append/2`. An appended workflow starts with one or - more jobs, which reuses the original `workflow_id`, and optionally builds a set of dependencies - to check against. - - In this example we disable deps checking with `check_deps: false`: - - ```elixir - def process(job) do - jobs = - job - |> Workflow.append(check_deps: false) - |> Workflow.add(:d, WorkerD.new(%{}), deps: [:a]) - |> Workflow.add(:e, WorkerE.new(%{}), deps: [:b]) - |> Workflow.add(:f, WorkerF.new(%{}), deps: [:c]) - |> Oban.insert_all() - - {:ok, jobs} - end - ``` - - The new jobs specify deps on preexisting jobs named `:a`, `:b`, and `:c`, but there isn't any - guarantee those jobs actually exist. That could lead to an incomplete workflow where the new - jobs may never complete. - - To be safe and check jobs while appending we'll fetch all of the previous jobs with `all_jobs/3` - and feed them in: - - ```elixir - def process(job) do - jobs = Workflow.all_jobs(job) - - jobs - |> Workflow.append() - |> Workflow.add(:d, WorkerD.new(%{}), deps: :a) - |> Workflow.add(:e, WorkerE.new(%{}), deps: :b) - |> Workflow.add(:f, WorkerF.new(%{}), deps: :c) - |> Oban.insert_all() - - :ok - end - ``` - - Now there isn't any risk of an incomplete workflow from missing dependencies, at the expense of - loading some extraneous jobs. - - Appending jobs is particularly useful for: - - - Extending workflows based on user interaction or external events - - Adding additional reporting or cleanup steps after seeing initial results - - Breaking large workflows into stages that are built incrementally - - ## Customization - - Workflows provide several customization options to control their behavior, from identification - to error handling. This section covers how to tailor workflows to your specific requirements. - - #### Workflow ID - - Every workflow needs a unique identifier. By default, `workflow_id` is a time-ordered, random - [UUIDv7][uuid], which ensures uniqueness for any period. For more control, you can provide a - custom ID: - - ```elixir - Workflow.new(workflow_id: "custom-but-still-unique-id") - ``` - - [uuid]: https://datatracker.ietf.org/doc/html/draft-peabody-dispatch-new-uuid-format-04#section-5.2 - - #### Workflow Name - - Additionally, workflows accept an optional name that describes their purpose: - - ```elixir - Workflow.new(workflow_name: "nightly-etl") - ``` - - While the `workflow_id` must be unique, the `workflow_name` doesn't have to be, so it can serve - as a general-purpose label for categorizing or grouping workflows. - - #### Context Key Format - - By default, cascade functions receive context with atom keys for top level dependencies. - However, _nested_ dependencies will have string keys. To prevent this inconsistency you can opt - in to atomizing string keys to atoms with the `atom_keys` option: - - * `atom_keys` — Atomize top level keys for cascade context. Defaults to `true`. - - Apply this option when creating a workflow: - - ```elixir - Workflow.new(atom_keys: true) - |> Workflow.add_cascade(:fetch, &MyApp.fetch/1) - |> Workflow.add_cascade(:process, &MyApp.process/1, deps: :fetch) - ``` - - With `atom_keys: false`, the `process/1` function receives `%{"fetch" => result}` instead of - `%{fetch: result}`. - - #### Dependency Handling - - By default, workflows use conservative dependency handling - if an upstream job is cancelled, - discarded, or deleted, dependent jobs are automatically cancelled. You can customize this - behavior with these options: - - * `ignore_cancelled` — Treat `cancelled` dependencies as completed rather than cancelling - downstream jobs. Defaults to `false`. - - * `ignore_discarded` — Treat `discarded` dependencies as completed rather than cancelling - downstream jobs. Defaults to `false`. - - * `ignore_deleted` — Treat `deleted` (typically pruned) dependencies as completed rather than - cancelling downstream jobs. Defaults to `false`. - - Apply these options to the entire workflow: - - ```elixir - Workflow.new(ignore_cancelled: true, ignore_deleted: true, ignore_discarded: true) - ``` - - Or configure individual jobs within a workflow: - - ```elixir - Workflow.new() - |> Workflow.add(:a, MyWorkflow.new(%{})) - |> Workflow.add(:b, MyWorkflow.new(%{}), deps: :a, ignore_cancelled: true) - |> Workflow.add(:c, MyWorkflow.new(%{}), deps: :b, ignore_discarded: true) - |> Workflow.add(:d, MyWorkflow.new(%{}), deps: :c, ignore_deleted: true) - ``` - - > #### Preventing Stuck Workflows {: .tip} - > - > Be sure that you're running the `DynamicLifeline` to rescue stuck workflows when upstream - > dependencies are deleted unexpectedly. - > - > ```elixir - > config :my_app, Oban, - > plugins: [Oban.Pro.Plugins.DynamicLifeline], - > ... - > ``` - - #### Cancellation Callbacks - - Workflow jobs are automatically `cancelled` when their upstream dependencies are `cancelled`, - `discarded`, or `deleted` (unless specifically configured with the `ignore_*` options mentioned - above). Since these workflow jobs are cancelled before execution, standard - `c:Oban.Pro.Worker.after_process/3` hooks won't be called. - - Instead, you can implement the optional `c:after_cancelled/2` callback specifically for - workflows: - - ```elixir - defmodule MyApp.Workflow do - use Oban.Pro.Worker - - @behaviour Oban.Pro.Workflow - - require Logger - - @impl Oban.Pro.Workflow - def after_cancelled(reason, job) do - Logger.warn("Workflow job #{job.id} cancelled because a dependency was #{reason}") - - # Perform any cleanup required - notify_monitoring_system(job, reason) - - :ok - end - end - ``` - - The `reason` parameter will be one of: - - - `:cancelled` - A dependency was cancelled - - `:discarded` - A dependency was discarded - - `:deleted` - A dependency was deleted - - #### Handling Failures with Status Checking - - For situations where you can't use the `after_cancelled` callback, or when you have many - different workers in a workflow, you can combine the `ignore_cancelled` option with status - checking in a downstream job. - - This pattern allows a final "cleanup" job to examine the state of all upstream jobs and take - appropriate action: - - ```elixir - Workflow.new(ignore_cancelled: true) - |> Workflow.add(:step_1, StepOne.new(%{})) - |> Workflow.add(:step_2, StepTwo.new(%{}), deps: :step_1) - |> Workflow.add(:step_3, StepThree.new(%{}), deps: :step_2) - |> Workflow.add(:report, Reporter.new(%{}), deps: ~w(step_1 step_2 step_3)) - ``` - - Then, in your final worker, examine the state of the upstream jobs using `status/1`: - - ```elixir - defmodule MyApp.Reporter do - use Oban.Pro.Worker - - @impl true - def process(job) do - status = Workflow.status(job) - - if status.counts.cancelled > 0 do - # Some results were cancelled - else - :ok - end - end - end - ``` - - This approach offers several advantages over hooks: - - - It works with any type of worker, including decorated functions and cascades - - You can centralize error handling in a single place - - The workflow continues to the reporting step even if some steps fail - - ## Introspection - - Workflow jobs are tied together through `meta` attributes. You can retrieve these jobs using - several functions, each with different optimization characteristics. - - ### Fetching Jobs - - There are several functions for retrieving jobs, `get_job/2` and `all_jobs/3`, which are - optimized for different use cases: - - ```elixir - # Get a single specific job by name - job_a = Workflow.get_job(job, :a) - - # Get all jobs in a workflow - all_jobs = Workflow.all_jobs(job) - - # Get only the dependencies of the current job - deps = Workflow.all_jobs(job, only_deps: true) - - # Get specific named jobs - [job_a, job_b] = Workflow.all_jobs(job, names: [:a, :b]) - - # Access from outside a worker using workflow_id - jobs = Workflow.all_jobs("some-uuid-1234-5678") - ``` - - ### Streaming Jobs - - For large workflows where loading all jobs at once would consume too much memory, use - `stream_jobs/3`: - - ```elixir - def process(job) do - {:ok, workflow_jobs} = - MyApp.Repo.transaction(fn -> - job - |> Oban.Pro.Workflow.stream_jobs() - |> Stream.filter(& &1.state == "completed") - |> Enum.to_list() - end) - - process_completed_jobs(workflow_jobs) - - :ok - end - ``` - - > #### Transaction Required {: .info} - > - > Streaming is provided by Ecto's `Repo.stream`, and must take place within a transaction. - > This approach lets you control the number of jobs loaded from the database, minimizing memory - > usage for large workflows. - - ### Retrieving Results - - To fetch recorded output from workflow jobs, without loading the full job first, use - `get_recorded/2` and `all_recorded/3`: - - ```elixir - # Get results from a specific job - result_a = Workflow.get_recorded(job, :a) - - # Get results from all jobs - results = Workflow.all_recorded(job) - - # Get only results from dependencies - dep_results = Workflow.all_recorded(job, only_deps: true) - ``` - - ### Checking Status - - To get detailed status information about a workflow, including the overall workflow state, job - counts, time measurement, and sub-workflow details, use `status/1`: - - ```elixir - status = Workflow.status(job) - ``` - - This outputs a map with details, somewhat like this: - - ```elixir - %{ - id: "some-uuid-1234-5678", - name: "my-workflow", - total: 3, - state: :completed, - duration: 72, - counts: %{ - cancelled: 0, - available: 0, - completed: 3, - scheduled: 0, - discarded: 0, - executing: 0, - retryable: 0 - }, - started_at: ~U[2025-04-10 16:34:34], - stopped_at: ~U[2025-04-10 16:34:34], - subs: %{} - } - ``` - - ## Visualization - - Workflows are a type of a [Directed Acyclic Graph][dag], which means we can represent them - visually as nodes (jobs) connected by edges (dependencies). By converting workflows into - standard graph description languages, we can create clear visual representations of job - execution patterns. - - ### Converting to Graph Formats - - Along with the general purpose `to_graph/1` that outputs a `digraph`, there are several built-in - visualization options: - - * `to_dot/1` - Generates [GraphViz][gv] DOT format - * `to_mermaid/1` - Produces [Mermaid][mermaid] flowchart markup - - For example, let's generate a mermaid flowchart from our account workflow: - - ```elixir - Workflow.to_mermaid(archive_account_workflow(123)) - ``` - - This produces the following mermaid output, where each vertex represents a job's name and the - label shows the worker: - - ```text - flowchart TD - backup_1[MyApp.BackupPost] --> delete[MyApp.DeleteAccount] - backup_2[MyApp.BackupPost] --> delete[MyApp.DeleteAccount] - backup_3[MyApp.BackupPost] --> delete[MyApp.DeleteAccount] - receipt[MyApp.FinalReceipt] --> backup_1[MyApp.BackupPost] - receipt[MyApp.FinalReceipt] --> backup_2[MyApp.BackupPost] - receipt[MyApp.FinalReceipt] --> backup_3[MyApp.BackupPost] - receipt[MyApp.FinalReceipt] --> email_1[MyApp.EmailSubscriber] - receipt[MyApp.FinalReceipt] --> email_2[MyApp.EmailSubscriber] - email_1[MyApp.EmailSubscriber] --> delete[MyApp.DeleteAccount] - email_2[MyApp.EmailSubscriber] --> delete[MyApp.DeleteAccount] - ``` - - ### Rendering Visualizations - - You can render these visualizations using the command line with a `mermaid` binary, directly in - [LiveBook][live] using [Kino][kino], or in documentation sites. - - For LiveBook, you can pipe a workflow through `to_mermaid/1` and then into `Kino.Mermaid`: - - ```elixir - workflow - |> Workflow.to_mermaid() - |> Kino.Mermaid.new() - ``` - - This generates an SVG diagram like this: - - ![Mermaid Diagram](assets/mermaid-workflow.svg) - - The visualization clearly shows how our workflow starts with a single `receipt` job, fans-out to - multiple `email` and `backup` jobs, and finally fans-in to the `delete` job—making complex - dependency relationships immediately apparent. - - [dag]: https://en.wikipedia.org/wiki/Directed_acyclic_graph - [mermaid]: https://mermaid.js.org/ - [gv]: https://graphviz.org - [live]: https://livebook.dev/ - [kino]: https://hexdocs.pm/kino/Kino.Mermaid.html - """ - - @behaviour Oban.Pro.Flusher - @behaviour Oban.Pro.Handler - - import Ecto.Query - - alias Ecto.Changeset - alias Oban.Pro.Stages.Chain - alias Oban.Pro.Utils - alias Oban.Pro.Workflow.Cascade - alias Oban.{Config, Job, Repo, Validation} - - require Logger - - # Types - - @type add_opt :: - {:deps, name() | [name()]} - | {:ignore_cancelled, boolean()} - | {:ignore_deleted, boolean()} - | {:ignore_discarded, boolean()} - - @type add_opts :: [add_opt()] - - @type add_cascade_opts :: [Job.option() | add_opt()] - - @type add_workflow_opts :: [deps: name() | [name()]] - - @type append_opts :: [ - check_deps: boolean(), - ignore_cancelled: boolean(), - ignore_deleted: boolean(), - ignore_discarded: boolean(), - workflow_id: String.t(), - workflow_name: String.t() - ] - - @type cancel_opts :: [names: [name()]] - - @type cancel_reason :: :deleted | :discarded | :cancelled - - @type cascade_capture :: (map() -> any()) | {Enum.t(), (any(), map() -> any())} - - @type fetch_opts :: [ - log: Logger.level(), - names: [name()], - only_deps: boolean(), - timeout: timeout(), - with_subs: boolean() - ] - - @type job_or_wid :: Job.t() | workflow_id() - - @type name :: atom() | String.t() - - @type new_opts :: [ - ignore_cancelled: boolean(), - ignore_deleted: boolean(), - ignore_discarded: boolean(), - workflow_id: String.t(), - workflow_name: String.t() - ] - - @type status :: %{ - id: workflow_id(), - name: String.t() | nil, - total: non_neg_integer(), - state: :executing | :completed | :cancelled | :discarded | :unknown, - counts: %{atom() => non_neg_integer()}, - duration: non_neg_integer() | nil, - started_at: DateTime.t() | nil, - stopped_at: DateTime.t() | nil, - subs: %{String.t() => map()} - } - - @type t :: %__MODULE__{ - id: workflow_id(), - changesets: [Job.changeset()], - check_deps: boolean(), - names: MapSet.t(), - opts: map() - } - - @type workflow_id :: String.t() - - # Callbacks - - @doc """ - Called after a workflow job is cancelled due to upstream jobs being `cancelled`, `deleted`, or - `discarded`. - - This callback is _only_ called when a job is cancelled because of an upstream dependency. It is - _never_ called after normal job execution. For that, use `c:Oban.Pro.Worker.after_process/3`. - """ - @callback after_cancelled(cancel_reason(), job :: Job.t()) :: :ok - - @optional_callbacks after_cancelled: 2 - - # Constants - - @all_states Enum.map(Job.states(), &to_string/1) - @hold_date ~U[3000-01-01 00:00:00.000000Z] - @incomplete_set MapSet.new(~w(available executing retryable scheduled)) - @chan_keys ~w(max_attempts meta priority queue schedule_in scheduled_at tags)a - @work_keys ~w(ignore_cancelled ignore_deleted ignore_discarded workflow_id workflow_name)a - - # Macros - - defstruct [ - :id, - changesets: [], - check_deps: true, - eliminations: %{}, - grafts: %{}, - names: MapSet.new(), - opts: %{}, - subs: %{} - ] - - defguardp is_job_or_wid(job_or_wid) when is_struct(job_or_wid, Job) or is_binary(job_or_wid) - - defguardp is_name(name) when (is_atom(name) and not is_nil(name)) or is_binary(name) - - # This is a replacement for `push`, which uses `array_append` and isn't compatible with jsonb - # arrays. The `||` operator works with both arrays and jsonb. - defmacrop concat_errors(column, error) do - quote do - fragment("? || ?", unquote(column), unquote(error)) - end - end - - defmacrop drop_hold(meta) do - quote do - fragment(~s(? || '{"on_hold":false}'), unquote(meta)) - end - end - - defmacrop in_workflow(meta, workflow_id) do - quote do - fragment( - "? \\? 'workflow_id' AND ?->>'workflow_id' = ?", - unquote(meta), - unquote(meta), - unquote(workflow_id) - ) - end - end - - defmacrop in_workflow_or_sup(meta, workflow_id) do - quote do - fragment( - "? \\? 'workflow_id' AND (?->>'workflow_id' = ? OR ?->>'sup_workflow_id' = ?)", - unquote(meta), - unquote(meta), - unquote(workflow_id), - unquote(meta), - unquote(workflow_id) - ) - end - end - - defmacrop in_held_workflow(meta, workflow_id) do - quote do - fragment( - "? \\? 'workflow_id' AND ?->>'workflow_id' = ? AND ?->>'on_hold' = 'true'", - unquote(meta), - unquote(meta), - unquote(workflow_id), - unquote(meta) - ) - end - end - - defmacrop in_held_sup_workflow(meta, workflow_id) do - quote do - fragment( - "? \\? 'sup_workflow_id' AND ?->>'sup_workflow_id' = ? AND ?->>'on_hold' = 'true'", - unquote(meta), - unquote(meta), - unquote(workflow_id), - unquote(meta) - ) - end - end - - defguardp is_legacy_scheduled(meta) - when is_map_key(meta, "orig_scheduled_at") and - :erlang.map_get("orig_scheduled_at", meta) < 9_999_999_999 - - # Telemetry - - @impl Oban.Pro.Handler - def on_start do - events = [ - [:oban, :engine, :cancel_all_jobs, :stop], - [:oban, :plugin, :stop] - ] - - :telemetry.attach_many("oban.workflow", events, &__MODULE__.handle_event/4, nil) - end - - @impl Oban.Pro.Handler - def on_stop do - :telemetry.detach("oban.workflow") - end - - @doc false - def handle_event([:oban, :plugin, _], _time, %{conf: conf, discarded_jobs: jobs}, nil) do - for %{meta: %{"workflow_id" => workflow_id}} <- jobs do - on_flush(workflow_id, conf) - end - end - - def handle_event([:oban, :engine, _, _], _time, %{conf: conf, jobs: jobs}, nil) do - for %{meta: %{"workflow_id" => workflow_id}} <- jobs do - on_flush(workflow_id, conf) - end - end - - def handle_event(_event, _time, _meta, _conf), do: :ok - - # Handling - - @doc false - @impl Oban.Pro.Flusher - def to_flush_mfa(job, conf) do - case job.meta do - %{"ancestor_ids" => ancestors, "workflow_id" => workflow_id} -> - {__MODULE__, :on_flush, [Enum.uniq([workflow_id | ancestors]), conf]} - - %{"sup_workflow_id" => sup_workflow_id, "workflow_id" => workflow_id} -> - {__MODULE__, :on_flush, [[sup_workflow_id, workflow_id], conf]} - - %{"on_hold" => _, "workflow_id" => workflow_id} -> - {__MODULE__, :on_flush, [workflow_id, conf]} - - _ -> - :ignore - end - end - - @doc false - def on_flush(workflow_id, conf) when is_binary(workflow_id) do - # The internal, external, and wildcard deps must be queried independently in order for the - # planner to utilize indexes properly. - lateral_query = - Job - |> select([j], j.state) - |> where( - [j], - # This uses a hardcoded reference to the `sf0` alias for the `parent_as(:elem)` source. - # This is to avoid using `parent_as(:elem).value`, which is valid syntax but - # incompatible with how CRDB handles jsonb references. Ecto requires a field reference - # on any source currently, and this is a workaround. - fragment("? \\? 'workflow_id'", j.meta) and - fragment( - """ - ( - (jsonb_typeof(sf0) = 'string' AND ?->>'workflow_id' = ?->>'workflow_id' AND ?->>'name' = sf0 #>>'{}') OR - (jsonb_typeof(sf0) = 'array' AND sf0->>1 <> '*' AND ?->>'workflow_id' = sf0->>0 AND ?->>'name' = sf0->>1) OR - (jsonb_typeof(sf0) = 'array' AND sf0->>1 = '*' AND ?->>'workflow_id' = sf0->>0) OR - (jsonb_typeof(sf0) = 'array' AND sf0->>1 = '*' AND ? \\? 'sup_workflow_id' AND ?->>'sup_workflow_id' = sf0->>0) - ) - """, - j.meta, - parent_as(:jobs).meta, - j.meta, - j.meta, - j.meta, - j.meta, - j.meta, - j.meta - ) - ) - - deps_query = - from v in fragment("jsonb_array_elements(?->'deps')", parent_as(:jobs).meta), - as: :elem, - inner_lateral_join: j in subquery(lateral_query), - on: true, - select: fragment("coalesce(array_agg(?), '{}')", j.state) - - # A complete set of conditions must be used for held workflows and sub workflows. The planner - # won't use the correct indexes when an "OR" is used. - query = - from(j in Job, as: :jobs) - |> select([j], {j, subquery(deps_query)}) - |> lock("FOR UPDATE SKIP LOCKED") - |> where( - [j], - (j.state == "scheduled" and in_held_workflow(j.meta, ^workflow_id)) or - (j.state == "scheduled" and in_held_sup_workflow(j.meta, ^workflow_id)) - ) - - conf - |> Repo.all(query) - |> Enum.map(&to_stage_operation/1) - |> Enum.group_by(&elem(&1, 0), &elem(&1, 1)) - |> Enum.map(&apply_operations(&1, conf)) - |> on_flush(conf) - end - - def on_flush(workflow_ids, conf) when is_list(workflow_ids) do - workflow_ids - |> List.flatten() - |> Enum.each(&on_flush(&1, conf)) - end - - @doc """ - Instantiate a new workflow struct with a unique workflow id. - - ## Examples - - Create a standard workflow without any options: - - Workflow.new() - - Create a workflow with a custom name: - - Workflow.new(workflow_name: "logistics") - - Create a workflow with a static id and some options: - - Workflow.new(workflow_id: "workflow-id", ignore_cancelled: true, ignore_discarded: true) - """ - @spec new(opts :: new_opts()) :: t() - def new(opts \\ []) do - validate!(opts) - - opts = - opts - |> Keyword.put(:workflow, true) - |> Keyword.put_new_lazy(:workflow_id, &Oban.Pro.UUIDv7.generate/0) - |> Map.new() - - %__MODULE__{id: opts.workflow_id, opts: opts} - end - - defp validate!(opts) do - Validation.validate_schema!(opts, - atom_keys: :boolean, - ignore_cancelled: :boolean, - ignore_deleted: :boolean, - ignore_discarded: :boolean, - workflow_id: :string, - workflow_name: :string - ) - end - - @doc """ - Add a named job to the workflow along with optional dependencies. - - ## Examples - - Add jobs to a workflow with dependencies: - - Workflow.new() - |> Workflow.add(:a, WorkerA.new(%{id: id})) - |> Workflow.add(:b, WorkerB.new(%{id: id}), deps: :a) - |> Workflow.add(:c, WorkerC.new(%{id: id}), deps: :a) - |> Workflow.add(:d, WorkerC.new(%{id: id}), deps: [:b, :c]) - """ - @doc since: "1.5.0" - @spec add(t(), name(), Job.changeset(), add_opts()) :: t() - def add(%__MODULE__{} = workflow, name, changeset, opts \\ []) when is_name(name) do - {deps, opts} = Keyword.pop(opts, :deps, []) - - deps = expand_deps(deps, workflow) - name = to_string(name) - - prevent_chain!(changeset, name) - prevent_dupe!(workflow, name) - confirm_deps!(workflow, deps) - - meta = - changeset - |> Changeset.get_change(:meta, %{}) - |> Map.put(:deps, deps) - |> Map.put(:name, name) - |> Map.put(:on_hold, Enum.any?(deps)) - |> Map.put(:workflow_id, workflow.id) - |> Map.merge(workflow.opts) - |> Map.merge(Map.new(opts)) - - meta = - if Changeset.get_field(changeset, :state) == "scheduled" do - orig_at = - changeset - |> Changeset.get_change(:scheduled_at) - |> DateTime.to_unix(:microsecond) - - Map.put(meta, :orig_scheduled_at, orig_at) - else - meta - end - - changeset = - if Enum.any?(deps) do - changeset - |> Changeset.put_change(:meta, meta) - |> Changeset.put_change(:state, "scheduled") - |> Changeset.put_change(:scheduled_at, @hold_date) - else - Changeset.put_change(changeset, :meta, meta) - end - - changesets = workflow.changesets ++ [changeset] - - %{workflow | changesets: changesets, names: MapSet.put(workflow.names, name)} - end - - defp expand_deps(deps, workflow, mode \\ :main) do - deps - |> List.wrap() - |> Enum.reduce([], fn name, acc -> - name = to_string(name) - - cond do - is_map_key(workflow.grafts, name) -> - grafter_name = "grafter_#{name}" - - grafter_dep = - if is_map_key(workflow.subs, grafter_name) do - [Map.get(workflow.subs, grafter_name), "*"] - else - grafter_name - end - - [grafter_dep | [[Map.get(workflow.subs, name), "*"] | acc]] - - is_map_key(workflow.eliminations, name) -> - Map.get(workflow.eliminations, name) ++ acc - - is_map_key(workflow.subs, name) -> - [[Map.get(workflow.subs, name), "*"] | acc] - - mode == :sub -> - [[workflow.id, name] | acc] - - true -> - [name | acc] - end - end) - |> Enum.reverse() - end - - @doc """ - Add a cascading function to the workflow with optional dependencies. - - Cascading functions receive a context map containing the workflow's shared context and any - results from their upstream dependencies. Results of the function are automatically recorded, - unless they're one of the following standard control tuples: - - - `{:cancel, reason}` - - `{:error, reason}` - - `{:snooze, seconds}` - - > #### Function Captures Only {: .warning} - > - > Cascading functions _must_ be captures like `&MyApp.foo/1` for external functions or `&foo/1` - > for local functions. Anonymous functions won't work. The function always receives a single - > map parameter containing all context and previous results. - - ## Cascading Sub-Workflow - - Cascades can be added as sub-workflows that efficiently fan out workloads across multiple items. - This works similar to `add_many/4`, but for cascading functions. - - To create a fan-out cascade, provide a tuple containing an enumerable (like a list, map, or - range) and a function capture with arity 2. The function will be called once for each item in - the enumerable, with the first argument receiving the current item and the second argument - receiving the cumulative context. - - ```elixir - user_ids = [101, 202, 303] - - Workflow.new() - |> Workflow.put_context(%{operation: "sync"}) - |> Workflow.add_cascade(:users, {user_ids, &MyApp.sync_user/2}) - |> Workflow.add_cascade(:notify, &MyApp.send_summary/1, deps: :users) - - def sync_user(user_id, context) do - # ... - end - ``` - - ## Options - - A subset of standard job options can be passed directly to customize the cascade jobs. The - available options are: - - * `:max_attempts` - * `:meta` - * `:priority` - * `:queue` - * `:schedule_in` - * `:scheduled_at` - * `:tags` - - Pass the additional options alongside `deps` or other `add/4` workflow options. - - ## Examples - - Add a cascading function to a workflow: - - Workflow.new() - |> Workflow.put_context(%{user_id: 123}) - |> Workflow.add_cascade(:activate, &MyApp.activate/1) - |> Workflow.add_cascade(:notify, &MyApp.notify/1, deps: :activate) - |> Oban.insert_all() - - Customize the cascade job: - - Workflow.new() - |> Workflow.add_cascade(:activate, &MyApp.activate/1, queue: :activation) - |> Workflow.add_cascade(:retrofit, &MyApp.retrofit/1, max_attempts: 3) - - Add cascade steps from a range: - - Workflow.new() - |> Workflow.put_context(%{user_id: 123}) - |> Workflow.add_cascade(:activations, {1..5, &MyApp.activate/2}) - - Add cascade steps from a map, using the map keys as custom names: - - accounts = %{admin: 1, user: 2, guest: 3} - - Workflow.new() - |> Workflow.put_context(%{action: "sync"}) - |> Workflow.add_cascade(:accounts, {accounts, &MyApp.sync_account/2}) - """ - @doc since: "1.6.0" - @spec add_cascade(t(), name(), cascade_capture(), add_cascade_opts()) :: t() - def add_cascade(workflow, name, fun, opts \\ []) - - def add_cascade(%__MODULE__{} = workflow, name, fun, opts) when is_function(fun) do - {changeset, opts} = build_cascade(fun, opts) - - add(workflow, name, changeset, opts) - end - - def add_cascade(%__MODULE__{} = workflow, name, {items, fun}, opts) - when is_name(name) and is_function(fun, 2) do - info = Function.info(fun) - - cond do - info[:arity] != 2 -> - raise ArgumentError, "context function must have arity of 2" - - info[:module] == :erl_eval -> - raise ArgumentError, "context function must be a capture and not anonymous" - - true -> - argf = &%{arg: &1, mod: inspect(info[:module]), fun: to_string(info[:name])} - decn = "#{inspect(info[:module])}.#{info[:name]}/#{info[:arity]}" - meta = %{decorated: true, decorated_name: decn} - - {chan_opts, opts} = Keyword.split(opts, @chan_keys) - {work_opts, opts} = Keyword.split(opts, @work_keys) - - chan_opts = Keyword.update(chan_opts, :meta, meta, &Map.merge(&1, meta)) - - sub = - items - |> with_names() - |> Enum.reduce(new(work_opts), fn {key, item}, acc -> - add(acc, to_string(key), Cascade.new(argf.(item), chan_opts)) - end) - - add_workflow(workflow, name, sub, opts) - end - end - - @doc """ - Adds a grafting point to the workflow where sub-workflows can be attached later. - - A grafter serves as a placeholder in the workflow topology that downstream jobs can - depend upon. Later, another job can dynamically generate a sub-workflow and attach it - at this grafting point using `graft/3`. - - The grafter can be defined using either a job changeset or a cascading function. - - ## Examples - - Add a grafting point with a job changeset: - - Workflow.new() - |> Workflow.add_graft(:users, MyWorker.new(%{})) - |> Workflow.add(:notify, WorkerC.new(%{}), deps: :users) - - Add a grafting point with a cascading function: - - Workflow.new() - |> Workflow.put_context(%{account_id: account_id}) - |> Workflow.add_graft(:users, &build_users_workflow/1) - |> Workflow.add_cascade(:notify, &send_summary/1, deps: :users) - - Add multiple grafting points with a fan-out tuple: - - Workflow.new() - |> Workflow.add_graft(:accounts, {account_ids, &sync_account/2}) - |> Workflow.add(:finalize, FinalizerWorker.new(%{}), deps: :accounts) - - Each function in a fan-out graft receives the item and context, and should call - `apply_graft/1` to dynamically create a sub-workflow: - - def sync_account(account_id, _context) do - {user_ids(account_id), &sync_user/2} - |> Workflow.apply_graft() - |> Oban.insert_all() - - :ok - end - """ - @doc since: "1.6.0" - @spec add_graft(t(), name(), Job.changeset() | cascade_capture(), add_opts()) :: t() - def add_graft(workflow, name, changeset_or_fun, opts \\ []) - - def add_graft(%__MODULE__{} = workflow, name, fun, opts) when is_function(fun) do - {changeset, opts} = build_cascade(fun, opts) - - add_graft(workflow, name, changeset, opts) - end - - def add_graft(%__MODULE__{} = workflow, name, {_items, fun} = cascade, opts) - when is_name(name) and is_function(fun, 2) do - name = to_string(name) - uuid = "grafted_#{Oban.Pro.UUIDv7.generate()}" - grafter_name = "grafter_#{name}" - - graft_meta = %{graft_id: uuid, graft_name: name} - opts = Keyword.update(opts, :meta, graft_meta, &Map.merge(&1, graft_meta)) - - workflow = %{ - workflow - | grafts: Map.put(workflow.grafts, name, grafter_name), - subs: Map.put(workflow.subs, name, uuid) - } - - add_cascade(workflow, grafter_name, cascade, opts) - end - - def add_graft(%__MODULE__{} = workflow, name, changeset, opts) when is_name(name) do - name = to_string(name) - uuid = "grafted_#{Oban.Pro.UUIDv7.generate()}" - grafter_name = "grafter_#{name}" - - meta = - changeset - |> Changeset.get_change(:meta, %{}) - |> Map.put(:graft_id, uuid) - |> Map.put(:graft_name, name) - - changeset = Changeset.put_change(changeset, :meta, meta) - - workflow = - workflow - |> Map.update!(:grafts, &Map.put(&1, name, grafter_name)) - |> Map.update!(:subs, &Map.put(&1, name, uuid)) - - add(workflow, grafter_name, changeset, opts) - end - - defp build_cascade(fun, opts) do - info = Function.info(fun) - - cond do - info[:arity] != 1 -> - raise ArgumentError, "context function must have arity 1" - - info[:module] == :erl_eval -> - raise ArgumentError, "context function must be a capture and not anonymous" - - true -> - args = %{mod: inspect(info[:module]), fun: to_string(info[:name])} - decn = "#{inspect(info[:module])}.#{info[:name]}/#{info[:arity]}" - meta = %{decorated: true, decorated_name: decn} - - {chan_opts, opts} = - opts - |> Keyword.update(:meta, meta, &Map.merge(&1, meta)) - |> Keyword.split(@chan_keys) - - {Cascade.new(args, chan_opts), opts} - end - end - - @doc """ - Add multiple named jobs as a sub workflow along with optional dependencies. - - ## Empty Sub-Workflows - - When an empty collection of jobs is provided, the sub-workflow step is skipped entirely. - Any dependencies on this step are automatically transferred to downstream steps to maintain - the correct execution order. - - Empty sub-workflows have no job results to retrieve or cascade outputs to reference. Consider - this when creating dependencies on sub-workflows that might be empty at runtime. - - ## Examples - - Add multiple jobs as a sub workflow with a list, where the keys will be the job's position, - starting at 0: - - Workflow.add_many(workflow, :sub_1, [WorkerA.new(%{}), WorkerB.new(%{})]) - - Add multiple jobs as a sub workflow with a map, where the keys are the job names: - - Workflow.add_many(workflow, :sub_1, %{a: WorkerA.new(%{}), b: WorkerB.new(%{})}) - - Add multiple jobs as sub workflows and add a dependency on them: - - Workflow.new() - |> Workflow.add_many(:a, changesets_a) - |> Workflow.add_many(:b, changesets_b) - |> Workflow.add(:c, WorkerC.new(%{}), deps: [:a, :b]) - - Workflow options pay be provided to name or otherwise customize the sub-workflow: - - Workflow.add_many(workflow, :sub_1, changesets, ignore_cancelled: true) - """ - @doc since: "1.6.0" - @spec add_many(t(), name(), Enum.t(), new_opts() | add_opts()) :: t() - def add_many(%__MODULE__{} = workflow, name, changesets, opts \\ []) when is_name(name) do - {work_opts, opts} = Keyword.split(opts, @work_keys) - - sub = - changesets - |> with_names() - |> Enum.reduce(new(work_opts), fn {key, chg}, acc -> add(acc, to_string(key), chg) end) - - add_workflow(workflow, name, sub, opts) - end - - defp with_names(items) when is_list(items), do: Enum.with_index(items, &{&2, &1}) - defp with_names(items) when is_struct(items), do: items |> Enum.to_list() |> with_names() - defp with_names(items) when is_map(items), do: items - - @doc """ - Add a sub workflow with a name and optional dependencies to another workflow. - - This function lets you compose workflows by adding one workflow as a dependency within another - workflow. Workflows may depend on jobs or other workflows, and jobs may depend on entire - workflows. If a job depends on a workflow, it won't execute until every job in the workflow is - in a final state (completed, cancelled, or discarded). - - ## Examples - - Add a sub workflow to a workflow with dependencies: - - sub_1 = - Workflow.new() - |> Workflow.add(:a, WorkerB.new(%{id: id})) - |> Workflow.add(:b, WorkerC.new(%{id: id})) - - sub_2 = - Workflow.new() - |> Workflow.add(:a, WorkerB.new(%{id: id})) - |> Workflow.add(:b, WorkerC.new(%{id: id})) - - Workflow.new() - |> Workflow.add(:a, WorkerA.new(%{id: id})) - |> Workflow.add_workflow(:b, sub_1, deps: :a) - |> Workflow.add_workflow(:c, sub_2, deps: :b) - |> Workflow.add(:d, WorkerD.new(%{id: id}), deps: [:b, :c]) - """ - @doc since: "1.6.0" - @spec add_workflow(t(), name(), t(), add_workflow_opts()) :: t() - def add_workflow(%_{} = workflow, name, %_{} = sub_workflow, opts \\ []) when is_name(name) do - {deps, opts} = Keyword.pop(opts, :deps, []) - - deps = expand_deps(deps, workflow, :sub) - name = to_string(name) - - prevent_dupe!(workflow, name) - confirm_deps!(workflow, deps) - - sub_changesets = - Enum.map(sub_workflow.changesets, fn changeset -> - meta_deps = get_in(changeset.changes, [:meta, :deps]) - - if deps != [] and meta_deps == [] do - old_ancestors = get_in(changeset.changes, [:meta, :ancestor_ids]) || [] - new_ancestors = Map.get(workflow.opts, :ancestor_ids, []) - all_ancestors = old_ancestors ++ [workflow.id] ++ new_ancestors - - opts - |> Map.new() - |> Map.put(:deps, deps) - |> Map.put(:sup_workflow_id, workflow.id) - |> Map.put(:sub_name, name) - |> Map.put(:workflow_id, sub_workflow.id) - |> Map.put(:ancestor_ids, all_ancestors) - |> put_hold(changeset) - else - new_ancestors = Map.get(workflow.opts, :ancestor_ids, []) - - meta = - if get_in(changeset.changes, [:meta, :sup_workflow_id]) do - old_ancestors = get_in(changeset.changes, [:meta, :ancestor_ids]) || [] - all_ancestors = old_ancestors ++ [workflow.id] ++ new_ancestors - - %{sup_workflow_id: workflow.id, ancestor_ids: all_ancestors} - else - %{ - sub_name: name, - sup_workflow_id: workflow.id, - workflow_id: sub_workflow.id, - ancestor_ids: [workflow.id] ++ new_ancestors - } - end - - Changeset.update_change(changeset, :meta, &Map.merge(&1, meta)) - end - end) - - subs = Map.put(workflow.subs, name, sub_workflow.id) - - case sub_changesets do - [_ | _] -> - changesets = workflow.changesets ++ sub_changesets - - %{workflow | changesets: changesets, subs: subs} - - [] -> - eliminations = Map.put(workflow.eliminations, name, deps) - - %{workflow | eliminations: eliminations, subs: subs} - end - end - - defp put_hold(meta, changeset) do - meta = - if Changeset.get_field(changeset, :state) == "scheduled" do - orig_at = - changeset - |> Changeset.get_change(:scheduled_at) - |> DateTime.to_unix(:microsecond) - - meta - |> Map.put(:on_hold, true) - |> Map.put(:orig_scheduled_at, orig_at) - else - Map.put(meta, :on_hold, true) - end - - changeset - |> Changeset.update_change(:meta, &Map.merge(&1, meta)) - |> Changeset.put_change(:state, "scheduled") - |> Changeset.put_change(:scheduled_at, @hold_date) - end - - @doc """ - Adds a context value to the workflow that can be accessed by all jobs. - - This is a convenience function that creates a recorded job named `:context` containing - any value that needs to be shared across multiple workflow jobs. - - ## Examples - - Add a context value to a workflow: - - Workflow.new() - |> Workflow.put_context(%{user_id: 123, action: "process"}) - |> Workflow.add(:job_a, WorkerA.new(%{})) - |> Workflow.add(:job_b, WorkerB.new(%{})) - |> Oban.insert_all() - - Later, retrieve the context in a worker: - - def process(job) do - context = Workflow.get_context(job) - # Use context map... - end - """ - @doc since: "1.6.0" - @spec put_context(t(), map()) :: t() - def put_context(%__MODULE__{} = workflow, value) when is_map(value) do - opts = [ - meta: %{context: true, recorded: true, return: Utils.encode64(value)}, - worker: Oban.Pro.Workers.Context - ] - - changeset = - %{} - |> Job.new(opts) - |> Changeset.put_change(:state, "completed") - |> Changeset.put_change(:completed_at, DateTime.utc_now()) - - add(workflow, :context, changeset) - end - - @doc """ - Gets a workflow's context value. - - This is a convenience function that internally calls `get_recorded/2` with the `:context` name. - - ## Examples - - Retrieve the context within a worker: - - def process(job) do - context = Workflow.get_context(job) - end - - Retrieve the context with a workflow id: - - Workflow.get_context("some-uuid-1234-5678") - - Retrieve the context using a custom Oban instance name: - - Workflow.get_context(MyApp.Oban, "some-uuid-1234-5678") - """ - @doc since: "1.6.0" - @spec get_context(Oban.name(), job_or_wid()) :: nil | map() - def get_context(oban_name \\ Oban, job_or_wid) do - get_recorded(oban_name, job_or_wid, :context) - end - - @doc """ - Instantiate a new workflow from one or more existing workflow jobs. - - ## Examples - - Append to a workflow seeded with all other jobs in the workflow: - - jobs - |> Workflow.append() - |> Workflow.add(:d, WorkerD.new(%{}), deps: [:a]) - |> Workflow.add(:e, WorkerE.new(%{}), deps: [:b]) - |> Oban.insert_all() - - Append to a workflow from a single job and bypass checking deps: - - job - |> Workflow.append(check_deps: false) - |> Workflow.add(:d, WorkerD.new(%{}), deps: [:a]) - |> Workflow.add(:e, WorkerE.new(%{}), deps: [:b]) - |> Oban.insert_all() - """ - @doc since: "1.5.0" - @spec append(Job.t() | [Job.t()], append_opts()) :: t() - def append(jobs, opts \\ []) - - def append([%Job{meta: %{"workflow_id" => _} = meta} | _] = jobs, opts) do - {check, opts} = Keyword.pop(opts, :check_deps, true) - - workflow_opts = - meta - |> Map.take(~w(workflow_id workflow_name)) - |> Keyword.new(fn {key, val} -> {String.to_existing_atom(key), val} end) - - # Preserve ancestor chain for nested sub-workflows - ancestors = - case meta do - %{"ancestor_ids" => ids} -> ids - %{"sup_workflow_id" => sup} -> [sup] - _ -> [] - end - - workflow = - opts - |> Keyword.merge(workflow_opts) - |> new() - - opts = Map.put(workflow.opts, :ancestor_ids, ancestors) - - %{workflow | check_deps: check, names: MapSet.new(jobs, & &1.meta["name"]), opts: opts} - end - - def append(job, opts), do: append([job], opts) - - @doc """ - Attaches a sub-workflow to a previously defined grafting point. - - Grafting points are defined using `add_graft/4` and serve as placeholders where sub-workflows can be - attached dynamically during execution. This function must be called from within a worker's - `process/1` callback, and only from a job that was defined using `add_graft/4`. - - ## Examples - - Graft a sub-workflow using a cascade: - - def process(job) do - user_ids = active_user_ids(job.args["account_id"]) - - {user_ids, &build_user_workflow/2} - |> Workflow.apply_graft() - |> Oban.insert_all() - - :ok - end - - Graft a sub-workflow using a workflow: - - def process(job) do - workflow = - job.args["account_id"] - |> active_user_ids() - |> Enum.reduce(Workflow.new(), fn user_id, workflow -> - Workflow.add(workflow, user_id, UserWorker.new(%{user_id: user_id})) - end) - - workflow - |> Workflow.apply_graft() - |> Oban.insert_all() - - :ok - end - - Add workflow-specific options when grafting: - - {user_ids, &build_user_workflow/2} - |> Workflow.apply_graft(workflow_name: "my-workflow") - |> Oban.insert_all() - """ - @doc since: "1.6.0" - @spec apply_graft(t() | cascade_capture(), new_opts() | add_cascade_opts()) :: t() - def apply_graft(cascade_or_workflow, opts \\ []) - - def apply_graft({_enum, fun} = cascade, opts) when is_function(fun) do - {job, graft_id, graft_name} = extract_graft!() - - job - |> append(check_deps: false) - |> add_cascade(graft_name, cascade, Keyword.put(opts, :workflow_id, graft_id)) - end - - def apply_graft(%__MODULE__{} = workflow, opts) do - {job, graft_id, graft_name} = extract_graft!() - - root_id = - case job.meta["workflow_id"] do - "grafted_" <> _ = parent_graft_id -> parent_graft_id - _ -> graft_id - end - - sub_names = expand_sub_names(workflow) - - changesets = - Enum.map(workflow.changesets, fn changeset -> - changeset = Changeset.update_change(changeset, :meta, &Map.put(&1, :workflow_id, root_id)) - - # Convert internal wildcard deps to explicit name deps so internal dependencies resolve - # correctly. - deps = get_in(changeset.changes, [:meta, :deps]) || [] - - updated_deps = - Enum.flat_map(deps, fn - [sub_id, "*"] -> - case Map.get(sub_names, sub_id) do - nil -> [[sub_id, "*"]] - names -> names - end - - other -> - [other] - end) - - if updated_deps != deps do - Changeset.update_change(changeset, :meta, &Map.put(&1, :deps, updated_deps)) - else - changeset - end - end) - - workflow = %{workflow | id: root_id, changesets: changesets} - - job - |> append(check_deps: false) - |> add_workflow(graft_name, workflow, opts) - end - - defp expand_sub_names(workflow) do - Enum.reduce(workflow.changesets, %{}, fn changeset, acc -> - meta = changeset.changes[:meta] || %{} - name = meta[:name] - - sub_id = if meta[:sup_workflow_id], do: meta[:workflow_id] - - if sub_id && name do - Map.update(acc, sub_id, [name], &[name | &1]) - else - acc - end - end) - end - - defp extract_graft! do - case Process.get(:oban_processing) do - {_module, %{meta: %{"graft_id" => _}} = job, _opts} -> - %{"graft_id" => graft_id, "graft_name" => graft_name} = job.meta - - {job, graft_id, graft_name} - - {_module, _job, _opts} -> - raise "current job isnt't marked for grafting with add_graft/4" - - _ -> - raise "current job isn't available, graft/1 must be called from process/1" - end - end - - @doc false - def all_jobs(job_or_wid), do: all_jobs(Oban, job_or_wid, []) - - @doc false - def all_jobs(job_or_wid, opts) when is_job_or_wid(job_or_wid) and is_list(opts) do - all_jobs(Oban, job_or_wid, opts) - end - - def all_jobs(oban_name, job_or_wid) when is_job_or_wid(job_or_wid) do - all_jobs(oban_name, job_or_wid, []) - end - - @doc """ - Get all jobs for a workflow, optionally filtered by upstream deps. - - ## Examples - - Retrieve all workflow jobs within a `process/1` function: - - def process(job) do - job - |> Workflow.all_jobs() - |> do_things_with_jobs() - - :ok - end - - Retrieve all of the current job's deps: - - jobs = Workflow.all_jobs(job, only_deps: true) - - Retrieve an explicit list of dependencies: - - [job_a, job_b] = Workflow.all_jobs(job, names: [:a, :b]) - - Include sub workflow jobs: - - [sub_a, sub_b] = Workflow.all_jobs(job, names: [:sub], with_subs: true) - - Retrieve all jobs using a `workflow_id`: - - jobs = Workflow.all_jobs("some-uuid-1234-5678") - - Retrieve only some named jobs using a `workflow_id`: - - [job_a, job_b] = Workflow.all_jobs("some-uuid-1234-5678", deps: [:a, :b]) - - Retrieve all jobs using a `workflow_id` and a custom Oban instance name: - - jobs = Workflow.all_jobs(MyApp.Oban, "some-uuid-1234-5678") - """ - @doc since: "1.5.0" - @spec all_jobs(Oban.name(), job_or_wid(), fetch_opts()) :: [Job.t()] - def all_jobs(oban_name, job_or_wid, opts) - - def all_jobs(_oban_name, %Job{conf: conf, meta: meta}, opts) do - %{"deps" => deps, "workflow_id" => workflow_id} = meta - - {query_opts, opts} = Keyword.split(opts, ~w(names only_deps with_subs)a) - - Repo.all(conf, workflow_query(workflow_id, deps, query_opts), opts) - end - - def all_jobs(oban_name, workflow_id, opts) when is_binary(workflow_id) do - if Keyword.has_key?(opts, :only_deps) do - raise RuntimeError, ":only_deps isn't a valid option with a workflow_id" - end - - {query_opts, opts} = Keyword.split(opts, ~w(names with_subs)a) - - oban_name - |> Oban.config() - |> Repo.all(workflow_query(workflow_id, [], query_opts), opts) - end - - @doc false - def all_recorded(job_or_wid), do: all_recorded(Oban, job_or_wid, []) - - @doc false - def all_recorded(job_or_wid, opts) when is_job_or_wid(job_or_wid) and is_list(opts) do - all_recorded(Oban, job_or_wid, opts) - end - - def all_recorded(oban_name, job_or_wid) when is_job_or_wid(job_or_wid) do - all_recorded(oban_name, job_or_wid, []) - end - - @doc """ - Get all recordings for workflow jobs, optionally filtered by name or relationship. - - The result is a map of the job's name and recorded output. Unrecorded jobs, or those without a - recording will be present in the map with a a `nil` value. - - ## Examples - - Retrieve results for all workflow jobs within a `process/1` function: - - def process(job) do - %{"a" => job_a_return, "b" => job_b_return} = Workflow.all_recorded(job) - - :ok - end - - Retrieve recordings for all of the current job's deps: - - recorded = Workflow.all_recorded(job, only_deps: true) - - Retrieve recordings for a list of dependencies: - - %{"a" => _, "b" => _} = Workflow.all_recorded(job, names: [:a, :b]) - - Retrieve recordings for some named jobs using the `workflow_id`: - - %{"a" => _, "b" => _} = Workflow.all_recorded("some-uuid-1234-5678", names: [:a, :b]) - - Retrieve recordings including results from grafted sub-workflows: - - recorded = Workflow.all_recorded(job, with_subs: true) - - Retrieve recordings for a specific grafted sub-workflow by name: - - %{"users" => user_results} = Workflow.all_recorded(job, names: [:users], with_subs: true) - """ - @doc since: "1.5.0" - @spec all_recorded(Oban.name(), job_or_wid(), fetch_opts()) :: map() - def all_recorded(oban_name, job_or_wid, opts) - - def all_recorded(_oban_name, %Job{conf: conf, meta: meta}, opts) do - %{"deps" => deps, "workflow_id" => workflow_id} = meta - - all_recorded(conf, workflow_id, deps, opts) - end - - def all_recorded(%Config{} = conf, workflow_id, opts) when is_binary(workflow_id) do - if Keyword.has_key?(opts, :only_deps) do - raise RuntimeError, ":only_deps isn't a valid option with a workflow_id" - end - - all_recorded(conf, workflow_id, [], opts) - end - - def all_recorded(oban_name, workflow_id, opts) when is_binary(workflow_id) do - oban_name - |> Oban.config() - |> all_recorded(workflow_id, [], opts) - end - - defp all_recorded(conf, workflow_id, deps, opts) when is_binary(workflow_id) do - {query_opts, opts} = Keyword.split(opts, ~w(names only_deps with_subs)a) - - with_subs = Keyword.get(query_opts, :with_subs, false) - - query = - workflow_id - |> workflow_query(deps, query_opts) - |> select([j], j.meta) - - conf - |> Repo.all(query, opts) - |> Enum.reject(&Map.get(&1, "context")) - |> Enum.reduce(%{}, fn meta, acc -> - case meta do - %{"name" => name, "sub_name" => work_name} when with_subs -> - acc - |> Map.put_new(work_name, %{}) - |> put_in([work_name, name], extract_recorded(meta)) - - %{"name" => name} -> - Map.put(acc, name, extract_recorded(meta)) - end - end) - end - - @doc false - def cancel_jobs(job_or_wid), do: cancel_jobs(Oban, job_or_wid, []) - - @doc false - def cancel_jobs(job_or_wid, opts) when is_job_or_wid(job_or_wid) and is_list(opts) do - cancel_jobs(Oban, job_or_wid, opts) - end - - def cancel_jobs(oban_name, job_or_wid) when is_job_or_wid(job_or_wid) do - cancel_jobs(oban_name, job_or_wid, []) - end - - @doc """ - Cancel one or more workflow jobs. - - This uses `Oban.cancel_all_jobs/2` internally, and adheres to the same cancellation rules. - Namely, it won't touch jobs in a `completed`, `cancelled`, or `discarded` state. - - Cancelling jobs via this function bulk-updates their state in the database without loading them - into memory or resolving their workers. As a result, **worker `after_process/3` hooks are not - called** for cancelled jobs. If you need to execute cleanup logic or other side effects when jobs - are cancelled, consider using workflow callbacks like `after_cancelled/2` or implement cleanup - logic at the application level. - - ## Examples - - Cancel jobs with a workflow `job` from a `process/1` function: - - def process(job) do - if should_stop_processing?(job.args) do - Workflow.cancel_jobs(job) - else - ... - end - end - - Cancel specific workflow jobs: - - Workflow.cancel_jobs("some-uuid-1234-5678", names: [:a, :b]) - - Cancel all jobs in a workflow by `workflow_id`: - - Workflow.cancel_jobs("some-uuid-1234-5678") - - Cancel jobs from anywhere with a custom Oban instance name: - - Workflow.cancel_jobs(MyApp.Oban, "some-uuid-1234-5678") - """ - @doc since: "1.5.0" - @spec cancel_jobs(Oban.name(), job_or_wid(), cancel_opts()) :: {:ok, integer()} - def cancel_jobs(oban_name, job_or_wid, opts) - - def cancel_jobs(_oban_name, %Job{conf: conf, meta: %{"workflow_id" => wid}}, opts) do - cancel_jobs(conf.name, wid, opts) - end - - def cancel_jobs(oban_name, wid, opts) when is_binary(wid) do - query_opts = - opts - |> Keyword.take([:names, :with_subs]) - |> Keyword.put_new(:with_subs, true) - - Oban.cancel_all_jobs(oban_name, workflow_query(wid, [], query_opts)) - end - - @doc """ - Retry all jobs in a workflow. - - By default, this function will retry all jobs that aren't `executing` or `available`. Jobs with - dependencies will be placed on hold to run again once their dependencies are complete. - - ## Examples - - Retry only failed jobs in a workflow from within a process function: - - def process(job) do - Workflow.retry_jobs(job) - end - - Retry a workflow by ID: - - Workflow.retry_jobs("some-uuid-1234-5678") - - Retry a workflow with a custom Oban instance: - - Workflow.retry_jobs(MyApp.Oban, "some-uuid-1234-5678") - """ - @doc since: "1.6.0" - @spec retry_jobs(Oban.name(), job_or_wid(), keyword()) :: {:ok, integer()} - def retry_jobs(oban_name \\ Oban, job_or_wid, opts \\ []) - - def retry_jobs(_oban_name, %Job{conf: conf, meta: %{"workflow_id" => wid}}, opts) do - retry_jobs(conf.name, wid, opts) - end - - def retry_jobs(oban_name, wid, opts) when is_binary(wid) do - conf = Oban.config(oban_name) - - Repo.transaction(conf, fn -> - jobs = all_jobs(oban_name, wid, opts) - - {with_deps, wout_deps} = - jobs - |> Enum.reject(& &1.meta["context"]) - |> Enum.split_with(fn %{meta: meta} -> Enum.any?(meta["deps"]) end) - - base = where(Job, [j], j.state not in ~w(available executing)) - - if Enum.any?(wout_deps) do - query = - base - |> where([j], j.id in ^Enum.map(wout_deps, & &1.id)) - |> update([j], - set: [ - state: "available", - max_attempts: fragment("GREATEST(?, ? + 1)", j.max_attempts, j.attempt), - scheduled_at: ^DateTime.utc_now(), - completed_at: nil, - cancelled_at: nil, - discarded_at: nil - ] - ) - - Repo.update_all(conf, query, []) - end - - if Enum.any?(with_deps) do - query = - base - |> where([j], j.id in ^Enum.map(with_deps, & &1.id)) - |> update([j], - set: [ - state: "scheduled", - max_attempts: fragment("GREATEST(?, ? + 1)", j.max_attempts, j.attempt), - meta: fragment("(? || ?) - 'orig_scheduled_at'", j.meta, ^%{on_hold: true}), - scheduled_at: ^DateTime.utc_now(), - completed_at: nil, - cancelled_at: nil, - discarded_at: nil - ] - ) - - Repo.update_all(conf, query, []) - end - - length(jobs) - end) - end - - @doc """ - Get a single workflow job by name. - - ## Examples - - Get the workflow job named `:step_1` from within `process/`: - - def process(job) do - case Workflow.get_job(job, :step_1) do - nil -> ... - dep_job -> use_the_job(dep_job) - end - end - - Get a named workflow job with the `workflow_id`: - - Workflow.get_job("some-uuid-1234-5678", :step_2) - - Get a named workflow job with the `workflow_id` and a custom Oban instance name: - - Workflow.get_job(MyApp.Oban, "some-uuid-1234-5678", :step_2) - """ - @doc since: "1.5.0" - @spec get_job(Oban.name(), job_or_wid(), name()) :: nil | Job.t() - def get_job(oban_name \\ Oban, job_or_wid, deps_name) - - def get_job(_oban_name, %Job{conf: conf, meta: %{"workflow_id" => wid}}, deps_name) do - get_job(conf.name, wid, deps_name) - end - - def get_job(oban_name, wid, deps_name) when is_binary(wid) do - case all_jobs(oban_name, wid, names: [deps_name]) do - [job | _] -> job - [] -> nil - end - end - - @doc """ - Fetch the recorded output from a workflow job. - - This function provides an optimized way to get recorded output with a single query. - - A `nil` value is returned when a dependency is incomplete, missing, unrecorded, or lacks - recorded output. - - ## Examples - - Get the recorded output from `:step_1` within `process/`: - - def process(job) do - case Workflow.get_recorded(job, :step_1) do - nil -> ... - val -> use_recorded_output(val) - end - end - - Get recorded output using the workflow id: - - Workflow.get_recorded("some-uuid-1234-5678", :step_2) - - Get recorded output using the `workflow_id` and a custom Oban instance name: - - Workflow.get_recorded(MyApp.Oban, "some-uuid-1234-5678", :step_2) - """ - @doc since: "1.5.0" - @spec get_recorded(Oban.name(), job_or_wid(), name()) :: any() - def get_recorded(oban_name \\ Oban, job_or_wid, deps_name) - - def get_recorded(_oban_name, %Job{conf: conf, meta: %{"workflow_id" => wid}}, deps_name) do - get_recorded(conf, wid, deps_name) - end - - def get_recorded(%Config{} = conf, wid, deps_name) when is_binary(wid) do - query = - wid - |> workflow_query([], names: [deps_name]) - |> select([j], j.meta) - - conf - |> Repo.one(query) - |> extract_recorded() - end - - def get_recorded(oban_name, wid, deps_name) when is_binary(wid) do - oban_name - |> Oban.config() - |> get_recorded(wid, deps_name) - end - - @doc false - def stream_jobs(job_or_wid), do: stream_jobs(Oban, job_or_wid, []) - - @doc false - def stream_jobs(job_or_wid, opts) when is_job_or_wid(job_or_wid) and is_list(opts) do - stream_jobs(Oban, job_or_wid, opts) - end - - def stream_jobs(oban_name, job_or_wid) when is_job_or_wid(job_or_wid) do - stream_jobs(oban_name, job_or_wid, []) - end - - @doc """ - Stream all jobs for a workflow. - - This function behaves identically to `all_jobs/3`, except it streams jobs lazily from within a - `Repo` transaction. - - ## Examples - - Stream with filtering to only preserve `completed` jobs: - - def process(job) do - {:ok, workflow_jobs} = - MyApp.Repo.transaction(fn -> - job - |> Workflow.stream_jobs() - |> Stream.filter(& &1.state == "completed") - |> Enum.to_list() - end) - - do_things_with_jobs(workflow_jobs) - - :ok - end - - Stream workflow jobs from anywhere using the `workflow_id`: - - MyApp.Repo.transaction(fn -> - "some-uuid-1234-5678" - |> Workflow.stream_jobs() - |> Enum.map(& &1.args["account_id"]) - end) - - Stream workflow jobs using a custom Oban instance name: - - MyApp.Repo.transaction(fn -> - MyApp.Oban - |> Workflow.stream_jobs("some-uuid-1234-5678") - |> Enum.map(& &1.args["account_id"]) - end) - """ - @doc since: "1.5.0" - @spec stream_jobs(Oban.name(), Job.t() | String.t(), fetch_opts()) :: Enum.t() - def stream_jobs(oban_name, job_or_wid, opts) - - def stream_jobs(_oban_name, %Job{conf: conf, meta: meta}, opts) do - %{"deps" => deps, "workflow_id" => wid} = meta - - {query_opts, opts} = Keyword.split(opts, [:names, :only_deps]) - - Repo.stream(conf, workflow_query(wid, deps, query_opts), opts) - end - - def stream_jobs(oban_name, wid, opts) when is_binary(wid) do - if Keyword.has_key?(opts, :only_deps) do - raise RuntimeError, ":only_deps isn't a valid option with a workflow_id" - end - - {query_opts, opts} = Keyword.split(opts, [:names]) - - oban_name - |> Oban.config() - |> Repo.stream(workflow_query(wid, [], query_opts), opts) - end - - @doc """ - Get detailed status information about a workflow. - - Returns a map with the following fields: - - - `:id` - The workflow's unique id - - `:name` - The name of the workflow, if it has one - - `:total` - Total number of jobs in the workflow - - `:state` - Overall status of the workflow (e.g. executing, completed, cancelled, discarded) - - `:counts` - Map of job state counts (e.g. `%{completed: 10, available: 2}`) - - `:duration` - Elapsed time of the workflow in milliseconds (nil if not yet ran) - - `:started_at` - When the first job in the workflow was inserted - - `:stopped_at` - When the last job in the workflow finished - - `:subs` - A map of status output for sub-worfklows where the key is the sub name and the value - is a `status/1` output - - An example of the full output: - - ```elixir - %{ - id: "some-uuid-1234-5678", - name: "my-workflow", - total: 3, - state: :completed, - duration: 72, - counts: %{ - cancelled: 0, - available: 0, - completed: 3, - scheduled: 0, - discarded: 0, - executing: 0, - retryable: 0 - }, - started_at: ~U[2025-04-10 16:34:34.114730Z], - stopped_at: ~U[2025-04-10 16:34:34.186935Z], - subs: %{} - } - ``` - - ## Examples - - Get a workflow's status using a job: - - status = Workflow.status(job) - - Get a workflow's status using the workflow_id: - - status = Workflow.status("some-uuid-1234-5678") - - Get a workflow's status using a custom Oban instance name: - - status = Workflow.status(MyApp.Oban, "some-uuid-1234-5678") - """ - @doc since: "1.6.0" - @spec status(Oban.name(), job_or_wid()) :: status() - def status(oban_name \\ Oban, job_or_wid) - - def status(_oban_name, %Job{conf: conf, meta: %{"workflow_id" => wid}}) do - status(conf, wid) - end - - def status(%Config{} = conf, workflow_id) when is_binary(workflow_id) do - query = - Job - |> where([j], in_workflow(j.meta, ^workflow_id)) - |> group_by([j], [j.meta["workflow_id"], j.meta["workflow_name"]]) - |> select([j], %{ - id: j.meta["workflow_id"], - name: j.meta["workflow_name"], - ignore_cancelled: fragment("array_agg(?)", j.meta["ignore_cancelled"]), - ignore_discarded: fragment("array_agg(?)", j.meta["ignore_discarded"]), - states: fragment("array_agg(?)", j.state), - min_scheduled: min(j.inserted_at), - max_completed: max(j.completed_at), - max_cancelled: max(j.cancelled_at), - max_discarded: max(j.discarded_at) - }) - - case Repo.one(conf, query) do - nil -> - %{ - id: workflow_id, - name: nil, - total: 0, - state: :unknown, - counts: Map.new(Job.states(), &{&1, 0}), - duration: nil, - started_at: nil, - stopped_at: nil, - subs: %{} - } - - result -> - expand_status(conf, workflow_id, result) - end - end - - def status(oban_name, workflow_id) when is_binary(workflow_id) do - oban_name - |> Oban.config() - |> status(workflow_id) - end - - defp expand_status(conf, workflow_id, result) do - total = length(result.states) - empty = Map.new(Job.states(), &{&1, 0}) - - counts = - [result.states, result.ignore_cancelled, result.ignore_discarded] - |> Enum.zip() - |> Enum.reduce(empty, fn tuple, acc -> - state = - case tuple do - {"cancelled", true, _} -> :completed - {"discarded", _, true} -> :completed - {state, _ican, _idisc} -> String.to_existing_atom(state) - end - - Map.update(acc, state, 1, &(&1 + 1)) - end) - - state = - cond do - counts.completed + counts.cancelled + counts.discarded < total -> :executing - counts.cancelled > 0 -> :cancelled - counts.discarded > 0 -> :discarded - true -> :completed - end - - stopped_at = - [result.max_completed, result.max_cancelled, result.max_discarded] - |> Enum.reject(&is_nil/1) - |> Enum.max(fn -> nil end) - - duration = - if stopped_at do - DateTime.diff(stopped_at, result.min_scheduled, :millisecond) - else - nil - end - - subs = - Job - |> where([j], fragment("? \\? 'sup_workflow_id'", j.meta)) - |> where([j], fragment("?->>'sup_workflow_id' = ?", j.meta, ^workflow_id)) - |> select([j], %{wid: j.meta["workflow_id"], name: j.meta["sub_name"]}) - |> distinct(true) - |> then(&Repo.all(conf, &1)) - |> Map.new(fn %{wid: wid, name: name} -> {name, status(conf, wid)} end) - - %{ - id: result.id, - name: result.name, - total: total, - state: state, - counts: counts, - duration: duration, - started_at: result.min_scheduled, - stopped_at: stopped_at, - subs: subs - } - end - - @doc """ - Converts the given workflow to a `:digraph`. - - The resulting digraph can be explored, evaluated, and then be converted to a number of other - formats. - - ## Examples - - Generate a digraph from a workflow: - - graph = Workflow.to_graph(workflow) - - Check the path between jobs in a workflow: - - workflow - |> Workflow.to_graph() - |> :digraph.get_path("step_1", "step_5") - """ - @doc since: "1.5.0" - @spec to_graph(flow :: t()) :: :digraph.graph() - def to_graph(%__MODULE__{changesets: changesets, subs: subs}) do - graph = :digraph.new([:acyclic]) - - changesets - |> Enum.group_by(& &1.changes.meta[:sub_name]) - |> Enum.each(fn - {nil, changesets} -> - add_graph_nodes(graph, changesets) - - {sub, changesets} -> - sub_graph = - [:acyclic] - |> :digraph.new() - |> add_graph_nodes(changesets) - - :digraph.add_vertex(graph, sub, sub_graph) - end) - - for %{changes: %{meta: meta}} <- changesets, dep <- meta.deps do - # This would also create edges in the top graph for values that shouldn't be there. - case dep do - [wid, "*"] -> - sub_name = Enum.find_value(subs, &if(elem(&1, 1) == wid, do: elem(&1, 0))) - dep_name = meta[:sub_name] || meta[:name] - - :digraph.add_edge(graph, sub_name, dep_name) - - [_id, dep] -> - :digraph.add_edge(graph, dep, meta.sub_name) - - dep -> - :digraph.add_edge(graph, dep, meta.name) - end - end - - graph - end - - defp add_graph_nodes(graph, changesets) do - for %{changes: %{meta: meta, worker: worker}} <- changesets do - label = Map.get(meta, :decorated_name, worker) - - :digraph.add_vertex(graph, meta.name, label) - end - - graph - end - - @doc """ - Converts the given workflow to a [graphviz](https://graphviz.org/) `dot` digraph. - - ## Examples - - Generate a dot digraph: - - Workflow.to_dot(workflow) - """ - @doc since: "1.5.0" - @spec to_dot(flow :: t()) :: String.t() - def to_dot(%__MODULE__{} = workflow) do - graph = to_graph(workflow) - nodes = :digraph.vertices(graph) - edges = :digraph.edges(graph) - - nodes_dot = visit_nodes(nodes, graph, :dot) - edges_dot = visit_edges(edges, graph, :dot) - - """ - digraph { - node [shape=box]; - #{nodes_dot} - #{edges_dot} - } - """ - end - - @doc """ - Generate a [Mermaid](https://mermaid.js.org/) flowchart in top-down orientation from a workflow. - - ## Examples - - Generate a flowchart: - - Workflow.to_mermaid(workflow) - """ - @doc since: "1.5.0" - @spec to_mermaid(flow :: t()) :: String.t() - def to_mermaid(%__MODULE__{} = workflow) do - graph = to_graph(workflow) - nodes = :digraph.vertices(graph) - edges = :digraph.edges(graph) - - nodes_mermaid = visit_nodes(nodes, graph, :mermaid) - edges_mermaid = visit_edges(edges, graph, :mermaid) - - """ - flowchart TD - #{nodes_mermaid} - #{edges_mermaid} - """ - end - - @doc false - def rescue_workflows(conf, opts \\ []) do - {limit, opts} = Keyword.pop(opts, :limit, 1_000) - - query = - Job - |> where([j], j.state == "scheduled") - |> where([j], fragment("? \\? 'workflow_id'", j.meta)) - |> where([j], fragment("?->>'on_hold' = 'true'", j.meta)) - |> select([j], fragment("?->>'workflow_id'", j.meta)) - |> distinct(true) - |> limit(^limit) - - conf - |> Repo.all(query, opts) - |> on_flush(conf) - end - - # Construction Helpers - - defp prevent_chain!(changeset, name) do - if Chain.chain?(changeset) do - raise "#{inspect(name)} is a chain, which isn't allowed in a workflow" - end - end - - defp prevent_dupe!(%{names: names, subs: subs}, name) do - if MapSet.member?(names, name) or is_map_key(subs, name) do - raise "#{inspect(name)} is already a member of the workflow" - end - end - - defp confirm_deps!(%{check_deps: true, names: names, grafts: grafts}, [_ | _] = deps) do - Enum.each(deps, fn - [_wd, "*"] -> - :ok - - [_wd, name] -> - if not MapSet.member?(names, name) do - raise "deps #{inspect(name)} is not a member of the workflow" - end - - name when is_binary(name) -> - if not (MapSet.member?(names, name) or is_map_key(grafts, name)) do - raise "deps #{inspect(name)} is not a member of the workflow" - end - end) - end - - defp confirm_deps!(_workflow, _deps), do: :ok - - # Visualization Helpers - - defp visit_nodes(nodes, graph, mode, prefix \\ nil) do - Enum.map_join(nodes, "\n", fn node -> - {_node, label} = :digraph.vertex(graph, node) - - if is_tuple(label) do - sub_nodes = visit_nodes(:digraph.vertices(label), label, mode, "#{node}_") - sub_edges = visit_edges(:digraph.edges(label), label, mode) - - format_subg(mode, node, sub_nodes, sub_edges) - else - format_node(mode, "#{prefix}#{node}", label) - end - end) - end - - defp visit_edges(edges, graph, mode) do - edges - |> Enum.map(fn edge -> - {_, in_vert, out_vert, _} = :digraph.edge(graph, edge) - - {in_vert, out_vert} - end) - |> Enum.uniq() - |> Enum.map_join("\n ", &format_edge(mode, &1)) - end - - defp format_node(:dot, node, label) do - ~s|"#{node}" [label="#{node} (#{label})"];| - end - - defp format_node(:mermaid, node, label) do - ~s|#{node}["#{node} (#{label})"]| - end - - defp format_edge(:dot, {in_vert, out_vert}) do - ~s|"#{in_vert}" -> "#{out_vert}";| - end - - defp format_edge(:mermaid, {in_vert, out_vert}) do - "#{in_vert} --> #{out_vert}" - end - - defp format_subg(:dot, node, sub_nodes, sub_edges) do - """ - subgraph "cluster_#{node}" { - label="#{node}"; - #{sub_nodes} - #{sub_edges} - } - """ - end - - defp format_subg(:mermaid, node, sub_nodes, sub_edges) do - """ - subgraph #{node} - #{sub_nodes} - #{sub_edges} - end - """ - end - - # Query Helpers - - defp workflow_query(wid, deps, opts) do - query = - Job - |> where([j], j.state in @all_states) - |> order_by(asc: :id) - - query = - if opts[:with_subs] do - where(query, [j], in_workflow_or_sup(j.meta, ^wid)) - else - where(query, [j], in_workflow(j.meta, ^wid)) - end - - cond do - opts[:with_subs] && is_list(opts[:names]) -> - names = Enum.map(opts[:names], &to_string/1) - - where( - query, - [j], - fragment("coalesce(?->>'sub_name', ?->>'name')", j.meta, j.meta) in ^names - ) - - is_list(opts[:names]) -> - names = Enum.map(opts[:names], &to_string/1) - - where(query, [j], j.meta["name"] in ^names) - - opts[:with_subs] && opts[:only_deps] -> - where( - query, - [j], - fragment("coalesce(?->>'sub_name', ?->>'name')", j.meta, j.meta) in ^deps - ) - - opts[:only_deps] -> - where(query, [j], j.meta["name"] in ^deps) - - true -> - query - end - end - - defp to_stage_operation({%{meta: meta} = job, deps_states}) do - # A grafted sub-workflow won't have jobs until the grafter runs. Ignore the "deleted" check - # for the sub-workflow and rely entirely on the grafter job itself. - deps = Enum.reject(meta["deps"], &match?(["grafted_" <> _, "*"], &1)) - - deps_set = - if length(deps_states) < length(deps) do - MapSet.new(["deleted" | deps_states]) - else - MapSet.new(deps_states) - end - - op = - cond do - MapSet.member?(deps_set, "deleted") and meta["ignore_deleted"] != true -> - {:cancelled, :deleted} - - MapSet.member?(deps_set, "cancelled") and meta["ignore_cancelled"] != true -> - {:cancelled, :cancelled} - - MapSet.member?(deps_set, "discarded") and meta["ignore_discarded"] != true -> - {:cancelled, :discarded} - - not MapSet.disjoint?(deps_set, @incomplete_set) -> - :ignore - - meta["context"] == true -> - :ignore - - is_legacy_scheduled(meta) -> - {:scheduled, DateTime.from_unix!(meta["orig_scheduled_at"])} - - is_map_key(meta, "orig_scheduled_at") -> - {:scheduled, DateTime.from_unix!(meta["orig_scheduled_at"], :microsecond)} - - true -> - :available - end - - {op, job} - end - - defp apply_operations({:ignore, _}, _), do: [] - - defp apply_operations({op, jobs}, conf) do - now = DateTime.utc_now() - ids = Enum.map(jobs, & &1.id) - - base = - Job - |> where([j], j.id in ^ids and j.state == "scheduled") - |> update([j], set: [meta: drop_hold(j.meta)]) - - query = - case op do - :available -> - base - |> select([j], nil) - |> update([j], set: [scheduled_at: ^now, state: "available"]) - - {:scheduled, at} -> - base - |> select([j], nil) - |> update([j], set: [scheduled_at: ^at, state: "scheduled"]) - - {:cancelled, reason} -> - error = %{ - kind: :error, - reason: Oban.Pro.WorkflowError.exception(reason), - stacktrace: [] - } - - formatted = Job.format_attempt(%Job{attempt: 1, unsaved_error: error}) - - base - |> select([j], j.meta["workflow_id"]) - |> update([j], - set: [ - cancelled_at: ^now, - state: "cancelled", - errors: concat_errors(j.errors, ^[formatted]) - ] - ) - end - - {_count, workflow_ids} = Repo.update_all(conf, query, set: []) - - with {:cancelled, reason} <- op, do: Enum.each(jobs, &apply_callback(reason, &1, conf)) - - workflow_ids - |> Enum.reject(&is_nil/1) - |> Enum.uniq() - end - - defp apply_callback(reason, job, conf) do - with {:ok, worker} <- Oban.Worker.from_string(job.worker), - true <- function_exported?(worker, :after_cancelled, 2), - {:ok, job} <- Oban.Pro.Worker.before_process(worker, job, worker.__opts__()) do - worker.after_cancelled(reason, %{job | conf: conf}) - end - catch - kind, value -> - Logger.error(fn -> - "[Oban.Pro.Workflow] callback error: " <> Exception.format(kind, value, __STACKTRACE__) - end) - - :ok - end - - # Recorded Helpers - - defp extract_recorded(meta) do - case meta do - %{"recorded" => true, "return" => value, "safe_decode" => true} -> - Utils.decode64(value, [:safe]) - - %{"recorded" => true, "return" => value} -> - Utils.decode64(value, []) - - _ -> - nil - end - end -end diff --git a/vendor/oban_pro/lib/oban/pro/workflow/cascade.ex b/vendor/oban_pro/lib/oban/pro/workflow/cascade.ex deleted file mode 100644 index 997614ba..00000000 --- a/vendor/oban_pro/lib/oban/pro/workflow/cascade.ex +++ /dev/null @@ -1,87 +0,0 @@ -defmodule Oban.Pro.Workflow.Cascade do - @moduledoc false - - use Oban.Pro.Worker, recorded: [limit: 256_000] - - alias Oban.Pro.Workflow - - args_schema do - field :mod, :string - field :fun, :string - field :arg, :term, default: :none - end - - @impl Oban.Pro.Worker - def process(%{args: %{arg: arg, mod: mod, fun: fun}, conf: conf, meta: meta}) do - mod = - mod - |> String.split(".") - |> Module.safe_concat() - - fun = String.to_existing_atom(fun) - - full = Map.merge(all_context(conf, meta), all_recorded(conf, meta)) - args = if arg == :none, do: [full], else: [arg, full] - - case apply(mod, fun, args) do - {:ok, _} = result -> result - {:error, _} = error -> error - {:snooze, _} = snooze -> snooze - {:cancel, _} = cancel -> cancel - other -> {:ok, other} - end - end - - defp all_context(conf, meta) do - sup = Map.get(meta, "sup_workflow_id") - wid = Map.get(meta, "workflow_id") - ancestors = Map.get(meta, "ancestor_ids", []) - - (Enum.reverse(ancestors) ++ [sup, wid]) - |> Enum.filter(& &1) - |> Enum.uniq() - |> Enum.reduce(%{}, fn id, acc -> - Map.merge(acc, Workflow.get_context(conf, id) || %{}) - end) - end - - defp all_recorded(_conf, %{"deps" => []}), do: %{} - - defp all_recorded(conf, %{"deps" => deps, "workflow_id" => wid} = meta) do - atomize? = Map.get(meta, "atom_keys", true) - - grouper = fn - [sup, name], acc -> Map.update(acc, sup, [name], &[name | &1]) - name, acc -> Map.update(acc, wid, [name], &[name | &1]) - end - - deps - |> Enum.reject(&match?("grafter_" <> _, &1)) - |> Enum.reduce(%{}, grouper) - |> Enum.reduce(%{}, fn {dep_wid, names}, acc -> - filter = if names == ["*"], do: [], else: [names: names] - filter = if dep_wid != wid, do: Keyword.put(filter, :with_subs, true), else: filter - - recorded = - conf - |> Workflow.all_recorded(dep_wid, filter) - |> then(fn map -> if atomize?, do: atomize_map(map), else: map end) - - Map.merge(acc, recorded) - end) - end - - defp atomize_map(map) when is_map(map) and not is_struct(map) do - Map.new(map, fn {key, val} -> {atomize_key(key), val} end) - end - - defp atomize_map(val), do: val - - defp atomize_key(key) when is_binary(key) do - String.to_existing_atom(key) - rescue - ArgumentError -> key - end - - defp atomize_key(key), do: key -end diff --git a/vendor/oban_pro/lib/pro.ex b/vendor/oban_pro/lib/pro.ex deleted file mode 100644 index 7e0248e0..00000000 --- a/vendor/oban_pro/lib/pro.ex +++ /dev/null @@ -1,3 +0,0 @@ -defmodule Oban.Pro do - @moduledoc false -end diff --git a/vendor/oban_pro/mix.exs b/vendor/oban_pro/mix.exs deleted file mode 100644 index 923203a1..00000000 --- a/vendor/oban_pro/mix.exs +++ /dev/null @@ -1,203 +0,0 @@ -defmodule Oban.Pro.MixProject do - use Mix.Project - - @version "1.6.14" - - def project do - [ - app: :oban_pro, - version: @version, - elixir: "~> 1.15", - start_permanent: Mix.env() == :prod, - elixirc_paths: elixirc_paths(Mix.env()), - prune_code_paths: false, - deps: deps(), - docs: docs(), - aliases: aliases(), - package: package(), - name: "Oban Pro", - description: "Oban Pro Component", - xref: [exclude: [Oban.JSON, Oban.Validation]], - - # Dialyzer - dialyzer: [ - plt_add_apps: [:ex_unit, :mix, :postgrex], - plt_core_path: "_build/#{Mix.env()}", - flags: [:error_handling, :missing_return, :no_opaque, :underspecs] - ] - ] - end - - def application do - [ - extra_applications: [:logger], - mod: {Oban.Pro.Application, []} - ] - end - - def cli do - [ - preferred_envs: [ - precommit: :test, - "test.ci": :test, - "test.reset": :test, - "test.rollback": :test, - "test.setup": :test - ] - ] - end - - def package do - [ - organization: "oban", - files: ~w(lib/pro.ex lib/oban .formatter.exs mix.exs), - licenses: ["Commercial"], - links: [] - ] - end - - defp elixirc_paths(:test), do: ["lib", "test/support"] - defp elixirc_paths(_env), do: ["lib"] - - defp deps do - [ - {:oban, "~> 2.19"}, - {:ecto_sql, "~> 3.10"}, - {:postgrex, "~> 0.16", optional: true}, - - # Test and Dev - {:stream_data, "~> 1.1", only: [:test, :dev]}, - {:tz, "~> 0.26", only: [:test, :dev]}, - {:benchee, "~> 1.0", only: [:test, :dev], runtime: false}, - {:credo, "~> 1.7.7-rc.0", only: [:test, :dev], runtime: false}, - {:dialyxir, "~> 1.0", only: [:test, :dev], runtime: false}, - - # Dev - {:ex_doc, "~> 0.21", only: :dev, runtime: false}, - {:makeup_diff, "~> 0.1", only: :dev, runtime: false}, - {:lys_publish, "~> 0.1", only: :dev, path: "../lys_publish", optional: true, runtime: false} - ] - end - - defp aliases do - [ - release: [ - "cmd git tag v#{@version}", - "cmd git push", - "cmd git push --tags", - "hex.publish package --yes", - "lys.publish" - ], - "test.reset": ["ecto.drop -r Oban.Pro.Repo --quiet", "test.setup"], - "test.rollback": ["ecto.rollback -r Oban.Pro.Repo --all --quiet"], - "test.setup": [ - "ecto.create -r Oban.Pro.Repo --quiet", - "ecto.migrate -r Oban.Pro.Repo --quiet" - ], - "test.ci": [ - "format --check-formatted", - "deps.unlock --check-unused", - "credo --strict", - "test --raise", - "dialyzer" - ], - precommit: [ - "format --check-formatted", - "deps.unlock --check-unused", - "credo --strict", - "test --raise" - ] - ] - end - - defp docs do - [ - main: "overview", - source_ref: "v#{@version}", - formatters: ["html"], - api_reference: false, - extra_section: "GUIDES", - extras: extras(), - groups_for_extras: groups_for_extras(), - groups_for_modules: groups_for_modules(), - homepage_url: "/", - logo: "assets/oban-pro-logo.svg", - assets: %{"assets" => "assets"}, - nest_modules_by_prefix: nest_modules_by_prefix(), - skip_undefined_reference_warnings_on: ["CHANGELOG.md"], - before_closing_body_tag: fn _ -> - """ - - """ - end - ] - end - - defp extras do - [ - "guides/introduction/overview.md", - "guides/introduction/installation.md", - "guides/introduction/adoption.md", - "guides/introduction/composition.md", - "guides/testing/testing.md", - "guides/testing/testing_workers.md", - "guides/deployment/ci_cd.md", - "guides/deployment/docker.md", - "guides/deployment/fly.md", - "guides/deployment/gigalixir.md", - "guides/deployment/heroku.md", - "guides/deployment/dependabot.md", - "guides/upgrading/v1.0.md", - "guides/upgrading/v1.4.md", - "guides/upgrading/v1.5.md", - "guides/upgrading/v1.6.md", - "CHANGELOG.md": [filename: "changelog", title: "Changelog"] - ] - end - - defp groups_for_extras do - [ - Introduction: ~r/guides\/introduction\/.?/, - Testing: ~r/guides\/testing\/.?/, - Deployment: ~r/guides\/deployment\/.?/, - Upgrading: ~r/guides\/upgrading\/.?/ - ] - end - - defp groups_for_modules do - [ - Extensions: [ - Oban.Pro.Decorator, - Oban.Pro.Relay, - Oban.Pro.Testing, - Oban.Pro.Worker - ], - Composition: [ - Oban.Pro.Batch, - Oban.Pro.Workflow - ], - Plugins: [ - Oban.Pro.Plugins.DynamicCron, - Oban.Pro.Plugins.DynamicLifeline, - Oban.Pro.Plugins.DynamicPartitioner, - Oban.Pro.Plugins.DynamicPrioritizer, - Oban.Pro.Plugins.DynamicPruner, - Oban.Pro.Plugins.DynamicQueues, - Oban.Pro.Plugins.DynamicScaler - ], - Scaling: [ - Oban.Pro.Cloud - ], - Workers: [ - Oban.Pro.Workers.Batch, - Oban.Pro.Workers.Chain, - Oban.Pro.Workers.Chunk, - Oban.Pro.Workers.Workflow - ] - ] - end - - defp nest_modules_by_prefix do - [Oban.Pro, Oban.Pro.Plugins, Oban.Pro.Workers] - end -end