graham/prop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

deps: re-vendor oban_pro 1.7.0 (revert hex-repo dep)

Browse source 
Previous commit (3c988a5f) switched oban_pro to the licensed hex
repo at deploy time. Reverting that — keep all three Pro packages
vendored so prod image builds don't depend on oban.pro reachability
or auth on every CI run.

oban_pro 1.7.0 dropped into vendor/oban_pro via
`mix hex.package fetch oban_pro 1.7.0 --repo=oban --unpack`. mix.exs
goes back to `path: "vendor/oban_pro"` (matching oban_met / oban_web,
which were already vendored — and stay vendored since the licensed
repo only has older versions of those: 0.1.11 / 2.10.6 vs the
1.1.0 / 2.12.1 we vendor).

Schema migration from 3c988a5f stays — 1.7.0 tables/indexes are
already applied. No code changes.
This commit is contained in:
Graham McIntire 2026-04-30 09:29:55 -05:00
parent 3c988a5ff3
commit e99bf06eb4
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
76 changed files with 21734 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
mix.exs
View file

@ -84,11 +84,13 @@ defmodule Microwaveprop.MixProject do
{:bandit, "~> 1.5"},
{:styler, "~> 1.11", only: [:dev, :test], runtime: false},
{:oban, "~> 2.21"},
# All three Oban Pro packages stay vendored so production deploys
# don't have to authenticate against the licensed oban.pro hex
# repo on every image build. Bumping a version is a fetch-and-
# extract step (`mix hex.package fetch oban_pro <ver> --repo=oban
# --unpack`) into vendor/, then commit.
{:oban_met, "~> 1.0", path: "vendor/oban_met", override: true},
# Oban Pro 1.7+ pulled from the licensed `oban` hex repo
# (configured via `mix hex.repo`). Pre-1.7 was vendored;
# vendor/oban_pro was removed when this bump landed.
{:oban_pro, "~> 1.7.0", repo: "oban"},
{:oban_pro, "~> 1.7", path: "vendor/oban_pro"},
{:oban_web, "~> 2.12", path: "vendor/oban_web", override: true},
{:credo, "~> 1.7", only: [:dev, :test], runtime: false},
{:nx, "~> 0.9", only: [:dev, :test]},

1
mix.lock
View file

@ -42,7 +42,6 @@
"nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"},
"nx": {:hex, :nx, "0.11.0", "d37723dbd6cfa274a5def6d6664f5680c32e2eb8a1ce25ec6d91751967fa0abf", [:mix], [{:complex, "~> 0.6", [hex: :complex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "36157b21239aeb251d6cbac23eb0eb3495a5e1e0cbc2e6df16afd2ede1575205"},
"oban": {:hex, :oban, "2.22.0", "d50fb8be22a58ca8839dbc69610e271773d851b7deae01acd8b62e7197bcd318", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.20", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6675164c11eadd82044732dd7b6e448aa6d6f429a41a1e7d6dbf5fd358f52f26"},
"oban_pro": {:hex, :oban_pro, "1.7.0", "e501c3957540dbd4b2d69c9045f04f4bd451d4f54d7cf185bf22c8c8cc56323c", [:mix], [{:ecto_sql, "~> 3.12", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:oban, "~> 2.21", [hex: :oban, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.22", [hex: :postgrex, repo: "hexpm", optional: true]}], "oban", "9e02b1abe356ee2c382d4a3606a59386158560c6ceab9aa95019d39363cde46a"},
"octo_fetch": {:hex, :octo_fetch, "0.5.0", "f50701568b9fc752656367f82cc134d5fbefff37c5a0e8ddfcceb02ceee3f5fc", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "6226cc3c14ca948ee9f25fb0446322e5c288e215da9beba7899b6b5f4cd3ccb0"},
"owl": {:hex, :owl, "0.13.0", "26010e066d5992774268f3163506972ddac0a7e77bfe57fa42a250f24d6b876e", [:mix], [{:ucwidth, "~> 0.2", [hex: :ucwidth, repo: "hexpm", optional: true]}], "hexpm", "59bf9d11ce37a4db98f57cb68fbfd61593bf419ec4ed302852b6683d3d2f7475"},
"peep": {:hex, :peep, "3.5.0", "9f6ead7b0f2c684494200c8fc02e7e62e8c459afe861b29bd859e4c96f402ed8", [:mix], [{:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:plug, "~> 1.16", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry_metrics, "~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "5a73a99c6e60062415efeb7e536a663387146463a3d3df1417da31fd665ac210"},

19
vendor/oban_pro/.formatter.exs vendored Normal file
View file

@ -0,0 +1,19 @@
locals_without_parens = [
args_schema: 1,
field: 2,
field: 3,
embeds_one: 2,
embeds_many: 2
]
[
import_deps: [:ecto, :ecto_sql, :oban, :stream_data],
export: [locals_without_parens: locals_without_parens],
locals_without_parens: locals_without_parens,
inputs: [
"{mix,.formatter}.exs",
"{config,lib}/**/*.{ex,exs}",
"test/**/*.{exs}",
"test/support/**/*.ex"
]
]

75
vendor/oban_pro/hex_metadata.config vendored Normal file
View file

@ -0,0 +1,75 @@
{<<"links">>,[]}.
{<<"name">>,<<"oban_pro">>}.
{<<"version">>,<<"1.7.0">>}.
{<<"description">>,<<"Oban Pro Component">>}.
{<<"elixir">>,<<"~> 1.15">>}.
{<<"files">>,
[<<"lib/oban">>,<<"lib/oban/pro">>,<<"lib/oban/pro/diagnostics.ex">>,
<<"lib/oban/pro/cron.ex">>,<<"lib/oban/pro/decorator.ex">>,
<<"lib/oban/pro/migrations">>,<<"lib/oban/pro/migrations/v1_4_0.ex">>,
<<"lib/oban/pro/migrations/v1_5_0.ex">>,
<<"lib/oban/pro/migrations/v1_6_0.ex">>,
<<"lib/oban/pro/migrations/v1_7_0.ex">>,
<<"lib/oban/pro/migrations/dynamic_partitioner.ex">>,
<<"lib/oban/pro/migrations/v1_0_0.ex">>,<<"lib/oban/pro/unique.ex">>,
<<"lib/oban/pro/batch.ex">>,<<"lib/oban/pro/chunk.ex">>,
<<"lib/oban/pro/producer.ex">>,<<"lib/oban/pro/worker.ex">>,
<<"lib/oban/pro/stages">>,<<"lib/oban/pro/stages/rate.ex">>,
<<"lib/oban/pro/stages/chain.ex">>,<<"lib/oban/pro/stages/encrypted.ex">>,
<<"lib/oban/pro/stages/recorded.ex">>,<<"lib/oban/pro/stages/hooks.ex">>,
<<"lib/oban/pro/stages/deadline.ex">>,
<<"lib/oban/pro/stages/executing.ex">>,
<<"lib/oban/pro/stages/structured.ex">>,<<"lib/oban/pro/plugins">>,
<<"lib/oban/pro/plugins/dynamic_prioritizer.ex">>,
<<"lib/oban/pro/plugins/dynamic_pruner.ex">>,
<<"lib/oban/pro/plugins/dynamic_scaler.ex">>,
<<"lib/oban/pro/plugins/dynamic_partitioner.ex">>,
<<"lib/oban/pro/plugins/dynamic_lifeline.ex">>,
<<"lib/oban/pro/plugins/dynamic_queues.ex">>,
<<"lib/oban/pro/plugins/dynamic_cron.ex">>,<<"lib/oban/pro/limiters">>,
<<"lib/oban/pro/limiters/local.ex">>,<<"lib/oban/pro/limiters/rate.ex">>,
<<"lib/oban/pro/limiters/rate">>,
<<"lib/oban/pro/limiters/rate/token_bucket.ex">>,
<<"lib/oban/pro/limiters/rate/algorithm.ex">>,
<<"lib/oban/pro/limiters/rate/fixed_window.ex">>,
<<"lib/oban/pro/limiters/rate/sliding_window.ex">>,
<<"lib/oban/pro/limiters/global.ex">>,<<"lib/oban/pro/queue.ex">>,
<<"lib/oban/pro/handler.ex">>,<<"lib/oban/pro/partition.ex">>,
<<"lib/oban/pro/cloud.ex">>,<<"lib/oban/pro/uuidv7.ex">>,
<<"lib/oban/pro/stage.ex">>,<<"lib/oban/pro/refresher.ex">>,
<<"lib/oban/pro/workflow.ex">>,<<"lib/oban/pro/testing.ex">>,
<<"lib/oban/pro/migration.ex">>,<<"lib/oban/pro/engines">>,
<<"lib/oban/pro/engines/smart.ex">>,<<"lib/oban/pro/workflow">>,
<<"lib/oban/pro/workflow/schema.ex">>,
<<"lib/oban/pro/workflow/cascade.ex">>,<<"lib/oban/pro/limiter.ex">>,
<<"lib/oban/pro/validation.ex">>,<<"lib/oban/pro/flusher.ex">>,
<<"lib/oban/pro/workers">>,<<"lib/oban/pro/workers/chain.ex">>,
<<"lib/oban/pro/workers/batch.ex">>,<<"lib/oban/pro/workers/chunk.ex">>,
<<"lib/oban/pro/workers/workflow.ex">>,<<"lib/oban/pro/application.ex">>,
<<"lib/oban/pro/utils.ex">>,<<"lib/oban/pro/relay.ex">>,
<<"lib/oban/pro/exceptions.ex">>,<<"lib/oban/pro/rate_limit.ex">>,
<<"lib/oban/pro.ex">>,<<"usage-rules">>,<<"usage-rules/workflows.md">>,
<<"usage-rules/chunks.md">>,<<"usage-rules/workers.md">>,
<<"usage-rules/testing.md">>,<<"usage-rules/batches.md">>,
<<"usage-rules/queues.md">>,<<"usage-rules/plugins.md">>,
<<"usage-rules/chains.md">>,<<".formatter.exs">>,<<"mix.exs">>,
<<"usage-rules.md">>]}.
{<<"app">>,<<"oban_pro">>}.
{<<"licenses">>,[<<"Commercial">>]}.
{<<"requirements">>,
[[{<<"name">>,<<"oban">>},
{<<"app">>,<<"oban">>},
{<<"optional">>,false},
{<<"requirement">>,<<"~> 2.21">>},
{<<"repository">>,<<"hexpm">>}],
[{<<"name">>,<<"ecto_sql">>},
{<<"app">>,<<"ecto_sql">>},
{<<"optional">>,false},
{<<"requirement">>,<<"~> 3.12">>},
{<<"repository">>,<<"hexpm">>}],
[{<<"name">>,<<"postgrex">>},
{<<"app">>,<<"postgrex">>},
{<<"optional">>,true},
{<<"requirement">>,<<"~> 0.22">>},
{<<"repository">>,<<"hexpm">>}]]}.
{<<"build_tools">>,[<<"mix">>]}.

3
vendor/oban_pro/lib/oban/pro.ex vendored Normal file
View file

@ -0,0 +1,3 @@
defmodule Oban.Pro do
@moduledoc false
end

29
vendor/oban_pro/lib/oban/pro/application.ex vendored Normal file
View file

@ -0,0 +1,29 @@
defmodule Oban.Pro.Application do
@moduledoc false
use Application
@handlers [
Oban.Pro.Batch,
Oban.Pro.Engines.Smart,
Oban.Pro.Migration,
Oban.Pro.Partition,
Oban.Pro.Relay,
Oban.Pro.Worker,
Oban.Pro.Workflow
]
@impl Application
def start(_type, _args) do
for handler <- @handlers, do: handler.on_start()
children = [Oban.Pro.Diagnostics, Oban.Pro.Refresher]
Supervisor.start_link(children, strategy: :one_for_one, name: __MODULE__)
end
@impl Application
def stop(_state) do
for handler <- @handlers, do: handler.on_stop()
end
end

873
vendor/oban_pro/lib/oban/pro/batch.ex vendored Normal file
View file

@ -0,0 +1,873 @@
defmodule Oban.Pro.Batch do
@moduledoc """
Batches link the execution of many jobs as a group and runs optional callbacks after jobs are
processed. This allows your application to coordinate the execution of any number of jobs in
parallel.
## Usage
Batches are composed of one or more Pro workers that are linked with a shared `batch_id` and
optional callbacks. As a simple example, let's define a worker that delivers daily emails:
```elixir
defmodule MyApp.EmailBatch do
use Oban.Pro.Worker, queue: :mailers
@behaviour Oban.Pro.Batch
@impl Oban.Pro.Worker
def process(%Job{args: %{"email" => email}}) do
MyApp.Mailer.daily_update(email)
end
@impl Oban.Pro.Batch
def batch_completed(_job) do
Logger.info("BATCH COMPLETE")
:ok
end
end
```
Now, create a batch with `new/1` by passing a list of job changesets:
```elixir
emails
|> Enum.map(&MyApp.EmailBatch.new(%{email: &1}))
|> Batch.new()
|> Oban.insert_all()
```
After all jobs in the batch are `completed`, the `batch_completed/1` callback will be
triggered.
## Handler Callbacks
After jobs in the batch are processed, a callback job may be inserted. There are four optional
batch handler callbacks that a worker may define:
| callback | enqueued after |
| --------------------- | -------------------------------------------------------- |
| `c:batch_attempted/1` | _all_ jobs `executed` at least once |
| `c:batch_cancelled/1` | the _first_ job is `cancelled` |
| `c:batch_completed/1` | _all_ jobs in the batch are `completed` |
| `c:batch_discarded/1` | the _first_ job is `discarded` |
| `c:batch_exhausted/1` | _all_ jobs are `completed`, `cancelled`, or `discarded` |
| `c:batch_retryable/1` | the _first_ job is `retryable` |
Each callback runs in a separate, isolated job, so it may be retried or discarded like any other
job. The callback function receives an `Oban.Job` struct with the `batch_id` in `meta` and
should return `:ok` (or another valid `Worker.result()`).
Here we'll implement each of the optional handler callbacks and have them print out the batch
status along with the `batch_id`:
```elixir
defmodule MyApp.BatchWorker do
use Oban.Pro.Worker
@behaviour Oban.Pro.Batch
@impl Oban.Pro.Worker
def process(_job), do: :ok
@impl Oban.Pro.Batch
def batch_attempted(%Job{meta: %{"batch_id" => batch_id}}) do
IO.inspect({:attempted, batch_id})
:ok
end
@impl Oban.Pro.Batch
def batch_cancelled(%Job{meta: %{"batch_id" => batch_id}}) do
IO.inspect({:cancelled, batch_id})
:ok
end
@impl Oban.Pro.Batch
def batch_completed(%Job{meta: %{"batch_id" => batch_id}}) do
IO.inspect({:completed, batch_id})
:ok
end
@impl Oban.Pro.Batch
def batch_discarded(%Job{meta: %{"batch_id" => batch_id}}) do
IO.inspect({:discarded, batch_id})
:ok
end
@impl Oban.Pro.Batch
def batch_exhausted(%Job{meta: %{"batch_id" => batch_id}}) do
IO.inspect({:exhausted, batch_id})
:ok
end
@impl Oban.Pro.Batch
def batch_retryable(%Job{meta: %{"batch_id" => batch_id}}) do
IO.inspect({:retryable, batch_id})
:ok
end
end
```
## Hybrid Batches
Batches may also be built from a variety of workers, though you must provide an explicit worker
for callbacks:
```elixir
mail_jobs = Enum.map(mail_args, &MyApp.MailWorker.new/1)
push_jobs = Enum.map(push_args, &MyApp.PushWorker.new/1)
[callback_worker: MyApp.CallbackWorker]
|> Batch.new()
|> Batch.add(mail_jobs)
|> Batch.add(push_jobs)
```
Without an explicit `callback_worker`, any worker in the batch may be used for callbacks. That
makes callback handling unpredictable and unexpected.
## Customizing Batches
Batches accept a variety of options for grouping and callback customization.
### Batch ID
By default a `batch_id` is generated as a time-ordered random [UUIDv7][uuid]. UUIDs are
sufficient to ensure uniqueness between workers and nodes for any time period. However, if you
require control, you can override `batch_id` generation at the worker level or pass a value
directly to the `new/2` function.
```elixir
Batch.new(batch_id: "custom-batch-id")
```
### Batch Name
Batches accept an optional name to describe the purpose of the batch, beyond the generated id or
individual jobs in it. While the `batch_id` must be unique, the `batch_name` doesn't have to be,
so it can be used as a general purpose label.
```elixir
Batch.new(batch_name: "nightly-etl")
```
### Callback Workers
For some batches, notably those with heterogeneous jobs, it's necessary to specify a different
worker for callbacks. That is easily accomplished by passing the `:callback_worker` option to
`new/2`:
```elixir
Batch.new(callback_worker: MyCallbackWorker)
```
The callback worker **must** be an `Oban.Pro.Worker` that defines one or more of the batch callback
handlers.
### Callback Options
By default, callback jobs have an empty `args` map and inherit other options from batch jobs.
With `callback_opts`, you can set standard job options for batch callback jobs (only `args`,
`max_attempts`, `meta`, `priority`, `queue`, and `tags` are allowed).
For example, here we're passing a webhook URLs as `args`:
```elixir
Batch.new(callback_opts: [args: %{webhook: "https://web.hook"}])
```
Here, we change the callback queue and knock the priority down:
```elixir
Batch.new(callback_opts: [queue: :callbacks, priority: 9])
```
Be careful to minimize `callback_opts` as they are stored in each batch job's meta.
## Fetching Batch Jobs
To pull more context from batch jobs, it's possible to load all jobs from the batch with
`all_jobs/2` and `stream_jobs/2`. The functions take a single batch job and returns a list or
stream of all non-callback jobs in the batch, which you can then operate on with `Enum` or
`Stream` functions.
As an example, imagine you have a batch that ran for a few thousand accounts and you'd like to
notify admins that the batch is complete.
```elixir
defmodule MyApp.BatchWorker do
use Oban.Pro.Worker
@behaviour Oban.Pro.Batch
@impl Oban.Pro.Batch
def batch_completed(%Job{} = job) do
{:ok, account_ids} =
job
|> Oban.Pro.Batch.all_jobs()
|> Enum.map(& &1.args["account_id"])
account_ids
|> MyApp.Accounts.all()
|> MyApp.Mailer.notify_admins_about_batch()
end
```
Fetching a thousand jobs may be alright, but for larger batches you don't want to load that much
data into memory. Instead, you can use `stream_jobs` to iterate through them lazily:
```elixir
{:ok, account_ids} =
MyApp.Repo.transaction(fn ->
job
|> Batch.stream_jobs()
|> Stream.map(& &1.args["account_id"])
|> Enum.to_list()
end)
```
Streaming is provided by Ecto's `Repo.stream`, and it must take place within a transaction.
While it may be overkill for small batches, for batches with tens or hundreds of thousands of
jobs, it will prevent massive memory spikes or the database grinding to a halt.
[uuid]: https://datatracker.ietf.org/doc/html/draft-peabody-dispatch-new-uuid-format-04#section-5.2
"""
@behaviour Oban.Pro.Flusher
@behaviour Oban.Pro.Handler
import Ecto.Query, only: [limit: 2, order_by: 2, select: 3, union_all: 2, where: 3]
alias Ecto.Changeset
alias Oban.{Job, Repo, Validation, Worker}
alias Oban.Pro.Engines.Smart
alias Oban.Pro.Workflow
require Logger
@type changeset :: Job.changeset()
@type callback_opts :: [
args: Job.args(),
max_attempts: pos_integer(),
meta: map(),
priority: 0..9,
queue: atom() | String.t(),
tags: Job.tags()
]
@type batch_opts :: [
batch_id: String.t(),
batch_name: String.t(),
callback_opts: callback_opts(),
callback_worker: module()
]
@type repo_opts :: [timeout: timeout()]
@type t :: %__MODULE__{changesets: Enumerable.t(changeset), opts: map()}
@callbacks_to_functions %{
"attempted" => :batch_attempted,
"cancelled" => :batch_cancelled,
"completed" => :batch_completed,
"discarded" => :batch_discarded,
"exhausted" => :batch_exhausted,
"retryable" => :batch_retryable
}
@callbacks_to_deprecated %{
"attempted" => :handle_attempted,
"cancelled" => :handle_cancelled,
"completed" => :handle_completed,
"discarded" => :handle_discarded,
"exhausted" => :handle_exhausted,
"retryable" => :handle_retryable
}
@callbacks_to_combined Map.merge(
@callbacks_to_functions,
@callbacks_to_deprecated,
fn _k, new, old -> [new, old] end
)
@callbacks_to_states %{
"attempted" => ~w(scheduled available executing),
"completed" => ~w(scheduled available executing retryable cancelled discarded),
"cancelled" => ~w(cancelled),
"discarded" => ~w(discarded),
"exhausted" => ~w(scheduled retryable available executing),
"retryable" => ~w(retryable)
}
@all_states Enum.map(Job.states(), &to_string/1)
@doc """
Called after all jobs in the batch were attempted at least once.
If a `batch_attempted/1` function is defined it is executed by an isolated callback job.
## Example
Print when all jobs were attempted:
def batch_attempted(%Job{meta: %{"batch_id" => batch_id}}) do
IO.inspect(batch_id, label: "Attempted")
end
"""
@callback batch_attempted(job :: Job.t()) :: Worker.result()
@doc """
Called after any jobs in the batch have a `cancelled` state.
If a `batch_cancelled/1` function is defined it is executed by an isolated callback job.
## Example
Print when any jobs are discarded:
def batch_cancelled(%Job{meta: %{"batch_id" => batch_id}}) do
IO.inspect(batch_id, label: "Cancelled")
end
"""
@callback batch_cancelled(job :: Job.t()) :: Worker.result()
@doc """
Called after all jobs in the batch have a `completed` state.
If a `batch_completed/1` function is defined it is executed by an isolated callback job.
## Example
Print when all jobs are completed:
def batch_completed(%Job{meta: %{"batch_id" => batch_id}}) do
IO.inspect(batch_id, label: "Completed")
end
"""
@callback batch_completed(job :: Job.t()) :: Worker.result()
@doc """
Called after any jobs in the batch have a `discarded` state.
If a `batch_discarded/1` function is defined it is executed by an isolated callback job.
## Example
Print when any jobs are discarded:
def batch_discarded(%Job{meta: %{"batch_id" => batch_id}}) do
IO.inspect(batch_id, label: "Discarded")
end
"""
@callback batch_discarded(job :: Job.t()) :: Worker.result()
@doc """
Called after all jobs in the batch have either a `cancelled`, `completed` or `discarded` state.
If a `batch_exhausted/1` function is defined it is executed by an isolated callback job.
## Example
Print when all jobs are completed or discarded:
def batch_exhausted(%Job{meta: %{"batch_id" => batch_id}}) do
IO.inspect(batch_id, label: "Exhausted")
end
"""
@callback batch_exhausted(job :: Job.t()) :: Worker.result()
@doc """
Called after any jobs in the batch have a `retryable` state.
If a `batch_retryable/1` function is defined it is executed by an isolated callback job.
## Example
Print when any jobs are retryable:
def batch_retryable(%Job{meta: %{"batch_id" => batch_id}}) do
IO.inspect(batch_id, label: "Retryable")
end
"""
@callback batch_retryable(job :: Job.t()) :: Worker.result()
@optional_callbacks batch_attempted: 1,
batch_cancelled: 1,
batch_completed: 1,
batch_discarded: 1,
batch_exhausted: 1,
batch_retryable: 1
defstruct changesets: [], opts: []
defguardp is_list_or_stream(enum) when is_list(enum) or is_struct(enum, Stream)
# Handler Callbacks
@impl Oban.Pro.Handler
def on_start do
events = [
[:oban, :engine, :cancel_all_jobs, :stop]
]
:telemetry.attach_many("oban.batch", events, &__MODULE__.handle_event/4, nil)
end
@impl Oban.Pro.Handler
def on_stop do
:telemetry.detach("oban.batch")
end
@doc false
def handle_event(_event, _timing, %{conf: conf, jobs: jobs}, _) do
for %{meta: %{"batch_id" => batch_id}} = job <- jobs do
on_flush(job, batch_id, conf)
end
end
# Constants
@doc false
def callbacks_to_functions, do: @callbacks_to_functions
@doc false
def callbacks_to_deprecated, do: @callbacks_to_deprecated
# Public Functions
@doc false
def new, do: new([], [])
@doc false
def new([%Changeset{} | _] = changesets), do: new(changesets, [])
def new(%Stream{} = changesets), do: new(changesets, [])
def new(opts) when is_list(opts), do: new([], opts)
@doc """
Build a new batch from a list or stream of job changesets and some options.
## Examples
Build an empty batch without any jobs or options:
Batch.new()
Build a batch from a list of changesets:
1..3
|> Enum.map(fn id -> MyWorker.new(%{id: id}) end)
|> Batch.new()
Build a batch from a stream:
stream_of_args
|> Stream.map(&MyWorker.new/1)
|> Batch.new()
Build a batch with callback options:
Batch.new(list_of_jobs, batch_id: "abc-123", callback_worker: MyApp.CallbackWorker)
"""
@spec new(Enumerable.t(changeset()), batch_opts()) :: t()
def new(changesets, opts) when is_list_or_stream(changesets) and is_list(opts) do
validate!(opts)
opts =
opts
|> Keyword.put(:batch, true)
|> Keyword.put_new(:callback_opts, [])
|> Keyword.update!(:callback_opts, &Map.new/1)
|> Keyword.put_new_lazy(:batch_id, &Oban.Pro.UUIDv7.generate/0)
|> Map.new()
changesets = Stream.map(changesets, &put_meta(&1, opts))
%__MODULE__{changesets: changesets, opts: opts}
end
defp put_meta(changeset, opts) do
meta =
changeset
|> Changeset.get_change(:meta, %{})
|> Map.merge(opts)
Changeset.put_change(changeset, :meta, meta)
end
defp validate!(opts) do
Validation.validate_schema!(opts,
batch_id: :string,
batch_name: :string,
callback_opts: :list,
callback_worker: :module
)
end
@doc """
Add one or more jobs to a batch.
## Examples
Add jobs to an existing batche:
Batch.add(batch, Enum.map(args, &MyWorker.new/1))
Add jobs to a batch one at a time:
Batch.new()
|> Batch.add(MyWorker.new(%{}))
|> Batch.add(MyWorker.new(%{}))
|> Batch.add(MyWorker.new(%{}))
"""
@spec add(t(), Enumerable.t(changeset())) :: t()
def add(batch, %Changeset{} = changeset), do: add(batch, [changeset])
def add(%__MODULE__{} = batch, changesets) when is_list_or_stream(changesets) do
changesets = Stream.map(changesets, &put_meta(&1, batch.opts))
%{batch | changesets: Stream.concat(batch.changesets, changesets)}
end
@doc """
Append to a batch from an existing batch job.
The `batch_id` and any other batch options are propagated to the newly created batch.
> #### Appending and Callbacks {: .warning}
>
> Batch callback jobs are _only inserted once_. Appending to a batch where callbacks were already
> triggered, e.g. `batch_completed`, won't re-trigger the callback.
## Examples
Build an empty batch from an existing batch job:
job
|> Batch.append()
|> Batch.add(MyWorker.new(%{}))
|> Batch.add(MyWorker.new(%{}))
|> Oban.insert_all()
Build a batch from an existing job with overriden options:
Batch.append(job, callback_worker: MyApp.OtherWorker)
"""
@spec append(Job.t(), batch_opts()) :: t()
def append(%Job{meta: %{"batch_id" => _} = meta}, opts \\ []) when is_list(opts) do
orig_opts =
for {key, val} <- meta,
key in ~w(batch_id batch_name callback_opts callback_worker),
do: {String.to_existing_atom(key), val}
orig_opts
|> Keyword.merge(opts)
|> new()
end
@doc """
Get all non-callback jobs from a batch.
## Examples
Fetch results from all jobs from within a `batch_completed/1` callback:
def batch_completed(%Job{} = job) do
results =
job
|> Batch.all_jobs()
|> Enum.map(&fetch_recorded/1)
...
end
Get all batch jobs from anywhere using the `batch_id`:
Batch.all_jobs("some-uuid-1234-5678")
Get all jobs from anywhere with a custom Oban instance name:
Batch.all_jobs(MyApp.Oban, "some-uuid-1234-5678")
"""
@spec all_jobs(Oban.name(), Job.t() | String.t(), [repo_opts()]) :: [Job.t()]
def all_jobs(name \\ Oban, job_or_batch_id, opts \\ [])
def all_jobs(_name, %Job{conf: conf, meta: %{"batch_id" => batch_id}}, opts) do
Repo.all(conf, batch_query(batch_id), opts)
end
def all_jobs(name, batch_id, opts) when is_binary(batch_id) do
name
|> Oban.config()
|> Repo.all(batch_query(batch_id), opts)
end
def all_jobs(%Job{} = job, opts, []), do: all_jobs(Oban, job, opts)
@doc """
Cancel all non-callback jobs in a batch.
Cancelling jobs via this function bulk-updates their state in the database without loading them
into memory or resolving their workers. As a result, **worker `after_process/3` hooks are not
called** for cancelled jobs. If you need to execute cleanup logic or other side effects when jobs
are cancelled, consider using batch callbacks like `batch_cancelled/1` or implement cleanup logic
at the application level.
## Examples
Cancel jobs with the `job` in a `process/1` function:
def process(job) do
if should_stop_processing?(job.args) do
Batch.cancel_jobs(job)
else
...
end
end
Cancel jobs from anywhere using the `batch_id`:
Batch.cancel_jobs("some-uuid-1234-5678")
Cancel jobs from anywhere with a custom Oban instance name:
Batch.cancel_jobs(MyApp.Oban, "some-uuid-1234-5678")
"""
@spec cancel_jobs(Oban.name(), Job.t() | String.t()) :: {:ok, non_neg_integer()}
def cancel_jobs(oban_name \\ Oban, job_or_batch_id)
def cancel_jobs(_oban_name, %Job{conf: conf, meta: %{"batch_id" => batch_id}}) do
cancel_jobs(conf.name, batch_id)
end
def cancel_jobs(oban_name, batch_id) when is_binary(batch_id) do
Oban.cancel_all_jobs(oban_name, batch_query(batch_id))
end
@doc """
Create a batch from a workflow.
All jobs in the workflow are augmented to also be part of a batch.
## Examples
Create a batch from a workflow without any extra options:
Workflow.new()
|> Workflow.add(:step_1, MyWorker.new(%{id: 123}))
|> Workflow.add(:step_2, MyWorker.new(%{id: 345}), deps: [:step_1])
|> Workflow.add(:step_3, MyWorker.new(%{id: 456}), deps: [:step_2])
|> Batch.from_workflow()
|> Oban.insert_all()
Create a batch with callback options:
Batch.from_workflow(workflow, callback_opts: [priority: 3], callback_worker: MyApp.Worker)
"""
@spec from_workflow(Workflow.t(), batch_opts()) :: t()
def from_workflow(%Workflow{changesets: changesets}, opts \\ []) do
new(changesets, opts)
end
@doc """
Stream all non-callback jobs from a batch.
## Examples
Stream all batch jobs from within a `batch_completed/1` callback:
def batch_completed(%Job{} = job) do
{:ok, account_ids} =
MyApp.Repo.transaction(fn ->
job
|> Batch.stream_jobs()
|> Enum.map(& &1.args["account_id"])
end)
# use_account_ids
end
Stream all batch jobs from anywhere using the `batch_id`:
MyApp.Repo.transaction(fn ->
"some-uuid-1234-5678"
|> Batch.stream_jobs()
|> Enum.map(& &1.args["account_id"])
end)
Stream all batch jobs using a custom Oban instance name:
MyApp.Repo.transaction(fn ->
MyApp.Oban
|> Batch.stream_jobs("some-uuid-1234-5678")
|> Enum.map(& &1.args["account_id"])
end)
"""
@spec stream_jobs(Oban.name(), Job.t() | String.t(), [repo_opts()]) :: Enum.t()
def stream_jobs(name \\ Oban, job_or_batch_id, opts \\ [])
def stream_jobs(_name, %Job{conf: conf, meta: %{"batch_id" => batch_id}}, opts) do
Repo.stream(conf, batch_query(batch_id), opts)
end
def stream_jobs(name, batch_id, opts) when is_binary(batch_id) do
name
|> Oban.config()
|> Repo.stream(batch_query(batch_id), opts)
end
def stream_jobs(%Job{} = job, opts, []), do: stream_jobs(Oban, job, opts)
defp batch_query(batch_id) do
Job
|> where([j], fragment("? \\? 'batch_id'", j.meta))
|> where([j], fragment("? ->> 'batch_id'", j.meta) == ^batch_id)
|> where([j], is_nil(fragment("? ->> 'callback'", j.meta)))
|> where([j], j.state in @all_states)
|> order_by(asc: :id)
end
# Event Handling
@doc false
@impl Oban.Pro.Flusher
def to_flush_mfa(job, conf) do
case job.meta do
%{"batch_id" => _, "callback" => _} -> :ignore
%{"batch_id" => batch_id} -> {__MODULE__, :on_flush, [job, batch_id, conf]}
_ -> :ignore
end
end
@doc false
def on_flush(job, batch_id, conf) do
batch_worker = job.meta["batch_callback_worker"] || job.meta["callback_worker"] || job.worker
with {:ok, worker} <- Worker.from_string(batch_worker),
supported = supported_callbacks(worker),
{:ok, {states, exists}} <- states_for_callbacks(supported, batch_id, conf) do
for callback <- supported,
callback not in exists,
callback_ready?(callback, states) do
insert_callback(callback, worker, job, conf)
end
end
end
defp supported_callbacks(worker) do
for {name, [new, old]} <- @callbacks_to_combined,
function_exported?(worker, new, 1) or function_exported?(worker, old, 1),
do: name
end
defp states_for_callbacks([], _batch_id, _conf), do: :ok
defp states_for_callbacks(callbacks, batch_id, conf) do
state_query =
callbacks
|> Enum.flat_map(&Map.fetch!(@callbacks_to_states, &1))
|> Enum.uniq()
|> Enum.reduce(:none, fn state, acc ->
query =
Job
|> select([_], [type(^state, :string)])
|> where([j], fragment("? \\? 'batch_id'", j.meta))
|> where([j], fragment("? ->> 'batch_id'", j.meta) == ^batch_id)
|> where([j], is_nil(fragment("? ->> 'callback'", j.meta)))
|> where([j], j.state == ^state)
|> limit(1)
if acc == :none, do: query, else: union_all(acc, ^query)
end)
exist_query =
Enum.reduce(callbacks, :none, fn callback, acc ->
query =
Job
|> select([_j], [type(^callback, :string)])
|> where([j], fragment("? \\? 'batch_id'", j.meta))
|> where([j], fragment("? ->> 'batch_id'", j.meta) == ^batch_id)
|> where([j], fragment("? ->> 'callback'", j.meta) == ^callback)
|> where([j], j.state in @all_states)
if acc == :none, do: query, else: union_all(acc, ^query)
end)
Repo.transaction(conf, fn ->
states = conf |> Repo.all(state_query) |> List.flatten()
exists = conf |> Repo.all(exist_query) |> List.flatten()
{states, exists}
end)
end
defp callback_ready?(callback, batch_states) do
ready? =
@callbacks_to_states
|> Map.fetch!(callback)
|> Enum.any?(&(&1 in batch_states))
# Other callbacks use a negated query to avoid counting `completed` jobs.
if callback in ~w(cancelled discarded retryable) do
ready?
else
not ready?
end
end
defp insert_callback(callback, worker, job, conf) do
call_opts =
job.meta
|> Map.get("callback_opts", %{})
|> Map.put_new("args", Map.get(job.meta, "batch_callback_args", %{}))
|> Map.put_new("meta", Map.get(job.meta, "batch_callback_meta", %{}))
|> Map.put_new("queue", Map.get(job.meta, "batch_callback_queue", job.queue))
unique = [
period: :infinity,
fields: [:worker, :queue, :meta],
keys: [:batch_id, :callback],
states: Job.states()
]
{args, call_opts} = Map.pop(call_opts, "args")
{meta, call_opts} = Map.pop(call_opts, "meta")
xtra_meta =
job.meta
|> Map.take(~w(batch_id batch_name))
|> Map.put("callback", callback)
|> Map.merge(meta)
opts =
call_opts
|> Keyword.new(fn {key, val} -> {String.to_existing_atom(key), val} end)
|> Keyword.put(:meta, xtra_meta)
|> Keyword.put(:unique, unique)
changeset = worker.new(args, opts)
if not changeset.valid? and Keyword.has_key?(changeset.errors, :args) do
changeset
|> structured_error_message()
|> Logger.error()
else
{:ok, Smart.insert_job(conf, changeset, [])}
end
end
defp structured_error_message(changeset) do
"""
[Oban.Pro.Batch] can't insert batch callback because it has invalid keys:
#{get_in(changeset.errors, [:args, Access.elem(0)])}
Use one of the following options to restore batch callbacks:
* Modify structured `keys` or `required` to allow the missing keys
* Include the required arguments with the `batch_callback_args` option
* Specify a different callback worker with the `batch_callback_worker` option
"""
end
end

660
vendor/oban_pro/lib/oban/pro/chunk.ex vendored Normal file
View file

@ -0,0 +1,660 @@
defmodule Oban.Pro.Chunk do
@moduledoc """
Chunk workers execute jobs together in groups based on a size or a timeout option, e.g. when
1000 jobs are available or after 10 minutes have elapsed.
Multiple chunks can run in parallel within a single queue, and each chunk may be composed of
many thousands of jobs. Aside from improved throughput for a single queue, chunks are ideal as
the initial stage of data-ingestion and data-processing pipelines.
## Usage
Let's define a worker that sends SMS messages in chunks, rather than individually:
```elixir
defmodule MyApp.ChunkWorker do
use Oban.Pro.Chunk, queue: :messages, size: 100, timeout: 1000
@impl true
def process([_ | _] = jobs) do
jobs
|> Enum.map(& &1.args)
|> MyApp.Messaging.send_batch()
:ok
end
end
```
Notice that we declared a `size` and a `timeout` along with a `queue` for the worker. If `size`
or `timeout` are omitted they fall back to their defaults: 100 `size` and 1000ms respectively.
To process _larger_ batches _less_ frequently, we can increase both values:
```elixir
use Oban.Pro.Chunk, size: 500, timeout: :timer.seconds(5)
```
Now chunks will run with up to 500 jobs or every 5 seconds, whichever comes first.
Unlike other Pro workers, a `Chunk` worker's `process/1` receives a list of jobs rather than a
single job struct. Fittingly, the [expected return values](#t:result/0) are different as well.
## Chunk Partitioning
By default, chunks are divided into groups based on the `queue` and `worker`. This means that
each chunk consists of workers belonging to the same queue, regardless of their `args` or
`meta`. However, this approach may not always be suitable. For instance, you may want to group
workers based on a specific argument such as `:account_id` instead of just the worker. In such
cases, you can use the [`:by`](#t:chunk_by/0) option to customize how chunks are partitioned.
Here are a few examples of the `:by` option that you can use to achieve fine-grained control over
chunk partitioning:
```elixir
# Explicitly chunk by :worker
use Oban.Pro.Chunk, by: :worker
# Chunk by a single args key without considering the worker
use Oban.Pro.Chunk, by: [args: :account_id]
# Chunk by multiple args keys without considering the worker
use Oban.Pro.Chunk, by: [args: [:account_id, :cohort]]
# Chunk by worker and a single args key
use Oban.Pro.Chunk, by: [:worker, args: :account_id]
# Chunk by a single meta key
use Oban.Pro.Chunk, by: [meta: :source_id]
```
When partitioning chunks of jobs, it's important to note that using only `:args` or `:meta`
without `:worker` may result in heterogeneous chunks of jobs from different workers.
Nevertheless, regardless of the partitioning method chunks always consist of jobs from the same
queue.
Here's a simple example of partitioning by `worker` and an `author_id` field to batch analysis
of recent messages per author:
```elixir
defmodule MyApp.LLMAnalysis do
use Oban.Pro.Chunk, by: [:worker, args: :author_id], size: 50, timeout: 30_000
@impl true
def process([%{args: %{"author_id" => author_id}} | _] = jobs) do
messages =
jobs
|> Enum.map(& &1.args["message_id"])
|> MyApp.Messages.all()
{:ok, sentiment} = MyApp.GPT.gauge_sentiment(messages)
MyApp.Messages.record_sentiment(author_id)
end
end
```
## Chunk Results and Error Handling
`Chunk` worker's result type is tailored to handling multiple jobs at once. The result types
allow you to succeed an entire chunk, or selectively apply different outcomes to individual
jobs. Here are the possible results:
* `:ok` — The chunk succeeded and all jobs can be marked complete
* `{:ok, result}` — Like `:ok`, but with a result for testing.
* `{:error, reason, jobs}` — One or more jobs in the chunk failed and may be retried, any
unlisted jobs are a success.
* `{:cancel, reason, jobs}` — One or more jobs in the chunk should be cancelled, any unlisted
jobs are a success.
* `{:snooze, period, jobs}` — One or more jobs in the chunk should be snoozed and rescheduled
to run again after the specified period (e.g. `10` for seconds or `{1, :minute}`), any
unlisted jobs are a success.
* `[error: {reason, jobs}, cancel: {reason, jobs}, ...]` — Selectively apply different
outcomes to different jobs. Supports `error`, `cancel`, `discard`, and `snooze` keys. For
snooze, use `snooze: {period, jobs}`. Any unlisted jobs are a success.
To illustrate using chunk results let's expand on the message processing example from earlier.
We'll extend it to complete the whole batch when all messages are delivered or cancel
undeliverable messages:
```elixir
def process([_ | _] = jobs) do
notifications =
jobs
|> Enum.map(& &1.args)
|> MyApp.Messaging.send_batch()
bad_token = fn %{response: response} -> response == :bad_token end
if Enum.any?(notifications, bad_token) do
cancels =
notifications
|> Enum.zip(jobs)
|> Enum.filter(fn {notification, _job} -> bad_token.(notification) end)
|> Enum.map(&elem(&1, 1))
{:cancel, :bad_token, cancels}
else
{:ok, notifications}
end
end
```
In the event of an ephemeral crash, like a network issue, the entire chunk may be retried if
there are any remaining attempts.
## Chunk Organization
Chunks are run by a leader job (which has nothing to do with peer leadership). When the leader
executes it determines whether a complete chunk is available or if enough time has elapsed to
run anyhow. If neither case applies, then the leader will sleep until the timeout elapses and
then execute with as many jobs as it can find.
Completion queries run every `1000ms` by default. You can use the `:sleep` option to control how
long the leader delays between queries to check for complete chunks:
```elixir
use Oban.Pro.Chunk, size: 50, sleep: 2_000, timeout: 10_000
```
## Run Chunks Immediately
A chunk won't run until either the full number of jobs are available (`size`), or the total time
has elapsed (`timeout`). In situations where there is a long gap between incoming jobs it may be
desirable to run immediately when the leader starts. This can be done with the `leading` option.
For example, to chunk all jobs that arrive within a 5 minute window, but run immediately if more
than 5 minutes have passed since the last chunk completed:
```elixir
use Oban.Pro.Chunk, size: 100, leading: true, timeout: :timer.minutes(5)
```
## Optimizing Chunks
Queues with high concurrency and low throughput may have multiple chunk leaders running
simultaneously rather than getting bundled into a single chunk. The solution is to reduce the
queue's global concurrency to a smaller number so that new chunk leader jobs don't start and the
existing chunk can run a bigger batch.
```elixir
queues: [
chunked_queue: [global_limit: 1]
]
```
A low global limit will reduce overall throughput. Partitioning the queue with the same options
as the chunk can preserve throughput by allowing one concurrent leader per chunk. For example,
if you were chunking by `:user_id` in args, you'd partition the queue with the same option:
```elixir
queues: [
chunked_queue: [local_limit: 20, global_limit: [allowed: 1, partition: [args: :user_id]]]
]
```
"""
import Ecto.Query
import DateTime, only: [utc_now: 0]
alias Oban.{CrashError, Job, Notifier, PerformError, Period, Repo, Validation}
alias Oban.Pro.Engines.Smart
alias Oban.Pro.Utils
@typedoc """
A single key or list of keys used to specify fields for partitioning.
"""
@type key_or_keys :: atom() | [atom()]
@typedoc """
Specifies how jobs are grouped into chunks.
Jobs are always partitioned by queue, but may also be grouped by worker, args, or meta fields.
"""
@type chunk_by ::
:worker
| {:args, key_or_keys()}
| {:meta, key_or_keys()}
| [:worker | {:args, key_or_keys()} | {:meta, key_or_keys()}]
@typedoc """
A sub-result for error, cancel, or discard outcomes within a keyword list result.
"""
@type sub_result :: {reason :: term(), [Job.t()]}
@typedoc """
A sub-result for snooze outcomes within a keyword list result.
"""
@type snooze_result :: {period :: Oban.Period.t(), [Job.t()]}
@typedoc """
The return value of `process/1`, indicating the outcome for jobs in the chunk.
Unlisted jobs are marked as completed. See the "Chunk Results and Error Handling" section for
details on each variant.
"""
@type result ::
:ok
| {:ok, term()}
| {:cancel | :discard | :error, reason :: term(), [Job.t()]}
| {:snooze, period :: Oban.Period.t(), [Job.t()]}
| [
cancel: sub_result(),
discard: sub_result(),
error: sub_result(),
snooze: snooze_result()
]
@typedoc """
Options for configuring chunk behavior.
"""
@type options :: [
by: chunk_by(),
leading: boolean(),
size: pos_integer(),
sleep: pos_integer(),
timeout: pos_integer()
]
@doc false
defmacro __using__(opts) do
{chunk_opts, other_opts} = Keyword.split(opts, ~w(by leading size sleep timeout)a)
quote do
Validation.validate!(unquote(chunk_opts), &Oban.Pro.Chunk.validate/1)
use Oban.Pro.Worker, unquote(other_opts)
alias Oban.Pro.Chunk
@chunk_meta %{
chunk: true,
chunk_by: Keyword.get(unquote(chunk_opts), :by, [:worker]),
chunk_leading: Keyword.get(unquote(chunk_opts), :leading, false),
chunk_size: Keyword.get(unquote(chunk_opts), :size, 100),
chunk_sleep: Keyword.get(unquote(chunk_opts), :sleep, 1000),
chunk_timeout: Keyword.get(unquote(chunk_opts), :timeout, 1000)
}
@impl Oban.Worker
def new(args, opts) when is_map(args) and is_list(opts) do
meta =
@chunk_meta
|> Map.merge(Keyword.get(opts, :meta, %{}))
|> Map.update!(:chunk_by, &Utils.normalize_by/1)
chunk_id = Chunk.to_chunk_id(meta.chunk_by, __MODULE__, args, opts)
meta = Map.put(meta, :chunk_id, chunk_id)
super(args, Keyword.put(opts, :meta, meta))
end
@impl Oban.Worker
def perform(%Job{} = job) do
opts = __opts__()
with {:ok, job} <- Oban.Pro.Worker.before_process(__MODULE__, job, opts) do
Process.put(:oban_processing, {__MODULE__, job, opts})
job
|> Chunk.maybe_process(__MODULE__)
|> Oban.Pro.Worker.after_process(__MODULE__, job, opts)
end
end
end
end
defmacrop in_chunk(meta, chunk_id) do
quote do
fragment(
"? \\? 'chunk_id' AND ?->>'chunk_id' = ?",
unquote(meta),
unquote(meta),
unquote(chunk_id)
)
end
end
defmacrop finished_at(job) do
quote do
type(
fragment(
"greatest(?, ?, ?)",
unquote(job).completed_at,
unquote(job).discarded_at,
unquote(job).cancelled_at
),
:utc_datetime_usec
)
end
end
@doc false
def validate(opts) do
Validation.validate_schema(opts,
by: {:custom, &Oban.Pro.Validation.validate_by/1},
leading: :boolean,
size: :pos_integer,
sleep: :timeout,
timeout: :timeout
)
end
@doc false
def to_chunk_id(chunk_by, worker, args, opts) do
chunk_data =
Enum.map([:queue | chunk_by], fn
:queue -> opts |> Keyword.get(:queue, "default") |> to_string()
:worker -> inspect(worker)
[:args, keys] -> Utils.take_keys(args, keys)
[:meta, keys] -> opts |> Keyword.get(:meta, %{}) |> Utils.take_keys(keys)
end)
Utils.hash64(chunk_data)
end
# Public Interface
@doc false
def maybe_process(%Job{conf: conf} = job, worker) do
cond do
full_timeout?(conf, job) ->
fetch_and_process(worker, job)
full_chunk?(conf, job) ->
fetch_and_process(worker, job)
true ->
job.meta
|> Map.get("chunk_timeout", 1000)
|> sleep_and_process(worker, job)
end
end
# Check Helpers
defp full_chunk?(conf, %{meta: %{"chunk_id" => chunk_id, "chunk_size" => size}}) do
limit = size - 1
query =
Job
|> select([j], j.id)
|> where([j], j.state == "available")
|> where([j], in_chunk(j.meta, ^chunk_id))
|> limit(^limit)
Repo.one(conf, from(oj in subquery(query), select: count(oj.id) >= ^limit))
end
defp full_timeout?(_, %{meta: %{"chunk_leading" => false}}), do: false
defp full_timeout?(conf, job) do
%{id: job_id, meta: %{"chunk_id" => chunk_id, "chunk_timeout" => timeout}} = job
query =
Job
|> where([j], j.id != ^job_id)
|> where([j], j.state in ~w(completed cancelled discarded))
|> where([j], in_chunk(j.meta, ^chunk_id))
|> order_by(desc: :id)
|> limit(1)
|> select([j], finished_at(j))
last_ts = Repo.one(conf, query)
last_ts &&
DateTime.utc_now()
|> DateTime.add(-timeout, :millisecond)
|> DateTime.after?(last_ts)
end
# Processing Helpers
defp sleep_and_process(total, worker, %Job{conf: conf, meta: meta} = job) do
sleep = Map.get(meta, "chunk_sleep", 1000)
Process.sleep(sleep)
if total - sleep < 0 or full_chunk?(conf, job) do
fetch_and_process(worker, job)
else
sleep_and_process(total - sleep, worker, job)
end
end
defp fetch_and_process(worker, %{conf: conf, meta: %{"chunk_size" => size}} = job) do
{:ok, chunk} = fetch_chunk(conf, job, size - 1)
{chunk, _errored} = prepare_chunk(chunk)
guard_cancel(conf, worker, job, chunk)
guard_timeout(conf, worker.timeout(job), worker, job, chunk)
try do
case worker.process([job | chunk]) do
:ok ->
ack_jobs(conf, worker, job, chunk, %{})
{:ok, result} ->
with :ok <- ack_jobs(conf, worker, job, chunk, %{}), do: {:ok, result}
{:cancel, reason, cancelled} ->
ops = Map.new(cancelled, &{&1.id, {:cancel, reason}})
ack_jobs(conf, worker, job, chunk, ops)
{:discard, reason, discarded} ->
ops = Map.new(discarded, &{&1.id, {:discard, reason}})
ack_jobs(conf, worker, job, chunk, ops)
{:error, reason, errored} ->
ops = Map.new(errored, &{&1.id, {:error, reason}})
ack_jobs(conf, worker, job, chunk, ops)
{:snooze, seconds, snoozed} ->
ops = Map.new(snoozed, &{&1.id, {:snooze, seconds}})
ack_jobs(conf, worker, job, chunk, ops)
[_ | _] = multiple ->
ops =
for {oper, {reason, jobs}} <- multiple,
job <- jobs,
into: %{},
do: {job.id, {oper, reason}}
ack_jobs(conf, worker, job, chunk, ops)
end
rescue
error ->
ops = Map.new(chunk, &{&1.id, {:raised, error}})
ack_jobs(conf, worker, job, chunk, ops, __STACKTRACE__)
reraise error, __STACKTRACE__
catch
kind, reason ->
error = CrashError.exception({kind, reason, __STACKTRACE__})
ops = Map.new(chunk, &{&1.id, {:raised, error}})
ack_jobs(conf, worker, job, chunk, ops, __STACKTRACE__)
reraise error, __STACKTRACE__
end
end
# This replicates the query used in Smart.fetch_jobs/3, without the meta tracking or any other
# complications. Any modifications to the original query must be replicated here. Another
defp fetch_chunk(conf, %{meta: %{"chunk_id" => chunk_id}} = job, limit) do
subset_query =
Job
|> select([:id])
|> where([j], j.state == "available")
|> where([j], in_chunk(j.meta, ^chunk_id))
|> order_by(asc: :priority, asc: :scheduled_at, asc: :id)
|> limit(^limit)
|> lock("FOR UPDATE SKIP LOCKED")
query =
Job
|> with_cte("subset", as: ^subset_query)
|> join(:inner, [j], x in fragment(~s("subset")), on: true)
|> where([j, x], j.id == x.id)
|> select([j, _], j)
attempted_by = job.attempted_by ++ ["chunk-#{job.id}"]
updates = [
set: [state: "executing", attempted_at: DateTime.utc_now(), attempted_by: attempted_by],
inc: [attempt: 1]
]
Repo.transaction(conf, fn ->
{_count, chunk} = Repo.update_all(conf, query, updates)
chunk
end)
end
defp prepare_chunk(chunk) do
Enum.reduce(chunk, {[], []}, fn job, {acc, err} ->
with {:ok, worker} <- Oban.Worker.from_string(job.worker),
{:ok, job} <- Oban.Pro.Worker.before_hook(worker, job, worker.__opts__()) do
{[job | acc], err}
else
{:error, reason} ->
{acc, [{job, reason} | err]}
end
end)
end
# Guards
# Only the leader job is registered as "running" by the queue producer. We listen for pkill
# messages for _other_ jobs in the chunk and apply that to all jobs. Without this, cancelling
# doesn't apply to the leader and chunk jobs are orphaned.
defp guard_cancel(conf, worker, job, chunk) do
# No supervision tree is running when draining, attempting to notify will raise an exception.
if Process.get(:oban_draining) do
:ok
else
parent = self()
Task.start(fn ->
ref = Process.monitor(parent)
:ok = Notifier.listen(conf.name, [:signal])
await_cancel(conf, ref, worker, job, chunk)
end)
end
end
defp await_cancel(conf, ref, worker, job, chunk) do
receive do
{:notification, :signal, %{"action" => "pkill", "job_id" => kill_id}} ->
if Enum.any?(chunk, &(&1.id == kill_id)) do
ops = Map.new(chunk, &{&1.id, {:cancel, :shutdown}})
ack_jobs(conf, worker, job, chunk, ops)
Notifier.notify(conf.name, :signal, %{action: "pkill", job_id: job.id})
else
await_cancel(conf, ref, worker, job, chunk)
end
{:DOWN, ^ref, :process, _pid, _reason} ->
:ok
end
end
# The executor uses :timer.exit_after/2 to kill jobs that exceed the timeout. The queue's
# producer then catches the DOWN message and uses that to record a job error. The producer
# isn't aware of the job's chunk, so we monitor the parent process and ack the chunk jobs.
defp guard_timeout(_conf, :infinity, _worker, _job, _chunk), do: :ok
defp guard_timeout(conf, timeout, worker, job, chunk) do
parent = self()
Task.start(fn ->
ref = Process.monitor(parent)
receive do
{:DOWN, ^ref, :process, _pid, %Oban.TimeoutError{} = error} ->
ops = Map.new(chunk, &{&1.id, {:raised, error}})
ack_jobs(conf, worker, job, chunk, ops)
{:DOWN, ^ref, :process, _pid, _reason} ->
:ok
after
timeout + 100 -> :ok
end
end)
end
# Acking
defp ack_jobs(conf, worker, host, chunk, ops, stacktrace \\ []) do
now = DateTime.utc_now()
to_ack = fn job ->
set =
case Map.get(ops, job.id) do
nil ->
%{state: "completed", completed_at: now}
{:cancel, reason} ->
error = format_error(job, worker, {:cancel, reason}, [])
%{state: "cancelled", cancelled_at: now, error: error}
{:discard, reason} ->
error = format_error(job, worker, {:discard, reason}, [])
%{state: "discarded", discarded_at: now, error: error}
{:error, reason} when job.attempt >= job.max_attempts ->
error = format_error(job, worker, {:discard, reason}, [])
%{state: "discarded", discarded_at: now, error: error}
{:error, reason} ->
error = format_error(job, worker, {:error, reason}, [])
%{state: "retryable", scheduled_at: backoff_at(worker, job), error: error}
{:raised, error} when job.attempt >= job.max_attempts ->
error = format_error(job, worker, error, stacktrace)
%{state: "discarded", discarded_at: now, error: error}
{:raised, error} ->
error = format_error(job, worker, error, stacktrace)
%{state: "retryable", scheduled_at: backoff_at(worker, job), error: error}
{:snooze, period} ->
scheduled_at = DateTime.add(now, Period.to_seconds(period), :second)
%{state: "scheduled", scheduled_at: scheduled_at}
end
{{:ack, conf.name, job.queue, job.id}, nil, [], set}
end
acks =
chunk
|> Enum.reject(&(&1.id == host.id))
|> Enum.map(to_ack)
with {:ok, _} <- Smart.ack_jobs(acks, conf), do: Map.get(ops, host.id, :ok)
end
defp backoff_at(worker, job) do
DateTime.add(utc_now(), worker.backoff(job), :second)
end
defp format_error(job, worker, reason, stacktrace) when is_tuple(reason) do
format_error(job, worker, PerformError.exception({worker, reason}), stacktrace)
end
defp format_error(job, _worker, error, stacktrace) do
{blamed, stacktrace} = Exception.blame(:error, error, stacktrace)
formatted = Exception.format(:error, blamed, stacktrace)
%{attempt: job.attempt, at: utc_now(), error: formatted}
end
end

23
vendor/oban_pro/lib/oban/pro/cloud.ex vendored Normal file
View file

@ -0,0 +1,23 @@
defmodule Oban.Pro.Cloud do
@moduledoc """
A behaviour for interacting with cloud hosting providers.
"""
@type conf :: term()
@type opts :: keyword()
@type quantity :: non_neg_integer()
@doc """
Executed once at runtime to gather, normalize, and transform options.
"""
@callback init(opts()) :: conf()
@doc """
Called to horizontally scale a cloud resource up or down.
Successful scaling requests must return a new `conf` to be used during the next call to
`scale/2`. That allows cloud modules to track responses for additional control and
introspection.
"""
@callback scale(quantity(), conf()) :: {:ok, conf()} | {:error, term()}
end

125
vendor/oban_pro/lib/oban/pro/cron.ex vendored Normal file
View file

@ -0,0 +1,125 @@
defmodule Oban.Pro.Cron do
@moduledoc false
use Ecto.Schema
import Ecto.Changeset
alias Oban.Cron.Expression
alias Oban.Worker
@primary_key {:name, :string, autogenerate: false}
schema "oban_crons" do
field :expression, :string
field :worker, :string
field :opts, :map
field :paused, :boolean
field :insertions, {:array, :utc_datetime_usec}
field :lock_version, :integer, default: 1
field :parsed, :any, virtual: true
timestamps(
inserted_at: :inserted_at,
updated_at: :updated_at,
type: :utc_datetime_usec
)
end
@permitted ~w(name expression insertions worker opts paused)a
@requried ~w(name expression worker)a
@allowed_opts ~w(args guaranteed max_attempts meta priority queue tags timezone)a
@doc false
def allowed_opts, do: @allowed_opts
@spec changeset({binary(), module()} | {binary(), module(), Keyword.t()}) :: Ecto.Changeset.t()
def changeset({expression, worker}) do
params = %{expression: expression, name: worker, worker: worker, opts: %{}}
changeset(%__MODULE__{}, params)
end
def changeset({expression, worker, opts}) do
{name, opts} = Keyword.pop(opts, :name, worker)
{paused, opts} = Keyword.pop(opts, :paused)
{insertions, opts} = Keyword.pop(opts, :insertions)
params = %{
expression: expression,
insertions: insertions,
name: name,
opts: Map.new(opts),
paused: paused,
worker: worker
}
changeset(%__MODULE__{}, params)
end
@spec changeset({Ecto.Schema.t(), map()}) :: Ecto.Changeset.t()
def changeset(schema, params) when is_map(params) do
params =
params
|> coerce_name(:name)
|> coerce_name(:worker)
|> merge_opts(schema)
schema
|> cast(params, @permitted)
|> validate_required(@requried)
|> validate_change(:expression, &expression_validator/2)
|> validate_change(:opts, &opts_validator/2)
|> optimistic_lock(:lock_version)
end
def update_changeset(schema, params) when is_map(params) do
if Map.has_key?(params, :expression) or Map.has_key?(params, :timezone) do
changeset(schema, Map.put(params, :insertions, []))
else
changeset(schema, params)
end
end
defp coerce_name(params, key) do
case params do
%{^key => value} when is_atom(value) ->
Map.put(params, key, Worker.to_string(value))
_ ->
params
end
end
defp merge_opts(params, schema) do
case Map.split(params, @allowed_opts) do
{opts, params} when map_size(opts) > 0 ->
opts = Map.new(opts, fn {key, val} -> {to_string(key), val} end)
Map.put(params, :opts, Map.merge(schema.opts, opts))
_ ->
params
end
end
# Validators
defp expression_validator(:expression, expression) do
Expression.parse!(expression)
[]
rescue
ArgumentError ->
[expression: "expected cron expression to be a parsable binary"]
end
defp opts_validator(:opts, opts) do
string_keys = Enum.map(@allowed_opts, &to_string/1)
if Enum.all?(opts, fn {key, _} -> to_string(key) in string_keys end) do
[]
else
[opts: "expected cron opts to be one of #{inspect(@allowed_opts)}"]
end
end
end

587
vendor/oban_pro/lib/oban/pro/decorator.ex vendored Normal file
View file

@ -0,0 +1,587 @@
defmodule Oban.Pro.Decorator do
@moduledoc """
The `Decorator` extension converts functions into Oban jobs with a simple annotation.
Decorated functions, such as those in contexts or other non-worker modules, can be executed as
fully fledged background jobs with retries, scheduling, and the other guarantees you'd expect
from Oban jobs.
## Usage
To get started, use the `Decorator` module, then annotate functions with the `@job` attribute.
Here's a simple example that uses `@job true` to decorate a function without any options:
```elixir
defmodule MyApp.Business do
use Oban.Pro.Decorator
@job true
def weekly_report(tps_id, opts) do
...
end
end
```
The `@job` attribute also accepts all standard `Oban.Worker` options, e.g. `max_attempts`,
`priority`, `queue`. This example swaps out `@job true` for a list of options:
```elixir
@job [max_attempts: 3, priority: 1, queue: :reports, recorded: true]
def weekly_report(tps_id, opts) do
...
end
```
Now you can build and insert a job by calling `insert_weekly_report/2`:
```elixir
{:ok, job} = MyApp.Business.insert_weekly_report(123, pdf: true, rtf: true)
```
Notice that the second argument is a keyword list of options, which isn't normally allowed in
job args because it's not JSON serializable.
## Generated Functions
Functions decorated with `@job` generate three variants of the original function:
* `new_*` — builds an `Oban.Job` changeset ready to be inserted for execution
* `insert_*` — builds and inserts an `Oban.Job` using `Oban.insert/2`
* `relay_*` — inserts a job, awaits execution, then returns the job's results
See the `t:comp_opts/0` typespec for the subset of job options that are supported at compile
time. Additional options are available at runtime, as described in the [Runtime
Options](#module-runtime-options) section below.
### Using New
The `new_` variant will build an `Oban.Job` changeset that's ready for insertion, but not
persisted to the database. This is identical to the output from calling `c:Oban.Worker.new/2` on
a standard worker.
```elixir
changeset = Business.new_weekly_report(123)
```
The returned changeset is perfect for bulk inserts via `Oban.insert_all/1`:
```elixir
[123, 456, 789]
|> Enum.map(&Business.new_weekly_report/1)
|> Oban.insert_all()
```
It can also be used to compose batches or workflows:
```elixir
alias Oban.Pro.Workflow
Workflow.new()
|> Workflow.add(:rep_1, Business.new_weekly_report(1))
|> Workflow.add(:rep_2, Business.new_weekly_report(2))
|> Workflow.add(:rep_3, Business.new_weekly_report(3))
|> Workflow.add(:fin, Business.new_finish_up(), deps: ~w(rep_1 rep_2 rep_3)a)
|> Oban.insert_all()
```
### Using Insert
The `insert_` variant builds a changeset and immediately calls `Oban.insert/3` to enqueue it.
The result is a success tuple containing the job, or a changeset with errors.
```elixir
{:ok, job} = Business.insert_weekly_report(123)
```
By default, jobs are inserted using the standard `Oban` instance. For applications that run
multiple Oban instances, or use a non-standard name, you can override the instance with the
`:oban` option:
```elixir
{:ok, job} = Business.insert_weekly_report(123, oban: SomeOban)
```
### Using Relay
The `relay_` variant builds a changeset, inserts it, then leverages `Oban.Pro.Relay` to await
execution and return a result:
```elixir
{:ok, result} = Business.relay_weekly_report(123)
```
The default timeout is a brief 5000ms, which doesn't account for scheduling or queueing time.
Provide an alternate timeout to wait longer:
```elixir
case Business.relay_weekly_report(123, timeout: 30_000) do
{:ok, result} -> IO.inspect(result, label: "RESULT")
{:error, reason} -> IO.inspect(reason, label: "ERROR")
end
```
Note that the `timeout` option only controls how long the local process will block while
awaiting a result. The job will keep executing regardless of the timeout period.
> #### Considerations and Caveats {: .info}
>
> Decorated functions are a convenient way to run functions in the background, and suitable in many
> situations. However, there are circumstances where they're unsuitable and you should exercise
> care:
>
> * Advanced worker functionality such as custom backoffs, hooks, structured args, or
> callbacks requires a dedicated worker module and isn't suitable for decoration.
>
> * Args of any type are safely serialized, but dumping large amounts of data may cause
> performance problems because it must be serialized, stored, and deserialized.
>
> * Changing function signatures while jobs are in-flight can cause jobs to fail, just like
> changing the shape of args passed to a `process/1` callback.
## Runtime Options
Each decorated function has an additional generated clause that accepts job options, e.g.
`new_weekly_report/1` also has a `new_weekly_report/2` variant.
All compile time options can be overridden at runtime. For example, to override the `queue` and
`max_attempts`:
```elixir
Business.insert_weekly_report(123, queue: "default", max_attempts: 10)
```
In addition to compile time options, runtime options accepted by `Oban.Job.new/2` (other than
`worker`) are also allowed. This makes it possible to schedule decorated jobs:
```elixir
Business.insert_weekly_report(123, schedule_in: {1, :minute})
```
See `t:full_opts/0` for the complete typespec of runtime options.
## Patterns and Guards
Each generated variant retains pattern matches and guards from the original function. That
allows expressive, defensive data validation before a job executes.
For example, use a guard to ensure the `id` argument is an integer:
```elixir
@job true
def notify_user(id) when is_integer(id), do: ...
```
Or, pattern match on a map with a `user_id` key and ensure the `id` is an integer:
```elixir
@job true
def notify_user(%{user_id: id}) when is_integer(id), do: ...
```
> #### Multiple Function Clauses Not Supported {: .warning}
>
> The `@job` decorator only works with single-clause functions. When decorating a function
> with multiple clauses, only the first clause is captured. Subsequent clauses are ignored for
> decoration, which will cause `FunctionClauseError` when calling the decorated variants.
>
> Instead of using multiple clauses, use a single function clause with pattern matching inside:
>
> ```diff
> @job true
> -def process_data(1, 2), do: :special_case
> -def process_data(a, b), do: {:ok, a + b}
> +def process_data(a, b) do
> + case {a, b} do
> + {1, 2} -> :special_case
> + {a, b} -> {:ok, a + b}
> + end
> +end
> ```
## Complex Types
Any Elixir term may be passed as an argument, not just JSON encodable values. That enables
passing native data-types such as tuples, keywords, or structs that can't easily be used in
regular jobs.
```elixir
@job true
def add_discount(%User{id: _}, amount: 10_000), do: ...
```
> #### Avoid Non-Portable Types {: .warning}
>
> Pass non-portable data types such as `pid`, `reference`, `port` with caution. There's no
> guarantee that a job will run on the same node and those specific values available.
> Furthermore, be careful passing anonymous functions because they are closures over the local
> environment.
## Return Values
Decorated jobs respect the same standard return types as `c:Oban.Pro.Worker.process/1`. That
means you can return an `{:error, reason}` tuple to signify an error, or `{:cancel, reason}` to
quietly cancel a job. However, because decorated functions weren't necessarily designed to be
executed in a job, unexpected returns are considered a success.
While there's no harm in returning `nil`, a struct, or some other non-standard value, it's best
to return an explicitly support a type such as `:ok` or `{:ok, value}`.
```diff
@job true
def update_account(user_id, params) do
user = Repo.get(User, user_id)
do_update(user, params)
- user
+ {:ok, user}
end
```
## Unique and Replace
Unique and replace options are available for decorated jobs. However, they have purposeful
limitations for compatibility with the decorated worker:
* `unique` — only supports `states`, `period`, and `timestamp` options. The `fields` and `keys`
options aren't supported.
* `replace` — expects the newer, per-state syntax, and it doesn't support replacing the `worker`
or `args`.
Both options can be defined at compilie in the `@job` annotation:
```elixir
@job [unique: [period: :infinity], replace: [scheduled: [:scheduled_at]]]
```
Or as runtime options:
```elixir
Business.send_notice(user, schedule_in: 60, replace: [scheduled: [:scheduled_at]])
```
## Limiting Decoration
To avoid generating unnecessary functions you can disable generating `new`, `insert`, or `relay`
functions via passing flags to `use`:
```elixir
use Oban.Pro.Decorator, new: false
use Oban.Pro.Decorator, insert: false
use Oban.Pro.Decorator, relay: false
```
Filtering options can be combined to restrict generation to one variant. For example, to only
generate `insert_*` functions:
```elixir
use Oban.Pro.Decorator, new: false, relay: false
```
## Testing Decorated Jobs
Testing decorated jobs is tricky because they're always enqueued with `Oban.Pro.Decorated` as
the worker. The assert helpers `Oban.Pro.Testing` have a `:decorated` option specifically to
make testing decorated jobs more convenient.
Pass a captured function with the original arity to the `decorated` option:
```elixir
assert_enqueued decorated: &Business.weekly_report/2
```
Use a list of to assert on args (not a map, as you would for a standard job):
```elixir
assert_enqueued args: [123, pdf: true], decorated: &Business.weekly_report/2
```
## Accessing the Decorated Job
The `Decorator` module provides a convenient way to access the currently executing job through
the `current_job/0` function. This can be useful when you need job context information during
execution.
In any module that uses `Oban.Pro.Decorator`, you can call the `current_job/0` function to
retrieve the current job. This is particularly useful when you need to access job metadata as
part of a workflow. For example:
```elixir
defmodule MyApp.Doubler do
use Oban.Pro.Decorator
alias Oban.Pro.Workflow
def invoke do
Workflow.new()
|> Workflow.add(:dbl_1, new_double(1))
|> Workflow.add(:dbl_2, new_double(2))
|> Workflow.add(:dbl_3, new_double(3))
|> Workflow.add(:print, new_print(), deps: [:dbl_1, :dbl_2, :dbl_3])
|> Oban.insert_all()
end
@job recorded: true
def double(value), do: value * 2
@job true
def print do
current_job()
|> Workflow.all_recorded(only_deps: true)
|> IO.inspect()
end
end
```
If `current_job/0` is called outside of a job context (not during job execution), it returns
`nil`.
"""
use Oban.Pro.Worker
alias __MODULE__, as: Decorator
alias Oban.Pro.Relay
alias Oban.Validation
args_schema do
field :mod, :string
field :fun, :string
field :arg, :term
end
@typedoc """
Options allowed in `@job` annotations at compile time.
"""
@type comp_opts :: [
max_attempts: pos_integer(),
oban: GenServer.name(),
priority: 0..9,
queue: atom() | binary(),
recorded: boolean() | keyword(),
replace: [Job.replace_by_state_option()],
tags: Job.tags(),
unique:
true
| [
period: timeout(),
states: atom() | [Job.unique_state()],
timestamp: :inserted_at | :scheduled_at
]
]
@typedoc """
Options allowed at runtime when calling generated `new_`, `insert_`, or `relay_` functions.
"""
@type full_opts :: [
max_attempts: pos_integer(),
meta: map(),
oban: GenServer.name(),
priority: 0..9,
queue: atom() | binary(),
recorded: boolean() | keyword(),
replace: [Job.replace_by_state_option()],
scheduled_at: DateTime.t(),
schedule_in: Job.schedule_in_option(),
tags: Job.tags(),
timeout: timeout(),
unique:
true
| [
period: Job.unique_period(),
states: Job.unique_state_group() | [Job.unique_state()],
timestamp: Job.unique_timestamp()
]
]
@doc false
defmacro __using__(opts) do
deco_opts = Keyword.merge([new: true, insert: true, relay: true], opts)
quote bind_quoted: [deco_opts: deco_opts] do
Module.put_attribute(__MODULE__, :oban_deco_opts, deco_opts)
Module.register_attribute(__MODULE__, :job, accumulate: true)
Module.register_attribute(__MODULE__, :oban_decorated, accumulate: true)
@on_definition Oban.Pro.Decorator
@before_compile Oban.Pro.Decorator
end
end
@doc false
defmacro __before_compile__(env) do
gen_opts = Module.get_attribute(env.module, :oban_deco_opts)
for {fun, orig_args, guards, opts} <- Module.get_attribute(env.module, :oban_decorated) do
validate!(opts)
{args, {_idx, renamed}} =
Macro.prewalk(orig_args, {0, %{}}, fn
{var, _meta, nil}, {idx, acc} ->
ast = {:"arg#{idx}", [], __MODULE__}
{ast, {idx + 1, Map.put(acc, var, ast)}}
other, idx_acc ->
{other, idx_acc}
end)
guards =
case guards do
[] ->
quote(do: [is_boolean(true)])
[_ | _] ->
Macro.prewalk(guards, fn
{var, _meta, nil} when is_map_key(renamed, var) -> Map.fetch!(renamed, var)
other -> other
end)
end
quote generated: true,
bind_quoted: [
new: :"new_#{fun}",
ins: :"insert_#{fun}",
rel: :"relay_#{fun}",
gen_opts: gen_opts,
fun: fun,
mod: env.module,
opts: opts,
args: Macro.escape(args, unquote: true),
args_opts: Macro.escape(args ++ [Macro.var(:oban_opts, nil)], unquote: true),
guards: Macro.escape(guards, unquote: true)
] do
def current_job do
with {_module, job, _opts} <- Process.get(:oban_processing), do: job
end
if gen_opts[:new] do
def unquote(new)(unquote_splicing(args)) when unquote_splicing(guards) do
Decorator.new(unquote(mod), unquote(fun), unquote(args), unquote(opts))
end
def unquote(new)(unquote_splicing(args_opts)) when unquote_splicing(guards) do
full_opts = Keyword.merge(unquote(opts), unquote(Macro.var(:oban_opts, nil)))
Decorator.new(unquote(mod), unquote(fun), unquote(args), full_opts)
end
end
if gen_opts[:insert] do
def unquote(ins)(unquote_splicing(args)) when unquote_splicing(guards) do
Decorator.insert(unquote(mod), unquote(fun), unquote(args), unquote(opts))
end
def unquote(ins)(unquote_splicing(args_opts)) when unquote_splicing(guards) do
full_opts = Keyword.merge(unquote(opts), unquote(Macro.var(:oban_opts, nil)))
Decorator.insert(unquote(mod), unquote(fun), unquote(args), full_opts)
end
end
if gen_opts[:relay] do
def unquote(rel)(unquote_splicing(args)) when unquote_splicing(guards) do
Decorator.relay(unquote(mod), unquote(fun), unquote(args), unquote(opts))
end
def unquote(rel)(unquote_splicing(args_opts)) when unquote_splicing(guards) do
full_opts = Keyword.merge(unquote(opts), unquote(Macro.var(:oban_opts, nil)))
Decorator.relay(unquote(mod), unquote(fun), unquote(args), full_opts)
end
end
end
end
end
@doc false
def __on_definition__(env, :def, fun, args, guards, _body) do
with [opts] <- Module.get_attribute(env.module, :job) do
opts = if is_list(opts), do: opts, else: []
Module.put_attribute(env.module, :oban_decorated, {fun, List.wrap(args), guards, opts})
end
Module.delete_attribute(env.module, :job)
end
def __on_definition__(_env, _kind, _fun, _args, _guards, _body), do: :ok
@doc false
def new(mod, fun, arg, opts) when is_atom(mod) and is_atom(fun) and is_list(arg) do
name = "#{inspect(mod)}.#{fun}/#{length(arg)}"
meta = %{decorated: true, decorated_name: name}
opts =
opts
|> Keyword.drop([:oban, :worker])
|> Keyword.update(:meta, meta, &Map.merge(&1, meta))
new(%{mod: inspect(mod), fun: to_string(fun), arg: arg}, opts)
end
@doc false
def insert(mod, fun, arg, opts) do
oban = Keyword.get(opts, :oban, Oban)
mod
|> new(fun, arg, opts)
|> then(&Oban.insert(oban, &1))
end
@doc false
def relay(mod, fun, arg, opts) do
oban = Keyword.get(opts, :oban, Oban)
{timeout, opts} = Keyword.pop(opts, :timeout, 5_000)
mod
|> new(fun, arg, opts)
|> then(&Relay.async(oban, &1))
|> Relay.await(timeout)
end
@impl Oban.Pro.Worker
def process(%{args: %{mod: mod, fun: fun, arg: arg}}) do
mod =
mod
|> String.split(".")
|> Module.safe_concat()
|> Code.ensure_loaded!()
fun = String.to_existing_atom(fun)
case apply(mod, fun, arg) do
:ok -> :ok
{:ok, _} = result -> result
{:error, _} = error -> error
{:snooze, _} = snooze -> snooze
{:cancel, _} = cancel -> cancel
other -> {:ok, other}
end
end
defp validate!(opts) do
Validation.validate_schema!(opts,
max_attempts: :pos_integer,
oban: :any,
priority: {:range, 0..9},
queue: {:or, [:atom, :string]},
recorded: {:or, [:boolean, :list]},
replace: {:custom, &Job.validate_replace/1},
tags: {:list, :string},
unique: {:custom, &validate_unique/1}
)
end
defp validate_unique(unique) do
cond do
is_list(unique) and Keyword.has_key?(unique, :fields) ->
{:error, "the :fields option isn't allowed for decorated jobs"}
is_list(unique) and Keyword.has_key?(unique, :keys) ->
{:error, "the :keys option isn't allowed for decorated jobs"}
true ->
Job.validate_unique(unique)
end
end
end

135
vendor/oban_pro/lib/oban/pro/diagnostics.ex vendored Normal file
View file

@ -0,0 +1,135 @@
defmodule Oban.Pro.Diagnostics do
@moduledoc false
use GenServer
alias __MODULE__, as: State
alias Oban.Notifier
defstruct [:timer, interval: :timer.seconds(5), subscribed: MapSet.new()]
@info_keys ~w(
current_stacktrace
heap_size
memory
message_queue_len
reductions
stack_size
status
)a
@doc false
def start_link(opts \\ []) do
state = struct!(State, opts)
GenServer.start_link(__MODULE__, state, name: __MODULE__)
end
# Callbacks
@impl GenServer
def init(state) do
Process.flag(:trap_exit, true)
{:ok, schedule_discover(state)}
end
@impl GenServer
def terminate(_reason, state) do
if is_reference(state.timer), do: Process.cancel_timer(state.timer)
:ok
end
@impl GenServer
def handle_call(:discover, _from, state) do
{:reply, :ok, discover_and_subscribe(state)}
end
@impl GenServer
def handle_info(:discover, state) do
{:noreply, discover_and_subscribe(state)}
end
def handle_info({:notification, :diagnostics, %{"job_id" => job_id}}, state) do
finder = fn conf ->
case Registry.lookup(Oban.Registry, {conf.name, {:executing, job_id}}) do
[{pid, _}] -> {conf, pid}
_ -> nil
end
end
case Enum.find_value(list_oban_instances(), finder) do
{conf, pid} ->
payload = %{"info" => process_info(pid), "job_id" => job_id, "node" => inspect(conf.node)}
Notifier.notify(conf, :diagnostics_reply, payload)
nil ->
:ok
end
{:noreply, state}
end
def handle_info(_message, state) do
{:noreply, state}
end
# Helpers
defp discover_and_subscribe(state) do
state =
Enum.reduce(list_oban_instances(), state, fn conf, acc ->
if MapSet.member?(acc.subscribed, conf.name) do
acc
else
:ok = Notifier.listen(conf.name, [:diagnostics])
%{acc | subscribed: MapSet.put(acc.subscribed, conf.name)}
end
end)
state
|> cancel_timer()
|> schedule_discover()
end
defp list_oban_instances do
match = [{{{:"$1", Oban.Notifier}, :_, :_}, [], [:"$1"]}]
Oban.Registry
|> Registry.select(match)
|> Enum.map(&Oban.config/1)
end
defp process_info(pid) do
case Process.info(pid, @info_keys) do
nil ->
%{}
info ->
info
|> Keyword.update!(:current_stacktrace, &format_stacktrace/1)
|> Keyword.update!(:status, &to_string/1)
|> Map.new(fn {key, val} -> {to_string(key), val} end)
end
end
defp format_stacktrace(nil), do: ""
defp format_stacktrace(stacktrace), do: Exception.format_stacktrace(stacktrace)
defp cancel_timer(%{timer: timer} = state) when is_reference(timer) do
Process.cancel_timer(timer)
%{state | timer: nil}
end
defp cancel_timer(state), do: state
defp schedule_discover(state) do
timer = Process.send_after(self(), :discover, state.interval)
%{state | timer: timer}
end
end

1987
vendor/oban_pro/lib/oban/pro/engines/smart.ex vendored Normal file
View file

File diff suppressed because it is too large Load diff

12
vendor/oban_pro/lib/oban/pro/exceptions.ex vendored Normal file
View file

@ -0,0 +1,12 @@
defmodule Oban.Pro.WorkflowError do
@moduledoc false
defexception [:message, :reason]
@impl Exception
def exception(reason) do
message = "upstream job was #{reason}, workflow can't complete"
%__MODULE__{message: message, reason: reason}
end
end

24
vendor/oban_pro/lib/oban/pro/flusher.ex vendored Normal file
View file

@ -0,0 +1,24 @@
defmodule Oban.Pro.Flusher do
@moduledoc false
alias Oban.{Config, Job}
@doc """
Construct an MFA for flushing accumulated operations after the Smart engine transaction commits.
By convention, the function should be named `:on_flush`.
"""
@callback to_flush_mfa(Job.t(), Config.t()) :: :ignore | mfa()
@doc """
Extract flush handlers from all registered flushing modules.
The flush order is important. Workflow flushing must complete before Batch flushing to ensure
batch wrapped workflow handlers are triggered.
"""
def get_flush_handlers(job, conf) do
[Oban.Pro.Workflow, Oban.Pro.Batch, Oban.Pro.Stages.Chain]
|> Enum.map(fn handler -> handler.to_flush_mfa(job, conf) end)
|> Enum.reject(&(&1 == :ignore))
end
end

13
vendor/oban_pro/lib/oban/pro/handler.ex vendored Normal file
View file

@ -0,0 +1,13 @@
defmodule Oban.Pro.Handler do
@moduledoc false
@doc """
Attach handler hooks.
"""
@callback on_start() :: any()
@doc """
Teardown handler hooks.
"""
@callback on_stop() :: any()
end

27
vendor/oban_pro/lib/oban/pro/limiter.ex vendored Normal file
View file

@ -0,0 +1,27 @@
defmodule Oban.Pro.Limiter do
@moduledoc false
alias Oban.{Config, Job}
alias Oban.Pro.Producer
@type changes :: %{
:conf => Config.t(),
:prod => Producer.t(),
:running => map(),
optional(atom()) => any()
}
@type demand :: non_neg_integer()
@type limit :: nil | demand | [{demand, term(), term()}]
@doc """
Calculate the current demand based on capacity and usage.
"""
@callback check(changes()) :: {:ok, limit()}
@doc """
Record demand usage.
"""
@callback track(Producer.meta(), [Job.t()]) :: Producer.meta()
end

102
vendor/oban_pro/lib/oban/pro/limiters/global.ex vendored Normal file
View file

@ -0,0 +1,102 @@
defmodule Oban.Pro.Limiters.Global do
@moduledoc false
@behaviour Oban.Pro.Limiter
alias Oban.Pro.Partition
@impl Oban.Pro.Limiter
def check(%{producer: producer} = changes) when is_map(producer.meta.global_limit) do
tracked =
[producer | changes.all_producers]
|> Enum.uniq_by(& &1.uuid)
|> Enum.map(& &1.meta.global_limit)
|> Enum.filter(&is_map/1)
|> Enum.reduce(%{}, &merge/2)
allowed = producer.meta.global_limit.allowed
tracked = update_legacy_tracked(tracked)
if is_nil(producer.meta.global_limit.partition) do
{:ok, combined_demands(allowed, tracked)}
else
{:ok, separate_demands(changes.conf, allowed, producer, tracked)}
end
end
def check(_changes), do: {:ok, nil}
defp merge(global_limit, acc) do
Map.merge(global_limit.tracked, acc, fn _, v1, v2 -> v1 + v2 end)
end
defp combined_demands(allowed, tracked) do
total = tracked |> Map.values() |> Enum.sum()
max(allowed - total, 0)
end
defp separate_demands(conf, allowed, producer, tracked) do
untracked =
for key <- Partition.available_keys(conf, producer.queue, producer.meta.local_limit),
not is_map_key(tracked, key),
into: %{},
do: {key, allowed}
allowed =
if producer.meta.global_limit.burst do
size =
tracked
|> Map.merge(untracked)
|> Map.delete("none")
|> map_size()
|> max(1)
producer.meta.local_limit
|> div(size)
|> max(allowed)
else
allowed
end
Enum.reduce(tracked, untracked, fn {key, total}, acc ->
case max(allowed - total, 0) do
demand when demand > 0 -> Map.put(acc, key, demand)
_ -> acc
end
end)
end
# Tracking must be updated after acking and before fetching in order to have accurate limits for
# the current fetch transaction.
def prepare_tracked(%{global_limit: global} = meta, keys) do
tracked =
Enum.reduce(keys, global.tracked, fn key, acc ->
if is_map_key(acc, key) do
Map.update!(acc, key, &(&1 - 1))
else
acc
end
end)
put_in(meta.global_limit.tracked, tracked)
end
@impl Oban.Pro.Limiter
def track(%{global_limit: global} = meta, jobs) when is_map(global) do
tracked = Enum.frequencies_by(jobs, &Partition.get_key(&1, "*"))
put_in(meta.global_limit.tracked, tracked)
end
def track(meta, _jobs), do: meta
defp update_legacy_tracked(%{"count" => count}), do: %{"none" => count}
defp update_legacy_tracked(tracked) do
Map.new(tracked, fn
{key, %{"count" => count}} -> {key, count}
tuple -> tuple
end)
end
end

13
vendor/oban_pro/lib/oban/pro/limiters/local.ex vendored Normal file
View file

@ -0,0 +1,13 @@
defmodule Oban.Pro.Limiters.Local do
@moduledoc false
@behaviour Oban.Pro.Limiter
@impl Oban.Pro.Limiter
def check(%{producer: producer, running: running}) do
{:ok, producer.meta.local_limit - map_size(running)}
end
@impl Oban.Pro.Limiter
def track(meta, _jobs), do: meta
end

98
vendor/oban_pro/lib/oban/pro/limiters/rate.ex vendored Normal file
View file

@ -0,0 +1,98 @@
defmodule Oban.Pro.Limiters.Rate do
@moduledoc false
@behaviour Oban.Pro.Limiter
alias Oban.Pro.Limiters.Rate.Algorithm
alias Oban.Pro.Partition
@impl Oban.Pro.Limiter
def check(%{producer: producer} = changes) when is_map(producer.meta.rate_limit) do
%{window_time: prev_time} = producer.meta.rate_limit
curr_time = unix_now()
windows =
[producer | changes.all_producers]
|> Enum.uniq_by(& &1.uuid)
|> Enum.map(& &1.meta.rate_limit)
|> Enum.filter(&(is_map(&1) and &1.window_time >= curr_time - &1.period))
|> Enum.reduce(%{}, &merge_windows/2)
windows = update_legacy_windows(windows)
demand = calculate_demand(producer.meta.rate_limit, windows, curr_time, prev_time, changes)
{:ok, demand}
end
def check(_changes), do: {:ok, nil}
defp calculate_demand(rate_limit, windows, curr_time, prev_time, changes) do
%{allowed: allowed, period: period} = rate_limit
algorithm = Algorithm.callback(rate_limit)
if is_nil(rate_limit.partition) do
algorithm.demand(windows, allowed, period, curr_time, prev_time)
else
producer = changes.producer
available_keys =
Partition.available_keys(changes.conf, producer.queue, producer.meta.local_limit)
algorithm.partition_demands(
windows,
allowed,
period,
curr_time,
prev_time,
available_keys
)
end
end
defp merge_windows(rate_limit, acc) do
Algorithm.callback(rate_limit).merge(rate_limit.windows, acc)
end
defp update_legacy_windows(%{"args" => _} = window), do: %{"*" => window}
defp update_legacy_windows(windows), do: windows
def flatten(%{rate_limit: rate_limit}) do
Algorithm.callback(rate_limit).flatten(rate_limit.windows)
end
@impl Oban.Pro.Limiter
def track(%{rate_limit: rate_limit} = meta, jobs) when is_map(rate_limit) do
new_counts = build_counts(jobs)
curr_time = unix_now()
{all_windows, next_time} =
Algorithm.callback(rate_limit).track(
rate_limit.windows,
new_counts,
rate_limit.period,
rate_limit.window_time,
curr_time,
rate_limit.allowed
)
%{meta | rate_limit: %{rate_limit | windows: all_windows, window_time: next_time}}
end
def track(meta, _jobs), do: meta
defp build_counts(jobs) do
Enum.reduce(jobs, %{}, fn job, acc ->
key = Partition.get_key(job, "*")
weight = Oban.Pro.Worker.get_weight(job)
Map.update(acc, key, weight, &(&1 + weight))
end)
end
defp unix_now do
DateTime.to_unix(DateTime.utc_now(), :second)
end
end

64
vendor/oban_pro/lib/oban/pro/limiters/rate/algorithm.ex vendored Normal file
View file

@ -0,0 +1,64 @@
defmodule Oban.Pro.Limiters.Rate.Algorithm do
@moduledoc false
alias Oban.Pro.Limiters.Rate.{FixedWindow, SlidingWindow, TokenBucket}
@type windows :: %{String.t() => map()}
@type counts :: %{String.t() => non_neg_integer()}
@doc """
Get the callback module for a rate limit's algorithm.
"""
@spec callback(%{:algorithm => atom(), optional(atom()) => term()}) :: module()
def callback(%{algorithm: :fixed_window}), do: FixedWindow
def callback(%{algorithm: :sliding_window}), do: SlidingWindow
def callback(%{algorithm: :token_bucket}), do: TokenBucket
def callback(_rate_limit), do: SlidingWindow
@doc """
Calculate available demand given current window state.
"""
@callback demand(
windows :: windows(),
allowed :: pos_integer(),
period :: pos_integer(),
curr_time :: integer(),
prev_time :: integer()
) :: non_neg_integer()
@doc """
Calculate per-partition demands for partitioned rate limiting.
"""
@callback partition_demands(
windows :: windows(),
allowed :: pos_integer(),
period :: pos_integer(),
curr_time :: integer(),
prev_time :: integer(),
available_keys :: [String.t()]
) :: %{String.t() => non_neg_integer()}
@doc """
Flatten windows into a list with a single, compact value regardless of partitions.
"""
@callback flatten(windows :: windows()) :: [map()]
@doc """
Merge window states from multiple producers.
"""
@callback merge(windows_a :: windows(), windows_b :: windows()) :: windows()
@doc """
Track job execution by updating window state.
Returns {updated_windows, next_window_time}.
"""
@callback track(
windows :: windows(),
new_counts :: counts(),
period :: pos_integer(),
prev_time :: integer(),
curr_time :: integer(),
allowed :: pos_integer()
) :: {windows(), integer()}
end

78
vendor/oban_pro/lib/oban/pro/limiters/rate/fixed_window.ex vendored Normal file
View file

@ -0,0 +1,78 @@
defmodule Oban.Pro.Limiters.Rate.FixedWindow do
@moduledoc false
# Fixed window simply resets the count when the period expires, providing a straightforward but
# potentially bursty rate limit.
@behaviour Oban.Pro.Limiters.Rate.Algorithm
@impl Oban.Pro.Limiters.Rate.Algorithm
def demand(windows, allowed, period, curr_time, prev_time) do
if window_expired?(curr_time, prev_time, period) do
allowed
else
total = Enum.reduce(windows, 0, fn {_key, val}, acc -> acc + val["count"] end)
max(allowed - total, 0)
end
end
@impl Oban.Pro.Limiters.Rate.Algorithm
def partition_demands(windows, allowed, period, curr_time, prev_time, available_keys) do
if window_expired?(curr_time, prev_time, period) do
Map.new(available_keys, &{&1, allowed})
else
untracked =
for key <- available_keys,
not is_map_key(windows, key),
into: %{},
do: {key, allowed}
Enum.reduce(windows, untracked, fn {key, val}, acc ->
case max(allowed - val["count"], 0) do
demand when demand > 0 -> Map.put(acc, key, demand)
_zero -> acc
end
end)
end
end
@impl Oban.Pro.Limiters.Rate.Algorithm
def flatten(windows) do
count = Enum.reduce(windows, 0, fn {_key, val}, acc -> acc + val["count"] end)
[%{"count" => count}]
end
@impl Oban.Pro.Limiters.Rate.Algorithm
def merge(windows_a, windows_b) do
Map.merge(windows_a, windows_b, fn _key, v1, v2 ->
%{"count" => v1["count"] + v2["count"]}
end)
end
@impl Oban.Pro.Limiters.Rate.Algorithm
def track(windows, new_counts, period, prev_time, curr_time, _allowed) do
new_counts = Map.new(new_counts, fn {key, count} -> {key, %{"count" => count}} end)
if window_expired?(curr_time, prev_time, period) do
{new_counts, curr_time}
else
all_windows =
Map.merge(windows, new_counts, fn _key, existing, new ->
%{"count" => existing["count"] + new["count"]}
end)
all_windows =
for {key, val} <- all_windows,
val["count"] > 0,
into: %{},
do: {key, val}
{all_windows, prev_time}
end
end
defp window_expired?(curr_time, prev_time, period) do
curr_time - prev_time >= period
end
end

98
vendor/oban_pro/lib/oban/pro/limiters/rate/sliding_window.ex vendored Normal file
View file

@ -0,0 +1,98 @@
defmodule Oban.Pro.Limiters.Rate.SlidingWindow do
@moduledoc false
# The sliding window uses two time buckets (current and previous) with weighted averaging to
# provide smooth rate limiting without the sharp edges of fixed windows.
@behaviour Oban.Pro.Limiters.Rate.Algorithm
@impl Oban.Pro.Limiters.Rate.Algorithm
def demand(windows, allowed, period, curr_time, prev_time) do
weight = div(max(period - (curr_time - prev_time), 0), period)
total =
Enum.reduce(windows, 0, fn {_key, val}, acc ->
acc + val["prev_count"] * weight + val["curr_count"]
end)
max(allowed - total, 0)
end
@impl Oban.Pro.Limiters.Rate.Algorithm
def partition_demands(windows, allowed, period, curr_time, prev_time, available_keys) do
weight = div(max(period - (curr_time - prev_time), 0), period)
untracked =
for key <- available_keys,
not is_map_key(windows, key),
into: %{},
do: {key, allowed}
Enum.reduce(windows, untracked, fn {key, val}, acc ->
%{"curr_count" => curr, "prev_count" => prev} = val
case max(allowed - (prev * weight + curr), 0) do
demand when demand > 0 -> Map.put(acc, key, demand)
_zero -> acc
end
end)
end
@impl Oban.Pro.Limiters.Rate.Algorithm
def flatten(windows) do
{pacc, cacc} =
Enum.reduce(windows, {0, 0}, fn {_key, val}, {pacc, cacc} ->
{pacc + val["prev_count"], cacc + val["curr_count"]}
end)
[%{"curr_count" => cacc, "prev_count" => pacc}]
end
@impl Oban.Pro.Limiters.Rate.Algorithm
def merge(windows_a, windows_b) do
Map.merge(windows_a, windows_b, fn _key, v1, v2 ->
%{"curr_count" => c1, "prev_count" => p1} = v1
%{"curr_count" => c2, "prev_count" => p2} = v2
%{"curr_count" => c1 + c2, "prev_count" => p1 + p2}
end)
end
@impl Oban.Pro.Limiters.Rate.Algorithm
def track(windows, new_counts, period, prev_time, curr_time, _allowed) do
unix_diff = curr_time - prev_time
{mode, next_time} =
cond do
unix_diff > period * 2 -> {:dump_prev, curr_time}
unix_diff > period -> {:swap_prev, curr_time}
true -> {:bump_curr, prev_time}
end
new_windows =
for {key, count} <- new_counts,
not is_map_key(windows, key),
into: %{},
do: {key, %{"curr_count" => count, "prev_count" => 0}}
all_windows =
for {key, window} <- windows,
not match?(%{"curr_count" => 0, "prev_count" => 0}, window),
into: new_windows,
do: {key, update_window(window, Map.get(new_counts, key, 0), mode)}
{all_windows, next_time}
end
defp update_window(window, next_count, mode) do
%{"curr_count" => curr_count, "prev_count" => prev_count} = window
{new_curr, new_prev} =
case mode do
:dump_prev -> {next_count, 0}
:swap_prev -> {next_count, curr_count}
:bump_curr -> {curr_count + next_count, prev_count}
end
%{"curr_count" => new_curr, "prev_count" => new_prev}
end
end

86
vendor/oban_pro/lib/oban/pro/limiters/rate/token_bucket.ex vendored Normal file
View file

@ -0,0 +1,86 @@
defmodule Oban.Pro.Limiters.Rate.TokenBucket do
@moduledoc false
# Tokens refill continuously at a rate of `allowed / period` per second. Available demand is the
# number of tokens in the bucket.
@behaviour Oban.Pro.Limiters.Rate.Algorithm
@impl Oban.Pro.Limiters.Rate.Algorithm
def demand(windows, allowed, _period, _curr_time, _prev_time) do
total_consumed = sum_consumed(windows)
max(allowed - total_consumed, 0)
end
@impl Oban.Pro.Limiters.Rate.Algorithm
def partition_demands(windows, allowed, _period, _curr_time, _prev_time, available_keys) do
untracked =
for key <- available_keys,
not is_map_key(windows, key),
into: %{},
do: {key, allowed}
Enum.reduce(windows, untracked, fn {key, val}, acc ->
case max(allowed - val["consumed"], 0) do
demand when demand > 0 -> Map.put(acc, key, demand)
_zero -> acc
end
end)
end
@impl Oban.Pro.Limiters.Rate.Algorithm
def flatten(windows) do
consumed = Enum.reduce(windows, 0, fn {_key, val}, acc -> acc + val["consumed"] end)
[%{"consumed" => consumed}]
end
@impl Oban.Pro.Limiters.Rate.Algorithm
def merge(windows_a, windows_b) do
Map.merge(windows_a, windows_b, fn _key, v1, v2 ->
%{"consumed" => v1["consumed"] + v2["consumed"]}
end)
end
@doc """
Track job execution by updating consumed tokens.
Refills tokens based on elapsed time, then adds new consumption.
Returns the updated windows map and the next window_time.
"""
@impl Oban.Pro.Limiters.Rate.Algorithm
def track(windows, new_counts, period, prev_time, curr_time, allowed) do
refill_amount = calculate_refill(allowed, period, curr_time, prev_time)
new_consumed = Map.new(new_counts, fn {key, count} -> {key, %{"consumed" => count}} end)
all_windows =
Map.merge(windows, new_consumed, fn _key, existing, new ->
updated_consumed = max(existing["consumed"] - refill_amount, 0) + new["consumed"]
%{"consumed" => trunc(updated_consumed)}
end)
all_windows =
Enum.reduce(new_consumed, all_windows, fn {key, val}, acc ->
Map.put_new(acc, key, val)
end)
all_windows =
for {key, val} <- all_windows,
val["consumed"] > 0,
into: %{},
do: {key, val}
{all_windows, curr_time}
end
defp calculate_refill(allowed, period, curr_time, prev_time) do
elapsed = curr_time - prev_time
refill_rate = allowed / period
min(elapsed * refill_rate, allowed)
end
defp sum_consumed(windows) do
Enum.reduce(windows, 0, fn {_key, val}, acc -> acc + val["consumed"] end)
end
end

413
vendor/oban_pro/lib/oban/pro/migration.ex vendored Normal file
View file

@ -0,0 +1,413 @@
defmodule Oban.Pro.Migration do
@moduledoc """
Migrations create and modify the database tables and indexes Oban Pro needs to function.
If you use prefixes, or have multiple instances with different prefixes, you can specify the
prefix and create multiple tables in one migration:
## Usage
To use migrations in your application you'll need to generate an `Ecto.Migration` that wraps
calls to `Oban.Pro.Migration`:
```bash
mix ecto.gen.migration add_oban_pro
```
Open the generated migration and delegate the `up/0` and `down/0` functions to
`Oban.Pro.Migration`:
```elixir
defmodule MyApp.Repo.Migrations.AddObanPro do
use Ecto.Migration
def up, do: Oban.Pro.Migration.up()
def down, do: Oban.Pro.Migration.down()
end
```
This will run all of the necessary migrations for your database.
Now, run the migration to create the table:
```bash
mix ecto.migrate
```
## Concurrent Migrations
For systems with many retained jobs, you may want to create indexes concurrently to avoid
blocking writes to your tables during migration. The `only` option allows splitting migrations
into separate steps for schema changes and index creation:
```bash
mix ecto.gen.migration add_oban_pro_schemas
mix ecto.gen.migration add_oban_pro_indexes
```
The `schemas` migration will add all necessary tables and columns within a DDL transaction:
```elixir
defmodule MyApp.Repo.Migrations.AddObanProSchemas do
use Ecto.Migration
def up, do: Oban.Pro.Migration.up(only: :schemas)
def down, do: Oban.Pro.Migration.down(only: :schemas)
end
```
Then the `indexes` migration will add indexes concurrently, outside of a transaction:
```elixir
defmodule MyApp.Repo.Migrations.AddObanProIndexes do
use Ecto.Migration
# Not needed with use advisory locks, i.e. `migration_lock: :pg_advisory_lock`
@disable_migration_lock true
@disable_ddl_transaction true
def up, do: Oban.Pro.Migration.up(only: :indexes)
def down, do: Oban.Pro.Migration.down(only: :indexes)
end
```
#### When to Use Split Migrations
Use the default, combined approach when:
1. Setting up a development or test environment
2. Working with small tables where index creation is quick
3. Simplicity is preferred over non-blocking operations
Choose split migrations when:
1. Operating on a database where blocking writes could cause downtime
2. Working with large tables where index creation would block for an extended period
## Upgrading
Like `Oban`, migrations are versioned and you may need to run the migration again when new
`oban_pro` versions are released. To do this, generate a new migration:
```bash
mix ecto.gen.migration upgrade_oban_pro
```
Open the generated migration in your editor and call the `up` and `down` functions on
`Oban.Pro.Migration`, with an optional version number:
```elixir
defmodule MyApp.Repo.Migrations.UpgradeObanPro do
use Ecto.Migration
def up, do: Oban.Pro.Migration.up(version: "1.6.0")
def down, do: Oban.Pro.Migration.down(version: "1.6.0")
end
```
> #### Differences from Oban : .tip
>
> Unlike Oban, the migration versions correspond directly to the `oban_pro` package version.
> In addition, running `down/0` will only step back one version, not all the way down.
## Isolation with Prefixes
Oban supports namespacing through PostgreSQL schemas, therefore so does Oban Pro. Specify a
prefix within your migration:
```elixir
defmodule MyApp.Repo.Migrations.AddPrefixedObanPro do
use Ecto.Migration
def up, do: Oban.Pro.Migration.up(prefix: "private")
def down, do: Oban.Pro.Migration.down(prefix: "private")
end
```
"""
@behaviour Oban.Pro.Handler
use Ecto.Migration
require Logger
@versions ~w(1.0.0 1.4.0 1.5.0 1.6.0 1.7.0)
@impl Oban.Pro.Handler
def on_start do
event = [:oban, :supervisor, :init]
:telemetry.attach("oban.migration", event, &__MODULE__.handle_event/4, nil)
end
@impl Oban.Pro.Handler
def on_stop do
:telemetry.detach("oban.migration")
end
@doc false
def handle_event(_event, _timing, %{conf: conf}, _) do
prefix = conf.prefix
case :persistent_term.get(__MODULE__, %{}) do
%{^prefix => _} ->
:ok
versions ->
:persistent_term.put(__MODULE__, Map.put(versions, prefix, true))
last = List.last(@versions)
with_dynamic_repo(conf, fn repo ->
with :off <- Keyword.get(repo.config(), :pool, :off),
curr <- migrated_version(%{prefix: prefix, repo: repo}) do
cond do
is_nil(curr) ->
Logger.warning(
"Oban Pro for prefix #{inspect(prefix)} isn't migrated. Run migrations to create " <>
"the tables and indexes necessary for Oban Pro to function."
)
not String.contains?(curr, last) ->
Logger.warning(
"Oban Pro for prefix #{inspect(prefix)} is migrated to #{inspect(curr)}, but the " <>
"latest is #{inspect(last)}. Run migrations to upgrade to the latest version."
)
true ->
:ok
end
end
end)
end
end
defp with_dynamic_repo(%{get_dynamic_repo: nil, repo: repo}, fun) do
fun.(repo)
end
defp with_dynamic_repo(%{get_dynamic_repo: get_dyn_fun, repo: repo}, fun) do
prev_repo =
try do
repo.get_dynamic_repo()
rescue
_ -> nil
end
dyna_repo =
case get_dyn_fun do
{mod, dyn_fun, arg} -> apply(mod, dyn_fun, arg)
dyn_fun when is_function(dyn_fun, 0) -> dyn_fun.()
end
try do
repo.put_dynamic_repo(dyna_repo)
fun.(repo)
after
if prev_repo, do: repo.put_dynamic_repo(prev_repo)
end
end
@doc """
Run the `up` changes for all migrations between the initial version and the current version.
## Example
Run all migrations up to the current version:
Oban.Pro.Migration.up()
Run migrations up to a specified version:
Oban.Pro.Migration.up(version: "1.4.0")
Run migrations in an alternate prefix:
Oban.Pro.Migration.up(prefix: "payments")
"""
def up(opts \\ []) when is_list(opts) do
opts = with_defaults(opts)
curr = migrated_version(opts)
indx = migrated_version_index(curr, opts)
cond do
is_nil(indx) -> inner_change(0..opts.next, :up, curr, opts)
indx < opts.next -> inner_change((indx + 1)..opts.next, :up, curr, opts)
true -> :ok
end
end
@doc """
Run the `down` changes for migrations between the current version and the previous version.
This behaviour differs from `Oban.Migration` by only stepping down **one version** per
invocation. You must explicitly specify version `1.0.0` to rollback all the way and drop initial
tables.
## Example
Run migrations down from the current version to the previous:
Oban.Pro.Migration.down()
Run migrations down to and including a specified version:
Oban.Pro.Migration.down(version: "1.4.0")
Run all down migrations back to the original:
Oban.Pro.Migration.down(version: "1.0.0")
Run migrations in an alternate prefix:
Oban.Pro.Migration.down(prefix: "payments")
"""
def down(opts \\ []) when is_list(opts) do
opts = with_defaults(opts)
curr = migrated_version(opts)
indx = migrated_version_index(curr, opts)
if is_integer(indx) and indx >= opts.next do
inner_change(indx..opts.next//-1, :down, curr, opts)
end
end
@doc false
def migrated_version(opts) when is_map(opts) do
repo = Map.get_lazy(opts, :repo, &repo/0)
opts = with_defaults(opts)
query = """
SELECT pg_catalog.obj_description(pg_class.oid, 'pg_class')
FROM pg_class
LEFT JOIN pg_namespace ON pg_namespace.oid = pg_class.relnamespace
WHERE pg_class.relname = 'oban_producers'
AND pg_namespace.nspname = '#{opts.escaped}'
"""
case repo.query(query, [], log: false) do
{:ok, %{rows: [[version]]}} when is_binary(version) -> version
_ -> nil
end
end
@doc false
def partitioned?(opts) do
query = """
SELECT EXISTS (
SELECT 1
FROM pg_tables
WHERE schemaname = '#{opts.prefix}' AND tablename = 'oban_jobs_completed'
)
"""
{:ok, %{rows: [[partitioned?]]}} = repo().query(query, [], log: false)
partitioned?
end
defp migrated_version_index(version, opts) do
[schemas_version, indexes_version] = normalize_recorded(version)
case opts.only do
[:schemas] -> find_version_index(schemas_version)
[:indexes] -> find_version_index(indexes_version)
_ -> find_version_index(schemas_version)
end
end
defp inner_change(_curr_index..next_index//_ = range, direction, current, opts) do
for index <- range do
version = Enum.at(@versions, index)
migration = Module.concat(Oban.Pro.Migrations, "V#{String.replace(version, ".", "")}")
{do_ddl, do_idx} =
if direction == :up do
{:up_ddl, :up_idx}
else
{:down_ddl, :down_idx}
end
if :schemas in opts.only, do: apply(migration, do_ddl, [opts])
if :indexes in opts.only, do: apply(migration, do_idx, [opts])
flush()
end
case direction do
:up -> record_version(next_index, current, opts)
:down -> record_version(next_index - 1, current, opts)
end
end
defp record_version(-1, _recorded, _opts), do: :ok
defp record_version(index, recorded, opts) do
[recorded_schemas, recorded_indexes] = normalize_recorded(recorded)
prefix = Enum.at(@versions, index)
versions =
case opts.only do
[:indexes] -> [recorded_schemas, "#{prefix}-indexes"]
[:schemas] -> ["#{prefix}-schemas", recorded_indexes]
_ -> ["#{prefix}-schemas", "#{prefix}-indexes"]
end
versions = versions |> Enum.filter(& &1) |> Enum.join(",")
execute "COMMENT ON TABLE #{opts.quoted}.oban_producers IS '#{versions}'"
end
defp with_defaults(opts) do
prefix = Access.get(opts, :prefix, "public")
unlogged = Access.get(opts, :unlogged, true)
version = Access.get(opts, :version, List.last(@versions))
generated_columns = Access.get(opts, :generated_columns, false)
only =
opts
|> Access.get(:only, [:schemas, :indexes])
|> List.wrap()
%{
concurrently: only == [:indexes],
escaped: String.replace(prefix, "'", "\\'"),
generated_columns: generated_columns,
next: find_version_index(version),
only: only,
prefix: prefix,
quoted: inspect(prefix),
unlogged: unlogged,
version: version
}
end
defp find_version_index(nil), do: nil
defp find_version_index(version) do
version = String.replace(version, ["-schemas", "-indexes"], "")
Enum.find_index(@versions, &(&1 == version))
end
defp normalize_recorded(nil), do: [nil, nil]
defp normalize_recorded(version) do
case String.split(version, ",", parts: 2) do
[schemas] when is_binary(schemas) ->
if String.ends_with?(schemas, "-schemas") do
[schemas, nil]
else
["#{schemas}-schemas", "#{schemas}-indexes"]
end
recorded ->
recorded
end
end
end

196
vendor/oban_pro/lib/oban/pro/migrations/dynamic_partitioner.ex vendored Normal file
View file

@ -0,0 +1,196 @@
defmodule Oban.Pro.Migrations.DynamicPartitioner do
@moduledoc false
use Ecto.Migration
alias Oban.Config
alias Oban.Pro.Plugins.DynamicPartitioner
def change(opts \\ []) do
case direction() do
:up -> up(opts)
:down -> down(opts)
end
end
def up(opts \\ []) do
prefix = Keyword.get(opts, :prefix, "public")
quoted = inspect(prefix)
if Keyword.get(opts, :create_schema, prefix != "public") do
execute "CREATE SCHEMA IF NOT EXISTS #{quoted}", ""
end
execute """
ALTER TABLE IF EXISTS #{quoted}.oban_jobs RENAME TO oban_jobs_old
"""
execute """
ALTER INDEX IF EXISTS #{quoted}.oban_jobs_pkey RENAME TO oban_jobs_pkey_old
"""
execute """
ALTER INDEX IF EXISTS #{quoted}.oban_jobs_args_index RENAME TO oban_jobs_args_index_old
"""
execute """
ALTER INDEX IF EXISTS #{quoted}.oban_jobs_meta_index RENAME TO oban_jobs_meta_index_old
"""
execute """
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_type
WHERE typname = 'oban_job_state'
AND typnamespace = '#{quoted}'::regnamespace::oid) THEN
CREATE TYPE #{quoted}.oban_job_state AS ENUM (
'available',
'suspended',
'scheduled',
'executing',
'retryable',
'completed',
'discarded',
'cancelled'
);
END IF;
END$$;
"""
create table(:oban_jobs,
primary_key: false,
prefix: prefix,
options: "PARTITION BY LIST (state)"
) do
add :id, :bigserial
add :state, :"#{quoted}.oban_job_state", null: false, default: "available"
add :queue, :text, null: false, default: "default"
add :worker, :text, null: false
add :attempt, :smallint, null: false, default: 0
add :max_attempts, :smallint, null: false, default: 20
add :priority, :smallint, null: false, default: 0
add :args, :map, null: false
add :meta, :map, null: false
add :attempted_by, {:array, :text}
add :errors, {:array, :map}, null: false, default: []
add :tags, {:array, :text}, null: false, default: []
add :inserted_at, :utc_datetime_usec,
null: false,
default: fragment("timezone('UTC', now())")
add :scheduled_at, :utc_datetime_usec,
null: false,
default: fragment("timezone('UTC', now())")
add :attempted_at, :utc_datetime_usec
add :cancelled_at, :utc_datetime_usec
add :completed_at, :utc_datetime_usec
add :discarded_at, :utc_datetime_usec
end
create_if_not_exists index(:oban_jobs, [:id], prefix: prefix)
create_if_not_exists index(:oban_jobs, [:args], using: :gin, prefix: prefix)
create_if_not_exists index(:oban_jobs, [:meta], using: :gin, prefix: prefix)
execute "COMMENT ON TABLE #{quoted}.oban_jobs IS '∞'", ""
create_if_not_exists table(:oban_peers, primary_key: false, prefix: prefix) do
add :name, :text, null: false, primary_key: true
add :node, :text, null: false
add :started_at, :utc_datetime_usec, null: false
add :expires_at, :utc_datetime_usec, null: false
end
execute "ALTER TABLE #{quoted}.oban_peers SET UNLOGGED", ""
# Collocate available and executing to prevent concurrency anomalies such as:
# > ERROR: tuple to be locked was already moved to another partition due to concurrent update
execute """
CREATE TABLE #{quoted}.oban_jobs_incomplete PARTITION OF #{quoted}.oban_jobs
FOR VALUES IN ('available', 'executing', 'scheduled', 'retryable')
"""
create_if_not_exists index(
:oban_jobs_incomplete,
[:state, :queue, :priority, :scheduled_at, :id],
prefix: prefix
)
date_partition? = fn ->
if Code.ensure_loaded?(Mix), do: Mix.env() in ~w(dev prod)a, else: true
end
{part_states, date_states} =
if Keyword.get_lazy(opts, :date_partition?, date_partition?) do
{[], ~w(completed cancelled discarded)}
else
{~w(completed cancelled discarded), []}
end
for state <- part_states do
execute """
CREATE TABLE #{quoted}.oban_jobs_#{state} PARTITION OF #{quoted}.oban_jobs
FOR VALUES IN ('#{state}')
"""
create_if_not_exists index("oban_jobs_#{state}", [:queue, :scheduled_at, :id],
prefix: prefix
)
end
for state <- date_states do
execute """
CREATE TABLE #{quoted}.oban_jobs_#{state} PARTITION OF #{quoted}.oban_jobs
FOR VALUES IN ('#{state}')
PARTITION BY RANGE (#{state}_at)
"""
date = Date.utc_today()
next = Date.add(date, 1)
safe = Calendar.strftime(date, "%Y%m%d")
execute """
CREATE TABLE IF NOT EXISTS #{quoted}.oban_jobs_#{state}_#{safe}
PARTITION OF #{quoted}.oban_jobs_#{state}
FOR VALUES FROM ('#{date} 00:00:00') TO ('#{next} 00:00:00')
"""
end
end
def down(opts \\ []) do
prefix = Keyword.get(opts, :prefix, "public")
quoted = inspect(prefix)
drop_if_exists table(:oban_jobs, prefix: prefix)
execute """
ALTER TABLE IF EXISTS #{quoted}.oban_jobs_old RENAME TO oban_jobs
"""
execute """
ALTER INDEX IF EXISTS #{quoted}.oban_jobs_pkey_old RENAME TO oban_jobs_pkey
"""
execute """
ALTER INDEX IF EXISTS #{quoted}.oban_jobs_args_index_old RENAME TO oban_jobs_args_index
"""
execute """
ALTER INDEX IF EXISTS #{quoted}.oban_jobs_meta_index_old RENAME TO oban_jobs_meta_index
"""
end
def backfill(opts \\ []) do
if direction() == :up do
# Ensure the oban_jobs_old table exists if this is ran in a single migration
flush()
DynamicPartitioner.backfill_jobs(conf(), opts)
end
end
defp conf, do: Config.new(repo: repo())
end

97
vendor/oban_pro/lib/oban/pro/migrations/v1_0_0.ex vendored Normal file
View file

@ -0,0 +1,97 @@
defmodule Oban.Pro.Migrations.DynamicCron do
@moduledoc false
use Ecto.Migration
def change(_opts \\ []), do: :ok
def up(_opts \\ []), do: :ok
def down(_opts \\ []), do: :ok
end
defmodule Oban.Pro.Migrations.DynamicQueues do
@moduledoc false
use Ecto.Migration
def change(_opts \\ []), do: :ok
def up(_opts \\ []), do: :ok
def down(_opts \\ []), do: :ok
end
defmodule Oban.Pro.Migrations.Producers do
@moduledoc false
use Ecto.Migration
def change(_opts \\ []), do: :ok
def up(_opts \\ []), do: :ok
def down(_opts \\ []), do: :ok
end
defmodule Oban.Pro.Migrations.V100 do
@moduledoc false
use Ecto.Migration
defmacrop utc_now do
quote do
fragment("timezone('utc', now())")
end
end
def up_ddl(opts) do
create_if_not_exists table(:oban_producers, primary_key: false, prefix: opts.prefix) do
add :uuid, :uuid, primary_key: true, null: false
add :name, :text, null: false
add :node, :text, null: false
add :queue, :text, null: false
add :meta, :map, null: false, default: %{}
add :started_at, :utc_datetime_usec, null: false, default: utc_now()
add :updated_at, :utc_datetime_usec, null: false, default: utc_now()
end
if opts.unlogged do
execute "ALTER TABLE #{opts.quoted}.oban_producers SET UNLOGGED"
end
create_if_not_exists table(:oban_crons, primary_key: false, prefix: opts.prefix) do
add :name, :text, primary_key: true, null: false
add :expression, :text, null: false
add :worker, :text, null: false
add :opts, :map, null: false
add :insertions, {:array, :utc_datetime_usec}, default: [], null: false
add :paused, :boolean, null: false, default: false
add :lock_version, :integer, default: 1
add :inserted_at, :utc_datetime_usec, null: false, default: utc_now()
add :updated_at, :utc_datetime_usec, null: false, default: utc_now()
end
create_if_not_exists table(:oban_queues, primary_key: false, prefix: opts.prefix) do
add :name, :text, primary_key: true, null: false
add :opts, :map, null: false, default: %{}
add :only, :map, null: false, default: %{}
add :lock_version, :integer, default: 1
add :inserted_at, :utc_datetime_usec, null: false, default: utc_now()
add :updated_at, :utc_datetime_usec, null: false, default: utc_now()
end
end
def up_idx(_opts), do: :ok
def down_ddl(opts) do
drop_if_exists table(:oban_queues, prefix: opts.prefix)
drop_if_exists table(:oban_crons, prefix: opts.prefix)
drop_if_exists table(:oban_producers, prefix: opts.prefix)
end
def down_idx(_opts), do: :ok
end

60
vendor/oban_pro/lib/oban/pro/migrations/v1_4_0.ex vendored Normal file
View file

@ -0,0 +1,60 @@
defmodule Oban.Pro.Migrations.Batch do
@moduledoc false
use Ecto.Migration
def change(_opts \\ []), do: :ok
def up(_opts \\ []), do: :ok
def down(_opts \\ []), do: :ok
end
defmodule Oban.Pro.Migrations.Workflow do
@moduledoc false
use Ecto.Migration
def change(_opts \\ []), do: :ok
def up(_opts \\ []), do: :ok
def down(_opts \\ []), do: :ok
end
defmodule Oban.Pro.Migrations.V140 do
@moduledoc false
use Ecto.Migration
def up_ddl(opts) do
alter table(:oban_crons, prefix: opts.prefix) do
add_if_not_exists :insertions, {:array, :utc_datetime_usec}, default: []
end
end
def up_idx(opts) do
create_if_not_exists index(
:oban_jobs,
[:state, "(meta->>'workflow_id')", "(meta->>'name')"],
concurrently: opts.concurrently,
name: :oban_jobs_state_meta_workflow_index,
prefix: opts.prefix,
where: "meta ? 'workflow_id'"
)
end
def down_ddl(opts) do
alter table(:oban_crons, prefix: opts.prefix) do
remove_if_exists :insertions, {:array, :utc_datetime_usec}
end
end
def down_idx(opts) do
drop_if_exists index(:oban_jobs, [],
name: :oban_jobs_state_meta_workflow_index,
concurrently: opts.concurrently,
prefix: opts.prefix
)
end
end

118
vendor/oban_pro/lib/oban/pro/migrations/v1_5_0.ex vendored Normal file
View file

@ -0,0 +1,118 @@
defmodule Oban.Pro.Migrations.V150 do
@moduledoc false
use Ecto.Migration
import Oban.Pro.Migration, only: [partitioned?: 1]
def up_ddl(opts) do
if not partitioned?(opts) do
execute """
CREATE OR REPLACE FUNCTION #{opts.quoted}.oban_state_to_bit(state #{opts.quoted}.oban_job_state)
RETURNS jsonb AS $$
SELECT CASE
WHEN state = 'scheduled' THEN '0'::jsonb
WHEN state = 'available' THEN '1'::jsonb
WHEN state = 'executing' THEN '2'::jsonb
WHEN state = 'retryable' THEN '3'::jsonb
WHEN state = 'completed' THEN '4'::jsonb
WHEN state = 'cancelled' THEN '5'::jsonb
WHEN state = 'discarded' THEN '6'::jsonb
END;
$$ LANGUAGE SQL IMMUTABLE STRICT
"""
if opts.generated_columns do
alter table(:oban_jobs, prefix: opts.prefix) do
add_if_not_exists :uniq_key, :text,
generated: """
ALWAYS AS (CASE WHEN meta->'uniq_bmp' @> #{opts.quoted}.oban_state_to_bit(state) THEN meta->>'uniq_key' END) STORED
"""
end
end
end
end
def up_idx(opts) do
cond do
partitioned?(opts) ->
create_if_not_exists index(
:oban_jobs,
["(meta->>'uniq_key')"],
concurrently: opts.concurrently,
name: :oban_jobs_unique_index,
prefix: opts.prefix,
where: "meta ? 'uniq_key'"
)
opts.generated_columns ->
create_if_not_exists index(
:oban_jobs,
[:uniq_key],
concurrently: opts.concurrently,
name: :oban_jobs_unique_index,
prefix: opts.prefix,
unique: true,
where: "uniq_key IS NOT NULL"
)
true ->
create_if_not_exists index(
:oban_jobs,
["(meta->>'uniq_key')"],
concurrently: opts.concurrently,
name: :oban_jobs_unique_index,
prefix: opts.prefix,
unique: true,
where:
"meta ? 'uniq_key' AND jsonb_contains(meta->'uniq_bmp', #{opts.quoted}.oban_state_to_bit(state))"
)
end
create_if_not_exists index(
:oban_jobs,
[:state, "(meta->>'batch_id')", "(meta->>'callback')"],
concurrently: opts.concurrently,
name: :oban_jobs_batch_index,
prefix: opts.prefix,
where: "meta ? 'batch_id'"
)
create_if_not_exists index(
:oban_jobs,
[:state, "(meta->>'chain_id')", "(meta->>'on_hold')"],
concurrently: opts.concurrently,
name: :oban_jobs_chain_index,
prefix: opts.prefix,
where: "meta ? 'chain_id'"
)
end
def down_ddl(opts) do
alter table(:oban_jobs, prefix: opts.prefix) do
remove_if_exists :uniq_key, :text
end
execute "DROP FUNCTION IF EXISTS #{opts.quoted}.oban_state_to_bit CASCADE"
end
def down_idx(opts) do
drop_if_exists index(:oban_jobs, [],
name: :oban_jobs_batch_index,
concurrently: opts.concurrently,
prefix: opts.prefix
)
drop_if_exists index(:oban_jobs, [],
name: :oban_jobs_chain_index,
concurrently: opts.concurrently,
prefix: opts.prefix
)
drop_if_exists index(:oban_jobs, [],
name: :oban_jobs_unique_index,
concurrently: opts.concurrently,
prefix: opts.prefix
)
end
end

122
vendor/oban_pro/lib/oban/pro/migrations/v1_6_0.ex vendored Normal file
View file

@ -0,0 +1,122 @@
defmodule Oban.Pro.Migrations.V160 do
@moduledoc false
use Ecto.Migration
def up_ddl(opts) do
# Partitioning
if opts.generated_columns do
alter table(:oban_jobs, prefix: opts.prefix) do
add_if_not_exists :partition_key, :text,
generated: """
ALWAYS AS (CASE WHEN meta ? 'partition_key' THEN meta->>'partition_key' END) STORED
"""
end
end
# Dynamic Queues
alter table(:oban_queues, prefix: opts.prefix) do
add_if_not_exists :hash, :text
end
end
def up_idx(opts) do
# Partitioning
if opts.generated_columns do
create_if_not_exists index(
:oban_jobs,
[:partition_key, :queue, :priority, :scheduled_at, :id],
concurrently: opts.concurrently,
name: :oban_jobs_partition_index,
prefix: opts.prefix,
where: "state = 'available' AND partition_key IS NOT NULL"
)
else
create_if_not_exists index(
:oban_jobs,
["(meta->>'partition_key')", :queue, :priority, :scheduled_at, :id],
concurrently: opts.concurrently,
name: :oban_jobs_partition_index,
prefix: opts.prefix,
where: "state = 'available' AND meta ? 'partition_key'"
)
end
# Workflows
drop_if_exists index(:oban_jobs, [],
concurrently: opts.concurrently,
name: :oban_jobs_state_meta_workflow_index,
prefix: opts.prefix
)
create_if_not_exists index(
:oban_jobs,
[
"(meta->>'workflow_id')",
:state,
"(meta->>'on_hold')",
"(meta->>'name')"
],
concurrently: opts.concurrently,
name: :oban_jobs_workflow_index,
prefix: opts.prefix,
where: "meta ? 'workflow_id'"
)
create_if_not_exists index(
:oban_jobs,
[
"(meta->>'sup_workflow_id')",
:state,
"(meta->>'on_hold')"
],
concurrently: opts.concurrently,
name: :oban_jobs_sup_workflow_index,
prefix: opts.prefix,
where: "meta ? 'sup_workflow_id'"
)
end
def down_ddl(opts) do
alter table(:oban_queues, prefix: opts.prefix) do
remove_if_exists :hash, :text
end
alter table(:oban_jobs, prefix: opts.prefix) do
remove_if_exists :partition_key, :text
end
end
def down_idx(opts) do
drop_if_exists index(:oban_jobs, [],
name: :oban_jobs_partition_index,
concurrently: opts.concurrently,
prefix: opts.prefix
)
drop_if_exists index(:oban_jobs, [],
name: :oban_jobs_workflow_index,
concurrently: opts.concurrently,
prefix: opts.prefix
)
drop_if_exists index(:oban_jobs, [],
name: :oban_jobs_sup_workflow_index,
concurrently: opts.concurrently,
prefix: opts.prefix
)
create_if_not_exists index(
:oban_jobs,
[:state, "(meta->>'workflow_id')", "(meta->>'name')"],
concurrently: opts.concurrently,
name: :oban_jobs_state_meta_workflow_index,
prefix: opts.prefix,
where: "meta ? 'workflow_id'"
)
end
end

416
vendor/oban_pro/lib/oban/pro/migrations/v1_7_0.ex vendored Normal file
View file

@ -0,0 +1,416 @@
defmodule Oban.Pro.Migrations.V170 do
@moduledoc false
use Ecto.Migration
import Oban.Pro.Migration, only: [partitioned?: 1]
def up_ddl(opts) do
execute """
CREATE OR REPLACE FUNCTION #{opts.quoted}.oban_state_to_bit(state #{opts.quoted}.oban_job_state)
RETURNS jsonb AS $$
SELECT CASE
WHEN state = 'scheduled' THEN '0'::jsonb
WHEN state = 'available' THEN '1'::jsonb
WHEN state = 'executing' THEN '2'::jsonb
WHEN state = 'retryable' THEN '3'::jsonb
WHEN state = 'completed' THEN '4'::jsonb
WHEN state = 'cancelled' THEN '5'::jsonb
WHEN state = 'discarded' THEN '6'::jsonb
WHEN state = 'suspended' THEN '7'::jsonb
END;
$$ LANGUAGE SQL IMMUTABLE STRICT
"""
create_if_not_exists table(:oban_workflows, primary_key: false, prefix: opts.prefix) do
add :id, :text, primary_key: true
add :name, :text
add :parent_id, :text
add :suspended, :integer, null: false, default: 0
add :available, :integer, null: false, default: 0
add :scheduled, :integer, null: false, default: 0
add :executing, :integer, null: false, default: 0
add :retryable, :integer, null: false, default: 0
add :completed, :integer, null: false, default: 0
add :cancelled, :integer, null: false, default: 0
add :discarded, :integer, null: false, default: 0
add :inserted_at, :utc_datetime_usec, null: false
add :started_at, :utc_datetime_usec
add :completed_at, :utc_datetime_usec
add :meta, :map, null: false, default: %{}
end
execute """
ALTER TABLE #{opts.quoted}.oban_workflows
ADD COLUMN IF NOT EXISTS state TEXT GENERATED ALWAYS AS (
CASE
WHEN (suspended + available + scheduled + executing + retryable) > 0 THEN 'executing'
WHEN cancelled > 0 AND NOT COALESCE((meta->>'ignore_cancelled')::boolean, false) THEN 'cancelled'
WHEN discarded > 0 AND NOT COALESCE((meta->>'ignore_discarded')::boolean, false) THEN 'discarded'
ELSE 'completed'
END
) STORED
"""
execute """
CREATE OR REPLACE FUNCTION #{opts.quoted}.oban_workflows_update() RETURNS TRIGGER AS $$
BEGIN
UPDATE #{opts.quoted}.oban_workflows wf SET
suspended = GREATEST(0, wf.suspended + COALESCE(d.suspended, 0)),
available = GREATEST(0, wf.available + COALESCE(d.available, 0)),
scheduled = GREATEST(0, wf.scheduled + COALESCE(d.scheduled, 0)),
executing = GREATEST(0, wf.executing + COALESCE(d.executing, 0)),
retryable = GREATEST(0, wf.retryable + COALESCE(d.retryable, 0)),
completed = GREATEST(0, wf.completed + COALESCE(d.completed, 0)),
cancelled = GREATEST(0, wf.cancelled + COALESCE(d.cancelled, 0)),
discarded = GREATEST(0, wf.discarded + COALESCE(d.discarded, 0)),
started_at = CASE
WHEN wf.started_at IS NULL AND d.executing > 0 THEN NOW()
ELSE wf.started_at
END,
completed_at = CASE
WHEN wf.completed_at IS NULL
AND (wf.suspended + wf.available + wf.scheduled + wf.executing + wf.retryable
+ COALESCE(d.suspended, 0) + COALESCE(d.available, 0) + COALESCE(d.scheduled, 0)
+ COALESCE(d.executing, 0) + COALESCE(d.retryable, 0)) <= 0
THEN NOW()
ELSE wf.completed_at
END
FROM (
SELECT
COALESCE(n.meta, o.meta)->>'workflow_id' AS workflow_id,
SUM(CASE WHEN n.state = 'suspended' THEN 1 WHEN o.state = 'suspended' THEN -1 ELSE 0 END) AS suspended,
SUM(CASE WHEN n.state = 'available' THEN 1 WHEN o.state = 'available' THEN -1 ELSE 0 END) AS available,
SUM(CASE WHEN n.state = 'scheduled' THEN 1 WHEN o.state = 'scheduled' THEN -1 ELSE 0 END) AS scheduled,
SUM(CASE WHEN n.state = 'executing' THEN 1 WHEN o.state = 'executing' THEN -1 ELSE 0 END) AS executing,
SUM(CASE WHEN n.state = 'retryable' THEN 1 WHEN o.state = 'retryable' THEN -1 ELSE 0 END) AS retryable,
SUM(CASE WHEN n.state = 'completed' THEN 1 WHEN o.state = 'completed' THEN -1 ELSE 0 END) AS completed,
SUM(CASE WHEN n.state = 'cancelled' THEN 1 WHEN o.state = 'cancelled' THEN -1 ELSE 0 END) AS cancelled,
SUM(CASE WHEN n.state = 'discarded' THEN 1 WHEN o.state = 'discarded' THEN -1 ELSE 0 END) AS discarded
FROM new_table n
FULL OUTER JOIN old_table o ON n.id = o.id
WHERE (n.meta ? 'workflow_id' OR o.meta ? 'workflow_id')
AND n.state IS DISTINCT FROM o.state
GROUP BY COALESCE(n.meta, o.meta)->>'workflow_id'
) d
WHERE wf.id = d.workflow_id;
RETURN NULL;
END;
$$ LANGUAGE plpgsql;
"""
execute """
CREATE OR REPLACE FUNCTION #{opts.quoted}.oban_workflows_delete() RETURNS TRIGGER AS $$
BEGIN
UPDATE #{opts.quoted}.oban_workflows wf SET
suspended = GREATEST(0, wf.suspended - COALESCE(d.suspended, 0)),
available = GREATEST(0, wf.available - COALESCE(d.available, 0)),
scheduled = GREATEST(0, wf.scheduled - COALESCE(d.scheduled, 0)),
executing = GREATEST(0, wf.executing - COALESCE(d.executing, 0)),
retryable = GREATEST(0, wf.retryable - COALESCE(d.retryable, 0)),
completed = GREATEST(0, wf.completed - COALESCE(d.completed, 0)),
cancelled = GREATEST(0, wf.cancelled - COALESCE(d.cancelled, 0)),
discarded = GREATEST(0, wf.discarded - COALESCE(d.discarded, 0))
FROM (
SELECT
o.meta->>'workflow_id' AS workflow_id,
COUNT(*) FILTER (WHERE o.state = 'suspended') AS suspended,
COUNT(*) FILTER (WHERE o.state = 'available') AS available,
COUNT(*) FILTER (WHERE o.state = 'scheduled') AS scheduled,
COUNT(*) FILTER (WHERE o.state = 'executing') AS executing,
COUNT(*) FILTER (WHERE o.state = 'retryable') AS retryable,
COUNT(*) FILTER (WHERE o.state = 'completed') AS completed,
COUNT(*) FILTER (WHERE o.state = 'cancelled') AS cancelled,
COUNT(*) FILTER (WHERE o.state = 'discarded') AS discarded
FROM old_table o
WHERE o.meta ? 'workflow_id'
GROUP BY o.meta->>'workflow_id'
) d
WHERE wf.id = d.workflow_id;
RETURN NULL;
END;
$$ LANGUAGE plpgsql;
"""
execute """
CREATE OR REPLACE TRIGGER oban_workflows_update_trigger
AFTER UPDATE ON #{opts.quoted}.oban_jobs
REFERENCING OLD TABLE AS old_table NEW TABLE AS new_table
FOR EACH STATEMENT
EXECUTE FUNCTION #{opts.quoted}.oban_workflows_update();
"""
execute """
CREATE OR REPLACE TRIGGER oban_workflows_delete_trigger
AFTER DELETE ON #{opts.quoted}.oban_jobs
REFERENCING OLD TABLE AS old_table
FOR EACH STATEMENT
EXECUTE FUNCTION #{opts.quoted}.oban_workflows_delete();
"""
end
def up_idx(opts) do
# Replace column-based indexes with expression indexes for unique and partition keys. This
# enables querying meta fields directly without relying on generated columns. Partitioned
# tables already use expression indexes from v1.5.0.
if not opts.generated_columns and not partitioned?(opts) do
drop_if_exists index(:oban_jobs, [],
name: :oban_jobs_unique_index,
concurrently: opts.concurrently,
prefix: opts.prefix
)
drop_if_exists index(:oban_jobs, [],
name: :oban_jobs_partition_index,
concurrently: opts.concurrently,
prefix: opts.prefix
)
create_if_not_exists index(
:oban_jobs,
["(meta->>'uniq_key')"],
concurrently: opts.concurrently,
name: :oban_jobs_unique_index,
prefix: opts.prefix,
unique: true,
where:
"meta ? 'uniq_key' AND jsonb_contains(meta->'uniq_bmp', #{opts.quoted}.oban_state_to_bit(state))"
)
create_if_not_exists index(
:oban_jobs,
["(meta->>'partition_key')", :queue, :priority, :scheduled_at, :id],
concurrently: opts.concurrently,
name: :oban_jobs_partition_index,
prefix: opts.prefix,
where: "state = 'available' AND meta ? 'partition_key'"
)
end
# Rename old workflow/chain indexes that use on_hold
execute "ALTER INDEX IF EXISTS #{opts.quoted}.oban_jobs_workflow_index RENAME TO oban_jobs_workflow_index_old"
execute "ALTER INDEX IF EXISTS #{opts.quoted}.oban_jobs_sup_workflow_index RENAME TO oban_jobs_sup_workflow_index_old"
execute "ALTER INDEX IF EXISTS #{opts.quoted}.oban_jobs_chain_index RENAME TO oban_jobs_chain_index_old"
# Create new indexes without on_hold
create_if_not_exists index(
:oban_jobs,
["(meta->>'workflow_id')", :state, "(meta->>'name')"],
concurrently: opts.concurrently,
name: :oban_jobs_workflow_index,
prefix: opts.prefix,
where: "meta ? 'workflow_id'"
)
create_if_not_exists index(
:oban_jobs,
["(meta->>'sup_workflow_id')", :state],
concurrently: opts.concurrently,
name: :oban_jobs_sup_workflow_index,
prefix: opts.prefix,
where: "meta ? 'sup_workflow_id'"
)
create_if_not_exists index(
:oban_jobs,
[:state, "(meta->>'chain_id')"],
concurrently: opts.concurrently,
name: :oban_jobs_chain_index,
prefix: opts.prefix,
where: "meta ? 'chain_id'"
)
create_if_not_exists index(
:oban_jobs,
[:state, "(meta->>'chunk_id')"],
concurrently: opts.concurrently,
name: :oban_jobs_chunk_index,
prefix: opts.prefix,
where: "meta ? 'chunk_id'"
)
# Replace the non-partial state timestamp indexes from core Oban with partial indexes.
# These partial indexes are smaller and more efficient for pruning operations.
drop_if_exists index(:oban_jobs, [],
name: :oban_jobs_state_cancelled_at_index,
concurrently: opts.concurrently,
prefix: opts.prefix
)
drop_if_exists index(:oban_jobs, [],
name: :oban_jobs_state_discarded_at_index,
concurrently: opts.concurrently,
prefix: opts.prefix
)
create_if_not_exists index(
:oban_jobs,
[:cancelled_at],
concurrently: opts.concurrently,
name: :oban_jobs_cancelled_at_index,
prefix: opts.prefix,
where: "state = 'cancelled'"
)
create_if_not_exists index(
:oban_jobs,
[:discarded_at],
concurrently: opts.concurrently,
name: :oban_jobs_discarded_at_index,
prefix: opts.prefix,
where: "state = 'discarded'"
)
create_if_not_exists index(
:oban_jobs,
[:completed_at],
concurrently: opts.concurrently,
name: :oban_jobs_completed_at_index,
prefix: opts.prefix,
where: "state = 'completed'"
)
# Add a staging index for efficiently finding scheduled/retryable jobs ready to run.
# This speeds up the stage_jobs operation which transitions jobs to 'available'.
create_if_not_exists index(
:oban_jobs,
[:scheduled_at, :id],
concurrently: opts.concurrently,
name: :oban_jobs_staging_index,
prefix: opts.prefix,
where: "state IN ('scheduled', 'retryable')"
)
# Workflow table indexes
create_if_not_exists index(
:oban_workflows,
[:state, :name],
concurrently: opts.concurrently,
prefix: opts.prefix
)
create_if_not_exists index(
:oban_workflows,
[:parent_id],
concurrently: opts.concurrently,
prefix: opts.prefix,
where: "parent_id IS NOT NULL"
)
create_if_not_exists index(
:oban_workflows,
[:name],
concurrently: opts.concurrently,
name: :oban_workflows_unique_name,
prefix: opts.prefix,
unique: true,
where: "state != 'completed' AND meta ? 'unique'"
)
end
def down_ddl(opts) do
execute "DROP TRIGGER IF EXISTS oban_workflows_update_trigger ON #{opts.quoted}.oban_jobs"
execute "DROP TRIGGER IF EXISTS oban_workflows_delete_trigger ON #{opts.quoted}.oban_jobs"
execute "DROP FUNCTION IF EXISTS #{opts.quoted}.oban_workflows_update()"
execute "DROP FUNCTION IF EXISTS #{opts.quoted}.oban_workflows_delete()"
drop_if_exists table(:oban_workflows, prefix: opts.prefix)
end
def down_idx(opts) do
drop_if_exists index(:oban_workflows, [:state, :name],
concurrently: opts.concurrently,
prefix: opts.prefix
)
drop_if_exists index(:oban_workflows, [:parent_id],
concurrently: opts.concurrently,
prefix: opts.prefix
)
drop_if_exists index(:oban_workflows, [],
name: :oban_workflows_unique_name,
concurrently: opts.concurrently,
prefix: opts.prefix
)
# Drop new indexes
drop_if_exists index(:oban_jobs, [],
name: :oban_jobs_workflow_index,
concurrently: opts.concurrently,
prefix: opts.prefix
)
drop_if_exists index(:oban_jobs, [],
name: :oban_jobs_sup_workflow_index,
concurrently: opts.concurrently,
prefix: opts.prefix
)
drop_if_exists index(:oban_jobs, [],
name: :oban_jobs_chain_index,
concurrently: opts.concurrently,
prefix: opts.prefix
)
drop_if_exists index(:oban_jobs, [],
name: :oban_jobs_chunk_index,
concurrently: opts.concurrently,
prefix: opts.prefix
)
# Drop the partial state timestamp indexes and restore the original non-partial indexes
drop_if_exists index(:oban_jobs, [],
name: :oban_jobs_staging_index,
concurrently: opts.concurrently,
prefix: opts.prefix
)
drop_if_exists index(:oban_jobs, [],
name: :oban_jobs_completed_at_index,
concurrently: opts.concurrently,
prefix: opts.prefix
)
drop_if_exists index(:oban_jobs, [],
name: :oban_jobs_discarded_at_index,
concurrently: opts.concurrently,
prefix: opts.prefix
)
drop_if_exists index(:oban_jobs, [],
name: :oban_jobs_cancelled_at_index,
concurrently: opts.concurrently,
prefix: opts.prefix
)
create_if_not_exists index(
:oban_jobs,
[:state, :cancelled_at],
concurrently: opts.concurrently,
name: :oban_jobs_state_cancelled_at_index,
prefix: opts.prefix
)
create_if_not_exists index(
:oban_jobs,
[:state, :discarded_at],
concurrently: opts.concurrently,
name: :oban_jobs_state_discarded_at_index,
prefix: opts.prefix
)
# Rename old indexes back
execute "ALTER INDEX IF EXISTS #{opts.quoted}.oban_jobs_workflow_index_old RENAME TO oban_jobs_workflow_index"
execute "ALTER INDEX IF EXISTS #{opts.quoted}.oban_jobs_sup_workflow_index_old RENAME TO oban_jobs_sup_workflow_index"
execute "ALTER INDEX IF EXISTS #{opts.quoted}.oban_jobs_chain_index_old RENAME TO oban_jobs_chain_index"
end
end

195
vendor/oban_pro/lib/oban/pro/partition.ex vendored Normal file
View file

@ -0,0 +1,195 @@
defmodule Oban.Pro.Partition do
@moduledoc false
@behaviour Oban.Pro.Handler
import Ecto.Query
alias Ecto.Changeset
alias Oban.{Config, Job, Repo}
alias Oban.Pro.{Producer, Queue, Utils}
@keys_cache_ttl Application.compile_env(:oban_pro, [__MODULE__, :keys_cache_ttl], 3_000)
@keys_limit_mlt Application.compile_env(:oban_pro, [__MODULE__, :keys_limit_multiplier], 3)
@conf_tab Module.concat(__MODULE__, Conf)
@keys_tab Module.concat(__MODULE__, Keys)
@impl Oban.Pro.Handler
def on_start do
:ets.new(@conf_tab, [:set, :named_table, :public, read_concurrency: true])
:ets.new(@keys_tab, [:set, :named_table, :public, read_concurrency: true])
# Clear the cache with a timeout, as not all queues are guaranteed to be running on a given
# node and the cache won't clear naturally.
:timer.apply_interval(@keys_cache_ttl, __MODULE__, :__clear_cache__, [@keys_cache_ttl])
:telemetry.attach(
"oban.pro.partition",
[:oban, :engine, :put_meta, :stop],
&__MODULE__.handle_event/4,
nil
)
end
@impl Oban.Pro.Handler
def on_stop do
:telemetry.detach("oban.pro.partition")
end
def __clear_cache__(ttl) do
now = System.monotonic_time(:millisecond)
:ets.select_delete(@keys_tab, [
{{:_, :_, :"$3"}, [{:>=, {:-, now, :"$3"}, ttl}], [true]}
])
end
def handle_event(_event, _measure, %{conf: conf, key: key, meta: meta}, nil) do
# Bust the partition cache on any change to a partitionable limit. This is rare enough an
# event that it's better to be safe.
if key in ~w(global_limit rate_limit)a do
:ets.delete(@conf_tab, {conf.name, meta.queue})
end
end
@spec get_key(Job.t() | Job.changeset(), any()) :: any()
def get_key(changeset, default \\ nil)
def get_key(%{changes: %{meta: %{partition_key: key}}}, _default), do: key
def get_key(%{meta: %{"partition_key" => key}}, _default), do: key
def get_key(_changeset, default), do: default
@spec with_partition_meta(Job.changeset(), Config.t()) :: Job.changeset()
def with_partition_meta(changeset, conf) do
queue = Changeset.get_field(changeset, :queue, "default")
case config_for_queue(conf, queue) do
:missing ->
changeset
partition ->
part_meta = %{partition: true, partition_key: gen_key(changeset, partition)}
meta =
changeset
|> Changeset.get_change(:meta, %{})
|> Map.merge(part_meta)
Changeset.put_change(changeset, :meta, meta)
end
end
@spec gen_key(Job.changeset(), %{fields: [atom()], keys: [atom()]}) :: String.t()
def gen_key(changeset, %{fields: fields, keys: keys}) do
fields
|> Enum.sort()
|> Enum.map(fn
:args -> Utils.take_keys(changeset, :args, keys)
:meta -> Utils.take_keys(changeset, :meta, keys)
:worker -> Changeset.get_field(changeset, :worker)
end)
|> Utils.hash64()
end
@spec available_keys(Config.t(), String.t(), pos_integer()) :: [String.t()]
def available_keys(conf, queue, limit) do
timed_cache(conf.name, queue, fn ->
query_limit = @keys_limit_mlt * limit
subquery =
Job
|> where([j], j.state == "available" and j.queue == ^queue)
|> where([j], fragment("meta \\? 'partition_key'"))
|> group_by([j], fragment("meta->>'partition_key'"))
|> order_by([j], asc: min(j.priority), asc: min(j.scheduled_at))
|> select([j], %{partition_key: fragment("meta->>'partition_key'")})
|> limit(^query_limit)
partitioned_keys =
subquery
|> subquery()
|> select([s], s.partition_key)
|> then(&Repo.all(conf, &1))
["none" | partitioned_keys]
end)
end
defp timed_cache(name, queue, fun) do
cache_key = {name, queue}
now = System.monotonic_time(:millisecond)
case :ets.lookup(@keys_tab, cache_key) do
[{^cache_key, keys, timestamp}] when now - timestamp < @keys_cache_ttl ->
keys
_ ->
keys = fun.()
:ets.insert(@keys_tab, {cache_key, keys, now})
keys
end
end
@doc """
We can't guarantee that the queue is running locally, running on a current node, or running at
all. This will check the following locations:
- From a locally running producer
- From another node's producer in the database
- From a DynamicQueues entry in the database
"""
@spec config_for_queue(Config.t(), String.t()) :: :missing | %{fields: list(), keys: list()}
def config_for_queue(conf, queue) do
# Partitioning isn't used for draining and there's no point in hitting the database just to
# find out there's no config available.
if Process.get(:oban_draining) do
:missing
else
case :ets.lookup(@conf_tab, {conf.name, queue}) do
[{_key, value}] ->
value
[] ->
value =
with :missing <- reg_config(conf, queue),
:missing <- pro_config(conf, queue),
do: dyn_config(conf, queue)
:ets.insert(@conf_tab, {{conf.name, queue}, value})
value
end
end
end
defp reg_config(conf, queue) do
case Registry.meta(Oban.Registry, {conf.name, {:producer, queue}}) do
{:ok, %{meta: %{global_limit: %{partition: %{} = partition}}}} -> partition
{:ok, %{meta: %{rate_limit: %{partition: %{} = partition}}}} -> partition
_other -> :missing
end
end
defp pro_config(conf, queue) do
query =
Producer
|> where(queue: ^queue)
|> order_by(desc: :started_at)
|> limit(1)
case Repo.one(conf, query) do
%{meta: %{global_limit: %{partition: %{} = partition}}} -> partition
%{meta: %{rate_limit: %{partition: %{} = partition}}} -> partition
_other -> :missing
end
end
defp dyn_config(conf, queue) do
case Repo.get_by(conf, Queue, name: queue) do
%{opts: %{global_limit: %{partition: %{} = partition}}} -> partition
%{opts: %{rate_limit: %{partition: %{} = partition}}} -> partition
_other -> :missing
end
end
end

851
vendor/oban_pro/lib/oban/pro/plugins/dynamic_cron.ex vendored Normal file
View file

@ -0,0 +1,851 @@
defmodule Oban.Pro.Plugins.DynamicCron do
@moduledoc """
Enhanced cron scheduling that's configurable at runtime across your entire cluster.
`DynamicCron` can configure cron scheduling before boot or during runtime, globally, with
scheduling guarantees and per-entry timezone overrides. It's an ideal solution for applications
that can't miss a cron job or must dynamically start and manage scheduled jobs at runtime.
## Using and Configuring
To begin using `DynamicCron`, add the module to your list of Oban plugins in `config.exs`:
```elixir
config :my_app, Oban,
plugins: [Oban.Pro.Plugins.DynamicCron]
...
```
By itself, without providing a crontab or dynamically inserting cron entries, the plugin doesn't
have anything to schedule. To get scheduling started, provide a list of `{cron, worker}` or
`{cron, worker, options}` tuples to the plugin. The syntax is identical to Oban's built-in
`:crontab` option, which means you can _probably_ copy an existing standard `:crontab` list into
the plugin's `:crontab`.
```elixir
plugins: [{
Oban.Pro.Plugins.DynamicCron,
crontab: [
{"* * * * *", MyApp.MinuteJob},
{"0 * * * *", MyApp.HourlyJob, queue: :scheduled},
{"0 0 * * *", MyApp.DailyJob, max_attempts: 1},
{"0 12 * * MON", MyApp.MondayWorker, tags: ["scheduled"]},
{"@daily", MyApp.AnotherDailyWorker}
]
}]
```
> #### Beware of Conflicts {: .warning}
>
> While the syntax is syntactically identical to Oban's basic `:crontab`, you can't copy an
> existing `:crontab` if you have multiple entries using the same worker. `DynamicCron`
> requires unique names for all entries, which defaults to the worker's module name. Multiple
> entries with the same worker will cause naming conflicts unless you provide explicit `:name`
> overrides for each duplicate worker.
Now, when dynamic cron initializes, it will persist those cron entries to the database and start
scheduling them according to their cron expression. Beyond standard cron syntax, `DynamicCron`
entries support naming, pausing, per-entry timezone overrides, guaranteed scheduling, and custom
job args and metadata—all covered in the sections below.
Each of the crontab entries are persisted to the database and referenced globally, by all the
other connected Oban instances. That allows us to insert, update, or delete cron entries at any
time. In fact, changing the schedule or options of an entry in the crontab provided to the
plugin will automatically update the persisted entry. To demonstrate, let's modify the
`MinuteJob` we specified so that it runs every other minute in the `:scheduled` queue:
```elixir
crontab: [
{"*/2 * * * *", MyApp.MinuteJob, queue: :scheduled},
...
]
```
Now it isn't really a "minute job" any more, and the name is no longer suitable. However, we
didn't provide a name for the entry and it's using the module name instead. To provide more
flexibility we can add a `:name` override, then we can update the worker's name as well:
```elixir
crontab: [
{"*/2 * * * *", MyApp.FrequentJob, name: "frequent", queue: :scheduled},
...
]
```
All entries are referenced by name, which defaults to the worker's name and must be unique. You
may define the same worker multiple times _as long as_ you provide a name override:
```elixir
crontab: [
{"*/3 * * * *", MyApp.BasicJob, name: "client-1", args: %{client_id: 1}},
{"*/3 * * * *", MyApp.BasicJob, name: "client-2", args: %{client_id: 2}},
...
]
```
To temporarily disable scheduling jobs you can set the `paused` flag:
```elixir
crontab: [
{"* * * * *", MyApp.BasicJob, paused: true},
...
]
```
To resume the job you must supply `paused: false` (or use `update/2` to resume it manually),
simply removing the `paused` option will have no effect.
```elixir
crontab: [
{"* * * * *", MyApp.BasicJob, paused: false},
...
]
```
It is also possible to delete a persisted entry during initialization by passing the `:delete`
option:
```elixir
crontab: [
{"* * * * *", MyApp.MinuteJob, delete: true},
...
]
```
One or more entries can be deleted this way. Deleting entries is idempotent, nothing will happen
if no matching entry can be found.
## Entry Options
Each crontab entry accepts the following options:
* `:name` - a unique identifier for the entry, defaults to the worker module name. Required when
the same worker is used in multiple entries.
* `:args` - a map of arguments passed to the worker's `perform/1` callback. Defaults to `%{}`.
* `:meta` - a map of metadata merged into the job's meta field alongside automatic cron metadata.
* `:queue` - the queue to insert the job into, as an atom or string.
* `:priority` - an integer from 0 to 9, where lower numbers are higher priority.
* `:max_attempts` - the maximum number of execution attempts before the job is discarded.
* `:tags` - a list of string tags for categorizing or searching jobs.
* `:timezone` - a timezone for evaluating the cron expression, overriding the plugin-level
timezone. See [Overriding the Timezone](#module-overriding-the-timezone).
* `:paused` - when `true`, the entry is persisted but won't schedule jobs until resumed.
* `:guaranteed` - when `true`, enables guaranteed scheduling for this entry. See
[Scheduling Guarantees](#module-scheduling-guarantees).
* `:delete` - when `true`, deletes the persisted entry during initialization. Only used in the
plugin's `:crontab` list, not with runtime functions.
In addition to standard five-field cron expressions, the plugin supports extended expressions
such as `@daily`, `@hourly`, `@weekly`, `@monthly`, `@yearly`, and `@reboot`. The `@reboot`
expression inserts a job once each time the plugin starts and won't re-fire until the next
restart.
## Automatic Synchronization
Synchronizing persisted entries manually requires two deploys: one to flag it with `delete:
true` and another to clean up the entry entirely. That extra step isn't ideal for applications
that don't insert or delete jobs at runtime.
To delete entries that are no longer listed in the crontab automatically set the `sync_mode`
option to `:automatic`:
```elixir
[
sync_mode: :automatic,
crontab: [
{"0 * * * *", MyApp.BasicJob},
{"0 0 * * *", MyApp.OtherJob}
]
]
```
To remove unwanted entries, simply delete them from the crontab:
```diff
crontab: [
{"0 * * * *", MyApp.BasicJob},
- {"0 0 * * *", MyApp.OtherJob}
]
```
With `:automatic` sync, the entry for `MyApp.OtherJob` will be deleted on the next deployment.
## Scheduling Guarantees
Depending on an application's restart timing or as the result of unexpected downtime, a job's
scheduled insert period may be missed. To compensate, enable `guaranteed` mode for the entire
crontab or on an individual basis.
Here's an example of enabling guaranteed insertion for the entire crontab:
```elixir
[
guaranteed: true,
crontab: [
{"0 * * * *", MyApp.HourlyJob},
{"0 0 * * *", MyApp.DailyJob},
{"0 0 1 * *", MyApp.MonthlyJob},
{"0 0 1 */2 *", MyApp.BiMonthlyJob}
]
]
```
Guaranteed mode may be enabled or disabled for individual jobs instead if it's a poor fit for
the entire crontab:
```elixir
[
crontab: [
{"@daily", MyApp.DailyJob, guaranteed: true}
{"@monthly", MyApp.MonthlyJob, guaranteed: false}
]
]
```
Guaranteed insertion is enforced by cron name and it's durable across option changes. Changing
an entry's `expression` or `timezone` will clear any prior insertions and reset the guarantee.
## Overriding the Timezone
Without any configuration the default timezone is `Etc/UTC`. You can override that for all cron
entries by passing a `timezone` option to the plugin:
```elixir
plugins: [{
Oban.Pro.Plugins.DynamicCron,
timezone: "America/Chicago",
# ...
```
You can also override the timezone for individual entries by passing it as an option to the
`crontab` list or to `DynamicCron.insert/1`:
```elixir
DynamicCron.insert([
{"0 0 * * *", MyApp.Pinger, name: "oslo", timezone: "Europe/Oslo"},
{"0 0 * * *", MyApp.Pinger, name: "chicago", timezone: "America/Chicago"},
{"0 0 * * *", MyApp.Pinger, name: "zagreb", timezone: "Europe/Zagreb"}
])
```
## Runtime Updates
Dynamic cron entries are persisted to the database, making it easy to manipulate them through
typical CRUD operations. The `DynamicCron` plugin provides convenience functions to simplify
those operations.
The `insert/1` function takes a list of one or more tuples with the same `{expression, worker}`
or `{expression, worker, options}` format as the plugin's `crontab` option:
```elixir
DynamicCron.insert([
{"0 0 * * *", MyApp.GenericWorker},
{"* * * * *", MyApp.ClientWorker, name: "client-1", args: %{client_id: 1}},
{"* * * * *", MyApp.ClientWorker, name: "client-2", args: %{client_id: 2}},
{"* * * * *", MyApp.ClientWorker, name: "client-3", args: %{client_id: 3}}
])
```
Be aware that `insert/1` acts like an "upsert", making it possible to modify existing entries if
the worker or name matches.
## Isolation and Namespacing
All `DynamicCron` functions have an alternate clause that accepts an Oban instance name as the
first argument. This is in line with base `Oban` functions such as `Oban.insert/2`, which allow
you to seamlessly work with multiple Oban instances and across multiple database prefixes. For
example, you can use `all/1` to list all cron entries for the instance named `ObanPrivate`:
```elixir
entries = DynamicCron.all(ObanPrivate)
```
Likewise, to insert a new entry using the configuration associated with the `ObanPrivate`
instance:
```elixir
{:ok, _} = DynamicCron.insert(ObanPrivate, [{"* * * * *", PrivateWorker}])
```
## Instrumenting with Telemetry
The `DynamicCron` plugin adds the following metadata to the `[:oban, :plugin, :stop]` event:
* `:jobs` - a list of jobs inserted during the scheduling pass
"""
@behaviour Oban.Plugin
use GenServer
import Ecto.Query, only: [order_by: 2, where: 2, where: 3]
alias Oban.Cron.Expression
alias Oban.{Job, Peer, Repo, Validation, Worker}
alias Oban.Plugins.Cron, as: CronPlugin
alias Oban.Pro.Cron, as: CronEntry
require Logger
@type cron_expr :: String.t()
@type cron_name :: String.t() | atom()
@type cron_opt :: [
args: Job.args(),
expression: cron_expr(),
guaranteed: boolean(),
max_attempts: pos_integer(),
paused: boolean(),
priority: 0..9,
meta: map(),
name: cron_name(),
queue: atom() | String.t(),
tags: Job.tags(),
timezone: String.t()
]
@type cron_input :: {cron_expr(), module()} | {cron_expr(), module(), cron_opt()}
@type sync_mode :: :manual | :automatic
@type option :: [
conf: Oban.Config.t(),
guaranteed: boolean(),
crontab: [cron_input()],
name: Oban.name(),
sync_mode: sync_mode(),
timezone: Calendar.time_zone(),
timeout: timeout()
]
defstruct [
:conf,
:timer,
crontab: [],
guaranteed: false,
rebooted: false,
sync_mode: :manual,
timezone: "Etc/UTC"
]
defguardp is_name(name) when is_binary(name) or (is_atom(name) and not is_nil(name))
defmacrop maybe_clear_insertions(cron, expression, timezone) do
quote do
fragment(
"CASE WHEN ? != ? OR COALESCE(?->>'timezone', 'NA') != COALESCE(?, 'NA') THEN '{}' ELSE ? END",
field(unquote(cron), :expression),
unquote(expression),
field(unquote(cron), :opts),
unquote(timezone),
field(unquote(cron), :insertions)
)
end
end
@doc false
def child_spec(args), do: super(args)
@impl Oban.Plugin
@spec start_link([option()]) :: GenServer.on_start()
def start_link(opts) do
{name, opts} =
opts
|> Keyword.update(:crontab, [], &normalize_crontab/1)
|> Keyword.delete(:insert_history)
|> Keyword.delete(:timeout)
|> Keyword.pop(:name)
GenServer.start_link(__MODULE__, struct!(__MODULE__, opts), name: name)
end
@impl Oban.Plugin
def validate(opts) do
Validation.validate_schema(opts,
conf: :any,
crontab: {:custom, &validate_crontab(&1, opts)},
guaranteed: :boolean,
insert_history: :pos_integer,
name: :any,
sync_mode: {:enum, [:manual, :automatic]},
timezone: :timezone,
timeout: :timeout
)
end
@impl Oban.Plugin
def format_logger_output(_conf, %{jobs: jobs}), do: %{jobs: Enum.map(jobs, & &1.id)}
@doc """
Used to retrieve all persisted cron entries.
The `all/0` function is provided as a convenience to inspect persisted entries.
## Examples
Return a list of cron schemas with raw attributes:
entries = DynamicCron.all()
"""
@spec all(term()) :: [Ecto.Schema.t()]
def all(oban_name \\ Oban) do
oban_name
|> Oban.config()
|> all_entries()
end
@doc """
Retrieve a single cron entry by name.
## Examples
Fetch an entry using the worker module as the name:
{:ok, entry} = DynamicCron.get(MyApp.Worker)
Fetch an entry using a custom name:
{:ok, entry} = DynamicCron.get("my-custom-cron")
Returns an error tuple if the entry doesn't exist:
{:error, "no cron entry named \\"missing\\" could be found"} = DynamicCron.get("missing")
"""
@spec get(term(), cron_name()) :: {:ok, Ecto.Schema.t()} | {:error, String.t()}
def get(oban_name \\ Oban, name) when is_name(name) do
oban_name
|> Oban.config()
|> fetch_entry(Worker.to_string(name))
end
@doc """
Insert cron entries into the database to start scheduling new jobs.
Be aware that `insert/1` acts like an "upsert", making it possible to modify existing entries if
the worker or name matches. Still, it is better to use `update/2` to make targeted updates.
## Examples
Insert a list of tuples with the same `{expression, worker}` or `{expression, worker, options}`
format as the plugin's `crontab` option.
DynamicCron.insert([
{"0 0 * * *", MyApp.GenericWorker},
{"* * * * *", MyApp.ClientWorker, name: "client-1", args: %{client_id: 1}},
{"* * * * *", MyApp.ClientWorker, name: "client-2", args: %{client_id: 2}},
{"* * * * *", MyApp.ClientWorker, name: "client-3", args: %{client_id: 3}}
])
"""
@spec insert(term(), [cron_input()]) :: {:ok, [Ecto.Schema.t()]} | {:error, Ecto.Changeset.t()}
def insert(oban_name \\ Oban, [_ | _] = crontab) do
conf = Oban.config(oban_name)
Repo.transaction(conf, fn -> insert_entries(conf, crontab) end)
rescue
error in [Ecto.InvalidChangesetError] ->
{:error, error.changeset}
end
@doc """
Update a single cron entry, as identified by worker or name.
Any option available when specifying an entry in the `crontab` list or when calling `insert/2`
can be updated—that includes the cron `expression` and the `worker`.
## Examples
The following call demonstrates updating every possible option:
{:ok, _} =
DynamicCron.update(
"cron-1",
expression: "1 * * * *",
max_attempts: 10,
name: "special-cron",
paused: false,
priority: 0,
queue: "dedicated",
tags: ["client", "scheduled"],
timezone: "Europe/Amsterdam",
worker: Other.Worker,
)
Naturally, individual options may be updated instead. For example, set `paused: true` to pause
an entry:
{:ok, _} = DynamicCron.update(MyApp.ClientWorker, paused: true)
Since `update/2` operates on a single entry at a time, it is possible to rename an entry without
doing a `delete`/`insert` dance:
{:ok, _} = DynamicCron.update(MyApp.ClientWorker, name: "client-worker")
Or, update an entry with a custom entry name already set:
{:ok, _} = DynamicCron.update("cron-1", name: "special-cron")
"""
@spec update(term(), cron_name(), cron_opt()) ::
{:ok, Ecto.Schema.t()} | {:error, Ecto.Changeset.t() | String.t()}
def update(oban_name \\ Oban, name, opts) when is_name(name) do
oban_name
|> Oban.config()
|> update_entry(Worker.to_string(name), opts)
end
@doc """
Delete individual entries, by worker or name.
Use `delete/1` to remove entries at runtime, rather than hard-coding the `:delete` flag into the
`crontab` list at compile time.
## Examples
With the worker as the entry name:
{:ok, _} = DynamicCron.delete(Worker)
With a custom name:
{:ok, _} = DynamicCron.delete("cron-1")
"""
@spec delete(term(), cron_name()) ::
{:ok, Ecto.Schema.t()}
| {:error, Ecto.Changeset.t() | String.t()}
def delete(oban_name \\ Oban, name) when is_name(name) do
oban_name
|> Oban.config()
|> delete_cron(Worker.to_string(name))
end
# Callbacks
@impl GenServer
def init(state) do
:telemetry.execute([:oban, :plugin, :init], %{}, %{conf: state.conf, plugin: __MODULE__})
{:ok, schedule_evaluate(state)}
end
@impl GenServer
def handle_call(:evaluate, _from, state) do
manage_entries(state)
reload_and_insert(state)
{:reply, :ok, %{state | rebooted: true}}
end
@impl GenServer
def handle_info(:evaluate, state) do
if Peer.leader?(state.conf) do
manage_entries(state)
reload_and_insert(state)
{:noreply, schedule_evaluate(%{state | rebooted: true})}
else
{:noreply, schedule_evaluate(state)}
end
end
def handle_info(message, state) do
Logger.warning(
message: "Received unexpected message: #{inspect(message)}",
source: :oban_pro,
module: __MODULE__
)
{:noreply, state}
end
# Validations
defp normalize_crontab(crontab) do
Enum.map(crontab, fn
{expr, worker} -> {expr, worker, []}
tuple -> tuple
end)
end
defp validate_crontab(crontab, opts) when is_list(crontab) and is_list(opts) do
automatic? = opts[:sync_mode] == :automatic
Validation.validate(:crontab, crontab, &validate_crontab(&1, automatic?))
end
defp validate_crontab({expression, worker, opts}, automatic?) do
with {:ok, _} <- CronPlugin.parse(expression) do
delete? = Keyword.get(opts, :delete, false)
cond do
not is_atom(worker) ->
{:error, "expected worker to be an atom, got: #{inspect(worker)}"}
not delete? and not Code.ensure_loaded?(worker) ->
{:error, "#{inspect(worker)} not found or can't be loaded"}
not delete? and not function_exported?(worker, :perform, 1) ->
{:error, "#{inspect(worker)} does not implement `perform/1` callback"}
not Keyword.keyword?(opts) ->
{:error, "options must be a keyword list, got: #{inspect(opts)}"}
not valid_entry?(opts, automatic?) ->
{:error,
"expected cron options to be one of #{inspect(CronEntry.allowed_opts())}, got: #{inspect(opts)}"}
not delete? and not valid_job?(worker, opts) ->
{:error, "expected valid job options, got: #{inspect(opts)}"}
true ->
:ok
end
end
end
defp validate_crontab({expression, worker}, automatic?) do
validate_crontab({expression, worker, []}, automatic?)
end
defp validate_crontab(invalid, _automatic?) do
{:error,
"expected crontab entry to be an {expression, worker} or " <>
"{expression, worker, options} tuple, got: #{inspect(invalid)}"}
end
defp valid_entry?(opts, automatic?) do
string_keys = Enum.map(CronEntry.allowed_opts(), &to_string/1)
ignore_keys =
if automatic? do
~w(guaranteed insertions name paused)a
else
~w(delete guaranteed insertions name paused)a
end
opts
|> Keyword.drop(ignore_keys)
|> Enum.all?(fn {key, _} -> to_string(key) in string_keys end)
end
defp valid_job?(worker, opts) do
args = Keyword.get(opts, :args, %{})
opts = Keyword.drop(opts, ~w(args delete guaranteed insertions name paused timezone)a)
worker.new(args, opts).valid?
end
# Scheduling
defp schedule_evaluate(state) do
timer = Process.send_after(self(), :evaluate, CronPlugin.interval_to_next_minute())
%{state | timer: timer}
end
# Updating
defp reload_and_insert(state) do
meta = %{conf: state.conf, plugin: __MODULE__}
:telemetry.span([:oban, :plugin], meta, fn ->
{:ok, jobs} =
state
|> reload_entries()
|> insert_scheduled_jobs(state)
{:ok, Map.put(meta, :jobs, jobs)}
end)
end
defp reload_entries(state) do
query =
CronEntry
|> where(paused: false)
|> order_by(asc: :inserted_at)
state.conf
|> Repo.all(query)
|> Enum.reduce([], fn entry, acc ->
with {:ok, parsed} <- Expression.parse(entry.expression),
{:ok, worker} <- Worker.from_string(entry.worker) do
opts = Keyword.new(entry.opts, fn {key, val} -> {String.to_existing_atom(key), val} end)
insertions = Enum.reject(entry.insertions, &is_nil/1)
entry = %{entry | insertions: insertions, opts: opts, parsed: parsed, worker: worker}
[entry | acc]
else
_ -> acc
end
end)
end
# Inserting
defp insert_scheduled_jobs(entries, state) do
entries = Enum.filter(entries, &(now?(&1, state) or missed?(&1, state)))
jobs =
Enum.map(entries, fn entry ->
{args, opts} = Keyword.pop(entry.opts, :args, %{})
{tz, opts} = Keyword.pop(opts, :timezone, state.timezone)
meta = %{cron: true, cron_expr: entry.expression, cron_name: entry.name, cron_tz: tz}
opts =
entry.worker.__opts__()
|> Worker.merge_opts(opts)
|> Keyword.drop([:guaranteed, :timezone])
|> Keyword.update(:meta, meta, &Map.merge(&1, meta))
entry.worker.new(args, opts)
end)
Repo.transaction(state.conf, fn ->
now = DateTime.utc_now()
query = where(CronEntry, [c], c.name in ^Enum.map(entries, & &1.name))
Repo.update_all(state.conf, query, set: [insertions: [now]])
Oban.insert_all(state.conf.name, jobs)
end)
end
defp now?(%{parsed: %{reboot?: true}}, %{rebooted: true}), do: false
defp now?(entry, state) do
datetime =
entry.opts
|> Keyword.get(:timezone, state.timezone)
|> DateTime.now!()
Expression.now?(entry.parsed, datetime)
end
defp missed?(entry, state) do
guaranteed? = Keyword.get(entry.opts, :guaranteed, state.guaranteed)
cond do
[] == entry.insertions ->
false
not guaranteed? ->
false
entry.parsed.reboot? ->
false
true ->
timezone = Keyword.get(entry.opts, :timezone, state.timezone)
last_inserted_at = List.first(entry.insertions)
expected_last_at =
entry.parsed
|> Expression.last_at(timezone)
|> DateTime.shift_zone!("Etc/UTC")
# Compare in UTC and ensure it has been at least 2 minutes, which safely handles DST where
# the same clock time can occur within 1 hour.
DateTime.compare(expected_last_at, last_inserted_at) == :gt and
DateTime.diff(expected_last_at, last_inserted_at, :minute) >= 2
end
end
# Entry Management
defp manage_entries(%{rebooted: true}), do: :ok
defp manage_entries(%{sync_mode: :automatic} = state) do
Repo.transaction(state.conf, fn ->
delete_missing(state.conf, state.crontab)
insert_entries(state.conf, state.crontab)
end)
end
defp manage_entries(state) do
Repo.transaction(state.conf, fn ->
{deletes, inserts} =
Enum.split_with(state.crontab, fn {_expr, _work, opts} ->
Keyword.get(opts, :delete)
end)
delete_entries(state.conf, deletes)
insert_entries(state.conf, inserts)
end)
end
defp insert_entries(conf, crontab) do
Enum.map(crontab, fn tuple ->
changeset = CronEntry.changeset(tuple)
replace =
for {key, val} <- changeset.changes,
key in ~w(expression worker opts paused)a,
val != %{},
do: {key, val}
expression = changeset.changes.expression
timezone = changeset.changes.opts[:timezone]
repo_opts = [
conflict_target: :name,
on_conflict:
CronEntry
|> Ecto.Query.update([_], set: ^replace)
|> Ecto.Query.update([c],
set: [insertions: maybe_clear_insertions(c, ^expression, ^timezone)]
)
]
Repo.insert!(conf, changeset, repo_opts)
end)
end
defp delete_entries(conf, crontab) do
Repo.delete_all(conf, where(CronEntry, [e], e.name in ^crontab_names(crontab)))
end
defp delete_missing(conf, crontab) do
Repo.delete_all(conf, where(CronEntry, [e], e.name not in ^crontab_names(crontab)))
end
defp crontab_names(crontab) do
Enum.map(crontab, fn {_expr, worker, opts} ->
opts
|> Keyword.get(:name, worker)
|> Worker.to_string()
end)
end
# Queries
defp all_entries(conf) do
Repo.all(conf, order_by(CronEntry, asc: :inserted_at))
end
defp update_entry(conf, name, params) do
with {:ok, entry} <- fetch_entry(conf, name) do
changeset = CronEntry.update_changeset(entry, Map.new(params))
Repo.update(conf, changeset)
end
end
defp delete_cron(conf, name) do
with {:ok, entry} <- fetch_entry(conf, name), do: Repo.delete(conf, entry)
end
defp fetch_entry(conf, name) do
case Repo.one(conf, where(CronEntry, name: ^name)) do
nil -> {:error, "no cron entry named #{inspect(name)} could be found"}
entry -> {:ok, entry}
end
end
end

459
vendor/oban_pro/lib/oban/pro/plugins/dynamic_lifeline.ex vendored Normal file
View file

@ -0,0 +1,459 @@
defmodule Oban.Pro.Plugins.DynamicLifeline do
@moduledoc """
The `DynamicLifeline` plugin uses producer records to periodically rescue orphaned jobs, i.e.
jobs that are stuck in the `executing` state because the node was shut down before the job could
finish. In addition, it performs the following maintenance tasks:
* Discard jobs left `available` with exhausted attempts due to rare edge cases
* Repair stuck workflows with deleted dependencies or missed scheduling events
* Repair stuck chains with deleted dependencies or missed scheduling events
* Repair jobs in partitioned queues that are missing a partition key
* Repair chunk jobs that are missing a computed `chunk_id`
Without `DynamicLifeline` you'll need to manually rescue stuck jobs or perform maintenance.
## Using the Plugin
To use the `DynamicLifeline` plugin, add the module to your list of Oban plugins in
`config.exs`:
```elixir
config :my_app, Oban,
engine: Oban.Pro.Engines.Smart,
plugins: [Oban.Pro.Plugins.DynamicLifeline]
...
```
No configuration is necessary—the defaults are tuned for most systems.
## Options
* `:repair_limit` — the maximum number of jobs each repair operation (workflows, chains,
partitions, chunks) will process per cycle. Defaults to `1000`.
* `:retry_exhausted` — when `true`, jobs that have exhausted their attempts are rescued back to
`available` with an incremented `max_attempts` instead of being discarded. Defaults to
`false`. See "Rescuing Exhausted Jobs" for details.
* `:timeout` — the maximum time allowed for each rescue query, in milliseconds. Increase this
if your system is under high load or produces a multitude of orphans. Defaults to `45_000`.
## Automatic Repairs
The plugin automatically performs several repair operations during each rescue cycle to keep
workflows, chains, partitions, and chunks healthy.
* **Workflow repair** — Jobs held waiting for dependencies that were deleted or missed a
scheduling event are released. This handles edge cases where workflow jobs get stuck due to
incomplete dependency resolution.
* **Chain repair** — Similar to workflows, chain jobs waiting on deleted or stuck predecessors
are released to continue processing.
* **Partition repair** — Jobs in partitioned queues that are missing a `partition_key` in their
metadata (e.g., jobs scheduled before a queue was partitioned) have their partition key
computed and set automatically.
* **Chunk repair** — Chunk jobs that are missing a `chunk_id` in their metadata have their
chunk ID computed from the job's `chunk_by` configuration and set automatically. This can
happen when chunk settings are changed after jobs are already enqueued.
Each repair operation processes up to `repair_limit` jobs per cycle.
## Identifying Rescued Jobs
Rescued jobs can be identified by a `rescued` value in `meta`. Each rescue increments the
`rescued` count by one.
## Rescuing Exhausted Jobs
When a job's `attempt` matches its `max_attempts` its retries are considered "exhausted".
Normally, the `DynamicLifeline` plugin transitions exhausted jobs to the `discarded` state and
they won't be retried again. It does this for a couple of reasons:
1. To ensure at-most-once semantics. Suppose a long-running job interacted with a non-idempotent
service and was shut down while waiting for a reply; you may not want that job to retry.
2. To prevent infinitely crashing BEAM nodes. Poorly behaving jobs may crash the node (through
NIFs, memory exhaustion, etc.) We don't want to repeatedly rescue and rerun a job that
repeatedly crashes the entire node.
When exhausted jobs are discarded, the `c:Oban.Pro.Worker.on_discarded/2` callback is called
with an `:exhausted` reason, which is useful for error reporting or notifications.
Discarding exhausted jobs may not always be desired. Use the `retry_exhausted` option if you'd
prefer to retry exhausted jobs when they are rescued, rather than discarding them:
```elixir
plugins: [{Oban.Pro.Plugins.DynamicLifeline, retry_exhausted: true}]
```
During rescues, with `retry_exhausted: true`, a job's `max_attempts` is incremented and it is
moved back to the `available` state.
## Instrumenting with Telemetry
The `DynamicLifeline` plugin adds the following metadata to the `[:oban, :plugin, :stop]` event:
* `:rescued_jobs` — a list of jobs transitioned back to `available`
* `:discarded_jobs` — a list of jobs transitioned to `discarded`
_Note: jobs only include `id`, `queue`, and `state` fields._
"""
@behaviour Oban.Plugin
use GenServer
import Ecto.Query
alias Oban.{Job, Peer, Repo, Validation}
alias Oban.Pro.Chunk
alias Oban.Pro.Engines.Smart
alias Oban.Pro.{Partition, Producer, Utils, Workflow}
alias Oban.Pro.Stages.{Chain, Hooks}
require Logger
@type option ::
{:conf, Oban.Config.t()}
| {:name, Oban.name()}
| {:repair_limit, pos_integer()}
| {:retry_exhausted, boolean()}
| {:rescue_interval, timeout()}
| {:timeout, timeout()}
defstruct [
:conf,
:rescue_timer,
repair_limit: 1000,
retry_exhausted: false,
rescue_interval: :timer.minutes(1),
timeout: :timer.seconds(45)
]
defmacrop attempted_join(attempted_by, uuid) do
quote do
fragment(
"array_length(?, 1) <> 1 and ? = uuid (?[2])",
unquote(attempted_by),
unquote(uuid),
unquote(attempted_by)
)
end
end
defmacrop coalesce_partition(meta) do
quote do
fragment(
"COALESCE(?->'global_limit'->'partition', ?->'rate_limit'->'partition')",
unquote(meta),
unquote(meta)
)
end
end
defmacrop track_rescued(meta) do
quote do
fragment(
"""
jsonb_set(?, '{rescued}', (COALESCE(?->>'rescued', '0')::int + 1)::text::jsonb)
""",
unquote(meta),
unquote(meta)
)
end
end
@doc false
def child_spec(args), do: super(args)
@impl Oban.Plugin
@spec start_link([option()]) :: GenServer.on_start()
def start_link(opts) do
{name, opts} = Keyword.pop(opts, :name)
case opts[:conf] do
%{engine: Smart} ->
GenServer.start_link(__MODULE__, struct!(__MODULE__, opts), name: name)
%{engine: engine} ->
raise RuntimeError, """
DynamicLifeline requires the Smart engine to run correctly, but you're using:
engine: #{inspect(engine)}
You can either switch to use the Smart or remove DynamicLifeline from your plugins.
"""
end
end
@impl Oban.Plugin
def validate(opts) do
Validation.validate_schema(opts,
conf: :any,
name: :any,
repair_limit: :pos_integer,
rescue_interval: :pos_integer,
retry_exhausted: :boolean,
timeout: :timeout
)
end
@impl Oban.Plugin
def format_logger_output(_conf, meta) do
for {key, val} <- meta,
key in ~w(discarded_jobs rescued_jobs)a,
into: %{},
do: {key, Enum.map(val, & &1.id)}
end
@impl GenServer
def init(state) do
Process.flag(:trap_exit, true)
:telemetry.execute([:oban, :plugin, :init], %{}, %{conf: state.conf, plugin: __MODULE__})
{:ok, schedule_rescue(state)}
end
@impl GenServer
def terminate(_reason, state) do
if is_reference(state.rescue_timer), do: Process.cancel_timer(state.rescue_timer)
:ok
end
@impl GenServer
def handle_info(:rescue, state) do
if Peer.leader?(state.conf) do
meta = %{conf: state.conf, plugin: __MODULE__}
:telemetry.span([:oban, :plugin], meta, fn ->
orphan_meta = rescue_orphaned(state)
:ok = repair_workflows(state)
:ok = repair_chains(state)
:ok = repair_partitions(state)
:ok = repair_chunk_ids(state)
{:ok, Map.merge(meta, orphan_meta)}
end)
end
{:noreply, schedule_rescue(state)}
end
def handle_info(message, state) do
Logger.warning(
message: "Received unexpected message: #{inspect(message)}",
source: :oban_pro,
module: __MODULE__
)
{:noreply, state}
end
# Scheduling
defp schedule_rescue(state) do
timer = Process.send_after(self(), :rescue, state.rescue_interval)
%{state | rescue_timer: timer}
end
# Orphan Rescuing
defp rescue_orphaned(state) do
{res_count, res_jobs} = transition_available(orphan_query(), state)
{dis_count, dis_jobs} = transition_discarded(orphan_query(), state)
{exh_count, exh_jobs} = transition_discarded(stuck_query(), state)
if state.retry_exhausted do
%{
discarded_count: 0,
discarded_jobs: [],
rescued_count: res_count + dis_count + exh_count,
rescued_jobs: res_jobs ++ dis_jobs ++ exh_jobs
}
else
%{
discarded_count: dis_count + exh_count,
discarded_jobs: dis_jobs ++ exh_jobs,
rescued_count: res_count,
rescued_jobs: res_jobs
}
end
end
defp orphan_query do
subquery =
Job
|> where([j], not is_nil(j.queue) and j.state == "executing")
|> join(:left, [j], p in Producer, on: attempted_join(j.attempted_by, p.uuid))
|> where([_, p], is_nil(p.uuid))
Job
|> join(:inner, [j], x in subquery(subquery), on: j.id == x.id)
|> select([_, x], map(x, [:id, :meta, :queue, :state]))
end
# We don't know how it's possible for a job to be available with attempts >= max_attempts, but
# we can clean up to prevent clogging a queue at least.
defp stuck_query do
Job
|> where([j], j.state == "available")
|> select([j], map(j, [:id, :meta, :queue, :state]))
end
defp transition_available(query, state) do
query =
query
|> where([j], j.attempt < j.max_attempts)
|> update([j], set: [meta: track_rescued(j.meta), state: "available"])
Repo.update_all(state.conf, query, [], timeout: state.timeout)
catch
:error, %Postgrex.Error{postgres: %{code: :unique_violation, detail: detail}} ->
Smart.clear_uniq_violation(state.conf, detail)
{0, []}
end
defp transition_discarded(query, %{retry_exhausted: true} = state) do
query =
query
|> where([j], j.attempt >= j.max_attempts)
|> update([j],
inc: [max_attempts: 1],
set: [meta: track_rescued(j.meta), state: "available"]
)
Repo.update_all(state.conf, query, [], timeout: state.timeout)
catch
:error, %Postgrex.Error{postgres: %{code: :unique_violation, detail: detail}} ->
Smart.clear_uniq_violation(state.conf, detail)
{0, []}
end
defp transition_discarded(query, state) do
{count, partial_jobs} =
Repo.update_all(
state.conf,
where(query, [j], j.attempt >= j.max_attempts),
[set: [state: "discarded", discarded_at: DateTime.utc_now()]],
timeout: state.timeout
)
if count > 0 do
ids = Enum.map(partial_jobs, & &1.id)
Task.start(fn ->
state.conf
|> Repo.all(where(Job, [j], j.id in ^ids), timeout: state.timeout)
|> Enum.each(&Hooks.execute_on_discarded(:exhausted, &1))
end)
end
{count, partial_jobs}
end
# Composition Repair
defp repair_workflows(state) do
Workflow.rescue_workflows(state.conf, timeout: state.timeout, limit: state.repair_limit)
end
defp repair_chains(state) do
Chain.rescue_chains(state.conf, timeout: state.timeout, limit: state.repair_limit)
end
# Partition Repair
defp repair_partitions(state) do
for {queue, partition} <- partitioned_queues(state) do
query =
Job
|> where([j], j.queue == ^queue)
|> where([j], j.state in ~w(available scheduled retryable))
|> where([j], fragment("NOT ? \\? 'partition_key'", j.meta))
|> limit(^state.repair_limit)
state.conf
|> Repo.all(query, timeout: state.timeout)
|> Enum.group_by(fn job -> gen_partition_key(job, partition) end)
|> Enum.each(fn {key, grouped_jobs} ->
ids = Enum.map(grouped_jobs, & &1.id)
Job
|> where([j], j.id in ^ids)
|> update([j], set: [meta: fragment("jsonb_set(?, '{partition_key}', ?)", j.meta, ^key)])
|> then(&Repo.update_all(state.conf, &1, [], timeout: state.timeout))
end)
end
:ok
end
defp gen_partition_key(job, partition) do
job
|> Ecto.Changeset.change()
|> Partition.gen_key(partition)
end
defp partitioned_queues(state) do
normalize_partition = fn partition ->
fields = partition |> Map.get("fields", []) |> Enum.map(&String.to_existing_atom/1)
%{fields: fields, keys: Map.get(partition, "keys", [])}
end
Producer
|> where([p], fragment("jsonb_typeof(?) = 'object'", coalesce_partition(p.meta)))
|> select([p], {p.queue, coalesce_partition(p.meta)})
|> distinct(true)
|> then(&Repo.all(state.conf, &1, timeout: state.timeout))
|> Enum.map(fn {queue, partition} -> {queue, normalize_partition.(partition)} end)
end
# Chunk Repair
defp repair_chunk_ids(state) do
query =
Job
|> where([j], j.state in ~w(available scheduled retryable))
|> where([j], fragment("? \\? 'chunk_by'", j.meta))
|> where([j], fragment("NOT ? \\? 'chunk_id'", j.meta))
|> limit(^state.repair_limit)
state.conf
|> Repo.all(query, timeout: state.timeout)
|> Enum.group_by(&gen_chunk_id/1)
|> Enum.each(fn {chunk_id, grouped_jobs} ->
ids = Enum.map(grouped_jobs, & &1.id)
Job
|> where([j], j.id in ^ids)
|> update([j], set: [meta: fragment("jsonb_set(?, '{chunk_id}', ?)", j.meta, ^chunk_id)])
|> then(&Repo.update_all(state.conf, &1, [], timeout: state.timeout))
end)
:ok
end
defp gen_chunk_id(job) do
chunk_by = normalize_chunk_by(job.meta["chunk_by"])
Chunk.to_chunk_id(chunk_by, job.worker, job.args, queue: job.queue, meta: job.meta)
end
defp normalize_chunk_by(chunk_by) do
Enum.map(chunk_by, fn
"worker" -> :worker
["args", keys] -> [:args, Enum.map(keys, &Utils.maybe_to_atom/1)]
["meta", keys] -> [:meta, Enum.map(keys, &Utils.maybe_to_atom/1)]
end)
end
end

674
vendor/oban_pro/lib/oban/pro/plugins/dynamic_partitioner.ex vendored Normal file
View file

@ -0,0 +1,674 @@
defmodule Oban.Pro.Plugins.DynamicPartitioner do
@moduledoc """
The `DynamicPartitioner` plugin manages a partitioned `oban_jobs` table for optimized query
performance, minimal database bloat, and efficiently pruned historic jobs.
> #### When to use Partitioned Tables {: .warning}
>
> Partitioning has performance tradeoffs because it doesn't support unique indexes and some
> queries need to check each partition. It is ideally **suited for high throughput**
> applications that run **tens of millions of jobs a day**. Be absolutely sure your application
> will benefit from partitioning.
Partitioning is only officially supported on Postgres 11 and higher. While older versions of
Postgres support partitioning, they have prohibitive technical limitations and your experience
may vary.
## Installation
Before running the `DynamicPartitioner` plugin, you must run a migration to create a partitioned
`oban_jobs` table to your database.
> #### Table Name Conflicts {: .info}
>
> Existing `oban_jobs` tables can't be converted to a partitioned table in place and require a
> transition stage. The migration will automatically handle table conflicts by renaming the
> existing table to `oban_jobs_old`. However, if the partitioned table is added to a different
> prefix without a conflict, then the original table is left untouched and both tables are named
> `oban_jobs`.
>
> See the [Backfilling and Migrating](##module-backfilling-and-migrating) section for strategies
> before running migrations.
```bash
mix ecto.gen.migration add_partitioned_oban_jobs
```
Open the generated migration in your editor and delegate to the dynamic partitions migration:
```elixir
defmodule MyApp.Repo.Migrations.AddPartitionedObanJobs do
use Ecto.Migration
defdelegate up, to: Oban.Pro.Migrations.DynamicPartitioner
defdelegate down, to: Oban.Pro.Migrations.DynamicPartitioner
end
```
As with the standard `oban_jobs` table, you can optionally provide a `prefix` to "namespace" the table
within your database. Here we specify a `"partitioned"` prefix:
```elixir
defmodule MyApp.Repo.Migrations.AddPartitionedObanJobs do
use Ecto.Migration
def up do
Oban.Pro.Migrations.DynamicPartitioner.up(prefix: "partitioned")
end
def down do
Oban.Pro.Migrations.DynamicPartitioner.down(prefix: "partitioned")
end
end
```
Run the migration to create the new table:
```bash
mix ecto.migrate
```
The new table is partitioned for optimal inserts, ready for you to backfill any existing jobs
and configure retention periods.
### Date Partitioning in Test Environments
To prevent testing errors after migration, the `completed`, `cancelled`, and `discarded` states
are sub-partitioned by date only in `:dev` and `:prod` environments.
You can explicitly enable date partitioning in other production-like environments with the
`date_partition?` flag:
```elixir
Oban.Pro.Migrations.DynamicPartitioner.change(date_partition?: true)
```
## Backfilling and Migrating
Below are several recommended strategies for backfilling original jobs into the new partitioned
table. They're listed in order of complexity, beginning with the least invasive approach.
### Strategy 1: Don't Backfill at All
The most performant strategy is to not backfill jobs at all. It's perfectly acceptable to leave
old jobs untouched if you **don't need them for uniqueness checks** or other observability
concerns. Though, in an active system, you'll still want to finish processing jobs in the
original table.
During the transition period, until all of the original jobs are processed, you'll run two
separate, entirely isolated, Oban instances:
1. **Original** — configured with the original prefix, `public` if you never changed it. This
instance will run all of the original queues, without any plugins and it won't insert new
jobs.
2. **Partitioned** — configured with the new prefix for the partitioned table, all of your
original queues, plugins, and any other options.
Here's an example of that configuration:
```elixir
queues = [
default: 10,
other_queue: 10,
and_another: 10
]
config :my_app, Oban.Original, queues: queues
config :my_app, Oban,
prefix: "partitioned",
queues: queues,
plugins: [
...
```
Now, start both Oban instances within your application's supervisor:
```diff
children = [
MyApp.Repo,
{Oban, Application.fetch_env!(:my_app, Oban)},
+ {Oban, Application.fetch_env!(:my_app, Oban.Original)},
...
]
```
New jobs will be inserted into the `partitioned` table while existing jobs keep processing
through the `Oban.Original` instance. Once the all original jobs have executed you're free to
remove the extra instance and [drop the original table](#module-cleaning-up).
### Strategy 2: Backfill Old Jobs
This strategy moves old jobs automatically as part of a migration. The `backfill/1` migration
helper, which delegates to `backfill_jobs/2`, helps move jobs in batches for each state.
By default, backfilling includes _all states_ and is intended for smaller tables, i.e. 50k-100k
total jobs.
Backfill by creating an additional migration with ddl transaction disabled:
```elixir
defmodule MyApp.Repo.Migrations.BackfillPartitionedObanJobs do
use Ecto.Migration
@disable_ddl_transaction true
def change do
Oban.Pro.Migrations.DynamicPartitioner.backfill()
end
end
```
Like all other migrations, `backfill/1` accepts options to control the table's prefix. You can
specify both old and new prefixes to handle situations where the partitioned table lives in a
different prefix:
```elixir
def change do
Oban.Pro.Migrations.DynamicPartitioner.backfill(new_prefix: "private", old_prefix: "public")
end
```
For larger tables, or applications that are sensitive to longer migrations, you can split
backfilling between migrations and prioritize in-flight jobs.
Use the `states` option to restrict backfilling to actively `executing` jobs:
```elixir
def change do
Oban.Pro.Migrations.DynamicPartitioner.backfill(states: ~w(executing))
end
```
For the remaining jobs, you can either use a secondary migration or manually call
`backfill_jobs/1` from your application code.
See `backfill_jobs/1` for the full range of backfill options including changing the batch size
and automatically sleeping between batches.
### Cleaning Up
After backfilling is complete you can drop the original `oban_jobs` table. Be _very_ careful to
ensure you're dropping the old job table! If the new and old prefix was the same, which it is by
default, then the table has `_old` appended.
```elixir
defmodule MyApp.Repo.Migrations.DropStandardObanJobs do
use Ecto.Migration
def change do
drop_if_exists table(:oban_jobs_old)
end
end
```
## Using and Configuring
After running the migration to partition tables, enable the plugin to manage sub-partitions:
```elixir
config :my_app, Oban,
plugins: [Oban.Pro.Plugins.DynamicPartitioner]
...
```
The plugin will preemptively create sub-partitions for finished job states (`completed`,
`cancelled`, `discarded`) as well as prune partitions older than the retention period. By
default, older jobs are retained for 3 days.
You can override the retention period for states individually. For example, to retain `completed`
jobs for 2 days, `cancelled` for 7, and `discarded` for 30:
```elixir
plugins: [{
Oban.Pro.Plugins.DynamicPartitioner,
retention: [completed: 2, cancelled: 7, discarded: 30]
}]
```
Pruning sub-partitions is an extremely fast operation akin to dropping a table. As a result,
there is zero lingering bloat. It's not advised that you use the `DynamicPruner`, unless
you're pruning a subset jobs aggressively after a few minutes, hours, etc.
```diff
plugins: [
Oban.Pro.Plugins.DynamicPartitioner,
- Oban.Plugins.Pruner,
- Oban.Pro.Plugins.DynamicPruner
]
```
`DynamicPartitioner` will warn you if the standard `Pruner` is enabled at the same time.
### Tuning Partition Management
The partitioner attempts once an hour to pre-create partitions two days in advance. That
schedule and buffer should be suitable for most applications. However, you can increase the
buffer period and set an alternate schedule if necessary.
For example, to increase the buffer to 3 days and run at 05:00 in the Europe/Paris timezone:
```elixir
plugins: [{
Oban.Pro.Plugins.DynamicPartitioner,
buffer: 3,
schedule: "0 5 * * *",
timezone: "Europe/Paris"
}]
```
## Instrumenting with Telemetry
The `DynamicPartitioner` plugin adds the following metadata to the `[:oban, :plugin, :stop]` event:
* `:created_count` — the number of partitions created
* `:deleted_count` - the number of partitions deleted
"""
@moduledoc deprecated: """
The complexity and edge cases introduced by dynamic partitioning outweigh the
benefits for most applications. This plugin will be removed in a future version.
"""
@behaviour Oban.Plugin
use GenServer
alias __MODULE__, as: State
alias Oban.{Config, Peer, Repo, Validation}
alias Oban.Cron.Expression
alias Oban.Plugins.{Cron, Pruner}
require Logger
@type retention :: [
completed: pos_integer(),
cancelled: pos_integer(),
discarded: pos_integer()
]
@type option ::
{:conf, Oban.Config.t()}
| {:name, GenServer.name()}
| {:retention, retention()}
| {:schedule, String.t()}
| {:timeout, timeout()}
| {:timezone, String.t()}
@retention [completed: 3, cancelled: 3, discarded: 3]
@states Keyword.keys(@retention)
defstruct [
:conf,
:name,
:retention,
:schedule,
:timer,
buffer: 2,
timeout: :timer.minutes(1),
timezone: "Etc/UTC"
]
@doc false
def child_spec(args), do: super(args)
@impl Oban.Plugin
def start_link(opts) do
GenServer.start_link(__MODULE__, opts, name: opts[:name])
end
@impl Oban.Plugin
def validate(opts) do
Validation.validate_schema(opts,
buffer: :pos_integer,
conf: {:custom, &validate_conf/1},
name: :any,
retention: {:custom, &validate_retention/1},
schedule: :schedule,
timeout: :timeout,
timezone: :timezone
)
end
@impl Oban.Plugin
def format_logger_output(_conf, meta) do
Map.take(meta, ~w(created_count deleted_count)a)
end
@sub_states ~w(cancelled completed discarded)
@defaults [
new_prefix: "public",
old_prefix: "public",
states: ~w(suspended executing available retryable scheduled cancelled discarded completed),
batch_size: 5_000,
batch_sleep: 0
]
@doc false
def backfill_jobs, do: backfill_jobs(Oban, [])
@doc """
Backfill jobs from a standard table into a newly partitioned table.
Backfilling is flexible enough to run against one or more job states, with arbitrary batch
sizes, and without transactional blocks. That allows repeated backfill runs in the face of
restarts or database errors.
Sub-partitions by date are created for final states (`completed`, `cancelled`, `discarded`)
automatically before jobs are moved.
## Options
* `:new_prefix` — The prefix where the new partitioned `oban_jobs` table resides. Defaults to
`public`.
* `:old_prefix` — The prefix where the standard `oban_jobs_old` table resides. Defaults to
`public`.
* `:batch_size` — The number of jobs to move (delete/insert) in a single query. Defaults to a
conservative 5,000 jobs per batch.
* `:batch_sleep` — The amount of time to sleep between backfill batches in order to minimize
load on the database. Defaults to 0, no downtime between batches.
* `:states` — A list of job states to backfill jobs from. Defaults to all states.
## Examples
Backfill old jobs across all states in the default `public` prefix:
DynamicPartitioner.backfill_jobs()
Restrict backfilling to incomplete job states:
DynamicPartitioner.backfill_jobs(states: ~w(executing available scheduled retryable))
Backfill to and from an alternate prefix:
DynamicPartitioner.backfill_jobs(old_prefix: "private", new_prefix: "private")
Backfill using larger batches with half a second between queries:
DynamicPartitioner.backfill_jobs(batch_size: 20_000, batch_sleep: 500)
"""
@spec backfill_jobs(name_or_conf :: Oban.name() | Config.t(), opts :: Keyword.t()) :: :ok
def backfill_jobs(conf_or_name, opts \\ [])
def backfill_jobs(%Config{} = conf, opts) when is_list(opts) do
opts =
opts
|> Keyword.validate!(@defaults)
|> Keyword.update!(:states, fn states -> Enum.map(states, &to_string/1) end)
|> Map.new()
old_table =
if opts.old_prefix == opts.new_prefix do
"oban_jobs_old"
else
"oban_jobs"
end
conf = %{conf | prefix: opts.new_prefix}
opts = Map.put(opts, :old_table, old_table)
for state <- opts.states do
backfill_partitions(conf, state, opts)
backfill_jobs(conf, state, opts)
end
:ok
end
def backfill_jobs(oban_name, opts) do
oban_name
|> Oban.config()
|> backfill_jobs(opts)
end
defp backfill_partitions(conf, state, opts) when state in @sub_states do
query = """
SELECT DISTINCT date_trunc('day', #{state}_at)::date
FROM #{opts.old_prefix}.#{opts.old_table}
WHERE state = '#{state}'
"""
{:ok, %{rows: rows}} = Repo.query(conf, query)
rows
|> List.flatten()
|> Enum.reject(&is_nil/1)
|> Enum.each(&create_sub_partition(conf, state, &1))
end
defp backfill_partitions(_conf, _state, _opts), do: :ok
defp backfill_jobs(conf, state, opts) do
query = """
WITH old_jobs AS (
DELETE FROM #{opts.old_prefix}.#{opts.old_table}
WHERE id IN (
SELECT id
FROM #{opts.old_prefix}.#{opts.old_table}
WHERE state = '#{state}'
LIMIT #{opts.batch_size}
)
RETURNING *
), new_jobs AS (
INSERT INTO #{opts.new_prefix}.oban_jobs (
id,
state,
queue,
worker,
attempt,
max_attempts,
priority,
args,
meta,
attempted_by,
errors,
tags,
inserted_at,
scheduled_at,
attempted_at,
cancelled_at,
completed_at,
discarded_at
) SELECT
id,
state,
queue,
worker,
attempt,
max_attempts,
priority,
args,
meta,
attempted_by,
errors,
tags,
inserted_at,
scheduled_at,
attempted_at,
cancelled_at,
completed_at,
discarded_at
FROM old_jobs
RETURNING 1
)
SELECT count(*) from new_jobs
"""
{:ok, %{rows: [[count]]}} = Repo.query(conf, query)
if count > 0 do
Process.sleep(opts.batch_sleep)
backfill_jobs(conf, state, opts)
end
end
@impl GenServer
def init(opts) do
opts =
opts
|> Keyword.update(:retention, @retention, &Keyword.merge(@retention, &1))
|> Keyword.update(:schedule, Expression.parse!("@hourly"), &Expression.parse!/1)
state =
State
|> struct!(opts)
|> schedule_manage()
:telemetry.execute([:oban, :plugin, :init], %{}, %{conf: state.conf, plugin: __MODULE__})
{:ok, state, {:continue, :start}}
end
@impl GenServer
def handle_continue(:start, state) do
manage_tables(state)
{:noreply, state}
end
@impl GenServer
def handle_info(:manage, state) do
datetime = DateTime.now!(state.timezone)
if Peer.leader?(state.conf) and Expression.now?(state.schedule, datetime) do
manage_tables(state)
end
{:noreply, schedule_manage(state)}
end
def handle_info(message, state) do
Logger.warning(
message: "Received unexpected message: #{inspect(message)}",
source: :oban,
module: __MODULE__
)
{:noreply, state}
end
defp manage_tables(%{conf: conf} = state) do
meta = %{conf: conf, plugin: __MODULE__}
:telemetry.span([:oban, :plugin], meta, fn ->
xact_opts = [retry: 1, timeout: state.timeout]
{:ok, created} = Repo.transaction(conf, fn -> create_sub_partitions(state) end, xact_opts)
{:ok, deleted} = Repo.transaction(conf, fn -> delete_sub_partitions(state) end, xact_opts)
counts = %{created_count: length(created), deleted_count: length(deleted)}
{:ok, Map.merge(meta, counts)}
end)
catch
kind, value ->
Logger.error(fn ->
"[#{__MODULE__}] management error: " <> Exception.format(kind, value, __STACKTRACE__)
end)
end
defp create_sub_partitions(%{buffer: buffer, conf: conf}) do
for state <- @states, days <- 0..buffer do
date = Date.add(Date.utc_today(), days)
create_sub_partition(conf, state, date)
end
end
defp create_sub_partition(conf, state, date) do
quoted = inspect(conf.prefix)
next = Date.add(date, 1)
safe = Calendar.strftime(date, "%Y%m%d")
command = """
CREATE TABLE IF NOT EXISTS #{quoted}.oban_jobs_#{state}_#{safe}
PARTITION OF #{quoted}.oban_jobs_#{state}
FOR VALUES FROM ('#{date} 00:00:00') TO ('#{next} 00:00:00')
"""
query!(conf, command)
end
defp delete_sub_partitions(%{conf: conf, retention: retention}) do
rules =
Map.new(retention, fn {state, days} ->
stamp =
Date.utc_today()
|> Date.add(-days)
|> Calendar.strftime("%Y%m%d")
|> String.to_integer()
{to_string(state), stamp}
end)
query = """
SELECT table_name
FROM information_schema.tables
WHERE table_schema = '#{conf.prefix}'
AND table_name ~ 'oban_jobs_(cancelled|completed|discarded)_.+'
"""
for [table] <- query!(conf, query),
<<"oban_jobs_", state::binary-size(9), "_", date::binary-size(8)>> = table,
String.to_integer(date) <= Map.fetch!(rules, state) do
query!(conf, "DROP TABLE IF EXISTS #{inspect(conf.prefix)}.#{table}")
end
end
defp query!(conf, command) do
case Repo.query(conf, command) do
{:ok, %{rows: rows}} -> rows
{:error, error} -> raise(error)
end
end
# Scheduling
defp schedule_manage(state) do
timer = Process.send_after(self(), :manage, Cron.interval_to_next_minute())
%{state | timer: timer}
end
# Validation
defp validate_conf(conf) do
cond do
conf.engine != Oban.Pro.Engines.Smart ->
{:error, "requires the Smart engine to run, got: #{inspect(conf.engine)}"}
Keyword.has_key?(conf.plugins, Pruner) ->
{:error, "pruning isn't compatibile with partitioning, found: #{Pruner}"}
true ->
:ok
end
end
defp validate_retention(retention) do
keys = Keyword.keys(retention)
vals = Keyword.values(retention)
cond do
not Keyword.keyword?(retention) ->
{:error, "expected :retention to be a keyword list, got: #{inspect(retention)}"}
Enum.any?(keys, &(&1 not in @states)) ->
{:error, "expected :retention keys to overlap #{inspect(@states)}, got: #{inspect(keys)}"}
Enum.any?(vals, &((is_integer(&1) and &1 <= 0) or not is_integer(&1))) ->
{:error, "expected :retention values to be positive integers, got: #{inspect(vals)}"}
true ->
:ok
end
end
end

313
vendor/oban_pro/lib/oban/pro/plugins/dynamic_prioritizer.ex vendored Normal file
View file

@ -0,0 +1,313 @@
defmodule Oban.Pro.Plugins.DynamicPrioritizer do
@moduledoc """
The `DynamicPrioritizer` plugin automatically adjusts job priorities to ensure all jobs are
eventually processed.
Using mixed priorities in a queue causes certain jobs to execute before others. For example, a
queue that processes jobs from various customers may prioritize customers that are in a higher
tier or plan. All high priority (`0`) jobs are guaranteed to run before any with lower priority
(`1..9`), which is wonderful for the higher tier customers but can lead to resource starvation.
When there is a constant flow of high priority jobs the lower priority jobs will never get the
chance to run.
The `DynamicPrioritizer` solves this by periodically boosting the priority of jobs that have
been waiting longer than a configured threshold. Each cycle, qualifying jobs have their priority
decremented by one (e.g. from `3` to `2`), gradually converging toward the highest priority.
## Using the Plugin
To use the `DynamicPrioritizer` plugin, add the module to your list of Oban plugins in
`config.exs`:
```elixir
config :my_app, Oban,
plugins: [Oban.Pro.Plugins.DynamicPrioritizer]
...
```
Without any additional options the plugin will automatically increase the priority of any jobs
that are `available` for 5 minutes or more, checking once per minute.
## Options
* `:after` — how long a job must be `available` before it becomes eligible for reprioritization,
in milliseconds. Set to `:infinity` to disable reprioritization by default (useful when
relying entirely on overrides). Defaults to `300_000` (5 minutes).
* `:interval` — the number of milliseconds between reprioritization cycles. Defaults to
`60_000` (1 minute).
* `:limit` — the maximum number of jobs to reprioritize per override or default group in each
cycle. Defaults to `10_000`.
* `:max_priority` — the highest priority (lowest number) that jobs can be boosted to. For
example, `max_priority: 2` prevents jobs from being boosted above priority `2`, reserving
`0` and `1` for genuinely high priority work. Defaults to `0`.
* `:queue_overrides` — a keyword list of per-queue wait thresholds that override `:after` for
specific queues. See "Queue and Worker Overrides" for details.
* `:worker_overrides` — a keyword list of per-worker wait thresholds that override `:after` for
specific workers. See "Queue and Worker Overrides" for details.
## Queue and Worker Overrides
The `:after` threshold applies globally to all queues and workers. Overrides let you fine tune
reprioritization for specific queues or workers.
Configure the `analysis` queue to nudge jobs after only 1 minute:
```elixir
plugins: [{
Oban.Pro.Plugins.DynamicPrioritizer,
queue_overrides: [analysis: :timer.minutes(1)]
}]
```
Disable reprioritization globally while enabling it for a single queue:
```elixir
plugins: [{
Oban.Pro.Plugins.DynamicPrioritizer,
after: :infinity,
queue_overrides: [analysis: :timer.minutes(1)]
}]
```
Override on a per-worker basis:
```elixir
plugins: [{
Oban.Pro.Plugins.DynamicPrioritizer,
worker_overrides: [
"MyApp.HighSLAWorker": :timer.seconds(30),
"MyApp.LowSLAWorker": :timer.minutes(10)
]
}]
```
Queue and worker overrides can be combined. Note that overrides are applied independently—a
job matching both a queue override and a worker override will be boosted by each:
```elixir
plugins: [{
Oban.Pro.Plugins.DynamicPrioritizer,
interval: :timer.minutes(2),
after: :timer.minutes(5),
queue_overrides: [media: :timer.minutes(10)],
worker_overrides: ["MyApp.HighSLAWorker": :timer.seconds(30)]
}]
```
## Instrumenting with Telemetry
The `DynamicPrioritizer` plugin adds the following metadata to the `[:oban, :plugin,
:stop]` event:
* `:reprioritized_count` — the number of jobs reprioritized
"""
@behaviour Oban.Plugin
use GenServer
import Ecto.Query, only: [join: 5, limit: 2, order_by: 2, where: 3]
alias Oban.{Job, Peer, Repo, Validation}
require Logger
@type option :: [
after: timeout(),
interval: timeout(),
limit: pos_integer(),
max_priority: 0..9,
queue_overrides: [{atom() | String.t(), timeout()}],
worker_overrides: [{atom() | String.t(), timeout()}]
]
defstruct [
:conf,
:timer,
after: :timer.minutes(5),
interval: :timer.seconds(60),
limit: 10_000,
max_priority: 0,
queue_overrides: [],
worker_overrides: []
]
@doc false
def child_spec(args), do: super(args)
# Callbacks
@impl Oban.Plugin
def start_link(opts) do
{name, opts} = Keyword.pop(opts, :name)
GenServer.start_link(__MODULE__, struct!(__MODULE__, opts), name: name)
end
@impl Oban.Plugin
def validate(opts) do
Validation.validate_schema(opts,
conf: :any,
name: :any,
after: :timeout,
interval: :pos_integer,
limit: :pos_integer,
max_priority: {:range, 0..9},
queue_overrides: {:custom, &validate_overrides(:queues, &1)},
worker_overrides: {:custom, &validate_overrides(:workers, &1)}
)
end
@impl Oban.Plugin
def format_logger_output(_conf, meta) do
Map.take(meta, ~w(reprioritized_count)a)
end
@impl GenServer
def init(state) do
Process.flag(:trap_exit, true)
:telemetry.execute([:oban, :plugin, :init], %{}, %{conf: state.conf, plugin: __MODULE__})
{:ok, schedule_reprioritization(state)}
end
@impl GenServer
def terminate(_reason, state) do
if is_reference(state.timer), do: Process.cancel_timer(state.timer)
:ok
end
@impl GenServer
def handle_info(:reprioritize, state) do
meta = %{conf: state.conf, plugin: __MODULE__}
:telemetry.span([:oban, :plugin], meta, fn ->
case reprioritize_starved_jobs(state) do
{:ok, count} when is_integer(count) ->
{:ok, Map.put(meta, :reprioritized_count, count)}
error ->
{:error, Map.put(meta, :error, error)}
end
end)
{:noreply, schedule_reprioritization(state)}
end
def handle_info(message, state) do
Logger.warning(
message: "Received unexpected message: #{inspect(message)}",
source: :oban_pro,
module: __MODULE__
)
{:noreply, state}
end
# Validation
defp validate_overrides(parent_key, overrides) do
Validation.validate(parent_key, overrides, &validate_override/1)
end
defp validate_override({key, value}) when is_atom(key) do
Validation.validate_timeout(key, value)
end
defp validate_override(option) do
{:error, "expected override option to be a tuple, got: #{inspect(option)}"}
end
# Scheduling
defp schedule_reprioritization(state) do
%{state | timer: Process.send_after(self(), :reprioritize, state.interval)}
end
# Queries
defp reprioritize_starved_jobs(state) do
if Peer.leader?(state.conf) do
Repo.transaction(state.conf, fn ->
queue_counts =
for {queue, period} <- state.queue_overrides do
state
|> base_query()
|> query_for_queues(:any, [to_string(queue)])
|> query_for_period(period)
|> update_all(state)
end
worker_counts =
for {worker, period} <- state.worker_overrides do
state
|> base_query()
|> query_for_workers(:any, [to_string(worker)])
|> query_for_period(period)
|> update_all(state)
end
default_count =
state
|> base_query()
|> query_for_queues(:not, string_keys(state.queue_overrides))
|> query_for_workers(:not, string_keys(state.worker_overrides))
|> query_for_period(state.after)
|> update_all(state)
[queue_counts, worker_counts]
|> List.flatten()
|> Enum.reduce(default_count, &(&1 + &2))
end)
else
{:ok, 0}
end
end
defp string_keys(keyword) do
for {key, _} <- keyword, do: to_string(key)
end
defp base_query(%{limit: limit, max_priority: max_priority}) do
Job
|> where([j], j.state == "available")
|> where([j], j.priority > ^max_priority)
|> order_by(asc: :id)
|> limit(^limit)
end
defp query_for_period(query, :infinity) do
where(query, [j], true == false)
end
defp query_for_period(query, period) do
where(query, [j], j.scheduled_at <= ^to_timestamp(period))
end
defp query_for_queues(query, :not, []), do: query
defp query_for_queues(query, :not, queues), do: where(query, [j], j.queue not in ^queues)
defp query_for_queues(query, :any, queues), do: where(query, [j], j.queue in ^queues)
defp query_for_workers(query, :not, []), do: query
defp query_for_workers(query, :not, workers), do: where(query, [j], j.worker not in ^workers)
defp query_for_workers(query, :any, workers), do: where(query, [j], j.worker in ^workers)
defp update_all(subquery, state) do
query = join(Job, :inner, [j], x in subquery(subquery), on: j.id == x.id)
state.conf
|> Repo.update_all(query, inc: [priority: -1])
|> elem(0)
end
defp to_timestamp(milliseconds) do
DateTime.add(DateTime.utc_now(), -milliseconds, :millisecond)
end
end

783
vendor/oban_pro/lib/oban/pro/plugins/dynamic_pruner.ex vendored Normal file
View file

@ -0,0 +1,783 @@
defmodule Oban.Pro.Plugins.DynamicPruner do
@moduledoc """
The `DynamicPruner` enhances the default Pruner plugin's behaviour by allowing you to customize
how jobs are retained in the jobs table. Where the Pruner operates on a fixed schedule and
treats all jobs the same, with the DynamicPruner, you can use a flexible CRON schedule and
provide custom rules for specific queues, workers, and job states.
## Using the Plugin
To start using the `DynamicPruner` add the module to your list of Oban plugins in `config.exs`:
```elixir
config :my_app, Oban,
plugins: [Oban.Pro.Plugins.DynamicPruner]
...
```
Without any additional options the pruner operates in maximum length mode (`max_len`) and
retains a conservative 1,000 `completed`, `cancelled`, or `discarded` jobs. To increase the
number of jobs retained you can provide your own `mode` configuration:
```elixir
plugins: [{Oban.Pro.Plugins.DynamicPruner, mode: {:max_len, 50_000}}]
```
Now the pruner will retain the most recent 50,000 jobs instead.
A fixed limit on the number of jobs isn't always ideal. Often you want to retain jobs based on
their age instead. For example, if your application needs to ensure that a duplicate job
hasn't been enqueued within the past 24 hours you need to retain jobs for at least 24 hours; a
fixed limit simply won't work. For that we can use maximum age (`max_age`) mode instead:
```elixir
plugins: [{Oban.Pro.Plugins.DynamicPruner, mode: {:max_age, 60 * 60 * 48}}]
```
Here we've specified `max_age` using seconds, where `60 * 60 * 48` is the number of seconds in
two days.
Calculating the number of seconds in a period isn't especially readable, particularly when you
have numerous `max_age` declarations in overrides (see below). For clarity you can specify the
age's time unit as `:second`, `:minute`, `:hour`, `:day`, `:week`, or `:month`. Here is the same 48 hour
configuration from above, but specified in terms of days:
```elixir
plugins: [{Oban.Pro.Plugins.DynamicPruner, mode: {:max_age, {2, :days}}}]
```
Now you can tell exactly how long jobs should be retained, without reverse calculating how many
seconds an expression represents.
## Options
* `:mode` — the pruning strategy, either `{:max_len, count}` to retain the most recent `count`
jobs, or `{:max_age, age}` to retain jobs newer than `age`. The age can be an integer in
seconds or a `{value, unit}` tuple where unit is `:second`, `:minute`, `:hour`, `:day`,
`:week`, or `:month` (plural forms also accepted). Defaults to `{:max_len, 1_000}`.
* `:queue_overrides` — a keyword list of `{queue, mode}` tuples that override the default mode
for specific queues. See [Providing Overrides](#module-providing-overrides).
* `:state_overrides` — a keyword list of `{state, mode}` tuples that override the default mode
for specific states (`:completed`, `:cancelled`, or `:discarded`). See
[Providing Overrides](#module-providing-overrides).
* `:worker_overrides` — a keyword list of `{worker, mode}` tuples that override the default
mode for specific workers. See [Providing Overrides](#module-providing-overrides).
* `:schedule` — a cron expression controlling when pruning runs. Defaults to `"* * * * *"`
(every minute). See [Setting a Schedule](#module-setting-a-schedule).
* `:timezone` — the timezone used to evaluate the cron schedule. Defaults to `"Etc/UTC"`.
* `:limit` — the maximum number of jobs deleted per pruning pass. Exists to prevent connection
timeouts and excessive table locks. Defaults to `10_000`. See
[Keeping Up With Inserts](#module-keeping-up-with-inserts).
* `:timeout` — the maximum time allowed for each pruning query, in milliseconds. Defaults to
`60_000`.
* `:preserve_workflows` — when `true`, jobs belonging to active workflows are retained
regardless of their age or position. Defaults to `true`. See
[Workflow Preservation](#module-workflow-preservation).
* `:before_delete` — an `{module, function, args}` callback invoked with job ids before they
are deleted, within the same transaction. See
[Executing a Callback Before Delete](#module-executing-a-callback-before-delete).
## Providing Overrides
The `mode` option is indiscriminate when determining which jobs to prune. It pays no attention
to which queue they are in, what worker the job is for, or which state they landed in. The
`DynamicPruner` allows you to specify per-queue, per-worker and per-state overrides that fine
tune pruning.
We'll start with a simple example of limiting the total number of retained jobs in the `events`
queue:
```elixir
plugins: [{
Oban.Pro.Plugins.DynamicPruner,
mode: {:max_age, {7, :days}},
queue_overrides: [events: {:max_len, 1_000}]
}]
```
With this configuration most jobs will be retained for seven days, but we'll only keep the
latest 1,000 jobs in the events queue. We can extend this further and override all of our queues
(and omit the default mode entirely):
```elixir
plugins: [{
Oban.Pro.Plugins.DynamicPruner,
queue_overrides: [
default: {:max_age, {6, :hours}},
analysis: {:max_age, {1, :day}},
events: {:max_age, {10, :minutes}},
mailers: {:max_age, {2, :weeks}},
media: {:max_age, {2, :months}}
]
}]
```
When pruning by queue isn't granular enough you can provide overrides by worker instead:
```elixir
plugins: [{
Oban.Pro.Plugins.DynamicPruner,
worker_overrides: [
"MyApp.BusyWorker": {:max_age, {1, :day}},
"MyApp.SecretWorker": {:max_age, {1, :second}},
"MyApp.HistoricWorker": {:max_age, {1, :month}}
]
}]
```
You can also override by state, which allows you to keep discarded jobs for inspection while
quickly purging cancelled or successfully completed jobs:
```elixir
plugins: [{
Oban.Pro.Plugins.DynamicPruner,
state_overrides: [
cancelled: {:max_age, {1, :hour}},
completed: {:max_age, {1, :day}},
discarded: {:max_age, {1, :month}}
]
}]
```
Naturally you can mix and match overrides to finely control job retention:
```elixir
plugins: [{
Oban.Pro.Plugins.DynamicPruner,
mode: {:max_age, {7, :days}},
queue_overrides: [events: {:max_age, {10, :minutes}}],
state_overrides: [discarded: {:max_age, {2, :days}}],
worker_overrides: ["MyApp.SecretWorker": {:max_age, {1, :second}}]
}]
```
### Override Precedence
Overrides are applied sequentially, in this order:
1. Queue
2. State
3. Worker
4. Default
Using the example above, jobs in the `events` queue are deleted first, followed by jobs in the
`discarded` state, then the `MyApp.SecretWorker` worker, and finally any other jobs older than 7
days that **weren't covered by any overrides**.
### Preventing Timeouts with Overrides
Worker override queries aren't able to use any of Oban's standard indexes. If you're processing
a high volume of jobs, pruning with worker overrides may be extremely slow due to sequential
scans. To prevent timeouts, and speed up pruning altogether, you should add a compound index to
the `oban_jobs` table:
```elixir
create_if_not_exists index(:oban_jobs, [:worker, :state, :id], concurrently: true)
```
## Retaining Jobs Forever
To retain jobs in a queue, state, or for a particular worker forever (without using something
like `{:max_age, {999, :years}}`), use `:infinity` as the length or duration:
```elixir
plugins: [{
Oban.Pro.Plugins.DynamicPruner,
mode: {:max_age, {7, :days}},
state_overrides: [
cancelled: {:max_len, :infinity},
discarded: {:max_age, :infinity}
]
}]
```
## Keeping Up With Inserts
With the default settings the `DynamicPruner` will only delete 10,000 jobs each time it prunes.
The limit exists to prevent connection timeouts and excessive table locks. A busy system can
easily insert more than 10,000 jobs per minute during standard operation. If you find that jobs
are accumulating despite active pruning you can override the `limit`.
Here we set the delete limit to 25,000 and give it 60 seconds to complete:
```elixir
plugins: [{
Oban.Pro.Plugins.DynamicPruner,
mode: {:max_len, 100_000},
limit: 25_000,
timeout: :timer.seconds(60)
}]
```
Deleting in PostgreSQL is _very_ fast, and the 10k default is rather conservative. Feel free to
increase the limit to a number that your system can handle.
## Setting a Schedule
By default, pruning happens at the top of every minute based on the CRON schedule `* * * * *`.
You're free to set any CRON schedule you prefer for greater control over when to prune. For
example, to prune once an hour instead:
```elixir
plugins: [{Oban.Pro.Plugins.DynamicPruner, schedule: "0 * * * *"}]
```
Or, to prune once a day at midnight in your local timezone:
```elixir
plugins: [{
Oban.Pro.Plugins.DynamicPruner,
limit: 100_000,
schedule: "0 0 * * *",
timezone: "America/Chicago",
timeout: :timer.minutes(1)
}]
```
Pruning less frequently can reduce load on your system, particularly if you're using
multiple [overrides](#providing-overrides). However, be sure to set a higher `limit` and
`timeout` (as shown above) to compensate for more accumulated jobs.
## Executing a Callback Before Delete
Sometimes jobs are a historic record of activity and it's desirable to operate on them _before_
they're deleted. For example, you may want to copy some jobs into cold storage prior to completion.
To accomplish this, specify a callback to execute before proceeding with the deletion:
```elixir
defmodule DeleteHandler do
def call(job_ids) do
# Use the ids at this point, from within a transaction
end
end
plugins: [{
Oban.Pro.Plugins.DynamicPruner,
mode: {:max_age, {7, :days}},
before_delete: {DeleteHandler, :call, []}
}]
```
The callback receives a list of the ids for the jobs that are about to be deleted. The callback
runs within the same transaction that's used for deletion, and you should keep it quick or move
heavy processing to an async process. Note that because it runs in the same transaction as
deletion, the jobs _won't be available after the callback exits_.
To pass in extra arguments as "configuration" you can provide args to the callback MFA:
```elixir
defmodule DeleteHandler do
import Ecto.Query
def call(job_ids, storage_name) do
jobs = MyApp.Repo.all(where(Oban.Job, [j], j.id in ^job_ids))
Storage.call(storage_name, jobs)
end
end
before_delete: {DeleteHandler, :call, [ColdStorage]}
```
## Workflow Preservation
By default, jobs that are part of an active workflow are retained to prevent partial deletion of
workflow data while the workflow is still running. A workflow is considered active if it
contains any jobs in an incomplete state.
This check includes linked sub-workflows and parent workflows, ensuring that all related jobs
across workflow hierarchies are preserved until the entire workflow completes.
To disable workflow preservation and prune all eligible jobs regardless of workflow status:
```elixir
plugins: [{
Oban.Pro.Plugins.DynamicPruner,
mode: {:max_age, {7, :days}},
preserve_workflows: false
}]
```
Disabling workflow preservation can improve pruning performance for systems with many workflows,
but may result in incomplete workflow job histories if workflows are pruned while still active.
## Implementation Notes
Some additional notes about pruning in general and nuances of the `DynamicPruner` plugin:
* Pruning is best-effort and performed out-of-band. This means that all limits are soft; jobs
beyond a specified age may not be pruned immediately after jobs complete.
* Pruning is only applied to jobs that are `completed`, `cancelled` or `discarded` (has reached
the maximum number of retries or has been manually killed). It'll never delete a new,
scheduled, or retryable job.
* Jobs that are part of an active workflow are retained regardless of their state. A workflow
is considered active if it contains any jobs in `suspended`, `scheduled`, `retryable`,
`available`, or `executing` states. This behavior is controlled by the `preserve_workflows`
option, which defaults to `true`.
* Only a single node will prune at any given time, which prevents potential deadlocks between
transactions.
## Instrumenting with Telemetry
The `DynamicPruner` plugin adds the following metadata to the `[:oban, :plugin, :stop]` event:
* `:pruned_count` - the total number of jobs that were deleted
* `:pruned_jobs` - the jobs that were deleted from the database
_Note: jobs only include `id`, `queue`, and `state` fields._
"""
@behaviour Oban.Plugin
use GenServer
import Ecto.Query
alias Oban.Cron.Expression
alias Oban.{Job, Peer, Repo, Validation}
alias Oban.Plugins.Cron
alias Oban.Pro.Workflow.Schema, as: Workflow
require Logger
@type time_unit ::
:second
| :seconds
| :minute
| :minutes
| :hour
| :hours
| :day
| :days
| :week
| :weeks
| :month
| :months
@type max_age :: pos_integer() | {pos_integer(), time_unit()}
@type max_len :: pos_integer()
@type mode :: {:max_age, max_age()} | {:max_len, max_len()}
@type state_override :: {:completed | :cancelled | :discarded, mode()}
@type queue_override :: {atom(), mode()}
@type worker_override :: {module(), mode()}
@type override :: state_override() | queue_override() | worker_override()
@type option ::
{:conf, Oban.Config.t()}
| {:schedule, String.t()}
| {:name, Oban.name()}
@modes [:max_age, :max_len]
@states [:completed, :cancelled, :discarded]
@time_units [
:second,
:seconds,
:minute,
:minutes,
:hour,
:hours,
:day,
:days,
:week,
:weeks,
:month,
:months
]
defstruct [
:before_delete,
:conf,
:schedule,
:timer,
limit: 10_000,
mode: {:max_len, 1_000},
preserve_workflows: true,
queue_overrides: [],
state_overrides: [],
timeout: :timer.seconds(60),
timezone: "Etc/UTC",
worker_overrides: []
]
defmacrop same_workflow(meta_a, meta_b) do
quote do
fragment(
"""
? \\? 'workflow_id' AND ?->> 'workflow_id' = ? ->> 'workflow_id'
""",
unquote(meta_a),
unquote(meta_a),
unquote(meta_b)
)
end
end
defmacrop same_sub_workflow(meta_a, meta_b) do
quote do
fragment(
"""
? \\? 'sup_workflow_id' AND ?->> 'sup_workflow_id' = ? ->> 'workflow_id'
""",
unquote(meta_a),
unquote(meta_a),
unquote(meta_b)
)
end
end
defmacrop same_sup_workflow(meta_a, meta_b) do
quote do
fragment(
"""
? \\? 'workflow_id' AND ?->> 'workflow_id' = ? ->> 'sup_workflow_id'
""",
unquote(meta_a),
unquote(meta_a),
unquote(meta_b)
)
end
end
@doc false
def child_spec(args), do: super(args)
@impl Oban.Plugin
def start_link(opts) do
{name, opts} = Keyword.pop(opts, :name)
opts =
opts
|> Keyword.put_new(:schedule, "* * * * *")
|> Keyword.update!(:schedule, &Expression.parse!/1)
GenServer.start_link(__MODULE__, struct!(__MODULE__, opts), name: name)
end
@impl Oban.Plugin
def validate(opts) do
opts
|> Keyword.delete(:by_state_timestamp)
|> Validation.validate_schema(
before_delete: {:custom, &validate_before_delete/1},
conf: :any,
limit: :pos_integer,
mode: {:custom, &validate_mode/1},
name: :any,
preserve_workflows: :boolean,
queue_overrides: {:custom, &validate_overrides(:queue_overrides, &1)},
state_overrides: {:custom, &validate_overrides(:state_overrides, &1)},
schedule: :schedule,
timeout: :timeout,
timezone: :timezone,
worker_overrides: {:custom, &validate_overrides(:worker_overrides, &1)}
)
end
@impl Oban.Plugin
def format_logger_output(_conf, meta) do
Map.take(meta, ~w(pruned_count)a)
end
@impl GenServer
def init(state) do
Process.flag(:trap_exit, true)
:telemetry.execute([:oban, :plugin, :init], %{}, %{conf: state.conf, plugin: __MODULE__})
{:ok, schedule_prune(state)}
end
@impl GenServer
def terminate(_reason, state) do
if is_reference(state.timer), do: Process.cancel_timer(state.timer)
:ok
end
@impl GenServer
def handle_info(:prune, state) do
meta = %{conf: state.conf, plugin: __MODULE__}
:telemetry.span([:oban, :plugin], meta, fn ->
with {:ok, jobs_extra} <- prune_jobs(state),
{:ok, workflows_extra} <- prune_workflows(state) do
extra =
meta
|> Map.merge(jobs_extra)
|> Map.merge(workflows_extra)
{:ok, extra}
end
end)
{:noreply, schedule_prune(state)}
end
def handle_info(message, state) do
Logger.warning(
message: "Received unexpected message: #{inspect(message)}",
source: :oban_pro,
module: __MODULE__
)
{:noreply, state}
end
# Scheduling
defp schedule_prune(state) do
timer = Process.send_after(self(), :prune, Cron.interval_to_next_minute())
%{state | timer: timer}
end
# Query
defp prune_jobs(state) do
{:ok, datetime} = DateTime.now(state.timezone)
if Peer.leader?(state.conf) and Expression.now?(state.schedule, datetime) do
string_queues = string_keys(state.queue_overrides)
string_states = string_keys(state.state_overrides)
string_workers = string_keys(state.worker_overrides)
queue_pruned =
for {queue, mode} <- state.queue_overrides do
Job
|> query_for_queues(:any, [to_string(queue)])
|> delete_for_mode(mode, state)
end
state_pruned =
for {queue_state, mode} <- state.state_overrides do
Job
|> query_for_states(:any, [to_string(queue_state)])
|> delete_for_mode(mode, state)
end
worker_pruned =
for {worker, mode} <- state.worker_overrides do
Job
|> query_for_workers(:any, [to_string(worker)])
|> delete_for_mode(mode, state)
end
default_pruned =
Job
|> query_for_queues(:not, string_queues)
|> query_for_states(:not, string_states)
|> query_for_workers(:not, string_workers)
|> delete_for_mode(state.mode, state)
pruned = List.flatten([queue_pruned, state_pruned, worker_pruned, default_pruned])
{:ok, %{pruned_count: length(pruned), pruned_jobs: pruned}}
else
{:ok, %{pruned_count: 0, pruned_jobs: []}}
end
end
defp string_keys(keyword) do
for {key, _} <- keyword, do: to_string(key)
end
defp query_for_queues(query, :not, []), do: query
defp query_for_queues(query, :not, queues), do: where(query, [j], j.queue not in ^queues)
defp query_for_queues(query, :any, queues), do: where(query, [j], j.queue in ^queues)
defp query_for_states(query, :not, []), do: query
defp query_for_states(query, :not, states), do: where(query, [j], j.state not in ^states)
defp query_for_states(query, :any, states), do: where(query, [j], j.state in ^states)
defp query_for_workers(query, :not, []), do: query
defp query_for_workers(query, :not, workers), do: where(query, [j], j.worker not in ^workers)
defp query_for_workers(query, :any, workers), do: where(query, [j], j.worker in ^workers)
defp delete_for_mode(_query, {_mode, :infinity}, _state), do: []
defp delete_for_mode(query, {:max_age, age}, state) do
time = to_timestamp(age)
query
|> select([j], %{id: j.id, queue: j.queue, state: j.state, rn: 100_000_000})
|> where(
[j],
(j.state == "completed" and j.completed_at < ^time) or
(j.state == "cancelled" and j.cancelled_at < ^time) or
(j.state == "discarded" and j.discarded_at < ^time)
)
|> delete_all(state.limit + 1, state)
end
defp delete_for_mode(query, {:max_len, len}, state) do
query
|> where([j], j.state in ~w(cancelled completed discarded))
|> select([j], %{
id: j.id,
queue: j.queue,
state: j.state,
rn: fragment("row_number() over (order by id desc)")
})
|> delete_all(len, state)
end
defp delete_all(query, offset, state) do
subquery =
query
|> from(as: :jobs)
|> maybe_preserve_workflows(state.preserve_workflows)
|> order_by(asc: :id)
|> limit(^state.limit)
query =
Job
|> join(:inner, [j], x in subquery(subquery), on: j.id == x.id and x.rn > ^offset)
|> select([_, x], map(x, [:id, :queue, :state]))
fun = fn ->
apply_before_delete(query, state)
Repo.delete_all(state.conf, query, timeout: state.timeout)
end
{:ok, {_count, deleted}} = Repo.transaction(state.conf, fun, timeout: state.timeout)
deleted
end
defp maybe_preserve_workflows(query, false), do: query
defp maybe_preserve_workflows(query, true) do
workflow_subquery =
Job
|> where([j], j.state in ~w(suspended scheduled retryable available executing))
|> where([j], fragment("? \\? 'workflow_id'", parent_as(:jobs).meta))
|> where(
[j],
same_workflow(j.meta, parent_as(:jobs).meta) or
same_sub_workflow(j.meta, parent_as(:jobs).meta) or
same_sup_workflow(j.meta, parent_as(:jobs).meta)
)
|> select(1)
where(query, [j], not exists(workflow_subquery))
end
defp apply_before_delete(query, state) do
with {mod, fun, args} <- state.before_delete do
ids =
state.conf
|> Repo.all(query)
|> Enum.map(& &1.id)
apply(mod, fun, [ids | args])
end
:ok
end
# Workflows
defp prune_workflows(state) do
query =
Workflow
|> where([wf], wf.state in ~w(completed cancelled discarded))
|> where([wf], wf.completed + wf.cancelled + wf.discarded == 0)
{count, _} = Repo.delete_all(state.conf, query, [])
{:ok, %{pruned_workflows_count: count}}
end
# Timestamp
defp to_timestamp(seconds) when is_integer(seconds) do
DateTime.add(DateTime.utc_now(), -seconds, :second)
end
defp to_timestamp({seconds, :second}), do: to_timestamp(seconds)
defp to_timestamp({seconds, :seconds}), do: to_timestamp(seconds)
defp to_timestamp({minutes, :minute}), do: to_timestamp(minutes * 60)
defp to_timestamp({minutes, :minutes}), do: to_timestamp({minutes, :minute})
defp to_timestamp({hours, :hour}), do: to_timestamp(hours * 60 * 60)
defp to_timestamp({hours, :hours}), do: to_timestamp({hours, :hour})
defp to_timestamp({days, :day}), do: to_timestamp(days * 24 * 60 * 60)
defp to_timestamp({days, :days}), do: to_timestamp({days, :day})
defp to_timestamp({weeks, :week}), do: to_timestamp(weeks * 7 * 24 * 60 * 60)
defp to_timestamp({weeks, :weeks}), do: to_timestamp({weeks, :week})
defp to_timestamp({months, :month}), do: to_timestamp(months * 30 * 24 * 60 * 60)
defp to_timestamp({months, :months}), do: to_timestamp({months, :month})
# Validations
defp validate_before_delete({mod, fun, args})
when is_atom(mod) and is_atom(fun) and is_list(args) do
cond do
not Code.ensure_loaded?(mod) ->
{:error, "module #{inspect(mod)} can't be loaded"}
not function_exported?(mod, fun, length(args) + 1) ->
{:error, "no function #{inspect(fun)} with arity #{length(args) + 1}"}
true ->
:ok
end
end
defp validate_before_delete(mfa) do
{:error, "expected :before_delete to be a {module, fun, args} tuple, got: #{inspect(mfa)}"}
end
defp validate_mode({mode, :infinity}) when mode in @modes, do: :ok
defp validate_mode({mode, int}) when is_integer(int) and mode in @modes, do: :ok
defp validate_mode({mode, {value, time_unit}})
when is_integer(value) and mode in @modes and time_unit in @time_units,
do: :ok
defp validate_mode(mode) do
{:error, "expected :mode to be {:max_len, length} or {:max_age, age}, got: #{inspect(mode)}"}
end
defp validate_overrides(:state_overrides = parent_key, overrides) do
Validation.validate(parent_key, overrides, fn
{state, mode} when state in @states ->
validate_mode(mode)
{state, _mode} ->
{:error, "expected state to be included in #{inspect(@states)}, got: #{inspect(state)}"}
override ->
{:error, "expected #{inspect(parent_key)} to contain a tuple, got: #{inspect(override)}"}
end)
end
defp validate_overrides(parent_key, overrides) do
Validation.validate(parent_key, overrides, fn
{key, mode} when is_atom(key) ->
validate_mode(mode)
override ->
{:error, "expected #{inspect(parent_key)} to contain a tuple, got: #{inspect(override)}"}
end)
end
end

757
vendor/oban_pro/lib/oban/pro/plugins/dynamic_queues.ex vendored Normal file
View file

@ -0,0 +1,757 @@
defmodule Oban.Pro.Plugins.DynamicQueues do
@moduledoc """
The `DynamicQueues` plugin extends Oban's basic queue management by persisting queue changes
across restarts, globally, across all connected nodes. It also boasts a declarative syntax for
specifying which nodes a queue will run on.
`DynamicQueues` are ideal for applications that dynamically start, stop, or modify queues at
runtime.
## Using and Configuring
To begin using `DynamicQueues`, add the module to your list of Oban plugins in `config.exs`:
```elixir
config :my_app, Oban,
plugins: [Oban.Pro.Plugins.DynamicQueues]
...
```
Without providing a list of queues the plugin doesn't have anything to run. The syntax for
specifying dynamic queues is identical to Oban's built-in `:queues` option, which means you can
copy them over:
```diff
- queues: [
- default: 20,
- mailers: [global_limit: 20],
- events: [local_limit: 30, rate_limit: [allowed: 100, period: 60]]
- ]
+ plugins: [{
+ Oban.Pro.Plugins.DynamicQueues,
+ queues: [
+ default: 20,
+ mailers: [global_limit: 20],
+ events: [local_limit: 30, rate_limit: [allowed: 100, period: 60]]
+ ]
+ }]
```
Now, when `DynamicQueues` initializes, it will persist all of the queues to the database and
start supervising them. The `queues` syntax is _nearly_ identical to Oban's standard queues,
with an important enhancement we'll look at shortly.
Each of the persisted queues are referenced globally, by all other connected Oban instances that
are running the `DynamicQueues` plugin. Changing the queue's name, pausing, scaling, or changing
any other options will automatically update queues across all nodes—and persist across restarts.
Persisted queues are referenced by name, so you can tweak a queue's options by changing the
definition within your config. For example, to bump the mailer's global limit up to `30`:
queues: [
mailers: [global_limit: 30],
...
]
That isn't especially interesting—after all, that's exactly how regular queues work! Dynamic
queues start to shine when you insert, update, or delete them _dynamically_, either through Oban
Web or your own application code.
## Options
* `:queues` — a keyword list of queue definitions, where each key is the queue name and the
value is either an integer local limit or a keyword list of queue options. See
[Using and Configuring](#module-using-and-configuring) for the full syntax.
* `:sync_mode` — controls how queues are synchronized during startup. Either `:manual`
(default) or `:automatic`. See [Synchronizing Queues](#module-synchronizing-queues).
* `:interval` — how often (in milliseconds) to poll the database for queue changes. Defaults
to `:infinity` (polling disabled). See [Enabling Polling Mode](#module-enabling-polling-mode).
### Synchronizing Queues
The `:sync_mode` option controls how DynamicQueues handles queue synchronization during startup.
There are two available modes:
* `:manual` - Only deletes queues explicitly marked with `delete: true` and inserts/updates
the remaining queues. This is the default.
* `:automatic` - Automatically deletes any queues that aren't defined in the configuration and
inserts/updates the remaining queues.
Here's how to configure each mode:
```elixir
# Manual mode (default)
config :my_app, Oban, plugins: [{DynamicQueues, sync_mode: :manual, queues: [...]}]
# Automatic mode
config :my_app, Oban, plugins: [{DynamicQueues, sync_mode: :automatic, queues: [...]}]
```
In manual mode, you must explicitly mark queues for deletion:
```elixir
queues: [
old_queue: [delete: true],
default: 20,
mailers: [limit: 10]
]
```
In automatic mode, any queue that exists in the database but isn't defined in the configuration
will be automatically deleted during startup. This is useful when you want to ensure your
runtime queue configuration exactly matches what's defined in your application config.
> #### Choosing a Sync Mode {: .info}
>
> Use `:manual` mode when you want fine-grained control over queue deletion and want to preserve
> dynamically created queues across restarts. Use `:automatic` mode when you want to ensure your
> queue configuration is the single source of truth and automatically clean up old queues.
>
> In either mode, changes to queues are persisted across restarts **until the configuration is
> changed**. For example, if you change the `global_limit` of a queue through the Web dashboard,
> that change will persist across restarts until you change the `global_limit` in your config.
### Limiting Where Queues Run
Dynamic queues can be configured to run on a subset of available nodes. This is especially
useful when you wish to restrict resource-intensive queues to only dedicated nodes. Restriction is
configured through the `only` option, which you use like this:
queues: [
basic: [local_limit: 10, only: {:node, :=~, "web|worker"}],
audio: [local_limit: 5, only: {:node, "worker.1"}],
video: [local_limit: 5, only: {:node, "worker.2"}],
learn: [local_limit: 5, only: {:sys_env, "EXLA", "CUDA"}],
store: [local_limit: 1, only: {:sys_env, "WAREHOUSE", true}]
]
In this example we've defined five queues, with the following restrictions:
* `basic` — will run on a node named `web` or `worker`
* `audio` — will only run on a node named `worker.1`
* `video` — will only run on a node named `worker.2`
* `learn` — will run wherever `EXLA=CUDA` is an environment variable
* `store` — will run wherever `WAREHOUSE=true` is an environment variable
Here are the various match modes, operators, and allowed patterns:
#### Modes
* `:node` — matches the node name as set in your Oban config. By default, `node`
is the node's id in a cluster, the hostname outside a cluster, or a `DYNO`
variable on Heroku.
* `:sys_env` — matches a single system environment variable as retrieved by
`System.get_env/1`
#### Operators
* `:==` — compares the pattern and runtime value for _equality_ as strings. This
is the default operator if nothing is specified.
* `:!=` — compares the pattern and runtime value for _inequality_ as strings
* `:=~` — treats the pattern as a regex and matches it against a runtime value
#### Patterns
* `boolean` — either `true` or `false`, which is stringified before comparison
* `string` — either a literal pattern or a regular expression, depending on the
supplied operator
## Runtime Updates
Dynamic queues are persisted to the database, making it easy to manipulate them directly through
CRUD operations, or indirectly with Oban's queue operations, i.e. `pause_queue/2`,
`scale_queue/2`.
Explicit queue options will be overwritten on restart, while omitted fields are retained. For
example, consider the following dynamic queue entry:
queues: [
default: [limit: 10],
...
]
Scaling that `default` queue up or down at runtime _wouldn't_ persist across a restart, because
the definition will overwrite the `limit`. However, pausing the queue or changing the
`global_limit` _would_ persist because they aren't included in the queue definition.
# pause, global_limit, etc. will persist, but limit won't
default: [limit: 10]
# pause will persist, but limit and global_limit won't
default: [limit: 10, global_limit: 20]
# neither limits nor pausing will persist
default: [limit: 10, global_limit: 20, paused: true]
See function documentation for `all/0`, `insert/1`, `update/2`, and `delete/1` for more
information about runtime updates.
## Enabling Polling Mode
In environments with restricted connectivity (where PubSub doesn't work) you can still use
DynamicQueues at runtime through polling mode. The polling interval is entirely up to you, as
it's disabled by default.
config :my_app, Oban,
plugins: [{Oban.Pro.Plugins.DynamicQueues, interval: :timer.minutes(1)}]
With the interval above each DynamicQueues instance will wake up every minute, check the
database for changes, and start new queues.
## Isolation and Namespacing
All DynamicQueues functions have an alternate clause that accepts an Oban instance name for the
first argument. This matches base `Oban` functions such as `Oban.pause_queue/2`, which allow you
to seamlessly work with multiple Oban instances and across multiple database prefixes. For
example, you can use `all/1` to list all queues for the instance named `ObanPrivate`:
queues = DynamicQueues.all(ObanPrivate)
Likewise, to insert a new queue using the configuration associated with the `ObanPrivate`
instance:
DynamicQueues.insert(ObanPrivate, private: limit: 10)
"""
@behaviour Oban.Plugin
use GenServer
import Ecto.Query
alias Ecto.Changeset
alias Oban.{Config, Midwife, Notifier, Peer, Registry, Repo, Validation}
alias Oban.Pro.Engines.Smart
alias Oban.Pro.{Queue, Utils}
require Logger
@type oban_name :: term()
@type operator :: :== | :!= | :=~
@type partition :: Smart.partition()
@type pattern :: boolean() | String.t()
@type period :: Smart.period()
@type sync_mode :: :manual | :automatic
@type sys_key :: String.t()
@type queue_name :: atom() | binary()
@type queue_input :: [{queue_name(), pos_integer() | queue_opts() | [queue_opts()]}]
@type queue_opts ::
{:local_limit, pos_integer()}
| {:global_limit, Smart.global_limit()}
| {:only, only()}
| {:paused, boolean()}
| {:rate_limit, Smart.rate_limit()}
@type only ::
{:node, pattern()}
| {:node, operator(), pattern()}
| {:sys_env, sys_key(), pattern()}
| {:sys_env, sys_key(), operator(), pattern()}
defstruct [
:conf,
:foreman_ref,
:timer,
interval: :infinity,
queues: [],
restart_attempts: 50,
restart_delay: 10,
sync_mode: :manual
]
defguardp is_name(name) when is_binary(name) or (is_atom(name) and not is_nil(name))
@impl Oban.Plugin
def start_link(opts) do
{name, opts} = Keyword.pop(opts, :name)
GenServer.start_link(__MODULE__, struct!(__MODULE__, opts), name: name)
end
@impl Oban.Plugin
def validate(opts) do
Validation.validate_schema(opts,
conf: {:custom, &validate_engine/1},
name: :any,
interval: :timeout,
queues: {:custom, &validate_queues/1},
sync_mode: {:enum, [:manual, :automatic]}
)
end
@doc """
Retrieve all persisted queues.
While it's possible to modify queues returned from `all/0`, it is recommended that you use
`update/2` to ensure options are cast and validated correctly.
## Examples
Retrieve a list of all queue schemas with persisted attributes:
DynamicQueues.all()
"""
@spec all(oban_name()) :: [Ecto.Schema.t()]
def all(oban_name \\ Oban) do
oban_name
|> Oban.config()
|> all_queues()
end
@doc """
Retrieve a single persisted queue.
## Examples
Retrieve the default queue's schema with persisted attributes:
DynamicQueues.get(:default)
"""
@spec get(oban_name(), queue_name()) :: nil | Ecto.Schema.t()
def get(oban_name \\ Oban, queue_name) do
oban_name
|> Oban.config()
|> Repo.get_by(Queue, name: to_string(queue_name))
end
@doc """
Persist a list of queue inputs, exactly like the `:queues` option passed as configuration.
Note that `insert/1` acts like an upsert, making it possible to modify queues if the name
matches. Still, it is better to use `update/2` to make targeted updates.
## Examples
Insert a variety of queues with standard and advanced options:
DynamicQueues.insert(
basic: 10,
audio: [global_limit: 10],
video: [global_limit: 10],
learn: [local_limit: 5, only: {:node, :=~, "learn"}]
)
"""
@spec insert(oban_name(), queue_input()) ::
{:ok, [Ecto.Schema.t()]} | {:error, Ecto.Changeset.t()}
def insert(oban_name \\ Oban, [_ | _] = entries) do
conf = Oban.config(oban_name)
case insert_queues(conf, entries) do
{:ok, changes} ->
inserted = Map.values(changes)
Enum.each(inserted, &notify(conf, :dyn_start, &1.name))
{:ok, inserted}
{:error, _name, changeset, _changes} ->
{:error, changeset}
end
end
@doc """
Modify a single queue's options.
Every option available when inserting queues can be updated.
## Examples
The following call demonstrates updating every possible option:
DynamicQueues.update(
:video,
local_limit: 5,
global_limit: 20,
rate_limit: [allowed: 10, period: 30, partition: :worker]],
only: {:node, :=~, "media"},
paused: false
)
Updating a single option won't remove other persisted options. If you'd like to
clear an option you must set them to `nil`:
DynamicQueues.update(:video, global_limit: nil)
Since `update/2` operates on a single queue, it is possible to rename a queue
without doing a `delete`/`insert` dance:
DynamicQueues.update(:video, name: :media)
"""
@spec update(oban_name(), queue_name(), [queue_opts()]) ::
{:ok, Ecto.Schema.t()} | {:error, Ecto.Changeset.t()}
def update(oban_name \\ Oban, name, opts) when is_name(name) do
conf = Oban.config(oban_name)
with {:ok, queue} <- fetch_queue(conf, to_string(name)),
{:ok, queue} <- update_queue(conf, queue, opts) do
if queue.name == to_string(name) do
opts =
opts
|> Map.new()
|> Map.delete(:only)
|> Map.merge(%{action: :scale, ident: :any, queue: name, update: false})
# Manually send the notification because Oban.scale_queue won't allow unknown fields
Notifier.notify(conf.name, :signal, opts)
else
notify(conf, :dyn_start, queue.name)
notify(conf, :dyn_stop, name)
end
{:ok, queue}
end
end
defp update_queue(conf, queue, opts) do
params =
case Keyword.pop(opts, :name) do
{nil, opts} -> %{opts: Map.new(opts)}
{name, []} -> %{name: name}
{name, opts} -> %{name: name, opts: Map.new(opts)}
end
changeset = Queue.changeset(queue, params)
Repo.update(conf, changeset)
rescue
Ecto.StaleEntryError ->
update_queue(conf, Repo.reload!(conf, queue), opts)
end
@doc """
Delete a queue by name at runtime, rather than using the `:delete` option into the `queues` list
in your configuration.
## Examples
Delete the "audio" queue:
{:ok, _} = DynamicQueues.delete(:audio)
"""
@spec delete(oban_name(), queue_name()) :: {:ok, Ecto.Schema.t()} | {:error, Ecto.Changeset.t()}
def delete(oban_name \\ Oban, name) when is_name(name) do
conf = Oban.config(oban_name)
with {:ok, queue} <- fetch_queue(conf, to_string(name)),
{:ok, queue} <- Repo.delete(conf, queue) do
notify(conf, :dyn_stop, queue.name)
{:ok, queue}
end
end
# Callbacks
@impl GenServer
def init(state) do
:telemetry.execute([:oban, :plugin, :init], %{}, %{conf: state.conf, plugin: __MODULE__})
ref =
state.conf.name
|> Registry.whereis(Foreman)
|> Process.monitor()
{:ok, %{state | foreman_ref: ref}, {:continue, :start}}
end
@impl GenServer
def handle_continue(:start, state) do
:ok = Notifier.listen(state.conf.name, [:signal])
case manage_queues(state) do
{:ok, _inserted} ->
state.conf
|> all_queues()
|> Enum.reject(&queue_running?(state.conf, &1))
|> Enum.each(&start_queue(state.conf, &1))
{:noreply, schedule_refresh(state)}
{:error, _name, changeset, _changes} ->
{:stop, %ArgumentError{message: changeset.errors}, state}
end
end
@impl GenServer
def handle_call(:flush, _from, state) do
{:reply, :ok, state}
end
@impl GenServer
def handle_info({:DOWN, ref, :process, _pid, _reason}, %{foreman_ref: ref} = state) do
state = restart_queues(state, state.restart_attempts)
{:noreply, state}
end
def handle_info({:notification, :signal, payload}, state) do
case payload do
%{"action" => "scale", "ident" => "any", "queue" => queue_name} ->
scale_queue(state.conf, queue_name, payload)
%{"action" => "pause", "ident" => "any", "queue" => queue_name} ->
scale_queue(state.conf, queue_name, %{"paused" => true})
%{"action" => "resume", "ident" => "any", "queue" => queue_name} ->
scale_queue(state.conf, queue_name, %{"paused" => false})
%{"action" => "dyn_start", "queue" => queue_name} ->
start_queue(state.conf, queue_name)
%{"action" => "dyn_stop", "queue" => queue_name} ->
stop_queue(state.conf, queue_name)
_ ->
:ignore
end
{:noreply, state}
end
def handle_info(:refresh, state) do
for queue <- all_queues(state.conf), not queue_running?(state.conf, queue) do
start_queue(state.conf, queue)
end
{:noreply, schedule_refresh(state)}
end
def handle_info(message, state) do
Logger.warning(
message: "Received unexpected message: #{inspect(message)}",
source: :oban_pro,
module: __MODULE__
)
{:noreply, state}
end
# Validations
defp validate_engine(%Config{engine: engine}) do
if engine == Smart do
:ok
else
{:error,
"""
DynamicQueues requires the Smart engine to run correctly, but you're using:
engine: #{inspect(engine)}
You can either use the Smart engine or remove DynamicQueue from your plugins.
"""}
end
end
defp validate_queues(queues) do
Validation.validate(:queues, queues, &validate_queue/1)
end
defp validate_queue({_name, delete: true}), do: :ok
defp validate_queue({name, _} = tuple) when is_atom(name) do
changeset = Queue.changeset(tuple)
if changeset.valid? do
:ok
else
exception = Utils.to_exception(changeset)
{:error, "expected #{inspect(name)} to be a valid queue, but: #{exception.message}"}
end
end
defp validate_queue(queue) do
{:error, "expected queue to be a keyword pair, got: #{inspect(queue)}"}
end
# Queries
defp all_queues(conf) do
Repo.all(conf, order_by(Queue, asc: :inserted_at))
end
defp manage_queues(%{sync_mode: :automatic} = state) do
Repo.transaction(state.conf, fn ->
delete_missing(state.conf, state.queues)
insert_queues(state.conf, state.queues)
end)
end
defp manage_queues(state) do
Repo.transaction(state.conf, fn ->
{deletes, inserts} =
Enum.split_with(state.queues, fn
{_name, opts} when is_list(opts) -> Keyword.get(opts, :delete)
{_name, _lim} -> false
end)
delete_queues(state.conf, deletes)
insert_queues(state.conf, inserts)
end)
end
defp delete_missing(conf, queues) do
Repo.delete_all(conf, where(Queue, [q], q.name not in ^queue_names(queues)))
end
defp delete_queues(conf, queues) do
Repo.delete_all(conf, where(Queue, [q], q.name in ^queue_names(queues)))
end
defp queue_names(queues), do: Enum.map(queues, &to_string(elem(&1, 0)))
defp insert_queues(conf, queues) do
changesets =
Map.new(queues, fn {name, opts} ->
changeset = Queue.changeset({name, opts})
{changeset.changes.name, {changeset, opts}}
end)
Repo.transaction(conf, fn ->
existing = existing_map(conf, Map.keys(changesets))
Enum.reduce(changesets, %{}, fn {name, {changeset, opts}}, acc ->
cond do
is_map_key(existing, name) and Map.get(existing, name).hash == changeset.changes.hash ->
acc
is_map_key(existing, name) ->
params = Queue.normalize({name, opts})
changeset =
existing
|> Map.get(name)
|> Queue.changeset(params)
Map.put(acc, name, Repo.update!(conf, changeset))
true ->
Map.put(acc, name, Repo.insert!(conf, changeset))
end
end)
end)
end
defp existing_map(conf, names) do
query = where(Queue, [q], q.name in ^names)
conf
|> Repo.all(query)
|> Map.new(&{&1.name, &1})
end
defp fetch_queue(conf, name) do
case Repo.one(conf, where(Queue, name: ^name)) do
nil -> {:error, "no queue named #{inspect(name)} could be found"}
queue -> {:ok, queue}
end
end
# Scheduling
defp schedule_refresh(%{interval: :infinity} = state), do: state
defp schedule_refresh(state) do
%{state | timer: Process.send_after(self(), :refresh, state.interval)}
end
# Notification
defp notify(conf, action, queue_name) do
Notifier.notify(conf.name, :signal, %{action: action, ident: :any, queue: queue_name})
end
# Supervision
defp restart_queues(state, 0) do
state
end
defp restart_queues(state, attempt) do
case Registry.whereis(state.conf.name, Foreman) do
pid when is_pid(pid) ->
ref = Process.monitor(pid)
state.conf
|> all_queues()
|> Enum.each(&start_queue(state.conf, &1))
%{state | foreman_ref: ref}
nil ->
Process.sleep(state.restart_delay)
restart_queues(state, attempt - 1)
end
end
defp queue_running?(conf, queue) do
conf.name
|> Registry.whereis({:supervisor, queue.name})
|> is_pid()
end
defp scale_queue(_conf, _, %{"update" => false}), do: :ok
defp scale_queue(conf, "*", %{"paused" => paused}) do
if Peer.leader?(conf) do
query =
from q in Queue,
update: [set: [opts: fragment("jsonb_set(?, '{paused}', ?)", q.opts, ^paused)]]
Repo.update_all(conf, query, [])
end
end
defp scale_queue(conf, queue_name, payload) do
with true <- Peer.leader?(conf),
{:ok, queue} <- fetch_queue(conf, queue_name) do
new_opts = Map.drop(payload, ["action", "ident", "queue"])
changeset =
queue
|> Queue.changeset(%{opts: new_opts})
|> Changeset.delete_change(:hash)
{:ok, _} = Repo.update(conf, changeset)
end
end
defp start_queue(conf, queue_name) when is_binary(queue_name) do
with {:ok, queue} <- fetch_queue(conf, queue_name), do: start_queue(conf, queue)
end
defp start_queue(conf, queue) do
if run_locally?(conf, queue.only) do
Midwife.start_queue(conf, Queue.to_keyword_opts(queue))
else
:ignore
end
end
defp stop_queue(conf, queue_name) when is_binary(queue_name) do
Midwife.stop_queue(conf, queue_name)
end
defp run_locally?(%{node: node}, %{mode: :node, op: op, value: value}) do
compare(node, op, value)
end
defp run_locally?(_conf, %{mode: :sys_env, op: op, key: key, value: value}) do
key
|> System.get_env()
|> to_string()
|> compare(op, value)
end
defp run_locally?(_conf, _only), do: true
defp compare(value_a, :==, value_b), do: value_a == value_b
defp compare(value_a, :!=, value_b), do: value_a != value_b
defp compare(value_a, :=~, value_b), do: value_a =~ Regex.compile!(value_b, "i")
end

669
vendor/oban_pro/lib/oban/pro/plugins/dynamic_scaler.ex vendored Normal file
View file

@ -0,0 +1,669 @@
defmodule Oban.Pro.Plugins.DynamicScaler do
@moduledoc """
The `DynamicScaler` examines queue throughput and issues commands to horizontally scale
cloud infrastructure to optimize processing. With auto-scaling you can spin up additional nodes
during high traffic events, and pare down to a single node during a lull. Beyond optimizing
throughput, scaling may save money in environments with little to no usage at off-peak times,
e.g. staging.
Horizontal scaling is applied at the _node_ level, not the _queue_ level, so you can distribute
processing over more physical hardware.
* Predictive Scaling — The optimal scale is calculated by predicting the future size of a queue
based on recent trends. Multiple samples are then used to prevent overreacting to changes in
queue depth or throughput. You provide an acceptable `range` of nodes and auto-scaling takes
care of the rest.
* Multi-Cloud — Cloud integration is provided by a simple, flexible, behaviour that you
implement for your specific environment and configure for each scaler.
* Queue Filtering — By default, all queues are considered for scale calculations. However, you
can restrict calculations to one or more critical queues.
* Multiple Scalers — Some systems may restrict work to specific node types, e.g. generating
exports or processing videos. Other hybrid systems may straddle multiple clouds. In either
case, you can configure multiple independent scalers driven by distinct queues.
* Non Linear — An optional `step` parameter allows you to conservatively scale up or down one
node at a time, or optimize for responsiveness and jump from the min to the max in a single
scaling period.
* Prevent Thrashing — A `cooldown` period skips scaling when there was recent scale activity to
prevent unnecessarily scaling nodes up or down. Nodes may take several minutes to start within
an environment, so the default `cooldown` period is 2 minutes.
## Using and Configuring
> #### Clouds {: .info}
>
> There are ample hosting platforms, aka "clouds", out there and we can't support them all!
> Before you can begin dynamic scaling you'll need to implement a `Cloud` module for your
> environment. Don't worry, we have [copy-and-paste examples](#module-cloud-modules) for some
> popular platforms and a guide to walk through [implementations for your
> environment](#module-writing-cloud-modules).
With a `cloud` module in hand you're ready to add the `DynamicScaler` plugin to your Oban
config:
config :my_app, Oban,
plugins: [
{Oban.Pro.Plugins.DynamicScaler, ...}
]
Then, add a scaler with a `range` to define the minimum and maximum nodes, and your `cloud`
strategy:
{DynamicScaler, scalers: [range: 1..5, cloud: MyApp.Cloud]}
Now, every minute, `DynamicScaler` will calculate the optimal number of nodes for your queue's
throughput and issue scaling commands accordingly.
## Options
* `:scalers` — one or more scaler configurations. A single scaler is a keyword list of scaler
options (see below). Multiple scalers are a list of keyword lists. Required.
* `:timeout` — the maximum time allowed for throughput queries, in milliseconds. Defaults to
`15_000`. See [Optimizing Throughput Queries](#module-optimizing-throughput-queries).
### Configuring Scalers
Each scaler accepts the following options:
* `:cloud` — A `module` or `{module, options}` tuple that interacts with the external cloud
during scale events. Required.
* `:range` — The range of compute units to scale between. For example, `1..3` declares a minimum
of 1 node and a maximum of 3. The minimum must be 0 or more, and the maximum must be 1 or at
least match the minimum. Required.
* `:cooldown` — The minimum time between scaling events. Defaults to 120 seconds.
* `:lookback` — The historic time to check queues. Defaults to 60 seconds.
* `:queues` — Either `:all` or a list of queues to consider when measuring throughput and
backlog.
* `:step` — Either `:none` or the maximum nodes to scale up or down at once. Defaults to
`:none`.
### Scaler Examples
Filter throughput queries to the `:media` queue:
scalers: [queues: :media, range: 1..3, cloud: MyApp.Cloud]
Filter throughput queries to both `:audio` and `:video` queues:
scalers: [queues: [:audio, :video], range: 1..3, cloud: MyApp.Cloud]
Configure scalers driven by different queues (note, queues _may not_ overlap):
scalers: [
[queues: :audio, range: 0..2, cloud: {MyApp.Cloud, asg: "my-audio-asg"}],
[queues: :video, range: 0..5, cloud: {MyApp.Cloud, asg: "my-video-asg"}]
]
Limit scaling to one node up or down at a time:
scalers: [range: 1..3, step: 1, cloud: MyApp.Cloud]
Wait at least 5 minutes (300 seconds) between scaling events:
scalers: [range: 1..3, cloud: MyApp.Cloud, cooldown: 300]
Increase the period used to calculate historic throughput to 90 seconds:
scalers: [range: 1..3, cloud: MyApp.Cloud, lookback: 90]
> ### Scaling Down to Zero Nodes {: .warning}
>
> It's possible to scale down to zero nodes in staging environments or production applications
> with periods of downtime. However, it is **only viable for multi-node setups** with dedicated
> worker nodes and another instance type that isn't controlled by `DynamicScaler`. Without a
> separate "web" node, or something that is always running, you run the risk of scaling down
> without the ability to scale back up.
## Cloud Modules
There are a lot of hosting platforms, aka "clouds" out there and we can't support them all! Even
with optional dependencies, it would be a mess of libraries that may not agree with your
application decisions. Instead, the `Oban.Pro.Cloud` behaviour defines two simple callbacks, and
integrating with platforms typically takes a single HTTP query or library call.
The following links contain gists of full implementations for popular cloud platforms. Feel free
to copy-and-paste to use them as-is or as the basis for your own cloud modules.
* [EC2](https://gist.github.com/sorentwo/ba4a07d3a011d212c19a5bb775a6c536)
* [Fly](https://gist.github.com/sorentwo/d6be222091db7ba3c5b50d8bcabca252)
* [GCP](https://gist.github.com/sorentwo/a54a1e2b37123fd627cca16f07aed951)
* [Gigalixir](https://gist.github.com/sorentwo/90b4a427819756f7ceb72254ad53d9bb)
* [Heroku](https://gist.github.com/sorentwo/54d99fb2ac05cb63ea1e30aa1935b6fc)
* [Kubernetes](https://gist.github.com/sorentwo/f5ba9048e1e91456d37a8c7d4b8e4d58)
Let us know if an integration for your platform is missing (which is rather likely) and you'd
like assistance. Otherwise, follow the guide below to write your own integration!
### Writing Cloud Modules
Cloud callback modules must define an `init/1` function to prepare configuration at runtime, and
a `scale/2` callback called with the desired number of nodes and the prepared configuration.
The following example demonstrates a complete callback module for scaling EC2 Auto Scaling
Groups on AWS using the [SetDesiredCapacity][sdc] action. It assumes you're using the
[ex_aws][exa] package with the proper credentials.
defmodule MyApp.ASG do
@behaviour Oban.Pro.Cloud
@impl Oban.Pro.Cloud
def init(opts), do: Map.new(opts)
@impl Oban.Pro.Cloud
def scale(desired, conf) do
params = %{
"Action" => "SetDesiredCapacity",
"AutoScalingGroupName" => conf.asg,
"DesiredCapacity" => desired,
"Version" => "2011-01-01"
}
query = %ExAws.Operation.Query{
path: "",
params: params,
service: :autoscaling,
action: :set_desired_capacity
}
with {:ok, _} <- ExAws.request(query), do: {:ok, conf}
end
end
You'd then use your cloud module as a scaler option:
{DynamicScaler, scalers: [range: 1..3, cloud: {MyApp.ASG, asg: "my-asg-name"}]}
Clouds can also pull from the application or system environment to build configuration. If your
module pulls from the environment exclusively, then you can pass the module name rather than a
tuple:
{DynamicScaler, scalers: [range: 1..3, cloud: MyApp.ASG]}
[sdc]: https://docs.aws.amazon.com/autoscaling/ec2/APIReference/API_SetDesiredCapacity.html
[exa]: https://github.com/ex-aws/ex_aws
### Optimizing Throughput Queries
While the scaler's throughput queries are optimized for a standard load, high throughput queues,
or systems that retain a large volume of jobs, may benefit from an additional index that aids
calculating throughput. Use the following migration to add an index if you find that scaling
queries are too slow or timing out:
@disable_ddl_transaction true
@disable_migration_lock true
def change do
create_if_not_exists index(
:oban_jobs,
[:state, :queue, :attempted_at, :attempted_by],
concurrently: true,
where: "attempted_at IS NOT NULL",
prefix: "public"
)
end
Alternatively, you can change the timeout used for scaler inspection queries:
{DynamicScaler, timeout: :timer.seconds(15), scalers: ...}
## Instrumenting with Telemetry
The `DynamicScaler` plugin adds the following metadata to the `[:oban, :plugin, :stop]` event:
* `:scaler` - details of the active scaler config with recent scaling values
* `:skipped` - the reason scaling is skipped, either `:recently_scaled` or `:already_scaled`
* `:error` — the reason returned from `scale/2` when scaling fails
When multiple `scalers` are configured one event is emitted for _each_ scaler.
"""
@behaviour Oban.Plugin
use GenServer
import DateTime, only: [utc_now: 0]
import Ecto.Query
alias __MODULE__, as: State
alias Oban.{Job, Notifier, Peer, Repo, Validation}
alias Oban.Plugins.Cron
require Logger
defmodule Scaler do
@moduledoc false
@enforce_keys [:cloud, :range]
defstruct [
:cloud,
:last_rate,
:last_scaled_at,
:last_scaled_to,
:last_size,
:range,
cooldown: 120,
lookback: 60,
queues: :all,
step: :none
]
def new(opts) do
opts
|> Keyword.update(:cloud, nil, &init_cloud/1)
|> Keyword.update(:queues, :all, &normalize_queues/1)
|> then(&struct!(__MODULE__, &1))
end
defp init_cloud(cmod) when is_atom(cmod), do: {cmod, cmod.init([])}
defp init_cloud({cmod, copt}), do: {cmod, cmod.init(copt)}
defp normalize_queues(:all), do: :all
defp normalize_queues(queues) do
queues
|> List.wrap()
|> Enum.map(&to_string/1)
end
end
defstruct [:conf, :name, :timer, scalers: [], timeout: :timer.seconds(15)]
defmacrop attempted_by_node(column, position) do
quote do
fragment("?[?]", unquote(column), unquote(position))
end
end
@doc false
def child_spec(args), do: super(args)
@impl Oban.Plugin
def start_link(opts) do
opts =
Keyword.update!(opts, :scalers, fn scalers ->
scalers
|> wrap_keyword()
|> Enum.map(&Scaler.new/1)
end)
state = struct!(State, opts)
GenServer.start_link(__MODULE__, state, name: state.name)
end
defp wrap_keyword([elem | _] = list) when is_tuple(elem), do: [list]
defp wrap_keyword(list), do: list
@impl Oban.Plugin
def validate(opts) do
Validation.validate_schema(opts,
conf: :ok,
name: :ok,
scalers: {:custom, &validate_scalers(wrap_keyword(&1))},
timeout: :timeout
)
end
@impl Oban.Plugin
def format_logger_output(_conf, %{skipped: reason}), do: %{skipped: reason}
def format_logger_output(_conf, %{error: reason}), do: %{error: inspect(reason)}
def format_logger_output(_conf, %{scaler: scaler}) do
cloud =
case scaler.cloud do
{module, _opts} -> module
module -> module
end
Map.from_struct(%{scaler | cloud: inspect(cloud), range: inspect(scaler.range)})
end
@doc false
def optimal_scale(size, rate, min..max//_) do
cond do
rate == 0 and size > 0 and min == 0 ->
1
size == 0 and rate > 0 and min == 0 ->
1
rate == 0 ->
min
true ->
optimal = size / rate
optimal
|> ceil()
|> max(min)
|> min(max)
end
end
@doc false
def clamped_scale(size, _last, _range, :none), do: size
def clamped_scale(size, size, _range, _step), do: size
def clamped_scale(size, last, _min..max//_, step) when size > last do
(last + step)
|> min(size)
|> min(max)
end
def clamped_scale(size, last, min.._max//_, step) when size < last do
(last - step)
|> max(size)
|> max(min)
end
# Callbacks
@impl GenServer
def init(state) do
:telemetry.execute([:oban, :plugin, :init], %{}, %{conf: state.conf, plugin: __MODULE__})
{:ok, state, {:continue, :start}}
end
@impl GenServer
def handle_continue(:start, state) do
:ok = Notifier.listen(state.conf.name, [:scaler])
{:noreply, schedule_scaling(state)}
end
@impl GenServer
def handle_call(:check_scale, _from, %State{} = state) do
{:reply, :ok, check_and_scale(state)}
end
@impl GenServer
def handle_info(:check_scale, %State{} = state) do
state =
if Peer.leader?(state.conf) do
check_and_scale(state)
else
state
end
{:noreply, schedule_scaling(state)}
end
def handle_info({:notification, :scaler, payload}, state) do
{queues, payload} = Map.pop!(payload, "queues")
if Peer.leader?(state.conf) do
{:noreply, state}
else
scalers =
Enum.map(state.scalers, fn scaler ->
if queues == to_scaler_id(scaler.queues) do
Enum.reduce(payload, scaler, fn
{"last_rate", val}, acc ->
%{acc | last_rate: val}
{"last_size", val}, acc ->
%{acc | last_size: val}
{"last_scaled_to", val}, acc ->
%{acc | last_scaled_to: val}
{"last_scaled_at", nil}, acc ->
%{acc | last_scaled_at: nil}
{"last_scaled_at", val}, acc ->
{:ok, at, _offset} = DateTime.from_iso8601(val)
%{acc | last_scaled_at: at}
end)
else
scaler
end
end)
{:noreply, %{state | scalers: scalers}}
end
end
def handle_info(message, state) do
Logger.warning(
message: "Received unexpected message: #{inspect(message)}",
source: :oban_pro,
module: __MODULE__
)
{:noreply, state}
end
defp to_scaler_id(:all), do: "all"
defp to_scaler_id(queues) when is_list(queues), do: Enum.map(queues, &to_string/1)
# Validations
defp validate_scalers([]), do: :ok
defp validate_scalers([head | tail]) do
with :ok <- Validation.validate(:scalers, head, &validate_scaler/1) do
validate_scalers(tail)
end
end
defp validate_scalers(scalers) do
{:error, "expected :scaler to be a list of keywords, got: #{inspect(scalers)}"}
end
defp validate_scaler({:cloud, cloud}) do
case cloud do
{cmod, copt} when is_atom(cmod) and is_list(copt) ->
:ok
cmod when is_atom(cmod) ->
:ok
_ ->
{:error,
"expected :cloud to be a module or {module, options} tuple, got: #{inspect(cloud)}"}
end
end
defp validate_scaler({:cooldown, cooldown}) do
Validation.validate_integer(:cooldown, cooldown, min: 0)
end
defp validate_scaler({:lookback, cooldown}) do
Validation.validate_integer(:lookback, cooldown)
end
defp validate_scaler({:queues, queues}) do
case queues do
:all ->
:ok
queue when is_atom(queue) and not is_nil(queue) ->
:ok
[queue | _] when is_atom(queue) or is_binary(queue) ->
:ok
_ ->
{:error, "expected :queues to be :all or a list of queue names, got: #{inspect(queues)}"}
end
end
defp validate_scaler({:range, range}) do
case range do
min..max//_ when min <= max and max > 0 ->
:ok
_min.._max//_ ->
{:error, "expected :range to have a min less than max, got: #{inspect(range)}"}
_ ->
{:error, "expected :range to be a range of integers, got: #{inspect(range)}"}
end
end
defp validate_scaler({:step, :none}), do: :ok
defp validate_scaler({:step, limit}) do
Validation.validate_integer(:step, limit)
end
# Necessary to seed testing, not documented
defp validate_scaler({:last_rate, _}), do: :ok
defp validate_scaler({:last_scaled_at, _}), do: :ok
defp validate_scaler({:last_scaled_to, _}), do: :ok
defp validate_scaler({:last_size, _}), do: :ok
defp validate_scaler(option), do: {:unknown, option, Scaler}
# Scheduling
defp schedule_scaling(state) do
timer = Process.send_after(self(), :check_scale, Cron.interval_to_next_minute())
%{state | timer: timer}
end
# Scaling
defp check_and_scale(%State{scalers: scalers} = state) do
scalers =
for scaler <- scalers do
meta = %{conf: state.conf, plugin: __MODULE__, scaler: scaler}
:telemetry.span([:oban, :plugin], meta, fn ->
prev_size = scaler.last_size
scaler =
scaler
|> record_size(state)
|> record_rate(state)
case apply_scale(scaler, prev_size) do
{:ok, scaler} ->
notify_scalers(scaler, state)
{scaler, Map.put(meta, :scaler, scaler)}
{:skipped, reason} ->
notify_scalers(scaler, state)
{scaler, Map.put(meta, :skipped, reason)}
{:error, reason} ->
{scaler, Map.put(meta, :error, reason)}
end
end)
end
%{state | scalers: scalers}
end
defp notify_scalers(scaler, %{conf: conf}) do
payload = Map.take(scaler, ~w(last_rate last_scaled_at last_scaled_to last_size queues)a)
Notifier.notify(conf, :scaler, payload)
end
defp record_size(scaler, state) do
next = DateTime.add(utc_now(), scaler.lookback, :second)
query =
scaler.queues
|> base_query()
|> where([j], j.state in ~w(available scheduled retryable) and j.scheduled_at <= ^next)
|> select(count())
%{scaler | last_size: Repo.one(state.conf, query, timeout: state.timeout)}
end
defp record_rate(scaler, state) do
since = DateTime.add(utc_now(), -scaler.lookback, :second)
query =
scaler.queues
|> base_query()
|> where([j], j.state in ~w(executing retryable completed cancelled discarded))
|> where([j], j.attempted_at > ^since)
|> group_by([j], attempted_by_node(j.attempted_by, 1))
|> select([j], {attempted_by_node(j.attempted_by, 1), count()})
case Repo.all(state.conf, query, timeout: state.timeout) do
[] ->
%{scaler | last_rate: 0, last_scaled_to: scaler.last_scaled_to || 0}
node_counts ->
size = length(node_counts)
rate =
node_counts
|> Enum.map(&elem(&1, 1))
|> Enum.sum()
|> div(size)
%{scaler | last_rate: rate, last_scaled_to: scaler.last_scaled_to || size}
end
end
defp base_query(queues) do
if queues == :all do
where(Job, [j], not is_nil(j.queue))
else
where(Job, [j], j.queue in ^queues)
end
end
defp apply_scale(scaler, prev_size) do
scale_to =
prev_size
|> next_size(scaler.last_size)
|> optimal_scale(scaler.last_rate, scaler.range)
|> clamped_scale(scaler.last_scaled_to, scaler.range, scaler.step)
cond do
recently_scaled?(scaler) ->
{:skipped, :recently_scaled}
already_scaled?(scale_to, scaler) ->
{:skipped, :already_scaled}
true ->
{cloud_mod, cloud_opt} = scaler.cloud
case cloud_mod.scale(scale_to, cloud_opt) do
{:ok, cloud_opt} ->
cloud = {cloud_mod, cloud_opt}
{:ok, %{scaler | cloud: cloud, last_scaled_at: utc_now(), last_scaled_to: scale_to}}
{:error, reason} ->
{:error, reason}
end
end
end
defp next_size(nil, size), do: size
defp next_size(prev, curr), do: max(curr + (curr - prev), 0)
defp recently_scaled?(%{last_scaled_at: nil}), do: false
defp recently_scaled?(%{cooldown: cooldown, last_scaled_at: scaled_at}) do
scaling_allowed_at = DateTime.add(scaled_at, cooldown)
DateTime.compare(utc_now(), scaling_allowed_at) != :gt
end
defp already_scaled?(scale_to, %{last_scaled_to: to}), do: scale_to == to
end

376
vendor/oban_pro/lib/oban/pro/producer.ex vendored Normal file
View file

@ -0,0 +1,376 @@
require Protocol
defmodule Oban.Pro.Producer do
@moduledoc false
use Ecto.Schema
import Ecto.Changeset
alias Ecto.Changeset
alias Oban.Pro.Utils
defmodule ListOrMap do
@moduledoc false
# This is a custom type to allow a graceful transition from the legacy rate limit windows
# structure to the new, flattened map.
@behaviour Ecto.Type
@impl true
def type, do: :any
@impl true
def cast(data) when is_map(data), do: {:ok, data}
def cast([data | _]), do: {:ok, data}
def cast(_data), do: :error
@impl true
def load(data) when is_map(data), do: {:ok, data}
def load([data | _]), do: {:ok, data}
def load([]), do: {:ok, %{}}
def load(_data), do: :error
@impl true
def dump(data) when is_map(data), do: {:ok, data}
def dump([data | _]), do: {:ok, data}
def dump([]), do: {:ok, %{}}
def dump(_data), do: :error
@impl true
def equal?(a, b), do: a == b
@impl true
def embed_as(_data), do: :dump
end
@type t :: %__MODULE__{
uuid: Ecto.UUID.t(),
name: binary(),
node: binary(),
queue: binary(),
meta: map(),
started_at: DateTime.t(),
updated_at: DateTime.t()
}
@type meta :: %__MODULE__.Meta{local_limit: pos_integer(), paused: boolean()}
@primary_key {:uuid, :binary_id, autogenerate: false}
schema "oban_producers" do
field :name, :string
field :node, :string
field :queue, :string
field :started_at, :utc_datetime_usec
field :updated_at, :utc_datetime_usec
field :ack_async, :boolean, virtual: true, default: true
field :ack_tab, :any, virtual: true
field :refresh_interval, :integer, virtual: true, default: :timer.seconds(30)
field :xact_delay, :integer, default: :timer.seconds(1), virtual: true
field :xact_retry, :integer, default: 5, virtual: true
field :xact_timeout, :integer, default: :timer.seconds(30), virtual: true
embeds_one :meta, Meta, on_replace: :update, primary_key: false do
@moduledoc false
field :local_limit, :integer, default: 10
field :paused, :boolean, default: false
field :shutdown_started_at, :utc_datetime_usec
embeds_one :global_limit, GlobalLimit, on_replace: :update, primary_key: false do
@moduledoc false
field :allowed, :integer
field :burst, :boolean, default: false
field :tracked, ListOrMap, default: %{}
embeds_one :partition, Partition, on_replace: :update, primary_key: false do
@moduledoc false
field :fields, {:array, Ecto.Enum}, values: [:args, :meta, :worker]
field :keys, {:array, :string}
end
end
embeds_one :rate_limit, RateLimit, on_replace: :update, primary_key: false do
@moduledoc false
field :algorithm, Ecto.Enum,
values: ~w(sliding_window fixed_window token_bucket)a,
default: :sliding_window
field :allowed, :integer
field :period, :integer
field :window_time, :integer, skip_default_validation: true
field :windows, ListOrMap, default: %{}
embeds_one :partition, Partition, on_replace: :update, primary_key: false do
@moduledoc false
field :fields, {:array, Ecto.Enum}, values: [:args, :meta, :worker]
field :keys, {:array, :string}
end
end
end
end
@spec new(map() | Keyword.t()) :: Changeset.t(t())
def new(params) when is_list(params) or is_map(params) do
params =
params
|> Map.new()
|> Map.put_new_lazy(:uuid, &Oban.Pro.UUIDv7.generate/0)
|> Map.update!(:name, &to_clean_string/1)
|> Map.update!(:node, &to_clean_string/1)
|> Map.update!(:queue, &to_clean_string/1)
%__MODULE__{}
|> cast(params, ~w(name node queue refresh_interval started_at updated_at uuid)a)
|> cast_embed(:meta, required: true, with: &meta_changeset/2)
|> validate_required(~w(name node queue)a)
|> validate_length(:name, min: 1)
|> validate_length(:node, min: 1)
|> validate_length(:queue, min: 1)
|> Utils.enforce_keys(params, __MODULE__)
end
@spec update_meta(t(), atom(), term()) :: Changeset.t()
def update_meta(schema, key, value) do
update_meta(schema, %{key => value})
end
@spec update_meta(t(), map()) :: Changeset.t()
def update_meta(schema, changes) do
meta =
schema.meta
|> meta_changeset(changes)
|> apply_changes()
|> Ecto.embedded_dump(:json)
schema
|> cast(%{meta: meta}, [])
|> cast_embed(:meta, with: &meta_changeset/2)
end
@allowed ~w(local_limit paused shutdown_started_at)a
@doc false
def meta_changeset(schema, params, allowed \\ @allowed) do
params =
params
|> Map.new()
|> Map.delete(:limit)
|> Map.put_new_lazy(:local_limit, fn -> default_local_limit(params, schema) end)
params =
if Map.get(params, :global_limit) do
Map.update!(params, :global_limit, &cast_global_limit/1)
else
params
end
params =
if Map.get(params, :rate_limit) do
Map.update!(params, :rate_limit, &cast_rate_limit/1)
else
params
end
schema
|> cast(params, allowed)
|> cast_embed(:global_limit, with: &global_changeset/2)
|> cast_embed(:rate_limit, with: &rate_changeset/2)
|> validate_required(~w(local_limit)a)
|> validate_number(:local_limit, greater_than: 0)
|> validate_single_partitioner()
|> Utils.enforce_keys(params, schema.__struct__)
end
@doc false
@spec default_local_limit(map(), Ecto.Schema.t()) :: non_neg_integer()
def default_local_limit(params, schema) do
cond do
is_integer(params[:limit]) ->
params[:limit]
is_integer(params[:global_limit]) ->
params[:global_limit]
is_integer(get_in(params, [:global_limit, :allowed])) ->
get_in(params, [:global_limit, :allowed])
true ->
schema.local_limit
end
end
@doc false
@spec validate_single_partitioner(Changeset.t()) :: Changeset.t()
def validate_single_partitioner(changeset) do
case {get_field(changeset, :global_limit), get_field(changeset, :rate_limit)} do
{%{partition: %{}}, %{partition: %{}}} ->
add_error(changeset, :global_limit, "only one limiter may have partitioning")
_ ->
changeset
end
end
@doc false
@spec global_changeset(Ecto.Schema.t(), integer() | map() | Keyword.t()) :: Changeset.t()
def global_changeset(schema, params) do
params = Map.update(params, :partition, nil, &normalize_partition/1)
schema
|> cast(params, ~w(allowed burst)a)
|> cast_embed(:partition, with: &partition_changeset/2)
|> validate_number(:allowed, greater_than: 0)
|> Utils.enforce_keys(params, schema.__struct__)
end
@doc false
@spec rate_changeset(Ecto.Schema.t(), map() | Keyword.t()) :: Changeset.t()
def rate_changeset(schema, params) do
params =
params
|> Map.update(:period, nil, &Utils.cast_period/1)
|> Map.update(:partition, nil, &normalize_partition/1)
|> Map.put_new_lazy(:window_time, fn -> DateTime.to_unix(DateTime.utc_now(), :second) end)
fields = schema.__struct__.__schema__(:fields) -- [:partition]
schema
|> cast(params, fields)
|> cast_embed(:partition, with: &partition_changeset/2)
|> validate_required(~w(allowed period)a)
|> validate_number(:allowed, greater_than: 0)
|> validate_number(:period, greater_than: 0)
|> clear_windows_on_algorithm_change()
|> Utils.enforce_keys(params, schema.__struct__)
end
defp clear_windows_on_algorithm_change(changeset) do
case get_change(changeset, :algorithm) do
nil -> changeset
_new_algorithm -> force_change(changeset, :windows, %{})
end
end
@doc false
@spec partition_changeset(Ecto.Schema.t(), map() | Keyword.t()) :: Changeset.t()
def partition_changeset(schema, params) do
params =
params
|> Map.new()
|> Map.update(:keys, [], &for(key <- &1, do: to_string(key)))
schema
|> cast(params, ~w(fields keys)a)
|> validate_required(~w(fields)a)
|> Utils.enforce_keys(params, schema.__struct__)
end
# Global Limit Helpers
@doc false
@spec cast_global_limit(pos_integer() | list() | map()) :: map()
def cast_global_limit(limit) when is_integer(limit) do
%{allowed: limit}
end
def cast_global_limit(opts), do: cast_rate_limit(opts)
# Rate Limit Helpers
@doc false
@spec cast_rate_limit(list() | map()) :: map()
def cast_rate_limit([[key | _] | _] = opts) when is_binary(key) do
opts
|> Enum.map(&List.to_tuple/1)
|> Map.new()
|> cast_rate_limit()
end
def cast_rate_limit(opts)
when is_map_key(opts, "algorithm") or
is_map_key(opts, "allowed") or
is_map_key(opts, "burst") or
is_map_key(opts, "period") or
is_map_key(opts, "partition") do
Map.new(opts, fn
{"algorithm", algorithm} ->
{:algorithm, String.to_existing_atom(algorithm)}
{"allowed", allowed} ->
{:allowed, allowed}
{"burst", allowed} ->
{:burst, allowed}
{"period", [time, unit]} ->
{:period, {time, unit}}
{"period", period} ->
{:period, period}
{"partition", [["fields", fields]]} ->
{:partition, fields: fields}
{"partition", [["fields", fields], ["keys", keys]]} ->
{:partition, fields: fields, keys: keys}
{"partition", field} when is_binary(field) ->
{:partition, fields: [field]}
end)
end
def cast_rate_limit(opts), do: opts
def normalize_partition(partition) do
cond do
is_nil(partition) ->
nil
is_map(partition) ->
partition
Keyword.keyword?(partition) and Keyword.has_key?(partition, :fields) ->
partition
true ->
partition
|> List.wrap()
|> Enum.reduce([fields: [], keys: []], fn
{field, keys}, opts when field in ~w(args meta)a ->
opts
|> Keyword.put(:keys, List.wrap(keys))
|> Keyword.update!(:fields, &[field | &1])
field, opts ->
Keyword.update!(opts, :fields, &[field | &1])
end)
end
end
# Helpers
defp to_clean_string(value) when is_reference(value) do
inspect(value)
end
defp to_clean_string(value) do
value
|> to_string()
|> String.trim_leading("Elixir.")
end
end
encoder = if Code.ensure_loaded?(JSON.Encoder), do: JSON.Encoder, else: Jason.Encoder
Protocol.derive(encoder, Oban.Pro.Producer, except: [:__meta__])
Protocol.derive(encoder, Oban.Pro.Producer.Meta.GlobalLimit)
Protocol.derive(encoder, Oban.Pro.Producer.Meta.GlobalLimit.Partition)
Protocol.derive(encoder, Oban.Pro.Producer.Meta.RateLimit)
Protocol.derive(encoder, Oban.Pro.Producer.Meta.RateLimit.Partition)

161
vendor/oban_pro/lib/oban/pro/queue.ex vendored Normal file
View file

@ -0,0 +1,161 @@
defmodule Oban.Pro.Queue do
@moduledoc false
use Ecto.Schema
import Ecto.Changeset
alias Oban.Pro.{Producer, Utils}
@allowed ~w(ack_async local_limit paused refresh_interval)a
@primary_key {:name, :string, autogenerate: false}
schema "oban_queues" do
field :lock_version, :integer, default: 1
field :hash, :string
embeds_one :only, Only, on_replace: :update, primary_key: false do
@moduledoc false
field :mode, Ecto.Enum, values: [:node, :sys_env]
field :op, Ecto.Enum, values: [:==, :!=, :=~]
field :key, :string
field :value, :string
end
embeds_one :opts, Opts, on_replace: :update, primary_key: false do
@moduledoc false
field :ack_async, :boolean
field :local_limit, :integer
field :paused, :boolean
field :refresh_interval, :integer
field :xact_delay, :integer
field :xact_retry, :integer
embeds_one :global_limit, GlobalLimit, on_replace: :update, primary_key: false do
@moduledoc false
field :allowed, :integer
field :burst, :boolean, default: false
embeds_one :partition, Partition, on_replace: :update, primary_key: false do
@moduledoc false
field :fields, {:array, Ecto.Enum}, values: [:args, :meta, :worker]
field :keys, {:array, :string}
end
end
embeds_one :rate_limit, RateLimit, on_replace: :update, primary_key: false do
@moduledoc false
field :algorithm, Ecto.Enum,
values: ~w(sliding_window fixed_window token_bucket)a,
default: :sliding_window
field :allowed, :integer
field :period, :integer
field :window_time, :integer, virtual: true, skip_default_validation: true
embeds_one :partition, Partition, on_replace: :update, primary_key: false do
@moduledoc false
field :fields, {:array, Ecto.Enum}, values: [:args, :meta, :worker]
field :keys, {:array, :string}
end
end
end
timestamps(inserted_at: :inserted_at, updated_at: :updated_at, type: :utc_datetime_usec)
end
def normalize(params) do
case params do
[_ | _] -> Map.new(params)
{name, [_ | _] = opts} -> %{name: name, opts: Map.new(opts)}
{name, limit} -> %{name: name, opts: %{local_limit: limit}}
end
end
def changeset(params) do
changeset(%__MODULE__{}, normalize(params))
end
def changeset(schema, params) do
params =
params
|> Map.replace_lazy(:name, fn _ -> to_string(params[:name]) end)
|> extract_only()
schema
|> cast(params, [:name])
|> cast_embed(:only, with: &only_changeset/2)
|> cast_embed(:opts, required: true, with: &opts_changeset/2)
|> validate_required([:name])
|> validate_length(:name, min: 1)
|> optimistic_lock(:lock_version)
|> inject_hash(params)
|> Utils.enforce_keys(params, __MODULE__)
end
defp extract_only(params) do
case params do
%{opts: %{only: _}} ->
{only, params} = pop_in(params, [:opts, :only])
Map.put(params, :only, cast_only(only))
_ ->
params
end
end
defp only_changeset(schema, params) do
schema
|> cast(params, ~w(mode op key value)a)
|> validate_required(~w(mode op value)a)
|> Utils.enforce_keys(params, __MODULE__.Only)
end
defp opts_changeset(schema, params) do
params = Map.new(params, fn {key, val} -> {Utils.maybe_to_atom(key), val} end)
Producer.meta_changeset(schema, params, @allowed)
end
defp inject_hash(changeset, params) do
hash =
params
|> Map.take(~w(only opts)a)
|> Enum.map_join(&:erlang.phash2/1)
|> Utils.hash64()
put_change(changeset, :hash, hash)
end
@spec to_keyword_opts(%{opts: map()} | map()) :: Keyword.t()
def to_keyword_opts(%__MODULE__{name: queue, opts: opts}) do
opts
|> Ecto.embedded_dump(:json)
|> Map.put(:queue, queue)
|> to_keyword_opts()
end
def to_keyword_opts(opts) do
for {key, val} <- opts, not is_nil(val), do: {Utils.maybe_to_atom(key), val}
end
# Helpers
defp cast_only({:node, value}), do: cast_only({:node, :==, value})
defp cast_only({:node, op, value}), do: %{mode: :node, op: op, value: to_string(value)}
defp cast_only({:sys_env, key, value}), do: cast_only({:sys_env, key, :==, value})
defp cast_only({:sys_env, key, op, value}) do
%{mode: :sys_env, op: op, key: key, value: to_string(value)}
end
defp cast_only(other), do: other
end

384
vendor/oban_pro/lib/oban/pro/rate_limit.ex vendored Normal file
View file

@ -0,0 +1,384 @@
defmodule Oban.Pro.RateLimit do
@moduledoc """
Programmatic API for inspecting and manipulating rate limit state.
Rate limits are normally managed automatically by the Smart engine as jobs execute. This module
provides functions to interact with rate limits outside of normal job execution: checking
available capacity, manually consuming tokens, and resetting state.
All functions in this module require the [Smart engine](`Oban.Pro.Engines.Smart`) and a queue
configured with `rate_limit` options.
## Checking Available Capacity
Use `available/2` to check how much capacity remains before performing operations:
```elixir
case Oban.Pro.RateLimit.available(:some_api) do
{:ok, capacity} when capacity >= 10 ->
make_api_calls(10)
{:ok, capacity} when capacity > 0 ->
make_api_calls(capacity)
{:ok, 0} ->
{:snooze, 60}
end
```
For partitioned rate limits, check a specific partition with a job's computed `partition_key`:
```elixir
Oban.Pro.RateLimit.available(:some_api, partition: job.meta["partition_key"])
```
## Manual Consumption
Use `consume/3` when you need to track rate-limited operations that happen outside of job
execution. This ensures the rate limit reflects all usage, not just job execution. Consumption
can even track operations entirely outside of Oban:
```elixir
def handle_webhook(conn, params) do
:ok = Oban.Pro.RateLimit.consume(:api_calls, 1)
result = ExternalAPI.process(params)
json(conn, result)
end
```
## Resetting State
Use `reset/2` to clear all rate limit tracking. This is primarily for recovery after
configuration changes when limits should be cleared:
```elixir
:ok = Oban.Pro.RateLimit.reset(:api_calls)
```
## See Also
* `Oban.Pro.Engines.Smart` — Rate limit configuration and algorithms
* `Oban.Pro.Worker` — Weighted jobs with the `weight/1` callback
"""
@moduledoc since: "1.7.0"
import Ecto.Query
import DateTime, only: [utc_now: 0]
alias Oban.Pro.Limiters.Rate.Algorithm
alias Oban.Pro.Producer
alias Oban.Repo
@type queue :: atom() | String.t()
@type option :: {:oban, Oban.name()} | {:partition, String.t()}
@type wait_option :: {:timeout, timeout()} | {:interval, pos_integer()}
@doc """
Manually consume rate limit capacity for a queue.
Consumption is applied to the producer with the most available capacity. If the requested
amount exceeds a single producer's capacity, consumption is spread across multiple producers.
## Options
* `:oban` - The Oban instance name. Defaults to `Oban`.
* `:partition` - The partition key to consume from. Defaults to `"*"` (the global partition).
* `:require_full` - When `true`, returns `{:error, :insufficient_capacity}` if the full
amount can't be consumed. Defaults to `false`, which consumes as much as available.
## Examples
Consume 5 units from the default partition
Oban.Pro.RateLimit.consume(:my_queue, 5)
Consume from a specific partition
Oban.Pro.RateLimit.consume(:my_queue, 3, partition: job.meta["partition_key"])
Use a named Oban instance
Oban.Pro.RateLimit.consume(:my_queue, 1, oban: MyApp.Oban)
"""
@spec consume(queue(), pos_integer(), [option()]) ::
:ok | {:error, :insufficient_capacity | :no_rate_limit | :queue_not_found}
def consume(queue, amount, opts \\ []) when is_integer(amount) and amount > 0 do
conf = fetch_conf(opts)
partition = Keyword.get(opts, :partition, "*")
require_full = Keyword.get(opts, :require_full, false)
func = fn ->
with {:ok, producers} <- fetch_producers(conf, queue, take_lock: true),
{:ok, rate_limit} <- fetch_rate_limit(hd(producers)) do
curr_time = unix_now()
callback = Algorithm.callback(rate_limit)
sorted =
producers
|> Enum.map(fn producer ->
rl = producer.meta.rate_limit
capacity =
callback.demand(rl.windows, rl.allowed, rl.period, curr_time, rl.window_time)
{producer, capacity}
end)
|> Enum.sort_by(&elem(&1, 1), :desc)
total_capacity = Enum.reduce(sorted, 0, fn {_, cap}, acc -> acc + cap end)
if require_full and total_capacity < amount do
{:error, :insufficient_capacity}
else
consume(sorted, conf, partition, amount, curr_time, callback)
:ok
end
end
end
with {:ok, result} <- Repo.transaction(conf, func), do: result
end
@doc """
Check the available rate limit capacity for a queue.
Returns the total available capacity across all producers for the queue, calculated by merging
window states from all active producers.
## Options
* `:oban` - The Oban instance name. Defaults to `Oban`.
* `:partition` - The partition key to check. Defaults to `"*"` (the global partition).
## Examples
Check capacity for a queue:
{:ok, capacity} = Oban.Pro.RateLimit.available(:my_queue)
Check capacity for a specific partition:
{:ok, capacity} = Oban.Pro.RateLimit.available(:my_queue, partition: job.meta["partition_key"])
"""
@spec available(queue(), [option()]) ::
{:ok, non_neg_integer()} | {:error, :no_rate_limit | :queue_not_found}
def available(queue, opts \\ []) do
conf = fetch_conf(opts)
partition = Keyword.get(opts, :partition, "*")
with {:ok, producers} <- fetch_producers(conf, queue),
{:ok, rate_limit} <- fetch_rate_limit(hd(producers)) do
curr_time = unix_now()
callback = Algorithm.callback(rate_limit)
windows =
producers
|> Enum.map(& &1.meta.rate_limit)
|> Enum.filter(&(&1.window_time >= curr_time - &1.period))
|> Enum.reduce(%{}, fn rl, acc -> callback.merge(rl.windows, acc) end)
demand =
if is_nil(rate_limit.partition) do
callback.demand(
windows,
rate_limit.allowed,
rate_limit.period,
curr_time,
rate_limit.window_time
)
else
demands =
callback.partition_demands(
windows,
rate_limit.allowed,
rate_limit.period,
curr_time,
rate_limit.window_time,
[partition]
)
Map.get(demands, partition, 0)
end
{:ok, demand}
end
end
@doc """
Execute a function after atomically reserving rate limit capacity.
This function waits for capacity to become available, atomically consumes the requested amount,
then executes the provided function. This prevents race conditions where multiple callers might
consume the same quota.
## Options
* `:oban` - The Oban instance name. Defaults to `Oban`.
* `:partition` - The partition key to check. Defaults to `"*"` (the global partition).
* `:timeout` - Maximum time to wait in milliseconds. Defaults to `5_000` (5 seconds).
* `:interval` - Polling interval in milliseconds. Defaults to `100`.
## Examples
Execute a function after reserving 5 units of capacity:
{:ok, result} = Oban.Pro.RateLimit.with_quota(:my_queue, 5, fn ->
ExternalAPI.batch_request(items)
end)
Handle timeout when capacity isn't available:
case Oban.Pro.RateLimit.with_quota(:my_queue, 5, &make_api_calls/0, timeout: 10_000) do
{:ok, result} -> handle_result(result)
{:error, :timeout} -> handle_timeout()
end
Reserve capacity on a specific partition:
{:ok, result} = Oban.Pro.RateLimit.with_quota(:my_queue, 1, fun, partition: "user_123")
"""
@spec with_quota(queue(), pos_integer(), (-> result), [option() | wait_option()]) ::
{:ok, result} | {:error, :timeout | :no_rate_limit | :queue_not_found}
when result: term()
def with_quota(queue, amount, fun, opts \\ []) when is_integer(amount) and amount > 0 do
{timeout, opts} = Keyword.pop(opts, :timeout, 5_000)
{interval, opts} = Keyword.pop(opts, :interval, 100)
deadline = System.monotonic_time(:millisecond) + timeout
with_quota_loop(queue, amount, fun, opts, deadline, interval)
end
defp with_quota_loop(queue, amount, fun, opts, deadline, interval) do
case consume(queue, amount, Keyword.put(opts, :require_full, true)) do
:ok ->
{:ok, fun.()}
{:error, :insufficient_capacity} ->
remaining = deadline - System.monotonic_time(:millisecond)
if remaining <= 0 do
{:error, :timeout}
else
Process.sleep(min(interval, remaining))
with_quota_loop(queue, amount, fun, opts, deadline, interval)
end
{:error, _reason} = error ->
error
end
end
@doc """
Reset the rate limit state for a queue.
This clears all window data and resets the window time for all producers on the queue. Tracking
is cleared across all partitions for partitioned queues.
## Options
* `:oban` - The Oban instance name. Defaults to `Oban`.
## Examples
Reset the rate limit for a queue:
:ok = Oban.Pro.RateLimit.reset(:my_queue)
Reset the rate limit for a queue:
:ok = Oban.Pro.RateLimit.reset(:my_queue, oban: MyApp.Oban)
"""
@spec reset(queue(), [option()]) :: :ok | {:error, :no_rate_limit | :queue_not_found}
def reset(queue, opts \\ []) do
conf = fetch_conf(opts)
func = fn ->
with {:ok, producers} <- fetch_producers(conf, queue, take_lock: true),
{:ok, _rate_lim} <- fetch_rate_limit(hd(producers)) do
curr_time = unix_now()
Enum.each(producers, fn producer ->
rate_limit = producer.meta.rate_limit
updated = %{rate_limit | windows: %{}, window_time: curr_time}
update_rate_limit(conf, producer, updated)
end)
end
end
with {:ok, result} <- Repo.transaction(conf, func), do: result
end
# Private Helpers
defp fetch_conf(opts) do
opts
|> Keyword.get(:oban, Oban)
|> Oban.config()
end
defp fetch_producers(conf, queue, opts \\ []) do
query =
if opts[:take_lock] do
Producer
|> where(queue: ^to_string(queue))
|> lock("FOR UPDATE")
else
where(Producer, queue: ^to_string(queue))
end
case Repo.all(conf, query) do
[_ | _] = producers -> {:ok, producers}
_ -> {:error, :queue_not_found}
end
end
defp fetch_rate_limit(producer) do
case producer.meta.rate_limit do
%{} = rate_limit -> {:ok, rate_limit}
nil -> {:error, :no_rate_limit}
end
end
defp consume([], _, _, _, _, _), do: :ok
defp consume(_, _, _, 0, _, _), do: :ok
defp consume([{producer, capacity} | rest], conf, partition, remaining, curr_time, callback) do
to_consume = min(capacity, remaining)
if to_consume > 0 do
rate_limit = producer.meta.rate_limit
new_counts = %{partition => to_consume}
{all_windows, next_time} =
callback.track(
rate_limit.windows,
new_counts,
rate_limit.period,
rate_limit.window_time,
curr_time,
rate_limit.allowed
)
updated = %{rate_limit | windows: all_windows, window_time: next_time}
update_rate_limit(conf, producer, updated)
end
consume(rest, conf, partition, remaining - to_consume, curr_time, callback)
end
defp update_rate_limit(conf, producer, rate_limit) do
meta = %{producer.meta | rate_limit: rate_limit}
Producer
|> where(uuid: ^producer.uuid)
|> then(&Repo.update_all(conf, &1, set: [meta: meta, updated_at: utc_now()]))
end
defp unix_now do
DateTime.to_unix(utc_now(), :second)
end
end

123
vendor/oban_pro/lib/oban/pro/refresher.ex vendored Normal file
View file

@ -0,0 +1,123 @@
defmodule Oban.Pro.Refresher do
@moduledoc false
# Centralized refreshing and cleaning of producer records across all Oban instances.
#
# The Refresher runs as a single process supervised by the Oban.Pro application. It refreshes
# producer records for all actively running queues on the current node.
use GenServer
import Ecto.Query, only: [where: 3]
alias __MODULE__, as: State
alias Oban.{Peer, Repo}
alias Oban.Pro.Producer
require Logger
defstruct [
:timer,
interval: :timer.seconds(15),
producers: %{},
producer_ttl: :timer.minutes(1)
]
@doc false
def start_link(opts \\ []) do
state = struct!(State, opts)
GenServer.start_link(__MODULE__, state, name: __MODULE__)
end
# GenServer Callbacks
@impl GenServer
def init(state) do
Process.flag(:trap_exit, true)
{:ok, schedule_refresh(state)}
end
@impl GenServer
def terminate(_reason, state) do
if is_reference(state.timer), do: Process.cancel_timer(state.timer)
:ok
end
@impl GenServer
def handle_call(:refresh, _from, state) do
{:noreply, state} = handle_info(:refresh, state)
{:reply, :ok, state}
end
@impl GenServer
def handle_info(:refresh, state) do
try do
refresh_producers(state)
cleanup_producers(state)
rescue
error ->
Logger.warning(
message: "Producer refresh failed",
source: :oban_pro,
module: __MODULE__,
reason: error
)
end
{:noreply, schedule_refresh(state)}
end
# Helpers
defp schedule_refresh(state) do
timer = Process.send_after(self(), :refresh, state.interval)
%{state | timer: timer}
end
defp refresh_producers(_state) do
match = [{{{:"$1", {:producer, :"$2"}}, :_, :_}, [], [[:"$1", :"$2"]]}]
Oban.Registry
|> Registry.select(match)
|> Enum.group_by(&hd/1, &Enum.at(&1, 1))
|> Enum.each(fn {oban_name, queues} ->
with {_pid, %{testing: :disabled} = conf} <- Oban.Registry.lookup(oban_name) do
now = DateTime.utc_now()
ids = Enum.flat_map(queues, &lookup_uuid(&1, oban_name))
refresh_query = where(Producer, [p], p.uuid in ^ids)
Repo.update_all(conf, refresh_query, set: [updated_at: now])
end
end)
end
defp cleanup_producers(state) do
match = [{{:"$1", :_, :"$2"}, [{:not, {:is_tuple, :"$1"}}], [:"$2"]}]
Oban.Registry
|> Registry.select(match)
|> Enum.filter(&Peer.leader?/1)
|> Enum.each(fn conf ->
now = DateTime.utc_now()
outdated_at = DateTime.add(now, -state.producer_ttl, :millisecond)
cleanup_query = where(Producer, [p], p.updated_at <= ^outdated_at)
Repo.delete_all(conf, cleanup_query)
end)
end
defp lookup_uuid(queue, oban_name) do
prod_name = {oban_name, {:producer, queue}}
case Registry.meta(Oban.Registry, prod_name) do
{:ok, %{uuid: uuid}} -> [uuid]
_error -> []
end
end
end

408
vendor/oban_pro/lib/oban/pro/relay.ex vendored Normal file
View file

@ -0,0 +1,408 @@
defmodule Oban.Pro.Relay do
@moduledoc """
The `Relay` extension lets you insert and await the results of jobs locally or remotely, across
any number of nodes, so you can seamlessly distribute jobs and await the results synchronously.
Think of `Relay` as persistent, distributed tasks.
`Relay` uses PubSub for to transmit results. That means it will work without Erlang distribution
or clustering, but it does require Oban to have functional PubSub. It doesn't matter which node
executes a job, the result will still be broadcast back.
Results are encoded using `term_to_binary/2` and decoded using `binary_to_term/2` using the
`:safe` option to prevent the creation of new atoms or function references. If you're returning
results with atoms you _must be sure_ that those atoms are defined locally, where the `await/2`
or `await_many/2` function is called.
## Usage
Use `async/1` to insert a job for asynchronous execution:
```elixir
relay =
%{id: 123}
|> MyApp.Worker.new()
|> Oban.Pro.Relay.async()
```
After inserting a job and constructing a relay, use `await/1` to await the job's execution and
return the result:
```elixir
{:ok, result} =
%{id: 123}
|> MyApp.Worker.new()
|> Oban.Pro.Relay.async()
|> Oban.Pro.Relay.await()
```
By default, `await/1` will timeout after 5 seconds and return an `{:error, :timeout}` tuple. The job
itself may continue to run, only the local process stops waiting on it. Pass an explicit timeout
to wait longer:
```elixir
{:ok, result} = Oban.Pro.Relay.await(relay, :timer.seconds(30))
```
Any successful result such as `:ok`, `{:ok, value}`, or `{:cancel, reason}` is passed back as
the await result. When the executed job returns an `{:error, reason}` tuple, raises an
exception, or crashes, the result comes through as an error tuple.
Use `await_many/1` when you need the results of multiple async relays:
```elixir
relayed =
1..3
|> Enum.map(&DoubleWorker.new(%{int: &1}))
|> Enum.map(&Oban.Pro.Relay.async/1)
|> Oban.Pro.Relay.await_many()
#=> [{:ok, 2}, {:ok, 4}, {:ok, 6}]
```
## Testing with Relay
Calls to `Relay.await/2` will block until the job runs. That will cause tests to hang when
testing in `:manual` mode until jobs are drained. Unit tests that wrap `Relay` processing can
switch to `:inline` mode so that jobs run immediately.
For example, assuming `Oban` is configured to run in `:manual` testing mode:
```elixir
defmodule MyWorker do
use Oban.Pro.Worker
alias Oban.Pro.Relay
@impl Oban.Pro.Worker
def process(%{args: %{"sub" => sub}}) do
results =
sub
|> Enum.map(&MyOtherWorker.new(%{id: &1})
|> Enum.map(&Relay.async/1)
|> Relay.await_many()
{:ok, results}
end
end
test "running multiple sub workers from perform_job/2" do
Oban.Testing.with_testing_mode(:inline, fn ->
assert {:ok, _} = perform_job(MyWorker, %{subs: [1, 2, 3]})
end)
end
```
## Usage with Chunks
`Relay` is intended for use with a single job and isn't suited to awaiting results from
`Oban.Pro.Chunk` jobs. That's because only one of the chunk's jobs (the leader) will
relay results back. Awaiting any other job in the chunk will time out without returning a proper
result.
"""
@behaviour Oban.Pro.Handler
alias Ecto.{Changeset, UUID}
alias Oban.{Job, Notifier}
alias Oban.Pro.{Unique, Utils}
require Logger
@type t :: %__MODULE__{job: Job.t(), pid: pid(), ref: UUID.t()}
@type await_result ::
:ok
| {:ok, term()}
| {:cancel, term()}
| {:discard, term()}
| {:error, :result_too_large | :timeout | Exception.t()}
| {:snooze, integer()}
defstruct [:job, :name, :pid, :ref]
@postgres_max_bytes 8000
@doc """
Insert a job for asynchronous execution.
The returned map contains the caller's pid and a unique ref that is used to await the results.
## Examples
The single arity version takes a job changeset and inserts it:
relay =
%{id: 123}
|> MyApp.Worker.new()
|> Oban.Pro.Relay.async()
When the Oban instance has a custom name, or an app has multiple Oban
instances, you can use the two arity version to select an instance:
changeset = MyApp.Worker.new(%{id: 123})
Oban.Pro.Relay.async(MyOban, changeset)
"""
@spec async(Oban.name(), Job.changeset()) :: t() | {:error, Job.changeset()}
def async(name \\ Oban, %Changeset{} = changeset) do
pid = self()
ref = if Unique.unique?(changeset), do: Unique.gen_key(changeset), else: UUID.generate()
meta =
changeset
|> Changeset.get_change(:meta, %{})
|> Map.put_new(:await_ref, ref)
changeset = Changeset.put_change(changeset, :meta, meta)
:ok = Notifier.listen(name, :relay)
with {:ok, job} <- Oban.insert(name, changeset) do
%__MODULE__{job: job, name: name, pid: pid, ref: ref}
end
end
@doc """
Await a relay's execution and return the result.
Any successful result such as `:ok`, `{:ok, value}`, or `{:cancel, reason}` is passed back as
the await result. When the executed job returns an `{:error, reason}` tuple, raises an
exception, or crashes, the result comes back as an error tuple with the exception.
By default, `await/1` will timeout after 5 seconds and return `{:error, :timeout}`. The job
itself may continue to run, only the local process stops waiting on it.
> #### Result Size Limits {: .warning}
>
> When using the default `Oban.Notifiers.Postgres` notifier for PubSub, any value larger than 8kb
> (compressed) can't be broadcast due to a Postgres `NOTIFY` limitation. Instead, awaiting will
> return an `{:error, :result_too_large}` tuple. The `Oban.Notifiers.PG` notifier doesn't have any
> such size limitation, but it requires Distributed Erlang.
## Options
* `:timeout` - the maximum time in milliseconds to wait for the result. Defaults to `5_000`.
* `:with_retries` - when `true`, wait through all retry attempts until the job reaches a final
state (`completed`, `cancelled`, `discarded`), or a timeout is reached. Defaults to `false`.
## Examples
Await a job:
{:ok, result} =
%{id: 123}
|> MyApp.Worker.new()
|> Oban.Pro.Relay.async()
|> Oban.Pro.Relay.await()
Increase the wait time with a timeout value:
%{id: 456}
|> MyApp.Worker.new()
|> Oban.Pro.Relay.async()
|> Oban.Pro.Relay.await(timeout: :timer.seconds(30))
Wait through all retry attempts:
%{id: 789}
|> MyApp.Worker.new()
|> Oban.Pro.Relay.async()
|> Oban.Pro.Relay.await(with_retries: true, timeout: :timer.minutes(1))
"""
@spec await(t(), timeout() | keyword()) :: await_result()
def await(relay, opts \\ [])
def await(relay, timeout) when timeout == :infinity or is_integer(timeout) do
await(relay, timeout: timeout)
end
def await(%{name: name, pid: pid, ref: ref}, opts) when is_list(opts) do
check_ownership!(pid)
timeout = Keyword.get(opts, :timeout, 5_000)
retries = Keyword.get(opts, :with_retries, false)
try do
await(ref, timeout, retries)
after
Notifier.unlisten(name, :relay)
end
end
defp await(ref, timeout, with_retries) do
receive do
{:notification, :relay, %{"ref" => ^ref, "result" => result} = payload} ->
# For backward compati. Existing nodes won't return these values in the payload
state = Map.get(payload, "state", "success")
attempt = Map.get(payload, "attempt", 1)
max_attempts = Map.get(payload, "max_attempts", 20)
decoded = Utils.decode64(result)
if with_retries and state == "failure" and attempt < max_attempts do
await(ref, timeout, with_retries)
else
decoded
end
after
timeout -> {:error, :timeout}
end
end
@doc """
Await replies from multiple relays and return the results.
It returns a list of the results in the same order as the relays supplied as the first argument.
Unlike `Task.await_many` or `Task.yield_many`, `await_many/2` may return partial results when
the timeout is reached. When a job hasn't finished executing the value will be `{:error,
:timeout}`
## Examples
Await multiple jobs without any errors or timeouts:
relayed =
1..3
|> Enum.map(&DoubleWorker.new(%{int: &1}))
|> Enum.map(&Oban.Pro.Relay.async(&1))
|> Oban.Pro.Relay.await_many()
#=> [{:ok, 2}, {:ok, 4}, {:ok, 6}]
Await multiple jobs with an error timeout:
relayed =
[1, 2, 300_000_000]
|> Enum.map(&SlowWorker.new(%{int: &1}))
|> Enum.map(&Oban.Pro.Relay.async(&1))
|> Oban.Pro.Relay.await_many(100)
#=> [{:ok, 2}, {:ok, 4}, {:error, :timeout}]
"""
@spec await_many([t()], timeout()) :: [await_result()]
def await_many([%__MODULE__{} | _] = relays, timeout \\ 5_000) do
awaited =
for %{pid: pid, ref: ref} <- relays, into: %{} do
check_ownership!(pid)
{ref, {:error, :timeout}}
end
error_ref = make_ref()
timer_ref = maybe_send_after(error_ref, timeout)
try do
await_many(relays, awaited, %{}, error_ref)
after
if is_reference(timer_ref), do: Process.cancel_timer(timer_ref)
receive do: (^error_ref -> :ok), after: (0 -> :ok)
end
after
relays
|> Enum.map(& &1.name)
|> Enum.uniq()
|> Enum.each(&Notifier.unlisten(&1, :relay))
end
defp await_many(relays, awaited, replied, _error_ref) when map_size(awaited) == 0 do
for %{ref: ref} <- relays, do: Map.fetch!(replied, ref)
end
defp await_many(relays, awaited, replied, error_ref) do
receive do
^error_ref ->
for %{ref: ref} <- relays, do: replied[ref] || awaited[ref]
{:notification, :relay, %{"ref" => ref, "result" => res}} ->
if Map.has_key?(awaited, ref) do
await_many(
relays,
Map.delete(awaited, ref),
Map.put(replied, ref, Utils.decode64(res)),
error_ref
)
else
await_many(relays, awaited, replied, error_ref)
end
end
end
@doc false
def handle_event([:oban, :job, _], _, %{conf: conf, job: job, state: state} = meta, _) do
with %{"await_ref" => aref} <- job.meta,
oban_pid when is_pid(oban_pid) <- Oban.whereis(conf.name) do
payload = %{
"ref" => aref,
"result" => extract_result(conf, meta),
"state" => state,
"attempt" => job.attempt,
"max_attempts" => job.max_attempts
}
Notifier.notify(conf, :relay, payload)
end
catch
kind, value ->
Logger.error(fn ->
"[Oban.Pro.Relay] handler error: " <> Exception.format(kind, value)
end)
:ok
end
@doc false
@impl Oban.Pro.Handler
def on_start do
events = [
[:oban, :job, :stop],
[:oban, :job, :exception]
]
:telemetry.attach_many("oban.pro.relay", events, &__MODULE__.handle_event/4, nil)
end
@doc false
@impl Oban.Pro.Handler
def on_stop do
:telemetry.detach("oban.pro.relay")
end
# Messaging
defp check_ownership!(pid) do
if pid != self() do
raise ArgumentError,
"relay must be awaited by #{inspect(pid)}, was awaited by #{inspect(self())}"
end
end
defp maybe_send_after(_error_ref, :infinity), do: nil
defp maybe_send_after(error_ref, timeout) do
Process.send_after(self(), error_ref, timeout)
end
# Result Handling
defp extract_result(conf, %{state: state, result: result})
when state in [:cancelled, :snoozed, :success] do
encoded = Utils.encode64(result)
if match?({Oban.Notifiers.Postgres, _}, conf.notifier) and
byte_size(encoded) > @postgres_max_bytes do
Utils.encode64({:error, :result_too_large})
else
encoded
end
end
defp extract_result(_conf, %{state: :discard, job: job}) do
Utils.encode64({:discard, job.unsaved_error.reason})
end
defp extract_result(_conf, %{state: :failure, error: error}) do
Utils.encode64({:error, error})
end
end

36
vendor/oban_pro/lib/oban/pro/stage.ex vendored Normal file
View file

@ -0,0 +1,36 @@
defmodule Oban.Pro.Stage do
@moduledoc false
alias Oban.Job
@type args :: Job.args()
@type changeset :: Job.changeset()
@type conf :: term()
@type job :: Job.t()
@type opts :: Keyword.t()
@type reason :: term()
@type result :: term()
@doc """
Initialize configuration passed to various before callbacks.
"""
@callback init(module(), opts()) :: {:ok, conf()} | {:error, reason()}
@doc """
Pre-process the args and/or opts for a job before calling `new/2`.
"""
@callback before_new(args(), opts(), conf()) :: {:ok, args(), opts()} | {:error, reason()}
@doc """
Pre-process a job before calling `process/1`.
"""
@callback before_process(job(), conf()) ::
{:ok, job()} | {:cancel, reason()} | {:error, reason()} | {:snooze, integer()}
@doc """
Post-process a job before returning from the wrapping `perform/1`
"""
@callback after_process(result(), job(), conf()) :: :ok
@optional_callbacks before_new: 3, before_process: 2, after_process: 3
end

240
vendor/oban_pro/lib/oban/pro/stages/chain.ex vendored Normal file
View file

@ -0,0 +1,240 @@
defmodule Oban.Pro.Stages.Chain do
@moduledoc false
@behaviour Oban.Pro.Flusher
@behaviour Oban.Pro.Stage
import Ecto.Query
alias Ecto.Changeset
alias Oban.{Job, Repo, Validation}
alias Oban.Pro.Utils
defstruct [:by, :worker, on_cancelled: :ignore, on_discarded: :ignore]
defmacrop drop_hold(meta) do
quote do
fragment(~s(? || '{"on_hold":false, "uniq_bmp":[]}'), unquote(meta))
end
end
defmacrop continue_state(prefix, meta, scheduled_at, now) do
quote do
fragment(
"""
(CASE
WHEN ? \\? 'orig_scheduled_at' THEN 'scheduled'
WHEN ? > ? THEN 'scheduled'
ELSE 'available'
END)::?.oban_job_state
""",
unquote(meta),
unquote(scheduled_at),
unquote(now),
unquote(prefix)
)
end
end
defmacrop continue_at(meta, scheduled_at, now) do
quote do
fragment(
"""
CASE
WHEN ? \\? 'orig_scheduled_at' THEN timezone('utc', to_timestamp(div((?->'orig_scheduled_at')::bigint, 1000000)::float))
WHEN ? > ? THEN ?
ELSE ?
END
""",
unquote(meta),
unquote(meta),
unquote(scheduled_at),
unquote(now),
unquote(scheduled_at),
unquote(now)
)
end
end
defmacrop same_chain(meta_a, meta_b) do
quote do
fragment("?->>'chain_id' = ?->>'chain_id'", unquote(meta_a), unquote(meta_b))
end
end
@spec chain?(Job.changeset()) :: boolean()
def chain?(changeset), do: match?(%{changes: %{meta: %{"chain_id" => _}}}, changeset)
@spec get_key(Job.changeset()) :: nil | String.t()
def get_key(%{changes: %{meta: %{"chain_id" => chain_id}}}), do: chain_id
def get_key(_changeset), do: nil
@spec continuable?(String.t(), map()) :: boolean()
def continuable?(state, meta) do
state == "completed" or
(state == "cancelled" and meta["on_cancelled"] != "hold") or
(state == "discarded" and meta["on_discarded"] != "hold")
end
@spec to_suspended(Job.changeset()) :: Job.changeset()
def to_suspended(changeset) do
Changeset.put_change(changeset, :state, "suspended")
end
@doc false
def rescue_chains(conf, opts \\ []) do
prev_query =
Job
|> where([j], j.state in ~w(available suspended scheduled retryable executing))
|> where([j], fragment("? \\? 'chain_id'", j.meta))
|> where([j], same_chain(j.meta, parent_as(:jobs).meta))
|> where([j], j.id < parent_as(:jobs).id)
subquery =
from(j in Job, as: :jobs)
|> where([j], fragment("? \\? 'chain_id'", j.meta))
|> where([j], not exists(prev_query))
|> order_by(asc: :id)
|> limit(1)
subquery =
if Utils.has_legacy_suspended?(conf) do
subquery
|> where([j], j.state == "scheduled")
|> where([j], fragment("?->>'on_hold' = 'true'", j.meta))
else
where(subquery, [j], j.state == "suspended")
end
now = DateTime.utc_now()
query =
Job
|> join(:inner, [j], x in subquery(subquery), on: j.id == x.id)
|> update([j],
set: [
meta: drop_hold(j.meta),
state: continue_state(literal(^conf.prefix), j.meta, j.scheduled_at, ^now),
scheduled_at: continue_at(j.meta, j.scheduled_at, ^now)
]
)
Repo.update_all(conf, query, [], opts)
:ok
end
@doc false
def to_chain_id(chain_by, worker, args, opts) do
chain_data =
Enum.map([:queue | chain_by], fn
:queue -> opts |> Keyword.get(:queue, "default") |> to_string()
:worker -> inspect(worker)
[:args, keys] -> take_keys(args, keys)
[:meta, keys] -> opts |> Keyword.get(:meta, %{}) |> take_keys(keys)
end)
Utils.hash64(chain_data)
end
# Stage
@impl Oban.Pro.Stage
def init(worker, opts) do
with :ok <- validate(opts) do
opts =
opts
|> Keyword.put_new(:by, :worker)
|> Keyword.update!(:by, &Utils.normalize_by/1)
|> Keyword.put(:worker, worker)
{:ok, struct!(__MODULE__, opts)}
end
end
@impl Oban.Pro.Stage
def before_new(args, opts, conf) do
conf = merge_conf_opts(conf, opts)
meta = %{
"chain" => true,
"chain_id" => to_chain_id(conf.by, conf.worker, args, opts),
"on_cancelled" => to_string(conf.on_cancelled),
"on_discarded" => to_string(conf.on_discarded)
}
opts = Keyword.update(opts, :meta, meta, &Map.merge(&1, meta))
{:ok, args, opts}
end
# Flushing
@impl Oban.Pro.Flusher
def to_flush_mfa(job, conf) do
case job do
%{meta: %{"chain_id" => _}} -> {__MODULE__, :on_flush, [job, conf]}
_ -> :ignore
end
end
def on_flush(%{state: state, meta: %{"chain_id" => chain_id} = meta}, conf) do
if continuable?(state, meta) do
subquery =
Job
|> select([j], j.id)
|> where([j], fragment("?->>'chain_id'", j.meta) == ^chain_id)
|> order_by(asc: :id)
|> limit(1)
|> lock("FOR UPDATE")
subquery =
if Utils.has_legacy_suspended?(conf) do
subquery
|> where([j], j.state == "scheduled")
|> where([j], fragment("?->>'on_hold' = 'true'", j.meta))
else
where(subquery, [j], j.state == "suspended")
end
now = DateTime.utc_now()
query =
Job
|> join(:inner, [j], x in subquery(subquery), on: j.id == x.id)
|> update([j],
set: [
meta: drop_hold(j.meta),
state: continue_state(literal(^conf.prefix), j.meta, j.scheduled_at, ^now),
scheduled_at: continue_at(j.meta, j.scheduled_at, ^now)
]
)
Repo.update_all(conf, query, [])
end
end
# Helpers
defp merge_conf_opts(conf, opts) do
case Keyword.fetch(opts, :chain) do
{:ok, chain} ->
Enum.reduce(chain, conf, fn {key, val}, acc -> Map.replace!(acc, key, val) end)
:error ->
conf
end
end
defp take_keys(map, keys) do
Enum.map(keys, fn key -> inspect(map[key] || map[to_string(key)]) end)
end
defp validate(opts) do
Validation.validate_schema(opts,
by: {:custom, &Oban.Pro.Validation.validate_by/1},
on_cancelled: {:enum, [:hold, :ignore]},
on_discarded: {:enum, [:hold, :ignore]}
)
end
end

131
vendor/oban_pro/lib/oban/pro/stages/deadline.ex vendored Normal file
View file

@ -0,0 +1,131 @@
defmodule Oban.Pro.Stages.Deadline do
@moduledoc false
@behaviour Oban.Pro.Stage
alias Oban.Pro.Stages.Hooks
alias Oban.Pro.Utils
alias Oban.{Registry, Validation}
@time_units ~w(
second
seconds
minute
minutes
hour
hours
day
days
week
weeks
)a
defstruct [:in, force: false]
@impl Oban.Pro.Stage
def init(worker \\ nil, period)
def init(worker, period) when is_integer(period) or is_tuple(period) do
init(worker, in: period)
end
def init(_worker, opts) when is_list(opts) do
with :ok <- validate(opts) do
conf =
opts
|> Keyword.update!(:in, &Utils.cast_period/1)
|> then(&struct(__MODULE__, &1))
{:ok, conf}
end
end
@impl Oban.Pro.Stage
def before_new(args, opts, conf) do
{deadline, opts} = Keyword.pop(opts, :deadline, conf.in)
seconds = Utils.cast_period(deadline)
now = DateTime.utc_now()
offset =
cond do
schedule_in = opts[:schedule_in] -> Utils.cast_period(schedule_in) + seconds
scheduled_at = opts[:scheduled_at] -> DateTime.diff(scheduled_at, now) + seconds
true -> seconds
end
at =
now
|> DateTime.add(offset)
|> DateTime.to_unix()
meta = %{"deadline_at" => at}
opts = Keyword.update(opts, :meta, meta, &Map.merge(&1, meta))
{:ok, args, opts}
end
@impl Oban.Pro.Stage
def before_process(%{meta: %{"deadline_at" => at}} = job, conf) do
now = DateTime.utc_now()
dat = DateTime.from_unix!(at)
if :gt == DateTime.compare(now, dat) do
{:cancel, "deadline at #{inspect(dat)} exired"}
else
if conf.force, do: force_timeout(now, dat, job)
{:ok, job}
end
end
def before_process(job, conf) do
opts = [scheduled_at: job.scheduled_at, meta: job.meta]
{:ok, _args, opts} = before_new(job.args, opts, conf)
meta = Keyword.fetch!(opts, :meta)
before_process(%{job | meta: meta}, conf)
end
defp force_timeout(now, dat, job) do
parent = self()
Task.start(fn ->
ref = Process.monitor(parent)
Process.send_after(self(), :cancel, DateTime.diff(dat, now, :millisecond))
receive do
{:DOWN, ^ref, :process, _pid, _reason} ->
Process.demonitor(ref, [:flush])
:cancel ->
with pid when is_pid(pid) <- Registry.whereis(job.conf.name, {:producer, job.queue}) do
payload = %{"action" => "pkill", "job_id" => job.id}
send(pid, {:notification, :signal, payload})
end
Hooks.execute_on_cancelled(:deadline, job)
end
end)
end
# Validation
defp validate(opts) do
Validation.validate_schema(opts,
in: {:custom, &validate_period/1},
force: :boolean
)
end
defp validate_period(period) when is_integer(period), do: :ok
defp validate_period({value, units}) when is_integer(value) and units in @time_units, do: :ok
defp validate_period(period) do
{:error, "expected deadline to be an integer in seconds or a tuple, got: #{inspect(period)}"}
end
end

83
vendor/oban_pro/lib/oban/pro/stages/encrypted.ex vendored Normal file
View file

@ -0,0 +1,83 @@
defmodule Oban.Pro.Stages.Encrypted do
@moduledoc false
@behaviour Oban.Pro.Stage
defstruct [:key, cipher: :aes_256_ctr, iv_bytes: 16]
@impl Oban.Pro.Stage
def init(_worker \\ nil, opts) do
case Keyword.fetch(opts, :key) do
{:ok, key} when is_binary(key) ->
{:ok, %__MODULE__{key: Base.decode64!(key)}}
{:ok, {_mod, _fun, _arg} = key} ->
{:ok, %__MODULE__{key: key}}
_ ->
{:error, "expected :key to be a binary or mfa, got: #{inspect(opts)}"}
end
end
@impl Oban.Pro.Stage
def before_new(args, opts, conf) do
iv = :crypto.strong_rand_bytes(conf.iv_bytes)
meta = %{"encrypted" => true, "iv" => Base.encode64(iv)}
opts =
opts
|> Keyword.put_new(:meta, %{})
|> Keyword.update!(:meta, &Map.merge(&1, meta))
{:ok, %{"data" => encode(args, iv, conf)}, opts}
end
@impl Oban.Pro.Stage
def before_process(%{args: args, meta: meta} = job, conf) do
with %{"encrypted" => true, "iv" => iv} <- meta,
%{"data" => data} <- args do
{:ok, %{job | args: decode(data, iv, conf)}}
else
_ ->
{:ok, job}
end
end
# Helpers
defp resolve_key({mod, fun, arg}) do
mod
|> apply(fun, arg)
|> Base.decode64!()
end
defp resolve_key(key), do: key
defp encode(data, iv, conf) do
key = resolve_key(conf.key)
json = Oban.JSON.encode!(data)
conf.cipher
|> :crypto.crypto_one_time(key, iv, json, true)
|> Base.encode64()
rescue
error -> reraise error, prune_args_from_stacktrace(__STACKTRACE__)
end
defp decode(data, iv, conf) do
key = resolve_key(conf.key)
conf.cipher
|> :crypto.crypto_one_time(key, Base.decode64!(iv), Base.decode64!(data), false)
|> Oban.JSON.decode!()
rescue
error -> reraise error, prune_args_from_stacktrace(__STACKTRACE__)
end
defp prune_args_from_stacktrace([{mod, fun, [_ | _] = args, info} | rest]) do
[{mod, fun, length(args), info} | rest]
end
defp prune_args_from_stacktrace(stacktrace), do: stacktrace
end

17
vendor/oban_pro/lib/oban/pro/stages/executing.ex vendored Normal file
View file

@ -0,0 +1,17 @@
defmodule Oban.Pro.Stages.Executing do
@moduledoc false
@behaviour Oban.Pro.Stage
@impl Oban.Pro.Stage
def init(_worker, _opts), do: {:ok, :ignore}
@impl Oban.Pro.Stage
def before_process(%{conf: conf} = job, _conf) when is_struct(conf) do
Registry.register(Oban.Registry, {conf.name, {:executing, job.id}}, nil)
{:ok, job}
end
def before_process(job, _conf), do: {:ok, job}
end

205
vendor/oban_pro/lib/oban/pro/stages/hooks.ex vendored Normal file
View file

@ -0,0 +1,205 @@
defmodule Oban.Pro.Stages.Hooks do
@moduledoc false
@behaviour Oban.Pro.Stage
import Ecto.Query, only: [where: 3]
alias Oban.{Job, Repo}
alias Oban.Pro.Utils
require Logger
@hooks [
after_cancelled: 2,
before_new: 2,
before_process: 1,
after_process: 3,
on_cancelled: 2,
on_discarded: 2
]
defstruct modules: []
@impl Oban.Pro.Stage
def init(worker, modules) do
{:ok, struct!(__MODULE__, modules: [worker | modules])}
end
@impl Oban.Pro.Stage
def before_new(args, opts, %{modules: modules}) do
(modules ++ get_hooks())
|> Enum.filter(&hook_exported?(&1, :before_new))
|> Enum.reduce_while({:ok, args, opts}, fn module, {:ok, args, opts} ->
case module.before_new(args, opts) do
{:ok, _, _} = ok -> {:cont, ok}
other -> {:halt, other}
end
end)
end
@impl Oban.Pro.Stage
def before_process(job, %{modules: modules}) do
(modules ++ get_hooks())
|> Enum.filter(&hook_exported?(&1, :before_process))
|> Enum.reduce_while({:ok, job}, fn module, {:ok, job} ->
case module.before_process(job) do
{:ok, _} = ok -> {:cont, ok}
other -> {:halt, other}
end
end)
end
def attach_hook(module) do
with :ok <- Utils.validate_opts([module], &validate_module/1) do
modules =
get_hooks()
|> Enum.concat([module])
|> Enum.uniq()
:persistent_term.put(:oban_pro_hooks, modules)
end
end
def detach_hook(module) do
case get_hooks() do
[_ | _] = modules ->
:persistent_term.put(:oban_pro_hooks, modules -- [module])
_ ->
:ok
end
end
# The `Workflow.after_cancelled` hook is deprecated and will be removed in a future version.
def execute_after_cancelled(reason, job) do
with {worker, job, opts} <- resolve_processing(job) do
execute_hook(worker, opts, :on_cancelled, [:dependency, job])
execute_hook(worker, opts, :after_cancelled, [reason, job])
end
end
def execute_on_cancelled(reason, job) do
with {worker, job, opts} <- resolve_processing(job) do
execute_hook(worker, opts, :on_cancelled, [reason, job])
end
end
def execute_on_discarded(reason, job) do
with {worker, job, opts} <- resolve_processing(job) do
execute_hook(worker, opts, :on_discarded, [reason, job])
end
end
def handle_event([:oban, :job, _], _timing, meta, _conf) do
with {worker, job, opts} <- resolve_processing(meta.job) do
execute_hook(worker, opts, :after_process, [to_hook_state(meta.state), job, meta[:result]])
end
end
def handle_event([:oban, :engine, :cancel_all_jobs, _], _timing, meta, _conf) do
global? = Enum.any?(get_hooks(), &hook_exported?(&1, :on_cancelled))
Task.start(fn ->
load_and_exec = fn partials ->
ids = Enum.map(partials, & &1.id)
meta.conf
|> Repo.all(where(Job, [j], j.id in ^ids))
|> Enum.each(&execute_on_cancelled(:manual, &1))
end
meta.jobs
|> Stream.filter(&(global? or hook_exported?(&1, :on_cancelled)))
|> Stream.chunk_every(100)
|> Stream.each(load_and_exec)
|> Stream.run()
end)
end
defp execute_hook(worker, opts, hook, args) do
mod_hooks = Keyword.get(opts, :hooks, [])
all_hooks = Enum.uniq([worker | mod_hooks] ++ get_hooks())
for module <- all_hooks, hook_exported?(module, hook) do
apply(module, hook, args)
end
catch
kind, value ->
Logger.error(fn ->
"[Oban.Pro.Worker] hook error: " <> Exception.format(kind, value, __STACKTRACE__)
end)
end
defp resolve_processing(job) do
with nil <- Process.get(:oban_processing),
{:ok, worker} <- Oban.Worker.from_string(job.worker),
opts = worker.__opts__(),
{:ok, job} <- Oban.Pro.Worker.before_hook(worker, job, opts) do
{worker, job, opts}
end
end
defp get_hooks, do: :persistent_term.get(:oban_pro_hooks, [])
defp to_hook_state(:cancelled), do: :cancel
defp to_hook_state(:success), do: :complete
defp to_hook_state(:discard), do: :discard
defp to_hook_state(:failure), do: :error
defp to_hook_state(:snoozed), do: :snooze
defp hook_exported?(worker, fun) when is_binary(worker) do
case Oban.Worker.from_string(worker) do
{:ok, module} -> hook_exported?(module, fun)
_ -> false
end
end
defp hook_exported?(module, fun) when is_atom(module) do
module
|> cached_hooks()
|> MapSet.member?(fun)
end
defp hook_exported?(%{worker: worker}, fun), do: hook_exported?(worker, fun)
defp hook_exported?(_map, _fun), do: false
defp cached_hooks(module) do
case :persistent_term.get({:oban_pro_hook_cache, module}, :not_cached) do
:not_cached ->
module
|> discover_hooks()
|> tap(&:persistent_term.put({:oban_pro_hook_cache, module}, &1))
hooks ->
hooks
end
end
defp discover_hooks(module) do
if Code.ensure_loaded?(module) do
for {fun, arity} <- @hooks, function_exported?(module, fun, arity), into: MapSet.new() do
fun
end
else
MapSet.new()
end
end
# Validation
defp validate_module(module) do
cond do
not Code.ensure_loaded?(module) ->
{:error, "unable to load callback module #{inspect(module)}"}
Enum.any?(@hooks, &function_exported?(module, elem(&1, 0), elem(&1, 1))) ->
:ok
true ->
names = Enum.map_join(@hooks, ", ", fn {fun, arity} -> "#{fun}/#{arity}" end)
{:error, "#{inspect(module)} doesn't implement known hooks: #{names}"}
end
end
end

36
vendor/oban_pro/lib/oban/pro/stages/rate.ex vendored Normal file
View file

@ -0,0 +1,36 @@
defmodule Oban.Pro.Stages.Rate do
@moduledoc false
@behaviour Oban.Pro.Stage
alias Oban.Validation
defstruct [:weight]
@impl Oban.Pro.Stage
def init(_worker, opts) when is_list(opts) do
with :ok <- validate(opts) do
{:ok, struct(__MODULE__, opts)}
end
end
@impl Oban.Pro.Stage
def before_new(args, opts, conf) do
{rate_opts, opts} = Keyword.pop(opts, :rate, [])
weight = Keyword.get(rate_opts, :weight, conf.weight)
if is_integer(weight) do
meta = %{"rate_weight" => weight}
opts = Keyword.update(opts, :meta, meta, &Map.merge(&1, meta))
{:ok, args, opts}
else
{:ok, args, opts}
end
end
defp validate(opts) do
Validation.validate_schema(opts, weight: :pos_integer)
end
end

77
vendor/oban_pro/lib/oban/pro/stages/recorded.ex vendored Normal file
View file

@ -0,0 +1,77 @@
defmodule Oban.Pro.Stages.Recorded do
@moduledoc false
@behaviour Oban.Pro.Stage
alias Oban.Pro.Utils
alias Oban.Validation
defstruct to: :return, limit: 64_000_000, safe_decode: false
@impl Oban.Pro.Stage
def init(_worker, true) do
{:ok, %__MODULE__{}}
end
def init(_worker, opts) do
with :ok <- validate(opts) do
{:ok, struct(__MODULE__, opts)}
end
end
@impl Oban.Pro.Stage
def before_new(args, opts, _conf) do
meta = %{"recorded" => true}
opts = Keyword.update(opts, :meta, meta, &Map.merge(&1, meta))
{:ok, args, opts}
end
@impl Oban.Pro.Stage
def after_process(result, %{meta: %{"recorded" => true}} = job, conf) do
with {:ok, return} <- result do
encoded = Utils.encode64(return)
cond do
job.conf.engine != Oban.Pro.Engines.Smart ->
{:error, "recording requires the Smart engine"}
byte_size(encoded) > conf.limit ->
{:error, "return is #{byte_size(encoded)} bytes, larger than the limit: #{conf.limit}"}
true ->
Process.put(:oban_meta_update, %{to_string(conf.to) => encoded})
:ok
end
end
end
def after_process(_result, _job, _conf), do: :ok
@spec fetch_recorded(Oban.Job.t(), conf :: map()) :: {:ok, term()} | {:error, term()}
def fetch_recorded(job, conf) do
to = to_string(conf.to)
case job.meta do
%{"recorded" => true, ^to => value} ->
{:ok, Utils.decode64(value, decode_opts(job.meta))}
_ ->
{:error, :missing}
end
end
# Helpers
defp validate(opts) do
Validation.validate_schema(opts,
to: {:or, [:atom, :string]},
limit: :pos_integer,
safe_decode: :boolean
)
end
defp decode_opts(%{"safe_decode" => true}), do: [:safe]
defp decode_opts(_meta), do: []
end

193
vendor/oban_pro/lib/oban/pro/stages/structured.ex vendored Normal file
View file

@ -0,0 +1,193 @@
defmodule Oban.Pro.Stages.Structured do
@moduledoc false
@behaviour Oban.Pro.Stage
alias Ecto.Changeset
alias Oban.Pro.Utils
defstruct [:worker]
defmodule Term do
@moduledoc false
use Ecto.Type
alias Oban.Pro.Utils
@impl Ecto.Type
def type, do: :string
@impl Ecto.Type
def cast("term-" <> data), do: {:ok, Utils.decode64(data)}
def cast(term), do: {:ok, "term-" <> Utils.encode64(term)}
@impl Ecto.Type
def dump(_term), do: {:ok, :noop}
@impl Ecto.Type
def load(_data), do: {:ok, :noop}
end
@impl Oban.Pro.Stage
def init(worker, _opts) do
if function_exported?(worker, :__args_schema__, 0) do
{:ok, %__MODULE__{worker: worker}}
else
{:ok, :ignore}
end
end
@impl Oban.Pro.Stage
def before_new(args, opts, :ignore), do: {:ok, args, opts}
def before_new(args, opts, conf) do
{:ok, changes} = gather_changes(conf.worker, args)
meta = %{structured: true}
opts = Keyword.update(opts, :meta, meta, &Map.merge(&1, meta))
{:ok, dump_changes(changes), opts}
catch
message -> {:error, message}
end
@impl Oban.Pro.Stage
def before_process(job, :ignore), do: {:ok, job}
def before_process(%{args: %module{}} = job, %{worker: module}), do: {:ok, job}
def before_process(%{args: args} = job, conf) do
{:ok, changes} = gather_changes(conf.worker, args)
{:ok, %{job | args: changes}}
catch
message -> {:error, message}
end
defp gather_changes(module, args) do
fields = module.__args_schema__()
struct = struct(module, [])
{fields, required, defaults, merge} = split_fields(fields, args)
keys = Map.keys(fields)
args =
if Enum.any?(Map.keys(args), &is_binary/1) do
defaults
|> Map.merge(args)
|> Map.new(fn {key, val} -> {to_string(key), val} end)
else
Map.merge(defaults, args)
end
changeset =
{struct, fields}
|> Changeset.cast(args, keys)
|> Changeset.validate_required(required)
|> validate_allowed(args, keys)
if changeset.valid? do
changeset
|> Changeset.apply_changes()
|> Map.merge(merge)
|> then(&{:ok, &1})
else
changeset
|> Utils.to_translated_errors()
|> Enum.map_join(", ", fn {field, error} -> "#{inspect(field)} #{error}" end)
|> throw()
end
end
defp split_fields(fields, args) do
Enum.reduce(fields, {%{}, [], %{}, %{}}, fn {name, opts}, {keep, required, defaults, merge} ->
required = if opts[:required], do: [name | required], else: required
defaults =
case Keyword.fetch(opts, :default) do
{:ok, default} -> Map.put(defaults, name, default)
:error -> defaults
end
case Map.new(opts) do
%{cardinality: :one, module: module, required: embed_required?, type: :embed} ->
data = args[name] || args[to_string(name)]
keep = Map.put(keep, name, :any)
if is_nil(data) and not embed_required? do
{keep, required, defaults, merge}
else
{:ok, embed} = gather_changes(module, data || %{})
{keep, required, defaults, Map.put(merge, name, embed)}
end
%{cardinality: :many, module: module, type: :embed} ->
embed_list =
for sub_arg <- args[name] || args[to_string(name)] || [] do
{:ok, embed} = gather_changes(module, sub_arg)
embed
end
{Map.put(keep, name, :any), required, defaults, Map.put(merge, name, embed_list)}
%{type: :enum} ->
enum = Ecto.ParameterizedType.init(Ecto.Enum, values: opts[:values])
{Map.put(keep, name, enum), required, defaults, merge}
%{type: :term} ->
{Map.put(keep, name, Term), required, defaults, merge}
%{type: :uuid} ->
{Map.put(keep, name, :binary_id), required, defaults, merge}
%{type: {:array, :enum}} ->
enum = Ecto.ParameterizedType.init(Ecto.Enum, values: opts[:values])
{Map.put(keep, name, {:array, enum}), required, defaults, merge}
%{type: {:array, :uuid}} ->
{Map.put(keep, name, {:array, :binary_id}), required, defaults, merge}
%{type: type} ->
{Map.put(keep, name, type), required, defaults, merge}
end
end)
end
defp validate_allowed(changeset, args, keys) do
keys = MapSet.new(keys, &to_string/1)
Enum.reduce(args, changeset, fn {key, _val}, changeset ->
if to_string(key) in keys do
changeset
else
Changeset.add_error(changeset, key, "is an unexpected key")
end
end)
end
defp dump_changes(struct) when is_struct(struct, Date), do: struct
defp dump_changes(struct) when is_struct(struct, DateTime), do: struct
defp dump_changes(struct) when is_struct(struct, Decimal), do: struct
defp dump_changes(struct) when is_struct(struct, NaiveDateTime), do: struct
defp dump_changes(struct) when is_struct(struct, Time), do: struct
defp dump_changes(struct) when is_struct(struct) do
for {key, val} <- Map.from_struct(struct), not is_nil(val), into: %{} do
{key, dump_changes(val)}
end
end
defp dump_changes(map) when is_map(map) do
Map.new(map, fn {key, val} -> {key, dump_changes(val)} end)
end
defp dump_changes(list) when is_list(list), do: Enum.map(list, &dump_changes/1)
defp dump_changes(term), do: term
end

1332
vendor/oban_pro/lib/oban/pro/testing.ex vendored Normal file
View file

File diff suppressed because it is too large Load diff

88
vendor/oban_pro/lib/oban/pro/unique.ex vendored Normal file
View file

@ -0,0 +1,88 @@
defmodule Oban.Pro.Unique do
@moduledoc false
alias Ecto.Changeset
alias Oban.Job
alias Oban.Pro.Utils
@states_to_ints %{
"scheduled" => 0,
"available" => 1,
"executing" => 2,
"retryable" => 3,
"completed" => 4,
"cancelled" => 5,
"discarded" => 6,
"suspended" => 7
}
@spec replace?(Job.changeset()) :: boolean()
def replace?(changeset), do: match?(%{changes: %{replace: _}}, changeset)
@spec unique?(Job.changeset()) :: boolean()
def unique?(changeset), do: match?(%{changes: %{unique: %{}}}, changeset)
@spec get_key(Job.changeset(), any()) :: any()
def get_key(changeset, default \\ nil)
def get_key(%{changes: %{meta: %{uniq_key: uniq_key}}}, _default), do: uniq_key
def get_key(_changeset, default), do: default
@spec get_states(Job.changeset()) :: [atom()]
def get_states(%{changes: %{unique: %{states: states}}}), do: states
# For backward compatibility, `unique` works with non-pro workers, so it can't be injected into
# the meta through a stage.
@spec with_uniq_meta(Job.changeset()) :: Job.changeset()
def with_uniq_meta(%{changes: %{unique: %{period: _}}} = changeset) do
uniq_meta = %{uniq: true, uniq_bmp: gen_bmp(changeset), uniq_key: gen_key(changeset)}
meta =
changeset
|> Changeset.get_change(:meta, %{})
|> Map.merge(uniq_meta)
Changeset.put_change(changeset, :meta, meta)
end
def with_uniq_meta(changeset), do: changeset
@spec gen_bmp(Job.changeset()) :: [integer()]
def gen_bmp(%{changes: %{unique: %{states: states}}}) do
for state <- states, do: Map.fetch!(@states_to_ints, to_string(state))
end
@spec gen_key(Job.changeset()) :: String.t()
def gen_key(%{changes: %{unique: unique}} = changeset) do
%{fields: fields, keys: keys, period: period, timestamp: timestamp} = unique
data =
fields
|> Enum.sort()
|> Enum.map(fn
:args -> Utils.take_keys(changeset, :args, keys)
:meta -> Utils.take_keys(changeset, :meta, keys)
field -> Changeset.get_field(changeset, field)
end)
data =
if period == :infinity do
data
else
[truncate(period, Changeset.get_field(changeset, timestamp)) | data]
end
Utils.hash64(data)
end
defp truncate(period, timestamp) do
now = timestamp || DateTime.utc_now()
now
|> DateTime.to_unix(:second)
|> rem(period)
|> then(&DateTime.add(now, -&1))
|> DateTime.truncate(:second)
|> DateTime.to_iso8601()
end
end

183
vendor/oban_pro/lib/oban/pro/utils.ex vendored Normal file
View file

@ -0,0 +1,183 @@
defmodule Oban.Pro.Utils do
@moduledoc false
alias Ecto.Changeset
@spec encode64(term(), [:compressed | :deterministic]) :: String.t()
def encode64(term, opts \\ [:compressed]) do
term
|> :erlang.term_to_binary(opts)
|> Base.encode64(padding: false)
end
@spec decode64(binary(), [:safe | :used]) :: term()
def decode64(bin, opts \\ [:safe]) do
bin
|> Base.decode64!(padding: false)
|> :erlang.binary_to_term(opts)
end
@spec hash64(iodata()) :: String.t()
def hash64(iodata) when is_binary(iodata) or is_list(iodata) do
:blake2s
|> :crypto.hash(iodata)
|> Base.encode64(padding: false)
end
@spec persistent_cache(tuple(), fun()) :: any()
def persistent_cache(key, fun) when is_function(fun, 0) do
case :persistent_term.get(key, nil) do
nil -> tap(fun.(), &:persistent_term.put(key, &1))
val -> val
end
end
@doc """
Check if any legacy suspended jobs exist (scheduled + on_hold pattern).
Result is cached in persistent_term per prefix and only checked once per application lifetime.
"""
@spec has_legacy_suspended?(Oban.Config.t()) :: boolean()
def has_legacy_suspended?(conf) do
import Ecto.Query
persistent_cache({__MODULE__, :legacy_suspended, conf.prefix}, fn ->
query =
from(job in Oban.Job,
where: fragment("? \\? 'workflow_id'", job.meta),
where: fragment("? ->> 'workflow_id' IS NOT NULL", job.meta),
where: job.state == "scheduled",
where: fragment("? ->> 'on_hold' = 'true'", job.meta),
select: true,
limit: 1
)
Oban.Repo.exists?(conf, query)
end)
end
@spec validate_opts(list(), fun()) :: :ok | {:error, term()}
def validate_opts(opts, validator) do
Enum.reduce_while(opts, :ok, fn opt, acc ->
case validator.(opt) do
{:error, _reason} = error -> {:halt, error}
_ -> {:cont, acc}
end
end)
end
@spec cast_period(pos_integer() | {atom(), pos_integer()}) :: pos_integer()
def cast_period({value, unit}) do
unit = to_string(unit)
cond do
unit in ~w(second seconds) -> value
unit in ~w(minute minutes) -> value * 60
unit in ~w(hour hours) -> value * 60 * 60
unit in ~w(day days) -> value * 24 * 60 * 60
true -> unit
end
end
def cast_period(period), do: period
@spec enforce_keys(Changeset.t(), map(), module()) :: Changeset.t()
def enforce_keys(changeset, params, schema) do
fields =
[:fields, :virtual_fields]
|> Enum.flat_map(&schema.__schema__/1)
|> Enum.map(&to_string/1)
Enum.reduce(params, changeset, fn {key, _val}, acc ->
if to_string(key) in fields do
acc
else
Changeset.add_error(acc, :base, "unknown field #{key} provided")
end
end)
end
@spec take_keys(Changeset.t(), atom(), list()) :: [any()]
def take_keys(changeset, field, keys) do
changeset
|> Changeset.get_field(field)
|> take_keys(keys)
end
@spec take_keys(map(), [atom()]) :: [any()]
def take_keys(map, _keys) when map_size(map) == 0, do: []
def take_keys(map, []) when is_map(map), do: take_keys(map, Map.keys(map))
def take_keys(map, keys) when is_map(map) and is_list(keys) do
map = Map.new(map, fn {key, val} -> {to_string(key), val} end)
keys
|> Enum.map(&to_string/1)
|> Enum.sort()
|> Enum.map(fn key ->
val = map |> Map.get(key) |> to_hash_val()
[key, val]
end)
end
if Application.compile_env(:oban_pro, [__MODULE__, :safe_hash], false) do
defp to_hash_val(val) when is_binary(val), do: val
defp to_hash_val(val) when is_number(val), do: to_string(val)
defp to_hash_val(val) when is_atom(val), do: to_string(val)
defp to_hash_val(val), do: inspect(val)
else
defp to_hash_val(val), do: val |> :erlang.phash2() |> Integer.to_string()
end
def normalize_by(by) do
by
|> List.wrap()
|> Enum.map(fn
{key, val} -> [key, val |> List.wrap() |> Enum.sort()]
field -> field
end)
end
@spec to_exception(Changeset.t()) :: Exception.t()
def to_exception(changeset) do
changeset
|> to_translated_errors()
|> Enum.reverse()
|> Enum.map(fn {field, message} -> ArgumentError.exception("#{field} #{message}") end)
|> List.first()
end
@spec to_translated_errors(Changeset.t()) :: Keyword.t()
def to_translated_errors(changeset) do
changeset
|> Changeset.traverse_errors(&translate_errors/1)
|> Enum.map(&extract_errors/1)
|> List.flatten()
end
## Conversions
@spec maybe_to_atom(atom() | String.t()) :: atom()
def maybe_to_atom(key) when is_binary(key), do: String.to_existing_atom(key)
def maybe_to_atom(key), do: key
# Helpers
defp translate_errors({msg, opt}) do
Regex.replace(~r"%{(\w+)}", msg, fn _, key ->
opt
|> Keyword.get(String.to_existing_atom(key), key)
|> to_string()
end)
end
defp extract_errors({_key, val}) when is_map(val) do
val
|> Map.to_list()
|> Enum.map(&extract_errors/1)
end
defp extract_errors({key, [message | _]}), do: {key, message}
end

42
vendor/oban_pro/lib/oban/pro/uuidv7.ex vendored Normal file
View file

@ -0,0 +1,42 @@
defmodule Oban.Pro.UUIDv7 do
@moduledoc false
# Derived from [bitwalker/uniq](https://github.com/bitwalker/uniq)
defmacrop biguint(n), do: quote(do: big - unsigned - integer - size(unquote(n)))
@spec generate() :: <<_::288>>
def generate do
time = System.system_time(:millisecond)
<<rand_a::12, _::6, rand_b::62>> = :crypto.strong_rand_bytes(10)
<<a1::4, a2::4, a3::4, a4::4, a5::4, a6::4, a7::4, a8::4, b1::4, b2::4, b3::4, b4::4, c1::4,
c2::4, c3::4, c4::4, d1::4, d2::4, d3::4, d4::4, e1::4, e2::4, e3::4, e4::4, e5::4, e6::4,
e7::4, e8::4, e9::4, e10::4, e11::4, e12::4>> =
<<time::biguint(48), 7::4, rand_a::12, <<2::2>>, rand_b::62>>
<<e(a1), e(a2), e(a3), e(a4), e(a5), e(a6), e(a7), e(a8), ?-, e(b1), e(b2), e(b3), e(b4), ?-,
e(c1), e(c2), e(c3), e(c4), ?-, e(d1), e(d2), e(d3), e(d4), ?-, e(e1), e(e2), e(e3), e(e4),
e(e5), e(e6), e(e7), e(e8), e(e9), e(e10), e(e11), e(e12)>>
end
@compile {:inline, e: 1}
defp e(0), do: ?0
defp e(1), do: ?1
defp e(2), do: ?2
defp e(3), do: ?3
defp e(4), do: ?4
defp e(5), do: ?5
defp e(6), do: ?6
defp e(7), do: ?7
defp e(8), do: ?8
defp e(9), do: ?9
defp e(10), do: ?a
defp e(11), do: ?b
defp e(12), do: ?c
defp e(13), do: ?d
defp e(14), do: ?e
defp e(15), do: ?f
end

22
vendor/oban_pro/lib/oban/pro/validation.ex vendored Normal file
View file

@ -0,0 +1,22 @@
defmodule Oban.Pro.Validation do
@moduledoc false
# Shared validations for use with `Oban.Validation.validate/2,3`
@doc """
Validate `by:` options for partitioning workers.
"""
def validate_by(:worker), do: :ok
def validate_by([{:args, key}]) when is_atom(key), do: :ok
def validate_by([{:meta, key}]) when is_atom(key), do: :ok
def validate_by([{:args, [key | _]}]) when is_atom(key), do: :ok
def validate_by([{:meta, [key | _]}]) when is_atom(key), do: :ok
def validate_by([:worker, {:args, key}]) when is_atom(key), do: :ok
def validate_by([:worker, {:meta, key}]) when is_atom(key), do: :ok
def validate_by([:worker, {:args, [key | _]}]) when is_atom(key), do: :ok
def validate_by([:worker, {:meta, [key | _]}]) when is_atom(key), do: :ok
def validate_by(fields) do
{:error, "expected :by to be :worker or an :args/:meta tuple, got: #{inspect(fields)}"}
end
end

1712
vendor/oban_pro/lib/oban/pro/worker.ex vendored Normal file
View file

File diff suppressed because it is too large Load diff

91
vendor/oban_pro/lib/oban/pro/workers/batch.ex vendored Normal file
View file

@ -0,0 +1,91 @@
defmodule Oban.Pro.Workers.Batch do
@moduledoc false
alias Oban.{Job, Worker}
@callback new_batch([Job.t() | Job.args()], [keyword()]) :: Oban.Pro.Batch.t()
@callback gen_id() :: String.t()
@callback handle_attempted(Job.t()) :: Worker.result()
@callback handle_cancelled(Job.t()) :: Worker.result()
@callback handle_completed(Job.t()) :: Worker.result()
@callback handle_discarded(Job.t()) :: Worker.result()
@callback handle_exhausted(Job.t()) :: Worker.result()
@callback handle_retryable(Job.t()) :: Worker.result()
@callback all_batch_jobs(Job.t(), [keyword()]) :: Enum.t()
@callback stream_batch_jobs(Job.t(), [keyword()]) :: Enum.t()
@optional_callbacks handle_attempted: 1,
handle_cancelled: 1,
handle_completed: 1,
handle_discarded: 1,
handle_exhausted: 1,
handle_retryable: 1
@doc false
defmacro __using__(opts) do
quote do
use Oban.Pro.Worker, unquote(opts)
alias Oban.Pro.Workers.Batch
@behaviour Oban.Pro.Workers.Batch
@batch_opts ~w(
batch_id
batch_name
batch_callback_args
batch_callback_meta
batch_callback_queue
batch_callback_worker
callback_opts
callback_worker
)a
@impl Batch
def new_batch([_ | _] = args_or_changesets, opts \\ []) when is_list(opts) do
{batch_opts, job_opts} = Keyword.split(opts, @batch_opts)
batch_opts =
Enum.reduce(batch_opts, [callback_opts: []], fn {key, val}, acc ->
case key do
:batch_callback_args -> put_in(acc, [:callback_opts, :args], val)
:batch_callback_meta -> put_in(acc, [:callback_opts, :meta], val)
:batch_callback_queue -> put_in(acc, [:callback_opts, :queue], val)
:batch_callback_worker -> Keyword.put(acc, :callback_worker, val)
_ -> Keyword.put(acc, key, val)
end
end)
case args_or_changesets do
[changeset | _] = changesets when is_struct(changeset) ->
Oban.Pro.Batch.new(changesets, batch_opts)
args ->
args
|> Enum.map(&new(&1, job_opts))
|> Oban.Pro.Batch.new(batch_opts)
end
end
@impl Batch
def gen_id, do: Oban.Pro.UUIDv7.generate()
@impl Batch
def all_batch_jobs(job, opts \\ []), do: Oban.Pro.Batch.all_jobs(Oban, job, opts)
@impl Batch
def stream_batch_jobs(job, opts \\ []), do: Oban.Pro.Batch.stream_jobs(Oban, job, opts)
defoverridable Oban.Pro.Workers.Batch
end
end
end

163
vendor/oban_pro/lib/oban/pro/workers/chain.ex vendored Normal file
View file

@ -0,0 +1,163 @@
defmodule Oban.Pro.Workers.Chain do
@moduledoc false
import Ecto.Query
alias Oban.{Job, Repo, Validation}
alias Oban.Pro.Utils
@default_meta %{
on_discarded: "ignore",
on_cancelled: "ignore",
chain_key: nil,
hold_snooze: 60,
wait_retry: 10,
wait_sleep: 1000,
wait_snooze: 5
}
@opts_keys ~w(by hold_snooze on_cancelled on_discarded wait_retry wait_sleep wait_snooze)a
@doc false
defmacro __using__(opts) do
{chain_opts, other_opts} = Keyword.split(opts, @opts_keys)
quote do
Validation.validate!(unquote(chain_opts), &Oban.Pro.Workers.Chain.validate/1)
use Oban.Pro.Worker, unquote(other_opts)
@base_meta unquote(chain_opts)
|> Keyword.delete(:by)
|> Map.new()
@impl Oban.Worker
def new(args, opts) when is_map(args) and is_list(opts) do
meta =
opts
|> Keyword.get(:meta, %{})
|> Map.merge(@base_meta)
|> Map.put_new(:chain_by, Keyword.fetch!(unquote(chain_opts), :by))
|> Map.update!(:chain_by, &Utils.normalize_by/1)
chain_key = Oban.Pro.Workers.Chain.to_chain_key(__MODULE__, args, meta)
chain_id = Oban.Pro.Stages.Chain.to_chain_id(meta.chain_by, __MODULE__, args, opts)
# Include the `chain_id` for compatibility with chains in v1.5
meta =
meta
|> Map.put(:chain_key, chain_key)
|> Map.put(:chain_id, chain_id)
super(args, Keyword.put(opts, :meta, meta))
end
@impl Oban.Worker
def perform(%Job{} = job) do
opts = __opts__()
with {:ok, job} <- Oban.Pro.Worker.before_process(__MODULE__, job, opts) do
job
|> Oban.Pro.Workers.Chain.maybe_process(__MODULE__)
|> Oban.Pro.Worker.after_process(__MODULE__, job, opts)
end
end
end
end
@doc false
def validate(opts) do
Validation.validate_schema(opts,
by: {:custom, &Oban.Pro.Validation.validate_by/1},
hold_snooze: :non_neg_integer,
on_cancelled: {:custom, &validate_on_action(:on_cancelled, &1)},
on_discarded: {:custom, &validate_on_action(:on_discarded, &1)},
wait_retry: :non_neg_integer,
wait_sleep: :timeout,
wait_snooze: :non_neg_integer
)
end
defp validate_on_action(_key, :halt), do: :ok
defp validate_on_action(_key, :hold), do: :ok
defp validate_on_action(_key, :ignore), do: :ok
defp validate_on_action(key, action) do
{:error, "expected #{key} to be :halt, :hold, or :ignore, got: #{inspect(action)}"}
end
@doc false
def to_chain_key(worker, args, meta) do
meta
|> Map.fetch!(:chain_by)
|> List.wrap()
|> Enum.reduce(%{}, fn
:worker, acc -> Map.put(acc, :worker, worker)
[:args, keys], acc -> Map.put(acc, :args, take_keys(args, keys))
[:meta, keys], acc -> Map.put(acc, :meta, take_keys(meta, keys))
end)
|> :erlang.phash2()
end
defp take_keys(map, keys) do
keys
|> Enum.sort()
|> Enum.map(fn key -> map[key] || map[to_string(key)] end)
end
@doc false
def maybe_process(job, worker, wait_count \\ 0)
# It's running in :inline testing mode and the job doesn't have an id.
def maybe_process(%Job{id: nil} = job, worker, _wait_count) do
worker.process(job)
end
def maybe_process(%Job{conf: conf, id: id, meta: meta} = job, worker, wait_count) do
meta = meta_with_defaults(meta)
query =
Job
|> select([j], {j.state, j.scheduled_at})
|> where([j], fragment("? @> ?", j.meta, ^%{chain_key: meta.chain_key}))
|> where([j], j.state != "completed")
|> where([j], j.id < ^id)
|> order_by(desc: :id)
|> limit(1)
case Repo.one(conf, query) do
{"executing", _at} ->
if wait_count >= meta.wait_retry do
{:snooze, meta.wait_snooze}
else
Process.sleep(meta.wait_sleep)
maybe_process(job, worker, wait_count + 1)
end
{state, _at} when state in ~w(cancelled discarded) ->
key = if state == "cancelled", do: :on_cancelled, else: :on_discarded
case Map.fetch!(meta, key) do
"halt" -> {:cancel, "chain halted by upstream job"}
"hold" -> {:snooze, meta.hold_snooze}
"ignore" -> worker.process(job)
end
{_state, scheduled_at} ->
seconds =
scheduled_at
|> DateTime.diff(DateTime.utc_now())
|> max(meta.wait_snooze)
{:snooze, seconds}
nil ->
worker.process(job)
end
end
defp meta_with_defaults(meta) do
Map.new(@default_meta, fn {key, val} -> {key, meta[to_string(key)] || val} end)
end
end

9
vendor/oban_pro/lib/oban/pro/workers/chunk.ex vendored Normal file
View file

@ -0,0 +1,9 @@
defmodule Oban.Pro.Workers.Chunk do
@moduledoc false
defmacro __using__(opts) do
quote do
use Oban.Pro.Chunk, unquote(opts)
end
end
end

202
vendor/oban_pro/lib/oban/pro/workers/workflow.ex vendored Normal file
View file

@ -0,0 +1,202 @@
defmodule Oban.Pro.Workers.Workflow do
@moduledoc false
import Ecto.Query, only: [group_by: 3, select: 3, where: 3]
alias Oban.{Job, Repo}
alias Oban.Pro.Workflow
@callback new_workflow(Workflow.new_opts()) :: Workflow.t()
@callback add(Workflow.t(), Workflow.name(), Job.changeset(), Workflow.add_opts()) ::
Workflow.t()
@callback after_cancelled(Workflow.cancel_reason(), Job.t()) :: :ok
@callback all_workflow_jobs(Job.t(), Workflow.fetch_opts()) :: [Job.t()]
@callback append_workflow(Job.t() | [Job.t()], Workflow.append_opts()) :: Workflow.t()
@callback gen_id() :: String.t()
@callback stream_workflow_jobs(Job.t(), Workflow.fetch_opts()) :: Enum.t()
@callback to_dot(Workflow.t()) :: String.t()
@optional_callbacks after_cancelled: 2
defmacro __using__(opts) do
quote do
use Oban.Pro.Worker, unquote(opts)
@behaviour Oban.Pro.Workers.Workflow
@impl Oban.Pro.Workers.Workflow
def new_workflow(opts \\ []) when is_list(opts) do
opts
|> Keyword.put_new(:workflow_id, gen_id())
|> Oban.Pro.Workflow.new()
end
@impl Oban.Pro.Workers.Workflow
def append_workflow(jobs, opts \\ []) do
Oban.Pro.Workflow.append(jobs, opts)
end
@impl Oban.Pro.Workers.Workflow
def add(workflow, name, changeset, opts \\ []) do
Oban.Pro.Workflow.add(workflow, name, changeset, opts)
end
@impl Oban.Pro.Workers.Workflow
def gen_id, do: Oban.Pro.UUIDv7.generate()
@impl Oban.Pro.Workers.Workflow
def to_dot(workflow) do
Oban.Pro.Workflow.to_dot(workflow)
end
@impl Oban.Pro.Workers.Workflow
def stream_workflow_jobs(%Job{} = job, opts \\ []) do
Oban.Pro.Workflow.stream_jobs(Oban, job, opts)
end
@impl Oban.Pro.Workers.Workflow
def all_workflow_jobs(%Job{} = job, opts \\ []) do
Oban.Pro.Workflow.all_jobs(Oban, job, opts)
end
@impl Oban.Worker
def perform(%Job{} = job) do
opts = __opts__()
with {:ok, job} <- Oban.Pro.Worker.before_process(__MODULE__, job, opts) do
job
|> Oban.Pro.Workers.Workflow.maybe_process(__MODULE__)
|> Oban.Pro.Worker.after_process(__MODULE__, job, opts)
end
end
defoverridable Oban.Pro.Workers.Workflow
end
end
@doc false
defdelegate add(workflow, name, changeset, opts \\ []), to: Workflow
@doc false
def all_jobs(job, opts \\ []), do: Workflow.all_jobs(Oban, job, opts)
@doc false
defdelegate append(job, opts \\ []), to: Workflow
@doc false
def gen_id, do: Oban.Pro.UUIDv7.generate()
@doc false
defdelegate new(opts \\ []), to: Workflow
@doc false
def stream_jobs(job, opts \\ []), do: Workflow.stream_jobs(Oban, job, opts)
@doc false
defdelegate to_dot(workflow), to: Workflow
@doc false
def maybe_process(%{meta: meta} = job, module) do
if is_map_key(meta, "workflow_id") and not is_map_key(meta, "on_hold") do
legacy_process(job, module)
else
module.process(job)
end
end
@legacy_meta %{
ignore_cancelled: false,
ignore_deleted: false,
ignore_discarded: false,
waiting_delay: :timer.seconds(1),
waiting_limit: 10,
waiting_snooze: 5
}
# This is necessary for backward compatibility
defp legacy_process(job, module, waiting_count \\ 0) do
meta = for {key, val} <- @legacy_meta, into: %{}, do: {key, job.meta[to_string(key)] || val}
case legacy_check_deps(job) do
:completed ->
module.process(job)
:available ->
{:snooze, meta.waiting_snooze}
:executing ->
if waiting_count >= meta.waiting_limit do
{:snooze, meta.waiting_snooze}
else
Process.sleep(meta.waiting_delay)
legacy_process(job, module, waiting_count + 1)
end
{:scheduled, scheduled_at} ->
seconds =
scheduled_at
|> DateTime.diff(DateTime.utc_now())
|> max(meta.waiting_snooze)
{:snooze, seconds}
:cancelled ->
if meta.ignore_cancelled do
module.process(job)
else
{:cancel, "upstream deps cancelled, workflow will never complete"}
end
:discarded ->
if meta.ignore_discarded do
module.process(job)
else
{:cancel, "upstream deps discarded, workflow will never complete"}
end
:deleted ->
if meta.ignore_deleted do
module.process(job)
else
{:cancel, "upstream deps deleted, workflow will never complete"}
end
end
end
defp legacy_check_deps(%{conf: conf, meta: %{"deps" => [_ | _]}} = job) do
%{"deps" => deps, "workflow_id" => workflow_id} = job.meta
deps_count = length(deps)
query =
Job
|> where([j], fragment("? @> ?", j.meta, ^%{workflow_id: workflow_id}))
|> where([j], fragment("?->>'name'", j.meta) in ^deps)
|> group_by([j], j.state)
|> select([j], {j.state, count(j.id), max(j.scheduled_at)})
conf
|> Repo.all(query)
|> Map.new(fn {state, count, sc_at} -> {state, {count, sc_at}} end)
|> case do
%{"completed" => {^deps_count, _}} -> :completed
%{"scheduled" => {_, scheduled_at}} -> {:scheduled, scheduled_at}
%{"retryable" => {_, scheduled_at}} -> {:scheduled, scheduled_at}
%{"executing" => _} -> :executing
%{"available" => _} -> :available
%{"cancelled" => _} -> :cancelled
%{"discarded" => _} -> :discarded
%{} -> :deleted
end
end
defp legacy_check_deps(_job), do: :completed
end

3125
vendor/oban_pro/lib/oban/pro/workflow.ex vendored Normal file
View file

File diff suppressed because it is too large Load diff

87
vendor/oban_pro/lib/oban/pro/workflow/cascade.ex vendored Normal file
View file

@ -0,0 +1,87 @@
defmodule Oban.Pro.Workflow.Cascade do
@moduledoc false
use Oban.Pro.Worker, recorded: [limit: 256_000]
alias Oban.Pro.Workflow
args_schema do
field :mod, :string
field :fun, :string
field :arg, :term, default: :none
end
@impl Oban.Pro.Worker
def process(%{args: %{arg: arg, mod: mod, fun: fun}, conf: conf, meta: meta}) do
mod =
mod
|> String.split(".")
|> Module.safe_concat()
fun = String.to_existing_atom(fun)
full = Map.merge(all_context(conf, meta), all_recorded(conf, meta))
args = if arg == :none, do: [full], else: [arg, full]
case apply(mod, fun, args) do
{:ok, _} = result -> result
{:error, _} = error -> error
{:snooze, _} = snooze -> snooze
{:cancel, _} = cancel -> cancel
other -> {:ok, other}
end
end
defp all_context(conf, meta) do
sup = Map.get(meta, "sup_workflow_id")
wid = Map.get(meta, "workflow_id")
ancestors = Map.get(meta, "ancestor_ids", [])
(Enum.reverse(ancestors) ++ [sup, wid])
|> Enum.filter(& &1)
|> Enum.uniq()
|> Enum.reduce(%{}, fn id, acc ->
Map.merge(acc, Workflow.get_context(conf, id) || %{})
end)
end
defp all_recorded(_conf, %{"deps" => []}), do: %{}
defp all_recorded(conf, %{"deps" => deps, "workflow_id" => wid} = meta) do
atomize? = Map.get(meta, "atom_keys", true)
grouper = fn
[sup, name], acc -> Map.update(acc, sup, [name], &[name | &1])
name, acc -> Map.update(acc, wid, [name], &[name | &1])
end
deps
|> Enum.reject(&match?("grafter_" <> _, &1))
|> Enum.reduce(%{}, grouper)
|> Enum.reduce(%{}, fn {dep_wid, names}, acc ->
filter = if names == ["*"], do: [], else: [names: names]
filter = if dep_wid != wid, do: Keyword.put(filter, :with_subs, true), else: filter
recorded =
conf
|> Workflow.all_recorded(dep_wid, filter)
|> then(fn map -> if atomize?, do: atomize_map(map), else: map end)
Map.merge(acc, recorded)
end)
end
defp atomize_map(map) when is_map(map) and not is_struct(map) do
Map.new(map, fn {key, val} -> {atomize_key(key), val} end)
end
defp atomize_map(val), do: val
defp atomize_key(key) when is_binary(key) do
String.to_existing_atom(key)
rescue
ArgumentError -> key
end
defp atomize_key(key), do: key
end

117
vendor/oban_pro/lib/oban/pro/workflow/schema.ex vendored Normal file
View file

@ -0,0 +1,117 @@
defmodule Oban.Pro.Workflow.Schema do
@moduledoc false
use Ecto.Schema
import Ecto.Changeset
@fields ~w(id name parent_id suspended available scheduled executing
retryable completed cancelled discarded inserted_at meta)a
@meta_fields ~w(ignore_cancelled ignore_discarded ignore_deleted unique)a
@states ~w(
suspended
available
scheduled
executing
retryable
completed
cancelled
discarded
)a
@counts Map.new(@states, &{&1, 0})
@primary_key {:id, :string, autogenerate: false}
schema "oban_workflows" do
field :name, :string
field :parent_id, :string
field :state, :string, read_after_writes: true
field :suspended, :integer, default: 0
field :available, :integer, default: 0
field :scheduled, :integer, default: 0
field :executing, :integer, default: 0
field :retryable, :integer, default: 0
field :completed, :integer, default: 0
field :cancelled, :integer, default: 0
field :discarded, :integer, default: 0
field :inserted_at, :utc_datetime_usec
field :started_at, :utc_datetime_usec
field :completed_at, :utc_datetime_usec
field :meta, :map, default: %{}
end
@doc false
def new(params) do
params = Map.put_new_lazy(params, :inserted_at, &DateTime.utc_now/0)
%__MODULE__{}
|> cast(params, @fields)
|> validate_required(~w(id inserted_at)a)
|> unique_constraint(:name, name: :oban_workflows_unique_name)
end
@doc false
def from_changesets({workflow_id, changesets}), do: from_changesets(workflow_id, changesets)
@doc false
def from_changesets(workflow_id, changesets) do
[first | _] = changesets
meta = get_field(first, :meta, %{})
{queues, workers} =
Enum.reduce(changesets, {MapSet.new(), MapSet.new()}, fn changeset, {queues, workers} ->
queue = to_string(get_field(changeset, :queue))
worker = get_field(changeset, :worker)
{MapSet.put(queues, queue), MapSet.put(workers, worker)}
end)
counts = count_by_state(changesets)
workflow_meta =
meta
|> Map.take(@meta_fields)
|> Map.put("queues", MapSet.to_list(queues))
|> Map.put("workers", MapSet.to_list(workers))
params =
counts
|> Map.put(:id, workflow_id)
|> Map.put(:name, build_name(meta, changesets))
|> Map.put(:parent_id, meta[:sup_workflow_id])
|> Map.put(:meta, workflow_meta)
new(params)
end
defp count_by_state(changesets) do
Enum.reduce(changesets, @counts, fn changeset, acc ->
state =
changeset
|> get_field(:state, "available")
|> String.to_existing_atom()
Map.update!(acc, state, &(&1 + 1))
end)
end
defp build_name(%{workflow_name: name}, _) when is_binary(name), do: name
defp build_name(_meta, changesets) do
Enum.find_value(changesets, fn changeset ->
meta = get_field(changeset, :meta)
worker = get_field(changeset, :worker)
cond do
name = meta[:decorated_name] -> name
worker != "Oban.Pro.Workers.Context" -> worker
true -> nil
end
end)
end
end

202
vendor/oban_pro/mix.exs vendored Normal file
View file

@ -0,0 +1,202 @@
defmodule Oban.Pro.MixProject do
use Mix.Project
@version "1.7.0"
def project do
[
app: :oban_pro,
version: @version,
elixir: "~> 1.15",
start_permanent: Mix.env() == :prod,
elixirc_paths: elixirc_paths(Mix.env()),
test_ignore_filters: [~r/fixture_app/],
prune_code_paths: false,
deps: deps(),
docs: docs(),
aliases: aliases(),
package: package(),
name: "Oban Pro",
description: "Oban Pro Component",
xref: [exclude: [Oban.JSON, Oban.Validation]],
# Dialyzer
dialyzer: [
plt_add_apps: [:ex_unit, :inets, :mix, :postgrex],
plt_core_path: "_build/#{Mix.env()}",
flags: [:error_handling, :missing_return, :no_opaque, :underspecs]
]
]
end
def application do
[
extra_applications: [:logger],
mod: {Oban.Pro.Application, []}
]
end
def cli do
[
preferred_envs: [
precommit: :test,
"test.ci": :test,
"test.reset": :test,
"test.rollback": :test,
"test.setup": :test
]
]
end
def package do
[
organization: "oban",
files: ~w(lib/oban usage-rules .formatter.exs mix.exs usage-rules.md),
licenses: ["Commercial"],
links: []
]
end
defp elixirc_paths(:test), do: ["lib", "test/support"]
defp elixirc_paths(_env), do: ["lib"]
defp deps do
[
{:oban, "~> 2.21"},
{:ecto_sql, "~> 3.12"},
{:postgrex, "~> 0.22", optional: true},
# Test and Dev
{:stream_data, "~> 1.1", only: [:test, :dev]},
{:tz, "~> 0.28", only: [:test, :dev]},
{:benchee, "~> 1.5", only: [:test, :dev], runtime: false},
{:credo, "~> 1.7.17", only: [:test, :dev], runtime: false},
{:dialyxir, "~> 1.4", only: [:test, :dev], runtime: false},
# Dev
{:ex_doc, "~> 0.40", only: :dev, runtime: false},
{:makeup_diff, "~> 0.1", only: :dev, runtime: false}
]
end
defp aliases do
[
release: [
"cmd git tag v#{@version}",
"cmd git push",
"cmd git push --tags",
"oban_pro.release",
"hex.publish package --yes"
],
"db.reset": ["ecto.drop -r Oban.Pro.Repo --quiet", "db.setup"],
"db.rollback": ["ecto.rollback -r Oban.Pro.Repo --all --quiet"],
"db.setup": [
"ecto.create -r Oban.Pro.Repo --quiet",
"ecto.migrate -r Oban.Pro.Repo --quiet"
],
"test.ci": [
fn _ -> System.put_env("CI", "true") end,
"format --check-formatted",
"deps.unlock --check-unused",
"credo --strict",
"test --raise",
"dialyzer"
],
"test.reset": ["db.reset"],
precommit: [
"format --check-formatted",
"deps.unlock --check-unused",
"credo --strict",
"test --raise"
]
]
end
defp docs do
[
main: "overview",
source_ref: "v#{@version}",
formatters: ["html"],
api_reference: false,
extra_section: "GUIDES",
extras: extras(),
groups_for_extras: groups_for_extras(),
groups_for_modules: groups_for_modules(),
homepage_url: "/",
logo: "assets/oban-pro-logo.svg",
assets: %{"assets" => "assets"},
nest_modules_by_prefix: nest_modules_by_prefix(),
skip_undefined_reference_warnings_on: ["CHANGELOG.md", "guides/upgrading/v1.5.md"],
before_closing_body_tag: fn _ ->
"""
<script>document.querySelector('footer.footer p').remove()</script>
"""
end
]
end
defp extras do
[
"guides/introduction/overview.md",
"guides/introduction/installation.md",
"guides/introduction/adoption.md",
"guides/introduction/composition.md",
"guides/testing/testing.md",
"guides/testing/testing_workers.md",
"guides/deployment/ci_cd.md",
"guides/deployment/docker.md",
"guides/deployment/fly.md",
"guides/deployment/gigalixir.md",
"guides/deployment/heroku.md",
"guides/deployment/dependabot.md",
"guides/upgrading/v1.0.md",
"guides/upgrading/v1.4.md",
"guides/upgrading/v1.5.md",
"guides/upgrading/v1.6.md",
"guides/upgrading/v1.7.md",
"CHANGELOG.md": [filename: "changelog", title: "Changelog"]
]
end
defp groups_for_extras do
[
Introduction: ~r/guides\/introduction\/.?/,
Testing: ~r/guides\/testing\/.?/,
Deployment: ~r/guides\/deployment\/.?/,
Upgrading: ~r/guides\/upgrading\/.?/
]
end
defp groups_for_modules do
[
Workers: [
Oban.Pro.Worker,
Oban.Pro.Decorator
],
Composition: [
Oban.Pro.Batch,
Oban.Pro.Chunk,
Oban.Pro.Workflow
],
Utilities: [
Oban.Pro.Cloud,
Oban.Pro.RateLimit,
Oban.Pro.Relay,
Oban.Pro.Testing
],
Plugins: [
Oban.Pro.Plugins.DynamicCron,
Oban.Pro.Plugins.DynamicLifeline,
Oban.Pro.Plugins.DynamicPartitioner,
Oban.Pro.Plugins.DynamicPrioritizer,
Oban.Pro.Plugins.DynamicPruner,
Oban.Pro.Plugins.DynamicQueues,
Oban.Pro.Plugins.DynamicScaler
]
]
end
defp nest_modules_by_prefix do
[Oban.Pro, Oban.Pro.Plugins]
end
end

9
vendor/oban_pro/usage-rules.md vendored Normal file
View file

@ -0,0 +1,9 @@
# Rules for Using Oban Pro
Oban Pro is a commercial extension to Oban that provides advanced job processing features for
Elixir applications. It adds composition primitives (workflows, batches, chains, chunks), smart
concurrency controls (global limits, rate limiting), and enhanced worker capabilities (structured
args, encryption, deadlines, hooks). Read the reference files before writing Oban Pro code. Do not
assume standard Oban patterns apply. Pro has its own idioms and best practices.
For full documentation: https://oban.pro/docs/pro/overview.html

43
vendor/oban_pro/usage-rules/batches.md vendored Normal file
View file

@ -0,0 +1,43 @@
# Batches
Batches group jobs together and run callbacks after processing.
```elixir
defmodule MyApp.EmailWorker do
use Oban.Pro.Worker
@behaviour Oban.Pro.Batch
@impl Oban.Pro.Worker
def process(%{args: %{"email" => email}}) do
MyApp.Mailer.deliver(email)
end
@impl Oban.Pro.Batch
def batch_completed(job) do
Logger.info("Batch #{job.meta["batch_id"]} completed")
:ok
end
end
```
## Creating batches
```elixir
alias Oban.Pro.Batch
emails
|> Enum.map(&EmailWorker.new(%{email: &1}))
|> Batch.new()
|> Oban.insert_all()
```
## Callbacks
| Callback | Triggered when |
|---------------------|---------------------------------------------|
| `batch_attempted/1` | All jobs executed at least once |
| `batch_completed/1` | All jobs completed successfully |
| `batch_exhausted/1` | All jobs completed, cancelled, or discarded |
| `batch_cancelled/1` | First job is cancelled |
| `batch_discarded/1` | First job is discarded |

34
vendor/oban_pro/usage-rules/chains.md vendored Normal file
View file

@ -0,0 +1,34 @@
# Chains
Chains ensure jobs run sequentially for the same entity. Use when order matters (e.g., webhook processing, account balance updates).
```elixir
defmodule MyApp.WebhookWorker do
use Oban.Pro.Worker,
queue: :webhooks,
chain: [by: [args: :account_id]]
@impl Oban.Pro.Worker
def process(%Oban.Job{args: args}) do
MyApp.Account.handle_webhook(args["account_id"], args["event"])
end
end
```
## Partitioning
```elixir
chain: [by: :worker] # All jobs of this worker chain together
chain: [by: [args: :account_id]] # Chain by account_id across workers
chain: [by: [:worker, args: :account_id]] # Chain by worker AND account_id
chain: [by: [args: [:account_id, :region]]] # Chain by multiple args
```
## Handling failures
```elixir
chain: [by: [args: :id], on_discarded: :hold, on_cancelled: :hold]
```
- `:ignore` (default) — continue processing downstream jobs
- `:hold` — stop the chain until the failed job is resolved

34
vendor/oban_pro/usage-rules/chunks.md vendored Normal file
View file

@ -0,0 +1,34 @@
# Chunks
Chunks process jobs in groups based on size or timeout. Use for bulk operations and batched API calls.
```elixir
defmodule MyApp.BulkIndexer do
use Oban.Pro.Chunk, size: 100, timeout: 5_000
@impl true
def process([_ | _] = jobs) do
jobs
|> Enum.map(& &1.args["document"])
|> MyApp.Search.bulk_index()
:ok
end
end
```
## Options
- `size: 100` — process up to 100 jobs at once (default)
- `timeout: 1000` — process after 1 second even if size not reached (default)
- `by: [args: :tenant_id]` — partition chunks by field
## Return values
```elixir
:ok # All jobs succeeded
{:ok, result} # All jobs succeeded with result
{:error, reason, failed_jobs} # Some jobs failed, others succeed
{:cancel, reason, jobs} # Cancel specific jobs
{:snooze, seconds, jobs} # Snooze specific jobs
```

237
vendor/oban_pro/usage-rules/plugins.md vendored Normal file
View file

@ -0,0 +1,237 @@
# Plugins
## Available Plugins
- `DynamicCron`: Runtime-configurable cron scheduling with cluster-wide coordination
- `DynamicLifeline`: Rescues orphaned jobs, repairs stuck workflows/chains
- `DynamicPrioritizer`: Automatically increases priority of waiting jobs
- `DynamicPruner`: Flexible job retention with per-queue/worker rules
- `DynamicQueues`: Persists queue configuration across restarts
- `DynamicScaler`: Auto-scales cloud infrastructure based on queue depth
## DynamicCron
Runtime-configurable cron with cluster-wide coordination. Unlike standard Oban cron, DynamicCron
supports runtime updates, guaranteed scheduling, and automatic sync.
### Configuration
```elixir
{Oban.Pro.Plugins.DynamicCron,
timezone: "Etc/UTC",
sync_mode: :automatic,
crontab: [
{"*/5 * * * *", MyApp.Workers.FrequentWorker},
{"0 0 * * *", MyApp.Workers.DailyWorker, guaranteed: true},
{"0 9 * * MON", MyApp.Workers.WeeklyWorker, timezone: "America/New_York"},
{"@reboot", MyApp.Workers.StartupWorker}
]}
```
### Options
- `timezone` — default timezone for all entries (defaults to `"Etc/UTC"`)
- `sync_mode` — `:automatic` syncs crontab entries to the database on startup, removing stale
entries; `:manual` requires explicit `insert/1` and `delete/1` calls
- `guaranteed` — per-entry option that ensures missed jobs are inserted when the system comes back
online (use for critical scheduled tasks)
### Runtime Management
```elixir
alias Oban.Pro.Plugins.DynamicCron
# Insert or update entries at runtime
DynamicCron.insert({"0 * * * *", MyApp.HourlyWorker})
# Delete an entry
DynamicCron.delete(MyApp.HourlyWorker)
# Get an entry's details
DynamicCron.get(MyApp.HourlyWorker)
# List all entries
DynamicCron.all()
```
## DynamicLifeline
Rescues orphaned jobs and repairs stuck workflows and chains. Essential for production systems
where nodes may crash or restart unexpectedly.
### Configuration
```elixir
{Oban.Pro.Plugins.DynamicLifeline, retry_discarded: false}
```
### Options
- `retry_discarded` — whether to retry discarded workflow/chain jobs when dependencies complete
Do not change the `rescue_interval` option. If you see it configured, remove it—the default is
carefully tuned and changing it causes problems.
### What It Rescues
- **Orphaned jobs** — jobs stuck in `executing` state after a node crash
- **Stuck workflows** — workflows blocked by completed dependencies that weren't properly tracked
- **Stuck chains** — chains that stopped progressing due to missed completion signals
- **Partition backfill** — repopulates rate limit and global limit partitions after restarts
## DynamicPrioritizer
Automatically increases priority of jobs that have been waiting too long. Prevents starvation
of low-priority jobs in busy systems.
### Configuration
```elixir
{Oban.Pro.Plugins.DynamicPrioritizer,
prioritizers: [
rate: [queue: :default, rate: 1, period: to_timeout(second: 30)],
once: [queue: :media, threshold: to_timeout(minute: 5), boost: 1]
]}
```
### Strategies
- `rate` — continuously boost priority at a fixed rate while waiting
- `once` — boost priority once after a threshold wait time
### Options
- `queue` — target queue (required)
- `rate` — priority boost per period (for `rate` strategy)
- `period` — how often to apply the boost (for `rate` strategy)
- `threshold` — wait time before boosting (for `once` strategy)
- `boost` — how much to increase priority (for `once` strategy, default: 1)
## DynamicPruner
Flexible job retention with per-queue and per-worker rules. More powerful than the standard
Oban pruner with fine-grained control over what gets deleted and when.
### Configuration
```elixir
{Oban.Pro.Plugins.DynamicPruner,
mode: {:max_age, {7, :days}},
queue_overrides: [
logs: {:max_age, {1, :day}},
critical: {:max_age, {30, :days}}
],
worker_overrides: [
MyApp.AuditWorker: {:max_age, {90, :days}},
MyApp.TempWorker: {:max_age, {1, :hour}}
]}
```
### Modes
- `{:max_age, {count, unit}}` — delete jobs older than the specified age
- `{:max_length, count}` — keep only the most recent N jobs
### Options
- `mode` — default retention rule for all jobs
- `queue_overrides` — per-queue retention rules
- `worker_overrides` — per-worker retention rules (takes precedence over queue rules)
- `state` — which job states to prune (default: `[:completed, :cancelled, :discarded]`)
## DynamicQueues
Persists queue configuration to the database for runtime management. Queues survive restarts
and can be modified without redeploying.
### Configuration
```elixir
{Oban.Pro.Plugins.DynamicQueues,
sync_mode: :automatic,
queues: [
default: 10,
mailers: [local_limit: 20, rate_limit: [allowed: 100, period: {1, :minute}]],
exports: [local_limit: 5, only: {:node, :=~, "worker"}]
]}
```
### Options
- `sync_mode` — `:automatic` syncs queue config to database on startup; `:manual` requires
explicit API calls
- `queues` — initial queue definitions (same format as standard Oban queues)
- `only` — restrict queue to matching nodes using `{:node, pattern}` or `{:sys_env, key, pattern}`;
supports operators `:==`, `:!=`, `:=~` for pattern matching
### Runtime Management
```elixir
alias Oban.Pro.Plugins.DynamicQueues
# Add or update a queue
DynamicQueues.insert(:reports, local_limit: 5)
# Pause a queue
DynamicQueues.pause(:reports)
# Resume a queue
DynamicQueues.resume(:reports)
# Scale a queue
DynamicQueues.scale(:reports, local_limit: 20)
# Delete a queue
DynamicQueues.delete(:reports)
# List all queues
DynamicQueues.all()
```
## DynamicScaler
Auto-scales cloud infrastructure based on queue depth and throughput. Scales nodes horizontally
to optimize processing during high traffic and save costs during quiet periods.
### Configuration
```elixir
{Oban.Pro.Plugins.DynamicScaler,
scalers: [
range: 1..5,
cloud: MyApp.Cloud
]}
```
### Options
- `cloud` — module implementing the `Oban.Pro.Cloud` behaviour (required)
- `range` — min..max nodes to scale between (required)
- `cooldown` — minimum time between scaling events (default: 120 seconds)
- `queues` — `:all` or list of queues to monitor (default: `:all`)
- `step` — `:none` or max nodes to scale at once (default: `:none`)
### Features
- **Predictive scaling** — calculates optimal scale based on recent trends, not just current depth
- **Multi-cloud** — implement the `Cloud` behaviour for your platform (AWS, Fly, Kubernetes, etc.)
- **Thrashing prevention** — cooldown period prevents rapid scale up/down cycles
### Cloud Behaviour
Implement `Oban.Pro.Cloud` for your platform:
```elixir
defmodule MyApp.Cloud do
@behaviour Oban.Pro.Cloud
@impl true
def init(opts), do: opts
@impl true
def current(_opts), do: {:ok, current_node_count()}
@impl true
def scale(count, _opts), do: scale_to(count)
end
```

54
vendor/oban_pro/usage-rules/queues.md vendored Normal file
View file

@ -0,0 +1,54 @@
# Queues
Consider a dedicated queue when you need global concurrency, rate limiting, isolation, or different SLAs.
## Global concurrency
Limit concurrent jobs across all nodes:
```elixir
# 10 jobs total across all nodes
my_queue: [global_limit: 10]
# 10 global, but max 3 per node
my_queue: [local_limit: 3, global_limit: 10]
# Exactly 1 job at a time cluster-wide
my_queue: [global_limit: 1]
```
## Rate limiting
Control throughput over time:
```elixir
# 60 jobs per minute
my_queue: [rate_limit: [allowed: 60, period: {1, :minute}]]
# 1000 jobs per hour
my_queue: [rate_limit: [allowed: 1000, period: {1, :hour}]]
# 10 jobs per 30 seconds
my_queue: [rate_limit: [allowed: 10, period: 30]]
```
Algorithms: `:sliding_window` (default, smooth), `:fixed_window` (simple resets), `:token_bucket` (allows bursts):
```elixir
my_queue: [rate_limit: [allowed: 100, period: {1, :minute}, algorithm: :token_bucket]]
```
## Partitioning
Apply limits per-partition instead of queue-wide (partition is nested inside the limit):
```elixir
# Global limit of 1 per worker
my_queue: [global_limit: [allowed: 1, partition: :worker]]
# Global limit of 1 per tenant_id from args
my_queue: [global_limit: [allowed: 1, partition: [args: :tenant_id]]]
# Rate limit per account_id
my_queue: [rate_limit: [allowed: 100, period: {1, :minute}, partition: [args: :account_id]]]
```

48
vendor/oban_pro/usage-rules/testing.md vendored Normal file
View file

@ -0,0 +1,48 @@
# Testing
Use `Oban.Pro.Testing` — a drop-in replacement for `Oban.Testing` with Pro helpers.
## Configuration
```elixir
# config/test.exs
config :my_app, Oban, testing: :manual
```
## Helpers
**Unit testing** — execute jobs without the database:
- `perform_job/3` — execute a single job
- `perform_callback/4` — execute batch callbacks
- `perform_chunk/3` — execute chunk workers
**Integration testing** — insert and execute jobs:
- `drain_jobs/1` — execute jobs across queues
- `run_workflow/2` — insert and execute a workflow
- `run_batch/2` — insert and execute a batch
- `run_chunk/2` — insert and execute chunked jobs
**Always use `drain_jobs/1` instead of `Oban.drain_queue/2`:**
- Drains all queues with `queue: :all`
- Handles workflows, batches, and chains correctly
- Supports recursive draining
- Returns completion counts
```elixir
defmodule MyApp.WorkerTest do
use MyApp.DataCase, async: false
use Oban.Pro.Testing, repo: MyApp.Repo
test "processes job" do
assert :ok == perform_job(SomeWorker, %{user_id: 123})
end
test "drains enqueued jobs" do
MyApp.enqueue_work()
assert %{completed: 3} = drain_jobs(queue: :all)
end
end
```

100
vendor/oban_pro/usage-rules/workers.md vendored Normal file
View file

@ -0,0 +1,100 @@
# Workers
Use `Oban.Pro.Worker` with `args_schema` for type-safe, validated job arguments:
```elixir
defmodule MyApp.SomeWorker do
use Oban.Pro.Worker, queue: :default, max_attempts: 3
args_schema do
field :user_id, :integer, required: true
field :status, :enum, values: ~w(pending active)a, default: :pending
field :metadata, :map
end
@impl Oban.Pro.Worker
def process(%Oban.Job{args: %__MODULE__{} = args}) do
:ok
end
end
```
## Structured Args
Validates data when calling `new/1`, before insertion. Notable types:
- `:enum` - Validates and casts to atoms: `field :status, :enum, values: ~w(pending completed)a`
- `:term` - Safely encodes any Elixir term
- `:uuid` - Alias for `:binary_id`
- `{:array, *}` - Lists of typed values
Nested structures with `embeds_one` and `embeds_many`:
```elixir
args_schema do
field :order_id, :integer, required: true
embeds_one :customer, required: true do
field :customer_id, :integer, required: true
end
embeds_many :line_items do
field :product_id, :integer, required: true
field :quantity, :integer, required: true
end
end
```
## Encrypted jobs
Protect sensitive args with transparent encryption:
```elixir
use Oban.Pro.Worker,
queue: :default,
encrypted: [key: {Application, :fetch_env!, [:my_app, :oban_key]}]
```
**Important:** Only `args` are encrypted, never put sensitive data in `meta`.
## Hooks
**Execution hooks:**
- `before_new/2` — modify args/opts before job creation
- `before_process/1` — runs before `process/1`
- `after_process/3` — runs after `process/1` with state (`:complete`, `:cancel`, `:discard`, `:error`, `:snooze`), job, and result
**External state hooks:**
- `on_cancelled/2` — job cancelled externally (manual, deadline, dependency)
- `on_discarded/2` — job discarded by DynamicLifeline
```elixir
@impl Oban.Pro.Worker
def after_process(state, job, result) do
if state in [:error, :discard], do: notify_error(job)
:ok
end
```
Attach hooks explicitly or globally:
```elixir
use Oban.Pro.Worker, hooks: [MyApp.ErrorHook]
# Global hooks (in application startup)
Oban.Pro.Worker.attach_hook(:errors, MyApp.ErrorHook)
```
## Deadlines
Cancel jobs that don't run within a time limit:
```elixir
use Oban.Pro.Worker, deadline: {1, :hour}
# Override at runtime
MyWorker.new(%{}, deadline: {30, :minutes})
# Force cancel already-running jobs
use Oban.Pro.Worker, deadline: [in: {10, :minutes}, force: true]
```

70
vendor/oban_pro/usage-rules/workflows.md vendored Normal file
View file

@ -0,0 +1,70 @@
# Workflows
Workflows compose jobs with dependency relationships: sequential, fan-out, and fan-in patterns.
```elixir
alias Oban.Pro.Workflow
Workflow.new()
|> Workflow.add(:extract, ExtractWorker.new(%{source: "db"}))
|> Workflow.add(:transform, TransformWorker.new(%{}), deps: :extract)
|> Workflow.add(:notify, NotifyWorker.new(%{}), deps: :extract) # fan-out
|> Workflow.add(:load, LoadWorker.new(%{}), deps: [:transform, :notify]) # fan-in
|> Oban.insert_all()
```
## Patterns
- **Sequential**: `deps: :previous_job`
- **Fan-out**: Multiple jobs depend on the same parent
- **Fan-in**: One job depends on multiple parents with `deps: [:a, :b, :c]`
## Accessing upstream results
Use `recorded: true` workers:
```elixir
def process(job) do
{:ok, data} = Workflow.fetch_recorded(job, :extract)
end
```
## Cascading Functions
Function captures that automatically receive context and upstream results:
```elixir
Workflow.new()
|> Workflow.put_context(%{source: source})
|> Workflow.add_cascade(:extract, &extract/1)
|> Workflow.add_cascade(:transform, &transform/1, deps: :extract)
|> Workflow.add_cascade(:load, &load/1, deps: :transform)
|> Oban.insert_all()
def extract(%{source: source}), do: %{records: fetch_records(source)}
def transform(%{extract: %{records: records}}), do: %{data: process(records)}
def load(%{transform: %{data: data}}), do: insert_all(data)
```
Fan-out with `{enumerable, function/2}`:
```elixir
Workflow.add_cascade(:accounts, {account_ids, &sync_account/2})
```
## Sub-workflows
Compose from reusable pieces:
```elixir
extract_flow =
Workflow.new()
|> Workflow.add(:fetch, FetchWorker.new(%{}))
|> Workflow.add(:parse, ParseWorker.new(%{}), deps: :fetch)
Workflow.new()
|> Workflow.add(:setup, SetupWorker.new(%{}))
|> Workflow.add_workflow(:extract, extract_flow, deps: :setup)
|> Workflow.add(:cleanup, CleanupWorker.new(%{}), deps: :extract)
|> Oban.insert_all()
```