diff --git a/lib/towerops/agents/agent_assignment.ex b/lib/towerops/agents/agent_assignment.ex index 971ee218..db69b74f 100644 --- a/lib/towerops/agents/agent_assignment.ex +++ b/lib/towerops/agents/agent_assignment.ex @@ -48,7 +48,7 @@ defmodule Towerops.Agents.AgentAssignment do assignment |> cast(attrs, [:agent_token_id, :device_id, :enabled]) |> validate_required([:agent_token_id, :device_id]) - |> unique_constraint([:device_id], name: "agent_assignments_equipment_id_index") + |> unique_constraint([:device_id], name: "agent_assignments_device_id_index") |> foreign_key_constraint(:agent_token_id) |> foreign_key_constraint(:device_id) end diff --git a/mix.exs b/mix.exs index 004373ac..29528d0b 100644 --- a/mix.exs +++ b/mix.exs @@ -85,7 +85,7 @@ defmodule Towerops.MixProject do {:jason, "~> 1.2"}, {:dns_cluster, "~> 0.2.0"}, {:libcluster, "~> 3.4"}, - {:oban_pro, "~> 1.6", path: "vendor/oban_pro"}, + {:oban_pro, "~> 1.7", path: "vendor/oban_pro"}, {:oban_met, "~> 1.0", path: "vendor/oban_met", override: true}, {:oban_web, "~> 2.11", path: "vendor/oban_web"}, {:bandit, "~> 1.5"}, diff --git a/mix.lock b/mix.lock index bbda083e..01467ebd 100644 --- a/mix.lock +++ b/mix.lock @@ -14,7 +14,7 @@ "cloak_ecto": {:hex, :cloak_ecto, "1.3.0", "0de127c857d7452ba3c3367f53fb814b0410ff9c680a8d20fbe8b9a3c57a1118", [:mix], [{:cloak, "~> 1.1.1", [hex: :cloak, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}], "hexpm", "314beb0c123b8a800418ca1d51065b27ba3b15f085977e65c0f7b2adab2de1cc"}, "comeonin": {:hex, :comeonin, "5.5.1", "5113e5f3800799787de08a6e0db307133850e635d34e9fab23c70b6501669510", [:mix], [], "hexpm", "65aac8f19938145377cee73973f192c5645873dcf550a8a6b18187d17c13ccdb"}, "credo": {:hex, :credo, "1.7.18", "5c5596bf7aedf9c8c227f13272ac499fe8eae6237bd326f2f07dfc173786f042", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "a189d164685fd945809e862fe76a7420c4398fa288d76257662aecb909d6b3e5"}, - "db_connection": {:hex, :db_connection, "2.9.0", "a6a97c5c958a2d7091a58a9be40caf41ab496b0701d21e1d1abff3fa27a7f371", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "17d502eacaf61829db98facf6f20808ed33da6ccf495354a41e64fe42f9c509c"}, + "db_connection": {:hex, :db_connection, "2.10.0", "8ff756471e41765bd5563b633f73e9a94bbc138816e8644bb17d0d91bf260a95", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "02cdd01b45efb1b550e68edbbea41be32de9b24bb07e1ea0e9cbc522ac377e54"}, "decimal": {:hex, :decimal, "2.3.0", "3ad6255aa77b4a3c4f818171b12d237500e63525c2fd056699967a3e7ea20f62", [:mix], [], "hexpm", "a4d66355cb29cb47c3cf30e71329e58361cfcb37c34235ef3bf1d7bf3773aeac"}, "dialyxir": {:hex, :dialyxir, "1.4.7", "dda948fcee52962e4b6c5b4b16b2d8fa7d50d8645bbae8b8685c3f9ecb7f5f4d", [:mix], [{:erlex, ">= 0.2.8", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b34527202e6eb8cee198efec110996c25c5898f43a4094df157f8d28f27d9efe"}, "dns_cluster": {:hex, :dns_cluster, "0.2.0", "aa8eb46e3bd0326bd67b84790c561733b25c5ba2fe3c7e36f28e88f384ebcb33", [:mix], [], "hexpm", "ba6f1893411c69c01b9e8e8f772062535a4cf70f3f35bcc964a324078d8c8240"}, @@ -57,7 +57,8 @@ "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, "nimble_totp": {:hex, :nimble_totp, "1.0.0", "79753bae6ce59fd7cacdb21501a1dbac249e53a51c4cd22b34fa8438ee067283", [:mix], [], "hexpm", "6ce5e4c068feecdb782e85b18237f86f66541523e6bad123e02ee1adbe48eda9"}, - "oban": {:hex, :oban, "2.21.1", "4b6af7b901ef9baca09e239b5a991ef2fa429cf5a13799bc429a131d610ff692", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.20", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8162a160924cf4a25905fed2a9242e7787d88e320e3b5b0dcf324eb17c51c4e6"}, + "oban": {:hex, :oban, "2.22.1", "9d2a38cec95070b31c1e274fae55f3925089f62159d8f3facabee0454d55b257", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.20", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "af2508c156c5b0ec30b21b0883babf7e2716af35ed5d264095896103fe3cea37"}, + "oban_pro": {:hex, :oban_pro, "1.7.0", "e501c3957540dbd4b2d69c9045f04f4bd451d4f54d7cf185bf22c8c8cc56323c", [:mix], [{:ecto_sql, "~> 3.12", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:oban, "~> 2.21", [hex: :oban, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.22", [hex: :postgrex, repo: "hexpm", optional: true]}], "oban", "9e02b1abe356ee2c382d4a3606a59386158560c6ceab9aa95019d39363cde46a"}, "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"}, "phoenix": {:hex, :phoenix, "1.8.5", "919db335247e6d4891764dc3063415b0d2457641c5f9b3751b5df03d8e20bbcf", [:mix], [{:bandit, "~> 1.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "83b2bb125127e02e9f475c8e3e92736325b5b01b0b9b05407bcb4083b7a32485"}, "phoenix_ecto": {:hex, :phoenix_ecto, "4.7.0", "75c4b9dfb3efdc42aec2bd5f8bccd978aca0651dbcbc7a3f362ea5d9d43153c6", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "1d75011e4254cb4ddf823e81823a9629559a1be93b4321a6a5f11a5306fbf4cc"}, diff --git a/priv/repo/migrations/20260430142200_upgrade_oban_to_v14.exs b/priv/repo/migrations/20260430142200_upgrade_oban_to_v14.exs new file mode 100644 index 00000000..cc636335 --- /dev/null +++ b/priv/repo/migrations/20260430142200_upgrade_oban_to_v14.exs @@ -0,0 +1,6 @@ +defmodule Towerops.Repo.Migrations.UpgradeObanToV14 do + use Ecto.Migration + + def up, do: Oban.Migration.up(version: 14) + def down, do: Oban.Migration.down(version: 12) +end diff --git a/priv/repo/migrations/20260430142241_upgrade_oban_pro_to_1_7.exs b/priv/repo/migrations/20260430142241_upgrade_oban_pro_to_1_7.exs new file mode 100644 index 00000000..2736a63e --- /dev/null +++ b/priv/repo/migrations/20260430142241_upgrade_oban_pro_to_1_7.exs @@ -0,0 +1,6 @@ +defmodule Towerops.Repo.Migrations.UpgradeObanProTo17 do + use Ecto.Migration + + def up, do: Oban.Pro.Migration.up(version: "1.7.0") + def down, do: Oban.Pro.Migration.down(version: "1.7.0") +end diff --git a/vendor/README.md b/vendor/README.md index 3976ece3..71fd3fdf 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -24,7 +24,7 @@ RUN mix deps.get --only $MIX_ENV ### Oban Pro -**Version**: 1.6.13 +**Version**: 1.7.0 **Source**: https://getoban.pro/repo (private) **License**: Commercial (licensed) @@ -32,7 +32,7 @@ Advanced Oban features including batching, workflows, dynamic partitioning, and **Configuration in mix.exs**: ```elixir -{:oban_pro, "~> 1.6", path: "vendor/oban_pro"} +{:oban_pro, "~> 1.7", path: "vendor/oban_pro"} ``` ### Oban Web diff --git a/vendor/oban_pro/.formatter.exs b/vendor/oban_pro/.formatter.exs index da6266a2..4792336e 100644 --- a/vendor/oban_pro/.formatter.exs +++ b/vendor/oban_pro/.formatter.exs @@ -10,5 +10,10 @@ locals_without_parens = [ import_deps: [:ecto, :ecto_sql, :oban, :stream_data], export: [locals_without_parens: locals_without_parens], locals_without_parens: locals_without_parens, - inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] + inputs: [ + "{mix,.formatter}.exs", + "{config,lib}/**/*.{ex,exs}", + "test/**/*.{exs}", + "test/support/**/*.ex" + ] ] diff --git a/vendor/oban_pro/.hex b/vendor/oban_pro/.hex index ae747695..799f1470 100644 Binary files a/vendor/oban_pro/.hex and b/vendor/oban_pro/.hex differ diff --git a/vendor/oban_pro/hex_metadata.config b/vendor/oban_pro/hex_metadata.config index a1b60886..22012455 100644 --- a/vendor/oban_pro/hex_metadata.config +++ b/vendor/oban_pro/hex_metadata.config @@ -1,19 +1,20 @@ {<<"links">>,[]}. {<<"name">>,<<"oban_pro">>}. -{<<"version">>,<<"1.6.13">>}. +{<<"version">>,<<"1.7.0">>}. {<<"description">>,<<"Oban Pro Component">>}. {<<"elixir">>,<<"~> 1.15">>}. {<<"files">>, - [<<"lib/pro.ex">>,<<"lib/oban">>,<<"lib/oban/pro">>, - <<"lib/oban/pro/diagnostics.ex">>,<<"lib/oban/pro/cron.ex">>, - <<"lib/oban/pro/decorator.ex">>,<<"lib/oban/pro/migrations">>, - <<"lib/oban/pro/migrations/v1_4_0.ex">>, + [<<"lib/oban">>,<<"lib/oban/pro">>,<<"lib/oban/pro/diagnostics.ex">>, + <<"lib/oban/pro/cron.ex">>,<<"lib/oban/pro/decorator.ex">>, + <<"lib/oban/pro/migrations">>,<<"lib/oban/pro/migrations/v1_4_0.ex">>, <<"lib/oban/pro/migrations/v1_5_0.ex">>, <<"lib/oban/pro/migrations/v1_6_0.ex">>, + <<"lib/oban/pro/migrations/v1_7_0.ex">>, <<"lib/oban/pro/migrations/dynamic_partitioner.ex">>, <<"lib/oban/pro/migrations/v1_0_0.ex">>,<<"lib/oban/pro/unique.ex">>, - <<"lib/oban/pro/batch.ex">>,<<"lib/oban/pro/producer.ex">>, - <<"lib/oban/pro/worker.ex">>,<<"lib/oban/pro/stages">>, + <<"lib/oban/pro/batch.ex">>,<<"lib/oban/pro/chunk.ex">>, + <<"lib/oban/pro/producer.ex">>,<<"lib/oban/pro/worker.ex">>, + <<"lib/oban/pro/stages">>,<<"lib/oban/pro/stages/rate.ex">>, <<"lib/oban/pro/stages/chain.ex">>,<<"lib/oban/pro/stages/encrypted.ex">>, <<"lib/oban/pro/stages/recorded.ex">>,<<"lib/oban/pro/stages/hooks.ex">>, <<"lib/oban/pro/stages/deadline.ex">>, @@ -27,6 +28,11 @@ <<"lib/oban/pro/plugins/dynamic_queues.ex">>, <<"lib/oban/pro/plugins/dynamic_cron.ex">>,<<"lib/oban/pro/limiters">>, <<"lib/oban/pro/limiters/local.ex">>,<<"lib/oban/pro/limiters/rate.ex">>, + <<"lib/oban/pro/limiters/rate">>, + <<"lib/oban/pro/limiters/rate/token_bucket.ex">>, + <<"lib/oban/pro/limiters/rate/algorithm.ex">>, + <<"lib/oban/pro/limiters/rate/fixed_window.ex">>, + <<"lib/oban/pro/limiters/rate/sliding_window.ex">>, <<"lib/oban/pro/limiters/global.ex">>,<<"lib/oban/pro/queue.ex">>, <<"lib/oban/pro/handler.ex">>,<<"lib/oban/pro/partition.ex">>, <<"lib/oban/pro/cloud.ex">>,<<"lib/oban/pro/uuidv7.ex">>, @@ -34,29 +40,36 @@ <<"lib/oban/pro/workflow.ex">>,<<"lib/oban/pro/testing.ex">>, <<"lib/oban/pro/migration.ex">>,<<"lib/oban/pro/engines">>, <<"lib/oban/pro/engines/smart.ex">>,<<"lib/oban/pro/workflow">>, + <<"lib/oban/pro/workflow/schema.ex">>, <<"lib/oban/pro/workflow/cascade.ex">>,<<"lib/oban/pro/limiter.ex">>, <<"lib/oban/pro/validation.ex">>,<<"lib/oban/pro/flusher.ex">>, <<"lib/oban/pro/workers">>,<<"lib/oban/pro/workers/chain.ex">>, <<"lib/oban/pro/workers/batch.ex">>,<<"lib/oban/pro/workers/chunk.ex">>, <<"lib/oban/pro/workers/workflow.ex">>,<<"lib/oban/pro/application.ex">>, <<"lib/oban/pro/utils.ex">>,<<"lib/oban/pro/relay.ex">>, - <<"lib/oban/pro/exceptions.ex">>,<<".formatter.exs">>,<<"mix.exs">>]}. + <<"lib/oban/pro/exceptions.ex">>,<<"lib/oban/pro/rate_limit.ex">>, + <<"lib/oban/pro.ex">>,<<"usage-rules">>,<<"usage-rules/workflows.md">>, + <<"usage-rules/chunks.md">>,<<"usage-rules/workers.md">>, + <<"usage-rules/testing.md">>,<<"usage-rules/batches.md">>, + <<"usage-rules/queues.md">>,<<"usage-rules/plugins.md">>, + <<"usage-rules/chains.md">>,<<".formatter.exs">>,<<"mix.exs">>, + <<"usage-rules.md">>]}. {<<"app">>,<<"oban_pro">>}. {<<"licenses">>,[<<"Commercial">>]}. {<<"requirements">>, [[{<<"name">>,<<"oban">>}, {<<"app">>,<<"oban">>}, {<<"optional">>,false}, - {<<"requirement">>,<<"~> 2.19">>}, + {<<"requirement">>,<<"~> 2.21">>}, {<<"repository">>,<<"hexpm">>}], [{<<"name">>,<<"ecto_sql">>}, {<<"app">>,<<"ecto_sql">>}, {<<"optional">>,false}, - {<<"requirement">>,<<"~> 3.10">>}, + {<<"requirement">>,<<"~> 3.12">>}, {<<"repository">>,<<"hexpm">>}], [{<<"name">>,<<"postgrex">>}, {<<"app">>,<<"postgrex">>}, {<<"optional">>,true}, - {<<"requirement">>,<<"~> 0.16">>}, + {<<"requirement">>,<<"~> 0.22">>}, {<<"repository">>,<<"hexpm">>}]]}. {<<"build_tools">>,[<<"mix">>]}. diff --git a/vendor/oban_pro/lib/pro.ex b/vendor/oban_pro/lib/oban/pro.ex similarity index 100% rename from vendor/oban_pro/lib/pro.ex rename to vendor/oban_pro/lib/oban/pro.ex diff --git a/vendor/oban_pro/lib/oban/pro/batch.ex b/vendor/oban_pro/lib/oban/pro/batch.ex index f63ed592..bf5ca253 100644 --- a/vendor/oban_pro/lib/oban/pro/batch.ex +++ b/vendor/oban_pro/lib/oban/pro/batch.ex @@ -240,8 +240,8 @@ defmodule Oban.Pro.Batch do import Ecto.Query, only: [limit: 2, order_by: 2, select: 3, union_all: 2, where: 3] alias Ecto.Changeset - alias Oban.Pro.Engines.Smart alias Oban.{Job, Repo, Validation, Worker} + alias Oban.Pro.Engines.Smart alias Oban.Pro.Workflow require Logger diff --git a/vendor/oban_pro/lib/oban/pro/chunk.ex b/vendor/oban_pro/lib/oban/pro/chunk.ex new file mode 100644 index 00000000..93597647 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/chunk.ex @@ -0,0 +1,660 @@ +defmodule Oban.Pro.Chunk do + @moduledoc """ + Chunk workers execute jobs together in groups based on a size or a timeout option, e.g. when + 1000 jobs are available or after 10 minutes have elapsed. + + Multiple chunks can run in parallel within a single queue, and each chunk may be composed of + many thousands of jobs. Aside from improved throughput for a single queue, chunks are ideal as + the initial stage of data-ingestion and data-processing pipelines. + + ## Usage + + Let's define a worker that sends SMS messages in chunks, rather than individually: + + ```elixir + defmodule MyApp.ChunkWorker do + use Oban.Pro.Chunk, queue: :messages, size: 100, timeout: 1000 + + @impl true + def process([_ | _] = jobs) do + jobs + |> Enum.map(& &1.args) + |> MyApp.Messaging.send_batch() + + :ok + end + end + ``` + + Notice that we declared a `size` and a `timeout` along with a `queue` for the worker. If `size` + or `timeout` are omitted they fall back to their defaults: 100 `size` and 1000ms respectively. + + To process _larger_ batches _less_ frequently, we can increase both values: + + ```elixir + use Oban.Pro.Chunk, size: 500, timeout: :timer.seconds(5) + ``` + + Now chunks will run with up to 500 jobs or every 5 seconds, whichever comes first. + + Unlike other Pro workers, a `Chunk` worker's `process/1` receives a list of jobs rather than a + single job struct. Fittingly, the [expected return values](#t:result/0) are different as well. + + ## Chunk Partitioning + + By default, chunks are divided into groups based on the `queue` and `worker`. This means that + each chunk consists of workers belonging to the same queue, regardless of their `args` or + `meta`. However, this approach may not always be suitable. For instance, you may want to group + workers based on a specific argument such as `:account_id` instead of just the worker. In such + cases, you can use the [`:by`](#t:chunk_by/0) option to customize how chunks are partitioned. + + Here are a few examples of the `:by` option that you can use to achieve fine-grained control over + chunk partitioning: + + ```elixir + # Explicitly chunk by :worker + use Oban.Pro.Chunk, by: :worker + + # Chunk by a single args key without considering the worker + use Oban.Pro.Chunk, by: [args: :account_id] + + # Chunk by multiple args keys without considering the worker + use Oban.Pro.Chunk, by: [args: [:account_id, :cohort]] + + # Chunk by worker and a single args key + use Oban.Pro.Chunk, by: [:worker, args: :account_id] + + # Chunk by a single meta key + use Oban.Pro.Chunk, by: [meta: :source_id] + ``` + + When partitioning chunks of jobs, it's important to note that using only `:args` or `:meta` + without `:worker` may result in heterogeneous chunks of jobs from different workers. + Nevertheless, regardless of the partitioning method chunks always consist of jobs from the same + queue. + + Here's a simple example of partitioning by `worker` and an `author_id` field to batch analysis + of recent messages per author: + + ```elixir + defmodule MyApp.LLMAnalysis do + use Oban.Pro.Chunk, by: [:worker, args: :author_id], size: 50, timeout: 30_000 + + @impl true + def process([%{args: %{"author_id" => author_id}} | _] = jobs) do + messages = + jobs + |> Enum.map(& &1.args["message_id"]) + |> MyApp.Messages.all() + + {:ok, sentiment} = MyApp.GPT.gauge_sentiment(messages) + + MyApp.Messages.record_sentiment(author_id) + end + end + ``` + + ## Chunk Results and Error Handling + + `Chunk` worker's result type is tailored to handling multiple jobs at once. The result types + allow you to succeed an entire chunk, or selectively apply different outcomes to individual + jobs. Here are the possible results: + + * `:ok` — The chunk succeeded and all jobs can be marked complete + + * `{:ok, result}` — Like `:ok`, but with a result for testing. + + * `{:error, reason, jobs}` — One or more jobs in the chunk failed and may be retried, any + unlisted jobs are a success. + + * `{:cancel, reason, jobs}` — One or more jobs in the chunk should be cancelled, any unlisted + jobs are a success. + + * `{:snooze, period, jobs}` — One or more jobs in the chunk should be snoozed and rescheduled + to run again after the specified period (e.g. `10` for seconds or `{1, :minute}`), any + unlisted jobs are a success. + + * `[error: {reason, jobs}, cancel: {reason, jobs}, ...]` — Selectively apply different + outcomes to different jobs. Supports `error`, `cancel`, `discard`, and `snooze` keys. For + snooze, use `snooze: {period, jobs}`. Any unlisted jobs are a success. + + To illustrate using chunk results let's expand on the message processing example from earlier. + We'll extend it to complete the whole batch when all messages are delivered or cancel + undeliverable messages: + + ```elixir + def process([_ | _] = jobs) do + notifications = + jobs + |> Enum.map(& &1.args) + |> MyApp.Messaging.send_batch() + + bad_token = fn %{response: response} -> response == :bad_token end + + if Enum.any?(notifications, bad_token) do + cancels = + notifications + |> Enum.zip(jobs) + |> Enum.filter(fn {notification, _job} -> bad_token.(notification) end) + |> Enum.map(&elem(&1, 1)) + + {:cancel, :bad_token, cancels} + else + {:ok, notifications} + end + end + ``` + + In the event of an ephemeral crash, like a network issue, the entire chunk may be retried if + there are any remaining attempts. + + ## Chunk Organization + + Chunks are run by a leader job (which has nothing to do with peer leadership). When the leader + executes it determines whether a complete chunk is available or if enough time has elapsed to + run anyhow. If neither case applies, then the leader will sleep until the timeout elapses and + then execute with as many jobs as it can find. + + Completion queries run every `1000ms` by default. You can use the `:sleep` option to control how + long the leader delays between queries to check for complete chunks: + + ```elixir + use Oban.Pro.Chunk, size: 50, sleep: 2_000, timeout: 10_000 + ``` + + ## Run Chunks Immediately + + A chunk won't run until either the full number of jobs are available (`size`), or the total time + has elapsed (`timeout`). In situations where there is a long gap between incoming jobs it may be + desirable to run immediately when the leader starts. This can be done with the `leading` option. + For example, to chunk all jobs that arrive within a 5 minute window, but run immediately if more + than 5 minutes have passed since the last chunk completed: + + ```elixir + use Oban.Pro.Chunk, size: 100, leading: true, timeout: :timer.minutes(5) + ``` + + ## Optimizing Chunks + + Queues with high concurrency and low throughput may have multiple chunk leaders running + simultaneously rather than getting bundled into a single chunk. The solution is to reduce the + queue's global concurrency to a smaller number so that new chunk leader jobs don't start and the + existing chunk can run a bigger batch. + + ```elixir + queues: [ + chunked_queue: [global_limit: 1] + ] + ``` + + A low global limit will reduce overall throughput. Partitioning the queue with the same options + as the chunk can preserve throughput by allowing one concurrent leader per chunk. For example, + if you were chunking by `:user_id` in args, you'd partition the queue with the same option: + + ```elixir + queues: [ + chunked_queue: [local_limit: 20, global_limit: [allowed: 1, partition: [args: :user_id]]] + ] + ``` + """ + + import Ecto.Query + import DateTime, only: [utc_now: 0] + + alias Oban.{CrashError, Job, Notifier, PerformError, Period, Repo, Validation} + alias Oban.Pro.Engines.Smart + alias Oban.Pro.Utils + + @typedoc """ + A single key or list of keys used to specify fields for partitioning. + """ + @type key_or_keys :: atom() | [atom()] + + @typedoc """ + Specifies how jobs are grouped into chunks. + + Jobs are always partitioned by queue, but may also be grouped by worker, args, or meta fields. + """ + @type chunk_by :: + :worker + | {:args, key_or_keys()} + | {:meta, key_or_keys()} + | [:worker | {:args, key_or_keys()} | {:meta, key_or_keys()}] + + @typedoc """ + A sub-result for error, cancel, or discard outcomes within a keyword list result. + """ + @type sub_result :: {reason :: term(), [Job.t()]} + + @typedoc """ + A sub-result for snooze outcomes within a keyword list result. + """ + @type snooze_result :: {period :: Oban.Period.t(), [Job.t()]} + + @typedoc """ + The return value of `process/1`, indicating the outcome for jobs in the chunk. + + Unlisted jobs are marked as completed. See the "Chunk Results and Error Handling" section for + details on each variant. + """ + @type result :: + :ok + | {:ok, term()} + | {:cancel | :discard | :error, reason :: term(), [Job.t()]} + | {:snooze, period :: Oban.Period.t(), [Job.t()]} + | [ + cancel: sub_result(), + discard: sub_result(), + error: sub_result(), + snooze: snooze_result() + ] + + @typedoc """ + Options for configuring chunk behavior. + """ + @type options :: [ + by: chunk_by(), + leading: boolean(), + size: pos_integer(), + sleep: pos_integer(), + timeout: pos_integer() + ] + + @doc false + defmacro __using__(opts) do + {chunk_opts, other_opts} = Keyword.split(opts, ~w(by leading size sleep timeout)a) + + quote do + Validation.validate!(unquote(chunk_opts), &Oban.Pro.Chunk.validate/1) + + use Oban.Pro.Worker, unquote(other_opts) + + alias Oban.Pro.Chunk + + @chunk_meta %{ + chunk: true, + chunk_by: Keyword.get(unquote(chunk_opts), :by, [:worker]), + chunk_leading: Keyword.get(unquote(chunk_opts), :leading, false), + chunk_size: Keyword.get(unquote(chunk_opts), :size, 100), + chunk_sleep: Keyword.get(unquote(chunk_opts), :sleep, 1000), + chunk_timeout: Keyword.get(unquote(chunk_opts), :timeout, 1000) + } + + @impl Oban.Worker + def new(args, opts) when is_map(args) and is_list(opts) do + meta = + @chunk_meta + |> Map.merge(Keyword.get(opts, :meta, %{})) + |> Map.update!(:chunk_by, &Utils.normalize_by/1) + + chunk_id = Chunk.to_chunk_id(meta.chunk_by, __MODULE__, args, opts) + meta = Map.put(meta, :chunk_id, chunk_id) + + super(args, Keyword.put(opts, :meta, meta)) + end + + @impl Oban.Worker + def perform(%Job{} = job) do + opts = __opts__() + + with {:ok, job} <- Oban.Pro.Worker.before_process(__MODULE__, job, opts) do + Process.put(:oban_processing, {__MODULE__, job, opts}) + + job + |> Chunk.maybe_process(__MODULE__) + |> Oban.Pro.Worker.after_process(__MODULE__, job, opts) + end + end + end + end + + defmacrop in_chunk(meta, chunk_id) do + quote do + fragment( + "? \\? 'chunk_id' AND ?->>'chunk_id' = ?", + unquote(meta), + unquote(meta), + unquote(chunk_id) + ) + end + end + + defmacrop finished_at(job) do + quote do + type( + fragment( + "greatest(?, ?, ?)", + unquote(job).completed_at, + unquote(job).discarded_at, + unquote(job).cancelled_at + ), + :utc_datetime_usec + ) + end + end + + @doc false + def validate(opts) do + Validation.validate_schema(opts, + by: {:custom, &Oban.Pro.Validation.validate_by/1}, + leading: :boolean, + size: :pos_integer, + sleep: :timeout, + timeout: :timeout + ) + end + + @doc false + def to_chunk_id(chunk_by, worker, args, opts) do + chunk_data = + Enum.map([:queue | chunk_by], fn + :queue -> opts |> Keyword.get(:queue, "default") |> to_string() + :worker -> inspect(worker) + [:args, keys] -> Utils.take_keys(args, keys) + [:meta, keys] -> opts |> Keyword.get(:meta, %{}) |> Utils.take_keys(keys) + end) + + Utils.hash64(chunk_data) + end + + # Public Interface + + @doc false + def maybe_process(%Job{conf: conf} = job, worker) do + cond do + full_timeout?(conf, job) -> + fetch_and_process(worker, job) + + full_chunk?(conf, job) -> + fetch_and_process(worker, job) + + true -> + job.meta + |> Map.get("chunk_timeout", 1000) + |> sleep_and_process(worker, job) + end + end + + # Check Helpers + + defp full_chunk?(conf, %{meta: %{"chunk_id" => chunk_id, "chunk_size" => size}}) do + limit = size - 1 + + query = + Job + |> select([j], j.id) + |> where([j], j.state == "available") + |> where([j], in_chunk(j.meta, ^chunk_id)) + |> limit(^limit) + + Repo.one(conf, from(oj in subquery(query), select: count(oj.id) >= ^limit)) + end + + defp full_timeout?(_, %{meta: %{"chunk_leading" => false}}), do: false + + defp full_timeout?(conf, job) do + %{id: job_id, meta: %{"chunk_id" => chunk_id, "chunk_timeout" => timeout}} = job + + query = + Job + |> where([j], j.id != ^job_id) + |> where([j], j.state in ~w(completed cancelled discarded)) + |> where([j], in_chunk(j.meta, ^chunk_id)) + |> order_by(desc: :id) + |> limit(1) + |> select([j], finished_at(j)) + + last_ts = Repo.one(conf, query) + + last_ts && + DateTime.utc_now() + |> DateTime.add(-timeout, :millisecond) + |> DateTime.after?(last_ts) + end + + # Processing Helpers + + defp sleep_and_process(total, worker, %Job{conf: conf, meta: meta} = job) do + sleep = Map.get(meta, "chunk_sleep", 1000) + + Process.sleep(sleep) + + if total - sleep < 0 or full_chunk?(conf, job) do + fetch_and_process(worker, job) + else + sleep_and_process(total - sleep, worker, job) + end + end + + defp fetch_and_process(worker, %{conf: conf, meta: %{"chunk_size" => size}} = job) do + {:ok, chunk} = fetch_chunk(conf, job, size - 1) + {chunk, _errored} = prepare_chunk(chunk) + + guard_cancel(conf, worker, job, chunk) + guard_timeout(conf, worker.timeout(job), worker, job, chunk) + + try do + case worker.process([job | chunk]) do + :ok -> + ack_jobs(conf, worker, job, chunk, %{}) + + {:ok, result} -> + with :ok <- ack_jobs(conf, worker, job, chunk, %{}), do: {:ok, result} + + {:cancel, reason, cancelled} -> + ops = Map.new(cancelled, &{&1.id, {:cancel, reason}}) + ack_jobs(conf, worker, job, chunk, ops) + + {:discard, reason, discarded} -> + ops = Map.new(discarded, &{&1.id, {:discard, reason}}) + ack_jobs(conf, worker, job, chunk, ops) + + {:error, reason, errored} -> + ops = Map.new(errored, &{&1.id, {:error, reason}}) + ack_jobs(conf, worker, job, chunk, ops) + + {:snooze, seconds, snoozed} -> + ops = Map.new(snoozed, &{&1.id, {:snooze, seconds}}) + ack_jobs(conf, worker, job, chunk, ops) + + [_ | _] = multiple -> + ops = + for {oper, {reason, jobs}} <- multiple, + job <- jobs, + into: %{}, + do: {job.id, {oper, reason}} + + ack_jobs(conf, worker, job, chunk, ops) + end + rescue + error -> + ops = Map.new(chunk, &{&1.id, {:raised, error}}) + ack_jobs(conf, worker, job, chunk, ops, __STACKTRACE__) + + reraise error, __STACKTRACE__ + catch + kind, reason -> + error = CrashError.exception({kind, reason, __STACKTRACE__}) + ops = Map.new(chunk, &{&1.id, {:raised, error}}) + ack_jobs(conf, worker, job, chunk, ops, __STACKTRACE__) + + reraise error, __STACKTRACE__ + end + end + + # This replicates the query used in Smart.fetch_jobs/3, without the meta tracking or any other + # complications. Any modifications to the original query must be replicated here. Another + defp fetch_chunk(conf, %{meta: %{"chunk_id" => chunk_id}} = job, limit) do + subset_query = + Job + |> select([:id]) + |> where([j], j.state == "available") + |> where([j], in_chunk(j.meta, ^chunk_id)) + |> order_by(asc: :priority, asc: :scheduled_at, asc: :id) + |> limit(^limit) + |> lock("FOR UPDATE SKIP LOCKED") + + query = + Job + |> with_cte("subset", as: ^subset_query) + |> join(:inner, [j], x in fragment(~s("subset")), on: true) + |> where([j, x], j.id == x.id) + |> select([j, _], j) + + attempted_by = job.attempted_by ++ ["chunk-#{job.id}"] + + updates = [ + set: [state: "executing", attempted_at: DateTime.utc_now(), attempted_by: attempted_by], + inc: [attempt: 1] + ] + + Repo.transaction(conf, fn -> + {_count, chunk} = Repo.update_all(conf, query, updates) + + chunk + end) + end + + defp prepare_chunk(chunk) do + Enum.reduce(chunk, {[], []}, fn job, {acc, err} -> + with {:ok, worker} <- Oban.Worker.from_string(job.worker), + {:ok, job} <- Oban.Pro.Worker.before_hook(worker, job, worker.__opts__()) do + {[job | acc], err} + else + {:error, reason} -> + {acc, [{job, reason} | err]} + end + end) + end + + # Guards + + # Only the leader job is registered as "running" by the queue producer. We listen for pkill + # messages for _other_ jobs in the chunk and apply that to all jobs. Without this, cancelling + # doesn't apply to the leader and chunk jobs are orphaned. + defp guard_cancel(conf, worker, job, chunk) do + # No supervision tree is running when draining, attempting to notify will raise an exception. + if Process.get(:oban_draining) do + :ok + else + parent = self() + + Task.start(fn -> + ref = Process.monitor(parent) + :ok = Notifier.listen(conf.name, [:signal]) + + await_cancel(conf, ref, worker, job, chunk) + end) + end + end + + defp await_cancel(conf, ref, worker, job, chunk) do + receive do + {:notification, :signal, %{"action" => "pkill", "job_id" => kill_id}} -> + if Enum.any?(chunk, &(&1.id == kill_id)) do + ops = Map.new(chunk, &{&1.id, {:cancel, :shutdown}}) + + ack_jobs(conf, worker, job, chunk, ops) + + Notifier.notify(conf.name, :signal, %{action: "pkill", job_id: job.id}) + else + await_cancel(conf, ref, worker, job, chunk) + end + + {:DOWN, ^ref, :process, _pid, _reason} -> + :ok + end + end + + # The executor uses :timer.exit_after/2 to kill jobs that exceed the timeout. The queue's + # producer then catches the DOWN message and uses that to record a job error. The producer + # isn't aware of the job's chunk, so we monitor the parent process and ack the chunk jobs. + defp guard_timeout(_conf, :infinity, _worker, _job, _chunk), do: :ok + + defp guard_timeout(conf, timeout, worker, job, chunk) do + parent = self() + + Task.start(fn -> + ref = Process.monitor(parent) + + receive do + {:DOWN, ^ref, :process, _pid, %Oban.TimeoutError{} = error} -> + ops = Map.new(chunk, &{&1.id, {:raised, error}}) + + ack_jobs(conf, worker, job, chunk, ops) + + {:DOWN, ^ref, :process, _pid, _reason} -> + :ok + after + timeout + 100 -> :ok + end + end) + end + + # Acking + + defp ack_jobs(conf, worker, host, chunk, ops, stacktrace \\ []) do + now = DateTime.utc_now() + + to_ack = fn job -> + set = + case Map.get(ops, job.id) do + nil -> + %{state: "completed", completed_at: now} + + {:cancel, reason} -> + error = format_error(job, worker, {:cancel, reason}, []) + %{state: "cancelled", cancelled_at: now, error: error} + + {:discard, reason} -> + error = format_error(job, worker, {:discard, reason}, []) + %{state: "discarded", discarded_at: now, error: error} + + {:error, reason} when job.attempt >= job.max_attempts -> + error = format_error(job, worker, {:discard, reason}, []) + %{state: "discarded", discarded_at: now, error: error} + + {:error, reason} -> + error = format_error(job, worker, {:error, reason}, []) + %{state: "retryable", scheduled_at: backoff_at(worker, job), error: error} + + {:raised, error} when job.attempt >= job.max_attempts -> + error = format_error(job, worker, error, stacktrace) + %{state: "discarded", discarded_at: now, error: error} + + {:raised, error} -> + error = format_error(job, worker, error, stacktrace) + %{state: "retryable", scheduled_at: backoff_at(worker, job), error: error} + + {:snooze, period} -> + scheduled_at = DateTime.add(now, Period.to_seconds(period), :second) + %{state: "scheduled", scheduled_at: scheduled_at} + end + + {{:ack, conf.name, job.queue, job.id}, nil, [], set} + end + + acks = + chunk + |> Enum.reject(&(&1.id == host.id)) + |> Enum.map(to_ack) + + with {:ok, _} <- Smart.ack_jobs(acks, conf), do: Map.get(ops, host.id, :ok) + end + + defp backoff_at(worker, job) do + DateTime.add(utc_now(), worker.backoff(job), :second) + end + + defp format_error(job, worker, reason, stacktrace) when is_tuple(reason) do + format_error(job, worker, PerformError.exception({worker, reason}), stacktrace) + end + + defp format_error(job, _worker, error, stacktrace) do + {blamed, stacktrace} = Exception.blame(:error, error, stacktrace) + + formatted = Exception.format(:error, blamed, stacktrace) + + %{attempt: job.attempt, at: utc_now(), error: formatted} + end +end diff --git a/vendor/oban_pro/lib/oban/pro/engines/smart.ex b/vendor/oban_pro/lib/oban/pro/engines/smart.ex index c16377c3..f014324c 100644 --- a/vendor/oban_pro/lib/oban/pro/engines/smart.ex +++ b/vendor/oban_pro/lib/oban/pro/engines/smart.ex @@ -57,8 +57,8 @@ defmodule Oban.Pro.Engines.Smart do Rate limiting controls the number of jobs that execute within a period of time. Rate limiting uses counts for the same queue from all other nodes in the cluster (with or - without Distributed Erlang). The limiter uses a sliding window over the configured period to - accurately approximate a limit. + without Distributed Erlang). By default, the limiter uses a sliding window over the configured + period to accurately approximate a limit, though other algorithms are available. Every job execution counts toward the rate limit, regardless of whether the job completes, errors, snoozes, etc. @@ -99,6 +99,75 @@ defmodule Oban.Pro.Engines.Smart do > will run 10 jobs concurrently. If that same queue had a `rate_limit` that allowed 5 jobs within > a period, then a single node is limited to 5 jobs. + ### Rate Limiting Algorithms + + Three rate limiting algorithms are available, each with different trade-offs: + + * `:sliding_window` (default) — Uses two time buckets with weighted averaging to provide smooth + rate limiting. Prevents bursting at window boundaries by gradually transitioning between + periods. + + * `:fixed_window` — Resets the count when each period expires. Simple and predictable, but can + allow bursting at window boundaries (e.g., `allowed` jobs at 11:59:59, then `allowed` more at + 12:00:01). + + * `:token_bucket` — Tokens refill continuously at a rate of `allowed / period` per second. + Allows controlled bursting up to `allowed` while maintaining the overall rate over time. Ideal + for APIs that permit short bursts but enforce sustained limits. + + Specify the algorithm with the `:algorithm` option: + + ```elixir + # Use fixed windows for simple, predictable resets + my_queue: [rate_limit: [allowed: 100, period: {1, :minute}, algorithm: :fixed_window]] + + # Use token bucket for APIs that allow bursting + my_queue: [rate_limit: [allowed: 60, period: {1, :minute}, algorithm: :token_bucket]] + ``` + + ### Weighted Jobs + + By default, each job consumes one unit of rate limit capacity. For jobs that vary in resource + usage, you can assign weights so that heavier jobs consume more capacity. + + There are three ways to assign weights, in order of precedence: + + 1. **Callback** — Define a `weight/1` callback for runtime calculation based on job data + 2. **Job option** — Set the weight when creating a specific job + 3. **Worker option** — Set a default weight for all jobs from a worker + + For simple rate adjustments, set a default heavier weight for the worker: + + ```elixir + defmodule MyApp.HeavyWorker do + use Oban.Pro.Worker, rate: [weight: 10] + end + ``` + + The default can be overridden by passing an option to the worker's `new/2` function: + + ```elixir + MyApp.HeavyWorker.new(args, rate: [weight: 5]) + ``` + + For variable rate consumption, you can calculate the weight dynamically with the `weight/1` + callback: + + ```elixir + defmodule MyApp.BatchWorker do + use Oban.Pro.Worker + + @impl Oban.Pro.Worker + def weight(%{args: args}), do: length(args["records"]) + + @impl Oban.Pro.Worker + def process(job), do: # ... + end + ``` + + Jobs without any weight configuration default to a weight of 1. See `Oban.Pro.Worker` for more + details on the `weight/1` callback. + ## Queue Partitioning In addition to global and rate limits at the queue level, you can partition a queue so that it's @@ -323,6 +392,69 @@ defmodule Oban.Pro.Engines.Smart do Oban.insert_all(lots_of_jobs, timeout: :infinity) ``` + ### Skipping Conflicts + + By default, when a unique conflict is detected during bulk insert, the conflicting job is still + returned with `conflict?: true` set. This requires updating the existing row to mark it as a + conflict, which involves row-level locking. + + For high-throughput scenarios where you don't need information about conflicting jobs, you can + use `on_conflict: :skip` to bypass locking entirely: + + ```elixir + Oban.insert_all(lots_of_unique_jobs, on_conflict: :skip) + ``` + + With this option: + + * Conflicting jobs are silently skipped without any database locks + * Only newly inserted jobs are returned in the result list + * Any `replace:` options on individual changesets are ignored + + This is ideal for "fire and forget" scenarios where you want to insert jobs as fast as possible + and don't need to track which jobs already existed. + + ### Automatic Spacing + + When inserting a large batch of jobs that can't all execute immediately, you can automatically + space them out over time using the `auto_space` option. This schedules each batch at increasing + intervals, preventing a flood of jobs from overwhelming your queue. + + ```elixir + Oban.insert_all(lots_of_jobs, batch_size: 1000, auto_space: 60) + ``` + + With `batch_size: 1000` and `auto_space: 60`, jobs are scheduled as follows: + + * Jobs 1–1000: scheduled immediately + * Jobs 1001–2000: scheduled 60 seconds from now + * Jobs 2001–3000: scheduled 120 seconds from now + * And so on... + + The `auto_space` value can be an integer (seconds) or a duration tuple: + + ```elixir + auto_space: 30 # 30 seconds between batches + auto_space: {1, :minute} # 1 minute between batches + auto_space: {5, :minutes} # 5 minutes between batches + ``` + + Note that `auto_space` overrides any `scheduled_at` values set on individual changesets. + + ### Per-Batch Transactions + + By default, all batches are inserted within a single transaction. If any batch fails, all + previously inserted jobs are rolled back. For scenarios where you'd rather commit successful + batches even if a later batch fails, use `transaction: :per_batch`: + + ```elixir + Oban.insert_all(lots_of_jobs, batch_size: 1000, transaction: :per_batch) + ``` + + With this option, each batch is committed independently. If batch 3 fails, jobs from batches 1 + and 2 remain in the database. Note that on failure, the function still raises an exception and + does not return references to the successfully inserted jobs. + ## Accurate Snooze Unlike the `Basic` engine which increments `attempts` and `max_attempts`, the Smart engine rolls @@ -353,9 +485,10 @@ defmodule Oban.Pro.Engines.Smart do alias Ecto.{Changeset, Multi} alias Oban.{Backoff, Config, Engine, Job, Repo} - alias Oban.Pro.Limiters.{Global, Local, Rate} alias Oban.Pro.{Flusher, Handler, Partition, Producer, Unique, Utils} + alias Oban.Pro.Limiters.{Global, Local, Rate} alias Oban.Pro.Stages.Chain + alias Oban.Pro.Workflow.Schema, as: WorkflowSchema require Logger @@ -402,6 +535,20 @@ defmodule Oban.Pro.Engines.Smart do end end + defmacrop merge_meta_array(meta, key) do + query = """ + jsonb_set(?, '{#{key}}', + (SELECT jsonb_agg(DISTINCT v) + FROM jsonb_array_elements_text( + COALESCE(?->'#{key}', '[]'::jsonb) || COALESCE(EXCLUDED.meta->'#{key}', '[]'::jsonb) + ) AS v)) + """ + + quote do + fragment(unquote(query), unquote(meta), unquote(meta)) + end + end + defmacrop xact_lock(pref_key, lock_key) do quote do fragment("pg_try_advisory_xact_lock(?::int, ?::int)", unquote(pref_key), unquote(lock_key)) @@ -578,15 +725,8 @@ defmodule Oban.Pro.Engines.Smart do |> Map.merge(meta) end - defp flatten_windows(%{rate_limit: %{windows: windows}} = meta) do - {pacc, cacc} = - Enum.reduce(windows, {0, 0}, fn {_key, map}, {pacc, cacc} -> - %{"prev_count" => prev, "curr_count" => curr} = map - - {pacc + prev, cacc + curr} - end) - - put_in(meta.rate_limit.windows, [%{"curr_count" => cacc, "prev_count" => pacc}]) + defp flatten_windows(%{rate_limit: %{windows: _}} = meta) do + put_in(meta.rate_limit.windows, Rate.flatten(meta)) end defp flatten_windows(meta), do: meta @@ -613,9 +753,7 @@ defmodule Oban.Pro.Engines.Smart do |> Multi.run(:all_producers, &all_producers/2) |> Multi.run(:ack_ids, &ack_jobs/2) |> Multi.run(:afh_ids, &run_flush_handlers/2) - |> Multi.run(:local_demand, &Local.check/2) - |> Multi.run(:global_demand, &Global.check/2) - |> Multi.run(:rate_demand, &Rate.check/2) + |> Multi.run(:demand, &check_demand/2) |> Multi.run(:jobs, &fetch_jobs/2) |> Multi.run(:tracked, &track_jobs(&1, &2, running)) @@ -758,7 +896,7 @@ defmodule Oban.Pro.Engines.Smart do def complete_job(%Config{} = conf, %Job{} = job) do set = [state: "completed", completed_at: utc_now()] - case Process.get(:oban_recorded) do + case Process.get(:oban_meta_update) do nil -> put_ack(conf, %{job | state: "completed"}, set) map -> put_ack(conf, %{job | state: "completed"}, set ++ [meta: map]) end @@ -802,11 +940,19 @@ defmodule Oban.Pro.Engines.Smart do snoozed = Map.get(job.meta, "snoozed", 0) orig_at = Map.get(job.meta, "orig_scheduled_at", to_unix(job.scheduled_at)) + meta = %{orig_scheduled_at: orig_at, snoozed: snoozed + 1} + + meta = + case Process.get(:oban_meta_update) do + nil -> meta + map -> Map.merge(meta, map) + end + put_ack(conf, %{job | state: "scheduled"}, attempt_change: -1, state: "scheduled", scheduled_at: seconds_from_now(seconds), - meta: %{orig_scheduled_at: orig_at, snoozed: snoozed + 1} + meta: meta ) :ok @@ -835,7 +981,7 @@ defmodule Oban.Pro.Engines.Smart do Job |> join(:inner, [j], x in subquery(subquery), on: j.id == x.id) |> update(set: [state: "cancelled", cancelled_at: ^utc_now()]) - |> select([_, x], map(x, [:id, :args, :attempted_by, :meta, :queue, :state, :worker])) + |> select([_, x], map(x, [:id, :attempted_by, :meta, :queue, :state, :worker])) {:ok, %{jobs: {_count, jobs}}} = Multi.new() @@ -849,13 +995,17 @@ defmodule Oban.Pro.Engines.Smart do @doc false # This is used to unblock a uniqueness violation by clearing out the uniq_bmp field. The - # generated `uniq_key` column uses the `uniq_key` from meta, but is only present when the job's - # state maps to a value in `uniq_bmp`. + # The unique constraint (either column-based or expression-based) uses `uniq_key` from meta, + # but is only enforced when the job's state maps to a value in `uniq_bmp`. # # We find conflicting jobs by matching the unique key in meta and filtering by the expected # states. Clearing their `uniq_bmp` disables the unique constraint, allowing the transition. def clear_uniq_violation(%Config{} = conf, detail, states \\ ["executing"]) do - with %{"key" => key} <- Regex.named_captures(~r/\(uniq_key\)=\((?.+)\)/, detail) do + # Match both column-based `(uniq_key)=(value)` and expression-based `((meta->>'uniq_key'))=(value)` + # formats. The exact spacing is critical. + pattern = ~r/\((?:uniq_key|\(meta ->> 'uniq_key'::text\))\)=\((?.+)\)/ + + with %{"key" => key} <- Regex.named_captures(pattern, detail) do query = Job |> where([j], j.state in ^states) @@ -872,7 +1022,7 @@ defmodule Oban.Pro.Engines.Smart do log_key = {__MODULE__, :uniq_violation_logged, key} - unless :persistent_term.get(log_key, false) do + if !:persistent_term.get(log_key, false) do :persistent_term.put(log_key, true) Logger.info(fn -> @@ -908,21 +1058,28 @@ defmodule Oban.Pro.Engines.Smart do @impl Engine def insert_all_jobs(conf, changesets, opts) do - fun = fn -> insert_all_batches(conf, changesets, opts) end + if opts[:transaction] == :per_batch do + insert_all_batches(conf, changesets, opts) + else + fun = fn -> insert_all_batches(conf, changesets, opts) end - {:ok, jobs} = Repo.transaction(conf, fun, opts) + {:ok, jobs} = Repo.transaction(conf, fun, opts) - jobs + jobs + end end defp insert_all_batches(conf, changesets, opts) do batch_size = Keyword.get(opts, :batch_size, @base_batch_size) + auto_space = Keyword.get(opts, :auto_space) changesets |> Stream.map(&Unique.with_uniq_meta/1) |> Stream.map(&Partition.with_partition_meta(&1, conf)) |> Stream.uniq_by(&Unique.get_key(&1, System.unique_integer())) |> Stream.chunk_every(batch_size) + |> Stream.with_index() + |> Stream.map(&apply_auto_space(&1, auto_space)) |> Enum.flat_map(fn changesets -> {uniq_map, link_map, replace?} = Enum.reduce(changesets, {%{}, %{}, false}, fn changeset, {uniq_map, link_map, replace?} -> @@ -955,7 +1112,8 @@ defmodule Oban.Pro.Engines.Smart do |> Multi.put(:uniq_map, uniq_map) |> Multi.put(:link_map, link_map) |> Multi.put(:replacements?, replace?) - |> Multi.run(:uniq_index?, &uniq_index?/2) + |> Multi.run(:uniq_mode, &uniq_mode/2) + |> Multi.run(:workflow_conflicts, &insert_workflows/2) |> Multi.run(:chain_changesets, &prepare_chains/2) |> Multi.run(:dupe_map, &find_dupes/2) |> Multi.run(:new_jobs, &insert_entries/2) @@ -967,20 +1125,134 @@ defmodule Oban.Pro.Engines.Smart do end) end - defp uniq_index?(_repo, %{conf: conf}) do + defp apply_auto_space({changesets, _index}, nil), do: changesets + + defp apply_auto_space({changesets, batch_index}, auto_space) do + space_seconds = Utils.cast_period(auto_space) + scheduled_at = DateTime.add(utc_now(), batch_index * space_seconds, :second) + + Enum.map(changesets, &Changeset.force_change(&1, :scheduled_at, scheduled_at)) + end + + defp uniq_mode(_repo, %{conf: conf}) do %{name: name, prefix: prefix} = conf - Utils.persistent_cache({__MODULE__, :uniq_index?, name}, fn -> + Utils.persistent_cache({__MODULE__, :uniq_mode, name}, fn -> query = - from("columns") - |> put_query_prefix("information_schema") - |> where(table_schema: ^prefix, table_name: "oban_jobs", column_name: "uniq_key") - |> select(true) + from("pg_indexes") + |> put_query_prefix("pg_catalog") + |> where(schemaname: ^prefix, tablename: "oban_jobs", indexname: "oban_jobs_unique_index") + |> where([i], fragment("? LIKE 'CREATE UNIQUE%'", i.indexdef)) + |> select([i], i.indexdef) - {:ok, Repo.one(conf, query) == true} + mode = + case Repo.one(conf, query) do + nil -> :none + indexdef -> if String.contains?(indexdef, "(uniq_key)"), do: :column, else: :expression + end + + {:ok, mode} end) end + defp insert_workflows(_repo, %{changesets: changesets, conf: conf}) do + {unique_workflows, regular_workflows} = + changesets + |> Enum.filter(&workflow_changeset?/1) + |> Enum.group_by(&get_workflow_id/1) + |> Enum.map(&WorkflowSchema.from_changesets/1) + |> Enum.split_with(&Changeset.get_field(&1, :meta)[:unique]) + + {existing_ids, taken_names} = lock_workflows(unique_workflows, conf) + + {old_unique, new_unique} = + Enum.split_with( + unique_workflows, + &MapSet.member?(existing_ids, Changeset.get_field(&1, :id)) + ) + + {conflicting, insertable} = + Enum.split_with(new_unique, fn changeset -> + name = Changeset.get_field(changeset, :name) + name != nil and MapSet.member?(taken_names, name) + end) + + Enum.each(regular_workflows ++ old_unique ++ insertable, &upsert_workflow(conf, &1)) + + conflicting_ids = + conflicting + |> Enum.map(&Changeset.get_field(&1, :id)) + |> MapSet.new() + + {:ok, conflicting_ids} + end + + defp lock_workflows([], _conf), do: {MapSet.new(), MapSet.new()} + + defp lock_workflows(workflows, conf) do + ids = Enum.map(workflows, &Changeset.get_field(&1, :id)) + + names = + workflows + |> Enum.map(&Changeset.get_field(&1, :name)) + |> Enum.reject(&is_nil/1) + + query = + WorkflowSchema + |> where([w], w.id in ^ids) + |> or_where( + [w], + w.name in ^names and w.state != "completed" and fragment("meta \\? 'unique'") + ) + |> select([w], {w.id, w.name}) + |> lock("FOR UPDATE") + + existing = Repo.all(conf, query) + + existing_ids = existing |> Enum.map(&elem(&1, 0)) |> MapSet.new() + + taken_names = + existing + |> Enum.map(&elem(&1, 1)) + |> Enum.reject(&is_nil/1) + |> MapSet.new() + + {existing_ids, taken_names} + end + + defp workflow_changeset?(changeset) do + changeset + |> Changeset.get_field(:meta, %{}) + |> is_map_key(:workflow_id) + end + + defp get_workflow_id(changeset) do + changeset + |> Changeset.get_field(:meta, %{}) + |> Map.get(:workflow_id) + end + + defp upsert_workflow(conf, changeset) do + on_conflict = + from(wf in WorkflowSchema, + update: [ + set: [ + suspended: fragment("? + EXCLUDED.suspended", wf.suspended), + available: fragment("? + EXCLUDED.available", wf.available), + scheduled: fragment("? + EXCLUDED.scheduled", wf.scheduled), + executing: fragment("? + EXCLUDED.executing", wf.executing), + retryable: fragment("? + EXCLUDED.retryable", wf.retryable), + completed: fragment("? + EXCLUDED.completed", wf.completed), + cancelled: fragment("? + EXCLUDED.cancelled", wf.cancelled), + discarded: fragment("? + EXCLUDED.discarded", wf.discarded), + meta: merge_meta_array(merge_meta_array(wf.meta, "queues"), "workers") + ] + ] + ) + + Repo.insert(conf, changeset, conflict_target: :id, on_conflict: on_conflict) + end + defp prepare_chains(_repo, %{link_map: link_map, conf: conf}) when map_size(link_map) > 0 do chain_ids = Map.keys(link_map) @@ -1004,7 +1276,6 @@ defmodule Oban.Pro.Engines.Smart do |> where([j], j.state != "completed") |> where([j], fragment("? \\? 'chain_id'", j.meta)) |> where([j], fragment("?->>'chain_id'", j.meta) == parent_as(:list).value) - |> where([j], fragment("?->>'on_hold'", j.meta) in ~w(true false)) |> order_by(desc: :id) |> lock("FOR UPDATE") |> limit(1) @@ -1030,7 +1301,7 @@ defmodule Oban.Pro.Engines.Smart do cond do index == 0 and is_nil(state) -> changeset index == 0 and Chain.continuable?(state, meta) -> changeset - true -> Chain.to_postponed(changeset) + true -> Chain.to_suspended(changeset) end end) end) @@ -1049,10 +1320,7 @@ defmodule Oban.Pro.Engines.Smart do @impl Engine def retry_all_jobs(%Config{} = conf, queryable) do - subquery = - queryable - |> where([j], j.state not in ["available", "executing"]) - |> where([j], not fragment("? @> ?", j.meta, ^%{on_hold: true})) + subquery = where(queryable, [j], j.state not in ~w(available executing suspended)) query = Job @@ -1142,11 +1410,11 @@ defmodule Oban.Pro.Engines.Smart do cond do Process.get(:oban_draining) -> - Process.delete(:oban_recorded) + Process.delete(:oban_meta_update) {:ok, jids} = ack_jobs([ack_entry], conf) - run_flush_handlers([ack_entry]) + run_flush_handlers([ack_entry], conf) if producer.ack_tab, do: del_acks(jids, producer) @@ -1175,21 +1443,23 @@ defmodule Oban.Pro.Engines.Smart do {:ok, jids} = ack_jobs(acks, conf) - run_flush_handlers(acks) - + run_flush_handlers(acks, conf) del_acks(jids, producer) - track_acks(acks, producer) end - defp run_flush_handlers(_repo, %{acks: acks}) do - {:ok, run_flush_handlers(acks)} + defp run_flush_handlers(_repo, %{acks: acks, conf: conf}) do + {:ok, run_flush_handlers(acks, conf)} end - defp run_flush_handlers(acks) do + defp run_flush_handlers(acks, conf) do mfas = for {_, _, all, _} <- acks, mfa <- all, is_tuple(mfa), uniq: true, do: mfa - Enum.each(mfas, fn {mod, fun, arg} -> apply(mod, fun, arg) end) + span(:flush, conf, %{count: length(mfas)}, fn -> + Enum.each(mfas, fn {mod, fun, arg} -> apply(mod, fun, arg) end) + + {:ok, length(mfas)} + end) end # Fetch Helpers @@ -1215,6 +1485,16 @@ defmodule Oban.Pro.Engines.Smart do end end + defp check_demand(_repo, %{conf: conf, producer: producer} = changes) do + span(:demand, conf, %{queue: producer.queue}, fn -> + {:ok, local} = Local.check(changes) + {:ok, global} = Global.check(changes) + {:ok, rate} = Rate.check(changes) + + {:ok, %{local: local, global: global, rate: rate}} + end) + end + defp any_flush_handlers?(tab) do match = {:_, :_, :"$1", :_} guard = [{:"/=", :"$1", []}] @@ -1244,44 +1524,50 @@ defmodule Oban.Pro.Engines.Smart do end defp fetch_jobs(_repo, %{conf: conf, producer: producer} = changes) do - subset_query = fetch_subquery(changes) + span(:fetch, conf, %{queue: producer.queue}, fn -> + subset_query = fetch_subquery(changes) - query = - Job - |> with_cte("subset", as: ^subset_query) - |> join(:inner, [j], x in fragment(~s("subset")), on: true) - |> where([j, x], j.id == x.id and j.state == "available" and j.attempt < j.max_attempts) - |> select([j, _], j) + query = + Job + |> with_cte("subset", as: ^subset_query) + |> join(:inner, [j], x in fragment(~s("subset")), on: true) + |> where([j, x], j.id == x.id and j.state == "available" and j.attempt < j.max_attempts) + |> select([j, _], j) - updates = [ - set: [state: "executing", attempted_at: utc_now(), attempted_by: [conf.node, producer.uuid]], - inc: [attempt: 1] - ] + updates = [ + set: [ + state: "executing", + attempted_at: utc_now(), + attempted_by: [conf.node, producer.uuid] + ], + inc: [attempt: 1] + ] - {_count, jobs} = Repo.update_all(conf, query, updates) + {_count, jobs} = Repo.update_all(conf, query, updates) - {:ok, jobs} + {:ok, jobs} + end) end - defp fetch_subquery(%{local_demand: local, producer: producer} = changes) do - case changes do - %{global_demand: nil, rate_demand: nil} -> + defp fetch_subquery(%{demand: %{local: local, global: global, rate: rate}, producer: producer}) do + case {global, rate} do + {nil, nil} -> fetch_subquery(producer, local) - %{global_demand: global, rate_demand: nil} when is_integer(global) -> + {global, nil} when is_integer(global) -> fetch_subquery(producer, min(local, global)) - %{global_demand: nil, rate_demand: rated} when is_integer(rated) -> - fetch_subquery(producer, min(local, rated)) + {nil, rate} when is_integer(rate) -> + fetch_subquery(producer, min(local, rate)) - %{global_demand: global, rate_demand: %{} = demands} -> + {global, %{} = demands} -> fetch_subquery(producer, demands, min(local, global || local)) - %{global_demand: %{} = demands, rate_demand: rated} -> - fetch_subquery(producer, demands, min(local, rated || local)) + {%{} = demands, rate} -> + fetch_subquery(producer, demands, min(local, rate || local)) - %{global_demand: global, rate_demand: rated} -> - limit = rated |> min(local) |> min(global) + {global, rate} -> + limit = rate |> min(local) |> min(global) fetch_subquery(producer, limit) end @@ -1308,7 +1594,8 @@ defmodule Oban.Pro.Engines.Smart do Job |> select([:id, :priority, :scheduled_at]) |> where([j], j.state == "available" and j.queue == ^producer.queue) - |> where([j], fragment("partition_key") == parent_as(:part).key) + |> where([j], fragment("meta \\? 'partition_key'")) + |> where([j], fragment("meta->>'partition_key'") == parent_as(:part).key) |> order_by([:priority, :scheduled_at, :id]) |> limit(parent_as(:part).lim) ), @@ -1327,27 +1614,34 @@ defmodule Oban.Pro.Engines.Smart do @ack_fields ~w(state cancelled_at completed_at discarded_at scheduled_at attempt_change error meta)a - defp ack_jobs(_repo, %{acks: acks, conf: conf}), do: ack_jobs(acks, conf) - defp ack_jobs([], _conf), do: {:ok, []} + @doc false + # This is public for sharing with chunk.ex, which needs to ack chunk jobs _without_ tracking + # them as normally executed jobs. The initial caluse will emit `fetch_jobs` telemetry, and is + # purposefully limited to calls in the fetch multi. + def ack_jobs(_repo, %{acks: acks, conf: conf}) do + span(:ack, conf, %{count: length(acks)}, fn -> ack_jobs(acks, conf) end) + end - defp ack_jobs(acks, conf) do + def ack_jobs([], _conf), do: {:ok, []} + + def ack_jobs(acks, conf) do [ids | params] = acks |> Enum.map(fn {{:ack, _, _, id}, _, _, set} -> [id | Enum.map(@ack_fields, &set[&1])] end) |> Enum.zip_with(&Function.identity/1) - case Repo.query(conf, ack_query(conf), [ids | params]) do + case Repo.query(conf, ack_query(conf.prefix), [ids | params]) do {:ok, %{rows: rows}} -> {:ok, List.flatten(rows)} error -> error end end - defp ack_query(conf) do - Utils.persistent_cache({__MODULE__, :ack_query, conf.prefix}, fn -> + defp ack_query(prefix) do + Utils.persistent_cache({__MODULE__, :ack_query, prefix}, fn -> """ WITH params AS ( SELECT unnest($1::bigint[]) AS id, - unnest($2::"#{conf.prefix}".oban_job_state[]) AS state, + unnest($2::"#{prefix}".oban_job_state[]) AS state, unnest($3::timestamp[]) AS cancelled_at, unnest($4::timestamp[]) AS completed_at, unnest($5::timestamp[]) AS discarded_at, @@ -1358,16 +1652,20 @@ defmodule Oban.Pro.Engines.Smart do ), locked AS ( SELECT oj.id - FROM "#{conf.prefix}"."oban_jobs" oj + FROM "#{prefix}"."oban_jobs" oj INNER JOIN params tmp ON oj.id = tmp.id FOR UPDATE OF oj ) - UPDATE "#{conf.prefix}"."oban_jobs" oj + UPDATE "#{prefix}"."oban_jobs" oj SET state = tmp.state, cancelled_at = COALESCE(tmp.cancelled_at, oj.cancelled_at), completed_at = COALESCE(tmp.completed_at, oj.completed_at), discarded_at = COALESCE(tmp.discarded_at, oj.discarded_at), - scheduled_at = COALESCE(tmp.scheduled_at, oj.scheduled_at), + scheduled_at = CASE + WHEN (tmp.meta ? 'wait_until') AND (oj.meta ? 'signal') + THEN oj.scheduled_at + ELSE COALESCE(tmp.scheduled_at, oj.scheduled_at) + END, attempt = COALESCE(tmp.attempt_change + oj.attempt, oj.attempt), errors = CASE WHEN tmp.error IS NULL THEN oj.errors ELSE oj.errors || tmp.error END, meta = CASE WHEN tmp.meta IS NULL THEN oj.meta ELSE oj.meta || tmp.meta END @@ -1443,7 +1741,7 @@ defmodule Oban.Pro.Engines.Smart do # Insert Helpers - defp find_dupes(_repo, %{conf: conf, uniq_index?: false, uniq_map: uniq_map}) + defp find_dupes(_repo, %{conf: conf, uniq_mode: :none, uniq_map: uniq_map}) when map_size(uniq_map) > 0 do {uniq_keys, uniq_states} = Enum.reduce(uniq_map, {[], []}, fn {key, changeset}, {key_acc, sta_acc} -> @@ -1481,7 +1779,7 @@ defmodule Oban.Pro.Engines.Smart do defp insert_entries(_repo, changes) do %{conf: conf, changesets: changesets, chain_changesets: chains} = changes - %{dupe_map: dupe_map, opts: opts} = changes + %{dupe_map: dupe_map, opts: opts, workflow_conflicts: workflow_conflicts} = changes changesets = if chains == [] do @@ -1493,6 +1791,7 @@ defmodule Oban.Pro.Engines.Smart do {entries, placeholders} = for changeset <- changesets, not is_map_key(dupe_map, Unique.get_key(changeset)), + not workflow_conflicted?(changeset, workflow_conflicts), reduce: {[], nil} do {entries, acc} -> map = Job.to_map(changeset) @@ -1528,42 +1827,65 @@ defmodule Oban.Pro.Engines.Smart do {:ok, jobs} end - defp conflict_opts(%{uniq_map: uniq_map, uniq_index?: uniq_index?}) do - if uniq_map == %{} or not uniq_index? do - [] - else - [ - conflict_target: {:unsafe_fragment, "(uniq_key) WHERE uniq_key IS NOT NULL"}, - on_conflict: update(Job, [j], set: [meta: merge_jsonb(j.meta, ^%{uniq_conflict: true})]) - ] + defp conflict_opts(%{uniq_map: uniq_map, uniq_mode: mode, opts: opts, conf: conf}) do + cond do + uniq_map == %{} or mode == :none -> + [on_conflict: :nothing] + + opts[:on_conflict] == :skip -> + [ + conflict_target: {:unsafe_fragment, conflict_target(mode, conf)}, + on_conflict: :nothing + ] + + true -> + [ + conflict_target: {:unsafe_fragment, conflict_target(mode, conf)}, + on_conflict: update(Job, [j], set: [meta: merge_jsonb(j.meta, ^%{uniq_conflict: true})]) + ] end end - defp apply_replacements(_repo, %{replacements?: false}), do: {:ok, []} + defp conflict_target(:column, _conf), do: "(uniq_key) WHERE uniq_key IS NOT NULL" - defp apply_replacements(_repo, %{conf: conf, opts: opts} = changes) do - updates = - for {_key, {id, state, %{changes: %{replace: replace} = changes}}} <- dupe_map(changes), - reduce: %{} do - acc -> - state = String.to_existing_atom(state) - rep_keys = Keyword.get(replace, state, []) - - changes - |> Map.take(rep_keys) - |> Enum.reduce(acc, fn {key, val}, sub_acc -> - Map.update(sub_acc, {key, val}, [id], &[id | &1]) - end) - end - - Enum.each(updates, fn {val, ids} -> - Repo.update_all(conf, where(Job, [j], j.id in ^ids), [set: [val]], opts) - end) - - {:ok, []} + defp conflict_target(:expression, conf) do + """ + ((meta->>'uniq_key')) WHERE meta ? 'uniq_key' AND jsonb_contains(meta->'uniq_bmp', #{conf.prefix}.oban_state_to_bit(state)) + """ end - defp dupe_map(%{dupe_map: dupe_map, uniq_index?: false}), do: dupe_map + defp apply_replacements(_repo, %{conf: conf, opts: opts} = changes) do + cond do + not changes.replacements? -> + {:ok, []} + + changes.opts[:on_conflict] == :skip -> + {:ok, []} + + true -> + updates = + for {_key, {id, state, %{changes: %{replace: replace} = changes}}} <- dupe_map(changes), + reduce: %{} do + acc -> + state = String.to_existing_atom(state) + rep_keys = Keyword.get(replace, state, []) + + changes + |> Map.take(rep_keys) + |> Enum.reduce(acc, fn {key, val}, sub_acc -> + Map.update(sub_acc, {key, val}, [id], &[id | &1]) + end) + end + + Enum.each(updates, fn {val, ids} -> + Repo.update_all(conf, where(Job, [j], j.id in ^ids), [set: [val]], opts) + end) + + {:ok, []} + end + end + + defp dupe_map(%{dupe_map: dupe_map, uniq_mode: :none}), do: dupe_map defp dupe_map(%{new_jobs: new_jobs, uniq_map: uniq_map}) do Enum.reduce(new_jobs, %{}, fn job, acc -> @@ -1577,7 +1899,9 @@ defmodule Oban.Pro.Engines.Smart do end) end - defp apply_conflicts(_repo, %{uniq_index?: false, dupe_map: dupe_map, new_jobs: new_jobs}) do + defp apply_conflicts(_repo, %{uniq_mode: :none} = changes) do + %{dupe_map: dupe_map, new_jobs: new_jobs} = changes + old_jobs = dupe_map |> Map.values() @@ -1587,17 +1911,52 @@ defmodule Oban.Pro.Engines.Smart do |> Map.replace!(:conflict?, true) end) - {:ok, old_jobs ++ new_jobs} + workflow_conflict_jobs = apply_workflow_conflicts(changes) + + {:ok, old_jobs ++ workflow_conflict_jobs ++ new_jobs} end - defp apply_conflicts(_repo, %{new_jobs: new_jobs}) do + defp apply_conflicts(_repo, changes) do + %{new_jobs: new_jobs} = changes + new_jobs = Enum.map(new_jobs, fn %{meta: %{"uniq_conflict" => true}} = job -> %{job | conflict?: true} job -> job end) - {:ok, new_jobs} + workflow_conflict_jobs = apply_workflow_conflicts(changes) + + {:ok, workflow_conflict_jobs ++ new_jobs} + end + + defp apply_workflow_conflicts(%{changesets: changesets, workflow_conflicts: conflicts}) do + changesets + |> Enum.filter(&workflow_conflicted?(&1, conflicts)) + |> Enum.map(fn changeset -> + changeset + |> Changeset.apply_action!(:insert) + |> Map.replace!(:conflict?, true) + end) + end + + defp workflow_conflicted?(changeset, workflow_conflicts) do + workflow_id = get_workflow_id(changeset) + + workflow_id != nil and MapSet.member?(workflow_conflicts, workflow_id) + end + + # Telemetry Helpers + + defp span(event, conf, meta, fun) do + meta = Map.put(meta, :conf, conf) + + :telemetry.span([:oban, :engine, :fetch_jobs, event], meta, fn -> + case fun.() do + {:ok, result} -> {{:ok, result}, Map.put(meta, :result, result)} + {:error, _} = error -> {error, meta} + end + end) end # Time Helpers diff --git a/vendor/oban_pro/lib/oban/pro/limiter.ex b/vendor/oban_pro/lib/oban/pro/limiter.ex index 35e20e30..8d2bfeda 100644 --- a/vendor/oban_pro/lib/oban/pro/limiter.ex +++ b/vendor/oban_pro/lib/oban/pro/limiter.ex @@ -1,8 +1,8 @@ defmodule Oban.Pro.Limiter do @moduledoc false - alias Oban.Pro.Producer alias Oban.{Config, Job} + alias Oban.Pro.Producer @type changes :: %{ :conf => Config.t(), @@ -14,12 +14,11 @@ defmodule Oban.Pro.Limiter do @type demand :: non_neg_integer() @type limit :: nil | demand | [{demand, term(), term()}] - @type repo :: Ecto.Repo.t() @doc """ Calculate the current demand based on capacity and usage. """ - @callback check(repo(), changes()) :: {:ok, limit()} + @callback check(changes()) :: {:ok, limit()} @doc """ Record demand usage. diff --git a/vendor/oban_pro/lib/oban/pro/limiters/global.ex b/vendor/oban_pro/lib/oban/pro/limiters/global.ex index 43fb7e10..c1e14551 100644 --- a/vendor/oban_pro/lib/oban/pro/limiters/global.ex +++ b/vendor/oban_pro/lib/oban/pro/limiters/global.ex @@ -6,7 +6,7 @@ defmodule Oban.Pro.Limiters.Global do alias Oban.Pro.Partition @impl Oban.Pro.Limiter - def check(_repo, %{producer: producer} = changes) when is_map(producer.meta.global_limit) do + def check(%{producer: producer} = changes) when is_map(producer.meta.global_limit) do tracked = [producer | changes.all_producers] |> Enum.uniq_by(& &1.uuid) @@ -24,7 +24,7 @@ defmodule Oban.Pro.Limiters.Global do end end - def check(_repo, _changes), do: {:ok, nil} + def check(_changes), do: {:ok, nil} defp merge(global_limit, acc) do Map.merge(global_limit.tracked, acc, fn _, v1, v2 -> v1 + v2 end) diff --git a/vendor/oban_pro/lib/oban/pro/limiters/local.ex b/vendor/oban_pro/lib/oban/pro/limiters/local.ex index a9b498d0..b504e7d0 100644 --- a/vendor/oban_pro/lib/oban/pro/limiters/local.ex +++ b/vendor/oban_pro/lib/oban/pro/limiters/local.ex @@ -4,7 +4,7 @@ defmodule Oban.Pro.Limiters.Local do @behaviour Oban.Pro.Limiter @impl Oban.Pro.Limiter - def check(_repo, %{producer: producer, running: running}) do + def check(%{producer: producer, running: running}) do {:ok, producer.meta.local_limit - map_size(running)} end diff --git a/vendor/oban_pro/lib/oban/pro/limiters/rate.ex b/vendor/oban_pro/lib/oban/pro/limiters/rate.ex index ab6c132f..0a61c2b8 100644 --- a/vendor/oban_pro/lib/oban/pro/limiters/rate.ex +++ b/vendor/oban_pro/lib/oban/pro/limiters/rate.ex @@ -3,120 +3,93 @@ defmodule Oban.Pro.Limiters.Rate do @behaviour Oban.Pro.Limiter + alias Oban.Pro.Limiters.Rate.Algorithm alias Oban.Pro.Partition @impl Oban.Pro.Limiter - def check(_repo, %{producer: producer} = changes) when is_map(producer.meta.rate_limit) do - %{allowed: allowed, period: period, window_time: prev_time} = producer.meta.rate_limit + def check(%{producer: producer} = changes) when is_map(producer.meta.rate_limit) do + %{window_time: prev_time} = producer.meta.rate_limit curr_time = unix_now() - weight = div(max(period - curr_time - prev_time, 0), period) windows = [producer | changes.all_producers] |> Enum.uniq_by(& &1.uuid) |> Enum.map(& &1.meta.rate_limit) |> Enum.filter(&(is_map(&1) and &1.window_time >= curr_time - &1.period)) - |> Enum.reduce(%{}, &merge/2) + |> Enum.reduce(%{}, &merge_windows/2) windows = update_legacy_windows(windows) - if is_nil(producer.meta.rate_limit.partition) do - {:ok, combined_demands(weight, allowed, windows)} + demand = calculate_demand(producer.meta.rate_limit, windows, curr_time, prev_time, changes) + + {:ok, demand} + end + + def check(_changes), do: {:ok, nil} + + defp calculate_demand(rate_limit, windows, curr_time, prev_time, changes) do + %{allowed: allowed, period: period} = rate_limit + + algorithm = Algorithm.callback(rate_limit) + + if is_nil(rate_limit.partition) do + algorithm.demand(windows, allowed, period, curr_time, prev_time) else - {:ok, separate_demands(changes.conf, weight, allowed, producer, windows)} + producer = changes.producer + + available_keys = + Partition.available_keys(changes.conf, producer.queue, producer.meta.local_limit) + + algorithm.partition_demands( + windows, + allowed, + period, + curr_time, + prev_time, + available_keys + ) end end - def check(_repo, _changes), do: {:ok, nil} - - defp merge(rate_limit, acc) do - Map.merge(rate_limit.windows, acc, fn _key, v1, v2 -> - %{"curr_count" => c1, "prev_count" => p1} = v1 - %{"curr_count" => c2, "prev_count" => p2} = v2 - %{"curr_count" => c1 + c2, "prev_count" => p1 + p2} - end) + defp merge_windows(rate_limit, acc) do + Algorithm.callback(rate_limit).merge(rate_limit.windows, acc) end - defp update_legacy_windows(%{"args" => _} = window), do: %{"none" => window} + defp update_legacy_windows(%{"args" => _} = window), do: %{"*" => window} defp update_legacy_windows(windows), do: windows - defp combined_demands(weight, allowed, windows) do - total = - Enum.reduce(windows, 0, fn {_, val}, acc -> - acc + (val["prev_count"] * weight + val["curr_count"]) - end) - - max(allowed - total, 0) - end - - defp separate_demands(conf, weight, allowed, producer, windows) do - untracked = - for key <- Partition.available_keys(conf, producer.queue, producer.meta.local_limit), - not is_map_key(windows, key), - into: %{}, - do: {key, allowed} - - Enum.reduce(windows, untracked, fn {key, val}, acc -> - %{"curr_count" => curr, "prev_count" => prev} = val - - case max(allowed - (prev * weight + curr), 0) do - demand when demand > 0 -> Map.put(acc, key, demand) - _ -> acc - end - end) + def flatten(%{rate_limit: rate_limit}) do + Algorithm.callback(rate_limit).flatten(rate_limit.windows) end @impl Oban.Pro.Limiter def track(%{rate_limit: rate_limit} = meta, jobs) when is_map(rate_limit) do - new_mapping = - Enum.reduce(jobs, %{}, fn job, acc -> - key = Partition.get_key(job, "*") - map = %{"curr_count" => 1, "prev_count" => 0} + new_counts = build_counts(jobs) + curr_time = unix_now() - Map.update(acc, key, map, &%{&1 | "curr_count" => &1["curr_count"] + 1}) - end) - - new_windows = - for {key, val} <- new_mapping, - not is_map_key(rate_limit.windows, key), - into: %{}, - do: {key, val} - - prev_unix = rate_limit.window_time - unix_diff = unix_now() - prev_unix - - {mode, next_time} = - cond do - unix_diff > rate_limit.period * 2 -> {:dump_prev, unix_now()} - unix_diff > rate_limit.period -> {:swap_prev, unix_now()} - true -> {:bump_curr, prev_unix} - end - - all_windows = - for {key, window} <- rate_limit.windows, - not match?(%{"curr_count" => 0, "prev_count" => 0}, window), - into: new_windows, - do: {key, put_next_count(key, window, mode, new_mapping)} + {all_windows, next_time} = + Algorithm.callback(rate_limit).track( + rate_limit.windows, + new_counts, + rate_limit.period, + rate_limit.window_time, + curr_time, + rate_limit.allowed + ) %{meta | rate_limit: %{rate_limit | windows: all_windows, window_time: next_time}} end def track(meta, _jobs), do: meta - defp put_next_count(key, window, mode, mapping) do - %{"curr_count" => curr_count, "prev_count" => prev_count} = window + defp build_counts(jobs) do + Enum.reduce(jobs, %{}, fn job, acc -> + key = Partition.get_key(job, "*") + weight = Oban.Pro.Worker.get_weight(job) - next_count = get_in(mapping, [key, "curr_count"]) || 0 - - {curr_count, prev_count} = - case mode do - :dump_prev -> {next_count, 0} - :swap_prev -> {next_count, curr_count} - :bump_curr -> {curr_count + next_count, prev_count} - end - - %{window | "curr_count" => curr_count, "prev_count" => prev_count} + Map.update(acc, key, weight, &(&1 + weight)) + end) end defp unix_now do diff --git a/vendor/oban_pro/lib/oban/pro/limiters/rate/algorithm.ex b/vendor/oban_pro/lib/oban/pro/limiters/rate/algorithm.ex new file mode 100644 index 00000000..e93fde1b --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/limiters/rate/algorithm.ex @@ -0,0 +1,64 @@ +defmodule Oban.Pro.Limiters.Rate.Algorithm do + @moduledoc false + + alias Oban.Pro.Limiters.Rate.{FixedWindow, SlidingWindow, TokenBucket} + + @type windows :: %{String.t() => map()} + @type counts :: %{String.t() => non_neg_integer()} + + @doc """ + Get the callback module for a rate limit's algorithm. + """ + @spec callback(%{:algorithm => atom(), optional(atom()) => term()}) :: module() + def callback(%{algorithm: :fixed_window}), do: FixedWindow + def callback(%{algorithm: :sliding_window}), do: SlidingWindow + def callback(%{algorithm: :token_bucket}), do: TokenBucket + def callback(_rate_limit), do: SlidingWindow + + @doc """ + Calculate available demand given current window state. + """ + @callback demand( + windows :: windows(), + allowed :: pos_integer(), + period :: pos_integer(), + curr_time :: integer(), + prev_time :: integer() + ) :: non_neg_integer() + + @doc """ + Calculate per-partition demands for partitioned rate limiting. + """ + @callback partition_demands( + windows :: windows(), + allowed :: pos_integer(), + period :: pos_integer(), + curr_time :: integer(), + prev_time :: integer(), + available_keys :: [String.t()] + ) :: %{String.t() => non_neg_integer()} + + @doc """ + Flatten windows into a list with a single, compact value regardless of partitions. + """ + @callback flatten(windows :: windows()) :: [map()] + + @doc """ + Merge window states from multiple producers. + """ + @callback merge(windows_a :: windows(), windows_b :: windows()) :: windows() + + @doc """ + Track job execution by updating window state. + + Returns {updated_windows, next_window_time}. + """ + @callback track( + windows :: windows(), + new_counts :: counts(), + period :: pos_integer(), + prev_time :: integer(), + curr_time :: integer(), + allowed :: pos_integer() + ) :: {windows(), integer()} +end diff --git a/vendor/oban_pro/lib/oban/pro/limiters/rate/fixed_window.ex b/vendor/oban_pro/lib/oban/pro/limiters/rate/fixed_window.ex new file mode 100644 index 00000000..21198a7d --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/limiters/rate/fixed_window.ex @@ -0,0 +1,78 @@ +defmodule Oban.Pro.Limiters.Rate.FixedWindow do + @moduledoc false + + # Fixed window simply resets the count when the period expires, providing a straightforward but + # potentially bursty rate limit. + + @behaviour Oban.Pro.Limiters.Rate.Algorithm + + @impl Oban.Pro.Limiters.Rate.Algorithm + def demand(windows, allowed, period, curr_time, prev_time) do + if window_expired?(curr_time, prev_time, period) do + allowed + else + total = Enum.reduce(windows, 0, fn {_key, val}, acc -> acc + val["count"] end) + max(allowed - total, 0) + end + end + + @impl Oban.Pro.Limiters.Rate.Algorithm + def partition_demands(windows, allowed, period, curr_time, prev_time, available_keys) do + if window_expired?(curr_time, prev_time, period) do + Map.new(available_keys, &{&1, allowed}) + else + untracked = + for key <- available_keys, + not is_map_key(windows, key), + into: %{}, + do: {key, allowed} + + Enum.reduce(windows, untracked, fn {key, val}, acc -> + case max(allowed - val["count"], 0) do + demand when demand > 0 -> Map.put(acc, key, demand) + _zero -> acc + end + end) + end + end + + @impl Oban.Pro.Limiters.Rate.Algorithm + def flatten(windows) do + count = Enum.reduce(windows, 0, fn {_key, val}, acc -> acc + val["count"] end) + + [%{"count" => count}] + end + + @impl Oban.Pro.Limiters.Rate.Algorithm + def merge(windows_a, windows_b) do + Map.merge(windows_a, windows_b, fn _key, v1, v2 -> + %{"count" => v1["count"] + v2["count"]} + end) + end + + @impl Oban.Pro.Limiters.Rate.Algorithm + def track(windows, new_counts, period, prev_time, curr_time, _allowed) do + new_counts = Map.new(new_counts, fn {key, count} -> {key, %{"count" => count}} end) + + if window_expired?(curr_time, prev_time, period) do + {new_counts, curr_time} + else + all_windows = + Map.merge(windows, new_counts, fn _key, existing, new -> + %{"count" => existing["count"] + new["count"]} + end) + + all_windows = + for {key, val} <- all_windows, + val["count"] > 0, + into: %{}, + do: {key, val} + + {all_windows, prev_time} + end + end + + defp window_expired?(curr_time, prev_time, period) do + curr_time - prev_time >= period + end +end diff --git a/vendor/oban_pro/lib/oban/pro/limiters/rate/sliding_window.ex b/vendor/oban_pro/lib/oban/pro/limiters/rate/sliding_window.ex new file mode 100644 index 00000000..7b9a8f17 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/limiters/rate/sliding_window.ex @@ -0,0 +1,98 @@ +defmodule Oban.Pro.Limiters.Rate.SlidingWindow do + @moduledoc false + + # The sliding window uses two time buckets (current and previous) with weighted averaging to + # provide smooth rate limiting without the sharp edges of fixed windows. + + @behaviour Oban.Pro.Limiters.Rate.Algorithm + + @impl Oban.Pro.Limiters.Rate.Algorithm + def demand(windows, allowed, period, curr_time, prev_time) do + weight = div(max(period - (curr_time - prev_time), 0), period) + + total = + Enum.reduce(windows, 0, fn {_key, val}, acc -> + acc + val["prev_count"] * weight + val["curr_count"] + end) + + max(allowed - total, 0) + end + + @impl Oban.Pro.Limiters.Rate.Algorithm + def partition_demands(windows, allowed, period, curr_time, prev_time, available_keys) do + weight = div(max(period - (curr_time - prev_time), 0), period) + + untracked = + for key <- available_keys, + not is_map_key(windows, key), + into: %{}, + do: {key, allowed} + + Enum.reduce(windows, untracked, fn {key, val}, acc -> + %{"curr_count" => curr, "prev_count" => prev} = val + + case max(allowed - (prev * weight + curr), 0) do + demand when demand > 0 -> Map.put(acc, key, demand) + _zero -> acc + end + end) + end + + @impl Oban.Pro.Limiters.Rate.Algorithm + def flatten(windows) do + {pacc, cacc} = + Enum.reduce(windows, {0, 0}, fn {_key, val}, {pacc, cacc} -> + {pacc + val["prev_count"], cacc + val["curr_count"]} + end) + + [%{"curr_count" => cacc, "prev_count" => pacc}] + end + + @impl Oban.Pro.Limiters.Rate.Algorithm + def merge(windows_a, windows_b) do + Map.merge(windows_a, windows_b, fn _key, v1, v2 -> + %{"curr_count" => c1, "prev_count" => p1} = v1 + %{"curr_count" => c2, "prev_count" => p2} = v2 + %{"curr_count" => c1 + c2, "prev_count" => p1 + p2} + end) + end + + @impl Oban.Pro.Limiters.Rate.Algorithm + def track(windows, new_counts, period, prev_time, curr_time, _allowed) do + unix_diff = curr_time - prev_time + + {mode, next_time} = + cond do + unix_diff > period * 2 -> {:dump_prev, curr_time} + unix_diff > period -> {:swap_prev, curr_time} + true -> {:bump_curr, prev_time} + end + + new_windows = + for {key, count} <- new_counts, + not is_map_key(windows, key), + into: %{}, + do: {key, %{"curr_count" => count, "prev_count" => 0}} + + all_windows = + for {key, window} <- windows, + not match?(%{"curr_count" => 0, "prev_count" => 0}, window), + into: new_windows, + do: {key, update_window(window, Map.get(new_counts, key, 0), mode)} + + {all_windows, next_time} + end + + defp update_window(window, next_count, mode) do + %{"curr_count" => curr_count, "prev_count" => prev_count} = window + + {new_curr, new_prev} = + case mode do + :dump_prev -> {next_count, 0} + :swap_prev -> {next_count, curr_count} + :bump_curr -> {curr_count + next_count, prev_count} + end + + %{"curr_count" => new_curr, "prev_count" => new_prev} + end +end diff --git a/vendor/oban_pro/lib/oban/pro/limiters/rate/token_bucket.ex b/vendor/oban_pro/lib/oban/pro/limiters/rate/token_bucket.ex new file mode 100644 index 00000000..4f6994c1 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/limiters/rate/token_bucket.ex @@ -0,0 +1,86 @@ +defmodule Oban.Pro.Limiters.Rate.TokenBucket do + @moduledoc false + + # Tokens refill continuously at a rate of `allowed / period` per second. Available demand is the + # number of tokens in the bucket. + + @behaviour Oban.Pro.Limiters.Rate.Algorithm + + @impl Oban.Pro.Limiters.Rate.Algorithm + def demand(windows, allowed, _period, _curr_time, _prev_time) do + total_consumed = sum_consumed(windows) + max(allowed - total_consumed, 0) + end + + @impl Oban.Pro.Limiters.Rate.Algorithm + def partition_demands(windows, allowed, _period, _curr_time, _prev_time, available_keys) do + untracked = + for key <- available_keys, + not is_map_key(windows, key), + into: %{}, + do: {key, allowed} + + Enum.reduce(windows, untracked, fn {key, val}, acc -> + case max(allowed - val["consumed"], 0) do + demand when demand > 0 -> Map.put(acc, key, demand) + _zero -> acc + end + end) + end + + @impl Oban.Pro.Limiters.Rate.Algorithm + def flatten(windows) do + consumed = Enum.reduce(windows, 0, fn {_key, val}, acc -> acc + val["consumed"] end) + + [%{"consumed" => consumed}] + end + + @impl Oban.Pro.Limiters.Rate.Algorithm + def merge(windows_a, windows_b) do + Map.merge(windows_a, windows_b, fn _key, v1, v2 -> + %{"consumed" => v1["consumed"] + v2["consumed"]} + end) + end + + @doc """ + Track job execution by updating consumed tokens. + + Refills tokens based on elapsed time, then adds new consumption. + Returns the updated windows map and the next window_time. + """ + @impl Oban.Pro.Limiters.Rate.Algorithm + def track(windows, new_counts, period, prev_time, curr_time, allowed) do + refill_amount = calculate_refill(allowed, period, curr_time, prev_time) + + new_consumed = Map.new(new_counts, fn {key, count} -> {key, %{"consumed" => count}} end) + + all_windows = + Map.merge(windows, new_consumed, fn _key, existing, new -> + updated_consumed = max(existing["consumed"] - refill_amount, 0) + new["consumed"] + %{"consumed" => trunc(updated_consumed)} + end) + + all_windows = + Enum.reduce(new_consumed, all_windows, fn {key, val}, acc -> + Map.put_new(acc, key, val) + end) + + all_windows = + for {key, val} <- all_windows, + val["consumed"] > 0, + into: %{}, + do: {key, val} + + {all_windows, curr_time} + end + + defp calculate_refill(allowed, period, curr_time, prev_time) do + elapsed = curr_time - prev_time + refill_rate = allowed / period + min(elapsed * refill_rate, allowed) + end + + defp sum_consumed(windows) do + Enum.reduce(windows, 0, fn {_key, val}, acc -> acc + val["consumed"] end) + end +end diff --git a/vendor/oban_pro/lib/oban/pro/migration.ex b/vendor/oban_pro/lib/oban/pro/migration.ex index a61d9e79..a128662f 100644 --- a/vendor/oban_pro/lib/oban/pro/migration.ex +++ b/vendor/oban_pro/lib/oban/pro/migration.ex @@ -134,7 +134,7 @@ defmodule Oban.Pro.Migration do require Logger - @versions ~w(1.0.0 1.4.0 1.5.0 1.6.0) + @versions ~w(1.0.0 1.4.0 1.5.0 1.6.0 1.7.0) @impl Oban.Pro.Handler def on_start do @@ -189,13 +189,18 @@ defmodule Oban.Pro.Migration do fun.(repo) end - defp with_dynamic_repo(%{get_dynamic_repo: dyn_fun, repo: repo}, fun) do - prev_repo = repo.get_dynamic_repo() + defp with_dynamic_repo(%{get_dynamic_repo: get_dyn_fun, repo: repo}, fun) do + prev_repo = + try do + repo.get_dynamic_repo() + rescue + _ -> nil + end dyna_repo = - case dyn_fun do - {mod, fun, arg} -> apply(mod, fun, arg) - fun when is_function(fun, 0) -> fun.() + case get_dyn_fun do + {mod, dyn_fun, arg} -> apply(mod, dyn_fun, arg) + dyn_fun when is_function(dyn_fun, 0) -> dyn_fun.() end try do @@ -203,10 +208,8 @@ defmodule Oban.Pro.Migration do fun.(repo) after - repo.put_dynamic_repo(prev_repo) + if prev_repo, do: repo.put_dynamic_repo(prev_repo) end - - fun.(repo) end @doc """ @@ -292,6 +295,21 @@ defmodule Oban.Pro.Migration do end end + @doc false + def partitioned?(opts) do + query = """ + SELECT EXISTS ( + SELECT 1 + FROM pg_tables + WHERE schemaname = '#{opts.prefix}' AND tablename = 'oban_jobs_completed' + ) + """ + + {:ok, %{rows: [[partitioned?]]}} = repo().query(query, [], log: false) + + partitioned? + end + defp migrated_version_index(version, opts) do [schemas_version, indexes_version] = normalize_recorded(version) @@ -349,6 +367,7 @@ defmodule Oban.Pro.Migration do prefix = Access.get(opts, :prefix, "public") unlogged = Access.get(opts, :unlogged, true) version = Access.get(opts, :version, List.last(@versions)) + generated_columns = Access.get(opts, :generated_columns, false) only = opts @@ -358,6 +377,7 @@ defmodule Oban.Pro.Migration do %{ concurrently: only == [:indexes], escaped: String.replace(prefix, "'", "\\'"), + generated_columns: generated_columns, next: find_version_index(version), only: only, prefix: prefix, @@ -379,9 +399,15 @@ defmodule Oban.Pro.Migration do defp normalize_recorded(version) do case String.split(version, ",", parts: 2) do - [<<_prefix::binary-size(5), "-schemas">> = schemas] -> [schemas, nil] - [version] -> ["#{version}-schemas", "#{version}-indexes"] - recorded -> recorded + [schemas] when is_binary(schemas) -> + if String.ends_with?(schemas, "-schemas") do + [schemas, nil] + else + ["#{schemas}-schemas", "#{schemas}-indexes"] + end + + recorded -> + recorded end end end diff --git a/vendor/oban_pro/lib/oban/pro/migrations/dynamic_partitioner.ex b/vendor/oban_pro/lib/oban/pro/migrations/dynamic_partitioner.ex index 2f01335d..2f97186c 100644 --- a/vendor/oban_pro/lib/oban/pro/migrations/dynamic_partitioner.ex +++ b/vendor/oban_pro/lib/oban/pro/migrations/dynamic_partitioner.ex @@ -45,6 +45,7 @@ defmodule Oban.Pro.Migrations.DynamicPartitioner do AND typnamespace = '#{quoted}'::regnamespace::oid) THEN CREATE TYPE #{quoted}.oban_job_state AS ENUM ( 'available', + 'suspended', 'scheduled', 'executing', 'retryable', diff --git a/vendor/oban_pro/lib/oban/pro/migrations/v1_5_0.ex b/vendor/oban_pro/lib/oban/pro/migrations/v1_5_0.ex index af7ac3e8..56d010ea 100644 --- a/vendor/oban_pro/lib/oban/pro/migrations/v1_5_0.ex +++ b/vendor/oban_pro/lib/oban/pro/migrations/v1_5_0.ex @@ -3,6 +3,8 @@ defmodule Oban.Pro.Migrations.V150 do use Ecto.Migration + import Oban.Pro.Migration, only: [partitioned?: 1] + def up_ddl(opts) do if not partitioned?(opts) do execute """ @@ -20,35 +22,51 @@ defmodule Oban.Pro.Migrations.V150 do $$ LANGUAGE SQL IMMUTABLE STRICT """ - alter table(:oban_jobs, prefix: opts.prefix) do - add_if_not_exists :uniq_key, :text, - generated: """ - ALWAYS AS (CASE WHEN meta->'uniq_bmp' @> #{opts.quoted}.oban_state_to_bit(state) THEN meta->>'uniq_key' END) STORED - """ + if opts.generated_columns do + alter table(:oban_jobs, prefix: opts.prefix) do + add_if_not_exists :uniq_key, :text, + generated: """ + ALWAYS AS (CASE WHEN meta->'uniq_bmp' @> #{opts.quoted}.oban_state_to_bit(state) THEN meta->>'uniq_key' END) STORED + """ + end end end end def up_idx(opts) do - if partitioned?(opts) do - create_if_not_exists index( - :oban_jobs, - ["(meta->>'uniq_key')"], - concurrently: opts.concurrently, - name: :oban_jobs_unique_index, - prefix: opts.prefix, - where: "meta ? 'uniq_key'" - ) - else - create_if_not_exists index( - :oban_jobs, - [:uniq_key], - concurrently: opts.concurrently, - name: :oban_jobs_unique_index, - prefix: opts.prefix, - unique: true, - where: "uniq_key IS NOT NULL" - ) + cond do + partitioned?(opts) -> + create_if_not_exists index( + :oban_jobs, + ["(meta->>'uniq_key')"], + concurrently: opts.concurrently, + name: :oban_jobs_unique_index, + prefix: opts.prefix, + where: "meta ? 'uniq_key'" + ) + + opts.generated_columns -> + create_if_not_exists index( + :oban_jobs, + [:uniq_key], + concurrently: opts.concurrently, + name: :oban_jobs_unique_index, + prefix: opts.prefix, + unique: true, + where: "uniq_key IS NOT NULL" + ) + + true -> + create_if_not_exists index( + :oban_jobs, + ["(meta->>'uniq_key')"], + concurrently: opts.concurrently, + name: :oban_jobs_unique_index, + prefix: opts.prefix, + unique: true, + where: + "meta ? 'uniq_key' AND jsonb_contains(meta->'uniq_bmp', #{opts.quoted}.oban_state_to_bit(state))" + ) end create_if_not_exists index( @@ -75,7 +93,7 @@ defmodule Oban.Pro.Migrations.V150 do remove_if_exists :uniq_key, :text end - execute "DROP FUNCTION IF EXISTS #{opts.quoted}.oban_state_to_bit" + execute "DROP FUNCTION IF EXISTS #{opts.quoted}.oban_state_to_bit CASCADE" end def down_idx(opts) do @@ -97,18 +115,4 @@ defmodule Oban.Pro.Migrations.V150 do prefix: opts.prefix ) end - - defp partitioned?(opts) do - query = """ - SELECT EXISTS ( - SELECT 1 - FROM pg_tables - WHERE schemaname = '#{opts.prefix}' AND tablename = 'oban_jobs_completed' - ) - """ - - {:ok, %{rows: [[partitioned?]]}} = repo().query(query, [], log: false) - - partitioned? - end end diff --git a/vendor/oban_pro/lib/oban/pro/migrations/v1_6_0.ex b/vendor/oban_pro/lib/oban/pro/migrations/v1_6_0.ex index add75afa..a56de1b0 100644 --- a/vendor/oban_pro/lib/oban/pro/migrations/v1_6_0.ex +++ b/vendor/oban_pro/lib/oban/pro/migrations/v1_6_0.ex @@ -6,11 +6,13 @@ defmodule Oban.Pro.Migrations.V160 do def up_ddl(opts) do # Partitioning - alter table(:oban_jobs, prefix: opts.prefix) do - add_if_not_exists :partition_key, :text, - generated: """ - ALWAYS AS (CASE WHEN meta ? 'partition_key' THEN meta->>'partition_key' END) STORED - """ + if opts.generated_columns do + alter table(:oban_jobs, prefix: opts.prefix) do + add_if_not_exists :partition_key, :text, + generated: """ + ALWAYS AS (CASE WHEN meta ? 'partition_key' THEN meta->>'partition_key' END) STORED + """ + end end # Dynamic Queues @@ -23,14 +25,25 @@ defmodule Oban.Pro.Migrations.V160 do def up_idx(opts) do # Partitioning - create_if_not_exists index( - :oban_jobs, - [:partition_key, :queue, :priority, :scheduled_at, :id], - concurrently: opts.concurrently, - name: :oban_jobs_partition_index, - prefix: opts.prefix, - where: "state = 'available' AND partition_key IS NOT NULL" - ) + if opts.generated_columns do + create_if_not_exists index( + :oban_jobs, + [:partition_key, :queue, :priority, :scheduled_at, :id], + concurrently: opts.concurrently, + name: :oban_jobs_partition_index, + prefix: opts.prefix, + where: "state = 'available' AND partition_key IS NOT NULL" + ) + else + create_if_not_exists index( + :oban_jobs, + ["(meta->>'partition_key')", :queue, :priority, :scheduled_at, :id], + concurrently: opts.concurrently, + name: :oban_jobs_partition_index, + prefix: opts.prefix, + where: "state = 'available' AND meta ? 'partition_key'" + ) + end # Workflows diff --git a/vendor/oban_pro/lib/oban/pro/migrations/v1_7_0.ex b/vendor/oban_pro/lib/oban/pro/migrations/v1_7_0.ex new file mode 100644 index 00000000..82125b8f --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/migrations/v1_7_0.ex @@ -0,0 +1,416 @@ +defmodule Oban.Pro.Migrations.V170 do + @moduledoc false + + use Ecto.Migration + + import Oban.Pro.Migration, only: [partitioned?: 1] + + def up_ddl(opts) do + execute """ + CREATE OR REPLACE FUNCTION #{opts.quoted}.oban_state_to_bit(state #{opts.quoted}.oban_job_state) + RETURNS jsonb AS $$ + SELECT CASE + WHEN state = 'scheduled' THEN '0'::jsonb + WHEN state = 'available' THEN '1'::jsonb + WHEN state = 'executing' THEN '2'::jsonb + WHEN state = 'retryable' THEN '3'::jsonb + WHEN state = 'completed' THEN '4'::jsonb + WHEN state = 'cancelled' THEN '5'::jsonb + WHEN state = 'discarded' THEN '6'::jsonb + WHEN state = 'suspended' THEN '7'::jsonb + END; + $$ LANGUAGE SQL IMMUTABLE STRICT + """ + + create_if_not_exists table(:oban_workflows, primary_key: false, prefix: opts.prefix) do + add :id, :text, primary_key: true + add :name, :text + add :parent_id, :text + + add :suspended, :integer, null: false, default: 0 + add :available, :integer, null: false, default: 0 + add :scheduled, :integer, null: false, default: 0 + add :executing, :integer, null: false, default: 0 + add :retryable, :integer, null: false, default: 0 + add :completed, :integer, null: false, default: 0 + add :cancelled, :integer, null: false, default: 0 + add :discarded, :integer, null: false, default: 0 + + add :inserted_at, :utc_datetime_usec, null: false + add :started_at, :utc_datetime_usec + add :completed_at, :utc_datetime_usec + + add :meta, :map, null: false, default: %{} + end + + execute """ + ALTER TABLE #{opts.quoted}.oban_workflows + ADD COLUMN IF NOT EXISTS state TEXT GENERATED ALWAYS AS ( + CASE + WHEN (suspended + available + scheduled + executing + retryable) > 0 THEN 'executing' + WHEN cancelled > 0 AND NOT COALESCE((meta->>'ignore_cancelled')::boolean, false) THEN 'cancelled' + WHEN discarded > 0 AND NOT COALESCE((meta->>'ignore_discarded')::boolean, false) THEN 'discarded' + ELSE 'completed' + END + ) STORED + """ + + execute """ + CREATE OR REPLACE FUNCTION #{opts.quoted}.oban_workflows_update() RETURNS TRIGGER AS $$ + BEGIN + UPDATE #{opts.quoted}.oban_workflows wf SET + suspended = GREATEST(0, wf.suspended + COALESCE(d.suspended, 0)), + available = GREATEST(0, wf.available + COALESCE(d.available, 0)), + scheduled = GREATEST(0, wf.scheduled + COALESCE(d.scheduled, 0)), + executing = GREATEST(0, wf.executing + COALESCE(d.executing, 0)), + retryable = GREATEST(0, wf.retryable + COALESCE(d.retryable, 0)), + completed = GREATEST(0, wf.completed + COALESCE(d.completed, 0)), + cancelled = GREATEST(0, wf.cancelled + COALESCE(d.cancelled, 0)), + discarded = GREATEST(0, wf.discarded + COALESCE(d.discarded, 0)), + started_at = CASE + WHEN wf.started_at IS NULL AND d.executing > 0 THEN NOW() + ELSE wf.started_at + END, + completed_at = CASE + WHEN wf.completed_at IS NULL + AND (wf.suspended + wf.available + wf.scheduled + wf.executing + wf.retryable + + COALESCE(d.suspended, 0) + COALESCE(d.available, 0) + COALESCE(d.scheduled, 0) + + COALESCE(d.executing, 0) + COALESCE(d.retryable, 0)) <= 0 + THEN NOW() + ELSE wf.completed_at + END + FROM ( + SELECT + COALESCE(n.meta, o.meta)->>'workflow_id' AS workflow_id, + SUM(CASE WHEN n.state = 'suspended' THEN 1 WHEN o.state = 'suspended' THEN -1 ELSE 0 END) AS suspended, + SUM(CASE WHEN n.state = 'available' THEN 1 WHEN o.state = 'available' THEN -1 ELSE 0 END) AS available, + SUM(CASE WHEN n.state = 'scheduled' THEN 1 WHEN o.state = 'scheduled' THEN -1 ELSE 0 END) AS scheduled, + SUM(CASE WHEN n.state = 'executing' THEN 1 WHEN o.state = 'executing' THEN -1 ELSE 0 END) AS executing, + SUM(CASE WHEN n.state = 'retryable' THEN 1 WHEN o.state = 'retryable' THEN -1 ELSE 0 END) AS retryable, + SUM(CASE WHEN n.state = 'completed' THEN 1 WHEN o.state = 'completed' THEN -1 ELSE 0 END) AS completed, + SUM(CASE WHEN n.state = 'cancelled' THEN 1 WHEN o.state = 'cancelled' THEN -1 ELSE 0 END) AS cancelled, + SUM(CASE WHEN n.state = 'discarded' THEN 1 WHEN o.state = 'discarded' THEN -1 ELSE 0 END) AS discarded + FROM new_table n + FULL OUTER JOIN old_table o ON n.id = o.id + WHERE (n.meta ? 'workflow_id' OR o.meta ? 'workflow_id') + AND n.state IS DISTINCT FROM o.state + GROUP BY COALESCE(n.meta, o.meta)->>'workflow_id' + ) d + WHERE wf.id = d.workflow_id; + + RETURN NULL; + END; + $$ LANGUAGE plpgsql; + """ + + execute """ + CREATE OR REPLACE FUNCTION #{opts.quoted}.oban_workflows_delete() RETURNS TRIGGER AS $$ + BEGIN + UPDATE #{opts.quoted}.oban_workflows wf SET + suspended = GREATEST(0, wf.suspended - COALESCE(d.suspended, 0)), + available = GREATEST(0, wf.available - COALESCE(d.available, 0)), + scheduled = GREATEST(0, wf.scheduled - COALESCE(d.scheduled, 0)), + executing = GREATEST(0, wf.executing - COALESCE(d.executing, 0)), + retryable = GREATEST(0, wf.retryable - COALESCE(d.retryable, 0)), + completed = GREATEST(0, wf.completed - COALESCE(d.completed, 0)), + cancelled = GREATEST(0, wf.cancelled - COALESCE(d.cancelled, 0)), + discarded = GREATEST(0, wf.discarded - COALESCE(d.discarded, 0)) + FROM ( + SELECT + o.meta->>'workflow_id' AS workflow_id, + COUNT(*) FILTER (WHERE o.state = 'suspended') AS suspended, + COUNT(*) FILTER (WHERE o.state = 'available') AS available, + COUNT(*) FILTER (WHERE o.state = 'scheduled') AS scheduled, + COUNT(*) FILTER (WHERE o.state = 'executing') AS executing, + COUNT(*) FILTER (WHERE o.state = 'retryable') AS retryable, + COUNT(*) FILTER (WHERE o.state = 'completed') AS completed, + COUNT(*) FILTER (WHERE o.state = 'cancelled') AS cancelled, + COUNT(*) FILTER (WHERE o.state = 'discarded') AS discarded + FROM old_table o + WHERE o.meta ? 'workflow_id' + GROUP BY o.meta->>'workflow_id' + ) d + WHERE wf.id = d.workflow_id; + + RETURN NULL; + END; + $$ LANGUAGE plpgsql; + """ + + execute """ + CREATE OR REPLACE TRIGGER oban_workflows_update_trigger + AFTER UPDATE ON #{opts.quoted}.oban_jobs + REFERENCING OLD TABLE AS old_table NEW TABLE AS new_table + FOR EACH STATEMENT + EXECUTE FUNCTION #{opts.quoted}.oban_workflows_update(); + """ + + execute """ + CREATE OR REPLACE TRIGGER oban_workflows_delete_trigger + AFTER DELETE ON #{opts.quoted}.oban_jobs + REFERENCING OLD TABLE AS old_table + FOR EACH STATEMENT + EXECUTE FUNCTION #{opts.quoted}.oban_workflows_delete(); + """ + end + + def up_idx(opts) do + # Replace column-based indexes with expression indexes for unique and partition keys. This + # enables querying meta fields directly without relying on generated columns. Partitioned + # tables already use expression indexes from v1.5.0. + if not opts.generated_columns and not partitioned?(opts) do + drop_if_exists index(:oban_jobs, [], + name: :oban_jobs_unique_index, + concurrently: opts.concurrently, + prefix: opts.prefix + ) + + drop_if_exists index(:oban_jobs, [], + name: :oban_jobs_partition_index, + concurrently: opts.concurrently, + prefix: opts.prefix + ) + + create_if_not_exists index( + :oban_jobs, + ["(meta->>'uniq_key')"], + concurrently: opts.concurrently, + name: :oban_jobs_unique_index, + prefix: opts.prefix, + unique: true, + where: + "meta ? 'uniq_key' AND jsonb_contains(meta->'uniq_bmp', #{opts.quoted}.oban_state_to_bit(state))" + ) + + create_if_not_exists index( + :oban_jobs, + ["(meta->>'partition_key')", :queue, :priority, :scheduled_at, :id], + concurrently: opts.concurrently, + name: :oban_jobs_partition_index, + prefix: opts.prefix, + where: "state = 'available' AND meta ? 'partition_key'" + ) + end + + # Rename old workflow/chain indexes that use on_hold + execute "ALTER INDEX IF EXISTS #{opts.quoted}.oban_jobs_workflow_index RENAME TO oban_jobs_workflow_index_old" + + execute "ALTER INDEX IF EXISTS #{opts.quoted}.oban_jobs_sup_workflow_index RENAME TO oban_jobs_sup_workflow_index_old" + + execute "ALTER INDEX IF EXISTS #{opts.quoted}.oban_jobs_chain_index RENAME TO oban_jobs_chain_index_old" + + # Create new indexes without on_hold + create_if_not_exists index( + :oban_jobs, + ["(meta->>'workflow_id')", :state, "(meta->>'name')"], + concurrently: opts.concurrently, + name: :oban_jobs_workflow_index, + prefix: opts.prefix, + where: "meta ? 'workflow_id'" + ) + + create_if_not_exists index( + :oban_jobs, + ["(meta->>'sup_workflow_id')", :state], + concurrently: opts.concurrently, + name: :oban_jobs_sup_workflow_index, + prefix: opts.prefix, + where: "meta ? 'sup_workflow_id'" + ) + + create_if_not_exists index( + :oban_jobs, + [:state, "(meta->>'chain_id')"], + concurrently: opts.concurrently, + name: :oban_jobs_chain_index, + prefix: opts.prefix, + where: "meta ? 'chain_id'" + ) + + create_if_not_exists index( + :oban_jobs, + [:state, "(meta->>'chunk_id')"], + concurrently: opts.concurrently, + name: :oban_jobs_chunk_index, + prefix: opts.prefix, + where: "meta ? 'chunk_id'" + ) + + # Replace the non-partial state timestamp indexes from core Oban with partial indexes. + # These partial indexes are smaller and more efficient for pruning operations. + drop_if_exists index(:oban_jobs, [], + name: :oban_jobs_state_cancelled_at_index, + concurrently: opts.concurrently, + prefix: opts.prefix + ) + + drop_if_exists index(:oban_jobs, [], + name: :oban_jobs_state_discarded_at_index, + concurrently: opts.concurrently, + prefix: opts.prefix + ) + + create_if_not_exists index( + :oban_jobs, + [:cancelled_at], + concurrently: opts.concurrently, + name: :oban_jobs_cancelled_at_index, + prefix: opts.prefix, + where: "state = 'cancelled'" + ) + + create_if_not_exists index( + :oban_jobs, + [:discarded_at], + concurrently: opts.concurrently, + name: :oban_jobs_discarded_at_index, + prefix: opts.prefix, + where: "state = 'discarded'" + ) + + create_if_not_exists index( + :oban_jobs, + [:completed_at], + concurrently: opts.concurrently, + name: :oban_jobs_completed_at_index, + prefix: opts.prefix, + where: "state = 'completed'" + ) + + # Add a staging index for efficiently finding scheduled/retryable jobs ready to run. + # This speeds up the stage_jobs operation which transitions jobs to 'available'. + create_if_not_exists index( + :oban_jobs, + [:scheduled_at, :id], + concurrently: opts.concurrently, + name: :oban_jobs_staging_index, + prefix: opts.prefix, + where: "state IN ('scheduled', 'retryable')" + ) + + # Workflow table indexes + create_if_not_exists index( + :oban_workflows, + [:state, :name], + concurrently: opts.concurrently, + prefix: opts.prefix + ) + + create_if_not_exists index( + :oban_workflows, + [:parent_id], + concurrently: opts.concurrently, + prefix: opts.prefix, + where: "parent_id IS NOT NULL" + ) + + create_if_not_exists index( + :oban_workflows, + [:name], + concurrently: opts.concurrently, + name: :oban_workflows_unique_name, + prefix: opts.prefix, + unique: true, + where: "state != 'completed' AND meta ? 'unique'" + ) + end + + def down_ddl(opts) do + execute "DROP TRIGGER IF EXISTS oban_workflows_update_trigger ON #{opts.quoted}.oban_jobs" + execute "DROP TRIGGER IF EXISTS oban_workflows_delete_trigger ON #{opts.quoted}.oban_jobs" + execute "DROP FUNCTION IF EXISTS #{opts.quoted}.oban_workflows_update()" + execute "DROP FUNCTION IF EXISTS #{opts.quoted}.oban_workflows_delete()" + drop_if_exists table(:oban_workflows, prefix: opts.prefix) + end + + def down_idx(opts) do + drop_if_exists index(:oban_workflows, [:state, :name], + concurrently: opts.concurrently, + prefix: opts.prefix + ) + + drop_if_exists index(:oban_workflows, [:parent_id], + concurrently: opts.concurrently, + prefix: opts.prefix + ) + + drop_if_exists index(:oban_workflows, [], + name: :oban_workflows_unique_name, + concurrently: opts.concurrently, + prefix: opts.prefix + ) + + # Drop new indexes + drop_if_exists index(:oban_jobs, [], + name: :oban_jobs_workflow_index, + concurrently: opts.concurrently, + prefix: opts.prefix + ) + + drop_if_exists index(:oban_jobs, [], + name: :oban_jobs_sup_workflow_index, + concurrently: opts.concurrently, + prefix: opts.prefix + ) + + drop_if_exists index(:oban_jobs, [], + name: :oban_jobs_chain_index, + concurrently: opts.concurrently, + prefix: opts.prefix + ) + + drop_if_exists index(:oban_jobs, [], + name: :oban_jobs_chunk_index, + concurrently: opts.concurrently, + prefix: opts.prefix + ) + + # Drop the partial state timestamp indexes and restore the original non-partial indexes + drop_if_exists index(:oban_jobs, [], + name: :oban_jobs_staging_index, + concurrently: opts.concurrently, + prefix: opts.prefix + ) + + drop_if_exists index(:oban_jobs, [], + name: :oban_jobs_completed_at_index, + concurrently: opts.concurrently, + prefix: opts.prefix + ) + + drop_if_exists index(:oban_jobs, [], + name: :oban_jobs_discarded_at_index, + concurrently: opts.concurrently, + prefix: opts.prefix + ) + + drop_if_exists index(:oban_jobs, [], + name: :oban_jobs_cancelled_at_index, + concurrently: opts.concurrently, + prefix: opts.prefix + ) + + create_if_not_exists index( + :oban_jobs, + [:state, :cancelled_at], + concurrently: opts.concurrently, + name: :oban_jobs_state_cancelled_at_index, + prefix: opts.prefix + ) + + create_if_not_exists index( + :oban_jobs, + [:state, :discarded_at], + concurrently: opts.concurrently, + name: :oban_jobs_state_discarded_at_index, + prefix: opts.prefix + ) + + # Rename old indexes back + execute "ALTER INDEX IF EXISTS #{opts.quoted}.oban_jobs_workflow_index_old RENAME TO oban_jobs_workflow_index" + + execute "ALTER INDEX IF EXISTS #{opts.quoted}.oban_jobs_sup_workflow_index_old RENAME TO oban_jobs_sup_workflow_index" + + execute "ALTER INDEX IF EXISTS #{opts.quoted}.oban_jobs_chain_index_old RENAME TO oban_jobs_chain_index" + end +end diff --git a/vendor/oban_pro/lib/oban/pro/partition.ex b/vendor/oban_pro/lib/oban/pro/partition.ex index 402df2b6..e30f9966 100644 --- a/vendor/oban_pro/lib/oban/pro/partition.ex +++ b/vendor/oban_pro/lib/oban/pro/partition.ex @@ -6,8 +6,8 @@ defmodule Oban.Pro.Partition do import Ecto.Query alias Ecto.Changeset - alias Oban.Pro.{Producer, Queue, Utils} alias Oban.{Config, Job, Repo} + alias Oban.Pro.{Producer, Queue, Utils} @keys_cache_ttl Application.compile_env(:oban_pro, [__MODULE__, :keys_cache_ttl], 3_000) @keys_limit_mlt Application.compile_env(:oban_pro, [__MODULE__, :keys_limit_multiplier], 3) @@ -100,10 +100,10 @@ defmodule Oban.Pro.Partition do subquery = Job |> where([j], j.state == "available" and j.queue == ^queue) - |> where([j], not is_nil(fragment("partition_key"))) - |> group_by([j], fragment("partition_key")) + |> where([j], fragment("meta \\? 'partition_key'")) + |> group_by([j], fragment("meta->>'partition_key'")) |> order_by([j], asc: min(j.priority), asc: min(j.scheduled_at)) - |> select([j], %{partition_key: fragment("partition_key")}) + |> select([j], %{partition_key: fragment("meta->>'partition_key'")}) |> limit(^query_limit) partitioned_keys = diff --git a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_cron.ex b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_cron.ex index 3a13d796..8777d2dd 100644 --- a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_cron.ex +++ b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_cron.ex @@ -18,7 +18,7 @@ defmodule Oban.Pro.Plugins.DynamicCron do By itself, without providing a crontab or dynamically inserting cron entries, the plugin doesn't have anything to schedule. To get scheduling started, provide a list of `{cron, worker}` or - `{cron, worker, options}` tuples to the plugin. The syntax is identical to Oban's built in + `{cron, worker, options}` tuples to the plugin. The syntax is identical to Oban's built-in `:crontab` option, which means you can _probably_ copy an existing standard `:crontab` list into the plugin's `:crontab`. @@ -35,7 +35,7 @@ defmodule Oban.Pro.Plugins.DynamicCron do }] ``` - > #### Beware of Conflicts {: .tip} + > #### Beware of Conflicts {: .warning} > > While the syntax is syntactically identical to Oban's basic `:crontab`, you can't copy an > existing `:crontab` if you have multiple entries using the same worker. `DynamicCron` @@ -44,8 +44,9 @@ defmodule Oban.Pro.Plugins.DynamicCron do > overrides for each duplicate worker. Now, when dynamic cron initializes, it will persist those cron entries to the database and start - scheduling them according to their CRON expression. The plugin's `crontab` format is nearly - identical to Oban's standard crontab, with a few important enhancements we'll look at soon. + scheduling them according to their cron expression. Beyond standard cron syntax, `DynamicCron` + entries support naming, pausing, per-entry timezone overrides, guaranteed scheduling, and custom + job args and metadata—all covered in the sections below. Each of the crontab entries are persisted to the database and referenced globally, by all the other connected Oban instances. That allows us to insert, update, or delete cron entries at any @@ -62,7 +63,7 @@ defmodule Oban.Pro.Plugins.DynamicCron do Now it isn't really a "minute job" any more, and the name is no longer suitable. However, we didn't provide a name for the entry and it's using the module name instead. To provide more - flexibility we can add a `:name` overrride, then we can update the worker's name as well: + flexibility we can add a `:name` override, then we can update the worker's name as well: ```elixir crontab: [ @@ -84,7 +85,6 @@ defmodule Oban.Pro.Plugins.DynamicCron do To temporarily disable scheduling jobs you can set the `paused` flag: - ```elixir crontab: [ {"* * * * *", MyApp.BasicJob, paused: true}, @@ -115,16 +115,50 @@ defmodule Oban.Pro.Plugins.DynamicCron do One or more entries can be deleted this way. Deleting entries is idempotent, nothing will happen if no matching entry can be found. + ## Entry Options + + Each crontab entry accepts the following options: + + * `:name` - a unique identifier for the entry, defaults to the worker module name. Required when + the same worker is used in multiple entries. + + * `:args` - a map of arguments passed to the worker's `perform/1` callback. Defaults to `%{}`. + + * `:meta` - a map of metadata merged into the job's meta field alongside automatic cron metadata. + + * `:queue` - the queue to insert the job into, as an atom or string. + + * `:priority` - an integer from 0 to 9, where lower numbers are higher priority. + + * `:max_attempts` - the maximum number of execution attempts before the job is discarded. + + * `:tags` - a list of string tags for categorizing or searching jobs. + + * `:timezone` - a timezone for evaluating the cron expression, overriding the plugin-level + timezone. See [Overriding the Timezone](#module-overriding-the-timezone). + + * `:paused` - when `true`, the entry is persisted but won't schedule jobs until resumed. + + * `:guaranteed` - when `true`, enables guaranteed scheduling for this entry. See + [Scheduling Guarantees](#module-scheduling-guarantees). + + * `:delete` - when `true`, deletes the persisted entry during initialization. Only used in the + plugin's `:crontab` list, not with runtime functions. + + In addition to standard five-field cron expressions, the plugin supports extended expressions + such as `@daily`, `@hourly`, `@weekly`, `@monthly`, `@yearly`, and `@reboot`. The `@reboot` + expression inserts a job once each time the plugin starts and won't re-fire until the next + restart. + ## Automatic Synchronization - Synchronizing persisted entries manually requires two deploys: one to flag it with `deleted: + Synchronizing persisted entries manually requires two deploys: one to flag it with `delete: true` and another to clean up the entry entirely. That extra step isn't ideal for applications that don't insert or delete jobs at runtime. To delete entries that are no longer listed in the crontab automatically set the `sync_mode` option to `:automatic`: - ```elixir [ sync_mode: :automatic, @@ -150,7 +184,7 @@ defmodule Oban.Pro.Plugins.DynamicCron do Depending on an application's restart timing or as the result of unexpected downtime, a job's scheduled insert period may be missed. To compensate, enable `guaranteed` mode for the entire - crontab or on an individual bases. + crontab or on an individual basis. Here's an example of enabling guaranteed insertion for the entire crontab: @@ -208,7 +242,7 @@ defmodule Oban.Pro.Plugins.DynamicCron do Dynamic cron entries are persisted to the database, making it easy to manipulate them through typical CRUD operations. The `DynamicCron` plugin provides convenience functions to simplify - working those operations. + those operations. The `insert/1` function takes a list of one or more tuples with the same `{expression, worker}` or `{expression, worker, options}` format as the plugin's `crontab` option: @@ -247,7 +281,7 @@ defmodule Oban.Pro.Plugins.DynamicCron do The `DynamicCron` plugin adds the following metadata to the `[:oban, :plugin, :stop]` event: - * `:jobs` - a list of jobs that were inserted into the database + * `:jobs` - a list of jobs inserted during the scheduling pass """ @behaviour Oban.Plugin @@ -257,9 +291,9 @@ defmodule Oban.Pro.Plugins.DynamicCron do import Ecto.Query, only: [order_by: 2, where: 2, where: 3] alias Oban.Cron.Expression + alias Oban.{Job, Peer, Repo, Validation, Worker} alias Oban.Plugins.Cron, as: CronPlugin alias Oban.Pro.Cron, as: CronEntry - alias Oban.{Job, Peer, Repo, Validation, Worker} require Logger @@ -368,6 +402,30 @@ defmodule Oban.Pro.Plugins.DynamicCron do |> all_entries() end + @doc """ + Retrieve a single cron entry by name. + + ## Examples + + Fetch an entry using the worker module as the name: + + {:ok, entry} = DynamicCron.get(MyApp.Worker) + + Fetch an entry using a custom name: + + {:ok, entry} = DynamicCron.get("my-custom-cron") + + Returns an error tuple if the entry doesn't exist: + + {:error, "no cron entry named \\"missing\\" could be found"} = DynamicCron.get("missing") + """ + @spec get(term(), cron_name()) :: {:ok, Ecto.Schema.t()} | {:error, String.t()} + def get(oban_name \\ Oban, name) when is_name(name) do + oban_name + |> Oban.config() + |> fetch_entry(Worker.to_string(name)) + end + @doc """ Insert cron entries into the database to start scheduling new jobs. diff --git a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_lifeline.ex b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_lifeline.ex index 8a78b3d3..99ca8451 100644 --- a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_lifeline.ex +++ b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_lifeline.ex @@ -4,10 +4,11 @@ defmodule Oban.Pro.Plugins.DynamicLifeline do jobs that are stuck in the `executing` state because the node was shut down before the job could finish. In addition, it performs the following maintenance tasks: - * Discard jobs left `available` with an attempt equal to max attempts + * Discard jobs left `available` with exhausted attempts due to rare edge cases * Repair stuck workflows with deleted dependencies or missed scheduling events * Repair stuck chains with deleted dependencies or missed scheduling events * Repair jobs in partitioned queues that are missing a partition key + * Repair chunk jobs that are missing a computed `chunk_id` Without `DynamicLifeline` you'll need to manually rescue stuck jobs or perform maintenance. @@ -18,30 +19,29 @@ defmodule Oban.Pro.Plugins.DynamicLifeline do ```elixir config :my_app, Oban, + engine: Oban.Pro.Engines.Smart, plugins: [Oban.Pro.Plugins.DynamicLifeline] ... ``` - There isn't any configuration necessary. By default, the plugin rescues orphaned jobs every 1 - minute. If necessary, you can override the rescue interval: + No configuration is necessary—the defaults are tuned for most systems. - ```elixir - plugins: [{Oban.Pro.Plugins.DynamicLifeline, rescue_interval: :timer.minutes(5)}] - ``` + ## Options - If your system is under high load or produces a multitude of orphans you may wish to increase - the query timeout beyond the `45s` default: + * `:repair_limit` — the maximum number of jobs each repair operation (workflows, chains, + partitions, chunks) will process per cycle. Defaults to `1000`. - ```elixir - plugins: [{Oban.Pro.Plugins.DynamicLifeline, timeout: :timer.minutes(1)}] - ``` + * `:retry_exhausted` — when `true`, jobs that have exhausted their attempts are rescued back to + `available` with an incremented `max_attempts` instead of being discarded. Defaults to + `false`. See "Rescuing Exhausted Jobs" for details. - Note that rescuing orphans relies on producer records as used by the `Smart` engine. + * `:timeout` — the maximum time allowed for each rescue query, in milliseconds. Increase this + if your system is under high load or produces a multitude of orphans. Defaults to `45_000`. - ## Repairing Workflows, Chains, and Partitions + ## Automatic Repairs - The plugin automatically repairs stuck workflows, chains, and partitioned jobs during each - rescue cycle. These repair operations ensure: + The plugin automatically performs several repair operations during each rescue cycle to keep + workflows, chains, partitions, and chunks healthy. * **Workflow repair** — Jobs held waiting for dependencies that were deleted or missed a scheduling event are released. This handles edge cases where workflow jobs get stuck due to @@ -54,12 +54,11 @@ defmodule Oban.Pro.Plugins.DynamicLifeline do metadata (e.g., jobs scheduled before a queue was partitioned) have their partition key computed and set automatically. - By default, each repair operation processes up to 1000 jobs per cycle. You can adjust this - limit with the `repair_limit` option: + * **Chunk repair** — Chunk jobs that are missing a `chunk_id` in their metadata have their + chunk ID computed from the job's `chunk_by` configuration and set automatically. This can + happen when chunk settings are changed after jobs are already enqueued. - ```elixir - plugins: [{Oban.Pro.Plugins.DynamicLifeline, repair_limit: 500}] - ``` + Each repair operation processes up to `repair_limit` jobs per cycle. ## Identifying Rescued Jobs @@ -79,6 +78,9 @@ defmodule Oban.Pro.Plugins.DynamicLifeline do NIFs, memory exhaustion, etc.) We don't want to repeatedly rescue and rerun a job that repeatedly crashes the entire node. + When exhausted jobs are discarded, the `c:Oban.Pro.Worker.on_discarded/2` callback is called + with an `:exhausted` reason, which is useful for error reporting or notifications. + Discarding exhausted jobs may not always be desired. Use the `retry_exhausted` option if you'd prefer to retry exhausted jobs when they are rescued, rather than discarding them: @@ -106,10 +108,11 @@ defmodule Oban.Pro.Plugins.DynamicLifeline do import Ecto.Query - alias Oban.Pro.Engines.Smart - alias Oban.Pro.Stages.Chain - alias Oban.Pro.{Partition, Producer, Workflow} alias Oban.{Job, Peer, Repo, Validation} + alias Oban.Pro.Chunk + alias Oban.Pro.Engines.Smart + alias Oban.Pro.{Partition, Producer, Utils, Workflow} + alias Oban.Pro.Stages.{Chain, Hooks} require Logger @@ -193,7 +196,6 @@ defmodule Oban.Pro.Plugins.DynamicLifeline do name: :any, repair_limit: :pos_integer, rescue_interval: :pos_integer, - rescue_after: :pos_integer, retry_exhausted: :boolean, timeout: :timeout ) @@ -234,6 +236,7 @@ defmodule Oban.Pro.Plugins.DynamicLifeline do :ok = repair_workflows(state) :ok = repair_chains(state) :ok = repair_partitions(state) + :ok = repair_chunk_ids(state) {:ok, Map.merge(meta, orphan_meta)} end) @@ -336,12 +339,25 @@ defmodule Oban.Pro.Plugins.DynamicLifeline do end defp transition_discarded(query, state) do - Repo.update_all( - state.conf, - where(query, [j], j.attempt >= j.max_attempts), - [set: [state: "discarded", discarded_at: DateTime.utc_now()]], - timeout: state.timeout - ) + {count, partial_jobs} = + Repo.update_all( + state.conf, + where(query, [j], j.attempt >= j.max_attempts), + [set: [state: "discarded", discarded_at: DateTime.utc_now()]], + timeout: state.timeout + ) + + if count > 0 do + ids = Enum.map(partial_jobs, & &1.id) + + Task.start(fn -> + state.conf + |> Repo.all(where(Job, [j], j.id in ^ids), timeout: state.timeout) + |> Enum.each(&Hooks.execute_on_discarded(:exhausted, &1)) + end) + end + + {count, partial_jobs} end # Composition Repair @@ -401,4 +417,43 @@ defmodule Oban.Pro.Plugins.DynamicLifeline do |> then(&Repo.all(state.conf, &1, timeout: state.timeout)) |> Enum.map(fn {queue, partition} -> {queue, normalize_partition.(partition)} end) end + + # Chunk Repair + + defp repair_chunk_ids(state) do + query = + Job + |> where([j], j.state in ~w(available scheduled retryable)) + |> where([j], fragment("? \\? 'chunk_by'", j.meta)) + |> where([j], fragment("NOT ? \\? 'chunk_id'", j.meta)) + |> limit(^state.repair_limit) + + state.conf + |> Repo.all(query, timeout: state.timeout) + |> Enum.group_by(&gen_chunk_id/1) + |> Enum.each(fn {chunk_id, grouped_jobs} -> + ids = Enum.map(grouped_jobs, & &1.id) + + Job + |> where([j], j.id in ^ids) + |> update([j], set: [meta: fragment("jsonb_set(?, '{chunk_id}', ?)", j.meta, ^chunk_id)]) + |> then(&Repo.update_all(state.conf, &1, [], timeout: state.timeout)) + end) + + :ok + end + + defp gen_chunk_id(job) do + chunk_by = normalize_chunk_by(job.meta["chunk_by"]) + + Chunk.to_chunk_id(chunk_by, job.worker, job.args, queue: job.queue, meta: job.meta) + end + + defp normalize_chunk_by(chunk_by) do + Enum.map(chunk_by, fn + "worker" -> :worker + ["args", keys] -> [:args, Enum.map(keys, &Utils.maybe_to_atom/1)] + ["meta", keys] -> [:meta, Enum.map(keys, &Utils.maybe_to_atom/1)] + end) + end end diff --git a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_partitioner.ex b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_partitioner.ex index 0c725f03..3d7db998 100644 --- a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_partitioner.ex +++ b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_partitioner.ex @@ -266,14 +266,19 @@ defmodule Oban.Pro.Plugins.DynamicPartitioner do * `:deleted_count` - the number of partitions deleted """ + @moduledoc deprecated: """ + The complexity and edge cases introduced by dynamic partitioning outweigh the + benefits for most applications. This plugin will be removed in a future version. + """ + @behaviour Oban.Plugin use GenServer alias __MODULE__, as: State + alias Oban.{Config, Peer, Repo, Validation} alias Oban.Cron.Expression alias Oban.Plugins.{Cron, Pruner} - alias Oban.{Config, Peer, Repo, Validation} require Logger @@ -336,7 +341,7 @@ defmodule Oban.Pro.Plugins.DynamicPartitioner do @defaults [ new_prefix: "public", old_prefix: "public", - states: ~w(executing available retryable scheduled cancelled discarded completed), + states: ~w(suspended executing available retryable scheduled cancelled discarded completed), batch_size: 5_000, batch_sleep: 0 ] diff --git a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_prioritizer.ex b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_prioritizer.ex index fff57c0e..4d939e73 100644 --- a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_prioritizer.ex +++ b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_prioritizer.ex @@ -1,6 +1,6 @@ defmodule Oban.Pro.Plugins.DynamicPrioritizer do @moduledoc """ - The DynamicPrioritizer plugin automatically adjusts job's priorities to ensure all jobs are + The `DynamicPrioritizer` plugin automatically adjusts job priorities to ensure all jobs are eventually processed. Using mixed priorities in a queue causes certain jobs to execute before others. For example, a @@ -10,9 +10,13 @@ defmodule Oban.Pro.Plugins.DynamicPrioritizer do When there is a constant flow of high priority jobs the lower priority jobs will never get the chance to run. + The `DynamicPrioritizer` solves this by periodically boosting the priority of jobs that have + been waiting longer than a configured threshold. Each cycle, qualifying jobs have their priority + decremented by one (e.g. from `3` to `2`), gradually converging toward the highest priority. + ## Using the Plugin - To use the `DynamicPrioritizer` plugin add the module to your list of Oban plugins in + To use the `DynamicPrioritizer` plugin, add the module to your list of Oban plugins in `config.exs`: ```elixir @@ -22,43 +26,36 @@ defmodule Oban.Pro.Plugins.DynamicPrioritizer do ``` Without any additional options the plugin will automatically increase the priority of any jobs - that are `available` for 5 minutes or more. To reprioritize after less time waiting you can - configure the `:after` time: + that are `available` for 5 minutes or more, checking once per minute. - ```elixir - plugins: [{Oban.Pro.Plugins.DynamicPrioritizer, after: :timer.minutes(2)}] - ``` + ## Options - Now lower job priorities are bumped after 2 minutes of waiting, and every minute after that. To - lower the reprioritization frequency you can change the `:interval` along with the `:after` - time: + * `:after` — how long a job must be `available` before it becomes eligible for reprioritization, + in milliseconds. Set to `:infinity` to disable reprioritization by default (useful when + relying entirely on overrides). Defaults to `300_000` (5 minutes). - ```elixir - plugins: [{ - Oban.Pro.Plugins.DynamicPrioritizer, - after: :timer.minutes(10), - interval: :timer.minutes(5) - }] - ``` + * `:interval` — the number of milliseconds between reprioritization cycles. Defaults to + `60_000` (1 minute). - Here we've specified that job priority will increase every 5 minutes after the first 10 minutes - of waiting. + * `:limit` — the maximum number of jobs to reprioritize per override or default group in each + cycle. Defaults to `10_000`. - It's also possible to limit the maximum priority jobs may transition to. Typically, jobs will - converge on `0` priority, as that's the highest priority. Set the `max_priority` option to - change to a lower priority: + * `:max_priority` — the highest priority (lowest number) that jobs can be boosted to. For + example, `max_priority: 2` prevents jobs from being boosted above priority `2`, reserving + `0` and `1` for genuinely high priority work. Defaults to `0`. - ```elixir - plugins: [{Oban.Pro.Plugins.DynamicPrioritizer, max_priority: 2}] - ``` + * `:queue_overrides` — a keyword list of per-queue wait thresholds that override `:after` for + specific queues. See "Queue and Worker Overrides" for details. - ## Providing Overrides + * `:worker_overrides` — a keyword list of per-worker wait thresholds that override `:after` for + specific workers. See "Queue and Worker Overrides" for details. - The `after` option applies to jobs for any workers across all queues. The `DynamicPrioritizer` - plugin allows you to specify per-queue and per-worker overrides that fine tune reprioritization. + ## Queue and Worker Overrides - Let's configure reprioritization for the `analysis` queue so that it nudges jobs after only 1 - minute: + The `:after` threshold applies globally to all queues and workers. Overrides let you fine tune + reprioritization for specific queues or workers. + + Configure the `analysis` queue to nudge jobs after only 1 minute: ```elixir plugins: [{ @@ -67,8 +64,7 @@ defmodule Oban.Pro.Plugins.DynamicPrioritizer do }] ``` - We can also effectively disable reprioritization for all other queues by setting the period to - `:infinity`: + Disable reprioritization globally while enabling it for a single queue: ```elixir plugins: [{ @@ -78,13 +74,11 @@ defmodule Oban.Pro.Plugins.DynamicPrioritizer do }] ``` - If per-queue overrides aren't enough we can override on a per-worker basis instead: + Override on a per-worker basis: ```elixir plugins: [{ Oban.Pro.Plugins.DynamicPrioritizer, - interval: :timer.seconds(15), - limit: 20_000, worker_overrides: [ "MyApp.HighSLAWorker": :timer.seconds(30), "MyApp.LowSLAWorker": :timer.minutes(10) @@ -92,7 +86,8 @@ defmodule Oban.Pro.Plugins.DynamicPrioritizer do }] ``` - Naturally you can mix and match overrides to finely control reprioritization: + Queue and worker overrides can be combined. Note that overrides are applied independently—a + job matching both a queue override and a worker override will be boosted by each: ```elixir plugins: [{ diff --git a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_pruner.ex b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_pruner.ex index 36991ada..36058a21 100644 --- a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_pruner.ex +++ b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_pruner.ex @@ -1,9 +1,9 @@ defmodule Oban.Pro.Plugins.DynamicPruner do @moduledoc """ - DynamicPruner enhances the default Pruner plugin's behaviour by allowing you to customize how - jobs are retained in the jobs table. Where the Pruner operates on a fixed schedule and treats - all jobs the same, with the DynamicPruner, you can use a flexible CRON schedule and provide - custom rules for specific queues, workers, and job states. + The `DynamicPruner` enhances the default Pruner plugin's behaviour by allowing you to customize + how jobs are retained in the jobs table. Where the Pruner operates on a fixed schedule and + treats all jobs the same, with the DynamicPruner, you can use a flexible CRON schedule and + provide custom rules for specific queues, workers, and job states. ## Using the Plugin @@ -26,7 +26,7 @@ defmodule Oban.Pro.Plugins.DynamicPruner do Now the pruner will retain the most recent 50,000 jobs instead. A fixed limit on the number of jobs isn't always ideal. Often you want to retain jobs based on - their age instead. For example, if you your application needs to ensure that a duplicate job + their age instead. For example, if your application needs to ensure that a duplicate job hasn't been enqueued within the past 24 hours you need to retain jobs for at least 24 hours; a fixed limit simply won't work. For that we can use maximum age (`max_age`) mode instead: @@ -39,7 +39,7 @@ defmodule Oban.Pro.Plugins.DynamicPruner do Calculating the number of seconds in a period isn't especially readable, particularly when you have numerous `max_age` declarations in overrides (see below). For clarity you can specify the - age's time unit as `:second`, `:minute`, `:hour`, `:day` or `:month`. Here is the same 48 hour + age's time unit as `:second`, `:minute`, `:hour`, `:day`, `:week`, or `:month`. Here is the same 48 hour configuration from above, but specified in terms of days: ```elixir @@ -49,6 +49,43 @@ defmodule Oban.Pro.Plugins.DynamicPruner do Now you can tell exactly how long jobs should be retained, without reverse calculating how many seconds an expression represents. + ## Options + + * `:mode` — the pruning strategy, either `{:max_len, count}` to retain the most recent `count` + jobs, or `{:max_age, age}` to retain jobs newer than `age`. The age can be an integer in + seconds or a `{value, unit}` tuple where unit is `:second`, `:minute`, `:hour`, `:day`, + `:week`, or `:month` (plural forms also accepted). Defaults to `{:max_len, 1_000}`. + + * `:queue_overrides` — a keyword list of `{queue, mode}` tuples that override the default mode + for specific queues. See [Providing Overrides](#module-providing-overrides). + + * `:state_overrides` — a keyword list of `{state, mode}` tuples that override the default mode + for specific states (`:completed`, `:cancelled`, or `:discarded`). See + [Providing Overrides](#module-providing-overrides). + + * `:worker_overrides` — a keyword list of `{worker, mode}` tuples that override the default + mode for specific workers. See [Providing Overrides](#module-providing-overrides). + + * `:schedule` — a cron expression controlling when pruning runs. Defaults to `"* * * * *"` + (every minute). See [Setting a Schedule](#module-setting-a-schedule). + + * `:timezone` — the timezone used to evaluate the cron schedule. Defaults to `"Etc/UTC"`. + + * `:limit` — the maximum number of jobs deleted per pruning pass. Exists to prevent connection + timeouts and excessive table locks. Defaults to `10_000`. See + [Keeping Up With Inserts](#module-keeping-up-with-inserts). + + * `:timeout` — the maximum time allowed for each pruning query, in milliseconds. Defaults to + `60_000`. + + * `:preserve_workflows` — when `true`, jobs belonging to active workflows are retained + regardless of their age or position. Defaults to `true`. See + [Workflow Preservation](#module-workflow-preservation). + + * `:before_delete` — an `{module, function, args}` callback invoked with job ids before they + are deleted, within the same transaction. See + [Executing a Callback Before Delete](#module-executing-a-callback-before-delete). + ## Providing Overrides The `mode` option is indiscriminate when determining which jobs to prune. It pays no attention @@ -150,7 +187,7 @@ defmodule Oban.Pro.Plugins.DynamicPruner do ## Retaining Jobs Forever To retain jobs in a queue, state, or for a particular worker forever (without using something - like `{:max_age, {999, :years}}` use `:infinity` as the length or duration: + like `{:max_age, {999, :years}}`), use `:infinity` as the length or duration: ```elixir plugins: [{ @@ -213,9 +250,9 @@ defmodule Oban.Pro.Plugins.DynamicPruner do ## Executing a Callback Before Delete Sometimes jobs are a historic record of activity and it's desirable to operate on them _before_ - they're deleted. For example, you may want copy some jobs into cold storage prior to completion. + they're deleted. For example, you may want to copy some jobs into cold storage prior to completion. - To accomplish this, specify a callback to execute before proceeding with the deletion.: + To accomplish this, specify a callback to execute before proceeding with the deletion: ```elixir defmodule DeleteHandler do @@ -252,6 +289,28 @@ defmodule Oban.Pro.Plugins.DynamicPruner do before_delete: {DeleteHandler, :call, [ColdStorage]} ``` + ## Workflow Preservation + + By default, jobs that are part of an active workflow are retained to prevent partial deletion of + workflow data while the workflow is still running. A workflow is considered active if it + contains any jobs in an incomplete state. + + This check includes linked sub-workflows and parent workflows, ensuring that all related jobs + across workflow hierarchies are preserved until the entire workflow completes. + + To disable workflow preservation and prune all eligible jobs regardless of workflow status: + + ```elixir + plugins: [{ + Oban.Pro.Plugins.DynamicPruner, + mode: {:max_age, {7, :days}}, + preserve_workflows: false + }] + ``` + + Disabling workflow preservation can improve pruning performance for systems with many workflows, + but may result in incomplete workflow job histories if workflows are pruned while still active. + ## Implementation Notes Some additional notes about pruning in general and nuances of the `DynamicPruner` plugin: @@ -264,8 +323,9 @@ defmodule Oban.Pro.Plugins.DynamicPruner do scheduled, or retryable job. * Jobs that are part of an active workflow are retained regardless of their state. A workflow - is considered active if it contains any jobs in `scheduled`, `retryable`, `available`, or - `executing` states. + is considered active if it contains any jobs in `suspended`, `scheduled`, `retryable`, + `available`, or `executing` states. This behavior is controlled by the `preserve_workflows` + option, which defaults to `true`. * Only a single node will prune at any given time, which prevents potential deadlocks between transactions. @@ -274,6 +334,7 @@ defmodule Oban.Pro.Plugins.DynamicPruner do The `DynamicPruner` plugin adds the following metadata to the `[:oban, :plugin, :stop]` event: + * `:pruned_count` - the total number of jobs that were deleted * `:pruned_jobs` - the jobs that were deleted from the database _Note: jobs only include `id`, `queue`, and `state` fields._ @@ -286,8 +347,9 @@ defmodule Oban.Pro.Plugins.DynamicPruner do import Ecto.Query alias Oban.Cron.Expression - alias Oban.Plugins.Cron alias Oban.{Job, Peer, Repo, Validation} + alias Oban.Plugins.Cron + alias Oban.Pro.Workflow.Schema, as: Workflow require Logger @@ -303,7 +365,7 @@ defmodule Oban.Pro.Plugins.DynamicPruner do | :week | :weeks | :month - | :month + | :months @type max_age :: pos_integer() | {pos_integer(), time_unit()} @@ -348,6 +410,7 @@ defmodule Oban.Pro.Plugins.DynamicPruner do :timer, limit: 10_000, mode: {:max_len, 1_000}, + preserve_workflows: true, queue_overrides: [], state_overrides: [], timeout: :timer.seconds(60), @@ -419,6 +482,7 @@ defmodule Oban.Pro.Plugins.DynamicPruner do limit: :pos_integer, mode: {:custom, &validate_mode/1}, name: :any, + preserve_workflows: :boolean, queue_overrides: {:custom, &validate_overrides(:queue_overrides, &1)}, state_overrides: {:custom, &validate_overrides(:state_overrides, &1)}, schedule: :schedule, @@ -454,7 +518,15 @@ defmodule Oban.Pro.Plugins.DynamicPruner do meta = %{conf: state.conf, plugin: __MODULE__} :telemetry.span([:oban, :plugin], meta, fn -> - with {:ok, extra} <- prune_jobs(state), do: {:ok, Map.merge(meta, extra)} + with {:ok, jobs_extra} <- prune_jobs(state), + {:ok, workflows_extra} <- prune_workflows(state) do + extra = + meta + |> Map.merge(jobs_extra) + |> Map.merge(workflows_extra) + + {:ok, extra} + end end) {:noreply, schedule_prune(state)} @@ -549,7 +621,7 @@ defmodule Oban.Pro.Plugins.DynamicPruner do |> select([j], %{id: j.id, queue: j.queue, state: j.state, rn: 100_000_000}) |> where( [j], - (j.state == "completed" and j.scheduled_at < ^time) or + (j.state == "completed" and j.completed_at < ^time) or (j.state == "cancelled" and j.cancelled_at < ^time) or (j.state == "discarded" and j.discarded_at < ^time) ) @@ -569,22 +641,10 @@ defmodule Oban.Pro.Plugins.DynamicPruner do end defp delete_all(query, offset, state) do - workflow_subquery = - Job - |> where([j], j.state in ~w(scheduled retryable available executing)) - |> where([j], fragment("? \\? 'workflow_id'", parent_as(:jobs).meta)) - |> where( - [j], - same_workflow(j.meta, parent_as(:jobs).meta) or - same_sub_workflow(j.meta, parent_as(:jobs).meta) or - same_sup_workflow(j.meta, parent_as(:jobs).meta) - ) - |> select(1) - subquery = query |> from(as: :jobs) - |> where([j], not exists(workflow_subquery)) + |> maybe_preserve_workflows(state.preserve_workflows) |> order_by(asc: :id) |> limit(^state.limit) @@ -604,6 +664,24 @@ defmodule Oban.Pro.Plugins.DynamicPruner do deleted end + defp maybe_preserve_workflows(query, false), do: query + + defp maybe_preserve_workflows(query, true) do + workflow_subquery = + Job + |> where([j], j.state in ~w(suspended scheduled retryable available executing)) + |> where([j], fragment("? \\? 'workflow_id'", parent_as(:jobs).meta)) + |> where( + [j], + same_workflow(j.meta, parent_as(:jobs).meta) or + same_sub_workflow(j.meta, parent_as(:jobs).meta) or + same_sup_workflow(j.meta, parent_as(:jobs).meta) + ) + |> select(1) + + where(query, [j], not exists(workflow_subquery)) + end + defp apply_before_delete(query, state) do with {mod, fun, args} <- state.before_delete do ids = @@ -617,6 +695,19 @@ defmodule Oban.Pro.Plugins.DynamicPruner do :ok end + # Workflows + + defp prune_workflows(state) do + query = + Workflow + |> where([wf], wf.state in ~w(completed cancelled discarded)) + |> where([wf], wf.completed + wf.cancelled + wf.discarded == 0) + + {count, _} = Repo.delete_all(state.conf, query, []) + + {:ok, %{pruned_workflows_count: count}} + end + # Timestamp defp to_timestamp(seconds) when is_integer(seconds) do diff --git a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_queues.ex b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_queues.ex index 77b34209..2e7e9b24 100644 --- a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_queues.ex +++ b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_queues.ex @@ -1,6 +1,6 @@ defmodule Oban.Pro.Plugins.DynamicQueues do @moduledoc """ - The `DynamicQueue` plugin extends Oban's basic queue management by persisting queue changes + The `DynamicQueues` plugin extends Oban's basic queue management by persisting queue changes across restarts, globally, across all connected nodes. It also boasts a declarative syntax for specifying which nodes a queue will run on. @@ -57,6 +57,18 @@ defmodule Oban.Pro.Plugins.DynamicQueues do queues start to shine when you insert, update, or delete them _dynamically_, either through Oban Web or your own application code. + ## Options + + * `:queues` — a keyword list of queue definitions, where each key is the queue name and the + value is either an integer local limit or a keyword list of queue options. See + [Using and Configuring](#module-using-and-configuring) for the full syntax. + + * `:sync_mode` — controls how queues are synchronized during startup. Either `:manual` + (default) or `:automatic`. See [Synchronizing Queues](#module-synchronizing-queues). + + * `:interval` — how often (in milliseconds) to poll the database for queue changes. Defaults + to `:infinity` (polling disabled). See [Enabling Polling Mode](#module-enabling-polling-mode). + ### Synchronizing Queues The `:sync_mode` option controls how DynamicQueues handles queue synchronization during startup. @@ -104,7 +116,7 @@ defmodule Oban.Pro.Plugins.DynamicQueues do ### Limiting Where Queues Run Dynamic queues can be configured to run on a subset of available nodes. This is especially - useful when wish to restrict resource-intensive queues to only dedicated nodes. Restriction is + useful when you wish to restrict resource-intensive queues to only dedicated nodes. Restriction is configured through the `only` option, which you use like this: queues: [ @@ -210,9 +222,9 @@ defmodule Oban.Pro.Plugins.DynamicQueues do import Ecto.Query alias Ecto.Changeset + alias Oban.{Config, Midwife, Notifier, Peer, Registry, Repo, Validation} alias Oban.Pro.Engines.Smart alias Oban.Pro.{Queue, Utils} - alias Oban.{Config, Midwife, Notifier, Peer, Registry, Repo, Validation} require Logger @@ -273,7 +285,7 @@ defmodule Oban.Pro.Plugins.DynamicQueues do @doc """ Retrieve all persisted queues. - While it's possible to modify queue's returned from `all/0`, it is recommended that you use + While it's possible to modify queues returned from `all/0`, it is recommended that you use `update/2` to ensure options are cast and validated correctly. ## Examples @@ -359,7 +371,7 @@ defmodule Oban.Pro.Plugins.DynamicQueues do ) Updating a single option won't remove other persisted options. If you'd like to - clear an uption you must set them to `nil`: + clear an option you must set them to `nil`: DynamicQueues.update(:video, global_limit: nil) @@ -415,7 +427,7 @@ defmodule Oban.Pro.Plugins.DynamicQueues do ## Examples - Delete ethe "audio" queue: + Delete the "audio" queue: {:ok, _} = DynamicQueues.delete(:audio) """ diff --git a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_scaler.ex b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_scaler.ex index 6f727d0a..82e9608a 100644 --- a/vendor/oban_pro/lib/oban/pro/plugins/dynamic_scaler.ex +++ b/vendor/oban_pro/lib/oban/pro/plugins/dynamic_scaler.ex @@ -7,18 +7,18 @@ defmodule Oban.Pro.Plugins.DynamicScaler do e.g. staging. Horizontal scaling is applied at the _node_ level, not the _queue_ level, so you can distribute - processing over more phyiscal hardware. + processing over more physical hardware. * Predictive Scaling — The optimal scale is calculated by predicting the future size of a queue based on recent trends. Multiple samples are then used to prevent overreacting to changes in - queue depth or throughput. Your provide an acceptible `range` of nodes and auto-scaling takes + queue depth or throughput. You provide an acceptable `range` of nodes and auto-scaling takes care of the rest. * Multi-Cloud — Cloud integration is provided by a simple, flexible, behaviour that you implement for your specific environment and configure for each scaler. * Queue Filtering — By default, all queues are considered for scale calculations. However, you - can restrict calculations to one or more ciritical queues. + can restrict calculations to one or more critical queues. * Multiple Scalers — Some systems may restrict work to specific node types, e.g. generating exports or processing videos. Other hybrid systems may straddle multiple clouds. In either @@ -58,10 +58,17 @@ defmodule Oban.Pro.Plugins.DynamicScaler do Now, every minute, `DynamicScaler` will calculate the optimal number of nodes for your queue's throughput and issue scaling commands accordingly. + ## Options + + * `:scalers` — one or more scaler configurations. A single scaler is a keyword list of scaler + options (see below). Multiple scalers are a list of keyword lists. Required. + + * `:timeout` — the maximum time allowed for throughput queries, in milliseconds. Defaults to + `15_000`. See [Optimizing Throughput Queries](#module-optimizing-throughput-queries). + ### Configuring Scalers - Scalers have options beyond `:cloud` and `:range` for more advanced systems or to constrain - resource usage. Here's a breakdown of all options, followed by specific examples of each. + Each scaler accepts the following options: * `:cloud` — A `module` or `{module, options}` tuple that interacts with the external cloud during scale events. Required. @@ -228,8 +235,8 @@ defmodule Oban.Pro.Plugins.DynamicScaler do import Ecto.Query alias __MODULE__, as: State - alias Oban.Plugins.Cron alias Oban.{Job, Notifier, Peer, Repo, Validation} + alias Oban.Plugins.Cron require Logger @@ -503,7 +510,7 @@ defmodule Oban.Pro.Plugins.DynamicScaler do {:error, "expected :range to have a min less than max, got: #{inspect(range)}"} _ -> - {:error, "exepcted :range to be a range of integers, got: #{inspect(range)}"} + {:error, "expected :range to be a range of integers, got: #{inspect(range)}"} end end diff --git a/vendor/oban_pro/lib/oban/pro/producer.ex b/vendor/oban_pro/lib/oban/pro/producer.ex index d7f9b7ca..0e5fade7 100644 --- a/vendor/oban_pro/lib/oban/pro/producer.ex +++ b/vendor/oban_pro/lib/oban/pro/producer.ex @@ -97,6 +97,10 @@ defmodule Oban.Pro.Producer do embeds_one :rate_limit, RateLimit, on_replace: :update, primary_key: false do @moduledoc false + field :algorithm, Ecto.Enum, + values: ~w(sliding_window fixed_window token_bucket)a, + default: :sliding_window + field :allowed, :integer field :period, :integer field :window_time, :integer, skip_default_validation: true @@ -235,15 +239,25 @@ defmodule Oban.Pro.Producer do |> Map.update(:partition, nil, &normalize_partition/1) |> Map.put_new_lazy(:window_time, fn -> DateTime.to_unix(DateTime.utc_now(), :second) end) + fields = schema.__struct__.__schema__(:fields) -- [:partition] + schema - |> cast(params, ~w(allowed period window_time)a) + |> cast(params, fields) |> cast_embed(:partition, with: &partition_changeset/2) |> validate_required(~w(allowed period)a) |> validate_number(:allowed, greater_than: 0) |> validate_number(:period, greater_than: 0) + |> clear_windows_on_algorithm_change() |> Utils.enforce_keys(params, schema.__struct__) end + defp clear_windows_on_algorithm_change(changeset) do + case get_change(changeset, :algorithm) do + nil -> changeset + _new_algorithm -> force_change(changeset, :windows, %{}) + end + end + @doc false @spec partition_changeset(Ecto.Schema.t(), map() | Keyword.t()) :: Changeset.t() def partition_changeset(schema, params) do @@ -280,11 +294,15 @@ defmodule Oban.Pro.Producer do end def cast_rate_limit(opts) - when is_map_key(opts, "allowed") or + when is_map_key(opts, "algorithm") or + is_map_key(opts, "allowed") or is_map_key(opts, "burst") or is_map_key(opts, "period") or is_map_key(opts, "partition") do Map.new(opts, fn + {"algorithm", algorithm} -> + {:algorithm, String.to_existing_atom(algorithm)} + {"allowed", allowed} -> {:allowed, allowed} diff --git a/vendor/oban_pro/lib/oban/pro/queue.ex b/vendor/oban_pro/lib/oban/pro/queue.ex index 21c976f8..24465039 100644 --- a/vendor/oban_pro/lib/oban/pro/queue.ex +++ b/vendor/oban_pro/lib/oban/pro/queue.ex @@ -50,6 +50,10 @@ defmodule Oban.Pro.Queue do embeds_one :rate_limit, RateLimit, on_replace: :update, primary_key: false do @moduledoc false + field :algorithm, Ecto.Enum, + values: ~w(sliding_window fixed_window token_bucket)a, + default: :sliding_window + field :allowed, :integer field :period, :integer field :window_time, :integer, virtual: true, skip_default_validation: true diff --git a/vendor/oban_pro/lib/oban/pro/rate_limit.ex b/vendor/oban_pro/lib/oban/pro/rate_limit.ex new file mode 100644 index 00000000..5e2918a5 --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/rate_limit.ex @@ -0,0 +1,384 @@ +defmodule Oban.Pro.RateLimit do + @moduledoc """ + Programmatic API for inspecting and manipulating rate limit state. + + Rate limits are normally managed automatically by the Smart engine as jobs execute. This module + provides functions to interact with rate limits outside of normal job execution: checking + available capacity, manually consuming tokens, and resetting state. + + All functions in this module require the [Smart engine](`Oban.Pro.Engines.Smart`) and a queue + configured with `rate_limit` options. + + ## Checking Available Capacity + + Use `available/2` to check how much capacity remains before performing operations: + + ```elixir + case Oban.Pro.RateLimit.available(:some_api) do + {:ok, capacity} when capacity >= 10 -> + make_api_calls(10) + + {:ok, capacity} when capacity > 0 -> + make_api_calls(capacity) + + {:ok, 0} -> + {:snooze, 60} + end + ``` + + For partitioned rate limits, check a specific partition with a job's computed `partition_key`: + + ```elixir + Oban.Pro.RateLimit.available(:some_api, partition: job.meta["partition_key"]) + ``` + + ## Manual Consumption + + Use `consume/3` when you need to track rate-limited operations that happen outside of job + execution. This ensures the rate limit reflects all usage, not just job execution. Consumption + can even track operations entirely outside of Oban: + + ```elixir + def handle_webhook(conn, params) do + :ok = Oban.Pro.RateLimit.consume(:api_calls, 1) + result = ExternalAPI.process(params) + json(conn, result) + end + ``` + + ## Resetting State + + Use `reset/2` to clear all rate limit tracking. This is primarily for recovery after + configuration changes when limits should be cleared: + + ```elixir + :ok = Oban.Pro.RateLimit.reset(:api_calls) + ``` + + ## See Also + + * `Oban.Pro.Engines.Smart` — Rate limit configuration and algorithms + * `Oban.Pro.Worker` — Weighted jobs with the `weight/1` callback + """ + @moduledoc since: "1.7.0" + + import Ecto.Query + import DateTime, only: [utc_now: 0] + + alias Oban.Pro.Limiters.Rate.Algorithm + alias Oban.Pro.Producer + alias Oban.Repo + + @type queue :: atom() | String.t() + @type option :: {:oban, Oban.name()} | {:partition, String.t()} + @type wait_option :: {:timeout, timeout()} | {:interval, pos_integer()} + + @doc """ + Manually consume rate limit capacity for a queue. + + Consumption is applied to the producer with the most available capacity. If the requested + amount exceeds a single producer's capacity, consumption is spread across multiple producers. + + ## Options + + * `:oban` - The Oban instance name. Defaults to `Oban`. + * `:partition` - The partition key to consume from. Defaults to `"*"` (the global partition). + * `:require_full` - When `true`, returns `{:error, :insufficient_capacity}` if the full + amount can't be consumed. Defaults to `false`, which consumes as much as available. + + ## Examples + + Consume 5 units from the default partition + + Oban.Pro.RateLimit.consume(:my_queue, 5) + + Consume from a specific partition + + Oban.Pro.RateLimit.consume(:my_queue, 3, partition: job.meta["partition_key"]) + + Use a named Oban instance + + Oban.Pro.RateLimit.consume(:my_queue, 1, oban: MyApp.Oban) + """ + @spec consume(queue(), pos_integer(), [option()]) :: + :ok | {:error, :insufficient_capacity | :no_rate_limit | :queue_not_found} + def consume(queue, amount, opts \\ []) when is_integer(amount) and amount > 0 do + conf = fetch_conf(opts) + partition = Keyword.get(opts, :partition, "*") + require_full = Keyword.get(opts, :require_full, false) + + func = fn -> + with {:ok, producers} <- fetch_producers(conf, queue, take_lock: true), + {:ok, rate_limit} <- fetch_rate_limit(hd(producers)) do + curr_time = unix_now() + callback = Algorithm.callback(rate_limit) + + sorted = + producers + |> Enum.map(fn producer -> + rl = producer.meta.rate_limit + + capacity = + callback.demand(rl.windows, rl.allowed, rl.period, curr_time, rl.window_time) + + {producer, capacity} + end) + |> Enum.sort_by(&elem(&1, 1), :desc) + + total_capacity = Enum.reduce(sorted, 0, fn {_, cap}, acc -> acc + cap end) + + if require_full and total_capacity < amount do + {:error, :insufficient_capacity} + else + consume(sorted, conf, partition, amount, curr_time, callback) + :ok + end + end + end + + with {:ok, result} <- Repo.transaction(conf, func), do: result + end + + @doc """ + Check the available rate limit capacity for a queue. + + Returns the total available capacity across all producers for the queue, calculated by merging + window states from all active producers. + + ## Options + + * `:oban` - The Oban instance name. Defaults to `Oban`. + * `:partition` - The partition key to check. Defaults to `"*"` (the global partition). + + ## Examples + + Check capacity for a queue: + + {:ok, capacity} = Oban.Pro.RateLimit.available(:my_queue) + + Check capacity for a specific partition: + + {:ok, capacity} = Oban.Pro.RateLimit.available(:my_queue, partition: job.meta["partition_key"]) + """ + @spec available(queue(), [option()]) :: + {:ok, non_neg_integer()} | {:error, :no_rate_limit | :queue_not_found} + def available(queue, opts \\ []) do + conf = fetch_conf(opts) + partition = Keyword.get(opts, :partition, "*") + + with {:ok, producers} <- fetch_producers(conf, queue), + {:ok, rate_limit} <- fetch_rate_limit(hd(producers)) do + curr_time = unix_now() + callback = Algorithm.callback(rate_limit) + + windows = + producers + |> Enum.map(& &1.meta.rate_limit) + |> Enum.filter(&(&1.window_time >= curr_time - &1.period)) + |> Enum.reduce(%{}, fn rl, acc -> callback.merge(rl.windows, acc) end) + + demand = + if is_nil(rate_limit.partition) do + callback.demand( + windows, + rate_limit.allowed, + rate_limit.period, + curr_time, + rate_limit.window_time + ) + else + demands = + callback.partition_demands( + windows, + rate_limit.allowed, + rate_limit.period, + curr_time, + rate_limit.window_time, + [partition] + ) + + Map.get(demands, partition, 0) + end + + {:ok, demand} + end + end + + @doc """ + Execute a function after atomically reserving rate limit capacity. + + This function waits for capacity to become available, atomically consumes the requested amount, + then executes the provided function. This prevents race conditions where multiple callers might + consume the same quota. + + ## Options + + * `:oban` - The Oban instance name. Defaults to `Oban`. + * `:partition` - The partition key to check. Defaults to `"*"` (the global partition). + * `:timeout` - Maximum time to wait in milliseconds. Defaults to `5_000` (5 seconds). + * `:interval` - Polling interval in milliseconds. Defaults to `100`. + + ## Examples + + Execute a function after reserving 5 units of capacity: + + {:ok, result} = Oban.Pro.RateLimit.with_quota(:my_queue, 5, fn -> + ExternalAPI.batch_request(items) + end) + + Handle timeout when capacity isn't available: + + case Oban.Pro.RateLimit.with_quota(:my_queue, 5, &make_api_calls/0, timeout: 10_000) do + {:ok, result} -> handle_result(result) + {:error, :timeout} -> handle_timeout() + end + + Reserve capacity on a specific partition: + + {:ok, result} = Oban.Pro.RateLimit.with_quota(:my_queue, 1, fun, partition: "user_123") + """ + @spec with_quota(queue(), pos_integer(), (-> result), [option() | wait_option()]) :: + {:ok, result} | {:error, :timeout | :no_rate_limit | :queue_not_found} + when result: term() + def with_quota(queue, amount, fun, opts \\ []) when is_integer(amount) and amount > 0 do + {timeout, opts} = Keyword.pop(opts, :timeout, 5_000) + {interval, opts} = Keyword.pop(opts, :interval, 100) + + deadline = System.monotonic_time(:millisecond) + timeout + + with_quota_loop(queue, amount, fun, opts, deadline, interval) + end + + defp with_quota_loop(queue, amount, fun, opts, deadline, interval) do + case consume(queue, amount, Keyword.put(opts, :require_full, true)) do + :ok -> + {:ok, fun.()} + + {:error, :insufficient_capacity} -> + remaining = deadline - System.monotonic_time(:millisecond) + + if remaining <= 0 do + {:error, :timeout} + else + Process.sleep(min(interval, remaining)) + + with_quota_loop(queue, amount, fun, opts, deadline, interval) + end + + {:error, _reason} = error -> + error + end + end + + @doc """ + Reset the rate limit state for a queue. + + This clears all window data and resets the window time for all producers on the queue. Tracking + is cleared across all partitions for partitioned queues. + + ## Options + + * `:oban` - The Oban instance name. Defaults to `Oban`. + + ## Examples + + Reset the rate limit for a queue: + + :ok = Oban.Pro.RateLimit.reset(:my_queue) + + Reset the rate limit for a queue: + + :ok = Oban.Pro.RateLimit.reset(:my_queue, oban: MyApp.Oban) + """ + @spec reset(queue(), [option()]) :: :ok | {:error, :no_rate_limit | :queue_not_found} + def reset(queue, opts \\ []) do + conf = fetch_conf(opts) + + func = fn -> + with {:ok, producers} <- fetch_producers(conf, queue, take_lock: true), + {:ok, _rate_lim} <- fetch_rate_limit(hd(producers)) do + curr_time = unix_now() + + Enum.each(producers, fn producer -> + rate_limit = producer.meta.rate_limit + updated = %{rate_limit | windows: %{}, window_time: curr_time} + + update_rate_limit(conf, producer, updated) + end) + end + end + + with {:ok, result} <- Repo.transaction(conf, func), do: result + end + + # Private Helpers + + defp fetch_conf(opts) do + opts + |> Keyword.get(:oban, Oban) + |> Oban.config() + end + + defp fetch_producers(conf, queue, opts \\ []) do + query = + if opts[:take_lock] do + Producer + |> where(queue: ^to_string(queue)) + |> lock("FOR UPDATE") + else + where(Producer, queue: ^to_string(queue)) + end + + case Repo.all(conf, query) do + [_ | _] = producers -> {:ok, producers} + _ -> {:error, :queue_not_found} + end + end + + defp fetch_rate_limit(producer) do + case producer.meta.rate_limit do + %{} = rate_limit -> {:ok, rate_limit} + nil -> {:error, :no_rate_limit} + end + end + + defp consume([], _, _, _, _, _), do: :ok + defp consume(_, _, _, 0, _, _), do: :ok + + defp consume([{producer, capacity} | rest], conf, partition, remaining, curr_time, callback) do + to_consume = min(capacity, remaining) + + if to_consume > 0 do + rate_limit = producer.meta.rate_limit + new_counts = %{partition => to_consume} + + {all_windows, next_time} = + callback.track( + rate_limit.windows, + new_counts, + rate_limit.period, + rate_limit.window_time, + curr_time, + rate_limit.allowed + ) + + updated = %{rate_limit | windows: all_windows, window_time: next_time} + + update_rate_limit(conf, producer, updated) + end + + consume(rest, conf, partition, remaining - to_consume, curr_time, callback) + end + + defp update_rate_limit(conf, producer, rate_limit) do + meta = %{producer.meta | rate_limit: rate_limit} + + Producer + |> where(uuid: ^producer.uuid) + |> then(&Repo.update_all(conf, &1, set: [meta: meta, updated_at: utc_now()])) + end + + defp unix_now do + DateTime.to_unix(utc_now(), :second) + end +end diff --git a/vendor/oban_pro/lib/oban/pro/refresher.ex b/vendor/oban_pro/lib/oban/pro/refresher.ex index d8125143..e87e1d94 100644 --- a/vendor/oban_pro/lib/oban/pro/refresher.ex +++ b/vendor/oban_pro/lib/oban/pro/refresher.ex @@ -11,8 +11,10 @@ defmodule Oban.Pro.Refresher do import Ecto.Query, only: [where: 3] alias __MODULE__, as: State - alias Oban.Pro.Producer alias Oban.{Peer, Repo} + alias Oban.Pro.Producer + + require Logger defstruct [ :timer, @@ -53,8 +55,18 @@ defmodule Oban.Pro.Refresher do @impl GenServer def handle_info(:refresh, state) do - refresh_producers(state) - cleanup_producers(state) + try do + refresh_producers(state) + cleanup_producers(state) + rescue + error -> + Logger.warning( + message: "Producer refresh failed", + source: :oban_pro, + module: __MODULE__, + reason: error + ) + end {:noreply, schedule_refresh(state)} end diff --git a/vendor/oban_pro/lib/oban/pro/relay.ex b/vendor/oban_pro/lib/oban/pro/relay.ex index 21a2f574..9c55a75c 100644 --- a/vendor/oban_pro/lib/oban/pro/relay.ex +++ b/vendor/oban_pro/lib/oban/pro/relay.ex @@ -96,7 +96,7 @@ defmodule Oban.Pro.Relay do ## Usage with Chunks `Relay` is intended for use with a single job and isn't suited to awaiting results from - `Oban.Pro.Workers.Chunk` jobs. That's because only one of the chunk's jobs (the leader) will + `Oban.Pro.Chunk` jobs. That's because only one of the chunk's jobs (the leader) will relay results back. Awaiting any other job in the chunk will time out without returning a proper result. """ @@ -104,8 +104,8 @@ defmodule Oban.Pro.Relay do @behaviour Oban.Pro.Handler alias Ecto.{Changeset, UUID} - alias Oban.Pro.{Unique, Utils} alias Oban.{Job, Notifier} + alias Oban.Pro.{Unique, Utils} require Logger @@ -372,7 +372,7 @@ defmodule Oban.Pro.Relay do # Messaging defp check_ownership!(pid) do - unless pid == self() do + if pid != self() do raise ArgumentError, "relay must be awaited by #{inspect(pid)}, was awaited by #{inspect(self())}" end diff --git a/vendor/oban_pro/lib/oban/pro/stages/chain.ex b/vendor/oban_pro/lib/oban/pro/stages/chain.ex index 68af9fba..f65109b5 100644 --- a/vendor/oban_pro/lib/oban/pro/stages/chain.ex +++ b/vendor/oban_pro/lib/oban/pro/stages/chain.ex @@ -10,8 +10,6 @@ defmodule Oban.Pro.Stages.Chain do alias Oban.{Job, Repo, Validation} alias Oban.Pro.Utils - @hold_date ~U[3000-01-01 00:00:00.000000Z] - defstruct [:by, :worker, on_cancelled: :ignore, on_discarded: :ignore] defmacrop drop_hold(meta) do @@ -20,29 +18,39 @@ defmodule Oban.Pro.Stages.Chain do end end - defmacrop continue_state(prefix, meta) do + defmacrop continue_state(prefix, meta, scheduled_at, now) do quote do fragment( """ - (CASE WHEN ? \\? 'orig_scheduled_at' THEN 'scheduled' ELSE 'available' END)::?.oban_job_state + (CASE + WHEN ? \\? 'orig_scheduled_at' THEN 'scheduled' + WHEN ? > ? THEN 'scheduled' + ELSE 'available' + END)::?.oban_job_state """, unquote(meta), + unquote(scheduled_at), + unquote(now), unquote(prefix) ) end end - defmacrop continue_at(meta, now) do + defmacrop continue_at(meta, scheduled_at, now) do quote do fragment( """ CASE - WHEN ? \\? 'orig_scheduled_at' THEN timezone('utc', to_timestamp(div((?->'orig_scheduled_at')::bigint, 1000000)::float)) - ELSE ? + WHEN ? \\? 'orig_scheduled_at' THEN timezone('utc', to_timestamp(div((?->'orig_scheduled_at')::bigint, 1000000)::float)) + WHEN ? > ? THEN ? + ELSE ? END """, unquote(meta), unquote(meta), + unquote(scheduled_at), + unquote(now), + unquote(scheduled_at), unquote(now) ) end @@ -68,53 +76,46 @@ defmodule Oban.Pro.Stages.Chain do (state == "discarded" and meta["on_discarded"] != "hold") end - @spec to_postponed(Job.changeset()) :: Job.changeset() - def to_postponed(changeset) do - meta = - if Changeset.get_field(changeset, :state) == "scheduled" do - orig_at = - changeset - |> Changeset.get_change(:scheduled_at) - |> DateTime.to_unix(:microsecond) - - %{"on_hold" => true, "orig_scheduled_at" => orig_at} - else - %{"on_hold" => true} - end - - changeset - |> Changeset.update_change(:meta, &Map.merge(&1, meta)) - |> Changeset.put_change(:state, "scheduled") - |> Changeset.put_change(:scheduled_at, @hold_date) + @spec to_suspended(Job.changeset()) :: Job.changeset() + def to_suspended(changeset) do + Changeset.put_change(changeset, :state, "suspended") end @doc false def rescue_chains(conf, opts \\ []) do prev_query = Job - |> where([j], j.state in ~w(available scheduled retryable executing)) + |> where([j], j.state in ~w(available suspended scheduled retryable executing)) |> where([j], fragment("? \\? 'chain_id'", j.meta)) |> where([j], same_chain(j.meta, parent_as(:jobs).meta)) - |> where([j], fragment("?->>'on_hold'", j.meta) in ~w(true false)) |> where([j], j.id < parent_as(:jobs).id) subquery = from(j in Job, as: :jobs) - |> where([j], j.state == "scheduled") |> where([j], fragment("? \\? 'chain_id'", j.meta)) - |> where([j], fragment("?->>'on_hold' = 'true'", j.meta)) |> where([j], not exists(prev_query)) |> order_by(asc: :id) |> limit(1) + subquery = + if Utils.has_legacy_suspended?(conf) do + subquery + |> where([j], j.state == "scheduled") + |> where([j], fragment("?->>'on_hold' = 'true'", j.meta)) + else + where(subquery, [j], j.state == "suspended") + end + + now = DateTime.utc_now() + query = Job |> join(:inner, [j], x in subquery(subquery), on: j.id == x.id) |> update([j], set: [ meta: drop_hold(j.meta), - state: continue_state(literal(^conf.prefix), j.meta), - scheduled_at: continue_at(j.meta, ^DateTime.utc_now()) + state: continue_state(literal(^conf.prefix), j.meta, j.scheduled_at, ^now), + scheduled_at: continue_at(j.meta, j.scheduled_at, ^now) ] ) @@ -159,8 +160,7 @@ defmodule Oban.Pro.Stages.Chain do "chain" => true, "chain_id" => to_chain_id(conf.by, conf.worker, args, opts), "on_cancelled" => to_string(conf.on_cancelled), - "on_discarded" => to_string(conf.on_discarded), - "on_hold" => false + "on_discarded" => to_string(conf.on_discarded) } opts = Keyword.update(opts, :meta, meta, &Map.merge(&1, meta)) @@ -183,22 +183,30 @@ defmodule Oban.Pro.Stages.Chain do subquery = Job |> select([j], j.id) - |> where([j], j.state == "scheduled") - |> where([j], fragment("? \\? 'chain_id'", j.meta)) |> where([j], fragment("?->>'chain_id'", j.meta) == ^chain_id) - |> where([j], fragment("?->>'on_hold'", j.meta) == "true") |> order_by(asc: :id) |> limit(1) |> lock("FOR UPDATE") + subquery = + if Utils.has_legacy_suspended?(conf) do + subquery + |> where([j], j.state == "scheduled") + |> where([j], fragment("?->>'on_hold' = 'true'", j.meta)) + else + where(subquery, [j], j.state == "suspended") + end + + now = DateTime.utc_now() + query = Job |> join(:inner, [j], x in subquery(subquery), on: j.id == x.id) |> update([j], set: [ meta: drop_hold(j.meta), - state: continue_state(literal(^conf.prefix), j.meta), - scheduled_at: continue_at(j.meta, ^DateTime.utc_now()) + state: continue_state(literal(^conf.prefix), j.meta, j.scheduled_at, ^now), + scheduled_at: continue_at(j.meta, j.scheduled_at, ^now) ] ) diff --git a/vendor/oban_pro/lib/oban/pro/stages/deadline.ex b/vendor/oban_pro/lib/oban/pro/stages/deadline.ex index 7a026206..3ff9e0d6 100644 --- a/vendor/oban_pro/lib/oban/pro/stages/deadline.ex +++ b/vendor/oban_pro/lib/oban/pro/stages/deadline.ex @@ -3,6 +3,7 @@ defmodule Oban.Pro.Stages.Deadline do @behaviour Oban.Pro.Stage + alias Oban.Pro.Stages.Hooks alias Oban.Pro.Utils alias Oban.{Registry, Validation} @@ -106,6 +107,8 @@ defmodule Oban.Pro.Stages.Deadline do send(pid, {:notification, :signal, payload}) end + + Hooks.execute_on_cancelled(:deadline, job) end end) end diff --git a/vendor/oban_pro/lib/oban/pro/stages/hooks.ex b/vendor/oban_pro/lib/oban/pro/stages/hooks.ex index 18490ae4..42d95443 100644 --- a/vendor/oban_pro/lib/oban/pro/stages/hooks.ex +++ b/vendor/oban_pro/lib/oban/pro/stages/hooks.ex @@ -3,10 +3,22 @@ defmodule Oban.Pro.Stages.Hooks do @behaviour Oban.Pro.Stage + import Ecto.Query, only: [where: 3] + + alias Oban.{Job, Repo} alias Oban.Pro.Utils require Logger + @hooks [ + after_cancelled: 2, + before_new: 2, + before_process: 1, + after_process: 3, + on_cancelled: 2, + on_discarded: 2 + ] + defstruct modules: [] @impl Oban.Pro.Stage @@ -17,7 +29,7 @@ defmodule Oban.Pro.Stages.Hooks do @impl Oban.Pro.Stage def before_new(args, opts, %{modules: modules}) do (modules ++ get_hooks()) - |> Enum.filter(&hook_exported?(&1, :before_new, 2)) + |> Enum.filter(&hook_exported?(&1, :before_new)) |> Enum.reduce_while({:ok, args, opts}, fn module, {:ok, args, opts} -> case module.before_new(args, opts) do {:ok, _, _} = ok -> {:cont, ok} @@ -29,7 +41,7 @@ defmodule Oban.Pro.Stages.Hooks do @impl Oban.Pro.Stage def before_process(job, %{modules: modules}) do (modules ++ get_hooks()) - |> Enum.filter(&hook_exported?(&1, :before_process, 1)) + |> Enum.filter(&hook_exported?(&1, :before_process)) |> Enum.reduce_while({:ok, job}, fn module, {:ok, job} -> case module.before_process(job) do {:ok, _} = ok -> {:cont, ok} @@ -59,65 +71,119 @@ defmodule Oban.Pro.Stages.Hooks do end end - def handle_event(_event, _timing, meta, _conf) do - %{job: %{unsaved_error: unsaved_error}, state: state} = meta + # The `Workflow.after_cancelled` hook is deprecated and will be removed in a future version. + def execute_after_cancelled(reason, job) do + with {worker, job, opts} <- resolve_processing(job) do + execute_hook(worker, opts, :on_cancelled, [:dependency, job]) + execute_hook(worker, opts, :after_cancelled, [reason, job]) + end + end + def execute_on_cancelled(reason, job) do + with {worker, job, opts} <- resolve_processing(job) do + execute_hook(worker, opts, :on_cancelled, [reason, job]) + end + end + + def execute_on_discarded(reason, job) do + with {worker, job, opts} <- resolve_processing(job) do + execute_hook(worker, opts, :on_discarded, [reason, job]) + end + end + + def handle_event([:oban, :job, _], _timing, meta, _conf) do with {worker, job, opts} <- resolve_processing(meta.job) do - hook_state = exec_to_hook_state(state) - module_hooks = Keyword.get(opts, :hooks, []) - global_hooks = get_hooks() + execute_hook(worker, opts, :after_process, [to_hook_state(meta.state), job, meta[:result]]) + end + end - job = %{job | unsaved_error: unsaved_error} - result = Map.get(meta, :result, nil) + def handle_event([:oban, :engine, :cancel_all_jobs, _], _timing, meta, _conf) do + global? = Enum.any?(get_hooks(), &hook_exported?(&1, :on_cancelled)) - ([worker | module_hooks] ++ global_hooks) - |> Enum.uniq() - |> Enum.each(fn module -> - cond do - hook_exported?(module, :after_process, 3) -> - module.after_process(hook_state, job, result) + Task.start(fn -> + load_and_exec = fn partials -> + ids = Enum.map(partials, & &1.id) - hook_exported?(module, :after_process, 2) -> - module.after_process(hook_state, job) + meta.conf + |> Repo.all(where(Job, [j], j.id in ^ids)) + |> Enum.each(&execute_on_cancelled(:manual, &1)) + end - true -> - :ok - end - end) + meta.jobs + |> Stream.filter(&(global? or hook_exported?(&1, :on_cancelled))) + |> Stream.chunk_every(100) + |> Stream.each(load_and_exec) + |> Stream.run() + end) + end + + defp execute_hook(worker, opts, hook, args) do + mod_hooks = Keyword.get(opts, :hooks, []) + all_hooks = Enum.uniq([worker | mod_hooks] ++ get_hooks()) + + for module <- all_hooks, hook_exported?(module, hook) do + apply(module, hook, args) end catch kind, value -> Logger.error(fn -> "[Oban.Pro.Worker] hook error: " <> Exception.format(kind, value, __STACKTRACE__) end) - after - Process.delete(:oban_processing) end defp resolve_processing(job) do - # The event may be triggered by a producer after the process was killed or experienced a - # linked crash. In that event, the processing data isn't in the dictionary and we need to - # resolve and apply before hooks to the job again. with nil <- Process.get(:oban_processing), {:ok, worker} <- Oban.Worker.from_string(job.worker), opts = worker.__opts__(), - {:ok, job} <- Oban.Pro.Worker.before_process(worker, job, opts) do - processing = {worker, job, opts} - Process.put(:oban_processing, processing) - processing + {:ok, job} <- Oban.Pro.Worker.before_hook(worker, job, opts) do + {worker, job, opts} end end defp get_hooks, do: :persistent_term.get(:oban_pro_hooks, []) - defp exec_to_hook_state(:cancelled), do: :cancel - defp exec_to_hook_state(:success), do: :complete - defp exec_to_hook_state(:discard), do: :discard - defp exec_to_hook_state(:failure), do: :error - defp exec_to_hook_state(:snoozed), do: :snooze + defp to_hook_state(:cancelled), do: :cancel + defp to_hook_state(:success), do: :complete + defp to_hook_state(:discard), do: :discard + defp to_hook_state(:failure), do: :error + defp to_hook_state(:snoozed), do: :snooze - defp hook_exported?(mod, fun, arity) do - Code.ensure_loaded?(mod) and function_exported?(mod, fun, arity) + defp hook_exported?(worker, fun) when is_binary(worker) do + case Oban.Worker.from_string(worker) do + {:ok, module} -> hook_exported?(module, fun) + _ -> false + end + end + + defp hook_exported?(module, fun) when is_atom(module) do + module + |> cached_hooks() + |> MapSet.member?(fun) + end + + defp hook_exported?(%{worker: worker}, fun), do: hook_exported?(worker, fun) + defp hook_exported?(_map, _fun), do: false + + defp cached_hooks(module) do + case :persistent_term.get({:oban_pro_hook_cache, module}, :not_cached) do + :not_cached -> + module + |> discover_hooks() + |> tap(&:persistent_term.put({:oban_pro_hook_cache, module}, &1)) + + hooks -> + hooks + end + end + + defp discover_hooks(module) do + if Code.ensure_loaded?(module) do + for {fun, arity} <- @hooks, function_exported?(module, fun, arity), into: MapSet.new() do + fun + end + else + MapSet.new() + end end # Validation @@ -127,21 +193,13 @@ defmodule Oban.Pro.Stages.Hooks do not Code.ensure_loaded?(module) -> {:error, "unable to load callback module #{inspect(module)}"} - function_exported?(module, :before_new, 2) -> - :ok - - function_exported?(module, :after_process, 2) -> - :ok - - function_exported?(module, :after_process, 3) -> - :ok - - function_exported?(module, :before_process, 1) -> + Enum.any?(@hooks, &function_exported?(module, elem(&1, 0), elem(&1, 1))) -> :ok true -> - {:error, - "#{inspect(module)} doesn't implement after_process/3, before_new/2, before_process/1"} + names = Enum.map_join(@hooks, ", ", fn {fun, arity} -> "#{fun}/#{arity}" end) + + {:error, "#{inspect(module)} doesn't implement known hooks: #{names}"} end end end diff --git a/vendor/oban_pro/lib/oban/pro/stages/rate.ex b/vendor/oban_pro/lib/oban/pro/stages/rate.ex new file mode 100644 index 00000000..283a17ec --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/stages/rate.ex @@ -0,0 +1,36 @@ +defmodule Oban.Pro.Stages.Rate do + @moduledoc false + + @behaviour Oban.Pro.Stage + + alias Oban.Validation + + defstruct [:weight] + + @impl Oban.Pro.Stage + def init(_worker, opts) when is_list(opts) do + with :ok <- validate(opts) do + {:ok, struct(__MODULE__, opts)} + end + end + + @impl Oban.Pro.Stage + def before_new(args, opts, conf) do + {rate_opts, opts} = Keyword.pop(opts, :rate, []) + + weight = Keyword.get(rate_opts, :weight, conf.weight) + + if is_integer(weight) do + meta = %{"rate_weight" => weight} + opts = Keyword.update(opts, :meta, meta, &Map.merge(&1, meta)) + + {:ok, args, opts} + else + {:ok, args, opts} + end + end + + defp validate(opts) do + Validation.validate_schema(opts, weight: :pos_integer) + end +end diff --git a/vendor/oban_pro/lib/oban/pro/stages/recorded.ex b/vendor/oban_pro/lib/oban/pro/stages/recorded.ex index 42215c3c..e324dedc 100644 --- a/vendor/oban_pro/lib/oban/pro/stages/recorded.ex +++ b/vendor/oban_pro/lib/oban/pro/stages/recorded.ex @@ -40,7 +40,7 @@ defmodule Oban.Pro.Stages.Recorded do {:error, "return is #{byte_size(encoded)} bytes, larger than the limit: #{conf.limit}"} true -> - Process.put(:oban_recorded, %{to_string(conf.to) => encoded}) + Process.put(:oban_meta_update, %{to_string(conf.to) => encoded}) :ok end diff --git a/vendor/oban_pro/lib/oban/pro/stages/structured.ex b/vendor/oban_pro/lib/oban/pro/stages/structured.ex index 99c93782..cb9f8a38 100644 --- a/vendor/oban_pro/lib/oban/pro/stages/structured.ex +++ b/vendor/oban_pro/lib/oban/pro/stages/structured.ex @@ -55,6 +55,8 @@ defmodule Oban.Pro.Stages.Structured do @impl Oban.Pro.Stage def before_process(job, :ignore), do: {:ok, job} + def before_process(%{args: %module{}} = job, %{worker: module}), do: {:ok, job} + def before_process(%{args: args} = job, conf) do {:ok, changes} = gather_changes(conf.worker, args) diff --git a/vendor/oban_pro/lib/oban/pro/testing.ex b/vendor/oban_pro/lib/oban/pro/testing.ex index baadf578..0e43004e 100644 --- a/vendor/oban_pro/lib/oban/pro/testing.ex +++ b/vendor/oban_pro/lib/oban/pro/testing.ex @@ -48,11 +48,11 @@ defmodule Oban.Pro.Testing do alias Ecto.Adapters.SQL.Sandbox alias Ecto.Changeset - alias Oban.Pro.Engines.Smart - alias Oban.Pro.Workers.Chunk - alias Oban.Pro.{Batch, Utils, Workflow} - alias Oban.Queue.Executor alias Oban.{Config, Job, Repo, Worker} + alias Oban.Pro.{Batch, Utils, Workflow} + alias Oban.Pro.Chunk + alias Oban.Pro.Engines.Smart + alias Oban.Queue.Executor @type repo_option :: {:log, false | Logger.level()} | {:prefix, String.t()} | {:repo, module()} @@ -877,9 +877,9 @@ defmodule Oban.Pro.Testing do @doc since: "0.11.0" @spec run_jobs([Job.changeset()], [drain_option()]) :: drain_result() def run_jobs([_ | _] = changesets, opts) when is_list(changesets) and is_list(opts) do - opts - |> init_conf() - |> Smart.insert_all_jobs(changesets, []) + conf = init_conf(opts) + + Oban.Engine.insert_all_jobs(conf, changesets, []) opts |> Keyword.put_new(:queue, :all) @@ -931,7 +931,7 @@ defmodule Oban.Pro.Testing do def run_workflow(%Workflow{} = workflow, opts \\ []) when is_list(opts) do conf = init_conf(opts) - Smart.insert_all_jobs(conf, workflow.changesets, []) + Oban.Engine.insert_all_jobs(conf, workflow.changesets, []) workflow_ids = workflow.changesets @@ -950,6 +950,7 @@ defmodule Oban.Pro.Testing do jobs = Job |> where([j], j.meta["workflow_id"] in ^workflow_ids) + |> order_by([j], asc_nulls_last: j.completed_at, asc: j.id) |> then(&Repo.all(conf, &1)) if Keyword.get(opts, :with_summary, true) do @@ -1184,10 +1185,7 @@ defmodule Oban.Pro.Testing do end defp stage_scheduled(conf) do - query = - Job - |> where([j], j.state in ["scheduled", "retryable"]) - |> where([j], not fragment("? @> ?", j.meta, ^%{on_hold: true})) + query = where(Job, [j], j.state in ["scheduled", "retryable"]) Repo.update_all(conf, query, set: [state: "available"]) end diff --git a/vendor/oban_pro/lib/oban/pro/unique.ex b/vendor/oban_pro/lib/oban/pro/unique.ex index cc142430..6c32453d 100644 --- a/vendor/oban_pro/lib/oban/pro/unique.ex +++ b/vendor/oban_pro/lib/oban/pro/unique.ex @@ -12,7 +12,8 @@ defmodule Oban.Pro.Unique do "retryable" => 3, "completed" => 4, "cancelled" => 5, - "discarded" => 6 + "discarded" => 6, + "suspended" => 7 } @spec replace?(Job.changeset()) :: boolean() diff --git a/vendor/oban_pro/lib/oban/pro/utils.ex b/vendor/oban_pro/lib/oban/pro/utils.ex index 92e28feb..e4badf10 100644 --- a/vendor/oban_pro/lib/oban/pro/utils.ex +++ b/vendor/oban_pro/lib/oban/pro/utils.ex @@ -32,6 +32,30 @@ defmodule Oban.Pro.Utils do end end + @doc """ + Check if any legacy suspended jobs exist (scheduled + on_hold pattern). + + Result is cached in persistent_term per prefix and only checked once per application lifetime. + """ + @spec has_legacy_suspended?(Oban.Config.t()) :: boolean() + def has_legacy_suspended?(conf) do + import Ecto.Query + + persistent_cache({__MODULE__, :legacy_suspended, conf.prefix}, fn -> + query = + from(job in Oban.Job, + where: fragment("? \\? 'workflow_id'", job.meta), + where: fragment("? ->> 'workflow_id' IS NOT NULL", job.meta), + where: job.state == "scheduled", + where: fragment("? ->> 'on_hold' = 'true'", job.meta), + select: true, + limit: 1 + ) + + Oban.Repo.exists?(conf, query) + end) + end + @spec validate_opts(list(), fun()) :: :ok | {:error, term()} def validate_opts(opts, validator) do Enum.reduce_while(opts, :ok, fn opt, acc -> diff --git a/vendor/oban_pro/lib/oban/pro/worker.ex b/vendor/oban_pro/lib/oban/pro/worker.ex index 13e7ed70..926b1be0 100644 --- a/vendor/oban_pro/lib/oban/pro/worker.ex +++ b/vendor/oban_pro/lib/oban/pro/worker.ex @@ -18,6 +18,8 @@ defmodule Oban.Pro.Worker do worker name to continue without exceptions using the new worker code. * [🪝 Worker Hooks](#module-worker-hooks) — callbacks triggered around job execution, defined as callback functions on the worker, or in a separate module for reuse. + * [⚖️ Rate Limiting Weight](#module-rate-limiting-weight) — assign custom weights to jobs to + control the rate limit quota consumed by a job. ## Usage @@ -473,15 +475,23 @@ defmodule Oban.Pro.Worker do ## Worker Hooks - Worker hooks are called before and after a job finishes executing. They can be defined as - callback functions on the worker, or in a separate module for reuse across workers. + Worker hooks are callbacks triggered at various points in a job's lifecycle. They can be + defined as callback functions on the worker, or in a separate module for reuse across workers. - Hooks are called synchronously, from within the job's process. + There are two categories of hooks: + + * **Execution hooks**—`c:before_new/2`, `c:before_process/1`, and `c:after_process/3` are + called during job creation and execution, synchronously within the job's process. + + * **External state hooks**—`c:on_cancelled/2` and `c:on_discarded/2` are called when a job's + state changes externally, outside of normal execution (e.g., manual cancellation or + rescue-based discard). ### Defining Hooks - There are three mechanisms for defining and attaching `c:before_new/2`, `c:before_process/1`, - and `c:after_process/3` hooks: + There are three mechanisms for defining and attaching execution hooks (`c:before_new/2`, + `c:before_process/1`, `c:after_process/3`) and external state hooks (`c:on_cancelled/2`, + `c:on_discarded/2`): 1. **Implicitly**—hooks are defined directly on the worker and they only run for that worker 2. **Explicitly**—hooks are listed when defining a worker and they run anywhere they are listed @@ -641,13 +651,118 @@ defmodule Oban.Pro.Worker do ```elixir :ok = Oban.Pro.Worker.attach_hook(MyApp.UserMetadataHook) ``` + + ### External State Hooks + + External state hooks are called when a job's state changes outside of normal execution. Unlike + execution hooks, these are triggered by external actions rather than the job running. + + * `c:on_cancelled/2`—called when a job is cancelled externally via `Oban.cancel_job/1`, + `Oban.cancel_all_jobs/2`, when a workflow dependency fails, or when a deadline expires + during execution (with `force: true`) + * `c:on_discarded/2`—called when `DynamicLifeline` discards a job that exhausted retries + while executing (i.e., the node shut down before the job could finish) + + These hooks are useful for cleanup, notifications, or compensating actions when jobs don't + complete normally. Like `c:after_process/3`, exceptions are caught and logged without + affecting the state change. + + > #### External vs Execution Hooks {: .info} + > + > A job that returns `{:cancel, reason}` from `c:process/1` triggers `c:after_process/3` with + > state `:cancel`, **not** `c:on_cancelled/2`. Similarly, a job that exhausts retries during + > normal execution triggers `c:after_process/3` with state `:discard`. + > + > The external hooks are for state changes that happen _outside_ of execution: manual + > cancellation, workflow dependency failures, deadline expiration, or rescue-based discards. + + Here's an example that notifies an external service when jobs are cancelled or discarded: + + ```elixir + defmodule MyApp.NotifyingWorker do + use Oban.Pro.Worker + + @impl Oban.Pro.Worker + def process(%{args: %{order_id: order_id}}) do + MyApp.Orders.process(order_id) + end + + @impl Oban.Pro.Worker + def on_cancelled(reason, job) do + MyApp.Notifications.job_cancelled(job.args.order_id, reason) + end + + @impl Oban.Pro.Worker + def on_discarded(:exhausted, job) do + MyApp.Notifications.job_abandoned(job.args.order_id) + end + end + ``` + + These callbacks can also be defined in shared hook modules and attached explicitly or globally, + just like execution hooks. + + ## Rate Limiting Weight + + When using rate limiting with the Smart engine, each job consumes one unit of rate limit + capacity by default. For jobs that vary in resource usage, you can assign custom weights so + that heavier jobs consume more capacity. + + There are three ways to assign weights, in order of precedence: + + 1. **Callback** — Define a `c:weight/1` callback for runtime calculation based on job data + 2. **Job option** — Set the weight when creating a specific job + 3. **Worker option** — Set a default weight for all jobs from a worker + + ### Setting Weight + + For simple rate weight adjustments, set a default weight for all of a worker's jobs: + + ```elixir + use Oban.Pro.HeavyWorker, rate: [weight: 10] + ``` + + Override the weight for specific jobs by passing an option to `new/2`: + + ```elixir + MyApp.HeavyWorker.new(%{id: 2}, rate: [weight: 5]) + ``` + + ### The Weight Callback + + The `c:weight/1` callback receives the job with encryption and structured args already applied, + allowing you to calculate weight based on the complete job data: + + ```elixir + defmodule MyApp.BatchWorker do + use Oban.Pro.Worker + + args_schema do + field :records, {:array, :map} + end + + @impl Oban.Pro.Worker + def weight(%{args: args}), do: length(args.records) + + @impl Oban.Pro.Worker + def process(%{args: args}) do + Enum.each(args.records, &process_record/1) + end + end + ``` + + Jobs without any weight configuration default to a weight of 1. + + See `Oban.Pro.Engines.Smart` for more details on rate limiting configuration and algorithms. """ @behaviour Oban.Pro.Handler - alias Oban.{Job, Validation, Worker} + import Ecto.Query, only: [where: 3, update: 3] + + alias Oban.{Job, Notifier, Period, Repo, Validation, Worker} alias Oban.Pro.Batch - alias Oban.Pro.Stages.{Chain, Deadline, Encrypted, Executing, Hooks, Recorded, Structured} + alias Oban.Pro.Stages.{Chain, Deadline, Encrypted, Executing, Hooks, Rate, Recorded, Structured} @typedoc """ Options to enable and configure `alias` mode. @@ -692,8 +807,21 @@ defmodule Oban.Pro.Worker do """ @type hook_state :: :cancel | :complete | :discard | :error | :snooze - @type args :: Job.args() - @type opts :: [Job.option()] + @typedoc """ + Reasons a job may be cancelled. + + * `:deadline` — The job exceeded its configured deadline while executing (with `force: true`). + * `:dependency` — A workflow or chain dependency failed, was cancelled, or was deleted. + * `:manual` — The job was cancelled via `Oban.cancel_job/1` or `Oban.cancel_all_jobs/2`. + """ + @type cancel_reason :: :deadline | :dependency | :manual + + @typedoc """ + Reasons a job may be discarded. + + * `:exhausted` — The job exhausted all retry attempts. + """ + @type discard_reason :: :exhausted @doc """ Called when executing a job. @@ -739,7 +867,7 @@ defmodule Oban.Pro.Worker do """ @callback after_process(hook_state(), job :: Job.t(), result :: nil | Worker.result()) :: :ok - @doc deprecated: "Use after_process/3 instead" + @doc deprecated: "Not functional, use after_process/3 instead" @callback after_process(hook_state(), job :: Job.t()) :: :ok @doc """ @@ -750,15 +878,103 @@ defmodule Oban.Pro.Worker do """ @callback fetch_recorded(job :: Job.t()) :: {:ok, term()} | {:error, :missing} + @doc """ + Called when a job is cancelled externally, either manually, due to a dependency failure, or + when a deadline expires during execution. + + This callback is triggered after the job's state has been updated to `cancelled` in the + database. It receives the cancellation reason and the full job struct with decryption and + structuring applied. + + The callback is intended for side-effects such as cleanup, notifications, or logging. + Any exceptions raised are caught and logged without affecting the cancellation. + + > #### When is this called? {: .info} + > + > This callback is called for _external_ cancellations only: + > + > * `:deadline` — The job exceeded its configured deadline while executing (with `force: true`) + > * `:dependency` — A workflow dependency was cancelled, discarded, or deleted + > * `:manual` — The job was cancelled via `Oban.cancel_job/1` or `Oban.cancel_all_jobs/2` + > + > Jobs that cancel themselves by returning `{:cancel, reason}` from `process/1` do **not** + > trigger this callback. Use `c:after_process/3` with the `:cancel` state instead. + + ## Example + + @impl Oban.Pro.Worker + def on_cancelled(:deadline, job) do + Logger.warning("Job \#{job.id} exceeded its deadline") + :ok + end + + def on_cancelled(:dependency, job) do + notify(job) + + :ok + end + + def on_cancelled(:manual, job) do + Logger.info("Job \#{job.id} was manually cancelled") + :ok + end + """ + @doc since: "1.7.0" + @callback on_cancelled(cancel_reason(), job :: Job.t()) :: :ok + + @doc """ + Called when a job is discarded after exhausting all retry attempts. + + This callback is triggered by `DynamicLifeline` after the job's state has been updated to + `discarded` in the database. It receives the discard reason and the full job struct with + decryption and structuring applied. + + The callback is intended for side-effects such as cleanup, notifications, or logging. + Any exceptions raised are caught and logged without affecting the discard. + + ## Example + + @impl Oban.Pro.Worker + def on_discarded(:exhausted, job) do + send_failure_alert(job) + + :ok + end + """ + @doc since: "1.7.0" + @callback on_discarded(discard_reason(), job :: Job.t()) :: :ok + + @doc """ + Calculate the rate limit weight for a job. + + This callback is called at dispatch time when jobs are fetched for execution. The weight + determines how much of the rate limit capacity a job consumes. Jobs without this callback + or without a `rate_weight` in meta default to a weight of 1. + + The job passed to this callback has encryption and structured args applied, so you can + access structured args directly. + + ## Example + + @impl Oban.Pro.Worker + def weight(%{args: args}), do: length(args.records) + """ + @doc since: "1.7.0" + @callback weight(job :: Job.t()) :: pos_integer() + @optional_callbacks after_process: 2, after_process: 3, before_new: 2, before_process: 1, - fetch_recorded: 1 + fetch_recorded: 1, + on_cancelled: 2, + on_discarded: 2, + weight: 1 @stages [ executing: Executing, encrypted: Encrypted, + rate: Rate, recorded: Recorded, structured: Structured, chain: Chain, @@ -772,6 +988,8 @@ defmodule Oban.Pro.Worker do @after_process Keyword.take(@stages, ~w(recorded)a) + @before_hook Keyword.take(@stages, ~w(encrypted structured)a) + @doc false defmacro __using__(opts) do opts = Macro.expand_literals(opts, %{__CALLER__ | function: {:__using__, 1}}) @@ -788,6 +1006,8 @@ defmodule Oban.Pro.Worker do @behaviour Oban.Worker @behaviour Oban.Pro.Worker + @before_compile Oban.Pro.Worker + @after_compile {Oban.Pro.Worker, :__define_aliases__} @after_compile {Oban.Pro.Worker, :__validate_opts__} @after_compile {Oban.Pro.Worker, :__validate_stages__} @@ -817,6 +1037,7 @@ defmodule Oban.Pro.Worker do def __stand_opts__, do: @stand_opts @doc false + @impl Worker def __opts__, do: @opts @impl Worker @@ -863,6 +1084,29 @@ defmodule Oban.Pro.Worker do end end + @doc false + defmacro __before_compile__(_env) do + quote do + defoverridable backoff: 1, timeout: 1 + + @impl Oban.Worker + def timeout(%Job{} = job) do + case Oban.Pro.Worker.before_hook(__MODULE__, job, __opts__()) do + {:ok, job} -> super(job) + _ -> super(job) + end + end + + @impl Oban.Worker + def backoff(%Job{} = job) do + case Oban.Pro.Worker.before_hook(__MODULE__, job, __opts__()) do + {:ok, job} -> super(job) + _ -> super(job) + end + end + end + end + # Compile Callbacks @doc false @@ -902,6 +1146,7 @@ defmodule Oban.Pro.Worker do quote bind_quoted: [worker: env.module] do @behaviour Oban.Worker + @impl Oban.Worker defdelegate __opts__, to: worker defdelegate __stage_opts__, to: worker @@ -1067,7 +1312,7 @@ defmodule Oban.Pro.Worker do defp check_type!(_name, type) when type in ~w(enum term uuid)a, do: :ok defp check_type!(name, type) do - unless Ecto.Type.base?(type) do + if !Ecto.Type.base?(type) do raise ArgumentError, "invalid type #{inspect(type)} for field #{inspect(name)}" end end @@ -1085,6 +1330,19 @@ defmodule Oban.Pro.Worker do end end + # Private Macros + + defmacrop maybe_schedule(job, now) do + quote do + fragment( + "CASE WHEN (? ->> 'wait_until') IS NOT NULL THEN ? ELSE ? END", + unquote(job).meta, + unquote(now), + unquote(job).scheduled_at + ) + end + end + # Global Hooks @doc """ @@ -1124,6 +1382,178 @@ defmodule Oban.Pro.Worker do @spec detach_hook(module()) :: :ok defdelegate detach_hook(module), to: Hooks + @doc """ + Deliver a signal to one or more jobs by id. + + The `payload` is merged into the target jobs' `meta` under the `"signal"` key. If a job is + parked inside `await_signal/2`, its `scheduled_at` is bumped to `now()` so it resumes + immediately. + + Signals are fire-and-forget. Delivery to a job in a terminal state, or a job that doesn't + exist, is a no-op and still returns `:ok`. When multiple signals arrive for the same target + before the job consumes them the last write wins. + + ## Examples + + Signal a single job by id: + + Oban.Pro.Worker.signal(job_id, %{decision: "approved"}) + + Signal several jobs at once: + + Oban.Pro.Worker.signal([job_a_id, job_b_id], %{decision: "approved"}) + + Signal from within a process callback using the job struct: + + Oban.Pro.Worker.signal(job, %{decision: "approved"}) + + Target a non-default Oban instance: + + Oban.Pro.Worker.signal(MyApp.Oban, job_id, %{decision: "approved"}) + """ + @doc since: "1.7.0" + @spec signal(Job.t() | pos_integer() | [pos_integer()], term()) :: :ok + def signal(job_or_ids, payload) + + def signal(%Job{id: id, conf: conf}, payload), do: signal(conf.name, id, payload) + + def signal(ids, payload) when is_integer(ids) or is_list(ids) do + signal(Oban, ids, payload) + end + + @spec signal(Oban.name(), pos_integer() | [pos_integer()], term()) :: :ok + def signal(oban_name, id_or_ids, payload) do + conf = Oban.config(oban_name) + now = DateTime.utc_now() + ids = List.wrap(id_or_ids) + + query = + Job + |> where([j], j.id in ^ids) + |> where([j], j.state in ~w(suspended available scheduled executing retryable)) + |> update([j], + set: [ + meta: fragment("jsonb_set(?, '{signal}', ?::jsonb)", j.meta, ^payload), + scheduled_at: maybe_schedule(j, ^now) + ] + ) + + Repo.transaction(conf, fn -> + Repo.update_all(conf, query, []) + + Notifier.notify(conf, :pro_signal, %{job_ids: ids, signal: payload}) + end) + + :ok + end + + @doc """ + Pause the current job until a matching signal arrives. + + Must be called from within a job. Returns `{:ok, payload}` when a signal is received or + `{:error, :timeout}` when the overall timeout elapses. Any code that appears after + `await_signal/2` in the enclosing `process/1` is skipped when the job snoozes. + + ## Options + + * `:wait_for` - Total time to wait for a signal, measured from the first snooze. Accepts + seconds as an integer or a period tuple like `{30, :minutes}` or `{1, :day}`. The deadline + is persisted on first snooze so subsequent retry attempts honor the same deadline. Defaults + to `:infinity`. + + * `:wait_timeout` - Milliseconds to block during execution before snoozing the job. Pass `0` + to skip the live wait and snooze immediately. Defaults to `5_000`. + + ## Examples + + def process(job) do + case Oban.Pro.Worker.await_signal(wait_for: {30, :minutes}) do + {:ok, payload} -> proceed(payload) + {:error, :timeout} -> {:cancel, "no decision"} + end + end + """ + @doc since: "1.7.0" + @spec await_signal(keyword()) :: {:ok, term()} | {:error, :timeout} + def await_signal(opts \\ []) do + case Process.get(:oban_processing) do + {_, %Job{id: id, meta: meta, conf: conf}, _} -> + wait_timeout = Keyword.get(opts, :wait_timeout, 5_000) + + wait_until = + Map.get_lazy(meta, "wait_until", fn -> + opts + |> Keyword.get(:wait_for, :infinity) + |> compute_deadline() + end) + + cond do + payload = meta["signal"] -> {:ok, payload} + past_deadline?(wait_until) -> {:error, :timeout} + true -> await_signal(conf.name, id, wait_until, wait_timeout) + end + + _ -> + raise ArgumentError, "await_signal/1 must be called within an executing job" + end + end + + defp await_signal(oban_name, id, wait_until, wait_timeout) do + :ok = Notifier.listen(oban_name, :pro_signal) + + try do + await_loop(id, wait_until, wait_timeout) + after + Notifier.unlisten(oban_name, :pro_signal) + end + end + + defp await_loop(id, wait_until, wait_timeout) do + effective = bounded_timeout(wait_until, wait_timeout) + + receive do + {:notification, :pro_signal, %{"job_ids" => ids, "signal" => signal}} -> + if id in ids do + {:ok, signal} + else + await_loop(id, wait_until, wait_timeout) + end + after + effective -> + cond do + past_deadline?(wait_until) -> + {:error, :timeout} + + wait_until == :infinity -> + throw({:__oban_pro_park__, %{wait_until: wait_until}, {1, :day}}) + + true -> + snooze = max(1, ceil_seconds(wait_until - System.system_time(:millisecond))) + throw({:__oban_pro_park__, %{wait_until: wait_until}, snooze}) + end + end + end + + defp compute_deadline(:infinity), do: :infinity + + defp compute_deadline(period) do + System.system_time(:millisecond) + Period.to_seconds(period) * 1_000 + end + + defp past_deadline?(:infinity), do: false + defp past_deadline?(wait_until), do: System.system_time(:millisecond) >= wait_until + + defp bounded_timeout(:infinity, wait_timeout), do: wait_timeout + + defp bounded_timeout(wait_until, wait_timeout) do + wait_until + |> Kernel.-(System.system_time(:millisecond)) + |> max(0) + |> min(wait_timeout) + end + + defp ceil_seconds(ms), do: div(ms - 1, 1000) + 1 + # Stages @doc false @@ -1161,6 +1591,15 @@ defmodule Oban.Pro.Worker do |> handle_or_process(job) |> after_process(worker, job, opts) end + catch + # Used by Workflow.await_signal/2 to break out of execution and immediately snooze the job + # with a meta update. + :throw, {:__oban_pro_park__, meta_updates, snooze} -> + meta = Map.merge(job.meta, meta_updates) + + Process.put(:oban_meta_update, meta_updates) + + after_process({:snooze, snooze}, worker, %{job | meta: meta}, opts) end @doc false @@ -1181,6 +1620,24 @@ defmodule Oban.Pro.Worker do end) end + @doc false + def before_hook(worker, job, opts) do + Enum.reduce_while(@before_hook, {:ok, job}, fn {key, mod}, {:ok, job} -> + case Keyword.fetch(opts, key) do + {:ok, stage_opts} -> + {:ok, conf} = mod.init(worker, stage_opts) + + case mod.before_process(job, conf) do + {:ok, _} = ok -> {:cont, ok} + other -> {:halt, other} + end + + :error -> + {:cont, {:ok, job}} + end + end) + end + defp handle_or_process(module, %{meta: %{"batch_id" => _, "callback" => callback}} = job) do new_callback = Map.fetch!(Batch.callbacks_to_functions(), callback) old_callback = Map.fetch!(Batch.callbacks_to_deprecated(), callback) @@ -1222,12 +1679,27 @@ defmodule Oban.Pro.Worker do end end + @doc false + def get_weight(job) do + with {:ok, worker} <- Worker.from_string(job.worker), + true <- function_exported?(worker, :weight, 1), + {:ok, job} <- before_hook(worker, job, worker.__opts__()) do + worker.weight(job) + else + _ -> Map.get(job.meta, "rate_weight", 1) + end + end + # Telemetry @doc false @impl Oban.Pro.Handler def on_start do - events = [[:oban, :job, :stop], [:oban, :job, :exception]] + events = [ + [:oban, :job, :stop], + [:oban, :job, :exception], + [:oban, :engine, :cancel_all_jobs, :stop] + ] :telemetry.attach_many("oban.pro.worker", events, &Hooks.handle_event/4, nil) end diff --git a/vendor/oban_pro/lib/oban/pro/workers/batch.ex b/vendor/oban_pro/lib/oban/pro/workers/batch.ex index ccf34606..45cd5170 100644 --- a/vendor/oban_pro/lib/oban/pro/workers/batch.ex +++ b/vendor/oban_pro/lib/oban/pro/workers/batch.ex @@ -1,12 +1,5 @@ defmodule Oban.Pro.Workers.Batch do - @moduledoc """ - A dedicated worker for grouping the execution of jobs. - - This worker is deprecated in favor of `Oban.Pro.Batch`, which is more flexible and supports more - options. - """ - - @moduledoc deprecated: "Use Oban.Pro.Batch instead" + @moduledoc false alias Oban.{Job, Worker} diff --git a/vendor/oban_pro/lib/oban/pro/workers/chain.ex b/vendor/oban_pro/lib/oban/pro/workers/chain.ex index 61bd5ecf..785fd075 100644 --- a/vendor/oban_pro/lib/oban/pro/workers/chain.ex +++ b/vendor/oban_pro/lib/oban/pro/workers/chain.ex @@ -1,12 +1,5 @@ defmodule Oban.Pro.Workers.Chain do - @moduledoc """ - A dedicated worker for linking jobs to run in strict sequential order. - - This worker is deprecated in favor of `Oban.Pro.Worker`, which is composable and allows better - parallelism. - """ - - @moduledoc deprecated: "Use Oban.Pro.Worker instead" + @moduledoc false import Ecto.Query diff --git a/vendor/oban_pro/lib/oban/pro/workers/chunk.ex b/vendor/oban_pro/lib/oban/pro/workers/chunk.ex index 773d096d..42e2444e 100644 --- a/vendor/oban_pro/lib/oban/pro/workers/chunk.ex +++ b/vendor/oban_pro/lib/oban/pro/workers/chunk.ex @@ -1,735 +1,9 @@ defmodule Oban.Pro.Workers.Chunk do - @moduledoc """ - Chunk workers execute jobs together in groups based on a size or a timeout option. e.g. when - 1000 jobs are available or after 10 minutes have ellapsed. + @moduledoc false - Multiple chunks can run in parallel within a single queue, and each chunk may be composed of - many thousands of jobs. Aside from improved throughput for a single queue, chunks are ideal as - the initial stage of data-ingestion and data-processing pipelines. - - ## Usage - - Let's define a worker that sends SMS messages in chunks, rather than individually: - - ```elixir - defmodule MyApp.ChunkWorker do - use Oban.Pro.Workers.Chunk, queue: :messages, size: 100, timeout: 1000 - - @impl true - def process([_ | _] = jobs) do - jobs - |> Enum.map(& &1.args) - |> MyApp.Messaging.send_batch() - - :ok - end - end - ``` - - Notice that we declared a `size` and a `timeout` along with a `queue` for the worker. If `size` - or `timeout` are omitted they fall back to their defaults: 100 `size` and 1000ms respectively. - - To process _larger_ batches _less_ frequently, we can increase both values: - - ```elixir - use Oban.Pro.Workers.Chunk, size: 500, timeout: :timer.seconds(5) - ``` - - Now chunks will run with up to 500 jobs or every 5 seconds, whichever comes first. - - Unlike other Pro workers, a `Chunk` worker's `process/1` receives a list of jobs rather than a - single job struct. Fittingly, the [expected return values](#t:result/0) are different as well. - - ## Chunk Partitioning - - By default, chunks are divided into groups based on the `queue` and `worker`. This means that - each chunk consists of workers belonging to the same queue, regardless of their `args` or - `meta`. However, this approach may not always be suitable. For instance, you may want to group - workers based on a specific argument such as `:account_id` instead of just the worker. In such - cases, you can use the [`:by`](#t:chunk_by/0) option to customize how chunks are partitioned. - - Here are a few examples of the `:by` option that you can use to achieve fine-grained control over - chunk partitioning: - - ```elixir - # Explicitly chunk by :worker - use Oban.Pro.Workers.Chunk, by: :worker - - # Chunk by a single args key without considering the worker - use Oban.Pro.Workers.Chunk, by: [args: :account_id] - - # Chunk by multiple args keys without considering the worker - use Oban.Pro.Workers.Chunk, by: [args: [:account_id, :cohort]] - - # Chunk by worker and a single args key - use Oban.Pro.Workers.Chunk, by: [:worker, args: :account_id] - - # Chunk by a single meta key - use Oban.Pro.Workers.Chunk, by: [meta: :source_id] - ``` - - When partitioning chunks of jobs, it's important to note that using only `:args` or `:meta` - without `:worker` may result in heterogeneous chunks of jobs from different workers. - Nevertheless, regardless of the partitioning method chunks always consist of jobs from the same - queue. - - Here's a simple example of partitioning by `worker` and an `author_id` field to batch analysis - of recent messages per author: - - ```elixir - defmodule MyApp.LLMAnalysis do - use Oban.Pro.Workers.Chunk, by: [:worker, args: :author_id], size: 50, timeout: 30_000 - - @impl true - def process([%{args: %{"author_id" => author_id}} | _] = jobs) do - messages = - jobs - |> Enum.map(& &1.args["message_id"]) - |> MyApp.Messages.all() - - {:ok, sentiment} = MyApp.GPT.gauge_sentiment(messages) - - MyApp.Messages.record_sentiment(author_id) - end - end - ``` - - ## Chunk Results and Error Handling - - `Chunk` worker's result type is tailored to handling multiple jobs at once. For reference, here - are the types and callback definition for `process/1`: - - The result types allow you to succeed an entire chunk, or selectively fail parts of it. Here are - each of the possible results: - - * `:ok` — The chunk succeeded and all jobs can be marked complete - - * `{:ok, result}` — Like `:ok`, but with a result for testing. - - * `{:error, reason, jobs}` — One or more jobs in the chunk failed and may be retried, any - unlisted jobs are a success. - - * `{:cancel, reason, jobs}` — One or more jobs in the chunk should be cancelled, any unlisted - jobs are a success. - - * `[error: {reason, jobs}, cancel: {reason, jobs}]` — Retry some jobs and cancel some other - jobs, leaving any jobs not in either list a success. - - To illustrate using chunk results let's expand on the message processing example from earlier. - We'll extend it to complete the whole batch when all messages are delivered or cancel - undeliverable messages: - - ```elixir - def process([_ | _] = jobs) do - notifications = - jobs - |> Enum.map(& &1.args) - |> MyApp.Messaging.send_batch() - - bad_token = fn %{response: response} -> response == :bad_token end - - if Enum.any?(notifications, bad_token) do - cancels = - notifications - |> Enum.zip(jobs) - |> Enum.filter(fn {notification, _job} -> bad_token.(notification) end) - |> Enum.map(&elem(&1, 1)) - - {:cancel, :bad_token, cancels} - else - {:ok, notifications} - end - end - ``` - - In the event of an ephemeral crash, like a network issue, the entire batch may be retried if - there are any remaining attempts. - - Cancelling any jobs in a chunk will cancel the _entire_ chunk, including the leader job. - - ## Chunk Organization - - Chunks are ran by a leader job (which has nothing to do with peer leadership). When the leader - executes it determines whether a complete chunk is available or if enough time has elapsed to - run anyhow. If neither case applies, then the leader will delay until the timeout elapsed and - execute with as many jobs as it can find. - - Completion queries run every `1000ms` by default. You can use the `:sleep` option to control how - long the leader delays between queries to check for complete chunks: - - ```elixir - use Oban.Pro.Workers.Chunk, size: 50, sleep: 2_000, timeout: 10_000 - ``` - - ## Run Chunks Immediately - - A chunk won't run until either the full number of jobs are available (`size`), or the total time - has elapsed (`timeout`). In situations where there is a long gap between incoming jobs it may be - desirable to run the chunk as soon as the chunk runs. This can be done with the `leading` - option. For example, to chunk all jobs that arrive within a 5 minute window, but run immediately - if more than 5 minutes have elapsed: - - ```elixir - use Oban.Pro.Workers.Chunk, size: 100, leading: true, timeout: :timer.minutes(5) - ``` - - ## Optimizing Chunks - - Queue's with high concurrency and low throughput may have multiple chunk leaders running - simultaneously rather than getting bundled into a single chunk. The solution is to reduce the - queue’s global concurrency to a smaller number so that new chunk leader jobs don’t start and the - existing chunk can run a bigger batch. - - ```elixir - queues: [ - chunked_queue: [global_limit: 1] - ] - ``` - - A low global limit will reduce overall throughput. Partitioning the queue with the same options - as the chunk can preserve throughput by allowing one concurrent leader per chunk. For example, - if you were chunking by `:user_id` in args, you'd partition the queue with the same option: - - ```elixir - queues: [ - chunked_queue: [local_limit: 20, global_limit: [allowed: 1, partition: [args: :user_id]]] - ] - ``` - """ - - import Ecto.Query - import DateTime, only: [utc_now: 0] - - alias Ecto.Multi - alias Oban.{CrashError, Job, Notifier, PerformError, Repo, Validation} - alias Oban.Pro.Utils - - @type key_or_keys :: atom() | [atom()] - - @type chunk_by :: - :worker - | {:args, key_or_keys()} - | {:meta, key_or_keys()} - | [:worker | {:args, key_or_keys()} | {:meta, key_or_keys()}] - - @type sub_result :: {reason :: term(), [Job.t()]} - - @type result :: - :ok - | {:ok, term()} - | {:cancel | :discard | :error, reason :: term(), [Job.t()]} - | [cancel: sub_result(), discard: sub_result(), error: sub_result()] - - @type options :: [ - by: chunk_by(), - leading: boolean(), - size: pos_integer(), - sleep: pos_integer(), - timeout: pos_integer() - ] - - @doc false defmacro __using__(opts) do - {chunk_opts, other_opts} = Keyword.split(opts, ~w(by leading size sleep timeout)a) - quote do - Validation.validate!(unquote(chunk_opts), &Oban.Pro.Workers.Chunk.validate/1) - - use Oban.Pro.Worker, unquote(other_opts) - - alias Oban.Pro.Workers.Chunk - - @doc false - def default_meta do - %{ - chunk_by: Keyword.get(unquote(chunk_opts), :by, [:worker]), - chunk_leading: Keyword.get(unquote(chunk_opts), :leading, false), - chunk_size: Keyword.get(unquote(chunk_opts), :size, 100), - chunk_sleep: Keyword.get(unquote(chunk_opts), :sleep, 1000), - chunk_timeout: Keyword.get(unquote(chunk_opts), :timeout, 1000) - } - end - - @impl Oban.Worker - def new(args, opts) when is_map(args) and is_list(opts) do - opts = - opts - |> Keyword.update(:meta, default_meta(), &Map.merge(default_meta(), &1)) - |> update_in([:meta, :chunk_by], &Utils.normalize_by/1) - - super(args, opts) - end - - @impl Oban.Worker - def perform(%Job{} = job) do - opts = __opts__() - - with {:ok, job} <- Oban.Pro.Worker.before_process(__MODULE__, job, opts) do - Process.put(:oban_processing, {__MODULE__, job, opts}) - - job - |> Chunk.maybe_process(__MODULE__) - |> Oban.Pro.Worker.after_process(__MODULE__, job, opts) - end - end + use Oban.Pro.Chunk, unquote(opts) end end - - @doc false - def validate(opts) do - Validation.validate_schema(opts, - by: {:custom, &Oban.Pro.Validation.validate_by/1}, - leading: :boolean, - size: :pos_integer, - sleep: :timeout, - timeout: :timeout - ) - end - - # Public Interface - - @doc false - def maybe_process(%Job{conf: conf} = job, worker) do - job = with_defaults(job, worker) - - cond do - full_timeout?(conf, job) -> - fetch_and_process(worker, job) - - full_chunk?(conf, job) -> - fetch_and_process(worker, job) - - true -> - job.meta - |> Map.get("chunk_timeout", 1000) - |> sleep_and_process(worker, job) - end - end - - defp with_defaults(%{meta: %{"chunk_size" => _}} = job, _worker), do: job - - defp with_defaults(job, worker) do - chunk_meta = - Map.new(worker.default_meta(), fn - {:chunk_by, val} -> {"chunk_by", Enum.map(val, &to_string/1)} - {key, val} -> {to_string(key), val} - end) - - %{job | meta: Map.merge(chunk_meta, job.meta)} - end - - # Check Helpers - - defp full_chunk?(conf, %{meta: %{"chunk_size" => size}} = job) do - limit = size - 1 - - query = - Job - |> select([j], j.id) - |> where([j], j.state == "available") - |> chunk_where(job) - |> limit(^limit) - - Repo.one(conf, from(oj in subquery(query), select: count(oj.id) >= ^limit)) - end - - defp full_timeout?(_, %{meta: %{"chunk_leading" => false}}), do: false - - defp full_timeout?(conf, %{meta: %{"chunk_timeout" => timeout}} = job) do - query = - Job - |> where([j], j.id != ^job.id) - |> where([j], j.state in ~w(completed cancelled discarded)) - |> chunk_where(job) - |> order_by(desc: :id) - |> limit(1) - |> select( - [j], - type( - fragment( - "greatest(?, ?, ?)", - j.completed_at, - j.discarded_at, - j.cancelled_at - ), - :utc_datetime_usec - ) - ) - - last_ts = Repo.one(conf, query) - - last_ts && - utc_now() - |> DateTime.add(-timeout, :millisecond) - |> DateTime.after?(last_ts) - end - - # Processing Helpers - - defp fetch_and_process(worker, %{conf: conf, meta: %{"chunk_size" => size}} = job) do - {:ok, chunk} = fetch_chunk(conf, job, size - 1) - {chunk, _errored} = prepare_chunk(chunk) - - guard_cancel(conf, worker, job, chunk) - guard_timeout(conf, worker.timeout(job), worker, job, chunk) - - try do - case worker.process([job | chunk]) do - :ok -> - ack_completed(conf, chunk, :ok) - - {:ok, result} -> - ack_completed(conf, chunk, {:ok, result}) - - {:cancel, reason, cancelled} -> - ack_cancelled(conf, worker, job, chunk, cancelled, reason) - - {:discard, reason, discarded} -> - ack_discarded(conf, worker, job, chunk, discarded, reason) - - {:error, reason, errored} -> - ack_errored(conf, worker, job, chunk, errored, reason) - - [_ | _] = multiple -> - ack_multiple(conf, worker, job, chunk, multiple) - end - rescue - error -> - ack_raised(conf, worker, chunk, job, error, __STACKTRACE__) - - reraise error, __STACKTRACE__ - catch - kind, reason -> - error = CrashError.exception({kind, reason, __STACKTRACE__}) - - ack_raised(conf, worker, chunk, job, error, __STACKTRACE__) - - reraise error, __STACKTRACE__ - end - end - - # This replicates the query used in Smart.fetch_jobs/3, without the meta tracking or any other - # complications. Any modifications to the original query must be replicated here. Another - defp fetch_chunk(conf, job, limit) do - subset_query = - Job - |> select([:id]) - |> where(state: "available") - |> chunk_where(job) - |> order_by(asc: :priority, asc: :scheduled_at, asc: :id) - |> limit(^limit) - |> lock("FOR UPDATE SKIP LOCKED") - - query = - Job - |> with_cte("subset", as: ^subset_query) - |> join(:inner, [j], x in fragment(~s("subset")), on: true) - |> where([j, x], j.id == x.id) - |> select([j, _], j) - - attempted_by = job.attempted_by ++ ["chunk-#{job.id}"] - - updates = [ - set: [state: "executing", attempted_at: utc_now(), attempted_by: attempted_by], - inc: [attempt: 1] - ] - - Repo.transaction(conf, fn -> - {_count, chunk} = Repo.update_all(conf, query, updates) - - chunk - end) - end - - defp prepare_chunk(chunk) do - Enum.reduce(chunk, {[], []}, fn job, {acc, err} -> - with {:ok, worker} <- Oban.Worker.from_string(job.worker), - {:ok, job} <- Oban.Pro.Worker.before_process(worker, job, worker.__opts__()) do - {[job | acc], err} - else - {:error, reason} -> - {acc, [{job, reason} | err]} - end - end) - end - - # Only the leader job is registered as "running" by the queue producer. We listen for pkill - # messages for _other_ jobs in the chunk and apply that to all jobs. Without this, cancelling - # doesn't apply to the leader and chunk jobs are orphaned. - defp guard_cancel(conf, worker, job, chunk) do - # No supervision tree is running when draining, attempting to notify will raise an exception. - if Process.get(:oban_draining) do - :ok - else - parent = self() - - Task.start(fn -> - ref = Process.monitor(parent) - :ok = Notifier.listen(conf.name, [:signal]) - - await_cancel(conf, ref, worker, job, chunk) - end) - end - end - - defp await_cancel(conf, ref, worker, job, chunk) do - receive do - {:notification, :signal, %{"action" => "pkill", "job_id" => kill_id}} -> - if Enum.any?(chunk, &(&1.id == kill_id)) do - reason = PerformError.exception({job.worker, {:cancel, :shutdown}}) - - ack_cancelled(conf, worker, job, chunk, chunk, reason) - - Notifier.notify(conf.name, :signal, %{action: "pkill", job_id: job.id}) - else - await_cancel(conf, ref, worker, job, chunk) - end - - {:DOWN, ^ref, :process, _pid, _reason} -> - :ok - end - end - - # The executor uses :timer.exit_after/2 to kill jobs that exceed the timeout. The queue's - # producer then catches the DOWN message and uses that to record a job error. The producer - # isn't aware of the job's chunk, so we monitor the parent process and ack the chunk jobs. - defp guard_timeout(_conf, :infinity, _worker, _job, _chunk), do: :ok - - defp guard_timeout(conf, timeout, worker, job, chunk) do - parent = self() - - Task.start(fn -> - ref = Process.monitor(parent) - - receive do - {:DOWN, ^ref, :process, _pid, %Oban.TimeoutError{} = reason} -> - ack_errored(conf, worker, job, chunk, chunk, reason) - - {:DOWN, ^ref, :process, _pid, _reason} -> - :ok - after - timeout + 100 -> :ok - end - end) - end - - defp sleep_and_process(total, worker, %Job{conf: conf, meta: meta} = job) do - sleep = Map.get(meta, "chunk_sleep", 1000) - - Process.sleep(sleep) - - if total - sleep < 0 or full_chunk?(conf, job) do - fetch_and_process(worker, job) - else - sleep_and_process(total - sleep, worker, job) - end - end - - # Chunk Helpers - - defp chunk_where(query, job) do - query = where(query, queue: ^job.queue) - - job.meta - |> Map.get("chunk_by") - |> Enum.reduce(query, fn - "worker", acc -> - where(acc, worker: ^job.worker) - - ["args", keys], acc -> - where(acc, [j], fragment("? @> ?", j.args, ^take_keys(job.args, keys))) - - ["meta", keys], acc -> - where(acc, [j], fragment("? @> ?", j.meta, ^take_keys(job.meta, keys))) - end) - end - - defp take_keys(args, keys) when is_struct(args) do - Map.take(args, Enum.map(keys, &String.to_existing_atom/1)) - end - - defp take_keys(map, keys), do: Map.take(map, keys) - - # Ack Helpers - - defp ack_completed(conf, jobs, result) do - Repo.update_all(conf, ids_query(jobs), com_ups()) - - result - end - - defp ack_raised(conf, worker, chunk, job, error, stacktrace) do - {dis, ers} = Enum.split_with(chunk, &exhausted?/1) - - opts = Repo.default_options(conf) - - multi = - Multi.new() - |> Multi.update_all(:err, ids_query(ers), err_ups(worker, job, error, stacktrace), opts) - |> Multi.update_all(:dis, ids_query(dis), dis_ups(worker, job, error, stacktrace), opts) - - Repo.transaction(conf, multi) - - :ok - end - - defp ack_errored(conf, worker, job, chunk, errored, reason) do - {ers, oks, set} = split_with_set(errored, chunk) - {dis, ers} = Enum.split_with(ers, &exhausted?/1) - - opts = Repo.default_options(conf) - - multi = - Multi.new() - |> Multi.update_all(:com, ids_query(oks), com_ups(), opts) - |> Multi.update_all(:err, ids_query(ers), err_ups(worker, job, {:error, reason}), opts) - |> Multi.update_all(:dis, ids_query(dis), dis_ups(worker, job, {:error, reason}), opts) - - with {:ok, _result} <- Repo.transaction(conf, multi) do - if MapSet.member?(set, job.id), do: {:error, reason}, else: :ok - end - end - - defp ack_cancelled(conf, worker, job, chunk, cancelled, reason) do - {can, oks, set} = split_with_set(cancelled, chunk) - - opts = Repo.default_options(conf) - - multi = - Multi.new() - |> Multi.update_all(:com, ids_query(oks), com_ups(), opts) - |> Multi.update_all(:can, ids_query(can), can_ups(worker, job, {:cancel, reason}), opts) - - with {:ok, _result} <- Repo.transaction(conf, multi) do - if MapSet.member?(set, job.id), do: {:cancel, reason}, else: :ok - end - end - - defp ack_discarded(conf, worker, job, chunk, discarded, reason) do - {dis, oks, set} = split_with_set(discarded, chunk) - - opts = Repo.default_options(conf) - - multi = - Multi.new() - |> Multi.update_all(:com, ids_query(oks), com_ups(), opts) - |> Multi.update_all(:dis, ids_query(dis), dis_ups(worker, job, {:discard, reason}), opts) - - with {:ok, _result} <- Repo.transaction(conf, multi) do - if MapSet.member?(set, job.id), do: {:discard, reason}, else: :ok - end - end - - defp ack_multiple(conf, worker, host, chunk, multiple) do - {oks, result} = split_multiple_result(host, chunk, multiple) - - opts = Repo.default_options(conf) - - group = fn {oper, reason, %{attempt: attempt}} -> {oper, reason, attempt} end - multi = Multi.update_all(Multi.new(), :com, ids_query(oks), com_ups(), opts) - - multi = - for({oper, {reas, jobs}} <- multiple, job <- jobs, job.id != host.id, do: {oper, reas, job}) - |> Enum.map(&maybe_discard/1) - |> Enum.group_by(group, &elem(&1, 2)) - |> Enum.with_index() - |> Enum.reduce(multi, fn {{{oper, reason, _}, jobs}, index}, acc -> - update = - case oper do - :cancel -> can_ups(worker, hd(jobs), {:cancel, reason}) - :discard -> dis_ups(worker, hd(jobs), {:discard, reason}) - :error -> err_ups(worker, hd(jobs), {:error, reason}) - end - - Multi.update_all(acc, {oper, index}, ids_query(jobs), update, opts) - end) - - with {:ok, _} <- Repo.transaction(conf, multi), do: result - end - - # Single Helpers - - defp ids_query(jobs) do - where(Job, [j], j.id in ^Enum.map(jobs, & &1.id)) - end - - defp split_with_set(sub_jobs, all_jobs) do - set = MapSet.new(sub_jobs, & &1.id) - - all_jobs - |> Enum.split_with(&MapSet.member?(set, &1.id)) - |> Tuple.insert_at(2, set) - end - - defp exhausted?(%Job{attempt: attempt, max_attempts: max}), do: attempt >= max - - # Multiple Helpers - - defp split_multiple_result(%{id: jid}, chunk, multiple) do - set = for {_, {_, jobs}} <- multiple, %{id: id} <- jobs, into: MapSet.new(), do: id - - result = - if MapSet.member?(set, jid) do - {oper, {reason, _}} = - Enum.find(multiple, fn {_, {_, jobs}} -> jid in Enum.map(jobs, & &1.id) end) - - {oper, reason} - else - :ok - end - - {Enum.reject(chunk, &MapSet.member?(set, &1.id)), result} - end - - defp maybe_discard({:error, reason, job}) when job.attempt >= job.max_attempts do - {:discard, reason, job} - end - - defp maybe_discard(tuple), do: tuple - - # Update Helpers - - defp com_ups do - [set: [state: "completed", completed_at: utc_now()]] - end - - defp can_ups(worker, job, reason) do - error = format_error(job, worker, reason, []) - - Keyword.new() - |> Keyword.put(:set, state: "cancelled", cancelled_at: utc_now()) - |> Keyword.put(:push, errors: error) - end - - defp err_ups(worker, job, error, stacktrace \\ []) do - error = format_error(job, worker, error, stacktrace) - - Keyword.new() - |> Keyword.put(:set, state: "retryable", scheduled_at: backoff_at(worker, job)) - |> Keyword.put(:push, errors: error) - end - - defp dis_ups(worker, job, error, stacktrace \\ []) do - error = format_error(job, worker, error, stacktrace) - - Keyword.new() - |> Keyword.put(:set, state: "discarded", discarded_at: utc_now()) - |> Keyword.put(:push, errors: error) - end - - defp backoff_at(worker, job) do - DateTime.add(utc_now(), worker.backoff(job), :second) - end - - defp format_error(job, worker, reason, stacktrace) when is_tuple(reason) do - format_error(job, worker, PerformError.exception({worker, reason}), stacktrace) - end - - defp format_error(job, _worker, error, stacktrace) do - {blamed, stacktrace} = Exception.blame(:error, error, stacktrace) - - formatted = Exception.format(:error, blamed, stacktrace) - - %{attempt: job.attempt, at: utc_now(), error: formatted} - end end diff --git a/vendor/oban_pro/lib/oban/pro/workers/workflow.ex b/vendor/oban_pro/lib/oban/pro/workers/workflow.ex index b94947ee..22920b7c 100644 --- a/vendor/oban_pro/lib/oban/pro/workers/workflow.ex +++ b/vendor/oban_pro/lib/oban/pro/workers/workflow.ex @@ -1,12 +1,5 @@ defmodule Oban.Pro.Workers.Workflow do - @moduledoc """ - A dedicated worker for linking the execution of jobs. - - This worker is deprecated in favor of `Oban.Pro.Workflow`, which is more flexible and supports - more options. - """ - - @moduledoc deprecated: "Use Oban.Pro.Workflow instead" + @moduledoc false import Ecto.Query, only: [group_by: 3, select: 3, where: 3] diff --git a/vendor/oban_pro/lib/oban/pro/workflow.ex b/vendor/oban_pro/lib/oban/pro/workflow.ex index 78f28120..9844de3e 100644 --- a/vendor/oban_pro/lib/oban/pro/workflow.ex +++ b/vendor/oban_pro/lib/oban/pro/workflow.ex @@ -47,74 +47,11 @@ defmodule Oban.Pro.Workflow do ![Simple Diagram](assets/simple-workflow.svg) - ## Common Patterns + This short example already covers the shapes most workflows reach for: sequential execution + (`:a → :b`), fan-out where one job triggers several (`:b → :c`, `:b → :d`), and fan-in where + multiple upstreams converge (`:c`, `:d → :e`). - Workflows support several execution patterns. These patterns form the building blocks of - workflow design, allowing you to create both simple linear processes and complex dependency - graphs. Each pattern serves a specific purpose in orchestrating job execution: - - > #### Example Workers {: .info} - > - > The following examples all use a generic `Worker` module to help focus on declaring - > dependencies. The patterns apply to any type of worker module or decorated function. - - ### Sequential Execution - - Jobs run one after another, waiting for `scheduled` or `retryable` delays: - - ```elixir - Workflow.new() - |> Workflow.add(:first, new_job()) - |> Workflow.add(:second, Worker.new(%{}), deps: :first) - |> Workflow.add(:third, Worker.new(%{}), deps: :second) - ``` - - ### Fan-Out (1-to-many) - - One job triggers multiple parallel jobs: - - ```elixir - Workflow.new() - |> Workflow.add(:parent, Worker.new(%{})) - |> Workflow.add(:child_1, Worker.new(%{}), deps: :parent) - |> Workflow.add(:child_2, Worker.new(%{}), deps: :parent) - |> Workflow.add(:child_3, Worker.new(%{}), deps: :parent) - ``` - - ### Fan-In (many-to-1) - - Multiple parallel jobs converge to a single job: - - ```elixir - Workflow.new() - |> Workflow.add(:step_1, Worker.new(%{})) - |> Workflow.add(:step_2, Worker.new(%{})) - |> Workflow.add(:step_3, Worker.new(%{})) - |> Workflow.add(:final, Worker.new(%{}), deps: [:step_1, :step_2, :step_3]) - ``` - - ### Diamond Pattern - - Combines fan-out and fan-in: - - ```elixir - Workflow.new() - |> Workflow.add(:start, Worker.new(%{})) - |> Workflow.add(:left, Worker.new(%{}), deps: :start) - |> Workflow.add(:right, Worker.new(%{}), deps: :start) - |> Workflow.add(:end, Worker.new(%{}), deps: [:left, :right]) - ``` - - These patterns can be combined to create complex workflows tailored to your specific business - processes. - - ## Advanced Patterns - - Once you're comfortable with basic workflows, you can leverage more advanced patterns to handle - complex scenarios. These patterns enable shared state, modular composition, simplified - dependencies, and dynamic generation of workflows. - - ### Sub-Workflows + ## Sub-Workflows Workflows can be nested hierarchically using sub workflows. This allows you to compose complex workflows from simpler ones, making it easier to organize and reuse workflow patterns. @@ -130,12 +67,12 @@ defmodule Oban.Pro.Workflow do alias Oban.Pro.Workflow extr_flow = - Workflow.new(workflow_name: "extract") + Workflow.new(name: "extract") |> Workflow.add(:extract, WorkerA.new(%{source: "database"})) |> Workflow.add(:transform, WorkerB.new(%{type: "normalize"}), deps: :extract) note_flow = - Workflow.new(workflow_name: "notify") + Workflow.new(name: "notify") |> Workflow.add(:prepare, WorkerA.new(%{template: "report"})) |> Workflow.add(:send, WorkerB.new(%{method: "email"}), deps: :prepare) @@ -158,7 +95,7 @@ defmodule Oban.Pro.Workflow do - Enabling reuse across applications - Simplifying the maintenance of complex job graphs - ### Sharing Results + ## Sharing Results Directed dependencies between jobs, paired with the `recorded` option, allow downstream jobs to fetch the output of upstream jobs. This is particularly useful for multi-step processes where @@ -245,7 +182,7 @@ defmodule Oban.Pro.Workflow do approach above works well, cascading functions provide an even more elegant way to share and transform data between workflow steps. - ### Cascading Functions + ## Cascading Functions Cascade mode allows you to build workflows using function captures that automatically receive context and previous step results. Each function receives a map containing the workflow context @@ -362,7 +299,7 @@ defmodule Oban.Pro.Workflow do - Implementing fan-out/fan-in patterns with minimal boilerplate - Maintaining clean separation between item processing and result aggregation - ### Dynamic Workflows + ## Dynamic Workflows Many workflows aren't static—the number of jobs and their interdependencies aren't known beforehand. You can generate workflows dynamically based on runtime conditions. @@ -515,12 +452,51 @@ defmodule Oban.Pro.Workflow do - Adding additional reporting or cleanup steps after seeing initial results - Breaking large workflows into stages that are built incrementally + ## Awaiting Signals + + Workflow jobs can pause mid-execution to wait for an external decision, then resume with the + signal payload. This turns workflows into durable state machines that can wait for human + approval, third-party callbacks, or any other out-of-band event without holding a process or + database connection open. + + Pause a job with `Oban.Pro.Worker.await_signal/1` and resume it with `signal/3,4` from anywhere— + another job, a Phoenix controller, an IEx session, etc: + + ```elixir + defmodule MyApp.Approval do + use Oban.Pro.Worker + + @impl Oban.Pro.Worker + def process(_job) do + case Oban.Pro.Worker.await_signal(wait_for: {24, :hours}) do + {:ok, %{"decision" => "approved"}} -> charge_card() + {:ok, %{"decision" => "rejected"}} -> {:cancel, :rejected} + {:error, :timeout} -> {:cancel, :no_decision} + end + end + end + ``` + + Then signal the job by its workflow name when the decision arrives: + + ```elixir + Workflow.signal(workflow_id, :approval, %{decision: "approved"}) + ``` + + A short live wait (`:wait_timeout`, default 5s) blocks during execution so signals that arrive + immediately resume in place. Past that window the job snoozes until either a signal arrives or + the overall `:wait_for` deadline elapses, freeing the worker process during the wait. Signals are + persisted, so a signal delivered before `await_signal/1` is reached is consumed on the next + call. + + See `Oban.Pro.Worker.await_signal/1` for the full set of options. + ## Customization Workflows provide several customization options to control their behavior, from identification to error handling. This section covers how to tailor workflows to your specific requirements. - #### Workflow ID + ### Workflow ID Every workflow needs a unique identifier. By default, `workflow_id` is a time-ordered, random [UUIDv7][uuid], which ensures uniqueness for any period. For more control, you can provide a @@ -532,18 +508,18 @@ defmodule Oban.Pro.Workflow do [uuid]: https://datatracker.ietf.org/doc/html/draft-peabody-dispatch-new-uuid-format-04#section-5.2 - #### Workflow Name + ### Workflow Name Additionally, workflows accept an optional name that describes their purpose: ```elixir - Workflow.new(workflow_name: "nightly-etl") + Workflow.new(name: "nightly-etl") ``` - While the `workflow_id` must be unique, the `workflow_name` doesn't have to be, so it can serve + While the `workflow_id` must be unique, the `name` doesn't have to be, so it can serve as a general-purpose label for categorizing or grouping workflows. - #### Context Key Format + ### Context Key Format By default, cascade functions receive context with atom keys for top level dependencies. However, _nested_ dependencies will have string keys. To prevent this inconsistency you can opt @@ -562,7 +538,7 @@ defmodule Oban.Pro.Workflow do With `atom_keys: false`, the `process/1` function receives `%{"fetch" => result}` instead of `%{fetch: result}`. - #### Dependency Handling + ### Dependency Handling By default, workflows use conservative dependency handling - if an upstream job is cancelled, discarded, or deleted, dependent jobs are automatically cancelled. You can customize this @@ -604,7 +580,7 @@ defmodule Oban.Pro.Workflow do > ... > ``` - #### Cancellation Callbacks + ### Cancellation Callbacks Workflow jobs are automatically `cancelled` when their upstream dependencies are `cancelled`, `discarded`, or `deleted` (unless specifically configured with the `ignore_*` options mentioned @@ -640,7 +616,7 @@ defmodule Oban.Pro.Workflow do - `:discarded` - A dependency was discarded - `:deleted` - A dependency was deleted - #### Handling Failures with Status Checking + ### Handling Failures with Status Checking For situations where you can't use the `after_cancelled` callback, or when you have many different workers in a workflow, you can combine the `ignore_cancelled` option with status @@ -857,12 +833,10 @@ defmodule Oban.Pro.Workflow do import Ecto.Query alias Ecto.Changeset - alias Oban.Pro.Stages.Chain - alias Oban.Pro.Utils - alias Oban.Pro.Workflow.Cascade alias Oban.{Config, Job, Repo, Validation} - - require Logger + alias Oban.Pro.Stages.{Chain, Hooks} + alias Oban.Pro.Utils + alias Oban.Pro.Workflow.{Cascade, Schema} # Types @@ -906,9 +880,12 @@ defmodule Oban.Pro.Workflow do @type name :: atom() | String.t() @type new_opts :: [ + id: String.t(), ignore_cancelled: boolean(), ignore_deleted: boolean(), ignore_discarded: boolean(), + name: String.t(), + unique: boolean(), workflow_id: String.t(), workflow_name: String.t() ] @@ -937,13 +914,7 @@ defmodule Oban.Pro.Workflow do # Callbacks - @doc """ - Called after a workflow job is cancelled due to upstream jobs being `cancelled`, `deleted`, or - `discarded`. - - This callback is _only_ called when a job is cancelled because of an upstream dependency. It is - _never_ called after normal job execution. For that, use `c:Oban.Pro.Worker.after_process/3`. - """ + @doc deprecated: "Use the universal `c:Oban.Pro.Worker.on_cancelled/2` callback instead" @callback after_cancelled(cancel_reason(), job :: Job.t()) :: :ok @optional_callbacks after_cancelled: 2 @@ -951,8 +922,7 @@ defmodule Oban.Pro.Workflow do # Constants @all_states Enum.map(Job.states(), &to_string/1) - @hold_date ~U[3000-01-01 00:00:00.000000Z] - @incomplete_set MapSet.new(~w(available executing retryable scheduled)) + @incomplete_set MapSet.new(~w(suspended available executing retryable scheduled)) @chan_keys ~w(max_attempts meta priority queue schedule_in scheduled_at tags)a @work_keys ~w(ignore_cancelled ignore_deleted ignore_discarded workflow_id workflow_name)a @@ -1001,17 +971,40 @@ defmodule Oban.Pro.Workflow do defmacrop in_workflow_or_sup(meta, workflow_id) do quote do fragment( - "? \\? 'workflow_id' AND (?->>'workflow_id' = ? OR ?->>'sup_workflow_id' = ?)", + "((? \\? 'workflow_id' AND ?->>'workflow_id' = ?) OR (? \\? 'sup_workflow_id' AND ?->>'sup_workflow_id' = ?))", unquote(meta), unquote(meta), unquote(workflow_id), unquote(meta), + unquote(meta), unquote(workflow_id) ) end end - defmacrop in_held_workflow(meta, workflow_id) do + defmacrop in_suspended(meta, workflow_id) do + quote do + fragment( + "? \\? 'workflow_id' AND ?->>'workflow_id' = ?", + unquote(meta), + unquote(meta), + unquote(workflow_id) + ) + end + end + + defmacrop in_suspended_sup(meta, workflow_id) do + quote do + fragment( + "? \\? 'sup_workflow_id' AND ?->>'sup_workflow_id' = ?", + unquote(meta), + unquote(meta), + unquote(workflow_id) + ) + end + end + + defmacrop in_legacy_suspended(meta, workflow_id) do quote do fragment( "? \\? 'workflow_id' AND ?->>'workflow_id' = ? AND ?->>'on_hold' = 'true'", @@ -1023,7 +1016,7 @@ defmodule Oban.Pro.Workflow do end end - defmacrop in_held_sup_workflow(meta, workflow_id) do + defmacrop in_legacy_suspended_sup(meta, workflow_id) do quote do fragment( "? \\? 'sup_workflow_id' AND ?->>'sup_workflow_id' = ? AND ?->>'on_hold' = 'true'", @@ -1035,10 +1028,6 @@ defmodule Oban.Pro.Workflow do end end - defguardp is_legacy_scheduled(meta) - when is_map_key(meta, "orig_scheduled_at") and - :erlang.map_get("orig_scheduled_at", meta) < 9_999_999_999 - # Telemetry @impl Oban.Pro.Handler @@ -1048,12 +1037,12 @@ defmodule Oban.Pro.Workflow do [:oban, :plugin, :stop] ] - :telemetry.attach_many("oban.workflow", events, &__MODULE__.handle_event/4, nil) + :telemetry.attach_many("oban.pro.workflow", events, &__MODULE__.handle_event/4, nil) end @impl Oban.Pro.Handler def on_stop do - :telemetry.detach("oban.workflow") + :telemetry.detach("oban.pro.workflow") end @doc false @@ -1083,7 +1072,7 @@ defmodule Oban.Pro.Workflow do %{"sup_workflow_id" => sup_workflow_id, "workflow_id" => workflow_id} -> {__MODULE__, :on_flush, [[sup_workflow_id, workflow_id], conf]} - %{"on_hold" => _, "workflow_id" => workflow_id} -> + %{"workflow_id" => workflow_id} -> {__MODULE__, :on_flush, [workflow_id, conf]} _ -> @@ -1091,58 +1080,132 @@ defmodule Oban.Pro.Workflow do end end + # CTE macros for dependency state lookup query + # These extract prefix/wid once and build clean SQL fragments + + # Extract values once from held to avoid repeated TOAST access + defmacrop held_prep_cte do + quote do + fragment(""" + SELECT h.id, h.meta, h.meta->>'workflow_id' AS job_wid, h.meta->'deps' AS deps + FROM "held" h + """) + end + end + + defmacrop workflow_jobs_cte(prefix, wid) do + quote do + fragment( + """ + SELECT meta->>'name' AS name, state + FROM ?.oban_jobs + WHERE meta \\? 'workflow_id' AND meta->>'workflow_id' = ? + """, + identifier(^unquote(prefix)), + ^unquote(wid) + ) + end + end + + defmacrop job_deps_cte do + quote do + fragment(""" + SELECT h.id AS job_id, h.job_wid, dep, jsonb_typeof(dep) AS dep_type + FROM held_prep h, jsonb_array_elements(h.deps) AS dep + """) + end + end + + defmacrop dep_lookup_cte(prefix, wid) do + quote do + fragment( + """ + SELECT jd.job_id, wj.state + FROM job_deps jd + LEFT JOIN "workflow_jobs" wj ON wj.name = jd.dep #>> '{}' + WHERE jd.dep_type = 'string' AND jd.job_wid = ? + + UNION ALL + + SELECT jd.job_id, oj.state + FROM "job_deps" jd + LEFT JOIN ?.oban_jobs oj ON + oj.meta \\? 'workflow_id' + AND oj.meta->>'workflow_id' = jd.job_wid + AND oj.meta->>'name' = jd.dep #>> '{}' + WHERE jd.dep_type = 'string' AND jd.job_wid <> ? + + UNION ALL + + SELECT jd.job_id, oj.state + FROM "job_deps" jd + LEFT JOIN ?.oban_jobs oj ON + oj.meta \\? 'workflow_id' + AND oj.meta->>'workflow_id' = jd.dep->>0 + AND oj.meta->>'name' = jd.dep->>1 + WHERE jd.dep_type = 'array' AND jd.dep->>1 <> '*' + + UNION ALL + + SELECT jd.job_id, oj.state + FROM "job_deps" jd + LEFT JOIN ?.oban_jobs oj ON + (oj.meta \\? 'workflow_id' AND oj.meta->>'workflow_id' = jd.dep->>0) + OR (oj.meta \\? 'sup_workflow_id' AND oj.meta->>'sup_workflow_id' = jd.dep->>0) + WHERE jd.dep_type = 'array' AND jd.dep->>1 = '*' + """, + ^unquote(wid), + identifier(^unquote(prefix)), + ^unquote(wid), + identifier(^unquote(prefix)), + identifier(^unquote(prefix)) + ) + end + end + + defmacrop job_states_cte do + quote do + fragment(""" + SELECT job_id, coalesce(array_agg(state::text) FILTER (WHERE state IS NOT NULL), ARRAY[]::text[]) AS states + FROM dep_lookup + GROUP BY job_id + """) + end + end + @doc false - def on_flush(workflow_id, conf) when is_binary(workflow_id) do - # The internal, external, and wildcard deps must be queried independently in order for the - # planner to utilize indexes properly. - lateral_query = - Job - |> select([j], j.state) - |> where( - [j], - # This uses a hardcoded reference to the `sf0` alias for the `parent_as(:elem)` source. - # This is to avoid using `parent_as(:elem).value`, which is valid syntax but - # incompatible with how CRDB handles jsonb references. Ecto requires a field reference - # on any source currently, and this is a workaround. - fragment("? \\? 'workflow_id'", j.meta) and - fragment( - """ - ( - (jsonb_typeof(sf0) = 'string' AND ?->>'workflow_id' = ?->>'workflow_id' AND ?->>'name' = sf0 #>>'{}') OR - (jsonb_typeof(sf0) = 'array' AND sf0->>1 <> '*' AND ?->>'workflow_id' = sf0->>0 AND ?->>'name' = sf0->>1) OR - (jsonb_typeof(sf0) = 'array' AND sf0->>1 = '*' AND ?->>'workflow_id' = sf0->>0) OR - (jsonb_typeof(sf0) = 'array' AND sf0->>1 = '*' AND ? \\? 'sup_workflow_id' AND ?->>'sup_workflow_id' = sf0->>0) - ) - """, - j.meta, - parent_as(:jobs).meta, - j.meta, - j.meta, - j.meta, - j.meta, - j.meta, - j.meta - ) - ) + def on_flush(wid, conf) when is_binary(wid) do + held_query = lock(Job, "FOR UPDATE SKIP LOCKED") - deps_query = - from v in fragment("jsonb_array_elements(?->'deps')", parent_as(:jobs).meta), - as: :elem, - inner_lateral_join: j in subquery(lateral_query), - on: true, - select: fragment("coalesce(array_agg(?), '{}')", j.state) + held_query = + if Utils.has_legacy_suspended?(conf) do + where( + held_query, + [j], + (j.state == "suspended" and in_suspended(j.meta, ^wid)) or + (j.state == "suspended" and in_suspended_sup(j.meta, ^wid)) or + (j.state == "scheduled" and in_legacy_suspended(j.meta, ^wid)) or + (j.state == "scheduled" and in_legacy_suspended_sup(j.meta, ^wid)) + ) + else + held_query + |> where([j], j.state == "suspended") + |> where([j], in_suspended(j.meta, ^wid) or in_suspended_sup(j.meta, ^wid)) + end + + prefix = conf.prefix - # A complete set of conditions must be used for held workflows and sub workflows. The planner - # won't use the correct indexes when an "OR" is used. query = - from(j in Job, as: :jobs) - |> select([j], {j, subquery(deps_query)}) - |> lock("FOR UPDATE SKIP LOCKED") - |> where( - [j], - (j.state == "scheduled" and in_held_workflow(j.meta, ^workflow_id)) or - (j.state == "scheduled" and in_held_sup_workflow(j.meta, ^workflow_id)) - ) + Job + |> with_cte("held", as: ^held_query) + |> with_cte("held_prep", as: held_prep_cte(), materialized: true) + |> with_cte("workflow_jobs", as: workflow_jobs_cte(prefix, wid), materialized: true) + |> with_cte("job_deps", as: job_deps_cte(), materialized: true) + |> with_cte("dep_lookup", as: dep_lookup_cte(prefix, wid)) + |> with_cte("job_states", as: job_states_cte()) + |> join(:inner, [j], h in fragment(~s("held")), on: j.id == h.id) + |> join(:left, [j, h], js in fragment(~s("job_states")), on: js.job_id == h.id) + |> select([j, h, js], {j.id, j.meta, js.states}) conf |> Repo.all(query) @@ -1177,6 +1240,11 @@ defmodule Oban.Pro.Workflow do """ @spec new(opts :: new_opts()) :: t() def new(opts \\ []) do + opts = + opts + |> expand_alias(:id, :workflow_id) + |> expand_alias(:name, :workflow_name) + validate!(opts) opts = @@ -1188,15 +1256,27 @@ defmodule Oban.Pro.Workflow do %__MODULE__{id: opts.workflow_id, opts: opts} end + defp expand_alias(opts, short, full) do + case Keyword.pop(opts, short) do + {nil, opts} -> opts + {value, opts} -> Keyword.put_new(opts, full, value) + end + end + defp validate!(opts) do Validation.validate_schema!(opts, atom_keys: :boolean, ignore_cancelled: :boolean, ignore_deleted: :boolean, ignore_discarded: :boolean, + unique: :boolean, workflow_id: :string, workflow_name: :string ) + + if opts[:unique] == true and is_nil(opts[:workflow_name]) do + raise ArgumentError, "the :name option is required when :unique is true" + end end @doc """ @@ -1229,7 +1309,6 @@ defmodule Oban.Pro.Workflow do |> Changeset.get_change(:meta, %{}) |> Map.put(:deps, deps) |> Map.put(:name, name) - |> Map.put(:on_hold, Enum.any?(deps)) |> Map.put(:workflow_id, workflow.id) |> Map.merge(workflow.opts) |> Map.merge(Map.new(opts)) @@ -1250,8 +1329,7 @@ defmodule Oban.Pro.Workflow do if Enum.any?(deps) do changeset |> Changeset.put_change(:meta, meta) - |> Changeset.put_change(:state, "scheduled") - |> Changeset.put_change(:scheduled_at, @hold_date) + |> Changeset.put_change(:state, "suspended") else Changeset.put_change(changeset, :meta, meta) end @@ -1465,7 +1543,7 @@ defmodule Oban.Pro.Workflow do end """ @doc since: "1.6.0" - @spec add_graft(t(), name(), Job.changeset() | cascade_capture(), add_opts()) :: t() + @spec add_graft(t(), name(), Job.changeset() | cascade_capture(), add_cascade_opts()) :: t() def add_graft(workflow, name, changeset_or_fun, opts \\ []) def add_graft(%__MODULE__{} = workflow, name, fun, opts) when is_function(fun) do @@ -1638,11 +1716,11 @@ defmodule Oban.Pro.Workflow do opts |> Map.new() - |> Map.put(:deps, deps) - |> Map.put(:sup_workflow_id, workflow.id) - |> Map.put(:sub_name, name) - |> Map.put(:workflow_id, sub_workflow.id) |> Map.put(:ancestor_ids, all_ancestors) + |> Map.put(:deps, deps) + |> Map.put(:sub_name, name) + |> Map.put(:sup_workflow_id, workflow.id) + |> Map.put(:workflow_id, sub_workflow.id) |> put_hold(changeset) else new_ancestors = Map.get(workflow.opts, :ancestor_ids, []) @@ -1682,24 +1760,9 @@ defmodule Oban.Pro.Workflow do end defp put_hold(meta, changeset) do - meta = - if Changeset.get_field(changeset, :state) == "scheduled" do - orig_at = - changeset - |> Changeset.get_change(:scheduled_at) - |> DateTime.to_unix(:microsecond) - - meta - |> Map.put(:on_hold, true) - |> Map.put(:orig_scheduled_at, orig_at) - else - Map.put(meta, :on_hold, true) - end - changeset |> Changeset.update_change(:meta, &Map.merge(&1, meta)) - |> Changeset.put_change(:state, "scheduled") - |> Changeset.put_change(:scheduled_at, @hold_date) + |> Changeset.put_change(:state, "suspended") end @doc """ @@ -2258,10 +2321,8 @@ defmodule Oban.Pro.Workflow do |> where([j], j.id in ^Enum.map(with_deps, & &1.id)) |> update([j], set: [ - state: "scheduled", + state: "suspended", max_attempts: fragment("GREATEST(?, ? + 1)", j.max_attempts, j.attempt), - meta: fragment("(? || ?) - 'orig_scheduled_at'", j.meta, ^%{on_hold: true}), - scheduled_at: ^DateTime.utc_now(), completed_at: nil, cancelled_at: nil, discarded_at: nil @@ -2275,6 +2336,50 @@ defmodule Oban.Pro.Workflow do end) end + @doc """ + Deliver a signal to one or more named jobs in a workflow. + + A thin wrapper over `Oban.Pro.Worker.signal/3` that resolves workflow job names before + delegating. See the `Oban.Pro.Worker` docs for the underlying semantics. + + ## Examples + + Signal a single named job: + + Workflow.signal(workflow_id, :approval, %{decision: "approved"}) + + Broadcast a signal to several named jobs: + + Workflow.signal(workflow_id, [:reviewer_a, :reviewer_b], %{decision: "approved"}) + + Target a non-default Oban instance: + + Workflow.signal(MyApp.Oban, workflow_id, :approval, %{decision: "approved"}) + """ + @doc since: "1.7.0" + @spec signal(binary(), name() | [name()], term()) :: :ok + def signal(workflow_id, name_or_names, payload) when is_binary(workflow_id) do + signal(Oban, workflow_id, name_or_names, payload) + end + + @spec signal(Oban.name(), binary(), name() | [name()], term()) :: :ok + def signal(oban_name, workflow_id, name_or_names, payload) when is_binary(workflow_id) do + conf = Oban.config(oban_name) + + names = + name_or_names + |> List.wrap() + |> Enum.map(&to_string/1) + + query = + Job + |> where([j], in_workflow(j.meta, ^workflow_id)) + |> where([j], fragment("?->>'name' = ANY(?)", j.meta, ^names)) + |> select([j], j.id) + + Oban.Pro.Worker.signal(oban_name, Repo.all(conf, query), payload) + end + @doc """ Get a single workflow job by name. @@ -2728,20 +2833,38 @@ defmodule Oban.Pro.Workflow do @doc false def rescue_workflows(conf, opts \\ []) do - {limit, opts} = Keyword.pop(opts, :limit, 1_000) + {chunk_size, opts} = Keyword.pop(opts, :chunk_size, 100) + {since_secs, opts} = Keyword.pop(opts, :max_age, 60) + + threshold = DateTime.add(DateTime.utc_now(), -since_secs, :second) query = - Job - |> where([j], j.state == "scheduled") - |> where([j], fragment("? \\? 'workflow_id'", j.meta)) - |> where([j], fragment("?->>'on_hold' = 'true'", j.meta)) - |> select([j], fragment("?->>'workflow_id'", j.meta)) - |> distinct(true) - |> limit(^limit) + Schema + |> select([w], w.id) + |> where([w], w.suspended > 0 and w.started_at < ^threshold) - conf - |> Repo.all(query, opts) - |> on_flush(conf) + workflow_ids = Repo.all(conf, query, opts) + + workflow_ids = + if Utils.has_legacy_suspended?(conf) do + legacy_query = + Job + |> select([j], fragment("?->>'workflow_id'", j.meta)) + |> where([j], j.state == "scheduled") + |> where([j], fragment("? \\? 'workflow_id'", j.meta)) + |> where([j], fragment("?->>'on_hold' = 'true'", j.meta)) + |> distinct(true) + + legacy_ids = Repo.all(conf, legacy_query, opts) + + Enum.uniq(workflow_ids ++ legacy_ids) + else + workflow_ids + end + + workflow_ids + |> Enum.chunk_every(chunk_size) + |> Enum.each(&on_flush(&1, conf)) end # Construction Helpers @@ -2885,7 +3008,7 @@ defmodule Oban.Pro.Workflow do end end - defp to_stage_operation({%{meta: meta} = job, deps_states}) do + defp to_stage_operation({job_id, meta, deps_states}) do # A grafted sub-workflow won't have jobs until the grafter runs. Ignore the "deleted" check # for the sub-workflow and rely entirely on the grafter job itself. deps = Enum.reject(meta["deps"], &match?(["grafted_" <> _, "*"], &1)) @@ -2914,9 +3037,6 @@ defmodule Oban.Pro.Workflow do meta["context"] == true -> :ignore - is_legacy_scheduled(meta) -> - {:scheduled, DateTime.from_unix!(meta["orig_scheduled_at"])} - is_map_key(meta, "orig_scheduled_at") -> {:scheduled, DateTime.from_unix!(meta["orig_scheduled_at"], :microsecond)} @@ -2924,19 +3044,22 @@ defmodule Oban.Pro.Workflow do :available end - {op, job} + {op, job_id} end defp apply_operations({:ignore, _}, _), do: [] - defp apply_operations({op, jobs}, conf) do + defp apply_operations({op, ids}, conf) do now = DateTime.utc_now() - ids = Enum.map(jobs, & &1.id) base = - Job - |> where([j], j.id in ^ids and j.state == "scheduled") - |> update([j], set: [meta: drop_hold(j.meta)]) + if Utils.has_legacy_suspended?(conf) do + Job + |> where([j], j.id in ^ids and j.state == "scheduled") + |> update([j], set: [meta: drop_hold(j.meta)]) + else + where(Job, [j], j.id in ^ids and j.state == "suspended") + end query = case op do @@ -2972,28 +3095,19 @@ defmodule Oban.Pro.Workflow do {_count, workflow_ids} = Repo.update_all(conf, query, set: []) - with {:cancelled, reason} <- op, do: Enum.each(jobs, &apply_callback(reason, &1, conf)) + with {:cancelled, reason} <- op do + Task.start(fn -> + conf + |> Repo.all(where(Job, [j], j.id in ^ids)) + |> Enum.each(&Hooks.execute_after_cancelled(reason, %{&1 | conf: conf})) + end) + end workflow_ids |> Enum.reject(&is_nil/1) |> Enum.uniq() end - defp apply_callback(reason, job, conf) do - with {:ok, worker} <- Oban.Worker.from_string(job.worker), - true <- function_exported?(worker, :after_cancelled, 2), - {:ok, job} <- Oban.Pro.Worker.before_process(worker, job, worker.__opts__()) do - worker.after_cancelled(reason, %{job | conf: conf}) - end - catch - kind, value -> - Logger.error(fn -> - "[Oban.Pro.Workflow] callback error: " <> Exception.format(kind, value, __STACKTRACE__) - end) - - :ok - end - # Recorded Helpers defp extract_recorded(meta) do diff --git a/vendor/oban_pro/lib/oban/pro/workflow/schema.ex b/vendor/oban_pro/lib/oban/pro/workflow/schema.ex new file mode 100644 index 00000000..7c366a2c --- /dev/null +++ b/vendor/oban_pro/lib/oban/pro/workflow/schema.ex @@ -0,0 +1,117 @@ +defmodule Oban.Pro.Workflow.Schema do + @moduledoc false + + use Ecto.Schema + + import Ecto.Changeset + + @fields ~w(id name parent_id suspended available scheduled executing + retryable completed cancelled discarded inserted_at meta)a + + @meta_fields ~w(ignore_cancelled ignore_discarded ignore_deleted unique)a + + @states ~w( + suspended + available + scheduled + executing + retryable + completed + cancelled + discarded + )a + + @counts Map.new(@states, &{&1, 0}) + + @primary_key {:id, :string, autogenerate: false} + schema "oban_workflows" do + field :name, :string + field :parent_id, :string + field :state, :string, read_after_writes: true + + field :suspended, :integer, default: 0 + field :available, :integer, default: 0 + field :scheduled, :integer, default: 0 + field :executing, :integer, default: 0 + field :retryable, :integer, default: 0 + field :completed, :integer, default: 0 + field :cancelled, :integer, default: 0 + field :discarded, :integer, default: 0 + + field :inserted_at, :utc_datetime_usec + field :started_at, :utc_datetime_usec + field :completed_at, :utc_datetime_usec + + field :meta, :map, default: %{} + end + + @doc false + def new(params) do + params = Map.put_new_lazy(params, :inserted_at, &DateTime.utc_now/0) + + %__MODULE__{} + |> cast(params, @fields) + |> validate_required(~w(id inserted_at)a) + |> unique_constraint(:name, name: :oban_workflows_unique_name) + end + + @doc false + def from_changesets({workflow_id, changesets}), do: from_changesets(workflow_id, changesets) + + @doc false + def from_changesets(workflow_id, changesets) do + [first | _] = changesets + meta = get_field(first, :meta, %{}) + + {queues, workers} = + Enum.reduce(changesets, {MapSet.new(), MapSet.new()}, fn changeset, {queues, workers} -> + queue = to_string(get_field(changeset, :queue)) + worker = get_field(changeset, :worker) + + {MapSet.put(queues, queue), MapSet.put(workers, worker)} + end) + + counts = count_by_state(changesets) + + workflow_meta = + meta + |> Map.take(@meta_fields) + |> Map.put("queues", MapSet.to_list(queues)) + |> Map.put("workers", MapSet.to_list(workers)) + + params = + counts + |> Map.put(:id, workflow_id) + |> Map.put(:name, build_name(meta, changesets)) + |> Map.put(:parent_id, meta[:sup_workflow_id]) + |> Map.put(:meta, workflow_meta) + + new(params) + end + + defp count_by_state(changesets) do + Enum.reduce(changesets, @counts, fn changeset, acc -> + state = + changeset + |> get_field(:state, "available") + |> String.to_existing_atom() + + Map.update!(acc, state, &(&1 + 1)) + end) + end + + defp build_name(%{workflow_name: name}, _) when is_binary(name), do: name + + defp build_name(_meta, changesets) do + Enum.find_value(changesets, fn changeset -> + meta = get_field(changeset, :meta) + worker = get_field(changeset, :worker) + + cond do + name = meta[:decorated_name] -> name + worker != "Oban.Pro.Workers.Context" -> worker + true -> nil + end + end) + end +end diff --git a/vendor/oban_pro/mix.exs b/vendor/oban_pro/mix.exs index 57441e20..738a96ee 100644 --- a/vendor/oban_pro/mix.exs +++ b/vendor/oban_pro/mix.exs @@ -1,7 +1,7 @@ defmodule Oban.Pro.MixProject do use Mix.Project - @version "1.6.13" + @version "1.7.0" def project do [ @@ -10,6 +10,7 @@ defmodule Oban.Pro.MixProject do elixir: "~> 1.15", start_permanent: Mix.env() == :prod, elixirc_paths: elixirc_paths(Mix.env()), + test_ignore_filters: [~r/fixture_app/], prune_code_paths: false, deps: deps(), docs: docs(), @@ -21,7 +22,7 @@ defmodule Oban.Pro.MixProject do # Dialyzer dialyzer: [ - plt_add_apps: [:ex_unit, :mix, :postgrex], + plt_add_apps: [:ex_unit, :inets, :mix, :postgrex], plt_core_path: "_build/#{Mix.env()}", flags: [:error_handling, :missing_return, :no_opaque, :underspecs] ] @@ -50,7 +51,7 @@ defmodule Oban.Pro.MixProject do def package do [ organization: "oban", - files: ~w(lib/pro.ex lib/oban .formatter.exs mix.exs), + files: ~w(lib/oban usage-rules .formatter.exs mix.exs usage-rules.md), licenses: ["Commercial"], links: [] ] @@ -61,21 +62,20 @@ defmodule Oban.Pro.MixProject do defp deps do [ - {:oban, "~> 2.19"}, - {:ecto_sql, "~> 3.10"}, - {:postgrex, "~> 0.16", optional: true}, + {:oban, "~> 2.21"}, + {:ecto_sql, "~> 3.12"}, + {:postgrex, "~> 0.22", optional: true}, # Test and Dev {:stream_data, "~> 1.1", only: [:test, :dev]}, - {:tz, "~> 0.26", only: [:test, :dev]}, - {:benchee, "~> 1.0", only: [:test, :dev], runtime: false}, - {:credo, "~> 1.7.7-rc.0", only: [:test, :dev], runtime: false}, - {:dialyxir, "~> 1.0", only: [:test, :dev], runtime: false}, + {:tz, "~> 0.28", only: [:test, :dev]}, + {:benchee, "~> 1.5", only: [:test, :dev], runtime: false}, + {:credo, "~> 1.7.17", only: [:test, :dev], runtime: false}, + {:dialyxir, "~> 1.4", only: [:test, :dev], runtime: false}, # Dev - {:ex_doc, "~> 0.21", only: :dev, runtime: false}, - {:makeup_diff, "~> 0.1", only: :dev, runtime: false}, - {:lys_publish, "~> 0.1", only: :dev, path: "../lys_publish", optional: true, runtime: false} + {:ex_doc, "~> 0.40", only: :dev, runtime: false}, + {:makeup_diff, "~> 0.1", only: :dev, runtime: false} ] end @@ -85,22 +85,24 @@ defmodule Oban.Pro.MixProject do "cmd git tag v#{@version}", "cmd git push", "cmd git push --tags", - "hex.publish package --yes", - "lys.publish" + "oban_pro.release", + "hex.publish package --yes" ], - "test.reset": ["ecto.drop -r Oban.Pro.Repo --quiet", "test.setup"], - "test.rollback": ["ecto.rollback -r Oban.Pro.Repo --all --quiet"], - "test.setup": [ + "db.reset": ["ecto.drop -r Oban.Pro.Repo --quiet", "db.setup"], + "db.rollback": ["ecto.rollback -r Oban.Pro.Repo --all --quiet"], + "db.setup": [ "ecto.create -r Oban.Pro.Repo --quiet", "ecto.migrate -r Oban.Pro.Repo --quiet" ], "test.ci": [ + fn _ -> System.put_env("CI", "true") end, "format --check-formatted", "deps.unlock --check-unused", "credo --strict", "test --raise", "dialyzer" ], + "test.reset": ["db.reset"], precommit: [ "format --check-formatted", "deps.unlock --check-unused", @@ -124,7 +126,7 @@ defmodule Oban.Pro.MixProject do logo: "assets/oban-pro-logo.svg", assets: %{"assets" => "assets"}, nest_modules_by_prefix: nest_modules_by_prefix(), - skip_undefined_reference_warnings_on: ["CHANGELOG.md"], + skip_undefined_reference_warnings_on: ["CHANGELOG.md", "guides/upgrading/v1.5.md"], before_closing_body_tag: fn _ -> """ @@ -151,6 +153,7 @@ defmodule Oban.Pro.MixProject do "guides/upgrading/v1.4.md", "guides/upgrading/v1.5.md", "guides/upgrading/v1.6.md", + "guides/upgrading/v1.7.md", "CHANGELOG.md": [filename: "changelog", title: "Changelog"] ] end @@ -166,16 +169,21 @@ defmodule Oban.Pro.MixProject do defp groups_for_modules do [ - Extensions: [ - Oban.Pro.Decorator, - Oban.Pro.Relay, - Oban.Pro.Testing, - Oban.Pro.Worker + Workers: [ + Oban.Pro.Worker, + Oban.Pro.Decorator ], Composition: [ Oban.Pro.Batch, + Oban.Pro.Chunk, Oban.Pro.Workflow ], + Utilities: [ + Oban.Pro.Cloud, + Oban.Pro.RateLimit, + Oban.Pro.Relay, + Oban.Pro.Testing + ], Plugins: [ Oban.Pro.Plugins.DynamicCron, Oban.Pro.Plugins.DynamicLifeline, @@ -184,20 +192,11 @@ defmodule Oban.Pro.MixProject do Oban.Pro.Plugins.DynamicPruner, Oban.Pro.Plugins.DynamicQueues, Oban.Pro.Plugins.DynamicScaler - ], - Scaling: [ - Oban.Pro.Cloud - ], - Workers: [ - Oban.Pro.Workers.Batch, - Oban.Pro.Workers.Chain, - Oban.Pro.Workers.Chunk, - Oban.Pro.Workers.Workflow ] ] end defp nest_modules_by_prefix do - [Oban.Pro, Oban.Pro.Plugins, Oban.Pro.Workers] + [Oban.Pro, Oban.Pro.Plugins] end end diff --git a/vendor/oban_pro/usage-rules.md b/vendor/oban_pro/usage-rules.md new file mode 100644 index 00000000..27b4c885 --- /dev/null +++ b/vendor/oban_pro/usage-rules.md @@ -0,0 +1,9 @@ +# Rules for Using Oban Pro + +Oban Pro is a commercial extension to Oban that provides advanced job processing features for +Elixir applications. It adds composition primitives (workflows, batches, chains, chunks), smart +concurrency controls (global limits, rate limiting), and enhanced worker capabilities (structured +args, encryption, deadlines, hooks). Read the reference files before writing Oban Pro code. Do not +assume standard Oban patterns apply. Pro has its own idioms and best practices. + +For full documentation: https://oban.pro/docs/pro/overview.html diff --git a/vendor/oban_pro/usage-rules/batches.md b/vendor/oban_pro/usage-rules/batches.md new file mode 100644 index 00000000..98d6a8af --- /dev/null +++ b/vendor/oban_pro/usage-rules/batches.md @@ -0,0 +1,43 @@ +# Batches + +Batches group jobs together and run callbacks after processing. + +```elixir +defmodule MyApp.EmailWorker do + use Oban.Pro.Worker + + @behaviour Oban.Pro.Batch + + @impl Oban.Pro.Worker + def process(%{args: %{"email" => email}}) do + MyApp.Mailer.deliver(email) + end + + @impl Oban.Pro.Batch + def batch_completed(job) do + Logger.info("Batch #{job.meta["batch_id"]} completed") + :ok + end +end +``` + +## Creating batches + +```elixir +alias Oban.Pro.Batch + +emails +|> Enum.map(&EmailWorker.new(%{email: &1})) +|> Batch.new() +|> Oban.insert_all() +``` + +## Callbacks + +| Callback | Triggered when | +|---------------------|---------------------------------------------| +| `batch_attempted/1` | All jobs executed at least once | +| `batch_completed/1` | All jobs completed successfully | +| `batch_exhausted/1` | All jobs completed, cancelled, or discarded | +| `batch_cancelled/1` | First job is cancelled | +| `batch_discarded/1` | First job is discarded | diff --git a/vendor/oban_pro/usage-rules/chains.md b/vendor/oban_pro/usage-rules/chains.md new file mode 100644 index 00000000..c162e0d6 --- /dev/null +++ b/vendor/oban_pro/usage-rules/chains.md @@ -0,0 +1,34 @@ +# Chains + +Chains ensure jobs run sequentially for the same entity. Use when order matters (e.g., webhook processing, account balance updates). + +```elixir +defmodule MyApp.WebhookWorker do + use Oban.Pro.Worker, + queue: :webhooks, + chain: [by: [args: :account_id]] + + @impl Oban.Pro.Worker + def process(%Oban.Job{args: args}) do + MyApp.Account.handle_webhook(args["account_id"], args["event"]) + end +end +``` + +## Partitioning + +```elixir +chain: [by: :worker] # All jobs of this worker chain together +chain: [by: [args: :account_id]] # Chain by account_id across workers +chain: [by: [:worker, args: :account_id]] # Chain by worker AND account_id +chain: [by: [args: [:account_id, :region]]] # Chain by multiple args +``` + +## Handling failures + +```elixir +chain: [by: [args: :id], on_discarded: :hold, on_cancelled: :hold] +``` + +- `:ignore` (default) — continue processing downstream jobs +- `:hold` — stop the chain until the failed job is resolved diff --git a/vendor/oban_pro/usage-rules/chunks.md b/vendor/oban_pro/usage-rules/chunks.md new file mode 100644 index 00000000..05ccc1fd --- /dev/null +++ b/vendor/oban_pro/usage-rules/chunks.md @@ -0,0 +1,34 @@ +# Chunks + +Chunks process jobs in groups based on size or timeout. Use for bulk operations and batched API calls. + +```elixir +defmodule MyApp.BulkIndexer do + use Oban.Pro.Chunk, size: 100, timeout: 5_000 + + @impl true + def process([_ | _] = jobs) do + jobs + |> Enum.map(& &1.args["document"]) + |> MyApp.Search.bulk_index() + + :ok + end +end +``` + +## Options + +- `size: 100` — process up to 100 jobs at once (default) +- `timeout: 1000` — process after 1 second even if size not reached (default) +- `by: [args: :tenant_id]` — partition chunks by field + +## Return values + +```elixir +:ok # All jobs succeeded +{:ok, result} # All jobs succeeded with result +{:error, reason, failed_jobs} # Some jobs failed, others succeed +{:cancel, reason, jobs} # Cancel specific jobs +{:snooze, seconds, jobs} # Snooze specific jobs +``` diff --git a/vendor/oban_pro/usage-rules/plugins.md b/vendor/oban_pro/usage-rules/plugins.md new file mode 100644 index 00000000..bf56ae55 --- /dev/null +++ b/vendor/oban_pro/usage-rules/plugins.md @@ -0,0 +1,237 @@ +# Plugins + +## Available Plugins + +- `DynamicCron`: Runtime-configurable cron scheduling with cluster-wide coordination +- `DynamicLifeline`: Rescues orphaned jobs, repairs stuck workflows/chains +- `DynamicPrioritizer`: Automatically increases priority of waiting jobs +- `DynamicPruner`: Flexible job retention with per-queue/worker rules +- `DynamicQueues`: Persists queue configuration across restarts +- `DynamicScaler`: Auto-scales cloud infrastructure based on queue depth + +## DynamicCron + +Runtime-configurable cron with cluster-wide coordination. Unlike standard Oban cron, DynamicCron +supports runtime updates, guaranteed scheduling, and automatic sync. + +### Configuration + +```elixir +{Oban.Pro.Plugins.DynamicCron, + timezone: "Etc/UTC", + sync_mode: :automatic, + crontab: [ + {"*/5 * * * *", MyApp.Workers.FrequentWorker}, + {"0 0 * * *", MyApp.Workers.DailyWorker, guaranteed: true}, + {"0 9 * * MON", MyApp.Workers.WeeklyWorker, timezone: "America/New_York"}, + {"@reboot", MyApp.Workers.StartupWorker} + ]} +``` + +### Options + +- `timezone` — default timezone for all entries (defaults to `"Etc/UTC"`) +- `sync_mode` — `:automatic` syncs crontab entries to the database on startup, removing stale + entries; `:manual` requires explicit `insert/1` and `delete/1` calls +- `guaranteed` — per-entry option that ensures missed jobs are inserted when the system comes back + online (use for critical scheduled tasks) + +### Runtime Management + +```elixir +alias Oban.Pro.Plugins.DynamicCron + +# Insert or update entries at runtime +DynamicCron.insert({"0 * * * *", MyApp.HourlyWorker}) + +# Delete an entry +DynamicCron.delete(MyApp.HourlyWorker) + +# Get an entry's details +DynamicCron.get(MyApp.HourlyWorker) + +# List all entries +DynamicCron.all() +``` + +## DynamicLifeline + +Rescues orphaned jobs and repairs stuck workflows and chains. Essential for production systems +where nodes may crash or restart unexpectedly. + +### Configuration + +```elixir +{Oban.Pro.Plugins.DynamicLifeline, retry_discarded: false} +``` + +### Options + +- `retry_discarded` — whether to retry discarded workflow/chain jobs when dependencies complete + +Do not change the `rescue_interval` option. If you see it configured, remove it—the default is +carefully tuned and changing it causes problems. + +### What It Rescues + +- **Orphaned jobs** — jobs stuck in `executing` state after a node crash +- **Stuck workflows** — workflows blocked by completed dependencies that weren't properly tracked +- **Stuck chains** — chains that stopped progressing due to missed completion signals +- **Partition backfill** — repopulates rate limit and global limit partitions after restarts + +## DynamicPrioritizer + +Automatically increases priority of jobs that have been waiting too long. Prevents starvation +of low-priority jobs in busy systems. + +### Configuration + +```elixir +{Oban.Pro.Plugins.DynamicPrioritizer, + prioritizers: [ + rate: [queue: :default, rate: 1, period: to_timeout(second: 30)], + once: [queue: :media, threshold: to_timeout(minute: 5), boost: 1] + ]} +``` + +### Strategies + +- `rate` — continuously boost priority at a fixed rate while waiting +- `once` — boost priority once after a threshold wait time + +### Options + +- `queue` — target queue (required) +- `rate` — priority boost per period (for `rate` strategy) +- `period` — how often to apply the boost (for `rate` strategy) +- `threshold` — wait time before boosting (for `once` strategy) +- `boost` — how much to increase priority (for `once` strategy, default: 1) + +## DynamicPruner + +Flexible job retention with per-queue and per-worker rules. More powerful than the standard +Oban pruner with fine-grained control over what gets deleted and when. + +### Configuration + +```elixir +{Oban.Pro.Plugins.DynamicPruner, + mode: {:max_age, {7, :days}}, + queue_overrides: [ + logs: {:max_age, {1, :day}}, + critical: {:max_age, {30, :days}} + ], + worker_overrides: [ + MyApp.AuditWorker: {:max_age, {90, :days}}, + MyApp.TempWorker: {:max_age, {1, :hour}} + ]} +``` + +### Modes + +- `{:max_age, {count, unit}}` — delete jobs older than the specified age +- `{:max_length, count}` — keep only the most recent N jobs + +### Options + +- `mode` — default retention rule for all jobs +- `queue_overrides` — per-queue retention rules +- `worker_overrides` — per-worker retention rules (takes precedence over queue rules) +- `state` — which job states to prune (default: `[:completed, :cancelled, :discarded]`) + +## DynamicQueues + +Persists queue configuration to the database for runtime management. Queues survive restarts +and can be modified without redeploying. + +### Configuration + +```elixir +{Oban.Pro.Plugins.DynamicQueues, + sync_mode: :automatic, + queues: [ + default: 10, + mailers: [local_limit: 20, rate_limit: [allowed: 100, period: {1, :minute}]], + exports: [local_limit: 5, only: {:node, :=~, "worker"}] + ]} +``` + +### Options + +- `sync_mode` — `:automatic` syncs queue config to database on startup; `:manual` requires + explicit API calls +- `queues` — initial queue definitions (same format as standard Oban queues) +- `only` — restrict queue to matching nodes using `{:node, pattern}` or `{:sys_env, key, pattern}`; + supports operators `:==`, `:!=`, `:=~` for pattern matching + +### Runtime Management + +```elixir +alias Oban.Pro.Plugins.DynamicQueues + +# Add or update a queue +DynamicQueues.insert(:reports, local_limit: 5) + +# Pause a queue +DynamicQueues.pause(:reports) + +# Resume a queue +DynamicQueues.resume(:reports) + +# Scale a queue +DynamicQueues.scale(:reports, local_limit: 20) + +# Delete a queue +DynamicQueues.delete(:reports) + +# List all queues +DynamicQueues.all() +``` + +## DynamicScaler + +Auto-scales cloud infrastructure based on queue depth and throughput. Scales nodes horizontally +to optimize processing during high traffic and save costs during quiet periods. + +### Configuration + +```elixir +{Oban.Pro.Plugins.DynamicScaler, + scalers: [ + range: 1..5, + cloud: MyApp.Cloud + ]} +``` + +### Options + +- `cloud` — module implementing the `Oban.Pro.Cloud` behaviour (required) +- `range` — min..max nodes to scale between (required) +- `cooldown` — minimum time between scaling events (default: 120 seconds) +- `queues` — `:all` or list of queues to monitor (default: `:all`) +- `step` — `:none` or max nodes to scale at once (default: `:none`) + +### Features + +- **Predictive scaling** — calculates optimal scale based on recent trends, not just current depth +- **Multi-cloud** — implement the `Cloud` behaviour for your platform (AWS, Fly, Kubernetes, etc.) +- **Thrashing prevention** — cooldown period prevents rapid scale up/down cycles + +### Cloud Behaviour + +Implement `Oban.Pro.Cloud` for your platform: + +```elixir +defmodule MyApp.Cloud do + @behaviour Oban.Pro.Cloud + + @impl true + def init(opts), do: opts + + @impl true + def current(_opts), do: {:ok, current_node_count()} + + @impl true + def scale(count, _opts), do: scale_to(count) +end +``` diff --git a/vendor/oban_pro/usage-rules/queues.md b/vendor/oban_pro/usage-rules/queues.md new file mode 100644 index 00000000..c0b93451 --- /dev/null +++ b/vendor/oban_pro/usage-rules/queues.md @@ -0,0 +1,54 @@ +# Queues + +Consider a dedicated queue when you need global concurrency, rate limiting, isolation, or different SLAs. + +## Global concurrency + +Limit concurrent jobs across all nodes: + +```elixir +# 10 jobs total across all nodes +my_queue: [global_limit: 10] + +# 10 global, but max 3 per node +my_queue: [local_limit: 3, global_limit: 10] + +# Exactly 1 job at a time cluster-wide +my_queue: [global_limit: 1] +``` + +## Rate limiting + +Control throughput over time: + +```elixir +# 60 jobs per minute +my_queue: [rate_limit: [allowed: 60, period: {1, :minute}]] + +# 1000 jobs per hour +my_queue: [rate_limit: [allowed: 1000, period: {1, :hour}]] + +# 10 jobs per 30 seconds +my_queue: [rate_limit: [allowed: 10, period: 30]] +``` + +Algorithms: `:sliding_window` (default, smooth), `:fixed_window` (simple resets), `:token_bucket` (allows bursts): + +```elixir +my_queue: [rate_limit: [allowed: 100, period: {1, :minute}, algorithm: :token_bucket]] +``` + +## Partitioning + +Apply limits per-partition instead of queue-wide (partition is nested inside the limit): + +```elixir +# Global limit of 1 per worker +my_queue: [global_limit: [allowed: 1, partition: :worker]] + +# Global limit of 1 per tenant_id from args +my_queue: [global_limit: [allowed: 1, partition: [args: :tenant_id]]] + +# Rate limit per account_id +my_queue: [rate_limit: [allowed: 100, period: {1, :minute}, partition: [args: :account_id]]] +``` diff --git a/vendor/oban_pro/usage-rules/testing.md b/vendor/oban_pro/usage-rules/testing.md new file mode 100644 index 00000000..2c5fc674 --- /dev/null +++ b/vendor/oban_pro/usage-rules/testing.md @@ -0,0 +1,48 @@ +# Testing + +Use `Oban.Pro.Testing` — a drop-in replacement for `Oban.Testing` with Pro helpers. + +## Configuration + +```elixir +# config/test.exs +config :my_app, Oban, testing: :manual +``` + +## Helpers + +**Unit testing** — execute jobs without the database: + +- `perform_job/3` — execute a single job +- `perform_callback/4` — execute batch callbacks +- `perform_chunk/3` — execute chunk workers + +**Integration testing** — insert and execute jobs: + +- `drain_jobs/1` — execute jobs across queues +- `run_workflow/2` — insert and execute a workflow +- `run_batch/2` — insert and execute a batch +- `run_chunk/2` — insert and execute chunked jobs + +**Always use `drain_jobs/1` instead of `Oban.drain_queue/2`:** + +- Drains all queues with `queue: :all` +- Handles workflows, batches, and chains correctly +- Supports recursive draining +- Returns completion counts + +```elixir +defmodule MyApp.WorkerTest do + use MyApp.DataCase, async: false + use Oban.Pro.Testing, repo: MyApp.Repo + + test "processes job" do + assert :ok == perform_job(SomeWorker, %{user_id: 123}) + end + + test "drains enqueued jobs" do + MyApp.enqueue_work() + assert %{completed: 3} = drain_jobs(queue: :all) + end +end +``` diff --git a/vendor/oban_pro/usage-rules/workers.md b/vendor/oban_pro/usage-rules/workers.md new file mode 100644 index 00000000..ebb2d646 --- /dev/null +++ b/vendor/oban_pro/usage-rules/workers.md @@ -0,0 +1,100 @@ +# Workers + +Use `Oban.Pro.Worker` with `args_schema` for type-safe, validated job arguments: + +```elixir +defmodule MyApp.SomeWorker do + use Oban.Pro.Worker, queue: :default, max_attempts: 3 + + args_schema do + field :user_id, :integer, required: true + field :status, :enum, values: ~w(pending active)a, default: :pending + field :metadata, :map + end + + @impl Oban.Pro.Worker + def process(%Oban.Job{args: %__MODULE__{} = args}) do + :ok + end +end +``` + +## Structured Args + +Validates data when calling `new/1`, before insertion. Notable types: + +- `:enum` - Validates and casts to atoms: `field :status, :enum, values: ~w(pending completed)a` +- `:term` - Safely encodes any Elixir term +- `:uuid` - Alias for `:binary_id` +- `{:array, *}` - Lists of typed values + +Nested structures with `embeds_one` and `embeds_many`: + +```elixir +args_schema do + field :order_id, :integer, required: true + embeds_one :customer, required: true do + field :customer_id, :integer, required: true + end + embeds_many :line_items do + field :product_id, :integer, required: true + field :quantity, :integer, required: true + end +end +``` + +## Encrypted jobs + +Protect sensitive args with transparent encryption: + +```elixir +use Oban.Pro.Worker, + queue: :default, + encrypted: [key: {Application, :fetch_env!, [:my_app, :oban_key]}] +``` + +**Important:** Only `args` are encrypted, never put sensitive data in `meta`. + +## Hooks + +**Execution hooks:** + +- `before_new/2` — modify args/opts before job creation +- `before_process/1` — runs before `process/1` +- `after_process/3` — runs after `process/1` with state (`:complete`, `:cancel`, `:discard`, `:error`, `:snooze`), job, and result + +**External state hooks:** + +- `on_cancelled/2` — job cancelled externally (manual, deadline, dependency) +- `on_discarded/2` — job discarded by DynamicLifeline + +```elixir +@impl Oban.Pro.Worker +def after_process(state, job, result) do + if state in [:error, :discard], do: notify_error(job) + :ok +end +``` + +Attach hooks explicitly or globally: + +```elixir +use Oban.Pro.Worker, hooks: [MyApp.ErrorHook] + +# Global hooks (in application startup) +Oban.Pro.Worker.attach_hook(:errors, MyApp.ErrorHook) +``` + +## Deadlines + +Cancel jobs that don't run within a time limit: + +```elixir +use Oban.Pro.Worker, deadline: {1, :hour} + +# Override at runtime +MyWorker.new(%{}, deadline: {30, :minutes}) + +# Force cancel already-running jobs +use Oban.Pro.Worker, deadline: [in: {10, :minutes}, force: true] +``` diff --git a/vendor/oban_pro/usage-rules/workflows.md b/vendor/oban_pro/usage-rules/workflows.md new file mode 100644 index 00000000..c93562a6 --- /dev/null +++ b/vendor/oban_pro/usage-rules/workflows.md @@ -0,0 +1,70 @@ +# Workflows + +Workflows compose jobs with dependency relationships: sequential, fan-out, and fan-in patterns. + +```elixir +alias Oban.Pro.Workflow + +Workflow.new() +|> Workflow.add(:extract, ExtractWorker.new(%{source: "db"})) +|> Workflow.add(:transform, TransformWorker.new(%{}), deps: :extract) +|> Workflow.add(:notify, NotifyWorker.new(%{}), deps: :extract) # fan-out +|> Workflow.add(:load, LoadWorker.new(%{}), deps: [:transform, :notify]) # fan-in +|> Oban.insert_all() +``` + +## Patterns + +- **Sequential**: `deps: :previous_job` +- **Fan-out**: Multiple jobs depend on the same parent +- **Fan-in**: One job depends on multiple parents with `deps: [:a, :b, :c]` + +## Accessing upstream results + +Use `recorded: true` workers: + +```elixir +def process(job) do + {:ok, data} = Workflow.fetch_recorded(job, :extract) +end +``` + +## Cascading Functions + +Function captures that automatically receive context and upstream results: + +```elixir +Workflow.new() +|> Workflow.put_context(%{source: source}) +|> Workflow.add_cascade(:extract, &extract/1) +|> Workflow.add_cascade(:transform, &transform/1, deps: :extract) +|> Workflow.add_cascade(:load, &load/1, deps: :transform) +|> Oban.insert_all() + +def extract(%{source: source}), do: %{records: fetch_records(source)} +def transform(%{extract: %{records: records}}), do: %{data: process(records)} +def load(%{transform: %{data: data}}), do: insert_all(data) +``` + +Fan-out with `{enumerable, function/2}`: + +```elixir +Workflow.add_cascade(:accounts, {account_ids, &sync_account/2}) +``` + +## Sub-workflows + +Compose from reusable pieces: + +```elixir +extract_flow = + Workflow.new() + |> Workflow.add(:fetch, FetchWorker.new(%{})) + |> Workflow.add(:parse, ParseWorker.new(%{}), deps: :fetch) + +Workflow.new() +|> Workflow.add(:setup, SetupWorker.new(%{})) +|> Workflow.add_workflow(:extract, extract_flow, deps: :setup) +|> Workflow.add(:cleanup, CleanupWorker.new(%{}), deps: :extract) +|> Oban.insert_all() +```