diff --git a/Dockerfile b/Dockerfile index 2d3f343b..9390dcf5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,6 +61,7 @@ ENV MIX_ENV="prod" # install mix dependencies (changes only when mix.exs/mix.lock change) COPY mix.exs mix.lock ./ +COPY vendor vendor RUN mix deps.get --only $MIX_ENV # copy compile-time config before compiling deps diff --git a/lib/microwaveprop_web/components/layouts.ex b/lib/microwaveprop_web/components/layouts.ex index d431723b..6320e82d 100644 --- a/lib/microwaveprop_web/components/layouts.ex +++ b/lib/microwaveprop_web/components/layouts.ex @@ -54,6 +54,13 @@ defmodule MicrowavepropWeb.Layouts do > Users + <.link + :if={@current_scope && @current_scope.user && @current_scope.user.is_admin} + href="/admin/oban" + class="btn btn-ghost btn-sm" + > + Oban + <%= if @current_scope && @current_scope.user do %> {@current_scope.user.callsign} diff --git a/lib/microwaveprop_web/router.ex b/lib/microwaveprop_web/router.ex index ac0e7b29..f4bb5992 100644 --- a/lib/microwaveprop_web/router.ex +++ b/lib/microwaveprop_web/router.ex @@ -2,6 +2,7 @@ defmodule MicrowavepropWeb.Router do use MicrowavepropWeb, :router import MicrowavepropWeb.UserAuth + import Oban.Web.Router import Phoenix.LiveDashboard.Router pipeline :browser do @@ -102,6 +103,8 @@ defmodule MicrowavepropWeb.Router do ecto_repos: [Microwaveprop.Repo], ecto_psql_extras_options: [long_running_queries: [threshold: "200 milliseconds"]], on_mount: [{MicrowavepropWeb.UserAuth, :require_admin}] + + oban_dashboard "/admin/oban", on_mount: [{MicrowavepropWeb.UserAuth, :require_admin}] end # Enable Swoosh mailbox preview in development diff --git a/mix.exs b/mix.exs index 9d58b8b3..9ae75542 100644 --- a/mix.exs +++ b/mix.exs @@ -66,7 +66,9 @@ defmodule Microwaveprop.MixProject do {:jason, "~> 1.2"}, {:bandit, "~> 1.5"}, {:styler, "~> 1.11", only: [:dev, :test], runtime: false}, - {:oban, "~> 2.19"}, + {:oban, "~> 2.21"}, + {:oban_met, "~> 1.0", path: "vendor/oban_met", override: true}, + {:oban_web, "~> 2.12", path: "vendor/oban_web"}, {:credo, "~> 1.7", only: [:dev, :test], runtime: false}, {:nx, "~> 0.9", only: [:dev, :test]}, {:axon, "~> 0.7", only: [:dev, :test]}, diff --git a/vendor/oban_met/.formatter.exs b/vendor/oban_met/.formatter.exs new file mode 100644 index 00000000..1b79ee44 --- /dev/null +++ b/vendor/oban_met/.formatter.exs @@ -0,0 +1,4 @@ +[ + import_deps: [:ecto_sql, :oban, :stream_data], + inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] +] diff --git a/vendor/oban_met/.hex b/vendor/oban_met/.hex new file mode 100644 index 00000000..67e377c4 Binary files /dev/null and b/vendor/oban_met/.hex differ diff --git a/vendor/oban_met/CHANGELOG.md b/vendor/oban_met/CHANGELOG.md new file mode 100644 index 00000000..83da1147 --- /dev/null +++ b/vendor/oban_met/CHANGELOG.md @@ -0,0 +1,113 @@ +# Changelog for Oban Met v1.0 + +## v1.1.0 — 2026-03-25 + +### Enhancements + +- [Met] Require a minimum of Oban v2.21 + + Oban v2.21 introduces the `suspended` state, which is now counted by the reporter along with the + other states. + +## v1.0.6 — 2026-02-19 + +### Enhancements + +- [Met] Add configurable time unit for timing metrics + + Metrics stored in sketches now support configurable time units via the `:sketch_time_unit` + compile-time option. By default, timing values remain in `:native` units for backward + compatibility. + + Setting `config :oban_met, sketch_time_unit: :millisecond` reduces sketch storage size by ~20% + and consolidates timing values into fewer bins by eliminating nanosecond-level measurement + noise. + +- [Cronitor] Share crontab options as a map rather than nested lists + + For compatibility with entries shared by python, we convert options to a + map structure rather than a list of lists. + +## v1.0.5 — 2025-12-10 + +### Bug Fixes + +- [Examiner] Normalize `limit` to `local_limit` for web consistency + + Oban checks return `limit`, while Pro stores them as `local_limit`. That causes inconsistent + data when displaying OSS queue details in `oban_web`. This stores `limit` as `local_limit` + consistently, rather than checking for both keys. + +## v1.0.4 — 2025-11-26 + +### Bug Fixes + +- [Met] Match on config to auto-start named instances + + The registry pattern only matched `Oban` instances, not modules created with `use Oban`. This + modifies the pattern to extract `Oban.Config` structs directly. + +- [Met] Remove use of struct update syntax deprecated in Elixir v1.19 + + +## v1.0.3 — 2025-05-06 + +### Enhancements + +- [Examiner] Use parallel execution for health checks. + + Minimize total time taken to scrape checks from producers by parallelizing calls. The worst case + time is now O(1) rather than O(n) when producers are busy. + +## v1.0.2 — 2025-03-14 + +This release requires Oban v2.19 because of a dependency on the new JSON module. + +### Bug Fixes + +- [Met] Log unexpected messages from trapping servers + + The `reporter` and `cronitor` have a small set of messages they expect. Rather than crashing on + unexpected messages, the processes now log a warning instead. + +- [Meta] Override application options with passed options + + Passed options weren't relayed to the underlying module and only application level options had + any effect. This ensures options from both locations are merged. + +- [Met] Use conditional JSON derive and Oban.JSON wrapper + +## v1.0.1 — 2025-01-16 + +### Bug Fixes + +- [Value] Implement `JSON.Encoder` for value types + + The encoder is necessary to work with official JSON encoded values as well as legacy Jason. + +## v1.0.0 — 2025-01-16 + +This is the official v1.0 release _and_ the package is now open source! + +### Enhancements + +- [Reporter] Count queries work with all official Oban engines and databases (Postgres, SQLite, + and MySQL). + +- [Migration] Add ability to explicitly create estimate function through a dedicated migration. + + For databases that don't allow the application connect to create new functions, this adds an + `Oban.Met.Migration` module and an option to disable auto-migrating in the reporter. + +### Bug Fixes + +- [Reporter] Skip queries with empty states or queues to prevent query errors. + + Querying with an empty list may cause the following error: + + ``` + (Postgrex.Error) ERROR 42809 (wrong_object_type) op ANY/ALL (array) + ``` + + This fixes the error by skipping queries for empty lists. This could happen if all of the states + were above the estimate threshold, or no active queues were found. diff --git a/vendor/oban_met/LICENSE.txt b/vendor/oban_met/LICENSE.txt new file mode 100644 index 00000000..f772ede7 --- /dev/null +++ b/vendor/oban_met/LICENSE.txt @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2025 The Oban Team + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/vendor/oban_met/README.md b/vendor/oban_met/README.md new file mode 100644 index 00000000..35883b60 --- /dev/null +++ b/vendor/oban_met/README.md @@ -0,0 +1,192 @@ +# Oban Met + +

+ + Hex Version + + + + Hex Docs + + + + CI Status + + + + Apache 2 License + +

+ + + +Met is a distributed, compacting, multidimensional, telemetry-powered time series datastore for +Oban that requires no configuration. It gathers data for queues, job counts, execution metrics, +active crontabs, historic metrics, and more. + +Met powers the charts and runtime details shown in the [Oban Web][web] dashboard. + +[web]: https://github.com/oban-bg/oban_web + +## Features + +- **🤖 Autonomous** - Supervises a collection of autonomous modules that dynamically start and stop alongside + Oban instances without any code changes. + +- **🎩 Distributed** - Metrics are shared between all connected nodes via pubsub. Leadership is used + to restrict expensive operations, such as performing counts, to a single node. + +- **📼 Recorded** - Telemetry events and scraped data are stored in-memory as time series data. + Values are stored as either gauges or space efficient "sketches". + +- **🪐 Multidimensional** - Metrics are stored with labels such as `node`, `queue`, `worker`, etc. + that can be filtered and grouped dynamically at runtime. + +- **🗜️ Compacting** - Time series values are periodically compacted into larger windows of time to + save space and optimize querying historic data. Compaction periods use safe defaults, but are + configurable. + +- **✏️ Estimating** - In supporting systems (Postgres), count queries use optimized estimates + automatically for tables with a large number of jobs. + +- **🔎 Queryable** - Historic metrics may be filtered and grouped by any label, sliced by + arbitrary time intervals, and numeric values aggregated at dynamic percentiles (e.g. P50, P99) + without pre-computed histogram buckets. + +- **🤝 Handoff** - Ephemeral data storage via data replication with handoff between nodes. All nodes have a + shared view of the cluster's data and new nodes are caught up when they come online. + +## Installation + +Oban Met is included with Oban Web and manual installation is only necessary in hybrid +environments (separate Web and Worker nodes). + +To receive metrics from non-web nodes in a system with separate "web" and "worker" applications +you must explicitly include `oban_met` as a dependency for "workers". + +```elixir +{:oban_met, "~> 1.0"} +``` + +## Usage + +No configuration is necessary and Oban Met will start automatically in a typical application. A +variety of options are provided for more complex or nuanced usage. + +### Auto Start + +Supervised Met instances start automatically along with Oban instances unless Oban is in testing +mode. You can disable auto-starting globally with application configuration: + +```elixir +config :oban_met, auto_start: false +``` + +Then, start instances as a child directly within your Oban app's plugins: + +```elixir +plugins: [ + Oban.Met, + ... +] +``` + +### Customizing Estimates + +Options for internal `Oban.Met` processes can be overridden from the plugin specification. Most +options are internal and not meant to be overridden, but one particularly useful option to tune is +the `estimate_limit`. The `estimate_limit` determines at which point state/queue counts switch +from using an accurate `count(*)` call to a much more efficient, but less accurate, estimate +function. + +The default limit is a conservative 50k, which may be too low for systems with insert spikes. This +declares an override to set the limit to 200k: + +```elixir +{Oban.Met, reporter: [estimate_limit: 200_000]} +``` + +### Customizing Check Interval + +The reporter periodically counts jobs in each state/queue combination to provide the counts +displayed in Oban Web. By default, counts are checked every 1 second. For systems where real-time +counts aren't critical, or to reduce database load, you can increase the interval: + +```elixir +{Oban.Met, reporter: [check_interval: :timer.seconds(5)]} +``` + +### Explicit Migrations + +Met will create the necessary estimate function automatically when possible. The migration isn't +necessary under normal circumstances, but is provided to avoid permission issues or allow full +control over database changes. + +```bash +mix ecto.gen.migration add_oban_met +``` + +Open the generated migration and delegate the `up/0` and `down/0` functions to +`Oban.Met.Migration`: + +```elixir +defmodule MyApp.Repo.Migrations.AddObanMet do + use Ecto.Migration + + def up, do: Oban.Met.Migration.up() + def down, do: Oban.Met.Migration.down() +end +``` + +Then, after disabling auto-start, configure the reporter not to auto-migrate if you run the +explicit migration: + +```elixir +{Oban.Met, reporter: [auto_migrate: false]} +``` + +### Sketch Time Unit + +Timing metrics (execution time, queue time) are stored in space-efficient quantile sketches. By +default, values are recorded in `:native` time units (nanoseconds on most systems), which provides +maximum precision but uses more space. + +For reduced storage size (~20% smaller) and better bin consolidation, you can configure sketches to +store timing in milliseconds: + +```elixir +config :oban_met, sketch_time_unit: :millisecond +``` + +This is a compile-time option that affects how timing values are binned in sketches. + +> #### Cluster Consistency {: .warning} +> +> All nodes in a cluster **must** use the same `sketch_time_unit` setting. Mixing units between +> nodes will produce incorrect aggregated metrics. +> +> After changing this setting, existing metrics will be invalid until compaction ages them out +> (approximately 2 hours with default compaction periods). + + + +## Contributing + +To run the test suite you must have PostgreSQL 12+. Once dependencies are installed, setup the +databases and run necessary migrations: + +```bash +mix test.setup +``` + +## Community + +There are a few places to connect and communicate with other Oban users: + +- Ask questions and discuss *#oban* on the [Elixir Forum][forum] +- [Request an invitation][invite] and join the *#oban* channel on Slack +- Learn about bug reports and upcoming features in the [issue tracker][issues] + +[invite]: https://elixir-slack.community/ +[forum]: https://elixirforum.com/ +[issues]: https://github.com/sorentwo/oban/issues diff --git a/vendor/oban_met/hex_metadata.config b/vendor/oban_met/hex_metadata.config new file mode 100644 index 00000000..c654163f --- /dev/null +++ b/vendor/oban_met/hex_metadata.config @@ -0,0 +1,28 @@ +{<<"links">>, + [{<<"Website">>,<<"https://oban.pro">>}, + {<<"Changelog">>, + <<"https://github.com/oban-bg/oban_met/blob/main/CHANGELOG.md">>}, + {<<"GitHub">>,<<"https://github.com/oban-bg/oban_met">>}]}. +{<<"name">>,<<"oban_met">>}. +{<<"version">>,<<"1.1.0">>}. +{<<"description">>, + <<"A distributed, compacting, multidimensional, telemetry-powered time series datastore">>}. +{<<"elixir">>,<<"~> 1.15">>}. +{<<"files">>, + [<<"lib">>,<<"lib/oban">>,<<"lib/oban/met">>,<<"lib/oban/met/examiner.ex">>, + <<"lib/oban/met/reporter.ex">>,<<"lib/oban/met/recorder.ex">>, + <<"lib/oban/met/cronitor.ex">>,<<"lib/oban/met/values">>, + <<"lib/oban/met/values/gauge.ex">>,<<"lib/oban/met/values/sketch.ex">>, + <<"lib/oban/met/value.ex">>,<<"lib/oban/met/migration.ex">>, + <<"lib/oban/met/listener.ex">>,<<"lib/oban/met/application.ex">>, + <<"lib/met.ex">>,<<".formatter.exs">>,<<"mix.exs">>,<<"README.md">>, + <<"CHANGELOG.md">>,<<"LICENSE.txt">>]}. +{<<"app">>,<<"oban_met">>}. +{<<"licenses">>,[<<"Apache-2.0">>]}. +{<<"requirements">>, + [[{<<"name">>,<<"oban">>}, + {<<"app">>,<<"oban">>}, + {<<"optional">>,false}, + {<<"requirement">>,<<"~> 2.21">>}, + {<<"repository">>,<<"hexpm">>}]]}. +{<<"build_tools">>,[<<"mix">>]}. diff --git a/vendor/oban_met/lib/met.ex b/vendor/oban_met/lib/met.ex new file mode 100644 index 00000000..bdd56adb --- /dev/null +++ b/vendor/oban_met/lib/met.ex @@ -0,0 +1,325 @@ +defmodule Oban.Met do + @external_resource readme = Path.join([__DIR__, "../README.md"]) + + @moduledoc readme + |> File.read!() + |> String.split("") + |> Enum.fetch!(1) + + use Supervisor + + alias Oban.Met.{Cronitor, Examiner, Listener, Recorder, Reporter, Value} + alias Oban.Registry + + @type counts :: %{optional(String.t()) => non_neg_integer()} + @type sub_counts :: %{optional(String.t()) => non_neg_integer() | counts()} + + @type series :: atom() | String.t() + @type value :: Value.t() + @type label :: String.t() + @type ts :: integer() + + @type filter_value :: label() | [label()] + + @type series_detail :: %{series: series(), labels: [label()], value: module()} + + @type operation :: :max | :sum | {:pct, float()} + + @type latest_opts :: [ + filters: keyword(filter_value()), + group: nil | label(), + lookback: pos_integer() + ] + + @type timeslice_opts :: [ + by: pos_integer(), + filters: keyword(filter_value()), + group: nil | label(), + label: nil | label(), + lookback: pos_integer(), + operation: operation(), + since: pos_integer() + ] + + @doc false + @spec child_spec(Keyword.t()) :: Supervisor.child_spec() + def child_spec(opts) do + conf = Keyword.fetch!(opts, :conf) + name = Registry.via(conf.name, __MODULE__) + + opts + |> super() + |> Map.put(:id, name) + end + + @doc """ + Start a Met supervisor for an Oban instance. + + `Oban.Met` typically starts supervisors automatically when Oban instances initialize. However, + starting a supervisor manually can be used if `auto_start` is disabled. + + ## Options + + These options are required; without them the supervisor won't start: + + * `:conf` — configuration for a running Oban instance, required + + * `:name` — an optional name for the supervisor, defaults to `Oban.Met` + + ## Example + + Start a supervisor for the default Oban instance: + + Oban.Met.start_link(conf: Oban.config()) + + Start a supervisor with a custom name: + + Oban.Met.start_link(conf: Oban.config(), name: MyApp.MetSup) + """ + @spec start_link(Keyword.t()) :: Supervisor.on_start() + def start_link(opts) when is_list(opts) do + conf = Keyword.fetch!(opts, :conf) + name = Registry.via(conf.name, __MODULE__) + + Supervisor.start_link(__MODULE__, opts, name: name) + end + + @doc """ + Retrieve stored producer checks. + + This mimics the output of the legacy `Oban.Web.Plugins.Stats.all_gossip/1` function. + + Checks are queried approximately every second and broadcast to all connected nodes, so each node + is a replica of checks from the entire cluster. Checks are stored for 30 seconds before being + purged. + + ## Output + + Checks are the result of `Oban.check_queue/1`, and the exact contents depends on which + `Oban.Engine` is in use. A `Basic` engine check will look similar to this: + + %{ + uuid: "2dde4c0f-53b8-4f59-9a16-a9487454292d", + limit: 10, + node: "me@local", + paused: false, + queue: "default", + running: [100, 102], + started_at: ~D[2020-10-07 15:31:00], + updated_at: ~D[2020-10-07 15:31:00] + } + + ## Examples + + Get all current checks: + + Oban.Met.checks() + + Get current checks for a non-standard Oban isntance: + + Oban.Met.checks(MyOban) + """ + @spec checks(Oban.name()) :: [map()] + def checks(oban \\ Oban) do + oban + |> Registry.via(Examiner) + |> Examiner.all_checks() + end + + @doc """ + Get a normalized, unified crontab from all connected nodes. + + ## Examples + + Get a merged crontab: + + Oban.Met.crontab() + [ + {"* * * * *", "Worker.A", []}, + {"* * * * *", "Worker.B", [["args", %{"mode" => "foo"}]]} + ] + + Get the crontab for a non-standard Oban instance: + + Oban.Met.crontab(MyOban) + """ + @spec crontab(Oban.name()) :: [{binary(), binary(), map()}] + def crontab(oban \\ Oban) do + oban + |> Registry.via(Cronitor) + |> Cronitor.merged_crontab() + end + + @doc """ + Get all stored, unique values for a particular label. + + ## Examples + + Get all known queues: + + Oban.Met.labels("queue") + ~w(alpha gamma delta) + + Get all known workers: + + Oban.Met.labels("worker") + ~w(MyApp.Worker MyApp.OtherWorker) + """ + @spec labels(Oban.name(), label(), keyword()) :: [label()] + def labels(oban \\ Oban, label, opts \\ []) do + oban + |> Registry.via(Recorder) + |> Recorder.labels(to_string(label), opts) + end + + @doc """ + Get all stored values for a series without any filtering. + """ + @spec lookup(Oban.name(), series()) :: [term()] + def lookup(oban \\ Oban, series) do + oban + |> Registry.via(Recorder) + |> Recorder.lookup(to_string(series)) + end + + @doc """ + Get the latest values for a gauge series, optionally subdivided by a label. + + Unlike queues and workers, states are static and constant, so they'll always show up in the + counts or subdivision maps. + + ## Gauge Series + + Latest counts only apply to `Gauge` series. There are two gauges available (as reported by + `series/1`: + + * `:exec_count` — jobs executing at that moment, including `node`, `queue`, `state`, and + `worker` labels. + + * `:full_count` — jobs in the database, including `queue`, and `state` labels. + + ## Examples + + Get the `:full_count` value without any grouping: + + Oban.Met.latest(:full_count) + %{"all" => 99} + + Group the `:full_count` value by state: + + Oban.Met.latest(:full_count, group: "state") + %{"available" => 9, "completed" => 80, "executing" => 5, ... + + Group results by queue: + + Oban.Met.latest(:exec_count, group: "queue") + %{"alpha" => 9, "gamma" => 3} + + Group results by node: + + Oban.Met.latest(:exec_count, group: "node") + %{"worker.1" => 6, "worker.2" => 5} + + Filter values by node: + + Oban.Met.latest(:exec_count, filters: [node: "worker.1"]) + %{"all" => 6} + + Filter values by queue and state: + + Oban.Met.latest(:exec_count, filters: [node: "worker.1", "worker.2"]) + """ + @spec latest(Oban.name(), series()) :: counts() | sub_counts() + def latest(oban \\ Oban, series, opts \\ []) do + oban + |> Registry.via(Recorder) + |> Recorder.latest(to_string(series), opts) + end + + @doc """ + List all recorded series along with their labels and value type. + + ## Examples + + Oban.Met.series() + [ + %{series: "exec_time", labels: ["state", "queue", "worker"], value: Sketch}, + %{series: "wait_time", labels: ["state", "queue", "worker"], value: Sketch}, + %{series: "exec_count", labels: ["state", "queue", "worker"], value: Gauge}, + %{series: "full_count", labels: ["state", "queue"], value: Gauge} + ] + """ + @spec series(Oban.name()) :: [series_detail()] + def series(oban \\ Oban) do + oban + |> Registry.via(Recorder) + |> Recorder.series() + end + + @doc """ + Summarize a series of data with an aggregate over a configurable window of time. + + ## Examples + + Retreive a 3 second timeslice of the `exec_time` sketch: + + Oban.Met.timeslice(Oban, :exec_time, lookback: 3) + [ + {2, 16771374649.128689, nil}, + {1, 24040058779.3428, nil}, + {0, 22191534459.516357, nil}, + ] + + Group `exec_time` slices by the `queue` label: + + Oban.Met.timeslice(Oban, :exec_time, group: "queue") + [ + {1, 9970235387.031698, "analysis"}, + {0, 11700429279.446463, "analysis"}, + {1, 23097311376.231316, "default"}, + {0, 23097311376.231316, "default"}, + {1, 1520977874.3348415, "events"}, + {0, 2558504265.2738624, "events"}, + ... + """ + @spec timeslice(Oban.name(), series(), timeslice_opts()) :: [{ts(), value(), label()}] + def timeslice(oban \\ Oban, series, opts \\ []) do + oban + |> Registry.via(Recorder) + |> Recorder.timeslice(to_string(series), opts) + end + + # Callbacks + + @impl Supervisor + def init(opts) do + conf = Keyword.fetch!(opts, :conf) + + children = [ + {Cronitor, with_opts(:cronitor, conf, opts)}, + {Examiner, with_opts(:examiner, conf, opts)}, + {Recorder, with_opts(:recorder, conf, opts)}, + {Listener, with_opts(:listener, conf, opts)}, + {Reporter, with_opts(:reporter, conf, opts)}, + {Task, fn -> :telemetry.execute([:oban, :met, :init], %{}, %{pid: self(), conf: conf}) end} + ] + + Supervisor.init(children, strategy: :one_for_one) + end + + defp with_opts(name, conf, opts) do + real_name = + name + |> to_string() + |> String.capitalize() + |> then(&Module.safe_concat([Oban, Met, &1])) + + init_opts = Keyword.get(opts, name, []) + + :oban_met + |> Application.get_env(name, []) + |> Keyword.merge(init_opts) + |> Keyword.put(:conf, conf) + |> Keyword.put(:name, Registry.via(conf.name, real_name)) + end +end diff --git a/vendor/oban_met/lib/oban/met/application.ex b/vendor/oban_met/lib/oban/met/application.ex new file mode 100644 index 00000000..38951188 --- /dev/null +++ b/vendor/oban_met/lib/oban/met/application.ex @@ -0,0 +1,83 @@ +defmodule Oban.Met.Application do + @moduledoc false + + use Application + + alias Oban.{Config, Registry} + + require Logger + + @main_sup Oban.Met.Supervisor + @task_sup Oban.Met.TaskSupervisor + @handler_id :oban_met_handler + + @impl Application + def start(_type, _args) do + :telemetry.attach(@handler_id, [:oban, :supervisor, :init], &__MODULE__.init_metrics/4, []) + + children = [ + {Task.Supervisor, name: @task_sup}, + {Task, &boot_metrics/0} + ] + + Supervisor.start_link(children, strategy: :one_for_one, name: @main_sup) + end + + @impl Application + def prep_stop(state) do + :telemetry.detach(@handler_id) + + state + end + + @doc false + def init_metrics(_event, _measure, %{conf: conf}, _conf) do + start_metrics(conf) + end + + @doc false + def boot_metrics do + guard = {:andalso, {:is_map, :"$1"}, {:==, {:map_get, :__struct__, :"$1"}, Config}} + + [{{:_, :_, :"$1"}, [guard], [:"$1"]}] + |> Registry.select() + |> Enum.each(&start_metrics/1) + end + + defp start_metrics(conf) do + opts = Application.get_all_env(:oban_met) + + if opts[:auto_start] and conf.testing in opts[:auto_testing_modes] do + spec = Oban.Met.child_spec(conf: conf) + + case Supervisor.start_child(@main_sup, spec) do + {:ok, _pid} -> + Task.Supervisor.start_child(@task_sup, fn -> watch_oban(spec, conf) end) + + :ok + + {:error, {:already_started, _pid}} -> + :ok + + {:error, {error, _stack}} -> + Logger.error("Unable to start Oban.Met supervisor: #{inspect(error)}") + end + end + end + + defp watch_oban(%{id: child_id}, conf) do + ref = + conf.name + |> Oban.whereis() + |> Process.monitor() + + receive do + {:DOWN, ^ref, :process, _pid, _reason} -> + Process.demonitor(ref, [:flush]) + + with :ok <- Supervisor.terminate_child(@main_sup, child_id) do + Supervisor.delete_child(@main_sup, child_id) + end + end + end +end diff --git a/vendor/oban_met/lib/oban/met/cronitor.ex b/vendor/oban_met/lib/oban/met/cronitor.ex new file mode 100644 index 00000000..ecabb58f --- /dev/null +++ b/vendor/oban_met/lib/oban/met/cronitor.ex @@ -0,0 +1,134 @@ +defmodule Oban.Met.Cronitor do + @moduledoc false + + use GenServer + + alias __MODULE__, as: State + alias Oban.Notifier + + require Logger + + defstruct [ + :conf, + :name, + :timer, + crontabs: %{}, + interval: :timer.seconds(15), + ttl: :timer.seconds(60) + ] + + @spec child_spec(Keyword.t()) :: Supervisor.child_spec() + def child_spec(opts) do + name = Keyword.get(opts, :name, __MODULE__) + + opts + |> super() + |> Map.put(:id, name) + end + + @spec start_link(Keyword.t()) :: GenServer.on_start() + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, name: opts[:name]) + end + + def merged_crontab(name) do + GenServer.call(name, :merged_crontab) + end + + # Callbacks + + @impl GenServer + def init(opts) do + Process.flag(:trap_exit, true) + + state = struct!(State, opts) + + Notifier.listen(state.conf.name, [:cronitor]) + + {:ok, state, {:continue, :start}} + end + + @impl GenServer + def handle_continue(:start, %State{} = state) do + handle_info(:share, state) + end + + @impl GenServer + def terminate(_reason, state) do + if is_reference(state.timer), do: Process.cancel_timer(state.timer) + + :ok + end + + @impl GenServer + def handle_call(:share, _from, %State{} = state) do + {:noreply, state} = handle_info(:share, state) + + {:reply, :ok, state} + end + + def handle_call(:merged_crontab, _from, %State{} = state) do + merged_crontab = + for {_key, {_ts, crontab}} <- state.crontabs, entry <- crontab, uniq: true, do: entry + + {:reply, merged_crontab, state} + end + + @impl GenServer + def handle_info(:share, %State{} = state) do + %{name: name, node: node, plugins: plugins} = state.conf + + crontab = + plugins + |> Keyword.get(Oban.Plugins.Cron, []) + |> Keyword.get(:crontab, []) + |> Enum.map(fn + {expr, work} -> {expr, inspect(work), %{}} + {expr, work, opts} -> {expr, inspect(work), Map.new(opts)} + end) + + payload = %{crontab: crontab, name: inspect(name), node: node} + + Notifier.notify(state.conf, :cronitor, payload) + + state = + state + |> purge_stale() + |> schedule_share() + + {:noreply, state} + end + + def handle_info({:notification, :cronitor, payload}, %State{} = state) do + %{"crontab" => crontab, "name" => name, "node" => node} = payload + + ts = System.system_time(:millisecond) + crontab = Enum.map(crontab, &List.to_tuple/1) + crontabs = Map.put(state.crontabs, {node, name}, {ts, crontab}) + + {:noreply, %{state | crontabs: crontabs}} + end + + def handle_info(message, state) do + Logger.warning( + message: "Received unexpected message: #{inspect(message)}", + source: :oban_met, + module: __MODULE__ + ) + + {:noreply, state} + end + + # Helpers + + defp purge_stale(%State{} = state) do + expires = System.system_time(:millisecond) - state.ttl + crontabs = Map.reject(state.crontabs, fn {_key, {ts, _tab}} -> ts < expires end) + + %{state | crontabs: crontabs} + end + + defp schedule_share(%State{} = state) do + %{state | timer: Process.send_after(self(), :share, state.interval)} + end +end diff --git a/vendor/oban_met/lib/oban/met/examiner.ex b/vendor/oban_met/lib/oban/met/examiner.ex new file mode 100644 index 00000000..48777b5c --- /dev/null +++ b/vendor/oban_met/lib/oban/met/examiner.ex @@ -0,0 +1,180 @@ +defmodule Oban.Met.Examiner do + @moduledoc false + + # Examiner uses notifications to periodically exchange queue state information between all + # interested nodes. + + # This module is more of a "producer queue checker", but that name stinks. + + use GenServer + + alias __MODULE__, as: State + alias Oban.Notifier + + require Logger + + @type name_or_table :: :ets.tab() | GenServer.name() + + defstruct [ + :conf, + :name, + :table, + :timer, + interval: 750, + ttl: :timer.seconds(30) + ] + + @spec child_spec(Keyword.t()) :: Supervisor.child_spec() + def child_spec(opts) do + name = Keyword.get(opts, :name, __MODULE__) + + opts + |> super() + |> Map.put(:id, name) + end + + @spec start_link(Keyword.t()) :: GenServer.on_start() + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, name: opts[:name]) + end + + @spec all_checks(name_or_table()) :: [map()] + def all_checks(name_or_table) do + name_or_table + |> table() + |> :ets.select([{{:_, :_, :"$1"}, [], [:"$1"]}]) + end + + @spec store(name_or_table(), map(), timestamp: integer()) :: :ok + def store(name_or_table, check, opts \\ []) when is_map(check) do + %{"node" => node, "name" => name, "queue" => queue} = check + + check = normalize_limit(check) + timestamp = Keyword.get(opts, :timestamp, System.system_time(:millisecond)) + + name_or_table + |> table() + |> :ets.insert({{node, name, queue}, timestamp, check}) + + :ok + end + + @spec purge(name_or_table(), pos_integer()) :: {:ok, non_neg_integer()} + def purge(name_or_table, ttl) when is_integer(ttl) and ttl > 0 do + expires = System.system_time(:millisecond) - ttl + pattern = [{{:_, :"$1", :_}, [{:<, :"$1", expires}], [true]}] + + deleted = + name_or_table + |> table() + |> :ets.select_delete(pattern) + + {:ok, deleted} + end + + # Callbacks + + @impl GenServer + def init(opts) do + Process.flag(:trap_exit, true) + + table = :ets.new(:checks, [:public, read_concurrency: true]) + state = struct!(State, Keyword.put(opts, :table, table)) + + Notifier.listen(state.conf.name, [:gossip]) + + Registry.register(Oban.Registry, state.name, table) + + {:ok, state, {:continue, :start}} + end + + @impl GenServer + def handle_continue(:start, %State{} = state) do + handle_info(:check, state) + end + + @impl GenServer + def terminate(_reason, state) do + if is_reference(state.timer), do: Process.cancel_timer(state.timer) + + :ok + end + + @impl GenServer + def handle_info(:check, %State{} = state) do + match = [{{{state.conf.name, {:producer, :_}}, :"$1", :_}, [], [:"$1"]}] + + checks = + Oban.Registry + |> Registry.select(match) + |> Task.async_stream(&safe_check(&1, state), on_timeout: :kill_task, ordered: false) + |> Enum.reduce([], fn + {:ok, check}, acc when is_map(check) -> [check | acc] + _, acc -> acc + end) + + if Enum.any?(checks), do: Notifier.notify(state.conf, :gossip, %{checks: checks}) + + purge(state.table, state.ttl) + + {:noreply, schedule_check(state)} + end + + def handle_info({:notification, :gossip, %{"checks" => checks}}, %State{} = state) do + Enum.each(checks, &store(state.table, &1)) + + {:noreply, state} + end + + def handle_info({:notification, :gossip, _payload}, state) do + {:noreply, state} + end + + def handle_info(message, state) do + Logger.warning( + message: "Received unexpected message: #{inspect(message)}", + source: :oban_met, + module: __MODULE__ + ) + + {:noreply, state} + end + + # Table + + defp table(tab) when is_reference(tab), do: tab + + defp table(name) do + [{_pid, table}] = Registry.lookup(Oban.Registry, name) + + table + end + + # Scheduling + + defp schedule_check(%State{} = state) do + %{state | timer: Process.send_after(self(), :check, state.interval)} + end + + # Checking + + defp safe_check(pid, state) do + if Process.alive?(pid) do + pid + |> GenServer.call(:check, state.interval) + |> sanitize_name() + end + catch + :exit, _ -> :error + end + + defp sanitize_name(%{name: name} = check) when is_binary(name), do: check + defp sanitize_name(%{name: name} = check), do: %{check | name: inspect(name)} + + defp normalize_limit(check) do + case Map.pop(check, "limit") do + {nil, check} -> check + {lim, check} -> Map.put(check, "local_limit", lim) + end + end +end diff --git a/vendor/oban_met/lib/oban/met/listener.ex b/vendor/oban_met/lib/oban/met/listener.ex new file mode 100644 index 00000000..c05d22eb --- /dev/null +++ b/vendor/oban_met/lib/oban/met/listener.ex @@ -0,0 +1,157 @@ +defmodule Oban.Met.Listener do + @moduledoc false + + # Track local telemetry events and periodically relay them to external recorders. + + use GenServer + + alias __MODULE__, as: State + alias Oban.Met.Values.{Gauge, Sketch} + alias Oban.Notifier + + @time_unit Application.compile_env(:oban_met, :sketch_time_unit, :native) + + defstruct [ + :conf, + :name, + :table, + :timer, + interval: :timer.seconds(1) + ] + + @spec child_spec(Keyword.t()) :: Supervisor.child_spec() + def child_spec(opts) do + name = Keyword.get(opts, :name, __MODULE__) + + %{super(opts) | id: name} + end + + @spec start_link(Keyword.t()) :: GenServer.on_start() + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, name: opts[:name]) + end + + @doc false + @spec report(GenServer.name()) :: :ok + def report(name) do + GenServer.call(name, :report) + end + + # Callbacks + + @impl GenServer + def init(opts) do + Process.flag(:trap_exit, true) + + table = :ets.new(:reporter, [:duplicate_bag, :public, write_concurrency: true]) + state = struct!(State, Keyword.put(opts, :table, table)) + + :telemetry.attach_many( + handler_id(state), + [[:oban, :job, :stop], [:oban, :job, :exception]], + &__MODULE__.handle_event/4, + {state.conf, table} + ) + + {:ok, schedule_report(state)} + end + + @impl GenServer + def terminate(_reason, %State{timer: timer} = state) do + if is_reference(timer), do: Process.cancel_timer(timer) + + :telemetry.detach(handler_id(state)) + + :ok + end + + @impl GenServer + def handle_info(:report, %State{conf: conf, table: table} = state) do + since = System.monotonic_time() + match = {:_, :"$2", :_} + guard = [{:<, :"$2", since}] + + objects = :ets.select(table, [{match, guard, [:"$_"]}]) + _delete = :ets.select_delete(table, [{match, guard, [true]}]) + + payload = %{ + metrics: objects_to_metrics(objects), + name: inspect(conf.name), + node: conf.node, + time: System.system_time(:second) + } + + Notifier.notify(conf, :metrics, payload) + + {:noreply, schedule_report(state)} + end + + defp objects_to_metrics(objects) do + objects + |> Enum.group_by(&elem(&1, 0), &elem(&1, 2)) + |> Enum.map(fn {{series, state, queue, worker}, values} -> + value = if series == :exec_count, do: Gauge.new(values), else: Sketch.new(values) + + %{series: series, state: state, queue: queue, worker: worker, value: value} + end) + end + + @impl GenServer + def handle_call(:report, _from, %State{} = state) do + handle_info(:report, state) + + {:reply, :ok, state} + end + + # Telemetry Events + + defp handler_id(state) do + "oban-met-recorder-#{inspect(state.name)}" + end + + @doc false + def handle_event([:oban, :job, _], measure, %{conf: conf} = meta, {conf, tab}) do + %{job: %{queue: queue, worker: worker}, state: state} = meta + %{duration: exec_time, queue_time: wait_time} = measure + + time = System.monotonic_time() + trst = trans_state(state) + + # Sketches don't support negative times (caused when the VM wakes from sleep), or zero values + # (infrequently caused by the same situation). Times are converted to the configured unit + # before clamping. + exec_time = exec_time |> convert_time() |> abs() |> max(1) + wait_time = wait_time |> convert_time() |> abs() |> max(1) + + :ets.insert(tab, [ + {{:exec_time, trst, queue, worker}, time, exec_time}, + {{:wait_time, trst, queue, worker}, time, wait_time}, + {{:exec_count, trst, queue, worker}, time, 1} + ]) + end + + def handle_event(_event, _measure, _meta, _conf), do: :ok + + if @time_unit == :native do + defp convert_time(value), do: value + else + defp convert_time(value), do: System.convert_time_unit(value, :native, @time_unit) + end + + # Scheduling + + defp schedule_report(state) do + timer = Process.send_after(self(), :report, state.interval) + + %{state | timer: timer} + end + + # For backward compatibility reasons, the telemetry event's state doesn't match the final job + # state. Here we translate the event state to the `t:Oban.Job.state`. + defp trans_state(:discard), do: :discarded + defp trans_state(:exhausted), do: :discarded + defp trans_state(:failure), do: :retryable + defp trans_state(:snoozed), do: :scheduled + defp trans_state(:success), do: :completed + defp trans_state(state), do: state +end diff --git a/vendor/oban_met/lib/oban/met/migration.ex b/vendor/oban_met/lib/oban/met/migration.ex new file mode 100644 index 00000000..d02f3bb6 --- /dev/null +++ b/vendor/oban_met/lib/oban/met/migration.ex @@ -0,0 +1,103 @@ +defmodule Oban.Met.Migration do + @moduledoc """ + Migrations that add estimate functionality. + + > #### Migrations Are Not Required {: .tip} + > + > Met will create the necessary estimate function automatically when possible. This migration + > isn't necessary under normal circumstances, but is provided to avoid permission issues or + > allow full control over database changes. + > + > See the section on [Explicit Migrations](installation.html) for more information. + + ## Usage + + To use migrations in your application you'll need to generate an `Ecto.Migration` that wraps + calls to `Oban.Met.Migration`: + + ```bash + mix ecto.gen.migration add_oban_met + ``` + + Open the generated migration and delegate the `up/0` and `down/0` functions to + `Oban.Met.Migration`: + + ```elixir + defmodule MyApp.Repo.Migrations.AddObanMet do + use Ecto.Migration + + def up, do: Oban.Met.Migration.up() + + def down, do: Oban.Met.Migration.down() + end + ``` + + This will run all of the necessary migrations for your database. + """ + + use Ecto.Migration + + @doc """ + Run the `up` migration. + + ## Example + + Run all migrations up to the current version: + + Oban.Met.Migration.up() + + Run migrations in an alternate prefix: + + Oban.Met.Migration.up(prefix: "payments") + """ + def up(opts \\ []) when is_list(opts) do + opts + |> Keyword.get(:prefix, "public") + |> oban_count_estimate() + |> execute() + end + + @doc """ + Run the `down` migration. + + ## Example + + Run all migrations up to the current version: + + Oban.Met.Migration.down() + + Run migrations in an alternate prefix: + + Oban.Met.Migration.down(prefix: "payments") + """ + def down(opts \\ []) when is_list(opts) do + prefix = Keyword.get(opts, :prefix, "public") + + execute "DROP FUNCTION IF EXISTS #{prefix}.oban_count_estimate(text, text)" + end + + # An `EXPLAIN` can only be executed as the top level of a query, or through an SQL function's + # EXECUTE as we're doing here. A named function helps the performance because it is prepared, + # and we have to support distributed databases that don't allow DO/END functions. + @doc false + def oban_count_estimate(prefix) do + """ + CREATE OR REPLACE FUNCTION #{prefix}.oban_count_estimate(state text, queue text) + RETURNS integer AS $func$ + DECLARE + plan jsonb; + BEGIN + EXECUTE 'EXPLAIN (FORMAT JSON) + SELECT id + FROM #{prefix}.oban_jobs + WHERE state = $1::#{prefix}.oban_job_state + AND queue = $2' + INTO plan + USING state, queue; + RETURN plan->0->'Plan'->'Plan Rows'; + END; + $func$ + LANGUAGE plpgsql + """ + end +end diff --git a/vendor/oban_met/lib/oban/met/recorder.ex b/vendor/oban_met/lib/oban/met/recorder.ex new file mode 100644 index 00000000..77bc0db9 --- /dev/null +++ b/vendor/oban_met/lib/oban/met/recorder.ex @@ -0,0 +1,393 @@ +defmodule Oban.Met.Recorder do + @moduledoc false + + # Aggregate metrics via pubsub for querying and compaction. + + use GenServer + + alias __MODULE__, as: State + alias Oban.Met.{Value, Values.Gauge, Values.Sketch} + alias Oban.{Notifier, Peer} + + @type series :: atom() | String.t() + @type value :: Value.t() + @type label :: String.t() + @type labels :: %{optional(String.t()) => label()} + @type ts :: integer() + @type period :: {pos_integer(), pos_integer()} + + @periods [{1, 300}, {5, 1_200}, {30, 3_600}, {60, 7_200}] + + @default_latest_opts [filters: [], group: nil, lookback: 2] + + @default_timeslice_opts [ + by: 1, + filters: [], + group: nil, + lookback: 60, + operation: :sum + ] + + defstruct [ + :compact_timer, + :conf, + :name, + :table, + compact_periods: @periods, + handoff: :awaiting + ] + + @spec child_spec(keyword()) :: Supervisor.child_spec() + def child_spec(opts) do + name = Keyword.get(opts, :name, __MODULE__) + + %{super(opts) | id: name} + end + + @spec start_link(keyword()) :: GenServer.on_start() + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, name: opts[:name]) + end + + @spec lookup(GenServer.name(), series()) :: [term()] + def lookup(name, series) do + match = {{to_string(series), :_, :_}, :_, :_, :_} + + :ets.select_reverse(table(name), [{match, [], [:"$_"]}]) + end + + @spec labels(GenServer.name(), label(), keyword()) :: [label()] + def labels(name, label, opts \\ []) when is_binary(label) do + opts = Keyword.validate!(opts, [:lookback, :since]) + + stime = Keyword.get(opts, :since, System.system_time(:second)) + lookback = Keyword.get(opts, :lookback, 120) + match = {{:_, :_, :"$2"}, :_, :"$1", :_} + guard = [{:andalso, {:is_map_key, label, :"$1"}, {:>=, :"$2", stime - lookback}}] + value = [{:map_get, label, :"$1"}] + + name + |> table() + |> :ets.select([{match, guard, value}]) + |> :lists.usort() + end + + @spec latest(GenServer.name(), series(), keyword()) :: %{optional(String.t()) => value()} + def latest(name, series, opts \\ []) do + opts = Keyword.validate!(opts, @default_latest_opts) + + group = Keyword.fetch!(opts, :group) + lookback = Keyword.fetch!(opts, :lookback) + filters = Keyword.fetch!(opts, :filters) + + name + |> table() + |> select(series, lookback, filters) + |> Enum.dedup_by(fn {{_, _, _}, _, labels, _} -> labels end) + |> Enum.group_by(fn {{_, _, _}, _, labels, _} -> labels[group] || "all" end) + |> Map.new(fn {group, metrics} -> + total = + metrics + |> Enum.map(&elem(&1, 3)) + |> Enum.reduce(&Value.merge/2) + |> Value.sum() + + {group, total} + end) + end + + @spec series(GenServer.name()) :: [map()] + def series(name) do + match = {{:"$1", :_, :_}, :_, :"$2", :"$3"} + + name + |> table() + |> :ets.select([{match, [], [:"$$"]}]) + |> Enum.group_by(&hd/1) + |> Enum.map(fn {series, [[_series, _labels, %vtype{}] | _] = metrics} -> + labels = + for [_series, labels, _value] <- metrics, + key <- Map.keys(labels), + uniq: true, + do: key + + %{series: series, labels: labels, type: vtype} + end) + |> Enum.sort_by(& &1.series) + end + + @spec timeslice(GenServer.name(), series(), keyword()) :: [{ts(), value(), label()}] + def timeslice(name, series, opts \\ []) do + opts = Keyword.validate!(opts, [:since] ++ @default_timeslice_opts) + + by = Keyword.fetch!(opts, :by) + group = Keyword.fetch!(opts, :group) + lookback = Keyword.fetch!(opts, :lookback) + operation = Keyword.fetch!(opts, :operation) + since = Keyword.get(opts, :since, System.system_time(:second)) + + name + |> table() + |> select(series, lookback, opts[:filters]) + |> Enum.reduce(%{}, &merge_group(&1, &2, group)) + |> Enum.reduce(%{}, &merge_chunk(&1, &2, since, by)) + |> Enum.sort_by(fn {{label, chunk}, _} -> {label, -chunk} end) + |> Enum.map(fn {{label, chunk}, value} -> + value = + case operation do + :sum -> Value.sum(value) + :max -> Value.quantile(value, 1.0) + {:pct, ntile} -> Value.quantile(value, ntile) + end + + {chunk, value, label} + end) + end + + defp merge_group({{_, _, ts}, _, labels, value}, acc, group) do + Map.update(acc, {labels[group], ts}, value, &Value.union(&1, value)) + end + + defp merge_chunk({{label, ts}, value}, acc, since, by) do + chunk = div(since - ts - 1, by) + + Map.update(acc, {label, chunk}, value, &Value.merge(&1, value)) + end + + def compact(name, periods) when is_list(periods) do + GenServer.call(name, {:compact, periods}) + end + + def store(name, series, value, labels, opts \\ []) do + time = Keyword.get(opts, :time, System.system_time(:second)) + + GenServer.call(name, {:store, {series, value, labels, time}}) + end + + # Callbacks + + @impl GenServer + def init(opts) do + table = + :ets.new(:metrics, [ + :compressed, + :ordered_set, + :protected, + read_concurrency: true + ]) + + state = + State + |> struct!(Keyword.put(opts, :table, table)) + |> schedule_compact() + + Registry.register(Oban.Registry, state.name, table) + + {:ok, state, {:continue, :start}} + end + + @impl GenServer + def handle_continue(:start, %State{conf: conf} = state) do + payload = %{ + syn: true, + module: __MODULE__, + name: inspect(conf.name), + node: conf.node + } + + Notifier.notify(conf.name, :handoff, payload) + Notifier.listen(conf.name, [:gossip, :handoff, :metrics]) + + {:noreply, state} + end + + @impl GenServer + def handle_call({:compact, periods}, _from, %State{table: table} = state) do + inner_compact(table, periods) + + {:reply, :ok, state} + end + + def handle_call({:store, params}, _from, %State{table: table} = state) do + {series, value, labels, time} = params + + inner_store(table, series, value, labels, time) + + {:reply, :ok, state} + end + + @impl GenServer + def handle_info({:notification, :handoff, %{"syn" => _}}, state) do + if Peer.leader?(state.conf) do + data = + state.table + |> :ets.tab2list() + |> :erlang.term_to_binary() + |> Base.encode64() + + payload = %{ + ack: true, + module: __MODULE__, + data: data, + node: state.conf.node, + name: inspect(state.conf.name) + } + + Notifier.notify(state.conf, :handoff, payload) + end + + {:noreply, state} + end + + def handle_info({:notification, :handoff, %{"ack" => _, "data" => data}}, %State{} = state) do + if state.handoff == :awaiting and not Peer.leader?(state.conf) do + data + |> Base.decode64!() + |> :erlang.binary_to_term() + |> then(&:ets.insert(state.table, &1)) + end + + {:noreply, %{state | handoff: :complete}} + end + + def handle_info({:notification, :metrics, %{"metrics" => _} = payload}, state) do + %{"metrics" => metrics, "node" => node, "time" => time} = payload + + for %{"series" => series, "value" => value} = metric <- metrics do + labels = + metric + |> Map.drop(~w(series value)) + |> Map.put("node", node) + + inner_store(state.table, series, from_map(value), labels, time) + end + + {:noreply, state} + end + + def handle_info({:notification, _channel, _payload}, state) do + {:noreply, state} + end + + def handle_info(:compact, %State{compact_periods: periods, table: table} = state) do + inner_compact(table, periods) + + :erlang.garbage_collect() + + {:noreply, schedule_compact(state), :hibernate} + end + + defp from_map(%{"size" => _} = value), do: Sketch.from_map(value) + defp from_map(value), do: Gauge.from_map(value) + + # Table + + defp table(name) do + case Registry.lookup(Oban.Registry, name) do + [{_pid, table}] -> + table + + _ -> + raise RuntimeError, "no table registered for #{inspect(name)}" + end + end + + defp inner_compact(table, periods) do + delete_outdated(table, periods) + + Enum.reduce(periods, System.system_time(:second), fn {step, duration}, ts -> + since = ts - duration + match = {{:_, :_, :"$1"}, :"$2", :_, :_} + guard = [{:andalso, {:>=, :"$2", since}, {:"=<", :"$1", ts}}] + + objects = :ets.select(table, [{match, guard, [:"$_"]}]) + _delete = :ets.select_delete(table, [{match, guard, [true]}]) + + objects + |> Enum.chunk_by(fn {{ser, lab, max}, _, _, _} -> {ser, lab, div(ts - max - 1, step)} end) + |> Enum.map(&compact_object/1) + |> then(&:ets.insert(table, &1)) + + since + end) + end + + defp compact_object([{{series, lab_key, _}, _, labels, _} | _] = metrics) do + {min_ts, max_ts} = + metrics + |> Enum.flat_map(fn {{_, _, max_ts}, min_ts, _, _} -> [max_ts, min_ts] end) + |> Enum.min_max() + + value = + metrics + |> Enum.map(&elem(&1, 3)) + |> Enum.reduce(&Value.merge/2) + + {{series, lab_key, max_ts}, min_ts, labels, value} + end + + defp delete_outdated(table, periods) do + systime = System.system_time(:second) + maximum = Enum.reduce(periods, 0, fn {_, duration}, acc -> duration + acc end) + + since = systime - maximum + match = {{:_, :_, :"$1"}, :_, :_, :_} + guard = [{:<, :"$1", since}] + + :ets.select_delete(table, [{match, guard, [true]}]) + end + + defp inner_store(table, series, value, labels, time) do + key = {to_string(series), :erlang.phash2(labels), time} + + value = + case :ets.lookup(table, key) do + [{_key, _time, _labels, old_value}] -> Value.union(old_value, value) + _ -> value + end + + :ets.insert(table, {key, time, labels, value}) + end + + # Scheduling + + defp schedule_compact(state) do + time = Time.utc_now() + + interval = + time + |> Time.add(60) + |> Map.put(:second, 0) + |> Time.diff(time) + |> Integer.mod(86_400) + |> System.convert_time_unit(:second, :millisecond) + + timer = Process.send_after(self(), :compact, interval) + + %{state | compact_timer: timer} + end + + # Fetching & Filtering + + defp select(table, series, since, filters) do + stime = System.system_time(:second) + match = {{to_string(series), :_, :"$2"}, :_, :"$1", :_} + guard = filters_to_guards(filters, {:>=, :"$2", stime - since}) + + :ets.select_reverse(table, [{match, [guard], [:"$_"]}]) + end + + defp filters_to_guards(nil, base), do: base + + defp filters_to_guards(filters, base) do + Enum.reduce(filters, base, fn {field, values}, and_acc -> + and_guard = + values + |> List.wrap() + |> Enum.map(fn value -> {:==, {:map_get, to_string(field), :"$1"}, value} end) + |> Enum.reduce(fn or_guard, or_acc -> {:orelse, or_guard, or_acc} end) + + {:andalso, and_guard, and_acc} + end) + end +end diff --git a/vendor/oban_met/lib/oban/met/reporter.ex b/vendor/oban_met/lib/oban/met/reporter.ex new file mode 100644 index 00000000..b64497f8 --- /dev/null +++ b/vendor/oban_met/lib/oban/met/reporter.ex @@ -0,0 +1,216 @@ +defmodule Oban.Met.Reporter do + @moduledoc false + + # Periodically count and report jobs by state and queue. + # + # Because exact counts are expensive, counts for states with jobs beyond a configurable + # threshold are estimated. This is a tradeoff that aims to preserve system resources at the + # expense of accuracy. + + use GenServer + + import Ecto.Query, only: [from: 2, group_by: 3, select: 3, where: 3] + + alias __MODULE__, as: State + alias Oban.{Job, Notifier, Peer, Repo} + alias Oban.Met.Migration + alias Oban.Met.Values.Gauge + alias Oban.Pro.Engines.Smart + + require Logger + + @empty_states %{ + "available" => [], + "cancelled" => [], + "completed" => [], + "discarded" => [], + "executing" => [], + "retryable" => [], + "scheduled" => [], + "suspended" => [] + } + + defstruct [ + :conf, + :name, + :queue_timer, + :check_timer, + auto_migrate: true, + checks: @empty_states, + check_counter: 0, + check_interval: :timer.seconds(1), + estimate_limit: 50_000, + function_created?: false, + queues: [] + ] + + @spec child_spec(Keyword.t()) :: Supervisor.child_spec() + def child_spec(opts) do + name = Keyword.get(opts, :name, __MODULE__) + + %{super(opts) | id: name} + end + + @spec start_link(Keyword.t()) :: GenServer.on_start() + def start_link(opts) do + conf = Keyword.fetch!(opts, :conf) + + opts = + if conf.repo.__adapter__() == Ecto.Adapters.Postgres do + opts + else + opts + |> Keyword.put(:auto_migrate, false) + |> Keyword.put(:estimate_limit, :infinity) + end + + state = struct!(State, opts) + + GenServer.start_link(__MODULE__, state, name: opts[:name]) + end + + # Callbacks + + @impl GenServer + def init(%State{} = state) do + Process.flag(:trap_exit, true) + + # Used to ensure testing helpers to auto-allow this module for sandbox access. + :telemetry.execute([:oban, :plugin, :init], %{}, %{conf: state.conf, plugin: __MODULE__}) + + {:ok, schedule_checks(state)} + end + + @impl GenServer + def terminate(_reason, %State{} = state) do + if is_reference(state.check_timer), do: Process.cancel_timer(state.check_timer) + + :ok + end + + @impl GenServer + def handle_info(:checkpoint, %State{conf: conf} = state) do + if Peer.leader?(conf.name) do + state = + state + |> create_estimate_function() + |> cache_queues() + + {:ok, checks} = checks(state) + + metrics = + for {_key, counts} <- checks, + count <- counts, + do: Map.update!(count, :value, &Gauge.new/1) + + payload = %{ + metrics: metrics, + name: inspect(conf.name), + node: conf.node, + time: System.system_time(:second) + } + + Notifier.notify(conf, :metrics, payload) + + {:noreply, + schedule_checks(%{state | check_counter: state.check_counter + 1, checks: checks})} + else + {:noreply, schedule_checks(state)} + end + end + + def handle_info(message, state) do + Logger.warning( + message: "Received unexpected message: #{inspect(message)}", + source: :oban_met, + module: __MODULE__ + ) + + {:noreply, state} + end + + # Scheduling + + defp schedule_checks(state) do + timer = Process.send_after(self(), :checkpoint, state.check_interval) + + %{state | check_timer: timer} + end + + # Checking + + defp create_estimate_function(%{auto_migrate: true, function_created?: false} = state) do + %{conf: %{prefix: prefix}} = state + + query = Migration.oban_count_estimate(prefix) + + Repo.query!(state.conf, query, []) + + %{state | function_created?: true} + end + + defp create_estimate_function(state), do: state + + defp cache_queues(state) do + if Integer.mod(state.check_counter, 60) == 0 do + source = if state.conf.engine == Smart, do: "oban_producers", else: "oban_jobs" + query = from(p in source, select: p.queue, distinct: true) + + %{state | queues: Repo.all(state.conf, query)} + else + state + end + end + + defp checks(%{estimate_limit: limit} = state) do + {count_states, guess_states} = + for {state, counts} <- state.checks, reduce: {[], []} do + {count_acc, guess_acc} -> + total = Enum.reduce(counts, 0, &(&2 + &1.value)) + + if total < limit do + {[state | count_acc], guess_acc} + else + {count_acc, [state | guess_acc]} + end + end + + count_query = count_query(count_states) + guess_query = guess_query(guess_states, state.queues, state.conf) + + Repo.transaction(state.conf, fn -> + count_counts = Repo.all(state.conf, count_query) + guess_counts = Repo.all(state.conf, guess_query) + + (count_counts ++ guess_counts) + |> Enum.group_by(& &1.state) + |> Enum.reduce(@empty_states, fn {state, counts}, acc -> + Map.put(acc, state, counts) + end) + end) + end + + defp count_query([]), do: where(Job, [_], false) + + defp count_query(states) when is_list(states) do + Job + |> select([j], %{series: :full_count, state: j.state, queue: j.queue, value: count(j.id)}) + |> where([j], j.state in ^states) + |> group_by([j], [j.state, j.queue]) + end + + defp guess_query([], _queues, _conf), do: where(Job, [_], false) + defp guess_query(_states, [], _conf), do: where(Job, [_], false) + + defp guess_query(states, queues, conf) when is_list(states) and is_list(queues) do + from(p in fragment("json_array_elements_text(?)", ^queues), + cross_join: x in fragment("json_array_elements_text(?)", ^states), + select: %{ + series: :full_count, + state: x.value, + queue: p.value, + value: fragment("?.oban_count_estimate(?, ?)", literal(^conf.prefix), x.value, p.value) + } + ) + end +end diff --git a/vendor/oban_met/lib/oban/met/value.ex b/vendor/oban_met/lib/oban/met/value.ex new file mode 100644 index 00000000..ea886651 --- /dev/null +++ b/vendor/oban_met/lib/oban/met/value.ex @@ -0,0 +1,44 @@ +defprotocol Oban.Met.Value do + @moduledoc """ + Tracked data access functions. + """ + + @doc """ + Add or append a new value to the data type. + """ + def add(struct, value) + + @doc """ + Merge two values into one. + """ + def merge(struct_1, struct_2) + + @doc """ + Compute the quantile for a value. + """ + def quantile(struct, quantile) + + @doc """ + Sum all data points for a value. + """ + def sum(struct) + + @doc """ + Union two values by reducing them into one. + """ + def union(struct_1, struct_2) +end + +for module <- [Oban.Met.Values.Gauge, Oban.Met.Values.Sketch] do + defimpl Oban.Met.Value, for: module do + defdelegate add(struct, value), to: @for + + defdelegate merge(struct_1, struct_2), to: @for + + defdelegate quantile(struct, quantile), to: @for + + defdelegate sum(struct), to: @for + + defdelegate union(struct_1, struct_2), to: @for + end +end diff --git a/vendor/oban_met/lib/oban/met/values/gauge.ex b/vendor/oban_met/lib/oban/met/values/gauge.ex new file mode 100644 index 00000000..fac3b205 --- /dev/null +++ b/vendor/oban_met/lib/oban/met/values/gauge.ex @@ -0,0 +1,141 @@ +defmodule Oban.Met.Values.Gauge do + @moduledoc """ + One or more non-negative integers captured over time. + """ + + alias __MODULE__, as: Gauge + + @encoder if Code.ensure_loaded?(JSON.Encoder), do: JSON.Encoder, else: Jason.Encoder + + @derive @encoder + defstruct data: 0 + + @type t :: %__MODULE__{data: [non_neg_integer()]} + + @doc """ + Initialize a new gauge. + + ## Examples + + iex> Gauge.new(1).data + [1] + + iex> Gauge.new([1, 2]).data + [1, 2] + """ + @spec new(non_neg_integer() | [non_neg_integer()]) :: t() + def new(data) when is_integer(data) and data > 0 do + %Gauge{data: [data]} + end + + def new(data) when is_list(data) do + %Gauge{data: data} + end + + @doc """ + Add to the gauge. + + ## Examples + + iex> 1 + ...> |> Gauge.new() + ...> |> Gauge.add(1) + ...> |> Gauge.add(1) + ...> |> Map.fetch!(:data) + [3] + """ + @spec add(t(), pos_integer()) :: t() + def add(%Gauge{data: [head | tail]}, value) when value > 0 do + %Gauge{data: [head + value | tail]} + end + + @doc """ + Merges two gauges into a one. + + ## Examples + + Merging two gauge retains all values: + + iex> gauge_1 = Gauge.new([1, 2]) + ...> gauge_2 = Gauge.new([2, 3]) + ...> Gauge.merge(gauge_1, gauge_2).data + [1, 2, 2, 3] + """ + @spec merge(t(), t()) :: t() + def merge(%Gauge{data: data_1}, %Gauge{data: data_2}) do + %Gauge{data: data_1 ++ data_2} + end + + @doc """ + Compute the quantile for a gauge. + + ## Examples + + With single values: + + iex> Gauge.quantile(Gauge.new([1, 2, 3]), 1.0) + 3 + + iex> Gauge.quantile(Gauge.new([1, 2, 3]), 0.5) + 2 + """ + @spec quantile(t(), float()) :: non_neg_integer() + def quantile(%Gauge{data: data}, ntile) do + length = length(data) - 1 + + data + |> Enum.sort() + |> Enum.at(floor(length * ntile)) + end + + @doc """ + Compute the sum for a gauge. + + ## Examples + + iex> Gauge.sum(Gauge.new(3)) + 3 + + iex> Gauge.sum(Gauge.new([1, 2, 3])) + 6 + """ + @spec sum(t()) :: non_neg_integer() + def sum(%Gauge{data: data}), do: Enum.sum(data) + + @doc """ + Union two gauges into a single value. + + ## Examples + + Merging two gauge results in a single value: + + iex> gauge_1 = Gauge.new([1, 2]) + ...> gauge_2 = Gauge.new([2, 3]) + ...> Gauge.union(gauge_1, gauge_2).data + [8] + """ + @spec union(t(), t()) :: t() + def union(%Gauge{data: data_1}, %Gauge{data: data_2}) do + data = + (data_1 ++ data_2) + |> Enum.sum() + |> List.wrap() + + %Gauge{data: data} + end + + @doc """ + Initialize a gauge struct from a stringified map, e.g. encoded JSON. + + ## Examples + + iex> Gauge.new(2) + ...> |> Oban.JSON.encode!() + ...> |> Oban.JSON.decode!() + ...> |> Gauge.from_map() + ...> |> Map.fetch!(:data) + [2] + """ + @spec from_map(%{optional(String.t()) => term()}) :: t() + def from_map(%{"data" => data}), do: %Gauge{data: data} +end diff --git a/vendor/oban_met/lib/oban/met/values/sketch.ex b/vendor/oban_met/lib/oban/met/values/sketch.ex new file mode 100644 index 00000000..35f1278d --- /dev/null +++ b/vendor/oban_met/lib/oban/met/values/sketch.ex @@ -0,0 +1,186 @@ +defmodule Oban.Met.Values.Sketch do + @moduledoc """ + A fast and fully mergeable quantile sketch with relative error guarantees. + + Derived from [DogSketch](https://github.com/moosecodebv/dog_sketch), based on DDSketch. This + variant has a hard-coded error rate of 0.02 for the sake of simplicity. + """ + + alias __MODULE__, as: Sketch + + @type t :: %__MODULE__{ + data: %{optional(pos_integer()) => pos_integer()}, + size: non_neg_integer() + } + + @encoder if Code.ensure_loaded?(JSON.Encoder), do: JSON.Encoder, else: Jason.Encoder + + @derive @encoder + defstruct data: %{}, size: 0 + + @error 0.02 + @gamma (1 + @error) / (1 - @error) + @inv_log_gamma 1.0 / :math.log(@gamma) + + @doc """ + Create a new sketch instance with an optional error rate. + + ## Examples + + iex> sketch = Sketch.new() + ...> Sketch.size(sketch) + 0 + + iex> sketch = Sketch.new(1) + ...> Sketch.size(sketch) + 1 + + iex> sketch = Sketch.new([1, 2, 3]) + ...> Sketch.size(sketch) + 3 + """ + @spec new(pos_integer() | [pos_integer()]) :: t() + def new(values \\ []) do + values + |> List.wrap() + |> Enum.reduce(%Sketch{}, &add(&2, &1)) + end + + @doc """ + Insert sample values into a sketch. + + ## Examples + + iex> Sketch.new() + ...> |> Sketch.add(1) + ...> |> Sketch.add(2) + ...> |> Sketch.add(3) + ...> |> Sketch.size() + 3 + """ + @spec add(t(), pos_integer()) :: t() + def add(%Sketch{} = sketch, value) when is_integer(value) and value > 0 do + bin = ceil(:math.log(value) * @inv_log_gamma) + data = Map.update(sketch.data, bin, 1, &(&1 + 1)) + + %{sketch | data: data, size: sketch.size + 1} + end + + @doc """ + Merge two sketch instances. + + ## Examples + + iex> sketch_1 = Sketch.new([1]) + ...> + ...> Sketch.new([2]) + ...> |> Sketch.merge(sketch_1) + ...> |> Sketch.size() + 2 + """ + @spec merge(t(), t()) :: t() + def merge(%Sketch{} = sketch_1, %Sketch{} = sketch_2) do + data = Map.merge(sketch_1.data, sketch_2.data, fn _, val_1, val_2 -> val_1 + val_2 end) + + %{sketch_1 | data: data, size: sketch_1.size + sketch_2.size} + end + + @doc """ + Compute the quantile value for a sketch. + + ## Examples + + Without any values: + + iex> Sketch.quantile(Sketch.new(), 0.5) + nil + + With recorded values: + + iex> Sketch.new() + ...> |> Sketch.add(1) + ...> |> Sketch.add(2) + ...> |> Sketch.add(3) + ...> |> Sketch.quantile(0.5) + ...> |> trunc() + 2 + """ + @spec quantile(t(), float()) :: nil | float() + def quantile(%Sketch{size: 0}, _ntile), do: nil + + def quantile(sketch, quantile) when quantile >= 0 and quantile <= 1 do + size_ntile = sketch.size * quantile + + index = + sketch.data + |> Enum.sort_by(&elem(&1, 0)) + |> Enum.reduce_while(0, fn {key, val}, size -> + if size + val >= size_ntile do + {:halt, key} + else + {:cont, size + val} + end + end) + + 2 * :math.pow(@gamma, index) / (@gamma + 1) + end + + @doc """ + Compute the sum for a sketch. Hardcoded to 0. + + ## Examples + + iex> Sketch.sum(Sketch.new([1, 2, 3, 3])) + 0.0 + """ + @spec sum(t()) :: float() + def sum(%Sketch{data: _data}), do: 0.0 + + @doc """ + Union two sketches into a single value. This is an alias for `merge/2`. + """ + @spec union(t(), t()) :: t() + def union(sketch_1, sketch_2), do: merge(sketch_1, sketch_2) + + @doc """ + Convert a sketch into a list of bins and values. + + ## Examples + + iex> Sketch.new() + ...> |> Sketch.add(1) + ...> |> Sketch.add(2) + ...> |> Sketch.add(3) + ...> |> Sketch.to_list() + ...> |> length() + 3 + """ + @spec to_list(t()) :: [float()] + def to_list(%Sketch{data: data}) do + for {key, val} <- data, do: {2 * :math.pow(@gamma, key) / (@gamma + 1), val} + end + + @doc false + @spec size(t()) :: non_neg_integer() + def size(%Sketch{size: size}), do: size + + @doc """ + Initialize a sketch struct from a stringified map, e.g. encoded JSON. + + ## Examples + + iex> Sketch.new([1, 2]) + ...> |> Oban.JSON.encode!() + ...> |> Oban.JSON.decode!() + ...> |> Sketch.from_map() + ...> |> Sketch.quantile(1.0) + ...> |> floor() + 2 + """ + @spec from_map(%{optional(String.t()) => term()}) :: t() + def from_map(%{"data" => data, "size" => size}) do + data = Map.new(data, fn {key, val} -> {String.to_integer(key), val} end) + + %Sketch{data: data, size: size} + end +end diff --git a/vendor/oban_met/mix.exs b/vendor/oban_met/mix.exs new file mode 100644 index 00000000..fd12d413 --- /dev/null +++ b/vendor/oban_met/mix.exs @@ -0,0 +1,131 @@ +defmodule Oban.Met.MixProject do + use Mix.Project + + @source_url "https://github.com/oban-bg/oban_met" + @version "1.1.0" + + def project do + [ + app: :oban_met, + version: @version, + elixir: "~> 1.15", + elixirc_paths: elixirc_paths(Mix.env()), + start_permanent: Mix.env() == :prod, + deps: deps(), + docs: docs(), + aliases: aliases(), + package: package(), + + # Description + name: "Oban Met", + description: + "A distributed, compacting, multidimensional, telemetry-powered time series datastore", + + # Dialyzer + dialyzer: [ + plt_add_apps: [:ex_unit], + plt_core_path: "_build/#{Mix.env()}", + flags: [:error_handling, :underspecs] + ] + ] + end + + def application do + [ + extra_applications: [:logger], + mod: {Oban.Met.Application, []}, + env: [auto_start: true, auto_testing_modes: [:disabled]] + ] + end + + def cli do + [ + preferred_envs: [ + "ecto.gen.migration": :test, + "test.ci": :test, + "test.reset": :test, + "test.setup": :test + ] + ] + end + + defp elixirc_paths(:test), do: ["lib", "test/support"] + defp elixirc_paths(_env), do: ["lib"] + + defp package do + [ + maintainers: ["Parker Selbert"], + licenses: ["Apache-2.0"], + files: ~w(lib .formatter.exs mix.exs README* CHANGELOG* LICENSE*), + links: %{ + Website: "https://oban.pro", + Changelog: "#{@source_url}/blob/main/CHANGELOG.md", + GitHub: @source_url + } + ] + end + + defp docs do + [ + main: "Oban.Met", + source_ref: "v#{@version}", + source_url: @source_url, + formatters: ["html"], + api_reference: false, + extra_section: "GUIDES", + extras: extras(), + groups_for_modules: groups_for_modules(), + skip_undefined_reference_warnings_on: ["CHANGELOG.md"] + ] + end + + defp extras do + [ + "CHANGELOG.md": [filename: "changelog", title: "Changelog"] + ] + end + + defp groups_for_modules do + [ + Values: [ + Oban.Met.Value, + Oban.Met.Values.Gauge, + Oban.Met.Values.Sketch + ] + ] + end + + defp deps do + [ + {:oban, "~> 2.21"}, + {:ecto_sqlite3, "~> 0.18", only: [:test, :dev]}, + {:postgrex, "~> 0.20", only: [:test, :dev]}, + {:stream_data, "~> 1.1", only: [:test, :dev]}, + {:benchee, "~> 1.3", only: [:test, :dev], runtime: false}, + {:credo, "~> 1.7", only: [:test, :dev], runtime: false}, + {:dialyxir, "~> 1.3", only: [:test, :dev], runtime: false}, + {:ex_doc, "~> 0.34", only: :dev, runtime: false}, + {:makeup_diff, "~> 0.1", only: :dev, runtime: false} + ] + end + + defp aliases do + [ + release: [ + "cmd git tag v#{@version} -f", + "cmd git push", + "cmd git push --tags", + "hex.publish --yes" + ], + "test.reset": ["ecto.drop --quiet", "test.setup"], + "test.setup": ["ecto.create --quiet", "ecto.migrate --quiet"], + "test.ci": [ + "format --check-formatted", + "deps.unlock --check-unused", + "credo", + "test --raise", + "dialyzer" + ] + ] + end +end diff --git a/vendor/oban_web/.formatter.exs b/vendor/oban_web/.formatter.exs new file mode 100644 index 00000000..48f82b0c --- /dev/null +++ b/vendor/oban_web/.formatter.exs @@ -0,0 +1,14 @@ +# Used by "mix format" +[ + import_deps: [ + :ecto, + :ecto_sql, + :phoenix + ], + inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs,heex}"], + export: [ + locals_without_parens: [oban_dashboard: 1, oban_dashboard: 2] + ], + locals_without_parens: [oban_dashboard: 1, oban_dashboard: 2], + plugins: [Phoenix.LiveView.HTMLFormatter] +] diff --git a/vendor/oban_web/.hex b/vendor/oban_web/.hex new file mode 100644 index 00000000..b2dee2fe Binary files /dev/null and b/vendor/oban_web/.hex differ diff --git a/vendor/oban_web/CHANGELOG.md b/vendor/oban_web/CHANGELOG.md new file mode 100644 index 00000000..68978f96 --- /dev/null +++ b/vendor/oban_web/CHANGELOG.md @@ -0,0 +1,237 @@ +# Changelog for Oban Web v2.12 + +This is a major release that overhauls job details, the queues table, queue details, introduces a +crons page, adds a workflows page, and adds a job creation sidebar. + +> #### Requirements {: .info} +> +> This release requires Oban v2.21+ and the new V14 migration due to important schema changes. For +> Pro users, v1.7+ is also required along with the v1.7.0 migration. + +## 🔀 Workflows Page + +There is a new page for viewing, filtering, and generally managing workflows. The table displays +workflow progress, activity counts, duration, and nested sub-workflows. Workflows can be filtered +by properties like name, workers, or status. + + + +Clicking into a workflow brings you to a detail view with an interactive graph showing jobs as +stateful nodes with dependencies. The graph supports panning, zooming, directional layout +toggling, and a default tracking mode that follows executing nodes. Sub-workflow nodes can be +expanded inline to reveal their internal jobs, or navigated to for direct management such as +retrying or cancelling. + +Workflow viewing remains fast on busy systems or with _large_ workflows thanks to the new +`oban_workflows` aggregate table in Pro v1.7. It's also compatible with Python Pro, though +cancel/retry actions require Elixir. + +## ⏰ Crons Page + +There's also a new page for viewing and managing cron entries. The table displays all static and +dynamic entries with history sparklines and activity details. + + + +The cron detail view includes natural language expressions like "Daily at 8:00 and 9:00" or +"Weekdays except Monday", along with cron entry specifics, and recent job history. + +For Pro users, `DynamicCron` entires can be created, edited, paused, resumed, or deleted directly +from a form on the details page. + +## 🔍 Job Details + +The job detail page is rebuilt with a full-width layout and a new timeline component that shows +the job state machine as a branching diagram rather than a linear progression. + + + +A scoped chart displays execution history for that worker's previous jobs and jobs in an +incomplete, non-executing state can now be edited directly. + +Executing Pro jobs display live diagnostics including process status, reductions, memory, and +current stacktrace. The diagnostics panel persists after job completion with a "Stale" indicator +showing the data is from when the job was running. + +## ➡️ Queues Table and Details + +The queues table is redesigned with a utilization gauge and a history sparkline showing 5-minute +throughput for each queue. The queue sidebar provided minimal value, and it was removed to make +room for the additional data displayed per-row. + + + +The queue detail page adds status badges for paused, partial, and terminating states, with +pause/resume, stop, and edit buttons in the header. Partitioning controls are expanded with meta +options and burst mode configuration. + +## v2.12.1 - 2026-03-25 + +### Bug Fixes + +- [Dashboard] Include `priv/timezones.txt` in hex package + + The timezones file wasn't included in the package files list, which broke compilation for + downloaded packages. + +## v2.12.0 - 2026-03-25 + +While all bug fixes are listed below, the enhancements section only covers a portion of the new +features. For enhancements, a video is worth many thousands of words. + +### Enhancements + +- [Dashboard] Preserve refresh changes between page changes + + The refresh reverted to the original value between changes because there wasn't a new liveview + connection. Now the stored refresh value is synced on change, and reloaded when the component + mounts. + +- [Dahboard] Add a 30s option for refreshing + + It's a simple addition that makes watching the cron page a bit more sensible. + +- [Dashboard] Serve dynamically loaded mask based svg icons + + Rather than manually defining SVG icons inline, SVG files are tracked and dynamically loaded + from a compiled assets module. + +- [Dashboard] Serve font as static asset instead of embedding + + Extract font out of the `app.css` and serve it as a stand-alone asset for better caching. + +- [Dashboard] Add help button to primary toolbar + + The keyboard shortcuts modal was only accessible via the ? key with no visual indication it + existed. Added a help dropdown to the toolbar that links to documentation and opens the + shortcuts modal. + +- [Jobs] Support suspended state in sidebar and details + + The latest Oban version adds a proper `suspended` state, so there's no more on_hold + psuedo-state. + +- [Jobs] Jobs rescued by any lifeline are detectable + + Change the language for orphans to correctly indicate that any lifeline may have rescued them. + +- [Jobs] Redesign timeline as state machine visualization + + Replace the linear horizontal timeline with a branching layout that accurately represents the + job state machine. Entry states (scheduled/retryable) flow into available, then executing, which + branches to terminal states (completed/cancelled/discarded). + +- [Jobs] Add diagnostics for executing jobs + + Actively executing `Oban.Pro.Worker` jobs now display diagnostics including process status, + reductions, memory, and the current stacktrace. + +- [Jobs] Add job editing to job detail page + + Job's in an incomplete and non-executing state can have their attributes edited. Internally, + `Oban.update_job/3` is used to perform the update, so standard validations still apply. + +- [Jobs] Refine layout and error display for job details + + Restructure information for job details to emphasize what's important (args, the most recent + error), while providing control over which information is displayed. + +- [Jobs] Show history chart component for job detail + + The new component uses exact historic information for the current job rather than the aggregate + metrics used for the primary job chart. + +- [Jobs] Add "New Job" drawer for creating jobs + + Jobs can now be created directly from the Jobs page using a slide-out drawer. The form includes + fields for worker, args, queue, priority, max attempts, scheduled time, and tags. After + creation, the user is navigated to the new job's detail page. + +- [Crons] Use name provided by crontab for entry job history + + The `cron_name` calculated for Elixir entries isn't compatible with those generated by Python. + The `oban-py` metrics now include a `name` option that is used to correctly match entries up + with historic jobs. + +- [Crons] Add cron parser for complex expressions + + Parse cron fields into structured data before describing them, enabling support for: + + - Combined DOM/DOW patterns like "The 1st, only on Mondays" + - Complement detection, "Daily except the 1st" or "except Tuesdays" + - Multiple hour values, "Daily at 8:00, 9:00, and 10:00" + - Weekday/weekend recognition + + It's also switched to a 24-hour time format for international consistency. + +- [Queues] Remove sidebar from queues page The sidebar filters (paused, terminating, modes, nodes) added little + + The sidebar filters (paused, terminating, modes, nodes) added little value for a typically small + dataset while consuming significant screen space. Filtering remains available via the search + component. + +- [Queues] Add history sparkline graph to queues table + + Display a 5-minute throughput history for each queue using a sparkline + visualization with 5-second rollups (60 data points). Hovering over bars + shows the job count and timestamp for that interval. + +- [Queues] Refine queue detail forms and layout + + Expand the queue form partitioning controls with "meta" options and burst mode. Also changes to + standard form inputs for numbers and select boxes within the edit form to simplify event + handling. + +- [Queues] Redesign queue details with actions and history + + Add status badges for paused, partial, and terminating states. Include pause/resume, stop, and + edit buttons in the header for quick access. Display queue execution history in a chart + alongside stats. + +- [Pages] Add empty states for workflows, crons, and queues + + Each page now shows a helpful message with an icon and documentation link when there's nothing + to display. The queues page distinguishes between having no queues configured versus filters + hiding all results. + +- [Pages] Improve dark mode color consistency and contrast + + Standardize border colors across form inputs and controls, align form input backgrounds, and + increase contrast for disabled elements. + +### Bug Fixes + +- [Dahboard] Fix instance switching when resolver returns a list + + Ensure the instance name and allowed instances are strings before comparison + +- [Dashboard] Poll registry instead of blocking on telemetry + + Replace telemetry-based init that could block for 15 seconds waiting for an init event that may + have already fired. Now polls the Oban registry, avoiding the race condition that caused slow + websocket reconnections. + +- [Dashboard] Cache sidebar counts to prevent flickering + + When the metric reporter's `check_interval` exceeds the 2s lookback window, counts briefly show + as zero between broadcasts. Cache previous non-empty counts in a centralized Metrics module and + return them when `Met.latest/3` returns an empty map. + +- [Dashboard] Automatically update theme when OS theme changes + + Listen for prefers-color-scheme media query changes so the theme updates in real-time when the + browser or OS switches between light and dark mode. + +- [Standalone] Use the Postgres notifier for standalone instance + + The PG notifier can't (easily) connect to an external cluster for notifications. Connection is + possible through the Postgres notifier. + diff --git a/vendor/oban_web/LICENSE.txt b/vendor/oban_web/LICENSE.txt new file mode 100644 index 00000000..227bf3fc --- /dev/null +++ b/vendor/oban_web/LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2025 The Oban Team + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/oban_web/README.md b/vendor/oban_web/README.md new file mode 100644 index 00000000..d85b2e75 --- /dev/null +++ b/vendor/oban_web/README.md @@ -0,0 +1,172 @@ +# Oban Web + +

+ + Hex Version + + + + Hex Docs + + + + CI Status + + + + Apache 2 License + +

+ + + +Oban Web is a view of [Oban's][oba] inner workings that you host directly within your application. +Powered by [Oban Metrics][met] and [Phoenix Live View][liv], it is distributed, lightweight, and +fully realtime. + +[oba]: https://github.com/oban-bg/oban +[met]: https://github.com/oban-bg/oban_met +[liv]: https://github.com/phoenixframework/phoenix_live_view + +

+ + + + + +

+ +## Features + +- **🐦‍🔥 Embedded LiveView** - Mount the dashboard directly in your application without any + external dependencies. + +- **📊 Realtime Charts** - Powered by a custom, distributed time-series data store that's compacted + for hours of efficient storage and filterable by node, queue, state, and worker. + +- **🛸 Live Updates** - Monitor background job activity across all queues and nodes in real + time, with customizable refresh rates and automatic pausing on blur. + +- **🔍 Powerful Filtering** - Intelligently filter jobs by worker, queue, args, tags and more with + auto-completed suggestions. + +- **🔬 Detailed Inspection** - View job details including when, where and how it was ran (or how + it failed to run). + +- **🔄 Batch Actions** - Cancel, delete and retry selected jobs or all jobs matching the current + filters. + +- **🎛️ Queue Controls** - Scale, pause, resume, and stop queues across all running nodes. Queues + running with [Oban Pro](https://oban.pro) can also edit global limits, rate limiting, and + partitioning. + +- **♊ Multiple Dashboards** - Switch between all running Oban instance from a single mount point, + or restrict access to some dashboards with exclusion controls. + +- **🔒 Access Control** - Allow admins to control queues and interract with jobs while restricting + other users to read-only use of the dashboard. + +- **🎬 Action Logging** - Use telemetry events to instrument and report all of a user's dashboard + activity. A telemetry-powered logger is provided for easy reporting. + +## Installation + +See the [installation guide](https://hexdocs.pm/oban_web/installation.html) for details on +installing and configuring Oban Web for your application. + +### Standalone Docker Image + +A standalone Docker image is available for monitoring Oban without embedding the dashboard in your +application: + +```bash +docker run -d \ + -e DATABASE_URL="postgres://user:pass@host:5432/myapp" \ + -p 4000:4000 \ + ghcr.io/oban-bg/oban-dash +``` + +See the [standalone guide](https://hexdocs.pm/oban_web/standalone.html) for configuration options +and details. + + + +## Contributing + +To run the Oban Web test suite you must have PostgreSQL 12+ and MySQL 8+ running. Once dependencies +are installed, setup the databases and run necessary migrations: + +```bash +mix test.setup +``` + +### Development Server + +For development, a single file server that generates a wide variety of fake jobs is built in: + +```bash +iex -S mix dev +``` + +### Python Development Workers + +A Python development app is available that runs alongside the Elixir dev server, sharing the same +database. Python workers use separate queues (inference, scraping, etl, webhooks, notifications, +transcoding, maintenance) and are tagged with "python" for easy filtering in the dashboard. + +First, install the Python dependencies (requires [uv](https://docs.astral.sh/uv/)): + +```bash +mix py.install +``` + +Then, with the Elixir dev server running in one terminal, start the Python workers in another: + +```bash +mix py.dev +``` + +Alternatively, run directly from the py directory: + +```bash +cd py && uv run py-dev +``` + +### Testing with Oban Pro + +Oban Pro is an optional dependency for development and testing. The test suite will automatically +skip Pro-dependent tests (tagged with `@tag :pro`) when Oban Pro is not available. + +To run tests with Pro features, you'll need access to a valid [Oban Pro](https://oban.pro) +license. First, authorize with the Oban repository: + +```bash +mix hex.repo add oban https://repo.oban.pro \ + --fetch-public-key SHA256:4/OSKi0NRF91QVVXlGAhb/BIMLnK8NHcx/EWs+aIWPc \ + --auth-key YOUR_AUTH_KEY +``` + +Then fetch dependencies and run the full test suite: + +```bash +mix deps.get +mix test +``` + +To explicitly run tests without Oban Pro: + +```bash +mix test --exclude pro +``` + +## Community + +There are a few places to connect and communicate with other Oban users: + +- Ask questions and discuss *#oban* on the [Elixir Forum][forum] +- [Request an invitation][invite] and join the *#oban* channel on Slack +- Learn about bug reports and upcoming features in the [issue tracker][issues] + +[invite]: https://elixir-slack.community/ +[forum]: https://elixirforum.com/ +[issues]: https://github.com/oban-bg/oban_web/issues diff --git a/vendor/oban_web/hex_metadata.config b/vendor/oban_web/hex_metadata.config new file mode 100644 index 00000000..41484ccd --- /dev/null +++ b/vendor/oban_web/hex_metadata.config @@ -0,0 +1,179 @@ +{<<"links">>, + [{<<"Website">>,<<"https://oban.pro">>}, + {<<"Changelog">>, + <<"https://github.com/oban-bg/oban_web/blob/main/CHANGELOG.md">>}, + {<<"GitHub">>,<<"https://github.com/oban-bg/oban_web">>}]}. +{<<"name">>,<<"oban_web">>}. +{<<"version">>,<<"2.12.1">>}. +{<<"description">>,<<"Dashboard for the Oban job orchestration framework">>}. +{<<"elixir">>,<<"~> 1.15">>}. +{<<"files">>, + [<<"lib">>,<<"lib/mix">>,<<"lib/mix/tasks">>, + <<"lib/mix/tasks/oban_web.install.ex">>,<<"lib/oban">>,<<"lib/oban/web">>, + <<"lib/oban/web/cron.ex">>,<<"lib/oban/web/dashboard_live.ex">>, + <<"lib/oban/web/page.ex">>,<<"lib/oban/web/telemetry.ex">>, + <<"lib/oban/web/cache.ex">>,<<"lib/oban/web/plugins">>, + <<"lib/oban/web/plugins/stats.ex">>,<<"lib/oban/web/queue.ex">>, + <<"lib/oban/web/metrics.ex">>,<<"lib/oban/web/queries">>, + <<"lib/oban/web/queries/query_helpers.ex">>, + <<"lib/oban/web/queries/workflow_query.ex">>, + <<"lib/oban/web/queries/cron_query.ex">>, + <<"lib/oban/web/queries/job_query.ex">>, + <<"lib/oban/web/queries/queue_query.ex">>,<<"lib/oban/web/router.ex">>, + <<"lib/oban/web/components">>,<<"lib/oban/web/components/sort.ex">>, + <<"lib/oban/web/components/icons.ex">>, + <<"lib/oban/web/components/sidebar_components.ex">>, + <<"lib/oban/web/components/core.ex">>, + <<"lib/oban/web/components/layouts.ex">>, + <<"lib/oban/web/components/layouts">>, + <<"lib/oban/web/components/layouts/live.html.heex">>, + <<"lib/oban/web/components/layouts/root.html.heex">>, + <<"lib/oban/web/components/form_components.ex">>, + <<"lib/oban/web/workflow.ex">>,<<"lib/oban/web/timezones.ex">>, + <<"lib/oban/web/live">>,<<"lib/oban/web/live/search_component.ex">>, + <<"lib/oban/web/live/workflows">>, + <<"lib/oban/web/live/workflows/table_component.ex">>, + <<"lib/oban/web/live/workflows/detail_component.ex">>, + <<"lib/oban/web/live/shortcuts_component.ex">>, + <<"lib/oban/web/live/theme_component.ex">>,<<"lib/oban/web/live/queues">>, + <<"lib/oban/web/live/queues/table_component.ex">>, + <<"lib/oban/web/live/queues/detail_component.ex">>, + <<"lib/oban/web/live/queues/detail_instance_component.ex">>, + <<"lib/oban/web/live/help_component.ex">>,<<"lib/oban/web/live/crons">>, + <<"lib/oban/web/live/crons/table_component.ex">>, + <<"lib/oban/web/live/crons/new_component.ex">>, + <<"lib/oban/web/live/crons/detail_component.ex">>, + <<"lib/oban/web/live/crons/helpers.ex">>,<<"lib/oban/web/live/jobs">>, + <<"lib/oban/web/live/jobs/table_component.ex">>, + <<"lib/oban/web/live/jobs/sidebar_component.ex">>, + <<"lib/oban/web/live/jobs/new_component.ex">>, + <<"lib/oban/web/live/jobs/detail_component.ex">>, + <<"lib/oban/web/live/jobs/history_chart_component.ex">>, + <<"lib/oban/web/live/jobs/timeline_component.ex">>, + <<"lib/oban/web/live/jobs/chart_component.ex">>, + <<"lib/oban/web/live/connectivity_component.ex">>, + <<"lib/oban/web/live/instances_component.ex">>, + <<"lib/oban/web/live/refresh_component.ex">>,<<"lib/oban/web/helpers.ex">>, + <<"lib/oban/web/colors.ex">>,<<"lib/oban/web/cron_expr.ex">>, + <<"lib/oban/web/application.ex">>,<<"lib/oban/web/utils.ex">>, + <<"lib/oban/web/timing.ex">>,<<"lib/oban/web/pages">>, + <<"lib/oban/web/pages/jobs_page.ex">>, + <<"lib/oban/web/pages/crons_page.ex">>, + <<"lib/oban/web/pages/queues_page.ex">>, + <<"lib/oban/web/pages/workflows_page.ex">>,<<"lib/oban/web/resolver.ex">>, + <<"lib/oban/web/authentication.ex">>,<<"lib/oban/web/helpers">>, + <<"lib/oban/web/helpers/queue_helper.ex">>,<<"lib/oban/web/assets.ex">>, + <<"lib/oban/web/search.ex">>,<<"lib/oban/web/exceptions.ex">>, + <<"lib/oban/web.ex">>,<<"priv/static">>,<<"priv/static/app.css">>, + <<"priv/static/icons">>,<<"priv/static/icons/solid">>, + <<"priv/static/icons/solid/pause-circle.svg">>, + <<"priv/static/icons/solid/play-circle.svg">>, + <<"priv/static/icons/solid/crossbones-circle.svg">>, + <<"priv/static/icons/special">>,<<"priv/static/icons/special/spinner.svg">>, + <<"priv/static/icons/outline">>, + <<"priv/static/icons/outline/arrow-top-right-on-square.svg">>, + <<"priv/static/icons/outline/queue-list.svg">>, + <<"priv/static/icons/outline/pause-circle.svg">>, + <<"priv/static/icons/outline/computer-desktop.svg">>, + <<"priv/static/icons/outline/cog.svg">>, + <<"priv/static/icons/outline/play-circle.svg">>, + <<"priv/static/icons/outline/x-circle.svg">>, + <<"priv/static/icons/outline/square-2x2.svg">>, + <<"priv/static/icons/outline/no-symbol.svg">>, + <<"priv/static/icons/outline/chevron-down.svg">>, + <<"priv/static/icons/outline/adjustments-horizontal.svg">>, + <<"priv/static/icons/outline/power.svg">>, + <<"priv/static/icons/outline/cog-8-tooth.svg">>, + <<"priv/static/icons/outline/sparkles.svg">>, + <<"priv/static/icons/outline/chevron-up.svg">>, + <<"priv/static/icons/outline/table-cells.svg">>, + <<"priv/static/icons/outline/arrow-path.svg">>, + <<"priv/static/icons/outline/ellipsis-horizontal-circle.svg">>, + <<"priv/static/icons/outline/chevron-right.svg">>, + <<"priv/static/icons/outline/clipboard.svg">>, + <<"priv/static/icons/outline/link.svg">>, + <<"priv/static/icons/outline/bars-arrow-down.svg">>, + <<"priv/static/icons/outline/arrow-right.svg">>, + <<"priv/static/icons/outline/pencil-square.svg">>, + <<"priv/static/icons/outline/bolt-circle.svg">>, + <<"priv/static/icons/outline/arrow-turn-down-right.svg">>, + <<"priv/static/icons/outline/arrow-path-rounded.svg">>, + <<"priv/static/icons/outline/indeterminate.svg">>, + <<"priv/static/icons/outline/check.svg">>, + <<"priv/static/icons/outline/percent-square.svg">>, + <<"priv/static/icons/outline/rectangle-group.svg">>, + <<"priv/static/icons/outline/x-mark.svg">>, + <<"priv/static/icons/outline/info-circle.svg">>, + <<"priv/static/icons/outline/arrow-right-circle.svg">>, + <<"priv/static/icons/outline/check-circle.svg">>, + <<"priv/static/icons/outline/camera.svg">>, + <<"priv/static/icons/outline/life-buoy.svg">>, + <<"priv/static/icons/outline/view-columns.svg">>, + <<"priv/static/icons/outline/bolt-slash.svg">>, + <<"priv/static/icons/outline/exclamation-circle.svg">>, + <<"priv/static/icons/outline/play-pause-circle.svg">>, + <<"priv/static/icons/outline/trash.svg">>, + <<"priv/static/icons/outline/hashtag.svg">>, + <<"priv/static/icons/outline/sun.svg">>, + <<"priv/static/icons/outline/bars-arrow-up.svg">>, + <<"priv/static/icons/outline/plus-circle.svg">>, + <<"priv/static/icons/outline/user-circle.svg">>, + <<"priv/static/icons/outline/user-group.svg">>, + <<"priv/static/icons/outline/clock.svg">>, + <<"priv/static/icons/outline/command-line.svg">>, + <<"priv/static/icons/outline/adjustments-vertical.svg">>, + <<"priv/static/icons/outline/question-mark-circle.svg">>, + <<"priv/static/icons/outline/map-pin.svg">>, + <<"priv/static/icons/outline/globe.svg">>, + <<"priv/static/icons/outline/arrow-left.svg">>, + <<"priv/static/icons/outline/magnifying-glass.svg">>, + <<"priv/static/icons/outline/minus-circle.svg">>, + <<"priv/static/icons/outline/lock-closed.svg">>, + <<"priv/static/icons/outline/arrow-trending-down.svg">>, + <<"priv/static/icons/outline/chevron-left.svg">>, + <<"priv/static/icons/outline/moon.svg">>, + <<"priv/static/icons/outline/chart-bar-square.svg">>, + <<"priv/static/icons/outline/square-stack.svg">>, + <<"priv/static/icons/outline/calendar-days.svg">>,<<"priv/static/fonts">>, + <<"priv/static/fonts/Inter.woff2">>,<<"priv/static/app.js">>, + <<"priv/timezones.txt">>,<<".formatter.exs">>,<<"mix.exs">>,<<"README.md">>, + <<"CHANGELOG.md">>,<<"LICENSE.txt">>]}. +{<<"app">>,<<"oban_web">>}. +{<<"licenses">>,[<<"Apache-2.0">>]}. +{<<"requirements">>, + [[{<<"name">>,<<"jason">>}, + {<<"app">>,<<"jason">>}, + {<<"optional">>,false}, + {<<"requirement">>,<<"~> 1.2">>}, + {<<"repository">>,<<"hexpm">>}], + [{<<"name">>,<<"phoenix">>}, + {<<"app">>,<<"phoenix">>}, + {<<"optional">>,false}, + {<<"requirement">>,<<"~> 1.7">>}, + {<<"repository">>,<<"hexpm">>}], + [{<<"name">>,<<"phoenix_html">>}, + {<<"app">>,<<"phoenix_html">>}, + {<<"optional">>,false}, + {<<"requirement">>,<<"~> 3.3 or ~> 4.0">>}, + {<<"repository">>,<<"hexpm">>}], + [{<<"name">>,<<"phoenix_pubsub">>}, + {<<"app">>,<<"phoenix_pubsub">>}, + {<<"optional">>,false}, + {<<"requirement">>,<<"~> 2.1">>}, + {<<"repository">>,<<"hexpm">>}], + [{<<"name">>,<<"phoenix_live_view">>}, + {<<"app">>,<<"phoenix_live_view">>}, + {<<"optional">>,false}, + {<<"requirement">>,<<"~> 1.0">>}, + {<<"repository">>,<<"hexpm">>}], + [{<<"name">>,<<"oban">>}, + {<<"app">>,<<"oban">>}, + {<<"optional">>,false}, + {<<"requirement">>,<<"~> 2.21">>}, + {<<"repository">>,<<"hexpm">>}], + [{<<"name">>,<<"oban_met">>}, + {<<"app">>,<<"oban_met">>}, + {<<"optional">>,false}, + {<<"requirement">>,<<"~> 1.1">>}, + {<<"repository">>,<<"hexpm">>}]]}. +{<<"build_tools">>,[<<"mix">>]}. diff --git a/vendor/oban_web/lib/mix/tasks/oban_web.install.ex b/vendor/oban_web/lib/mix/tasks/oban_web.install.ex new file mode 100644 index 00000000..1e675a9f --- /dev/null +++ b/vendor/oban_web/lib/mix/tasks/oban_web.install.ex @@ -0,0 +1,134 @@ +defmodule Mix.Tasks.ObanWeb.Install.Docs do + @moduledoc false + + def short_doc do + "Installs Oban Web into your Phoenix application" + end + + def example do + "mix oban_web.install" + end + + def long_doc do + """ + #{short_doc()} + + This task configures your Phoenix application to use the Oban Web dashboard: + + * Adds the required `Oban.Web.Router` import + * Sets up the dashboard route at "/oban" within the :dev_routes conditional + + ## Example + + ```bash + #{example()} + ``` + """ + end +end + +if Code.ensure_loaded?(Igniter) do + defmodule Mix.Tasks.ObanWeb.Install do + @shortdoc "#{__MODULE__.Docs.short_doc()}" + + @moduledoc __MODULE__.Docs.long_doc() + use Igniter.Mix.Task + + @impl Igniter.Mix.Task + def info(_argv, _composing_task) do + %Igniter.Mix.Task.Info{ + group: :oban, + installs: [{:oban, "~> 2.0"}], + example: __MODULE__.Docs.example() + } + end + + @impl Igniter.Mix.Task + def igniter(igniter) do + case Igniter.Libs.Phoenix.select_router(igniter) do + {igniter, nil} -> + Igniter.add_warning(igniter, """ + No Phoenix router found, Phoenix Liveview is needed for Oban Web + """) + + {igniter, router} -> + update_router(igniter, router) + end + end + + defp update_router(igniter, router) do + zipper = &do_update_router(igniter, &1) + + case Igniter.Project.Module.find_and_update_module(igniter, router, zipper) do + {:ok, igniter} -> + igniter + + {:error, igniter} -> + Igniter.add_warning(igniter, """ + Something went wrong, please check the Oban Web install docs for manual setup instructions + """) + end + end + + defp do_update_router(igniter, zipper) do + web_module = Igniter.Libs.Phoenix.web_module(igniter) + app_name = Igniter.Project.Application.app_name(igniter) + + with {:ok, zipper} <- add_import(zipper, web_module) do + add_route(zipper, app_name) + end + end + + defp add_import(zipper, web_module) do + with {:ok, zipper} <- Igniter.Code.Module.move_to_use(zipper, web_module) do + {:ok, Igniter.Code.Common.add_code(zipper, "\nimport Oban.Web.Router")} + end + end + + defp add_route(zipper, app_name) do + matcher = &dev_routes?(&1, app_name) + + with {:ok, zipper} <- Igniter.Code.Function.move_to_function_call(zipper, :if, 2, matcher), + {:ok, zipper} <- Igniter.Code.Common.move_to_do_block(zipper) do + {:ok, + Igniter.Code.Common.add_code(zipper, """ + scope "/" do + pipe_through :browser + + oban_dashboard "/oban" + end + """)} + end + end + + defp dev_routes?(zipper, app_name) do + case Igniter.Code.Function.move_to_nth_argument(zipper, 0) do + {:ok, zipper} -> + Igniter.Code.Function.function_call?(zipper, {Application, :compile_env}, 2) and + Igniter.Code.Function.argument_equals?(zipper, 0, app_name) and + Igniter.Code.Function.argument_equals?(zipper, 1, :dev_routes) + + _ -> + false + end + end + end +else + defmodule Mix.Tasks.ObanWeb.Install do + @shortdoc "#{__MODULE__.Docs.short_doc()} | Install `igniter` to use" + + @moduledoc __MODULE__.Docs.long_doc() + + use Mix.Task + + def run(_argv) do + Mix.shell().error(""" + The task 'ObanWeb.Task.Install' requires igniter. Please install igniter and try again. + + For more information, see: https://hexdocs.pm/igniter/readme.html#installation + """) + + exit({:shutdown, 1}) + end + end +end diff --git a/vendor/oban_web/lib/oban/web.ex b/vendor/oban_web/lib/oban/web.ex new file mode 100644 index 00000000..7708aca3 --- /dev/null +++ b/vendor/oban_web/lib/oban/web.ex @@ -0,0 +1,53 @@ +defmodule Oban.Web do + @moduledoc false + + alias Oban.Web.Layouts + + def html do + quote do + @moduledoc false + + import Phoenix.Controller, only: [get_csrf_token: 0, view_module: 1, view_template: 1] + + unquote(html_helpers()) + end + end + + def live_view do + quote do + @moduledoc false + + use Phoenix.LiveView, layout: {Layouts, :live} + + unquote(html_helpers()) + end + end + + def live_component do + quote do + @moduledoc false + + use Phoenix.LiveComponent + + unquote(html_helpers()) + end + end + + defp html_helpers do + quote do + use Phoenix.Component + + import Oban.Web.Helpers + import Phoenix.HTML + import Phoenix.LiveView.Helpers + + alias Oban.Web.Components.{Core, Icons} + alias Phoenix.LiveView.JS + end + end + + @doc false + defmacro __using__(which) when is_atom(which) do + apply(__MODULE__, which, []) + end +end diff --git a/vendor/oban_web/lib/oban/web/application.ex b/vendor/oban_web/lib/oban/web/application.ex new file mode 100644 index 00000000..a947e5be --- /dev/null +++ b/vendor/oban_web/lib/oban/web/application.ex @@ -0,0 +1,12 @@ +defmodule Oban.Web.Application do + @moduledoc false + + use Application + + @impl Application + def start(_type, _args) do + children = [Oban.Web.Cache] + + Supervisor.start_link(children, strategy: :one_for_one, name: __MODULE__) + end +end diff --git a/vendor/oban_web/lib/oban/web/assets.ex b/vendor/oban_web/lib/oban/web/assets.ex new file mode 100644 index 00000000..d523b6b8 --- /dev/null +++ b/vendor/oban_web/lib/oban/web/assets.ex @@ -0,0 +1,97 @@ +defmodule Oban.Web.Assets do + @moduledoc false + + @behaviour Plug + + import Plug.Conn + + @static_path Application.app_dir(:oban_web, ["priv", "static"]) + + # Icons + + icon_files = + for dir <- ["outline", "solid", "special"], + base = Path.join([@static_path, "icons", dir]), + file <- File.ls!(base), + String.ends_with?(file, ".svg") do + path = Path.join(base, file) + Module.put_attribute(__MODULE__, :external_resource, path) + {Path.join(dir, file), File.read!(path)} + end + + @icons Map.new(icon_files) + + # Font + + @external_resource font_path = Path.join(@static_path, "fonts/Inter.woff2") + + @font File.read!(font_path) + + # CSS + + @external_resource css_path = Path.join(@static_path, "app.css") + + @css File.read!(css_path) + + # JS + + phoenix_js_paths = + for app <- ~w(phoenix phoenix_html phoenix_live_view)a do + path = Application.app_dir(app, ["priv", "static", "#{app}.js"]) + Module.put_attribute(__MODULE__, :external_resource, path) + path + end + + @external_resource js_path = Path.join(@static_path, "app.js") + + @js """ + #{for path <- phoenix_js_paths, do: path |> File.read!() |> String.replace("//# sourceMappingURL=", "// ")} + #{File.read!(js_path)} + """ + + @impl Plug + def init(asset), do: asset + + @impl Plug + def call(conn, :css) do + serve_asset(conn, @css, "text/css") + end + + def call(conn, :js) do + serve_asset(conn, @js, "text/javascript") + end + + def call(conn, :font) do + serve_asset(conn, @font, "font/woff2") + end + + def call(conn, :icon) do + icon_path = Enum.join(conn.path_params["path"], "/") + + case Map.get(@icons, icon_path) do + nil -> + conn + |> put_resp_header("content-type", "text/plain") + |> send_resp(404, "Icon not found") + |> halt() + + contents -> + serve_asset(conn, contents, "image/svg+xml") + end + end + + defp serve_asset(conn, contents, content_type) do + conn + |> put_resp_header("content-type", content_type) + |> put_resp_header("cache-control", "public, max-age=31536000, immutable") + |> put_private(:plug_skip_csrf_protection, true) + |> send_resp(200, contents) + |> halt() + end + + for {key, val} <- [css: @css, js: @js] do + md5 = Base.encode16(:crypto.hash(:md5, val), case: :lower) + + def current_hash(unquote(key)), do: unquote(md5) + end +end diff --git a/vendor/oban_web/lib/oban/web/authentication.ex b/vendor/oban_web/lib/oban/web/authentication.ex new file mode 100644 index 00000000..6a361290 --- /dev/null +++ b/vendor/oban_web/lib/oban/web/authentication.ex @@ -0,0 +1,30 @@ +defmodule Oban.Web.Authentication do + @moduledoc false + + import Phoenix.Component + import Phoenix.LiveView + + alias Oban.Web.{Resolver, Telemetry} + + def on_mount(:default, params, session, socket) do + %{"oban" => oban, "resolver" => resolver, "user" => user} = session + + conf = if Oban.whereis(oban), do: Oban.config(oban), else: nil + socket = assign(socket, conf: conf, user: user) + + Telemetry.action(:mount, socket, [params: params], fn -> + case Resolver.call_with_fallback(resolver, :resolve_access, [user]) do + {:forbidden, path} -> + socket = + socket + |> put_flash(:error, "Access forbidden") + |> redirect(to: path) + + {:halt, socket} + + _ -> + {:cont, socket} + end + end) + end +end diff --git a/vendor/oban_web/lib/oban/web/cache.ex b/vendor/oban_web/lib/oban/web/cache.ex new file mode 100644 index 00000000..e46d0cbc --- /dev/null +++ b/vendor/oban_web/lib/oban/web/cache.ex @@ -0,0 +1,53 @@ +defmodule Oban.Web.Cache do + @moduledoc false + + use GenServer + + defstruct [:name, purge_interval: :timer.seconds(300)] + + def start_link(opts) do + opts = Keyword.put_new(opts, :name, __MODULE__) + + GenServer.start_link(__MODULE__, opts, name: opts[:name]) + end + + def fetch(name \\ __MODULE__, key, fun) do + if cache_enabled?() do + case :ets.lookup(name, key) do + [{^key, val}] -> + val + + [] -> + tap(fun.(), &:ets.insert(name, {key, &1})) + end + else + fun.() + end + end + + @impl GenServer + def init(opts) do + state = struct!(__MODULE__, opts) + + :ets.new(state.name, [:set, :public, :compressed, :named_table]) + + {:ok, schedule_purge(state)} + end + + @impl GenServer + def handle_info(:purge, state) do + :ets.delete_all_objects(state.name) + + {:noreply, schedule_purge(state)} + end + + defp cache_enabled? do + Process.get(:cache_enabled, Application.get_env(:oban_web, :cache)) + end + + defp schedule_purge(state) do + Process.send_after(self(), :purge, state.purge_interval) + + state + end +end diff --git a/vendor/oban_web/lib/oban/web/colors.ex b/vendor/oban_web/lib/oban/web/colors.ex new file mode 100644 index 00000000..446571a9 --- /dev/null +++ b/vendor/oban_web/lib/oban/web/colors.ex @@ -0,0 +1,93 @@ +defmodule Oban.Web.Colors do + @moduledoc false + + # Hex colors for SVGs and inline styles (must match assets/js/lib/colors.js) + @hex %{ + blue: "#60a5fa", + cyan: "#22d3ee", + emerald: "#34d399", + gray: "#9ca3af", + indigo: "#818cf8", + rose: "#fb7185", + violet: "#a78bfa", + yellow: "#facc15" + } + + # State to color name mapping + @state_color_names %{ + "suspended" => :gray, + "scheduled" => :indigo, + "available" => :blue, + "retryable" => :yellow, + "executing" => :emerald, + "completed" => :cyan, + "cancelled" => :violet, + "discarded" => :rose + } + + # Tailwind classes per state for templates + @state_classes %{ + "suspended" => {"border-gray-400", "bg-gray-400/10", "text-gray-600 dark:text-gray-400"}, + "scheduled" => + {"border-indigo-400", "bg-indigo-400/10", "text-indigo-600 dark:text-indigo-400"}, + "available" => {"border-blue-400", "bg-blue-400/10", "text-blue-600 dark:text-blue-400"}, + "retryable" => + {"border-yellow-400", "bg-yellow-400/10", "text-yellow-600 dark:text-yellow-400"}, + "executing" => + {"border-emerald-400", "bg-emerald-400/10", "text-emerald-600 dark:text-emerald-400"}, + "completed" => {"border-cyan-400", "bg-cyan-400/10", "text-cyan-600 dark:text-cyan-400"}, + "cancelled" => + {"border-violet-400", "bg-violet-400/10", "text-violet-600 dark:text-violet-400"}, + "discarded" => {"border-rose-400", "bg-rose-400/10", "text-rose-600 dark:text-rose-400"} + } + + @inactive_classes { + "border-gray-300 dark:border-gray-600", + "bg-gray-100 dark:bg-gray-800", + "text-gray-400 dark:text-gray-500" + } + + @doc """ + Returns the hex color for a state (for SVGs and inline styles). + """ + def state_hex(state) when is_binary(state) do + color_name = Map.get(@state_color_names, state, :gray) + Map.fetch!(@hex, color_name) + end + + def state_hex(_state), do: Map.fetch!(@hex, :gray) + + @doc """ + Returns Tailwind classes for a state: {border, background, text}. + """ + def state_classes(state) when is_binary(state) do + Map.get(@state_classes, state, @inactive_classes) + end + + def state_classes(_state), do: @inactive_classes + + @doc """ + Returns Tailwind classes for inactive state. + """ + def inactive_classes, do: @inactive_classes + + @doc """ + Returns the background class for a state (e.g., "bg-cyan-400"). + """ + def state_bg_class(state) when is_binary(state) do + color_name = Map.get(@state_color_names, state, :gray) + "bg-#{color_name}-400" + end + + def state_bg_class(_state), do: "bg-gray-400" + + @doc """ + Returns the text class for a state (e.g., "text-cyan-400"). + """ + def state_text_class(state) when is_binary(state) do + color_name = Map.get(@state_color_names, state, :gray) + "text-#{color_name}-400" + end + + def state_text_class(_state), do: "text-gray-400" +end diff --git a/vendor/oban_web/lib/oban/web/components/core.ex b/vendor/oban_web/lib/oban/web/components/core.ex new file mode 100644 index 00000000..80fbc0f7 --- /dev/null +++ b/vendor/oban_web/lib/oban/web/components/core.ex @@ -0,0 +1,424 @@ +defmodule Oban.Web.Components.Core do + use Oban.Web, :html + + attr :click, :string, required: true + attr :danger, :boolean, default: false + attr :disabled, :boolean, default: false + attr :label, :string, required: true + attr :target, :any + slot :icon + slot :title + + def action_button(assigns) do + class = + cond do + assigns.disabled -> + "text-gray-400" + + assigns.danger -> + "text-red-500 group-hover:text-red-600 hover:bg-gray-100 dark:hover:bg-gray-950" + + true -> + "text-gray-500 group-hover:text-gray-600 hover:bg-gray-100 dark:hover:bg-gray-950" + end + + assigns = assign(assigns, :class, class) + + ~H""" + + """ + end + + slot :inner_block, required: true + attr :name, :string, required: true + attr :options, :list, required: true + attr :selected, :any, required: true + attr :title, :string, required: true + attr :disabled, :boolean, default: false + attr :target, :any, default: "myself" + + def dropdown_button(assigns) do + ~H""" +
+ + + +
+ """ + end + + attr :name, :any, required: true + attr :value, :any, required: true + attr :selected, :any, required: true + attr :target, :any + + defp dropdown_option(assigns) do + class = + if assigns.selected == assigns.value do + "text-blue-500 dark:text-blue-400" + else + "text-gray-500 dark:text-gray-400 " + end + + assigns = assign(assigns, :class, class) + + ~H""" +
  • + <%= if to_string(@value) == to_string(@selected) do %> + + <% else %> + + <% end %> + + + {@value |> to_string() |> String.replace("_", " ")} + +
    • + """ + end + + attr :checked, :boolean, default: false + attr :click, :string, required: true + attr :myself, :any, required: true + attr :value, :string, required: true + + def row_checkbox(assigns) do + style = + if assigns.checked do + "border-blue-500 bg-blue-500" + else + "border-gray-400 dark:border-gray-600 group-hover:bg-gray-400 dark:group-hover:bg-gray-600" + end + + assigns = assign(assigns, :style, style) + + ~H""" + + """ + end + + attr :checked, :atom, required: true + attr :click, :string, required: true + attr :myself, :any, required: true + + def all_checkbox(assigns) do + style = + if assigns.checked in [:all, :some] do + "border-blue-500 bg-blue-500" + else + "border-gray-400 dark:border-gray-600 group-hover:bg-gray-400 dark:group-hover:bg-gray-600" + end + + assigns = assign(assigns, :style, style) + + ~H""" + + """ + end + + @doc """ + A status badge with icon that expands to show label on hover. + """ + attr :icon, :string, required: true + attr :id, :string, default: nil + attr :label, :string, required: true + + def status_badge(assigns) do + ~H""" +
    + + <.badge_icon name={@icon} /> + + {@label} + + +
    + """ + end + + defp badge_icon(%{name: "camera"} = assigns), do: ~H[] + + defp badge_icon(%{name: "life_buoy"} = assigns), + do: ~H[] + + defp badge_icon(%{name: "lock_closed"} = assigns), + do: ~H[] + + defp badge_icon(%{name: "sparkles"} = assigns), + do: ~H[] + + defp badge_icon(%{name: "table_cells"} = assigns), + do: ~H[] + + defp badge_icon(%{name: "square_2x2"} = assigns), + do: ~H[] + + defp badge_icon(%{name: "rectangle_group"} = assigns), + do: ~H[] + + defp badge_icon(%{name: "user_group"} = assigns), + do: ~H[] + + defp badge_icon(%{name: "link"} = assigns), + do: ~H[] + + defp badge_icon(%{name: "power"} = assigns), + do: ~H[] + + defp badge_icon(%{name: "pause_circle"} = assigns), + do: ~H[] + + defp badge_icon(%{name: "play_pause_circle"} = assigns), + do: ~H[] + + @doc """ + An icon-only button that expands to show label on hover. Supports disabled state. + """ + attr :id, :string, required: true + attr :icon, :string, required: true + attr :label, :string, required: true + attr :color, :string, required: true + attr :disabled, :boolean, default: false + attr :confirm, :string, default: nil + attr :tooltip, :string, default: nil + attr :rest, :global + + def icon_button(assigns) do + color_classes = + case {assigns.color, assigns.disabled} do + {_, true} -> + "text-gray-400 dark:text-gray-500 bg-gray-100 dark:bg-gray-800 border-gray-200 dark:border-gray-700 cursor-not-allowed" + + {"yellow", false} -> + "text-gray-600 dark:text-gray-400 bg-white dark:bg-gray-800 border-gray-300 dark:border-gray-700 cursor-pointer hover:text-yellow-600 hover:border-yellow-600" + + {"blue", false} -> + "text-gray-600 dark:text-gray-400 bg-white dark:bg-gray-800 border-gray-300 dark:border-gray-700 cursor-pointer hover:text-blue-500 hover:border-blue-600" + + {"red", false} -> + "text-gray-600 dark:text-gray-400 bg-white dark:bg-gray-800 border-gray-300 dark:border-gray-700 cursor-pointer hover:text-red-500 hover:border-red-600" + + {"violet", false} -> + "text-gray-600 dark:text-gray-400 bg-white dark:bg-gray-800 border-gray-300 dark:border-gray-700 cursor-pointer hover:text-violet-500 hover:border-violet-600" + end + + icon_color = + if assigns.disabled do + "text-gray-400 dark:text-gray-500" + else + case assigns.color do + "yellow" -> "text-gray-500 group-hover:text-yellow-500" + "blue" -> "text-gray-500 group-hover:text-blue-500" + "red" -> "text-gray-500 group-hover:text-red-500" + "violet" -> "text-gray-500 group-hover:text-violet-500" + end + end + + assigns = assign(assigns, color_classes: color_classes, icon_color: icon_color) + + ~H""" + + """ + end + + defp button_icon(%{name: "arrow_path"} = assigns), do: ~H[] + + defp button_icon(%{name: "pause_circle"} = assigns), + do: ~H[] + + defp button_icon(%{name: "pencil_square"} = assigns), + do: ~H[] + + defp button_icon(%{name: "play_circle"} = assigns), do: ~H[] + defp button_icon(%{name: "trash"} = assigns), do: ~H[] + defp button_icon(%{name: "x_circle"} = assigns), do: ~H[] + + # Sparkline + + attr :id, :string, required: true + attr :history, :map, required: true + attr :max_value, :integer, default: nil + attr :bar_width, :integer, default: 4 + attr :count, :integer, default: 60 + attr :gap, :integer, default: 1 + attr :height, :integer, default: 16 + attr :class, :string, default: nil + + def sparkline(assigns) do + history = assigns.history + count = assigns.count + bar_width = assigns.bar_width + gap = assigns.gap + height = assigns.height + max_index = count - 1 + + max_value = + if assigns.max_value do + max(assigns.max_value, 1) + else + history + |> Map.values() + |> Enum.reduce(1, fn %{count: c}, acc -> max(c, acc) end) + end + + now = System.system_time(:millisecond) + + {bars, tooltip_data} = + for slot <- 0..max_index, reduce: {[], []} do + {bars_acc, tool_acc} -> + index = max_index - slot + timestamp = now - index * 5 * 1000 + x = slot * (bar_width + gap) + + case Map.get(history, index) do + %{count: c} -> + bar_height = min(c / max_value, 1.0) * height + bar = %{x: x, height: max(bar_height, 0)} + tooltip = %{timestamp: timestamp, count: c} + + {[bar | bars_acc], [tooltip | tool_acc]} + + nil -> + tooltip = %{timestamp: timestamp, count: 0} + + {bars_acc, [tooltip | tool_acc]} + end + end + + bars = Enum.reverse(bars) + tooltip_data = Enum.reverse(tooltip_data) + + placeholders = + for slot <- 0..max_index do + %{x: slot * (bar_width + gap)} + end + + width = count * (bar_width + gap) + + assigns = + assigns + |> assign(bars: bars, placeholders: placeholders, width: width) + |> assign(tooltip_data: tooltip_data) + + ~H""" + + + + + """ + end +end diff --git a/vendor/oban_web/lib/oban/web/components/form_components.ex b/vendor/oban_web/lib/oban/web/components/form_components.ex new file mode 100644 index 00000000..84abf97c --- /dev/null +++ b/vendor/oban_web/lib/oban/web/components/form_components.ex @@ -0,0 +1,192 @@ +defmodule Oban.Web.FormComponents do + @moduledoc false + + use Oban.Web, :html + + attr :label, :string, required: true + attr :name, :string, required: true + attr :value, :any, default: nil + attr :type, :string, default: "text" + attr :colspan, :string, default: nil + attr :placeholder, :string, default: nil + attr :required, :boolean, default: false + attr :disabled, :boolean, default: false + attr :hint, :string, default: nil + attr :rows, :integer, default: 2 + + def form_field(assigns) do + ~H""" +
    + + <%= if @type == "textarea" do %> + + <% else %> + + <% end %> +
    + """ + end + + attr :label, :string, required: true + attr :name, :string, required: true + attr :value, :any, default: nil + attr :options, :list, required: true + attr :disabled, :boolean, default: false + + def select_field(assigns) do + ~H""" +
    + + +
    + """ + end + + attr :label, :string, required: true + attr :name, :string, required: true + attr :checked, :boolean, default: false + attr :disabled, :boolean, default: false + attr :hint, :string, default: nil + attr :colspan, :string, default: nil + + def checkbox_field(assigns) do + ~H""" +
    + + +
    + """ + end + + # Form Parsing Helpers + + @doc """ + Parses and trims a string, returning nil for empty values. + """ + def parse_string(nil), do: nil + def parse_string(""), do: nil + def parse_string(str), do: String.trim(str) + + @doc """ + Parses a string to an integer, returning nil for empty or invalid values. + """ + def parse_int(nil), do: nil + def parse_int(""), do: nil + + def parse_int(str) when is_binary(str) do + case Integer.parse(str) do + {num, ""} -> num + _ -> nil + end + end + + @doc """ + Parses a comma-separated string into a list of trimmed tags. Returns nil for empty input. + """ + def parse_tags(nil), do: nil + def parse_tags(""), do: nil + + def parse_tags(str) when is_binary(str) do + str + |> String.split(",") + |> Enum.map(&String.trim/1) + |> Enum.reject(&(&1 == "")) + |> case do + [] -> nil + tags -> tags + end + end + + @doc """ + Parses a JSON string into a map, returning nil for empty or invalid values. + """ + def parse_json(nil), do: nil + def parse_json(""), do: nil + + def parse_json(str) when is_binary(str) do + case Oban.JSON.decode!(str) do + map when is_map(map) -> map + _ -> nil + end + rescue + _ -> nil + end + + @doc """ + Builds queue options from a list of queue structs. + """ + def queue_options(queues) do + queues + |> Enum.map(fn %{name: name} -> {name, name} end) + |> Enum.sort_by(&elem(&1, 0)) + end +end diff --git a/vendor/oban_web/lib/oban/web/components/icons.ex b/vendor/oban_web/lib/oban/web/components/icons.ex new file mode 100644 index 00000000..98997ba2 --- /dev/null +++ b/vendor/oban_web/lib/oban/web/components/icons.ex @@ -0,0 +1,162 @@ +defmodule Oban.Web.Components.Icons do + @moduledoc false + + use Phoenix.Component + + @icons_path Application.app_dir(:oban_web, ["priv", "static", "icons"]) + + @outline_path Path.join(@icons_path, "outline") + @solid_path Path.join(@icons_path, "solid") + + @outline_icons @outline_path + |> File.ls!() + |> Enum.filter(&String.ends_with?(&1, ".svg")) + |> Enum.map(&String.trim_trailing(&1, ".svg")) + + @solid_icons @solid_path + |> File.ls!() + |> Enum.filter(&String.ends_with?(&1, ".svg")) + |> Enum.map(&String.trim_trailing(&1, ".svg")) + + for icon <- @outline_icons do + @external_resource Path.join(@outline_path, "#{icon}.svg") + + func_name = icon |> String.replace("-", "_") |> String.to_atom() + + @doc false + attr :class, :any, default: "w-6 h-6" + attr :rest, :global + + def unquote(func_name)(assigns) do + assigns = assign(assigns, :icon_path, "outline/#{unquote(icon)}.svg") + + render_icon(assigns) + end + end + + for icon <- @solid_icons do + @external_resource Path.join(@solid_path, "#{icon}.svg") + + func_name = + icon + |> String.replace("-", "_") + |> Kernel.<>("_solid") + |> String.to_atom() + + @doc false + attr :class, :any, default: "w-6 h-6" + attr :rest, :global + + def unquote(func_name)(assigns) do + assigns = assign(assigns, :icon_path, "solid/#{unquote(icon)}.svg") + + render_icon(assigns) + end + end + + # Inline Icons + + attr :rest, :global, default: %{class: "w-6 h-6"} + + def oban_pro_logo(assigns) do + ~H""" + + + + + + """ + end + + attr :rest, :global, default: %{class: "w-6 h-6"} + + def spinner(assigns) do + ~H""" + + + + + """ + end + + defp render_icon(assigns) do + icon_url = icon_url(assigns.icon_path) + class = normalize_class(assigns.class) + assigns = assigns |> assign(:icon_url, icon_url) |> assign(:normalized_class, class) + + ~H""" +