aprs.me/lib/aprsme/circuit_breaker.ex

defmodule Aprsme.CircuitBreaker do
  @moduledoc """
  Circuit breaker implementation for external service calls to prevent cascading failures
  """
  use GenServer

  alias Aprsme.LogSanitizer

  require Logger

  @type state :: :closed | :open | :half_open

  @default_failure_threshold 5
  @default_timeout 5000
  @default_recovery_timeout 30_000

  defstruct state: :closed,
            failure_count: 0,
            failure_threshold: @default_failure_threshold,
            timeout: @default_timeout,
            last_failure_time: nil,
            recovery_timeout: @default_recovery_timeout

  @type service_state :: %__MODULE__{
          state: state(),
          failure_count: non_neg_integer(),
          failure_threshold: pos_integer(),
          timeout: pos_integer(),
          last_failure_time: DateTime.t() | nil,
          recovery_timeout: pos_integer()
        }

  # Public API

  @doc """
  Start the circuit breaker GenServer
  """
  def start_link(opts \\ []) do
    GenServer.start_link(__MODULE__, opts, name: __MODULE__)
  end

  @doc """
  Execute a function with circuit breaker protection
  """
  @spec call(atom(), function(), non_neg_integer()) :: {:ok, any()} | {:error, :circuit_open | :timeout | any()}
  def call(service_name, fun, timeout \\ @default_timeout) when is_function(fun, 0) do
    case get_state(service_name) do
      :open ->
        Logger.warning("Circuit breaker open for service",
          circuit_breaker:
            LogSanitizer.log_data(
              service: service_name,
              state: :open,
              action: :rejected
            )
        )

        {:error, :circuit_open}

      state when state in [:closed, :half_open] ->
        attempt_call(service_name, fun, timeout)
    end
  end

  @doc """
  Get the current state of a service circuit breaker
  """
  @spec get_state(atom()) :: state()
  def get_state(service_name) do
    GenServer.call(__MODULE__, {:get_state, service_name})
  end

  @doc """
  Get statistics for a service circuit breaker
  """
  @spec get_stats(atom()) :: service_state()
  def get_stats(service_name) do
    GenServer.call(__MODULE__, {:get_stats, service_name})
  end

  @doc """
  Manually reset a circuit breaker
  """
  @spec reset(atom()) :: :ok
  def reset(service_name) do
    GenServer.call(__MODULE__, {:reset, service_name})
  end

  # GenServer implementation

  @impl true
  def init(_opts) do
    {:ok, %{}}
  end

  @impl true
  def handle_call({:get_state, service_name}, _from, services) do
    service_state = get_service_state(services, service_name)
    current_state = calculate_current_state(service_state)

    # Persist state transitions (e.g., :open -> :half_open) so that
    # concurrent callers see the same state and don't all independently
    # enter half_open.
    services =
      if current_state == service_state.state do
        services
      else
        updated_state = %{service_state | state: current_state}
        Map.put(services, service_name, updated_state)
      end

    {:reply, current_state, services}
  end

  @impl true
  def handle_call({:get_stats, service_name}, _from, services) do
    service_state = get_service_state(services, service_name)
    {:reply, service_state, services}
  end

  @impl true
  def handle_call({:reset, service_name}, _from, services) do
    default_state = default_service_state()
    new_services = Map.put(services, service_name, default_state)

    Logger.info("Circuit breaker reset",
      circuit_breaker:
        LogSanitizer.log_data(
          service: service_name,
          action: :manual_reset
        )
    )

    {:reply, :ok, new_services}
  end

  @impl true
  def handle_call({:record_success, service_name}, _from, services) do
    service_state = get_service_state(services, service_name)
    updated_state = %{service_state | failure_count: 0, state: :closed}
    new_services = Map.put(services, service_name, updated_state)
    {:reply, :ok, new_services}
  end

  @impl true
  def handle_call({:record_failure, service_name}, _from, services) do
    service_state = get_service_state(services, service_name)
    new_failure_count = service_state.failure_count + 1

    new_state =
      if new_failure_count >= service_state.failure_threshold do
        :open
      else
        service_state.state
      end

    updated_state = %{
      service_state
      | failure_count: new_failure_count,
        state: new_state,
        last_failure_time: DateTime.utc_now()
    }

    if new_state == :open do
      Logger.warning("Circuit breaker opened due to failure threshold",
        circuit_breaker:
          LogSanitizer.log_data(
            service: service_name,
            failure_count: new_failure_count,
            threshold: service_state.failure_threshold,
            state: :open
          )
      )
    end

    new_services = Map.put(services, service_name, updated_state)
    {:reply, :ok, new_services}
  end

  # Private functions

  defp attempt_call(service_name, fun, timeout) do
    task = Task.async(fn -> fun.() end)

    try do
      result = Task.await(task, timeout)
      record_success(service_name)
      {:ok, result}
    catch
      :exit, {:timeout, _} ->
        _ = Task.shutdown(task, :brutal_kill)
        record_failure(service_name)

        Logger.warning("Service call timed out",
          circuit_breaker:
            LogSanitizer.log_data(
              service: service_name,
              timeout_ms: timeout,
              action: :timeout_failure
            )
        )

        {:error, :timeout}

      kind, error ->
        record_failure(service_name)

        Logger.error("Service call failed",
          circuit_breaker:
            LogSanitizer.log_data(
              service: service_name,
              error_kind: kind,
              error: LogSanitizer.sanitize(error)
            )
        )

        {:error, error}
    end
  end

  defp record_success(service_name) do
    GenServer.call(__MODULE__, {:record_success, service_name})
  end

  defp record_failure(service_name) do
    GenServer.call(__MODULE__, {:record_failure, service_name})
  end

  defp get_service_state(services, service_name) do
    Map.get(services, service_name, default_service_state())
  end

  defp default_service_state do
    %__MODULE__{}
  end

  defp calculate_current_state(%{
         state: :open,
         last_failure_time: last_failure_time,
         recovery_timeout: recovery_timeout
       }) do
    if last_failure_time && DateTime.diff(DateTime.utc_now(), last_failure_time, :millisecond) >= recovery_timeout do
      :half_open
    else
      :open
    end
  end

  defp calculate_current_state(%{state: state}), do: state
end