towerops/lib/towerops/netbox/sync.ex
Graham McIntie c790794191 Fix tests, credo issues, and Gaiia sync bugs
- Fix CLOAK_KEY placeholder in nix shell (invalid base64)
- Fix error HTML test assertion to match actual template
- Fix Gaiia sync: read 'subnet' field instead of 'block' for IP blocks
- Fix Gaiia sync: extract nested status name from subscription objects
- Fix security headers tests for environment detection
- Fix mobile auth tests to use raw_token
- Fix LiveView test assertions to match actual template content
- Fix SNMP config test expectations for test environment
- Refactor: resolve all Credo complexity/nesting issues
  - Extract helper functions in NetBox sync, VISP sync, Sonar sync
  - Flatten nested conditionals in settings, integrations, alerts
  - Use with/validate_present pattern for connection testing
2026-02-15 11:55:49 -06:00

248 lines
8 KiB
Elixir

defmodule Towerops.NetBox.Sync do
@moduledoc """
Orchestrates syncing data from NetBox into the local database.
Pulls devices and sites from NetBox via the REST API, upserts them
into local tables, and updates integration sync status.
"""
alias Towerops.Devices
alias Towerops.Integrations
alias Towerops.NetBox.Client
alias Towerops.Sites
require Logger
@doc """
Main entry point: syncs enabled entity types for the given integration.
Returns `{:ok, %{devices: n, sites: n}}` or `{:error, reason}`.
"""
def sync_organization(%Integrations.Integration{} = integration) do
url = integration.credentials["url"]
token = integration.credentials["api_token"]
org_id = integration.organization_id
sync_devices? = integration.credentials["sync_devices"] in [true, "true"]
sync_sites? = integration.credentials["sync_sites"] in [true, "true"]
results = %{devices: 0, sites: 0}
with {:ok, results} <- maybe_sync_sites(results, sync_sites?, url, token, org_id, integration),
{:ok, results} <- maybe_sync_devices(results, sync_devices?, url, token, org_id, integration) do
message =
"Synced #{results.sites} sites, #{results.devices} devices from NetBox"
Integrations.update_sync_status(integration, "success", message)
{:ok, results}
else
{:error, reason} ->
Integrations.update_sync_status(integration, "failed", humanize_sync_error(reason))
{:error, reason}
end
end
defp maybe_sync_sites(results, false, _url, _token, _org_id, _integration), do: {:ok, results}
defp maybe_sync_sites(results, true, url, token, org_id, integration) do
params = build_site_params(integration.credentials)
case Client.list_sites(url, token, params) do
{:ok, %{"results" => sites}} ->
count = upsert_sites(org_id, sites)
Logger.info("NetBox sync: upserted #{count} sites for org #{org_id}")
{:ok, %{results | sites: count}}
{:error, reason} ->
{:error, reason}
end
end
defp maybe_sync_devices(results, false, _url, _token, _org_id, _integration), do: {:ok, results}
defp maybe_sync_devices(results, true, url, token, org_id, integration) do
params = build_device_params(integration.credentials)
case Client.list_devices(url, token, params) do
{:ok, %{"results" => devices}} ->
count = upsert_devices(org_id, devices)
Logger.info("NetBox sync: upserted #{count} devices for org #{org_id}")
{:ok, %{results | devices: count}}
{:error, reason} ->
{:error, reason}
end
end
# --- Sites ---
defp upsert_sites(org_id, netbox_sites) do
existing = Sites.list_organization_sites(org_id)
site_by_name = Map.new(existing, &{String.downcase(&1.name), &1})
Enum.reduce(netbox_sites, 0, fn nb_site, count ->
case upsert_site(org_id, nb_site, site_by_name) do
{:ok, _} ->
count + 1
{:error, reason} ->
Logger.warning("NetBox sync: failed to upsert site #{nb_site["name"]}: #{inspect(reason)}")
count
end
end)
end
defp upsert_site(org_id, nb_site, site_by_name) do
name = nb_site["name"] || ""
key = String.downcase(name)
latitude = nb_site |> get_in(["latitude"]) |> parse_decimal()
longitude = nb_site |> get_in(["longitude"]) |> parse_decimal()
attrs = %{
"name" => name,
"latitude" => latitude,
"longitude" => longitude,
"organization_id" => org_id
}
case Map.get(site_by_name, key) do
nil -> Sites.create_site(attrs)
existing -> Sites.update_site(existing, attrs)
end
end
# --- Devices ---
defp upsert_devices(org_id, netbox_devices) do
existing = Devices.list_organization_devices(org_id)
device_by_name = Map.new(existing, &{String.downcase(&1.name), &1})
device_by_ip =
existing
|> Enum.filter(& &1.ip_address)
|> Map.new(&{&1.ip_address, &1})
# Build site lookup for assignment
sites = Sites.list_organization_sites(org_id)
site_by_name = Map.new(sites, &{String.downcase(&1.name), &1})
Enum.reduce(netbox_devices, 0, fn nb_device, count ->
case upsert_device(org_id, nb_device, device_by_name, device_by_ip, site_by_name) do
{:ok, _} ->
count + 1
{:error, reason} ->
Logger.warning("NetBox sync: failed to upsert device #{nb_device["name"]}: #{inspect(reason)}")
count
end
end)
end
defp upsert_device(org_id, nb_device, device_by_name, device_by_ip, site_by_name) do
name = nb_device["name"] || ""
ip = extract_ip(nb_device["primary_ip4"] || nb_device["primary_ip"])
site_name = get_in(nb_device, ["site", "name"])
attrs = %{
"name" => name,
"ip_address" => ip,
"device_role" => get_in(nb_device, ["role", "name"]) || get_in(nb_device, ["device_role", "name"]),
"status" => map_status(get_in(nb_device, ["status", "value"]) || nb_device["status"]),
"serial_number" => nb_device["serial"],
"site_id" => resolve_site_id(site_name, site_by_name),
"organization_id" => org_id
}
existing = find_existing_device(name, ip, device_by_name, device_by_ip)
case existing do
nil -> Devices.create_device(attrs)
device -> Devices.update_device(device, attrs)
end
end
defp resolve_site_id(nil, _site_by_name), do: nil
defp resolve_site_id(site_name, site_by_name) do
case Map.get(site_by_name, String.downcase(site_name)) do
nil -> nil
site -> site.id
end
end
defp find_existing_device(name, ip, device_by_name, device_by_ip) do
Map.get(device_by_name, String.downcase(name)) ||
(ip && Map.get(device_by_ip, ip))
end
# --- Helpers ---
defp extract_ip(nil), do: nil
defp extract_ip(%{"address" => addr}) when is_binary(addr) do
# NetBox returns "10.0.0.1/24" format
addr |> String.split("/") |> List.first()
end
defp extract_ip(_), do: nil
defp map_status("active"), do: "up"
defp map_status("offline"), do: "down"
defp map_status("planned"), do: "pending"
defp map_status("staged"), do: "pending"
defp map_status("decommissioning"), do: "down"
defp map_status(other), do: other
defp parse_decimal(nil), do: nil
defp parse_decimal(val) when is_float(val), do: Decimal.from_float(val)
defp parse_decimal(val) when is_binary(val) do
case Decimal.parse(val) do
{dec, _} -> dec
:error -> nil
end
end
defp parse_decimal(val) when is_integer(val), do: Decimal.new(val)
defp parse_decimal(_), do: nil
defp build_device_params(credentials) do
%{}
|> maybe_put("role", credentials["device_role_filter"])
|> maybe_put("site", credentials["site_filter"])
|> maybe_put("tag", credentials["tag_filter"])
end
defp maybe_put(params, _key, nil), do: params
defp maybe_put(params, _key, ""), do: params
defp maybe_put(params, key, value), do: Map.put(params, key, value)
defp build_site_params(credentials) do
case credentials["tag_filter"] do
nil -> %{}
"" -> %{}
tag -> %{"tag" => tag}
end
end
defp humanize_sync_error(:unauthorized), do: "Authentication failed — check your NetBox API token"
defp humanize_sync_error(:forbidden), do: "Access denied — your API token may not have sufficient permissions"
defp humanize_sync_error(:not_found), do: "API endpoint not found — check your NetBox URL"
defp humanize_sync_error(:timeout), do: "Connection timed out — NetBox may be temporarily unavailable"
defp humanize_sync_error(%Req.TransportError{reason: :econnrefused}), do: "Connection refused — unable to reach NetBox"
defp humanize_sync_error(%Req.TransportError{reason: reason}), do: "Connection error: #{reason}"
defp humanize_sync_error(reason) when is_binary(reason), do: reason
defp humanize_sync_error({:unexpected_status, status}), do: "NetBox returned unexpected HTTP #{status}"
defp humanize_sync_error(reason) do
Logger.warning("NetBox sync failed with unexpected error: #{inspect(reason)}")
"An unexpected error occurred during sync: #{inspect(reason)}"
end
end