- Fix CLOAK_KEY placeholder in nix shell (invalid base64) - Fix error HTML test assertion to match actual template - Fix Gaiia sync: read 'subnet' field instead of 'block' for IP blocks - Fix Gaiia sync: extract nested status name from subscription objects - Fix security headers tests for environment detection - Fix mobile auth tests to use raw_token - Fix LiveView test assertions to match actual template content - Fix SNMP config test expectations for test environment - Refactor: resolve all Credo complexity/nesting issues - Extract helper functions in NetBox sync, VISP sync, Sonar sync - Flatten nested conditionals in settings, integrations, alerts - Use with/validate_present pattern for connection testing
248 lines
8 KiB
Elixir
248 lines
8 KiB
Elixir
defmodule Towerops.NetBox.Sync do
|
|
@moduledoc """
|
|
Orchestrates syncing data from NetBox into the local database.
|
|
|
|
Pulls devices and sites from NetBox via the REST API, upserts them
|
|
into local tables, and updates integration sync status.
|
|
"""
|
|
|
|
alias Towerops.Devices
|
|
alias Towerops.Integrations
|
|
alias Towerops.NetBox.Client
|
|
alias Towerops.Sites
|
|
|
|
require Logger
|
|
|
|
@doc """
|
|
Main entry point: syncs enabled entity types for the given integration.
|
|
|
|
Returns `{:ok, %{devices: n, sites: n}}` or `{:error, reason}`.
|
|
"""
|
|
def sync_organization(%Integrations.Integration{} = integration) do
|
|
url = integration.credentials["url"]
|
|
token = integration.credentials["api_token"]
|
|
org_id = integration.organization_id
|
|
|
|
sync_devices? = integration.credentials["sync_devices"] in [true, "true"]
|
|
sync_sites? = integration.credentials["sync_sites"] in [true, "true"]
|
|
|
|
results = %{devices: 0, sites: 0}
|
|
|
|
with {:ok, results} <- maybe_sync_sites(results, sync_sites?, url, token, org_id, integration),
|
|
{:ok, results} <- maybe_sync_devices(results, sync_devices?, url, token, org_id, integration) do
|
|
message =
|
|
"Synced #{results.sites} sites, #{results.devices} devices from NetBox"
|
|
|
|
Integrations.update_sync_status(integration, "success", message)
|
|
{:ok, results}
|
|
else
|
|
{:error, reason} ->
|
|
Integrations.update_sync_status(integration, "failed", humanize_sync_error(reason))
|
|
{:error, reason}
|
|
end
|
|
end
|
|
|
|
defp maybe_sync_sites(results, false, _url, _token, _org_id, _integration), do: {:ok, results}
|
|
|
|
defp maybe_sync_sites(results, true, url, token, org_id, integration) do
|
|
params = build_site_params(integration.credentials)
|
|
|
|
case Client.list_sites(url, token, params) do
|
|
{:ok, %{"results" => sites}} ->
|
|
count = upsert_sites(org_id, sites)
|
|
Logger.info("NetBox sync: upserted #{count} sites for org #{org_id}")
|
|
{:ok, %{results | sites: count}}
|
|
|
|
{:error, reason} ->
|
|
{:error, reason}
|
|
end
|
|
end
|
|
|
|
defp maybe_sync_devices(results, false, _url, _token, _org_id, _integration), do: {:ok, results}
|
|
|
|
defp maybe_sync_devices(results, true, url, token, org_id, integration) do
|
|
params = build_device_params(integration.credentials)
|
|
|
|
case Client.list_devices(url, token, params) do
|
|
{:ok, %{"results" => devices}} ->
|
|
count = upsert_devices(org_id, devices)
|
|
Logger.info("NetBox sync: upserted #{count} devices for org #{org_id}")
|
|
{:ok, %{results | devices: count}}
|
|
|
|
{:error, reason} ->
|
|
{:error, reason}
|
|
end
|
|
end
|
|
|
|
# --- Sites ---
|
|
|
|
defp upsert_sites(org_id, netbox_sites) do
|
|
existing = Sites.list_organization_sites(org_id)
|
|
site_by_name = Map.new(existing, &{String.downcase(&1.name), &1})
|
|
|
|
Enum.reduce(netbox_sites, 0, fn nb_site, count ->
|
|
case upsert_site(org_id, nb_site, site_by_name) do
|
|
{:ok, _} ->
|
|
count + 1
|
|
|
|
{:error, reason} ->
|
|
Logger.warning("NetBox sync: failed to upsert site #{nb_site["name"]}: #{inspect(reason)}")
|
|
count
|
|
end
|
|
end)
|
|
end
|
|
|
|
defp upsert_site(org_id, nb_site, site_by_name) do
|
|
name = nb_site["name"] || ""
|
|
key = String.downcase(name)
|
|
latitude = nb_site |> get_in(["latitude"]) |> parse_decimal()
|
|
longitude = nb_site |> get_in(["longitude"]) |> parse_decimal()
|
|
|
|
attrs = %{
|
|
"name" => name,
|
|
"latitude" => latitude,
|
|
"longitude" => longitude,
|
|
"organization_id" => org_id
|
|
}
|
|
|
|
case Map.get(site_by_name, key) do
|
|
nil -> Sites.create_site(attrs)
|
|
existing -> Sites.update_site(existing, attrs)
|
|
end
|
|
end
|
|
|
|
# --- Devices ---
|
|
|
|
defp upsert_devices(org_id, netbox_devices) do
|
|
existing = Devices.list_organization_devices(org_id)
|
|
device_by_name = Map.new(existing, &{String.downcase(&1.name), &1})
|
|
|
|
device_by_ip =
|
|
existing
|
|
|> Enum.filter(& &1.ip_address)
|
|
|> Map.new(&{&1.ip_address, &1})
|
|
|
|
# Build site lookup for assignment
|
|
sites = Sites.list_organization_sites(org_id)
|
|
site_by_name = Map.new(sites, &{String.downcase(&1.name), &1})
|
|
|
|
Enum.reduce(netbox_devices, 0, fn nb_device, count ->
|
|
case upsert_device(org_id, nb_device, device_by_name, device_by_ip, site_by_name) do
|
|
{:ok, _} ->
|
|
count + 1
|
|
|
|
{:error, reason} ->
|
|
Logger.warning("NetBox sync: failed to upsert device #{nb_device["name"]}: #{inspect(reason)}")
|
|
count
|
|
end
|
|
end)
|
|
end
|
|
|
|
defp upsert_device(org_id, nb_device, device_by_name, device_by_ip, site_by_name) do
|
|
name = nb_device["name"] || ""
|
|
ip = extract_ip(nb_device["primary_ip4"] || nb_device["primary_ip"])
|
|
site_name = get_in(nb_device, ["site", "name"])
|
|
|
|
attrs = %{
|
|
"name" => name,
|
|
"ip_address" => ip,
|
|
"device_role" => get_in(nb_device, ["role", "name"]) || get_in(nb_device, ["device_role", "name"]),
|
|
"status" => map_status(get_in(nb_device, ["status", "value"]) || nb_device["status"]),
|
|
"serial_number" => nb_device["serial"],
|
|
"site_id" => resolve_site_id(site_name, site_by_name),
|
|
"organization_id" => org_id
|
|
}
|
|
|
|
existing = find_existing_device(name, ip, device_by_name, device_by_ip)
|
|
|
|
case existing do
|
|
nil -> Devices.create_device(attrs)
|
|
device -> Devices.update_device(device, attrs)
|
|
end
|
|
end
|
|
|
|
defp resolve_site_id(nil, _site_by_name), do: nil
|
|
|
|
defp resolve_site_id(site_name, site_by_name) do
|
|
case Map.get(site_by_name, String.downcase(site_name)) do
|
|
nil -> nil
|
|
site -> site.id
|
|
end
|
|
end
|
|
|
|
defp find_existing_device(name, ip, device_by_name, device_by_ip) do
|
|
Map.get(device_by_name, String.downcase(name)) ||
|
|
(ip && Map.get(device_by_ip, ip))
|
|
end
|
|
|
|
# --- Helpers ---
|
|
|
|
defp extract_ip(nil), do: nil
|
|
|
|
defp extract_ip(%{"address" => addr}) when is_binary(addr) do
|
|
# NetBox returns "10.0.0.1/24" format
|
|
addr |> String.split("/") |> List.first()
|
|
end
|
|
|
|
defp extract_ip(_), do: nil
|
|
|
|
defp map_status("active"), do: "up"
|
|
defp map_status("offline"), do: "down"
|
|
defp map_status("planned"), do: "pending"
|
|
defp map_status("staged"), do: "pending"
|
|
defp map_status("decommissioning"), do: "down"
|
|
defp map_status(other), do: other
|
|
|
|
defp parse_decimal(nil), do: nil
|
|
defp parse_decimal(val) when is_float(val), do: Decimal.from_float(val)
|
|
|
|
defp parse_decimal(val) when is_binary(val) do
|
|
case Decimal.parse(val) do
|
|
{dec, _} -> dec
|
|
:error -> nil
|
|
end
|
|
end
|
|
|
|
defp parse_decimal(val) when is_integer(val), do: Decimal.new(val)
|
|
defp parse_decimal(_), do: nil
|
|
|
|
defp build_device_params(credentials) do
|
|
%{}
|
|
|> maybe_put("role", credentials["device_role_filter"])
|
|
|> maybe_put("site", credentials["site_filter"])
|
|
|> maybe_put("tag", credentials["tag_filter"])
|
|
end
|
|
|
|
defp maybe_put(params, _key, nil), do: params
|
|
defp maybe_put(params, _key, ""), do: params
|
|
defp maybe_put(params, key, value), do: Map.put(params, key, value)
|
|
|
|
defp build_site_params(credentials) do
|
|
case credentials["tag_filter"] do
|
|
nil -> %{}
|
|
"" -> %{}
|
|
tag -> %{"tag" => tag}
|
|
end
|
|
end
|
|
|
|
defp humanize_sync_error(:unauthorized), do: "Authentication failed — check your NetBox API token"
|
|
|
|
defp humanize_sync_error(:forbidden), do: "Access denied — your API token may not have sufficient permissions"
|
|
|
|
defp humanize_sync_error(:not_found), do: "API endpoint not found — check your NetBox URL"
|
|
|
|
defp humanize_sync_error(:timeout), do: "Connection timed out — NetBox may be temporarily unavailable"
|
|
|
|
defp humanize_sync_error(%Req.TransportError{reason: :econnrefused}), do: "Connection refused — unable to reach NetBox"
|
|
|
|
defp humanize_sync_error(%Req.TransportError{reason: reason}), do: "Connection error: #{reason}"
|
|
|
|
defp humanize_sync_error(reason) when is_binary(reason), do: reason
|
|
|
|
defp humanize_sync_error({:unexpected_status, status}), do: "NetBox returned unexpected HTTP #{status}"
|
|
|
|
defp humanize_sync_error(reason) do
|
|
Logger.warning("NetBox sync failed with unexpected error: #{inspect(reason)}")
|
|
"An unexpected error occurred during sync: #{inspect(reason)}"
|
|
end
|
|
end
|