towerops/lib/towerops/gaiia/sync.ex
Graham McIntire 485748d253 feat(gaiia): site-based auto-matching + automatic site mapping during sync
Two improvements to Gaiia reconciliation:

1. Auto-match now uses two-phase matching:
   - Phase 1: IP match (high confidence, same as before)
   - Phase 2: Site match — if a device is at a Towerops site linked to a
     Gaiia network site, match it to unmapped inventory assigned to that site

2. Gaiia sync now automatically maps network sites to Towerops sites by
   name during sync. When a Gaiia network site name matches a Towerops
   site name, `site_id` is set on the `gaiia_network_sites` row. This
   enables the site-based matching in phase 2.

Flash message now shows breakdown: "Auto-matched 47 devices (34 by IP,
13 by site). 17 remaining."
2026-05-11 14:17:41 -05:00

257 lines
8.2 KiB
Elixir

defmodule Towerops.Gaiia.Sync do
@moduledoc """
Orchestrates syncing data from the Gaiia GraphQL API into the local database.
Pulls accounts, network sites, inventory items, and billing subscriptions
from Gaiia, upserts them into local cache tables, and updates integration
sync status.
"""
alias Towerops.Gaiia
alias Towerops.Gaiia.Client
alias Towerops.Gaiia.SubscriberMatching
alias Towerops.Integrations
require Logger
@doc """
Main entry point: syncs all entity types for the given integration.
Returns `{:ok, %{accounts: n, network_sites: n, inventory_items: n, subscriptions: n}}`
or `{:error, reason}`.
"""
def sync_organization(%Integrations.Integration{} = integration) do
api_key = integration.credentials["api_key"]
org_id = integration.organization_id
opts = client_opts(integration)
with {:ok, accounts} <- Client.list_accounts(api_key, opts),
accounts_count = upsert_nodes(org_id, accounts, :account),
{:ok, sites} <- Client.list_network_sites(api_key, opts),
sites_count = upsert_nodes(org_id, sites, :network_site),
{:ok, items} <- Client.list_inventory_items(api_key, opts),
items_count = upsert_nodes(org_id, items, :inventory_item),
{:ok, subs} <- Client.list_billing_subscriptions(api_key, opts) do
subs_count = upsert_nodes(org_id, subs, :billing_subscription)
message =
"Synced #{accounts_count} accounts, #{sites_count} sites, #{items_count} inventory items, #{subs_count} subscriptions"
Integrations.update_sync_status(integration, "success", message)
persist_billing_totals(integration)
SubscriberMatching.refresh_device_links(org_id)
Gaiia.SiteAggregation.compute_and_store(org_id)
{:ok,
%{
accounts: accounts_count,
network_sites: sites_count,
inventory_items: items_count,
subscriptions: subs_count
}}
else
{:error, reason} ->
Integrations.update_sync_status(integration, "failed", humanize_sync_error(reason))
{:error, reason}
end
end
defp upsert_nodes(org_id, nodes, type) do
Enum.reduce(nodes, 0, fn node, count ->
case upsert_node(org_id, node, type) do
{:ok, _} ->
count + 1
{:error, changeset} ->
Logger.warning("Failed to upsert Gaiia #{type}: #{inspect(changeset.errors)}")
count
end
end)
end
defp upsert_node(org_id, node, :account) do
Gaiia.upsert_account(org_id, map_account(node))
end
defp upsert_node(org_id, node, :network_site) do
Gaiia.upsert_network_site(org_id, map_network_site(node, org_id))
end
defp upsert_node(org_id, node, :inventory_item) do
Gaiia.upsert_inventory_item(org_id, map_inventory_item(node))
end
defp upsert_node(org_id, node, :billing_subscription) do
Gaiia.upsert_billing_subscription(org_id, map_billing_subscription(node))
end
defp map_account(node) do
subs = get_in(node, ["billingSubscriptions", "edges"]) || []
active_count = Enum.count(subs, &(&1["node"]["status"] == "ACTIVE"))
%{
gaiia_id: node["id"],
readable_id: to_string(node["readableId"]),
name: node["name"],
status: get_in(node, ["status", "name"]),
account_type: get_in(node, ["type", "name"]),
address: map_address(node["physicalAddress"]),
subscription_count: active_count,
raw_data: node
}
end
defp map_network_site(node, org_id) do
ip_block_edges = get_in(node, ["ipBlocks", "edges"]) || []
ip_blocks = Enum.map(ip_block_edges, & &1["node"]["block"])
gaiia_name = node["name"]
%{
gaiia_id: node["id"],
name: gaiia_name,
address: map_address(node["address"]),
ip_blocks: ip_blocks,
site_id: find_matching_site_id(org_id, gaiia_name),
raw_data: node
}
end
defp find_matching_site_id(org_id, gaiia_name) when is_binary(gaiia_name) and gaiia_name != "" do
import Ecto.Query
alias Towerops.Repo
alias Towerops.Sites.Site
search = "%#{Towerops.QueryHelpers.sanitize_like(gaiia_name)}%"
Repo.one(
from(s in Site,
where: s.organization_id == ^org_id,
where: ilike(s.name, ^search) or ilike(s.name, ^gaiia_name),
limit: 1,
select: s.id
)
)
end
defp find_matching_site_id(_org_id, _name), do: nil
defp map_inventory_item(node) do
assignation = node["assignation"]
{account_id, site_id} =
case assignation do
%{"assigneeType" => "ACCOUNT", "assignee" => %{"id" => id}} -> {id, nil}
%{"assigneeType" => "NETWORK_SITE", "assignee" => %{"id" => id}} -> {nil, id}
_ -> {nil, nil}
end
%{
gaiia_id: node["id"],
name: get_in(node, ["model", "name"]),
status: node["status"],
serial_number: nil,
mac_address: extract_mac_address(node),
ip_address: node["ipAddressV4"],
model_name: get_in(node, ["model", "name"]),
manufacturer_name: get_in(node, ["model", "manufacturer", "name"]),
category: get_in(node, ["model", "category", "name"]),
assigned_account_gaiia_id: account_id,
assigned_network_site_gaiia_id: site_id,
raw_data: node
}
end
defp extract_mac_address(node) do
edges = get_in(node, ["fields", "edges"]) || []
Enum.find_value(edges, fn
%{"node" => %{"data" => data, "modelField" => %{"name" => field_name}}}
when is_binary(data) and is_binary(field_name) ->
if String.contains?(String.downcase(field_name), "mac"), do: data
_ ->
nil
end)
end
defp map_billing_subscription(node) do
price_cents = get_in(node, ["productVersion", "price"])
mrr_amount =
if is_integer(price_cents),
do: Decimal.div(Decimal.new(price_cents), 100)
%{
gaiia_id: node["id"],
account_gaiia_id: get_in(node, ["entity", "id"]),
status: node["status"],
product_name: get_in(node, ["productVersion", "product", "name"]),
mrr_amount: mrr_amount,
currency: get_in(node, ["productVersion", "currency"]),
raw_data: node
}
end
defp map_address(nil), do: nil
defp map_address(addr) do
%{
"line1" => addr["line1"],
"city" => addr["locality"],
"state" => addr["region"],
"zip" => addr["postalCode"],
"country" => addr["country"],
"latitude" => addr["latitude"],
"longitude" => addr["longitude"]
}
end
defp persist_billing_totals(integration) do
org_id = integration.organization_id
subscriber_count = Gaiia.count_active_subscribers(org_id)
total_mrr = Gaiia.sum_active_mrr(org_id)
Integrations.update_billing_totals(integration, subscriber_count, total_mrr)
end
defp client_opts(integration) do
case integration.credentials["endpoint"] do
nil -> []
endpoint -> [endpoint: endpoint]
end
end
defp humanize_sync_error(:unauthorized), do: "Authentication failed — check your API key"
defp humanize_sync_error(:forbidden), do: "Access denied — your API key may not have sufficient permissions"
defp humanize_sync_error(%Req.TransportError{reason: :econnrefused}), do: "Connection refused — unable to reach Gaiia"
defp humanize_sync_error(%Req.TransportError{reason: reason}) do
Logger.warning("Gaiia sync transport error: #{inspect(reason)}")
"Connection error — unable to establish a secure connection to Gaiia"
end
defp humanize_sync_error(reason) when is_binary(reason) do
if String.contains?(reason, ["CA trust store", "cacert", "certificate"]) do
"SSL certificate verification failed — unable to establish a secure connection"
else
Logger.warning("Sync failed with raw error: #{reason}")
"An unexpected error occurred during sync"
end
end
defp humanize_sync_error({:graphql_errors, errors}) when is_list(errors) do
messages = Enum.map_join(errors, "; ", &(&1["message"] || inspect(&1)))
"GraphQL errors: #{messages}"
end
defp humanize_sync_error({:rate_limited, retry_after}), do: "Rate limited by Gaiia — retry after #{retry_after}s"
defp humanize_sync_error({:unexpected_status, status}), do: "Gaiia returned unexpected HTTP #{status}"
defp humanize_sync_error(reason) do
Logger.warning("Gaiia sync failed with unexpected error: #{inspect(reason)}")
"An unexpected error occurred during sync"
end
end