feat: add profile export/import workflow via API
Add export and import functionality for device SNMP profiles: Export (local): - New mix task: mix export_profiles --librenms-path ~/dev/librenms --output profiles.json - Exports all profiles to JSON format - Can filter specific profiles with --profiles flag Import (production): - New API endpoint: POST /api/v1/admin/profiles/import - Requires superuser authentication via browser session - Processes imports in background via Exq worker (maintenance queue) - Returns job ID for tracking New modules: - Mix.Tasks.ExportProfiles - Export profiles to JSON - ProfileImportWorker - Background job for importing profiles - ProfilesController - API endpoint for bulk import - RequireSuperuser plug - Restricts access to superusers - Importer.import_profile_from_data/2 - Import from data structures This enables bulk profile management without SSH access to production.
This commit is contained in:
parent
28e9be773d
commit
e0bcd4feda
6 changed files with 395 additions and 1 deletions
179
lib/mix/tasks/export_profiles.ex
Normal file
179
lib/mix/tasks/export_profiles.ex
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
defmodule Mix.Tasks.ExportProfiles do
|
||||
@moduledoc """
|
||||
Exports device SNMP profiles to JSON format for uploading to production.
|
||||
|
||||
## Usage
|
||||
|
||||
# Export all profiles to JSON
|
||||
mix export_profiles --librenms-path ~/dev/librenms --output profiles.json
|
||||
|
||||
# Export specific profiles
|
||||
mix export_profiles --librenms-path ~/dev/librenms --profiles epmp,airos-af --output profiles.json
|
||||
|
||||
## Options
|
||||
|
||||
--librenms-path PATH - Path to external installation (required)
|
||||
--output PATH - Output JSON file path (default: profiles.json)
|
||||
--profiles LIST - Export only specific profiles (comma-separated, optional)
|
||||
|
||||
## Output Format
|
||||
|
||||
The exported JSON file contains all profile data in a format that can be uploaded
|
||||
to the production server via API:
|
||||
|
||||
POST /api/v1/admin/profiles/import
|
||||
Content-Type: application/json
|
||||
Authorization: Bearer YOUR_TOKEN
|
||||
|
||||
"""
|
||||
use Mix.Task
|
||||
|
||||
@requirements ["app.start"]
|
||||
|
||||
@impl Mix.Task
|
||||
def run(args) do
|
||||
{opts, _, _} =
|
||||
OptionParser.parse(args,
|
||||
strict: [
|
||||
librenms_path: :string,
|
||||
output: :string,
|
||||
profiles: :string
|
||||
]
|
||||
)
|
||||
|
||||
librenms_path = opts[:librenms_path] || guess_librenms_path()
|
||||
output_file = opts[:output] || "profiles.json"
|
||||
profile_filter = parse_profile_filter(opts[:profiles])
|
||||
|
||||
if librenms_path do
|
||||
Mix.shell().info("Exporting profiles from: #{librenms_path}")
|
||||
export_profiles(librenms_path, output_file, profile_filter)
|
||||
else
|
||||
Mix.shell().error("Please specify --librenms-path option")
|
||||
print_usage()
|
||||
end
|
||||
end
|
||||
|
||||
defp export_profiles(librenms_path, output_file, profile_filter) do
|
||||
detection_dir = Path.join(librenms_path, "resources/definitions/os_detection")
|
||||
discovery_dir = Path.join(librenms_path, "resources/definitions/os_discovery")
|
||||
|
||||
detection_files = Path.wildcard(Path.join(detection_dir, "*.yaml"))
|
||||
|
||||
# Filter profiles if specified
|
||||
detection_files =
|
||||
if profile_filter do
|
||||
Enum.filter(detection_files, fn file ->
|
||||
os_name = Path.basename(file, ".yaml")
|
||||
os_name in profile_filter
|
||||
end)
|
||||
else
|
||||
detection_files
|
||||
end
|
||||
|
||||
Mix.shell().info("Found #{length(detection_files)} profiles to export")
|
||||
Mix.shell().info("Reading YAML files...")
|
||||
|
||||
# Read and parse all profiles
|
||||
profiles =
|
||||
detection_files
|
||||
|> Enum.map(fn detection_file ->
|
||||
os_name = Path.basename(detection_file, ".yaml")
|
||||
discovery_file = Path.join(discovery_dir, "#{os_name}.yaml")
|
||||
|
||||
Mix.shell().info(" Reading: #{os_name}")
|
||||
|
||||
detection_data = read_yaml_file(detection_file)
|
||||
|
||||
discovery_data =
|
||||
if File.exists?(discovery_file) do
|
||||
read_yaml_file(discovery_file)
|
||||
end
|
||||
|
||||
%{
|
||||
os: os_name,
|
||||
detection: detection_data,
|
||||
discovery: discovery_data
|
||||
}
|
||||
end)
|
||||
|> Enum.reject(&is_nil(&1.detection))
|
||||
|
||||
Mix.shell().info("\nWriting to: #{output_file}")
|
||||
|
||||
# Write JSON file
|
||||
json_data = Jason.encode!(profiles, pretty: true)
|
||||
File.write!(output_file, json_data)
|
||||
|
||||
file_size = format_bytes(File.stat!(output_file).size)
|
||||
Mix.shell().info("✓ Exported #{length(profiles)} profiles (#{file_size})")
|
||||
Mix.shell().info("\nUpload to production (requires superuser session):")
|
||||
|
||||
Mix.shell().info("""
|
||||
|
||||
curl -X POST https://towerops.net/api/v1/admin/profiles/import \\
|
||||
-H "Cookie: _towerops_key=YOUR_SESSION_COOKIE" \\
|
||||
-H "Content-Type: application/json" \\
|
||||
-d @#{output_file}
|
||||
|
||||
Note: You must be logged in as a superuser. Get the session cookie from
|
||||
your browser's developer tools (Application > Cookies > _towerops_key).
|
||||
""")
|
||||
end
|
||||
|
||||
defp read_yaml_file(file_path) do
|
||||
case YamlElixir.read_from_file(file_path) do
|
||||
{:ok, data} ->
|
||||
data
|
||||
|
||||
{:error, reason} ->
|
||||
Mix.shell().error(" Failed to read #{file_path}: #{inspect(reason)}")
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_profile_filter(nil), do: nil
|
||||
|
||||
defp parse_profile_filter(profiles_string) do
|
||||
profiles_string
|
||||
|> String.split(",", trim: true)
|
||||
|> Enum.map(&String.trim/1)
|
||||
end
|
||||
|
||||
defp guess_librenms_path do
|
||||
home = System.get_env("HOME")
|
||||
possible_paths = ["~/dev/librenms", "#{home}/dev/librenms", "/opt/librenms"]
|
||||
|
||||
possible_paths
|
||||
|> Enum.find(fn path ->
|
||||
expanded = Path.expand(path)
|
||||
File.dir?(expanded)
|
||||
end)
|
||||
|> case do
|
||||
nil -> nil
|
||||
path -> Path.expand(path)
|
||||
end
|
||||
end
|
||||
|
||||
defp format_bytes(bytes) when bytes < 1024, do: "#{bytes} B"
|
||||
defp format_bytes(bytes) when bytes < 1024 * 1024, do: "#{Float.round(bytes / 1024, 1)} KB"
|
||||
defp format_bytes(bytes), do: "#{Float.round(bytes / (1024 * 1024), 1)} MB"
|
||||
|
||||
defp print_usage do
|
||||
Mix.shell().info("""
|
||||
Usage:
|
||||
mix export_profiles [options]
|
||||
|
||||
Options:
|
||||
--librenms-path PATH Path to external installation (required)
|
||||
--output PATH Output JSON file (default: profiles.json)
|
||||
--profiles LIST Export specific profiles (comma-separated)
|
||||
|
||||
Examples:
|
||||
# Export all profiles
|
||||
mix export_profiles --librenms-path ~/dev/librenms --output profiles.json
|
||||
|
||||
# Export specific profiles
|
||||
mix export_profiles --librenms-path ~/dev/librenms --profiles epmp,unifi,airos-af
|
||||
""")
|
||||
end
|
||||
end
|
||||
|
|
@ -36,6 +36,30 @@ defmodule Towerops.DeviceProfiles.Importer do
|
|||
end)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Imports a device profile from parsed data structures (instead of files).
|
||||
|
||||
## Parameters
|
||||
- detection_data: Parsed YAML map from os_detection
|
||||
- discovery_data: Parsed YAML map from os_discovery (optional)
|
||||
|
||||
## Returns
|
||||
- {:ok, profile} on success
|
||||
- {:error, reason} on failure
|
||||
"""
|
||||
def import_profile_from_data(detection_data, discovery_data \\ nil) when is_map(detection_data) do
|
||||
Repo.transaction(fn ->
|
||||
with {:ok, profile} <- create_profile_from_detection(detection_data),
|
||||
{:ok, profile} <- maybe_import_discovery_data(profile, discovery_data) do
|
||||
profile
|
||||
else
|
||||
{:error, reason} ->
|
||||
Logger.error("Failed to import profile: #{inspect(reason)}")
|
||||
Repo.rollback(reason)
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Imports all profiles from a directory.
|
||||
"""
|
||||
|
|
@ -193,6 +217,12 @@ defmodule Towerops.DeviceProfiles.Importer do
|
|||
end
|
||||
end
|
||||
|
||||
defp maybe_import_discovery_data(profile, nil), do: {:ok, profile}
|
||||
|
||||
defp maybe_import_discovery_data(profile, discovery_data) when is_map(discovery_data) do
|
||||
import_discovery_definitions(profile, discovery_data)
|
||||
end
|
||||
|
||||
defp import_discovery_definitions(profile, data) do
|
||||
# Import MIB if specified
|
||||
if data["mib"] do
|
||||
|
|
|
|||
74
lib/towerops/workers/profile_import_worker.ex
Normal file
74
lib/towerops/workers/profile_import_worker.ex
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
defmodule Towerops.Workers.ProfileImportWorker do
|
||||
@moduledoc """
|
||||
Background worker for importing device SNMP profiles.
|
||||
|
||||
Enqueued when:
|
||||
- Superuser uploads profiles via API endpoint POST /api/v1/admin/profiles/import
|
||||
|
||||
Queue: maintenance
|
||||
"""
|
||||
|
||||
alias Towerops.DeviceProfiles
|
||||
|
||||
require Logger
|
||||
|
||||
@doc """
|
||||
Imports device profiles from JSON data.
|
||||
|
||||
## Parameters
|
||||
- profiles: List of profile maps with detection and discovery data
|
||||
- job_id: Unique identifier for tracking the import job
|
||||
|
||||
## Returns
|
||||
- :ok on completion (logs individual profile successes/failures)
|
||||
"""
|
||||
def perform(profiles, job_id) when is_list(profiles) do
|
||||
Logger.info("Starting profile import job #{job_id} with #{length(profiles)} profiles")
|
||||
|
||||
results =
|
||||
Enum.map(profiles, fn profile_data ->
|
||||
os_name = profile_data["os"]
|
||||
|
||||
case import_profile(profile_data) do
|
||||
{:ok, profile} ->
|
||||
Logger.info("[#{job_id}] Successfully imported profile: #{profile.os}")
|
||||
{:ok, profile}
|
||||
|
||||
{:error, reason} ->
|
||||
Logger.error("[#{job_id}] Failed to import profile #{os_name}: #{inspect(reason)}")
|
||||
{:error, os_name, reason}
|
||||
end
|
||||
end)
|
||||
|
||||
{successes, failures} = Enum.split_with(results, &match?({:ok, _}, &1))
|
||||
|
||||
Logger.info("[#{job_id}] Import complete: #{length(successes)} succeeded, #{length(failures)} failed")
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
# Private functions
|
||||
|
||||
defp import_profile(%{"os" => os_name, "detection" => detection_data} = data) do
|
||||
# Delete existing profile first (force replace)
|
||||
if profile = DeviceProfiles.get_profile(os_name) do
|
||||
DeviceProfiles.delete_profile(profile)
|
||||
end
|
||||
|
||||
# Import directly from data structures
|
||||
case Towerops.DeviceProfiles.Importer.import_profile_from_data(
|
||||
detection_data,
|
||||
data["discovery"]
|
||||
) do
|
||||
{:ok, profile} ->
|
||||
{:ok, profile}
|
||||
|
||||
{:error, reason} ->
|
||||
{:error, reason}
|
||||
end
|
||||
end
|
||||
|
||||
defp import_profile(_invalid_data) do
|
||||
{:error, "Missing required fields: os and detection"}
|
||||
end
|
||||
end
|
||||
75
lib/towerops_web/controllers/api/v1/profiles_controller.ex
Normal file
75
lib/towerops_web/controllers/api/v1/profiles_controller.ex
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
defmodule ToweropsWeb.Api.V1.ProfilesController do
|
||||
@moduledoc """
|
||||
API controller for managing device SNMP profiles.
|
||||
|
||||
All endpoints require API token authentication.
|
||||
"""
|
||||
use ToweropsWeb, :controller
|
||||
|
||||
require Logger
|
||||
|
||||
@doc """
|
||||
POST /api/v1/profiles/import
|
||||
|
||||
Imports device profiles from JSON data exported via `mix export_profiles`.
|
||||
|
||||
Request body:
|
||||
[
|
||||
{
|
||||
"os": "epmp",
|
||||
"detection": { ... }, # YAML detection data
|
||||
"discovery": { ... } # YAML discovery data (optional)
|
||||
},
|
||||
...
|
||||
]
|
||||
|
||||
Response:
|
||||
{
|
||||
"success": 15,
|
||||
"failed": 2,
|
||||
"errors": [
|
||||
{"os": "profile_name", "reason": "error message"}
|
||||
]
|
||||
}
|
||||
|
||||
Example:
|
||||
curl -X POST https://towerops.net/api/v1/profiles/import \\
|
||||
-H "Authorization: Bearer YOUR_TOKEN" \\
|
||||
-H "Content-Type: application/json" \\
|
||||
-d @profiles.json
|
||||
"""
|
||||
def import_bulk(conn, %{"_json" => profiles}) when is_list(profiles) do
|
||||
Logger.info("Queueing import for #{length(profiles)} device profiles via API")
|
||||
|
||||
# Generate unique job ID for tracking
|
||||
job_id = Ecto.UUID.generate()
|
||||
|
||||
# Enqueue background job
|
||||
case Exq.enqueue(Exq, "maintenance", Towerops.Workers.ProfileImportWorker, [profiles, job_id]) do
|
||||
{:ok, _job_jid} ->
|
||||
Logger.info("Profile import job #{job_id} queued successfully")
|
||||
|
||||
json(conn, %{
|
||||
status: "queued",
|
||||
job_id: job_id,
|
||||
profile_count: length(profiles),
|
||||
message: "Import job queued successfully. #{length(profiles)} profiles will be imported in the background."
|
||||
})
|
||||
|
||||
{:error, reason} ->
|
||||
Logger.error("Failed to queue profile import: #{inspect(reason)}")
|
||||
|
||||
conn
|
||||
|> put_status(:internal_server_error)
|
||||
|> json(%{error: "Failed to queue import job", reason: inspect(reason)})
|
||||
end
|
||||
end
|
||||
|
||||
def import_bulk(conn, _params) do
|
||||
conn
|
||||
|> put_status(:bad_request)
|
||||
|> json(%{error: "Invalid request format. Expected array of profile objects."})
|
||||
end
|
||||
|
||||
# Private functions (removed - logic moved to ProfileImportWorker)
|
||||
end
|
||||
26
lib/towerops_web/plugs/require_superuser.ex
Normal file
26
lib/towerops_web/plugs/require_superuser.ex
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
defmodule ToweropsWeb.Plugs.RequireSuperuser do
|
||||
@moduledoc """
|
||||
Plug that requires the current user to be a superuser.
|
||||
|
||||
Must be used after authentication plug that sets :current_user assign.
|
||||
Returns 403 Forbidden if user is not a superuser.
|
||||
"""
|
||||
|
||||
import Phoenix.Controller, only: [json: 2]
|
||||
import Plug.Conn
|
||||
|
||||
def init(opts), do: opts
|
||||
|
||||
def call(conn, _opts) do
|
||||
user = conn.assigns[:current_user]
|
||||
|
||||
if user && user.is_superuser do
|
||||
conn
|
||||
else
|
||||
conn
|
||||
|> put_status(:forbidden)
|
||||
|> json(%{error: "Superuser access required"})
|
||||
|> halt()
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -6,6 +6,8 @@ defmodule ToweropsWeb.Router do
|
|||
import Phoenix.LiveDashboard.Router
|
||||
import ToweropsWeb.UserAuth
|
||||
|
||||
alias ToweropsWeb.Api.V1
|
||||
|
||||
pipeline :browser do
|
||||
plug :accepts, ["html"]
|
||||
plug :fetch_session
|
||||
|
|
@ -70,13 +72,21 @@ defmodule ToweropsWeb.Router do
|
|||
end
|
||||
|
||||
# API v1 routes (requires API token authentication)
|
||||
scope "/api/v1", ToweropsWeb.Api.V1 do
|
||||
scope "/api/v1", V1 do
|
||||
pipe_through :api_v1
|
||||
|
||||
resources "/sites", SitesController, except: [:new, :edit]
|
||||
resources "/devices", DevicesController, except: [:new, :edit]
|
||||
end
|
||||
|
||||
# Admin API routes (requires superuser authentication via browser session)
|
||||
scope "/api/v1/admin", V1 do
|
||||
pipe_through [:api, :require_authenticated_user, ToweropsWeb.Plugs.RequireSuperuser]
|
||||
|
||||
# Profile management (superuser only)
|
||||
post "/profiles/import", ProfilesController, :import_bulk
|
||||
end
|
||||
|
||||
# WebAuthn API routes
|
||||
scope "/api/webauthn", ToweropsWeb do
|
||||
pipe_through [:browser]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue