graham/towerops
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

support prod ip db import

Browse source
This commit is contained in:
Graham McIntire 2026-01-28 13:20:53 -06:00
parent f240d429c9
commit 7ecc986bcd
5 changed files with 234 additions and 134 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
CLAUDE.md
View file

@ -300,28 +300,26 @@ See `vendor/README.md` for update instructions.
make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/
```
2. **Production Import via Mix Task**:
2. **Production Import via API** (requires superuser API token):
```bash
# Using DATABASE_URL environment variable
make geoip-import-prod DIR=~/Downloads/GeoLite2-City-CSV_20260127/ DATABASE_URL=postgresql://user:pass@host/db
# Using Makefile
make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/ TOWEROPS_KEY=your_superuser_api_key
# Or using command-line flag
DATABASE_URL=postgresql://user:pass@host/db mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/
# Or directly with mix
TOWEROPS_KEY=your_superuser_api_key mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/
```
3. **Production Import via API** (requires superuser API token):
```bash
curl -X POST https://towerops.net/admin/api/geoip/import \
-H "Authorization: Bearer {superuser_api_token}" \
-H "Content-Type: application/json" \
-d '{"directory": "/path/to/GeoLite2-City-CSV_20260127"}'
```
**How It Works**:
- Development (no `TOWEROPS_KEY`): Direct database import via Ecto
- Production (`TOWEROPS_KEY` set): Processes CSVs locally, sends data to API in batches of 5,000 records
- API endpoint: `POST /admin/api/geoip/import` (superuser only)
- No need to upload large CSV files to production - only processed data is sent over HTTPS
- Default API URL: `https://towerops.net` (override with `TOWEROPS_URL` env var)
**Implementation Notes**:
- API endpoint: `POST /admin/api/geoip/import` (superuser only)
- Mix task supports custom DATABASE_URL for importing to any PostgreSQL database
- Uses batch inserts of 5,000 rows to stay within PostgreSQL's 65,535 parameter limit
- Automatically filters orphaned IP blocks (those without matching location references)
- Mix task processes CSVs locally and sends batches via API when `TOWEROPS_KEY` is set
- CSV files are NOT committed to git - imported separately in each environment
- Download GeoLite2-City CSV from MaxMind: https://dev.maxmind.com/geoip/geolite2-free-geolocation-data
- Required files: `GeoLite2-City-Blocks-IPv4.csv`, `GeoLite2-City-Locations-en.csv`

32
Makefile
View file

@ -1,4 +1,4 @@
.PHONY: help build push deploy restart login clean compile-mibs geoip-import geoip-import-prod
.PHONY: help build push deploy restart login clean compile-mibs geoip-import
# Variables
REGISTRY := registry.gitlab.com/towerops/towerops
@ -78,25 +78,21 @@ info: ## Show current image information
compile-mibs: ## Pre-compile MIB files to JSON for runtime loading
@python3 scripts/compile_mibs.py
geoip-import: ## Import MaxMind GeoLite2 City CSV database to local database (specify DIR=path/to/csv)
geoip-import: ## Import MaxMind GeoLite2 City CSV database (specify DIR=path/to/csv, optionally TOWEROPS_KEY for production)
@if [ -z "$(DIR)" ]; then \
echo "Error: Please specify DIR variable"; \
echo "Usage: make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/"; \
echo ""; \
echo "Usage (local):"; \
echo " make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/"; \
echo ""; \
echo "Usage (production via API):"; \
echo " make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/ TOWEROPS_KEY=your_superuser_api_key"; \
exit 1; \
fi
@echo "Importing GeoIP database from $(DIR)..."
@mix geoip.import $(DIR)
geoip-import-prod: ## Import MaxMind GeoLite2 City CSV to production (specify DIR=path/to/csv and DATABASE_URL=postgresql://...)
@if [ -z "$(DIR)" ]; then \
echo "Error: Please specify DIR variable"; \
echo "Usage: make geoip-import-prod DIR=~/Downloads/GeoLite2-City-CSV_20260127/ DATABASE_URL=postgresql://..."; \
exit 1; \
@if [ -n "$(TOWEROPS_KEY)" ]; then \
echo "Importing GeoIP database to production via API..."; \
TOWEROPS_KEY=$(TOWEROPS_KEY) mix geoip.import $(DIR); \
else \
echo "Importing GeoIP database to local database..."; \
mix geoip.import $(DIR); \
fi
@if [ -z "$(DATABASE_URL)" ]; then \
echo "Error: Please specify DATABASE_URL variable"; \
echo "Usage: make geoip-import-prod DIR=~/Downloads/GeoLite2-City-CSV_20260127/ DATABASE_URL=postgresql://..."; \
exit 1; \
fi
@echo "Importing GeoIP database from $(DIR) to production..."
@DATABASE_URL=$(DATABASE_URL) mix geoip.import $(DIR)

149
lib/mix/tasks/geoip.import.ex
View file

@ -46,40 +46,35 @@ defmodule Mix.Tasks.Geoip.Import do
@impl Mix.Task
def run(args) do
{opts, remaining_args} = parse_args(args)
{_opts, remaining_args} = parse_args(args)
case remaining_args do
[directory] ->
# Start app if using default repo, otherwise start minimal dependencies
database_url = opts[:database_url] || System.get_env("DATABASE_URL")
# Check if TOWEROPS_KEY is set (production API import)
api_key = System.get_env("TOWEROPS_KEY")
if database_url do
# Start minimal dependencies for custom database connection
if api_key do
# Production: Send batches to API
Mix.Task.run("app.config")
{:ok, _} = Application.ensure_all_started(:postgrex)
{:ok, _} = Application.ensure_all_started(:ecto_sql)
{:ok, _} = Application.ensure_all_started(:req)
# Start dynamic repo with custom database URL
import_with_custom_repo(directory, database_url)
import_via_api(directory, api_key)
else
# Use default repo (development)
# Development: Direct database import
Mix.Task.run("app.start")
import_from_directory(directory)
end
_ ->
Mix.shell().error("""
Usage: mix geoip.import <directory> [--database-url URL]
Usage: mix geoip.import <directory>
Examples:
# Import to local database
# Import to local database (development)
mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/
# Import to production database
DATABASE_URL=postgresql://user:pass@host/db mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/
# Import to production database with flag
mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ --database-url postgresql://user:pass@host/db
# Import to production via API
TOWEROPS_KEY=your_superuser_api_key mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/
""")
end
end
@ -94,29 +89,115 @@ defmodule Mix.Tasks.Geoip.Import do
{opts, remaining}
end
defp import_with_custom_repo(directory, database_url) do
Mix.shell().info("Connecting to custom database: #{sanitize_url(database_url)}")
defp import_via_api(directory, api_key) do
directory = Path.expand(directory)
blocks_file = Path.join(directory, "GeoLite2-City-Blocks-IPv4.csv")
locations_file = Path.join(directory, "GeoLite2-City-Locations-en.csv")
# Define a dynamic repo module for the import
repo_config = [
database: nil,
url: database_url,
pool_size: 2
]
# Determine API URL (default to production)
api_url = System.get_env("TOWEROPS_URL", "https://towerops.net")
# Start the repo
{:ok, _pid} = Repo.start_link(repo_config)
Mix.shell().info("Importing GeoIP database to #{api_url} via API...")
# Run the import
import_from_directory(directory)
# Parse and send locations first
Mix.shell().info(" Processing locations file...")
{locations_count, valid_geoname_ids} = send_locations_to_api(locations_file, api_url, api_key)
Mix.shell().info(" Sent #{locations_count} locations")
# Stop the repo
Supervisor.stop(Repo)
# Parse and send blocks
Mix.shell().info(" Processing blocks file (this may take several minutes)...")
blocks_count = send_blocks_to_api(blocks_file, valid_geoname_ids, api_url, api_key)
Mix.shell().info(" Sent #{blocks_count} IP blocks")
Mix.shell().info("✓ GeoIP database imported successfully to production!")
Mix.shell().info(" #{locations_count} locations")
Mix.shell().info(" #{blocks_count} IP blocks")
end
defp sanitize_url(url) do
# Remove password from URL for logging
String.replace(url, ~r/:([^@]+)@/, ":***@")
defp send_locations_to_api(file, api_url, api_key) do
file
|> File.stream!()
|> Stream.drop(1)
|> Stream.map(&parse_location_line/1)
|> Stream.reject(&is_nil/1)
|> Stream.chunk_every(@batch_size)
|> Enum.reduce({0, MapSet.new()}, fn batch, {count, geoname_ids} ->
# Convert batch to JSON-serializable format
data =
Enum.map(batch, fn location ->
%{
"geoname_id" => location.geoname_id,
"country_code" => location.country_code,
"country_name" => location.country_name,
"city_name" => location.city_name,
"subdivision_1_name" => location.subdivision_1_name,
"subdivision_2_name" => location.subdivision_2_name,
"latitude" => location.latitude,
"longitude" => location.longitude
}
end)
# Send to API
truncate = count == 0
send_batch_to_api(api_url, api_key, "locations", data, truncate)
# Track geoname_ids
new_ids = MapSet.new(batch, & &1.geoname_id)
{count + length(batch), MapSet.union(geoname_ids, new_ids)}
end)
end
defp send_blocks_to_api(file, valid_geoname_ids, api_url, api_key) do
file
|> File.stream!()
|> Stream.drop(1)
|> Stream.map(&parse_block_line/1)
|> Stream.reject(&is_nil/1)
|> Stream.filter(fn block -> MapSet.member?(valid_geoname_ids, block.geoname_id) end)
|> Stream.chunk_every(@batch_size)
|> Enum.reduce(0, fn batch, count ->
# Convert batch to JSON-serializable format
data =
Enum.map(batch, fn block ->
%{
"network" => block.network,
"start_ip_int" => block.start_ip_int,
"end_ip_int" => block.end_ip_int,
"geoname_id" => block.geoname_id,
"registered_country_geoname_id" => block.registered_country_geoname_id
}
end)
# Send to API
truncate = count == 0
send_batch_to_api(api_url, api_key, "blocks", data, truncate)
count + length(batch)
end)
end
defp send_batch_to_api(api_url, api_key, batch_type, data, truncate) do
url = "#{api_url}/admin/api/geoip/import"
body = %{
"batch_type" => batch_type,
"data" => data,
"truncate" => truncate
}
response =
Req.post!(url,
json: body,
headers: [{"authorization", "Bearer #{api_key}"}],
retry: :transient
)
if response.status != 200 do
Mix.shell().error("API request failed: #{inspect(response.body)}")
raise "Failed to send batch to API"
end
response.body
end
defp import_from_directory(directory) do

150
lib/towerops_web/controllers/api/v1/geoip_controller.ex
View file

@ -9,76 +9,55 @@ defmodule ToweropsWeb.Api.V1.GeoipController do
use ToweropsWeb, :controller
alias Mix.Tasks.Geoip.Import
require Logger
@doc """
Import GeoIP database from MaxMind GeoLite2-City CSV files.
Import GeoIP database batch from processed CSV data.
Requires superuser API token.
## Request
POST /api/v1/geoip/import
POST /admin/api/geoip/import
Content-Type: application/json
{
"directory": "/path/to/GeoLite2-City-CSV_20260127"
"batch_type": "locations", // or "blocks"
"data": [...], // Array of location or block records
"truncate": true // Optional: truncate table before inserting (first batch only)
}
## Response
202 Accepted
{
"status": "ok",
"message": "GeoIP import started in background"
}
Or:
200 OK
{
"status": "ok",
"message": "GeoIP database imported successfully",
"locations_count": 130000,
"blocks_count": 3500000
"inserted": 5000
}
"""
def import_database(conn, %{"directory" => directory}) do
def import_database(conn, %{"batch_type" => batch_type, "data" => data} = params) do
with :ok <- require_superuser(conn) do
Logger.info("GeoIP import requested by #{conn.assigns[:current_user].email} for directory: #{directory}")
truncate = Map.get(params, "truncate", false)
# Expand the path to handle ~ and relative paths
expanded_directory = Path.expand(directory)
Logger.info(
"GeoIP import batch requested by #{conn.assigns[:current_user].email}: #{batch_type}, #{length(data)} records, truncate=#{truncate}"
)
# Verify directory exists
if File.dir?(expanded_directory) do
# Run import synchronously (it's fast enough with batching)
case run_import(expanded_directory) do
{:ok, locations_count, blocks_count} ->
Logger.info("GeoIP import completed: #{locations_count} locations, #{blocks_count} blocks")
case import_batch(batch_type, data, truncate) do
{:ok, inserted_count} ->
conn
|> put_status(:ok)
|> json(%{
status: "ok",
inserted: inserted_count
})
conn
|> put_status(:ok)
|> json(%{
status: "ok",
message: "GeoIP database imported successfully",
locations_count: locations_count,
blocks_count: blocks_count
})
{:error, reason} ->
Logger.error("GeoIP batch import failed: #{inspect(reason)}")
{:error, reason} ->
Logger.error("GeoIP import failed: #{inspect(reason)}")
conn
|> put_status(:internal_server_error)
|> json(%{error: "Import failed: #{inspect(reason)}"})
end
else
conn
|> put_status(:bad_request)
|> json(%{error: "Directory does not exist: #{directory}"})
conn
|> put_status(:internal_server_error)
|> json(%{error: "Import failed: #{inspect(reason)}"})
end
end
end
@ -86,7 +65,9 @@ defmodule ToweropsWeb.Api.V1.GeoipController do
def import_database(conn, _params) do
conn
|> put_status(:bad_request)
|> json(%{error: "Missing required parameter: directory"})
|> json(%{
error: "Missing required parameters: batch_type and data"
})
end
defp require_superuser(conn) do
@ -106,32 +87,71 @@ defmodule ToweropsWeb.Api.V1.GeoipController do
end
end
defp run_import(directory) do
alias Towerops.GeoIP.Block
defp import_batch("locations", data, truncate) do
alias Towerops.GeoIP.Location
alias Towerops.Repo
blocks_file = Path.join(directory, "GeoLite2-City-Blocks-IPv4.csv")
locations_file = Path.join(directory, "GeoLite2-City-Locations-en.csv")
# Verify files exist
if File.exists?(blocks_file) && File.exists?(locations_file) do
# Truncate existing data
Repo.delete_all(Block)
if truncate do
Repo.delete_all(Location)
# Load locations
{locations_count, valid_geoname_ids} = Import.load_locations(locations_file)
# Load blocks
blocks_count = Import.load_blocks(blocks_file, valid_geoname_ids)
{:ok, locations_count, blocks_count}
else
{:error, "Required CSV files not found in directory"}
end
now = DateTime.truncate(DateTime.utc_now(), :second)
entries =
Enum.map(data, fn location ->
%{
geoname_id: location["geoname_id"],
country_code: location["country_code"],
country_name: location["country_name"],
city_name: location["city_name"],
subdivision_1_name: location["subdivision_1_name"],
subdivision_2_name: location["subdivision_2_name"],
latitude: location["latitude"],
longitude: location["longitude"],
inserted_at: now,
updated_at: now
}
end)
{inserted, _} = Repo.insert_all(Location, entries)
{:ok, inserted}
rescue
e ->
{:error, Exception.message(e)}
end
defp import_batch("blocks", data, truncate) do
alias Towerops.GeoIP.Block
alias Towerops.Repo
if truncate do
Repo.delete_all(Block)
end
now = DateTime.truncate(DateTime.utc_now(), :second)
entries =
Enum.map(data, fn block ->
%{
id: Ecto.UUID.generate(),
network: block["network"],
start_ip_int: block["start_ip_int"],
end_ip_int: block["end_ip_int"],
geoname_id: block["geoname_id"],
registered_country_geoname_id: block["registered_country_geoname_id"],
inserted_at: now,
updated_at: now
}
end)
{inserted, _} = Repo.insert_all(Block, entries)
{:ok, inserted}
rescue
e ->
{:error, Exception.message(e)}
end
defp import_batch(_type, _data, _truncate) do
{:error, "Invalid batch_type. Must be 'locations' or 'blocks'"}
end
end

11
lib/towerops_web/user_auth.ex
View file

@ -321,18 +321,23 @@ defmodule ToweropsWeb.UserAuth do
if there's no existing return path to avoid overwriting.
"""
def store_return_to_for_liveview(conn, _opts) do
# Skip authentication-related paths (login, register, reset password, confirm)
# Skip authentication-related and public paths
skip_paths = [
"/users/log-in",
"/users/register",
"/users/reset-password",
"/users/confirm"
"/users/confirm",
"/docs/api",
"/privacy",
"/terms",
"/help"
]
should_store =
conn.method == "GET" &&
is_nil(get_session(conn, :user_return_to)) &&
!Enum.any?(skip_paths, &String.starts_with?(conn.request_path, &1))
!Enum.any?(skip_paths, &String.starts_with?(conn.request_path, &1)) &&
conn.request_path != "/"
if should_store do
put_session(conn, :user_return_to, current_path(conn))