diff --git a/CLAUDE.md b/CLAUDE.md index e5492a5a..2ec64305 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -278,6 +278,54 @@ See `vendor/README.md` for update instructions. - All impersonation events are logged to audit_logs table via `Towerops.Admin.create_audit_log/1` - Stop impersonation: "Stop Impersonating" link appears in user menu when impersonating +### GeoIP Database Management + +**Purpose**: Import MaxMind GeoLite2-City database for IP-based country/city detection (used for GDPR cookie consent). + +**Files**: +- Controller: `lib/towerops_web/controllers/api/v1/geoip_controller.ex` +- Mix Task: `lib/mix/tasks/geoip.import.ex` +- Schemas: `lib/towerops/geoip/location.ex`, `lib/towerops/geoip/block.ex` +- Lookup: `lib/towerops/geoip.ex` +- Migration: `priv/repo/migrations/*_create_geoip_tables.exs` + +**Database Tables**: +- `geoip_locations` (~130,000 rows) - Countries, cities, regions with coordinates +- `geoip_blocks` (~3.5M rows) - IP ranges mapped to locations + +**Import Methods**: + +1. **Local Import (Development)**: +```bash +make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/ +``` + +2. **Production Import via Mix Task**: +```bash +# Using DATABASE_URL environment variable +make geoip-import-prod DIR=~/Downloads/GeoLite2-City-CSV_20260127/ DATABASE_URL=postgresql://user:pass@host/db + +# Or using command-line flag +DATABASE_URL=postgresql://user:pass@host/db mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ +``` + +3. **Production Import via API** (requires superuser API token): +```bash +curl -X POST https://towerops.net/admin/api/geoip/import \ + -H "Authorization: Bearer {superuser_api_token}" \ + -H "Content-Type: application/json" \ + -d '{"directory": "/path/to/GeoLite2-City-CSV_20260127"}' +``` + +**Implementation Notes**: +- API endpoint: `POST /admin/api/geoip/import` (superuser only) +- Mix task supports custom DATABASE_URL for importing to any PostgreSQL database +- Uses batch inserts of 5,000 rows to stay within PostgreSQL's 65,535 parameter limit +- Automatically filters orphaned IP blocks (those without matching location references) +- CSV files are NOT committed to git - imported separately in each environment +- Download GeoLite2-City CSV from MaxMind: https://dev.maxmind.com/geoip/geolite2-free-geolocation-data +- Required files: `GeoLite2-City-Blocks-IPv4.csv`, `GeoLite2-City-Locations-en.csv` + ## Project-Specific Constraints Key constraints from AGENTS.md (see that file for complete details): diff --git a/Makefile b/Makefile index a3c2c2f7..295dbb21 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help build push deploy restart login clean compile-mibs geoip-import +.PHONY: help build push deploy restart login clean compile-mibs geoip-import geoip-import-prod # Variables REGISTRY := registry.gitlab.com/towerops/towerops @@ -78,7 +78,7 @@ info: ## Show current image information compile-mibs: ## Pre-compile MIB files to JSON for runtime loading @python3 scripts/compile_mibs.py -geoip-import: ## Import MaxMind GeoLite2 City CSV database (specify DIR=path/to/csv) +geoip-import: ## Import MaxMind GeoLite2 City CSV database to local database (specify DIR=path/to/csv) @if [ -z "$(DIR)" ]; then \ echo "Error: Please specify DIR variable"; \ echo "Usage: make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/"; \ @@ -86,3 +86,17 @@ geoip-import: ## Import MaxMind GeoLite2 City CSV database (specify DIR=path/to/ fi @echo "Importing GeoIP database from $(DIR)..." @mix geoip.import $(DIR) + +geoip-import-prod: ## Import MaxMind GeoLite2 City CSV to production (specify DIR=path/to/csv and DATABASE_URL=postgresql://...) + @if [ -z "$(DIR)" ]; then \ + echo "Error: Please specify DIR variable"; \ + echo "Usage: make geoip-import-prod DIR=~/Downloads/GeoLite2-City-CSV_20260127/ DATABASE_URL=postgresql://..."; \ + exit 1; \ + fi + @if [ -z "$(DATABASE_URL)" ]; then \ + echo "Error: Please specify DATABASE_URL variable"; \ + echo "Usage: make geoip-import-prod DIR=~/Downloads/GeoLite2-City-CSV_20260127/ DATABASE_URL=postgresql://..."; \ + exit 1; \ + fi + @echo "Importing GeoIP database from $(DIR) to production..." + @DATABASE_URL=$(DATABASE_URL) mix geoip.import $(DIR) diff --git a/lib/mix/tasks/geoip.import.ex b/lib/mix/tasks/geoip.import.ex index d00a5b4e..bfe056cb 100644 --- a/lib/mix/tasks/geoip.import.ex +++ b/lib/mix/tasks/geoip.import.ex @@ -46,21 +46,79 @@ defmodule Mix.Tasks.Geoip.Import do @impl Mix.Task def run(args) do - case args do + {opts, remaining_args} = parse_args(args) + + case remaining_args do [directory] -> - Mix.Task.run("app.start") - import_from_directory(directory) + # Start app if using default repo, otherwise start minimal dependencies + database_url = opts[:database_url] || System.get_env("DATABASE_URL") + + if database_url do + # Start minimal dependencies for custom database connection + Mix.Task.run("app.config") + {:ok, _} = Application.ensure_all_started(:postgrex) + {:ok, _} = Application.ensure_all_started(:ecto_sql) + + # Start dynamic repo with custom database URL + import_with_custom_repo(directory, database_url) + else + # Use default repo (development) + Mix.Task.run("app.start") + import_from_directory(directory) + end _ -> Mix.shell().error(""" - Usage: mix geoip.import + Usage: mix geoip.import [--database-url URL] - Example: + Examples: + # Import to local database mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ + + # Import to production database + DATABASE_URL=postgresql://user:pass@host/db mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ + + # Import to production database with flag + mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ --database-url postgresql://user:pass@host/db """) end end + defp parse_args(args) do + {opts, remaining, _invalid} = + OptionParser.parse(args, + strict: [database_url: :string], + aliases: [d: :database_url] + ) + + {opts, remaining} + end + + defp import_with_custom_repo(directory, database_url) do + Mix.shell().info("Connecting to custom database: #{sanitize_url(database_url)}") + + # Define a dynamic repo module for the import + repo_config = [ + database: nil, + url: database_url, + pool_size: 2 + ] + + # Start the repo + {:ok, _pid} = Repo.start_link(repo_config) + + # Run the import + import_from_directory(directory) + + # Stop the repo + Supervisor.stop(Repo) + end + + defp sanitize_url(url) do + # Remove password from URL for logging + String.replace(url, ~r/:([^@]+)@/, ":***@") + end + defp import_from_directory(directory) do directory = Path.expand(directory) blocks_file = Path.join(directory, "GeoLite2-City-Blocks-IPv4.csv")