support prod ip db import
This commit is contained in:
parent
d1fd1d4a62
commit
f240d429c9
3 changed files with 127 additions and 7 deletions
48
CLAUDE.md
48
CLAUDE.md
|
|
@ -278,6 +278,54 @@ See `vendor/README.md` for update instructions.
|
|||
- All impersonation events are logged to audit_logs table via `Towerops.Admin.create_audit_log/1`
|
||||
- Stop impersonation: "Stop Impersonating" link appears in user menu when impersonating
|
||||
|
||||
### GeoIP Database Management
|
||||
|
||||
**Purpose**: Import MaxMind GeoLite2-City database for IP-based country/city detection (used for GDPR cookie consent).
|
||||
|
||||
**Files**:
|
||||
- Controller: `lib/towerops_web/controllers/api/v1/geoip_controller.ex`
|
||||
- Mix Task: `lib/mix/tasks/geoip.import.ex`
|
||||
- Schemas: `lib/towerops/geoip/location.ex`, `lib/towerops/geoip/block.ex`
|
||||
- Lookup: `lib/towerops/geoip.ex`
|
||||
- Migration: `priv/repo/migrations/*_create_geoip_tables.exs`
|
||||
|
||||
**Database Tables**:
|
||||
- `geoip_locations` (~130,000 rows) - Countries, cities, regions with coordinates
|
||||
- `geoip_blocks` (~3.5M rows) - IP ranges mapped to locations
|
||||
|
||||
**Import Methods**:
|
||||
|
||||
1. **Local Import (Development)**:
|
||||
```bash
|
||||
make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/
|
||||
```
|
||||
|
||||
2. **Production Import via Mix Task**:
|
||||
```bash
|
||||
# Using DATABASE_URL environment variable
|
||||
make geoip-import-prod DIR=~/Downloads/GeoLite2-City-CSV_20260127/ DATABASE_URL=postgresql://user:pass@host/db
|
||||
|
||||
# Or using command-line flag
|
||||
DATABASE_URL=postgresql://user:pass@host/db mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/
|
||||
```
|
||||
|
||||
3. **Production Import via API** (requires superuser API token):
|
||||
```bash
|
||||
curl -X POST https://towerops.net/admin/api/geoip/import \
|
||||
-H "Authorization: Bearer {superuser_api_token}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"directory": "/path/to/GeoLite2-City-CSV_20260127"}'
|
||||
```
|
||||
|
||||
**Implementation Notes**:
|
||||
- API endpoint: `POST /admin/api/geoip/import` (superuser only)
|
||||
- Mix task supports custom DATABASE_URL for importing to any PostgreSQL database
|
||||
- Uses batch inserts of 5,000 rows to stay within PostgreSQL's 65,535 parameter limit
|
||||
- Automatically filters orphaned IP blocks (those without matching location references)
|
||||
- CSV files are NOT committed to git - imported separately in each environment
|
||||
- Download GeoLite2-City CSV from MaxMind: https://dev.maxmind.com/geoip/geolite2-free-geolocation-data
|
||||
- Required files: `GeoLite2-City-Blocks-IPv4.csv`, `GeoLite2-City-Locations-en.csv`
|
||||
|
||||
## Project-Specific Constraints
|
||||
|
||||
Key constraints from AGENTS.md (see that file for complete details):
|
||||
|
|
|
|||
18
Makefile
18
Makefile
|
|
@ -1,4 +1,4 @@
|
|||
.PHONY: help build push deploy restart login clean compile-mibs geoip-import
|
||||
.PHONY: help build push deploy restart login clean compile-mibs geoip-import geoip-import-prod
|
||||
|
||||
# Variables
|
||||
REGISTRY := registry.gitlab.com/towerops/towerops
|
||||
|
|
@ -78,7 +78,7 @@ info: ## Show current image information
|
|||
compile-mibs: ## Pre-compile MIB files to JSON for runtime loading
|
||||
@python3 scripts/compile_mibs.py
|
||||
|
||||
geoip-import: ## Import MaxMind GeoLite2 City CSV database (specify DIR=path/to/csv)
|
||||
geoip-import: ## Import MaxMind GeoLite2 City CSV database to local database (specify DIR=path/to/csv)
|
||||
@if [ -z "$(DIR)" ]; then \
|
||||
echo "Error: Please specify DIR variable"; \
|
||||
echo "Usage: make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/"; \
|
||||
|
|
@ -86,3 +86,17 @@ geoip-import: ## Import MaxMind GeoLite2 City CSV database (specify DIR=path/to/
|
|||
fi
|
||||
@echo "Importing GeoIP database from $(DIR)..."
|
||||
@mix geoip.import $(DIR)
|
||||
|
||||
geoip-import-prod: ## Import MaxMind GeoLite2 City CSV to production (specify DIR=path/to/csv and DATABASE_URL=postgresql://...)
|
||||
@if [ -z "$(DIR)" ]; then \
|
||||
echo "Error: Please specify DIR variable"; \
|
||||
echo "Usage: make geoip-import-prod DIR=~/Downloads/GeoLite2-City-CSV_20260127/ DATABASE_URL=postgresql://..."; \
|
||||
exit 1; \
|
||||
fi
|
||||
@if [ -z "$(DATABASE_URL)" ]; then \
|
||||
echo "Error: Please specify DATABASE_URL variable"; \
|
||||
echo "Usage: make geoip-import-prod DIR=~/Downloads/GeoLite2-City-CSV_20260127/ DATABASE_URL=postgresql://..."; \
|
||||
exit 1; \
|
||||
fi
|
||||
@echo "Importing GeoIP database from $(DIR) to production..."
|
||||
@DATABASE_URL=$(DATABASE_URL) mix geoip.import $(DIR)
|
||||
|
|
|
|||
|
|
@ -46,21 +46,79 @@ defmodule Mix.Tasks.Geoip.Import do
|
|||
|
||||
@impl Mix.Task
|
||||
def run(args) do
|
||||
case args do
|
||||
{opts, remaining_args} = parse_args(args)
|
||||
|
||||
case remaining_args do
|
||||
[directory] ->
|
||||
Mix.Task.run("app.start")
|
||||
import_from_directory(directory)
|
||||
# Start app if using default repo, otherwise start minimal dependencies
|
||||
database_url = opts[:database_url] || System.get_env("DATABASE_URL")
|
||||
|
||||
if database_url do
|
||||
# Start minimal dependencies for custom database connection
|
||||
Mix.Task.run("app.config")
|
||||
{:ok, _} = Application.ensure_all_started(:postgrex)
|
||||
{:ok, _} = Application.ensure_all_started(:ecto_sql)
|
||||
|
||||
# Start dynamic repo with custom database URL
|
||||
import_with_custom_repo(directory, database_url)
|
||||
else
|
||||
# Use default repo (development)
|
||||
Mix.Task.run("app.start")
|
||||
import_from_directory(directory)
|
||||
end
|
||||
|
||||
_ ->
|
||||
Mix.shell().error("""
|
||||
Usage: mix geoip.import <directory>
|
||||
Usage: mix geoip.import <directory> [--database-url URL]
|
||||
|
||||
Example:
|
||||
Examples:
|
||||
# Import to local database
|
||||
mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/
|
||||
|
||||
# Import to production database
|
||||
DATABASE_URL=postgresql://user:pass@host/db mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/
|
||||
|
||||
# Import to production database with flag
|
||||
mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ --database-url postgresql://user:pass@host/db
|
||||
""")
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_args(args) do
|
||||
{opts, remaining, _invalid} =
|
||||
OptionParser.parse(args,
|
||||
strict: [database_url: :string],
|
||||
aliases: [d: :database_url]
|
||||
)
|
||||
|
||||
{opts, remaining}
|
||||
end
|
||||
|
||||
defp import_with_custom_repo(directory, database_url) do
|
||||
Mix.shell().info("Connecting to custom database: #{sanitize_url(database_url)}")
|
||||
|
||||
# Define a dynamic repo module for the import
|
||||
repo_config = [
|
||||
database: nil,
|
||||
url: database_url,
|
||||
pool_size: 2
|
||||
]
|
||||
|
||||
# Start the repo
|
||||
{:ok, _pid} = Repo.start_link(repo_config)
|
||||
|
||||
# Run the import
|
||||
import_from_directory(directory)
|
||||
|
||||
# Stop the repo
|
||||
Supervisor.stop(Repo)
|
||||
end
|
||||
|
||||
defp sanitize_url(url) do
|
||||
# Remove password from URL for logging
|
||||
String.replace(url, ~r/:([^@]+)@/, ":***@")
|
||||
end
|
||||
|
||||
defp import_from_directory(directory) do
|
||||
directory = Path.expand(directory)
|
||||
blocks_file = Path.join(directory, "GeoLite2-City-Blocks-IPv4.csv")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue