diff --git a/CLAUDE.md b/CLAUDE.md index 2ec64305..a532f2fd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -300,28 +300,26 @@ See `vendor/README.md` for update instructions. make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/ ``` -2. **Production Import via Mix Task**: +2. **Production Import via API** (requires superuser API token): ```bash -# Using DATABASE_URL environment variable -make geoip-import-prod DIR=~/Downloads/GeoLite2-City-CSV_20260127/ DATABASE_URL=postgresql://user:pass@host/db +# Using Makefile +make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/ TOWEROPS_KEY=your_superuser_api_key -# Or using command-line flag -DATABASE_URL=postgresql://user:pass@host/db mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ +# Or directly with mix +TOWEROPS_KEY=your_superuser_api_key mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ ``` -3. **Production Import via API** (requires superuser API token): -```bash -curl -X POST https://towerops.net/admin/api/geoip/import \ - -H "Authorization: Bearer {superuser_api_token}" \ - -H "Content-Type: application/json" \ - -d '{"directory": "/path/to/GeoLite2-City-CSV_20260127"}' -``` +**How It Works**: +- Development (no `TOWEROPS_KEY`): Direct database import via Ecto +- Production (`TOWEROPS_KEY` set): Processes CSVs locally, sends data to API in batches of 5,000 records +- API endpoint: `POST /admin/api/geoip/import` (superuser only) +- No need to upload large CSV files to production - only processed data is sent over HTTPS +- Default API URL: `https://towerops.net` (override with `TOWEROPS_URL` env var) **Implementation Notes**: -- API endpoint: `POST /admin/api/geoip/import` (superuser only) -- Mix task supports custom DATABASE_URL for importing to any PostgreSQL database - Uses batch inserts of 5,000 rows to stay within PostgreSQL's 65,535 parameter limit - Automatically filters orphaned IP blocks (those without matching location references) +- Mix task processes CSVs locally and sends batches via API when `TOWEROPS_KEY` is set - CSV files are NOT committed to git - imported separately in each environment - Download GeoLite2-City CSV from MaxMind: https://dev.maxmind.com/geoip/geolite2-free-geolocation-data - Required files: `GeoLite2-City-Blocks-IPv4.csv`, `GeoLite2-City-Locations-en.csv` diff --git a/Makefile b/Makefile index 295dbb21..3b667d90 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help build push deploy restart login clean compile-mibs geoip-import geoip-import-prod +.PHONY: help build push deploy restart login clean compile-mibs geoip-import # Variables REGISTRY := registry.gitlab.com/towerops/towerops @@ -78,25 +78,21 @@ info: ## Show current image information compile-mibs: ## Pre-compile MIB files to JSON for runtime loading @python3 scripts/compile_mibs.py -geoip-import: ## Import MaxMind GeoLite2 City CSV database to local database (specify DIR=path/to/csv) +geoip-import: ## Import MaxMind GeoLite2 City CSV database (specify DIR=path/to/csv, optionally TOWEROPS_KEY for production) @if [ -z "$(DIR)" ]; then \ echo "Error: Please specify DIR variable"; \ - echo "Usage: make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/"; \ + echo ""; \ + echo "Usage (local):"; \ + echo " make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/"; \ + echo ""; \ + echo "Usage (production via API):"; \ + echo " make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/ TOWEROPS_KEY=your_superuser_api_key"; \ exit 1; \ fi - @echo "Importing GeoIP database from $(DIR)..." - @mix geoip.import $(DIR) - -geoip-import-prod: ## Import MaxMind GeoLite2 City CSV to production (specify DIR=path/to/csv and DATABASE_URL=postgresql://...) - @if [ -z "$(DIR)" ]; then \ - echo "Error: Please specify DIR variable"; \ - echo "Usage: make geoip-import-prod DIR=~/Downloads/GeoLite2-City-CSV_20260127/ DATABASE_URL=postgresql://..."; \ - exit 1; \ + @if [ -n "$(TOWEROPS_KEY)" ]; then \ + echo "Importing GeoIP database to production via API..."; \ + TOWEROPS_KEY=$(TOWEROPS_KEY) mix geoip.import $(DIR); \ + else \ + echo "Importing GeoIP database to local database..."; \ + mix geoip.import $(DIR); \ fi - @if [ -z "$(DATABASE_URL)" ]; then \ - echo "Error: Please specify DATABASE_URL variable"; \ - echo "Usage: make geoip-import-prod DIR=~/Downloads/GeoLite2-City-CSV_20260127/ DATABASE_URL=postgresql://..."; \ - exit 1; \ - fi - @echo "Importing GeoIP database from $(DIR) to production..." - @DATABASE_URL=$(DATABASE_URL) mix geoip.import $(DIR) diff --git a/lib/mix/tasks/geoip.import.ex b/lib/mix/tasks/geoip.import.ex index bfe056cb..b96ad0a2 100644 --- a/lib/mix/tasks/geoip.import.ex +++ b/lib/mix/tasks/geoip.import.ex @@ -46,40 +46,35 @@ defmodule Mix.Tasks.Geoip.Import do @impl Mix.Task def run(args) do - {opts, remaining_args} = parse_args(args) + {_opts, remaining_args} = parse_args(args) case remaining_args do [directory] -> - # Start app if using default repo, otherwise start minimal dependencies - database_url = opts[:database_url] || System.get_env("DATABASE_URL") + # Check if TOWEROPS_KEY is set (production API import) + api_key = System.get_env("TOWEROPS_KEY") - if database_url do - # Start minimal dependencies for custom database connection + if api_key do + # Production: Send batches to API Mix.Task.run("app.config") - {:ok, _} = Application.ensure_all_started(:postgrex) - {:ok, _} = Application.ensure_all_started(:ecto_sql) + {:ok, _} = Application.ensure_all_started(:req) - # Start dynamic repo with custom database URL - import_with_custom_repo(directory, database_url) + import_via_api(directory, api_key) else - # Use default repo (development) + # Development: Direct database import Mix.Task.run("app.start") import_from_directory(directory) end _ -> Mix.shell().error(""" - Usage: mix geoip.import [--database-url URL] + Usage: mix geoip.import Examples: - # Import to local database + # Import to local database (development) mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ - # Import to production database - DATABASE_URL=postgresql://user:pass@host/db mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ - - # Import to production database with flag - mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ --database-url postgresql://user:pass@host/db + # Import to production via API + TOWEROPS_KEY=your_superuser_api_key mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ """) end end @@ -94,29 +89,115 @@ defmodule Mix.Tasks.Geoip.Import do {opts, remaining} end - defp import_with_custom_repo(directory, database_url) do - Mix.shell().info("Connecting to custom database: #{sanitize_url(database_url)}") + defp import_via_api(directory, api_key) do + directory = Path.expand(directory) + blocks_file = Path.join(directory, "GeoLite2-City-Blocks-IPv4.csv") + locations_file = Path.join(directory, "GeoLite2-City-Locations-en.csv") - # Define a dynamic repo module for the import - repo_config = [ - database: nil, - url: database_url, - pool_size: 2 - ] + # Determine API URL (default to production) + api_url = System.get_env("TOWEROPS_URL", "https://towerops.net") - # Start the repo - {:ok, _pid} = Repo.start_link(repo_config) + Mix.shell().info("Importing GeoIP database to #{api_url} via API...") - # Run the import - import_from_directory(directory) + # Parse and send locations first + Mix.shell().info(" Processing locations file...") + {locations_count, valid_geoname_ids} = send_locations_to_api(locations_file, api_url, api_key) + Mix.shell().info(" Sent #{locations_count} locations") - # Stop the repo - Supervisor.stop(Repo) + # Parse and send blocks + Mix.shell().info(" Processing blocks file (this may take several minutes)...") + blocks_count = send_blocks_to_api(blocks_file, valid_geoname_ids, api_url, api_key) + Mix.shell().info(" Sent #{blocks_count} IP blocks") + + Mix.shell().info("✓ GeoIP database imported successfully to production!") + Mix.shell().info(" #{locations_count} locations") + Mix.shell().info(" #{blocks_count} IP blocks") end - defp sanitize_url(url) do - # Remove password from URL for logging - String.replace(url, ~r/:([^@]+)@/, ":***@") + defp send_locations_to_api(file, api_url, api_key) do + file + |> File.stream!() + |> Stream.drop(1) + |> Stream.map(&parse_location_line/1) + |> Stream.reject(&is_nil/1) + |> Stream.chunk_every(@batch_size) + |> Enum.reduce({0, MapSet.new()}, fn batch, {count, geoname_ids} -> + # Convert batch to JSON-serializable format + data = + Enum.map(batch, fn location -> + %{ + "geoname_id" => location.geoname_id, + "country_code" => location.country_code, + "country_name" => location.country_name, + "city_name" => location.city_name, + "subdivision_1_name" => location.subdivision_1_name, + "subdivision_2_name" => location.subdivision_2_name, + "latitude" => location.latitude, + "longitude" => location.longitude + } + end) + + # Send to API + truncate = count == 0 + send_batch_to_api(api_url, api_key, "locations", data, truncate) + + # Track geoname_ids + new_ids = MapSet.new(batch, & &1.geoname_id) + {count + length(batch), MapSet.union(geoname_ids, new_ids)} + end) + end + + defp send_blocks_to_api(file, valid_geoname_ids, api_url, api_key) do + file + |> File.stream!() + |> Stream.drop(1) + |> Stream.map(&parse_block_line/1) + |> Stream.reject(&is_nil/1) + |> Stream.filter(fn block -> MapSet.member?(valid_geoname_ids, block.geoname_id) end) + |> Stream.chunk_every(@batch_size) + |> Enum.reduce(0, fn batch, count -> + # Convert batch to JSON-serializable format + data = + Enum.map(batch, fn block -> + %{ + "network" => block.network, + "start_ip_int" => block.start_ip_int, + "end_ip_int" => block.end_ip_int, + "geoname_id" => block.geoname_id, + "registered_country_geoname_id" => block.registered_country_geoname_id + } + end) + + # Send to API + truncate = count == 0 + send_batch_to_api(api_url, api_key, "blocks", data, truncate) + + count + length(batch) + end) + end + + defp send_batch_to_api(api_url, api_key, batch_type, data, truncate) do + url = "#{api_url}/admin/api/geoip/import" + + body = %{ + "batch_type" => batch_type, + "data" => data, + "truncate" => truncate + } + + response = + Req.post!(url, + json: body, + headers: [{"authorization", "Bearer #{api_key}"}], + retry: :transient + ) + + if response.status != 200 do + Mix.shell().error("API request failed: #{inspect(response.body)}") + raise "Failed to send batch to API" + end + + response.body end defp import_from_directory(directory) do diff --git a/lib/towerops_web/controllers/api/v1/geoip_controller.ex b/lib/towerops_web/controllers/api/v1/geoip_controller.ex index 420c262e..98ef9201 100644 --- a/lib/towerops_web/controllers/api/v1/geoip_controller.ex +++ b/lib/towerops_web/controllers/api/v1/geoip_controller.ex @@ -9,76 +9,55 @@ defmodule ToweropsWeb.Api.V1.GeoipController do use ToweropsWeb, :controller - alias Mix.Tasks.Geoip.Import - require Logger @doc """ - Import GeoIP database from MaxMind GeoLite2-City CSV files. + Import GeoIP database batch from processed CSV data. Requires superuser API token. ## Request - POST /api/v1/geoip/import + POST /admin/api/geoip/import Content-Type: application/json { - "directory": "/path/to/GeoLite2-City-CSV_20260127" + "batch_type": "locations", // or "blocks" + "data": [...], // Array of location or block records + "truncate": true // Optional: truncate table before inserting (first batch only) } ## Response - 202 Accepted - { - "status": "ok", - "message": "GeoIP import started in background" - } - - Or: - 200 OK { "status": "ok", - "message": "GeoIP database imported successfully", - "locations_count": 130000, - "blocks_count": 3500000 + "inserted": 5000 } """ - def import_database(conn, %{"directory" => directory}) do + def import_database(conn, %{"batch_type" => batch_type, "data" => data} = params) do with :ok <- require_superuser(conn) do - Logger.info("GeoIP import requested by #{conn.assigns[:current_user].email} for directory: #{directory}") + truncate = Map.get(params, "truncate", false) - # Expand the path to handle ~ and relative paths - expanded_directory = Path.expand(directory) + Logger.info( + "GeoIP import batch requested by #{conn.assigns[:current_user].email}: #{batch_type}, #{length(data)} records, truncate=#{truncate}" + ) - # Verify directory exists - if File.dir?(expanded_directory) do - # Run import synchronously (it's fast enough with batching) - case run_import(expanded_directory) do - {:ok, locations_count, blocks_count} -> - Logger.info("GeoIP import completed: #{locations_count} locations, #{blocks_count} blocks") + case import_batch(batch_type, data, truncate) do + {:ok, inserted_count} -> + conn + |> put_status(:ok) + |> json(%{ + status: "ok", + inserted: inserted_count + }) - conn - |> put_status(:ok) - |> json(%{ - status: "ok", - message: "GeoIP database imported successfully", - locations_count: locations_count, - blocks_count: blocks_count - }) + {:error, reason} -> + Logger.error("GeoIP batch import failed: #{inspect(reason)}") - {:error, reason} -> - Logger.error("GeoIP import failed: #{inspect(reason)}") - - conn - |> put_status(:internal_server_error) - |> json(%{error: "Import failed: #{inspect(reason)}"}) - end - else - conn - |> put_status(:bad_request) - |> json(%{error: "Directory does not exist: #{directory}"}) + conn + |> put_status(:internal_server_error) + |> json(%{error: "Import failed: #{inspect(reason)}"}) end end end @@ -86,7 +65,9 @@ defmodule ToweropsWeb.Api.V1.GeoipController do def import_database(conn, _params) do conn |> put_status(:bad_request) - |> json(%{error: "Missing required parameter: directory"}) + |> json(%{ + error: "Missing required parameters: batch_type and data" + }) end defp require_superuser(conn) do @@ -106,32 +87,71 @@ defmodule ToweropsWeb.Api.V1.GeoipController do end end - defp run_import(directory) do - alias Towerops.GeoIP.Block + defp import_batch("locations", data, truncate) do alias Towerops.GeoIP.Location alias Towerops.Repo - blocks_file = Path.join(directory, "GeoLite2-City-Blocks-IPv4.csv") - locations_file = Path.join(directory, "GeoLite2-City-Locations-en.csv") - - # Verify files exist - if File.exists?(blocks_file) && File.exists?(locations_file) do - # Truncate existing data - Repo.delete_all(Block) + if truncate do Repo.delete_all(Location) - - # Load locations - {locations_count, valid_geoname_ids} = Import.load_locations(locations_file) - - # Load blocks - blocks_count = Import.load_blocks(blocks_file, valid_geoname_ids) - - {:ok, locations_count, blocks_count} - else - {:error, "Required CSV files not found in directory"} end + + now = DateTime.truncate(DateTime.utc_now(), :second) + + entries = + Enum.map(data, fn location -> + %{ + geoname_id: location["geoname_id"], + country_code: location["country_code"], + country_name: location["country_name"], + city_name: location["city_name"], + subdivision_1_name: location["subdivision_1_name"], + subdivision_2_name: location["subdivision_2_name"], + latitude: location["latitude"], + longitude: location["longitude"], + inserted_at: now, + updated_at: now + } + end) + + {inserted, _} = Repo.insert_all(Location, entries) + {:ok, inserted} rescue e -> {:error, Exception.message(e)} end + + defp import_batch("blocks", data, truncate) do + alias Towerops.GeoIP.Block + alias Towerops.Repo + + if truncate do + Repo.delete_all(Block) + end + + now = DateTime.truncate(DateTime.utc_now(), :second) + + entries = + Enum.map(data, fn block -> + %{ + id: Ecto.UUID.generate(), + network: block["network"], + start_ip_int: block["start_ip_int"], + end_ip_int: block["end_ip_int"], + geoname_id: block["geoname_id"], + registered_country_geoname_id: block["registered_country_geoname_id"], + inserted_at: now, + updated_at: now + } + end) + + {inserted, _} = Repo.insert_all(Block, entries) + {:ok, inserted} + rescue + e -> + {:error, Exception.message(e)} + end + + defp import_batch(_type, _data, _truncate) do + {:error, "Invalid batch_type. Must be 'locations' or 'blocks'"} + end end diff --git a/lib/towerops_web/user_auth.ex b/lib/towerops_web/user_auth.ex index b2367ada..c00cc541 100644 --- a/lib/towerops_web/user_auth.ex +++ b/lib/towerops_web/user_auth.ex @@ -321,18 +321,23 @@ defmodule ToweropsWeb.UserAuth do if there's no existing return path to avoid overwriting. """ def store_return_to_for_liveview(conn, _opts) do - # Skip authentication-related paths (login, register, reset password, confirm) + # Skip authentication-related and public paths skip_paths = [ "/users/log-in", "/users/register", "/users/reset-password", - "/users/confirm" + "/users/confirm", + "/docs/api", + "/privacy", + "/terms", + "/help" ] should_store = conn.method == "GET" && is_nil(get_session(conn, :user_return_to)) && - !Enum.any?(skip_paths, &String.starts_with?(conn.request_path, &1)) + !Enum.any?(skip_paths, &String.starts_with?(conn.request_path, &1)) && + conn.request_path != "/" if should_store do put_session(conn, :user_return_to, current_path(conn))