support prod ip db import

CLAUDE.md

@@ -300,28 +300,26 @@ See `vendor/README.md` for update instructions.
 make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/
 ```
 
-2. **Production Import via Mix Task**:
+2. **Production Import via API** (requires superuser API token):
 ```bash
-# Using DATABASE_URL environment variable
-make geoip-import-prod DIR=~/Downloads/GeoLite2-City-CSV_20260127/ DATABASE_URL=postgresql://user:pass@host/db
+# Using Makefile
+make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/ TOWEROPS_KEY=your_superuser_api_key
 
-# Or using command-line flag
-DATABASE_URL=postgresql://user:pass@host/db mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/
+# Or directly with mix
+TOWEROPS_KEY=your_superuser_api_key mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/
 ```
 
-3. **Production Import via API** (requires superuser API token):
-```bash
-curl -X POST https://towerops.net/admin/api/geoip/import \
-  -H "Authorization: Bearer {superuser_api_token}" \
-  -H "Content-Type: application/json" \
-  -d '{"directory": "/path/to/GeoLite2-City-CSV_20260127"}'
-```
+**How It Works**:
+- Development (no `TOWEROPS_KEY`): Direct database import via Ecto
+- Production (`TOWEROPS_KEY` set): Processes CSVs locally, sends data to API in batches of 5,000 records
+- API endpoint: `POST /admin/api/geoip/import` (superuser only)
+- No need to upload large CSV files to production - only processed data is sent over HTTPS
+- Default API URL: `https://towerops.net` (override with `TOWEROPS_URL` env var)
 
 **Implementation Notes**:
-- API endpoint: `POST /admin/api/geoip/import` (superuser only)
-- Mix task supports custom DATABASE_URL for importing to any PostgreSQL database
 - Uses batch inserts of 5,000 rows to stay within PostgreSQL's 65,535 parameter limit
 - Automatically filters orphaned IP blocks (those without matching location references)
+- Mix task processes CSVs locally and sends batches via API when `TOWEROPS_KEY` is set
 - CSV files are NOT committed to git - imported separately in each environment
 - Download GeoLite2-City CSV from MaxMind: https://dev.maxmind.com/geoip/geolite2-free-geolocation-data
 - Required files: `GeoLite2-City-Blocks-IPv4.csv`, `GeoLite2-City-Locations-en.csv`

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help build push deploy restart login clean compile-mibs geoip-import geoip-import-prod
+.PHONY: help build push deploy restart login clean compile-mibs geoip-import
 
 # Variables
 REGISTRY := registry.gitlab.com/towerops/towerops
@@ -78,25 +78,21 @@ info: ## Show current image information
 compile-mibs: ## Pre-compile MIB files to JSON for runtime loading
 	@python3 scripts/compile_mibs.py
 
-geoip-import: ## Import MaxMind GeoLite2 City CSV database to local database (specify DIR=path/to/csv)
+geoip-import: ## Import MaxMind GeoLite2 City CSV database (specify DIR=path/to/csv, optionally TOWEROPS_KEY for production)
 	@if [ -z "$(DIR)" ]; then \
 		echo "Error: Please specify DIR variable"; \
-		echo "Usage: make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/"; \
+		echo ""; \
+		echo "Usage (local):"; \
+		echo "  make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/"; \
+		echo ""; \
+		echo "Usage (production via API):"; \
+		echo "  make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/ TOWEROPS_KEY=your_superuser_api_key"; \
 		exit 1; \
 	fi
-	@echo "Importing GeoIP database from $(DIR)..."
-	@mix geoip.import $(DIR)
-
-geoip-import-prod: ## Import MaxMind GeoLite2 City CSV to production (specify DIR=path/to/csv and DATABASE_URL=postgresql://...)
-	@if [ -z "$(DIR)" ]; then \
-		echo "Error: Please specify DIR variable"; \
-		echo "Usage: make geoip-import-prod DIR=~/Downloads/GeoLite2-City-CSV_20260127/ DATABASE_URL=postgresql://..."; \
-		exit 1; \
+	@if [ -n "$(TOWEROPS_KEY)" ]; then \
+		echo "Importing GeoIP database to production via API..."; \
+		TOWEROPS_KEY=$(TOWEROPS_KEY) mix geoip.import $(DIR); \
+	else \
+		echo "Importing GeoIP database to local database..."; \
+		mix geoip.import $(DIR); \
 	fi
-	@if [ -z "$(DATABASE_URL)" ]; then \
-		echo "Error: Please specify DATABASE_URL variable"; \
-		echo "Usage: make geoip-import-prod DIR=~/Downloads/GeoLite2-City-CSV_20260127/ DATABASE_URL=postgresql://..."; \
-		exit 1; \
-	fi
-	@echo "Importing GeoIP database from $(DIR) to production..."
-	@DATABASE_URL=$(DATABASE_URL) mix geoip.import $(DIR)

lib/mix/tasks/geoip.import.ex

@@ -46,40 +46,35 @@ defmodule Mix.Tasks.Geoip.Import do
 
   @impl Mix.Task
   def run(args) do
-    {opts, remaining_args} = parse_args(args)
+    {_opts, remaining_args} = parse_args(args)
 
     case remaining_args do
       [directory] ->
-        # Start app if using default repo, otherwise start minimal dependencies
-        database_url = opts[:database_url] || System.get_env("DATABASE_URL")
+        # Check if TOWEROPS_KEY is set (production API import)
+        api_key = System.get_env("TOWEROPS_KEY")
 
-        if database_url do
-          # Start minimal dependencies for custom database connection
+        if api_key do
+          # Production: Send batches to API
           Mix.Task.run("app.config")
-          {:ok, _} = Application.ensure_all_started(:postgrex)
-          {:ok, _} = Application.ensure_all_started(:ecto_sql)
+          {:ok, _} = Application.ensure_all_started(:req)
 
-          # Start dynamic repo with custom database URL
-          import_with_custom_repo(directory, database_url)
+          import_via_api(directory, api_key)
         else
-          # Use default repo (development)
+          # Development: Direct database import
           Mix.Task.run("app.start")
           import_from_directory(directory)
         end
 
       _ ->
         Mix.shell().error("""
-        Usage: mix geoip.import <directory> [--database-url URL]
+        Usage: mix geoip.import <directory>
 
         Examples:
-          # Import to local database
+          # Import to local database (development)
           mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/
 
-          # Import to production database
-          DATABASE_URL=postgresql://user:pass@host/db mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/
-
-          # Import to production database with flag
-          mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ --database-url postgresql://user:pass@host/db
+          # Import to production via API
+          TOWEROPS_KEY=your_superuser_api_key mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/
         """)
     end
   end
@@ -94,29 +89,115 @@ defmodule Mix.Tasks.Geoip.Import do
     {opts, remaining}
   end
 
-  defp import_with_custom_repo(directory, database_url) do
-    Mix.shell().info("Connecting to custom database: #{sanitize_url(database_url)}")
+  defp import_via_api(directory, api_key) do
+    directory = Path.expand(directory)
+    blocks_file = Path.join(directory, "GeoLite2-City-Blocks-IPv4.csv")
+    locations_file = Path.join(directory, "GeoLite2-City-Locations-en.csv")
 
-    # Define a dynamic repo module for the import
-    repo_config = [
-      database: nil,
-      url: database_url,
-      pool_size: 2
-    ]
+    # Determine API URL (default to production)
+    api_url = System.get_env("TOWEROPS_URL", "https://towerops.net")
 
-    # Start the repo
-    {:ok, _pid} = Repo.start_link(repo_config)
+    Mix.shell().info("Importing GeoIP database to #{api_url} via API...")
 
-    # Run the import
-    import_from_directory(directory)
+    # Parse and send locations first
+    Mix.shell().info("  Processing locations file...")
+    {locations_count, valid_geoname_ids} = send_locations_to_api(locations_file, api_url, api_key)
+    Mix.shell().info("  Sent #{locations_count} locations")
 
-    # Stop the repo
-    Supervisor.stop(Repo)
+    # Parse and send blocks
+    Mix.shell().info("  Processing blocks file (this may take several minutes)...")
+    blocks_count = send_blocks_to_api(blocks_file, valid_geoname_ids, api_url, api_key)
+    Mix.shell().info("  Sent #{blocks_count} IP blocks")
+
+    Mix.shell().info("✓ GeoIP database imported successfully to production!")
+    Mix.shell().info("  #{locations_count} locations")
+    Mix.shell().info("  #{blocks_count} IP blocks")
   end
 
-  defp sanitize_url(url) do
-    # Remove password from URL for logging
-    String.replace(url, ~r/:([^@]+)@/, ":***@")
+  defp send_locations_to_api(file, api_url, api_key) do
+    file
+    |> File.stream!()
+    |> Stream.drop(1)
+    |> Stream.map(&parse_location_line/1)
+    |> Stream.reject(&is_nil/1)
+    |> Stream.chunk_every(@batch_size)
+    |> Enum.reduce({0, MapSet.new()}, fn batch, {count, geoname_ids} ->
+      # Convert batch to JSON-serializable format
+      data =
+        Enum.map(batch, fn location ->
+          %{
+            "geoname_id" => location.geoname_id,
+            "country_code" => location.country_code,
+            "country_name" => location.country_name,
+            "city_name" => location.city_name,
+            "subdivision_1_name" => location.subdivision_1_name,
+            "subdivision_2_name" => location.subdivision_2_name,
+            "latitude" => location.latitude,
+            "longitude" => location.longitude
+          }
+        end)
+
+      # Send to API
+      truncate = count == 0
+      send_batch_to_api(api_url, api_key, "locations", data, truncate)
+
+      # Track geoname_ids
+      new_ids = MapSet.new(batch, & &1.geoname_id)
+      {count + length(batch), MapSet.union(geoname_ids, new_ids)}
+    end)
+  end
+
+  defp send_blocks_to_api(file, valid_geoname_ids, api_url, api_key) do
+    file
+    |> File.stream!()
+    |> Stream.drop(1)
+    |> Stream.map(&parse_block_line/1)
+    |> Stream.reject(&is_nil/1)
+    |> Stream.filter(fn block -> MapSet.member?(valid_geoname_ids, block.geoname_id) end)
+    |> Stream.chunk_every(@batch_size)
+    |> Enum.reduce(0, fn batch, count ->
+      # Convert batch to JSON-serializable format
+      data =
+        Enum.map(batch, fn block ->
+          %{
+            "network" => block.network,
+            "start_ip_int" => block.start_ip_int,
+            "end_ip_int" => block.end_ip_int,
+            "geoname_id" => block.geoname_id,
+            "registered_country_geoname_id" => block.registered_country_geoname_id
+          }
+        end)
+
+      # Send to API
+      truncate = count == 0
+      send_batch_to_api(api_url, api_key, "blocks", data, truncate)
+
+      count + length(batch)
+    end)
+  end
+
+  defp send_batch_to_api(api_url, api_key, batch_type, data, truncate) do
+    url = "#{api_url}/admin/api/geoip/import"
+
+    body = %{
+      "batch_type" => batch_type,
+      "data" => data,
+      "truncate" => truncate
+    }
+
+    response =
+      Req.post!(url,
+        json: body,
+        headers: [{"authorization", "Bearer #{api_key}"}],
+        retry: :transient
+      )
+
+    if response.status != 200 do
+      Mix.shell().error("API request failed: #{inspect(response.body)}")
+      raise "Failed to send batch to API"
+    end
+
+    response.body
   end
 
   defp import_from_directory(directory) do

lib/towerops_web/controllers/api/v1/geoip_controller.ex

@@ -9,84 +9,65 @@ defmodule ToweropsWeb.Api.V1.GeoipController do
 
   use ToweropsWeb, :controller
 
-  alias Mix.Tasks.Geoip.Import
-
   require Logger
 
   @doc """
-  Import GeoIP database from MaxMind GeoLite2-City CSV files.
+  Import GeoIP database batch from processed CSV data.
 
   Requires superuser API token.
 
   ## Request
 
-      POST /api/v1/geoip/import
+      POST /admin/api/geoip/import
       Content-Type: application/json
 
       {
-        "directory": "/path/to/GeoLite2-City-CSV_20260127"
+        "batch_type": "locations",  // or "blocks"
+        "data": [...],  // Array of location or block records
+        "truncate": true  // Optional: truncate table before inserting (first batch only)
       }
 
   ## Response
 
-      202 Accepted
-      {
-        "status": "ok",
-        "message": "GeoIP import started in background"
-      }
-
-  Or:
-
       200 OK
       {
         "status": "ok",
-        "message": "GeoIP database imported successfully",
-        "locations_count": 130000,
-        "blocks_count": 3500000
+        "inserted": 5000
       }
   """
-  def import_database(conn, %{"directory" => directory}) do
+  def import_database(conn, %{"batch_type" => batch_type, "data" => data} = params) do
     with :ok <- require_superuser(conn) do
-      Logger.info("GeoIP import requested by #{conn.assigns[:current_user].email} for directory: #{directory}")
+      truncate = Map.get(params, "truncate", false)
 
-      # Expand the path to handle ~ and relative paths
-      expanded_directory = Path.expand(directory)
-
-      # Verify directory exists
-      if File.dir?(expanded_directory) do
-        # Run import synchronously (it's fast enough with batching)
-        case run_import(expanded_directory) do
-          {:ok, locations_count, blocks_count} ->
-            Logger.info("GeoIP import completed: #{locations_count} locations, #{blocks_count} blocks")
+      Logger.info(
+        "GeoIP import batch requested by #{conn.assigns[:current_user].email}: #{batch_type}, #{length(data)} records, truncate=#{truncate}"
+      )
 
+      case import_batch(batch_type, data, truncate) do
+        {:ok, inserted_count} ->
           conn
           |> put_status(:ok)
           |> json(%{
             status: "ok",
-              message: "GeoIP database imported successfully",
-              locations_count: locations_count,
-              blocks_count: blocks_count
+            inserted: inserted_count
           })
 
         {:error, reason} ->
-            Logger.error("GeoIP import failed: #{inspect(reason)}")
+          Logger.error("GeoIP batch import failed: #{inspect(reason)}")
 
           conn
           |> put_status(:internal_server_error)
           |> json(%{error: "Import failed: #{inspect(reason)}"})
       end
-      else
-        conn
-        |> put_status(:bad_request)
-        |> json(%{error: "Directory does not exist: #{directory}"})
-      end
     end
   end
 
   def import_database(conn, _params) do
     conn
     |> put_status(:bad_request)
-    |> json(%{error: "Missing required parameter: directory"})
+    |> json(%{
+      error: "Missing required parameters: batch_type and data"
+    })
   end
 
   defp require_superuser(conn) do
@@ -106,32 +87,71 @@ defmodule ToweropsWeb.Api.V1.GeoipController do
     end
   end
 
-  defp run_import(directory) do
-    alias Towerops.GeoIP.Block
+  defp import_batch("locations", data, truncate) do
     alias Towerops.GeoIP.Location
     alias Towerops.Repo
 
-    blocks_file = Path.join(directory, "GeoLite2-City-Blocks-IPv4.csv")
-    locations_file = Path.join(directory, "GeoLite2-City-Locations-en.csv")
-
-    # Verify files exist
-    if File.exists?(blocks_file) && File.exists?(locations_file) do
-      # Truncate existing data
-      Repo.delete_all(Block)
+    if truncate do
       Repo.delete_all(Location)
-
-      # Load locations
-      {locations_count, valid_geoname_ids} = Import.load_locations(locations_file)
-
-      # Load blocks
-      blocks_count = Import.load_blocks(blocks_file, valid_geoname_ids)
-
-      {:ok, locations_count, blocks_count}
-    else
-      {:error, "Required CSV files not found in directory"}
     end
+
+    now = DateTime.truncate(DateTime.utc_now(), :second)
+
+    entries =
+      Enum.map(data, fn location ->
+        %{
+          geoname_id: location["geoname_id"],
+          country_code: location["country_code"],
+          country_name: location["country_name"],
+          city_name: location["city_name"],
+          subdivision_1_name: location["subdivision_1_name"],
+          subdivision_2_name: location["subdivision_2_name"],
+          latitude: location["latitude"],
+          longitude: location["longitude"],
+          inserted_at: now,
+          updated_at: now
+        }
+      end)
+
+    {inserted, _} = Repo.insert_all(Location, entries)
+    {:ok, inserted}
   rescue
     e ->
       {:error, Exception.message(e)}
   end
+
+  defp import_batch("blocks", data, truncate) do
+    alias Towerops.GeoIP.Block
+    alias Towerops.Repo
+
+    if truncate do
+      Repo.delete_all(Block)
+    end
+
+    now = DateTime.truncate(DateTime.utc_now(), :second)
+
+    entries =
+      Enum.map(data, fn block ->
+        %{
+          id: Ecto.UUID.generate(),
+          network: block["network"],
+          start_ip_int: block["start_ip_int"],
+          end_ip_int: block["end_ip_int"],
+          geoname_id: block["geoname_id"],
+          registered_country_geoname_id: block["registered_country_geoname_id"],
+          inserted_at: now,
+          updated_at: now
+        }
+      end)
+
+    {inserted, _} = Repo.insert_all(Block, entries)
+    {:ok, inserted}
+  rescue
+    e ->
+      {:error, Exception.message(e)}
+  end
+
+  defp import_batch(_type, _data, _truncate) do
+    {:error, "Invalid batch_type. Must be 'locations' or 'blocks'"}
+  end
 end

lib/towerops_web/user_auth.ex

@@ -321,18 +321,23 @@ defmodule ToweropsWeb.UserAuth do
   if there's no existing return path to avoid overwriting.
   """
   def store_return_to_for_liveview(conn, _opts) do
-    # Skip authentication-related paths (login, register, reset password, confirm)
+    # Skip authentication-related and public paths
     skip_paths = [
       "/users/log-in",
       "/users/register",
       "/users/reset-password",
-      "/users/confirm"
+      "/users/confirm",
+      "/docs/api",
+      "/privacy",
+      "/terms",
+      "/help"
     ]
 
     should_store =
       conn.method == "GET" &&
         is_nil(get_session(conn, :user_return_to)) &&
-        !Enum.any?(skip_paths, &String.starts_with?(conn.request_path, &1))
+        !Enum.any?(skip_paths, &String.starts_with?(conn.request_path, &1)) &&
+        conn.request_path != "/"
 
     if should_store do
       put_session(conn, :user_return_to, current_path(conn))