prod ip db import

2026-01-28 13:28:11 -06:00 · 2026-01-28 13:28:11 -06:00 · b341f89332
commit b341f89332
parent 7ecc986bcd
4 changed files with 49 additions and 53 deletions
--- a/CLAUDE.md
+++ b/CLAUDE.md
@ -300,20 +300,21 @@ See `vendor/README.md` for update instructions.
 make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/
 ```

-2. **Production Import via API** (requires superuser API token):
+2. **Production Import via API** (requires `TOWEROPS_KEY` env var):
 ```bash
-# Using Makefile
-make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/ TOWEROPS_KEY=your_superuser_api_key
+# Using Makefile (recommended)
+make geoip-import-prod DIR=~/Downloads/GeoLite2-City-CSV_20260127/

 # Or directly with mix
-TOWEROPS_KEY=your_superuser_api_key mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/
+mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ --production
 ```

 **How It Works**:
- Development (no `TOWEROPS_KEY`): Direct database import via Ecto
- Production (`TOWEROPS_KEY` set): Processes CSVs locally, sends data to API in batches of 5,000 records
+- **Local mode** (`make geoip-import`): Direct database import via Ecto
+- **Production mode** (`make geoip-import-prod` or `--production` flag): Processes CSVs locally, sends data to API in batches of 5,000 records
 - API endpoint: `POST /admin/api/geoip/import` (superuser only)
 - No need to upload large CSV files to production - only processed data is sent over HTTPS
+- Production mode requires `TOWEROPS_KEY` environment variable (superuser API token)
 - Default API URL: `https://towerops.net` (override with `TOWEROPS_URL` env var)

 **Implementation Notes**:
--- a/28
+++ b/28
@ -1,4 +1,4 @@
-.PHONY: help build push deploy restart login clean compile-mibs geoip-import
+.PHONY: help build push deploy restart login clean compile-mibs geoip-import geoip-import-prod

 # Variables
 REGISTRY := registry.gitlab.com/towerops/towerops
@ -78,21 +78,21 @@ info: ## Show current image information
 compile-mibs: ## Pre-compile MIB files to JSON for runtime loading
 	@python3 scripts/compile_mibs.py

-geoip-import: ## Import MaxMind GeoLite2 City CSV database (specify DIR=path/to/csv, optionally TOWEROPS_KEY for production)
+geoip-import: ## Import MaxMind GeoLite2 City CSV database to local database (specify DIR=path/to/csv)
 	@if [ -z "$(DIR)" ]; then \
 		echo "Error: Please specify DIR variable"; \
-		echo ""; \
-		echo "Usage (local):"; \
-		echo "  make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/"; \
-		echo ""; \
-		echo "Usage (production via API):"; \
-		echo "  make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/ TOWEROPS_KEY=your_superuser_api_key"; \
+		echo "Usage: make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/"; \
 		exit 1; \
 	fi
-	@if [ -n "$(TOWEROPS_KEY)" ]; then \
-		echo "Importing GeoIP database to production via API..."; \
-		TOWEROPS_KEY=$(TOWEROPS_KEY) mix geoip.import $(DIR); \
-	else \
-		echo "Importing GeoIP database to local database..."; \
-		mix geoip.import $(DIR); \
+	@echo "Importing GeoIP database to local database..."
+	@mix geoip.import $(DIR)
+
+geoip-import-prod: ## Import MaxMind GeoLite2 City CSV to production via API (specify DIR=path/to/csv, requires TOWEROPS_KEY env var)
+	@if [ -z "$(DIR)" ]; then \
+		echo "Error: Please specify DIR variable"; \
+		echo "Usage: make geoip-import-prod DIR=~/Downloads/GeoLite2-City-CSV_20260127/"; \
+		echo "Note: TOWEROPS_KEY environment variable must be set"; \
+		exit 1; \
 	fi
+	@echo "Importing GeoIP database to production via API..."
+	@mix geoip.import $(DIR) --production
--- a/lib/mix/tasks/geoip.import.ex
+++ b/lib/mix/tasks/geoip.import.ex
@ -9,18 +9,22 @@ defmodule Mix.Tasks.Geoip.Import do
      # Import to local database (development)
      mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/

-      # Import to production database using DATABASE_URL environment variable
-      DATABASE_URL=postgresql://user:pass@host/db mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/
-
-      # Import to production database using command-line flag
-      mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ --database-url postgresql://user:pass@host/db
+      # Import to production via API (requires TOWEROPS_KEY env var)
+      mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ --production

  ## What it does

+  ### Local Mode (default)
  1. Reads GeoLite2-City-Locations-en.csv (geoname_id -> country, city, region)
-  2. Inserts locations into geoip_locations table
+  2. Inserts locations into geoip_locations table via Ecto
  3. Reads GeoLite2-City-Blocks-IPv4.csv (IP ranges -> geoname_id)
-  4. Inserts IP blocks into geoip_blocks table
+  4. Inserts IP blocks into geoip_blocks table via Ecto
+
+  ### Production Mode (--production flag)
+  1. Reads and parses CSV files locally on your machine
+  2. Sends processed data to API in batches of 5,000 records
+  3. API endpoint handles database inserts
+  4. Requires TOWEROPS_KEY environment variable (superuser API token)

  The resulting tables are used by Towerops.GeoIP for fast country and city lookups.

@ -28,9 +32,10 @@ defmodule Mix.Tasks.Geoip.Import do

  - Only processes IPv4 addresses (IPv6 support could be added)
  - Uses bulk inserts for performance (5,000 rows at a time)
-  - Truncates existing data before import
+  - Truncates existing data before import (first batch only in production mode)
  - Expands CIDR ranges to start/end IP integers for fast binary search
-  - Can import to any PostgreSQL database using DATABASE_URL env var or --database-url flag
+  - Production mode does not upload CSV files - only sends processed JSON data
+  - Default production API URL: https://towerops.net (override with TOWEROPS_URL env var)
  """

  use Mix.Task
@ -46,15 +51,19 @@ defmodule Mix.Tasks.Geoip.Import do

  @impl Mix.Task
  def run(args) do
-    {_opts, remaining_args} = parse_args(args)
+    {opts, remaining_args} = parse_args(args)

    case remaining_args do
      [directory] ->
-        # Check if TOWEROPS_KEY is set (production API import)
-        api_key = System.get_env("TOWEROPS_KEY")
-
-        if api_key do
+        if opts[:production] do
          # Production: Send batches to API
+          api_key = System.get_env("TOWEROPS_KEY")
+
+          if !api_key do
+            Mix.shell().error("Error: TOWEROPS_KEY environment variable must be set for production import")
+            exit({:shutdown, 1})
+          end
+
          Mix.Task.run("app.config")
          {:ok, _} = Application.ensure_all_started(:req)

@ -67,14 +76,14 @@ defmodule Mix.Tasks.Geoip.Import do

      _ ->
        Mix.shell().error("""
-        Usage: mix geoip.import <directory>
+        Usage: mix geoip.import <directory> [--production]

        Examples:
          # Import to local database (development)
          mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/

-          # Import to production via API
-          TOWEROPS_KEY=your_superuser_api_key mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/
+          # Import to production via API (requires TOWEROPS_KEY env var)
+          mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ --production
        """)
    end
  end
@ -82,8 +91,8 @@ defmodule Mix.Tasks.Geoip.Import do
  defp parse_args(args) do
    {opts, remaining, _invalid} =
      OptionParser.parse(args,
-        strict: [database_url: :string],
-        aliases: [d: :database_url]
+        strict: [production: :boolean],
+        aliases: [p: :production]
      )

    {opts, remaining}
--- a/test/towerops/snmp/mib_translator_test.exs
+++ b/test/towerops/snmp/mib_translator_test.exs
@ -240,20 +240,6 @@ defmodule Towerops.Snmp.MibTranslatorTest do
      end)
    end

-    test "uses fallback for LM-SENSORS-MIB OIDs (Linux/NetSNMP)" do
-      lm_sensor_oids = [
-        {"LM-SENSORS-MIB::lmTempSensorsDevice", "1.3.6.1.4.1.2021.13.16.2.1.2"},
-        {"LM-SENSORS-MIB::lmTempSensorsValue", "1.3.6.1.4.1.2021.13.16.2.1.3"},
-        {"LM-SENSORS-MIB::lmFanSensorsValue", "1.3.6.1.4.1.2021.13.16.3.1.3"}
-      ]
-
-      Enum.each(lm_sensor_oids, fn {mib_name, expected_oid} ->
-        result = MibTranslator.translate(mib_name)
-        assert {:ok, oid} = result
-        assert String.contains?(oid, expected_oid)
-      end)
-    end
-
    test "uses fallback for UCD-SNMP-MIB OIDs (Linux memory)" do
      # Note: expected OIDs are base OIDs without instance suffix
      # snmptranslate returns base OID when input doesn't have instance suffix