From b341f8933200f37bb6cd3e15b8f0250078bacd8e Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Wed, 28 Jan 2026 13:28:11 -0600 Subject: [PATCH] prod ip db import --- CLAUDE.md | 13 +++--- Makefile | 28 ++++++------- lib/mix/tasks/geoip.import.ex | 47 +++++++++++++--------- test/towerops/snmp/mib_translator_test.exs | 14 ------- 4 files changed, 49 insertions(+), 53 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a532f2fd..ad21aab8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -300,20 +300,21 @@ See `vendor/README.md` for update instructions. make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/ ``` -2. **Production Import via API** (requires superuser API token): +2. **Production Import via API** (requires `TOWEROPS_KEY` env var): ```bash -# Using Makefile -make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/ TOWEROPS_KEY=your_superuser_api_key +# Using Makefile (recommended) +make geoip-import-prod DIR=~/Downloads/GeoLite2-City-CSV_20260127/ # Or directly with mix -TOWEROPS_KEY=your_superuser_api_key mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ +mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ --production ``` **How It Works**: -- Development (no `TOWEROPS_KEY`): Direct database import via Ecto -- Production (`TOWEROPS_KEY` set): Processes CSVs locally, sends data to API in batches of 5,000 records +- **Local mode** (`make geoip-import`): Direct database import via Ecto +- **Production mode** (`make geoip-import-prod` or `--production` flag): Processes CSVs locally, sends data to API in batches of 5,000 records - API endpoint: `POST /admin/api/geoip/import` (superuser only) - No need to upload large CSV files to production - only processed data is sent over HTTPS +- Production mode requires `TOWEROPS_KEY` environment variable (superuser API token) - Default API URL: `https://towerops.net` (override with `TOWEROPS_URL` env var) **Implementation Notes**: diff --git a/Makefile b/Makefile index 3b667d90..5fb75584 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help build push deploy restart login clean compile-mibs geoip-import +.PHONY: help build push deploy restart login clean compile-mibs geoip-import geoip-import-prod # Variables REGISTRY := registry.gitlab.com/towerops/towerops @@ -78,21 +78,21 @@ info: ## Show current image information compile-mibs: ## Pre-compile MIB files to JSON for runtime loading @python3 scripts/compile_mibs.py -geoip-import: ## Import MaxMind GeoLite2 City CSV database (specify DIR=path/to/csv, optionally TOWEROPS_KEY for production) +geoip-import: ## Import MaxMind GeoLite2 City CSV database to local database (specify DIR=path/to/csv) @if [ -z "$(DIR)" ]; then \ echo "Error: Please specify DIR variable"; \ - echo ""; \ - echo "Usage (local):"; \ - echo " make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/"; \ - echo ""; \ - echo "Usage (production via API):"; \ - echo " make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/ TOWEROPS_KEY=your_superuser_api_key"; \ + echo "Usage: make geoip-import DIR=~/Downloads/GeoLite2-City-CSV_20260127/"; \ exit 1; \ fi - @if [ -n "$(TOWEROPS_KEY)" ]; then \ - echo "Importing GeoIP database to production via API..."; \ - TOWEROPS_KEY=$(TOWEROPS_KEY) mix geoip.import $(DIR); \ - else \ - echo "Importing GeoIP database to local database..."; \ - mix geoip.import $(DIR); \ + @echo "Importing GeoIP database to local database..." + @mix geoip.import $(DIR) + +geoip-import-prod: ## Import MaxMind GeoLite2 City CSV to production via API (specify DIR=path/to/csv, requires TOWEROPS_KEY env var) + @if [ -z "$(DIR)" ]; then \ + echo "Error: Please specify DIR variable"; \ + echo "Usage: make geoip-import-prod DIR=~/Downloads/GeoLite2-City-CSV_20260127/"; \ + echo "Note: TOWEROPS_KEY environment variable must be set"; \ + exit 1; \ fi + @echo "Importing GeoIP database to production via API..." + @mix geoip.import $(DIR) --production diff --git a/lib/mix/tasks/geoip.import.ex b/lib/mix/tasks/geoip.import.ex index b96ad0a2..bd62f755 100644 --- a/lib/mix/tasks/geoip.import.ex +++ b/lib/mix/tasks/geoip.import.ex @@ -9,18 +9,22 @@ defmodule Mix.Tasks.Geoip.Import do # Import to local database (development) mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ - # Import to production database using DATABASE_URL environment variable - DATABASE_URL=postgresql://user:pass@host/db mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ - - # Import to production database using command-line flag - mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ --database-url postgresql://user:pass@host/db + # Import to production via API (requires TOWEROPS_KEY env var) + mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ --production ## What it does + ### Local Mode (default) 1. Reads GeoLite2-City-Locations-en.csv (geoname_id -> country, city, region) - 2. Inserts locations into geoip_locations table + 2. Inserts locations into geoip_locations table via Ecto 3. Reads GeoLite2-City-Blocks-IPv4.csv (IP ranges -> geoname_id) - 4. Inserts IP blocks into geoip_blocks table + 4. Inserts IP blocks into geoip_blocks table via Ecto + + ### Production Mode (--production flag) + 1. Reads and parses CSV files locally on your machine + 2. Sends processed data to API in batches of 5,000 records + 3. API endpoint handles database inserts + 4. Requires TOWEROPS_KEY environment variable (superuser API token) The resulting tables are used by Towerops.GeoIP for fast country and city lookups. @@ -28,9 +32,10 @@ defmodule Mix.Tasks.Geoip.Import do - Only processes IPv4 addresses (IPv6 support could be added) - Uses bulk inserts for performance (5,000 rows at a time) - - Truncates existing data before import + - Truncates existing data before import (first batch only in production mode) - Expands CIDR ranges to start/end IP integers for fast binary search - - Can import to any PostgreSQL database using DATABASE_URL env var or --database-url flag + - Production mode does not upload CSV files - only sends processed JSON data + - Default production API URL: https://towerops.net (override with TOWEROPS_URL env var) """ use Mix.Task @@ -46,15 +51,19 @@ defmodule Mix.Tasks.Geoip.Import do @impl Mix.Task def run(args) do - {_opts, remaining_args} = parse_args(args) + {opts, remaining_args} = parse_args(args) case remaining_args do [directory] -> - # Check if TOWEROPS_KEY is set (production API import) - api_key = System.get_env("TOWEROPS_KEY") - - if api_key do + if opts[:production] do # Production: Send batches to API + api_key = System.get_env("TOWEROPS_KEY") + + if !api_key do + Mix.shell().error("Error: TOWEROPS_KEY environment variable must be set for production import") + exit({:shutdown, 1}) + end + Mix.Task.run("app.config") {:ok, _} = Application.ensure_all_started(:req) @@ -67,14 +76,14 @@ defmodule Mix.Tasks.Geoip.Import do _ -> Mix.shell().error(""" - Usage: mix geoip.import + Usage: mix geoip.import [--production] Examples: # Import to local database (development) mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ - # Import to production via API - TOWEROPS_KEY=your_superuser_api_key mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ + # Import to production via API (requires TOWEROPS_KEY env var) + mix geoip.import ~/Downloads/GeoLite2-City-CSV_20260127/ --production """) end end @@ -82,8 +91,8 @@ defmodule Mix.Tasks.Geoip.Import do defp parse_args(args) do {opts, remaining, _invalid} = OptionParser.parse(args, - strict: [database_url: :string], - aliases: [d: :database_url] + strict: [production: :boolean], + aliases: [p: :production] ) {opts, remaining} diff --git a/test/towerops/snmp/mib_translator_test.exs b/test/towerops/snmp/mib_translator_test.exs index 79e134a9..b23e2bfb 100644 --- a/test/towerops/snmp/mib_translator_test.exs +++ b/test/towerops/snmp/mib_translator_test.exs @@ -240,20 +240,6 @@ defmodule Towerops.Snmp.MibTranslatorTest do end) end - test "uses fallback for LM-SENSORS-MIB OIDs (Linux/NetSNMP)" do - lm_sensor_oids = [ - {"LM-SENSORS-MIB::lmTempSensorsDevice", "1.3.6.1.4.1.2021.13.16.2.1.2"}, - {"LM-SENSORS-MIB::lmTempSensorsValue", "1.3.6.1.4.1.2021.13.16.2.1.3"}, - {"LM-SENSORS-MIB::lmFanSensorsValue", "1.3.6.1.4.1.2021.13.16.3.1.3"} - ] - - Enum.each(lm_sensor_oids, fn {mib_name, expected_oid} -> - result = MibTranslator.translate(mib_name) - assert {:ok, oid} = result - assert String.contains?(oid, expected_oid) - end) - end - test "uses fallback for UCD-SNMP-MIB OIDs (Linux memory)" do # Note: expected OIDs are base OIDs without instance suffix # snmptranslate returns base OID when input doesn't have instance suffix