312 lines
12 KiB
Elixir
312 lines
12 KiB
Elixir
defmodule Mix.Tasks.Geoip.ImportTest do
|
|
use Towerops.DataCase, async: true
|
|
|
|
import ExUnit.CaptureIO
|
|
import Mix.Tasks.Geoip.Import, only: [load_locations: 1, load_blocks: 2, run: 1]
|
|
|
|
alias Towerops.GeoIP.Block
|
|
alias Towerops.GeoIP.Location
|
|
alias Towerops.Repo
|
|
|
|
setup do
|
|
# Create temporary directory for test CSV files
|
|
temp_dir = Path.join(System.tmp_dir!(), "geoip_test_#{System.unique_integer([:positive])}")
|
|
File.mkdir_p!(temp_dir)
|
|
|
|
on_exit(fn ->
|
|
File.rm_rf!(temp_dir)
|
|
end)
|
|
|
|
{:ok, temp_dir: temp_dir}
|
|
end
|
|
|
|
describe "load_locations/1" do
|
|
test "imports locations from CSV file", %{temp_dir: temp_dir} do
|
|
csv_file = Path.join(temp_dir, "locations.csv")
|
|
|
|
File.write!(csv_file, """
|
|
geoname_id,locale_code,continent_code,continent_name,country_iso_code,country_name,subdivision_1_iso_code,subdivision_1_name,subdivision_2_iso_code,subdivision_2_name,city_name,metro_code,time_zone,is_in_european_union
|
|
5391959,en,NA,North America,US,United States,CA,California,,,San Francisco,807,America/Los_Angeles,0
|
|
6167865,en,NA,North America,CA,Canada,ON,Ontario,,,Toronto,0,America/Toronto,0
|
|
2643743,en,EU,Europe,GB,United Kingdom,ENG,England,,,London,0,Europe/London,1
|
|
""")
|
|
|
|
{count, geoname_ids} = load_locations(csv_file)
|
|
|
|
assert count == 3
|
|
assert MapSet.size(geoname_ids) == 3
|
|
assert MapSet.member?(geoname_ids, 5_391_959)
|
|
assert MapSet.member?(geoname_ids, 6_167_865)
|
|
assert MapSet.member?(geoname_ids, 2_643_743)
|
|
|
|
# Verify database records
|
|
locations = Repo.all(Location)
|
|
assert length(locations) == 3
|
|
|
|
san_francisco = Enum.find(locations, &(&1.geoname_id == 5_391_959))
|
|
assert san_francisco.country_code == "US"
|
|
assert san_francisco.country_name == "United States"
|
|
assert san_francisco.city_name == "San Francisco"
|
|
assert san_francisco.subdivision_1_name == "California"
|
|
end
|
|
|
|
test "skips locations without country code", %{temp_dir: temp_dir} do
|
|
csv_file = Path.join(temp_dir, "locations.csv")
|
|
|
|
File.write!(csv_file, """
|
|
geoname_id,locale_code,continent_code,continent_name,country_iso_code,country_name,subdivision_1_iso_code,subdivision_1_name,subdivision_2_iso_code,subdivision_2_name,city_name,metro_code,time_zone,is_in_european_union
|
|
5391959,en,NA,North America,US,United States,CA,California,,,San Francisco,807,America/Los_Angeles,0
|
|
9999999,en,NA,North America,,,,,,,,0,,0
|
|
""")
|
|
|
|
{count, geoname_ids} = load_locations(csv_file)
|
|
|
|
assert count == 1
|
|
assert MapSet.size(geoname_ids) == 1
|
|
assert MapSet.member?(geoname_ids, 5_391_959)
|
|
end
|
|
|
|
test "skips locations without geoname_id", %{temp_dir: temp_dir} do
|
|
csv_file = Path.join(temp_dir, "locations.csv")
|
|
|
|
File.write!(csv_file, """
|
|
geoname_id,locale_code,continent_code,continent_name,country_iso_code,country_name,subdivision_1_iso_code,subdivision_1_name,subdivision_2_iso_code,subdivision_2_name,city_name,metro_code,time_zone,is_in_european_union
|
|
5391959,en,NA,North America,US,United States,CA,California,,,San Francisco,807,America/Los_Angeles,0
|
|
,en,NA,North America,CA,Canada,ON,Ontario,,,Toronto,0,America/Toronto,0
|
|
""")
|
|
|
|
{count, geoname_ids} = load_locations(csv_file)
|
|
|
|
assert count == 1
|
|
assert MapSet.size(geoname_ids) == 1
|
|
end
|
|
|
|
test "handles empty optional fields", %{temp_dir: temp_dir} do
|
|
csv_file = Path.join(temp_dir, "locations.csv")
|
|
|
|
File.write!(csv_file, """
|
|
geoname_id,locale_code,continent_code,continent_name,country_iso_code,country_name,subdivision_1_iso_code,subdivision_1_name,subdivision_2_iso_code,subdivision_2_name,city_name,metro_code,time_zone,is_in_european_union
|
|
5391959,en,NA,North America,US,,,,,,,,America/Los_Angeles,0
|
|
""")
|
|
|
|
{count, _geoname_ids} = load_locations(csv_file)
|
|
assert count == 1
|
|
|
|
location = Repo.get_by!(Location, geoname_id: 5_391_959)
|
|
assert location.country_name == nil
|
|
assert location.city_name == nil
|
|
assert location.subdivision_1_name == nil
|
|
assert location.subdivision_2_name == nil
|
|
end
|
|
|
|
test "processes large batches correctly", %{temp_dir: temp_dir} do
|
|
csv_file = Path.join(temp_dir, "locations.csv")
|
|
|
|
# Generate 10,000 locations (more than batch size of 5,000)
|
|
header =
|
|
"geoname_id,locale_code,continent_code,continent_name,country_iso_code,country_name,subdivision_1_iso_code,subdivision_1_name,subdivision_2_iso_code,subdivision_2_name,city_name,metro_code,time_zone,is_in_european_union\n"
|
|
|
|
rows =
|
|
Enum.map(1..10_000, fn i ->
|
|
"#{i},en,NA,North America,US,United States,CA,California,,,City#{i},807,America/Los_Angeles,0"
|
|
end)
|
|
|
|
File.write!(csv_file, header <> Enum.join(rows, "\n") <> "\n")
|
|
|
|
{count, geoname_ids} = load_locations(csv_file)
|
|
|
|
assert count == 10_000
|
|
assert MapSet.size(geoname_ids) == 10_000
|
|
assert Repo.aggregate(Location, :count) == 10_000
|
|
end
|
|
end
|
|
|
|
describe "load_blocks/2" do
|
|
test "imports IP blocks from CSV file", %{temp_dir: temp_dir} do
|
|
csv_file = Path.join(temp_dir, "blocks.csv")
|
|
|
|
# First, create some locations
|
|
location1 = insert(:geoip_location, geoname_id: 5_391_959, country_code: "US")
|
|
location2 = insert(:geoip_location, geoname_id: 6_167_865, country_code: "CA")
|
|
|
|
valid_geoname_ids = MapSet.new([location1.geoname_id, location2.geoname_id])
|
|
|
|
File.write!(csv_file, """
|
|
network,geoname_id,registered_country_geoname_id,represented_country_geoname_id,is_anonymous_proxy,is_satellite_provider,postal_code,latitude,longitude,accuracy_radius
|
|
1.0.0.0/24,5391959,5391959,,,0,94102,37.7749,-122.4194,1000
|
|
1.0.1.0/24,6167865,6167865,,,0,M5H,43.6532,-79.3832,1000
|
|
1.0.2.0/24,5391959,5391959,,,0,94102,37.7749,-122.4194,1000
|
|
""")
|
|
|
|
count = load_blocks(csv_file, valid_geoname_ids)
|
|
|
|
assert count == 3
|
|
|
|
# Verify database records
|
|
blocks = Repo.all(Block)
|
|
assert length(blocks) == 3
|
|
|
|
block1 = Enum.find(blocks, &(&1.network == "1.0.0.0/24"))
|
|
assert block1.geoname_id == 5_391_959
|
|
assert block1.start_ip_int == 16_777_216
|
|
assert block1.end_ip_int == 16_777_471
|
|
end
|
|
|
|
test "filters out blocks with invalid geoname_id", %{temp_dir: temp_dir} do
|
|
csv_file = Path.join(temp_dir, "blocks.csv")
|
|
|
|
location = insert(:geoip_location, geoname_id: 5_391_959, country_code: "US")
|
|
valid_geoname_ids = MapSet.new([location.geoname_id])
|
|
|
|
File.write!(csv_file, """
|
|
network,geoname_id,registered_country_geoname_id,represented_country_geoname_id,is_anonymous_proxy,is_satellite_provider,postal_code,latitude,longitude,accuracy_radius
|
|
1.0.0.0/24,5391959,5391959,,,0,94102,37.7749,-122.4194,1000
|
|
1.0.1.0/24,9999999,9999999,,,0,M5H,43.6532,-79.3832,1000
|
|
""")
|
|
|
|
count = load_blocks(csv_file, valid_geoname_ids)
|
|
|
|
assert count == 1
|
|
assert Repo.aggregate(Block, :count) == 1
|
|
end
|
|
|
|
test "uses registered_country_geoname_id as fallback", %{temp_dir: temp_dir} do
|
|
csv_file = Path.join(temp_dir, "blocks.csv")
|
|
|
|
location = insert(:geoip_location, geoname_id: 5_391_959, country_code: "US")
|
|
valid_geoname_ids = MapSet.new([location.geoname_id])
|
|
|
|
File.write!(csv_file, """
|
|
network,geoname_id,registered_country_geoname_id,represented_country_geoname_id,is_anonymous_proxy,is_satellite_provider,postal_code,latitude,longitude,accuracy_radius
|
|
1.0.0.0/24,,5391959,,,0,94102,37.7749,-122.4194,1000
|
|
""")
|
|
|
|
count = load_blocks(csv_file, valid_geoname_ids)
|
|
|
|
assert count == 1
|
|
|
|
block = Repo.one!(Block)
|
|
assert block.geoname_id == 5_391_959
|
|
assert block.registered_country_geoname_id == 5_391_959
|
|
end
|
|
|
|
test "skips blocks with no valid geoname_id", %{temp_dir: temp_dir} do
|
|
csv_file = Path.join(temp_dir, "blocks.csv")
|
|
valid_geoname_ids = MapSet.new()
|
|
|
|
File.write!(csv_file, """
|
|
network,geoname_id,registered_country_geoname_id,represented_country_geoname_id,is_anonymous_proxy,is_satellite_provider,postal_code,latitude,longitude,accuracy_radius
|
|
1.0.0.0/24,,,,,0,94102,37.7749,-122.4194,1000
|
|
""")
|
|
|
|
count = load_blocks(csv_file, valid_geoname_ids)
|
|
|
|
assert count == 0
|
|
assert Repo.aggregate(Block, :count) == 0
|
|
end
|
|
|
|
test "calculates IP ranges correctly", %{temp_dir: temp_dir} do
|
|
csv_file = Path.join(temp_dir, "blocks.csv")
|
|
|
|
location = insert(:geoip_location, geoname_id: 5_391_959, country_code: "US")
|
|
valid_geoname_ids = MapSet.new([location.geoname_id])
|
|
|
|
File.write!(csv_file, """
|
|
network,geoname_id,registered_country_geoname_id,represented_country_geoname_id,is_anonymous_proxy,is_satellite_provider,postal_code,latitude,longitude,accuracy_radius
|
|
192.168.1.0/24,5391959,5391959,,,0,94102,37.7749,-122.4194,1000
|
|
10.0.0.0/8,5391959,5391959,,,0,94102,37.7749,-122.4194,1000
|
|
172.16.0.0/12,5391959,5391959,,,0,94102,37.7749,-122.4194,1000
|
|
""")
|
|
|
|
count = load_blocks(csv_file, valid_geoname_ids)
|
|
assert count == 3
|
|
|
|
blocks = Repo.all(Block)
|
|
|
|
# 192.168.1.0/24 = 192.168.1.0 to 192.168.1.255
|
|
block1 = Enum.find(blocks, &(&1.network == "192.168.1.0/24"))
|
|
assert block1.start_ip_int == 3_232_235_776
|
|
assert block1.end_ip_int == 3_232_236_031
|
|
|
|
# 10.0.0.0/8 = 10.0.0.0 to 10.255.255.255
|
|
block2 = Enum.find(blocks, &(&1.network == "10.0.0.0/8"))
|
|
assert block2.start_ip_int == 167_772_160
|
|
assert block2.end_ip_int == 184_549_375
|
|
|
|
# 172.16.0.0/12 = 172.16.0.0 to 172.31.255.255
|
|
block3 = Enum.find(blocks, &(&1.network == "172.16.0.0/12"))
|
|
assert block3.start_ip_int == 2_886_729_728
|
|
assert block3.end_ip_int == 2_887_778_303
|
|
end
|
|
|
|
test "processes large batches correctly", %{temp_dir: temp_dir} do
|
|
csv_file = Path.join(temp_dir, "blocks.csv")
|
|
|
|
location = insert(:geoip_location, geoname_id: 5_391_959, country_code: "US")
|
|
valid_geoname_ids = MapSet.new([location.geoname_id])
|
|
|
|
# Generate 10,000 blocks (more than batch size of 5,000)
|
|
header =
|
|
"network,geoname_id,registered_country_geoname_id,represented_country_geoname_id,is_anonymous_proxy,is_satellite_provider,postal_code,latitude,longitude,accuracy_radius\n"
|
|
|
|
rows =
|
|
Enum.map(1..10_000, fn i ->
|
|
# Use sequential /32 blocks
|
|
octet1 = div(i, 256 * 256)
|
|
octet2 = rem(div(i, 256), 256)
|
|
octet3 = rem(i, 256)
|
|
"#{octet1}.#{octet2}.#{octet3}.0/32,5391959,5391959,,,0,94102,37.7749,-122.4194,1000"
|
|
end)
|
|
|
|
File.write!(csv_file, header <> Enum.join(rows, "\n") <> "\n")
|
|
|
|
count = load_blocks(csv_file, valid_geoname_ids)
|
|
|
|
assert count == 10_000
|
|
assert Repo.aggregate(Block, :count) == 10_000
|
|
end
|
|
end
|
|
|
|
describe "run/1 with invalid arguments" do
|
|
test "shows usage help when no directory is provided" do
|
|
output = capture_io(:stderr, fn -> run([]) end)
|
|
assert output =~ "Usage: mix geoip.import <directory>"
|
|
end
|
|
|
|
test "shows usage help when too many arguments are provided" do
|
|
output = capture_io(:stderr, fn -> run(["dir1", "dir2"]) end)
|
|
assert output =~ "Usage: mix geoip.import <directory>"
|
|
end
|
|
|
|
test "requires TOWEROPS_KEY env var for production mode" do
|
|
System.delete_env("TOWEROPS_KEY")
|
|
|
|
assert catch_exit(
|
|
capture_io(:stderr, fn ->
|
|
run(["~/geoip", "--production"])
|
|
end)
|
|
) == {:shutdown, 1}
|
|
end
|
|
end
|
|
|
|
# Helper to insert location for tests
|
|
defp insert(:geoip_location, attrs) do
|
|
defaults = %{
|
|
country_code: "US",
|
|
country_name: "United States",
|
|
city_name: "Test City",
|
|
subdivision_1_name: nil,
|
|
subdivision_2_name: nil,
|
|
latitude: nil,
|
|
longitude: nil,
|
|
inserted_at: Towerops.Time.now(),
|
|
updated_at: Towerops.Time.now()
|
|
}
|
|
|
|
attrs = Map.merge(defaults, Map.new(attrs))
|
|
|
|
%Location{}
|
|
|> Ecto.Changeset.change(attrs)
|
|
|> Repo.insert!()
|
|
end
|
|
end
|