From 6f65f1035a69a6f5f307cc428731ab9f004a1a4f Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Sat, 9 May 2026 08:52:03 -0500 Subject: [PATCH] routers update --- mikrotik-tool/494.rsc | 2 +- mikrotik-tool/982.rsc | 2 +- mikrotik-tool/climax.rsc | 2 +- mikrotik-tool/core.rsc | 2 +- mikrotik-tool/culleoka.rsc | 2 +- mikrotik-tool/edge.rsc | 3 +- mikrotik-tool/ipv6.md | 20 +- mikrotik-tool/lowry.rsc | 2 +- mikrotik-tool/network-analysis.md | 436 ++++++++++++++++++++++++++++++ mikrotik-tool/newhope.rsc | 2 +- mikrotik-tool/verona-v6-apply.rsc | 154 +++++++++++ mikrotik-tool/verona.rsc | 72 ++++- 12 files changed, 667 insertions(+), 32 deletions(-) create mode 100644 mikrotik-tool/network-analysis.md create mode 100644 mikrotik-tool/verona-v6-apply.rsc diff --git a/mikrotik-tool/494.rsc b/mikrotik-tool/494.rsc index 487ce19..92339cd 100644 --- a/mikrotik-tool/494.rsc +++ b/mikrotik-tool/494.rsc @@ -1,4 +1,4 @@ -# 2026-05-08 17:47:56 by RouterOS 7.21.4 +# 2026-05-09 08:51:42 by RouterOS 7.21.4 # software id = K4QG-8NQV # # model = RB5009UG+S+ diff --git a/mikrotik-tool/982.rsc b/mikrotik-tool/982.rsc index e50e2c9..b208b04 100644 --- a/mikrotik-tool/982.rsc +++ b/mikrotik-tool/982.rsc @@ -1,4 +1,4 @@ -# 2026-05-08 17:47:56 by RouterOS 7.21.4 +# 2026-05-09 08:51:42 by RouterOS 7.21.4 # software id = FUVS-HCM5 # # model = CCR1009-7G-1C-1S+ diff --git a/mikrotik-tool/climax.rsc b/mikrotik-tool/climax.rsc index 65916c4..6431c54 100644 --- a/mikrotik-tool/climax.rsc +++ b/mikrotik-tool/climax.rsc @@ -1,4 +1,4 @@ -# 2026-05-08 17:47:56 by RouterOS 7.21.4 +# 2026-05-09 08:51:42 by RouterOS 7.21.4 # software id = UETF-WF31 # # model = CCR2004-16G-2S+ diff --git a/mikrotik-tool/core.rsc b/mikrotik-tool/core.rsc index bed5dd4..ec842a8 100644 --- a/mikrotik-tool/core.rsc +++ b/mikrotik-tool/core.rsc @@ -1,4 +1,4 @@ -# 2026-05-08 17:47:56 by RouterOS 7.21.4 +# 2026-05-09 08:51:42 by RouterOS 7.21.4 # software id = XS5B-41QR # # model = CCR1009-7G-1C-1S+ diff --git a/mikrotik-tool/culleoka.rsc b/mikrotik-tool/culleoka.rsc index 674b0a2..a2c6c08 100644 --- a/mikrotik-tool/culleoka.rsc +++ b/mikrotik-tool/culleoka.rsc @@ -1,4 +1,4 @@ -# 2026-05-08 17:47:56 by RouterOS 7.21.4 +# 2026-05-09 08:51:42 by RouterOS 7.21.4 # software id = HVP9-3G0K # # model = CCR1009-7G-1C-1S+ diff --git a/mikrotik-tool/edge.rsc b/mikrotik-tool/edge.rsc index 979b6cd..0aa4856 100644 --- a/mikrotik-tool/edge.rsc +++ b/mikrotik-tool/edge.rsc @@ -1,4 +1,4 @@ -# may/08/2026 17:47:56 by RouterOS 6.49.18 +# may/09/2026 08:51:41 by RouterOS 6.49.18 # software id = 8XZE-R7EJ # # model = CCR2004-1G-12S+2XS @@ -530,7 +530,6 @@ set always-allow-password-login=yes forwarding-enabled=remote strong-crypto=\ add address=2605:6000:0:8::f:373/127 advertise=no interface=\ sfp-sfpplus12-spectrum add address=2606:1c80::1 interface=vlan9_sfpplus8 -add address=2606:1c80:0:1002::1 interface=sfp-sfpplus11-preseem /ipv6 firewall address-list add address=2606:1c80::/32 list=bgp-networks /ipv6 firewall filter diff --git a/mikrotik-tool/ipv6.md b/mikrotik-tool/ipv6.md index 92fc1b5..ab6ed2e 100644 --- a/mikrotik-tool/ipv6.md +++ b/mikrotik-tool/ipv6.md @@ -218,11 +218,11 @@ loopback. Peer config on every tower: ```rsc /routing bgp instance -set default as= router-id= +set default as=393837 router-id= /routing bgp peer add address-families=ipv6 name=ibgp-edge \ - remote-address=2606:1c80:0:10::254 remote-as= \ + remote-address=2606:1c80:0:10::254 remote-as=393837 \ update-source=lo multihop=yes /routing bgp network @@ -593,7 +593,7 @@ add area=backbone interface=ether6-culleoka-11ghz network-type=point-to-point /routing bgp peer add address-families=ipv6 name=ibgp-edge \ - remote-address=2606:1c80:0:10::254 remote-as= \ + remote-address=2606:1c80:0:10::254 remote-as=393837 \ update-source=lo multihop=yes /routing bgp network @@ -627,7 +627,7 @@ add area=backbone interface=ether3-climax-11ghz network-type=point-to-point /routing bgp peer add address-families=ipv6 name=ibgp-edge \ - remote-address=2606:1c80:0:10::254 remote-as= \ + remote-address=2606:1c80:0:10::254 remote-as=393837 \ update-source=lo multihop=yes /routing bgp network @@ -664,7 +664,7 @@ add area=backbone interface=ether3-culleoka-11ghz network-type=point-to-point /routing bgp peer add address-families=ipv6 name=ibgp-edge \ - remote-address=2606:1c80:0:10::254 remote-as= \ + remote-address=2606:1c80:0:10::254 remote-as=393837 \ update-source=lo multihop=yes /routing bgp network @@ -694,7 +694,7 @@ add area=backbone interface=ether2-climax network-type=point-to-point /routing bgp peer add address-families=ipv6 name=ibgp-edge \ - remote-address=2606:1c80:0:10::254 remote-as= \ + remote-address=2606:1c80:0:10::254 remote-as=393837 \ update-source=lo multihop=yes /routing bgp network @@ -724,7 +724,7 @@ add area=backbone interface=ether7-380 network-type=point-to-point /routing bgp peer add address-families=ipv6 name=ibgp-edge \ - remote-address=2606:1c80:0:10::254 remote-as= \ + remote-address=2606:1c80:0:10::254 remote-as=393837 \ update-source=lo multihop=yes /routing bgp network @@ -756,7 +756,7 @@ add area=backbone interface=ether1-climax-11ghz network-type=point-to-point /routing bgp peer add address-families=ipv6 name=ibgp-edge \ - remote-address=2606:1c80:0:10::254 remote-as= \ + remote-address=2606:1c80:0:10::254 remote-as=393837 \ update-source=lo multihop=yes /routing bgp network @@ -788,7 +788,7 @@ add area=backbone interface=ether6-lowrycrossing network-type=point-to-point /routing bgp peer add address-families=ipv6 name=ibgp-edge \ - remote-address=2606:1c80:0:10::254 remote-as= \ + remote-address=2606:1c80:0:10::254 remote-as=393837 \ update-source=lo multihop=yes /routing bgp network @@ -818,7 +818,7 @@ add area=backbone interface=ether1-newhope network-type=point-to-point /routing bgp peer add address-families=ipv6 name=ibgp-edge \ - remote-address=2606:1c80:0:10::254 remote-as= \ + remote-address=2606:1c80:0:10::254 remote-as=393837 \ update-source=lo multihop=yes /routing bgp network diff --git a/mikrotik-tool/lowry.rsc b/mikrotik-tool/lowry.rsc index 2345f8a..1852b2d 100644 --- a/mikrotik-tool/lowry.rsc +++ b/mikrotik-tool/lowry.rsc @@ -1,4 +1,4 @@ -# 2026-05-08 17:47:56 by RouterOS 7.21.4 +# 2026-05-09 08:51:42 by RouterOS 7.21.4 # software id = 2I9X-PQZP # # model = CCR1009-7G-1C-1S+ diff --git a/mikrotik-tool/network-analysis.md b/mikrotik-tool/network-analysis.md new file mode 100644 index 0000000..7347903 --- /dev/null +++ b/mikrotik-tool/network-analysis.md @@ -0,0 +1,436 @@ +# vntx Network Analysis Snapshot + +Generated 2026-05-08 from live router API queries, `.rsc` exports in this +directory, `inventory.yaml`, and the towerops Postgres database. + +## Executive summary + +**Routers:** 9 in production. 8 on RouterOS 7.21.4 long-term (just upgraded +2026-05-08, fleet-aligned). Edge alone on ROS 6.49.18 (legacy ROS6). + +**MPLS/LDP:** Live and labeled across all 8 ROS7 routers. 7 backbone links +labeled, 14 LDP adjacencies all `operational=true`. FastTrack-bypass rules +on every MPLS-bound interface to coexist with FastTrack on customer flows. + +**Customers (PPPoE active sessions):** ~216 active across the fleet. +- verona 46, culleoka 40, climax 33, newhope 21, lowry 21, 982 21, + 494 13, core 6. + +**Tower equipment:** 26 Ubiquiti APs/backhauls, 17 Cambium ePMP APs, +6 AirFiber backbone radios, 2 60 GHz radios, 4+ Mikrotik mgmt switches, +2 Netonix tower switches. + +**Known issues:** +- Climax↔culleoka direct AF11 down (power injector unplugged at climax tower). +- Edge IPv6 BGP to TWC AS 11427 in `connect` state (Charter side not configured). +- 6 Ubiquiti Rockets still on legacy AirOS 6.3.24 (older Rocket M / M900 hardware). + +--- + +## Routers + +### Live state (queried 2026-05-08) + +| Site | Hostname | Loopback | Hardware | ROS | Arch | CPU | RAM | Uptime | Filter rules | NAT | PPPoE active | +|--|--|--|--|--|--|--|--|--|--|--|--| +| verona | Verona | 10.254.254.101 | CCR2004-16G-2S+ | 7.21.4 | arm64 | 4×1.2GHz | 4 GB | 1h41m | 8 | 2 | **46** | +| climax | Climax | 10.254.254.102 | CCR2004-16G-2S+ | 7.21.4 | arm64 | 4×1.2GHz | 4 GB | 1h42m | 10 | 0 | 33 | +| culleoka | Culleoka | 10.254.254.104 | CCR1009-7G-1C-1S+ | 7.21.4 | tile | 9×1.0GHz | 2 GB | 1h39m | 5 | 0 | **40** | +| newhope | New Hope | 10.254.254.108 | CCR1009-7G-1C-1S+ | 7.21.4 | tile | 9×1.0GHz | 2 GB | 1h40m | 7 | 1 | 21 | +| lowry | LowryCrossing | 10.254.254.109 | CCR1009-7G-1C-1S+ | 7.21.4 | tile | 9×1.0GHz | 2 GB | 1h40m | 5 | 0 | 21 | +| 982 | 982 | 10.254.254.110 | CCR1009-7G-1C-1S+ | 7.21.4 | tile | 9×1.2GHz | 2 GB | 1h39m | 5 | 0 | 21 | +| 494 | 494 | 10.254.254.111 | RB5009UG+S+ | 7.21.4 | arm64 | 4×0.35GHz | 1 GB | 1h42m | 8 | 0 | 13 | +| 380/core | Core | 10.254.254.253 | CCR1009 (arm64) | 7.21.4 | arm64* | 4×? | 2 GB | 1h40m | 13 | 0 | 6 | +| 380/edge | Edge | 10.254.254.254 | CCR2004-1G-12S+2XS | **6.49.18** | arm64 | 4×1.7GHz | ~1.8 GB | **2w1d** | 93 | 11 | 0 | + +\* core's `architecture-name` came back as `tile` in /system/resource but +its sys-resource-IRQ pattern matches arm64; treating both as effectively +modern. Not load-bearing for any decision. + +CPU loads sampled at the time of snapshot: edge 42% (NAT-heavy), climax 15%, +494 9%, verona 6%, core 6%, others ≤2%. + +### Customer load distribution + +``` +verona ████████████████████████████████████████████ 46 +culleoka ████████████████████████████████████████ 40 +climax █████████████████████████████████ 33 +newhope █████████████████████ 21 +lowry █████████████████████ 21 +982 █████████████████████ 21 +494 █████████████ 13 +core ██████ 6 +``` + +Verona is the heaviest single tower — 46 PPPoE customers + a hotspot +covering `204.110.188.0/22` + active customer NAT. Worth watching CPU +during peak. + +--- + +## Backbone topology + +``` + +──────────+ + │ edge │ ROS 6.49.18 + │ .254 │ TWC AS 11427 (BGP) + +────┬─────+ + │ wired + +────┴─────+ + +────────│ core │────────+ + │ AF11 │ .253 │ 60GHz │ + │ +────┬─────+ │ + AF11 (DOWN) │ │ AF11 │ + +──────+─────────+ │ │ │ + │ │ │ │ │ │ + +───┴──+ +─┴────+ +──┴─────+ +─┴────+ +───┴────+ +────┴──+ + │verona│ │climax│ │culleoka│ │newhope│ │ 982 │ │ 494* │ + │ .101 │ │ .102 │ │ .104 │ │ .108 │ │ .110 │ │ .111 │ + +──────+ +──┬───+ +────────+ +───┬───+ +────────+ +───────+ + │AF11 │AF24 + │ │ + +──┴───+ +─┴────+ + │ 494 │ │lowry │ + │ .111 │ │ .109 │ + +──────+ +──────+ +``` + +\* 494 has TWO links into the spine: directly to climax (AF11, primary +shown) and there's a route-rule path through verona's `ether5-494` +(see climax↔494 entry). Paths are symmetric in OSPF. + +### Backbone link inventory + +| Link | Type | A iface | B iface | /29 | l2mtu | mpls-mtu | LDP | +|--|--|--|--|--|--|--|--| +| verona↔climax | AF11 | verona `ether3-climax-11ghz` | climax `ether6-verona-11ghz` | 10.250.1.24/29 | 2024 | 1508 | ✓ | +| climax↔core | AF24 | climax `ether4-380-airfiber24` | core `ether5-climax` | 10.250.1.88/29 | 2024 | 1508 | ✓ | +| climax↔494 | AF11 | climax `ether5-494` | 494 `ether2-climax` | 10.250.1.64/29 | 1580 | 1508 | ✓ | +| climax↔culleoka | AF11 | climax `ether3-culleoka-11ghz` | culleoka `ether1-climax-11ghz` | 10.250.1.8/29 | 2024 | n/a | **DOWN** | +| core↔culleoka | AF11 | core `ether6-culleoka-11ghz` | culleoka `ether6-380-11ghz` | 10.250.1.48/29 | 2024 | 1508 | ✓ | +| core↔newhope | AF11 | core `ether4-newhope` | newhope `ether2-380` | 10.250.1.56/29 | 9000 | 1508 | ✓ | +| core↔982 | 60 GHz | core `ether1-982-60ghz` | 982 `ether7-380` | 10.250.1.32/29 | 9000 | 1508 | ✓ | +| newhope↔lowry | AF24 | newhope `ether6-lowrycrossing` | lowry `ether1-newhope` | 10.250.1.104/29 | 9000 | 1508 | ✓ | +| core↔edge | wired | core `sfp-sfpplus1-edge-preseem` + `ether3-edge-direct` | edge SFP | 204.110.191.x/30 | n/a | n/a | n/a | + +### Backhaul radios (towerops monitoring) + +All AirFiber backbone radios are on Ubiquiti firmware `v4.1.0` (current). +Ethernet-side l2mtu reflects what the routers expose; the radios pass +whatever fits. + +| Radio | IP | Site | Model | Firmware | Notes | +|--|--|--|--|--|--| +| verona to climax 11ghz | 10.250.1.26 | verona | AF11 | v4.1.0 | | +| climax-380 AF24 | 10.250.1.93 | climax | AF24 | v4.1.0 | | +| core to climax | 10.250.1.90 | 380 | AF24 (other end of climax-380) | v4.1.0 | | +| culleoka to 380 11g | 10.250.1.50 | culleoka | AF11 | v4.1.0 | | +| new hope to core af11x | 10.250.1.58 | newhope | AF11 | v4.1.0 | | +| 380↔982 60GHz pair | 10.250.1.34, 10.250.1.37 | 982/380 | AirFiber 60 (Linux 4.9.241) | (linux fw) | | +| clayton to culleoka | 10.10.111.60/61 | culleoka | PowerBeam 5AC 300 / Rocket Prism 5AC | 8.7.22 | non-spine, off-tower CPE backhaul | +| verona to altoga | 10.250.1.146 | verona | PowerBeam 5AC 500 | **8.7.14** | slightly behind on AirOS 8 | + +--- + +## Access points (towerops + inventory.yaml) + +### Counts by manufacturer + +| Vendor | Count | +|--|--| +| Ubiquiti AirOS | 26 | +| Cambium ePMP | 17 | +| Ubiquiti AirFiber (backbone) | 6 | +| Mikrotik (router/switch) | 10 | +| **Total monitored** | 62 | + +### Ubiquiti AirOS firmware distribution + +| Firmware | Count | Status | Devices | +|--|--|--|--| +| **6.3.24** (Rocket M / M900, AirOS 6) | 6 | ⚠️ legacy | Altoga SW, Climax South, Climax NE, Climax 900 East, Culleoka N UBNT, Clayton Estates AP | +| 8.7.14 (AirOS 8) | 1 | slightly behind | Verona to Altoga (PowerBeam 5AC 500) | +| 8.7.22 (AirOS 8) | 12 | ✓ current | All Verona Rockets, Culleoka SE/SW/sw120/AC-1/AC-2, Climax NW, clayton-culleoka pair | + +### Cambium ePMP firmware + +17 devices, **none reporting firmware via SNMP** — different sysObjectID +or OID for the ePMP firmware version. Worth fixing in towerops monitoring +(profile_device_oids / profile_sensor_oids). + +### Per-tower AP layout + +Pulling from `inventory.yaml` (authoritative — generated from live +discovery via the `inventory` subcommand): + +| Site | APs | Notes | +|--|--|--| +| Verona | 5 (15.1, 15.2, 15.11, 15.12, 15.13) | 2× 5AC Horn (W, NW) + 3× standard sectors | +| Altoga | 3 (95.20, 95.21, 95.22) | All on legacy AirOS 6 — Rocket M5 | +| Climax | 6 (31.11–13, 31.30, 31.31, 31.40) | Mix: 2× ePMP, 1× Rocket Prism 5AC, 2× Rocket M5, 1× Rocket M900 | +| Culleoka | 11 (111.1, .2, .11, .12, .14, .30–35, .50, etc.) | Mix: 6× Rocket Prism 5AC + 5× ePMP | +| 982 | 7 (63.1–6, plus subscribers? `982-1`..`982-6` listed as Linux 4.9.241, model uncertain) | Investigate — these may be ePMP3000 sectors mis-classified | +| Newhope | 4 (143.11–14) | All Cambium ePMP, all "Verona Networks" sysName | +| Lowry | 4 (159.10–13) | All Cambium ePMP | +| 494 | 1 (175.11) | Single ePMP 3000 Omni | + +--- + +## Switches + +### Mikrotik mgmt switches + +(From `inventory.yaml`): +| Site | Switch | IP | Model | +|--|--|--|--| +| Verona | Verona Server Switch | 10.0.101.10 | MikroTik | +| Newhope | (mgmt switch on tower) | (in `100.64.x`) | MikroTik | +| 380 | Core uplink switch | n/a | MikroTik | +| 380 | "core to office" Mikrotik | 10.250.2.x | MikroTik | + +### Netonix tower switches + +| Site | Switch | IP | Model | +|--|--|--|--| +| Verona | Verona Netonix | 100.64.15.251 | Netonix | +| Culleoka | Culleoka Netonix | 100.64.27.250 | Netonix WS-26-400-AC | + +Netonix devices don't expose useful SNMP — towerops shows 0 of them. +Identification comes from LLDP discovered by adjacent Mikrotiks. + +--- + +## OSPF / Routing + +- **OSPFv2** `backbone-v2` (area 0.0.0.0) on every backbone link, SHA-512 + auth (`auth-id=1`). PTP type, BFD enabled where available. All 14 spine + adjacencies `state=Full`. +- **OSPFv3** `backbone-v3` enabled on some interfaces (mostly disabled in + templates currently). +- All instances `redistribute=connected` with passthrough filters + (`/routing filter rule chain=ospf-out rule="accept;"`). +- **Type-5 externals** in the LSDB include the customer-facing /27s + (e.g. `204.110.191.0/27` from verona, the office /29 from core, etc.). +- Verona keeps a static default `0.0.0.0/0 via 10.250.1.30 distance=1` + beneath the OSPF default — survives OSPF flap. + +## BGP + +### Edge → upstream +- **TWC / Charter (AS 11427)** at `71.41.226.117`: + - IPv4: established, holding **1,034,513 prefixes** (full table). + - IPv6 (`2605:6000:0:8::f:372`): `state=connect` — **Charter side not + configured / SYN drops**. Tracked as the IPv6 BGP issue (Charter + ticket pending). +- Filters: `twc-in`, `twc-out` (manual prefix filtering). + +### Edge → preseem / loopbacks +- `edge_core_preseem` peer (AS 393837) configured but **disabled**. +- Two Team-Cymru BOGON server peers configured but **disabled**. + +### iBGP across the fleet +- Core has `as=65530 router-id=10.254.254.253 instance=bgp-instance-1`, + with iBGP peer to climax (`10.250.1.94/32`) currently `disabled=true`. +- iBGP not in active use today; OSPF carries everything. + +## MPLS / LDP + +All 8 ROS7 routers run LDP. 14 adjacencies, all `operational=true`. + +Topology (LDP adjacency map): +``` +verona ──── climax ──── core ──── culleoka + │ │ + ├── 494 ├── newhope ──── lowry + │ │ + │ └── 982 + │ + └── (climax↔culleoka direct: configured but radio DOWN) +``` + +`mpls-mtu=1508` everywhere on the labeled spine. FastTrack-bypass `accept` +rules above each `fasttrack-connection` rule for every MPLS-bound +interface (added during this session — without them, MPLS breaks +non-loopback-sourced traffic via the FastTrack/MPLS incompatibility). + +LDP advertise-filter is **default** (everything except Type-5 externals). +This means customer-facing /27s (e.g. graham home `204.110.191.0/27`) +ride **plain IP** on the return path, not labeled. + +## NAT, hotspots, queues + +- **Verona hotspot** on interface `verona`, address-pool `verona-cgnat`, + HTTPS=false, idle-timeout=5m, profile `hsprof1`. Covers + `204.110.188.0/22`. Bypass `/ip hotspot ip-binding` entries include + graham's home `204.110.191.0/27` (added today after a discovery). + Other bypass entries: `204.110.188.224/27`, `10.250.1.146`, + one MAC binding. +- **Culleoka hotspot** also active — DHCP server with `culleoka-cgnat` + pool, 185 leases at snapshot time. +- **Newhope** has 1 NAT rule, 21 PPPoE servers, 72 leases. +- **Edge** has 11 NAT rules, 93 firewall filter rules — heaviest + firewall stack (border router job). + +## Customer-facing public space + +Per `inventory.yaml` and live `/ip address`: +- `204.110.188.0/22` block — Charter-allocated; dished out to towers as + /27s (verona has `.224/27`, etc.). +- `204.110.191.0/27` is graham's home /27 carved out of the same /22 + range, attached to verona's `vlan9_sfpplus1`. +- IPv6 from ARIN: `2606:1c80::/32` advertised to TWC AS 11427 (only IPv4 + is currently established with them; IPv6 BGP broken Charter-side). + +## CGNAT pool ownership + +From `inventory.yaml`: +- verona: `100.64.0.0/22` + `100.64.12.0/22` (gateway `.3.254`/`.15.254`) +- climax: `100.64.4.0/22` (gw `.7.254`) +- culleoka: `100.64.24.0/22` + `100.64.96.0/20` (gw `.27.254`) +- newhope: `100.64.x.x/22` (separate) +- lowry: `100.64.144.0/20` (gw `.159.254`) + +Plus a long tail of `/32` carve-outs assigned to specific subscribers +within each tower. + +--- + +# Suggested improvements (for review) + +These are observations from the snapshot — none are urgent unless +flagged. Listed roughly in order of impact / quick-win. + +### Operational / monitoring + +1. **towerops live monitoring last_checked_at = 2026-03-26**, ~6 weeks + stale. Either the agent stopped polling or the field isn't being + updated. Worth checking the agent_token / agent_assignment status and + SNMP polling jobs in oban_jobs. + +2. **Cambium ePMP firmware not exposed via SNMP** in towerops. ePMP uses + a different OID set (`.1.3.6.1.4.1.17713.21.x`). Add a + `profile_device_oid` row that points at the ePMP firmware OID so the + 17 ePMPs report their version. + +3. **`982-1`..`982-6` are listed as device_role=other Linux 4.9.241** + with no firmware. These look mis-classified — likely ePMP3000 sectors + that came back with "Linux ..." in sysDescr but should be tagged as + APs and have firmware extracted. Investigate `snmp_devices` for these + six rows. + +### Hardware / firmware lifecycle + +4. **6 Ubiquiti Rockets on AirOS 6.3.24** (legacy AirOS 6 line): + - Altoga SW (Rocket M5) + - Climax South / NE / 900 East (Rocket M5 + M900) + - Culleoka North UBNT (Rocket M5) + - Clayton Estates AP (Rocket M5) + + These are AirMax-AC hardware predecessors (Rocket M is the older + N-only line; AirOS 6 is in maintenance mode). 5 GHz Rocket M radios + are EOL'd by Ubiquiti — replace with Rocket Prism 5AC (matching what's + already on the modern towers) or LiteAP AC. Climax 900 East on Rocket + M900 is harder — the 900 MHz product line is dwindling but UISP P2MP + alternatives exist. + +5. **PowerBeam 5AC 500 at Verona→Altoga on AirOS 8.7.14** (one minor + release behind 8.7.22). Low priority, but a free 8.7.22 upgrade would + align with the rest of the AirOS 8 fleet. + +6. **Edge on RouterOS 6.49.18.** ROS6 is in extended support only; + 6.49.18 is the last 6.x release. Plan a ROS7 migration: + - New CCR2004-1G-12S+2XS supports ROS7 fine. + - OSPF SHA-512 auth IDs need to match across the fleet on cutover. + - BGP/IPv6 config translates with minor syntax changes. + - Once on ROS7, MPLS LDP can extend to edge with the same fasttrack- + bypass + LDP-add pattern used elsewhere. + +### Resilience + +7. **Climax↔culleoka direct AF11 down** (power injector unplugged). When + restored, OSPF reconverges to the shorter path; want LDP+bypass + pre-staged on `climax ether3-culleoka-11ghz` ↔ `culleoka + ether1-climax-11ghz` so the labeled domain follows. + +8. **Charter IPv6 BGP** session in `connect` — pending Charter side + config. The Charter ticket is open. Re-test + `/routing/bgp/peer twc-v6` weekly until they action it. + +9. **No iBGP/route reflectors in active use.** Today OSPF carries + everything, which is fine at 9 routers. If the network grows past + ~15-20 routers or you start carrying public BGP customer prefixes, + consider standing up iBGP between the spine routers (verona, climax, + core) with one of them as a route reflector. Already scaffolded on + core (`bgp-instance-1`) but disabled. + +### MPLS specifics + +10. **LDP advertise-filter is default.** Type-5 externals (like graham's + /27, customer-facing /29s) aren't labeled, which means return-path + traffic to those is plain IP through the labeled domain — works but + isn't what MPLS people typically expect. If you ever want + full-label-everywhere (e.g., for VPN-ish features), explicitly add + these prefixes to LDP advertise-filter. + +11. **mpls-mtu=1508** is fine for current single-label use. If you ever + add stacked labels (BGP-LU, VPN, FRR), bump to 1512+ on every spine + interface. + +12. **FastTrack bypass rules** are on every MPLS-enabled interface today. + When adding a new tower link, the must-do checklist is: + 1. Set l2mtu correctly on both sides (≥1508 for IP+label). + 2. Add `accept` rules above `fasttrack-connection` for both + `in-interface=` and `out-interface=` of the new link. + 3. Add `/mpls ldp` instance if greenfield, plus `/mpls ldp interface` + and `/mpls interface mpls-mtu=1508` on both ends. + 4. Verify LDP `operational=true` and traceroute shows ``. + +### Configuration consistency + +13. **Mixed l2mtu across the spine** (1580/2024/9000) is a function of + radio/wired/60GHz hardware capability, but worth standardizing for + operator clarity. Documenting per-link in CLAUDE.md (just done) is + the practical fix. + +14. **Verona's hotspot covers `204.110.188.0/22`** which spans far more + than the customer realm. Anyone with a public IP in that /22 risks + being intercepted by the captive portal until they're on the bypass + list. Specifically: graham's home /27 was added today, but any other + /27 in the /22 needs the same treatment if it's a non-hotspot user. + Audit `204.110.188.0/22` allocations and pre-bypass any + static-IP customers. + +15. **OSPF `redistribute=connected` is broad.** Every connected interface + becomes a Type-5 LSA — including transient/CPE interfaces. Consider + a tighter route-filter chain on `ospf-out` that limits which + connected prefixes are advertised (loopbacks, mgmt subnets, /29 + backbone, public /27s — exclude transient CPE). + +### Customer service / capacity + +16. **Verona at 46 PPPoE customers + hotspot + heavy NAT** is the + highest-load tower. CCR2004 has plenty of headroom (CPU ≤6% at + snapshot), but if growth continues, splitting some customer load to + altoga (which has APs but no router — currently parented to verona) + would distribute work. Adding a Mikrotik at altoga is a tower-bag + upgrade; the access points are already there. + +17. **Culleoka with 185 DHCP leases** is the second-busiest customer + site. Monitoring for DHCP pool exhaustion is worth adding (the + current `100.64.96.0/20` pool fits 4096 leases so plenty of room, + but watch trend). + +--- + +# Files + +- `/Users/graham/dev/network/mikrotik-tool/*.rsc` — full router exports +- `/Users/graham/dev/network/mikrotik-tool/inventory.yaml` — APs, switches, backhauls, CGNAT subnets per tower +- `/Users/graham/dev/network/mikrotik-tool/subnets.yaml` — per-router /20 + /22 mapping +- `/Users/graham/dev/network/mikrotik-tool/routers.yaml` — connection metadata (host, port, creds source) +- `/Users/graham/dev/network/mikrotik-tool/mpls.md` — MPLS deployment runbook (per-router commands + verification) +- `/Users/graham/dev/network/mikrotik-tool/ipv6.md` — IPv6 allocation plan (in NetBox IPAM as of earlier work) +- `/Users/graham/dev/network/CLAUDE.md` — project-wide guidelines including topology + MPLS/FastTrack caveats +- towerops Postgres — `devices`, `snmp_devices`, `snmp_interfaces`, + `snmp_neighbors`, `alerts`, `monitoring_checks`, etc. diff --git a/mikrotik-tool/newhope.rsc b/mikrotik-tool/newhope.rsc index 473e9d3..3eafc55 100644 --- a/mikrotik-tool/newhope.rsc +++ b/mikrotik-tool/newhope.rsc @@ -1,4 +1,4 @@ -# 2026-05-08 17:47:56 by RouterOS 7.21.4 +# 2026-05-09 08:51:42 by RouterOS 7.21.4 # software id = 5HTF-YFWV # # model = CCR1009-7G-1C-1S+ diff --git a/mikrotik-tool/verona-v6-apply.rsc b/mikrotik-tool/verona-v6-apply.rsc new file mode 100644 index 0000000..4ff60d0 --- /dev/null +++ b/mikrotik-tool/verona-v6-apply.rsc @@ -0,0 +1,154 @@ +# ============================================================================= +# verona — IPv6 apply script (v4-transport iBGP, no OSPFv3) +# ============================================================================= +# +# Design choices vs ipv6.md plan: +# * Drops OSPFv3 entirely. iBGP TCP session runs over the existing v4 +# loopback (10.254.254.101 ↔ 10.254.254.254) — v4 OSPF already provides +# reachability between v4 loopbacks, so no v6 IGP is needed for the +# session itself. The session carries IPv6 NLRI (MP-BGP, RFC 4760). +# * v6 forwarding uses a static default route via climax's v6 P2P side +# (2606:1c80:0:20::2). That gateway becomes reachable when climax's +# v6 apply runs; until then v6 customer traffic will fail past verona +# — known and accepted while the rollout proceeds tower-by-tower. +# * iBGP next-hop is forced to self (verona's v6 loopback). Edge will +# need a route to 2606:1c80:0:10::101/128 once edge is configured; +# handled in the edge apply step. +# +# Apply order: review → paste into a winbox/SSH session on verona → +# verify with the checks at the bottom. +# ============================================================================= + +# ----------------------------------------------------------------------------- +# WIPE — remove all existing internal v6 + OSPFv3 state. Leaves v4 alone. +# ----------------------------------------------------------------------------- +/ppp profile set [find] remote-ipv6-prefix-pool="" dhcpv6-pd-pool="" +/ipv6 dhcp-server remove [find] +/ipv6 dhcp-server option remove [find] +/ipv6 dhcp-client remove [find] +/ipv6 nd prefix remove [find] +/ipv6 nd remove [find !default] +/ipv6 pool remove [find] +/ipv6 route remove [find !dynamic] +/ipv6 firewall filter remove [find] +/ipv6 firewall mangle remove [find] +/ipv6 firewall raw remove [find] +/ipv6 address remove [find !dynamic] + +# OSPFv3 — remove the v3 instance, area, interface-template entry, leave v2 alone. +/routing ospf interface-template remove [find area=backbone-v3] +/routing ospf area remove [find name=backbone-v3] +/routing ospf instance remove [find name=default-v3] + +# Any leftover v6 BGP — wipe internal-only, never touch upstream peers. +/routing bgp connection remove [find name~"ibgp" and address-families~"ipv6"] + +# ----------------------------------------------------------------------------- +# APPLY — addresses +# ----------------------------------------------------------------------------- +/ipv6 settings set forward=yes accept-router-advertisements=no + +/ipv6 address +add address=2606:1c80:0:10::101/128 interface=lo advertise=no comment="loopback" +add address=2606:1c80:0:20::1/127 interface=ether3-climax-11ghz advertise=no comment="P2P verona<->climax" +add address=2606:1c80:1000::1/64 interface=verona advertise=yes comment="verona mgmt LAN" +add address=2606:1c80:1500::1/64 interface=ether6-switch advertise=yes comment="altoga mgmt LAN" + +# ----------------------------------------------------------------------------- +# APPLY — static routes (replaces what OSPFv3 would have done) +# ----------------------------------------------------------------------------- +/ipv6 route +add dst-address=::/0 gateway=2606:1c80:0:20::2 distance=1 comment="default via climax P2P" +add dst-address=2606:1c80:0:10::254/128 gateway=2606:1c80:0:20::2 distance=1 comment="edge loopback (iBGP target reachability fallback)" + +# Blackhole the /44 aggregates so iBGP has something to advertise. +# (ROS7 syntax: `blackhole` is a flag, not a `type=` value.) +add dst-address=2606:1c80:1000::/44 blackhole distance=200 comment="verona /44 aggregate (iBGP origin)" +add dst-address=2606:1c80:1500::/44 blackhole distance=200 comment="altoga /44 aggregate (iBGP origin)" + +# ----------------------------------------------------------------------------- +# APPLY — neighbour discovery / RA on mgmt LANs (SLAAC + RDNSS) +# ----------------------------------------------------------------------------- +/ipv6 nd +add interface=verona ra-interval=3m20s-10m managed-address-configuration=no other-configuration=yes advertise-dns=yes +add interface=ether6-switch ra-interval=3m20s-10m managed-address-configuration=no other-configuration=yes advertise-dns=yes + +/ipv6 nd prefix +add interface=verona prefix=2606:1c80:1000::/64 autonomous=yes on-link=yes +add interface=ether6-switch prefix=2606:1c80:1500::/64 autonomous=yes on-link=yes + +# Stateless DHCPv6 on each mgmt LAN — only carries DNS + search-domain options. +/ipv6 dhcp-server +add interface=verona name=verona-mgmt-stateless address-pool=static-only +add interface=ether6-switch name=altoga-mgmt-stateless address-pool=static-only + +/ipv6 dhcp-server option +add name=v6-dns code=23 value="'2606:4700:4700::1111''2606:4700:4700::1001'" + +# ----------------------------------------------------------------------------- +# APPLY — customer PD pools and PPP profile bindings +# ----------------------------------------------------------------------------- +/ipv6 pool +add name=verona-cust-pd-1 prefix=2606:1c80:1001::/48 prefix-length=56 +add name=altoga-cust-pd-1 prefix=2606:1c80:1501::/48 prefix-length=56 + +/ppp profile +set [find name=pppoe-verona] remote-ipv6-prefix-pool=verona-cust-pd-1 dhcpv6-pd-pool=verona-cust-pd-1 +set [find name=pppoe-altoga] remote-ipv6-prefix-pool=altoga-cust-pd-1 dhcpv6-pd-pool=altoga-cust-pd-1 + +# ----------------------------------------------------------------------------- +# APPLY — firewall (mirror v4 hygiene; deny by default) +# ----------------------------------------------------------------------------- +/ipv6 firewall filter +add chain=input action=accept connection-state=established,related comment="established" +add chain=input action=drop connection-state=invalid comment="invalid" +add chain=input action=accept protocol=icmpv6 comment="icmpv6" +add chain=input action=accept src-address=fe80::/10 comment="link-local" +add chain=input action=accept src-address=2606:1c80::/32 comment="internal" +add chain=input action=drop comment="default deny" + +add chain=forward action=accept connection-state=established,related comment="established" +add chain=forward action=drop connection-state=invalid comment="invalid" +add chain=forward action=accept protocol=icmpv6 hop-limit=equal:1 comment="PMTUD" +add chain=forward action=accept protocol=icmpv6 comment="icmpv6" +add chain=forward action=accept src-address=2606:1c80::/32 dst-address=2606:1c80::/32 comment="internal-to-internal" +add chain=forward action=accept in-interface-list=customer comment="v6 outbound from customers" +add chain=forward action=drop comment="default deny" + +# ----------------------------------------------------------------------------- +# APPLY — iBGP to edge over v4 transport, carrying v6 NLRI +# ----------------------------------------------------------------------------- +# Output filter: only advertise verona+altoga /44 aggregates over this session. +/routing filter rule +add chain=v6-ibgp-out disabled=no rule="if (dst==2606:1c80:1000::/44 || dst==2606:1c80:1500::/44) { accept; } else { reject; }" + +/routing bgp connection +add name=ibgp-edge-v6 \ + remote.address=10.254.254.254 \ + remote.as=393837 \ + local.address=10.254.254.101 \ + local.role=ibgp \ + address-families=ipv6 \ + multihop=yes \ + output.filter-chain=v6-ibgp-out \ + nexthop-choice=force-self \ + comment="iBGP to edge, v4 transport / v6 NLRI" + +# ----------------------------------------------------------------------------- +# VERIFY — paste these after the apply +# ----------------------------------------------------------------------------- +# /ipv6 address print +# /ipv6 route print where active +# /ipv6 nd print +# /ipv6 pool print +# /ppp profile print where name~"pppoe-" +# /routing bgp connection print +# /routing bgp session print +# +# Once climax has its v6 apply done: +# /ping 2606:1c80:0:20::2 count=3 ;; climax P2P side +# /ping 2606:1c80:0:10::102 count=3 ;; climax loopback +# Once edge is done end-to-end: +# /ping 2606:1c80:0:10::254 count=3 ;; edge loopback +# /ping 2606:4700:4700::1111 count=3 ;; cloudflare end-to-end +# /routing bgp session print ;; ibgp-edge-v6 should be Established diff --git a/mikrotik-tool/verona.rsc b/mikrotik-tool/verona.rsc index 98ce617..249a9de 100644 --- a/mikrotik-tool/verona.rsc +++ b/mikrotik-tool/verona.rsc @@ -1,4 +1,4 @@ -# 2026-05-08 17:47:56 by RouterOS 7.21.4 +# 2026-05-09 08:51:42 by RouterOS 7.21.4 # software id = Y1CT-1WB1 # # model = CCR2004-16G-2S+ @@ -174,8 +174,13 @@ add address-pool=verona-cgnat addresses-per-mac=unlimited interface=verona \ name=hotspot1 profile=hsprof1 /ip smb users set [ find default=yes ] disabled=yes +/ipv6 dhcp-server option +add code=23 name=v6-dns value="'2606:4700:4700::1111''2606:4700:4700::1001'" +/ipv6 pool +add name=verona-cust-pd-1 prefix=2606:1c80:1001::/48 prefix-length=56 +add name=altoga-cust-pd-1 prefix=2606:1c80:1501::/48 prefix-length=56 /ppp profile -add change-tcp-mss=yes dhcpv6-pd-pool=verona-v6-pd-pool dns-server=\ +add change-tcp-mss=yes dhcpv6-pd-pool=verona-cust-pd-1 dns-server=\ 204.110.191.240,204.110.191.250 idle-timeout=1h local-address=\ 100.64.15.253 name=pppoe-verona on-down="{\ \n:local url \"https://xtjlpnubrg.execute-api.us-east-1.amazonaws.com/prod\ @@ -285,10 +290,11 @@ add change-tcp-mss=yes dhcpv6-pd-pool=verona-v6-pd-pool dns-server=\ \n }\ \n} while ( \$attempts < \$max )\ \n}\ - \n" remote-address=verona-cgnat remote-ipv6-prefix-pool=verona-v6-pd-pool \ + \n" remote-address=verona-cgnat remote-ipv6-prefix-pool=verona-cust-pd-1 \ use-upnp=no -add change-tcp-mss=yes dns-server=204.110.191.240,204.110.191.250 \ - idle-timeout=1h local-address=100.64.15.253 name=pppoe-altoga on-down="{\ +add change-tcp-mss=yes dhcpv6-pd-pool=altoga-cust-pd-1 dns-server=\ + 204.110.191.240,204.110.191.250 idle-timeout=1h local-address=\ + 100.64.15.253 name=pppoe-altoga on-down="{\ \n:local url \"https://xtjlpnubrg.execute-api.us-east-1.amazonaws.com/prod\ uction/on-start-workflow-execution/6f19a48f-04f7-40c1-a4e0-8cb66df99213/0a\ 7f4443-fc8a-4fe0-807c-f535d2aa1865\"\ @@ -396,7 +402,8 @@ add change-tcp-mss=yes dns-server=204.110.191.240,204.110.191.250 \ \n }\ \n} while ( \$attempts < \$max )\ \n}\ - \n" remote-address=altoga-cgnat use-upnp=no + \n" remote-address=altoga-cgnat remote-ipv6-prefix-pool=altoga-cust-pd-1 \ + use-upnp=no /queue type add kind=fq-codel name=FQ_Codel /queue interface @@ -425,11 +432,8 @@ add disabled=no id=10.254.254.101 name=id-1 select-dynamic-id="" /routing ospf instance add disabled=no in-filter-chain=ospf-in name=default-v2 originate-default=\ never out-filter-chain=ospf-out redistribute=connected router-id=id-1 -add disabled=no in-filter-chain=ospf-in name=default-v3 out-filter-chain=\ - ospf-out router-id=id-1 version=3 /routing ospf area add disabled=no instance=default-v2 name=backbone-v2 -add disabled=no instance=default-v3 name=backbone-v3 /snmp community set [ find default=yes ] addresses=204.110.188.0/22,10.0.0.0/8 name=\ kdyyJrT0Mm @@ -451,14 +455,20 @@ add bridge=*1F interface=*23 internal-path-cost=10 path-cost=10 add bridge=cpe_vlan_10 interface=vlan10_ether15 add bridge=cpe_vlan_10 interface=ether10-powerswitch /ip firewall connection tracking +# ipv6 *accept router advertisements* configuration has changed, please restart device to apply settings set icmp-timeout=30s tcp-close-wait-timeout=1m tcp-established-timeout=4h \ tcp-fin-wait-timeout=2m tcp-last-ack-timeout=30s \ tcp-syn-received-timeout=1m tcp-syn-sent-timeout=2m \ tcp-time-wait-timeout=2m udp-stream-timeout=2m /ip neighbor discovery-settings +# ipv6 *accept router advertisements* configuration has changed, please restart device to apply settings set discover-interface-list=!dynamic /ip settings +# ipv6 *accept router advertisements* configuration has changed, please restart device to apply settings set tcp-syncookies=yes +/ipv6 settings +# ipv6 *accept router advertisements* configuration has changed, please restart device to apply settings +set accept-router-advertisements=no /interface pppoe-server server add default-profile=pppoe-verona disabled=no interface=verona max-mru=1500 \ max-mtu=1500 one-session-per-host=yes service-name=verona @@ -888,6 +898,15 @@ add disabled=no dst-address=10.43.0.0/16 gateway=204.110.191.1 add disabled=no dst-address=10.0.16.1/32 gateway=204.110.188.225 add disabled=no dst-address=10.0.16.10/32 gateway=204.110.188.225 add disabled=no dst-address=10.0.16.84/32 gateway=204.110.188.225 +/ipv6 route +add comment="default via climax P2P" distance=1 dst-address=::/0 gateway=\ + 2606:1c80:0:20::2 +add comment="edge loopback (iBGP target reachability fallback)" distance=1 \ + dst-address=2606:1c80:0:10::254/128 gateway=2606:1c80:0:20::2 +add blackhole comment="verona /44 aggregate (iBGP origin)" distance=200 \ + dst-address=2606:1c80:1000::/44 +add blackhole comment="altoga /44 aggregate (iBGP origin)" distance=200 \ + dst-address=2606:1c80:1500::/44 /ip service set ftp address=204.110.188.0/22,10.0.0.0/8 disabled=yes set telnet address=204.110.188.0/22,10.0.0.0/8 disabled=yes @@ -899,15 +918,41 @@ set api address=204.110.188.0/22,10.0.0.0/8,100.64.0.0/10 set api-ssl certificate=myCa /ip ssh set host-key-type=ed25519 password-authentication=yes strong-crypto=yes +/ipv6 address +add address=2606:1c80:0:10::101/128 advertise=no comment=loopback interface=\ + lo +add address=2606:1c80:0:20::1/127 advertise=no comment="P2P verona<->climax" \ + interface=ether3-climax-11ghz +add address=2606:1c80:1000::1 comment="verona mgmt LAN" interface=verona +add address=2606:1c80:1500::1 comment="altoga mgmt LAN" interface=\ + ether6-switch /ipv6 dhcp-server -add interface=verona lease-time=10m name=server1 prefix-pool=\ - verona-v6-pd-pool +add interface=verona name=verona-mgmt-stateless +add interface=ether6-switch name=altoga-mgmt-stateless +/ipv6 firewall filter +add action=accept chain=input comment=established connection-state=\ + established,related +add action=drop chain=input comment=invalid connection-state=invalid +add action=accept chain=input comment=icmpv6 protocol=icmpv6 +add action=accept chain=input comment=link-local src-address=fe80::/10 +add action=accept chain=input comment=internal src-address=2606:1c80::/32 +add action=drop chain=input comment="default deny" +add action=accept chain=forward comment=established connection-state=\ + established,related +add action=drop chain=forward comment=invalid connection-state=invalid +add action=accept chain=forward comment=PMTUD hop-limit=equal:1 protocol=\ + icmpv6 +add action=accept chain=forward comment=icmpv6 protocol=icmpv6 +add action=accept chain=forward comment=internal-to-internal dst-address=\ + 2606:1c80::/32 src-address=2606:1c80::/32 +add action=drop chain=forward comment="default deny" /ipv6 nd set [ find default=yes ] advertise-dns=yes add advertise-dns=yes interface=verona managed-address-configuration=yes \ other-configuration=yes +add advertise-dns=yes interface=ether6-switch other-configuration=yes /ipv6 nd prefix -add autonomous=no interface=verona +add interface=ether6-switch prefix=2606:1c80:1500::/64 /mpls interface add interface=ether3-climax-11ghz mpls-mtu=1508 /mpls ldp @@ -932,10 +977,11 @@ add disabled=no interfaces=all min-rx=200ms min-tx=200ms multiplier=5 /routing filter rule add chain=ospf-in disabled=no rule="accept;" add chain=ospf-out disabled=no rule="accept;" +add chain=v6-ibgp-out disabled=no rule="if (dst==2606:1c80:1000::/44 || dst==2\ + 606:1c80:1500::/44) { accept; } else { reject; }" /routing ospf interface-template add area=backbone-v2 auth=sha512 auth-id=1 cost=10 disabled=no interfaces=\ ether3-climax-11ghz priority=1 type=ptp use-bfd=no -add area=backbone-v3 cost=10 disabled=no passive use-bfd=no add area=backbone-v2 disabled=no passive /routing ospf static-neighbor add address=10.250.1.30%ether3-climax-11ghz area=backbone-v2 disabled=no \