From 56590d40e19be81750068db942d9c6d80383e788 Mon Sep 17 00:00:00 2001 From: Joshua Gilman Date: Fri, 21 Aug 2026 18:07:42 -0700 Subject: [PATCH 1/2] feat(routeros): add VLAN 30 and lab LACP bonds on sw-core01 Storage VLAN 30 is L2-only on the three lab 802.3ad bonds and nas01 sfp-sfpplus7. Apply in two steps: destroy the six per-port bridge rows first, then create the bonds. RouterOS rejects enslaving a current bridge port, and tofu cannot order those orphan destroys before the bond creates. --- routeros/sw-core01/README.md | 44 +++++++++++++++++++++++++++++++++++ routeros/sw-core01/bonding.tf | 34 +++++++++++++++++++++++++++ routeros/sw-core01/bridge.tf | 44 ++++++++++++++--------------------- 3 files changed, 95 insertions(+), 27 deletions(-) create mode 100644 routeros/sw-core01/bonding.tf diff --git a/routeros/sw-core01/README.md b/routeros/sw-core01/README.md index be47219..9bbb62a 100644 --- a/routeros/sw-core01/README.md +++ b/routeros/sw-core01/README.md @@ -55,6 +55,50 @@ CI never decrypts secrets and never contacts the device. Run `just plan` before every change, after any RouterOS upgrade, and ad hoc. There is no CI drift job. +## First apply of lab LACP + VLAN 30 + +RouterOS rejects enslaving an interface that is still a bridge port. +terraform-routeros v1.99.1 bonding Create is a plain REST POST; it does +not unslave first. `depends_on` cannot name the six `sfp-sfpplus1`–`6` +bridge-port resources this change deletes, so OpenTofu will not destroy +those orphans before creating the bonds. + +An untargeted plan of this change is **7 to add, 6 to destroy**. Do not +apply that mixed plan: RouterOS will reject the bond creates while the +old port rows still exist. + +1. Snapshot, then destroy only the six per-port rows (they are already + absent from this configuration and remain only in state): + + ```sh + just snapshot + tofu apply \ + -target=routeros_interface_bridge_port.sfp_sfpplus1 \ + -target=routeros_interface_bridge_port.sfp_sfpplus2 \ + -target=routeros_interface_bridge_port.sfp_sfpplus3 \ + -target=routeros_interface_bridge_port.sfp_sfpplus4 \ + -target=routeros_interface_bridge_port.sfp_sfpplus5 \ + -target=routeros_interface_bridge_port.sfp_sfpplus6 + ``` + + Expected plan: **6 to destroy**. + +2. Create the bonds, their bridge ports, and VLAN 30: + + ```sh + just snapshot + just plan + just apply + ``` + + Expected plan: **7 to add** (3 bonds, 3 bond bridge ports, 1 VLAN 30 + row). No changes to `bridge-lab`, port 8, VLAN 10, VLAN 40, or the + mgmt address/route. + +VLAN 30 is L2-only. It is tagged on `bond-lab01`–`03` and `sfp-sfpplus7` +(nas01). It is not tagged on `bridge-lab` or the gw01 trunk +(`sfp-sfpplus8`). + ## Notes - Ethernet names stay at factory (`ether1`, `sfp-sfpplus1`–`8`). Roles and diff --git a/routeros/sw-core01/bonding.tf b/routeros/sw-core01/bonding.tf new file mode 100644 index 0000000..f0aed2e --- /dev/null +++ b/routeros/sw-core01/bonding.tf @@ -0,0 +1,34 @@ +# RouterOS rejects enslaving an interface that is still a bridge port +# ("already added as bridge port"). terraform-routeros v1.99.1 bonding +# Create is a plain REST POST (DefaultCreate); it does not remove the +# slaves from /interface/bridge/port first. depends_on cannot name the +# six sfp-sfpplus1..6 bridge-port resources this change deletes, and +# OpenTofu will not order those orphan destroys before these creates. +# Apply in two steps as described in README.md. + +resource "routeros_interface_bonding" "bond_lab01" { + name = "bond-lab01" + slaves = [routeros_interface_ethernet.sfp_sfpplus1.name, routeros_interface_ethernet.sfp_sfpplus2.name] + mode = "802.3ad" + transmit_hash_policy = "layer-3-and-4" + lacp_rate = "1sec" + comment = "lab01 SFP pair (PHY-012 + PHY-013)" +} + +resource "routeros_interface_bonding" "bond_lab02" { + name = "bond-lab02" + slaves = [routeros_interface_ethernet.sfp_sfpplus3.name, routeros_interface_ethernet.sfp_sfpplus4.name] + mode = "802.3ad" + transmit_hash_policy = "layer-3-and-4" + lacp_rate = "1sec" + comment = "lab02 SFP pair (PHY-014 + PHY-015)" +} + +resource "routeros_interface_bonding" "bond_lab03" { + name = "bond-lab03" + slaves = [routeros_interface_ethernet.sfp_sfpplus5.name, routeros_interface_ethernet.sfp_sfpplus6.name] + mode = "802.3ad" + transmit_hash_policy = "layer-3-and-4" + lacp_rate = "1sec" + comment = "lab03 SFP pair (PHY-016 + PHY-017)" +} diff --git a/routeros/sw-core01/bridge.tf b/routeros/sw-core01/bridge.tf index 87f9944..e21347f 100644 --- a/routeros/sw-core01/bridge.tf +++ b/routeros/sw-core01/bridge.tf @@ -3,44 +3,23 @@ resource "routeros_interface_bridge" "lab" { vlan_filtering = true } -resource "routeros_interface_bridge_port" "sfp_sfpplus1" { +resource "routeros_interface_bridge_port" "bond_lab01" { bridge = routeros_interface_bridge.lab.name - interface = routeros_interface_ethernet.sfp_sfpplus1.name + interface = routeros_interface_bonding.bond_lab01.name frame_types = "admit-only-vlan-tagged" ingress_filtering = true } -resource "routeros_interface_bridge_port" "sfp_sfpplus2" { +resource "routeros_interface_bridge_port" "bond_lab02" { bridge = routeros_interface_bridge.lab.name - interface = routeros_interface_ethernet.sfp_sfpplus2.name + interface = routeros_interface_bonding.bond_lab02.name frame_types = "admit-only-vlan-tagged" ingress_filtering = true } -resource "routeros_interface_bridge_port" "sfp_sfpplus3" { +resource "routeros_interface_bridge_port" "bond_lab03" { bridge = routeros_interface_bridge.lab.name - interface = routeros_interface_ethernet.sfp_sfpplus3.name - frame_types = "admit-only-vlan-tagged" - ingress_filtering = true -} - -resource "routeros_interface_bridge_port" "sfp_sfpplus4" { - bridge = routeros_interface_bridge.lab.name - interface = routeros_interface_ethernet.sfp_sfpplus4.name - frame_types = "admit-only-vlan-tagged" - ingress_filtering = true -} - -resource "routeros_interface_bridge_port" "sfp_sfpplus5" { - bridge = routeros_interface_bridge.lab.name - interface = routeros_interface_ethernet.sfp_sfpplus5.name - frame_types = "admit-only-vlan-tagged" - ingress_filtering = true -} - -resource "routeros_interface_bridge_port" "sfp_sfpplus6" { - bridge = routeros_interface_bridge.lab.name - interface = routeros_interface_ethernet.sfp_sfpplus6.name + interface = routeros_interface_bonding.bond_lab03.name frame_types = "admit-only-vlan-tagged" ingress_filtering = true } @@ -68,6 +47,17 @@ resource "routeros_interface_bridge_vlan" "vlan10" { ] } +resource "routeros_interface_bridge_vlan" "vlan30" { + bridge = routeros_interface_bridge.lab.name + vlan_ids = ["30"] + tagged = [ + routeros_interface_bonding.bond_lab01.name, + routeros_interface_bonding.bond_lab02.name, + routeros_interface_bonding.bond_lab03.name, + routeros_interface_ethernet.sfp_sfpplus7.name, + ] +} + resource "routeros_interface_bridge_vlan" "vlan40" { bridge = routeros_interface_bridge.lab.name vlan_ids = ["40"] From e221db290bfa0e95ea5b90eeb950778b16e65b93 Mon Sep 17 00:00:00 2001 From: Joshua Gilman Date: Fri, 21 Aug 2026 18:46:45 -0700 Subject: [PATCH 2/2] fix(routeros): use 30secs LACP rate to match observed IncusOS PDU cadence Hosts transmit one LACPDU per 30s despite the expected fast rate; 1sec partner timeout (3s) expired mid-interval and churned ports out of the aggregate. 30secs gives the standard 90s timeout and stable aggregation, verified live on all three bonds. --- routeros/sw-core01/bonding.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/routeros/sw-core01/bonding.tf b/routeros/sw-core01/bonding.tf index f0aed2e..1948888 100644 --- a/routeros/sw-core01/bonding.tf +++ b/routeros/sw-core01/bonding.tf @@ -11,7 +11,7 @@ resource "routeros_interface_bonding" "bond_lab01" { slaves = [routeros_interface_ethernet.sfp_sfpplus1.name, routeros_interface_ethernet.sfp_sfpplus2.name] mode = "802.3ad" transmit_hash_policy = "layer-3-and-4" - lacp_rate = "1sec" + lacp_rate = "30secs" comment = "lab01 SFP pair (PHY-012 + PHY-013)" } @@ -20,7 +20,7 @@ resource "routeros_interface_bonding" "bond_lab02" { slaves = [routeros_interface_ethernet.sfp_sfpplus3.name, routeros_interface_ethernet.sfp_sfpplus4.name] mode = "802.3ad" transmit_hash_policy = "layer-3-and-4" - lacp_rate = "1sec" + lacp_rate = "30secs" comment = "lab02 SFP pair (PHY-014 + PHY-015)" } @@ -29,6 +29,6 @@ resource "routeros_interface_bonding" "bond_lab03" { slaves = [routeros_interface_ethernet.sfp_sfpplus5.name, routeros_interface_ethernet.sfp_sfpplus6.name] mode = "802.3ad" transmit_hash_policy = "layer-3-and-4" - lacp_rate = "1sec" + lacp_rate = "30secs" comment = "lab03 SFP pair (PHY-016 + PHY-017)" }