diff --git a/api/src/main/java/com/cloud/network/Network.java b/api/src/main/java/com/cloud/network/Network.java index 2f0bcdd5ef9a..d7b467bab5a7 100644 --- a/api/src/main/java/com/cloud/network/Network.java +++ b/api/src/main/java/com/cloud/network/Network.java @@ -43,7 +43,7 @@ public interface Network extends ControlledEntity, StateObject, InternalIdentity, Identity, Serializable, Displayable { enum GuestType { - Shared, Isolated, L2; + Shared, Isolated, L2, L3; public static GuestType fromValue(String type) { if (StringUtils.isBlank(type)) { @@ -54,6 +54,8 @@ public static GuestType fromValue(String type) { return Isolated; } else if (type.equalsIgnoreCase("L2")) { return L2; + } else if (type.equalsIgnoreCase("L3")) { + return L3; } else { throw new InvalidParameterValueException("Unexpected Guest type : " + type); } diff --git a/api/src/main/java/com/cloud/network/Networks.java b/api/src/main/java/com/cloud/network/Networks.java index 61a1c820723f..03aa13c340d9 100644 --- a/api/src/main/java/com/cloud/network/Networks.java +++ b/api/src/main/java/com/cloud/network/Networks.java @@ -131,6 +131,23 @@ public URI toUri(T value) { } } }, + /** + * Direct Routed (L3) networks: the id is a label naming the per-network bridge on the + * hypervisor (brdr-<id>), not an encapsulation — nothing appears on the wire. + */ + Routed("routed", Long.class) { + @Override + public URI toUri(T value) { + try { + if (value.toString().contains("://")) + return new URI(value.toString()); + else + return new URI("routed://" + value.toString()); + } catch (URISyntaxException e) { + throw new CloudRuntimeException("Unable to convert to broadcast URI: " + value); + } + } + }, UnDecided(null, null), OpenDaylight("opendaylight", String.class), TUNGSTEN("tf", String.class), diff --git a/api/src/main/java/com/cloud/offering/NetworkOffering.java b/api/src/main/java/com/cloud/offering/NetworkOffering.java index 5000a4f8c626..b3ab6961fd75 100644 --- a/api/src/main/java/com/cloud/offering/NetworkOffering.java +++ b/api/src/main/java/com/cloud/offering/NetworkOffering.java @@ -81,6 +81,7 @@ enum RoutingMode { public final static String DefaultL2NetworkOfferingVlan = "DefaultL2NetworkOfferingVlan"; public final static String DefaultL2NetworkOfferingConfigDrive = "DefaultL2NetworkOfferingConfigDrive"; public final static String DefaultL2NetworkOfferingConfigDriveVlan = "DefaultL2NetworkOfferingConfigDriveVlan"; + public final static String DefaultL3NetworkOffering = "DefaultL3NetworkOffering"; /** * @return name for the network offering. diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java index 79fb5f6d01cf..e32ab4e77719 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java @@ -340,10 +340,10 @@ public Long getPhysicalNetworkId() { } } if (physicalNetworkId != null) { - if ((offering.getGuestType() == GuestType.Shared) || (offering.getGuestType() == GuestType.L2)) { + if ((offering.getGuestType() == GuestType.Shared) || (offering.getGuestType() == GuestType.L2) || (offering.getGuestType() == GuestType.L3)) { return physicalNetworkId; } else { - throw new InvalidParameterValueException("Physical network ID can be specified for networks of guest IP type " + GuestType.Shared + " or " + GuestType.L2 + " only."); + throw new InvalidParameterValueException(String.format("Physical network ID can be specified for networks of guest IP type %s, %s or %s only.", GuestType.Shared, GuestType.L2, GuestType.L3)); } } else { if (zoneId == null) { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RemoveIpFromVmNicCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RemoveIpFromVmNicCmd.java index f4c4d82b30d4..f8bdfe718d1b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RemoveIpFromVmNicCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RemoveIpFromVmNicCmd.java @@ -124,6 +124,16 @@ public NetworkType getNetworkType() { } + /** + * A Direct Routed (L3) network needs the agent told when a secondary IP goes away, so the + * host route and neighbour entry are removed - otherwise the host keeps routing an address + * the Instance no longer owns, and the routing daemon keeps advertising it. + */ + private boolean isDirectRoutedNetwork() { + Network ntwk = _entityMgr.findById(Network.class, getNetworkId()); + return ntwk != null && Network.GuestType.L3.equals(ntwk.getGuestType()); + } + private boolean isZoneSGEnabled() { Network ntwk = _entityMgr.findById(Network.class, getNetworkId()); DataCenter dc = _entityMgr.findById(DataCenter.class, ntwk.getDataCenterId()); @@ -144,7 +154,7 @@ public void execute() throws InvalidParameterValueException { secIp = nicSecIp.getIp6Address(); } - if (isZoneSGEnabled()) { + if (isZoneSGEnabled() || isDirectRoutedNetwork()) { //remove the security group rules for this secondary ip boolean success = false; success = _securityGroupService.securityGroupRulesForVmSecIp(nicSecIp.getNicId(), secIp, false); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmdTest.java index 3f5b75828025..8b1b8618094b 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmdTest.java @@ -251,7 +251,7 @@ public void testGetPhysicalNetworkIdForNonSharedNet() { try { cmd.getPhysicalNetworkId(); } catch (Exception e) { - Assert.assertTrue(e.getMessage().startsWith("Physical network ID can be specified for networks of guest IP type Shared or L2 only.")); + Assert.assertTrue(e.getMessage().startsWith("Physical network ID can be specified for networks of guest IP type Shared, L2 or L3 only.")); } } diff --git a/core/src/main/java/com/cloud/agent/api/NetworkRulesVmSecondaryIpCommand.java b/core/src/main/java/com/cloud/agent/api/NetworkRulesVmSecondaryIpCommand.java index c0752a1e97e5..ed6949a48095 100644 --- a/core/src/main/java/com/cloud/agent/api/NetworkRulesVmSecondaryIpCommand.java +++ b/core/src/main/java/com/cloud/agent/api/NetworkRulesVmSecondaryIpCommand.java @@ -28,12 +28,28 @@ public class NetworkRulesVmSecondaryIpCommand extends Command { private String vmSecIp; private String vmMac; private String action; + private boolean directRouted; + private boolean applySecurityGroupRules = true; public NetworkRulesVmSecondaryIpCommand(String vmName, VirtualMachine.Type type) { this.vmName = vmName; this.type = type; } + public NetworkRulesVmSecondaryIpCommand(String vmName, String vmMac, String secondaryIp, boolean action, boolean directRouted, boolean applySecurityGroupRules) { + this(vmName, vmMac, secondaryIp, action); + this.directRouted = directRouted; + this.applySecurityGroupRules = applySecurityGroupRules; + } + + public boolean isDirectRouted() { + return directRouted; + } + + public boolean isApplySecurityGroupRules() { + return applySecurityGroupRules; + } + public NetworkRulesVmSecondaryIpCommand(String vmName, String vmMac, String secondaryIp, boolean action) { this.vmName = vmName; this.vmMac = vmMac; diff --git a/docs/design/direct-routed-networks.md b/docs/design/direct-routed-networks.md new file mode 100644 index 000000000000..4e4787977c9a --- /dev/null +++ b/docs/design/direct-routed-networks.md @@ -0,0 +1,1511 @@ +# Direct Routed Networks + +**Status:** design agreed — open items are implementation and verification only +**Branch:** `direct-routed-network` +**Author:** Wido den Hollander +**Last updated:** 2026-09-04 + +> **Revision 2026-09-04.** The isolation model changed: direct routed networks now live on a +> **dedicated physical network with isolation method `ROUTED`**, and every network carries a +> broadcast domain of the new type **`routed://`**, whose id names the network's bridge +> (`brdr-`). This reverses two earlier decisions — "no isolation method" and "no broadcast +> domain" — recorded with rationale in §6.7 and §9.2.1 and in the decision log (§16). + +--- + +## 1. Summary + +A new guest network type in which the **hypervisor performs L3 routing for the guest**. There is +no Virtual Router, no NAT, and **no DHCP** — the defining characteristic of this network type. + +The operator creates a network and adds a subnet to it, as for a Shared network. CloudStack then +allocates **individual addresses** out of that subnet to guest NICs, and hands each NIC: + +* its IPv4 address as a **/32** and its IPv6 address as a **/128** +* a **shared, host-independent gateway**: `169.254.0.1` (IPv4) and `fe80::1` (IPv6), configured on + the network's bridge on every hypervisor +* the whole configuration via **ConfigDrive / cloud-init** — mandatory, since without DHCP or RA + there is no other way for the guest to learn its address + +The hypervisor's CloudStack agent installs, per guest address, a host route and a static neighbour +entry on the guest bridge, binding the address to that guest's MAC. This reuses the existing +`modifymacip.sh` hook unchanged (§9.1). A routing daemon on the host advertises those addresses to +the fabric. **The routing daemon is out of scope for this feature** — see §10. + +Direct routed networks are the network operator's choice, expressed in the zone design: they live +on a **dedicated physical network whose isolation method is `ROUTED`**. Each network carries a +broadcast domain of the new type **`routed://`** — for example `routed://5828` — and that id +names the network's bridge on every hypervisor: `brdr-5828`. The id is not an encapsulation and +never appears on the wire; it is the stable, operator-controlled handle that ties a network to +host-side routing policy. The operator either picks it at network creation or lets CloudStack +allocate one from the range configured on the physical network (§6.7, §9.2). + +## 2. Motivation + +Compared with what CloudStack offers today: + +| | Shared | Isolated (NATTED) | Isolated (ROUTED, 4.20+) | **Direct Routed** | +|---|---|---|---|---| +| VR in data path | for DHCP/DNS | yes | yes | **no** | +| DHCP | yes | yes | yes | **no** | +| VM gets routable IP | yes | no (NAT) | yes | yes | +| Guest netmask | subnet mask | subnet mask | subnet mask | **/32, /128** | +| Shared broadcast domain | yes | yes | yes | **no** | +| Guest-to-guest | switched | switched | switched | **routed by host** | +| Isolation ID | VLAN/VXLAN | VLAN/VXLAN | VLAN/VXLAN | **routed id — a label, no encapsulation** | + +What this closes: + +* **No VR in the data path.** `ROUTED` networks (4.20) removed NAT but kept a VR in the path and an + L2 segment per network. Throughput, failover, and per-network VR footprint remain concerns. +* **No shared broadcast domain between networks.** Each network gets its own uplink-less bridge + (§9.2), so there is no L2 path of any kind between networks — no ARP spoofing, no rogue DHCP + server, no rogue RA reaching another tenant. Isolation is topological, so it does not depend on a + rule set being correct, and an administrator can turn security groups off for a network without + weakening it. Within a single network guests still share a bridge; that residual exposure is + intra-tenant and is documented in §12.3. +* **No encapsulation at all.** No VLAN on the wire, no VXLAN, no tunnel. The routed id in + `routed://` is a **label, not a fabric resource**: it exists only to name the per-network + bridge (`brdr-`), consumes nothing on the switches, and is either chosen by the operator or + allocated from a range the operator configures on the `ROUTED` physical network (§6.7, §9.2.1). + Guest network count is bounded only by that range, and each network still gets a real L2 boundary. +* **Per-network routing policy on the hypervisor.** Each network is a distinct, named L3 interface + (`brdr-5828`), so the operator can apply different route-maps, redistribution filters, policy + routing or QoS per network in the host's own configuration. CloudStack does not need to know or + care; it is entirely a local network design decision (§9.2). Because the operator can pick the + routed id at network creation, the interface name is **plannable in advance** — host policy can + exist before the network does (§6.7.2). +* **IP mobility.** Because the gateway is identical on every hypervisor, the guest's network + configuration is entirely host-independent. A VM can start, stop, and migrate anywhere in the + routing domain with no reconfiguration — its address follows it as an advertised route. +* **Fits L3-to-the-host fabrics.** Operators already running BGP or OSPF on the hypervisor get an + addressing model that matches their fabric instead of fighting it with stretched VLANs. +* **Denser subnet use.** No broadcast domain means no network or broadcast address to reserve — + every address in the subnet is usable (§6.3.1). + +## 3. Non-goals + +* Not a replacement for `NetworkMode.ROUTED` + BGP-per-network (4.20). That stays. +* No NAT, source NAT, static NAT, port forwarding, LB, or VPN in this network type. +* No VR at all — not even for DHCP/DNS/UserData. +* No DHCP, DHCPv6, or SLAAC/RA for guest addressing. +* No support for guests that cannot consume ConfigDrive (§6.5). +* **No routing-daemon management.** CloudStack does not install, configure, or monitor FRR/BIRD + (§10). +* No per-tenant VRFs, and therefore no overlapping subnets between networks (§6.3.2). +* **Never part of a VPC** — VPC already covers BGP-routed subnets with a VR; this is the different + case of a public address on the Instance itself (§6.6). + +## 4. Terminology + +| Term | Meaning | +|---|---| +| Direct routed network | The new guest network type described here | +| `ROUTED` | The new isolation method; a physical network carrying it is where direct routed networks live (§6.7) | +| Routed id | The number in the network's `routed://` broadcast URI — operator-chosen at creation, or allocated from the `ROUTED` physical network's id range (§6.7.2) | +| `brdr-` | Bridge-DirectRouted: the uplink-less per-network bridge, named from the network's routed id; created by `modifybrdr.sh` | +| Shared gateway | `169.254.0.1` / `fe80::1`, present on **every** `brdr-*` bridge on **every** host | +| Host route | `ip route replace /32 dev `, installed by `modifymacip.sh` | +| Static neighbour | `ip neigh replace lladdr dev nud permanent` | +| Routing daemon | FRR/BIRD/other, run and configured by the operator — not by CloudStack | + +## 5. Network model + +### 5.1 Addressing + +* Guest NIC: `203.0.113.55/32`, `2001:db8:1::55/128` +* Guest default route: `default via 169.254.0.1 dev eth0 onlink` and `default via fe80::1 dev eth0` +* Host guest bridge: `169.254.0.1/32` and `fe80::1/64` — identical on every host +* Host, per guest address: a /32 (or /128) route and a permanent neighbour entry, both on the + **bridge** (§9.1.2) + +### 5.2 Packet walk — guest to elsewhere + +1. The guest has no on-link neighbours (it is a /32) → everything goes to the default route. +2. The guest ARPs for `169.254.0.1` — permitted because the route is `onlink` — or ND-solicits + `fe80::1`. The host bridge answers. +3. The host routes the packet per its own routing table, out to the fabric. + +### 5.3 Packet walk — fabric to guest + +1. The fabric has learned `203.0.113.55/32` from this host via BGP/OSPF. +2. The packet arrives; the host matches `203.0.113.55/32 dev `. +3. The host does **not** need to ARP — `modifymacip.sh` installed a permanent neighbour entry + mapping the address to the guest's MAC. The frame is handed to the bridge, which forwards it to + the port where that MAC was learned. +4. Delivered. + +Static neighbour entries rather than ARP are deliberate: they remove a resolution round-trip from +VM start, make host→guest delivery independent of whether the guest answers ARP, and close off +ARP-based address takeover between guests on the same host. + +Note the address is pinned to a **MAC**, not to a port — the MAC-to-port mapping comes from ordinary +bridge FDB learning. See §12.1 for why that is still sound, and what it depends on. + +### 5.4 Packet walk — guest to guest, same host + +Between guests of **different** networks, the two addresses are on different bridges (§9.2), so the +host routes between them and there is no L2 path at all. + +Between guests of the **same** network, both host routes are local to one bridge and the host +hairpins via that bridge. Either way the traffic passes through the host's forwarding table, and +therefore through security group filtering where those are in use (§12.2). + +Remaining asymmetry: same-host traffic never reaches the fabric, so fabric-level policy does not see +it. Since security groups are optional here, same-network guest-to-guest traffic on one host may be +unfiltered — accepted for v1, see §12.3. + +### 5.5 What the network object looks like + +Like a **Shared** network, not like L2: the network has a subnet. The operator supplies the CIDR, +and optionally an IPv6 prefix, via the usual IP-range mechanism (`vlan` rows — `vlan_gateway`, +`vlan_netmask`, `ip4_range`, `ip6_gateway`, `ip6_cidr`, `ip6_range` in +`engine/schema/src/main/java/com/cloud/dc/VlanVO.java`). + +The difference is what CloudStack does with it: the subnet is an **allocation pool that is routed to +the hypervisors**, not a broadcast domain. Guests never see the subnet mask or its gateway. + +**The subnet gateway is required, and ignored. DECIDED for v1.** + +`vlan_gateway` / `ip6_gateway` are meaningless for this network type — nothing reads them, because +the guest's gateway is always the shared link-local address and the whole subnet is routed to the +hosts. `createVlanIpRange` requires a gateway today, and it stays required: relaxing it would mean +touching validation shared with every other network type for no functional gain. + +The cost is a small wart — the operator has to nominate an address in the subnet that will never be +configured anywhere or answer anything. Two practical notes: + +* It should be **documented as unused**, so nobody wastes time debugging why traffic is not reaching + it, and so nobody assumes reserving it is necessary. +* Whichever address is given still gets consumed from the allocation pool unless explicitly kept + out of the IP range. Since every address in the subnet is otherwise usable (§6.3.1), an operator + can simply give the range as the full subnet and point the gateway at an address inside it — but + confirm existing validation does not reject a gateway that falls within the range. + +Making it optional remains available later if it proves annoying; it is a validation change, not a +model change, so nothing here forecloses it. + +## 6. Design decisions + +### 6.1 How is the network type modelled? **DECIDED — `GuestType.L3`** + +`Network.GuestType` becomes `Shared, Isolated, L2, L3` +(`api/src/main/java/com/cloud/network/Network.java:45`). + +Chosen because it is self-describing and symmetrical with the existing `L2`, and because every +existing `GuestType.L2` / `Shared` branch then becomes an obvious place to decide what `L3` should +do. The cost is accepted: a new enum value touches API responses, UI and upgrade, and roughly a +dozen files special-case `L2` with a similar number special-casing `Shared`. + +Rejected alternatives, for the record: + +* **`GuestType.Shared` plus a flag or `NetworkMode` on the offering** — no new enum, and it would + inherit Shared's subnet and allocation handling for free, but it overloads `NetworkMode.ROUTED` + which already means "VR routes, no NAT". Every Shared code path would have to ask "but is it the + routed kind?", which is worse than a new value. +* **`GuestType.DirectRouted`** — explicit, but mixes a topology concept (`L2`) with a routing one in + the same enum. + +Files to review — those branching on `GuestType.L2`: + +* `server/src/main/java/com/cloud/network/NetworkServiceImpl.java` +* `server/src/main/java/com/cloud/network/NetworkModelImpl.java` +* `server/src/main/java/com/cloud/network/guru/GuestNetworkGuru.java` +* `engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java` +* `engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java` +* `server/src/main/java/com/cloud/vm/UserVmManagerImpl.java` +* `api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java` +* `engine/schema/src/main/java/com/cloud/offerings/dao/NetworkOfferingDaoImpl.java` +* `plugins/network-elements/vxlan/.../VxlanGuestNetworkGuru.java` +* `server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java` + +…plus the `GuestType.Shared` branches, which is where subnet and IP-range handling lives. + +### 6.2 Which gateway address? **DECIDED — static `169.254.0.1` / `fe80::1`** + +Fixed, not configurable. Requirements met: identical on every host, never globally routable, cannot +collide with guest address space. + +`fe80::1` is unambiguously correct — link-local next-hops are the norm in IPv6 and need no special +handling. + +`169.254.0.1` requires `onlink` on the guest route because it falls outside the guest's /32 (§8.1), +and that is fine: **the address is a /32, so the guest has to treat its gateway as on-link whatever +that gateway is.** There is no configuration of the gateway address that would avoid needing +`onlink`, so making it configurable would buy nothing — any guest that can work at all here can use +an on-link next-hop. + +An earlier draft left open whether to add a global setting for operators whose images might refuse a +link-local next-hop. Closed: no setting. Keeping it fixed preserves the "guest configuration is +completely host-independent" property by construction, and removes a knob that could only ever be set +wrong. + +Consequences: + +* §8.1's rule — emit `on-link: true` when the gateway is in `169.254.0.0/16` — is now permanently + sufficient. There is no need to generalise it to "gateway outside the address's own prefix", + because the gateway will never be anything else. +* `modifybrdr.sh` keeps its `-4` / `-6` options with these values as defaults, **deliberately**. + Nothing calls them with anything other than the defaults today, and they are not a CloudStack + setting — but they cost nothing, make the script testable in isolation, and mean a future change of + heart is a caller-side change rather than a script rewrite. They should not be removed as dead + options. + +### 6.3 Where do the addresses come from? **DECIDED** + +Users add a subnet to the network; CloudStack assigns **individual** IPv4 and IPv6 addresses out of +it — but the two families work fundamentally differently, and both mechanisms are reused unchanged: + +* IPv4 → drawn from a **pre-populated pool**: `createVlanIpRange` writes every address of the range + into `user_ip_address` (`engine/schema/src/main/java/com/cloud/network/dao/IPAddressVO.java`) and + allocation marks rows. +* IPv6 → **computed, never pooled**: the address is calculated from the subnet and the NIC's MAC + (EUI-64) at allocation time and stored only on the NIC — see §6.3.4. +* Subnet definition → `vlan` rows, as for Shared networks + +This is the mechanism Shared networks already use: `DirectNetworkGuru.allocateDirectIp()` +(`server/src/main/java/com/cloud/network/guru/DirectNetworkGuru.java:316`) → +`IpAddressManagerImpl.allocateDirectIp()` +(`server/src/main/java/com/cloud/network/IpAddressManagerImpl.java:2434`). + +Consequences, all good: + +* No new tables, no new allocation logic, no new capacity accounting. +* Existing IP reservation, `listPublicIpAddresses`, and quota/usage machinery apply. +* Requested-IP (`ipaddress=` on `deployVirtualMachine`) works for free. + +The one thing the new guru **must** override: `allocateDirectIp()` sets the NIC's gateway and +netmask from the vlan row (lines 2459–2461). For this network type they must instead be forced to +`255.255.255.255` / `169.254.0.1` and `/128` / `fe80::1`. + +#### 6.3.1 Network and broadcast addresses are usable **DECIDED — must work** + +There is no broadcast domain, so the first and last address of a subnet are ordinary, routable, +assignable addresses. `203.0.113.0` and `203.0.113.255` in a /24 are just addresses; nothing +broadcasts to them, and every guest is a /32 behind the host's routing table. + +**They must be assignable.** The gain is proportionally largest exactly where address space is +tightest, which is the common case for this feature: + +| Subnet | Addresses | Usable today (minus network, broadcast, gateway) | Usable here | Gain | +|---|---|---|---|---| +| /29 | 8 | 5 | 7 | +40% | +| /28 | 16 | 13 | 15 | +15% | +| /27 | 32 | 29 | 31 | +7% | +| /24 | 256 | 253 | 255 | +0.8% | + +(The gateway is still deducted because §5.5 keeps it required, even though nothing uses it.) + +**Verified during implementation: no code change was needed.** The earlier draft assumed the +exclusion lived in the `NetUtils` CIDR helpers (`getIpRangeFromCidr()` and friends, whose +`start = (ip & netmask) + 1` / `end - 2` arithmetic does skip the two addresses) and would need +inclusive variants. Tracing the actual path an operator-supplied range takes showed those helpers +are only used where CloudStack *derives* a range from a CIDR — Isolated networks. The explicit +start/end path this network type uses (Shared-style `createVlanIpRange`) validates with +`sameSubnet()` — a plain bitmask comparison that `.0` and `.255` pass — plus start≤end and +gateway-not-in-range, and `savePublicIPRange()` then iterates the range without exclusions into +`user_ip_address`. Allocation is row-based from that table and never re-derives from the CIDR. + +**`.0` and `.255` are therefore already assignable end-to-end on this path.** The smoke test should +still assert it (an Instance actually receiving `.0` or `.255` and working), since this rests on +tracing rather than an existing guarantee anyone maintains. + +#### 6.3.2 Subnets must be unique across the routing domain **DECIDED — constraint** + +Because every direct routed network on a host shares one routing table, and all subnets are +advertised into one fabric, **subnets cannot overlap between networks**. Two tenants cannot both +use `10.0.0.0/24`. + +This follows from §9.2: networks separate tenants in the API and UI, not in address space. There is +no VRF, no per-tenant routing table, and no NAT to hide behind. Addresses must be unique and +routable within the routing domain. + +Implications: + +* Tenants cannot bring their own overlapping RFC1918 space. Operators assign from a pool they + control — public space, or private space that is unique zone-wide. +* **Validation must reject a new subnet that overlaps an existing direct routed subnet anywhere in + the zone. Required — an overlap is an address conflict, not a policy preference.** Two networks + sharing a subnet would produce duplicate /32s in one host routing table and duplicate + advertisements into the fabric, with traffic delivered to whichever Instance the host resolved + last. +* **Implemented and verified:** the IPv6 vlan overlap check was already zone-wide + (`_vlanDao.listByZone()`). IPv4 was not — the `user_ip_address` unique key is + `(public_ip_address, source_network_id)`, i.e. per network — so `createVlanAndPublicIpRange` now + calls the existing zone-wide `checkOverlapPublicIpRange()` for L3 networks. Shared networks keep + their historical behaviour, where the same IPv4 range in two VLANs is legitimate. +* This is the main user-visible limitation of "networks separate tenants administratively only" and + must be explicit in the documentation. +* Per-tenant VRFs would lift the restriction but mean per-VRF routing tables on the host and + per-VRF sessions in the routing daemon — well beyond v1 (§15). + +#### 6.3.3 Route scale **DECIDED — out of scope** + +Each guest address is advertised as an individual /32 or /128, so a zone with 50k Instances puts 50k +routes into the fabric. This is inherent to the design: any aggregation scheme pins addresses to +hosts and breaks migration (§11), so per-address advertisement is deliberate. + +**How that scales is the operator's concern, not CloudStack's.** Fabric capacity, aggregation and +route policy are local network design — the same boundary already drawn around the routing daemon in +§10. CloudStack states no supported ceiling, because it has no way to know one: the answer depends +entirely on the hardware and topology in front of it. + +For context rather than as a commitment: route counts in the order of 100k are not usually a problem +for modern equipment. Operators running L3-to-the-host fabrics are typically already carrying host +routes at that scale. + +#### 6.3.4 IPv6 is calculated from subnet + MAC, never drawn from a stored pool **DECIDED** + +**The IPv6 address of a NIC is computed with EUI-64 from the network's IPv6 subnet and the NIC's +MAC address, at allocation time. CloudStack never stores IPv6 addresses that are *to be* allocated; +it stores only what *was* allocated, on the NIC itself (`nics.ip6_address`).** This applies to +regular Instances and to SystemVMs alike. + +Verified in the code — this is already how CloudStack behaves, on both paths this feature uses: + +* **Guest NICs** (the `DirectNetworkGuru` lineage): `IpAddressManagerImpl.allocateDirectIp()` + (`:2479`) calls `Ipv6AddressManagerImpl.setNicIp6Address()` + (`server/src/main/java/com/cloud/network/Ipv6AddressManagerImpl.java:206`), which computes + `NetUtils.EUI64Address(network.getIp6Cidr(), nic.getMacAddress())` and sets the result on the + `NicProfile`. `DirectRoutedNetworkGuru.applyDirectRoutedAddressing()` then reshapes it to `/128` + + `fe80::1`. +* **Public NICs** (SystemVMs, §8.5): `PublicNetworkGuru.getIp()` calls + `Ipv6ServiceImpl.updateNicIpv6()` (`Ipv6ServiceImpl.java:440`), which selects the range's + `ip6_cidr`, narrows it to a /64 if it is larger, and computes + `NetUtils.EUI64Address(ipv6Network, nicMacAddress)` (`:236`); the reservation is a placeholder + NIC, i.e. again a NIC row, not a pool entry. +* **`user_ipv6_address` is not an allocation pool.** No production code path inserts rows into it + (verified by search: only reads — taken-checks and counts — remain). §6.3's earlier draft listed + it as the IPv6 counterpart of `user_ip_address`; that was wrong and is corrected above. The table + stays untouched by this feature. + +Why this is the right model, stated so it survives review: + +* **Recomputable = host- and database-independent.** Subnet + MAC always yields the same address: + what SLAAC would have produced, what the guest can verify, and what migration preserves for free. +* **Uniqueness needs no coordination.** MACs are unique per network, subnets are unique zone-wide + (§6.3.2), so EUI-64 addresses collide nowhere — with no counter, no lock and no pool to maintain. +* **Nothing to pre-populate or garbage-collect.** A /64 has 2⁶⁴ addresses; a pool table for that is + a non-starter anyway. The IPv4 pool model stays where it belongs, on IPv4. +* A user-requested IPv6 (`ipaddress6=` on deploy) is rejected when it is EUI-64-shaped + (`Ipv6AddressManagerImpl.acquireGuestIpv6Address()`, `:112`), so a manual address can never + collide with a computed one. + +Consequences: + +* **The IPv6 subnet must be /64 or larger.** `NetUtils.EUI64Address()` throws for prefixes longer + than /64 (`NetUtils.java:1731`) — the interface identifier needs 64 bits. Today an L3 network's + IPv6 CIDR skips the Shared-network "/64 exactly" check, which leaves a longer prefix (e.g. /80) + to blow up at Instance deploy instead of at network creation. **Validation to add:** reject an + IPv6 CIDR with a prefix longer than /64 at `createNetwork`/`createVlanIpRange` time for L3 + networks. +* The IPv6 usable-address accounting of §6.3.1 is moot for v6 — there is no pool whose ends could + be excluded; every EUI-64 result inside the subnet is valid. + +### 6.4 Service/provider matrix for the offering **DECIDED** + +`ConfigDriveNetworkElement` advertises `UserData`, `Dhcp`, `Dns` +(`server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java:203`). This type uses +two of the three. + +| Service | Provider | Note | +|---|---|---| +| `UserData` | `ConfigDrive` | **mandatory** | +| `Dns` | `ConfigDrive` | optional but **strongly recommended** — the only way a guest learns its resolvers unless its template already has them (§6.5) | +| `SecurityGroup` | `SecurityGroupProvider` | optional (§12.2) | +| `Dhcp` | — | **not supported, and not needed** | +| `SourceNat`, `StaticNat`, `PortForwarding`, `Lb`, `Firewall`, `Vpn`, `NetworkACL`, `Gateway` | — | not supported | + +* `specifyIpRanges` → `true` (the operator supplies the subnet) +* `specifyVlan` → the operator's choice, exactly as for Shared offerings: `true` means the routed + id is given at network creation (via the existing `vlan` parameter), `false` means CloudStack + allocates one from the `ROUTED` physical network's id range (§6.7.2) +* `NetworkMode` → not applicable; reject `NATTED`/`ROUTED` for this guest type. (The *isolation + method* `ROUTED` (§6.7) is a different axis: `NetworkMode.ROUTED` says how a VR routes, the + isolation method says which physical network these VR-less networks live on.) + +**Why DNS is recommended rather than mandatory.** There is no VR and no resolver on the host, so +`network_data.json` `services` is the only channel by which CloudStack can tell a guest its DNS +servers (§8.3). An offering without it leaves Instances with addresses and routing but no name +resolution — unless the template already carries resolvers, which is unusual but legitimate and the +operator's call (§6.5). Note this depends on the §8.2 gate being fixed first. + +**Why DHCP is not merely unsupported but unnecessary.** ConfigDrive delivers the address, netmask, +gateway and routes directly (§8.1). DHCP would have nothing left to hand out, and offering it would +reintroduce exactly the L2 broadcast dependency this network type removes. + +Validation to add: + +* reject the offering unless `UserData` is provided by `ConfigDrive`; `Dns` is permitted but not + required (§6.5) +* reject `Dhcp` outright for this guest type +* `NetworkServiceImpl.java:657` currently rejects DNS on L2 networks. This type *requires* DNS, so + that branch must distinguish L2 from L3 rather than treating them alike. + +### 6.5 ConfigDrive is mandatory, DNS is not **DECIDED** + +**`UserData` via ConfigDrive is mandatory. `Dns` is optional but strongly recommended.** + +ConfigDrive itself cannot be optional: with no DHCP and no RA, it is the only channel that carries +the address, netmask, gateway and routes. A guest that ignores it comes up with no addresses at all, +and nothing in CloudStack reports an error. + +DNS is a different matter. A template may already have resolvers baked in, or be configured by +whatever provisions it afterwards. That is unusual, but it is the operator's call, not something the +network type needs to enforce — so an offering may omit `Dns`. The documentation should recommend +`UserData` + `Dns` together, since omitting DNS leaves an Instance with connectivity but no name +resolution unless its template handles it. + +**This has a hard prerequisite — see §8.2.** `ConfigDriveBuilder.needForGeneratingNetworkData()` +currently writes network data only when the network supports `Dhcp` **or** `Dns`. Since this type +never has `Dhcp`, an offering without `Dns` would today produce an **empty `network_data.json` and an +Instance with no addressing at all** — a far worse outcome than missing resolvers. Making `Dns` +optional therefore requires changing that gate first; it is not merely a validation relaxation. + +**No warning is raised when a template ignores ConfigDrive. DECIDED.** Whether cloud-init is present +and configured inside the guest is the operator's responsibility, not something CloudStack should +police. There is no reliable way to detect it from outside the Instance in any case, so any check +would be a guess presented as a fact. Documented as a requirement of the network type; not enforced, +not warned about. + +### 6.6 VPC support **DECIDED — never** + +**Not out of scope for v1; out of scope permanently.** Standalone networks only. + +The purpose of this network type is to route a public IPv4 and IPv6 address directly to an Instance. +A VPC is the opposite proposition: a private, self-contained address space with a VR, tiers holding +their own CIDRs, and ACLs between them. Every one of those is something this type deliberately +removes, so "direct routed inside a VPC" would not be a reduced VPC — it would be a contradiction. + +The case that *sounds* like it overlaps is already covered: **VPC supports BGP routing with subnets +today** (`NetworkMode.ROUTED`, 4.20). An operator who wants tenant subnets advertised into the fabric +with a VR in the path should use that. This feature addresses the different case where the Instance +itself holds the public address and there is no VR at all. + +Keeping the two apart is deliberate. They are not two settings of one feature, and treating them as +such would make both harder to reason about. + +### 6.7 What makes a network direct routed **DECIDED — revised 2026-09-04** + +**The offering's guest type (`L3`) on a physical network whose isolation method is `ROUTED`.** +The network operator is in the lead: direct routed is a property of the zone's fabric — hosts that +route, a routing daemon, an id plan for the bridges — so it is declared where fabric properties are +declared, on a **dedicated physical network**, not inferred from an offering that any admin can +create. Every direct routed network then carries a broadcast domain of the new type +**`routed://`** (`BroadcastDomainType.Routed`, scheme `routed`), and that id names the bridge: +`routed://5828` → `brdr-5828` (§9.2). + +* **Guru selection follows the standard contract.** `canHandle()` tests the zone type, + `isMyTrafficType()`, `offering.getGuestType() == GuestType.L3` **and** `isMyIsolationMethod()`, + exactly like its siblings (`DirectNetworkGuru.java:147`, `VxlanGuestNetworkGuru.java:56`). The + guru registers `new IsolationMethod("ROUTED")`. +* **A dedicated physical network.** The operator creates a guest physical network with isolation + method `ROUTED` (zone wizard or `createPhysicalNetwork isolationmethods=ROUTED`), gives it an id + range (the physical network's existing VLAN/VNI range field — here it is the routed-id pool), and + steers L3 offerings to it with tags, as for any other physical network. The traffic label is + irrelevant for these networks — the bridges have no uplink (§9.3) — but the physical network is + where the id range, the tags and the operational statement "this zone routes to the host" live. +* **The isolation method is named `ROUTED`, deliberately.** It shares the word with + `NetworkMode.ROUTED` (4.20) but sits on a different axis: the network mode describes how a + Virtual Router routes an Isolated/VPC network; the isolation method marks the physical network + that carries these VR-less, hypervisor-routed networks. The documentation must state the + distinction once, plainly. + +**This reverses the earlier decision** ("the offering, and nothing else" — no isolation method, no +broadcast domain, bridge named from `networks.id`). What changed the call: + +* **Operator opt-in became explicit.** Previously any admin creating an L3 offering implicitly + asserted that every host in the zone runs a routing daemon. A physical network with `ROUTED` makes + that a deliberate, zone-level act by the network operator, and an L3 network simply cannot be + designed in a zone whose operator has not made it. +* **The id became operator property.** `networks.id` was unique but uncontrollable: bridge names + could only be discovered after creation, never planned. With `routed://` the id is chosen by + the operator or drawn from a range the operator sized — host routing policy per `brdr-` can be + written before the network exists (§6.7.2). +* **The agent gets an explicit signal.** `BroadcastDomainType.Routed` on the `NicTO` replaces + inferring "this is direct routed" from the /32-plus-link-local-gateway address form (§9.1.3). +* **No new plumbing after all.** The earlier draft rejected this model as "an allocation mechanism, + a range to configure, and a choice to make". Working through the code showed all three already + exist for Shared networks — the `vlan` parameter, the physical network's vnet range and + `_dcDao.allocateVnet()`, and `specifyVlan` — and are reused verbatim (§6.7.2). What remains new is + one enum value and one registered isolation method. + +Offering creation rejects `Dhcp` for this guest type (§6.4), so a direct routed offering cannot be +built with DHCP in the first place. + +#### 6.7.1 Changing a network's offering afterwards **ACCEPTED** + +`NetworkServiceImpl.canUpgrade()` (`server/src/main/java/com/cloud/network/NetworkServiceImpl.java:4193`) +gates `updateNetwork`'s offering change on SecurityGroup parity, tag equality, `specifyVlan` +equality, `NetworkMode` equality, and `canMoveToPhysicalNetwork()`. It does **not** compare guest +type, so an administrator can in principle move a network onto an offering of a different type — for +example one without ConfigDrive, leaving Instances with no way to get their addressing. + +**Accepted, not guarded.** This is an administrative action with an obvious cause and effect; adding +a special case to `canUpgrade()` for this type is not worth the complexity. Worth a line in the +documentation, nothing more. + +One incidental note from reading that method: the **SecurityGroup parity check** means security +groups cannot be toggled on a live network by swapping offerings — enabling them is a choice made +when the network is created. + +#### 6.7.2 Where the routed id comes from **DECIDED — Shared-network mechanics, reused** + +The id lifecycle is exactly the Shared network's VLAN lifecycle, with a different URI scheme. No +new allocation mechanism is introduced: + +* **`specifyVlan=true` on the offering** — the operator passes the id at network creation through + the existing `vlan` parameter: `createNetwork ... vlan=5828` → `broadcast_uri=routed://5828` → + `brdr-5828`. This is the flagship path: the operator plans the id space and can pre-provision + per-bridge routing policy on the hosts. +* **`specifyVlan=false`** — CloudStack allocates a free id from the `ROUTED` physical network's + range at creation (`_dcDao.allocateVnet()`, the same call Shared networks without `specifyVlan` + use — `NetworkServiceImpl.commitNetwork()`), and releases it when the network is deleted (the + existing release path in `NetworkOrchestrator`). + +The value must be numeric — `routed://` carries a number, nothing else — and the existing +Shared-network checks apply unchanged: an operator-specified id must not fall inside the dynamic +range (`_dcDao.findVnet()`), and must not collide with another network's broadcast URI. + +**Uniqueness must be zone-wide, not per physical network.** Bridge names are global on a host, so +two networks with routed id 5828 anywhere in the zone would share `brdr-5828` and merge their L2 +domains. The existing zone-wide URI overlap check +(`_networksDao.listByZoneAndUriAndGuestType()`) covers guest-vs-guest; with a single `ROUTED` +physical network per zone — the expected deployment — it is equivalent to the per-physnet check +anyway. **Guest networks and public ranges (§8.5) share the same id space** and are guarded in +both directions: creating a guest network whose routed id a public range already carries is +rejected, and so is creating a public range whose id a guest network holds. + +Either way the URI is set at creation and **stable for the network's life**: the bridge name never +changes, which is what makes it something host policy can reference. + +### 6.8 Hypervisor support + +**KVM only for v1.** The host must program routes and neighbour entries and run a routing daemon; +that is only realistic where the host OS is ours to configure. Other hypervisors: reject at network +creation with a clear error. + +## 7. API and data model changes + +### 7.1 API + +* `createPhysicalNetwork` / `updatePhysicalNetwork` — accept `ROUTED` as an isolation method; the + physical network's VLAN/VNI range field doubles as the routed-id pool (§6.7.2). UI: add `ROUTED` + to the isolation method choices (zone wizard and physical network form). +* `createNetworkOffering` — accept the new `guestiptype`; validate the service matrix (§6.4); + `specifyVlan` is a free choice (§6.7.2). +* `createNetwork` — accept the subnet as for Shared networks; the existing `vlan` parameter carries + the routed id when the offering has `specifyVlan=true` (§6.7.2); `createVlanIpRange` gateway + handling per §5.5; zone-wide overlap validation per §6.3.2. +* `listNetworks` / `NetworkResponse` — expose the type and the `broadcasturi` (`routed://5828`), + which is how a user reads off the bridge name. The network's CIDR is meaningful (it is the + pool) and should be shown; its gateway is not. +* `listNics` / `NicResponse` — report the /32 and /128 plus the shared gateway. `netmask` is + already a dotted quad, so `255.255.255.255` needs no schema change. +* `listPublicIpAddresses` — works as for Shared networks. +* New APIs: **none.** + +### 7.2 Database + +* `network_offerings.guest_type` — new enum value (`L3`). +* `networks.broadcast_domain_type` — new enum value (`Routed`); `broadcast_uri` holds + `routed://`. Both columns are strings, so no schema change. +* `physical_network_isolation_methods` — new value `ROUTED`; a plain string, no schema change. +* `op_dc_vnet_alloc` — reused unchanged as the routed-id pool for `specifyVlan=false` (§6.7.2). +* `nics` — all needed columns exist: `ip4_address`, `netmask`, `gateway`, `ip6_address`, + `ip6_cidr`, `ip6_gateway` (`engine/schema/src/main/java/com/cloud/vm/NicVO.java:55-108`). +* `user_ip_address`, `vlan` — reused unchanged. `user_ipv6_address` stays untouched: IPv6 is + computed from subnet + MAC and stored only on the NIC (§6.3.4). +* Upgrade: new enum values only. No data migration. + +### 7.3 New components + +* **Network guru — a subclass of `DirectNetworkGuru`. DECIDED.** `DirectNetworkGuru` already + implements "operator defines a subnet, CloudStack assigns individual v4 and v6 addresses", which + is exactly the allocation behaviour wanted, so the allocate/release lifecycle is inherited rather + than duplicated. The subclass registers `IsolationMethod("ROUTED")` and overrides: + * `canHandle()` — accept `GuestType.L3` on a physical network with isolation method `ROUTED` + (`isMyIsolationMethod()`, as its siblings do — §6.7) + * the `NicProfile` after allocation — force `255.255.255.255` / `169.254.0.1` and `/128` / + `fe80::1` over the vlan row's values (`IpAddressManagerImpl.allocateDirectIp()` lines 2459–2461, + §6.3) + * `design()` — broadcast domain type `Routed`, broadcast URI `routed://` from the + operator-specified or allocated id (§6.7.2, §9.2.1) + + The known cost of subclassing is inheriting `DirectNetworkGuru`'s Shared-network assumptions. + Accepted: the alternative is duplicating the address lifecycle, which is the part most likely to + drift and the part where bugs are least visible. **TODO:** during implementation, note any + inherited behaviour that only makes sense for `GuestType.Shared` and override it explicitly rather + than letting it apply by accident. +* **Network element** — none new. `ConfigDriveNetworkElement` covers UserData/DNS; the security + group element covers filtering; host routes ride on NIC plug/unplug (§9), not on element + `implement()`. + +## 8. Guest configuration via ConfigDrive + +### 8.1 `on-link` for the link-local gateway **DECIDED** + +`ConfigDriveBuilder.getNetworksJsonArrayForNic()` +(`engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java:365`) +today emits OpenStack **network_data.json v1**: + +```json +{"id":"eth0","ip_address":"...","netmask":"...","link":"eth0","type":"ipv4", + "routes":[{"gateway":"...","netmask":"0.0.0.0","network":"0.0.0.0"}]} +``` + +A /32 address with a gateway outside its own subnet cannot be expressed here — v1 has no way to +mark a next-hop as on-link, and the guest kernel will reject the resulting config with +`Nexthop has invalid gateway`. + +**Resolved — verified against cloud-init: no on-link plumbing is needed at all.** + +Two findings settled this, one from code and one from testing: + +1. The ISO is labelled `config-2` (`VirtualMachineManager.VmConfigDriveLabel`), so cloud-init + selects its **OpenStack/ConfigDrive datasource**, which takes network configuration from + `openstack/latest/network_data.json`. A `network-config` file (Network Config v2) is only read + by the NoCloud datasource (label `cidata`) and would be ignored on the default boot path. +2. **Verified by Wido:** when consuming `network_data.json`, cloud-init itself detects an IPv4 + gateway inside `169.254.0.0/16` and sets the on-link flag on the route it renders. The existing + v1 emission — a plain default route via the gateway — therefore works unchanged. + +So ConfigDrive's route generation is **not modified at all**. What did change (§8.2) is only the +gate: network data is now always generated for a direct routed NIC, since ConfigDrive is its only +addressing channel. + +Emitting a Network Config v2 `network-config` file for NoCloud-configured images was implemented +and then removed — deferred to a **later PR** (§15). A v1 host-route-to-the-gateway workaround was +likewise removed as redundant: cloud-init handles the case natively, and extra routes with a +`0.0.0.0` gateway are the kind of thing that can confuse a renderer. + +The guest-side requirement is accordingly not "Network Config v2 support" but **a cloud-init recent +enough to apply on-link for link-local gateways from OpenStack network data** — which the smoke +test must pin down to a minimum version for the documentation. + +Target guest config: + +```yaml +version: 2 +ethernets: + eth0: + match: {macaddress: "02:00:...:5a"} + addresses: [203.0.113.55/32, "2001:db8:1::55/128"] + routes: + - to: default + via: 169.254.0.1 + on-link: true + - to: default + via: "fe80::1" + nameservers: + addresses: [...] +``` + +Notes and follow-ups: + +* The on-link trigger — the gateway matching `169.254.0.0/16` — lives in **cloud-init**, not in + CloudStack. It is a property of the address, so it stays correct however the guest receives it. +* IPv6 needs no `on-link` — `fe80::1` is link-local by definition and always on-link. +* **The guest requirement is a cloud-init recent enough to set on-link for link-local IPv4 + gateways when consuming OpenStack network data.** Guests whose cloud-init predates that are not + supported on this network type, in the same way that guests without ConfigDrive are not (§6.5). +* **TODO:** verify against the images used in the smoke tests, and state the minimum cloud-init + version in the documentation so the requirement is visible to operators rather than discovered at + boot. + +### 8.2 network_data generation is gated on DHCP or DNS **REQUIRED CHANGE** + +```java +static boolean needForGeneratingNetworkData(Map> supportedServices) { + return supportedServices.values().stream() + .anyMatch(services -> services.contains(Network.Service.Dhcp) + || services.contains(Network.Service.Dns)); +} +``` +(`engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java:266`, +called from `writeNetworkData()` at `:252`.) + +If neither service is supported, `writeNetworkData()` writes an empty `{}` and **the guest receives +no network configuration whatsoever** — no address, no netmask, no gateway, no routes. + +That gate is wrong for this network type. It equates "does this network have DHCP or DNS?" with +"does this NIC need its addressing written into ConfigDrive?", which held while ConfigDrive was a +supplement to a VR but does not hold when ConfigDrive is the *only* channel. This type never has +`Dhcp`, and §6.5 makes `Dns` optional, so the two conditions can both be false while the NIC still +very much needs its /32 written. + +**Required:** extend the condition so network data is generated whenever the NIC belongs to a direct +routed network — or, more generally, whenever the NIC has an address to convey and no other means of +conveying it. Until that lands, a `Dns`-less offering is not merely degraded, it is non-functional. + +Worth a code comment either way, because the coupling is invisible from the offering side. + +### 8.3 DNS **DECIDED — per network, falling back to the zone** + +DNS servers reach the guest through `network_data.json` `services` +(`getServicesJsonArrayForNic`). No resolver on the host, no VR. + +Resolvers come from the network when set, and from the zone otherwise — so an operator configures +DNS once per zone and overrides it only on the networks that need something different. + +**This already works and needs no new code.** `NetworkModelImpl.getNetworkIp4Dns()` and +`getNetworkIp6Dns()` (`server/src/main/java/com/cloud/network/NetworkModelImpl.java:3023` and +`:3037`) implement exactly that precedence: network `dns1`/`dns2` if set, then the VPC, then the +zone. The VPC branch is simply never reached here (§6.6). The `networks` table already carries +`dns1`, `dns2`, `ip6_dns1`, `ip6_dns2`, and `createNetwork`/`updateNetwork` already expose them. + +Note the interaction with §6.5: DNS is optional on the offering. If the offering omits the `Dns` +service, these values are never written to ConfigDrive regardless of being configured on the network +or zone. + +### 8.4 Metadata service **DECIDED — none in v1** + +With no VR there is no `data-server` at the gateway and no link-local metadata endpoint. **The +ConfigDrive ISO is the only source of metadata and user data**, and there is no +`169.254.169.254`-style HTTP endpoint. + +**Note this explicitly for operators**, because it is a real behavioural difference from other +network types rather than an omission. Tooling that expects to `curl 169.254.169.254` — cloud-init +in some configurations, Kubernetes cloud providers, various agents — will not work unmodified. A +template that reads its metadata from ConfigDrive is fine; one that assumes the HTTP endpoint is not. + +A host-side responder on the `brdr-*` bridge is entirely feasible later: the gateway address is +already there, the host already routes for the guest, and the data is already assembled for the ISO. +It is deferred rather than ruled out — **a candidate for v2** (§15). + +### 8.5 SystemVMs: boot args, not ConfigDrive **REQUIRED CHANGES — dual-stack in v1** + +Console proxy and secondary storage VM are still needed in a direct routed zone, and their public +interface is directly routed like everything else — a `/32` + `169.254.0.1` on-link gateway **and** +a `/128` + `fe80::1`. **IPv6 for the SystemVMs must work from the start**; it is not deferred. + +SystemVMs do not consume ConfigDrive: their addressing is passed down from the hypervisor as boot +arguments, parsed by `systemvm/debian/opt/cloud/bin/setup/common.sh` and applied by +`setup_common()`. For both VM types the call is `setup_common eth0 eth1 eth2` (`init.sh:166`), so +`eth2` is the public interface **and** the default-gateway device. + +Walking the chain end to end, four gaps stand between the host-route NIC form and a working +systemvm: + +1. **The IPv4 default route needs `onlink`.** The values already flow — both builders pass + `eth2ip`, `eth2mask` and `gateway` straight from the `NicProfile` + (`ConsoleProxyManagerImpl.finalizeVirtualMachineProfile()`, `SecondaryStorageManagerImpl` + likewise), so a NIC stamped in host-route form arrives correctly. But `setup_common()` then runs + `ip route add default via $GW dev $gwdev` (`common.sh:399`), which the kernel rejects with + `Nexthop has invalid gateway` when `$GW` lies outside the /32. Append `onlink` when `$GW` falls + in `169.254.0.0/16` — the same trigger rule §8.1 leans on in cloud-init, applied by hand here + because a systemvm has no cloud-init. The `/etc/network/interfaces` stanza itself (address + + `netmask 255.255.255.255`, no gateway line) is fine as is, and the VMware ping-the-gateway + workaround right after the route works once the on-link route exists. +2. **IPv6 boot args are never sent for CPVM/SSVM.** Only the VR's builder emits them + (`VirtualNetworkApplianceManagerImpl.java:1935–1946`: `ethip6`, `ethip6prelen`, + `ip6gateway`). The CPVM and SSVM builders emit IPv4 only and must add the same three appends for + NICs that carry IPv6, from values the guru already stamped — prefix length via + `NetUtils.getIp6CidrSize(nic.getIPv6Cidr())` (→ 128), `ip6gateway` from the default NIC. The + consumer side already exists: `common.sh` parses `eth0ip6`/`eth2ip6`(+`prelen`) and + `ip6gateway` → `IP6GW` today. +3. **The systemvm never installs an IPv6 default route.** `IP6GW` is parsed and then consumed + nowhere; `setup_interface_ipv6()` (`common.sh:139`) writes only the address and prefix length + and leans on `accept_ra 1` — and on a direct routed bridge **no RA ever arrives** (`accept_ra=0` + on the bridge, §9.2, and nothing sends them). Add next to the v4 default in `setup_common()`: + `ip -6 route replace default via $IP6GW dev $gwdev` whenever `IP6GW` is set. `fe80::1` is + link-local, so `dev` is mandatory and no on-link handling exists or is needed — a link-local + next-hop is on-link by definition. *Note this gap predates the feature:* a classic VLAN public + network with IPv6 also leaves CPVM/SSVM without a v6 default unless a fabric router happens to + send RAs. The fix is useful independently of direct routed and should not be gated on it. +4. **The public NIC must be stamped and plugged like a guest's.** `PublicNetworkGuru.getIp()` + (`PublicNetworkGuru.java:146–156`) sets gateway/netmask from the public range's vlan row and + hard-codes a `Vlan`/`Vxlan` broadcast URI on the NIC. For a direct routed public range the NIC + must instead receive the host-route form (as `DirectRoutedNetworkGuru.applyDirectRoutedAddressing()` + does for guests) plus `BroadcastDomainType.Routed` and a `routed://` URI — which is also + exactly what steers `BridgeVifDriver`'s Public branch into the brdr-bridge + `modifymacip.sh` + path instead of the public bridge (today that handling sits only in the Guest branch). **IPv6 + is computed directly in the guru** — `NetUtils.EUI64Address(range ip6_cidr, NIC MAC)` per + §6.3.4, then the /128 + `fe80::1` form. Deliberately *not* via `ipv6Service.updateNicIpv6()`: + auditing that path showed it is gated on the public network offering's internet protocol + (never set on the system public offering, so a no-op in practice) and reserves through **one + placeholder NIC per network** — on the shared Public network every SystemVM would receive the + *same* address. Neither the gate nor the reservation decides anything here: the MAC already + makes the address unique. Mechanism: the public IP range's vlan row carries `routed://` as + its tag, which `getIp()` copies into the broadcast URI; the zone's public network carries one + routed id (and thus one `brdr-`) of its own, and systemvm /32s and /128s are advertised by + the host's routing daemon exactly like guest addresses. + +Minor, noted for completeness: `setup_interface_ipv6()` writes `accept_ra 1` — harmless on an +RA-less bridge, but the static default must not depend on it (and may be set to 0 for direct +routed interfaces later); SSVM's apache vhost binds `$ETH2_IP` (IPv4) — pre-existing; the SSVM +serves its HTTP endpoints on IPv4, while its own outbound traffic and reachability for +management are dual-stack. + +## 9. Hypervisor (KVM) implementation + +### 9.1 The agent's entire job + +Per guest address, on NIC plug: install a static neighbour entry and a host route. On unplug / VM +stop / migrate-away: remove them. **That is the whole contract.** Nothing else on the host is +CloudStack's concern. + +#### 9.1.1 This already exists — reuse `modifymacip.sh` **DECIDED** + +The mechanism was added to `main` by `4816e059383` ("KVM: add configurable MAC/IP script hook for +static ARP/NDP and routes", PR #13495, 2026-07-10) for the VXLAN/EVPN static MAC-IP work. It does +almost exactly what this design needs: + +* `scripts/vm/network/vnet/modifymacip.sh` — `-o add -b -m [-4 ]... [-6 ]...` + runs `ip neigh replace lladdr dev nud permanent` and + `ip route replace /32 dev `, and the `-6` equivalents with `/128`. `-o delete -b + -m ` discovers the addresses to remove by querying the neighbour table for that MAC, so no + state file is needed. +* `BridgeVifDriver.executeMacIpScript()` + (`plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java:437` + for add, `:418` for delete) is already wired into NIC plug (`:291`) and unplug (`:298`). +* It also passes the MAC-derived IPv6 link-local automatically + (`NetUtils.ipv6LinkLocal(mac)`), handles secondary IPs, and sets + `net.ipv6.conf..disable_ipv6=0` before installing NDP entries. +* Gated by the agent property `vm.network.macip.static` + (`AgentProperties.VM_NETWORK_MACIP_STATIC`, default `false`). + +So §9.1 is largely an integration exercise rather than new code. Four gaps to close. + +#### 9.1.2 Bridge, not tap **DECIDED** + +Routes and neighbour entries go on the **bridge**, exactly as `modifymacip.sh` and the existing +`BridgeVifDriver` hook already do (`intf.getBrName()` is passed at `:291`). **No change to the +script or the hook for this.** The same code serves both the EVPN case and this one; there is no +reason to write it twice. + +Consequence to be aware of: the host route pins each address to the bridge, and the static neighbour +entry pins it to a **MAC** — but the MAC-to-port mapping comes from ordinary bridge FDB learning, so +the routing table alone is not an anti-spoofing boundary. §12.1 sets out why the design is still +sound (short version: the per-network bridge is the isolation boundary, so MAC-to-port pinning only +matters within one account's own network), and §12.3 for what remains exposed. + +Per-tap routes were considered and rejected for v1. Recorded for completeness in case the anti-spoof +story needs tightening later: `-b` is passed verbatim as the `dev` argument to every `ip` command, +and the delete path's `ip neigh show dev ` works on a tap too, so targeting a tap would need no +script change at all — only a rename of `-b` to something less misleading. + +#### 9.1.3 Gating is per NIC, on the broadcast type **DECIDED — revised 2026-09-04** + +`vm.network.macip.static` is resolved once in `configure()` (`BridgeVifDriver.java:87`) and is +all-or-nothing for the host. A host runs direct routed guests *and* ordinary bridged guests side by +side, so the behaviour is decided **per NIC** — not from a host property. + +**The selector is now explicit.** With §6.7 giving every direct routed network a +`routed://` broadcast domain, the `NicTO` carries `broadcastType == BroadcastDomainType.Routed` +and the URI itself — populated for every NIC by `HypervisorGuruBase.toNicTO()`. That is the test: +one decision, used twice. It picks the bridge (`brdr-` from the URI's value, §9.2.1) **and** +gates the MAC/IP hook. They are not separate decisions — if the driver has chosen a `brdr-` bridge, +the hook applies. + +**The earlier inferred contract is superseded.** Before the revision these NICs were +`BroadcastDomainType.Native` — indistinguishable from other untagged cases — so the agent inferred +"direct routed" from the address form no other network type produces: netmask `255.255.255.255` +(and/or a `/128`) with a gateway in `169.254.0.0/16`. That inference was flagged at the time as an +implicit contract a future change could trip over; the broadcast type closes it. The address form +remains meaningful where it is genuinely about addressing — cloud-init's `on-link` handling (§8.1) +keys on the link-local gateway, and `ConfigDriveBuilder` recognises the host-route shape — but the +agent's *routing-behaviour* decisions (bridge choice, MAC/IP hook, `--directrouted` to +`security_group.py`) all key on `BroadcastDomainType.Routed`. + +The agent property `vm.network.macip.static` stays as an independent host-wide opt-in for the EVPN +use case and is unaffected. + +#### 9.1.4 Silent failures are kept **DECIDED — accepted** + +Both `executeMacIpScript()` overloads catch everything and only log, deliberately — "managing host +neighbour/route entries is best-effort and must never break VM lifecycle operations" +(`BridgeVifDriver.java:432`). + +**That behaviour is kept unchanged.** No new failure handling for this network type in v1, which +also means the existing EVPN path cannot be regressed by this feature. + +The consequence, stated so it is not a surprise later: if the route or neighbour install fails, the +Instance starts normally and appears healthy, but has **no connectivity at all**, and nothing in +CloudStack reports why. Diagnosis means looking at the agent log for the warning from +`executeMacIpScript()`, or checking `ip route` / `ip neigh` on the host. + +Making it fatal to the NIC plug, or raising an alert, remains available later (§15) and would be a +small change — the call sites already distinguish success from failure, they simply do not act on it. + +#### 9.1.5 Secondary IPs **IMPLEMENTED** + +CloudStack lets an Instance hold secondary IPv4 and IPv6 addresses on a NIC, and they need the +same treatment as the primary: a host route and a static neighbour entry, or the address is not +reachable. + +**At Instance start this already worked.** `modifymacip.sh` accepts repeated `-4`/`-6`, and +`BridgeVifDriver` passes `nic.getNicSecIps()` alongside the primary addresses, so every address the +NIC holds is installed on plug. Unplug removes them all by MAC. + +**Adding or removing one on a *running* Instance did not, and was fixed.** That path +(`NetworkRulesVmSecondaryIpCommand`) only ever updated ipsets and ebtables, and the management +server only sent it when security groups were in play — three separate gates +(`SecurityGroupManagerImpl`: Instance in no security group, network without the SG service; +`NetworkServiceImpl.configureNicSecondaryIp` / `RemoveIpFromVmNicCmd`: zone without SG). On a +Direct Routed network with security groups disabled, none of them fired, so a newly added secondary +IP stayed dark until the Instance was restarted — and a removed one kept being routed and +advertised. + +The command now carries `directRouted` and `applySecurityGroupRules`; the agent installs or removes +the host route and neighbour entry for the address whenever the former is set, independently of +`canBridgeFirewall`, and skips the security group script when the latter is not. `modifymacip.sh` +gained per-address delete (`-o delete` with `-4`/`-6` removes just those; without them it keeps its +delete-everything-for-this-MAC behaviour, which is what unplug uses). + +**The ipset-based dispatch pays off here.** Because to-Instance traffic on L3 matches +`--match-set dst` (§12.2), and secondary IPs are added to that same ipset, security group +dispatch covers a new secondary IP with no rule changes at all. + +#### 9.1.6 Not a gap: the shared gateway addresses + +`modifymacip.sh` does not configure `169.254.0.1` / `fe80::1` — that is `modifybrdr.sh`'s job when it +creates the network's bridge (§9.2). The two scripts have a clean split: `modifybrdr.sh` owns the +bridge and its gateway addresses, `modifymacip.sh` owns per-guest routes and neighbour entries on it. + +Minor robustness note: because the delete path derives addresses from the neighbour table, a route +leaks if its neighbour entry has already been flushed. Reconciliation (§9.6) covers this. + +### 9.2 One bridge per network **DECIDED** + +**Each direct routed network gets its own bridge on every hypervisor that runs one of its +Instances**, named `brdr-` (Bridge-DirectRouted) — for example `brdr-5828`. + +The `` is the network's **routed id** — the value of its `routed://` broadcast domain, +operator-chosen or allocated from the `ROUTED` physical network's range at creation, and stable for +the network's life (§6.7.2, §9.2.1). + +The bridge is created and removed by a new script, +`scripts/vm/network/vnet/modifybrdr.sh`, modelled on `modifyvxlan.sh`: + +``` +modifybrdr.sh -o add -n [-4 ] [-6 ] +modifybrdr.sh -o delete -b +``` + +On `add` it creates the bridge if absent (STP off, `forward_delay 0` — there is no uplink, so no +loop to detect and no reason to hold ports down at Instance start), enables IPv4/IPv6 forwarding on +it, disables RA acceptance, and configures the gateway addresses. On `delete` it removes the bridge, +but only after confirming nothing is still attached — an Instance may have started on the network +while the last one was stopping. The whole script runs under `flock`, like `modifyvxlan.sh`, because +concurrent Instance starts on one network will race to create the bridge. + +Consequences: + +* **Networks are isolated from each other at layer 2 by the topology**, not by filtering. This is + the first reason for the design: a guest on `brdr-5828` has no L2 path of any kind to a guest on + `brdr-5829`, and no rule set has to be correct for that to hold (§12). +* **Each network becomes a named L3 interface on the hypervisor.** This is the second reason, and it + is an operational one: `brdr-5828` is something the operator can attach local policy to. Different + route-maps or redistribution filters per network, per-network policy routing, QoS, or later a VRF + per bridge — all expressible in the host's own network configuration, matched on interface name, + with no involvement from CloudStack. The routing daemon is already the operator's (§10); giving + each network its own interface is what makes per-network routing decisions possible at all. And + because the operator can choose the id (§6.7.2), that policy can be written **before** the network + exists. +* **Bridge name is identical on every host** — it derives only from the routed id — which is what + keeps migration a no-op for the guest. +* **The vif driver now needs work.** The earlier shared-bridge design could ride on + `BridgeVifDriver`'s existing `brname = trafficLabel` fallback + (`plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java:255`); + that no longer applies. The driver must take the routed id from the NIC's broadcast URI + (`BroadcastDomainType.getValue(nic.getBroadcastUri())`) and invoke `modifybrdr.sh` on plug, and on + unplug when the last interface leaves — the same shape as its existing `createVnetBr()` handling + for VXLAN. + +#### 9.2.1 How the agent learns the bridge name **DECIDED — revised 2026-09-04** + +**`NicTO` carries the broadcast URI.** The URI (`routed://5828`) and broadcast type +(`BroadcastDomainType.Routed`) are populated for every NIC by `HypervisorGuruBase.toNicTO()`, the +same way VLAN and VXLAN NICs receive theirs. + +`BridgeVifDriver` selects on the broadcast type (§9.1.3), extracts the routed id from the URI and +passes it to `modifybrdr.sh`, which creates the bridge and **prints the name it chose** — the agent +uses whatever comes back. On unplug the agent asks the same script (`-o delete -b `), which +answers `notmine`, `kept` or `deleted`; `notmine` sends the agent down its regular unplug path. +**How the bridges are named is known only to the script**; no `brdr-` prefix appears anywhere in +Java. + +**This reverses the earlier decision**, which named the bridge from `networks.id` carried in +`NicTO.networkId`, precisely to avoid a broadcast domain, an isolation method and an id allocation. +The reversal rationale is in §6.7; the agent-side consequence is only that the number now comes from +the broadcast URI instead of the network-id field — the script contract (id in, name out, naming +private to the script) is unchanged. + +Consequences: + +* The network's `broadcast_uri` is `routed://` and its broadcast domain type is `Routed`, set + at creation and stable for the network's life. An auto-allocated id is released on network + deletion (§6.7.2); an operator-specified one was never in the pool. +* `specifyVlan` on the offering selects between operator-specified and allocated ids (§6.4, §6.7.2). +* An operator who wants `brdr-5828` to be a *specific* number **can have it** — create the network + from a `specifyVlan=true` offering with `vlan=5828`. Per-network host routing policy (§9.2) can + therefore be provisioned before the network exists. + +Per bridge, `modifybrdr.sh` sets: + +* `169.254.0.1/32` and `fe80::1/64` +* `net.ipv4.conf..forwarding=1`, `net.ipv6.conf..forwarding=1` +* `net.ipv6.conf..disable_ipv6=0` and `accept_ra=0` +* `arp_ignore=1` / `arp_announce=2` — see §9.2.1 +* **no physical uplink** — see §9.3, this is mandatory +* proxy ARP: **not needed** — the gateway addresses are local to the bridge, so the bridge answers + guest ARP/ND directly, and host→guest neighbour entries are static rather than resolved + +#### 9.2.2 The same gateway address on many bridges **DECIDED** + +Every `brdr-*` bridge on a host carries the *same* `169.254.0.1` and `fe80::1`. This is intended, +and it is what makes a guest's configuration identical no matter which network or host it lands on. + +For IPv6 it is unremarkable: link-local addresses are per-link and scoped by interface, so `fe80::1` +on twenty bridges is normal and correct. + +For IPv4 the sysctls are what make it correct, and `modifybrdr.sh` sets both: + +* `arp_ignore=1` — answer ARP only for addresses configured on the interface the request arrived on, + so a request reaching `brdr-5828` is never answered on behalf of `brdr-5829` +* `arp_announce=2` — always source ARP from the address of the interface the request goes out of + +Each bridge is its own L2 domain, so the ARP exchange stays within the right one regardless; the +sysctls remove the cases where the host might otherwise answer or source from the wrong interface. +The host's local route table gains one `local 169.254.0.1` entry per bridge, which is harmless — the +host never originates traffic from that address, it only replies on-link. + +### 9.3 The bridges have no physical uplink **DECIDED — correctness requirement** + +Unlike `cloudbr0`, a `brdr-*` bridge has **no physical port**. It is purely host-local: the only L3 +presence on it is `169.254.0.1` / `fe80::1`, and all guest traffic leaves the host via the host's own +routed uplink rather than being bridged. `modifybrdr.sh` never enslaves an interface, so this holds +by construction as long as nothing else adds one. + +If a bridge did have an uplink onto a shared L2 segment, two things break: + +1. **Duplicate gateway addresses.** Every hypervisor configures `169.254.0.1` and `fe80::1` on every + `brdr-*` bridge. Put those on a common segment and every host answers ARP/ND for the same + addresses; guests would resolve the gateway to an arbitrary host's MAC. +2. **Isolation collapses.** Guests of that network across all hosts would share one broadcast domain, + and the per-network bridge would stop being a boundary. + +### 9.4 Layer 2 isolation comes from the bridges **DECIDED** + +Separate bridges per network are the isolation mechanism. A guest on `brdr-5828` cannot send a frame of +any kind — ARP, raw L2, rogue RA, anything — to a guest on `brdr-5829`. There is no shared broadcast +domain, no shared FDB, and no path that filtering would have to police. + +This replaces the earlier shared-bridge design, in which separate networks were only administrative +and L2 separation had to be recovered with filtering. **No libvirt nwfilter is used**: the +`no-mac-spoofing` / `clean-traffic` approach was for the shared-bridge model and is dropped entirely. + +Two consequences worth stating plainly: + +* **Isolation between networks needs no filtering at all.** The boundary is the bridge rather than + a rule set. That was the motivation for this change. +* **Guests within one network still share a bridge**, so spoofing between them remains possible + (§12.3). Since a network belongs to one account, that is intra-tenant exposure rather than + cross-tenant. + +Bridge port isolation (`bridge link set dev vnetX isolated on`) remains available as a further step +if intra-network isolation is ever wanted; it is not applied in v1 (§12.3, §15). + +### 9.5 Sysctls + +Routing happens on the bridge, so the bridge is the L3 input interface for guest traffic and these +apply per `brdr-*` bridge rather than per tap. `modifybrdr.sh` sets them at creation: + +* `net.ipv4.conf..forwarding=1`, `net.ipv6.conf..forwarding=1` +* `net.ipv6.conf..disable_ipv6=0` — also set by `modifymacip.sh` before installing NDP entries +* `net.ipv6.conf..accept_ra=0` — a guest must never be able to send an RA the host acts on +* `net.ipv4.conf..arp_ignore=1` / `arp_announce=2` — required because every `brdr-*` bridge on + the host carries the same gateway address (§9.2.2) + +**`rp_filter=1` (strict) is set on every `brdr-*` bridge. DECIDED.** An Instance may only send from +an address that routes back out of the bridge it arrived on — which, given per-guest /32 routes, is +its own address. Source spoofing is therefore blocked at the host. + +Set **on the bridge only, never on `all`**, so no other interface on the host changes behaviour. The +kernel takes `max(conf.all.rp_filter, conf..rp_filter)`, so a per-bridge value of 1 is effective +regardless of what `all` is, without touching the uplink — which matters, because strict mode on a +host uplink can break legitimately asymmetric fabric routing. + +Strict mode is safe for the paths this design creates: + +* an Instance's own traffic — source `S`, route `S/32 dev brdr-N`, arrives on `brdr-N` → passes +* same-network hairpin (§5.4) — arrives on `brdr-N` from an address routed via `brdr-N` → passes +* cross-network — arrives on `brdr-N`, forwarded out `brdr-M`; the check is on ingress only → passes + +**IPv4 only.** The kernel has no IPv6 `rp_filter`, so IPv6 source spoofing is not bounded by this and +must be handled by security groups (§12.2) where it matters. + +### 9.6 No reconciliation on agent restart **DECIDED** + +**The agent does not scan or rebuild routes and neighbour entries at startup.** The existing hook +fires only on NIC plug/unplug (`BridgeVifDriver.java:291,298`), and that is sufficient. + +The reasoning, which is worth writing down because "kernel state is not persistent" invites the +opposite conclusion: + +* **Host reboot** clears the routes — but it also destroys every Instance on that host. The + management server starts them again, each start goes through the plug path, and the routes are + reinstalled as a side effect. There is nothing to reconcile, because there is nothing running whose + state could be missing. +* **Agent restart without a host reboot** does not clear anything. Routes and neighbour entries live + in the kernel, not in the agent, so running Instances keep working across an agent restart with no + action at all. +* **While the agent is down**, CloudStack cannot stop or migrate Instances on that host, so host + state cannot drift from what the management server believes. + +`modifymacip.sh` uses `ip route replace` / `ip neigh replace`, so any repeated plug is idempotent +regardless. + +**Residual, accepted:** if an Instance disappears without an unplug — libvirt kills it, or it crashes +in a way that skips the normal path — its route and neighbour entry linger. The practical risk is not +the stale kernel state itself but that the routing daemon keeps advertising that /32, so if the +Instance is started on another host the fabric may see the address from two places. Rare, and not +worth a startup sweep to prevent; noted so it is recognisable if it ever shows up. + +## 10. Routing daemon — explicitly out of scope + +**CloudStack does not install, configure, or monitor the routing daemon.** + +The contract is one-directional: *CloudStack guarantees the correct routes and neighbour entries +exist in the host kernel.* The operator configures their daemon to redistribute them, e.g. with +FRR: + +``` +router bgp 65001 + address-family ipv4 unicast + redistribute kernel + address-family ipv6 unicast + redistribute kernel +``` + +Why this is the right boundary: + +* Identical behaviour for FRR, BIRD, or anything else that redistributes kernel routes. +* Identical behaviour for BGP, OSPF, or IS-IS. +* No daemon version coupling, no config-file ownership conflict with the operator's automation, no + `vtysh`/reload dependency in the agent. +* Nothing new to fail: if the route is in the kernel the guest works locally, and advertisement is + the fabric's business. + +Consequences to accept and document: + +* We ship **documentation and reference configuration**, not code. Redistribution filtering via + route-maps is the operator's job. +* **No feedback loop.** CloudStack cannot tell whether a guest's address is reachable from the + fabric. Accepted as a known gap for v1; a health signal from the agent is listed under future work + (§15) rather than treated as an open question. +* This is the **opposite** choice from the 4.20 BGP work, where CloudStack manages peers + (`BgpPeerVO`, `SetBgpPeersCommand`, `systemvm/debian/opt/cloud/bin/cs/CsBgpPeers.py`). The doc + should say why explicitly — reviewers will ask. +* Design should not preclude an optional managed mode later, but v1 does not have one. + +## 11. Live migration + +The guest's configuration is host-independent, so the guest needs no reconfiguration. What moves is +host state: + +1. Destination host installs the route and neighbour entry when the NIC is plugged. +2. Source host removes both when the NIC goes away. +3. Each host's routing daemon advertises/withdraws; the fabric reconverges. + +To work through: + +* **Ordering.** Install-then-remove is safer than remove-then-install; a transient duplicate + advertisement is less harmful than a black hole. Confirm the plug/unplug sequence during + migration actually gives that ordering. +* **Convergence gap.** Traffic may be black-holed until the fabric reconverges. **TODO:** quantify + on a normal iBGP/OSPF setup — is sub-second realistic? +* **No GARP needed.** Normally a migrating VM sends a gratuitous ARP to update switch tables. Here + there is no L2 path to update, and the destination host's neighbour entry is installed statically + by the agent rather than learned — so this problem disappears. Verify. +* **Per-address advertisement is mandatory.** Any aggregation scheme that pins prefixes to hosts + breaks migration. This is the price of §6.3.3. +* **Routing domain boundaries.** Migration works as long as source and destination are in the same + routing domain. **Left to the operator, not validated by CloudStack** — consistent with §10 and + §6.3.3, where fabric topology is local network design. CloudStack has no model of routing domains + and inventing one to police migration would be a larger change than the problem warrants. + +## 12. Security + +### 12.1 The isolation model + +The boundary between networks is **topological**: one bridge per network (§9.2, §9.4), with no +uplink and therefore no path between bridges except through the host's routing table. Nothing has to +be configured correctly for that to hold, and nothing degrades if filtering is disabled. + +Within one network the guests still share a bridge, so the properties there are weaker: + +* **No L3 adjacency.** Each guest is a /32 behind the host's routing table; there is no + subnet-mates relationship to exploit even between guests on the same bridge. +* **Static neighbour entries** prevent ARP-based address takeover on the host→guest path: the + IP-to-MAC mapping is asserted by `modifymacip.sh`, never learned (§5.3). The host is therefore not + susceptible to a guest claiming another guest's address. +* **`rp_filter=1` on the bridge** (§9.5) confines an Instance to sending from its own /32, since + that is the only address routed back out of that bridge. IPv4 only. + +What remains open inside a network is set out in §12.3. Because a network belongs to one account, +that is intra-tenant exposure. + +### 12.2 Security groups — supported, via the unified script rules **DECIDED** + +Security groups are supported on L3 networks, programmed by `security_group.py` as for Shared +networks. There is **no separate L3 rule function**: the classic and Direct Routed paths share one +implementation, parameterized only by how each direction identifies the Instance, plus small +conditionals for what does not exist on L3 (DHCP, DHCPv6, router advertisements towards guests). + +| Direction | Classic bridge | Direct Routed bridge | +|---|---|---| +| from the Instance | `-m physdev --physdev-in ` | same rule, identical | +| towards the Instance | `-m physdev --physdev-is-bridged --physdev-out ` | `-m set --match-set dst` | + +**The `--physdev-is-bridged` question, settled in kernel source** (`net/netfilter/xt_physdev.c`): + +* On `--physdev-in` rules the flag was **removable**. Match-time semantics: a bridged-then-routed + packet carries bridge info with the ingress port set and no bridged egress port, so a plain + `--physdev-in` matches it while `--physdev-is-bridged` excludes it. On classic bridges removing + the flag changes nothing observable — the BF- framework hook (which keeps the flag) already + restricts what reaches the per-VM chains to bridged traffic. Removing it is what lets both paths + share the from-Instance rules verbatim. The golden test was regenerated once, deliberately, for + exactly this diff. +* On `--physdev-out` rules the flag **stays**: for a routed packet no bridge info exists at all at + FORWARD time (`nf_bridge_info_exists()` is false and every physdev variant returns false), so the + kernel is structurally incapable of identifying the bridged egress port there. Removing the flag + would change nothing and merely deviate from upstream convention. This is why the to-Instance + direction matches the destination against the Instance's ipsets instead — exact, since L3 + addresses are /32s and /128s. + +The IPv6 source-spoof drop in the shared rules doubles as the missing IPv6 `rp_filter` (§9.5). +Teardown uses one awk pattern on the chain names, which also covers rules created by older versions +that still carry the flag on `physdev-in` lines. + +**Framework setup is shared too.** `enable_bridge_netfilter()`, `create_bridge_fw_chains()` and +`add_notrack_ipset_rules()` were extracted from `add_fw_framework()` and are used by both it and +`add_l3_fw_framework()`; the two differ only in their FORWARD hooks (classic gates on +`physdev-is-bridged` and consults chain reference counts; L3 jumps unconditionally, with the same +default-deny backstop). A second golden test pins `add_fw_framework`'s command stream, proving the +extraction left it byte-identical — 44 commands, unchanged. + +**How the script knows: the Agent tells it.** `security_group.py` performs no classification of its +own — no bridge-name check, no gateway inspection. The Agent already identifies these NICs for the +bridge and MAC/IP hook (§9.1.3), so it passes `--directrouted` on `default_network_rules` and +`add_network_rules`; the script's own re-entry points thread the flag through. This keeps the +decision in exactly one host-side place and leaves the script with no inference to get wrong. + +One path cannot receive the flag: `network_rules_for_rebooted_vm` runs without the Agent. It +restores dispatch rules in whichever form the Instance's per-VM chain already uses (destination +ipset for Direct Routed, `physdev-out` for bridged), so a rebooted Instance is reprogrammed +correctly either way. + +**Investigated and rejected: libvirt nwfilters.** An nwfilter-based implementation was built and +then discarded. Its ebtables layer would have served anti-spoofing well (it sees routed delivery), +but stateful ingress cannot work: libvirt's own to-Instance iptables hook is hard-coded as +`-m physdev --physdev-is-bridged --physdev-out` (`src/nwfilter/nwfilter_ebiptables_driver.c`), the +same structural blindness to routed delivery — inside libvirt, where it cannot be patched or +augmented with destination matching. The script approach filters correctly in both directions and, +after the unification above, without duplicate code. + +### 12.3 Residual risk within one network **ACCEPTED for v1** + +Guests of the same network share a bridge. With security groups enabled, RA and gateway +impersonation between them are handled by the shared rules (ebtables ARP pinning, NDP source +checks, RA drop) and source spoofing is bounded by the ipset checks plus `rp_filter` (§9.5). An +operator who disables security groups accepts intra-tenant spoofing between guests of that one +network — a deliberate operator choice; isolation between *tenants* is topological and unaffected +(§12.1). The host itself is immune either way: its neighbour entries are static (§5.3). + +### 12.4 Sharp edge + +**Guests are directly reachable from the fabric.** No NAT, no VR firewall. Whatever the fabric +permits reaches the guest, subject only to the guest's security groups if they are in use. This is a +meaningful change in default posture versus an Isolated network and must be prominent in the +documentation. + +## 13. Orchestration touchpoints + +Checklist to work through: + +- [ ] `NetworkOrchestrator.allocate()` / `prepare()` / `release()` — address lifecycle +- [ ] `NetworkOrchestrator` and `VirtualMachineManagerImpl` — `L2` and `Shared` branches +- [ ] `UserVmManagerImpl` — `addNicToVm`, `updateDefaultNic`, IP change +- [ ] `NetworkModelImpl` — capability lookups, `getNetworkTag`, `isSecurityGroupSupportedInNetwork` +- [ ] `IpAddressManagerImpl.allocateDirectIp()` — gateway/netmask override for /32 and /128 +- [ ] `Networks.BroadcastDomainType` — new `Routed("routed", Long.class)` value (§6.7) +- [ ] `BridgeVifDriver` — select on `BroadcastDomainType.Routed`; gate the MAC/IP hook and bridge + selection on it. Script and bridge targeting reused unchanged (§9.1.3) +- [ ] `scripts/vm/network/vnet/modifybrdr.sh` — per-network bridge lifecycle (§9.2) +- [ ] `BridgeVifDriver` — take the routed id from the NIC's broadcast URI, call `modifybrdr.sh` on + plug and last unplug +- [ ] Guru registers `IsolationMethod("ROUTED")`; `canHandle()` on guest type + isolation method; + `design()` sets `Routed` + `routed://` (§6.7) +- [ ] `NetworkOrchestrator.encodeVlanIdIntoBroadcastUri()` — `ROUTED` physical network → + `routed://` URI; broadcast domain type derived from the URI scheme, not hard-coded `Vlan` + (§6.7.2) +- [ ] `NetworkServiceImpl.commitNetwork()` — extend the shared-without-`specifyVlan` vnet + auto-allocation to `GuestType.L3`; extend the matching release on deletion in + `NetworkOrchestrator` (§6.7.2) +- [ ] Offering validation — require ConfigDrive `UserData`, reject `Dhcp`; allow `specifyVlan` + either way (§6.4) +- [ ] `ConfigDriveBuilder.needForGeneratingNetworkData()` — must not gate on Dhcp/Dns (§8.2) +- [ ] CPVM/SSVM boot args — emit `ethip6`/`ethip6prelen`/`ip6gateway` as the VR builder + already does (§8.5) +- [ ] `systemvm/.../setup/common.sh` — `onlink` on the v4 default route for link-local gateways; + install the v6 default route from `IP6GW` instead of relying on RA (§8.5) +- [ ] `PublicNetworkGuru` / `createVlanIpRange` — direct routed public range for systemvm IPs, + dual-stack: host-route NIC form (v4 and v6, EUI-64 computed in the guru), `Routed` broadcast + URI; `BridgeVifDriver` Public branch handles it; routed ids guarded against guest/public + collisions in both directions (§8.5, §6.7.2) +- [ ] `createNetwork`/`createVlanIpRange` — reject an IPv6 CIDR with a prefix longer than /64 for + L3 networks; EUI-64 needs 64 interface-identifier bits (§6.3.4) +- [ ] `NetworkServiceImpl.java:657` — stop rejecting DNS for this type as it does for L2 (§6.4) +- [ ] `security_group.py` unified rules — verify on a real host that both directions match live + traffic on classic and Direct Routed bridges, and that ARP for `169.254.0.1` / ND for + `fe80::1` pass (§12.2) +- [ ] `createVlanIpRange` — zone-wide subnet overlap validation (§6.3.2) +- [ ] `NetUtils` — inclusive range variants so `.0` and `.255` are assignable (§6.3.1) +- [ ] VM snapshot / restore, VM import (`UnmanagedVMsManagerImpl`), template creation +- [ ] Network restart — no-op without a VR? +- [ ] IP capacity reporting and usage records — is a directly routed address a billable public IP? +- [ ] UI: `ROUTED` in the physical network isolation method options (zone wizard, + `phynetworks.js`); network creation wizard (the `vlan` field labelled as routed id for L3 + offerings); network detail page, NIC display, offering creation + +## 14. Upgrade and compatibility + +* Additive: new enum values (`GuestType.L3`, `BroadcastDomainType.Routed`), new isolation method + string, new offering type. No change to existing networks, no data migration. +* The pre-revision implementation on this branch (L3 networks with `Native` broadcast domain and an + empty `broadcast_uri`) was never released, so no migration from it is provided; development + deployments recreate their L3 networks. +* **Agent version gating is out of scope. DECIDED.** No capability flag, no version check, and no + management-server logic to keep Instances of this type away from agents that predate it. Operators + are expected to upgrade their agents as part of upgrading CloudStack, as they already are. The + failure mode if they do not is that `modifybrdr.sh` is missing on the old host and the Instance + fails to get connectivity — visible in the agent log, consistent with §9.1.4. +* Downgrade unsupported once networks of this type exist, as usual. + +## 15. Future work / explicitly deferred + +* Non-KVM hypervisors +* Optional CloudStack-managed routing daemon configuration +* **Host-side metadata service** (`169.254.169.254`) — a v2 candidate; the gateway address and the + data are both already present, so it is a natural addition (§8.4) +* Multiple addresses per NIC — additional /32s fit the model naturally, but v1 is one v4 + one v6 +* Reachability/health feedback from the routing daemon +* Network Config v2 `network-config` emission for NoCloud-configured images — a later PR (§8.1) +* Making a failed route/neighbour install fatal or alert-raising, rather than silent (§9.1.4) +* Closing the §12.3 intra-network spoofing gaps — bridge port isolation (§9.4) first, or libvirt + nwfilter if a narrower fix is preferred +* **Per-tenant VRFs**, which would lift the non-overlapping-subnet constraint of §6.3.2 + +## 16. Decision log and open questions + +### Revised 2026-09-04 — isolation model + +Three decisions from the 2026-07-30 design were reversed together; rationale in §6.7: + +* **Isolation method:** was "none to register"; now the guru registers `IsolationMethod("ROUTED")` + and direct routed networks live on a dedicated physical network carrying it. The network operator + opts the zone in explicitly. +* **Broadcast domain:** was "`Native`, empty URI"; now `BroadcastDomainType.Routed` with + `routed://`, set at creation and stable for the network's life. +* **Bridge naming:** was `brdr-` (unchoosable); now `brdr-`, with the id + operator-specified (`specifyVlan=true` + `vlan` parameter) or allocated from the physical + network's range (`specifyVlan=false`) — the Shared network's VLAN mechanics, reused (§6.7.2). + +The agent's inferred /32-plus-link-local-gateway gating became explicit gating on the broadcast +type as a consequence (§9.1.3). Entries below are updated in place; superseded wording is kept in +the relevant sections as "reverses the earlier decision" notes. + +### Settled + +* Network type is "no DHCP", not "like L2" (§1) +* IPv4 addresses come from the `user_ip_address` pool with an operator-supplied subnet (§6.3); + subnets must not overlap zone-wide (§6.3.2) +* **IPv6 is calculated (EUI-64) from the subnet and the NIC's MAC** — never drawn from a stored + pool; only the allocated result is stored, on the NIC. Already how both the guest and the public + path behave; requires the subnet to be /64 or larger, to be validated at creation (§6.3.4) +* New `GuestType.L3`, chosen over overloading `Shared`/`NetworkMode` (§6.1) +* Gateway is a static, non-configurable `169.254.0.1` / `fe80::1`; a /32 means the guest must treat + its gateway as on-link regardless, so configurability would buy nothing (§6.2) +* ConfigDrive emits `on-link: true` for gateways in `169.254.0.0/16` (§8.1) +* The agent writes only routes and neighbour entries; FRR is out of scope (§9.1, §10) +* That work is done by reusing `modifymacip.sh` + the `BridgeVifDriver` hook already in main from + `4816e059383` / PR #13495, rather than new code (§9.1.1) +* Routes and neighbour entries go on the **bridge**, not the tap — the script and hook are reused + unchanged (§9.1.2) +* **One bridge per network**, named `brdr-`, created and removed by the new + `scripts/vm/network/vnet/modifybrdr.sh` (§9.2) +* **Isolation method `ROUTED` on a dedicated physical network** selects the guru; the network's + broadcast domain is `routed://` and the id is operator-specified or allocated from the + physical network's range — reversing the earlier "no isolation method, no broadcast domain" + decision (§6.7, §6.7.2, §9.2.1; revision block above) +* Per-network bridges also give the operator a named interface per network to hang local routing + policy off — a deliberate benefit, not just a side effect (§9.2) +* Those bridges have no physical uplink (§9.3) +* L2 isolation between networks is topological — separate bridges — not filtering. **No libvirt + nwfilter is used**; the earlier `no-mac-spoofing` / `clean-traffic` plan is dropped (§9.4) +* Security groups are supported on L3 via **one unified rule implementation** shared with classic + bridges: `--physdev-is-bridged` dropped from `physdev-in` rules (verified harmless in kernel + source), destination-ipset matching towards the Instance. nwfilter was investigated and rejected + — libvirt's own to-Instance hook has the same physdev-is-bridged blindness (§12.2) +* Within one network, RA and gateway-impersonation protection depends on security groups being + enabled; leaving them off is a deliberate operator choice (§12.3) +* The `--physdev-is-bridged` rework is **required** — L3 filtering must work — and must be strictly + additive so existing Basic-zone and Shared-network rules are unchanged (§12.2) +* A network is direct routed when its offering's guest type is `L3` **and** it lives on a + physical network with isolation method `ROUTED` — guru selection follows the standard + isolation-method contract. Changing a network's offering afterwards is not guarded (§6.7, §6.7.1) +* The offering requires ConfigDrive `UserData`; `Dns` is optional but strongly recommended, and + `SecurityGroup` is optional. `Dhcp` is rejected — not just unsupported but unnecessary (§6.4, §6.5) +* `rp_filter=1` is set on each `brdr-*` bridge, bounding IPv4 source spoofing to the Instance's own + address; IPv6 has no kernel equivalent (§9.5) +* **No reconciliation at agent startup** — a host reboot destroys the Instances too, and an agent + restart does not clear kernel state (§9.6) +* The MAC/IP hook is gated **per NIC, on `BroadcastDomainType.Routed`** rather than a host + property; the same test picks the bridge. (Originally inferred from the /32 + link-local-gateway + address form; superseded by the explicit broadcast type — §9.1.3) +* The same `169.254.0.1` on every `brdr-*` bridge is **correct as designed**; `arp_ignore=1` and + `arp_announce=2` handle it (§9.2.2) +* The guru is a **subclass of `DirectNetworkGuru`**, inheriting the address lifecycle rather than + duplicating it (§7.3) +* **No on-link plumbing in ConfigDrive** — cloud-init itself sets on-link for an IPv4 gateway in + `169.254.0.0/16` when consuming `network_data.json` (verified); the v2 `network-config` file is + deferred to a later PR (§8.1, §15) +* DNS is **per network, falling back to the zone**; already implemented by + `NetworkModelImpl.getNetworkIp4Dns()` (§8.3) +* **No metadata service** in v1 — ConfigDrive only, no `169.254.169.254`; a v2 candidate (§8.4) +* Route/neighbour install failures **stay silent** for v1, as they are for EVPN (§9.1.4) +* Route scale is **out of scope** — fabric capacity and aggregation are local network design; + ~100k routes is not usually a problem on modern equipment, but CloudStack states no ceiling (§6.3.3) +* Network and broadcast addresses **must be assignable** — verified already true on the explicit + start/end range path; the `NetUtils` exclusion only affects CIDR-derived Isolated ranges (§6.3.1) +* The subnet's gateway stays **required and ignored**; relaxing validation shared with other + network types is not worth it for v1 (§5.5) +* The `--physdev-is-bridged` rework **lands in v1**; security groups are not shipped half-working + (§12.2) +* Zone-wide subnet overlap validation is **required** — an overlap is an address conflict (§6.3.2) +* **No warning** when a template ignores ConfigDrive; cloud-init inside the guest is the operator's + responsibility (§6.5) +* **Agent version gating is out of scope** — operators upgrade agents with CloudStack (§14) +* KVM only for v1 (§6.8); **VPC never** — it is a separate use case already served by VPC's own + BGP-routed subnets (§6.6) +* **SystemVMs stay, configured via boot args, dual-stack from the start** — the v4 default route + gains `onlink` for link-local gateways, the CPVM/SSVM builders emit the IPv6 args the VR builder + already emits, the systemvm installs the v6 default from `ip6gateway` instead of hoping for an + RA, and the public NIC is stamped/plugged in the same host-route + `routed://` form as a guest + NIC — its IPv6 coming from the same EUI-64 computation (§8.5, §6.3.4) + +### Still open + +**None.** Every design question raised in this document has been decided. + +What remains is implementation work, tracked in §13 and as `TODO` markers in the sections above. +Three of those are verification rather than coding, and are the ones most likely to change a +decision if they come out badly: + +* §12.2 — confirm on a real host that the unified rules match live traffic in both directions on + both bridge types, and that ARP for the gateway and neighbour discovery pass +* §6.3.1 — confirm nothing downstream of `createVlanIpRange` re-derives the usable range and + re-excludes `.0` and `.255` +* §6.3.2 — confirm the existing overlap checks are zone-wide, and widen them if not diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java index 109a44488ec4..8ebd251d5b71 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java @@ -218,6 +218,8 @@ void prepare(VirtualMachineProfile profile, DeployDestination dest, ReservationC boolean isSharedNetworkWithoutSpecifyVlan(NetworkOffering offering); + boolean isL3NetworkWithoutSpecifyVlan(NetworkOffering offering); + boolean shutdownNetwork(long networkId, ReservationContext context, boolean cleanupElements); boolean destroyNetwork(long networkId, ReservationContext context, boolean forced); diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index 8af75562b31c..e60de309cc49 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -557,6 +557,17 @@ public boolean configure(final String name, final Map params) th sgProviders.add(Provider.SecurityGroupProvider); defaultSharedSGEnabledNetworkOfferingProviders.put(Service.SecurityGroup, sgProviders); + // Direct Routed (L3) networks have no Virtual Router and no DHCP: Instances learn their + // addressing from ConfigDrive, which also carries UserData and the DNS configuration. + final Map> defaultL3NetworkOfferingProviders = new HashMap<>(); + final Set configDriveProvider = new HashSet<>(); + configDriveProvider.add(Provider.ConfigDrive); + defaultL3NetworkOfferingProviders.put(Service.UserData, configDriveProvider); + defaultL3NetworkOfferingProviders.put(Service.Dns, configDriveProvider); + final Set l3SecurityGroupProvider = new HashSet<>(); + l3SecurityGroupProvider.add(Provider.SecurityGroupProvider); + defaultL3NetworkOfferingProviders.put(Service.SecurityGroup, l3SecurityGroupProvider); + tungstenProvider.add(Provider.Tungsten); final Map> defaultTungstenSharedSGEnabledNetworkOfferingProviders = new HashMap<>(); defaultTungstenSharedSGEnabledNetworkOfferingProviders.put(Service.Connectivity, tungstenProvider); @@ -620,6 +631,17 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { null, true, false, false, false, false, null, null, null, true, null, null, false); } + //#3b - Direct Routed (L3) network offering: ConfigDrive for UserData and DNS, + // Security Groups for Instance isolation. Created on upgrade and fresh install + // alike, since this block runs on every management server start. The routed id + // is allocated from the ROUTED physical network's range (no specifyVlan). + if (_networkOfferingDao.findByUniqueName(NetworkOffering.DefaultL3NetworkOffering) == null) { + offering = _configMgr.createNetworkOffering(NetworkOffering.DefaultL3NetworkOffering, + "Offering for Direct Routed (L3) networks - public IPs routed directly to Instances, configuration via ConfigDrive (UserData and DNS), Security Groups enabled, no Virtual Router and no DHCP", + TrafficType.Guest, null, false, Availability.Optional, null, defaultL3NetworkOfferingProviders, true, Network.GuestType.L3, false, null, true, + null, true, false, null, false, null, true, false, false, false, false, null, null, null, true, null, null, false); + } + if (_networkOfferingDao.findByUniqueName(NetworkOffering.DEFAULT_TUNGSTEN_SHARED_NETWORK_OFFERING_WITH_SGSERVICE) == null) { offering = _configMgr.createNetworkOffering(NetworkOffering.DEFAULT_TUNGSTEN_SHARED_NETWORK_OFFERING_WITH_SGSERVICE, "Offering for Tungsten Shared Security group enabled networks", TrafficType.Guest, null, true, Availability.Optional, null, defaultTungstenSharedSGEnabledNetworkOfferingProviders, true, Network.GuestType.Shared, false, null, true, @@ -2940,7 +2962,7 @@ private Network createGuestNetwork(final long networkOfferingId, final String na final boolean vlanSpecified = vlanId != null; if (vlanSpecified != ntwkOff.isSpecifyVlan()) { if (vlanSpecified) { - if (!isSharedNetworkWithoutSpecifyVlan(ntwkOff) && !isPrivateGatewayWithoutSpecifyVlan(ntwkOff)) { + if (!isSharedNetworkWithoutSpecifyVlan(ntwkOff) && !isL3NetworkWithoutSpecifyVlan(ntwkOff) && !isPrivateGatewayWithoutSpecifyVlan(ntwkOff)) { throw new InvalidParameterValueException("Can't specify vlan; corresponding offering says specifyVlan=false"); } } else { @@ -2950,13 +2972,22 @@ private Network createGuestNetwork(final long networkOfferingId, final String na if (vlanSpecified) { URI uri = encodeVlanIdIntoBroadcastUri(vlanId, pNtwk); + // A routed id names a bridge on every host (brdr-): a guest network and a + // public range sharing one id would merge their L2 domains. Public ranges store + // the routed:// URI as their vlan tag; reject any id one already carries. + // (Guest-vs-guest collisions are caught by the zone-wide URI check below.) + if (BroadcastDomainType.getSchemeValue(uri) == BroadcastDomainType.Routed + && _vlanDao.findByZoneAndVlanId(zoneId, uri.toString()) != null) { + throw new InvalidParameterValueException(String.format( + "The routed id %s is already used by a public IP range in zone %s", vlanId, zone.getName())); + } // Aux: generate secondary URI for secondary VLAN ID (if provided) for performing checks URI secondaryUri = StringUtils.isNotBlank(isolatedPvlan) ? BroadcastDomainType.fromString(isolatedPvlan) : null; - if (isSharedNetworkWithoutSpecifyVlan(ntwkOff) || isPrivateGatewayWithoutSpecifyVlan(ntwkOff)) { + if (isSharedNetworkWithoutSpecifyVlan(ntwkOff) || isL3NetworkWithoutSpecifyVlan(ntwkOff) || isPrivateGatewayWithoutSpecifyVlan(ntwkOff)) { bypassVlanOverlapCheck = true; } //don't allow to specify vlan tag used by physical network for dynamic vlan allocation - if (!(bypassVlanOverlapCheck && (ntwkOff.getGuestType() == GuestType.Shared || isPrivateNetwork)) + if (!(bypassVlanOverlapCheck && (ntwkOff.getGuestType() == GuestType.Shared || ntwkOff.getGuestType() == GuestType.L3 || isPrivateNetwork)) && _dcDao.findVnet(zoneId, pNtwk.getId(), BroadcastDomainType.getValue(uri)).size() > 0) { throw new InvalidParameterValueException("The VLAN tag to use for new guest network, " + vlanId + " is already being used for dynamic vlan allocation for the guest network in zone " + zone.getName()); @@ -3161,14 +3192,21 @@ public Network doInTransaction(final TransactionStatus status) { uri = encodeVlanIdIntoBroadcastUri(vlanIdFinal, pNtwk); } - if (_networksDao.listByPhysicalNetworkPvlan(physicalNetworkId, uri.toString()).size() > 0) { + // The PVLAN overlap check only understands vlan:// and vxlan:// URIs. A + // routed:// URI is a bridge label on a ROUTED physical network, where no + // PVLAN network can exist; its overlap checks (zone-wide URI, public + // ranges, vnet range) have already run above. + final boolean isRoutedUri = uri != null && BroadcastDomainType.getSchemeValue(uri) == BroadcastDomainType.Routed; + if (!isRoutedUri && _networksDao.listByPhysicalNetworkPvlan(physicalNetworkId, uri.toString()).size() > 0) { throw new InvalidParameterValueException(String.format( "Network with vlan %s already exists or overlaps with other network pvlans in zone %s", vlanIdFinal, zone)); } userNetwork.setBroadcastUri(uri); - if (!vlanIdFinal.equalsIgnoreCase(Vlan.UNTAGGED)) { + if (uri != null && BroadcastDomainType.getSchemeValue(uri) == BroadcastDomainType.Routed) { + userNetwork.setBroadcastDomainType(BroadcastDomainType.Routed); + } else if (!vlanIdFinal.equalsIgnoreCase(Vlan.UNTAGGED)) { userNetwork.setBroadcastDomainType(BroadcastDomainType.Vlan); } else { userNetwork.setBroadcastDomainType(BroadcastDomainType.Native); @@ -3233,6 +3271,19 @@ public boolean isSharedNetworkWithoutSpecifyVlan(NetworkOffering offering) { return !offering.isSpecifyVlan(); } + /** + * An L3 (Direct Routed) network whose offering does not carry specifyVlan gets its routed id + * allocated from the physical network's vnet range at creation, the same way a Shared network + * without specifyVlan gets its VLAN — and released the same way on deletion. + */ + @Override + public boolean isL3NetworkWithoutSpecifyVlan(NetworkOffering offering) { + if (offering == null || offering.getTrafficType() != TrafficType.Guest || offering.getGuestType() != GuestType.L3) { + return false; + } + return !offering.isSpecifyVlan(); + } + private boolean isPrivateGatewayWithoutSpecifyVlan(NetworkOffering ntwkOff) { return ntwkOff.getId() == _networkOfferingDao.findByUniqueName(NetworkOffering.SystemPrivateGatewayNetworkOfferingWithoutVlan).getId(); } @@ -3250,9 +3301,14 @@ protected URI encodeVlanIdIntoBroadcastUri(String vlanId, PhysicalNetwork pNtwk) if (!pNtwk.getIsolationMethods().isEmpty() && StringUtils.isNotBlank(pNtwk.getIsolationMethods().get(0))) { String isolationMethod = pNtwk.getIsolationMethods().get(0).toLowerCase(); String vxlan = BroadcastDomainType.Vxlan.toString().toLowerCase(); + String routed = BroadcastDomainType.Routed.toString().toLowerCase(); if (isolationMethod.equals(vxlan)) { return BroadcastDomainType.encodeStringIntoBroadcastUri(vlanId, BroadcastDomainType.Vxlan); } + if (isolationMethod.equals(routed)) { + // The routed id is a bridge-name label, not an encapsulation: it must be numeric + return BroadcastDomainType.encodeStringIntoBroadcastUri(vlanId, BroadcastDomainType.Routed); + } } return BroadcastDomainType.fromString(vlanId); } @@ -3668,8 +3724,11 @@ protected Pair> deleteVlansInNetwork(final NetworkVO netwo logger.debug("Deleted ip range for private network {}", network); } - // release vlans of user-shared networks without specifyvlan - if (isSharedNetworkWithoutSpecifyVlan(_networkOfferingDao.findById(network.getNetworkOfferingId()))) { + // release vlans of user-shared networks without specifyvlan, and the routed ids of + // L3 (Direct Routed) networks without specifyvlan — both were allocated from the + // physical network's vnet range at creation + final NetworkOffering deletedNetworkOffering = _networkOfferingDao.findById(network.getNetworkOfferingId()); + if (isSharedNetworkWithoutSpecifyVlan(deletedNetworkOffering) || isL3NetworkWithoutSpecifyVlan(deletedNetworkOffering)) { logger.debug("Releasing vnet for the network {}", network); _dcDao.releaseVnet(BroadcastDomainType.getValue(network.getBroadcastUri()), network.getDataCenterId(), network.getPhysicalNetworkId(), network.getAccountId(), network.getReservationId()); diff --git a/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java b/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java index 15febbe972c4..9b27cd726591 100644 --- a/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java +++ b/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java @@ -48,6 +48,7 @@ import com.cloud.network.NetworkModel; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.NetUtils; import com.cloud.utils.script.Script; import com.google.gson.JsonArray; import com.google.gson.JsonElement; @@ -249,8 +250,15 @@ static void writeVmMetadata(List vmData, String tempDirName, File open */ static void writeNetworkData(List nics, Map> supportedServices, File openStackFolder) { JsonObject finalNetworkData = new JsonObject(); - if (needForGeneratingNetworkData(supportedServices)) { + // A direct routed NIC always needs its network data written: ConfigDrive is the only + // channel that carries its addressing, whatever services the offering does or does not + // have. For every other NIC the historical gate (Dhcp or Dns supported) is unchanged. + boolean generateForAllNics = needForGeneratingNetworkData(supportedServices); + if (generateForAllNics || nics.stream().anyMatch(ConfigDriveBuilder::isDirectRoutedNic)) { for (NicProfile nic : nics) { + if (!generateForAllNics && !isDirectRoutedNic(nic)) { + continue; + } List supportedService = supportedServices.get(nic.getId()); JsonObject networkData = getNetworkDataJsonObjectForNic(nic, supportedService); @@ -267,6 +275,24 @@ static boolean needForGeneratingNetworkData(Map> sup return supportedServices.values().stream().anyMatch(services -> services.contains(Network.Service.Dhcp) || services.contains(Network.Service.Dns)); } + /** + * A NIC on a Direct Routed (L3) network is recognised by the form of its addressing, not by a + * flag: an IPv4 host netmask with a link-local gateway, or an IPv6 /128 with the fixed + * link-local gateway. No other network type produces this combination. ConfigDrive is the + * only channel that carries such a NIC's network configuration (there is no DHCP and no RA), + * so network data must always be generated for it, whatever services the offering carries. + */ + static boolean isDirectRoutedNic(NicProfile nic) { + if (nic == null) { + return false; + } + boolean directRoutedIpv4 = StringUtils.isNotBlank(nic.getIPv4Address()) && NetUtils.IPV4_HOST_NETMASK.equals(nic.getIPv4Netmask()) + && StringUtils.isNotBlank(nic.getIPv4Gateway()) && NetUtils.isIpWithInCidrRange(nic.getIPv4Gateway(), NetUtils.getLinkLocalCIDR()); + boolean directRoutedIpv6 = StringUtils.isNotBlank(nic.getIPv6Address()) && StringUtils.isNotBlank(nic.getIPv6Cidr()) + && nic.getIPv6Cidr().endsWith("/" + NetUtils.IPV6_HOST_PREFIX_LENGTH) && NetUtils.getIpv6LinkLocalGateway().equals(nic.getIPv6Gateway()); + return directRoutedIpv4 || directRoutedIpv6; + } + /** * Writes an empty JSON file named vendor_data.json in openStackFolder * diff --git a/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java b/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java index 03ceac843997..9b785ae95900 100644 --- a/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java +++ b/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java @@ -52,6 +52,7 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; +import com.google.gson.JsonArray; import com.google.gson.JsonObject; @RunWith(MockitoJUnitRunner.class) @@ -659,4 +660,96 @@ public void testWriteNetworkDataEmptyJson() throws Exception { Assert.assertEquals(expectedJsonObject, actualJson); folder.delete(); } -} + + private NicProfile directRoutedNicProfile() { + NicProfile nic = new NicProfile(); + nic.setId(1L); + nic.setDeviceId(0); + nic.setMacAddress("02:00:4c:5f:00:01"); + nic.setIPv4Address("203.0.113.55"); + nic.setIPv4Netmask("255.255.255.255"); + nic.setIPv4Gateway("169.254.0.1"); + nic.setIPv6Address("2001:db8:1::55"); + nic.setIPv6Cidr("2001:db8:1::55/128"); + nic.setIPv6Gateway("fe80::1"); + nic.setIPv4Dns1("8.8.8.8"); + return nic; + } + + private NicProfile sharedNicProfile() { + NicProfile nic = new NicProfile(); + nic.setId(1L); + nic.setDeviceId(0); + nic.setMacAddress("02:00:4c:5f:00:02"); + nic.setIPv4Address("10.1.1.55"); + nic.setIPv4Netmask("255.255.255.0"); + nic.setIPv4Gateway("10.1.1.1"); + return nic; + } + + @Test + public void isDirectRoutedNicRecognisesHostRouteForm() { + Assert.assertTrue(ConfigDriveBuilder.isDirectRoutedNic(directRoutedNicProfile())); + } + + @Test + public void isDirectRoutedNicRecognisesIpv6OnlyForm() { + NicProfile nic = directRoutedNicProfile(); + nic.setIPv4Address(null); + nic.setIPv4Netmask(null); + nic.setIPv4Gateway(null); + Assert.assertTrue(ConfigDriveBuilder.isDirectRoutedNic(nic)); + } + + @Test + public void isDirectRoutedNicRejectsOrdinaryNics() { + Assert.assertFalse(ConfigDriveBuilder.isDirectRoutedNic(sharedNicProfile())); + Assert.assertFalse(ConfigDriveBuilder.isDirectRoutedNic(null)); + // a /32 with an ordinary gateway is not direct routed + NicProfile hostMaskOnly = sharedNicProfile(); + hostMaskOnly.setIPv4Netmask("255.255.255.255"); + Assert.assertFalse(ConfigDriveBuilder.isDirectRoutedNic(hostMaskOnly)); + } + + @Test + public void ordinaryNicRouteGenerationIsUnchanged() { + JsonArray networks = ConfigDriveBuilder.getNetworksJsonArrayForNic(sharedNicProfile()); + JsonObject ipv4Network = networks.get(0).getAsJsonObject(); + JsonArray routes = ipv4Network.getAsJsonArray("routes"); + Assert.assertEquals(1, routes.size()); + JsonObject defaultRoute = routes.get(0).getAsJsonObject(); + Assert.assertEquals("0.0.0.0", defaultRoute.get("network").getAsString()); + Assert.assertEquals("0.0.0.0", defaultRoute.get("netmask").getAsString()); + Assert.assertEquals("10.1.1.1", defaultRoute.get("gateway").getAsString()); + } + + @Test + public void networkDataIsGeneratedForDirectRoutedNicWithoutDhcpOrDns() throws Exception { + TemporaryFolder folder = new TemporaryFolder(); + folder.create(); + try { + Map> userDataOnly = Map.of(1L, List.of(Network.Service.UserData)); + ConfigDriveBuilder.writeNetworkData(List.of(directRoutedNicProfile()), userDataOnly, folder.getRoot()); + String json = FileUtils.readFileToString(new File(folder.getRoot(), "network_data.json"), com.cloud.utils.StringUtils.getPreferredCharset()); + Assert.assertTrue("direct routed nic must appear in network_data.json", json.contains("203.0.113.55")); + Assert.assertTrue(json.contains("169.254.0.1")); + } finally { + folder.delete(); + } + } + + @Test + public void networkDataStaysEmptyForOrdinaryNicWithoutDhcpOrDns() throws Exception { + TemporaryFolder folder = new TemporaryFolder(); + folder.create(); + try { + Map> userDataOnly = Map.of(1L, List.of(Network.Service.UserData)); + ConfigDriveBuilder.writeNetworkData(List.of(sharedNicProfile()), userDataOnly, folder.getRoot()); + String json = FileUtils.readFileToString(new File(folder.getRoot(), "network_data.json"), com.cloud.utils.StringUtils.getPreferredCharset()); + Assert.assertEquals("historical gate must be preserved for ordinary nics", "{}", json); + } finally { + folder.delete(); + } + } + +} \ No newline at end of file diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java index eefe491e8b27..591932537f89 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java @@ -49,9 +49,22 @@ public class BridgeVifDriver extends VifDriverBase { private String _modifyVlanPath; private String _modifyVxlanPath; private String _macIpScriptPath; + private boolean _macIpStaticEnabled; + private String _modifyBrdrPath; private String _controlCidr = NetUtils.getLinkLocalCIDR(); private Long libvirtVersion; + /** + * A NIC on a Direct Routed (L3) network is recognised by its broadcast domain: routed://, + * stamped by the management server. The id is a label naming the per-network bridge + * (brdr-), never an encapsulation. An earlier revision inferred this from the address + * form (/32 + link-local gateway); the explicit broadcast type supersedes that implicit + * contract, and works for guest and (systemvm) public NICs alike. + */ + public static boolean isDirectRoutedNic(NicTO nic) { + return nic != null && nic.getBroadcastType() == Networks.BroadcastDomainType.Routed; + } + private static boolean isVxlanOrNetris(String protocol) { return protocol.equals(Networks.BroadcastDomainType.Vxlan.scheme()) || protocol.equals(Networks.BroadcastDomainType.Netris.scheme()); } @@ -84,14 +97,19 @@ public void configure(Map params) throws ConfigurationException throw new ConfigurationException("Unable to find " + vxlanScript); } - if (Boolean.TRUE.equals(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VM_NETWORK_MACIP_STATIC))) { - _macIpScriptPath = Script.findScript(networkScriptsDir, "modifymacip.sh"); + // Resolved unconditionally: Direct Routed (L3) NICs need the MAC/IP script regardless of + // the host-wide property, which remains the opt-in for running it on every NIC (EVPN). + _macIpScriptPath = Script.findScript(networkScriptsDir, "modifymacip.sh"); + _macIpStaticEnabled = Boolean.TRUE.equals(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VM_NETWORK_MACIP_STATIC)); + if (_macIpStaticEnabled) { if (_macIpScriptPath == null) { throw new ConfigurationException("Unable to find modifymacip.sh"); } logger.info("VM network MAC/IP static script configured: {}", _macIpScriptPath); } + _modifyBrdrPath = Script.findScript(networkScriptsDir, "modifybrdr.sh"); + libvirtVersion = (Long) params.get("libvirtVersion"); if (libvirtVersion == null) { libvirtVersion = 0L; @@ -247,7 +265,12 @@ public LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicA networkRateKBps = getNetworkRateKbps(nic); } - if (nic.getType() == Networks.TrafficType.Guest) { + // Direct Routed NICs — guest Instances and the public NICs of SystemVMs alike — go on + // the network's own uplink-less bridge, never on a shared guest/public bridge. + if ((nic.getType() == Networks.TrafficType.Guest || nic.getType() == Networks.TrafficType.Public) && isDirectRoutedNic(nic)) { + String brName = createDirectRoutedBridge(nic); + intf.defBridgeNet(brName, null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), networkRateKBps); + } else if (nic.getType() == Networks.TrafficType.Guest) { if (isBroadcastTypeVlanOrVxlan(nic) && isValidProtocolAndVnetId(vNetId, protocol)) { if (trafficLabel != null && !trafficLabel.isEmpty()) { logger.debug("creating a vNet dev and bridge for guest traffic per traffic label " + trafficLabel); @@ -295,17 +318,98 @@ public LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicA } intf.setLinkStateUp(nic.isEnabled()); - executeMacIpScript(intf.getBrName(), nic.getMac(), nic.getIp(), nic.getIp6Address(), nic.getNicSecIps()); + // Host-wide property (EVPN use case) runs the MAC/IP script for every NIC; a Direct + // Routed NIC needs it regardless, since the host route and static neighbour entry are + // what deliver its traffic. + if (_macIpStaticEnabled || isDirectRoutedNic(nic)) { + executeMacIpScript(intf.getBrName(), nic.getMac(), nic.getIp(), nic.getIp6Address(), nic.getNicSecIps()); + } return intf; } @Override public void unplug(LibvirtVMDef.InterfaceDef iface, boolean deleteBr) { - executeMacIpScript(iface.getBrName(), iface.getMacAddress()); + // How the bridges of Direct Routed (L3) networks are named is known only to + // modifybrdr.sh; the agent classifies an interface at unplug by asking it. "notmine" + // means this is not such a bridge and the regular unplug handling applies. + boolean directRouted = deleteDirectRoutedBridge(iface.getBrName()); + if (_macIpStaticEnabled || directRouted) { + executeMacIpScript(iface.getBrName(), iface.getMacAddress()); + } + if (directRouted) { + return; + } deleteVnetBr(iface.getBrName(), deleteBr); } + /** + * Ensures the per-network bridge for a Direct Routed NIC exists, with the shared gateway + * addresses and sysctls applied. Idempotent and flock'd in the script itself. A failure here + * is fatal to the NIC plug: without the bridge the domain XML would reference a nonexistent + * device and the Instance would fail to start with a far less useful error. + */ + private String createDirectRoutedBridge(NicTO nic) throws InternalErrorException { + if (_modifyBrdrPath == null) { + throw new InternalErrorException("Unable to find modifybrdr.sh: this host cannot run Instances on Direct Routed (L3) networks"); + } + // The routed id — the value of the network's routed:// broadcast domain — names the + // bridge. It is operator-controlled and stable for the network's life. + String routedId = nic.getBroadcastUri() != null ? Networks.BroadcastDomainType.getValue(nic.getBroadcastUri()) : null; + if (StringUtils.isBlank(routedId)) { + throw new InternalErrorException("Direct Routed NIC " + nic.getMac() + " carries no routed:// broadcast URI; cannot derive its bridge"); + } + Script command = new Script(_modifyBrdrPath, _timeout, logger); + command.add("-o", "add"); + command.add("-n", routedId); + // The gateway addresses come from the NIC, whose values the management server stamped at + // allocation (NetUtils.getLinkLocalGateway() / getIpv6LinkLocalGateway()) — the script has + // no defaults of its own, so the addresses are defined in exactly one place. + if (StringUtils.isNotBlank(nic.getGateway())) { + command.add("-4", nic.getGateway()); + } + if (StringUtils.isNotBlank(nic.getIp6Gateway())) { + command.add("-6", nic.getIp6Gateway()); + } + // The script prints the bridge name it created: how these bridges are named is known + // only to modifybrdr.sh, never to the agent. + OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); + String result = command.execute(parser); + if (result != null || StringUtils.isBlank(parser.getLine())) { + throw new InternalErrorException("Failed to create bridge for routed id " + routedId + ": " + result); + } + return parser.getLine().trim(); + } + + /** + * Asks modifybrdr.sh to remove the bridge if it is one of its own and nothing is attached to + * it any more. Returns whether the bridge belongs to a Direct Routed network at all — + * "notmine" means it does not, and the caller falls back to the regular unplug handling. + * Best-effort beyond that: the script keeps the bridge while other Instances of the network + * still use it, and a leftover empty bridge is harmless and re-used on the next plug. + */ + private boolean deleteDirectRoutedBridge(String brName) { + if (_modifyBrdrPath == null || brName == null) { + return false; + } + try { + Script command = new Script(_modifyBrdrPath, _timeout, logger); + command.add("-o", "delete"); + command.add("-b", brName); + OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); + String result = command.execute(parser); + String verdict = parser.getLine() != null ? parser.getLine().trim() : ""; + if (result != null) { + logger.warn("Failed to delete bridge {}: {}", brName, result); + return false; + } + return !"notmine".equals(verdict); + } catch (Exception e) { + logger.warn("Failed to delete bridge {}", brName, e); + return false; + } + } + @Override public void attach(LibvirtVMDef.InterfaceDef iface) { Script.runSimpleBashScript("ip link set " + iface.getDevName() + " master " + iface.getBrName()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 8e8ae3143127..5ccbc876043c 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -427,6 +427,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv private boolean imageServerTlsEnabled = false; private String imageServerListenAddress; private String securityGroupPath; + private String macIpPath; private String ovsPvlanDhcpHostPath; private String ovsPvlanVmPath; private String routerProxyPath; @@ -1195,6 +1196,8 @@ public boolean configure(final String name, final Map params) th } securityGroupPath = Script.findScript(networkScriptsDir, "security_group.py"); + // Not fatal when absent: only Direct Routed networks need it, and a host may run none + macIpPath = Script.findScript(networkScriptsDir, "modifymacip.sh"); if (securityGroupPath == null) { throw new ConfigurationException("Unable to find the security_group.py"); } @@ -5724,6 +5727,10 @@ public boolean defaultNetworkRules(final Connect conn, final String vmName, fina if (checkBeforeApply) { cmd.add("--check"); } + // The Agent is the only component that knows the network type; the script never infers it + if (BridgeVifDriver.isDirectRoutedNic(nic)) { + cmd.add("--directrouted"); + } final String result = cmd.execute(); if (result != null) { return false; @@ -5852,7 +5859,7 @@ private Answer listLvmVolumes(String localPath, int startIndex, int pageSize) { } public boolean addNetworkRules(final String vmName, final String vmId, final String guestIP, final String guestIP6, final String sig, final String seq, final String mac, final String rules, final String vif, final String brname, - final String secIps) { + final String secIps, final boolean directRouted) { if (!canBridgeFirewall) { return false; } @@ -5875,6 +5882,9 @@ public boolean addNetworkRules(final String vmName, final String vmId, final Str if (newRules != null && !newRules.isEmpty()) { cmd.add("--rules", newRules); } + if (directRouted) { + cmd.add("--directrouted"); + } final String result = cmd.execute(); if (result != null) { return false; @@ -5883,11 +5893,27 @@ public boolean addNetworkRules(final String vmName, final String vmId, final Str } public boolean configureNetworkRulesVMSecondaryIP(final Connect conn, final String vmName, final String vmMac, final String secIp, final String action) { + return configureNetworkRulesVMSecondaryIP(conn, vmName, vmMac, secIp, action, false, true); + } - if (!canBridgeFirewall) { + public boolean configureNetworkRulesVMSecondaryIP(final Connect conn, final String vmName, final String vmMac, final String secIp, final String action, + final boolean directRouted, final boolean applySecurityGroupRules) { + + // On a Direct Routed network the address only reaches the Instance once the host has a + // route and a neighbour entry for it. Security groups may be disabled there, so this is + // done regardless of canBridgeFirewall, and before any firewall rules. + if (directRouted && !configureDirectRoutedSecondaryIp(conn, vmName, vmMac, secIp, action)) { return false; } + if (!applySecurityGroupRules) { + return true; + } + + if (!canBridgeFirewall) { + return directRouted; + } + final Script cmd = new Script(securityGroupPath, timeout, LOGGER); cmd.add("network_rules_vmSecondaryIp"); cmd.add("--vmname", vmName); @@ -5902,6 +5928,39 @@ public boolean configureNetworkRulesVMSecondaryIP(final Connect conn, final Stri return true; } + /** + * Adds or removes the host route and static neighbour entry for a secondary IP of an + * Instance on a Direct Routed network, so the address is reachable without restarting it. + */ + private boolean configureDirectRoutedSecondaryIp(final Connect conn, final String vmName, final String vmMac, final String secIp, final String action) { + if (macIpPath == null) { + LOGGER.warn("Unable to find modifymacip.sh, cannot configure secondary IP {} for {}", secIp, vmName); + return false; + } + String brName = null; + for (final InterfaceDef intf : getInterfaces(conn, vmName)) { + if (vmMac.equalsIgnoreCase(intf.getMacAddress())) { + brName = intf.getBrName(); + break; + } + } + if (brName == null) { + LOGGER.warn("Unable to find the interface of {} with MAC {}", vmName, vmMac); + return false; + } + final Script cmd = new Script(macIpPath, timeout, LOGGER); + cmd.add("-o", "-A".equals(action) ? "add" : "delete"); + cmd.add("-b", brName); + cmd.add("-m", vmMac); + cmd.add(NetUtils.isValidIp6(secIp) ? "-6" : "-4", secIp); + final String result = cmd.execute(); + if (result != null) { + LOGGER.warn("Failed to configure secondary IP {} for {}: {}", secIp, vmName, result); + return false; + } + return true; + } + public boolean setupTungstenVRouter(final String oper, final String inf, final String subnet, final String route, final String vrf) { final Script cmd = new Script(setupTungstenVrouterPath, timeout, LOGGER); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtNetworkRulesVmSecondaryIpCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtNetworkRulesVmSecondaryIpCommandWrapper.java index 890558ca3651..018d3b929284 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtNetworkRulesVmSecondaryIpCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtNetworkRulesVmSecondaryIpCommandWrapper.java @@ -39,7 +39,7 @@ public Answer execute(final NetworkRulesVmSecondaryIpCommand command, final Libv final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper(); final Connect conn = libvirtUtilitiesHelper.getConnectionByVmName(command.getVmName()); - result = libvirtComputingResource.configureNetworkRulesVMSecondaryIP(conn, command.getVmName(), command.getVmMac(), command.getVmSecIp(), command.getAction()); + result = libvirtComputingResource.configureNetworkRulesVMSecondaryIP(conn, command.getVmName(), command.getVmMac(), command.getVmSecIp(), command.getAction(), command.isDirectRouted(), command.isApplySecurityGroupRules()); } catch (final LibvirtException e) { logger.debug("Could not configure VM secondary IP! => " + e.getLocalizedMessage()); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtSecurityGroupRulesCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtSecurityGroupRulesCommandWrapper.java index 33164264ae80..fc65e1ac8ca3 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtSecurityGroupRulesCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtSecurityGroupRulesCommandWrapper.java @@ -28,6 +28,7 @@ import com.cloud.agent.api.SecurityGroupRuleAnswer; import com.cloud.agent.api.SecurityGroupRulesCmd; import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.hypervisor.kvm.resource.BridgeVifDriver; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef; import com.cloud.resource.CommandWrapper; @@ -59,8 +60,16 @@ public Answer execute(final SecurityGroupRulesCmd command, final LibvirtComputin return new SecurityGroupRuleAnswer(command, false, e.toString()); } + // The rules are programmed for the Instance's first NIC (vif/brname above); its network + // type decides which dispatch form the script must use. + boolean directRouted = false; + final VirtualMachineTO vmTO = command.getVmTO(); + if (vmTO != null && vmTO.getNics() != null && vmTO.getNics().length > 0) { + directRouted = BridgeVifDriver.isDirectRoutedNic(vmTO.getNics()[0]); + } + final boolean result = libvirtComputingResource.addNetworkRules(command.getVmName(), Long.toString(command.getVmId()), command.getGuestIp(), command.getGuestIp6(), command.getSignature(), - Long.toString(command.getSeqNum()), command.getGuestMac(), command.stringifyRules(), vif, brname, command.getSecIpsString()); + Long.toString(command.getSeqNum()), command.getGuestMac(), command.stringifyRules(), vif, brname, command.getSecIpsString(), directRouted); if (!result) { logger.warn("Failed to program network rules for vm " + command.getVmName()); diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index b62f0adbeb58..81172692f07b 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -2814,7 +2814,8 @@ public void testNetworkRulesVmSecondaryIpCommand() { } catch (final LibvirtException e) { fail(e.getMessage()); } - when(libvirtComputingResourceMock.configureNetworkRulesVMSecondaryIP(conn, command.getVmName(), command.getVmMac(), command.getVmSecIp(), command.getAction())).thenReturn(true); + when(libvirtComputingResourceMock.configureNetworkRulesVMSecondaryIP(conn, command.getVmName(), command.getVmMac(), command.getVmSecIp(), command.getAction(), + command.isDirectRouted(), command.isApplySecurityGroupRules())).thenReturn(true); final LibvirtRequestWrapper wrapper = LibvirtRequestWrapper.getInstance(); assertNotNull(wrapper); @@ -2828,7 +2829,8 @@ public void testNetworkRulesVmSecondaryIpCommand() { fail(e.getMessage()); } verify(libvirtComputingResourceMock, times(1)).getLibvirtUtilitiesHelper(); - verify(libvirtComputingResourceMock, times(1)).configureNetworkRulesVMSecondaryIP(conn, command.getVmName(), command.getVmMac(), command.getVmSecIp(), command.getAction()); + verify(libvirtComputingResourceMock, times(1)).configureNetworkRulesVMSecondaryIP(conn, command.getVmName(), command.getVmMac(), command.getVmSecIp(), command.getAction(), + command.isDirectRouted(), command.isApplySecurityGroupRules()); } @SuppressWarnings("unchecked") @@ -3448,7 +3450,7 @@ public void testSecurityGroupRulesCmdTrue() { when(libvirtComputingResourceMock.applyDefaultNetworkRules(conn, vm, true)).thenReturn(true); when(libvirtComputingResourceMock.addNetworkRules(command.getVmName(), Long.toString(command.getVmId()), command.getGuestIp(), command.getGuestIp6(), command.getSignature(), - Long.toString(command.getSeqNum()), command.getGuestMac(), command.stringifyRules(), vif, brname, command.getSecIpsString())).thenReturn(true); + Long.toString(command.getSeqNum()), command.getGuestMac(), command.stringifyRules(), vif, brname, command.getSecIpsString(), false)).thenReturn(true); final LibvirtRequestWrapper wrapper = LibvirtRequestWrapper.getInstance(); assertNotNull(wrapper); diff --git a/scripts/vm/network/security_group.py b/scripts/vm/network/security_group.py index 235e6342feea..1286fa95fd11 100755 --- a/scripts/vm/network/security_group.py +++ b/scripts/vm/network/security_group.py @@ -527,7 +527,7 @@ def ebtables_rules_vmip(vmname, vmmac, ips, action): except: logging.debug("Failed to program ebtables rules for secondary ip %s for vm %s with action %s" % (ip, vmname, action)) -def check_default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, sec_ips, is_first_nic=False): +def check_default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, sec_ips, is_first_nic=False, direct_routed=False): brfw = get_br_fw(brname) vmchain_default = '-'.join(vm_name.split('-')[:-1]) + "-def" try: @@ -536,7 +536,7 @@ def check_default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brna rules = None if not rules: logging.debug("iptables rules do not exist, programming default rules for %s %s" % (vm_name,vif)) - default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, sec_ips, is_first_nic) + default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, sec_ips, is_first_nic, direct_routed) else: vmchain_in = vm_name + "-in" try: @@ -551,9 +551,84 @@ def check_default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brna ebtables_rules_vmip(vm_name, vm_mac, ips, "-I") return True -def default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, sec_ips, is_first_nic=False): - if not add_fw_framework(brname): - return False +def enable_bridge_netfilter(): + """ Make bridged traffic traverse ip(6)tables and arptables, which every rule below relies + on. Shared by all bridge types. """ + try: + execute("modprobe br_netfilter") + except: + logging.debug("failed to load kernel module br_netfilter") + + try: + execute("sysctl -w net.bridge.bridge-nf-call-arptables=1") + execute("sysctl -w net.bridge.bridge-nf-call-iptables=1") + execute("sysctl -w net.bridge.bridge-nf-call-ip6tables=1") + except: + logging.warning("failed to turn on bridge netfilter") + + +def create_bridge_fw_chains(brfw): + """ Create this bridge's BF chains in both ip and ip6 tables if they are not there yet. """ + for ipt in ['iptables', 'ip6tables']: + for chain in [brfw, brfw + "-OUT", brfw + "-IN"]: + try: + execute("%s -L %s" % (ipt, chain)) + except: + execute("%s -N %s" % (ipt, chain)) + + +def add_notrack_ipset_rules(): + """ Addresses in the notrack ipsets are excluded from connection tracking. Host wide and + idempotent, so every bridge type sets this up the same way. """ + execute(f"ipset -! create {NOTRACK_IPV4_IPSET} hash:ip family inet") + execute(f"ipset -! create {NOTRACK_IPV6_IPSET} hash:ip family inet6") + + for ipt, ipset_name in [('iptables', NOTRACK_IPV4_IPSET), ('ip6tables', NOTRACK_IPV6_IPSET)]: + for direction in ['dst', 'src']: + try: + execute(f"{ipt} -t raw -n -L PREROUTING | grep -q 'match-set {ipset_name} {direction} NOTRACK'") + except: + execute(f"{ipt} -t raw -A PREROUTING -m set --match-set {ipset_name} {direction} -j NOTRACK") + + +def add_l3_fw_framework(brname): + """ FORWARD hooks for a Direct Routed (L3) bridge. Guest traffic on these bridges is routed + by the host, not bridged, so the classic hooks - which reach the BF chains only for + --physdev-is-bridged traffic and drop the rest - would drop everything. These hooks jump + unconditionally for traffic on the bridge, with the same default-deny backstop. """ + enable_bridge_netfilter() + + brfw = get_br_fw(brname) + create_bridge_fw_chains(brfw) + + for ipt in ['iptables', 'ip6tables']: + # Idempotent hook installation; -C fails when the rule is absent + for rule in ["-i %s -j DROP" % brname, + "-o %s -j DROP" % brname, + "-i %s -j %s" % (brname, brfw + "-IN"), + "-o %s -j %s" % (brname, brfw + "-OUT")]: + try: + execute("%s -C FORWARD %s" % (ipt, rule)) + except: + execute("%s -I FORWARD %s" % (ipt, rule)) + + add_notrack_ipset_rules() + + return True + + +def default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, sec_ips, is_first_nic=False, direct_routed=False): + """ direct_routed marks an Instance on a Direct Routed (L3) network, whose traffic the host + routes rather than bridges. The Agent decides this - it is the only component that knows the + network type - and passes --directrouted; the script never infers it. """ + l3 = direct_routed + + if l3: + if not add_l3_fw_framework(brname): + return False + else: + if not add_fw_framework(brname): + return False vmName = vm_name brfw = get_br_fw(brname) @@ -568,6 +643,22 @@ def default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, se vmipsetName = ipset_chain_name(vm_name) vmipsetName6 = vmipsetName + '-6' + # Direction matchers. Traffic from the Instance is identified by the bridge port it entered + # through; --physdev-is-bridged is deliberately absent there: bridged-then-routed packets + # (Direct Routed networks) carry no bridged egress port yet must still match, and on classic + # bridges the BF- framework hook already restricts what reaches these chains to bridged + # traffic, so dropping the flag changes nothing for them. Traffic towards the Instance cannot + # be identified by bridge port on a routed path at all - the kernel only knows the egress + # port for bridged packets - so Direct Routed networks match the destination against the + # Instance's ipsets instead. + from_vm = "-m physdev --physdev-in " + vif + if l3: + to_vm4 = "-m set --match-set " + vmipsetName + " dst" + to_vm6 = "-m set --match-set " + vmipsetName6 + " dst" + else: + to_vm4 = "-m physdev --physdev-is-bridged --physdev-out " + vif + to_vm6 = to_vm4 + if is_first_nic: delete_rules_for_vm_in_bridge_firewall_chain(vmName) destroy_ebtables_rules(vmName, vif) @@ -611,26 +702,28 @@ def default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, se add_to_ipset(vmipsetName6, ip6s, action) try: - execute("iptables -A " + brfw + "-OUT" + " -m physdev --physdev-is-bridged --physdev-out " + vif + " -j " + vmchain_default) - execute("iptables -A " + brfw + "-IN" + " -m physdev --physdev-is-bridged --physdev-in " + vif + " -j " + vmchain_default) - #allow dhcp - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-in " + vif + " -p udp --dport 67 --sport 68 -j ACCEPT") - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-out " + vif + " -p udp --dport 68 --sport 67 -j ACCEPT") - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-in " + vif + " -p udp --sport 67 -j DROP") + execute("iptables -A " + brfw + "-OUT " + to_vm4 + " -j " + vmchain_default) + execute("iptables -A " + brfw + "-IN " + from_vm + " -j " + vmchain_default) + if not l3: + #allow dhcp + execute("iptables -A " + vmchain_default + " " + from_vm + " -p udp --dport 67 --sport 68 -j ACCEPT") + execute("iptables -A " + vmchain_default + " " + to_vm4 + " -p udp --dport 68 --sport 67 -j ACCEPT") + execute("iptables -A " + vmchain_default + " " + from_vm + " -p udp --sport 67 -j DROP") #don't let vm spoof its ip address if vm_ip: - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-in " + vif + " -m set ! --match-set " + vmipsetName + " src -j DROP") - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-out " + vif + " -m set ! --match-set " + vmipsetName + " dst -j DROP") - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-in " + vif + " -m set --match-set " + vmipsetName + " src -p udp --dport 53 -j ACCEPT") - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-in " + vif + " -m set --match-set " + vmipsetName + " src -p tcp --dport 53 -j ACCEPT") - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-in " + vif + " -m set --match-set " + vmipsetName + " src -j " + vmchain_egress) - - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-out " + vif + " -j " + vmchain) - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-in " + vif + " -m state --state ESTABLISHED,RELATED -j ACCEPT") - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-out " + vif + " -m state --state ESTABLISHED,RELATED -j ACCEPT") - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-in " + vif + " -j DROP") - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-out " + vif + " -j DROP") + execute("iptables -A " + vmchain_default + " " + from_vm + " -m set ! --match-set " + vmipsetName + " src -j DROP") + if not l3: + execute("iptables -A " + vmchain_default + " " + to_vm4 + " -m set ! --match-set " + vmipsetName + " dst -j DROP") + execute("iptables -A " + vmchain_default + " " + from_vm + " -m set --match-set " + vmipsetName + " src -p udp --dport 53 -j ACCEPT") + execute("iptables -A " + vmchain_default + " " + from_vm + " -m set --match-set " + vmipsetName + " src -p tcp --dport 53 -j ACCEPT") + execute("iptables -A " + vmchain_default + " " + from_vm + " -m set --match-set " + vmipsetName + " src -j " + vmchain_egress) + + execute("iptables -A " + vmchain_default + " " + to_vm4 + " -j " + vmchain) + execute("iptables -A " + vmchain_default + " " + from_vm + " -m state --state ESTABLISHED,RELATED -j ACCEPT") + execute("iptables -A " + vmchain_default + " " + to_vm4 + " -m state --state ESTABLISHED,RELATED -j ACCEPT") + execute("iptables -A " + vmchain_default + " " + from_vm + " -j DROP") + execute("iptables -A " + vmchain_default + " " + to_vm4 + " -j DROP") execute("iptables -A " + vmchain + " -j RETURN") except: logging.debug("Failed to program default rules for vm " + vm_name) @@ -645,57 +738,63 @@ def default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, se logging.debug("Failed to log default network rules, ignoring") try: - execute('ip6tables -A ' + brfw + '-OUT' + ' -m physdev --physdev-is-bridged --physdev-out ' + vif + ' -j ' + vmchain_default) - execute('ip6tables -A ' + brfw + '-IN' + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -j ' + vmchain_default) + execute('ip6tables -A ' + brfw + '-OUT ' + to_vm6 + ' -j ' + vmchain_default) + execute('ip6tables -A ' + brfw + '-IN ' + from_vm + ' -j ' + vmchain_default) - # Allow Instances to receive Router Advertisements, send out solicitations, but block any outgoing Advertisement from a Instance - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-out ' + vif + ' --src fe80::/64 --dst ff02::1 -p icmpv6 --icmpv6-type router-advertisement -m hl --hl-eq 255 -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' --dst ff02::2 -p icmpv6 --icmpv6-type router-solicitation -m hl --hl-eq 255 -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -p icmpv6 --icmpv6-type router-advertisement -j DROP') + # Allow Instances to receive Router Advertisements, send out solicitations, but block any outgoing Advertisement from a Instance. + # On a Direct Routed network no router advertises anything, so only the block applies. + if not l3: + execute('ip6tables -A ' + vmchain_default + ' ' + to_vm6 + ' --src fe80::/64 --dst ff02::1 -p icmpv6 --icmpv6-type router-advertisement -m hl --hl-eq 255 -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' --dst ff02::2 -p icmpv6 --icmpv6-type router-solicitation -m hl --hl-eq 255 -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -p icmpv6 --icmpv6-type router-advertisement -j DROP') # Allow neighbor solicitations and advertisements - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -p icmpv6 --icmpv6-type neighbor-solicitation -m hl --hl-eq 255 -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-out ' + vif + ' -p icmpv6 --icmpv6-type neighbor-solicitation -m hl --hl-eq 255 -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -p icmpv6 --icmpv6-type neighbor-advertisement -m set --match-set ' + vmipsetName6 + ' src -m hl --hl-eq 255 -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-out ' + vif + ' -p icmpv6 --icmpv6-type neighbor-advertisement -m hl --hl-eq 255 -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -p icmpv6 --icmpv6-type neighbor-solicitation -m hl --hl-eq 255 -j ACCEPT') + if not l3: + execute('ip6tables -A ' + vmchain_default + ' ' + to_vm6 + ' -p icmpv6 --icmpv6-type neighbor-solicitation -m hl --hl-eq 255 -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -p icmpv6 --icmpv6-type neighbor-advertisement -m set --match-set ' + vmipsetName6 + ' src -m hl --hl-eq 255 -j ACCEPT') + if not l3: + execute('ip6tables -A ' + vmchain_default + ' ' + to_vm6 + ' -p icmpv6 --icmpv6-type neighbor-advertisement -m hl --hl-eq 255 -j ACCEPT') # Packets to allow as per RFC4890 - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -p icmpv6 --icmpv6-type packet-too-big -m set --match-set ' + vmipsetName6 + ' src -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-out ' + vif + ' -p icmpv6 --icmpv6-type packet-too-big -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -p icmpv6 --icmpv6-type packet-too-big -m set --match-set ' + vmipsetName6 + ' src -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + to_vm6 + ' -p icmpv6 --icmpv6-type packet-too-big -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -p icmpv6 --icmpv6-type destination-unreachable -m set --match-set ' + vmipsetName6 + ' src -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-out ' + vif + ' -p icmpv6 --icmpv6-type destination-unreachable -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -p icmpv6 --icmpv6-type destination-unreachable -m set --match-set ' + vmipsetName6 + ' src -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + to_vm6 + ' -p icmpv6 --icmpv6-type destination-unreachable -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -p icmpv6 --icmpv6-type time-exceeded -m set --match-set ' + vmipsetName6 + ' src -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-out ' + vif + ' -p icmpv6 --icmpv6-type time-exceeded -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -p icmpv6 --icmpv6-type time-exceeded -m set --match-set ' + vmipsetName6 + ' src -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + to_vm6 + ' -p icmpv6 --icmpv6-type time-exceeded -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -p icmpv6 --icmpv6-type parameter-problem -m set --match-set ' + vmipsetName6 + ' src -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-out ' + vif + ' -p icmpv6 --icmpv6-type parameter-problem -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -p icmpv6 --icmpv6-type parameter-problem -m set --match-set ' + vmipsetName6 + ' src -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + to_vm6 + ' -p icmpv6 --icmpv6-type parameter-problem -j ACCEPT') # MLDv2 discovery packets - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -p icmpv6 --dst ff02::16 -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -p icmpv6 --dst ff02::16 -j ACCEPT') - # Allow Instances to send out DHCPv6 client messages, but block server messages - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -p udp --sport 546 --dst ff02::1:2 --src ' + str(ipv6_link_local) + ' -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-out ' + vif + ' -p udp --src fe80::/64 --dport 546 --dst ' + str(ipv6_link_local) + ' -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -p udp --sport 547 ! --dst fe80::/64 -j DROP') + # Allow Instances to send out DHCPv6 client messages, but block server messages. Not on + # Direct Routed networks: there is no DHCP there. + if not l3: + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -p udp --sport 546 --dst ff02::1:2 --src ' + str(ipv6_link_local) + ' -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + to_vm6 + ' -p udp --src fe80::/64 --dport 546 --dst ' + str(ipv6_link_local) + ' -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -p udp --sport 547 ! --dst fe80::/64 -j DROP') # Always allow outbound DNS over UDP and TCP - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -p udp --dport 53 -m set --match-set ' + vmipsetName6 + ' src -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -p tcp --dport 53 -m set --match-set ' + vmipsetName6 + ' src -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -p udp --dport 53 -m set --match-set ' + vmipsetName6 + ' src -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -p tcp --dport 53 -m set --match-set ' + vmipsetName6 + ' src -j ACCEPT') # Prevent source address spoofing - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -m set ! --match-set ' + vmipsetName6 + ' src -j DROP') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -m set ! --match-set ' + vmipsetName6 + ' src -j DROP') # Send proper traffic to the egress chain of the Instance - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -m set --match-set ' + vmipsetName6 + ' src -j ' + vmchain_egress) + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -m set --match-set ' + vmipsetName6 + ' src -j ' + vmchain_egress) - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-out ' + vif + ' -j ' + vmchain) + execute('ip6tables -A ' + vmchain_default + ' ' + to_vm6 + ' -j ' + vmchain) - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -m state --state ESTABLISHED,RELATED -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-out ' + vif + ' -m state --state ESTABLISHED,RELATED -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -j DROP') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-out ' + vif + ' -j DROP') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -m state --state ESTABLISHED,RELATED -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + to_vm6 + ' -m state --state ESTABLISHED,RELATED -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -j DROP') + execute('ip6tables -A ' + vmchain_default + ' ' + to_vm6 + ' -j DROP') # Drop all other traffic into the Instance execute('ip6tables -A ' + vmchain + ' -j RETURN') @@ -747,21 +846,19 @@ def delete_rules_for_vm_in_bridge_firewall_chain(vmName): vmchain = iptables_chain_name(vm_name) - delcmd = """iptables-save | awk '/BF(.*)physdev-is-bridged(.*)%s/ { sub(/-A/, "-D", $1) ; print }'""" % vmchain - delcmds = [_f for _f in execute(delcmd).split('\n') if _f] - for cmd in delcmds: - try: - execute("iptables " + cmd) - except: - logging.exception("Ignoring failure to delete rules for vm " + vmName) - - delcmd = """ip6tables-save | awk '/BF(.*)physdev-is-bridged(.*)%s/ { sub(/-A/, "-D", $1) ; print }'""" % vmchain - delcmds = [_f for _f in execute(delcmd).split('\n') if _f] - for cmd in delcmds: - try: - execute('ip6tables ' + cmd) - except: - logging.exception("Ignoring failure to delete rules for vm " + vmName) + # Dispatch rules for this Instance in the BF- chains take several forms: physdev-in (from + # the Instance; historically with --physdev-is-bridged, which rules created by older + # versions still carry), physdev-out with --physdev-is-bridged (towards the Instance on a + # classic bridge) and ipset destination matches (towards the Instance on a Direct Routed + # bridge). One pattern on the chain names covers all of them. + for ipt in ['iptables', 'ip6tables']: + delcmd = """%s-save | awk '/BF(.*)%s/ { sub(/-A/, "-D", $1) ; print }'""" % (ipt, vmchain) + delcmds = [_f for _f in execute(delcmd).split('\n') if _f] + for cmd in delcmds: + try: + execute(ipt + ' ' + cmd) + except: + logging.exception("Ignoring failure to delete rules for vm " + vmName) def rewrite_rule_log_for_vm(vm_name, new_domid): @@ -844,13 +941,31 @@ def network_rules_for_rebooted_vm(vmName): vmchain = iptables_chain_name(vm_name) vmchain_default = '-'.join(vmchain.split('-')[:-1]) + "-def" + # This path runs without the Agent, so the --directrouted flag is not available here. The + # Instance's existing OUT dispatch rule tells us which form to restore: a Direct Routed + # Instance is dispatched by destination ipset, a bridged one by physdev-out. Captured before + # the rules are deleted above? No - delete_rules_for_vm_in_bridge_firewall_chain has already + # run, so read it from the per-VM chain, which is untouched by that function. + vmipsetName = ipset_chain_name(vm_name) + try: + l3 = bool(execute("""iptables-save | grep -w %s | grep -q -- '--match-set %s dst' && echo yes""" % (vmchain_default, vmipsetName)).strip()) + except: + l3 = False + vifs = get_vifs(vmName) logging.debug(vifs, brName) for v in vifs: - execute("iptables -A " + get_br_fw(brName) + "-IN " + " -m physdev --physdev-is-bridged --physdev-in " + v + " -j " + vmchain_default) - execute("iptables -A " + get_br_fw(brName) + "-OUT " + " -m physdev --physdev-is-bridged --physdev-out " + v + " -j " + vmchain_default) - execute("ip6tables -A " + get_br_fw(brName) + "-IN " + " -m physdev --physdev-is-bridged --physdev-in " + v + " -j " + vmchain_default) - execute("ip6tables -A " + get_br_fw(brName) + "-OUT " + " -m physdev --physdev-is-bridged --physdev-out " + v + " -j " + vmchain_default) + from_vm = "-m physdev --physdev-in " + v + if l3: + to_vm4 = "-m set --match-set " + vmipsetName + " dst" + to_vm6 = "-m set --match-set " + vmipsetName + "-6 dst" + else: + to_vm4 = "-m physdev --physdev-is-bridged --physdev-out " + v + to_vm6 = to_vm4 + execute("iptables -A " + get_br_fw(brName) + "-IN " + from_vm + " -j " + vmchain_default) + execute("iptables -A " + get_br_fw(brName) + "-OUT " + to_vm4 + " -j " + vmchain_default) + execute("ip6tables -A " + get_br_fw(brName) + "-IN " + from_vm + " -j " + vmchain_default) + execute("ip6tables -A " + get_br_fw(brName) + "-OUT " + to_vm6 + " -j " + vmchain_default) #change antispoof rule in vmchain try: @@ -1105,7 +1220,7 @@ def parse_network_rules(rules): return ret -def add_network_rules(vm_name, vm_id, vm_ip, vm_ip6, signature, seqno, vmMac, rules, vif, brname, sec_ips): +def add_network_rules(vm_name, vm_id, vm_ip, vm_ip6, signature, seqno, vmMac, rules, vif, brname, sec_ips, direct_routed=False): try: vmName = vm_name domId = get_vm_id(vmName) @@ -1128,7 +1243,7 @@ def add_network_rules(vm_name, vm_id, vm_ip, vm_ip6, signature, seqno, vmMac, ru execute('ip6tables -F ' + chain) except: logging.debug("Error flushing iptables rules for " + vm_name + ". Presuming firewall rules deleted, re-initializing." ) - default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vmMac, vif, brname, sec_ips, True) + default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vmMac, vif, brname, sec_ips, True, direct_routed) egressrule_v4 = 0 egressrule_v6 = 0 @@ -1312,78 +1427,16 @@ def get_br_fw(brname): def add_fw_framework(brname): - try: - execute("modprobe br_netfilter") - except: - logging.debug("failed to load kernel module br_netfilter") - - try: - execute("sysctl -w net.bridge.bridge-nf-call-arptables=1") - execute("sysctl -w net.bridge.bridge-nf-call-iptables=1") - execute("sysctl -w net.bridge.bridge-nf-call-ip6tables=1") - except: - logging.warning("failed to turn on bridge netfilter") + enable_bridge_netfilter() brfw = get_br_fw(brname) - try: - execute("iptables -L " + brfw) - except: - execute("iptables -N " + brfw) - - brfwout = brfw + "-OUT" - try: - execute("iptables -L " + brfwout) - except: - execute("iptables -N " + brfwout) + create_bridge_fw_chains(brfw) brfwin = brfw + "-IN" - try: - execute("iptables -L " + brfwin) - except: - execute("iptables -N " + brfwin) - - try: - execute('ip6tables -L ' + brfw) - except: - execute('ip6tables -N ' + brfw) - brfwout = brfw + "-OUT" - try: - execute('ip6tables -L ' + brfwout) - except: - execute('ip6tables -N ' + brfwout) - - brfwin = brfw + "-IN" - try: - execute('ip6tables -L ' + brfwin) - except: - execute('ip6tables -N ' + brfwin) - physdev = get_bridge_physdev(brname) - # Add notrack ipset. The IP addresses from it will be excluded from connection tracking - execute(f"ipset -! create {NOTRACK_IPV4_IPSET} hash:ip family inet") - execute(f"ipset -! create {NOTRACK_IPV6_IPSET} hash:ip family inet6") - - # Create IPv4 rules that disable connection tracking - try: - execute(f"iptables -t raw -n -L PREROUTING | grep -q 'match-set {NOTRACK_IPV4_IPSET} dst NOTRACK'") - except: - execute(f"iptables -t raw -A PREROUTING -m set --match-set {NOTRACK_IPV4_IPSET} dst -j NOTRACK") - try: - execute(f"iptables -t raw -n -L PREROUTING | grep -q 'match-set {NOTRACK_IPV4_IPSET} src NOTRACK'") - except: - execute(f"iptables -t raw -A PREROUTING -m set --match-set {NOTRACK_IPV4_IPSET} src -j NOTRACK") - - # Create IPv6 rules that disable connection tracking - try: - execute(f"ip6tables -t raw -n -L PREROUTING | grep -q 'match-set {NOTRACK_IPV6_IPSET} dst NOTRACK'") - except: - execute(f"ip6tables -t raw -A PREROUTING -m set --match-set {NOTRACK_IPV6_IPSET} dst -j NOTRACK") - try: - execute(f"ip6tables -t raw -n -L PREROUTING | grep -q 'match-set {NOTRACK_IPV6_IPSET} src NOTRACK'") - except: - execute(f"ip6tables -t raw -A PREROUTING -m set --match-set {NOTRACK_IPV6_IPSET} src -j NOTRACK") + add_notrack_ipset_rules() try: refs = int(execute("""iptables -n -L %s | awk '/%s(.*)references/ {gsub(/\(/, "") ;print $3}'""" % (brfw,brfw)).strip()) @@ -1418,7 +1471,7 @@ def add_fw_framework(brname): return False return False -def verify_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, sec_ips): +def verify_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, sec_ips, direct_routed=False): if vm_name is None or vm_ip is None or vm_mac is None: print("vm_name, vm_ip and vm_mac must be specifed") sys.exit(1) @@ -1665,6 +1718,7 @@ def verify_expected_rules_in_order(expected_rules, rules): parser.add_argument("--privnic", dest="privnic") parser.add_argument("--isFirstNic", action="store_true", dest="isFirstNic") parser.add_argument("--check", action="store_true", dest="check") + parser.add_argument("--directrouted", action="store_true", dest="directRouted") args = parser.parse_args() cmd = args.command logging.debug("Executing command: %s", cmd) @@ -1679,9 +1733,9 @@ def verify_expected_rules_in_order(expected_rules, rules): if cmd == "can_bridge_firewall": can_bridge_firewall(args.privnic) elif cmd == "default_network_rules" and args.check: - check_default_network_rules(args.vmName, args.vmID, args.vmIP, args.vmIP6, args.vmMAC, args.vif, args.brname, args.nicSecIps, args.isFirstNic) + check_default_network_rules(args.vmName, args.vmID, args.vmIP, args.vmIP6, args.vmMAC, args.vif, args.brname, args.nicSecIps, args.isFirstNic, args.directRouted) elif cmd == "default_network_rules" and not args.check: - default_network_rules(args.vmName, args.vmID, args.vmIP, args.vmIP6, args.vmMAC, args.vif, args.brname, args.nicSecIps, args.isFirstNic) + default_network_rules(args.vmName, args.vmID, args.vmIP, args.vmIP6, args.vmMAC, args.vif, args.brname, args.nicSecIps, args.isFirstNic, args.directRouted) elif cmd == "destroy_network_rules_for_vm": if not args.vmIP: destroy_network_rules_for_vm(args.vmName, args.vif) @@ -1692,7 +1746,7 @@ def verify_expected_rules_in_order(expected_rules, rules): elif cmd == "get_rule_logs_for_vms": get_rule_logs_for_vms() elif cmd == "add_network_rules": - add_network_rules(args.vmName, args.vmID, args.vmIP, args.vmIP6, args.sig, args.seq, args.vmMAC, args.rules, args.vif, args.brname, args.nicSecIps) + add_network_rules(args.vmName, args.vmID, args.vmIP, args.vmIP6, args.sig, args.seq, args.vmMAC, args.rules, args.vif, args.brname, args.nicSecIps, args.directRouted) elif cmd == "network_rules_vmSecondaryIp": network_rules_vmSecondaryIp(args.vmName, args.vmMAC, args.nicSecIps, args.action) elif cmd == "cleanup_rules": @@ -1700,7 +1754,7 @@ def verify_expected_rules_in_order(expected_rules, rules): elif cmd == "post_default_network_rules": post_default_network_rules(args.vmName, args.vmID, args.vmIP, args.vmMAC, args.vif, args.brname, args.dhcpSvr, args.hostIp, args.hostMacAddr) elif cmd == "verify_network_rules": - verify_network_rules(args.vmName, args.vmID, args.vmIP, args.vmIP6, args.vmMAC, args.vif, args.brname, args.nicSecIps) + verify_network_rules(args.vmName, args.vmID, args.vmIP, args.vmIP6, args.vmMAC, args.vif, args.brname, args.nicSecIps, args.directRouted) else: logging.debug("Unknown command: " + cmd) sys.exit(1) diff --git a/scripts/vm/network/tests/golden_add_fw_framework.txt b/scripts/vm/network/tests/golden_add_fw_framework.txt new file mode 100644 index 000000000000..6ecfe18702ea --- /dev/null +++ b/scripts/vm/network/tests/golden_add_fw_framework.txt @@ -0,0 +1,44 @@ +modprobe br_netfilter +sysctl -w net.bridge.bridge-nf-call-arptables=1 +sysctl -w net.bridge.bridge-nf-call-iptables=1 +sysctl -w net.bridge.bridge-nf-call-ip6tables=1 +iptables -L BF-cloudbr0 +iptables -N BF-cloudbr0 +iptables -L BF-cloudbr0-OUT +iptables -N BF-cloudbr0-OUT +iptables -L BF-cloudbr0-IN +iptables -N BF-cloudbr0-IN +ip6tables -L BF-cloudbr0 +ip6tables -N BF-cloudbr0 +ip6tables -L BF-cloudbr0-OUT +ip6tables -N BF-cloudbr0-OUT +ip6tables -L BF-cloudbr0-IN +ip6tables -N BF-cloudbr0-IN +ipset -! create cs_notrack hash:ip family inet +ipset -! create cs_notrack6 hash:ip family inet6 +iptables -t raw -n -L PREROUTING | grep -q 'match-set cs_notrack dst NOTRACK' +iptables -t raw -A PREROUTING -m set --match-set cs_notrack dst -j NOTRACK +iptables -t raw -n -L PREROUTING | grep -q 'match-set cs_notrack src NOTRACK' +iptables -t raw -A PREROUTING -m set --match-set cs_notrack src -j NOTRACK +ip6tables -t raw -n -L PREROUTING | grep -q 'match-set cs_notrack6 dst NOTRACK' +ip6tables -t raw -A PREROUTING -m set --match-set cs_notrack6 dst -j NOTRACK +ip6tables -t raw -n -L PREROUTING | grep -q 'match-set cs_notrack6 src NOTRACK' +ip6tables -t raw -A PREROUTING -m set --match-set cs_notrack6 src -j NOTRACK +iptables -n -L BF-cloudbr0 | awk '/BF-cloudbr0(.*)references/ {gsub(/\(/, "") ;print $3}' +iptables -n -L BF-cloudbr0-IN | awk '/BF-cloudbr0-IN(.*)references/ {gsub(/\(/, "") ;print $3}' +iptables -n -L BF-cloudbr0-OUT | awk '/BF-cloudbr0-OUT(.*)references/ {gsub(/\(/, "") ;print $3}' +ip6tables -n -L BF-cloudbr0 | awk '/BF-cloudbr0(.*)references/ {gsub(/\(/, "") ;print $3}' +iptables -I FORWARD -i cloudbr0 -j DROP +iptables -I FORWARD -o cloudbr0 -j DROP +iptables -I FORWARD -i cloudbr0 -m physdev --physdev-is-bridged -j BF-cloudbr0 +iptables -I FORWARD -o cloudbr0 -m physdev --physdev-is-bridged -j BF-cloudbr0 +iptables -A BF-cloudbr0 -m physdev --physdev-is-bridged --physdev-is-in -j BF-cloudbr0-IN +iptables -A BF-cloudbr0 -m physdev --physdev-is-bridged --physdev-is-out -j BF-cloudbr0-OUT +iptables -A BF-cloudbr0 -m physdev --physdev-is-bridged --physdev-out eth0 -j ACCEPT +ip6tables -I FORWARD -i cloudbr0 -j DROP +ip6tables -I FORWARD -o cloudbr0 -j DROP +ip6tables -I FORWARD -i cloudbr0 -m physdev --physdev-is-bridged -j BF-cloudbr0 +ip6tables -I FORWARD -o cloudbr0 -m physdev --physdev-is-bridged -j BF-cloudbr0 +ip6tables -A BF-cloudbr0 -m physdev --physdev-is-bridged --physdev-is-in -j BF-cloudbr0-IN +ip6tables -A BF-cloudbr0 -m physdev --physdev-is-bridged --physdev-is-out -j BF-cloudbr0-OUT +ip6tables -A BF-cloudbr0 -m physdev --physdev-is-bridged --physdev-out eth0 -j ACCEPT diff --git a/scripts/vm/network/tests/golden_default_network_rules.txt b/scripts/vm/network/tests/golden_default_network_rules.txt new file mode 100644 index 000000000000..891b53aaaac5 --- /dev/null +++ b/scripts/vm/network/tests/golden_default_network_rules.txt @@ -0,0 +1,92 @@ +iptables-save |grep physdev-is-bridged |grep FORWARD |grep BF |grep '\-o' | grep -w cloudbr0|awk '{print $9}' | head -1 +iptables -N i-2-7-VM +ip6tables -N i-2-7-VM +iptables -N i-2-7-VM-eg +ip6tables -N i-2-7-VM-eg +iptables -N i-2-7-def +ip6tables -N i-2-7-def +ipset -F i-2-7-VM +ipset -X i-2-7-VM +ipset -N i-2-7-VM iphash family inet +ipset -F i-2-7-VM-6 +ipset -X i-2-7-VM-6 +ipset -N i-2-7-VM-6 hash:net family inet6 +ipset -! -A i-2-7-VM 10.1.1.55 +ipset -! -A i-2-7-VM 10.1.1.55 +ipset -! -A i-2-7-VM-6 fd00::55 +ipset -! -A i-2-7-VM-6 fe80::4cff:fe5f:1 +iptables -A BF-cloudbr0-OUT -m physdev --physdev-is-bridged --physdev-out vnet3 -j i-2-7-def +iptables -A BF-cloudbr0-IN -m physdev --physdev-in vnet3 -j i-2-7-def +iptables -A i-2-7-def -m physdev --physdev-in vnet3 -p udp --dport 67 --sport 68 -j ACCEPT +iptables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -p udp --dport 68 --sport 67 -j ACCEPT +iptables -A i-2-7-def -m physdev --physdev-in vnet3 -p udp --sport 67 -j DROP +iptables -A i-2-7-def -m physdev --physdev-in vnet3 -m set ! --match-set i-2-7-VM src -j DROP +iptables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -m set ! --match-set i-2-7-VM dst -j DROP +iptables -A i-2-7-def -m physdev --physdev-in vnet3 -m set --match-set i-2-7-VM src -p udp --dport 53 -j ACCEPT +iptables -A i-2-7-def -m physdev --physdev-in vnet3 -m set --match-set i-2-7-VM src -p tcp --dport 53 -j ACCEPT +iptables -A i-2-7-def -m physdev --physdev-in vnet3 -m set --match-set i-2-7-VM src -j i-2-7-VM-eg +iptables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -j i-2-7-VM +iptables -A i-2-7-def -m physdev --physdev-in vnet3 -m state --state ESTABLISHED,RELATED -j ACCEPT +iptables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -m state --state ESTABLISHED,RELATED -j ACCEPT +iptables -A i-2-7-def -m physdev --physdev-in vnet3 -j DROP +iptables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -j DROP +iptables -A i-2-7-VM -j RETURN +ebtables -t nat -N i-2-7-VM-in +ebtables -t nat -N i-2-7-VM-out +ebtables -t nat -N i-2-7-VM-in-ips +ebtables -t nat -N i-2-7-VM-out-ips +ebtables -t nat -N i-2-7-VM-in-src +ebtables -t nat -N i-2-7-VM-out-dst +ebtables -t nat -A PREROUTING -i vnet3 -j i-2-7-VM-in +ebtables -t nat -A POSTROUTING -o vnet3 -j i-2-7-VM-out +ebtables -t nat -A i-2-7-VM-in-ips -j DROP +ebtables -t nat -A i-2-7-VM-out-ips -j DROP +ebtables -t nat -A i-2-7-VM-in-src -j DROP +ebtables -t nat -A i-2-7-VM-out-dst -p ARP --arp-op Reply -j DROP +ebtables -t nat -A i-2-7-VM-in -j i-2-7-VM-in-src +ebtables -t nat -I i-2-7-VM-in-src -s 02:00:4c:5f:00:01 -j RETURN +ebtables -t nat -A i-2-7-VM-in -p ARP -j i-2-7-VM-in-ips +ebtables -t nat -I i-2-7-VM-in-ips -p ARP -s 02:00:4c:5f:00:01 --arp-mac-src 02:00:4c:5f:00:01 --arp-ip-src 10.1.1.55 -j RETURN +ebtables -t nat -A i-2-7-VM-in -p ARP --arp-op Request -j ACCEPT +ebtables -t nat -A i-2-7-VM-in -p ARP --arp-op Reply -j ACCEPT +ebtables -t nat -A i-2-7-VM-in -p ARP -j DROP +ebtables -t nat -A i-2-7-VM-out -p ARP --arp-op Reply -j i-2-7-VM-out-dst +ebtables -t nat -I i-2-7-VM-out-dst -p ARP --arp-op Reply --arp-mac-dst 02:00:4c:5f:00:01 -j RETURN +ebtables -t nat -A i-2-7-VM-out -p ARP -j i-2-7-VM-out-ips +ebtables -t nat -I i-2-7-VM-out-ips -p ARP --arp-ip-dst 10.1.1.55 -j RETURN +ebtables -t nat -A i-2-7-VM-out -p ARP --arp-op Request -j ACCEPT +ebtables -t nat -A i-2-7-VM-out -p ARP --arp-op Reply -j ACCEPT +ebtables -t nat -A i-2-7-VM-out -p ARP -j DROP +ebtables -t nat -I i-2-7-VM-in-ips -p ARP -s 02:00:4c:5f:00:01 --arp-mac-src 02:00:4c:5f:00:01 --arp-ip-src 10.1.1.55 -j RETURN +ebtables -t nat -I i-2-7-VM-out-ips -p ARP --arp-ip-dst 10.1.1.55 -j RETURN +ip6tables -A BF-cloudbr0-OUT -m physdev --physdev-is-bridged --physdev-out vnet3 -j i-2-7-def +ip6tables -A BF-cloudbr0-IN -m physdev --physdev-in vnet3 -j i-2-7-def +ip6tables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 --src fe80::/64 --dst ff02::1 -p icmpv6 --icmpv6-type router-advertisement -m hl --hl-eq 255 -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 --dst ff02::2 -p icmpv6 --icmpv6-type router-solicitation -m hl --hl-eq 255 -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p icmpv6 --icmpv6-type router-advertisement -j DROP +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p icmpv6 --icmpv6-type neighbor-solicitation -m hl --hl-eq 255 -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -p icmpv6 --icmpv6-type neighbor-solicitation -m hl --hl-eq 255 -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p icmpv6 --icmpv6-type neighbor-advertisement -m set --match-set i-2-7-VM-6 src -m hl --hl-eq 255 -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -p icmpv6 --icmpv6-type neighbor-advertisement -m hl --hl-eq 255 -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p icmpv6 --icmpv6-type packet-too-big -m set --match-set i-2-7-VM-6 src -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -p icmpv6 --icmpv6-type packet-too-big -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p icmpv6 --icmpv6-type destination-unreachable -m set --match-set i-2-7-VM-6 src -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -p icmpv6 --icmpv6-type destination-unreachable -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p icmpv6 --icmpv6-type time-exceeded -m set --match-set i-2-7-VM-6 src -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -p icmpv6 --icmpv6-type time-exceeded -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p icmpv6 --icmpv6-type parameter-problem -m set --match-set i-2-7-VM-6 src -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -p icmpv6 --icmpv6-type parameter-problem -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p icmpv6 --dst ff02::16 -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p udp --sport 546 --dst ff02::1:2 --src fe80::4cff:fe5f:1 -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -p udp --src fe80::/64 --dport 546 --dst fe80::4cff:fe5f:1 -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p udp --sport 547 ! --dst fe80::/64 -j DROP +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p udp --dport 53 -m set --match-set i-2-7-VM-6 src -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p tcp --dport 53 -m set --match-set i-2-7-VM-6 src -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -m set ! --match-set i-2-7-VM-6 src -j DROP +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -m set --match-set i-2-7-VM-6 src -j i-2-7-VM-eg +ip6tables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -j i-2-7-VM +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -m state --state ESTABLISHED,RELATED -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -m state --state ESTABLISHED,RELATED -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -j DROP +ip6tables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -j DROP +ip6tables -A i-2-7-VM -j RETURN diff --git a/scripts/vm/network/tests/test_security_group.py b/scripts/vm/network/tests/test_security_group.py new file mode 100755 index 000000000000..84db7e58a9e7 --- /dev/null +++ b/scripts/vm/network/tests/test_security_group.py @@ -0,0 +1,245 @@ +#!/usr/bin/python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" Unit tests for security_group.py rule generation. + +The golden test pins the exact command stream the classic (bridged) path produces. It was +regenerated once, deliberately, when --physdev-is-bridged was removed from the --physdev-in +rules: the flag restricted them to bridged traffic, which is all that can reach these chains on +a classic bridge anyway (the BF- framework hook gates on it), while bridged-then-routed traffic +on Direct Routed networks must match them too. --physdev-out rules keep the flag: the kernel +cannot know the bridged egress port for routed packets, so removing it there changes nothing. +Regenerate only when a change to the classic rules is intended: + + python3 tests/test_security_group.py --regenerate +""" + +import importlib.util +import os +import sys +import unittest +from unittest import mock + +HERE = os.path.dirname(os.path.abspath(__file__)) +SCRIPT = os.path.join(HERE, os.pardir, 'security_group.py') +GOLDEN = os.path.join(HERE, 'golden_default_network_rules.txt') +GOLDEN_FRAMEWORK = os.path.join(HERE, 'golden_add_fw_framework.txt') + +VM_ARGS = dict(vm_name="i-2-7-VM", vm_id="7", vm_ip="10.1.1.55", vm_ip6="fd00::55", + vm_mac="02:00:4c:5f:00:01", vif="vnet3", sec_ips="0:") + + +def load_script(): + sys.modules.setdefault('libvirt', mock.MagicMock()) + spec = importlib.util.spec_from_file_location("security_group", SCRIPT) + sg = importlib.util.module_from_spec(spec) + spec.loader.exec_module(sg) + return sg + + +def capture_framework(sg, fn, brname): + """ Run a framework function with every probe failing (so create/insert branches are taken) + and chain references reported as zero (so hooks are installed). """ + captured = [] + + def fake_execute(cmd): + captured.append(cmd) + if ' -L ' in cmd and 'awk' not in cmd: + raise Exception("absent") + if 'grep -q' in cmd: + raise Exception("absent") + if ' -C ' in cmd: + raise Exception("absent") + if 'awk' in cmd and 'references' in cmd: + return "0" + return "" + + sg.execute = fake_execute + sg.get_br_fw = lambda b: "BF-" + b + sg.get_bridge_physdev = lambda b: "eth0" + fn(brname) + return captured + + +def capture_default_network_rules(sg, brname, direct_routed): + """ Run default_network_rules with all side effects stubbed, returning the commands the + script would have executed. """ + captured = [] + + def fake_execute(cmd): + captured.append(cmd) + return "" + + sg.execute = fake_execute + sg.add_fw_framework = lambda brname: True + sg.add_l3_fw_framework = lambda brname: True + sg.get_vm_id = lambda name: "7" + sg.write_rule_log_for_vm = lambda *a, **k: True + sg.write_secip_log_for_vm = lambda *a, **k: True + sg.delete_rules_for_vm_in_bridge_firewall_chain = lambda name: None + sg.destroy_ebtables_rules = lambda name, vif: None + + ok = sg.default_network_rules(VM_ARGS['vm_name'], VM_ARGS['vm_id'], VM_ARGS['vm_ip'], VM_ARGS['vm_ip6'], + VM_ARGS['vm_mac'], VM_ARGS['vif'], brname, VM_ARGS['sec_ips'], + is_first_nic=True, direct_routed=direct_routed) + return ok, captured + + +class TestClassicRulesUnchanged(unittest.TestCase): + """ The classic path must stay byte-identical: it serves Basic zones and every Shared + network in every existing deployment. """ + + def test_default_network_rules_matches_golden(self): + sg = load_script() + ok, captured = capture_default_network_rules(sg, "cloudbr0", direct_routed=False) + self.assertTrue(ok) + with open(GOLDEN) as f: + golden = f.read().splitlines() + self.assertEqual(golden, captured) + + def test_classic_physdev_direction_flags(self): + """ from-Instance rules identify the bridge port without --physdev-is-bridged (so the + same rules work for bridged-then-routed traffic); towards-Instance rules keep it, since + the kernel only knows the bridged egress port for bridged packets. """ + sg = load_script() + ok, captured = capture_default_network_rules(sg, "cloudbr0", direct_routed=False) + self.assertTrue(ok) + for cmd in captured: + if '--physdev-in' in cmd: + self.assertNotIn('--physdev-is-bridged', cmd, cmd) + if '--physdev-out' in cmd: + self.assertIn('--physdev-is-bridged', cmd, cmd) + + +class TestDirectRoutedRules(unittest.TestCase): + + def setUp(self): + self.sg = load_script() + ok, self.captured = capture_default_network_rules(self.sg, "brdr-42", direct_routed=True) + self.assertTrue(ok) + self.iptables = [c for c in self.captured if c.startswith('iptables ') or c.startswith('ip6tables ')] + + def test_no_physdev_is_bridged_anywhere(self): + # The defining property of the routed path: physdev-is-bridged matches only bridged + # traffic, which routed traffic is not. + for cmd in self.captured: + self.assertNotIn('--physdev-is-bridged', cmd, cmd) + + def test_no_dhcp_rules(self): + for cmd in self.iptables: + for marker in ['dport 67', 'sport 67', 'dport 68', 'sport 546', 'sport 547', 'dport 546']: + self.assertNotIn(marker, cmd, cmd) + + def test_from_instance_dispatch_uses_bridge_port(self): + self.assertIn("iptables -A BF-brdr-42-IN -m physdev --physdev-in vnet3 -j i-2-7-def", self.captured) + self.assertIn("ip6tables -A BF-brdr-42-IN -m physdev --physdev-in vnet3 -j i-2-7-def", self.captured) + + def test_towards_instance_dispatch_uses_ipset_destination(self): + self.assertIn("iptables -A BF-brdr-42-OUT -m set --match-set i-2-7-VM dst -j i-2-7-def", self.captured) + self.assertIn("ip6tables -A BF-brdr-42-OUT -m set --match-set i-2-7-VM-6 dst -j i-2-7-def", self.captured) + + def test_source_spoofing_dropped_both_families(self): + self.assertIn("iptables -A i-2-7-def -m physdev --physdev-in vnet3 -m set ! --match-set i-2-7-VM src -j DROP", self.captured) + self.assertIn("ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -m set ! --match-set i-2-7-VM-6 src -j DROP", self.captured) + + def test_router_advertisements_from_instance_dropped(self): + self.assertIn("ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p icmpv6 --icmpv6-type router-advertisement -j DROP", self.captured) + + def test_neighbor_discovery_between_instances_allowed(self): + self.assertIn("ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p icmpv6 --icmpv6-type neighbor-solicitation -m hl --hl-eq 255 -j ACCEPT", self.captured) + self.assertIn("ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p icmpv6 --icmpv6-type neighbor-advertisement -m set --match-set i-2-7-VM-6 src -m hl --hl-eq 255 -j ACCEPT", self.captured) + + def test_default_deny_in_both_directions_and_families(self): + self.assertIn("iptables -A i-2-7-def -m physdev --physdev-in vnet3 -j DROP", self.captured) + self.assertIn("iptables -A i-2-7-def -m set --match-set i-2-7-VM dst -j DROP", self.captured) + self.assertIn("ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -j DROP", self.captured) + self.assertIn("ip6tables -A i-2-7-def -m set --match-set i-2-7-VM-6 dst -j DROP", self.captured) + + def test_user_rule_chains_are_wired(self): + self.assertIn("iptables -A i-2-7-def -m physdev --physdev-in vnet3 -m set --match-set i-2-7-VM src -j i-2-7-VM-eg", self.captured) + self.assertIn("iptables -A i-2-7-def -m set --match-set i-2-7-VM dst -j i-2-7-VM", self.captured) + + def test_gateway_arp_protection_unchanged(self): + # The ebtables rules are shared with the classic path: they pin the ARP *source* to the + # Instance's own address and never filter the ARP target, so resolving the gateway works. + ebtables = [c for c in self.captured if c.startswith('ebtables')] + self.assertTrue(any('--arp-ip-src 10.1.1.55' in c for c in ebtables)) + self.assertFalse(any('arp-ip-dst 169.254' in c for c in ebtables)) + + +class TestSharedFrameworkHelpers(unittest.TestCase): + """ enable_bridge_netfilter, create_bridge_fw_chains and add_notrack_ipset_rules are shared + by the classic and Direct Routed frameworks. The golden proves extracting them left the + classic framework byte-identical. """ + + def test_add_fw_framework_matches_golden(self): + sg = load_script() + captured = capture_framework(sg, sg.add_fw_framework, "cloudbr0") + with open(GOLDEN_FRAMEWORK) as f: + golden = f.read().splitlines() + self.assertEqual(golden, captured) + + def test_both_frameworks_share_the_helpers(self): + sg = load_script() + classic = capture_framework(sg, sg.add_fw_framework, "cloudbr0") + l3 = capture_framework(sg, sg.add_l3_fw_framework, "brdr-42") + for stream in (classic, l3): + self.assertIn("modprobe br_netfilter", stream) + self.assertTrue(any('ipset -! create' in c for c in stream)) + self.assertTrue(any('-j NOTRACK' in c for c in stream)) + # the L3 hooks jump unconditionally; the classic ones gate on physdev-is-bridged + self.assertTrue(any('-I FORWARD -i brdr-42 -j BF-brdr-42-IN' in c for c in l3)) + for cmd in l3: + if 'FORWARD' in cmd: + self.assertNotIn('physdev', cmd, cmd) + self.assertTrue(any('physdev-is-bridged' in c and 'FORWARD' in c for c in classic)) + + +class TestDirectRoutedFlagPlumbing(unittest.TestCase): + """ The Agent decides the network type and passes --directrouted; the script never infers + it from the bridge name, the gateway, or anything else. """ + + def test_flag_reaches_default_network_rules(self): + sg = load_script() + self.assertFalse(hasattr(sg, 'is_direct_routed_bridge'), + "the script must not classify bridges itself") + _, classic = capture_default_network_rules(sg, "brdr-42", direct_routed=False) + # Same bridge name, flag off: the classic rules must be produced + self.assertTrue(any('--physdev-is-bridged --physdev-out' in c for c in classic)) + + def test_cli_exposes_directrouted(self): + with open(SCRIPT) as f: + source = f.read() + self.assertIn('"--directrouted"', source) + self.assertIn('dest="directRouted"', source) + + +def regenerate_golden(): + sg = load_script() + ok, captured = capture_default_network_rules(sg, "cloudbr0", direct_routed=False) + assert ok + with open(GOLDEN, 'w') as f: + f.write("\n".join(captured) + "\n") + print("wrote %d commands to %s" % (len(captured), GOLDEN)) + + +if __name__ == '__main__': + if '--regenerate' in sys.argv: + regenerate_golden() + else: + unittest.main() diff --git a/scripts/vm/network/vnet/modifybrdr.sh b/scripts/vm/network/vnet/modifybrdr.sh new file mode 100755 index 000000000000..7b6c1fd4ae5b --- /dev/null +++ b/scripts/vm/network/vnet/modifybrdr.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# modifybrdr.sh -- Manage per-network bridges for Direct Routed Networks +# +# One bridge is created per network, named brdr- +# (Bridge-DirectRouted). The routed id is the value of the network's +# routed:// broadcast domain -- chosen by the operator at network creation +# or allocated by CloudStack from the ROUTED physical network's range -- and is +# stable for the network's life. The bridge carries the shared, +# host-independent gateway addresses and is identical on every hypervisor, so +# an Instance needs no reconfiguration when it migrates. +# +# The bridge never has a physical uplink: all Instance traffic is routed by +# the hypervisor, not bridged. Separate bridges are what keep networks +# isolated from one another at layer 2, and they give the operator a named +# interface per network to hang local routing policy off. +# +# The gateway addresses are always passed in by the CloudStack agent, taken +# from the Instance's NIC; this script carries no defaults of its own, so the +# addresses are defined in exactly one place (NetUtils on the management +# server). +# +# This script is the only place that knows how these bridges are named. 'add' +# prints the bridge name on stdout for the agent to use, and 'delete' takes a +# bridge name and answers on stdout: +# notmine - not a bridge this script manages; the agent falls back to its +# regular unplug handling +# kept - ours, but other Instances still use it +# deleted - ours, removed +# +# Usage: +# add: modifybrdr.sh -o add -n -4 and/or -6 +# delete: modifybrdr.sh -o delete -b + +usage() { + echo "Usage: $0 -o add -n [-4 ] [-6 ] | -o delete -b " +} + +OP= +ROUTED_ID= +BRNAME= +IPV4_GATEWAY= +IPV6_GATEWAY= + +while getopts 'o:n:b:4:6:' OPTION; do + case $OPTION in + o) oflag=1 + OP="$OPTARG" + ;; + n) ROUTED_ID="$OPTARG" + ;; + b) BRNAME="$OPTARG" + ;; + 4) IPV4_GATEWAY="$OPTARG" + ;; + 6) IPV6_GATEWAY="$OPTARG" + ;; + ?) usage + exit 2 + ;; + esac +done + +if [[ "$oflag" != "1" ]]; then + usage + exit 2 +fi + +if [[ "$OP" == "add" ]]; then + if [[ ! "$ROUTED_ID" =~ ^[0-9]+$ ]]; then + echo "Routed id must be numeric: ${ROUTED_ID}" + exit 2 + fi + + if [[ -z "$IPV4_GATEWAY" && -z "$IPV6_GATEWAY" ]]; then + echo "At least one of -4 or -6 must be given for add" + usage + exit 2 + fi + + BRNAME="brdr-${ROUTED_ID}" + + # Linux caps interface names at 15 characters + if [[ ${#BRNAME} -gt 15 ]]; then + echo "Bridge name ${BRNAME} exceeds the 15 character interface name limit" + exit 2 + fi +elif [[ "$OP" == "delete" ]]; then + if [[ -z "$BRNAME" ]]; then + usage + exit 2 + fi + + # Not one of ours: tell the agent so it can fall back to its regular + # unplug handling. This is what keeps the bridge naming knowledge in + # this script and nowhere else. + if [[ "$BRNAME" != brdr-* ]]; then + echo "notmine" + exit 0 + fi +else + usage + exit 2 +fi + +addBr() { + if [[ ! -d /sys/class/net/${BRNAME} ]]; then + # No STP and no forwarding delay: the bridge has no uplink, so there is + # no loop to detect and no reason to hold ports down when an Instance starts + ip link add name ${BRNAME} type bridge stp_state 0 forward_delay 0 + ip link set ${BRNAME} up + fi + + # The bridge routes on behalf of every Instance attached to it + sysctl -qw net.ipv4.conf.${BRNAME}.forwarding=1 + sysctl -qw net.ipv6.conf.${BRNAME}.disable_ipv6=0 + sysctl -qw net.ipv6.conf.${BRNAME}.forwarding=1 + + # Never act on a router advertisement sent by an Instance + sysctl -qw net.ipv6.conf.${BRNAME}.accept_ra=0 + + # The same gateway address is configured on every brdr bridge on this host. + # Only answer ARP for the address on the interface the request arrived on, + # and always source ARP from that interface's own address. + sysctl -qw net.ipv4.conf.${BRNAME}.arp_ignore=1 + sysctl -qw net.ipv4.conf.${BRNAME}.arp_announce=2 + + if [[ -n "${IPV4_GATEWAY}" ]]; then + ip address replace ${IPV4_GATEWAY}/32 dev ${BRNAME} + fi + if [[ -n "${IPV6_GATEWAY}" ]]; then + ip -6 address replace ${IPV6_GATEWAY}/64 dev ${BRNAME} + fi + + # Strict reverse path filtering: an Instance may only send from an address + # that is routed back out of this bridge, which is its own /32. Set on the + # bridge only, never on 'all', so no other interface changes behaviour. + # Note this is IPv4 only; the kernel has no IPv6 equivalent. + sysctl -qw net.ipv4.conf.${BRNAME}.rp_filter=1 + +} + +deleteBr() { + if [[ ! -d /sys/class/net/${BRNAME} ]]; then + echo "deleted" + return 0 + fi + + # An Instance may have been started on this network while the last one was + # being stopped; leave the bridge alone if anything is still attached + if [[ -n "$(ls -A /sys/class/net/${BRNAME}/brif 2>/dev/null)" ]]; then + echo "kept" + return 0 + fi + + ip link set ${BRNAME} down + ip link delete ${BRNAME} type bridge + echo "deleted" +} + +# +# Add a lockfile to prevent this script from running twice on the same host +# this can cause a race condition +# + +LOCKFILE=/var/run/cloud/brdr.lock + +# ensures that parent directories exists and prepares the lock file +mkdir -p "${LOCKFILE%/*}" + +( + flock -x -w 10 200 || exit 1 + if [[ "$OP" == "add" ]]; then + addBr + + if [[ $? -gt 0 ]]; then + exit 1 + fi + + # The agent uses this as the bridge name; naming lives here only + echo "${BRNAME}" + elif [[ "$OP" == "delete" ]]; then + deleteBr + fi +) 200>${LOCKFILE} diff --git a/scripts/vm/network/vnet/modifymacip.sh b/scripts/vm/network/vnet/modifymacip.sh index c8d0b0e290ca..d6cd82ae5eae 100755 --- a/scripts/vm/network/vnet/modifymacip.sh +++ b/scripts/vm/network/vnet/modifymacip.sh @@ -24,8 +24,11 @@ # # Both -4 and -6 may be specified multiple times to cover primary and secondary # addresses (e.g. link-local + global unicast for IPv6). -# On delete the bridge neighbour table is queried for all entries matching the -# MAC address; no separate state file is required. +# On delete without any -4/-6, the bridge neighbour table is queried for all +# entries matching the MAC address and every one of them is removed; no separate +# state file is required. This is what an unplug uses. Given -4/-6 addresses, +# delete removes only those, which is what removing a single secondary IP from a +# running Instance needs. usage() { echo "Usage: $0 -o -b -m [-4 ] ... [-6 ] ..." @@ -69,7 +72,24 @@ add_entries() { fi } +delete_listed_entries() { + for addr in "${IPV4_LIST[@]}"; do + ip neigh del "${addr}" dev "${BRIDGE}" 2>/dev/null || true + ip route del "${addr}/32" dev "${BRIDGE}" 2>/dev/null || true + done + + for addr in "${IPV6_LIST[@]}"; do + ip -6 neigh del "${addr}" dev "${BRIDGE}" 2>/dev/null || true + ip -6 route del "${addr}/128" dev "${BRIDGE}" 2>/dev/null || true + done +} + delete_entries() { + if [[ "${#IPV4_LIST[@]}" -gt 0 || "${#IPV6_LIST[@]}" -gt 0 ]]; then + delete_listed_entries + return 0 + fi + # Find all IPv4 neighbour entries on the bridge matching this MAC and remove them while read -r addr; do ip neigh del "${addr}" dev "${BRIDGE}" 2>/dev/null || true diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index c68dc390df2a..ff869e01c981 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -5560,6 +5560,16 @@ public Vlan createVlanAndPublicIpRange(final CreateVlanIpRangeCmd cmd) throws In checkOverlapPrivateIpRange(zoneId, startIP, endIP); } + // On an L3 (Direct Routed) network an overlap is an address conflict, not a policy + // preference: all subnets share one host routing table and are advertised into one + // fabric, so the check must be zone-wide. The IPv6 vlan check above already is; for + // IPv4 the user_ip_address unique key is per network, hence this explicit check. + // Shared networks keep their historical behaviour: the same IPv4 range in two VLANs + // is legitimate there. + if (ipv4 && network.getGuestType() == GuestType.L3) { + checkOverlapPublicIpRange(zoneId, startIP, endIP); + } + long reservedIpAddressesAmount = 0L; if (forVirtualNetwork && vlanOwner != null) { reservedIpAddressesAmount = NetUtils.ip2Long(endIP) - NetUtils.ip2Long(startIP) + 1; @@ -5858,7 +5868,9 @@ public Vlan createVlanAndPublicIpRange(final long zoneId, final long networkId, } } - boolean isSharedNetworkWithoutSpecifyVlan = _networkMgr.isSharedNetworkWithoutSpecifyVlan(_networkOfferingDao.findById(network.getNetworkOfferingId())); + final NetworkOffering networkOffering = _networkOfferingDao.findById(network.getNetworkOfferingId()); + boolean isSharedNetworkWithoutSpecifyVlan = _networkMgr.isSharedNetworkWithoutSpecifyVlan(networkOffering); + boolean isL3NetworkWithoutSpecifyVlan = _networkMgr.isL3NetworkWithoutSpecifyVlan(networkOffering); if (ipv4) { final String newCidr = NetUtils.getCidrFromGatewayAndNetmask(vlanGateway, vlanNetmask); @@ -5916,8 +5928,11 @@ public Vlan createVlanAndPublicIpRange(final long zoneId, final long networkId, } } - // Check if the vlan is being used - if (isSharedNetworkWithoutSpecifyVlan) { + // Check if the vlan is being used. An auto-allocated id (Shared VLAN or L3 routed id, + // offering without specifyVlan) always lies inside the physical network's vnet range — + // findVnet matches any id in the range, taken or free — so the in-range check below + // would reject every such network; both cases are exempt. + if (isSharedNetworkWithoutSpecifyVlan || isL3NetworkWithoutSpecifyVlan) { bypassVlanOverlapCheck = true; } if (!bypassVlanOverlapCheck && !forExternalProvider && !_zoneDao.findVnet(zoneId, physicalNetworkId, BroadcastDomainType.getValue(BroadcastDomainType.fromString(vlanId))).isEmpty()) { @@ -5925,6 +5940,15 @@ public Vlan createVlanAndPublicIpRange(final long zoneId, final long networkId, + zone.getName()); } + // A routed id names a bridge on every host (brdr-): a public range and a guest + // network sharing one id would merge their L2 domains. The guest-side creation rejects + // ids held by public ranges; this is the same guard in the other direction. + if (vlanId != null && vlanId.startsWith(BroadcastDomainType.Routed.scheme() + "://") + && !_networkDao.listByZoneAndUriAndGuestType(zoneId, vlanId, null).isEmpty()) { + throw new InvalidParameterValueException(String.format( + "The routed id %s is already used by a guest network in zone %s", vlanId, zone.getName())); + } + String ipRange = null; if (ipv4) { @@ -5999,6 +6023,10 @@ private String getNetworkVlanId(Network network, boolean connectivityWithoutVlan if (network.getBroadcastDomainType() != BroadcastDomainType.Vlan) { networkVlanId = networkVlanId.split("-")[0]; } + } else if (BroadcastDomainType.getSchemeValue(uri) == BroadcastDomainType.Routed) { + // Direct Routed (L3) networks: the routed id plays the role the VLAN tag plays + // for Shared networks, so IP ranges added later stay consistent with it + networkVlanId = BroadcastDomainType.getValue(uri); } } return networkVlanId; @@ -7214,7 +7242,7 @@ public NetworkOffering createNetworkOffering(final NetworkOfferingBaseCmd cmd) { } if (guestType == null) { - throw new InvalidParameterValueException("Invalid \"type\" parameter is given; can have Shared and Isolated values"); + throw new InvalidParameterValueException("Invalid \"type\" parameter is given; supported values are " + Arrays.toString(Network.GuestType.values())); } if (internetProtocol != null) { @@ -7271,9 +7299,9 @@ public NetworkOffering createNetworkOffering(final NetworkOfferingBaseCmd cmd) { } if (service == Service.SecurityGroup) { - // allow security group service for Shared networks only - if (guestType != GuestType.Shared) { - throw new InvalidParameterValueException("Security group service is supported for network offerings with guest ip type " + GuestType.Shared); + // allow security group service for Shared and L3 (Direct Routed) networks only + if (guestType != GuestType.Shared && guestType != GuestType.L3) { + throw new InvalidParameterValueException(String.format("Security group service is supported for network offerings with guest ip type %s or %s", GuestType.Shared, GuestType.L3)); } final Set sgProviders = new HashSet<>(); sgProviders.add(Provider.SecurityGroupProvider); @@ -7369,6 +7397,10 @@ public NetworkOffering createNetworkOffering(final NetworkOfferingBaseCmd cmd) { // validate providers combination here _networkModel.canProviderSupportServices(providerCombinationToVerify); + if (guestType == GuestType.L3) { + validateL3NetworkOffering(serviceProviderMap, networkMode, specifyVlan, specifyIpRanges, forVpc); + } + // validate the LB service capabilities specified in the network // offering final Map lbServiceCapabilityMap = cmd.getServiceCapabilities(Service.Lb); @@ -7471,6 +7503,49 @@ public NetworkOffering createNetworkOffering(final NetworkOfferingBaseCmd cmd) { return offering; } + /** + * Validates a network offering for the L3 (Direct Routed) guest type. There is no Virtual + * Router and no DHCP on these networks: the Instance learns its /32 (and /128) address, + * on-link gateway and routes exclusively from ConfigDrive. UserData via ConfigDrive is + * therefore mandatory. Dns is optional (a template may carry its own resolvers) but must be + * provided by ConfigDrive when present. SecurityGroup is the only other permitted service. + */ + protected void validateL3NetworkOffering(final Map> serviceProviderMap, final NetworkOffering.NetworkMode networkMode, + final boolean specifyVlan, final boolean specifyIpRanges, final Boolean forVpc) { + if (Boolean.TRUE.equals(forVpc)) { + throw new InvalidParameterValueException(String.format("VPC is not supported for network offerings with guest type %s", GuestType.L3)); + } + if (networkMode != null) { + throw new InvalidParameterValueException(String.format("Network mode can not be specified for network offerings with guest type %s", GuestType.L3)); + } + // specifyVlan is a free choice: with it the operator supplies the routed id (routed://, + // naming the per-network bridge) at network creation via the vlan parameter; without it + // CloudStack allocates one from the ROUTED physical network's vnet range. + if (!specifyIpRanges) { + throw new InvalidParameterValueException(String.format("Network offerings with guest type %s must specify IP ranges", GuestType.L3)); + } + if (serviceProviderMap.containsKey(Service.Dhcp)) { + throw new InvalidParameterValueException(String.format("DHCP is not supported (and not needed) for network offerings with guest type %s; addressing is delivered via ConfigDrive", GuestType.L3)); + } + final Set allowedL3Services = new HashSet<>(Arrays.asList(Service.UserData, Service.Dns, Service.SecurityGroup)); + for (final Service service : serviceProviderMap.keySet()) { + if (!allowedL3Services.contains(service)) { + throw new InvalidParameterValueException(String.format("Service %s is not supported for network offerings with guest type %s; supported services are %s", + service.getName(), GuestType.L3, StringUtils.join(allowedL3Services.stream().map(Service::getName).toArray(), ", "))); + } + } + final Set configDriveOnly = Collections.singleton(Provider.ConfigDrive); + final Set userDataProviders = serviceProviderMap.get(Service.UserData); + if (!configDriveOnly.equals(userDataProviders)) { + throw new InvalidParameterValueException(String.format("UserData with provider %s is mandatory for network offerings with guest type %s; it is the only channel that carries the Instance's network configuration", + Provider.ConfigDrive.getName(), GuestType.L3)); + } + final Set dnsProviders = serviceProviderMap.get(Service.Dns); + if (dnsProviders != null && !configDriveOnly.equals(dnsProviders)) { + throw new InvalidParameterValueException(String.format("DNS on network offerings with guest type %s must use provider %s", GuestType.L3, Provider.ConfigDrive.getName())); + } + } + public static NetworkOffering.RoutingMode verifyRoutingMode(String routingModeString) { NetworkOffering.RoutingMode routingMode = null; if (routingModeString != null) { diff --git a/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index d07641eacc8a..3eadc9f36254 100644 --- a/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -1235,8 +1235,16 @@ public boolean finalizeVirtualMachineProfile(VirtualMachineProfile profile, Depl buf.append(" eth").append(deviceId).append("mask=").append(nic.getIPv4Netmask()); } + if (nic.getIPv6Address() != null) { + buf.append(" eth").append(deviceId).append("ip6=").append(nic.getIPv6Address()); + buf.append(" eth").append(deviceId).append("ip6prelen=").append(NetUtils.getIp6CidrSize(nic.getIPv6Cidr())); + } + if (nic.isDefaultNic()) { buf.append(" gateway=").append(nic.getIPv4Gateway()); + if (nic.getIPv6Gateway() != null) { + buf.append(" ip6gateway=").append(nic.getIPv6Gateway()); + } } if (nic.getTrafficType() == TrafficType.Management) { diff --git a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java index 2853fa96330d..7d23ca9a9733 100644 --- a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java @@ -868,6 +868,19 @@ public boolean stop() { protected NetworkServiceImpl() { } + /** + * True when the NIC belongs to a Direct Routed (L3) network, where secondary IPs need a host + * route and neighbour entry on the hypervisor whether or not security groups are in use. + */ + protected boolean isDirectRoutedNic(long nicId) { + NicVO nic = _nicDao.findById(nicId); + if (nic == null) { + return false; + } + Network network = _networksDao.findById(nic.getNetworkId()); + return network != null && GuestType.L3.equals(network.getGuestType()); + } + @Override @ActionEvent(eventType = EventTypes.EVENT_NIC_SECONDARY_IP_CONFIGURE, eventDescription = "Configuring secondary IP " + "rules", async = true) public boolean configureNicSecondaryIp(NicSecondaryIp secIp, boolean isZoneSgEnabled) { @@ -877,9 +890,11 @@ public boolean configureNicSecondaryIp(NicSecondaryIp secIp, boolean isZoneSgEna secondaryIp = secIp.getIp6Address(); } - if (isZoneSgEnabled) { + // A Direct Routed network needs the agent told regardless of the zone's security group + // setting: the host route and neighbour entry are what make the address reachable. + if (isZoneSgEnabled || isDirectRoutedNic(secIp.getNicId())) { success = _securityGroupService.securityGroupRulesForVmSecIp(secIp.getNicId(), secondaryIp, true); - logger.info("Associated IP address to NIC : " + secIp.getIp4Address()); + logger.info("Associated IP address to NIC : " + secondaryIp); } else { success = true; } @@ -1558,6 +1573,16 @@ public Network createGuestNetwork(CreateNetworkCmd cmd) throws InsufficientCapac ACLType aclType = getAclType(caller, cmd.getAclType(), ntwkOff); + // Fail here with a clear message rather than deep in setupNetwork(), where no guru + // claiming the network produces an opaque error. The physical network is the network + // operator's opt-in: without ROUTED it carries no direct routed networks. + if (ntwkOff.getGuestType() == GuestType.L3 + && (pNtwk.getIsolationMethods() == null || !pNtwk.getIsolationMethods().contains("ROUTED"))) { + throw new InvalidParameterValueException(String.format( + "Networks of guest type %s can only be created on a physical network with isolation method ROUTED; physical network %s carries %s", + GuestType.L3, pNtwk.getName(), pNtwk.getIsolationMethods())); + } + if (ntwkOff.getGuestType() != GuestType.Shared && (!StringUtils.isAllBlank(routerIPv4, routerIPv6))) { throw new InvalidParameterValueException("Router IP can be specified only for Shared networks"); } @@ -1605,8 +1630,8 @@ public Network createGuestNetwork(CreateNetworkCmd cmd) throws InsufficientCapac } } - // Start and end IP address are mandatory for shared networks. - if (ntwkOff.getGuestType() == GuestType.Shared && vpcId == null) { + // Start and end IP address are mandatory for shared and L3 (Direct Routed) networks. + if ((ntwkOff.getGuestType() == GuestType.Shared || ntwkOff.getGuestType() == GuestType.L3) && vpcId == null) { if (!AllowEmptyStartEndIpAddress.valueIn(owner.getAccountId()) && (startIP == null && endIP == null) && (startIPv6 == null && endIPv6 == null)) { @@ -1655,12 +1680,19 @@ public Network createGuestNetwork(CreateNetworkCmd cmd) throws InsufficientCapac endIPv6 = startIPv6; } _networkModel.checkIp6Parameters(startIPv6, endIPv6, ip6Gateway, ip6Cidr); - if (!GuestType.Shared.equals(ntwkOff.getGuestType())) { + if (!GuestType.Shared.equals(ntwkOff.getGuestType()) && !GuestType.L3.equals(ntwkOff.getGuestType())) { _networkModel.checkIp6CidrSizeEqualTo64(ip6Cidr); } + // IPv6 addresses are calculated with EUI-64 from the subnet and the NIC's MAC, which + // needs 64 interface-identifier bits: a prefix longer than /64 would fail at Instance + // deploy (NetUtils.EUI64Address throws), so reject it at network creation instead. + if (GuestType.L3.equals(ntwkOff.getGuestType()) && NetUtils.getIp6CidrSize(ip6Cidr) > 64) { + throw new InvalidParameterValueException(String.format( + "The IPv6 subnet of a %s network must be /64 or larger: addresses are derived with EUI-64 from the subnet and the NIC MAC", GuestType.L3)); + } - if (zone.getNetworkType() != NetworkType.Advanced || ntwkOff.getGuestType() != Network.GuestType.Shared) { - throw new InvalidParameterValueException("Can only support create IPv6 network with advance shared network!"); + if (zone.getNetworkType() != NetworkType.Advanced || (ntwkOff.getGuestType() != Network.GuestType.Shared && ntwkOff.getGuestType() != Network.GuestType.L3)) { + throw new InvalidParameterValueException(String.format("Can only support create IPv6 network with advanced %s or %s network!", GuestType.Shared, GuestType.L3)); } if(StringUtils.isAllBlank(ip6Dns1, ip6Dns2, zone.getIp6Dns1(), zone.getIp6Dns2())) { @@ -1699,7 +1731,7 @@ public Network createGuestNetwork(CreateNetworkCmd cmd) throws InsufficientCapac if (!_accountMgr.isRootAdmin(caller.getId())) { throw new InvalidParameterValueException("Only ROOT admin is allowed to create Private VLAN network"); } - if (zone.getNetworkType() != NetworkType.Advanced || ntwkOff.getGuestType() == GuestType.Isolated) { + if (zone.getNetworkType() != NetworkType.Advanced || ntwkOff.getGuestType() == GuestType.Isolated || ntwkOff.getGuestType() == GuestType.L3) { throw new InvalidParameterValueException("Can only support create Private VLAN network with advanced shared or L2 network!"); } if (ipv6) { @@ -1722,7 +1754,7 @@ public Network createGuestNetwork(CreateNetworkCmd cmd) throws InsufficientCapac } // Ignore vlanId if it is passed but specifyvlan=false in network offering - if (ntwkOff.getGuestType() == GuestType.Shared && ! ntwkOff.isSpecifyVlan() && vlanId != null) { + if ((ntwkOff.getGuestType() == GuestType.Shared || ntwkOff.getGuestType() == GuestType.L3) && ! ntwkOff.isSpecifyVlan() && vlanId != null) { throw new InvalidParameterValueException("Cannot specify vlanId when create a network from network offering with specifyvlan=false"); } @@ -1778,9 +1810,11 @@ public Network createGuestNetwork(CreateNetworkCmd cmd) throws InsufficientCapac } } - // Vlan is created in 1 cases - works in Advance zone only: - // 1) GuestType is Shared + // Vlan is created in these cases - works in Advance zone only: + // 1) GuestType is Shared or L3 (Direct Routed) + // 2) GuestType is Isolated without SourceNat boolean createVlan = (startIP != null && endIP != null && zone.getNetworkType() == NetworkType.Advanced && ((ntwkOff.getGuestType() == Network.GuestType.Shared) + || (ntwkOff.getGuestType() == Network.GuestType.L3) || (ntwkOff.getGuestType() == GuestType.Isolated && !areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat)))); if (!createVlan) { @@ -1797,9 +1831,9 @@ public Network createGuestNetwork(CreateNetworkCmd cmd) throws InsufficientCapac - if (GuestType.Shared == ntwkOff.getGuestType()) { + if (GuestType.Shared == ntwkOff.getGuestType() || GuestType.L3 == ntwkOff.getGuestType()) { if (!ntwkOff.isSpecifyIpRanges()) { - throw new CloudRuntimeException("The 'specifyipranges' parameter should be true for Shared Networks"); + throw new CloudRuntimeException(String.format("The 'specifyipranges' parameter should be true for %s Networks", ntwkOff.getGuestType())); } if (ipv4 && Objects.isNull(startIP)) { throw new CloudRuntimeException("IPv4 address range needs to be provided"); @@ -2018,7 +2052,7 @@ private static ACLType getAclType(String aclTypeStr, NetworkOffering ntwkOff) { } private ACLType getAclType(Account caller, NetworkOffering ntwkOff, ACLType aclType) { - if (ntwkOff.getGuestType() == GuestType.Isolated || ntwkOff.getGuestType() == GuestType.L2) { + if (ntwkOff.getGuestType() == GuestType.Isolated || ntwkOff.getGuestType() == GuestType.L2 || ntwkOff.getGuestType() == GuestType.L3) { aclType = ACLType.Account; } else if (ntwkOff.getGuestType() == GuestType.Shared) { if (_accountMgr.isRootAdmin(caller.getId())) { @@ -2326,15 +2360,18 @@ public Network doInTransaction(TransactionStatus status) throws InsufficientCapa } String vlanId = vlanIdFinal; - if (createVlan && vlanId == null && ntwkOff.getGuestType() == Network.GuestType.Shared && ! ntwkOff.isSpecifyVlan()) { + if (createVlan && vlanId == null && (ntwkOff.getGuestType() == Network.GuestType.Shared || ntwkOff.getGuestType() == Network.GuestType.L3) + && ! ntwkOff.isSpecifyVlan()) { if (associatedNetwork != null) { // Get vlanId from associated network vlanId = associatedNetwork.getBroadcastUri().toString(); } else { - // Allocate a vnet to shared network with specifyvlan=false + // Allocate a vnet to a Shared network with specifyvlan=false, or a routed id + // to an L3 (Direct Routed) network — both come from the physical network's + // vnet range and are released when the network is deleted. vlanId = _dcDao.allocateVnet(zoneId, physicalNetworkId, owner.getAccountId(), null, GuestNetworkGuru.UseSystemGuestVlans.valueIn(owner.getAccountId())); if (vlanId == null) { - throw new InvalidParameterValueException("Cannot allocate a vnet for this Shared network"); + throw new InvalidParameterValueException(String.format("Cannot allocate a vnet for this %s network", ntwkOff.getGuestType())); } } } diff --git a/server/src/main/java/com/cloud/network/guru/DirectRoutedNetworkGuru.java b/server/src/main/java/com/cloud/network/guru/DirectRoutedNetworkGuru.java new file mode 100644 index 000000000000..4ace6481df44 --- /dev/null +++ b/server/src/main/java/com/cloud/network/guru/DirectRoutedNetworkGuru.java @@ -0,0 +1,202 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.guru; + +import org.apache.commons.lang3.StringUtils; + +import com.cloud.dc.DataCenter; +import com.cloud.dc.DataCenter.NetworkType; +import com.cloud.deploy.DeployDestination; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientAddressCapacityException; +import com.cloud.exception.InsufficientVirtualNetworkCapacityException; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.network.Network; +import com.cloud.network.Network.GuestType; +import com.cloud.network.Network.State; +import com.cloud.network.Networks.BroadcastDomainType; +import com.cloud.network.Networks.Mode; +import com.cloud.network.PhysicalNetwork; +import com.cloud.network.PhysicalNetwork.IsolationMethod; +import com.cloud.network.dao.NetworkVO; +import com.cloud.network.dao.PhysicalNetworkVO; +import com.cloud.offering.NetworkOffering; +import com.cloud.user.Account; +import com.cloud.utils.net.NetUtils; +import com.cloud.vm.NicProfile; +import com.cloud.vm.ReservationContext; +import com.cloud.vm.VirtualMachineProfile; + +/** + * Network guru for L3 (Direct Routed) guest networks: the hypervisor routes a public IPv4 + * and/or IPv6 address directly to the Instance. There is no Virtual Router, no NAT and no + * DHCP; the Instance learns its addressing exclusively from ConfigDrive. + * + * Address allocation is inherited from {@link DirectNetworkGuru}: the operator supplies a + * subnet, CloudStack assigns individual addresses out of it. What differs is the form those + * addresses take on the NIC — an IPv4 address is a /32 and an IPv6 address a /128, with the + * shared, host-independent on-link gateway (169.254.0.1 / fe80::1) that every hypervisor + * carries on the network's bridge. + * + * These networks live on a dedicated physical network whose isolation method is ROUTED — the + * network operator's explicit, zone-level opt-in. Each network carries a broadcast domain of + * type routed://<id>; the id is a label naming the per-network, uplink-less bridge on every + * host (brdr-<id>), never an encapsulation. It is chosen by the operator at network creation + * (specifyVlan offerings, via the vlan parameter) or allocated from the physical network's vnet + * range (otherwise), and is stable for the network's life. + */ +public class DirectRoutedNetworkGuru extends DirectNetworkGuru { + + public DirectRoutedNetworkGuru() { + super(); + // Selection follows the standard guru contract: a network is direct routed when its + // offering's guest type is L3 AND its physical network carries the ROUTED isolation + // method. The id conveyed in routed:// consumes nothing on the wire. + _isolationMethods = new IsolationMethod[] {new IsolationMethod("ROUTED")}; + } + + @Override + protected boolean canHandle(NetworkOffering offering, DataCenter dc, PhysicalNetwork physnet) { + if (dc.getNetworkType() == NetworkType.Advanced && isMyTrafficType(offering.getTrafficType()) && offering.getGuestType() == GuestType.L3 + && isMyIsolationMethod(physnet)) { + return true; + } + logger.trace("We only take care of {} guest networks in zones of type {} on physical networks with isolation method ROUTED", GuestType.L3, NetworkType.Advanced); + return false; + } + + @Override + public Network design(NetworkOffering offering, DeploymentPlan plan, Network userSpecified, String name, Long vpcId, Account owner) { + DataCenter dc = _dcDao.findById(plan.getDataCenterId()); + PhysicalNetworkVO physnet = _physicalNetworkDao.findById(plan.getPhysicalNetworkId()); + + if (!canHandle(offering, dc, physnet)) { + return null; + } + + if (vpcId != null) { + throw new InvalidParameterValueException(String.format("%s networks are never part of a VPC", GuestType.L3)); + } + + // Mode.Static: the Instance is statically configured via ConfigDrive, there is no DHCP. + // BroadcastDomainType.Routed: routed:// names the per-network bridge (brdr-); + // no VLAN/VXLAN is consumed and nothing appears on the wire. + NetworkVO config = new NetworkVO(offering.getTrafficType(), Mode.Static, BroadcastDomainType.Routed, offering.getId(), State.Allocated, + plan.getDataCenterId(), plan.getPhysicalNetworkId(), false); + + if (userSpecified != null) { + if ((userSpecified.getCidr() == null && userSpecified.getGateway() != null) || (userSpecified.getCidr() != null && userSpecified.getGateway() == null)) { + throw new InvalidParameterValueException("cidr and gateway must be specified together."); + } + + // The routed id — operator-specified or allocated from the physical network's vnet + // range — arrives as the broadcast URI, stamped by the orchestrator before design. + // It is set once here and never changes: the bridge name derives from it. + if (userSpecified.getBroadcastUri() != null) { + if (BroadcastDomainType.getSchemeValue(userSpecified.getBroadcastUri()) != BroadcastDomainType.Routed) { + throw new InvalidParameterValueException(String.format("A %s network requires a routed:// broadcast domain, got %s", + GuestType.L3, userSpecified.getBroadcastUri())); + } + config.setBroadcastUri(userSpecified.getBroadcastUri()); + config.setState(State.Setup); + } + + if ((userSpecified.getIp6Cidr() == null && userSpecified.getIp6Gateway() != null) || + (userSpecified.getIp6Cidr() != null && userSpecified.getIp6Gateway() == null)) { + throw new InvalidParameterValueException("cidrv6 and gatewayv6 must be specified together."); + } + + // The subnet is an allocation pool routed to the hypervisors, not a broadcast domain. + // Its gateway is stored but never used: the Instance's gateway is always the shared + // link-local address. + if (userSpecified.getCidr() != null) { + config.setCidr(userSpecified.getCidr()); + config.setGateway(userSpecified.getGateway()); + } + + if (userSpecified.getIp6Cidr() != null) { + config.setIp6Cidr(userSpecified.getIp6Cidr()); + config.setIp6Gateway(userSpecified.getIp6Gateway()); + } + + if (userSpecified.getPublicMtu() != null) { + config.setPublicMtu(userSpecified.getPublicMtu()); + } + if (userSpecified.getPrivateMtu() != null) { + config.setPrivateMtu(userSpecified.getPrivateMtu()); + } + + if (StringUtils.isNotBlank(userSpecified.getDns1())) { + config.setDns1(userSpecified.getDns1()); + } + if (StringUtils.isNotBlank(userSpecified.getDns2())) { + config.setDns2(userSpecified.getDns2()); + } + if (StringUtils.isNotBlank(userSpecified.getIp6Dns1())) { + config.setIp6Dns1(userSpecified.getIp6Dns1()); + } + if (StringUtils.isNotBlank(userSpecified.getIp6Dns2())) { + config.setIp6Dns2(userSpecified.getIp6Dns2()); + } + } + + return config; + } + + @Override + public NicProfile allocate(Network network, NicProfile nic, VirtualMachineProfile vm) throws InsufficientVirtualNetworkCapacityException, + InsufficientAddressCapacityException, ConcurrentOperationException { + NicProfile profile = super.allocate(network, nic, vm); + applyDirectRoutedAddressing(profile); + return profile; + } + + @Override + public void reserve(NicProfile nic, Network network, VirtualMachineProfile vm, DeployDestination dest, ReservationContext context) + throws InsufficientVirtualNetworkCapacityException, InsufficientAddressCapacityException, ConcurrentOperationException { + super.reserve(nic, network, vm, dest, context); + applyDirectRoutedAddressing(nic); + } + + /** + * The inherited allocation ({@code IpAddressManagerImpl.allocateDirectIp()}) sets the NIC's + * gateway and netmask from the vlan row, as a Shared network needs. Here the address is a + * host route, not a subnet membership: force the /32 (or /128) form and the shared on-link + * gateway over whatever the vlan row provided. This is also the signature by which the KVM + * agent recognises a direct routed NIC. + */ + protected void applyDirectRoutedAddressing(NicProfile nic) { + if (nic == null) { + return; + } + // The inherited allocation labels the NIC's isolation URI vlan:// from the vlan + // row, as a Shared network needs; here there is no VLAN — the only isolation-shaped + // fact about this NIC is its routed:// broadcast domain. + if (nic.getBroadCastUri() != null && BroadcastDomainType.getSchemeValue(nic.getBroadCastUri()) == BroadcastDomainType.Routed) { + nic.setIsolationUri(nic.getBroadCastUri()); + } + if (nic.getIPv4Address() != null) { + nic.setIPv4Netmask(NetUtils.IPV4_HOST_NETMASK); + nic.setIPv4Gateway(NetUtils.getLinkLocalGateway()); + } + if (nic.getIPv6Address() != null) { + nic.setIPv6Cidr(nic.getIPv6Address() + "/" + NetUtils.IPV6_HOST_PREFIX_LENGTH); + nic.setIPv6Gateway(NetUtils.getIpv6LinkLocalGateway()); + } + } +} diff --git a/server/src/main/java/com/cloud/network/guru/PublicNetworkGuru.java b/server/src/main/java/com/cloud/network/guru/PublicNetworkGuru.java index b7e4f334622c..b857ecc30317 100644 --- a/server/src/main/java/com/cloud/network/guru/PublicNetworkGuru.java +++ b/server/src/main/java/com/cloud/network/guru/PublicNetworkGuru.java @@ -18,6 +18,8 @@ import javax.inject.Inject; +import org.apache.commons.lang3.StringUtils; + import com.cloud.dc.dao.VlanDetailsDao; import com.cloud.network.vpc.dao.VpcDao; import com.cloud.network.vpc.dao.VpcOfferingDao; @@ -57,11 +59,13 @@ import com.cloud.utils.db.TransactionCallbackNoReturn; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.NetUtils; import com.cloud.vm.Nic.ReservationStrategy; import com.cloud.vm.NicProfile; import com.cloud.vm.ReservationContext; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; +import com.googlecode.ipv6.IPv6Address; public class PublicNetworkGuru extends AdapterBase implements NetworkGuru { @@ -145,7 +149,23 @@ protected void getIp(NicProfile nic, DataCenter dc, VirtualMachineProfile vm, Ne nic.setIPv4Address(ip.getAddress().toString()); nic.setIPv4Gateway(ip.getGateway()); nic.setIPv4Netmask(ip.getNetmask()); - if (network.getBroadcastDomainType() == BroadcastDomainType.Vxlan) { + nic.setFormat(AddressFormat.Ip4); + nic.setReservationId(String.valueOf(ip.getVlanTag())); + // Before the branch below: setRoutedRangeIpv6() derives the EUI-64 address from this + // MAC and upgrades the format to DualStack, so both must already be on the profile. + nic.setMacAddress(ip.getMacAddress()); + if (isRoutedRange(ip)) { + // Direct routed public range (SystemVMs on a ROUTED physical network): the + // address is a host route, not a subnet membership. Same form as a guest NIC on + // a direct routed network: /32 with the shared on-link gateway, and the + // routed:// broadcast domain that names the bridge on the host. + nic.setIPv4Netmask(NetUtils.IPV4_HOST_NETMASK); + nic.setIPv4Gateway(NetUtils.getLinkLocalGateway()); + nic.setIsolationUri(BroadcastDomainType.Routed.toUri(ip.getVlanTag())); + nic.setBroadcastUri(BroadcastDomainType.Routed.toUri(ip.getVlanTag())); + nic.setBroadcastType(BroadcastDomainType.Routed); + setRoutedRangeIpv6(nic, dc, ip, network); + } else if (network.getBroadcastDomainType() == BroadcastDomainType.Vxlan) { nic.setIsolationUri(BroadcastDomainType.Vxlan.toUri(ip.getVlanTag())); nic.setBroadcastUri(BroadcastDomainType.Vxlan.toUri(ip.getVlanTag())); nic.setBroadcastType(BroadcastDomainType.Vxlan); @@ -154,9 +174,6 @@ protected void getIp(NicProfile nic, DataCenter dc, VirtualMachineProfile vm, Ne nic.setBroadcastUri(BroadcastDomainType.Vlan.toUri(ip.getVlanTag())); nic.setBroadcastType(BroadcastDomainType.Vlan); } - nic.setFormat(AddressFormat.Ip4); - nic.setReservationId(String.valueOf(ip.getVlanTag())); - nic.setMacAddress(ip.getMacAddress()); } Pair dns = networkModel.getNetworkIp4Dns(network, dc); @@ -166,6 +183,41 @@ protected void getIp(NicProfile nic, DataCenter dc, VirtualMachineProfile vm, Ne ipv6Service.updateNicIpv6(nic, dc, network); } + /** + * A public IP range on a ROUTED physical network is created with routed:// as its "vlan"; + * the tag flowing through the vlan row is what marks the addresses it holds as direct routed. + */ + private boolean isRoutedRange(PublicIp ip) { + String vlanTag = ip.getVlanTag(); + return vlanTag != null && vlanTag.startsWith(BroadcastDomainType.Routed.scheme() + "://"); + } + + /** + * IPv6 on a direct routed public range is computed with EUI-64 from the range's subnet and + * the NIC's MAC — the same stateless model guest NICs use (no pool, no reservation, only the + * result on the NIC) — and takes host-route form: a /128 with the shared link-local gateway. + * + * Deliberately not {@code ipv6Service.updateNicIpv6()}: that path is gated on the public + * network offering's internet protocol, and its per-network placeholder reservation would + * hand every SystemVM on the shared Public network the same address. Neither the gate nor + * the reservation has anything to decide here — the MAC makes the address unique. + */ + private void setRoutedRangeIpv6(NicProfile nic, DataCenter dc, PublicIp ip, Network network) { + String ip6Cidr = ip.vlan() != null ? ip.vlan().getIp6Cidr() : null; + if (StringUtils.isBlank(ip6Cidr) || nic.getIPv6Address() != null) { + return; + } + IPv6Address ipv6Address = NetUtils.EUI64Address(ip6Cidr, nic.getMacAddress()); + logger.info("Calculated IPv6 address {} using EUI-64 for direct routed public NIC {}", ipv6Address, nic); + nic.setIPv6Address(ipv6Address.toString()); + nic.setIPv6Cidr(ipv6Address + "/" + NetUtils.IPV6_HOST_PREFIX_LENGTH); + nic.setIPv6Gateway(NetUtils.getIpv6LinkLocalGateway()); + nic.setFormat(AddressFormat.DualStack); + Pair ip6Dns = networkModel.getNetworkIp6Dns(network, dc); + nic.setIPv6Dns1(ip6Dns.first()); + nic.setIPv6Dns2(ip6Dns.second()); + } + @Override public void updateNicProfile(NicProfile profile, Network network) { DataCenter dc = _dcDao.findById(network.getDataCenterId()); diff --git a/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java b/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java index 585b65aa4d94..6ca249e34fc3 100644 --- a/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java +++ b/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java @@ -1451,18 +1451,27 @@ public boolean securityGroupRulesForVmSecIp(long nicId, String secondaryIp, bool // Verify permissions _accountMgr.checkAccess(caller, null, false, vm); + Network network = _networkModel.getNetwork(nic.getNetworkId()); + + // On a Direct Routed network the host needs a route and a static neighbour entry for the + // secondary IP before it is reachable at all. That is independent of security groups, + // which are optional there and which the Instance may not be using, so the agent is told + // either way - otherwise the address would stay dark until the Instance was restarted. + boolean directRouted = Network.GuestType.L3.equals(network.getGuestType()); + // Validate parameters List vmSgGrps = getSecurityGroupsForVm(vmId); + boolean applySecurityGroupRules = true; if (vmSgGrps.isEmpty()) { logger.debug("Vm is not in any Security group "); - return true; - } - - //If network does not support SG service, no need add SG rules for secondary ip - Network network = _networkModel.getNetwork(nic.getNetworkId()); - if (!_networkModel.isSecurityGroupSupportedInNetwork(network)) { + applySecurityGroupRules = false; + } else if (!_networkModel.isSecurityGroupSupportedInNetwork(network)) { logger.debug("Network " + network + " is not enabled with security group service, "+ "so not applying SG rules for secondary ip"); + applySecurityGroupRules = false; + } + + if (!applySecurityGroupRules && !directRouted) { return true; } @@ -1473,7 +1482,8 @@ public boolean securityGroupRulesForVmSecIp(long nicId, String secondaryIp, bool } //create command for the to add ip in ipset and arptables rules - NetworkRulesVmSecondaryIpCommand cmd = new NetworkRulesVmSecondaryIpCommand(vmName, vmMac, secondaryIp, ruleAction); + NetworkRulesVmSecondaryIpCommand cmd = new NetworkRulesVmSecondaryIpCommand(vmName, vmMac, secondaryIp, ruleAction, + directRouted, applySecurityGroupRules); logger.debug("Asking agent to configure rules for vm secondary ip"); Commands cmds = null; diff --git a/server/src/main/resources/META-INF/cloudstack/server-network/spring-server-network-context.xml b/server/src/main/resources/META-INF/cloudstack/server-network/spring-server-network-context.xml index 97ed36e515c7..f3d518cd185c 100644 --- a/server/src/main/resources/META-INF/cloudstack/server-network/spring-server-network-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/server-network/spring-server-network-context.xml @@ -48,6 +48,9 @@ + + + diff --git a/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java b/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java index 9a0b150780e4..da5b740bd7aa 100644 --- a/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java +++ b/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java @@ -1410,4 +1410,82 @@ public void testGetExternalNetworkProviderReturnsNullWhenNoExternalProviders() { mapWithEmptySet.put(Network.Service.Firewall, Collections.emptySet()); Assert.assertNull(ConfigurationManagerImpl.getExternalNetworkProvider(null, mapWithEmptySet)); } + + private Map> validL3ServiceProviderMap() { + Map> map = new HashMap<>(); + map.put(Network.Service.UserData, Collections.singleton(Network.Provider.ConfigDrive)); + map.put(Network.Service.Dns, Collections.singleton(Network.Provider.ConfigDrive)); + map.put(Network.Service.SecurityGroup, Collections.singleton(Network.Provider.SecurityGroupProvider)); + return map; + } + + @Test + public void validateL3NetworkOfferingAcceptsUserDataDnsAndSecurityGroup() { + configurationManagerImplSpy.validateL3NetworkOffering(validL3ServiceProviderMap(), null, false, true, false); + } + + @Test + public void validateL3NetworkOfferingAcceptsOfferingWithoutDns() { + Map> map = validL3ServiceProviderMap(); + map.remove(Network.Service.Dns); + configurationManagerImplSpy.validateL3NetworkOffering(map, null, false, true, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3NetworkOfferingRejectsDhcp() { + Map> map = validL3ServiceProviderMap(); + map.put(Network.Service.Dhcp, Collections.singleton(Network.Provider.ConfigDrive)); + configurationManagerImplSpy.validateL3NetworkOffering(map, null, false, true, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3NetworkOfferingRejectsMissingUserData() { + Map> map = validL3ServiceProviderMap(); + map.remove(Network.Service.UserData); + configurationManagerImplSpy.validateL3NetworkOffering(map, null, false, true, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3NetworkOfferingRejectsUserDataWithoutProvider() { + Map> map = validL3ServiceProviderMap(); + map.put(Network.Service.UserData, Collections.emptySet()); + configurationManagerImplSpy.validateL3NetworkOffering(map, null, false, true, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3NetworkOfferingRejectsNonConfigDriveDns() { + Map> map = validL3ServiceProviderMap(); + map.put(Network.Service.Dns, Collections.singleton(Network.Provider.VirtualRouter)); + configurationManagerImplSpy.validateL3NetworkOffering(map, null, false, true, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3NetworkOfferingRejectsUnsupportedService() { + Map> map = validL3ServiceProviderMap(); + map.put(Network.Service.SourceNat, Collections.singleton(Network.Provider.VirtualRouter)); + configurationManagerImplSpy.validateL3NetworkOffering(map, null, false, true, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3NetworkOfferingRejectsNetworkMode() { + configurationManagerImplSpy.validateL3NetworkOffering(validL3ServiceProviderMap(), NetworkOffering.NetworkMode.ROUTED, false, true, false); + } + + @Test + public void validateL3NetworkOfferingAcceptsSpecifyVlan() { + // specifyVlan selects who picks the routed id: the operator (true, via the vlan + // parameter at network creation) or CloudStack (false, allocated from the ROUTED + // physical network's vnet range). Both are valid offerings. + configurationManagerImplSpy.validateL3NetworkOffering(validL3ServiceProviderMap(), null, true, true, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3NetworkOfferingRejectsWithoutSpecifyIpRanges() { + configurationManagerImplSpy.validateL3NetworkOffering(validL3ServiceProviderMap(), null, false, false, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3NetworkOfferingRejectsVpc() { + configurationManagerImplSpy.validateL3NetworkOffering(validL3ServiceProviderMap(), null, false, true, true); + } } diff --git a/server/src/test/java/com/cloud/network/guru/DirectRoutedNetworkGuruTest.java b/server/src/test/java/com/cloud/network/guru/DirectRoutedNetworkGuruTest.java new file mode 100644 index 000000000000..07ab289110c2 --- /dev/null +++ b/server/src/test/java/com/cloud/network/guru/DirectRoutedNetworkGuruTest.java @@ -0,0 +1,197 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.guru; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.dc.DataCenter.NetworkType; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.network.Network; +import com.cloud.network.Network.GuestType; +import com.cloud.network.Networks.BroadcastDomainType; +import com.cloud.network.Networks.Mode; +import com.cloud.network.Networks.TrafficType; +import com.cloud.network.dao.NetworkVO; +import com.cloud.network.dao.PhysicalNetworkDao; +import com.cloud.network.dao.PhysicalNetworkVO; +import com.cloud.offering.NetworkOffering; +import com.cloud.user.Account; +import com.cloud.vm.NicProfile; + +@RunWith(MockitoJUnitRunner.Silent.class) +public class DirectRoutedNetworkGuruTest { + + @InjectMocks + protected DirectRoutedNetworkGuru guru = new DirectRoutedNetworkGuru(); + + @Mock + DataCenterDao dcDao; + @Mock + PhysicalNetworkDao physicalNetworkDao; + + @Mock + NetworkOffering offering; + @Mock + DataCenterVO dc; + @Mock + PhysicalNetworkVO physicalNetwork; + @Mock + DeploymentPlan plan; + @Mock + Account owner; + + @Before + public void setUp() { + lenient().when(dc.getNetworkType()).thenReturn(NetworkType.Advanced); + lenient().when(offering.getTrafficType()).thenReturn(TrafficType.Guest); + lenient().when(offering.getGuestType()).thenReturn(GuestType.L3); + lenient().when(physicalNetwork.getIsolationMethods()).thenReturn(Arrays.asList("ROUTED")); + lenient().when(plan.getDataCenterId()).thenReturn(1L); + lenient().when(plan.getPhysicalNetworkId()).thenReturn(1L); + lenient().when(dcDao.findById(1L)).thenReturn(dc); + lenient().when(physicalNetworkDao.findById(1L)).thenReturn(physicalNetwork); + } + + @Test + public void canHandleAcceptsL3OnRoutedPhysicalNetwork() { + assertTrue(guru.canHandle(offering, dc, physicalNetwork)); + } + + @Test + public void canHandleRejectsBasicZone() { + when(dc.getNetworkType()).thenReturn(NetworkType.Basic); + assertFalse(guru.canHandle(offering, dc, physicalNetwork)); + } + + @Test + public void canHandleRejectsOtherGuestTypes() { + for (GuestType type : new GuestType[] {GuestType.Shared, GuestType.Isolated, GuestType.L2}) { + when(offering.getGuestType()).thenReturn(type); + assertFalse("guru must not claim guest type " + type, guru.canHandle(offering, dc, physicalNetwork)); + } + } + + @Test + public void canHandleRejectsPhysicalNetworkWithoutRoutedIsolation() { + for (List methods : Arrays.asList(Arrays.asList("VLAN"), Arrays.asList("VXLAN"), Collections.emptyList())) { + when(physicalNetwork.getIsolationMethods()).thenReturn(methods); + assertFalse("guru must not claim a physical network with isolation methods " + methods, guru.canHandle(offering, dc, physicalNetwork)); + } + } + + @Test + public void designProducesRoutedStaticNetwork() { + Network network = guru.design(offering, plan, null, "test", null, owner); + assertNotNull(network); + NetworkVO config = (NetworkVO)network; + assertEquals(BroadcastDomainType.Routed, config.getBroadcastDomainType()); + assertEquals(Mode.Static, config.getMode()); + assertNull(config.getBroadcastUri()); + } + + @Test + public void designCarriesRoutedBroadcastUri() throws URISyntaxException { + Network userSpecified = mock(Network.class); + when(userSpecified.getBroadcastUri()).thenReturn(new URI("routed://5828")); + + Network network = guru.design(offering, plan, userSpecified, "test", null, owner); + assertNotNull(network); + NetworkVO config = (NetworkVO)network; + assertEquals(BroadcastDomainType.Routed, config.getBroadcastDomainType()); + assertEquals("routed://5828", config.getBroadcastUri().toString()); + assertEquals(Network.State.Setup, config.getState()); + } + + @Test(expected = InvalidParameterValueException.class) + public void designRejectsNonRoutedBroadcastUri() throws URISyntaxException { + Network userSpecified = mock(Network.class); + when(userSpecified.getBroadcastUri()).thenReturn(new URI("vlan://5828")); + guru.design(offering, plan, userSpecified, "test", null, owner); + } + + @Test(expected = InvalidParameterValueException.class) + public void designRejectsVpc() { + guru.design(offering, plan, null, "test", 42L, owner); + } + + @Test + public void designReturnsNullForOtherGuestTypes() { + when(offering.getGuestType()).thenReturn(GuestType.Shared); + assertNull(guru.design(offering, plan, null, "test", null, owner)); + } + + @Test + public void applyDirectRoutedAddressingForcesHostRouteForm() { + NicProfile nic = new NicProfile(); + nic.setIPv4Address("203.0.113.55"); + nic.setIPv4Netmask("255.255.255.0"); + nic.setIPv4Gateway("203.0.113.1"); + nic.setIPv6Address("2001:db8:1::55"); + nic.setIPv6Cidr("2001:db8:1::/64"); + nic.setIPv6Gateway("2001:db8:1::1"); + + guru.applyDirectRoutedAddressing(nic); + + assertEquals("255.255.255.255", nic.getIPv4Netmask()); + assertEquals("169.254.0.1", nic.getIPv4Gateway()); + assertEquals("2001:db8:1::55/128", nic.getIPv6Cidr()); + assertEquals("fe80::1", nic.getIPv6Gateway()); + } + + @Test + public void applyDirectRoutedAddressingLeavesAbsentFamiliesAlone() { + NicProfile nic = new NicProfile(); + nic.setIPv4Address("203.0.113.55"); + nic.setIPv4Netmask("255.255.255.0"); + nic.setIPv4Gateway("203.0.113.1"); + + guru.applyDirectRoutedAddressing(nic); + + assertEquals("255.255.255.255", nic.getIPv4Netmask()); + assertEquals("169.254.0.1", nic.getIPv4Gateway()); + assertNull(nic.getIPv6Cidr()); + assertNull(nic.getIPv6Gateway()); + } + + @Test + public void applyDirectRoutedAddressingIsNullSafe() { + guru.applyDirectRoutedAddressing(null); + } +} diff --git a/server/src/test/java/com/cloud/network/guru/PublicNetworkGuruTest.java b/server/src/test/java/com/cloud/network/guru/PublicNetworkGuruTest.java new file mode 100644 index 000000000000..b6f38a2853da --- /dev/null +++ b/server/src/test/java/com/cloud/network/guru/PublicNetworkGuruTest.java @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.guru; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.when; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.dc.DataCenter; +import com.cloud.dc.Vlan.VlanType; +import com.cloud.dc.VlanVO; +import com.cloud.network.IpAddressManager; +import com.cloud.network.Ipv6Service; +import com.cloud.network.Network; +import com.cloud.network.NetworkModel; +import com.cloud.network.Networks.AddressFormat; +import com.cloud.network.Networks.BroadcastDomainType; +import com.cloud.network.addr.PublicIp; +import com.cloud.network.dao.IPAddressVO; +import com.cloud.user.Account; +import com.cloud.utils.Pair; +import com.cloud.utils.net.Ip; +import com.cloud.utils.net.NetUtils; +import com.cloud.vm.Nic.ReservationStrategy; +import com.cloud.vm.NicProfile; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachineProfile; + +@RunWith(MockitoJUnitRunner.Silent.class) +public class PublicNetworkGuruTest { + + @InjectMocks + protected PublicNetworkGuru guru = new PublicNetworkGuru(); + + @Mock + IpAddressManager ipAddrMgr; + @Mock + Ipv6Service ipv6Service; + @Mock + NetworkModel networkModel; + + @Mock + DataCenter dc; + @Mock + Network network; + @Mock + VirtualMachineProfile vm; + @Mock + Account owner; + + private static final long MAC = 0x1e003c000102L; + + @Before + public void setUp() { + lenient().when(vm.getType()).thenReturn(VirtualMachine.Type.ConsoleProxy); + lenient().when(vm.getOwner()).thenReturn(owner); + lenient().when(networkModel.getNetworkIp4Dns(network, dc)).thenReturn(new Pair<>(null, null)); + lenient().when(networkModel.getNetworkIp6Dns(network, dc)).thenReturn(new Pair<>(null, null)); + } + + private PublicIp publicIp(String vlanTag, String ip6Cidr) { + IPAddressVO addr = new IPAddressVO(new Ip("2.57.59.68"), 1L, 5L, 1L, false); + VlanVO vlan = new VlanVO(VlanType.VirtualNetwork, vlanTag, "2.57.59.65", "255.255.255.192", 1L, + "2.57.59.66-2.57.59.94", 200L, 200L, ip6Cidr == null ? null : "2a00:f10:402:2::1", ip6Cidr, null); + return new PublicIp(addr, vlan, MAC); + } + + private NicProfile allocateOn(PublicIp ip) throws Exception { + when(ipAddrMgr.assignPublicIpAddress(anyLong(), any(), any(), any(), any(), any(), anyBoolean(), anyBoolean())).thenReturn(ip); + NicProfile nic = new NicProfile(ReservationStrategy.Create, null, null, null, null); + guru.getIp(nic, dc, vm, network); + return nic; + } + + // Regression: setRoutedRangeIpv6() runs inside getIp()'s routed branch and needs the NIC's + // MAC (EUI-64) — the MAC must be on the profile before that branch, not after it (NPE), and + // the DualStack format it sets must not be clobbered back to Ip4 afterwards. + @Test + public void getIpOnRoutedRangeComputesEui64Ipv6FromNicMac() throws Exception { + PublicIp ip = publicIp("routed://534", "2a00:f10:402:2::/64"); + NicProfile nic = allocateOn(ip); + + assertEquals(ip.getMacAddress(), nic.getMacAddress()); + assertEquals("2.57.59.68", nic.getIPv4Address()); + assertEquals(NetUtils.IPV4_HOST_NETMASK, nic.getIPv4Netmask()); + assertEquals(NetUtils.getLinkLocalGateway(), nic.getIPv4Gateway()); + assertEquals("routed://534", nic.getBroadCastUri().toString()); + assertEquals(BroadcastDomainType.Routed, nic.getBroadcastType()); + assertEquals(NetUtils.EUI64Address("2a00:f10:402:2::/64", ip.getMacAddress()).toString(), nic.getIPv6Address()); + assertEquals(NetUtils.getIpv6LinkLocalGateway(), nic.getIPv6Gateway()); + assertEquals(AddressFormat.DualStack, nic.getFormat()); + } + + @Test + public void getIpOnVlanRangeKeepsClassicShape() throws Exception { + when(network.getBroadcastDomainType()).thenReturn(BroadcastDomainType.Vlan); + PublicIp ip = publicIp("50", null); + NicProfile nic = allocateOn(ip); + + assertEquals(ip.getMacAddress(), nic.getMacAddress()); + assertEquals("2.57.59.68", nic.getIPv4Address()); + assertEquals("255.255.255.192", nic.getIPv4Netmask()); + assertEquals("2.57.59.65", nic.getIPv4Gateway()); + assertEquals("vlan://50", nic.getBroadCastUri().toString()); + assertNull(nic.getIPv6Address()); + assertEquals(AddressFormat.Ip4, nic.getFormat()); + } +} diff --git a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java index 1239dd23e7ec..97710768a7dc 100644 --- a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java +++ b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java @@ -1049,6 +1049,11 @@ public boolean isSharedNetworkWithoutSpecifyVlan(NetworkOffering offering) { return false; } + @Override + public boolean isL3NetworkWithoutSpecifyVlan(NetworkOffering offering) { + return false; + } + @Override public IpAddress updateIP(Long id, String customId, Boolean displayIp) { // TODO Auto-generated method stub diff --git a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java index 9d4c73111595..084166dce083 100644 --- a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java +++ b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java @@ -1194,8 +1194,16 @@ public boolean finalizeVirtualMachineProfile(VirtualMachineProfile profile, Depl buf.append(" eth").append(deviceId).append("mask=").append(nic.getIPv4Netmask()); } + if (nic.getIPv6Address() != null) { + buf.append(" eth").append(deviceId).append("ip6=").append(nic.getIPv6Address()); + buf.append(" eth").append(deviceId).append("ip6prelen=").append(NetUtils.getIp6CidrSize(nic.getIPv6Cidr())); + } + if (nic.isDefaultNic()) { buf.append(" gateway=").append(nic.getIPv4Gateway()); + if (nic.getIPv6Gateway() != null) { + buf.append(" ip6gateway=").append(nic.getIPv6Gateway()); + } } if (nic.getTrafficType() == TrafficType.Management) { String mgmt_cidr = _configDao.getValue(Config.ManagementNetwork.key()); diff --git a/systemvm/debian/opt/cloud/bin/setup/common.sh b/systemvm/debian/opt/cloud/bin/setup/common.sh index ef1576ab588c..2cf91201f45e 100755 --- a/systemvm/debian/opt/cloud/bin/setup/common.sh +++ b/systemvm/debian/opt/cloud/bin/setup/common.sh @@ -396,7 +396,26 @@ setup_common() { gwdev="eth0" fi - ip route add default via $GW dev $gwdev + onlink="" + case "$GW" in + 169.254.*) + # Direct routed public interface: the address is a /32, so the shared + # link-local gateway lies outside it and the kernel rejects the route + # unless it is marked on-link. Same rule cloud-init applies for guest + # Instances; systemvms have no cloud-init, so it is applied here. + onlink="onlink" + ;; + esac + ip route add default via $GW dev $gwdev $onlink + + # The IPv6 default route is installed statically when a gateway was passed + # down: on a direct routed bridge no router advertisement ever arrives, so + # relying on accept_ra would leave the systemvm without v6 connectivity. + # A link-local gateway (fe80::1) needs the dev and is on-link by definition. + if [ -n "$IP6GW" ] + then + ip -6 route replace default via $IP6GW dev $gwdev + fi fi # Workaround to activate vSwitch under VMware diff --git a/test/integration/smoke/test_l3_networks.py b/test/integration/smoke/test_l3_networks.py new file mode 100644 index 000000000000..275d6180c165 --- /dev/null +++ b/test/integration/smoke/test_l3_networks.py @@ -0,0 +1,237 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" Tests for L3 (Direct Routed) guest networks: the hypervisor routes a public IPv4/IPv6 + address directly to the Instance - no Virtual Router, no NAT, no DHCP. Instances receive + a /32 (or /128) with the shared link-local gateway, delivered via ConfigDrive. +""" + +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.cloudstackException import CloudstackAPIException +from marvin.lib.base import (Account, + Network, + NetworkOffering, + PhysicalNetwork, + ServiceOffering, + VirtualMachine, + Zone) +from marvin.lib.common import (get_domain, + get_template, + get_zone) +from marvin.lib.utils import cleanup_resources +from nose.plugins.attrib import attr + + +class TestL3Networks(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + testClient = super(TestL3Networks, cls).getClsTestClient() + cls.apiclient = testClient.getApiClient() + cls.services = testClient.getParsedTestDataConfig() + + cls.domain = get_domain(cls.apiclient) + cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests()) + cls.template = get_template(cls.apiclient, cls.zone.id, cls.services["ostype"]) + + cls._cleanup = [] + cls.hypervisor = testClient.getHypervisorInfo() + cls.skip = False + + zone = Zone(cls.zone.__dict__) + if zone.networktype != 'Advanced': + cls.skip = True + return + + cls.services["virtual_machine"]["zoneid"] = cls.zone.id + cls.services["virtual_machine"]["template"] = cls.template.id + + # The network operator's opt-in: L3 networks live on a dedicated physical network + # with isolation method ROUTED. Its "vlan" range is the routed-id pool for networks + # whose offering does not carry specifyVlan; its tag steers L3 offerings to it. + cls.physical_network = PhysicalNetwork.create( + cls.apiclient, + {"name": "l3-direct-routed"}, + zoneid=cls.zone.id, + isolationmethods="ROUTED" + ) + cls._cleanup.append(cls.physical_network) + cls.physical_network.addTrafficType(cls.apiclient, "Guest") + cls.physical_network.update( + cls.apiclient, + vlan="5800-5899", + tags="l3routed", + state="Enabled" + ) + + cls.service_offering = ServiceOffering.create( + cls.apiclient, + cls.services["service_offering"] + ) + cls._cleanup.append(cls.service_offering) + + cls.network_offering = NetworkOffering.create( + cls.apiclient, + cls.services["l3_network_offering"] + ) + cls._cleanup.append(cls.network_offering) + cls.network_offering.update(cls.apiclient, state='Enabled') + + cls.network_offering_specifyid = NetworkOffering.create( + cls.apiclient, + cls.services["l3_network_offering_specifyid"] + ) + cls._cleanup.append(cls.network_offering_specifyid) + cls.network_offering_specifyid.update(cls.apiclient, state='Enabled') + + cls.account = Account.create( + cls.apiclient, + cls.services["account"], + admin=True, + domainid=cls.domain.id + ) + cls._cleanup.append(cls.account) + + @classmethod + def tearDownClass(cls): + try: + cleanup_resources(cls.apiclient, reversed(cls._cleanup)) + except Exception as e: + raise Exception("Warning: Exception during class cleanup : %s" % e) + + def setUp(self): + if self.skip: + self.skipTest("L3 networks require an Advanced zone, skipping") + self.cleanup = [] + + def tearDown(self): + try: + cleanup_resources(self.apiclient, reversed(self.cleanup)) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + def create_l3_network(self, startip="203.0.113.10", endip="203.0.113.50"): + services = dict(self.services["l3_network"]) + services["startip"] = startip + services["endip"] = endip + return Network.create( + self.apiclient, + services, + zoneid=self.zone.id, + networkofferingid=self.network_offering.id, + accountid=self.account.name, + domainid=self.account.domainid + ) + + @attr(tags=["advanced", "smoke"], required_hardware="false") + def test_01_create_l3_network(self): + """ An L3 network is created like a Shared network: with a subnet. The subnet is an + allocation pool routed to the hypervisors, not a broadcast domain. The network + carries a routed:// broadcast domain - here allocated from the ROUTED + physical network's range - whose id names the bridge (brdr-) on the hosts. """ + network = self.create_l3_network() + self.cleanup.append(network) + + self.assertEqual(network.type, "L3", "network type should be L3") + self.assertEqual(network.broadcastdomaintype, "Routed", "an L3 network has a routed broadcast domain") + self.assertTrue(network.broadcasturi.startswith("routed://"), + "the broadcast URI must be routed://, got %s" % network.broadcasturi) + allocated_id = int(network.broadcasturi.replace("routed://", "")) + self.assertTrue(5800 <= allocated_id <= 5899, + "the routed id must come from the physical network's range, got %d" % allocated_id) + self.assertIn(network.state, ["Setup", "Allocated"], "unexpected network state") + + @attr(tags=["advanced", "smoke"], required_hardware="false") + def test_01b_create_l3_network_with_operator_specified_id(self): + """ With a specifyVlan offering the operator picks the routed id at creation via the + vlan parameter, so bridge names (brdr-) are plannable before the network + exists. The id must lie outside the physical network's dynamic range. """ + services = dict(self.services["l3_network"]) + network = Network.create( + self.apiclient, + services, + zoneid=self.zone.id, + networkofferingid=self.network_offering_specifyid.id, + accountid=self.account.name, + domainid=self.account.domainid, + vlan="5928" + ) + self.cleanup.append(network) + self.assertEqual(network.broadcasturi, "routed://5928", + "the operator-specified routed id must be carried verbatim, got %s" % network.broadcasturi) + + @attr(tags=["advanced", "smoke"], required_hardware="false") + def test_02_deploy_vm_in_l3_network(self): + """ The Instance's NIC carries its address as a host route: a /32 with the shared, + host-independent link-local gateway. The subnet's own gateway is never used. """ + network = self.create_l3_network() + self.cleanup.append(network) + + virtual_machine = VirtualMachine.create( + self.apiclient, + self.services["virtual_machine"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.service_offering.id, + networkids=[network.id] + ) + self.cleanup.append(virtual_machine) + + self.assertEqual(virtual_machine.state, "Running") + nic = virtual_machine.nic[0] + self.assertEqual(nic.netmask, "255.255.255.255", "an L3 NIC address is a host route (/32)") + self.assertEqual(nic.gateway, "169.254.0.1", "an L3 NIC uses the shared link-local gateway") + self.assertTrue(nic.ipaddress.startswith("203.0.113."), "the address must come from the network's subnet") + + @attr(tags=["advanced", "smoke"], required_hardware="false") + def test_03_l3_offering_rejects_dhcp(self): + """ DHCP is not supported and not needed on L3 networks: ConfigDrive carries the + address, netmask, gateway and routes, so DHCP would have nothing left to hand out. """ + services = dict(self.services["l3_network_offering"]) + services["name"] = "Test L3 offering with Dhcp - must fail" + services["supportedservices"] = "UserData,Dns,Dhcp" + services["serviceProviderList"] = { + "UserData": "ConfigDrive", + "Dns": "ConfigDrive", + "Dhcp": "ConfigDrive" + } + with self.assertRaises(Exception): + NetworkOffering.create(self.apiclient, services) + + @attr(tags=["advanced", "smoke"], required_hardware="false") + def test_04_l3_offering_requires_userdata(self): + """ UserData via ConfigDrive is mandatory: it is the only channel that carries the + Instance's network configuration. """ + services = dict(self.services["l3_network_offering"]) + services["name"] = "Test L3 offering without UserData - must fail" + services["supportedservices"] = "Dns" + services["serviceProviderList"] = {"Dns": "ConfigDrive"} + with self.assertRaises(Exception): + NetworkOffering.create(self.apiclient, services) + + @attr(tags=["advanced", "smoke"], required_hardware="false") + def test_05_l3_subnets_may_not_overlap_zone_wide(self): + """ All L3 subnets share one host routing table and one routing fabric, so an overlap + is an address conflict, not a policy preference. The check is zone wide. """ + network = self.create_l3_network(startip="203.0.113.10", endip="203.0.113.30") + self.cleanup.append(network) + + try: + overlapping = self.create_l3_network(startip="203.0.113.20", endip="203.0.113.40") + self.cleanup.append(overlapping) + self.fail("creating an L3 network overlapping another must fail") + except (CloudstackAPIException, Exception): + pass diff --git a/tools/marvin/marvin/config/test_data.py b/tools/marvin/marvin/config/test_data.py index e3d4022cf0f9..75fb25dc2160 100644 --- a/tools/marvin/marvin/config/test_data.py +++ b/tools/marvin/marvin/config/test_data.py @@ -221,6 +221,44 @@ "endip": "172.16.15.41", "acltype": "Account" }, + "l3_network_offering": { + "name": "Test L3 Direct Routed - Network offering", + "displaytext": "Test L3 Direct Routed - Network offering", + "guestiptype": "L3", + "supportedservices": "UserData,Dns", + "specifyIpRanges": "True", + "specifyVlan": "False", + "traffictype": "GUEST", + "availability": "Optional", + "tags": "l3routed", + "serviceProviderList": { + "UserData": "ConfigDrive", + "Dns": "ConfigDrive" + } + }, + "l3_network_offering_specifyid": { + "name": "Test L3 Direct Routed - Network offering, operator-specified id", + "displaytext": "Test L3 Direct Routed - Network offering, operator-specified id", + "guestiptype": "L3", + "supportedservices": "UserData,Dns", + "specifyIpRanges": "True", + "specifyVlan": "True", + "traffictype": "GUEST", + "availability": "Optional", + "tags": "l3routed", + "serviceProviderList": { + "UserData": "ConfigDrive", + "Dns": "ConfigDrive" + } + }, + "l3_network": { + "name": "Test L3 Direct Routed Network", + "displaytext": "Test L3 Direct Routed Network", + "gateway": "203.0.113.1", + "netmask": "255.255.255.0", + "startip": "203.0.113.10", + "endip": "203.0.113.50" + }, "l2-network_offering": { "name": "Test L2 - Network offering", "displaytext": "Test L2 - Network offering", diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 83f7d182285a..e1715fcc7dbf 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -1554,6 +1554,7 @@ "label.kvm": "KVM", "label.kvmnetworklabel": "KVM traffic label", "label.l2": "L2", +"label.l3": "L3 (Direct Routed)", "label.l2gatewayserviceuuid": "L2 Gateway Service UUID", "label.l3gatewayserviceuuid": "L3 Gateway Service UUID", "label.label": "Label", @@ -2303,6 +2304,7 @@ "label.rootdisksize": "Root disk size (GB)", "label.routenexthop": "Route next hop", "label.routenexthoptype": "Route next hop type", +"label.routedid": "Routed ID", "label.routeprefix": "Route prefix", "label.router.health.check.last.updated": "Last updated", "label.router.health.check.name": "Check name", @@ -3690,6 +3692,7 @@ "message.error.required.input": "Please enter input", "message.error.reset.config": "Unable to reset config to default value", "message.error.retrieve.kubeconfig": "Unable to retrieve Kubernetes Cluster config", +"message.error.routedid": "Please enter the Routed ID for this network; the selected network offering requires the operator to choose it", "message.error.routing.policy.term": "Community need to have the following format number:number", "message.error.s3nfs.path": "Please enter S3 NFS Path", "message.error.s3nfs.server": "Please enter S3 NFS Server", @@ -3821,6 +3824,9 @@ "message.ip.v6.prefix.delete": "IPv6 prefix deleted", "message.iso.arch": "Please select an ISO architecture", "message.iso.desc": "Disc image containing data or bootable media for OS.", +"message.isolationmethod.routed.description": "Direct Routed (L3): public IPv4/IPv6 subnets are routed directly to Instances over a per-network bridge on every host. No VLAN or tunnel on the wire, no Virtual Router and no DHCP - Instances receive their address, routes and DNS through ConfigDrive. Only guest networks created from an L3 network offering (such as DefaultL3NetworkOffering) can be created on this physical network. KVM only.", +"message.isolationmethod.vlan.description": "Guest networks are isolated from each other by 802.1Q VLAN tags on the physical network.", +"message.isolationmethod.vxlan.description": "Guest networks are isolated from each other by VXLAN tunnels (one VNI per network) carried over the physical network.", "message.kubernetes.cluster.add.nodes": "Please confirm that you want to add the following nodes to the cluster", "message.kubernetes.cluster.delete": "Please confirm that you want to destroy the Cluster.", "message.kubeconfig.cluster.not.available": "Kubernetes Cluster kubeconfig not available currently.", @@ -3907,6 +3913,9 @@ "message.note.about.keypair.permissions.title": "Note about API key pair rule permissions", "message.note.about.keypair.permissions.body": "During the creation of API key pairs, it is possible to define a corresponding set of rule permissions. If a rule set is defined, the API key pair will only have access to APIs for which access has been explicitly granted (i.e., APIs whose corresponding rules are marked as allowed). On the other hand, if no rule set is specified, the API key pair permissions will follow and adapt to the permission set of the user's account role.", "message.offering.internet.protocol.warning": "WARNING: IPv6 supported Networks use static routing and will require upstream routes to be configured manually.", +"message.create.l3.network": "The hypervisor routes a public IPv4/IPv6 address directly to each Instance on this network: no Virtual Router, no NAT and no DHCP. The subnet is an allocation pool routed to the hypervisors; its gateway is required but never used, since Instances always use the shared link-local gateway. Instances receive their configuration via ConfigDrive.", +"message.success.create.l3.network": "Successfully created L3 (Direct Routed) network", +"message.offering.l3": "The hypervisor routes a public IPv4/IPv6 address directly to each Instance: no Virtual Router, no NAT and no DHCP. UserData via ConfigDrive is always enabled, since it is the only channel that carries the Instance's network configuration. DNS is strongly recommended; Security Groups are optional.", "message.offering.ipv6.warning": "Please refer documentation for creating IPv6 enabled Network/VPC offering IPv6 support in CloudStack - Isolated Networks and VPC Network Tiers", "message.oobm.configured": "Successfully configured out-of-band management for host", "message.ovf.configurations": "OVF configurations available for the selected appliance. Please select the desired value. Incompatible compute offerings will get disabled.", @@ -3945,6 +3954,7 @@ "message.releasing.dedicated.zone": "Releasing dedicated Zone...", "message.remote.access.vpn.iprange.description": "The range of IP addresses to allocate to VPN clients. The first IP in the range will be taken by the VPN server. (Optional)", "message.remove.annotation": "Are you sure you want to delete the comment?", +"message.routedid.description": "Numeric ID for this Direct Routed (L3) network, chosen by the operator. The ID names the network's bridge on every KVM host (brdr-) and must be unique within the zone; it is only a label, nothing appears on the wire. Choosing it yourself makes the bridge name plannable, so host routing policy can be prepared before the network exists. Network offerings without 'specify VLAN' allocate the ID automatically from the ROUTED physical network's range instead.", "message.remove.egress.rule.failed": "Removing egress rule failed", "message.remove.egress.rule.processing": "Deleting egress rule...", "message.remove.failed": "Removing failed", diff --git a/ui/src/config/section/infra/phynetworks.js b/ui/src/config/section/infra/phynetworks.js index 0863eff6ec0b..b705c1fb55b9 100644 --- a/ui/src/config/section/infra/phynetworks.js +++ b/ui/src/config/section/infra/phynetworks.js @@ -57,7 +57,7 @@ export default { args: ['name', 'zoneid', 'isolationmethods', 'vlan', 'tags', 'networkspeed', 'broadcastdomainrange'], mapping: { isolationmethods: { - options: ['VLAN', 'VXLAN', 'GRE', 'STT', 'BCF_SEGMENT', 'SSP', 'ODL', 'L3VPN', 'VCS', 'NSX', 'NETRIS'] + options: ['VLAN', 'VXLAN', 'ROUTED', 'GRE', 'STT', 'BCF_SEGMENT', 'SSP', 'ODL', 'L3VPN', 'VCS', 'NSX', 'NETRIS'] } } }, diff --git a/ui/src/views/infra/zone/PhysicalNetworksTab.vue b/ui/src/views/infra/zone/PhysicalNetworksTab.vue index cccb8719805f..a7c9fee7c3a5 100644 --- a/ui/src/views/infra/zone/PhysicalNetworksTab.vue +++ b/ui/src/views/infra/zone/PhysicalNetworksTab.vue @@ -109,7 +109,7 @@ }" v-focus="true" :placeholder="apiParams.isolationmethods.description"> - {{ i }} + {{ i }} @@ -200,10 +200,18 @@ export default { }, computed: { isolationMethods () { - return ['VLAN', 'VXLAN', 'GRE', 'STT', 'BCF_SEGMENT', 'SSP', 'ODL', 'L3VPN', 'VCS', 'NSX', 'NETRIS'] + return ['VLAN', 'VXLAN', 'ROUTED', 'GRE', 'STT', 'BCF_SEGMENT', 'SSP', 'ODL', 'L3VPN', 'VCS', 'NSX', 'NETRIS'] } }, methods: { + isolationMethodDescription (method) { + const keys = { + VLAN: 'message.isolationmethod.vlan.description', + VXLAN: 'message.isolationmethod.vxlan.description', + ROUTED: 'message.isolationmethod.routed.description' + } + return method in keys ? this.$t(keys[method]) : method + }, fetchData () { this.fetchLoading = true getAPI('listPhysicalNetworks', { zoneid: this.resource.id }).then(json => { diff --git a/ui/src/views/infra/zone/ZoneWizardPhysicalNetworkSetupStep.vue b/ui/src/views/infra/zone/ZoneWizardPhysicalNetworkSetupStep.vue index 88f681de3796..b8050ffd8326 100644 --- a/ui/src/views/infra/zone/ZoneWizardPhysicalNetworkSetupStep.vue +++ b/ui/src/views/infra/zone/ZoneWizardPhysicalNetworkSetupStep.vue @@ -56,8 +56,9 @@ :filterOption="(input, option) => { return option.value.toLowerCase().indexOf(input.toLowerCase()) >= 0 }" > - VLAN - VXLAN + VLAN + VXLAN + ROUTED GRE STT BCF_SEGMENT diff --git a/ui/src/views/network/CreateL3NetworkForm.vue b/ui/src/views/network/CreateL3NetworkForm.vue new file mode 100644 index 000000000000..7e2fd2a76a9a --- /dev/null +++ b/ui/src/views/network/CreateL3NetworkForm.vue @@ -0,0 +1,423 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + diff --git a/ui/src/views/network/CreateNetwork.vue b/ui/src/views/network/CreateNetwork.vue index 46ed9d2e039f..385bb1bba98e 100644 --- a/ui/src/views/network/CreateNetwork.vue +++ b/ui/src/views/network/CreateNetwork.vue @@ -34,6 +34,13 @@ @refresh-data="refreshParent" @refresh="handleRefresh"/> + + + {{ $t('label.l2') }} + + {{ $t('label.l3') }} + {{ $t('label.shared') }} @@ -93,7 +96,7 @@ - +