From 6174c777947a5fa5cfe68ad3b218c1842fad2efb Mon Sep 17 00:00:00 2001 From: Ramgopal Nagaboina Date: Wed, 2 Sep 2026 10:56:43 -0400 Subject: [PATCH] ha: reserve cluster capacity for HA failover Adds cluster.ha.failover.capacity.reservethreshold (default 1.0 = disabled). When set below 1.0, FirstFitPlanner also excludes any cluster whose allocated+requested cpu/memory would cross the reserve from new deployments, so headroom stays free for HA-triggered restarts. The threshold is per-cluster scope aware. --- .../deploy/DeploymentClusterPlanner.java | 10 +++ .../com/cloud/deploy/FirstFitPlanner.java | 16 ++++- .../com/cloud/vm/FirstFitPlannerTest.java | 68 +++++++++++++++++++ 3 files changed, 93 insertions(+), 1 deletion(-) diff --git a/api/src/main/java/com/cloud/deploy/DeploymentClusterPlanner.java b/api/src/main/java/com/cloud/deploy/DeploymentClusterPlanner.java index 9471c3d5c84c..66bddd628a53 100644 --- a/api/src/main/java/com/cloud/deploy/DeploymentClusterPlanner.java +++ b/api/src/main/java/com/cloud/deploy/DeploymentClusterPlanner.java @@ -30,6 +30,7 @@ public interface DeploymentClusterPlanner extends DeploymentPlanner { static final String ClusterCPUCapacityDisableThresholdCK = "cluster.cpu.allocated.capacity.disablethreshold"; static final String ClusterMemoryCapacityDisableThresholdCK = "cluster.memory.allocated.capacity.disablethreshold"; static final String ClusterThresholdEnabledCK = "cluster.threshold.enabled"; + static final String ClusterHAFailoverReserveThresholdCK = "cluster.ha.failover.capacity.reservethreshold"; static final ConfigKey ClusterCPUCapacityDisableThreshold = new ConfigKey( @@ -56,6 +57,15 @@ public interface DeploymentClusterPlanner extends DeploymentPlanner { "Enable/Disable cluster thresholds. If disabled, an instance can start in a cluster even though the threshold may be crossed.", false, ConfigKey.Scope.Global); + // Reserve spare cluster capacity for HA failover. + static final ConfigKey ClusterHAFailoverReserveThreshold = + new ConfigKey( + Float.class, + ClusterHAFailoverReserveThresholdCK, + "Alert", + "1.0", + "Fraction (0-1) of a cluster's CPU/memory allocation above which the allocator stops placing new instances in the cluster, reserving the remaining capacity for HA failover restarts. 1.0 (the default) disables the reservation. For example, 0.8 keeps about 20% of the cluster's CPU and memory free for HA-triggered restarts, which are not subject to this reserve. Set it below the corresponding cluster.*.allocated.capacity.disablethreshold to take effect.", + true, ConfigKey.Scope.Cluster, null); static final ConfigKey VmAllocationAlgorithm = new ConfigKey<>( String.class, diff --git a/server/src/main/java/com/cloud/deploy/FirstFitPlanner.java b/server/src/main/java/com/cloud/deploy/FirstFitPlanner.java index 3aab852ba7fc..b15d81e31a95 100644 --- a/server/src/main/java/com/cloud/deploy/FirstFitPlanner.java +++ b/server/src/main/java/com/cloud/deploy/FirstFitPlanner.java @@ -388,6 +388,20 @@ protected void removeClustersCrossingThreshold(List clusterListForVmAlloca logger.warn(warnMessageForClusterReachedCapacityThreshold); } + // Exclude clusters that would cross the HA failover reserve threshold, reserving + // capacity for HA-triggered restarts. Threshold is Cluster-scoped, resolved per-cluster by the DAO. + long haRequested = (capacity == Capacity.CAPACITY_TYPE_CPU) ? cpu_requested : ram_requested; + List clustersCrossingHAReserve = capacityDao.listClustersCrossingThreshold( + capacity, plan.getDataCenterId(), ClusterHAFailoverReserveThreshold.key(), haRequested); + if (clustersCrossingHAReserve != null && !clustersCrossingHAReserve.isEmpty()) { + avoid.addClusterList(clustersCrossingHAReserve); + clusterListForVmAllocation.removeAll(clustersCrossingHAReserve); + logger.warn(String.format( + "HA admission control: excluding clusters %s from new deployments; their %s allocation would cross the HA failover reserve threshold [%s], reserving capacity for HA failover", + clustersCrossingHAReserve, CapacityVO.getCapacityName(capacity), + ClusterHAFailoverReserveThreshold.key())); + } + } } @@ -694,6 +708,6 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { - return new ConfigKey[] {ClusterCPUCapacityDisableThreshold, ClusterMemoryCapacityDisableThreshold, ClusterThresholdEnabled, VmAllocationAlgorithm}; + return new ConfigKey[] {ClusterCPUCapacityDisableThreshold, ClusterMemoryCapacityDisableThreshold, ClusterThresholdEnabled, ClusterHAFailoverReserveThreshold, VmAllocationAlgorithm}; } } diff --git a/server/src/test/java/com/cloud/vm/FirstFitPlannerTest.java b/server/src/test/java/com/cloud/vm/FirstFitPlannerTest.java index 5b877cc66169..7d95a1d004e6 100644 --- a/server/src/test/java/com/cloud/vm/FirstFitPlannerTest.java +++ b/server/src/test/java/com/cloud/vm/FirstFitPlannerTest.java @@ -246,6 +246,74 @@ public void checkClusterReorderingForStartVMWithThresholdCheckDisabled() throws assertTrue("Reordered cluster list does not have clusters exceeding threshold", (clusterList.containsAll(clustersCrossingThreshold))); } + // VMware HA admission-control parity: with the HA failover reserve threshold set below + // 1.0, a cluster reported as crossing that reserve is excluded from a NEW deployment so headroom + // stays free for HA restarts. Uses config-key strings (not the new ConfigKey symbol) so the test + // fails-before (old code has no HA reserve) and passes-after. + @Test + public void checkHAFailoverReserveExcludesClusterOnDeploy() throws InsufficientServerCapacityException { + VirtualMachineProfileImpl vmProfile = mock(VirtualMachineProfileImpl.class); + DataCenterDeployment plan = mock(DataCenterDeployment.class); + ExcludeList avoids = mock(ExcludeList.class); + initializeForTest(vmProfile, plan, avoids); + List haReserved = initializeForHAFailoverReserve(); + + Map details = new HashMap(); + details.put("deployvm", "true"); + when(vmDetailsDao.listDetailsKeyPairs(vmProfile.getVirtualMachine().getId())).thenReturn(details); + + try { + List clusterList = planner.orderClusters(vmProfile, plan, avoids); + assertTrue("HA failover reserve must exclude the reserved cluster from deployment", (!clusterList.containsAll(haReserved))); + } finally { + // shared Spring mock: don't leak the deployvm detail (keyed by the default vm id) into other tests + Mockito.reset(vmDetailsDao); + } + } + + @Test + public void checkHAFailoverReserveDisabledByDefault() throws InsufficientServerCapacityException { + VirtualMachineProfileImpl vmProfile = mock(VirtualMachineProfileImpl.class); + DataCenterDeployment plan = mock(DataCenterDeployment.class); + ExcludeList avoids = mock(ExcludeList.class); + initializeForTest(vmProfile, plan, avoids); + // keep the (cached, non-dynamic) threshold flag consistent with the other threshold tests + when(configDepot.getConfigStringValue("cluster.threshold.enabled", ConfigKey.Scope.Global, null)).thenReturn(Boolean.FALSE.toString()); + // no cluster crosses any threshold; HA reserve is off by default (1.0) so cluster 3 is not excluded + when(capacityDao.listClustersCrossingThreshold(Mockito.anyShort(), Mockito.anyLong(), + Mockito.anyString(), Mockito.anyLong())).thenReturn(new ArrayList()); + + Map details = new HashMap(); + details.put("deployvm", "true"); + when(vmDetailsDao.listDetailsKeyPairs(vmProfile.getVirtualMachine().getId())).thenReturn(details); + + try { + List clusterList = planner.orderClusters(vmProfile, plan, avoids); + assertTrue("With HA reserve disabled (default), cluster 3 is not excluded", clusterList.contains(3L)); + } finally { + // shared Spring mock: don't leak the deployvm detail (keyed by the default vm id) into other tests + Mockito.reset(vmDetailsDao); + } + } + + private List initializeForHAFailoverReserve() { + // keep the (cached, non-dynamic) threshold flag consistent with the other threshold tests + when(configDepot.getConfigStringValue("cluster.threshold.enabled", ConfigKey.Scope.Global, null)).thenReturn(Boolean.FALSE.toString()); + // NOTE (regression guard for the per-cluster-scope bug the review caught): deliberately leave + // the GLOBAL HA reserve threshold at its 1.0 default and let only the DAO report cluster 3 as + // crossing the (per-cluster) reserve. This passes ONLY if the reserve is applied by consulting + // the DAO unconditionally (per-cluster aware) rather than gating on the global .value(). + when(capacityDao.listClustersCrossingThreshold(Mockito.anyShort(), Mockito.anyLong(), + Mockito.eq("cluster.cpu.allocated.capacity.disablethreshold"), Mockito.anyLong())).thenReturn(new ArrayList()); + when(capacityDao.listClustersCrossingThreshold(Mockito.anyShort(), Mockito.anyLong(), + Mockito.eq("cluster.memory.allocated.capacity.disablethreshold"), Mockito.anyLong())).thenReturn(new ArrayList()); + List haReserved = new ArrayList(); + haReserved.add(3L); + when(capacityDao.listClustersCrossingThreshold(Mockito.anyShort(), Mockito.anyLong(), + Mockito.eq("cluster.ha.failover.capacity.reservethreshold"), Mockito.anyLong())).thenReturn(haReserved); + return haReserved; + } + @Test public void testGetClusterOrderCapacityType() {