Skip to content

server: reserve cluster capacity for HA failover (HA admission control) #14042

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
nagaboinaramgopal wants to merge 1 commit into
base: main
Choose a base branch
from
+93 −1
Open

server: reserve cluster capacity for HA failover (HA admission control) #14042

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions api/src/main/java/com/cloud/deploy/DeploymentClusterPlanner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ public interface DeploymentClusterPlanner extends DeploymentPlanner {
static final String ClusterCPUCapacityDisableThresholdCK = "cluster.cpu.allocated.capacity.disablethreshold";
static final String ClusterMemoryCapacityDisableThresholdCK = "cluster.memory.allocated.capacity.disablethreshold";
static final String ClusterThresholdEnabledCK = "cluster.threshold.enabled";
static final String ClusterHAFailoverReserveThresholdCK = "cluster.ha.failover.capacity.reservethreshold";

static final ConfigKey<Float> ClusterCPUCapacityDisableThreshold =
new ConfigKey<Float>(
Expand All @@ -56,6 +57,15 @@ public interface DeploymentClusterPlanner extends DeploymentPlanner {
"Enable/Disable cluster thresholds. If disabled, an instance can start in a cluster even though the threshold may be crossed.",
false,
ConfigKey.Scope.Global);
// Reserve spare cluster capacity for HA failover.
static final ConfigKey<Float> ClusterHAFailoverReserveThreshold =
new ConfigKey<Float>(
Float.class,
ClusterHAFailoverReserveThresholdCK,
"Alert",
"1.0",
"Fraction (0-1) of a cluster's CPU/memory allocation above which the allocator stops placing new instances in the cluster, reserving the remaining capacity for HA failover restarts. 1.0 (the default) disables the reservation. For example, 0.8 keeps about 20% of the cluster's CPU and memory free for HA-triggered restarts, which are not subject to this reserve. Set it below the corresponding cluster.*.allocated.capacity.disablethreshold to take effect.",
true, ConfigKey.Scope.Cluster, null);

static final ConfigKey<String> VmAllocationAlgorithm = new ConfigKey<>(
String.class,
Expand Down
16 changes: 15 additions & 1 deletion server/src/main/java/com/cloud/deploy/FirstFitPlanner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -388,6 +388,20 @@ protected void removeClustersCrossingThreshold(List<Long> clusterListForVmAlloca
logger.warn(warnMessageForClusterReachedCapacityThreshold);
}

// Exclude clusters that would cross the HA failover reserve threshold, reserving
// capacity for HA-triggered restarts. Threshold is Cluster-scoped, resolved per-cluster by the DAO.
long haRequested = (capacity == Capacity.CAPACITY_TYPE_CPU) ? cpu_requested : ram_requested;
List<Long> clustersCrossingHAReserve = capacityDao.listClustersCrossingThreshold(
capacity, plan.getDataCenterId(), ClusterHAFailoverReserveThreshold.key(), haRequested);
if (clustersCrossingHAReserve != null && !clustersCrossingHAReserve.isEmpty()) {
avoid.addClusterList(clustersCrossingHAReserve);
clusterListForVmAllocation.removeAll(clustersCrossingHAReserve);
logger.warn(String.format(
"HA admission control: excluding clusters %s from new deployments; their %s allocation would cross the HA failover reserve threshold [%s], reserving capacity for HA failover",
clustersCrossingHAReserve, CapacityVO.getCapacityName(capacity),
ClusterHAFailoverReserveThreshold.key()));
}

}
}

Expand Down Expand Up @@ -694,6 +708,6 @@ public String getConfigComponentName() {

@Override
public ConfigKey<?>[] getConfigKeys() {
return new ConfigKey<?>[] {ClusterCPUCapacityDisableThreshold, ClusterMemoryCapacityDisableThreshold, ClusterThresholdEnabled, VmAllocationAlgorithm};
return new ConfigKey<?>[] {ClusterCPUCapacityDisableThreshold, ClusterMemoryCapacityDisableThreshold, ClusterThresholdEnabled, ClusterHAFailoverReserveThreshold, VmAllocationAlgorithm};
}
}
68 changes: 68 additions & 0 deletions server/src/test/java/com/cloud/vm/FirstFitPlannerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,74 @@ public void checkClusterReorderingForStartVMWithThresholdCheckDisabled() throws
assertTrue("Reordered cluster list does not have clusters exceeding threshold", (clusterList.containsAll(clustersCrossingThreshold)));
}

// VMware HA admission-control parity: with the HA failover reserve threshold set below
// 1.0, a cluster reported as crossing that reserve is excluded from a NEW deployment so headroom
// stays free for HA restarts. Uses config-key strings (not the new ConfigKey symbol) so the test
// fails-before (old code has no HA reserve) and passes-after.
@Test
public void checkHAFailoverReserveExcludesClusterOnDeploy() throws InsufficientServerCapacityException {
VirtualMachineProfileImpl vmProfile = mock(VirtualMachineProfileImpl.class);
DataCenterDeployment plan = mock(DataCenterDeployment.class);
ExcludeList avoids = mock(ExcludeList.class);
initializeForTest(vmProfile, plan, avoids);
List<Long> haReserved = initializeForHAFailoverReserve();

Map<String, String> details = new HashMap<String, String>();
details.put("deployvm", "true");
when(vmDetailsDao.listDetailsKeyPairs(vmProfile.getVirtualMachine().getId())).thenReturn(details);

try {
List<Long> clusterList = planner.orderClusters(vmProfile, plan, avoids);
assertTrue("HA failover reserve must exclude the reserved cluster from deployment", (!clusterList.containsAll(haReserved)));
} finally {
// shared Spring mock: don't leak the deployvm detail (keyed by the default vm id) into other tests
Mockito.reset(vmDetailsDao);
}
}

@Test
public void checkHAFailoverReserveDisabledByDefault() throws InsufficientServerCapacityException {
VirtualMachineProfileImpl vmProfile = mock(VirtualMachineProfileImpl.class);
DataCenterDeployment plan = mock(DataCenterDeployment.class);
ExcludeList avoids = mock(ExcludeList.class);
initializeForTest(vmProfile, plan, avoids);
// keep the (cached, non-dynamic) threshold flag consistent with the other threshold tests
when(configDepot.getConfigStringValue("cluster.threshold.enabled", ConfigKey.Scope.Global, null)).thenReturn(Boolean.FALSE.toString());
// no cluster crosses any threshold; HA reserve is off by default (1.0) so cluster 3 is not excluded
when(capacityDao.listClustersCrossingThreshold(Mockito.anyShort(), Mockito.anyLong(),
Mockito.anyString(), Mockito.anyLong())).thenReturn(new ArrayList<Long>());

Map<String, String> details = new HashMap<String, String>();
details.put("deployvm", "true");
when(vmDetailsDao.listDetailsKeyPairs(vmProfile.getVirtualMachine().getId())).thenReturn(details);

try {
List<Long> clusterList = planner.orderClusters(vmProfile, plan, avoids);
assertTrue("With HA reserve disabled (default), cluster 3 is not excluded", clusterList.contains(3L));
} finally {
// shared Spring mock: don't leak the deployvm detail (keyed by the default vm id) into other tests
Mockito.reset(vmDetailsDao);
}
}

private List<Long> initializeForHAFailoverReserve() {
// keep the (cached, non-dynamic) threshold flag consistent with the other threshold tests
when(configDepot.getConfigStringValue("cluster.threshold.enabled", ConfigKey.Scope.Global, null)).thenReturn(Boolean.FALSE.toString());
// NOTE (regression guard for the per-cluster-scope bug the review caught): deliberately leave
// the GLOBAL HA reserve threshold at its 1.0 default and let only the DAO report cluster 3 as
// crossing the (per-cluster) reserve. This passes ONLY if the reserve is applied by consulting
// the DAO unconditionally (per-cluster aware) rather than gating on the global .value().
when(capacityDao.listClustersCrossingThreshold(Mockito.anyShort(), Mockito.anyLong(),
Mockito.eq("cluster.cpu.allocated.capacity.disablethreshold"), Mockito.anyLong())).thenReturn(new ArrayList<Long>());
when(capacityDao.listClustersCrossingThreshold(Mockito.anyShort(), Mockito.anyLong(),
Mockito.eq("cluster.memory.allocated.capacity.disablethreshold"), Mockito.anyLong())).thenReturn(new ArrayList<Long>());
List<Long> haReserved = new ArrayList<Long>();
haReserved.add(3L);
when(capacityDao.listClustersCrossingThreshold(Mockito.anyShort(), Mockito.anyLong(),
Mockito.eq("cluster.ha.failover.capacity.reservethreshold"), Mockito.anyLong())).thenReturn(haReserved);
return haReserved;
}


@Test
public void testGetClusterOrderCapacityType() {
Expand Down