CLOUDSTACK-9175: [VMware DRS] Adding new host to DRS cluster does not participate in load balancing.

Summary: When a new host is added to a cluster, Cloudstack doesn't create all the port groups (created by cloudstack earlier in other hosts) present in the cluster. Since the new host doesn't have all the necessary networking port groups of cloudstack, it is not eligible to participate in DRS load balancing or HA.
Solution: When adding a host to the cluster in Cloudstack, use VMware API to find the list of unique port groups on a previously added host (older host in the cluster) if exists and then create them on the new host.

Author: Suresh Kumar Anaparti

engine/schema/src/com/cloud/host/dao/HostDao.java

@@ -72,6 +72,8 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status, Stat
 
     List<HostVO> findHypervisorHostInCluster(long clusterId);
 
+    HostVO findOldestExistentHypervisorHostInCluster(long clusterId);
+
     /**
      * @param type
      * @param clusterId

engine/schema/src/com/cloud/host/dao/HostDaoImpl.java

@@ -1122,6 +1122,23 @@ public List<HostVO> findHypervisorHostInCluster(long clusterId) {
         return listBy(sc);
     }
 
+    @Override
+    public HostVO findOldestExistentHypervisorHostInCluster(long clusterId) {
+        SearchCriteria<HostVO> sc = TypeClusterStatusSearch.create();
+        sc.setParameters("type", Host.Type.Routing);
+        sc.setParameters("cluster", clusterId);
+        sc.setParameters("status", Status.Up);
+        sc.setParameters("resourceState", ResourceState.Enabled);
+        Filter orderByFilter = new Filter(HostVO.class, "created", true, null, null);
+
+        List<HostVO> hosts = search(sc, orderByFilter, null, false);
+        if (hosts != null && hosts.size() > 0) {
+            return hosts.get(0);
+        }
+
+        return null;
+    }
+
     @Override
     public List<Long> listAllHosts(long zoneId) {
         SearchCriteria<Long> sc = HostIdSearch.create();

plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java

@@ -21,6 +21,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.net.URLDecoder;
 import java.rmi.RemoteException;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -71,6 +72,8 @@
 import com.cloud.exception.DiscoveryException;
 import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.exception.ResourceInUseException;
+import com.cloud.host.dao.HostDao;
+import com.cloud.host.HostVO;
 import com.cloud.host.Host;
 import com.cloud.host.Status;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
@@ -121,6 +124,7 @@
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.script.Script;
 import com.cloud.utils.ssh.SshHelper;
+import com.cloud.utils.UriUtils;
 import com.cloud.vm.DomainRouterVO;
 
 public class VmwareManagerImpl extends ManagerBase implements VmwareManager, VmwareStorageMount, Listener, VmwareDatacenterService {
@@ -139,6 +143,8 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw
     @Inject
     private NetworkModel _netMgr;
     @Inject
+    private HostDao _hostDao;
+    @Inject
     private ClusterDao _clusterDao;
     @Inject
     private ClusterDetailsDao _clusterDetailsDao;
@@ -377,6 +383,29 @@ private void prepareHost(HostMO hostMo, String privateTrafficLabel) throws Excep
         }
     }
 
+    private HostMO getOldestExistentHostInCluster(Long clusterId, VmwareContext serviceContext) throws Exception {
+        HostVO host = _hostDao.findOldestExistentHypervisorHostInCluster(clusterId);
+        if (host == null) {
+            return null;
+        }
+
+        ManagedObjectReference morSrcHost = HypervisorHostHelper.getHypervisorHostMorFromGuid(host.getGuid());
+        if (morSrcHost == null) {
+            Map<String, String> clusterDetails = _clusterDetailsDao.findDetails(clusterId);
+            if (clusterDetails.get("url") == null) {
+                return null;
+            }
+
+            URI uriForHost = new URI(UriUtils.encodeURIComponent(clusterDetails.get("url") + "/" + host.getName()));
+            morSrcHost = serviceContext.getHostMorByPath(URLDecoder.decode(uriForHost.getPath(), "UTF-8"));
+            if (morSrcHost == null) {
+                return null;
+            }
+        }
+
+        return new HostMO(serviceContext, morSrcHost);
+    }
+
     @Override
     public List<ManagedObjectReference> addHostToPodCluster(VmwareContext serviceContext, long dcId, Long podId, Long clusterId, String hostInventoryPath)
             throws Exception {
@@ -429,6 +458,11 @@ public List<ManagedObjectReference> addHostToPodCluster(VmwareContext serviceCon
                 // For ESX host, we need to enable host firewall to allow VNC access
                 HostMO hostMo = new HostMO(serviceContext, mor);
                 prepareHost(hostMo, privateTrafficLabel);
+                HostMO olderHostMo = getOldestExistentHostInCluster(clusterId, serviceContext);
+                if (olderHostMo != null) {
+                    hostMo.copyPortGroupsFromHost(olderHostMo);
+                }
+
                 returnedHostList.add(mor);
                 return returnedHostList;
             } else {

vmware-base/src/com/cloud/hypervisor/vmware/mo/HostMO.java

@@ -24,6 +24,7 @@
 import java.util.regex.Pattern;
 
 import org.apache.log4j.Logger;
+import org.apache.commons.collections.CollectionUtils;
 
 import com.google.gson.Gson;
 import com.vmware.vim25.AboutInfo;
@@ -48,6 +49,7 @@
 import com.vmware.vim25.HostNetworkTrafficShapingPolicy;
 import com.vmware.vim25.HostOpaqueNetworkInfo;
 import com.vmware.vim25.HostPortGroup;
+import com.vmware.vim25.HostPortGroupPort;
 import com.vmware.vim25.HostPortGroupSpec;
 import com.vmware.vim25.HostRuntimeInfo;
 import com.vmware.vim25.HostSystemConnectionState;
@@ -127,6 +129,43 @@ public HostPortGroupSpec getHostPortGroupSpec(String portGroupName) throws Excep
         return null;
     }
 
+    public List<HostPortGroupSpec> getHostPortGroupSpecs() throws Exception {
+        HostNetworkInfo hostNetInfo = getHostNetworkInfo();
+        if (hostNetInfo == null) {
+            return null;
+        }
+
+        List<HostPortGroup> portGroups = hostNetInfo.getPortgroup();
+        if (CollectionUtils.isEmpty(portGroups)) {
+            return null;
+        }
+
+        List<HostPortGroupSpec> portGroupSpecs = new ArrayList<HostPortGroupSpec>();
+        for (HostPortGroup portGroup : portGroups) {
+            if (!isVMKernelPort(portGroup)) {
+                portGroupSpecs.add(portGroup.getSpec());
+            }
+        }
+
+        return portGroupSpecs;
+    }
+
+    private boolean isVMKernelPort(HostPortGroup portGroup) {
+        assert (portGroup != null);
+        List<HostPortGroupPort> ports = portGroup.getPort();
+        if (CollectionUtils.isEmpty(ports)) {
+            return false;
+        }
+
+        for (HostPortGroupPort port : ports) {
+            if (port.getType().equalsIgnoreCase("host")) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
     @Override
     public String getHyperHostName() throws Exception {
         return getName();
@@ -1110,4 +1149,39 @@ public String getNetworkName(String netMorVal) throws Exception {
         }
         return networkName;
     }
+
+    public void createPortGroup(HostPortGroupSpec spec) throws Exception {
+        if (spec == null) {
+            return;
+        }
+
+        synchronized (_mor.getValue().intern()) {
+            HostNetworkSystemMO hostNetMo = getHostNetworkSystemMO();
+            if (hostNetMo == null) {
+                return;
+            }
+
+            ManagedObjectReference morNetwork = getNetworkMor(spec.getName());
+            if (morNetwork == null) {
+                hostNetMo.addPortGroup(spec);
+            }
+        }
+    }
+
+    public void copyPortGroupsFromHost(HostMO srcHost) throws Exception {
+        if (srcHost == null) {
+            return;
+        }
+
+        List<HostPortGroupSpec> portGroupSpecs = srcHost.getHostPortGroupSpecs();
+        if (CollectionUtils.isEmpty(portGroupSpecs)) {
+            s_logger.debug("No port groups in the host: " + srcHost.getName());
+            return;
+        }
+
+        for (HostPortGroupSpec spec : portGroupSpecs) {
+            s_logger.debug("Creating port group: " + spec.getName() + " in the host: " + getName());
+            createPortGroup(spec);
+        }
+    }
 }

vmware-base/src/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java

@@ -1584,6 +1584,30 @@ public void action(Long param) {
         }
     }
 
+    public static ManagedObjectReference getHypervisorHostMorFromGuid(String guid) {
+        if (guid == null) {
+            return null;
+        }
+
+        String[] tokens = guid.split("@");
+        if (tokens == null || tokens.length != 2) {
+            s_logger.error("Invalid content in host guid");
+            return null;
+        }
+
+        String[] hostTokens = tokens[0].split(":");
+        if (hostTokens == null || hostTokens.length != 2) {
+            s_logger.error("Invalid content in host guid");
+            return null;
+        }
+
+        ManagedObjectReference morHyperHost = new ManagedObjectReference();
+        morHyperHost.setType(hostTokens[0]);
+        morHyperHost.setValue(hostTokens[1]);
+
+        return morHyperHost;
+    }
+
     public static String getScsiController(Pair<String, String> controllerInfo, String recommendedController) {
         String rootDiskController = controllerInfo.first();
         String dataDiskController = controllerInfo.second();