NSX: Support internal LB service in NSX

api/src/main/java/com/cloud/network/VirtualRouterProvider.java

@@ -21,7 +21,7 @@
 
 public interface VirtualRouterProvider extends InternalIdentity, Identity {
     public enum Type {
-        VirtualRouter, ElasticLoadBalancerVm, VPCVirtualRouter, InternalLbVm, NetScalerVm
+        VirtualRouter, ElasticLoadBalancerVm, VPCVirtualRouter, InternalLbVm, NetScalerVm, Nsx
     }
 
     public Type getType();

api/src/main/java/com/cloud/offering/NetworkOffering.java

@@ -58,6 +58,7 @@ public enum NsxMode {
     public final static String DefaultSharedNetworkOfferingWithSGService = "DefaultSharedNetworkOfferingWithSGService";
     public static final String DEFAULT_TUNGSTEN_SHARED_NETWORK_OFFERING_WITH_SGSERVICE = "DefaultTungstenSharedNetworkOfferingWithSGService";
     public static final String DEFAULT_NAT_NSX_OFFERING_FOR_VPC = "DefaultNATNSXNetworkOfferingForVpc";
+    public static final String DEFAULT_NAT_NSX_OFFERING_FOR_VPC_WITH_ILB = "DefaultNATNSXNetworkOfferingForVpcWithInternalLB";
     public static final String DEFAULT_ROUTED_NSX_OFFERING_FOR_VPC = "DefaultRoutedNSXNetworkOfferingForVpc";
     public static final String DEFAULT_NAT_NSX_OFFERING = "DefaultNATNSXNetworkOffering";
     public static final String DEFAULT_ROUTED_NSX_OFFERING = "DefaultRoutedNSXNetworkOffering";

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -825,6 +825,7 @@ public class ApiConstants {
     public static final String FOR_VPC = "forvpc";
     public static final String FOR_NSX = "fornsx";
     public static final String NSX_SUPPORT_LB = "nsxsupportlb";
+    public static final String NSX_SUPPORTS_INTERNAL_LB = "nsxsupportsinternallb";
     public static final String FOR_TUNGSTEN = "fortungsten";
     public static final String SHRINK_OK = "shrinkok";
     public static final String NICIRA_NVP_DEVICE_ID = "nvpdeviceid";
@@ -963,6 +964,7 @@ public class ApiConstants {
     public static final String SUPPORTS_REGION_LEVEL_VPC = "supportsregionLevelvpc";
     public static final String SUPPORTS_STRECHED_L2_SUBNET = "supportsstrechedl2subnet";
     public static final String SUPPORTS_PUBLIC_ACCESS = "supportspublicaccess";
+    public static final String SUPPORTS_INTERNAL_LB = "supportsinternallb";
     public static final String SUPPORTS_VM_AUTOSCALING = "supportsvmautoscaling";
     public static final String REGION_LEVEL_VPC = "regionlevelvpc";
     public static final String STRECHED_L2_SUBNET = "strechedl2subnet";

api/src/main/java/org/apache/cloudstack/api/command/admin/network/CreateNetworkOfferingCmd.java

@@ -160,6 +160,12 @@ public class CreateNetworkOfferingCmd extends BaseCmd {
             since = "4.20.0")
     private Boolean nsxSupportsLbService;
 
+    @Parameter(name = ApiConstants.NSX_SUPPORTS_INTERNAL_LB,
+            type = CommandType.BOOLEAN,
+            description = "true if network offering for NSX network offering supports Internal Load balancer service.",
+            since = "4.20.0")
+    private Boolean nsxSupportsInternalLbService;
+
     @Parameter(name = ApiConstants.FOR_TUNGSTEN,
             type = CommandType.BOOLEAN,
             description = "true if network offering is meant to be used for Tungsten-Fabric, false otherwise.")
@@ -306,6 +312,10 @@ public boolean getNsxSupportsLbService() {
         return BooleanUtils.isTrue(nsxSupportsLbService);
     }
 
+    public boolean getNsxSupportsInternalLbService() {
+        return BooleanUtils.isTrue(nsxSupportsInternalLbService);
+    }
+
     public Boolean getForTungsten() {
         return forTungsten;
     }

api/src/main/java/org/apache/cloudstack/api/response/NetworkOfferingResponse.java

@@ -135,6 +135,10 @@ public class NetworkOfferingResponse extends BaseResponseWithAnnotations {
     @Param(description = "true if network offering supports public access for guest networks", since = "4.10.0")
     private Boolean supportsPublicAccess;
 
+    @SerializedName(ApiConstants.SUPPORTS_INTERNAL_LB)
+    @Param(description = "true if network offering supports public access for guest networks", since = "4.20.0")
+    private Boolean supportsInternalLb;
+
     @SerializedName(ApiConstants.DOMAIN_ID)
     @Param(description = "the domain ID(s) this disk offering belongs to. Ignore this information as it is not currently applicable.")
     private String domainId;
@@ -259,6 +263,10 @@ public void setSupportsPublicAccess(Boolean supportsPublicAccess) {
         this.supportsPublicAccess = supportsPublicAccess;
     }
 
+    public void setSupportsInternalLb(Boolean supportsInternalLb) {
+        this.supportsInternalLb = supportsInternalLb;
+    }
+
     public String getDomainId() {
         return domainId;
     }

plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxResource.java

@@ -308,7 +308,9 @@ private Answer executeRequest(CreateNsxTier1GatewayCommand cmd) {
 
     private Answer executeRequest(DeleteNsxTier1GatewayCommand cmd) {
         String tier1Id = NsxControllerUtils.getTier1GatewayName(cmd.getDomainId(), cmd.getAccountId(), cmd.getZoneId(), cmd.getNetworkResourceId(), cmd.isResourceVpc());
+        String lbName = NsxControllerUtils.getLoadBalancerName(tier1Id);
         try {
+            nsxApiClient.deleteLoadBalancer(lbName);
             nsxApiClient.deleteTier1Gateway(tier1Id);
         } catch (Exception e) {
             return new NsxAnswer(cmd, new CloudRuntimeException(e.getMessage()));

plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java

@@ -707,8 +707,7 @@ public void deleteNsxLbResources(String tier1GatewayName, long lbId) {
             LBPoolListResult lbPoolListResult = lbPools.list(null, null, null, null, null, null);
             if (CollectionUtils.isEmpty(lbVsListResult.getResults()) && CollectionUtils.isEmpty(lbPoolListResult.getResults())) {
                 String lbName = getLoadBalancerName(tier1GatewayName);
-                LbServices lbServices = (LbServices) nsxService.apply(LbServices.class);
-                lbServices.delete(lbName, true);
+                deleteLoadBalancer(lbName);
             }
 
         } catch (Error error) {
@@ -719,6 +718,11 @@ public void deleteNsxLbResources(String tier1GatewayName, long lbId) {
         }
     }
 
+    public void deleteLoadBalancer(String lbName) {
+        LbServices lbServices = (LbServices) nsxService.apply(LbServices.class);
+        lbServices.delete(lbName, true);
+    }
+
     private String getLbPoolPath(String lbPoolName) {
         try {
             LbPools lbPools = (LbPools) nsxService.apply(LbPools.class);

plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java

@@ -44,14 +44,17 @@
 import com.cloud.network.Networks;
 import com.cloud.network.PhysicalNetworkServiceProvider;
 import com.cloud.network.PublicIpAddress;
+import com.cloud.network.VirtualRouterProvider;
 import com.cloud.network.dao.IPAddressDao;
 import com.cloud.network.dao.IPAddressVO;
 import com.cloud.network.dao.LoadBalancerVMMapDao;
 import com.cloud.network.dao.LoadBalancerVMMapVO;
 import com.cloud.network.dao.NetworkDao;
 import com.cloud.network.dao.NetworkVO;
 import com.cloud.network.dao.PhysicalNetworkDao;
+import com.cloud.network.dao.PhysicalNetworkServiceProviderDao;
 import com.cloud.network.dao.PhysicalNetworkVO;
+import com.cloud.network.dao.VirtualRouterProviderDao;
 import com.cloud.network.element.DhcpServiceProvider;
 import com.cloud.network.element.DnsServiceProvider;
 import com.cloud.network.element.FirewallServiceProvider;
@@ -60,9 +63,12 @@
 import com.cloud.network.element.NetworkACLServiceProvider;
 import com.cloud.network.element.PortForwardingServiceProvider;
 import com.cloud.network.element.StaticNatServiceProvider;
+import com.cloud.network.element.VirtualRouterElement;
+import com.cloud.network.element.VirtualRouterProviderVO;
 import com.cloud.network.element.VpcProvider;
 import com.cloud.network.lb.LoadBalancingRule;
 import com.cloud.network.rules.FirewallRule;
+import com.cloud.network.rules.LoadBalancerContainer;
 import com.cloud.network.rules.PortForwardingRule;
 import com.cloud.network.rules.StaticNat;
 import com.cloud.network.vpc.NetworkACLItem;
@@ -82,6 +88,8 @@
 import com.cloud.uservm.UserVm;
 import com.cloud.utils.Pair;
 import com.cloud.utils.component.AdapterBase;
+import com.cloud.utils.db.QueryBuilder;
+import com.cloud.utils.db.SearchCriteria;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.vm.NicProfile;
 import com.cloud.vm.ReservationContext;
@@ -90,6 +98,10 @@
 import com.cloud.vm.dao.VMInstanceDao;
 import net.sf.ehcache.config.InvalidConfigurationException;
 import org.apache.cloudstack.StartupNsxCommand;
+import org.apache.cloudstack.api.command.admin.internallb.ConfigureInternalLoadBalancerElementCmd;
+import org.apache.cloudstack.api.command.admin.internallb.CreateInternalLoadBalancerElementCmd;
+import org.apache.cloudstack.api.command.admin.internallb.ListInternalLoadBalancerElementsCmd;
+import org.apache.cloudstack.network.element.InternalLoadBalancerElementService;
 import org.apache.cloudstack.resource.NsxLoadBalancerMember;
 import org.apache.cloudstack.resource.NsxNetworkRule;
 import org.apache.cloudstack.resource.NsxOpObject;
@@ -111,7 +123,7 @@
 @Component
 public class NsxElement extends AdapterBase implements  DhcpServiceProvider, DnsServiceProvider, VpcProvider,
         StaticNatServiceProvider, IpDeployer, PortForwardingServiceProvider, NetworkACLServiceProvider,
-        LoadBalancingServiceProvider, FirewallServiceProvider, ResourceStateAdapter, Listener {
+        LoadBalancingServiceProvider, FirewallServiceProvider, InternalLoadBalancerElementService, ResourceStateAdapter, Listener {
 
 
     @Inject
@@ -142,6 +154,10 @@ public class NsxElement extends AdapterBase implements  DhcpServiceProvider, Dns
     VpcDao vpcDao;
     @Inject
     LoadBalancerVMMapDao lbVmMapDao;
+    @Inject
+    VirtualRouterProviderDao vrProviderDao;
+    @Inject
+    PhysicalNetworkServiceProviderDao pNtwkSvcProviderDao;
 
     private static final Logger LOGGER = Logger.getLogger(NsxElement.class);
 
@@ -159,7 +175,16 @@ private static Map<Network.Service, Map<Network.Capability, String>> initCapabil
         capabilities.put(Network.Service.Dns, dnsCapabilities);
 
         capabilities.put(Network.Service.StaticNat, null);
-        capabilities.put(Network.Service.Lb, null);
+
+        // Set capabilities for LB service
+        Map<Network.Capability, String> lbCapabilities = new HashMap<Network.Capability, String>();
+        lbCapabilities.put(Network.Capability.SupportedLBAlgorithms, "roundrobin,leastconn");
+        lbCapabilities.put(Network.Capability.SupportedLBIsolation, "dedicated");
+        lbCapabilities.put(Network.Capability.SupportedProtocols, "tcp, udp");
+        lbCapabilities.put(Network.Capability.SupportedStickinessMethods, VirtualRouterElement.getHAProxyStickinessCapability());
+        lbCapabilities.put(Network.Capability.LbSchemes, String.join(",", LoadBalancerContainer.Scheme.Internal.name(), LoadBalancerContainer.Scheme.Public.name()));
+
+        capabilities.put(Network.Service.Lb, lbCapabilities);
         capabilities.put(Network.Service.PortForwarding, null);
         capabilities.put(Network.Service.NetworkACL, null);
 
@@ -633,7 +658,8 @@ public boolean applyLBRules(Network network, List<LoadBalancingRule> rules) thro
                     .setNetworkResourceName(nsxObject.getNetworkResourceName())
                     .setVpcResource(nsxObject.isVpcResource())
                     .setMemberList(lbMembers)
-                    .setPublicIp(publicIp.getAddress().addr())
+                    .setPublicIp(LoadBalancerContainer.Scheme.Public == loadBalancingRule.getScheme() ?
+                            publicIp.getAddress().addr() : loadBalancingRule.getSourceIp().addr())
                     .setPublicPort(String.valueOf(loadBalancingRule.getSourcePortStart()))
                     .setPrivatePort(String.valueOf(loadBalancingRule.getDefaultPortStart()))
                     .setRuleId(loadBalancingRule.getId())
@@ -778,4 +804,74 @@ protected List<String> transformCidrListValues(List<String> sourceCidrList) {
         }
         return list;
     }
+
+    @Override
+    public VirtualRouterProvider configureInternalLoadBalancerElement(long id, boolean enable) {
+        VirtualRouterProviderVO element = vrProviderDao.findById(id);
+        if (element == null || element.getType() != VirtualRouterProvider.Type.Nsx) {
+            throw new InvalidParameterValueException("Can't find " + getName() + " " +
+                    "element with network service provider id " + id + " to be used as a provider for " +
+                    getName());
+        }
+
+        element.setEnabled(enable);
+        element = vrProviderDao.persist(element);
+
+        return element;
+    }
+
+    @Override
+    public VirtualRouterProvider addInternalLoadBalancerElement(long ntwkSvcProviderId) {
+        VirtualRouterProviderVO element = vrProviderDao.findByNspIdAndType(ntwkSvcProviderId, VirtualRouterProvider.Type.Nsx);
+        if (element != null) {
+            LOGGER.debug("There is already an " + getName() + " with service provider id " + ntwkSvcProviderId);
+            return null;
+        }
+
+        PhysicalNetworkServiceProvider provider = pNtwkSvcProviderDao.findById(ntwkSvcProviderId);
+        if (provider == null || !provider.getProviderName().equalsIgnoreCase(getName())) {
+            throw new InvalidParameterValueException("Invalid network service provider is specified");
+        }
+
+        element = new VirtualRouterProviderVO(ntwkSvcProviderId, VirtualRouterProvider.Type.Nsx);
+        element = vrProviderDao.persist(element);
+        return element;
+    }
+
+    @Override
+    public VirtualRouterProvider getInternalLoadBalancerElement(long id) {
+        VirtualRouterProvider provider = vrProviderDao.findById(id);
+        if (provider == null || provider.getType() != VirtualRouterProvider.Type.Nsx) {
+            throw new InvalidParameterValueException("Unable to find " + getName() + " by id");
+        }
+        return provider;
+    }
+
+    @Override
+    public List<? extends VirtualRouterProvider> searchForInternalLoadBalancerElements(Long id, Long ntwkSvsProviderId, Boolean enabled) {
+        QueryBuilder<VirtualRouterProviderVO> sc = QueryBuilder.create(VirtualRouterProviderVO.class);
+        if (id != null) {
+            sc.and(sc.entity().getId(), SearchCriteria.Op.EQ, id);
+        }
+        if (ntwkSvsProviderId != null) {
+            sc.and(sc.entity().getNspId(), SearchCriteria.Op.EQ, ntwkSvsProviderId);
+        }
+        if (enabled != null) {
+            sc.and(sc.entity().isEnabled(), SearchCriteria.Op.EQ, enabled);
+        }
+
+        //return only Internal LB elements
+        sc.and(sc.entity().getType(), SearchCriteria.Op.EQ, VirtualRouterProvider.Type.Nsx);
+
+        return sc.list();
+    }
+
+    @Override
+    public List<Class<?>> getCommands() {
+        List<Class<?>> cmdList = new ArrayList<Class<?>>();
+        cmdList.add(CreateInternalLoadBalancerElementCmd.class);
+        cmdList.add(ConfigureInternalLoadBalancerElementCmd.class);
+        cmdList.add(ListInternalLoadBalancerElementsCmd.class);
+        return cmdList;
+    }
 }

server/src/main/java/com/cloud/api/query/dao/NetworkOfferingJoinDaoImpl.java

@@ -108,6 +108,7 @@ public NetworkOfferingResponse newNetworkOfferingResponse(NetworkOffering offeri
         networkOfferingResponse.setConcurrentConnections(offering.getConcurrentConnections());
         networkOfferingResponse.setSupportsStrechedL2Subnet(offering.isSupportingStrechedL2());
         networkOfferingResponse.setSupportsPublicAccess(offering.isSupportingPublicAccess());
+        networkOfferingResponse.setSupportsInternalLb(offering.isInternalLb());
         networkOfferingResponse.setCreated(offering.getCreated());
         if (offering.getGuestType() != null) {
             networkOfferingResponse.setGuestIpType(offering.getGuestType().toString());

server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -5956,6 +5956,7 @@ public NetworkOffering createNetworkOffering(final CreateNetworkOfferingCmd cmd)
         Boolean forNsx = cmd.isForNsx();
         Boolean forTungsten = cmd.getForTungsten();
         String nsxMode = cmd.getNsxMode();
+        boolean nsxSupportInternalLbSvc = cmd.getNsxSupportsInternalLbService();
         Integer maxconn = null;
         boolean enableKeepAlive = false;
         String servicePackageuuid = cmd.getServicePackageId();
@@ -6274,6 +6275,11 @@ public NetworkOffering createNetworkOffering(final CreateNetworkOfferingCmd cmd)
 
         final NetworkOfferingVO offering = createNetworkOffering(name, displayText, trafficType, tags, specifyVlan, availability, networkRate, serviceProviderMap, false, guestType, false,
                 serviceOfferingId, conserveMode, serviceCapabilityMap, specifyIpRanges, isPersistent, details, egressDefaultPolicy, maxconn, enableKeepAlive, forVpc, forTungsten, forNsx, nsxMode, domainIds, zoneIds, enable, internetProtocol);
+        if (Boolean.TRUE.equals(forNsx) && nsxSupportInternalLbSvc) {
+            offering.setInternalLb(true);
+            offering.setPublicLb(false);
+            _networkOfferingDao.update(offering.getId(), offering);
+        }
         CallContext.current().setEventDetails(" Id: " + offering.getId() + " Name: " + name);
         CallContext.current().putContextParameter(NetworkOffering.class, offering.getId());
         return offering;