<fix>[core]: cap client keep-alive below agent socket timeout

core/src/main/java/org/zstack/core/CoreGlobalProperty.java

@@ -48,6 +48,8 @@ public class CoreGlobalProperty {
     public static int REST_FACADE_MAX_PER_ROUTE;
     @GlobalProperty(name = "RESTFacade.maxTotal", defaultValue = "128")
     public static int REST_FACADE_MAX_TOTAL;
+    @GlobalProperty(name = "RESTFacade.keepAliveTimeMillis", defaultValue = "5000")
+    public static int REST_FACADE_KEEPALIVE_TIME_MILLIS;
     /**
      * When set RestServer.maskSensitiveInfo to true, sensitive info will be
      * masked see @NoLogging.

core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java

@@ -2,6 +2,8 @@
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.http.HttpStatus;
+import org.apache.http.conn.ConnectionKeepAliveStrategy;
+import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy;
 import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
 import org.apache.http.impl.nio.client.HttpAsyncClients;
 import org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager;
@@ -189,8 +191,16 @@ private static AsyncRestTemplate createAsyncRestTemplate(int readTimeout, int co
         connectionManager.setDefaultMaxPerRoute(maxPerRoute);
         connectionManager.setMaxTotal(maxTotal);
 
+        // cap the agent-advertised keep-alive to keepAliveMs; also set defaults when the agent sends none (duration < 0)
+        ConnectionKeepAliveStrategy keepAliveStrategy = (response, context) -> {
+            long serverDuration = DefaultConnectionKeepAliveStrategy.INSTANCE.getKeepAliveDuration(response, context);
+            long defaults = CoreGlobalProperty.REST_FACADE_KEEPALIVE_TIME_MILLIS;
+            return (serverDuration < 0 || serverDuration > defaults) ? defaults : serverDuration;
+        };
+
         CloseableHttpAsyncClient httpAsyncClient = HttpAsyncClients.custom()
                 .setConnectionManager(connectionManager)
+                .setKeepAliveStrategy(keepAliveStrategy)
                 .build();
 
         HttpComponentsAsyncClientHttpRequestFactory cf = new HttpComponentsAsyncClientHttpRequestFactory(httpAsyncClient);

test/src/test/groovy/org/zstack/test/integration/core/rest/RestFacadeKeepAliveCase.groovy

@@ -0,0 +1,68 @@
+package org.zstack.test.integration.core.rest
+
+import org.springframework.http.HttpEntity
+import org.zstack.core.CoreGlobalProperty
+import org.zstack.core.rest.RESTFacadeImpl
+import org.zstack.header.errorcode.ErrorCode
+import org.zstack.header.rest.AsyncRESTCallback
+import org.zstack.test.integration.ZStackTest
+import org.zstack.testlib.EnvSpec
+import org.zstack.testlib.SubCase
+import org.zstack.testlib.WebBeanConstructor
+import org.zstack.utils.URLBuilder
+
+import java.util.concurrent.TimeUnit
+
+class RestFacadeKeepAliveCase extends SubCase {
+    EnvSpec env
+    String BASE_URL = "/test-keep-alive"
+
+    @Override
+    void clean() {
+        env.delete()
+    }
+
+    @Override
+    void setup() {
+        useSpring(ZStackTest.springSpec)
+    }
+
+    @Override
+    void environment() {
+        env = env {
+        }
+    }
+
+    @Override
+    void test() {
+        env.create {
+            testAsyncPostStillWorks()
+        }
+    }
+
+    void testAsyncPostStillWorks() {
+        logger.info("Test 001")
+        RESTFacadeImpl restf = bean(RESTFacadeImpl.class)
+
+        env.simulator(BASE_URL) { HttpEntity<String> e ->
+            return e.toString()
+        }
+
+        boolean success = false
+        String url = URLBuilder.buildHttpUrl("127.0.0.1", WebBeanConstructor.port, BASE_URL)
+        restf.asyncJsonPost(url, "ping", new AsyncRESTCallback(null) {
+            @Override
+            void fail(ErrorCode err) {
+            }
+
+            @Override
+            void success(HttpEntity<String> responseEntity) {
+                success = true
+            }
+        }, TimeUnit.MILLISECONDS, 5000)
+
+        retryInSecs {
+            assert success
+        }
+    }
+}