engine: Add timeout to communication with network provider

Previously there was no explicit timeout for communication to external
network provider. This way there was a long timeout applied to the
communication. Since some objects are locked during the communication,
some objects was locked for a long period of time if the external
network provided did not respond.

An explicit timeout for the communication is added, to avoid locking for
a long period of time.

The library com.woorea.openstack is used for the communication with
external network providers. The version of this library is increased to
3.1.2, which introduces the possibility to add the timeout for
communication.

Change-Id: Ic2eb1380273c811bf659ac067e6ebb65cbf31b99
Bug-Url: https://bugzilla.redhat.com/1380356
Signed-off-by: Dominik Holler <dholler@redhat.com>

backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/provider/network/openstack/BaseNetworkProviderProxy.java

@@ -10,7 +10,12 @@
 
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.params.HttpConnectionParams;
+import org.apache.http.params.HttpParams;
 import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.jboss.resteasy.client.ClientExecutor;
+import org.jboss.resteasy.client.core.executors.ApacheHttpClient4Executor;
 import org.ovirt.engine.core.bll.provider.ProviderValidator;
 import org.ovirt.engine.core.bll.provider.network.NetworkProviderProxy;
 import org.ovirt.engine.core.common.businessentities.OpenstackNetworkProviderProperties;
@@ -22,6 +27,8 @@
 import org.ovirt.engine.core.common.businessentities.network.ProviderNetwork;
 import org.ovirt.engine.core.common.businessentities.network.VmNic;
 import org.ovirt.engine.core.common.businessentities.network.VnicProfile;
+import org.ovirt.engine.core.common.config.Config;
+import org.ovirt.engine.core.common.config.ConfigValues;
 import org.ovirt.engine.core.common.errors.EngineError;
 import org.ovirt.engine.core.common.errors.EngineException;
 import org.ovirt.engine.core.utils.NetworkUtils;
@@ -31,6 +38,7 @@
 import com.woorea.openstack.base.client.HttpMethod;
 import com.woorea.openstack.base.client.OpenStackRequest;
 import com.woorea.openstack.base.client.OpenStackResponseException;
+import com.woorea.openstack.connector.RESTEasyConnector;
 import com.woorea.openstack.keystone.utils.KeystoneTokenProvider;
 import com.woorea.openstack.quantum.Quantum;
 import com.woorea.openstack.quantum.model.Networks;
@@ -65,9 +73,24 @@ public BaseNetworkProviderProxy(Provider<P> provider) {
         this.provider = provider;
     }
 
+    class CustomizedRESTEasyConnector extends RESTEasyConnector {
+
+        @Override
+        protected ClientExecutor createClientExecutor() {
+            int socketTimeOut = Config.<Integer> getValue(ConfigValues.ExternalNetworkProviderTimeout) * 1000;
+            int connectionTimeOut = Config.<Integer> getValue(ConfigValues.ExternalNetworkProviderConnectionTimeout) * 1000;
+
+            DefaultHttpClient httpClient = new DefaultHttpClient();
+            HttpParams params = httpClient.getParams();
+            HttpConnectionParams.setConnectionTimeout(params, connectionTimeOut);
+            HttpConnectionParams.setSoTimeout(params, socketTimeOut);
+            return new ApacheHttpClient4Executor(httpClient);
+        }
+    }
+
     private Quantum getClient() {
         if (client == null) {
-            client = new Quantum(provider.getUrl() + API_VERSION);
+            client = new Quantum(provider.getUrl() + API_VERSION, new CustomizedRESTEasyConnector());
             if (provider.isRequiringAuthentication()) {
                 setClientTokenProvider(client);
             }

backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java

@@ -1983,5 +1983,21 @@ public enum ConfigValues {
     @DefaultValueAttribute("true")
     AgentChannelNamingSupported,
 
+    /**
+     * Timeout in seconds for the completion of calls to external network providers.
+     */
+    @TypeConverterAttribute(Integer.class)
+    @DefaultValueAttribute("30")
+    ExternalNetworkProviderTimeout,
+
+    /**
+     * Timeout in seconds for establishment of connections with external network providers. This
+     * should be quite small, a few seconds at most, as it the TCP handshake with
+     * network providers should be very quick in most networks.
+     */
+    @TypeConverterAttribute(Integer.class)
+    @DefaultValueAttribute("20")
+    ExternalNetworkProviderConnectionTimeout,
+
     Invalid
 }

ovirt-engine.spec.in

@@ -103,7 +103,7 @@
 %global vdsm_uid 36
 %global vdsm_user vdsm
 
-%global openstack_java_version 3.1.1
+%global openstack_java_version 3.1.2
 
 # Macro to create an user:
 #

pom.xml

@@ -94,7 +94,7 @@
     <archaius.version>0.4.1</archaius.version>
     <rxjava.version>1.0.8</rxjava.version>
     <!-- OpenStack -->
-    <openstack-client.version>3.1.1</openstack-client.version>
+    <openstack-client.version>3.1.2</openstack-client.version>
 
     <!-- Plugins Versions -->
     <maven-surefire-plugin.version>2.12.3</maven-surefire-plugin.version>