Merge pull request #374 from buffer/proxy-connect-timeout

[Thug API] Proxy connect timeout

docs/source/api.rst

@@ -506,6 +506,26 @@ Thug API interface definition is reported below for convenience.
             @return: None
             """
 
+        def get_proxy_connect_timeout():
+            """
+            get_proxy_connect_timeout
+
+            Get the proxy connect timeout (in seconds)
+
+            @return: the proxy connect timeout (in seconds)
+            """
+
+        def set_proxy_connect_timeout(timeout):
+            """
+            set_proxy_connect_timeout
+
+            Set the proxy connect timeout (in seconds)
+
+            @param timeout: the proxy connect timeout (in seconds)
+            @type timeout: C{int}
+            @return: None
+            """
+
         def get_timeout():
             """
             get_timeout

tests/ThugAPI/test_ThugAPI.py

@@ -202,6 +202,15 @@ def test_connect_timeout(self):
         self.thug_api.set_connect_timeout(20)
         assert self.thug_api.get_connect_timeout() in (20,)
 
+    def test_proxy_connect_timeout(self):
+        assert self.thug_api.get_proxy_connect_timeout() in (5,)
+
+        self.thug_api.set_proxy_connect_timeout(10)
+        assert self.thug_api.get_proxy_connect_timeout() in (10,)
+
+        self.thug_api.set_proxy_connect_timeout("foo")
+        assert self.thug_api.get_proxy_connect_timeout() in (10,)
+
     def test_broken_url(self):
         assert not self.thug_api.get_broken_url()
 

thug/DOM/HTTPSession.py

@@ -41,7 +41,9 @@ def __init__(self, proxy=None):
         self.filecount = 0
 
     def __check_proxy_alive(self, hostname, port):
-        s = socket.create_connection((hostname, port), 5.0)
+        s = socket.create_connection(
+            (hostname, port), timeout=log.ThugOpts.proxy_connect_timeout
+        )
         s.close()
 
     def __do_init_proxy(self, proxy):

thug/ThugAPI/IThugAPI.py

@@ -482,6 +482,26 @@ def set_connect_timeout(timeout):
         @return: None
         """
 
+    def get_proxy_connect_timeout():
+        """
+        get_proxy_connect_timeout
+
+        Get the proxy connect timeout (in seconds)
+
+        @return: the proxy connect timeout (in seconds)
+        """
+
+    def set_proxy_connect_timeout(timeout):
+        """
+        set_proxy_connect_timeout
+
+        Set the proxy connect timeout (in seconds)
+
+        @param timeout: the proxy connect timeout (in seconds)
+        @type timeout: C{int}
+        @return: None
+        """
+
     def get_timeout():
         """
         get_timeout

thug/ThugAPI/ThugAPI.py

@@ -295,6 +295,12 @@ def get_connect_timeout(self):
     def set_connect_timeout(self, timeout):
         log.ThugOpts.connect_timeout = timeout
 
+    def get_proxy_connect_timeout(self):
+        return log.ThugOpts.proxy_connect_timeout
+
+    def set_proxy_connect_timeout(self, timeout):
+        log.ThugOpts.proxy_connect_timeout = timeout
+
     def get_broken_url(self):
         return log.ThugOpts.broken_url
 

thug/ThugAPI/ThugOpts.py

@@ -44,6 +44,7 @@ def __init__(self):
         self.extensive = False
         self._threshold = 0
         self._connect_timeout = 10
+        self._proxy_connect_timeout = 5
         self._timeout = 600
         self.ast_debug = False
         self.http_debug = 0
@@ -281,6 +282,24 @@ def set_connect_timeout(self, timeout):
 
     connect_timeout = property(get_connect_timeout, set_connect_timeout)
 
+    def get_proxy_connect_timeout(self):
+        return self._proxy_connect_timeout
+
+    def set_proxy_connect_timeout(self, timeout):
+        try:
+            seconds = int(timeout)
+        except ValueError:
+            log.warning(
+                "[WARNING] Ignoring invalid proxy connect timeout value (should be an integer)"
+            )
+            return
+
+        self._proxy_connect_timeout = seconds
+
+    proxy_connect_timeout = property(
+        get_proxy_connect_timeout, set_proxy_connect_timeout
+    )
+
     def get_timeout(self):
         return self._timeout
 

thug/thug.py

@@ -44,72 +44,73 @@ def usage(self):
         thug [ options ] url
 
     Options:
-        -h, --help              \tDisplay this help information
-        -V, --version           \tDisplay Thug version
-        -i, --list-ua           \tDisplay available user agents
-        -u, --useragent=        \tSelect a user agent (use option -b for values, default: winxpie60)
-        -e, --events=           \tEnable comma-separated specified DOM events handling
-        -w, --delay=            \tSet a maximum setTimeout/setInterval delay value (in milliseconds)
-        -n, --logdir=           \tSet the log output directory
-        -o, --output=           \tLog to a specified file
-        -r, --referer           \tSpecify a referer
-        -p, --proxy=            \tSpecify a proxy (see below for format and supported schemes)
-        -m, --attachment        \tSet the attachment mode
-        -l, --local             \tAnalyze a locally saved page
-        -x, --local-nofetch     \tAnalyze a locally saved page and prevent remote content fetching
-        -v, --verbose           \tEnable verbose mode
-        -d, --debug             \tEnable debug mode
-        -q, --quiet             \tDisable console logging
-        -g, --http-debug        \tEnable HTTP debug mode
-        -t, --threshold         \tMaximum pages to fetch
-        -j, --extensive         \tExtensive fetch of linked pages
-        -O, --connect-timeout   \tSet the connect timeout (in seconds, default: 10 seconds)
-        -T, --timeout=          \tSet the analysis timeout (in seconds, default: 600 seconds)
-        -c, --broken-url        \tSet the broken URL mode
-        -z, --web-tracking      \tEnable web client tracking inspection
-        -b, --async-prefetch    \tEnable async prefetching mode
-        -k, --no-honeyagent     \tDisable HoneyAgent support
-        -a, --image-processing  \tEnable image processing analysis
-        -f, --screenshot        \tEnable screenshot capturing
-        -E, --awis              \tEnable AWS Alexa Web Information Service (AWIS)
-        -s, --no-down-prevent   \tDisable download prevention mechanism
+        -h, --help                  \tDisplay this help information
+        -V, --version               \tDisplay Thug version
+        -i, --list-ua               \tDisplay available user agents
+        -u, --useragent=            \tSelect a user agent (use option -b for values, default: winxpie60)
+        -e, --events=               \tEnable comma-separated specified DOM events handling
+        -w, --delay=                \tSet a maximum setTimeout/setInterval delay value (in milliseconds)
+        -n, --logdir=               \tSet the log output directory
+        -o, --output=               \tLog to a specified file
+        -r, --referer               \tSpecify a referer
+        -p, --proxy=                \tSpecify a proxy (see below for format and supported schemes)
+        -m, --attachment            \tSet the attachment mode
+        -l, --local                 \tAnalyze a locally saved page
+        -x, --local-nofetch         \tAnalyze a locally saved page and prevent remote content fetching
+        -v, --verbose               \tEnable verbose mode
+        -d, --debug                 \tEnable debug mode
+        -q, --quiet                 \tDisable console logging
+        -g, --http-debug            \tEnable HTTP debug mode
+        -t, --threshold             \tMaximum pages to fetch
+        -j, --extensive             \tExtensive fetch of linked pages
+        -O, --connect-timeout       \tSet the connect timeout (in seconds, default: 10 seconds)
+        -Y, --proxy-connect-timeout \tSet the proxy connect timeout (in seconds, default: 5 seconds)
+        -T, --timeout=              \tSet the analysis timeout (in seconds, default: 600 seconds)
+        -c, --broken-url            \tSet the broken URL mode
+        -z, --web-tracking          \tEnable web client tracking inspection
+        -b, --async-prefetch        \tEnable async prefetching mode
+        -k, --no-honeyagent         \tDisable HoneyAgent support
+        -a, --image-processing      \tEnable image processing analysis
+        -f, --screenshot            \tEnable screenshot capturing
+        -E, --awis                  \tEnable AWS Alexa Web Information Service (AWIS)
+        -s, --no-down-prevent       \tDisable download prevention mechanism
 
         Plugins:
-        -A, --adobepdf=         \tSpecify Adobe Acrobat Reader version (default: 9.1.0)
-        -P, --no-adobepdf       \tDisable Adobe Acrobat Reader plugin
-        -S, --shockwave=        \tSpecify Shockwave Flash version (default: 10.0.64.0)
-        -R, --no-shockwave      \tDisable Shockwave Flash plugin
-        -J, --javaplugin=       \tSpecify JavaPlugin version (default: 1.6.0.32)
-        -K, --no-javaplugin     \tDisable Java plugin
-        -L, --silverlight       \tSpecify SilverLight version (default: 4.0.50826.0)
-        -N, --no-silverlight    \tDisable SilverLight plugin
+        -A, --adobepdf=             \tSpecify Adobe Acrobat Reader version (default: 9.1.0)
+        -P, --no-adobepdf           \tDisable Adobe Acrobat Reader plugin
+        -S, --shockwave=            \tSpecify Shockwave Flash version (default: 10.0.64.0)
+        -R, --no-shockwave          \tDisable Shockwave Flash plugin
+        -J, --javaplugin=           \tSpecify JavaPlugin version (default: 1.6.0.32)
+        -K, --no-javaplugin         \tDisable Java plugin
+        -L, --silverlight           \tSpecify SilverLight version (default: 4.0.50826.0)
+        -N, --no-silverlight        \tDisable SilverLight plugin
 
         Classifiers:
-        --htmlclassifier=       \tSpecify a list of additional (comma separated) HTML classifier rule files
-        --urlclassifier=        \tSpecify a list of additional (comma separated) URL classifier rule files
-        --jsclassifier=         \tSpecify a list of additional (comma separated) JS classifier rule files
-        --vbsclassifier=        \tSpecify a list of additional (comma separated) VBS classifier rule files
-        --sampleclassifier=     \tSpecify a list of additional (comma separated) Sample classifier rule files
-        --textclassifier=       \tSpecify a list of additional (comma separated) Text classifier rule files
-        --cookieclassifier=     \tSpecify a list of additional (comma separated) Cookie classifier rule files
-        --imageclassifier=      \tSpecify a list of additional (comma separated) Image classifier rule files
-        --htmlfilter=           \tSpecify a list of additional (comma separated) HTML filter files
-        --urlfilter=            \tSpecify a list of additional (comma separated) URL filter files
-        --jsfilter=             \tSpecify a list of additional (comma separated) JS filter files
-        --vbsfilter=            \tSpecify a list of additional (comma separated) VBS filter files
-        --samplefilter=         \tSpecify a list of additional (comma separated) Sample filter files
-        --textfilter=           \tSpecify a list of additional (comma separated) Text filter files
-        --cookiefilter=         \tSpecify a list of additional (comma separated) Cookie filter files
-        --imagefilter=          \tSpecify a list of additional (comma separated) Image filter files
+        --htmlclassifier=           \tSpecify a list of additional (comma separated) HTML classifier rule files
+        --urlclassifier=            \tSpecify a list of additional (comma separated) URL classifier rule files
+        --jsclassifier=             \tSpecify a list of additional (comma separated) JS classifier rule files
+        --vbsclassifier=            \tSpecify a list of additional (comma separated) VBS classifier rule files
+        --sampleclassifier=         \tSpecify a list of additional (comma separated) Sample classifier rule files
+        --textclassifier=           \tSpecify a list of additional (comma separated) Text classifier rule files
+        --cookieclassifier=         \tSpecify a list of additional (comma separated) Cookie classifier rule files
+        --imageclassifier=          \tSpecify a list of additional (comma separated) Image classifier rule files
+        --htmlfilter=               \tSpecify a list of additional (comma separated) HTML filter files
+        --urlfilter=                \tSpecify a list of additional (comma separated) URL filter files
+        --jsfilter=                 \tSpecify a list of additional (comma separated) JS filter files
+        --vbsfilter=                \tSpecify a list of additional (comma separated) VBS filter files
+        --samplefilter=             \tSpecify a list of additional (comma separated) Sample filter files
+        --textfilter=               \tSpecify a list of additional (comma separated) Text filter files
+        --cookiefilter=             \tSpecify a list of additional (comma separated) Cookie filter files
+        --imagefilter=              \tSpecify a list of additional (comma separated) Image filter files
 
         Logging:
-        -F, --file-logging      \tEnable file logging mode (default: disabled)
-        -Z, --json-logging      \tEnable JSON logging mode (default: disabled)
-        -W, --features-logging  \tEnable features logging mode (default: disabled)
-        -G, --elasticsearch-logging\tEnable ElasticSearch logging mode (default: disabled)
-        -D, --mongodb-address=  \tSpecify address and port of the MongoDB instance (format: host:port)
-        -Y, --no-code-logging   \tDisable code logging
-        -U, --no-cert-logging   \tDisable SSL/TLS certificate logging
+        -F, --file-logging          \tEnable file logging mode (default: disabled)
+        -Z, --json-logging          \tEnable JSON logging mode (default: disabled)
+        -W, --features-logging      \tEnable features logging mode (default: disabled)
+        -G, --elasticsearch-logging \tEnable ElasticSearch logging mode (default: disabled)
+        -D, --mongodb-address=      \tSpecify address and port of the MongoDB instance (format: host:port)
+        -Y, --no-code-logging       \tDisable code logging
+        -U, --no-cert-logging       \tDisable SSL/TLS certificate logging
 
     Proxy Format:
         scheme://[username:password@]host:port (supported schemes: http, socks4, socks5)
@@ -141,7 +142,7 @@ def analyze(self):
         try:
             options, args = getopt.getopt(
                 self.args,
-                "hViu:e:w:n:o:r:p:mzbkafEslxvdqgA:PS:RJ:KL:Nt:jO:T:cFZWGYUD:",
+                "hViu:e:w:n:o:r:p:mzbkafEslxvdqgA:PS:RJ:KL:Nt:jO:Y:T:cFZWGYUD:",
                 [
                     "help",
                     "version",
@@ -178,6 +179,7 @@ def analyze(self):
                     "threshold=",
                     "extensive",
                     "connect-timeout=",
+                    "proxy-connect-timeout=",
                     "timeout=",
                     "broken-url",
                     "htmlclassifier=",
@@ -367,6 +369,11 @@ def analyze(self):
                 "--connect-timeout",
             ):
                 self.set_connect_timeout(option[1])
+            elif option[0] in (
+                "-Y",
+                "--proxy-connect-timeout",
+            ):
+                self.set_proxy_connect_timeout(option[1])
             elif option[0] in (
                 "-T",
                 "--timeout",