git-svn-id: http://proj.badc.rl.ac.uk/svn/ndg-security/trunk/ndg_http…

…sclient@7972 051b1e3e-aa0c-0410-b6c2-bfbade6052be

.project

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>ndg_httpsclient</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.python.pydev.PyDevBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.python.pydev.pythonNature</nature>
+	</natures>
+</projectDescription>

.pydevproject

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?eclipse-pydev version="1.0"?>
+
+<pydev_project>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.6</pydev_property>
+<pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
+<path>/ndg_httpsclient</path>
+</pydev_pathproperty>
+</pydev_project>

LICENSE

@@ -0,0 +1,26 @@
+Copyright (c) 2011, Science & Technology Facilities Council (STFC)
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without 
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice, 
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+    * Neither the name of the Science & Technology Facilities Council (STFC) 
+      nor the names of its contributors may be used to endorse or promote 
+      products derived from this software without specific prior written 
+      permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README

@@ -0,0 +1,39 @@
+================
+urllib2pyopenssl
+================
+
+Description
+===========
+This is a library to enable urllib2 to be used with SSL sockets from pyOpenSSL instead of the built in ssl library. A script is provided to exercise it:
+
+urllib2pyopenssl_get::
+ - Utility to fetch data using HTTP or HTTPS GET from a specified URL.
+
+Prerequisites
+=============
+This has been developed and tested for Python 2.6 (compiled with HTTPS support).
+pyOpenSSL
+
+
+Installation
+============
+Installation can be performed using easy_install, e.g.::
+  easy_install urllib2pyopenssl-0.1.0-py2.6.egg
+
+Running urllib2pyopenssl_get
+============================
+Parameter::
+    url                   The URL of the resource to be fetched
+
+Options::
+    -h, --help            Show help message and exit.
+    -c FILE, --certificate=FILE
+                          Certificate file - defaults to $HOME/credentials.pem
+    -k FILE, --private-key=FILE
+                          Private key file - defaults to the certificate file
+    -t DIR, --ca-certificate-dir=DIR
+                          Trusted CA certificate file directory.
+    -d, --debug           Print debug information - this may be useful in solving problems with HTTP
+                          or HTTPS access to a server.
+    -f FILE, --fetch=FILE Output file
+    -v, --verify-peer     Verify peer certificate.

ndg/__init__.py

@@ -0,0 +1,19 @@
+"""ndg_httpsclient - PyOpenSSL utility to make a httplib-like interface suitable
+for use with urllib2
+
+This is a setuptools namespace_package.  DO NOT place any other
+code in this file!  There is no guarantee that it will be installed
+with easy_install.  See:
+
+http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+
+... for details.
+"""
+__author__ = "P J Kershaw"
+__date__ = "06/01/12"
+__copyright__ = "(C) 2012 Science and Technology Facilities Council"
+__license__ = "BSD - see LICENSE file in top-level directory"
+__contact__ = "Philip.Kershaw@stfc.ac.uk"
+__revision__ = '$Id$'
+
+__import__('pkg_resources').declare_namespace(__name__)

ndg/httpsclient/__init__.py

@@ -0,0 +1,9 @@
+"""ndg_httpsclient - PyOpenSSL utility to make a httplib-like interface suitable
+for use with urllib2
+"""
+__author__ = "P J Kershaw (STFC) and Richard Wilkinson (Tessella)"
+__date__ = "09/12/11"
+__copyright__ = "(C) 2011 Science and Technology Facilities Council"
+__license__ = "BSD - see LICENSE file in top-level directory"
+__contact__ = "Philip.Kershaw@stfc.ac.uk"
+__revision__ = '$Id$'

ndg/httpsclient/get.py

@@ -0,0 +1,177 @@
+import cookielib
+import httplib
+import logging
+from optparse import OptionParser
+import os
+import urllib2
+import urlparse
+
+from OpenSSL import SSL
+
+from urllib2pyopenssl.urllib2_build_opener import urllib2_build_opener
+from urllib2pyopenssl.https import HTTPSContextHandler
+import urllib2pyopenssl.ssl_context_util as ssl_context_util
+from urllib2pyopenssl.ssl_peer_verification import ServerSSLCertVerification
+
+def fetch_from_url(url, config):
+    """Returns data retrieved from a URL.
+    @param url - URL to attempt to open
+    @param config - configuration
+    @return data retrieved from URL or None
+    """
+    (return_code, return_message, response) = open_url(url, config)
+    if return_code and return_code == httplib.OK:
+        return_data = response.read()
+        response.close()
+        return return_data
+    else:
+        raise Exception(return_message)
+
+def fetch_from_url_to_file(url, config, output_file):
+    """Writes data retrieved from a URL to a file.
+    @param url - URL to attempt to open
+    @param config - configuration
+    @param output_file - output file
+    @return tuple (
+        returned HTTP status code or 0 if an error occurred
+        returned message
+        boolean indicating whether access was successful
+    )
+    """
+    (return_code, return_message, response) = open_url(url, config)
+    if return_code == httplib.OK:
+        return_data = response.read()
+        response.close()
+        outfile = open(output_file, "w")
+        outfile.write(return_data)
+        outfile.close()
+    return (return_code, return_message, (return_code == httplib.OK))
+
+def open_url(url, config):
+    """Attempts to open a connection to a specified URL.
+    @param url - URL to attempt to open
+    @param config - configuration
+    @return tuple (
+        returned HTTP status code or 0 if an error occurred
+        returned message or error description
+        response object
+    )
+    """
+    debuglevel = 1 if config.debug else 0
+
+    # Set up handlers for URL opener.
+    cj = cookielib.CookieJar()
+    cookie_handler = urllib2.HTTPCookieProcessor(cj)
+
+    handlers = [cookie_handler]
+
+    if config.debug:
+        http_handler = urllib2.HTTPHandler(debuglevel=debuglevel)
+        https_handler = HTTPSContextHandler(config.ssl_context, debuglevel=debuglevel)
+        handlers.extend([http_handler, https_handler])
+
+    # Explicitly remove proxy handling if the host is one listed in the value of the no_proxy
+    # environment variable because urllib2 does use proxy settings set via http_proxy and
+    # https_proxy, but does not take the no_proxy value into account.
+    if not _should_use_proxy(url):
+        handlers.append(urllib2.ProxyHandler({}))
+        if config.debug:
+            print "Not using proxy"
+
+    opener = urllib2_build_opener(config.ssl_context, *handlers)
+
+    # Open the URL and check the response.
+    return_code = 0
+    return_message = ''
+    response = None
+    try:
+        response = opener.open(url)
+        if response.url == url:
+            return_message = response.msg
+            return_code = response.code
+        else:
+            return_message = ('Redirected (%s  %s)' % (response.code, response.url))
+        if config.debug:
+            for index, cookie in enumerate(cj):
+                print index, '  :  ', cookie        
+    except urllib2.HTTPError, exc:
+        return_code = exc.code
+        return_message = ("Error: %s" % exc.msg)
+        if config.debug:
+            print exc.code, exc.msg
+    except Exception, exc:
+        return_message = ("Error: %s" % exc.__str__())
+        if config.debug:
+            print exc.__class__, exc.__str__()
+    return (return_code, return_message, response)
+
+def _should_use_proxy(url):
+    """Determines whether a proxy should be used to open a connection to the specified URL, based on
+        the value of the no_proxy environment variable.
+    @param url - URL string
+    """
+    no_proxy = os.environ.get('no_proxy', '')
+
+    urlObj = urlparse.urlparse(url)
+    for np in [h.strip() for h in no_proxy.split(',')]:
+        if urlObj.hostname == np:
+            return False
+
+    return True
+
+class Configuration(object):
+    """Checker configuration.
+    """
+    def __init__(self, ssl_context, debug):
+        """
+        @param key_file - file containing the user's private key
+        @param cert_file - file containing the user's certificate
+        @param debug - if True, output debugging information
+        """
+        self.ssl_context = ssl_context
+        self.debug = debug
+
+def main():
+    '''Utility to fetch data using HTTP or HTTPS GET from a specified URL.
+    '''
+    parser = OptionParser(usage="%prog [options] url")
+    parser.add_option("-k", "--private-key", dest="key_file", metavar="FILE",
+                      default=None,
+                      help="Private key file.")
+    parser.add_option("-c", "--certificate", dest="cert_file", metavar="FILE",
+                      default=os.path.expanduser("~/credentials.pem"),
+                      help="Certificate file.")
+    parser.add_option("-t", "--ca-certificate-dir", dest="ca_dir", metavar="PATH",
+                      default=None,
+                      help="Trusted CA certificate file directory.")
+    parser.add_option("-d", "--debug", action="store_true", dest="debug", default=False,
+                      help="Print debug information.")
+    parser.add_option("-f", "--fetch", dest="output_file", metavar="FILE",
+                      default=None, help="Output file.")
+    parser.add_option("-v", "--verify-peer", action="store_true", dest="verify_peer", default=False,
+                      help="Verify peer certificate.")
+    (options, args) = parser.parse_args()
+    if len(args) != 1:
+        parser.error("Incorrect number of arguments")
+
+    url = args[0]
+    # If a private key file is not specified, the key is assumed to be stored in the certificate file.
+    ssl_context = ssl_context_util.make_ssl_context(
+        options.key_file if options.key_file and os.path.exists(options.key_file) else None,
+        options.cert_file if options.cert_file and os.path.exists(options.cert_file) else None,
+        None,
+        options.ca_dir if options.ca_dir and os.path.exists(options.ca_dir) else None,
+        options.verify_peer, url)
+
+    config = Configuration(ssl_context, options.debug)
+    if options.output_file:
+        (return_code, return_message, success) = fetch_from_url_to_file(url, config,
+            options.output_file)
+        print return_code, return_message
+    else:
+        data = fetch_from_url(url, config)
+        print data
+
+if __name__=='__main__':
+    logging.basicConfig()
+    main()