used defusedxml for parse and fromstring

geonode/base/management/commands/load_thesaurus.py

@@ -18,7 +18,7 @@
 #
 #########################################################################
 
-import xml.etree.ElementTree as etree
+from defusedxml import lxml as dlxml
 
 from django.core.management.base import BaseCommand, CommandError
 
@@ -83,7 +83,7 @@ def load_thesaurus(self, input_file, name, store):
             'skos': 'http://www.w3.org/2004/02/skos/core#'
         }
 
-        tfile = etree.parse(input_file)
+        tfile = dlxml.parse(input_file)
         root = tfile.getroot()
 
         scheme = root.find('skos:ConceptScheme', ns)

geonode/catalogue/backends/generic.py

@@ -27,7 +27,7 @@
 from owslib.csw import CatalogueServiceWeb, namespaces
 from owslib.util import http_post
 from urlparse import urlparse
-from lxml import etree
+from defusedxml import lxml as dlxml
 from geonode.catalogue.backends.base import BaseCatalogueBackend
 
 logger = logging.getLogger(__name__)
@@ -100,7 +100,7 @@ def login(self):
                 urllib2.HTTPCookieProcessor(),
                 urllib2.HTTPRedirectHandler())
             response = self.opener.open(request)
-            doc = etree.fromstring(response.read())
+            doc = dlxml.fromstring(response.read())
             assert doc.tag == 'ok', "GeoNetwork login failed!"
             self.connected = True
 
@@ -174,7 +174,7 @@ def csw_gen_xml(self, layer, template):
 
     def csw_gen_anytext(self, xml):
         """ get all element data from an XML document """
-        xml = etree.fromstring(xml)
+        xml = dlxml.fromstring(xml)
         return ' '.join([value.strip() for value in xml.xpath('//text()')])
 
     def csw_request(self, layer, template):
@@ -201,7 +201,7 @@ def create_from_layer(self, layer):
             # set layer.uuid based on what GeoNetwork returns
             # this is needed for inserting FGDC metadata in GN
 
-            exml = etree.fromstring(response.read())
+            exml = dlxml.fromstring(response.read())
             identifier = exml.find(
                 '{%s}InsertResult/{%s}BriefRecord/identifier' %
                 (namespaces['csw'], namespaces['csw'])).text
@@ -255,7 +255,7 @@ def set_metadata_privs(self, uuid, privileges):
             # get the id of the data.
             request = urllib2.Request(get_dbid_url)
             response = self.urlopen(request)
-            doc = etree.fromstring(response.read())
+            doc = dlxml.fromstring(response.read())
             data_dbid = doc.find(
                 'metadata/{http://www.fao.org/geonetwork}info/id').text
 
@@ -295,7 +295,7 @@ def _geonetwork_get_group_ids(self):
             self.base, urllib.urlencode({'type': 'groups'}))
         request = urllib2.Request(get_groups_url)
         response = self.urlopen(request)
-        doc = etree.fromstring(response.read())
+        doc = dlxml.fromstring(response.read())
         groups = {}
         for gp in doc.findall('groups/group'):
             groups[gp.find('name').text.lower()] = gp.attrib['id']
@@ -311,7 +311,7 @@ def _geonetwork_get_operation_ids(self):
             self.base, urllib.urlencode({'type': 'operations'}))
         request = urllib2.Request(get_ops_url)
         response = self.urlopen(request)
-        doc = etree.fromstring(response.read())
+        doc = dlxml.fromstring(response.read())
         ops = {}
         for op in doc.findall('operations/operation'):
             ops[op.find('name').text.lower()] = op.attrib['id']

geonode/catalogue/backends/pycsw_local.py

@@ -19,7 +19,7 @@
 #########################################################################
 
 import os
-from lxml import etree
+from defusedxml import lxml as dlxml
 from django.conf import settings
 from ConfigParser import SafeConfigParser
 from owslib.iso import MD_Metadata
@@ -76,7 +76,7 @@ def get_record(self, uuid):
         if len(results) < 1:
             return None
 
-        result = etree.fromstring(results).find('{http://www.isotc211.org/2005/gmd}MD_Metadata')
+        result = dlxml.fromstring(results).find('{http://www.isotc211.org/2005/gmd}MD_Metadata')
 
         if result is None:
             return None
@@ -96,7 +96,7 @@ def search_records(self, keywords, start, limit, bbox):
         with self.catalogue:
             lresults = self._csw_local_dispatch(keywords, keywords, start + 1, limit, bbox)
             # serialize XML
-            e = etree.fromstring(lresults)
+            e = dlxml.fromstring(lresults)
 
             self.catalogue.records = \
                 [MD_Metadata(x) for x in e.findall('//{http://www.isotc211.org/2005/gmd}MD_Metadata')]

geonode/catalogue/metadataxsl/views.py

@@ -22,6 +22,7 @@
 import logging
 
 from lxml import etree
+from defusedxml import lxml as dlxml
 
 from django.shortcuts import get_object_or_404
 from django.http import HttpResponse
@@ -53,7 +54,7 @@ def prefix_xsl_line(req, id):
     try:
         # generate an XML document (GeoNode's default is ISO)
         if resource.metadata_uploaded and resource.metadata_uploaded_preserve:
-            md_doc = etree.tostring(etree.fromstring(resource.metadata_xml))
+            md_doc = etree.tostring(dlxml.fromstring(resource.metadata_xml))
         else:
             md_doc = catalogue.catalogue.csw_gen_xml(resource, 'catalogue/full_metadata.xml')
         xml = md_doc

geonode/catalogue/models.py

@@ -24,6 +24,7 @@
 from django.conf import settings
 from django.db.models import signals
 from lxml import etree
+from defusedxml import lxml as dlxml
 from geonode.layers.models import Layer
 from geonode.documents.models import Document
 from geonode.catalogue import get_catalogue
@@ -85,7 +86,7 @@ def catalogue_post_save(instance, sender, **kwargs):
 
         # generate an XML document (GeoNode's default is ISO)
         if instance.metadata_uploaded and instance.metadata_uploaded_preserve:
-            md_doc = etree.tostring(etree.fromstring(instance.metadata_xml))
+            md_doc = etree.tostring(dlxml.fromstring(instance.metadata_xml))
         else:
             md_doc = catalogue.catalogue.csw_gen_xml(instance, 'catalogue/full_metadata.xml')
 

geonode/catalogue/views.py

@@ -22,6 +22,7 @@
 import os
 import logging
 import xml.etree.ElementTree as ET
+from defusedxml import lxml as dlxml
 from django.conf import settings
 from django.http import HttpResponse, HttpResponseRedirect
 from django.shortcuts import render
@@ -173,7 +174,7 @@ def csw_global_dispatch(request):
             ET.register_namespace(prefix, uri)
 
         if access_token and not access_token.is_expired():
-            tree = ET.fromstring(content)
+            tree = dlxml.fromstring(content)
             for online_resource in tree.findall(
                     '*//gmd:CI_OnlineResource', spaces):
                 try:

geonode/geoserver/helpers.py

@@ -54,6 +54,7 @@
 from geoserver.workspace import Workspace
 from gsimporter import Client
 from lxml import etree
+from defusedxml import lxml as dlxml
 from owslib.wcs import WebCoverageService
 from owslib.wms import WebMapService
 from geonode import GeoNodeException
@@ -188,7 +189,7 @@ def extract_name_from_sld(gs_catalog, sld, sld_file=None):
             dom = etree.XML(sld)
         elif sld_file and isfile(sld_file):
             sld = open(sld_file, "r").read()
-            dom = etree.parse(sld_file)
+            dom = dlxml.parse(sld_file)
     except Exception:
         logger.exception("The uploaded SLD file is not valid XML")
         raise Exception(
@@ -318,7 +319,7 @@ def set_layer_style(saved_layer, title, sld, base_file=None):
             etree.XML(sld)
         elif base_file and isfile(base_file):
             sld = open(base_file, "r").read()
-            etree.parse(base_file)
+            dlxml.parse(base_file)
     except Exception:
         logger.exception("The uploaded SLD file is not valid XML")
         # raise Exception("The uploaded SLD file is not valid XML")
@@ -913,7 +914,7 @@ def set_attributes_from_geoserver(layer, overwrite=False):
             # The code below will fail if http_client cannot be imported  or
             # WFS not supported
             req, body = http_client.get(dft_url)
-            doc = etree.fromstring(body)
+            doc = dlxml.fromstring(body)
             path = ".//{xsd}extension/{xsd}sequence/{xsd}element".format(
                 xsd="{http://www.w3.org/2001/XMLSchema}")
             attribute_map = [[n.attrib["name"], n.attrib["type"]] for n in doc.findall(
@@ -961,7 +962,7 @@ def set_attributes_from_geoserver(layer, overwrite=False):
         })
         try:
             req, body = http_client.get(dc_url)
-            doc = etree.fromstring(body)
+            doc = dlxml.fromstring(body)
             path = ".//{wcs}Axis/{wcs}AvailableKeys/{wcs}Key".format(
                 wcs="{http://www.opengis.net/wcs/1.1.1}")
             attribute_map = [[n.text, "raster"] for n in doc.findall(path)]
@@ -1575,7 +1576,7 @@ def wps_execute_layer_attribute_statistics(layer_name, field):
         headers=headers,
         user=ogc_server_settings.credentials.username)
 
-    exml = etree.fromstring(content)
+    exml = dlxml.fromstring(content)
 
     result = {}
 
@@ -1618,13 +1619,13 @@ def _stylefilterparams_geowebcache_layer(layer_name):
     # check/write GWC filter parameters
     import xml.etree.ElementTree as ET
     body = None
-    tree = ET.fromstring(_)
+    tree = dlxml.fromstring(_)
     param_filters = tree.findall('parameterFilters')
     if param_filters and len(param_filters) > 0:
         if not param_filters[0].findall('styleParameterFilter'):
             style_filters_xml = "<styleParameterFilter><key>STYLES</key>\
                 <defaultValue></defaultValue></styleParameterFilter>"
-            style_filters_elem = ET.fromstring(style_filters_xml)
+            style_filters_elem = dlxml.fromstring(style_filters_xml)
             param_filters[0].append(style_filters_elem)
             body = ET.tostring(tree)
     if body:
@@ -1694,7 +1695,7 @@ def style_update(request, url):
             style_name = os.path.basename(request.path)
         else:
             try:
-                tree = ET.ElementTree(ET.fromstring(request.body))
+                tree = ET.ElementTree(dlxml.fromstring(request.body))
                 elm_namedlayer_name = tree.findall(
                     './/{http://www.opengis.net/sld}Name')[0]
                 elm_user_style_name = tree.findall(

geonode/geoserver/views.py

@@ -24,6 +24,7 @@
 import logging
 import traceback
 from lxml import etree
+from defusedxml import lxml as dlxml
 from os.path import isfile
 
 from urlparse import urlsplit, urljoin
@@ -776,7 +777,7 @@ def get_capabilities(request, layerid=None, user=None,
                         namespaces = {'wms': 'http://www.opengis.net/wms',
                                       'xlink': 'http://www.w3.org/1999/xlink',
                                       'xsi': 'http://www.w3.org/2001/XMLSchema-instance'}
-                        layercap = etree.fromstring(layercap)
+                        layercap = dlxml.fromstring(layercap)
                         rootdoc = etree.ElementTree(layercap)
                         format_online_resource(workspace, layername, rootdoc, namespaces)
                         service_name = rootdoc.find('.//wms:Service/wms:Name', namespaces)

geonode/layers/metadata.py

@@ -24,7 +24,7 @@
 # Standard Modules
 import logging
 import datetime
-from lxml import etree
+from defusedxml import lxml as dlxml
 
 # Geonode functionality
 from geonode import GeoNodeException
@@ -42,7 +42,7 @@ def set_metadata(xml):
 
     # check if document is XML
     try:
-        exml = etree.fromstring(xml)
+        exml = dxml.fromstring(xml)
     except Exception as err:
         raise GeoNodeException(
             'Uploaded XML document is not XML: %s' % str(err))

geonode/layers/views.py

@@ -471,12 +471,12 @@ def sld_definition(style):
         if wms_capabilities_resp.status_code >= 200 and wms_capabilities_resp.status_code < 400:
             wms_capabilities = wms_capabilities_resp.getvalue()
             if wms_capabilities:
-                import xml.etree.ElementTree as ET
+                from defusedxml import lxml as dlxml
                 namespaces = {'wms': 'http://www.opengis.net/wms',
                               'xlink': 'http://www.w3.org/1999/xlink',
                               'xsi': 'http://www.w3.org/2001/XMLSchema-instance'}
 
-                e = ET.fromstring(wms_capabilities)
+                e = dlxml.fromstring(wms_capabilities)
                 for atype in e.findall(
                         "./[wms:Name='%s']/wms:Dimension[@name='time']" % (layer.alternate), namespaces):
                     dim_name = atype.get('name')
@@ -584,12 +584,12 @@ def sld_definition(style):
         if wms_capabilities_resp.status_code >= 200 and wms_capabilities_resp.status_code < 400:
             wms_capabilities = wms_capabilities_resp.getvalue()
             if wms_capabilities:
-                import xml.etree.ElementTree as ET
+                from defusedxml import lxml as dlxml
                 namespaces = {'wms': 'http://www.opengis.net/wms',
                               'xlink': 'http://www.w3.org/1999/xlink',
                               'xsi': 'http://www.w3.org/2001/XMLSchema-instance'}
 
-                e = ET.fromstring(wms_capabilities)
+                e = dlxml.fromstring(wms_capabilities)
                 for atype in e.findall(
                         "./[wms:Name='%s']/wms:Dimension[@name='time']" % (layer.alternate), namespaces):
                     dim_name = atype.get('name')

geonode/maps/tests.py

@@ -22,7 +22,7 @@
 from geonode.tests.base import GeoNodeBaseTestSupport
 
 from datetime import datetime
-from lxml import etree
+from defusedxml import lxml as dlxml
 
 from django.core.urlresolvers import reverse
 
@@ -265,7 +265,7 @@ def test_map_to_wmc(self):
         self.assertEquals(response.status_code, 200)
 
         # check specific XPaths
-        wmc = etree.fromstring(response.content)
+        wmc = dlxml.fromstring(response.content)
 
         namespace = '{http://www.opengis.net/context}'
         title = '{ns}General/{ns}Title'.format(ns=namespace)

geonode/maps/views.py

@@ -897,12 +897,12 @@ def sld_definition(style):
             if wms_capabilities_resp.status_code >= 200 and wms_capabilities_resp.status_code < 400:
                 wms_capabilities = wms_capabilities_resp.getvalue()
                 if wms_capabilities:
-                    import xml.etree.ElementTree as ET
+                    from defusedxml import lxml as dlxml
                     namespaces = {'wms': 'http://www.opengis.net/wms',
                                   'xlink': 'http://www.w3.org/1999/xlink',
                                   'xsi': 'http://www.w3.org/2001/XMLSchema-instance'}
 
-                    e = ET.fromstring(wms_capabilities)
+                    e = dlxml.fromstring(wms_capabilities)
                     for atype in e.findall(
                             "./[wms:Name='%s']/wms:Dimension[@name='time']" % (layer.alternate), namespaces):
                         dim_name = atype.get('name')

geonode/monitoring/tests.py

@@ -26,7 +26,7 @@
 
 import os
 import pytz
-from xml.etree.ElementTree import fromstring
+from defusedxml import lxml as dlxml
 import json
 import xmljson
 from decimal import Decimal
@@ -56,8 +56,8 @@
 req_err_xml = open(req_err_path, 'rt').read()
 req_xml = open(req_path, 'rt').read()
 
-req_big = xmljson.yahoo.data(fromstring(req_xml))
-req_err_big = xmljson.yahoo.data(fromstring(req_err_xml))
+req_big = xmljson.yahoo.data(dlxml.fromstring(req_xml))
+req_err_big = xmljson.yahoo.data(dlxml.fromstring(req_err_xml))
 
 
 @override_settings(USE_TZ=True)

geonode/monitoring/utils.py

@@ -35,7 +35,7 @@
 from bs4 import BeautifulSoup as bs
 from requests.auth import HTTPBasicAuth
 from datetime import datetime, timedelta
-from xml.etree import ElementTree as etree
+from defusedxml import lxml as dlxml
 
 from django.conf import settings
 from django.db.models.fields.related import RelatedField
@@ -179,7 +179,7 @@ def get_request(self, href, format=format):
         except (ValueError, TypeError,):
             # traceback.print_exc()
             try:
-                data = etree.fromstring(r.content)
+                data = dlxml.fromstring(r.content)
             except Exception as err:
                 log.debug("Cannot parse xml contents for %s: %s", href, err, exc_info=err)
                 data = bs(r.content)

geonode/qgis_server/gis_tools.py

@@ -23,7 +23,7 @@
 from urllib import urlencode, urlretrieve
 from os.path import splitext
 from math import atan, degrees, sinh, pi
-from lxml import etree
+from defusedxml import lxml as dlxml
 
 from django.conf import settings as geonode_config
 
@@ -63,7 +63,7 @@ def set_attributes(layer, overwrite=False):
         try:
             temp_file = urlretrieve(dft_url)[0]
             with open(temp_file, 'r') as wfs_file:
-                doc = etree.fromstring(wfs_file.read())
+                doc = dlxml.fromstring(wfs_file.read())
 
             path = './/{xsd}extension/{xsd}sequence/{xsd}element'.format(
                 xsd='{http://www.w3.org/2001/XMLSchema}')