From 25d620f569316089bcec9e481739297145f1b6fd Mon Sep 17 00:00:00 2001 From: Alessio Fabiani Date: Thu, 24 May 2018 12:02:37 +0200 Subject: [PATCH] - Backport fixes from master branch --- .travis.yml | 4 +- Dockerfile | 2 +- README | 8 +- docs/organizational/contribute/README | 8 +- .../customize/setup_admin.txt | 16 +- .../admin/install/custom_install.txt | 22 +- docs/tutorials/admin/socialaccounts/index.txt | 28 +- .../install_geonode_application.txt | 18 +- .../geonode_update/26_to_27/index.txt | 4 +- .../install_and_admin/quick_install.txt | 8 +- .../setup_on_centos/install_geonode.txt | 2 +- geonode/__init__.py | 1 + geonode/api/api.py | 2 +- geonode/api/resourcebase_api.py | 6 +- geonode/api/views.py | 2 +- geonode/base/__init__.py | 2 +- geonode/base/fields.py | 2 +- geonode/base/populate_test_data.py | 29 +- geonode/base/templatetags/base_tags.py | 6 +- geonode/client/admin_actions.py | 112 ++++---- .../templates/geoext/maps/map_include.html | 1 + geonode/client/utils.py | 10 +- geonode/context_processors.py | 1 + geonode/contrib/dynamic/postgis.py | 2 +- geonode/contrib/exif/utils.py | 4 +- geonode/contrib/metadataxsl/tests.py | 1 + geonode/contrib/metadataxsl/views.py | 4 +- geonode/contrib/monitoring/collector.py | 12 +- geonode/contrib/monitoring/models.py | 15 +- geonode/contrib/monitoring/tests.py | 12 +- geonode/contrib/monitoring/utils.py | 4 +- geonode/contrib/nlp/utils.py | 6 +- geonode/contrib/slack/utils.py | 8 +- geonode/documents/forms.py | 5 +- geonode/documents/renderers.py | 2 +- geonode/documents/search_indexes.py | 2 +- geonode/documents/tests.py | 34 +-- geonode/geoserver/helpers.py | 60 ++-- geonode/geoserver/ows.py | 6 +- geonode/geoserver/signals.py | 4 +- geonode/geoserver/tests.py | 1 + geonode/geoserver/upload.py | 3 +- geonode/geoserver/views.py | 10 +- geonode/groups/views.py | 1 + geonode/layers/forms.py | 2 +- geonode/layers/models.py | 3 +- geonode/layers/search_indexes.py | 2 +- .../layers/templates/layers/layer_detail.html | 37 ++- geonode/layers/tests.py | 24 +- geonode/layers/utils.py | 16 +- geonode/layers/views.py | 14 +- geonode/maps/forms.py | 2 +- geonode/maps/models.py | 2 +- geonode/maps/qgis_server_views.py | 8 +- geonode/maps/search_indexes.py | 2 +- geonode/maps/templates/maps/map_detail.html | 216 +++++++------- geonode/maps/tests.py | 20 +- geonode/maps/views.py | 10 +- geonode/messaging/consumer.py | 4 +- geonode/messaging/producer.py | 2 +- geonode/messaging/tests.py | 2 +- geonode/people/__init__.py | 1 + geonode/people/admin.py | 2 + geonode/proxy/views.py | 8 +- geonode/qgis_server/__init__.py | 1 + geonode/qgis_server/context_processors.py | 1 + geonode/qgis_server/gis_tools.py | 2 +- geonode/qgis_server/helpers.py | 2 +- geonode/qgis_server/models.py | 6 +- geonode/qgis_server/signals.py | 8 +- geonode/qgis_server/views.py | 16 +- geonode/qgis_server/xml_utilities.py | 2 +- geonode/security/utils.py | 10 +- geonode/security/views.py | 6 +- geonode/services/__init__.py | 1 + geonode/services/serviceprocessors/arcgis.py | 2 +- geonode/services/serviceprocessors/wms.py | 2 +- geonode/settings.py | 11 +- geonode/social/__init__.py | 1 + geonode/social/signals.py | 4 +- geonode/tests/base.py | 2 +- geonode/tests/bdd/e2e/conftest.py | 4 +- geonode/tests/integration.py | 31 +- geonode/tests/suite/runner.py | 5 +- geonode/tests/utils.py | 8 +- geonode/upload/__init__.py | 15 + geonode/upload/files.py | 28 +- geonode/upload/models.py | 2 +- geonode/upload/upload.py | 223 +++------------ geonode/upload/upload_validators.py | 14 +- geonode/upload/utils.py | 268 +++++++++++++++++- geonode/upload/views.py | 12 +- geonode/utils.py | 10 +- requirements.txt | 13 +- requirements_docs.txt | 2 +- scripts/cloud/fabfile.py | 2 +- scripts/jenkins/jenkins-geonode-deb-dev.sh | 5 +- scripts/jenkins/jenkins-geonode-deb.sh | 5 +- 98 files changed, 902 insertions(+), 689 deletions(-) diff --git a/.travis.yml b/.travis.yml index dc4629d8318..ac0910d0ab9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -83,8 +83,8 @@ install: - sudo apt-get update - sudo apt-get install -y --force-yes oracle-java8-installer ant maven2 --no-install-recommends - sudo update-java-alternatives --set java-8-oracle - - pip install -r requirements.txt - - pip install -e . + - pip install -r requirements.txt --upgrade + - pip install -e . --upgrade - pip install pygdal==`gdal-config --version` - pip install codecov diff --git a/Dockerfile b/Dockerfile index 7fd61fdca78..fbe441dac6a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ MAINTAINER GeoNode development team COPY requirements.txt /usr/src/app/ RUN pip install --upgrade pip -RUN pip install -r requirements.txt +RUN pip install -r requirements.txt --upgrade RUN python manage.py makemigrations --settings=geonode.settings RUN python manage.py migrate --settings=geonode.settings diff --git a/README b/README index f2a499734d1..a27b0ca7e49 100644 --- a/README +++ b/README @@ -66,7 +66,7 @@ Or if you want to use the provided Makefile:: # complete development setup make develop - + .. note:: For development you need to add ``geonode`` alias into ``/etc/hosts/`` file as following:: $ sudo vim /etc/hosts @@ -108,8 +108,8 @@ Ubuntu development build instructions using an isolated virtual environment (tes cd geonode # Install pip dependencies - pip install -r requirements.txt --no-deps - pip install -e . + pip install -r requirements.txt --upgrade + pip install -e . --upgrade --no-cache pip install pygdal==1.11.3.3 For Ubuntu 16.04 you should create your virtual-env like this instead (so with site packages):: @@ -278,7 +278,7 @@ Mac OSX Development Build Instructions:: npm install -g grunt-cli #Install pip dependencies - pip install -e . + pip install -e . --upgrade --no-cache #Paver handles dependencies for Geonode, first setup (this will download and update your python dependencies - ensure you're in a virtualenv) paver setup diff --git a/docs/organizational/contribute/README b/docs/organizational/contribute/README index 7930ee5edfe..ea0bdb29b27 100644 --- a/docs/organizational/contribute/README +++ b/docs/organizational/contribute/README @@ -62,7 +62,7 @@ Or if you want to use the provided Makefile:: # complete development setup make develop - + .. note:: For development you need to add ``geonode`` alias into ``/etc/hosts/`` file as following:: $ sudo vim /etc/hosts @@ -102,8 +102,8 @@ Ubuntu development build instructions using an isolated virtual environment (tes cd geonode # Install pip dependencies - pip install -r requirements.txt --no-deps - pip install -e . + pip install -r requirements.txt --upgrade + pip install -e . --upgrade --no-cache pip install pygdal==1.11.3.3 For Ubuntu 16.04 you should create your virtual-env like this instead (so with site packages):: @@ -266,7 +266,7 @@ Mac OSX Development Build Instructions:: npm install -g grunt-cli #Install pip dependencies - pip install -e . + pip install -e . --upgrade --no-cache #Paver handles dependencies for Geonode, first setup (this will download and update your python dependencies - ensure you're in a virtualenv) paver setup diff --git a/docs/tutorials/admin/customize_lookfeel/customize/setup_admin.txt b/docs/tutorials/admin/customize_lookfeel/customize/setup_admin.txt index 9b08b0fbc8e..73a98f6a136 100644 --- a/docs/tutorials/admin/customize_lookfeel/customize/setup_admin.txt +++ b/docs/tutorials/admin/customize_lookfeel/customize/setup_admin.txt @@ -10,11 +10,11 @@ If you are working remotely, you should first connect to the machine that has yo .. code-block:: console :linenos: - + $ apt-get install python-django $ django-admin startproject my_geonode --template=https://github.com/GeoNode/geonode-project/archive/master.zip -epy,rst $ cd my_geonode - $ sudo pip install -e . + $ sudo pip install -e . --upgrade --no-cache $ python manage.py migrate .. note:: You should NOT use the name *geonode* for your project as it will conflict with your default geonode package name. @@ -25,11 +25,11 @@ Make sure that the directories are reachable and have the correct rights for the .. code-block:: console :linenos: - + $ sudo chown -Rf geonode: * $ sudo chmod -Rf 775 my_geonode -If you have a brand new installation of GeoNode, rename the **/home/geonode/geonode/local_settings.py.sample** to **local_settings.py** and edit it's content by setting the SITEURL and SITENAME. This file will be your main settings file for your project. It inherits all the settings from the original one plus you can override the ones that you need. +If you have a brand new installation of GeoNode, rename the **/home/geonode/geonode/local_settings.py.sample** to **local_settings.py** and edit it's content by setting the SITEURL and SITENAME. This file will be your main settings file for your project. It inherits all the settings from the original one plus you can override the ones that you need. .. note:: You can also decide to copy the **/home/geonode/geonode/local_settings.py.sample** to **/path/to/my_geonode/my_geonode/local_settings.py** in order to keep all the custom settings confined into the new project. @@ -82,7 +82,7 @@ Edit the file /etc/apache2/sites-available/geonode.conf and modify the **Documen Allow from all IndexOptions FancyIndexing - + ... Then regenerate the static **JavaScript** and **CSS** files from **/path/to/my_geonode/** and restart apache @@ -144,7 +144,7 @@ It is recommended that you immediately put your new project under source code re :linenos: $ sudo git remote add origin - + #. Add your project files to the repository: @@ -161,7 +161,7 @@ It is recommended that you immediately put your new project under source code re # Those two command must be issued ONLY once $ sudo git config --global user.email "my@email" $ sudo git config --global user.name "myuser" - + $ sudo git commit -am "Initial commit" #. Push to the remote repository: @@ -237,7 +237,7 @@ python module may not be available) is: .. code-block: python :linenos: - + try: from local_settings import * except: diff --git a/docs/tutorials/admin/install/custom_install.txt b/docs/tutorials/admin/install/custom_install.txt index 202ee99dc61..3b5a4047a9a 100644 --- a/docs/tutorials/admin/install/custom_install.txt +++ b/docs/tutorials/admin/install/custom_install.txt @@ -45,7 +45,7 @@ This includes: * Python development libraries * PostgreSQL database * The GDAL, GEOS, and OGR geospatial software libraries - + For detailed information on dependencies please refer to the section :ref:`dependencies`. For convenience, appropriate commands to retrieve these dependencies on Ubuntu @@ -100,7 +100,7 @@ installation-configuration of GeoNode: #. `Configure using the ubuntu config file`_. This is the recommended way, using the provided .sh scripts #. `Configure manually`_, doing each step by hand as opposed to using the .sh scripts - + In the end you will have the same running instance of Geonode. Configure Manually @@ -120,7 +120,7 @@ Download the code using git. $ cd geonode # Installs django etc. - $ sudo pip install -e . + $ sudo pip install -e . --upgrade --no-cache # Downloads geoserver.war $ sudo paver setup @@ -192,7 +192,7 @@ To create the databases (the geonode one and the spatial for shapefile imports): $ psql -d geonode_data -c 'GRANT ALL ON spatial_ref_sys TO PUBLIC;' $ exit -This creates databases called *geonode* and *geonode_data* +This creates databases called *geonode* and *geonode_data* (which automatically has the postgis extension as well!) with owner *geonode*. @@ -491,7 +491,7 @@ pre-installed. However, some manual configuration may still be needed in case of not standard installation. -**Configure GeoServer** +**Configure GeoServer** Configure GeoServer with the location of the GeoNode site, used for authentication (so that GeoServer can recognize logins from the main site). @@ -508,12 +508,12 @@ so: GEONODE_BASE_URL http://localhost/ - + The ```` tag should enclose the URL to the Django application homepage. -And secondly, update the value of the ``baseUrl`` tag in +And secondly, update the value of the ``baseUrl`` tag in ``data/security/auth/geonodeAuthProvider/config.xml``: .. code-block:: xml @@ -544,16 +544,16 @@ Download the code using git. $ git clone https://github.com/GeoNode/geonode.git $ cd geonode -* Installer configuration file *(located in ``package/support/config-ubuntu.sh``)* +* Installer configuration file *(located in ``package/support/config-ubuntu.sh``)* This is the first thing to edit. * The installer script *(located in ``package/install.sh``)* Second thing to edit. - + #. First open the ``package/support/config-ubuntu.sh`` installer configuration file. Now provide the details of your installation, such as where your web server looks for documents. -#. Second, open ``install.sh`` and configure to your needs. +#. Second, open ``install.sh`` and configure to your needs. .. note:: you might have to change the postgresql and postgis versions in this file, according to the versions you use! @@ -573,6 +573,6 @@ Once edited (if necessary) run the ``./install.sh`` file with the following comm # create the IP address to your address $ sudo geonode-updateip youraddress - + GeoNode can now be accessed at http://localhost diff --git a/docs/tutorials/admin/socialaccounts/index.txt b/docs/tutorials/admin/socialaccounts/index.txt index 80d7669aad9..2353ddaf54e 100644 --- a/docs/tutorials/admin/socialaccounts/index.txt +++ b/docs/tutorials/admin/socialaccounts/index.txt @@ -23,7 +23,7 @@ Base concepts and objects * `django-invitations `_ has also been integrated as a dependency of geonode and is used for managing invitations to new users. This functionality was previously provided by geonode-user-accounts; * `django-allauth `_ has been extended in order to provide the following additional features: - + - Automatically registering an e-mail with a user when the e-mail is used to connect to a social account; - Automatically extract information from the user's social account and use that to enhance the user's profile fields on geonode. This was implemented in a pluggable way, allowing custom installs to configure it for other providers; - Allow approval of new registrations by staff members before allowing new users to login. This functionality was previously provided by geonode-user-accounts. @@ -33,7 +33,7 @@ Base concepts and objects .. image:: img/001_screenshot.png * When properly configured, the login and register pages now display the possibility to login with social accounts - + .. image:: img/002_screenshot.png @@ -44,8 +44,8 @@ Installation .. code:: python - pip install -r requirements.txt - pip install -e . + pip install -r requirements.txt --upgrade + pip install -e . --upgrade --no-cache pip uninstall geonode-user-accounts -y pip uninstall django-user-accounts -y @@ -157,9 +157,9 @@ Usage This will require a persoanl or business account, which can access to the ``developers`` sections of LinkedIn and Facebook and create and configure new ``Applications``. That account won't be visibile to the GeoNode users. This is needed only to generate OAuth2 ``Client ID`` and ``Client Secret`` Authorization Keys. - + In the following sections we will see in details how to configure them for both LinkedIn and Facebook. - + LinkedIn Application ++++++++++++++++++++ (ref.: http://django-allauth.readthedocs.io/en/latest/providers.html) @@ -188,10 +188,10 @@ LinkedIn Application 5. Add OAuth 2.0 Authorized Redirect URLs: .. code:: python - + http://geonode.geo-solutions.it/account/linkedin_oauth2/login/callback/ http://geonode.geo-solutions.it/account/linkedin/login/callback/ - + .. image:: img/008_socialaouth.png 6. Save @@ -243,10 +243,10 @@ Facebook Application 5. Cut and Paste the ``App ID`` and ``Secret Key`` on the related fields .. code:: python - + ClientID <--> App Id Client Secret <--> Secret Key - + .. image:: img/018_socialaouth.png 6. Save @@ -298,7 +298,7 @@ Facebook Application 18. Add the valid ``redirect URIs``: .. code:: python - + http://geonode.geo-solutions.it/account/facebook/login/callback/ http://geondoe.geo-solutions.it/account/login/ @@ -312,9 +312,9 @@ Login by using Existing Accounts on GeoNode ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If you want to enable an already existing user account to login through social apps, you need to associate it to social accounts. - + Usually this could be done only by the current user, since this operation requires authentication on its social accounts. - + In order to do that you need to go to the User Profile Settings .. image:: img/032_socialaouth.png @@ -325,7 +325,7 @@ Login by using Existing Accounts on GeoNode .. image:: img/033_socialaouth.png And actually connect them - + .. image:: img/034_socialaouth.png .. image:: img/035_socialaouth.png diff --git a/docs/tutorials/install_and_admin/geonode_install/install_geonode_application.txt b/docs/tutorials/install_and_admin/geonode_install/install_geonode_application.txt index a458c3a6e99..47e98f82963 100644 --- a/docs/tutorials/install_and_admin/geonode_install/install_geonode_application.txt +++ b/docs/tutorials/install_and_admin/geonode_install/install_geonode_application.txt @@ -37,7 +37,7 @@ packages .. code-block:: bash $ sudo apt-get update - + $ sudo apt-get install python-virtualenv python-dev libxml2 libxml2-dev libxslt1-dev zlib1g-dev libjpeg-dev libpq-dev git default-jdk $ sudo apt-get install build-essential openssh-server gettext nano vim unzip zip patch git-core postfix @@ -50,7 +50,7 @@ packages $ sudo apt-get install gcc apache2 libapache2-mod-wsgi libgeos-dev libjpeg-dev libpng-dev libpq-dev libproj-dev libxml2-dev libxslt-dev $ sudo apt-add-repository ppa:ubuntugis/ubuntugis-testing && sudo apt-get update && sudo apt-get upgrade $ sudo apt-get install gdal-bin libgdal20 libgdal-dev - $ sudo apt-get install python-gdal python-pycurl python-imaging python-pastescript python-psycopg2 python-urlgrabber + $ sudo apt-get install python-gdal python-pycurl python-imaging python-pastescript python-psycopg2 python-urlgrabber $ sudo apt-get install postgresql postgis postgresql-9.5-postgis-scripts postgresql-contrib $ sudo apt-get install tomcat8 @@ -101,7 +101,7 @@ First of all we need to prepare a new Python Virtual Environment: Let's activate the new `geonode` Python Virtual Environment: .. code-block:: bash - + $ workon geonode Move into the `geonode` home folder @@ -113,12 +113,12 @@ Move into the `geonode` home folder We are going to install GeoNode as a dependency of a **Customized DJango Project** .. note:: - A custom project is a DJango application with *ad hoc* configuration and folders, which allows you to + A custom project is a DJango application with *ad hoc* configuration and folders, which allows you to extend the original **GeoNode** code without actually dealing or modifying the main source code. - + This will allow you to easily customize your GeoNode instance, modify the theme, add new functionalities and so on, and also being able to keep updated with the GeoNode latest source code. - + For more deails please check https://github.com/GeoNode/geonode-project/tree/master .. code-block:: bash @@ -132,7 +132,7 @@ Let's install the GeoNode dependencies and packages into the Python Virtual Envi $ cd my_geonode - # Find the closest pygdal version. + # Find the closest pygdal version. # Example: 2.2.1 ... 2.2.1.3, ... $ gdal-config --version && pip install pygdal== @@ -142,8 +142,8 @@ Let's install the GeoNode dependencies and packages into the Python Virtual Envi -e git://github.com/GeoNode/geonode.git@2.8.0#egg=geonode pygdal==2.2.1.3 - $ pip install -r requirements.txt - $ pip install -e . + $ pip install -r requirements.txt --upgrade + $ pip install -e . --upgrade --no-cache In the next section we are going to setup PostgreSQL Databases for GeoNode and finalize the setup diff --git a/docs/tutorials/install_and_admin/geonode_update/26_to_27/index.txt b/docs/tutorials/install_and_admin/geonode_update/26_to_27/index.txt index d121f8bb7d8..78511009826 100644 --- a/docs/tutorials/install_and_admin/geonode_update/26_to_27/index.txt +++ b/docs/tutorials/install_and_admin/geonode_update/26_to_27/index.txt @@ -51,8 +51,8 @@ Upgrade Development Environment .. code-block:: bash pip install pip --upgrade - pip install -r requirements.txt - pip install -e . + pip install -r requirements.txt --upgrade + pip install -e . --upgrade --no-cache # WARNING: your GDAL version might be different. Use the right one accordingly to gdal-config --version pip install pygdal==2.2.1.3 diff --git a/docs/tutorials/install_and_admin/quick_install.txt b/docs/tutorials/install_and_admin/quick_install.txt index 3269be44684..dec65dc7d2d 100644 --- a/docs/tutorials/install_and_admin/quick_install.txt +++ b/docs/tutorials/install_and_admin/quick_install.txt @@ -97,8 +97,8 @@ This option installs geonode in a virtual environment. This option is very usefu cd geonode # Install pip dependencies - pip install -r requirements.txt --no-deps - pip install -e . + pip install -r requirements.txt --upgrade + pip install -e . --upgrade --no-cache sudo add-apt-repository ppa:ubuntugis/ppa && sudo apt-get update sudo apt-get install gdal-bin @@ -204,7 +204,7 @@ Install Python native dependencies, this command will look for and install binar Install GeoNode in the local virtualenv:: - pip install -e . + pip install -e . --upgrade --no-cache You have two options to set up the GEOS and GDAL libraries. Either create an environment variable or add the variables to your local_settings.py file as below. @@ -269,7 +269,7 @@ On a production environment, remember to refresh also the "static_root" folder:: Install pip dependencies:: - pip install -e . + pip install -e . --upgrade --no-cache Paver handles dependencies for Geonode, first setup (this will download and update your python dependencies - ensure you're in a virtualenv):: diff --git a/docs/tutorials/install_and_admin/setup_on_centos/install_geonode.txt b/docs/tutorials/install_and_admin/setup_on_centos/install_geonode.txt index 57bdbb272de..52b7aab1489 100644 --- a/docs/tutorials/install_and_admin/setup_on_centos/install_geonode.txt +++ b/docs/tutorials/install_and_admin/setup_on_centos/install_geonode.txt @@ -54,7 +54,7 @@ Move the sources into project folder:: Navigate to sources folder and install required packages:: cd /opt/apps/geonode/geonode - sudo pip install -e . + sudo pip install -e . --upgrade --no-cache .. _geonode_install_settings: diff --git a/geonode/__init__.py b/geonode/__init__.py index 1a6699a7fcb..d9955e7a086 100644 --- a/geonode/__init__.py +++ b/geonode/__init__.py @@ -22,6 +22,7 @@ __version__ = (2, 7, 7, 'unstable', 0) + default_app_config = "geonode.apps.AppConfig" diff --git a/geonode/api/api.py b/geonode/api/api.py index c87070560bc..adc8c7dbfe9 100644 --- a/geonode/api/api.py +++ b/geonode/api/api.py @@ -544,7 +544,7 @@ def populate_object(self, style): """:type: geonode.qgis_server.QGISServerLayer""" style.layer = qgis_layer.layer style.type = 'qml' - except: + except BaseException: pass return style diff --git a/geonode/api/resourcebase_api.py b/geonode/api/resourcebase_api.py index aa84465be5f..daeafcdb165 100644 --- a/geonode/api/resourcebase_api.py +++ b/geonode/api/resourcebase_api.py @@ -790,13 +790,13 @@ def populate_object(self, obj): # Default style try: obj.qgis_default_style = obj.qgis_layer.default_style - except: + except BaseException: obj.qgis_default_style = None # Styles try: obj.qgis_styles = obj.qgis_layer.styles - except: + except BaseException: obj.qgis_styles = [] return obj @@ -843,7 +843,7 @@ def patch_detail(self, request, **kwargs): layer_id = kwargs['id'] layer = Layer.objects.get(id=layer_id) - except: + except BaseException: return http.HttpBadRequest(reason=reason) from geonode.qgis_server.views import default_qml_style diff --git a/geonode/api/views.py b/geonode/api/views.py index e962897dcbd..b0aa8968572 100644 --- a/geonode/api/views.py +++ b/geonode/api/views.py @@ -77,7 +77,7 @@ def verify_token(request): if (request.POST and request.POST['token']): try: token = verify_access_token(request.POST['token']) - except Exception, e: + except Exception as e: return HttpResponse( json.dumps({ 'error': str(e) diff --git a/geonode/base/__init__.py b/geonode/base/__init__.py index 0e5351c9c2d..9afef92b283 100644 --- a/geonode/base/__init__.py +++ b/geonode/base/__init__.py @@ -25,7 +25,7 @@ class BaseAppConfig(NotificationsAppConfigBase): name = 'geonode.base' NOTIFICATIONS = (("request_download_resourcebase", _("Request to download a resource"), - _("A request for downloading a resource was sent")), + _("A request for downloading a resource was sent")), ) diff --git a/geonode/base/fields.py b/geonode/base/fields.py index e7753648304..3bcaab47df7 100644 --- a/geonode/base/fields.py +++ b/geonode/base/fields.py @@ -41,7 +41,7 @@ def __init__(self, label=None, required=True, help_text=None, widget=None): tkl = tk.keyword.filter(lang='en') choices_list.append((tkl[0].id, tkl[0].label)) fields_list.append(forms.MultipleChoiceField(choices=tuple(choices_list))) - except: + except BaseException: tb = traceback.format_exc() logger.error(tb) diff --git a/geonode/base/populate_test_data.py b/geonode/base/populate_test_data.py index f8829db8efb..4fbd62d4ecf 100644 --- a/geonode/base/populate_test_data.py +++ b/geonode/base/populate_test_data.py @@ -76,7 +76,7 @@ def reconnect_signals(): # django's dumpdata imgfile = StringIO.StringIO('GIF87a\x01\x00\x01\x00\x80\x01\x00\x00\x00\x00ccc,\x00' - '\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;') + '\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;') f = SimpleUploadedFile('test_img_file.gif', imgfile.read(), 'image/gif') @@ -139,15 +139,20 @@ def callable(): next_date = get_test_date() - layer_data = [('CA', 'abstract1', 'CA', 'geonode:CA', world_extent, next_date(), ('populartag', 'here'), elevation), - ('layer2', 'abstract2', 'layer2', 'geonode:layer2', world_extent, next_date(), ('populartag',), elevation), - ('uniquetitle', 'something here', 'mylayer', 'geonode:mylayer', world_extent, next_date(), ('populartag',), elevation), # flake8: noqa - ('common blar', 'lorem ipsum', 'foo', 'geonode:foo', world_extent, next_date(), ('populartag', 'layertagunique'), location), # flake8: noqa - ('common double it', 'whatever', 'whatever', 'geonode:whatever', [0, 1, 0, 1], next_date(), ('populartag',), location), # flake8: noqa - ('common double time', 'else', 'fooey', 'geonode:fooey', [0, 5, 0, 5], next_date(), ('populartag',), location), # flake8: noqa - ('common bar', 'uniqueabstract', 'quux', 'geonode:quux', [0, 10, 0, 10], next_date(), ('populartag',), biota), # flake8: noqa - ('common morx', 'lorem ipsum', 'fleem', 'geonode:fleem', [0, 50, 0, 50], next_date(), ('populartag',), biota), # flake8: noqa + ('layer2', 'abstract2', 'layer2', 'geonode:layer2', world_extent, next_date(), ('populartag',), elevation), + ('uniquetitle', 'something here', 'mylayer', 'geonode:mylayer', + world_extent, next_date(), ('populartag',), elevation), # flake8: noqa + ('common blar', 'lorem ipsum', 'foo', 'geonode:foo', world_extent, + next_date(), ('populartag', 'layertagunique'), location), # flake8: noqa + ('common double it', 'whatever', 'whatever', 'geonode:whatever', [ + 0, 1, 0, 1], next_date(), ('populartag',), location), # flake8: noqa + ('common double time', 'else', 'fooey', 'geonode:fooey', [ + 0, 5, 0, 5], next_date(), ('populartag',), location), # flake8: noqa + ('common bar', 'uniqueabstract', 'quux', 'geonode:quux', [ + 0, 10, 0, 10], next_date(), ('populartag',), biota), # flake8: noqa + ('common morx', 'lorem ipsum', 'fleem', 'geonode:fleem', [ + 0, 50, 0, 50], next_date(), ('populartag',), biota), # flake8: noqa ] document_data = [('lorem ipsum', 'common lorem ipsum', ('populartag',), world_extent, biota), @@ -269,7 +274,7 @@ def remove_models(obj_ids, type=None): for id in m_ids: m = Map.objects.get(pk=id) m.delete() - except: + except BaseException: pass elif type == 'layer': try: @@ -277,7 +282,7 @@ def remove_models(obj_ids, type=None): for id in l_ids: l = Layer.objects.get(pk=id) l.delete() - except: + except BaseException: pass elif type == 'document': try: @@ -285,7 +290,7 @@ def remove_models(obj_ids, type=None): for id in d_ids: d = Document.objects.get(pk=id) d.delete() - except: + except BaseException: pass diff --git a/geonode/base/templatetags/base_tags.py b/geonode/base/templatetags/base_tags.py index 4e0c373cbb2..d2f796f8aaf 100644 --- a/geonode/base/templatetags/base_tags.py +++ b/geonode/base/templatetags/base_tags.py @@ -106,7 +106,7 @@ def facets(context): kws = HierarchicalKeyword.objects.filter(name__iexact=keyword) for kw in kws: treeqs = treeqs | HierarchicalKeyword.get_tree(kw) - except: + except BaseException: # Ignore keywords not actually used? pass @@ -163,7 +163,7 @@ def facets(context): kws = HierarchicalKeyword.objects.filter(name__iexact=keyword) for kw in kws: treeqs = treeqs | HierarchicalKeyword.get_tree(kw) - except: + except BaseException: # Ignore keywords not actually used? pass @@ -248,7 +248,7 @@ def facets(context): kws = HierarchicalKeyword.objects.filter(name__iexact=keyword) for kw in kws: treeqs = treeqs | HierarchicalKeyword.get_tree(kw) - except: + except BaseException: # Ignore keywords not actually used? pass diff --git a/geonode/client/admin_actions.py b/geonode/client/admin_actions.py index 2ef4227266e..dcfb63c4d4e 100644 --- a/geonode/client/admin_actions.py +++ b/geonode/client/admin_actions.py @@ -65,7 +65,7 @@ def enable_theme(self, request, queryset): request, "'" + theme.name + "' Theme Activation Failed!", level=messages.WARNING) - except: + except BaseException: self.message_user( request, "Exception occurred while trying to activate theme: " + theme.name, @@ -126,7 +126,7 @@ def disable_theme(self, request, queryset): theme.is_enabled = False theme.save() self.message_user(request, "Disabled Theme: " + theme.name) - except: + except BaseException: self.message_user( request, "Exception occurred while trying to deactivate theme: " + theme.name, @@ -164,70 +164,70 @@ def disable_theme(self, request, queryset): def refresh_theme(self, request, queryset): - """ - Refresh the selected Themes - """ - siteObjs = queryset.filter(is_enabled=True).all() - if len(siteObjs) == 0: - self.message_user( - request, - "Please select at least one active custom theme to disable!", - level=messages.ERROR) - return - - if request.POST.get("post"): - value = None - for theme in siteObjs: - try: - if theme.is_enabled: - value = deactivate_theme(theme) - value = activate_theme(theme) + """ + Refresh the selected Themes + """ + siteObjs = queryset.filter(is_enabled=True).all() + if len(siteObjs) == 0: + self.message_user( + request, + "Please select at least one active custom theme to disable!", + level=messages.ERROR) + return - """ + if request.POST.get("post"): + value = None + for theme in siteObjs: + try: + if theme.is_enabled: + value = deactivate_theme(theme) + value = activate_theme(theme) + + """ Final checks if not exception has been raised until now """ - if value: - self.message_user(request, "Enabled Theme: " + theme.name) - else: - self.message_user( - request, - "'" + theme.name + "' Theme Activation Failed!", - level=messages.WARNING) - except: - self.message_user( - request, - "Exception occurred while trying to activate theme: " + theme.name, - level=messages.ERROR) - tb = traceback.format_exc() + if value: + self.message_user(request, "Enabled Theme: " + theme.name) + else: self.message_user( request, - tb, + "'" + theme.name + "' Theme Activation Failed!", level=messages.WARNING) - logger.debug(tb) - return - out = StringIO.StringIO() - call_command( - 'collectstatic', - '--noinput', - stdout=out) - value = out.getvalue() - if not value: + except BaseException: + self.message_user( + request, + "Exception occurred while trying to activate theme: " + theme.name, + level=messages.ERROR) + tb = traceback.format_exc() self.message_user( request, - "Collectstatic Regeneration Failed!", + tb, level=messages.WARNING) - else: - context = { - "objects_name": "Themes", - 'title': "Refresh GeoNode Custom Themes", - 'action_exec': "refresh_theme", - 'cancellable_themes': siteObjs, - 'action_checkbox_name': helpers.ACTION_CHECKBOX_NAME, - } - return TemplateResponse( + logger.debug(tb) + return + out = StringIO.StringIO() + call_command( + 'collectstatic', + '--noinput', + stdout=out) + value = out.getvalue() + if not value: + self.message_user( request, - 'admin/themes/confirm_cancel.html', - context=context) + "Collectstatic Regeneration Failed!", + level=messages.WARNING) + else: + context = { + "objects_name": "Themes", + 'title': "Refresh GeoNode Custom Themes", + 'action_exec': "refresh_theme", + 'cancellable_themes': siteObjs, + 'action_checkbox_name': helpers.ACTION_CHECKBOX_NAME, + } + return TemplateResponse( + request, + 'admin/themes/confirm_cancel.html', + context=context) enable_theme.short_description = "Activate Theme" diff --git a/geonode/client/templates/geoext/maps/map_include.html b/geonode/client/templates/geoext/maps/map_include.html index 69242d0fa51..1c0cff7bec8 100644 --- a/geonode/client/templates/geoext/maps/map_include.html +++ b/geonode/client/templates/geoext/maps/map_include.html @@ -4,6 +4,7 @@ + - + {% block thumbnail_script %} + + {% endblock thumbnail_script %} -{% include "_permissions_form_js.html" %} - - + }); + }; + {% endverbatim %} + }); + {% if DISPLAY_SOCIAL %} + (function() { + var po = document.createElement('script'); po.type = 'text/javascript'; po.async = true; + po.src = 'https://apis.google.com/js/plusone.js'; + var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(po, s); + })(); + {% endif %} + $("#comment_submit_btn").click(function(event) { + $.ajax({ + type: "POST", + url: $("#form_post_comment").attr('action'), + data: $("#form_post_comment").serialize(), + success: function() { + $('#form_post_comment_div').modal().find('form')[0].reset(); + $('#form_post_comment_div').modal('hide'); + $('#comments_section').load(window.location.pathname + ' #comments_section', + function(){$(this).children().unwrap()}) + } + }); + return false; + }); + + + {% block thumbnail_script %} + + {% endblock thumbnail_script %} -{% endblock %} + {% if GEONODE_SECURITY_ENABLED %} + {% include "_permissions_form_js.html" %} + {% endif %} +{% endblock extra_script %} diff --git a/geonode/maps/tests.py b/geonode/maps/tests.py index ea198bc7c41..2b07dd57cf1 100644 --- a/geonode/maps/tests.py +++ b/geonode/maps/tests.py @@ -823,9 +823,9 @@ def test_moderated_upload(self): content_type="text/json") self.assertEquals(response.status_code, 200) map_id = int(json.loads(response.content)['id']) - l = Map.objects.get(id=map_id) + _l = Map.objects.get(id=map_id) - self.assertTrue(l.is_published) + self.assertTrue(_l.is_published) with self.settings(ADMIN_MODERATE_UPLOADS=True): self.client.login(username=self.user, password=self.passwd) @@ -835,9 +835,9 @@ def test_moderated_upload(self): content_type="text/json") self.assertEquals(response.status_code, 200) map_id = int(json.loads(response.content)['id']) - l = Map.objects.get(id=map_id) + _l = Map.objects.get(id=map_id) - self.assertFalse(l.is_published) + self.assertFalse(_l.is_published) class MapsNotificationsTestCase(NotificationsTestsHelper): @@ -863,17 +863,17 @@ def testMapsNotifications(self): content_type="text/json") self.assertEquals(response.status_code, 200) map_id = int(json.loads(response.content)['id']) - l = Map.objects.get(id=map_id) + _l = Map.objects.get(id=map_id) self.assertTrue(self.check_notification_out('map_created', self.u)) - l.title = 'test notifications 2' - l.save() + _l.title = 'test notifications 2' + _l.save() self.assertTrue(self.check_notification_out('map_updated', self.u)) from dialogos.models import Comment - lct = ContentType.objects.get_for_model(l) + lct = ContentType.objects.get_for_model(_l) comment = Comment(author=self.u, name=self.u.username, - content_type=lct, object_id=l.id, - content_object=l, comment='test comment') + content_type=lct, object_id=_l.id, + content_object=_l, comment='test comment') comment.save() self.assertTrue(self.check_notification_out('map_comment', self.u)) diff --git a/geonode/maps/views.py b/geonode/maps/views.py index c4fe28a6452..c5bb9caf647 100644 --- a/geonode/maps/views.py +++ b/geonode/maps/views.py @@ -1098,7 +1098,7 @@ def perm_filter(layer): if j_layer["service"] is None: j_layers.remove(j_layer) continue - if (len([l for l in j_layers if l == j_layer])) > 1: + if (len([_l for _l in j_layers if _l == j_layer])) > 1: j_layers.remove(j_layer) mapJson = json.dumps(j_map) @@ -1141,7 +1141,7 @@ def perm_filter(layer): else: # we need to add the layer only once if len( - [l for l in downloadable_layers if l.name == lyr.name]) == 0: + [_l for _l in downloadable_layers if _l.name == lyr.name]) == 0: downloadable_layers.append(lyr) return render(request, template, context={ @@ -1274,7 +1274,7 @@ def snaplayer_config(layer, sources, user, access_token): snapshot = get_object_or_404(MapSnapshot, pk=decodedid) if snapshot.map == map_obj.map: config = json.loads(clean_config(snapshot.config)) - layers = [l for l in config["map"]["layers"]] + layers = [_l for _l in config["map"]["layers"]] sources = config["sources"] maplayers = [] for ordering, layer in enumerate(layers): @@ -1290,10 +1290,10 @@ def snaplayer_config(layer, sources, user, access_token): # map_obj, layer, config["sources"][layer["source"]], ordering)) config['map']['layers'] = [ snaplayer_config( - l, + _l, sources, user, - access_token) for l in maplayers] + access_token) for _l in maplayers] else: config = map_obj.viewer_json(user, access_token) return config diff --git a/geonode/messaging/consumer.py b/geonode/messaging/consumer.py index ebd498c5711..e722075accf 100644 --- a/geonode/messaging/consumer.py +++ b/geonode/messaging/consumer.py @@ -132,7 +132,7 @@ def on_geoserver_catalog(self, body, message): try: _update_layer_data(body, self.last_message) self.last_message = json.loads(body) - except: + except BaseException: logger.info("Could not encode message {!r}".format(body)) message.ack() logger.debug("on_geoserver_catalog: finished") @@ -143,7 +143,7 @@ def on_geoserver_data(self, body, message): try: _update_layer_data(body, self.last_message) self.last_message = json.loads(body) - except: + except BaseException: logger.info("Could not encode message {!r}".format(body)) message.ack() logger.debug("on_geoserver_data: finished") diff --git a/geonode/messaging/producer.py b/geonode/messaging/producer.py index 68839b184a3..5447cbf8d1b 100644 --- a/geonode/messaging/producer.py +++ b/geonode/messaging/producer.py @@ -66,7 +66,7 @@ def sync_if_local_memory(func, *args, **kwargs): worker = Consumer(connection, max_messages) try: worker.run(timeout=broker_socket_timeout) - except: + except BaseException: tb = traceback.format_exc() msg = "Exception while publishing message: {}".format(tb) logger.error(msg) diff --git a/geonode/messaging/tests.py b/geonode/messaging/tests.py index 756d94a1dee..294035dd833 100644 --- a/geonode/messaging/tests.py +++ b/geonode/messaging/tests.py @@ -42,5 +42,5 @@ def test_consumer(self): try: worker = Consumer(connection) self.assertTrue(worker is not None) - except: + except BaseException: self.fail("could not create a Consumer.") diff --git a/geonode/people/__init__.py b/geonode/people/__init__.py index 83d37fb65a2..bb683d31e00 100644 --- a/geonode/people/__init__.py +++ b/geonode/people/__init__.py @@ -33,4 +33,5 @@ class PeopleAppConfig(NotificationsAppConfigBase): def ready(self): super(PeopleAppConfig, self).ready() + default_app_config = 'geonode.people.PeopleAppConfig' diff --git a/geonode/people/admin.py b/geonode/people/admin.py index 602f9eb8678..d38e8c7cf0f 100644 --- a/geonode/people/admin.py +++ b/geonode/people/admin.py @@ -92,6 +92,7 @@ def get_form(self, request, obj=None, **kwargs): 'form': self.add_form, 'fields': admin.utils.flatten_fieldsets(self.add_fieldsets), }) + defaults.update(kwargs) return super(ProfileAdmin, self).get_form(request, obj, **defaults) @@ -130,6 +131,7 @@ def add_view(self, request, form_url='', extra_context=None): if extra_context is None: extra_context = {} username_field = self.model._meta.get_field(self.model.USERNAME_FIELD) + defaults = { 'auto_populated_fields': (), 'username_help_text': username_field.help_text, diff --git a/geonode/proxy/views.py b/geonode/proxy/views.py index eb388db8e9f..9e71a09a30a 100644 --- a/geonode/proxy/views.py +++ b/geonode/proxy/views.py @@ -274,12 +274,12 @@ def download(request, resourceid, sender=Layer): sld_file = open(sld_file_path, "w") sld_file.write(sld_remote_content.strip()) sld_file.close() - except: + except BaseException: traceback.print_exc() tb = traceback.format_exc() logger.debug(tb) - except: + except BaseException: traceback.print_exc() tb = traceback.format_exc() logger.debug(tb) @@ -304,7 +304,7 @@ def download(request, resourceid, sender=Layer): response = requests.get(link.url, stream=True, timeout=TIMEOUT) response.raw.decode_content = True shutil.copyfileobj(response.raw, link_file) - except: + except BaseException: traceback.print_exc() tb = traceback.format_exc() logger.debug(tb) @@ -315,7 +315,7 @@ def download(request, resourceid, sender=Layer): link_file = open(link_file, "w") link_file.write(link.url.strip()) link_file.close() - except: + except BaseException: traceback.print_exc() tb = traceback.format_exc() logger.debug(tb) diff --git a/geonode/qgis_server/__init__.py b/geonode/qgis_server/__init__.py index 3b3a490ef53..dfac570d150 100644 --- a/geonode/qgis_server/__init__.py +++ b/geonode/qgis_server/__init__.py @@ -20,4 +20,5 @@ BACKEND_PACKAGE = 'geonode.qgis_server' + default_app_config = "geonode.qgis_server.apps.AppConfig" diff --git a/geonode/qgis_server/context_processors.py b/geonode/qgis_server/context_processors.py index 392878847d7..ea0efee7519 100644 --- a/geonode/qgis_server/context_processors.py +++ b/geonode/qgis_server/context_processors.py @@ -56,5 +56,6 @@ def qgis_server_urls(request): 'MOSAIC_ENABLED', False), ) + defaults['style_upload_form'] = QGISLayerStyleUploadForm() return defaults diff --git a/geonode/qgis_server/gis_tools.py b/geonode/qgis_server/gis_tools.py index 403d8920403..cde4bcb07a1 100644 --- a/geonode/qgis_server/gis_tools.py +++ b/geonode/qgis_server/gis_tools.py @@ -71,7 +71,7 @@ def set_attributes(layer, overwrite=False): [n.attrib['name'], n.attrib['type']] for n in doc.findall( path) if n.attrib.get('name') and n.attrib.get('type')] - except: + except BaseException: attribute_map = [] else: attribute_map = [] diff --git a/geonode/qgis_server/helpers.py b/geonode/qgis_server/helpers.py index cf630551cf2..2c8adfedd91 100644 --- a/geonode/qgis_server/helpers.py +++ b/geonode/qgis_server/helpers.py @@ -856,7 +856,7 @@ def style_list(layer, internal=True, generating_qgis_capabilities=False): try: if not qgis_layer.default_style: set_default_style = True - except: + except BaseException: set_default_style = True if set_default_style and styles_obj: diff --git a/geonode/qgis_server/models.py b/geonode/qgis_server/models.py index 8491cf82eca..5ae043871a3 100644 --- a/geonode/qgis_server/models.py +++ b/geonode/qgis_server/models.py @@ -182,7 +182,7 @@ def get_self_resource(self): # Associate this model with resource try: return self.layer.get_self_resource() - except: + except BaseException: return None class Meta: @@ -315,7 +315,7 @@ def get_self_resource(self): qgis_layer = self.layer_styles.first() """:type: QGISServerLayer""" return qgis_layer.get_self_resource() - except: + except BaseException: return None class Meta: @@ -377,7 +377,7 @@ def get_self_resource(self): # Associate this model with resource try: return self.layer.get_self_resource() - except: + except BaseException: return None class Meta: diff --git a/geonode/qgis_server/signals.py b/geonode/qgis_server/signals.py index 931fb9ee62a..323e36fdee2 100644 --- a/geonode/qgis_server/signals.py +++ b/geonode/qgis_server/signals.py @@ -436,15 +436,15 @@ def qgis_server_post_save_map(instance, sender, **kwargs): # Geonode map supports local layers and remote layers # Remote layers were provided from other OGC services, so we don't # deal with it at the moment. - local_layers = [l for l in map_layers if l.local] + local_layers = [_l for _l in map_layers if _l.local] layers = [] for layer in local_layers: try: - l = Layer.objects.get(alternate=layer.name) - if not l.qgis_layer: + _l = Layer.objects.get(alternate=layer.name) + if not _l.qgis_layer: raise QGISServerLayer.DoesNotExist - layers.append(l) + layers.append(_l) except Layer.DoesNotExist: msg = 'No Layer found for typename: {0}'.format(layer.name) logger.debug(msg) diff --git a/geonode/qgis_server/views.py b/geonode/qgis_server/views.py index 336639013a7..25c068f18a7 100644 --- a/geonode/qgis_server/views.py +++ b/geonode/qgis_server/views.py @@ -214,7 +214,7 @@ def legend(request, layername, layertitle=False, style=None): if not qgis_layer.default_style: try: style_list(layer, internal=False) - except: + except BaseException: print 'Failed to fetch styles' # refresh values @@ -306,7 +306,7 @@ def tile(request, layername, z, x, y, style=None): if not qgis_layer.default_style: try: style_list(layer, internal=False) - except: + except BaseException: print 'Failed to fetch styles' # refresh values @@ -555,7 +555,7 @@ def qml_style(request, layername, style_name=None): styles_obj = None try: styles_obj = style_list(layer, internal=False) - except: + except BaseException: print 'Failed to fetch styles' styles_dict = [] @@ -654,7 +654,7 @@ def qml_style(request, layername, style_name=None): if not (response.status_code == 200 and response.content == 'OK'): try: style_list(layer, internal=False) - except: + except BaseException: print 'Failed to fetch styles' return TemplateResponse( @@ -679,7 +679,7 @@ def qml_style(request, layername, style_name=None): qgis_style.save() alert_message = 'Successfully add style %s' % style_name - except: + except BaseException: alert_message = 'Failed to fetch styles' return TemplateResponse( @@ -708,7 +708,7 @@ def qml_style(request, layername, style_name=None): try: style = layer.qgis_layer.styles.get(name=style_name) shutil.rmtree(style.style_tile_cache_path) - except: + except BaseException: pass style_url = style_remove_url(layer, style_name) @@ -721,7 +721,7 @@ def qml_style(request, layername, style_name=None): alert_message = '%s is not an existing style' % style_name try: style_list(layer, internal=False) - except: + except BaseException: print 'Failed to fetch styles' return TemplateResponse( @@ -743,7 +743,7 @@ def qml_style(request, layername, style_name=None): style_list(layer, internal=False) alert_message = 'Successfully deleted style %s' % style_name - except: + except BaseException: alert_message = 'Failed to fetch styles' return TemplateResponse( diff --git a/geonode/qgis_server/xml_utilities.py b/geonode/qgis_server/xml_utilities.py index dd039cede47..00db8d4f766 100644 --- a/geonode/qgis_server/xml_utilities.py +++ b/geonode/qgis_server/xml_utilities.py @@ -105,7 +105,7 @@ def insert_xml_element(root, element_path): # if a parent is missing insert it at the right place try: element = ElementTree.SubElement(parent, tag) - except: + except BaseException: # In some cases we can't add parent because the tag name is # not specific pass diff --git a/geonode/security/utils.py b/geonode/security/utils.py index 01f139e4b78..01b3cb69cfc 100644 --- a/geonode/security/utils.py +++ b/geonode/security/utils.py @@ -57,7 +57,7 @@ def get_visible_resources(queryset, is_admin = user.is_superuser if user else False try: is_manager = user.groupmember_set.all().filter(role='manager').exists() - except: + except BaseException: is_manager = False # Get the list of objects the user has access to @@ -68,18 +68,18 @@ def get_visible_resources(queryset, manager_groups = [] try: group_list_all = user.group_list_all().values('group') - except: + except BaseException: pass try: manager_groups = Group.objects.filter( name__in=user.groupmember_set.filter(role="manager").values_list("group__slug", flat=True)) - except: + except BaseException: pass try: anonymous_group = Group.objects.get(name='anonymous') if anonymous_group and anonymous_group not in groups: groups.append(anonymous_group) - except: + except BaseException: pass filter_set = queryset @@ -181,7 +181,7 @@ def get_geofence_rules_count(): rules_objs = json.loads(r.text) rules_count = rules_objs['count'] return int(rules_count) - except: + except BaseException: tb = traceback.format_exc() logger.debug(tb) return -1 diff --git a/geonode/security/views.py b/geonode/security/views.py index 933738b6b6a..ad927523c79 100644 --- a/geonode/security/views.py +++ b/geonode/security/views.py @@ -89,7 +89,7 @@ def resource_permissions(request, resource_id): status=200, content_type='text/plain' ) - except: + except BaseException: success = False message = "Error updating permissions :(" return HttpResponse( @@ -158,7 +158,7 @@ def request_permissions(request): json.dumps({'success': 'ok', }), status=200, content_type='text/plain') - except: + except BaseException: return HttpResponse( json.dumps({'error': 'error delivering notification'}), status=400, @@ -188,5 +188,5 @@ def send_email_owner_on_view(owner, viewer, layer_id, geonode_email="email@geo.n " was seen by {2}").format(layer.name, layer.uuid, viewer) try: send_mail(subject_email, msg, geonode_email, [owner_email, ]) - except: + except BaseException: pass diff --git a/geonode/services/__init__.py b/geonode/services/__init__.py index 3b161cc4c87..5fe78d0a45b 100644 --- a/geonode/services/__init__.py +++ b/geonode/services/__init__.py @@ -18,4 +18,5 @@ # ######################################################################### + default_app_config = "geonode.services.apps.ServicesAppConfig" diff --git a/geonode/services/serviceprocessors/arcgis.py b/geonode/services/serviceprocessors/arcgis.py index ef7aaf9692b..81fb697505f 100644 --- a/geonode/services/serviceprocessors/arcgis.py +++ b/geonode/services/serviceprocessors/arcgis.py @@ -139,7 +139,7 @@ def get_resources(self): """ try: return self._parse_layers(self.parsed_service.layers) - except: + except BaseException: return None def _parse_layers(self, layers): diff --git a/geonode/services/serviceprocessors/wms.py b/geonode/services/serviceprocessors/wms.py index 6fbdec80f2d..c7dc6b5d59a 100644 --- a/geonode/services/serviceprocessors/wms.py +++ b/geonode/services/serviceprocessors/wms.py @@ -168,7 +168,7 @@ def get_resources(self): try: contents_gen = self.parsed_service.contents.itervalues() return (r for r in contents_gen if not any(r.children)) - except: + except BaseException: return None def harvest_resource(self, resource_id, geonode_service): diff --git a/geonode/settings.py b/geonode/settings.py index 7f30a70c195..44a69b92055 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -740,7 +740,7 @@ # 'datastore', 'DATASTORE': '', 'PG_GEOGIG': False, - #'CACHE': ".cache" # local cache file to for HTTP requests + # 'CACHE': ".cache" # local cache file to for HTTP requests 'TIMEOUT': 10 # number of seconds to allow for HTTP requests } } @@ -1241,7 +1241,7 @@ """ # Disabling the heartbeat because workers seems often disabled in flower, # thanks to http://stackoverflow.com/a/14831904/654755 -BROKER_HEARTBEAT=0 +BROKER_HEARTBEAT = 0 # Avoid long running and retried tasks to be run over-and-over again. BROKER_TRANSPORT_OPTIONS = { @@ -1272,8 +1272,7 @@ CELERY_ACKS_LATE = True # Set this to False in order to run async -# CELERY_TASK_ALWAYS_EAGER = False if ASYNC_SIGNALS else True -CELERY_TASK_ALWAYS_EAGER = True +CELERY_TASK_ALWAYS_EAGER = False if ASYNC_SIGNALS else True CELERY_TASK_IGNORE_RESULT = True # I use these to debug kombu crashes; we get a more informative message. @@ -1492,10 +1491,10 @@ SOCIALACCOUNT_AUTO_SIGNUP = False # Uncomment this to enable Linkedin and Facebook login -#INSTALLED_APPS += ( +# INSTALLED_APPS += ( # 'allauth.socialaccount.providers.linkedin_oauth2', # 'allauth.socialaccount.providers.facebook', -#) +# ) SOCIALACCOUNT_PROVIDERS = { 'linkedin_oauth2': { diff --git a/geonode/social/__init__.py b/geonode/social/__init__.py index e064138a169..f5615c7b4a3 100644 --- a/geonode/social/__init__.py +++ b/geonode/social/__init__.py @@ -18,4 +18,5 @@ # ######################################################################### + default_app_config = "geonode.social.apps.SocialConfig" diff --git a/geonode/social/signals.py b/geonode/social/signals.py index f69198b2caf..014e52f677a 100644 --- a/geonode/social/signals.py +++ b/geonode/social/signals.py @@ -127,7 +127,7 @@ def activity_post_modify_object(sender, instance, created=None, **kwargs): raw_action=raw_action, ) # except ModelNotActionable: - except: + except BaseException: logger.debug('The activity received a non-actionable Model or None as the actor/action.') @@ -253,7 +253,7 @@ def json_serializer_producer(dictionary): y = str(y) # check datetime object # TODO: Use instanceof - if type(y) == datetime.datetime: + if isinstance(y, datetime.datetime): y = str(y) output[x] = y diff --git a/geonode/tests/base.py b/geonode/tests/base.py index c1c0e492ad4..f2a6c350e52 100644 --- a/geonode/tests/base.py +++ b/geonode/tests/base.py @@ -22,7 +22,7 @@ try: from django.utils.decorators import classproperty -except: +except BaseException: class classproperty(object): def __init__(self, method=None): diff --git a/geonode/tests/bdd/e2e/conftest.py b/geonode/tests/bdd/e2e/conftest.py index 7163ef08601..19bfc0a02fd 100644 --- a/geonode/tests/bdd/e2e/conftest.py +++ b/geonode/tests/bdd/e2e/conftest.py @@ -46,12 +46,12 @@ def en_browser(browser, bdd_server): try: # kill the specific phantomjs child proc en_browser.service.process.send_signal(signal.SIGTERM) - except: + except BaseException: pass try: # quit the node proc en_browser.quit() - except: + except BaseException: pass diff --git a/geonode/tests/integration.py b/geonode/tests/integration.py index 1c463a0c4c4..700583e95de 100644 --- a/geonode/tests/integration.py +++ b/geonode/tests/integration.py @@ -96,6 +96,7 @@ def zip_dir(basedir, archivename): zfn = absfn[len(basedir)+len(os.sep):] # XXX: relative path z.write(absfn, zfn) + """ HOW TO RUN THE TESTS -------------------- @@ -177,7 +178,8 @@ def test_layer_upload(self): class GeoNodeMapTest(GeoNodeLiveTestSupport): - """Tests geonode.maps app/module + """ + Tests geonode.maps app/module """ # geonode.maps.utils @@ -565,13 +567,13 @@ def test_bad_shapefile(self): try: # with self.assertRaises(GeoNodeException): thefile = file_upload(thefile, overwrite=True) - except GeoNodeException, e: + except GeoNodeException as e: self.assertEqual(str(e), "Invalid Projection. Layer is missing CRS!") finally: # Clean up and completely delete the layer try: thefile.delete() - except: + except BaseException: pass @timeout_decorator.timeout(LOCAL_TIMEOUT) @@ -950,7 +952,7 @@ def test_layer_replace(self): raster_layer.delete() if new_vector_layer: new_vector_layer.delete() - except: + except BaseException: # tb = traceback.format_exc() # logger.warning(tb) pass @@ -973,6 +975,7 @@ def test_importlayer_mgmt_command(self): self.assertIsNotNone(lyr) self.assertEqual(lyr.name, "test_san_andres_y_providencia_administrative") self.assertEqual(lyr.title, "Test San Andres y Providencia Administrative") + default_keywords = [ u'import', u'san andreas', @@ -996,7 +999,8 @@ def test_importlayer_mgmt_command(self): class GeoNodePermissionsTest(GeoNodeLiveTestSupport): - """Tests GeoNode permissions and its integration with GeoServer + """ + Tests GeoNode permissions and its integration with GeoServer """ """ @@ -1181,7 +1185,8 @@ def test_unpublished(self): class GeoNodeThumbnailTest(GeoNodeLiveTestSupport): - """Tests thumbnails behavior for layers and maps. + """ + Tests thumbnails behavior for layers and maps. """ @timeout_decorator.timeout(LOCAL_TIMEOUT) @@ -1241,7 +1246,8 @@ def test_map_thumbnail(self): class GeoNodeMapPrintTest(GeoNodeLiveTestSupport): - """Tests geonode.maps print + """ + Tests geonode.maps print """ @timeout_decorator.timeout(LOCAL_TIMEOUT) @@ -1332,7 +1338,8 @@ def testPrintProxy(self): class GeoNodeGeoServerSync(GeoNodeLiveTestSupport): - """Tests GeoNode/GeoServer syncronization + """ + Tests GeoNode/GeoServer syncronization """ @on_ogc_backend(geoserver.BACKEND_PACKAGE) @@ -1383,7 +1390,8 @@ def test_set_attributes_from_geoserver(self): class GeoNodeGeoServerCapabilities(GeoNodeLiveTestSupport): - """Tests GeoNode/GeoServer GetCapabilities per layer, user, category and map + """ + Tests GeoNode/GeoServer GetCapabilities per layer, user, category and map """ @on_ogc_backend(geoserver.BACKEND_PACKAGE) @@ -1650,6 +1658,7 @@ def test_add_delete_styles(self): self.assertValidJSONResponse(resp) obj = self.deserialize(resp) # Take default style url from Layer detail info + default_style_url = obj['default_style'] resp = self.api_client.get(default_style_url) self.assertValidJSONResponse(resp) @@ -1740,7 +1749,9 @@ def test_add_delete_styles(self): class GeoTIFFIOTest(GeoNodeLiveTestSupport): - """Tests integration of geotiff.io""" + """ + Tests integration of geotiff.io + """ def testLink(self): thefile = os.path.join(gisdata.RASTER_DATA, 'test_grid.tif') diff --git a/geonode/tests/suite/runner.py b/geonode/tests/suite/runner.py index 00f96d331f2..a7454222e0f 100644 --- a/geonode/tests/suite/runner.py +++ b/geonode/tests/suite/runner.py @@ -404,6 +404,7 @@ def dependency_ordered(self, test_databases, dependencies): while test_databases: changed = False + deferred = [] # Try to find a DB that has all its dependencies met @@ -553,7 +554,7 @@ def from_django_result(self, result_obj): self.failures = self._format_failures(result_obj.failures) try: self.output = result_obj.stream.read() - except: + except BaseException: pass return self @@ -562,7 +563,7 @@ def from_trial_result(self, result_obj): self.failures = self._format_failures(result_obj.failures) try: self.output = result_obj.stream.read() - except: + except BaseException: pass return self diff --git a/geonode/tests/utils.py b/geonode/tests/utils.py index fa32d5c70f4..d7ee04e370e 100644 --- a/geonode/tests/utils.py +++ b/geonode/tests/utils.py @@ -121,7 +121,7 @@ def test_set_attributes_creates_attributes(self): disconnected_post_save = signals.post_save.disconnect(geoserver_post_save, sender=Layer) # Create dummy layer to attach attributes to - l = Layer.objects.create( + _l = Layer.objects.create( name='dummy_layer', bbox_x0=-180, bbox_x1=180, @@ -144,13 +144,13 @@ def test_set_attributes_creates_attributes(self): expected_results = copy.deepcopy(attribute_map) # set attributes for resource - set_attributes(l, attribute_map) + set_attributes(_l, attribute_map) # 2 items in attribute_map should translate into 2 Attribute instances - self.assertEquals(l.attributes.count(), len(expected_results)) + self.assertEquals(_l.attributes.count(), len(expected_results)) # The name and type should be set as provided by attribute map - for a in l.attributes: + for a in _l.attributes: self.assertIn([a.attribute, a.attribute_type], expected_results) diff --git a/geonode/upload/__init__.py b/geonode/upload/__init__.py index b0fb2f81c70..e88286bf382 100644 --- a/geonode/upload/__init__.py +++ b/geonode/upload/__init__.py @@ -17,3 +17,18 @@ # along with this program. If not, see . # ######################################################################### + + +class UploadException(Exception): + + '''A handled exception meant to be presented to the user''' + + @staticmethod + def from_exc(msg, ex): + args = [msg] + args.extend(ex.args) + return UploadException(*args) + + +class LayerNotReady(Exception): + pass diff --git a/geonode/upload/files.py b/geonode/upload/files.py index 60a04eee8e3..99d82ca71ae 100644 --- a/geonode/upload/files.py +++ b/geonode/upload/files.py @@ -28,6 +28,8 @@ from geoserver.resource import FeatureType from geoserver.resource import Coverage +from django.utils.translation import ugettext as _ + from UserList import UserList import zipfile import os @@ -117,8 +119,9 @@ def __repr__(self): TYPE_UNKNOWN = FileType("unknown", None, None) +_keep_original_data = ('kmz', 'zip-mosaic') _tif_extensions = ("tif", "tiff", "geotif", "geotiff") -_mosaics_extensions = ("properties", "shp") +_mosaics_extensions = ("properties", "shp", "aux") types = [ FileType("Shapefile", "shp", vector, @@ -127,8 +130,8 @@ def __repr__(self): aliases=_tif_extensions[1:]), FileType( "ImageMosaic", "zip-mosaic", raster, - aliases=_tif_extensions[1:], - auxillary_file_exts=("properties", "aux") + _tif_extensions[1:] + aliases=_tif_extensions, + auxillary_file_exts=_mosaics_extensions + _tif_extensions ), FileType("ASCII Text File", "asc", raster, auxillary_file_exts=('prj')), @@ -246,6 +249,8 @@ def get_scan_hint(valid_extensions): result = "kml-overlay" elif "kmz" in valid_extensions: result = "kmz" + elif "zip-mosaic" in valid_extensions: + result = "zip-mosaic" else: result = None return result @@ -253,10 +258,12 @@ def get_scan_hint(valid_extensions): def scan_file(file_name, scan_hint=None): '''get a list of SpatialFiles for the provided file''' + if not os.path.exists(file_name): + raise Exception(_("Could not access to uploaded data.")) dirname = os.path.dirname(file_name) if zipfile.is_zipfile(file_name): - paths, kept_zip = _process_zip(file_name, dirname) + paths, kept_zip = _process_zip(file_name, dirname, scan_hint=scan_hint) archive = file_name if kept_zip else None else: paths = [os.path.join(dirname, p) for p in os.listdir(dirname)] @@ -284,8 +291,8 @@ def scan_file(file_name, scan_hint=None): if len(found) == 1: found[0].xml_files = xml_files else: - raise Exception("One or more XML files was provided, but no " + - "matching files were found for them.") + raise Exception(_("One or more XML files was provided, but no " + + "matching files were found for them.")) # detect slds and assign if a single upload is found sld_files = _find_file_type(safe_paths, extension='.sld') @@ -293,12 +300,12 @@ def scan_file(file_name, scan_hint=None): if len(found) == 1: found[0].sld_files = sld_files else: - raise Exception("One or more SLD files was provided, but no " + - "matching files were found for them.") + raise Exception(_("One or more SLD files was provided, but no " + + "matching files were found for them.")) return SpatialFiles(dirname, found, archive=archive) -def _process_zip(zip_path, destination_dir): +def _process_zip(zip_path, destination_dir, scan_hint=None): """Perform sanity checks on uploaded zip file This function will check if the zip file's contents have legal names. @@ -308,10 +315,9 @@ def _process_zip(zip_path, destination_dir): It will also check if an .sld file exists inside the zip and extract it """ - safe_zip_path = _rename_files([zip_path])[0] with zipfile.ZipFile(safe_zip_path, "r") as zip_handler: - if safe_zip_path.endswith(".kmz"): + if scan_hint in _keep_original_data: extracted_paths = _extract_zip(zip_handler, destination_dir) else: extracted_paths = _sanitize_zip_contents( diff --git a/geonode/upload/models.py b/geonode/upload/models.py index 0460dfff03d..d9f096103ee 100644 --- a/geonode/upload/models.py +++ b/geonode/upload/models.py @@ -126,7 +126,7 @@ def delete(self, cascade=True): if session: try: session.delete() - except: + except BaseException: logging.exception('error deleting upload session') if self.upload_dir and path.exists(self.upload_dir): shutil.rmtree(self.upload_dir) diff --git a/geonode/upload/upload.py b/geonode/upload/upload.py index 0d809c3b62d..1fb6f638993 100644 --- a/geonode/upload/upload.py +++ b/geonode/upload/upload.py @@ -51,18 +51,16 @@ from gsimporter import BadRequest from geonode import GeoNodeException +from geonode.upload import UploadException, LayerNotReady from geonode.base.models import SpatialRepresentationType, TopicCategory from ..people.utils import get_default_user from ..layers.models import Layer, UploadSession from ..layers.metadata import set_metadata from ..layers.utils import get_valid_layer_name, resolve_regions -from ..geoserver.helpers import (mosaic_delete_first_granule, - set_time_dimension, - set_time_info, +from ..geoserver.helpers import (set_time_info, set_layer_style, gs_catalog, - gs_uploader, - ogc_server_settings) + gs_uploader) from . import signals from . import utils from .models import Upload @@ -75,21 +73,6 @@ def _log(msg, *args): logger.debug(msg, *args) -class UploadException(Exception): - - '''A handled exception meant to be presented to the user''' - - @staticmethod - def from_exc(msg, ex): - args = [msg] - args.extend(ex.args) - return UploadException(*args) - - -class LayerNotReady(Exception): - pass - - class UploaderSession(object): """All objects held must be able to survive a good pickling""" @@ -336,7 +319,7 @@ def save_step(user, layer, spatial_files, overwrite=True, mosaic=False, # Is it a regular file or an ImageMosaic? # if mosaic_time_regex and mosaic_time_value: if mosaic: # we want to ingest as ImageMosaic - target_store = import_imagemosaic_granules( + target_store, files_to_upload = utils.import_imagemosaic_granules( spatial_files, append_to_mosaic_opts, append_to_mosaic_name, @@ -351,30 +334,49 @@ def save_step(user, layer, spatial_files, overwrite=True, mosaic=False, upload.append_to_mosaic_name = append_to_mosaic_name upload.mosaic_time_regex = mosaic_time_regex upload.mosaic_time_value = mosaic_time_value + # moving forward with a regular Importer session + if len(files_to_upload) > 1: + import_session = gs_uploader.upload_files( + files_to_upload[1:], + use_url=False, + # import_id=next_id, + target_store=target_store + ) + else: + import_session = gs_uploader.upload_files( + files_to_upload, + use_url=False, + # import_id=next_id, + target_store=target_store + ) + next_id = import_session.id if import_session else None + if not next_id: + error_msg = 'No valid Importer Session could be found' else: - target_store = None - # moving forward with a regular Importer session - import_session = gs_uploader.upload_files( - files_to_upload, - use_url=False, - import_id=next_id, - mosaic=len(spatial_files) > 1, - target_store=target_store - ) + # moving forward with a regular Importer session + import_session = gs_uploader.upload_files( + files_to_upload, + use_url=False, + import_id=next_id, + mosaic=False, + target_store=None + ) upload.import_id = import_session.id upload.save() # any unrecognized tasks/files must be deleted or we can't proceed import_session.delete_unrecognized_tasks() - if not import_session.tasks: - error_msg = 'No valid upload files could be found' - elif import_session.tasks[0].state == 'NO_FORMAT' \ - or import_session.tasks[0].state == 'BAD_FORMAT': - error_msg = 'There may be a problem with the data provided - ' \ - 'we could not identify its format' + if not mosaic: + if not import_session.tasks: + error_msg = 'No valid upload files could be found' + if import_session.tasks: + if import_session.tasks[0].state == 'NO_FORMAT' \ + or import_session.tasks[0].state == 'BAD_FORMAT': + error_msg = 'There may be a problem with the data provided - ' \ + 'we could not identify its format' - if len(import_session.tasks) > 1: + if not mosaic and len(import_session.tasks) > 1: error_msg = "Only a single upload is supported at the moment" if not error_msg and import_session.tasks: @@ -879,6 +881,7 @@ def _store_file(saved_layer, identifier=value.lower(), defaults={'description': '', 'gn_description': value}) key = 'category' + defaults[key] = value else: defaults[key] = value @@ -942,149 +945,3 @@ def _store_file(saved_layer, cat.reload() return saved_layer - - -def import_imagemosaic_granules( - spatial_files, - append_to_mosaic_opts, - append_to_mosaic_name, - mosaic_time_regex, - mosaic_time_value, - time_presentation, - time_presentation_res, - time_presentation_default_value, - time_presentation_reference_value): - - # The very first step is to rename the granule by adding the selected regex - # matching value to the filename. - - f = spatial_files[0].base_file - dirname = os.path.dirname(f) - basename = os.path.basename(f) - - head, tail = os.path.splitext(basename) - dst_file = os.path.join( - dirname, - head.replace( - "_", - "-") + - "_" + - mosaic_time_value + - tail) - os.rename(f, dst_file) - spatial_files[0].base_file = dst_file - - # We use the GeoServer REST APIs in order to create the ImageMosaic - # and later add the granule through the GeoServer Importer. - - # 1. Create a zip file containing the ImageMosaic .properties files - db = ogc_server_settings.datastore_db - db_engine = 'postgis' if \ - 'postgis' in db['ENGINE'] else db['ENGINE'] - - if not db_engine == 'postgis': - raise UploadException("Unsupported DataBase for Mosaics!") - - context = { - "abs_path_flag": "True", - "time_attr": "time", - "aux_metadata_flag": "False", - "mosaic_time_regex": mosaic_time_regex, - "db_host": db['HOST'], - "db_port": db['PORT'], - "db_name": db['NAME'], - "db_user": db['USER'], - "db_password": db['PASSWORD'], - "db_conn_timeout": db['CONN_TOUT'] or "10", - "db_conn_min": db['CONN_MIN'] or "1", - "db_conn_max": db['CONN_MAX'] or "5", - "db_conn_validate": db['CONN_VALIDATE'] or "true", - } - - if mosaic_time_regex: - indexer_template = """AbsolutePath={abs_path_flag} -TimeAttribute={time_attr} -Schema= the_geom:Polygon,location:String,{time_attr}:java.util.Date -PropertyCollectors=TimestampFileNameExtractorSPI[timeregex]({time_attr}) -CheckAuxiliaryMetadata={aux_metadata_flag} -SuggestedSPI=it.geosolutions.imageioimpl.plugins.tiff.TIFFImageReaderSpi""" - - timeregex_template = """regex=(?<=_)({mosaic_time_regex})""" - - with open(dirname + '/timeregex.properties', 'w') as timeregex_prop_file: - timeregex_prop_file.write(timeregex_template.format(**context)) - - else: - indexer_template = """AbsolutePath={abs_path_flag} -Schema= the_geom:Polygon,location:String,{time_attr} -CheckAuxiliaryMetadata={aux_metadata_flag} -SuggestedSPI=it.geosolutions.imageioimpl.plugins.tiff.TIFFImageReaderSpi""" - - datastore_template = """SPI=org.geotools.data.postgis.PostgisNGDataStoreFactory -host={db_host} -port={db_port} -database={db_name} -user={db_user} -passwd={db_password} -Loose\ bbox=true -Estimated\ extends=false -validate\ connections={db_conn_validate} -Connection\ timeout={db_conn_timeout} -min\ connections={db_conn_min} -max\ connections={db_conn_max}""" - - with open(dirname + '/indexer.properties', 'w') as indexer_prop_file: - indexer_prop_file.write(indexer_template.format(**context)) - - with open(dirname + '/datastore.properties', 'w') as datastore_prop_file: - datastore_prop_file.write(datastore_template.format(**context)) - - if not append_to_mosaic_opts: - - z = zipfile.ZipFile(dirname + '/' + head + '.zip', "w") - - z.write(dst_file, arcname=head + "_" + mosaic_time_value + tail) - z.write(dirname + '/indexer.properties', arcname='indexer.properties') - z.write( - dirname + - '/datastore.properties', - arcname='datastore.properties') - if mosaic_time_regex: - z.write( - dirname + '/timeregex.properties', - arcname='timeregex.properties') - - z.close() - - # 2. Send a "create ImageMosaic" request to GeoServer through gs_config - cat = gs_catalog - cat._cache.clear() - # - name = name of the ImageMosaic (equal to the base_name) - # - data = abs path to the zip file - # - configure = parameter allows for future configuration after harvesting - name = head - data = open(dirname + '/' + head + '.zip', 'rb') - cat.create_imagemosaic(name, data) - - # configure time as LIST - if mosaic_time_regex: - set_time_dimension( - cat, - name, - time_presentation, - time_presentation_res, - time_presentation_default_value, - time_presentation_reference_value) - - # - since GeoNode will uploade the first granule again through the Importer, we need to / - # delete the one created by the gs_config - mosaic_delete_first_granule(cat, name) - - return head - else: - cat = gs_catalog - cat._cache.clear() - cat.reset() - # cat.reload() - - return append_to_mosaic_name diff --git a/geonode/upload/upload_validators.py b/geonode/upload/upload_validators.py index bbed148500f..1a28923f4da 100644 --- a/geonode/upload/upload_validators.py +++ b/geonode/upload/upload_validators.py @@ -297,7 +297,9 @@ def dupes(_a): raise forms.ValidationError( _("No multiple rasters allowed")) else: - if not allow_multiple or 'sld' in other_extensions or 'xml' in other_extensions: + if not allow_multiple or ( + 'properties' not in other_extensions and ( + 'sld' in other_extensions or 'xml' in other_extensions)): return valid_extensions else: return ("zip-mosaic",) @@ -310,7 +312,9 @@ def validate_raster_zip(zip_django_file): with zipfile.ZipFile(zip_django_file) as zip_handler: contents = zip_handler.namelist() valid_extensions = validate_raster(contents, allow_multiple=True) - if valid_extensions and "zip-mosaic" not in valid_extensions: - return ("zip",) - else: - return ("zip-mosaic",) + if valid_extensions: + if "zip-mosaic" not in valid_extensions: + return ("zip",) + else: + return ("zip-mosaic",) + return None diff --git a/geonode/upload/utils.py b/geonode/upload/utils.py index b0aade7f82a..7a7585d1795 100644 --- a/geonode/upload/utils.py +++ b/geonode/upload/utils.py @@ -33,10 +33,15 @@ from django.core.urlresolvers import reverse from django.http import HttpResponse, HttpResponseRedirect from django.shortcuts import render -from geoserver.catalog import FailedRequestError +from django.utils.translation import ugettext as _ +from geoserver.catalog import FailedRequestError, ConflictingDataError +from geonode.upload import UploadException from geonode.utils import json_response as do_json_response, unzip_file -from geonode.geoserver.helpers import gs_catalog, gs_uploader, ogc_server_settings - +from geonode.geoserver.helpers import (gs_catalog, + gs_uploader, + ogc_server_settings, + get_store, + set_time_dimension) # mosaic_delete_first_granule ogr.UseExceptions() @@ -84,8 +89,8 @@ def _log(msg, *args): # at the moment, the various time support transformations require the database if _ALLOW_TIME_STEP and not _ASYNC_UPLOAD: - raise Exception( - "To support the time step, you must enable the OGC_SERVER DATASTORE option") + raise Exception(_( + "To support the time step, you must enable the OGC_SERVER DATASTORE option")) _geoserver_down_error_msg = """ GeoServer is not responding. Please try again later and sorry for the inconvenience. @@ -196,6 +201,7 @@ def get_kml_doc(kml_bytes): 'shp': ('srs', 'check', 'time', 'run', 'final'), 'csv': ('csv', 'srs', 'check', 'time', 'run', 'final'), 'tif': ('run', 'final'), + 'zip-mosaic': ('run', 'final'), 'asc': ('run', 'final'), 'kml': ('run', 'final'), 'kml-overlay': ('run', 'final'), @@ -251,7 +257,7 @@ def get_next_step(upload_session, offset=1): try: pages = _pages[upload_session.upload_type] except KeyError as e: - raise Exception('Unsupported file type: %s' % e.message) + raise Exception(_('Unsupported file type: %s' % e.message)) index = -1 if upload_session.completed_step and upload_session.completed_step != 'save': index = pages.index(upload_session.completed_step) @@ -554,16 +560,16 @@ def run_import(upload_session, async=_ASYNC_UPLOAD): import_execution_requested = False if import_session.state == 'INCOMPLETE': if task.state != 'ERROR': - raise Exception('unknown item state: %s' % task.state) + raise Exception(_('unknown item state: %s' % task.state)) elif import_session.state == 'PENDING' and task.target.store_type == 'coverageStore': if task.state == 'READY': import_session.commit(async) import_execution_requested = True if task.state == 'ERROR': progress = task.get_progress() - raise Exception( + raise Exception(_( 'error during import: %s' % - progress.get('message')) + progress.get('message'))) # if a target datastore is configured, ensure the datastore exists # in geoserver and set the uploader target appropriately @@ -663,9 +669,9 @@ def create_geoserver_db_featurestore( headers, body = response if 400 <= int(headers['status']) < 600: - raise FailedRequestError( + raise FailedRequestError(_( "Error code (%s) from GeoServer: %s" % - (headers['status'], body)) + (headers['status'], body))) ds = cat.create_datastore(store_name) ds.type = "GeoGig" @@ -746,3 +752,243 @@ def make_geogig_rest_payload(author_name='admin', payload["parentDirectory"] = \ ogc_server_settings.GEOGIG_DATASTORE_DIR return payload + + +""" + - ImageMosaics Management +""" + + +def _get_time_regex(spatial_files, base_file_name): + head, tail = os.path.splitext(base_file_name) + + # 1. Look for 'timeregex.properties' files among auxillary_files + regex = None + format = None + for aux in spatial_files[0].auxillary_files: + basename = os.path.basename(aux) + aux_head, aux_tail = os.path.splitext(basename) + if 'timeregex' == aux_head and '.properties' == aux_tail: + with open(aux, 'r') as timeregex_prop_file: + rr = timeregex_prop_file.read() + if rr and rr.split(","): + rrff = rr.split(",") + regex = rrff[0].split("=")[1] + if len(rrff) > 1: + for rf in rrff: + if 'format' in rf: + format = rf.split("=")[1] + break + if regex: + time_regexp = re.compile(regex) + if time_regexp.match(head): + time_tokens = time_regexp.match(head).groups() + if time_tokens: + return regex, format + return None, None + + +def import_imagemosaic_granules( + spatial_files, + append_to_mosaic_opts, + append_to_mosaic_name, + mosaic_time_regex, + mosaic_time_value, + time_presentation, + time_presentation_res, + time_presentation_default_value, + time_presentation_reference_value): + + # The very first step is to rename the granule by adding the selected regex + # matching value to the filename. + + f = spatial_files[0].base_file + dirname = os.path.dirname(f) + basename = os.path.basename(f) + head, tail = os.path.splitext(basename) + + if not mosaic_time_regex: + mosaic_time_regex, mosaic_time_format = _get_time_regex(spatial_files, basename) + + # 0. A Time Regex is mandartory to validate the files + if not mosaic_time_regex: + raise UploadException(_("Could not find any valid Time Regex for the Mosaic files.")) + + for spatial_file in spatial_files: + f = spatial_file.base_file + basename = os.path.basename(f) + head, tail = os.path.splitext(basename) + regexp = re.compile(mosaic_time_regex) + if regexp.match(head).groups(): + mosaic_time_value = regexp.match(head).groups()[0] + head = head.replace(regexp.match(head).groups()[0], '{mosaic_time_value}') + if mosaic_time_value: + dst_file = os.path.join( + dirname, + head.replace('{mosaic_time_value}', mosaic_time_value) + tail) + os.rename(f, dst_file) + spatial_file.base_file = dst_file + + # We use the GeoServer REST APIs in order to create the ImageMosaic + # and later add the granule through the GeoServer Importer. + head = head.replace('{mosaic_time_value}', '') + head = re.sub('^[^a-zA-z]*|[^a-zA-Z]*$', '', head) + + # 1. Create a zip file containing the ImageMosaic .properties files + # 1a. Let's check and prepare the DB based DataStore + cat = gs_catalog + workspace = cat.get_workspace(settings.DEFAULT_WORKSPACE) + db = ogc_server_settings.datastore_db + db_engine = 'postgis' if \ + 'postgis' in db['ENGINE'] else db['ENGINE'] + + if not db_engine == 'postgis': + raise UploadException(_("Unsupported DataBase for Mosaics!")) + + # dsname = ogc_server_settings.DATASTORE + dsname = db['NAME'] + + ds_exists = False + try: + ds = get_store(cat, dsname, workspace=workspace) + ds_exists = (ds is not None) + except FailedRequestError: + ds = cat.create_datastore(dsname, workspace=workspace) + db = ogc_server_settings.datastore_db + db_engine = 'postgis' if \ + 'postgis' in db['ENGINE'] else db['ENGINE'] + ds.connection_parameters.update( + {'validate connections': 'true', + 'max connections': '10', + 'min connections': '1', + 'fetch size': '1000', + 'host': db['HOST'], + 'port': db['PORT'] if isinstance( + db['PORT'], basestring) else str(db['PORT']) or '5432', + 'database': db['NAME'], + 'user': db['USER'], + 'passwd': db['PASSWORD'], + 'dbtype': db_engine} + ) + cat.save(ds) + ds = get_store(cat, dsname, workspace=workspace) + ds_exists = (ds is not None) + + if not ds_exists: + raise UploadException(_("Unsupported DataBase for Mosaics!")) + + context = { + "abs_path_flag": "True", + "time_attr": "time", + "aux_metadata_flag": "False", + "mosaic_time_regex": mosaic_time_regex, + "db_host": db['HOST'], + "db_port": db['PORT'], + "db_name": db['NAME'], + "db_user": db['USER'], + "db_password": db['PASSWORD'], + "db_conn_timeout": db['CONN_TOUT'] if 'CONN_TOUT' in db else "10", + "db_conn_min": db['CONN_MIN'] if 'CONN_MIN' in db else "1", + "db_conn_max": db['CONN_MAX'] if 'CONN_MAX' in db else "5", + "db_conn_validate": db['CONN_VALIDATE'] if 'CONN_VALIDATE' in db else "true", + } + + if mosaic_time_regex: + indexer_template = """AbsolutePath={abs_path_flag} +TimeAttribute={time_attr} +Schema= the_geom:Polygon,location:String,{time_attr}:java.util.Date +PropertyCollectors=TimestampFileNameExtractorSPI[timeregex]({time_attr}) +CheckAuxiliaryMetadata={aux_metadata_flag} +SuggestedSPI=it.geosolutions.imageioimpl.plugins.tiff.TIFFImageReaderSpi""" + + timeregex_template = """regex=(?<=_)({mosaic_time_regex})""" + + if not os.path.exists(dirname + '/timeregex.properties'): + with open(dirname + '/timeregex.properties', 'w') as timeregex_prop_file: + timeregex_prop_file.write(timeregex_template.format(**context)) + else: + indexer_template = """AbsolutePath={abs_path_flag} +Schema= the_geom:Polygon,location:String,{time_attr} +CheckAuxiliaryMetadata={aux_metadata_flag} +SuggestedSPI=it.geosolutions.imageioimpl.plugins.tiff.TIFFImageReaderSpi""" + + datastore_template = """SPI=org.geotools.data.postgis.PostgisNGDataStoreFactory +host={db_host} +port={db_port} +database={db_name} +user={db_user} +passwd={db_password} +Loose\ bbox=true +Estimated\ extends=false +validate\ connections={db_conn_validate} +Connection\ timeout={db_conn_timeout} +min\ connections={db_conn_min} +max\ connections={db_conn_max}""" + + if not os.path.exists(dirname + '/indexer.properties'): + with open(dirname + '/indexer.properties', 'w') as indexer_prop_file: + indexer_prop_file.write(indexer_template.format(**context)) + + if not os.path.exists(dirname + '/datastore.properties'): + with open(dirname + '/datastore.properties', 'w') as datastore_prop_file: + datastore_prop_file.write(datastore_template.format(**context)) + + files_to_upload = [] + if not append_to_mosaic_opts and spatial_files: + z = zipfile.ZipFile(dirname + '/' + head + '.zip', "w") + for spatial_file in spatial_files: + f = spatial_file.base_file + dst_basename = os.path.basename(f) + dst_head, dst_tail = os.path.splitext(dst_basename) + if not files_to_upload: + # Let's import only the first granule + z.write(spatial_file.base_file, arcname=dst_head + dst_tail) + files_to_upload.append(spatial_file.base_file) + if os.path.exists(dirname + '/indexer.properties'): + z.write(dirname + '/indexer.properties', arcname='indexer.properties') + if os.path.exists(dirname + '/datastore.properties'): + z.write( + dirname + + '/datastore.properties', + arcname='datastore.properties') + if mosaic_time_regex: + z.write( + dirname + '/timeregex.properties', + arcname='timeregex.properties') + z.close() + + # 2. Send a "create ImageMosaic" request to GeoServer through gs_config + cat._cache.clear() + # - name = name of the ImageMosaic (equal to the base_name) + # - data = abs path to the zip file + # - configure = parameter allows for future configuration after harvesting + name = head + data = open(dirname + '/' + head + '.zip', 'rb') + try: + cat.create_imagemosaic(name, data) + except ConflictingDataError: + # Trying to append granules to an existing mosaic + pass + + # configure time as LIST + if mosaic_time_regex: + set_time_dimension( + cat, + name, + workspace, + time_presentation, + time_presentation_res, + time_presentation_default_value, + time_presentation_reference_value) + + # - since GeoNode will upload the first granule again through the Importer, we need to / + # delete the one created by the gs_config + # mosaic_delete_first_granule(cat, name) + if len(spatial_files) > 1: + spatial_files = spatial_files[0] + return head, files_to_upload + else: + cat._cache.clear() + cat.reset() + # cat.reload() + return append_to_mosaic_name, files_to_upload diff --git a/geonode/upload/views.py b/geonode/upload/views.py index 8f758cfbfe6..6bfb0707bd0 100644 --- a/geonode/upload/views.py +++ b/geonode/upload/views.py @@ -62,8 +62,9 @@ UploadFileForm, ) from .models import Upload, UploadFile -from .files import get_scan_hint -from .files import scan_file +from .files import (get_scan_hint, + scan_file +) from .utils import ( _SUPPORTED_CRS, _ASYNC_UPLOAD, @@ -134,7 +135,7 @@ def _select_relevant_files(allowed_extensions, files): result = [] for django_file in files: extension = os.path.splitext(django_file.name)[-1].lower()[1:] - if extension in allowed_extensions: + if extension in allowed_extensions or get_scan_hint(allowed_extensions): already_selected = django_file.name in (f.name for f in result) if not already_selected: result.append(django_file) @@ -166,9 +167,10 @@ def save_step_view(req, session): name, ext = os.path.splitext(os.path.basename(base_file)) logger.debug('Name: {0}, ext: {1}'.format(name, ext)) logger.debug("base_file: {}".format(base_file)) + scan_hint = get_scan_hint(form.cleaned_data["valid_extensions"]) spatial_files = scan_file( base_file, - scan_hint=get_scan_hint(form.cleaned_data["valid_extensions"]) + scan_hint=scan_hint ) logger.info("spatial_files: {}".format(spatial_files)) import_session = save_step( @@ -176,7 +178,7 @@ def save_step_view(req, session): name, spatial_files, overwrite=False, - mosaic=form.cleaned_data['mosaic'], + mosaic=form.cleaned_data['mosaic'] or scan_hint == 'zip-mosaic', append_to_mosaic_opts=form.cleaned_data['append_to_mosaic_opts'], append_to_mosaic_name=form.cleaned_data['append_to_mosaic_name'], mosaic_time_regex=form.cleaned_data['mosaic_time_regex'], diff --git a/geonode/utils.py b/geonode/utils.py index 76c33c4c440..736cadf96c9 100755 --- a/geonode/utils.py +++ b/geonode/utils.py @@ -1100,7 +1100,7 @@ def designals(): if signalname in signals_store: try: signaltype = getattr(models.signals, signalname) - except: + except BaseException: continue logger.debug("RETRIEVE: %s: %d" % (signalname, len(signaltype.receivers))) @@ -1294,19 +1294,19 @@ def copy_tree(src, dst, symlinks=False, ignore=None): if os.path.exists(d): try: os.remove(d) - except: + except BaseException: try: shutil.rmtree(d) - except: + except BaseException: pass try: shutil.copytree(s, d, symlinks, ignore) - except: + except BaseException: pass else: try: shutil.copy2(s, d) - except: + except BaseException: pass except Exception: traceback.print_exc() diff --git a/requirements.txt b/requirements.txt index 13bfc3818c4..223268a5118 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,9 +36,9 @@ click==6.7 # python-click (6.7 in our ppa) coreapi==2.3.3 # python-coreapi (2.3.3 in our ppa) coreschema==0.0.4 # python-coreschema (0.0.4 in our ppa) autoflake<=0.7 # python-autoflake (0.7 in our ppa) -flake8<=2.6.2 # python-flake8 (2.5.4) FIXME -pycodestyle<=2.0.0 # missing TODO -pyflakes<=1.2.3 # python-pyflakes (1.6.0 in our ppa) FIXME +flake8>=2.6.2,<=3.5.0 # python-flake8 (2.5.4) FIXME +pycodestyle<=2.3.0 # missing TODO +pyflakes>=1.2.3,<=1.6.0 # python-pyflakes (1.6.0 in our ppa) FIXME pep8<=1.7.1 # python-pep8 (1.7.1 in our ppa) boto<=2.38.0 # python-boto (2.38.0) six<1.11.0 # https://github.com/benjaminp/six/issues/210 (1.10.0 in ppa) @@ -47,7 +47,8 @@ tqdm==4.23.3 # Django Apps dj-pagination<=2.3.2 # python-dj-pagination (2.3.2 in our ppa) -django-extensions>=1.2.5,<=2.0.3 # python-django-extensions (2.0.3 in our ppa) +django-celery-monitor<=1.1.2 +django-extensions>=1.2.5,<=2.0.7 # python-django-extensions (2.0.3 in our ppa) django-jsonfield<=1.0.1 # python-django-jsonfield (0.9.15, 1.0.1 in our ppa) django-jsonfield-compat<=0.4.4 # python-django-jsonfield-compat (0.4.4 in our ppa) django-leaflet<=0.23.0 # python-django-leaflet (0.23.0 in our ppa) @@ -113,7 +114,7 @@ geonode-agon-ratings==0.3.8 # python-geonode-agon-ratings (0.3.8 in our ppa) arcrest>=10.0 # TODO geonode-dialogos==0.9 # python-geonode-dialogos (0.9 in our ppa) gsconfig<2.0.0 # python-gsconfig (1.0.8 in our ppa) -gn-gsimporter<2.0.0 # python-gn-gsimporter (1.0.2 in our ppa) +gn-gsimporter>=1.0.3,<2.0.0 # python-gn-gsimporter (1.0.2 in our ppa) gisdata==0.5.4 # python-gisdata (0.5.4 in our ppa) # haystack/elasticsearch, uncomment to use @@ -145,7 +146,7 @@ idna<=2.6 # (2.6 in our ppa) ipaddress<=1.0.18 # (1.0.18 in our ppa) jdcal<=1.3 # (1.3 in our ppa) kombu==4.1.0 # python-kombu (4.1.0 in our ppa) -mccabe<=0.5.3 # (0.4.0 in our ppa) FIXME +mccabe>=0.5.3,<=0.6.1 # (0.4.0 in our ppa) FIXME mock<=2.0.0 # (1.3.0 in ppa) FIXME numpy<=1.13.1 # (1.11.0 in ppa) FIXME odfpy<=1.3.6 # python-odfpy (1.3.6 in our ppa) diff --git a/requirements_docs.txt b/requirements_docs.txt index f61d239b3d7..e6b5d0c249a 100644 --- a/requirements_docs.txt +++ b/requirements_docs.txt @@ -86,7 +86,7 @@ geonode-dialogos==0.9 geonode-user-messages==0.1.14 gisdata==0.5.4 glob2==0.6 -gn-gsimporter==1.0.2 +gn-gsimporter>=1.0.3 gsconfig==1.0.8 httplib2==0.10.3 idna==2.6 diff --git a/scripts/cloud/fabfile.py b/scripts/cloud/fabfile.py index 2abdc9c158a..964b3a43119 100644 --- a/scripts/cloud/fabfile.py +++ b/scripts/cloud/fabfile.py @@ -126,7 +126,7 @@ def deploy_project(project): put(projdir,PYLIBS,use_sudo=True) put('requirements.txt',GEONODEDIR,use_sudo=True) with cd(GEONODEDIR), prefix(ACT): - sudo('pip install -r requirements.txt --no-deps') + sudo('pip install -r requirements.txt --upgrade') sudo('rm requirements.txt') put('%s/%s.apache' % (project,project),'/etc/apache2/sites-available/%s' % project, use_sudo=True) sed('/etc/apache2/sites-available/%s' % project, 'REPLACE_WITH_SITEDIR', PYLIBS, use_sudo=True) diff --git a/scripts/jenkins/jenkins-geonode-deb-dev.sh b/scripts/jenkins/jenkins-geonode-deb-dev.sh index 958bbb94d30..6a48cd70284 100644 --- a/scripts/jenkins/jenkins-geonode-deb-dev.sh +++ b/scripts/jenkins/jenkins-geonode-deb-dev.sh @@ -22,8 +22,9 @@ fi # Setup and Build GeoNode git clean -dxff -pip install -e . -#pip install -r requirements.txt --no-deps +pip install pip --upgrade +pip install -r requirements.txt --upgrade +pip install -e . --upgrade paver setup # Make the Debian package (locally) diff --git a/scripts/jenkins/jenkins-geonode-deb.sh b/scripts/jenkins/jenkins-geonode-deb.sh index 27a6f14918f..7b4821f8fd4 100644 --- a/scripts/jenkins/jenkins-geonode-deb.sh +++ b/scripts/jenkins/jenkins-geonode-deb.sh @@ -22,8 +22,9 @@ fi # Setup and Build GeoNode git clean -dxff -pip install -e . -#pip install -r requirements.txt --no-deps +pip install pip --upgrade +pip install -r requirements.txt --upgrade +pip install -e . --upgrade paver setup # Make the Debian package (locally)