[Fixes #7194] Append data to an existing layer (#7195)

* [Fixes #7194] Add new append_layer html

* [Fixes #7194] Test helper utility for create single layer

* [Fixes #7194] Add append data to existing edit layer html

* [Fixes #7194] Flake8 formatting

* [Fixes #7194] New layer_append url map

* [Fixes #7194]Validation helper function for replace and appen layer

* [Fixes #7194] Add view for layer_append feature

* [Fixes #7194] Test coverage for new validation function

(cherry picked from commit b404e0d)

geonode/base/populate_test_data.py

@@ -317,6 +317,36 @@ def dump_models(path=None):
         f.write(result)
 
 
+def create_single_layer(name):
+    get_user_model().objects.create(
+                    username='admin',
+                    is_superuser=True,
+                    first_name='admin')
+    test_datetime = datetime.strptime('2020-01-01', '%Y-%m-%d')
+    user = get_user_model().objects.get(username='AnonymousUser')
+    ll = (name, 'lorem ipsum', name, f'geonode:{name}', [
+                    0, 22, 0, 22], test_datetime, ('populartag',), "farming")
+    title, abstract, name, alternate, (bbox_x0, bbox_x1, bbox_y0, bbox_y1), start, kws, category = ll
+    layer = Layer(
+        title=title,
+        abstract=abstract,
+        name=name,
+        alternate=alternate,
+        bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)),
+        ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)),
+        srid='EPSG:4326',
+        uuid=str(uuid4()),
+        owner=user,
+        temporal_extent_start=test_datetime,
+        temporal_extent_end=test_datetime,
+        date=start,
+        storeType="dataStore",
+    )
+    layer.save()
+    layer.set_default_permissions()
+    return layer
+
+
 if __name__ == '__main__':
     create_models()
     dump_models()

geonode/layers/templates/layers/layer_append.html

@@ -0,0 +1,133 @@
+{% extends "upload/layer_upload_base.html" %}
+{% load i18n %}
+{% load staticfiles %}
+{% load dialogos_tags %}
+{% load pinax_ratings_tags %}
+{% load bootstrap_tags %}
+{% load pagination_tags %}
+{% load base_tags %}
+{% load guardian_tags %}
+
+{% block title %} {% trans "Append to Layer"  %} - {{ block.super }}  {% endblock %}
+
+{% block body_class %}{% trans "layers append" %}{% endblock %}
+
+
+{% block head %}
+
+{{ block.super }}
+{% endblock %}
+
+{% block body_outer %}
+<div class="page-header">
+  <a href="{% url "layer_browse" %}?limit={{ CLIENT_RESULTS_LIMIT }}" class="btn btn-primary pull-right" style="margin-left: 3px;">{% trans "Explore Layers" %}</a>
+  <a href="{% url 'layer_detail' layername=resource.service_typename %}" class="btn btn-primary pull-right" style="margin-left: 3px;">{% trans "Return to Layer" %}</a>
+  <h2 class="page-title">{% trans "Append to Layer" %} <small><a href="{{ resource.get_absolute_url }}">{{ resource.title }}</a><small></h2>
+</div>
+<div class="row">
+  <div class="col-md-8">
+    {% if incomplete %}
+    <section class="widget" id="incomplete-download-list">
+      <h3>{% trans "Incomplete Uploads" %}</h3>
+      <p>{% trans "You have the following incomplete uploads" %}:</p>
+      {% for u in incomplete %}
+      <div class="clearfix uip" id="incomplete-{{ u.import_id }}">
+        <div class="pull-left">{{ u.name }}, {% trans "last updated on" %} {{ u.date }}</div>
+        <div class="upload_actions pull-right">
+          <a class="btn btn-mini" href="#" id="resume-{{ u.import_id }}">{% trans "Resume" %}</a>
+          <a class="btn btn-mini" href="#" id="delete-{{ u.import_id }}"><i class="icon-trash"></i> {% trans "Delete" %}</a>
+        </div>
+      </div>
+      {% endfor %}
+    </section>
+    <div id="confirm-delete" class="hidden alert alert-warning">
+      {% trans "Are you sure you want to delete this upload?" %}
+      <a href="#y" class="btn btn-danger">{% trans "Delete" %}</a>
+      <a href="#n" class="btn btn-default">{% trans "Cancel" %}</a>
+      <a href="#yy">{% trans "Delete, and don't ask me again." %}</a>
+    </div>
+    {% endif %}
+
+    {% block additional_info %}{% endblock %}
+
+    {% if errors %}
+    <div id="errors" class="alert alert-danger">
+      {% for error in errors %}
+      <p>{{ error }}</p>
+      {% endfor %}
+    </div>
+    {% endif %}
+
+    <div id="upload-status"></div>
+
+    <section id="drop-zone">
+      <h3><i class="fa fa-cloud-upload"></i><br />{% trans "Drop files here" %}</h3>
+    </section>
+
+    <p>{% trans " or select them one by one:" %}</p>
+
+    <form id="file-uploader" method="post" enctype="multipart/form-data">
+      <!-- UI change to hide the list of previously uploaded files from the user -->
+      <input type="file" id="file-input" style="display: none;" multiple>
+      <input class="btn btn-default" type="button" value="{% trans "Choose Files" %}" onclick="document.getElementById('file-input').click();">
+    </form>
+
+    <section class="widget">
+      <ul id="global-errors"></ul>
+      <h4>{% trans "Files to be uploaded" %}</h4>
+      <div id="file-queue"></div>
+      <div class="checkbox" style="display:none;" id="metadata_uploaded_preserve_check">
+          {% trans "Preserve Metadata XML" %} <input type="checkbox" name="metadata_uploaded_preserve" id="id_metadata_uploaded_preserve"/>
+      </div>
+    </section>
+
+    <section class="charset">
+      <p>{% trans "Select the charset or leave default" %}</p>
+      <select id="charset">
+        {% for charset in charsets %}
+        {% if charset.0 == 'UTF-8' %}
+        <option selected='selected' value={{ charset.0 }}>{{ charset.1 }}</option>
+        {% else %}
+        <option value={{ charset.0 }}>{{ charset.1 }}</option>
+        {% endif %}
+        {% endfor %}
+      </select>
+    </section>
+
+    <section>
+      <a href="#" id="clear-button" class="btn btn-default">{% trans "Clear" %}</a>
+      <a href="#" id="upload-button" class="btn btn-danger">{% trans "Append to Layer" %}</a>
+    </section>
+  </div>
+
+  {% if GEOSERVER_BASE_URL %}
+    {% get_obj_perms request.user for resource.layer as "layer_perms" %}
+  {% endif %}
+
+</div>
+{% endblock %}
+
+
+{% block extra_script %}
+{{ block.super }}
+<script data-main="{% static 'geonode/js/upload/main.js' %}"
+  src="{% static 'lib/js/require.js' %}">
+</script>
+<script type="text/javascript">
+{% autoescape off %}
+
+  csrf_token =  "{{ csrf_token }}",
+  form_target = "{% url "layer_append" resource.service_typename %}",
+  time_enabled = {{ TIME_ENABLED|lower  }},
+  mosaic_enabled = {{ MOSAIC_ENABLED|lower  }},
+  userLookup = "{% url "account_ajax_lookup" %}"
+
+{% endautoescape %}
+
+</script>
+{% if GEONODE_SECURITY_ENABLED %}
+{% with resource=layer %}
+{% include "_permissions_form_js.html" %}
+{% endwith %}
+{% endif %}
+{% endblock extra_script %}

geonode/layers/templates/layers/layer_detail.html

@@ -514,6 +514,9 @@ <h4>{% trans "Layer" %}</h4>
                   {% if "change_resourcebase" in perms_list and resource.storeType != "remoteStore" %}
                   <a class="btn btn-default btn-block btn-xs" href="{% url "layer_replace" resource.service_typename %}">{% trans "Replace" %}</a>
                   {% endif %}
+                  {% if "change_resourcebase" in perms_list and resource.storeType != "remoteStore" %}
+                  <a class="btn btn-default btn-block btn-xs" href="{% url "layer_append" resource.service_typename %}">{% trans "Append Data" %}</a>
+                  {% endif %}
                   {% if "change_layer_data" in layer_perms and OGC_SERVER.default.BACKEND == 'geonode.geoserver' %}
                     {% if layer_type != "raster" and resource.storeType != "remoteStore" %}
                     <a class="btn btn-default btn-block btn-xs" href="{% url "new_map" %}?layer={{resource.service_typename}}&storeType={{resource.storeType}}">{% trans "Edit data" %}</a>

geonode/layers/tests.py

@@ -17,6 +17,8 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 #########################################################################
+from collections import namedtuple
+
 from geonode.tests.base import GeoNodeBaseTestSupport
 from django.test import TestCase
 import io
@@ -55,9 +57,9 @@
     get_files,
     get_valid_name,
     get_valid_layer_name,
-    surrogate_escape_string)
+    surrogate_escape_string, validate_input_source)
 from geonode.people.utils import get_valid_user
-from geonode.base.populate_test_data import all_public
+from geonode.base.populate_test_data import all_public, create_single_layer
 from geonode.base.models import TopicCategory, License, Region, Link
 from geonode.layers.forms import JSONField, LayerUploadForm
 from geonode.utils import check_ogc_backend, set_resource_default_links
@@ -1701,3 +1703,109 @@ def test_layer_will_override_the_uuid_if_handler_is_defined(self):
         expected = "abc:abc-1234-abc"
         actual = Layer.objects.get(id=self.sut.id)
         self.assertEqual(expected, actual.uuid)
+
+
+class TestalidateInputSource(TestCase):
+
+    def setUp(self):
+        self.maxDiff = None
+        self.layer = create_single_layer('single_point')
+        self.r = namedtuple('GSCatalogRes', ['resource'])
+
+    def tearDown(self):
+        self.layer.delete()
+
+    def test_will_raise_exception_for_replace_vector_layer_with_raster(self):
+        layer = Layer.objects.filter(name="single_point")[0]
+        filename = "/tpm/filename.tif"
+        files = ["/opt/file1.shp", "/opt/file2.ccc"]
+        with self.assertRaises(Exception) as e:
+            validate_input_source(layer, filename, files, action_type="append")
+        expected = "You are attempting to append a vector layer with a raster."
+        self.assertEqual(expected, e.exception.args[0])
+
+    def test_will_raise_exception_for_replace_layer_with_unknown_format(self):
+        layer = Layer.objects.filter(name="single_point")[0]
+        filename = "/tpm/filename.ccc"
+        files = ["/opt/file1.shp", "/opt/file2.ccc"]
+        with self.assertRaises(Exception) as e:
+            validate_input_source(layer, filename, files, action_type="append")
+        expected = "You are attempting to append a vector layer with an unknown format."
+        self.assertEqual(expected, e.exception.args[0])
+
+    def test_will_raise_exception_for_replace_layer_with_different_file_name(self):
+        layer = Layer.objects.get(name="single_point")
+        file_path = gisdata.VECTOR_DATA
+        filename = os.path.join(file_path, "san_andres_y_providencia_highway.shp")
+        files = {
+            "shp": filename,
+            "dbf": f"{file_path}/san_andres_y_providencia_highway.sbf",
+            "prj": f"{file_path}/san_andres_y_providencia_highway.prj",
+            "shx": f"{file_path}/san_andres_y_providencia_highway.shx",
+        }
+        with self.assertRaises(Exception) as e:
+            validate_input_source(layer, filename, files, action_type="append")
+        expected = (
+            "Some error occurred while trying to access the uploaded schema: "
+            "Please ensure the name is consistent with the file you are trying to append."
+        )
+        self.assertEqual(expected, e.exception.args[0])
+
+    def test_will_raise_exception_for_not_existing_layer_in_the_catalog(self):
+        layer = Layer.objects.filter(name="single_point")[0]
+        file_path = gisdata.VECTOR_DATA
+        filename = os.path.join(file_path, "single_point.shp")
+        files = {
+            "shp": filename,
+            "dbf": f"{file_path}/single_point.sbf",
+            "prj": f"{file_path}/single_point.prj",
+            "shx": f"{file_path}/single_point.shx",
+        }
+        with self.assertRaises(Exception) as e:
+            validate_input_source(layer, filename, files, action_type="append")
+        expected = (
+            "Some error occurred while trying to access the uploaded schema: "
+            "The selected Layer does not exists in the catalog."
+        )
+        self.assertEqual(expected, e.exception.args[0])
+
+    @patch("geonode.layers.utils.gs_catalog")
+    def test_will_raise_exception_if_schema_is_not_equal_between_catalog_and_file(self, catalog):
+        attr = namedtuple('GSCatalogAttr', ['attributes'])
+        attr.attributes = []
+        self.r.resource = attr
+        catalog.get_layer.return_value = self.r
+        layer = Layer.objects.filter(name="single_point")[0]
+        file_path = gisdata.VECTOR_DATA
+        filename = os.path.join(file_path, "single_point.shp")
+        files = {
+            "shp": filename,
+            "dbf": f"{file_path}/single_point.sbf",
+            "prj": f"{file_path}/single_point.prj",
+            "shx": f"{file_path}/single_point.shx",
+        }
+        with self.assertRaises(Exception) as e:
+            validate_input_source(layer, filename, files, action_type="append")
+        expected = (
+            "Some error occurred while trying to access the uploaded schema: "
+            "Please ensure that the layer structure is consistent with the file you are trying to append."
+        )
+        self.assertEqual(expected, e.exception.args[0])
+
+    @patch("geonode.layers.utils.gs_catalog")
+    def test_validation_will_pass_for_valid_append(self, catalog):
+        attr = namedtuple('GSCatalogAttr', ['attributes'])
+        attr.attributes = ['label']
+        self.r.resource = attr
+        catalog.get_layer.return_value = self.r
+        layer = Layer.objects.filter(name="single_point")[0]
+        file_path = gisdata.VECTOR_DATA
+        filename = os.path.join(file_path, "single_point.shp")
+        files = {
+            "shp": filename,
+            "dbf": f"{file_path}/single_point.sbf",
+            "prj": f"{file_path}/single_point.prj",
+            "shx": f"{file_path}/single_point.shx",
+        }
+        actual = validate_input_source(layer, filename, files, action_type="append")
+        self.assertTrue(actual)

geonode/layers/urls.py

@@ -54,6 +54,8 @@
         name="layer_granule_remove"),
     url(r'^(?P<layername>[^/]*)/replace$',
         views.layer_replace, name="layer_replace"),
+    url(r'^(?P<layername>[^/]*)/append$',
+        views.layer_append, name="layer_append"),
     url(r'^(?P<layername>[^/]*)/thumbnail$',
         views.layer_thumbnail, name='layer_thumbnail'),
     url(r'^(?P<layername>[^/]*)/get$', views.get_layer, name='get_layer'),