Refactor Import Temporary Layer #226, Value mapping #323

src/gp/import_feature_class.py

@@ -13,11 +13,12 @@
 
 # create a data class to store 'src_field', 'dest_field', and optional 'map' values
 class ImportFieldMap:
-    def __init__(self, src_field: str, dest_field: str=None, map: dict = None, parent=None):
+    def __init__(self, src_field: str, dest_field: str=None, map: dict = None, parent=None, direct_copy=False):
         self.src_field = src_field
         self.dest_field = dest_field
         self.map = map
         self.parent = parent
+        self.direct_copy = direct_copy
 
 class ImportFeatureClass(QgsTask):
     """
@@ -171,6 +172,10 @@ def run(self):
                                 value = None
                             if field_map.dest_field == 'display_label':
                                 value = str(src_feature.GetFID()) if field_map.src_field == src_fid_field_name else value
+                            if field_map.direct_copy is True:
+                                # we need to copy the value directly to the output field
+                                dst_feature.SetField(field_map.dest_field, value)
+                                continue
                             if field_map.dest_field is not None:
                                 if field_map.parent is not None:
                                     # this is a child field. we need to add it to the parent

src/gp/import_temp_layer.py

@@ -1,9 +1,12 @@
 import os
+import json
+from typing import List
 
-from qgis.core import QgsTask, QgsMessageLog, Qgis, QgsDataProvider, QgsVectorLayer, QgsField, QgsVectorFileWriter, QgsCoordinateTransform, QgsCoordinateReferenceSystem, QgsProject
+from qgis.core import QgsTask, QgsMessageLog, Qgis, QgsDataProvider, QgsVectorLayer, QgsFields, QgsWkbTypes, QgsCoordinateTransformContext, QgsField, QgsVectorFileWriter, QgsCoordinateTransform, QgsCoordinateReferenceSystem, QgsProject
 from qgis.PyQt.QtCore import pyqtSignal, QVariant
 
-from ..model.db_item import DBItem
+from .import_feature_class import ImportFieldMap
+from ..gp.feature_class_functions import layer_path_parser
 
 MESSAGE_CATEGORY = 'QRiS_ImportTemporaryLayer'
 
@@ -16,45 +19,56 @@ class ImportTemporaryLayer(QgsTask):
     """
 
     # Signal to notify when done and return the PourPoint and whether it should be added to the map
-    import_complete = pyqtSignal(bool)
+    import_complete = pyqtSignal(bool, int, int, int)
 
-    def __init__(self, source_layer: QgsVectorLayer, output_dataset_path: str, output_layer_name: str, attributes: dict=None, mask_clip_id=None, proj_gpkg=None):
+    def __init__(self, source_layer: QgsVectorLayer, dest_path: str, attributes: dict=None, field_map: List[ImportFieldMap]=None, clip_mask_id=None, attribute_filter: str=None, proj_gpkg=None):
         super().__init__(f'Import Temporary Layer', QgsTask.CanCancel)
 
         self.source_layer = source_layer.clone()
-        self.mask_clip_id = mask_clip_id
-        self.output_path = output_dataset_path
-        self.output_fc_name = output_layer_name
+        self.clip_mask_id = clip_mask_id
+        self.dest_path = dest_path
         self.attributes = attributes
         self.proj_gpkg = proj_gpkg
+        self.field_map = field_map
+        self.attribute_filter = attribute_filter
+
+        self.in_feats = 0
+        self.out_feats = 0
+        self.skipped_feats = 0
 
     def run(self):
 
         self.setProgress(0)
 
         try:
-            output_dir = os.path.dirname(self.output_path)
-            if not os.path.isdir(output_dir):
-                os.makedirs(output_dir)
+
+            dst_path, dst_layer_name, _dst_layer_id = layer_path_parser(self.dest_path)
+
+            base_path = os.path.dirname(dst_path)
+            if not os.path.exists(base_path):
+                os.makedirs(base_path)
 
             # Set up Transform Context
             context = QgsProject.instance().transformContext()
 
             options = QgsVectorFileWriter.SaveVectorOptions()
             options.driverName = 'GPKG'
-            options.layerName = self.output_fc_name
+            options.layerName = dst_layer_name
 
             epgs_4326 = QgsCoordinateReferenceSystem('EPSG:4326')
             out_transform = QgsCoordinateTransform(self.source_layer.sourceCrs(), epgs_4326, QgsProject.instance().transformContext())
 
             # Logic to set the write/update mode depending on if data source and/or layers are present
             options.actionOnExistingFile = QgsVectorFileWriter.AppendToLayerNoNewFields
             if options.driverName == 'GPKG':
-                if os.path.exists(self.output_path):
-                    output_layer = QgsVectorLayer(self.output_path)
+                if os.path.exists(dst_path):
+                    output_layer = QgsVectorLayer(dst_path)
                     sublayers = [subLayer.split(QgsDataProvider.SUBLAYER_SEPARATOR)[1] for subLayer in output_layer.dataProvider().subLayers()]
                     if options.layerName not in sublayers:
                         options.actionOnExistingFile = QgsVectorFileWriter.CreateOrOverwriteLayer
+                else:
+                    # If the file does not exist, we need to create it
+                    options.actionOnExistingFile = QgsVectorFileWriter.CreateOrOverwriteFile
 
             # add the event_id field to the source layer
             if self.attributes is not None:
@@ -65,30 +79,90 @@ def run(self):
                 self.source_layer.dataProvider().addAttributes(fields)
                 self.source_layer.updateFields()
 
-            if self.mask_clip_id is not None:
+            if self.clip_mask_id is not None:
                 clip_layer = QgsVectorLayer(f'{self.proj_gpkg}|layername=aoi_features')
-                clip_layer.setSubsetString(f'mask_id = {self.mask_clip_id}')
+                clip_layer.setSubsetString(f'mask_id = {self.clip_mask_id}')
                 clip_transform = QgsCoordinateTransform(clip_layer.sourceCrs(), self.source_layer.sourceCrs(), QgsProject.instance().transformContext())
                 clip_feat = clip_layer.getFeatures()
                 clip_feat = next(clip_feat)
                 clip_geom = clip_feat.geometry()
                 clip_geom.transform(clip_transform)
 
+            if self.attribute_filter is not None:
+                self.source_layer.selectByExpression(self.attribute_filter)
+
+            self.in_feats = self.source_layer.featureCount()
+
+            # add the metadata field to the source layer
+            # Check first
+            if self.source_layer.fields().lookupField('metadata') == -1:
+                field = QgsField('metadata', QVariant.String)
+                self.source_layer.dataProvider().addAttributes([field])
+                self.source_layer.updateFields()
+
+            # make sure any direct copy fields are in the output layer
+            if self.field_map is not None:
+                for field_map in self.field_map:
+                    if field_map.direct_copy is True:
+                        if self.source_layer.fields().lookupField(field_map.dest_field) == -1:
+                            field = QgsField(field_map.dest_field, QVariant.String)
+                            self.source_layer.dataProvider().addAttributes([field])
+                            self.source_layer.updateFields()
+
             self.source_layer.startEditing()
+            self.out_feats = 0
             for feat in self.source_layer.getFeatures():
                 if self.attributes is not None:
                     for field_name, field_value in self.attributes.items():
                         feat[field_name] = field_value
+                if self.field_map is not None:
+                    metadata = {}
+                    field_map: ImportFieldMap = None
+                    for field_map in self.field_map:
+                        value = feat[field_map.src_field]
+                        # change empty stringd to None
+                        if value == '':
+                            value = None
+                        if field_map.direct_copy is True:
+                            # we need to copy the value directly to the output field
+                            feat[field_map.dest_field] = value
+                            continue
+                        # if field_map.dest_field == 'display_label':
+                        #     value = str(feat.id()) if field_map.src_field == src_fid_field_name else value
+                        if field_map.dest_field is not None:
+                            if field_map.parent is not None:
+                                # this is a child field. we need to add it to the parent
+                                if field_map.parent not in metadata:
+                                    metadata[field_map.parent] = {}
+                                metadata[field_map.parent].update({field_map.dest_field: value})
+                            else:
+                                metadata.update({field_map.dest_field: value})
+                        if field_map.map is not None:
+                            # there is a value map. we need to map the value to the output fields in the metadata
+                            value_map: dict = field_map.map[value]
+                            if field_map.parent is not None:
+                                # this is a child field. we need to add it to the parent
+                                if field_map.parent not in metadata:
+                                    metadata[field_map.parent] = {}
+                                for dest_field, out_value in value_map.items():
+                                    metadata[field_map.parent].update({dest_field: out_value})
+                            else:
+                                for dest_field, out_value in value_map.items():
+                                    metadata.update({dest_field: out_value})
+                    if metadata:
+                        feat['metadata'] = json.dumps(metadata)
+
                 geom = feat.geometry()
-                if self.mask_clip_id is not None:
+                if self.clip_mask_id is not None:
                     geom = geom.intersection(clip_geom)
                 geom.transform(out_transform)
                 feat.setGeometry(geom)
                 self.source_layer.updateFeature(feat)
+                self.out_feats += 1
             self.source_layer.commitChanges()
 
             # Write vector layer to file
-            error = QgsVectorFileWriter.writeAsVectorFormatV3(self.source_layer, self.output_path, context, options)
+            error = QgsVectorFileWriter.writeAsVectorFormatV3(self.source_layer, dst_path, context, options)
 
             if error[0] != QgsVectorFileWriter.NoError:
                 self.exception = Exception(str(error))
@@ -120,7 +194,7 @@ def finished(self, result: bool):
                 QgsMessageLog.logMessage(f'Feature Class copy exception: {self.exception}', MESSAGE_CATEGORY, Qgis.Critical)
                 # raise self.exception
 
-        self.import_complete.emit(result)
+        self.import_complete.emit(result, self.in_feats, self.out_feats, self.skipped_feats)
 
     def cancel(self):
         QgsMessageLog.logMessage(

src/view/frm_cross_sections.py

@@ -22,7 +22,7 @@ class FrmCrossSections(QtWidgets.QDialog):
 
     def __init__(self, parent, project: Project, import_source_path: str = None, cross_sections: CrossSections = None, output_features: Dict[float, QgsFeature] = None, metadata: dict = None):
 
-        self.project = project
+        self.qris_project = project
         self.cross_sections = cross_sections
         self.import_source_path = import_source_path
         self.output_features = output_features
@@ -70,7 +70,7 @@ def __init__(self, parent, project: Project, import_source_path: str = None, cro
                 self.cboAttribute.setCurrentIndex(self.attribute_model.getItemIndex(self.no_attribute))
         if show_mask_clip:
             # Masks (filtered to just AOI)
-            self.masks = {id: mask for id, mask in self.project.masks.items() if mask.mask_type.id == AOI_MASK_TYPE_ID}
+            self.masks = {id: mask for id, mask in self.qris_project.masks.items() if mask.mask_type.id == AOI_MASK_TYPE_ID}
             no_clipping = DBItem('None', 0, 'None - Retain full dataset extent')
             self.masks[0] = no_clipping
             self.masks_model = DBItemModel(self.masks)
@@ -100,7 +100,7 @@ def accept(self):
 
         if self.cross_sections is not None:
             try:
-                self.cross_sections.update(self.project.project_file, self.txtName.text(), self.txtDescription.toPlainText(), metadata)
+                self.cross_sections.update(self.qris_project.project_file, self.txtName.text(), self.txtDescription.toPlainText(), metadata)
                 super(FrmCrossSections, self).accept()
             except Exception as ex:
                 if 'unique' in str(ex).lower():
@@ -113,8 +113,8 @@ def accept(self):
 
         else:
             try:
-                self.cross_sections = insert_cross_sections(self.project.project_file, self.txtName.text(), self.txtDescription.toPlainText(), metadata)
-                self.project.cross_sections[self.cross_sections.id] = self.cross_sections
+                self.cross_sections = insert_cross_sections(self.qris_project.project_file, self.txtName.text(), self.txtDescription.toPlainText(), metadata)
+                self.qris_project.cross_sections[self.cross_sections.id] = self.cross_sections
             except Exception as ex:
                 if 'unique' in str(ex).lower():
                     QtWidgets.QMessageBox.warning(self, 'Duplicate Name', "A cross sections layer with the name '{}' already exists. Please choose a unique name.".format(self.txtName.text()))
@@ -134,20 +134,21 @@ def accept(self):
                     if self.cboAttribute.isVisible() and self.cboAttribute.currentData(QtCore.Qt.UserRole) != self.no_attribute:
                         attributes['display_label'] = self.cboAttribute.currentData(QtCore.Qt.UserRole).name
                     if self.layer_id == 'memory':
-                        task = ImportTemporaryLayer(self.import_source_path, self.project.project_file, 'cross_section_features', attributes, clip_mask_id, proj_gpkg=self.project.project_file)
+                        fc_name = f'{self.qris_project.project_file}|layername=cross_section_features'
+                        task = ImportTemporaryLayer(self.import_source_path, fc_name, attributes, clip_mask_id=clip_mask_id, proj_gpkg=self.qris_project.project_file)
                         result = task.run()
                         self.on_import_complete(result)
                         # this is getting stuck if run as a task:
                         # task.import_complete.connect(self.on_import_complete)
                         # QgsApplication.taskManager().addTask(task)
                     else:
-                        import_existing(self.import_source_path, self.project.project_file, 'cross_section_features', self.cross_sections.id, 'cross_section_id', attributes, clip_mask_id)
+                        import_existing(self.import_source_path, self.qris_project.project_file, 'cross_section_features', self.cross_sections.id, 'cross_section_id', attributes, clip_mask_id)
                         super(FrmCrossSections, self).accept()
                 elif self.output_features is not None:
-                    out_layer = QgsVectorLayer(f'{self.project.project_file}|layername=cross_section_features')
+                    out_layer = QgsVectorLayer(f'{self.qris_project.project_file}|layername=cross_section_features')
                     clip_geom = None
                     if clip_mask_id is not None:
-                        clip_layer = QgsVectorLayer(f'{self.project.project_file}|layername=aoi_features')
+                        clip_layer = QgsVectorLayer(f'{self.qris_project.project_file}|layername=aoi_features')
                         clip_layer.setSubsetString(f'mask_id = {clip_mask_id}')
                         clip_feats = clip_layer.getFeatures()
                         clip_feat = QgsFeature()
@@ -166,7 +167,7 @@ def accept(self):
 
             except Exception as ex:
                 try:
-                    self.cross_sections.delete(self.project.project_file)
+                    self.cross_sections.delete(self.qris_project.project_file)
                 except Exception as ex_del:
                     QtWidgets.QMessageBox.warning(self, 'Error attempting to delete cross sections after the importing of features failed.', str(ex_del))
                 QtWidgets.QMessageBox.warning(self, 'Error Importing Cross Sections Features', str(ex))