[Import] Output errors, duplicates csvs directly from the user job table

CRM/Contact/Import/Form/DataSource.php

@@ -212,15 +212,12 @@ public function postProcess() {
 
     $this->instantiateDataSource();
 
-    // We should have the data in the DB now, parse it
-    $importTableName = $this->get('importTableName');
-    $this->_prepareImportTable($importTableName);
     $mapper = [];
 
     $parser = new CRM_Contact_Import_Parser_Contact($mapper);
     $parser->setMaxLinesToProcess(100);
     $parser->setUserJobID($this->getUserJobID());
-    $parser->run($importTableName,
+    $parser->run(NULL,
       [],
       CRM_Import_Parser::MODE_MAPFIELD,
       $this->getSubmittedValue('contactType'),
@@ -253,45 +250,6 @@ private function instantiateDataSource(): void {
     $dataSource->postProcess($this->_params, $db, $this);
   }
 
-  /**
-   * Add a PK and status column to the import table so we can track our progress.
-   * Returns the name of the primary key and status columns
-   *
-   * @param $db
-   * @param string $importTableName
-   *
-   * @return array
-   */
-  private function _prepareImportTable($importTableName) {
-    /* TODO: Add a check for an existing _status field;
-     *  if it exists, create __status instead and return that
-     */
-
-    $statusFieldName = '_status';
-    $primaryKeyName = '_id';
-
-    $this->set('primaryKeyName', $primaryKeyName);
-    $this->set('statusFieldName', $statusFieldName);
-
-    /* Make sure the PK is always last! We rely on this later.
-     * Should probably stop doing that at some point, but it
-     * would require moving to associative arrays rather than
-     * relying on numerical order of the fields. This could in
-     * turn complicate matters for some DataSources, which
-     * would also not be good. Decisions, decisions...
-     */
-
-    $alterQuery = "ALTER TABLE $importTableName
-                       ADD COLUMN $statusFieldName VARCHAR(32)
-                            DEFAULT 'NEW' NOT NULL,
-                       ADD COLUMN ${statusFieldName}Msg TEXT,
-                       ADD COLUMN $primaryKeyName INT PRIMARY KEY NOT NULL
-                               AUTO_INCREMENT";
-    CRM_Core_DAO::executeQuery($alterQuery);
-
-    return ['status' => $statusFieldName, 'pk' => $primaryKeyName];
-  }
-
   /**
    * General function for handling invalid configuration.
    *

CRM/Contact/Import/Form/MapField.php

@@ -75,7 +75,6 @@ public function defaultFromColumnName($columnName) {
    */
   public function preProcess() {
     $this->_mapperFields = $this->getAvailableFields();
-    $this->_importTableName = $this->get('importTableName');
     $this->_contactSubType = $this->getSubmittedValue('contactSubType');
     //format custom field names, CRM-2676
     $contactType = $this->getContactType();
@@ -114,7 +113,7 @@ public function preProcess() {
     $this->assign('columnNames', $this->getColumnHeaders());
     //$this->_columnCount = $this->get( 'columnCount' );
     $this->assign('columnCount', $this->_columnCount);
-    $this->_dataValues = array_values($this->getDataRows(2));
+    $this->_dataValues = array_values($this->getDataRows([], 2));
     $this->assign('dataValues', $this->_dataValues);
   }
 
@@ -557,7 +556,7 @@ public function submit($params, $mapperKeys) {
     );
     $parser->setUserJobID($this->getUserJobID());
 
-    $parser->run($this->_importTableName,
+    $parser->run(NULL,
       $mapper,
       CRM_Import_Parser::MODE_PREVIEW,
       NULL,

CRM/Contact/Import/Form/Preview.php

@@ -29,10 +29,11 @@ class CRM_Contact_Import_Form_Preview extends CRM_Import_Form_Preview {
 
   /**
    * Set variables up before form is built.
+   *
+   * @throws \API_Exception
+   * @throws \CRM_Core_Exception
    */
   public function preProcess() {
-    $mapper = $this->get('mapper');
-    $invalidRowCount = $this->get('invalidRowCount');
     $conflictRowCount = $this->get('conflictRowCount');
     $mismatchCount = $this->get('unMatchCount');
     $columnNames = $this->get('columnNames');
@@ -60,11 +61,12 @@ public function preProcess() {
       $this->set('tag', $tag);
     }
 
-    if ($invalidRowCount) {
-      $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contact_Import_Parser_Contact';
-      $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams));
-    }
+    $this->assign('downloadErrorRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::ERROR));
+    $this->assign('invalidRowCount', $this->getRowCount(CRM_Import_Parser::ERROR));
+    $this->assign('validRowCount', $this->getRowCount(CRM_Import_Parser::VALID));
+    $this->assign('totalRowCount', $this->getRowCount([]));
 
+    // @todo conflict rows are still being output in the parser & not updating the temp table - fix
     if ($conflictRowCount) {
       $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Contact_Import_Parser_Contact';
       $this->set('downloadConflictRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams));
@@ -75,23 +77,9 @@ public function preProcess() {
       $this->set('downloadMismatchRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams));
     }
 
-    $properties = array(
-      'columnCount',
-      'totalRowCount',
-      'validRowCount',
-      'invalidRowCount',
-      'conflictRowCount',
-      'downloadErrorRecordsUrl',
-      'downloadConflictRecordsUrl',
-      'downloadMismatchRecordsUrl',
-    );
-
     $this->assign('mapper', $this->getMappedFieldLabels());
 
-    foreach ($properties as $property) {
-      $this->assign($property, $this->get($property));
-    }
-    $this->assign('dataValues', $this->getDataRows(2));
+    $this->assign('dataValues', $this->getDataRows([], 2));
 
     $this->setStatusUrl();
   }
@@ -213,15 +201,14 @@ public function postProcess() {
       'mapFields' => $this->getAvailableFields(),
       'contactType' => $this->get('contactType'),
       'contactSubType' => $this->getSubmittedValue('contactSubType'),
-      'primaryKeyName' => $this->get('primaryKeyName'),
-      'statusFieldName' => $this->get('statusFieldName'),
+      'primaryKeyName' => '_id',
+      'statusFieldName' => '_status',
       'statusID' => $this->get('statusID'),
       'totalRowCount' => $this->get('totalRowCount'),
       'userJobID' => $this->getUserJobID(),
     );
 
-    $tableName = $this->get('importTableName');
-    $importJob = new CRM_Contact_Import_ImportJob($tableName);
+    $importJob = new CRM_Contact_Import_ImportJob();
     $importJob->setJobParams($importJobParams);
 
     // If ACL applies to the current user, update cache before running the import.
@@ -264,9 +251,7 @@ public function postProcess() {
 
       $this->set('errorFile', $errorFile);
 
-      $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contact_Import_Parser_Contact';
-      $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams));
-
+      // @todo - these should use the new url but are not reliably updating the table yet.
       $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Contact_Import_Parser_Contact';
       $this->set('downloadConflictRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams));
 

CRM/Contact/Import/Form/Summary.php

@@ -27,7 +27,7 @@ public function preProcess() {
     // set the error message path to display
     $this->assign('errorFile', $this->get('errorFile'));
 
-    $totalRowCount = $this->get('totalRowCount');
+    $totalRowCount = $this->getRowCount();
     $relatedCount = $this->get('relatedCount');
     $totalRowCount += $relatedCount;
 
@@ -81,9 +81,6 @@ public function preProcess() {
     $this->assign('dupeActionString', $dupeActionString);
 
     $properties = [
-      'totalRowCount',
-      'validRowCount',
-      'invalidRowCount',
       'conflictRowCount',
       'downloadConflictRecordsUrl',
       'downloadErrorRecordsUrl',
@@ -98,6 +95,10 @@ public function preProcess() {
     foreach ($properties as $property) {
       $this->assign($property, $this->get($property));
     }
+    $this->assign('totalRowCount', $this->getRowCount());
+    $this->assign('validRowCount', $this->getRowCount(CRM_Import_Parser::VALID));
+    $this->assign('invalidRowCount', $this->getRowCount(CRM_Import_Parser::ERROR));
+    $this->assign('downloadDuplicateRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::DUPLICATE));
 
     $session = CRM_Core_Session::singleton();
     $session->pushUserContext(CRM_Utils_System::url('civicrm/import/contact', 'reset=1'));

CRM/Contact/Import/ImportJob.php

@@ -50,37 +50,6 @@ class CRM_Contact_Import_ImportJob {
 
   protected $_userJobID;
 
-  /**
-   * @param string|null $tableName
-   * @param string|null $createSql
-   * @param bool $createTable
-   *
-   * @throws \CRM_Core_Exception
-   */
-  public function __construct($tableName = NULL, $createSql = NULL, $createTable = FALSE) {
-    $dao = new CRM_Core_DAO();
-    $db = $dao->getDatabaseConnection();
-
-    if ($createTable) {
-      if (!$createSql) {
-        throw new CRM_Core_Exception(ts('Either an existing table name or an SQL query to build one are required'));
-      }
-      if ($tableName) {
-        // Drop previous table if passed in and create new one.
-        $db->query("DROP TABLE IF EXISTS $tableName");
-      }
-      $table = CRM_Utils_SQL_TempTable::build()->setDurable();
-      $tableName = $table->getName();
-      $table->createWithQuery($createSql);
-    }
-
-    if (!$tableName) {
-      throw new CRM_Core_Exception(ts('Import Table is required.'));
-    }
-
-    $this->_tableName = $tableName;
-  }
-
   /**
    * @return null|string
    */
@@ -92,19 +61,9 @@ public function getTableName() {
    * Has the job completed.
    *
    * @return bool
-   * @throws Exception
    */
-  public function isComplete() {
-    if (!$this->_statusFieldName) {
-      throw new CRM_Core_Exception("Could not get name of the import status field");
-    }
-    $query = "SELECT * FROM $this->_tableName
-                  WHERE  $this->_statusFieldName = 'NEW' LIMIT 1";
-    $result = CRM_Core_DAO::executeQuery($query);
-    if ($result->fetch()) {
-      return FALSE;
-    }
-    return TRUE;
+  public function isComplete(): bool {
+    return $this->_parser->isComplete();
   }
 
   /**
@@ -220,7 +179,7 @@ public function runImport(&$form, $timeout = 55) {
       $parserParameters['relatedContactWebsiteType']
     );
     $this->_parser->setUserJobID($this->_userJobID);
-    $this->_parser->run($this->_tableName, $mapperFields,
+    $this->_parser->run(NULL, $mapperFields,
       CRM_Import_Parser::MODE_IMPORT,
       $this->_contactType,
       $this->_primaryKeyName,

CRM/Contact/Import/Parser/Contact.php

@@ -2583,15 +2583,7 @@ public function run(
     $this->_conflicts = [];
     $this->_unparsedAddresses = [];
 
-    // Transitional support for deprecating table_name (and other fields)
-    // form input - the goal is to load them from userJob - but eventually
-    // we will just load the datasource object and this code will not know the
-    // table name.
-    if (!$tableName && $this->userJobID) {
-      $tableName = $this->getUserJob()['metadata']['DataSource']['table_name'];
-    }
-
-    $this->_tableName = $tableName;
+    $this->_tableName = $tableName = $this->getUserJob()['metadata']['DataSource']['table_name'];
     $this->_primaryKeyName = '_id';
     $this->_statusFieldName = '_status';
 
@@ -2609,7 +2601,7 @@ public function run(
     // get the contents of the temp. import table
     $query = "SELECT * FROM $tableName";
     if ($mode == self::MODE_IMPORT) {
-      $query .= " WHERE $statusFieldName = 'NEW'";
+      $query .= " WHERE _status = 'NEW'";
     }
     if ($this->_maxLinesToProcess > 0) {
       // Note this would only be the case in MapForm mode, where it is set to 100
@@ -2723,15 +2715,6 @@ public function run(
         }
       }
 
-      if ($this->_invalidRowCount) {
-        // removed view url for invlaid contacts
-        $headers = array_merge([
-          ts('Line Number'),
-          ts('Reason'),
-        ], $customHeaders);
-        $this->_errorFileName = self::errorFileName(self::ERROR);
-        self::exportCSV($this->_errorFileName, $headers, $this->_errors);
-      }
       if ($this->_conflictCount) {
         $headers = array_merge([
           ts('Line Number'),
@@ -2740,15 +2723,6 @@ public function run(
         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
       }
-      if ($this->_duplicateCount) {
-        $headers = array_merge([
-          ts('Line Number'),
-          ts('View Contact URL'),
-        ], $customHeaders);
-
-        $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
-        self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
-      }
       if ($this->_unMatchCount) {
         $headers = array_merge([
           ts('Line Number'),
@@ -3072,9 +3046,6 @@ public function set($store, $mode = self::MODE_SUMMARY) {
         $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
     }
 
-    if ($this->_invalidRowCount) {
-      $store->set('errorsFileName', $this->_errorFileName);
-    }
     if ($this->_conflictCount) {
       $store->set('conflictsFileName', $this->_conflictFileName);
     }
@@ -3092,11 +3063,7 @@ public function set($store, $mode = self::MODE_SUMMARY) {
       if ($this->_duplicateCount) {
         $store->set('duplicatesFileName', $this->_duplicateFileName);
       }
-      if ($this->_unparsedAddressCount) {
-        $store->set('errorsFileName', $this->_errorFileName);
-      }
     }
-    //echo "$this->_totalCount,$this->_invalidRowCount,$this->_conflictCount,$this->_duplicateCount";
   }
 
   /**
@@ -3651,4 +3618,17 @@ public function getMappedRow(array $values): array {
     return $params;
   }
 
+  /**
+   * Is the job complete.
+   *
+   * This function transitionally accesses the table from the userJob
+   * directly - but the function should be moved to the dataSource class.
+   *
+   * @throws \API_Exception
+   */
+  public function isComplete() {
+    $tableName = $this->getUserJob()['metadata']['DataSource']['table_name'];
+    return (bool) CRM_Core_DAO::singleValueQuery("SELECT count(*) FROM $tableName WHERE _status = 'NEW' LIMIT 1");
+  }
+
 }

CRM/Core/xml/Menu/Import.xml

@@ -17,6 +17,12 @@
      <page_callback>CRM_Contact_Import_Controller</page_callback>
      <weight>410</weight>
   </item>
+  <item>
+    <path>civicrm/import/outcome</path>
+    <title>Import results</title>
+    <access_arguments>access CiviCRM</access_arguments>
+    <page_callback>CRM_Import_Forms::outputCSV</page_callback>
+  </item>
   <item>
      <path>civicrm/import/activity</path>
      <title>Import Activities</title>