Skip to content

Commit

Permalink
Merge pull request #23292 from eileenmcnaughton/import_output_table
Browse files Browse the repository at this point in the history 
[Import] Output errors, duplicates csvs directly from the user job table
  • Loading branch information
@monishdeb
monishdeb authored May 5, 2022
2 parents 9087689 + d3bb0df commit a863d40
Show file tree
Hide file tree
Showing 15 changed files with 176 additions and 211 deletions.
44 changes: 1 addition & 43 deletions CRM/Contact/Import/Form/DataSource.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,15 +212,12 @@ public function postProcess() {

$this->instantiateDataSource();

// We should have the data in the DB now, parse it
$importTableName = $this->get('importTableName');
$this->_prepareImportTable($importTableName);
$mapper = [];

$parser = new CRM_Contact_Import_Parser_Contact($mapper);
$parser->setMaxLinesToProcess(100);
$parser->setUserJobID($this->getUserJobID());
$parser->run($importTableName,
$parser->run(NULL,
[],
CRM_Import_Parser::MODE_MAPFIELD,
$this->getSubmittedValue('contactType'),
Expand Down Expand Up @@ -253,45 +250,6 @@ private function instantiateDataSource(): void {
$dataSource->postProcess($this->_params, $db, $this);
}

/**
* Add a PK and status column to the import table so we can track our progress.
* Returns the name of the primary key and status columns
*
* @param $db
* @param string $importTableName
*
* @return array
*/
private function _prepareImportTable($importTableName) {
/* TODO: Add a check for an existing _status field;
* if it exists, create __status instead and return that
*/

$statusFieldName = '_status';
$primaryKeyName = '_id';

$this->set('primaryKeyName', $primaryKeyName);
$this->set('statusFieldName', $statusFieldName);

/* Make sure the PK is always last! We rely on this later.
* Should probably stop doing that at some point, but it
* would require moving to associative arrays rather than
* relying on numerical order of the fields. This could in
* turn complicate matters for some DataSources, which
* would also not be good. Decisions, decisions...
*/

$alterQuery = "ALTER TABLE $importTableName
ADD COLUMN $statusFieldName VARCHAR(32)
DEFAULT 'NEW' NOT NULL,
ADD COLUMN ${statusFieldName}Msg TEXT,
ADD COLUMN $primaryKeyName INT PRIMARY KEY NOT NULL
AUTO_INCREMENT";
CRM_Core_DAO::executeQuery($alterQuery);

return ['status' => $statusFieldName, 'pk' => $primaryKeyName];
}

/**
* General function for handling invalid configuration.
*
Expand Down
5 changes: 2 additions & 3 deletions CRM/Contact/Import/Form/MapField.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@ public function defaultFromColumnName($columnName) {
*/
public function preProcess() {
$this->_mapperFields = $this->getAvailableFields();
$this->_importTableName = $this->get('importTableName');
$this->_contactSubType = $this->getSubmittedValue('contactSubType');
//format custom field names, CRM-2676
$contactType = $this->getContactType();
Expand Down Expand Up @@ -114,7 +113,7 @@ public function preProcess() {
$this->assign('columnNames', $this->getColumnHeaders());
//$this->_columnCount = $this->get( 'columnCount' );
$this->assign('columnCount', $this->_columnCount);
$this->_dataValues = array_values($this->getDataRows(2));
$this->_dataValues = array_values($this->getDataRows([], 2));
$this->assign('dataValues', $this->_dataValues);
}

Expand Down Expand Up @@ -557,7 +556,7 @@ public function submit($params, $mapperKeys) {
);
$parser->setUserJobID($this->getUserJobID());

$parser->run($this->_importTableName,
$parser->run(NULL,
$mapper,
CRM_Import_Parser::MODE_PREVIEW,
NULL,
Expand Down
41 changes: 13 additions & 28 deletions CRM/Contact/Import/Form/Preview.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,11 @@ class CRM_Contact_Import_Form_Preview extends CRM_Import_Form_Preview {

/**
* Set variables up before form is built.
*
* @throws \API_Exception
* @throws \CRM_Core_Exception
*/
public function preProcess() {
$mapper = $this->get('mapper');
$invalidRowCount = $this->get('invalidRowCount');
$conflictRowCount = $this->get('conflictRowCount');
$mismatchCount = $this->get('unMatchCount');
$columnNames = $this->get('columnNames');
Expand Down Expand Up @@ -60,11 +61,12 @@ public function preProcess() {
$this->set('tag', $tag);
}

if ($invalidRowCount) {
$urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contact_Import_Parser_Contact';
$this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams));
}
$this->assign('downloadErrorRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::ERROR));
$this->assign('invalidRowCount', $this->getRowCount(CRM_Import_Parser::ERROR));
$this->assign('validRowCount', $this->getRowCount(CRM_Import_Parser::VALID));
$this->assign('totalRowCount', $this->getRowCount([]));

// @todo conflict rows are still being output in the parser & not updating the temp table - fix
if ($conflictRowCount) {
$urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Contact_Import_Parser_Contact';
$this->set('downloadConflictRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams));
Expand All @@ -75,23 +77,9 @@ public function preProcess() {
$this->set('downloadMismatchRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams));
}

$properties = array(
'columnCount',
'totalRowCount',
'validRowCount',
'invalidRowCount',
'conflictRowCount',
'downloadErrorRecordsUrl',
'downloadConflictRecordsUrl',
'downloadMismatchRecordsUrl',
);

$this->assign('mapper', $this->getMappedFieldLabels());

foreach ($properties as $property) {
$this->assign($property, $this->get($property));
}
$this->assign('dataValues', $this->getDataRows(2));
$this->assign('dataValues', $this->getDataRows([], 2));

$this->setStatusUrl();
}
Expand Down Expand Up @@ -213,15 +201,14 @@ public function postProcess() {
'mapFields' => $this->getAvailableFields(),
'contactType' => $this->get('contactType'),
'contactSubType' => $this->getSubmittedValue('contactSubType'),
'primaryKeyName' => $this->get('primaryKeyName'),
'statusFieldName' => $this->get('statusFieldName'),
'primaryKeyName' => '_id',
'statusFieldName' => '_status',
'statusID' => $this->get('statusID'),
'totalRowCount' => $this->get('totalRowCount'),
'userJobID' => $this->getUserJobID(),
);

$tableName = $this->get('importTableName');
$importJob = new CRM_Contact_Import_ImportJob($tableName);
$importJob = new CRM_Contact_Import_ImportJob();
$importJob->setJobParams($importJobParams);

// If ACL applies to the current user, update cache before running the import.
Expand Down Expand Up @@ -264,9 +251,7 @@ public function postProcess() {

$this->set('errorFile', $errorFile);

$urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contact_Import_Parser_Contact';
$this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams));

// @todo - these should use the new url but are not reliably updating the table yet.
$urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Contact_Import_Parser_Contact';
$this->set('downloadConflictRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams));

Expand Down
9 changes: 5 additions & 4 deletions CRM/Contact/Import/Form/Summary.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public function preProcess() {
// set the error message path to display
$this->assign('errorFile', $this->get('errorFile'));

$totalRowCount = $this->get('totalRowCount');
$totalRowCount = $this->getRowCount();
$relatedCount = $this->get('relatedCount');
$totalRowCount += $relatedCount;

Expand Down Expand Up @@ -81,9 +81,6 @@ public function preProcess() {
$this->assign('dupeActionString', $dupeActionString);

$properties = [
'totalRowCount',
'validRowCount',
'invalidRowCount',
'conflictRowCount',
'downloadConflictRecordsUrl',
'downloadErrorRecordsUrl',
Expand All @@ -98,6 +95,10 @@ public function preProcess() {
foreach ($properties as $property) {
$this->assign($property, $this->get($property));
}
$this->assign('totalRowCount', $this->getRowCount());
$this->assign('validRowCount', $this->getRowCount(CRM_Import_Parser::VALID));
$this->assign('invalidRowCount', $this->getRowCount(CRM_Import_Parser::ERROR));
$this->assign('downloadDuplicateRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::DUPLICATE));

$session = CRM_Core_Session::singleton();
$session->pushUserContext(CRM_Utils_System::url('civicrm/import/contact', 'reset=1'));
Expand Down
47 changes: 3 additions & 44 deletions CRM/Contact/Import/ImportJob.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,37 +50,6 @@ class CRM_Contact_Import_ImportJob {

protected $_userJobID;

/**
* @param string|null $tableName
* @param string|null $createSql
* @param bool $createTable
*
* @throws \CRM_Core_Exception
*/
public function __construct($tableName = NULL, $createSql = NULL, $createTable = FALSE) {
$dao = new CRM_Core_DAO();
$db = $dao->getDatabaseConnection();

if ($createTable) {
if (!$createSql) {
throw new CRM_Core_Exception(ts('Either an existing table name or an SQL query to build one are required'));
}
if ($tableName) {
// Drop previous table if passed in and create new one.
$db->query("DROP TABLE IF EXISTS $tableName");
}
$table = CRM_Utils_SQL_TempTable::build()->setDurable();
$tableName = $table->getName();
$table->createWithQuery($createSql);
}

if (!$tableName) {
throw new CRM_Core_Exception(ts('Import Table is required.'));
}

$this->_tableName = $tableName;
}

/**
* @return null|string
*/
Expand All @@ -92,19 +61,9 @@ public function getTableName() {
* Has the job completed.
*
* @return bool
* @throws Exception
*/
public function isComplete() {
if (!$this->_statusFieldName) {
throw new CRM_Core_Exception("Could not get name of the import status field");
}
$query = "SELECT * FROM $this->_tableName
WHERE $this->_statusFieldName = 'NEW' LIMIT 1";
$result = CRM_Core_DAO::executeQuery($query);
if ($result->fetch()) {
return FALSE;
}
return TRUE;
public function isComplete(): bool {
return $this->_parser->isComplete();
}

/**
Expand Down Expand Up @@ -220,7 +179,7 @@ public function runImport(&$form, $timeout = 55) {
$parserParameters['relatedContactWebsiteType']
);
$this->_parser->setUserJobID($this->_userJobID);
$this->_parser->run($this->_tableName, $mapperFields,
$this->_parser->run(NULL, $mapperFields,
CRM_Import_Parser::MODE_IMPORT,
$this->_contactType,
$this->_primaryKeyName,
Expand Down
50 changes: 15 additions & 35 deletions CRM/Contact/Import/Parser/Contact.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2583,15 +2583,7 @@ public function run(
$this->_conflicts = [];
$this->_unparsedAddresses = [];

// Transitional support for deprecating table_name (and other fields)
// form input - the goal is to load them from userJob - but eventually
// we will just load the datasource object and this code will not know the
// table name.
if (!$tableName && $this->userJobID) {
$tableName = $this->getUserJob()['metadata']['DataSource']['table_name'];
}

$this->_tableName = $tableName;
$this->_tableName = $tableName = $this->getUserJob()['metadata']['DataSource']['table_name'];
$this->_primaryKeyName = '_id';
$this->_statusFieldName = '_status';

Expand All @@ -2609,7 +2601,7 @@ public function run(
// get the contents of the temp. import table
$query = "SELECT * FROM $tableName";
if ($mode == self::MODE_IMPORT) {
$query .= " WHERE $statusFieldName = 'NEW'";
$query .= " WHERE _status = 'NEW'";
}
if ($this->_maxLinesToProcess > 0) {
// Note this would only be the case in MapForm mode, where it is set to 100
Expand Down Expand Up @@ -2723,15 +2715,6 @@ public function run(
}
}

if ($this->_invalidRowCount) {
// removed view url for invlaid contacts
$headers = array_merge([
ts('Line Number'),
ts('Reason'),
], $customHeaders);
$this->_errorFileName = self::errorFileName(self::ERROR);
self::exportCSV($this->_errorFileName, $headers, $this->_errors);
}
if ($this->_conflictCount) {
$headers = array_merge([
ts('Line Number'),
Expand All @@ -2740,15 +2723,6 @@ public function run(
$this->_conflictFileName = self::errorFileName(self::CONFLICT);
self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
}
if ($this->_duplicateCount) {
$headers = array_merge([
ts('Line Number'),
ts('View Contact URL'),
], $customHeaders);

$this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
}
if ($this->_unMatchCount) {
$headers = array_merge([
ts('Line Number'),
Expand Down Expand Up @@ -3072,9 +3046,6 @@ public function set($store, $mode = self::MODE_SUMMARY) {
$store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
}

if ($this->_invalidRowCount) {
$store->set('errorsFileName', $this->_errorFileName);
}
if ($this->_conflictCount) {
$store->set('conflictsFileName', $this->_conflictFileName);
}
Expand All @@ -3092,11 +3063,7 @@ public function set($store, $mode = self::MODE_SUMMARY) {
if ($this->_duplicateCount) {
$store->set('duplicatesFileName', $this->_duplicateFileName);
}
if ($this->_unparsedAddressCount) {
$store->set('errorsFileName', $this->_errorFileName);
}
}
//echo "$this->_totalCount,$this->_invalidRowCount,$this->_conflictCount,$this->_duplicateCount";
}

/**
Expand Down Expand Up @@ -3651,4 +3618,17 @@ public function getMappedRow(array $values): array {
return $params;
}

/**
* Is the job complete.
*
* This function transitionally accesses the table from the userJob
* directly - but the function should be moved to the dataSource class.
*
* @throws \API_Exception
*/
public function isComplete() {
$tableName = $this->getUserJob()['metadata']['DataSource']['table_name'];
return (bool) CRM_Core_DAO::singleValueQuery("SELECT count(*) FROM $tableName WHERE _status = 'NEW' LIMIT 1");
}

}
6 changes: 6 additions & 0 deletions CRM/Core/xml/Menu/Import.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,12 @@
<page_callback>CRM_Contact_Import_Controller</page_callback>
<weight>410</weight>
</item>
<item>
<path>civicrm/import/outcome</path>
<title>Import results</title>
<access_arguments>access CiviCRM</access_arguments>
<page_callback>CRM_Import_Forms::outputCSV</page_callback>
</item>
<item>
<path>civicrm/import/activity</path>
<title>Import Activities</title>
Expand Down
Loading

0 comments on commit a863d40

Please sign in to comment.