Export merge to household - fix DB error relating to fields too long for table.

CRM/Export/BAO/Export.php

@@ -41,15 +41,6 @@ class CRM_Export_BAO_Export {
   // CRM-7675
   const EXPORT_ROW_COUNT = 100000;
 
-  /**
-   * Key representing the head of household in the relationship array.
-   *
-   * e.g. ['8_b_a' => 'Household Member Is', '8_a_b = 'Household Member Of'.....]
-   *
-   * @var
-   */
-  protected static $relationshipTypes = [];
-
   /**
    * Get default return property for export based on mode
    *
@@ -226,12 +217,11 @@ public static function exportComponents(
 
     $processor = new CRM_Export_BAO_ExportProcessor($exportMode, $fields, $queryOperator, $mergeSameHousehold, $isPostalOnly);
     $returnProperties = array();
-    self::$relationshipTypes = $processor->getRelationshipTypes();
 
     if ($fields) {
       foreach ($fields as $key => $value) {
         $fieldName = CRM_Utils_Array::value(1, $value);
-        if (!$fieldName) {
+        if (!$fieldName || $processor->isHouseholdMergeRelationshipTypeKey($fieldName)) {
           continue;
         }
 
@@ -341,17 +331,7 @@ public static function exportComponents(
         $returnProperties['id'] = 1;
       }
 
-      foreach ($returnProperties as $key => $value) {
-        if (!$processor->isRelationshipTypeKey($key)) {
-          foreach ($processor->getHouseholdRelationshipTypes() as $householdRelationshipType) {
-            if (!in_array($key, ['location_type', 'im_provider'])) {
-              $returnProperties[$householdRelationshipType][$key] = $value;
-            }
-          }
-          // @todo - don't use returnProperties above.
-          $processor->setHouseholdMergeReturnProperties($returnProperties[$householdRelationshipType]);
-        }
-      }
+      $processor->setHouseholdMergeReturnProperties(array_diff_key($returnProperties, array_fill_keys(['location_type', 'im_provider'], 1)));
     }
 
     self::buildRelatedContactArray($selectAll, $ids, $processor, $componentTable);
@@ -475,6 +455,9 @@ public static function exportComponents(
         $count++;
         $rowsThisIteration++;
         $row = $processor->buildRow($query, $iterationDAO, $outputColumns, $metadata, $paymentDetails, $addPaymentHeader, $paymentTableId);
+        if ($row === FALSE) {
+          continue;
+        }
 
         // add component info
         // write the row to a file
@@ -500,13 +483,6 @@ public static function exportComponents(
         self::mergeSameAddress($exportTempTable, $sqlColumns, $exportParams);
       }
 
-      // merge the records if they have corresponding households
-      if ($mergeSameHousehold) {
-        foreach ($processor->getHouseholdRelationshipTypes() as $householdRelationshipType) {
-          self::mergeSameHousehold($exportTempTable, $sqlColumns, $householdRelationshipType);
-        }
-      }
-
       // call export hook
       CRM_Utils_Hook::export($exportTempTable, $headerRows, $sqlColumns, $exportMode, $componentTable, $ids);
 
@@ -996,82 +972,6 @@ public static function _buildMasterCopyArray($sql, $exportParams, $sharedAddress
     return $merge;
   }
 
-  /**
-   * Merge household record into the individual record
-   * if exists
-   *
-   * @param string $exportTempTable
-   *   Temporary temp table that stores the records.
-   * @param array $sqlColumns
-   *   Array of names of the table columns of the temp table.
-   * @param string $prefix
-   *   Name of the relationship type that is prefixed to the table columns.
-   */
-  public static function mergeSameHousehold($exportTempTable, &$sqlColumns, $prefix) {
-    $prefixColumn = $prefix . '_';
-    $replaced = array();
-
-    // name map of the non standard fields in header rows & sql columns
-    $mappingFields = array(
-      'civicrm_primary_id' => 'id',
-      'provider_id' => 'im_service_provider',
-    );
-
-    //figure out which columns are to be replaced by which ones
-    foreach ($sqlColumns as $columnNames => $dontCare) {
-      if ($rep = CRM_Utils_Array::value($columnNames, $mappingFields)) {
-        $replaced[$columnNames] = CRM_Utils_String::munge($prefixColumn . $rep, '_', 64);
-      }
-      else {
-        $householdColName = CRM_Utils_String::munge($prefixColumn . $columnNames, '_', 64);
-
-        if (!empty($sqlColumns[$householdColName])) {
-          $replaced[$columnNames] = $householdColName;
-        }
-      }
-    }
-    $query = "UPDATE $exportTempTable SET ";
-
-    $clause = array();
-    foreach ($replaced as $from => $to) {
-      $clause[] = "$from = $to ";
-      unset($sqlColumns[$to]);
-    }
-    $query .= implode(",\n", $clause);
-    $query .= " WHERE {$replaced['civicrm_primary_id']} != ''";
-
-    CRM_Core_DAO::executeQuery($query);
-
-    //drop the table columns that store redundant household info
-    $dropQuery = "ALTER TABLE $exportTempTable ";
-    foreach ($replaced as $householdColumns) {
-      $dropClause[] = " DROP $householdColumns ";
-    }
-    $dropQuery .= implode(",\n", $dropClause);
-
-    CRM_Core_DAO::executeQuery($dropQuery);
-
-    // also drop the temp table if exists
-    $sql = "DROP TABLE IF EXISTS {$exportTempTable}_temp";
-    CRM_Core_DAO::executeQuery($sql);
-
-    // clean up duplicate records
-    $query = "
-CREATE TABLE {$exportTempTable}_temp SELECT *
-FROM {$exportTempTable}
-GROUP BY civicrm_primary_id ";
-
-    CRM_Core_DAO::disableFullGroupByMode();
-    CRM_Core_DAO::executeQuery($query);
-    CRM_Core_DAO::reenableFullGroupByMode();
-
-    $query = "DROP TABLE $exportTempTable";
-    CRM_Core_DAO::executeQuery($query);
-
-    $query = "ALTER TABLE {$exportTempTable}_temp RENAME TO {$exportTempTable}";
-    CRM_Core_DAO::executeQuery($query);
-  }
-
   /**
    * @param $exportTempTable
    * @param $headerRows

CRM/Export/BAO/ExportProcessor.php

@@ -104,6 +104,13 @@ class CRM_Export_BAO_ExportProcessor {
    */
   protected $relationshipReturnProperties = [];
 
+  /**
+   * IDs of households that have already been exported.
+   *
+   * @var array
+   */
+  protected $exportedHouseholds = [];
+
   /**
    * Get return properties by relationship.
    * @return array
@@ -245,6 +252,30 @@ public function getRelationshipValue($relationshipType, $contactID, $field) {
     return isset($this->relatedContactValues[$relationshipType][$contactID][$field]) ? $this->relatedContactValues[$relationshipType][$contactID][$field] : '';
   }
 
+  /**
+   * Get the id of the related household.
+   *
+   * @param int $contactID
+   * @param string $relationshipType
+   *
+   * @return int
+   */
+  public function getRelatedHouseholdID($contactID, $relationshipType) {
+    return $this->relatedContactValues[$relationshipType][$contactID]['id'];
+  }
+
+  /**
+   * Has the household already been exported.
+   *
+   * @param int $housholdContactID
+   *
+   * @return bool
+   */
+  public function isHouseholdExported($housholdContactID) {
+    return isset($this->exportedHouseholds[$housholdContactID]);
+
+  }
+
   /**
    * @return bool
    */
@@ -606,18 +637,30 @@ public function getMetadata() {
    * @param $addPaymentHeader
    * @param $paymentTableId
    *
-   * @return array
+   * @return array|bool
    */
   public function buildRow($query, $iterationDAO, $outputColumns, $metadata, $paymentDetails, $addPaymentHeader, $paymentTableId) {
     $phoneTypes = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Phone', 'phone_type_id');
     $imProviders = CRM_Core_PseudoConstant::get('CRM_Core_DAO_IM', 'provider_id');
 
     $row = [];
+    $householdMergeRelationshipType = $this->getHouseholdMergeTypeForRow($iterationDAO->contact_id);
+    if ($householdMergeRelationshipType) {
+      $householdID = $this->getRelatedHouseholdID($iterationDAO->contact_id, $householdMergeRelationshipType);
+      if ($this->isHouseholdExported($householdID)) {
+        return FALSE;
+      }
+      foreach (array_keys($outputColumns) as $column) {
+        $row[$column] = $this->getRelationshipValue($householdMergeRelationshipType, $iterationDAO->contact_id, $column);
+      }
+      $this->markHouseholdExported($householdID);
+      return $row;
+    }
+
     $query->convertToPseudoNames($iterationDAO);
 
     //first loop through output columns so that we return what is required, and in same order.
     foreach ($outputColumns as $field => $value) {
-
       // add im_provider to $dao object
       if ($field == 'im_provider' && property_exists($iterationDAO, 'provider_id')) {
         $iterationDAO->im_provider = $iterationDAO->provider_id;
@@ -645,20 +688,7 @@ public function buildRow($query, $iterationDAO, $outputColumns, $metadata, $paym
       }
 
       if ($this->isRelationshipTypeKey($field)) {
-        foreach (array_keys($value) as $property) {
-          if ($property === 'location') {
-            // @todo just undo all this nasty location wrangling!
-            foreach ($value['location'] as $locationKey => $locationFields) {
-              foreach (array_keys($locationFields) as $locationField) {
-                $fieldKey = str_replace(' ', '_', $locationKey . '-' . $locationField);
-                $row[$field . '_' . $fieldKey] = $this->getRelationshipValue($field, $iterationDAO->contact_id, $fieldKey);
-              }
-            }
-          }
-          else {
-            $row[$field . '_' . $property] = $this->getRelationshipValue($field, $iterationDAO->contact_id, $property);
-          }
-        }
+        $this->buildRelationshipFieldsForRow($row, $iterationDAO->contact_id, $value, $field);
       }
       else {
         $row[$field] = $this->getTransformedFieldValue($field, $iterationDAO, $fieldValue, $metadata, $paymentDetails);
@@ -692,6 +722,33 @@ public function buildRow($query, $iterationDAO, $outputColumns, $metadata, $paym
     return $row;
   }
 
+  /**
+   * If this row has a household whose details we should use get the relationship type key.
+   *
+   * @param $contactID
+   *
+   * @return bool
+   */
+  public function getHouseholdMergeTypeForRow($contactID) {
+    if (!$this->isMergeSameHousehold()) {
+      return FALSE;
+    }
+    foreach ($this->getHouseholdRelationshipTypes() as $relationshipType) {
+      if (isset($this->relatedContactValues[$relationshipType][$contactID])) {
+        return $relationshipType;
+      }
+    }
+  }
+
+  /**
+   * Mark the given household as already exported.
+   *
+   * @param $householdID
+   */
+  public function markHouseholdExported($householdID) {
+    $this->exportedHouseholds[$householdID] = $householdID;
+  }
+
   /**
    * @param $field
    * @param $iterationDAO
@@ -1276,4 +1333,27 @@ public function getWhereParams() {
     return $params;
   }
 
+  /**
+   * @param $row
+   * @param $contactID
+   * @param $value
+   * @param $field
+   */
+  protected function buildRelationshipFieldsForRow(&$row, $contactID, $value, $field) {
+    foreach (array_keys($value) as $property) {
+      if ($property === 'location') {
+        // @todo just undo all this nasty location wrangling!
+        foreach ($value['location'] as $locationKey => $locationFields) {
+          foreach (array_keys($locationFields) as $locationField) {
+            $fieldKey = str_replace(' ', '_', $locationKey . '-' . $locationField);
+            $row[$field . '_' . $fieldKey] = $this->getRelationshipValue($field, $contactID, $fieldKey);
+          }
+        }
+      }
+      else {
+        $row[$field . '_' . $property] = $this->getRelationshipValue($field, $contactID, $property);
+      }
+    }
+  }
+
 }

tests/phpunit/CRM/Export/BAO/ExportTest.php

@@ -562,7 +562,6 @@ public function testExportRelationshipsMergeToHousehold() {
    * Test exporting relationships.
    */
   public function testExportRelationshipsMergeToHouseholdAllFields() {
-    $this->markTestIncomplete('Does not yet work under CI due to mysql limitation (number of columns in table). Works on some boxes');
     list($householdID) = $this->setUpHousehold();
     list($tableName) = CRM_Export_BAO_Export::exportComponents(
       FALSE,
@@ -583,12 +582,13 @@ public function testExportRelationshipsMergeToHouseholdAllFields() {
     );
     $dao = CRM_Core_DAO::executeQuery("SELECT * FROM {$tableName}");
     while ($dao->fetch()) {
+      $this->assertEquals('Unit Test household', $dao->display_name);
       $this->assertEquals('Portland', $dao->city);
       $this->assertEquals('ME', $dao->state_province);
       $this->assertEquals($householdID, $dao->civicrm_primary_id);
       $this->assertEquals($householdID, $dao->civicrm_primary_id);
-      $this->assertEquals('Unit Test Household', $dao->addressee);
-      $this->assertEquals('Unit Test Household', $dao->display_name);
+      $this->assertEquals('Unit Test household', $dao->addressee);
+      $this->assertEquals(1, $dao->N);
     }
   }