Skip to content

Commit

Permalink
Compress setting sql columns into output specification
Browse files Browse the repository at this point in the history
  • Loading branch information
@eileenmcnaughton
eileenmcnaughton committed Dec 3, 2018
1 parent c32be90 commit 05adae4
Show file tree
Hide file tree
Showing 3 changed files with 274 additions and 192 deletions.
51 changes: 5 additions & 46 deletions CRM/Export/BAO/Export.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -444,11 +444,8 @@ public static function exportComponents(
list($outputColumns, $metadata) = self::getExportStructureArrays($returnProperties, $processor);
$headerRows = $processor->getHeaderRows();


// add payment headers if required
if ($addPaymentHeader && $processor->isExportPaymentFields()) {
// @todo rather than do this for every single row do it before the loop starts.
// where other header definitions take place.
$headerRows = array_merge($headerRows, $processor->getPaymentHeaders());
foreach (array_keys($processor->getPaymentHeaders()) as $paymentHdr) {
$processor->setSqlColumnDefn($paymentHdr);
Expand Down Expand Up @@ -1228,46 +1225,26 @@ public static function componentPaymentFields() {
* yet find a way to comment them for posterity.
*/
public static function getExportStructureArrays($returnProperties, $processor) {
$metadata = $outputColumns = array();
$phoneTypes = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Phone', 'phone_type_id');
$imProviders = CRM_Core_PseudoConstant::get('CRM_Core_DAO_IM', 'provider_id');
$outputColumns = $metadata = array();
$queryFields = $processor->getQueryFields();
foreach ($returnProperties as $key => $value) {
if (($key != 'location' || !is_array($value)) && !$processor->isRelationshipTypeKey($key)) {
$outputColumns[$key] = $value;
$processor->addOutputSpecification($key);
$processor->setSqlColumnDefn($key);
}
elseif ($processor->isRelationshipTypeKey($key)) {
$outputColumns[$key] = $value;
$field = $key;
foreach ($value as $relationField => $relationValue) {
// below block is same as primary block (duplicate)
if (isset($queryFields[$relationField]['title'])) {
if (!$processor->isHouseholdMergeRelationshipTypeKey($field)) {
// Do not add to header row if we are only generating for merge reasons.
$processor->addOutputSpecification($relationField, $key);
}
$processor->setSqlColumnDefn($field . '-' . $relationField);
$processor->addOutputSpecification($relationField, $key);
}
elseif (is_array($relationValue) && $relationField == 'location') {
// fix header for location type case
foreach ($relationValue as $ltype => $val) {
foreach (array_keys($val) as $fld) {
$type = explode('-', $fld);

$hdr = "{$ltype}-" . $queryFields[$type[0]]['title'];

if (!empty($type[1])) {
if (CRM_Utils_Array::value(0, $type) == 'phone') {
$hdr .= "-" . CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_Phone', 'phone_type_id', $type[1]);
}
elseif (CRM_Utils_Array::value(0, $type) == 'im') {
$hdr .= "-" . CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_IM', 'provider_id', $type[1]);
}
}
$processor->addOutputSpecification($field, $key, $ltype, CRM_Utils_Array::value(1, $type));
$processor->setSqlColumnDefn($field . '-' . $hdr);
$processor->addOutputSpecification($type[0], $key, $ltype, CRM_Utils_Array::value(1, $type));
}
}
}
Expand All @@ -1279,31 +1256,13 @@ public static function getExportStructureArrays($returnProperties, $processor) {
$type = explode('-', $locationFieldName);

$actualDBFieldName = $type[0];
$outputFieldName = $locationType . '-' . $queryFields[$actualDBFieldName]['title'];
$daoFieldName = CRM_Utils_String::munge($locationType) . '-' . $actualDBFieldName;

if (!empty($type[1])) {
$daoFieldName .= "-" . $type[1];
if ($actualDBFieldName == 'phone') {
$outputFieldName .= "-" . CRM_Utils_Array::value($type[1], $phoneTypes);
}
elseif ($actualDBFieldName == 'im') {
$outputFieldName .= "-" . CRM_Utils_Array::value($type[1], $imProviders);
}
}
if ($type[0] == 'im_provider') {
// Warning: shame inducing hack.
$metadata[$daoFieldName]['pseudoconstant']['var'] = 'imProviders';
}

$processor->addOutputSpecification($outputFieldName, NULL, $locationType, CRM_Utils_Array::value(1, $type));
$processor->setSqlColumnDefn($outputFieldName);
if ($actualDBFieldName == 'country' || $actualDBFieldName == 'world_region') {
$metadata[$daoFieldName] = array('context' => 'country');
}
if ($actualDBFieldName == 'state_province') {
$metadata[$daoFieldName] = array('context' => 'province');
}
$processor->addOutputSpecification($actualDBFieldName, NULL, $locationType, CRM_Utils_Array::value(1, $type));
$metadata[$daoFieldName] = $processor->getMetaDataForField($actualDBFieldName);
$outputColumns[$daoFieldName] = TRUE;
}
}
Expand Down
175 changes: 149 additions & 26 deletions CRM/Export/BAO/ExportProcessor.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,12 @@ public function getQueryFields() {
* @param array $queryFields
*/
public function setQueryFields($queryFields) {
// legacy hacks - we add these to queryFields because this
// pseudometadata is currently required.
$queryFields['im_provider']['pseudoconstant']['var'] = 'imProviders';
$queryFields['country']['context'] = 'country';
$queryFields['world_region']['context'] = 'country';
$queryFields['state_province']['context'] = 'province';
$this->queryFields = $queryFields;
}

Expand Down Expand Up @@ -450,32 +456,54 @@ public function runQuery($params, $order, $returnProperties) {
* @param int $entityTypeID phone_type_id or provider_id for phone or im fields.
*/
public function addOutputSpecification($key, $relationshipType = NULL, $locationType = NULL, $entityTypeID = NULL) {
$label = $this->getHeaderForRow($key);
$labelPrefix = $fieldPrefix = [];
if ($relationshipType) {
$labelPrefix[] = $this->getRelationshipTypes()[$relationshipType];
$fieldPrefix[] = $relationshipType;
}
if ($locationType) {
$labelPrefix[] = $fieldPrefix[] = $locationType;
}
$entityLabel = '';
if ($entityTypeID) {
if ($key === 'phone') {
$labelPrefix[] = $fieldPrefix[] = CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_Phone', 'phone_type_id', $entityTypeID);
$entityLabel = CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_Phone', 'phone_type_id', $entityTypeID);
}
if ($key === 'im') {
$labelPrefix[] = $fieldPrefix[] = CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_IM', 'provider_id', $entityTypeID);
$entityLabel = CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_IM', 'provider_id', $entityTypeID);
}
}
$index = $this->getMungedFieldName(($fieldPrefix ? (implode('-', $fieldPrefix) . '-') : '') . $key);
$this->outputSpecification[$index]['header'] = ($labelPrefix ? (implode('-', $labelPrefix) . '-') : '') . $label;

// These oddly constructed keys are for legacy reasons. Altering them will affect test success
// but in time it may be good to rationalise them.
$label = $this->getOutputSpecificationLabel($key, $relationshipType, $locationType, $entityLabel);
$index = $this->getOutputSpecificationIndex($key, $relationshipType, $locationType, $entityLabel);
$fieldKey = $this->getOutputSpecificationFieldKey($key, $relationshipType, $locationType, $entityLabel);

$this->outputSpecification[$index]['header'] = $label;
$this->outputSpecification[$index]['sql_columns'] = $this->getSqlColumnDefinition($fieldKey, $key);

if ($relationshipType && $this->isHouseholdMergeRelationshipTypeKey($relationshipType)) {
$this->setColumnAsCalculationOnly($index);
}
$this->outputSpecification[$index]['metadata'] = $this->getMetaDataForField($key);
}

/**
* Get the metadata for the given field.
*
* @param $key
*
* @return array
*/
public function getMetaDataForField($key) {
$mappings = ['contact_id' => 'id'];
if (isset($this->getQueryFields()[$key])) {
return $this->getQueryFields()[$key];
}
if (isset($mappings[$key])) {
return $this->getQueryFields()[$mappings[$key]];
}
return [];
}

/**
* @param $key
*/
public function setSqlColumnDefn($key) {
$this->outputSpecification[$this->getMungedFieldName($key)]['sql_columns'] = $this->getSqlColumnDefinition($key);
$this->outputSpecification[$this->getMungedFieldName($key)]['sql_columns'] = $this->getSqlColumnDefinition($key, $this->getMungedFieldName($key));
}

/**
Expand Down Expand Up @@ -515,6 +543,16 @@ public function getSQLColumns() {
return $sqlColumns;
}

/**
* @return array
*/
public function getMetadata() {
$metadata = [];
foreach ($this->outputSpecification as $key => $spec) {
$metadata[$key] = $spec['metadata'];
}
return $metadata;
}

/**
* Build the row for output.
Expand Down Expand Up @@ -973,17 +1011,17 @@ public function getValidLocationFields() {
/**
* Get the sql column definition for the given field.
*
* @param $field
* @param string $fieldName
* @param string $columnName
*
* @return mixed
*/
public function getSqlColumnDefinition($field) {
$fieldName = $this->getMungedFieldName($field);
public function getSqlColumnDefinition($fieldName, $columnName) {

// early exit for master_id, CRM-12100
// in the DB it is an ID, but in the export, we retrive the display_name of the master record
// also for current_employer, CRM-16939
if ($fieldName == 'master_id' || $fieldName == 'current_employer') {
if ($columnName == 'master_id' || $columnName == 'current_employer') {
return "$fieldName varchar(128)";
}

Expand All @@ -995,20 +1033,20 @@ public function getSqlColumnDefinition($field) {
$queryFields = $this->getQueryFields();
$lookUp = ['prefix_id', 'suffix_id'];
// set the sql columns
if (isset($queryFields[$field]['type'])) {
switch ($queryFields[$field]['type']) {
if (isset($queryFields[$columnName]['type'])) {
switch ($queryFields[$columnName]['type']) {
case CRM_Utils_Type::T_INT:
case CRM_Utils_Type::T_BOOLEAN:
if (in_array($field, $lookUp)) {
if (in_array($columnName, $lookUp)) {
return "$fieldName varchar(255)";
}
else {
return "$fieldName varchar(16)";
}

case CRM_Utils_Type::T_STRING:
if (isset($queryFields[$field]['maxlength'])) {
return "$fieldName varchar({$queryFields[$field]['maxlength']})";
if (isset($queryFields[$columnName]['maxlength'])) {
return "$fieldName varchar({$queryFields[$columnName]['maxlength']})";
}
else {
return "$fieldName varchar(255)";
Expand Down Expand Up @@ -1052,12 +1090,12 @@ public function getSqlColumnDefinition($field) {
}
else {
// set the sql columns for custom data
if (isset($queryFields[$field]['data_type'])) {
if (isset($queryFields[$columnName]['data_type'])) {

switch ($queryFields[$field]['data_type']) {
switch ($queryFields[$columnName]['data_type']) {
case 'String':
// May be option labels, which could be up to 512 characters
$length = max(512, CRM_Utils_Array::value('text_length', $queryFields[$field]));
$length = max(512, CRM_Utils_Array::value('text_length', $queryFields[$columnName]));
return "$fieldName varchar($length)";

case 'Country':
Expand Down Expand Up @@ -1094,4 +1132,89 @@ public function getMungedFieldName($field) {
return $fieldName;
}

/**
* In order to respect the history of this class we need to index kinda illogically.
*
* On the bright side - this stuff is tested within a nano-byte of it's life.
*
* e.g '2-a-b_Home-City'
*
* @param string $key
* @param string $relationshipType
* @param string $locationType
* @param $entityLabel
*
* @return string
*/
protected function getOutputSpecificationIndex($key, $relationshipType, $locationType, $entityLabel) {
if ($locationType || $entityLabel || $key === 'im') {
// Just cos that's the history...
if ($key !== 'master_id') {
$key = $this->getHeaderForRow($key);
}
}
if (!$relationshipType || $key !== 'id') {
$key = $this->getMungedFieldName($key);
}
return $this->getMungedFieldName(
($relationshipType ? ($relationshipType . '_') : '')
. ($locationType ? ($locationType . '_') : '')
. $key
. ($entityLabel ? ('_' . $entityLabel) : '')
);
}

/**
* Get the compiled label for the column.
*
* e.g 'Gender', 'Employee Of-Home-city'
*
* @param string $key
* @param string $relationshipType
* @param string $locationType
* @param string $entityLabel
*
* @return string
*/
protected function getOutputSpecificationLabel($key, $relationshipType, $locationType, $entityLabel) {
return ($relationshipType ? $this->getRelationshipTypes()[$relationshipType] . '-' : '')
. ($locationType ? $locationType . '-' : '')
. $this->getHeaderForRow($key)
. ($entityLabel ? '-' . $entityLabel : '');
}

/**
* Get the mysql field name key.
*
* This key is locked in by tests but the reasons for the specific conventions -
* ie. headings are used for keying fields in some cases, are likely
* accidental rather than deliberate.
*
* This key is used for the output sql array.
*
* @param string $key
* @param $relationshipType
* @param $locationType
* @param $entityLabel
*
* @return string
*/
protected function getOutputSpecificationFieldKey($key, $relationshipType, $locationType, $entityLabel) {
if ($relationshipType || $entityLabel || $key === 'im') {
if ($key !== 'state_province' && $key !== 'id') {
$key = $this->getHeaderForRow($key);
}
}
if (!$relationshipType || $key !== 'id') {
$key = $this->getMungedFieldName($key);
}
$fieldKey = $this->getMungedFieldName(
($relationshipType ? ($relationshipType . '_') : '')
. ($locationType ? ($locationType . '_') : '')
. $key
. ($entityLabel ? ('_' . $entityLabel) : '')
);
return $fieldKey;
}

}
Loading

0 comments on commit 05adae4

Please sign in to comment.