From 153d2896a0496bf2ec9f0db265714050539f74b9 Mon Sep 17 00:00:00 2001 From: "deb.monish" Date: Fri, 23 Mar 2018 14:27:37 +0530 Subject: [PATCH] Exporting Contacts using "Export PRIMARY fields" and "Merge Household Members into their Households" produces db error --- CRM/Contact/BAO/Query.php | 4 +- CRM/Export/BAO/Export.php | 304 ++++++++++++-------- tests/phpunit/CRM/Export/BAO/ExportTest.php | 2 +- 3 files changed, 185 insertions(+), 125 deletions(-) diff --git a/CRM/Contact/BAO/Query.php b/CRM/Contact/BAO/Query.php index 69c5655c5c77..3668ffbd0aef 100644 --- a/CRM/Contact/BAO/Query.php +++ b/CRM/Contact/BAO/Query.php @@ -4708,10 +4708,11 @@ public static function filterCountryFromValuesIfStateExists(&$formValues) { * @param array $selectClauses * @param array $groupBy - Columns already included in GROUP By clause. * @param string $aggregateFunction + * @param bool $appendAlias - Append select column alias as it's name itself * * @return string */ - public static function appendAnyValueToSelect($selectClauses, $groupBy, $aggregateFunction = 'ANY_VALUE') { + public static function appendAnyValueToSelect($selectClauses, $groupBy, $aggregateFunction = 'ANY_VALUE', $appendAlias = FALSE) { if (!CRM_Utils_SQL::disableFullGroupByMode()) { $groupBy = array_map('trim', (array) $groupBy); $aggregateFunctions = '/(ROUND|AVG|COUNT|GROUP_CONCAT|SUM|MAX|MIN|IF)[[:blank:]]*\(/i'; @@ -4722,6 +4723,7 @@ public static function appendAnyValueToSelect($selectClauses, $groupBy, $aggrega $val = ($aggregateFunction == 'GROUP_CONCAT') ? str_replace($selectColumn, "$aggregateFunction(DISTINCT {$selectColumn})", $val) : str_replace($selectColumn, "$aggregateFunction({$selectColumn})", $val); + $val .= $appendAlias ? " as $selectColumn " : ''; } } } diff --git a/CRM/Export/BAO/Export.php b/CRM/Export/BAO/Export.php index 70a1a14e7358..94ecb3ea1bf6 100644 --- a/CRM/Export/BAO/Export.php +++ b/CRM/Export/BAO/Export.php @@ -566,6 +566,8 @@ public static function exportComponents( $query->_sort = $order; list($select, $from, $where, $having) = $query->query(); + $allRelContactArray = $relationQuery = $relationType = array(); + if ($mergeSameHousehold == 1) { if (empty($returnProperties['id'])) { $returnProperties['id'] = 1; @@ -574,6 +576,10 @@ public static function exportComponents( //also merge Head of Household $relationKeyMOH = CRM_Utils_Array::key('Household Member of', $contactRelationshipTypes); $relationKeyHOH = CRM_Utils_Array::key('Head of Household for', $contactRelationshipTypes); + $relationType = [ + $relationKeyMOH => NULL, + $relationKeyHOH => NULL, + ]; foreach ($returnProperties as $key => $value) { if (!array_key_exists($key, $contactRelationshipTypes)) { @@ -588,8 +594,6 @@ public static function exportComponents( unset($returnProperties[$relationKeyHOH]['im_provider']); } - $allRelContactArray = $relationQuery = array(); - foreach ($contactRelationshipTypes as $rel => $dnt) { if ($relationReturnProperties = CRM_Utils_Array::value($rel, $returnProperties)) { $allRelContactArray[$rel] = array(); @@ -783,7 +787,6 @@ public static function exportComponents( //first loop through output columns so that we return what is required, and in same order. foreach ($outputColumns as $field => $value) { - // add im_provider to $dao object if ($field == 'im_provider' && property_exists($iterationDAO, 'provider_id')) { $iterationDAO->im_provider = $iterationDAO->provider_id; @@ -825,6 +828,9 @@ public static function exportComponents( $row[$field] = $iterationDAO->pledge_next_pay_amount + $iterationDAO->pledge_outstanding_amount; } elseif (array_key_exists($field, $contactRelationshipTypes)) { + if (!array_key_exists($field, $relationType)) { + $relationType[$field] = NULL; + } $relDAO = CRM_Utils_Array::value($iterationDAO->contact_id, $allRelContactArray[$field]); $relationQuery[$field]->convertToPseudoNames($relDAO); self::fetchRelationshipDetails($relDAO, $value, $field, $row); @@ -921,6 +927,9 @@ public static function exportComponents( if ($setHeader) { $exportTempTable = self::createTempTable($sqlColumns); + foreach ($relationType as $type => &$dontCare) { + $relationType[$type] = self::createTempTable($sqlColumns); + } } //build header only once @@ -956,7 +965,7 @@ public static function exportComponents( // output every $tempRowCount rows if ($count % $tempRowCount == 0) { - self::writeDetailsToTable($exportTempTable, $componentDetails, $sqlColumns); + self::writeDetailsToTable($exportTempTable, $componentDetails, $sqlColumns, $relationType, $returnProperties); $componentDetails = array(); } } @@ -967,7 +976,7 @@ public static function exportComponents( } if ($exportTempTable) { - self::writeDetailsToTable($exportTempTable, $componentDetails, $sqlColumns); + self::writeDetailsToTable($exportTempTable, $componentDetails, $sqlColumns, $relationType, $returnProperties); // if postalMailing option is checked, exclude contacts who are deceased, have // "Do not mail" privacy setting, or have no street address @@ -984,8 +993,7 @@ public static function exportComponents( // merge the records if they have corresponding households if ($mergeSameHousehold) { - self::mergeSameHousehold($exportTempTable, $headerRows, $sqlColumns, $relationKeyMOH); - self::mergeSameHousehold($exportTempTable, $headerRows, $sqlColumns, $relationKeyHOH); + self::mergeSameHousehold($exportTempTable, $sqlColumns, $relationType); } // call export hook @@ -1012,6 +1020,119 @@ public static function exportComponents( } } + /** + * Get the values of linked household contact + * + * @param obj $relDAO + * @param array $value + * @param string $relPrefix + * @param array $row + */ + public static function fetchRelationshipDetails($relDAO, $value, $relPrefix, &$row) { + $row[$relPrefix] = []; + $i18n = CRM_Core_I18n::singleton(); + $phoneTypes = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Phone', 'phone_type_id'); + $imProviders = CRM_Core_PseudoConstant::get('CRM_Core_DAO_IM', 'provider_id'); + foreach ($value as $relationField => $relationValue) { + if (is_object($relDAO) && property_exists($relDAO, $relationField)) { + $fieldValue = $relDAO->$relationField; + if ($relationField == 'phone_type_id') { + $fieldValue = $phoneTypes[$relationValue]; + } + elseif ($relationField == 'provider_id') { + $fieldValue = CRM_Utils_Array::value($relationValue, $imProviders); + } + // CRM-13995 + elseif (is_object($relDAO) && in_array($relationField, array( + 'email_greeting', + 'postal_greeting', + 'addressee', + )) + ) { + //special case for greeting replacement + $fldValue = "{$relationField}_display"; + $fieldValue = $relDAO->$fldValue; + } + } + elseif (is_object($relDAO) && $relationField == 'state_province') { + $fieldValue = CRM_Core_PseudoConstant::stateProvince($relDAO->state_province_id); + } + elseif (is_object($relDAO) && $relationField == 'country') { + $fieldValue = CRM_Core_PseudoConstant::country($relDAO->country_id); + } + else { + $fieldValue = ''; + } + + if (is_object($relDAO) && $relationField == 'id') { + $row[$relPrefix][$relationField] = $relDAO->contact_id; + } + elseif (is_array($relationValue) && $relationField == 'location') { + foreach ($relationValue as $ltype => $val) { + foreach (array_keys($val) as $fld) { + $type = explode('-', $fld); + $fldValue = "{$ltype}-" . $type[0]; + if (!empty($type[1])) { + $fldValue .= "-" . $type[1]; + } + // CRM-3157: localise country, region (both have ‘country’ context) + // and state_province (‘province’ context) + switch (TRUE) { + case (!is_object($relDAO)): + $row[$relPrefix . '_' . $fldValue] = ''; + break; + + case in_array('country', $type): + case in_array('world_region', $type): + $row[$relPrefix][$fldValue] = $i18n->crm_translate($relDAO->$fldValue, + array('context' => 'country') + ); + break; + + case in_array('state_province', $type): + $row[$relPrefix][$fldValue] = $i18n->crm_translate($relDAO->$fldValue, + array('context' => 'province') + ); + break; + + default: + $row[$relPrefix][$fldValue] = $relDAO->$fldValue; + break; + } + } + } + } + elseif (isset($fieldValue) && $fieldValue != '') { + //check for custom data + if ($cfID = CRM_Core_BAO_CustomField::getKeyID($relationField)) { + $row[$relPrefix][$relationField] = CRM_Core_BAO_CustomField::displayValue($fieldValue, $cfID); + } + else { + //normal relationship fields + // CRM-3157: localise country, region (both have ‘country’ context) and state_province (‘province’ context) + switch ($relationField) { + case 'country': + case 'world_region': + $row[$relPrefix][$relationField] = $i18n->crm_translate($fieldValue, array('context' => 'country')); + break; + + case 'state_province': + $row[$relPrefix][$relationField] = $i18n->crm_translate($fieldValue, array('context' => 'province')); + break; + + default: + $row[$relPrefix][$relationField] = $fieldValue; + break; + } + } + } + else { + // if relation field is empty or null + $row[$relPrefix][$relationField] = ''; + } + } + } + /** * Name of the export file based on mode. * @@ -1267,10 +1388,12 @@ public static function sqlColumnDefn($query, &$sqlColumns, $field) { /** * @param string $tableName - * @param $details - * @param $sqlColumns + * @param array $details + * @param array $sqlColumns + * @param array $relationTypes + * @param array $returnProperties */ - public static function writeDetailsToTable($tableName, &$details, &$sqlColumns) { + public static function writeDetailsToTable($tableName, &$details, &$sqlColumns, $relationTypes, $returnProperties) { if (empty($details)) { return; } @@ -1287,10 +1410,35 @@ public static function writeDetailsToTable($tableName, &$details, &$sqlColumns) $sqlClause = array(); + $sqlColumnString = '(id, ' . implode(',', array_keys($sqlColumns)) . ')'; + foreach ($details as $dontCare => $row) { $id++; $valueString = array($id); - foreach ($row as $dontCare => $value) { + foreach ($row as $key1 => $value) { + if (array_key_exists($key1, $relationTypes)) { + // consider only those entries to pass, which have linked relationship contact + if (!empty($row[$key1]) && !empty($row[$key1]['id'])) { + $values = [$row['id']]; + foreach ($row[$key1] as $key2 => $value) { + if ($key2 == 'id') { + continue; + } + else { + $values[] = is_null($value) ? '' : "'" . CRM_Core_DAO::escapeString($value) . "'"; + } + } + $values[] = $row[$key1]['id']; + $sql = sprintf(" INSERT INTO %s (%s, civicrm_primary_id) VALUES ( %s ) ", + $relationTypes[$key1], + implode(',', array_keys($returnProperties[$key1])), + implode(",\n", $values) + ); + CRM_Core_DAO::executeQuery($sql); + } + unset($row[$key1]); + continue; + } if (empty($value)) { $valueString[] = "''"; } @@ -1301,8 +1449,6 @@ public static function writeDetailsToTable($tableName, &$details, &$sqlColumns) $sqlClause[] = '(' . implode(',', $valueString) . ')'; } - $sqlColumnString = '(id, ' . implode(',', array_keys($sqlColumns)) . ')'; - $sqlValueString = implode(",\n", $sqlClause); $sql = " @@ -1646,83 +1792,56 @@ public static function _buildMasterCopyArray($sql, $exportParams, $sharedAddress * * @param string $exportTempTable * Temporary temp table that stores the records. - * @param array $headerRows - * Array of headers for the export file. * @param array $sqlColumns * Array of names of the table columns of the temp table. - * @param string $prefix - * Name of the relationship type that is prefixed to the table columns. + * @param array $relationTempTables + * Name of the temp tables that holds the relationship records */ - public static function mergeSameHousehold($exportTempTable, &$headerRows, &$sqlColumns, $prefix) { - $prefixColumn = $prefix . '_'; + public static function mergeSameHousehold($exportTempTable, &$sqlColumns, $relationTempTables) { $allKeys = array_keys($sqlColumns); $replaced = array(); - $headerRows = array_values($headerRows); - - // name map of the non standard fields in header rows & sql columns - $mappingFields = array( - 'civicrm_primary_id' => 'id', - 'contact_source' => 'source', - 'current_employer_id' => 'employer_id', - 'contact_is_deleted' => 'is_deleted', - 'name' => 'address_name', - 'provider_id' => 'im_service_provider', - 'phone_type_id' => 'phone_type', - ); - //figure out which columns are to be replaced by which ones - foreach ($sqlColumns as $columnNames => $dontCare) { - if ($rep = CRM_Utils_Array::value($columnNames, $mappingFields)) { - $replaced[$columnNames] = CRM_Utils_String::munge($prefixColumn . $rep, '_', 64); - } - else { - $householdColName = CRM_Utils_String::munge($prefixColumn . $columnNames, '_', 64); - - if (!empty($sqlColumns[$householdColName])) { - $replaced[$columnNames] = $householdColName; + foreach ($relationTempTables as $relationType => $tempTable) { + foreach ($sqlColumns as $columnNames => $dontCare) { + if (!empty($sqlColumns[$columnNames])) { + $replaced["temp." . $columnNames] = "$relationType.$columnNames"; } } - } - $query = "UPDATE $exportTempTable SET "; - $clause = array(); - foreach ($replaced as $from => $to) { - $clause[] = "$from = $to "; - unset($sqlColumns[$to]); - if ($key = CRM_Utils_Array::key($to, $allKeys)) { - unset($headerRows[$key]); + $query = "UPDATE $exportTempTable temp + INNER JOIN $tempTable $relationType ON $relationType.id = temp.civicrm_primary_id + SET "; + $clause = array(); + foreach ($replaced as $from => $to) { + $clause[] = "$from = $to "; } - } - $query .= implode(",\n", $clause); - $query .= " WHERE {$replaced['civicrm_primary_id']} != ''"; + $query .= implode(",\n", $clause); + $query .= " WHERE temp.civicrm_primary_id != '' "; - CRM_Core_DAO::executeQuery($query); + CRM_Core_DAO::executeQuery($query); - //drop the table columns that store redundant household info - $dropQuery = "ALTER TABLE $exportTempTable "; - foreach ($replaced as $householdColumns) { - $dropClause[] = " DROP $householdColumns "; + $sql = "DROP TABLE IF EXISTS $tempTable"; + CRM_Core_DAO::executeQuery($sql); } - $dropQuery .= implode(",\n", $dropClause); - - CRM_Core_DAO::executeQuery($dropQuery); // also drop the temp table if exists $sql = "DROP TABLE IF EXISTS {$exportTempTable}_temp"; CRM_Core_DAO::executeQuery($sql); // clean up duplicate records + $select = CRM_Contact_BAO_Query::appendAnyValueToSelect($allKeys, "civicrm_primary_id", 'GROUP_CONCAT', TRUE); $query = " -CREATE TABLE {$exportTempTable}_temp SELECT * +CREATE TABLE {$exportTempTable}_temp +$select FROM {$exportTempTable} GROUP BY civicrm_primary_id "; - CRM_Core_DAO::executeQuery($query); $query = "DROP TABLE $exportTempTable"; CRM_Core_DAO::executeQuery($query); $query = "ALTER TABLE {$exportTempTable}_temp RENAME TO {$exportTempTable}"; + CRM_Core_DAO::executeQuery($query); } @@ -1914,73 +2033,12 @@ public static function setHeaderRows($field, $headerRows, $sqlColumns, $query, $ $headerRows[] = $query->_fields['activity'][$field]['title']; } } - elseif (array_key_exists($field, $contactRelationshipTypes)) { - foreach ($value as $relationField => $relationValue) { - // below block is same as primary block (duplicate) - if (isset($relationQuery[$field]->_fields[$relationField]['title'])) { - if ($relationQuery[$field]->_fields[$relationField]['name'] == 'name') { - $headerName = $field . '-' . $relationField; - } - else { - if ($relationField == 'current_employer') { - $headerName = $field . '-' . 'current_employer'; - } - else { - $headerName = $field . '-' . $relationQuery[$field]->_fields[$relationField]['name']; - } - } - - $headerRows[] = $headerName; - - self::sqlColumnDefn($query, $sqlColumns, $headerName); - } - elseif ($relationField == 'phone_type_id') { - $headerName = $field . '-' . 'Phone Type'; - $headerRows[] = $headerName; - self::sqlColumnDefn($query, $sqlColumns, $headerName); - } - elseif ($relationField == 'provider_id') { - $headerName = $field . '-' . 'Im Service Provider'; - $headerRows[] = $headerName; - self::sqlColumnDefn($query, $sqlColumns, $headerName); - } - elseif ($relationField == 'state_province_id') { - $headerName = $field . '-' . 'state_province_id'; - $headerRows[] = $headerName; - self::sqlColumnDefn($query, $sqlColumns, $headerName); - } - elseif (is_array($relationValue) && $relationField == 'location') { - // fix header for location type case - foreach ($relationValue as $ltype => $val) { - foreach (array_keys($val) as $fld) { - $type = explode('-', $fld); - - $hdr = "{$ltype}-" . $relationQuery[$field]->_fields[$type[0]]['title']; - - if (!empty($type[1])) { - if (CRM_Utils_Array::value(0, $type) == 'phone') { - $hdr .= "-" . CRM_Utils_Array::value($type[1], $phoneTypes); - } - elseif (CRM_Utils_Array::value(0, $type) == 'im') { - $hdr .= "-" . CRM_Utils_Array::value($type[1], $imProviders); - } - } - $headerName = $field . '-' . $hdr; - $headerRows[] = $headerName; - self::sqlColumnDefn($query, $sqlColumns, $headerName); - } - } - } - } - self::manipulateHeaderRows($headerRows, $contactRelationshipTypes); - } elseif ($selectedPaymentFields && array_key_exists($field, self::componentPaymentFields())) { $headerRows[] = CRM_Utils_Array::value($field, self::componentPaymentFields()); } else { $headerRows[] = $field; } - self::sqlColumnDefn($query, $sqlColumns, $field); return array($headerRows, $sqlColumns); diff --git a/tests/phpunit/CRM/Export/BAO/ExportTest.php b/tests/phpunit/CRM/Export/BAO/ExportTest.php index e5cadf81e882..24d5fe59bb77 100644 --- a/tests/phpunit/CRM/Export/BAO/ExportTest.php +++ b/tests/phpunit/CRM/Export/BAO/ExportTest.php @@ -48,7 +48,7 @@ public function testExportComponentsNull() { NULL, NULL, FALSE, - FALSE, + TRUE, array( 'exportOption' => 1, 'suppress_csv_for_testing' => TRUE,