From eeb6d4354028af88fe2f7ce821e0d1ab7c79cc34 Mon Sep 17 00:00:00 2001 From: "deb.monish" Date: Fri, 23 Mar 2018 14:27:37 +0530 Subject: [PATCH] Exporting Contacts using "Export PRIMARY fields" and "Merge Household Members into their Households" produces db error --- CRM/Contact/BAO/Query.php | 4 +- CRM/Export/BAO/Export.php | 223 ++++++++------------ tests/phpunit/CRM/Export/BAO/ExportTest.php | 2 +- 3 files changed, 88 insertions(+), 141 deletions(-) diff --git a/CRM/Contact/BAO/Query.php b/CRM/Contact/BAO/Query.php index 69c5655c5c77..3668ffbd0aef 100644 --- a/CRM/Contact/BAO/Query.php +++ b/CRM/Contact/BAO/Query.php @@ -4708,10 +4708,11 @@ public static function filterCountryFromValuesIfStateExists(&$formValues) { * @param array $selectClauses * @param array $groupBy - Columns already included in GROUP By clause. * @param string $aggregateFunction + * @param bool $appendAlias - Append select column alias as it's name itself * * @return string */ - public static function appendAnyValueToSelect($selectClauses, $groupBy, $aggregateFunction = 'ANY_VALUE') { + public static function appendAnyValueToSelect($selectClauses, $groupBy, $aggregateFunction = 'ANY_VALUE', $appendAlias = FALSE) { if (!CRM_Utils_SQL::disableFullGroupByMode()) { $groupBy = array_map('trim', (array) $groupBy); $aggregateFunctions = '/(ROUND|AVG|COUNT|GROUP_CONCAT|SUM|MAX|MIN|IF)[[:blank:]]*\(/i'; @@ -4722,6 +4723,7 @@ public static function appendAnyValueToSelect($selectClauses, $groupBy, $aggrega $val = ($aggregateFunction == 'GROUP_CONCAT') ? str_replace($selectColumn, "$aggregateFunction(DISTINCT {$selectColumn})", $val) : str_replace($selectColumn, "$aggregateFunction({$selectColumn})", $val); + $val .= $appendAlias ? " as $selectColumn " : ''; } } } diff --git a/CRM/Export/BAO/Export.php b/CRM/Export/BAO/Export.php index 70a1a14e7358..0ee3ecd3586b 100644 --- a/CRM/Export/BAO/Export.php +++ b/CRM/Export/BAO/Export.php @@ -566,6 +566,8 @@ public static function exportComponents( $query->_sort = $order; list($select, $from, $where, $having) = $query->query(); + $allRelContactArray = $relationQuery = $relationType = array(); + if ($mergeSameHousehold == 1) { if (empty($returnProperties['id'])) { $returnProperties['id'] = 1; @@ -574,6 +576,10 @@ public static function exportComponents( //also merge Head of Household $relationKeyMOH = CRM_Utils_Array::key('Household Member of', $contactRelationshipTypes); $relationKeyHOH = CRM_Utils_Array::key('Head of Household for', $contactRelationshipTypes); + $relationType = [ + $relationKeyMOH => NULL, + $relationKeyHOH => NULL, + ]; foreach ($returnProperties as $key => $value) { if (!array_key_exists($key, $contactRelationshipTypes)) { @@ -588,8 +594,6 @@ public static function exportComponents( unset($returnProperties[$relationKeyHOH]['im_provider']); } - $allRelContactArray = $relationQuery = array(); - foreach ($contactRelationshipTypes as $rel => $dnt) { if ($relationReturnProperties = CRM_Utils_Array::value($rel, $returnProperties)) { $allRelContactArray[$rel] = array(); @@ -783,7 +787,6 @@ public static function exportComponents( //first loop through output columns so that we return what is required, and in same order. foreach ($outputColumns as $field => $value) { - // add im_provider to $dao object if ($field == 'im_provider' && property_exists($iterationDAO, 'provider_id')) { $iterationDAO->im_provider = $iterationDAO->provider_id; @@ -825,6 +828,9 @@ public static function exportComponents( $row[$field] = $iterationDAO->pledge_next_pay_amount + $iterationDAO->pledge_outstanding_amount; } elseif (array_key_exists($field, $contactRelationshipTypes)) { + if (!array_key_exists($field, $relationType)) { + $relationType[$field] = NULL; + } $relDAO = CRM_Utils_Array::value($iterationDAO->contact_id, $allRelContactArray[$field]); $relationQuery[$field]->convertToPseudoNames($relDAO); self::fetchRelationshipDetails($relDAO, $value, $field, $row); @@ -921,6 +927,9 @@ public static function exportComponents( if ($setHeader) { $exportTempTable = self::createTempTable($sqlColumns); + foreach ($relationType as $type => &$dontCare) { + $relationType[$type] = self::createTempTable($sqlColumns); + } } //build header only once @@ -956,7 +965,7 @@ public static function exportComponents( // output every $tempRowCount rows if ($count % $tempRowCount == 0) { - self::writeDetailsToTable($exportTempTable, $componentDetails, $sqlColumns); + self::writeDetailsToTable($exportTempTable, $componentDetails, $sqlColumns, $relationType, $returnProperties); $componentDetails = array(); } } @@ -967,7 +976,7 @@ public static function exportComponents( } if ($exportTempTable) { - self::writeDetailsToTable($exportTempTable, $componentDetails, $sqlColumns); + self::writeDetailsToTable($exportTempTable, $componentDetails, $sqlColumns, $relationType, $returnProperties); // if postalMailing option is checked, exclude contacts who are deceased, have // "Do not mail" privacy setting, or have no street address @@ -984,8 +993,7 @@ public static function exportComponents( // merge the records if they have corresponding households if ($mergeSameHousehold) { - self::mergeSameHousehold($exportTempTable, $headerRows, $sqlColumns, $relationKeyMOH); - self::mergeSameHousehold($exportTempTable, $headerRows, $sqlColumns, $relationKeyHOH); + self::mergeSameHousehold($exportTempTable, $sqlColumns, $relationType); } // call export hook @@ -1267,10 +1275,12 @@ public static function sqlColumnDefn($query, &$sqlColumns, $field) { /** * @param string $tableName - * @param $details - * @param $sqlColumns + * @param array $details + * @param array $sqlColumns + * @param array $relationTypes + * @param array $returnProperties */ - public static function writeDetailsToTable($tableName, &$details, &$sqlColumns) { + public static function writeDetailsToTable($tableName, &$details, &$sqlColumns, $relationTypes, $returnProperties) { if (empty($details)) { return; } @@ -1287,10 +1297,35 @@ public static function writeDetailsToTable($tableName, &$details, &$sqlColumns) $sqlClause = array(); + $sqlColumnString = '(id, ' . implode(',', array_keys($sqlColumns)) . ')'; + foreach ($details as $dontCare => $row) { $id++; $valueString = array($id); - foreach ($row as $dontCare => $value) { + foreach ($row as $key1 => $value) { + if (array_key_exists($key1, $relationTypes)) { + // consider only those entries to pass, which have linked relationship contact + if (!empty($row[$key1]) && !empty($row[$key1]['id'])) { + $values = [$row['id']]; + foreach ($row[$key1] as $key2 => $value) { + if ($key2 == 'id') { + continue; + } + else { + $values[] = is_null($value) ? '' : "'" . CRM_Core_DAO::escapeString($value) . "'"; + } + } + $values[] = $row[$key1]['id']; + $sql = sprintf(" INSERT INTO %s (%s, civicrm_primary_id) VALUES ( %s ) ", + $relationTypes[$key1], + implode(',', array_keys($returnProperties[$key1])), + implode(",\n", $values) + ); + CRM_Core_DAO::executeQuery($sql); + } + unset($row[$key1]); + continue; + } if (empty($value)) { $valueString[] = "''"; } @@ -1301,8 +1336,6 @@ public static function writeDetailsToTable($tableName, &$details, &$sqlColumns) $sqlClause[] = '(' . implode(',', $valueString) . ')'; } - $sqlColumnString = '(id, ' . implode(',', array_keys($sqlColumns)) . ')'; - $sqlValueString = implode(",\n", $sqlClause); $sql = " @@ -1646,83 +1679,56 @@ public static function _buildMasterCopyArray($sql, $exportParams, $sharedAddress * * @param string $exportTempTable * Temporary temp table that stores the records. - * @param array $headerRows - * Array of headers for the export file. * @param array $sqlColumns * Array of names of the table columns of the temp table. - * @param string $prefix - * Name of the relationship type that is prefixed to the table columns. + * @param array $relationTempTables + * Name of the temp tables that holds the relationship records */ - public static function mergeSameHousehold($exportTempTable, &$headerRows, &$sqlColumns, $prefix) { - $prefixColumn = $prefix . '_'; + public static function mergeSameHousehold($exportTempTable, &$sqlColumns, $relationTempTables) { $allKeys = array_keys($sqlColumns); $replaced = array(); - $headerRows = array_values($headerRows); - - // name map of the non standard fields in header rows & sql columns - $mappingFields = array( - 'civicrm_primary_id' => 'id', - 'contact_source' => 'source', - 'current_employer_id' => 'employer_id', - 'contact_is_deleted' => 'is_deleted', - 'name' => 'address_name', - 'provider_id' => 'im_service_provider', - 'phone_type_id' => 'phone_type', - ); - //figure out which columns are to be replaced by which ones - foreach ($sqlColumns as $columnNames => $dontCare) { - if ($rep = CRM_Utils_Array::value($columnNames, $mappingFields)) { - $replaced[$columnNames] = CRM_Utils_String::munge($prefixColumn . $rep, '_', 64); - } - else { - $householdColName = CRM_Utils_String::munge($prefixColumn . $columnNames, '_', 64); - - if (!empty($sqlColumns[$householdColName])) { - $replaced[$columnNames] = $householdColName; + foreach ($relationTempTables as $relationType => $tempTable) { + foreach ($sqlColumns as $columnNames => $dontCare) { + if (!empty($sqlColumns[$columnNames])) { + $replaced["temp." . $columnNames] = "$relationType.$columnNames"; } } - } - $query = "UPDATE $exportTempTable SET "; - $clause = array(); - foreach ($replaced as $from => $to) { - $clause[] = "$from = $to "; - unset($sqlColumns[$to]); - if ($key = CRM_Utils_Array::key($to, $allKeys)) { - unset($headerRows[$key]); + $query = "UPDATE $exportTempTable temp + INNER JOIN $tempTable $relationType ON $relationType.id = temp.civicrm_primary_id + SET "; + $clause = array(); + foreach ($replaced as $from => $to) { + $clause[] = "$from = $to "; } - } - $query .= implode(",\n", $clause); - $query .= " WHERE {$replaced['civicrm_primary_id']} != ''"; + $query .= implode(",\n", $clause); + $query .= " WHERE temp.civicrm_primary_id != '' "; - CRM_Core_DAO::executeQuery($query); + CRM_Core_DAO::executeQuery($query); - //drop the table columns that store redundant household info - $dropQuery = "ALTER TABLE $exportTempTable "; - foreach ($replaced as $householdColumns) { - $dropClause[] = " DROP $householdColumns "; + $sql = "DROP TABLE IF EXISTS $tempTable"; + CRM_Core_DAO::executeQuery($sql); } - $dropQuery .= implode(",\n", $dropClause); - - CRM_Core_DAO::executeQuery($dropQuery); // also drop the temp table if exists $sql = "DROP TABLE IF EXISTS {$exportTempTable}_temp"; CRM_Core_DAO::executeQuery($sql); // clean up duplicate records + $select = CRM_Contact_BAO_Query::appendAnyValueToSelect($allKeys, "civicrm_primary_id", 'GROUP_CONCAT', TRUE); $query = " -CREATE TABLE {$exportTempTable}_temp SELECT * +CREATE TABLE {$exportTempTable}_temp +$select FROM {$exportTempTable} GROUP BY civicrm_primary_id "; - CRM_Core_DAO::executeQuery($query); $query = "DROP TABLE $exportTempTable"; CRM_Core_DAO::executeQuery($query); $query = "ALTER TABLE {$exportTempTable}_temp RENAME TO {$exportTempTable}"; + CRM_Core_DAO::executeQuery($query); } @@ -1914,73 +1920,12 @@ public static function setHeaderRows($field, $headerRows, $sqlColumns, $query, $ $headerRows[] = $query->_fields['activity'][$field]['title']; } } - elseif (array_key_exists($field, $contactRelationshipTypes)) { - foreach ($value as $relationField => $relationValue) { - // below block is same as primary block (duplicate) - if (isset($relationQuery[$field]->_fields[$relationField]['title'])) { - if ($relationQuery[$field]->_fields[$relationField]['name'] == 'name') { - $headerName = $field . '-' . $relationField; - } - else { - if ($relationField == 'current_employer') { - $headerName = $field . '-' . 'current_employer'; - } - else { - $headerName = $field . '-' . $relationQuery[$field]->_fields[$relationField]['name']; - } - } - - $headerRows[] = $headerName; - - self::sqlColumnDefn($query, $sqlColumns, $headerName); - } - elseif ($relationField == 'phone_type_id') { - $headerName = $field . '-' . 'Phone Type'; - $headerRows[] = $headerName; - self::sqlColumnDefn($query, $sqlColumns, $headerName); - } - elseif ($relationField == 'provider_id') { - $headerName = $field . '-' . 'Im Service Provider'; - $headerRows[] = $headerName; - self::sqlColumnDefn($query, $sqlColumns, $headerName); - } - elseif ($relationField == 'state_province_id') { - $headerName = $field . '-' . 'state_province_id'; - $headerRows[] = $headerName; - self::sqlColumnDefn($query, $sqlColumns, $headerName); - } - elseif (is_array($relationValue) && $relationField == 'location') { - // fix header for location type case - foreach ($relationValue as $ltype => $val) { - foreach (array_keys($val) as $fld) { - $type = explode('-', $fld); - - $hdr = "{$ltype}-" . $relationQuery[$field]->_fields[$type[0]]['title']; - - if (!empty($type[1])) { - if (CRM_Utils_Array::value(0, $type) == 'phone') { - $hdr .= "-" . CRM_Utils_Array::value($type[1], $phoneTypes); - } - elseif (CRM_Utils_Array::value(0, $type) == 'im') { - $hdr .= "-" . CRM_Utils_Array::value($type[1], $imProviders); - } - } - $headerName = $field . '-' . $hdr; - $headerRows[] = $headerName; - self::sqlColumnDefn($query, $sqlColumns, $headerName); - } - } - } - } - self::manipulateHeaderRows($headerRows, $contactRelationshipTypes); - } elseif ($selectedPaymentFields && array_key_exists($field, self::componentPaymentFields())) { $headerRows[] = CRM_Utils_Array::value($field, self::componentPaymentFields()); } else { $headerRows[] = $field; } - self::sqlColumnDefn($query, $sqlColumns, $field); return array($headerRows, $sqlColumns); @@ -2065,17 +2010,18 @@ public static function getExportStructureArrays($returnProperties, $query, $phon } /** - * Get the values of linked household contact. + * Get the values of linked household contact * - * @param CRM_Core_DAO $relDAO + * @param obj $relDAO * @param array $value - * @param string $field + * @param string $relPrefix * @param array $row */ - private static function fetchRelationshipDetails($relDAO, $value, $field, &$row) { + public static function fetchRelationshipDetails($relDAO, $value, $relPrefix, &$row) { + $row[$relPrefix] = []; + $i18n = CRM_Core_I18n::singleton(); $phoneTypes = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Phone', 'phone_type_id'); $imProviders = CRM_Core_PseudoConstant::get('CRM_Core_DAO_IM', 'provider_id'); - $i18n = CRM_Core_I18n::singleton(); foreach ($value as $relationField => $relationValue) { if (is_object($relDAO) && property_exists($relDAO, $relationField)) { $fieldValue = $relDAO->$relationField; @@ -2106,10 +2052,9 @@ private static function fetchRelationshipDetails($relDAO, $value, $field, &$row) else { $fieldValue = ''; } - $field = $field . '_'; if (is_object($relDAO) && $relationField == 'id') { - $row[$field . $relationField] = $relDAO->contact_id; + $row[$relPrefix][$relationField] = $relDAO->contact_id; } elseif (is_array($relationValue) && $relationField == 'location') { foreach ($relationValue as $ltype => $val) { @@ -2123,24 +2068,24 @@ private static function fetchRelationshipDetails($relDAO, $value, $field, &$row) // and state_province (‘province’ context) switch (TRUE) { case (!is_object($relDAO)): - $row[$field . '_' . $fldValue] = ''; + $row[$relPrefix . '_' . $fldValue] = ''; break; case in_array('country', $type): case in_array('world_region', $type): - $row[$field . '_' . $fldValue] = $i18n->crm_translate($relDAO->$fldValue, + $row[$relPrefix][$fldValue] = $i18n->crm_translate($relDAO->$fldValue, array('context' => 'country') ); break; case in_array('state_province', $type): - $row[$field . '_' . $fldValue] = $i18n->crm_translate($relDAO->$fldValue, + $row[$relPrefix][$fldValue] = $i18n->crm_translate($relDAO->$fldValue, array('context' => 'province') ); break; default: - $row[$field . '_' . $fldValue] = $relDAO->$fldValue; + $row[$relPrefix][$fldValue] = $relDAO->$fldValue; break; } } @@ -2149,7 +2094,7 @@ private static function fetchRelationshipDetails($relDAO, $value, $field, &$row) elseif (isset($fieldValue) && $fieldValue != '') { //check for custom data if ($cfID = CRM_Core_BAO_CustomField::getKeyID($relationField)) { - $row[$field . $relationField] = CRM_Core_BAO_CustomField::displayValue($fieldValue, $cfID); + $row[$relPrefix][$relationField] = CRM_Core_BAO_CustomField::displayValue($fieldValue, $cfID); } else { //normal relationship fields @@ -2157,22 +2102,22 @@ private static function fetchRelationshipDetails($relDAO, $value, $field, &$row) switch ($relationField) { case 'country': case 'world_region': - $row[$field . $relationField] = $i18n->crm_translate($fieldValue, array('context' => 'country')); + $row[$relPrefix][$relationField] = $i18n->crm_translate($fieldValue, array('context' => 'country')); break; case 'state_province': - $row[$field . $relationField] = $i18n->crm_translate($fieldValue, array('context' => 'province')); + $row[$relPrefix][$relationField] = $i18n->crm_translate($fieldValue, array('context' => 'province')); break; default: - $row[$field . $relationField] = $fieldValue; + $row[$relPrefix][$relationField] = $fieldValue; break; } } } else { // if relation field is empty or null - $row[$field . $relationField] = ''; + $row[$relPrefix][$relationField] = ''; } } } diff --git a/tests/phpunit/CRM/Export/BAO/ExportTest.php b/tests/phpunit/CRM/Export/BAO/ExportTest.php index e5cadf81e882..24d5fe59bb77 100644 --- a/tests/phpunit/CRM/Export/BAO/ExportTest.php +++ b/tests/phpunit/CRM/Export/BAO/ExportTest.php @@ -48,7 +48,7 @@ public function testExportComponentsNull() { NULL, NULL, FALSE, - FALSE, + TRUE, array( 'exportOption' => 1, 'suppress_csv_for_testing' => TRUE,