From 21ca2cb6a709353da455601231e247a827de4f03 Mon Sep 17 00:00:00 2001 From: eileen Date: Mon, 20 Mar 2017 17:34:00 +1300 Subject: [PATCH] CRM-19385 - remove dependence on ids in cache tables. This removes the use of the primary id keys in the cache tables. The upgrade part of this has gone stale a few times but I feel sure that this part is tested & confirmed and committing this part will make testing easier --- CRM/Contact/BAO/Contact/Permission.php | 2 +- CRM/Contact/BAO/Group.php | 2 +- CRM/Contact/BAO/Query.php | 6 +- CRM/Contact/Form/Search/Custom/FullText.php | 10 ++- CRM/Core/BAO/Cache.php | 7 +- CRM/Core/BAO/PrevNextCache.php | 3 +- CRM/Core/CodeGen/Specification.php | 8 +- CRM/Core/Reference/Basic.php | 9 ++- CRM/Group/Page/Group.php | 2 +- CRM/Upgrade/Incremental/Base.php | 16 ++++ .../Form/Search/Custom/FullTextTest.php | 74 +++++++++++++++++++ 11 files changed, 117 insertions(+), 22 deletions(-) create mode 100644 tests/phpunit/CRM/Contact/Form/Search/Custom/FullTextTest.php diff --git a/CRM/Contact/BAO/Contact/Permission.php b/CRM/Contact/BAO/Contact/Permission.php index a37e40e7f8bc..73c379e0dfff 100644 --- a/CRM/Contact/BAO/Contact/Permission.php +++ b/CRM/Contact/BAO/Contact/Permission.php @@ -221,7 +221,7 @@ public static function cache($userID, $type = CRM_Core_Permission::VIEW, $force // run a query to see if the cache is filled $sql = " -SELECT count(id) +SELECT count(*) FROM civicrm_acl_contact_cache WHERE user_id = %1 AND $operationClause diff --git a/CRM/Contact/BAO/Group.php b/CRM/Contact/BAO/Group.php index aab9bc9cd9f7..c58b9ade639f 100644 --- a/CRM/Contact/BAO/Group.php +++ b/CRM/Contact/BAO/Group.php @@ -1019,7 +1019,7 @@ public static function getGroupList(&$params) { } // Exclude deleted contacts $where .= " and c.id = g.contact_id AND c.is_deleted = 0"; - $dao = CRM_Core_DAO::executeQuery("SELECT g.group_id, COUNT(g.id) as `count` FROM $table g, civicrm_contact c WHERE $where GROUP BY g.group_id"); + $dao = CRM_Core_DAO::executeQuery("SELECT g.group_id, COUNT(*) as `count` FROM $table g, civicrm_contact c WHERE $where GROUP BY g.group_id"); while ($dao->fetch()) { $values[$dao->group_id]['count'] = $dao->count; } diff --git a/CRM/Contact/BAO/Query.php b/CRM/Contact/BAO/Query.php index 764f57325bc1..2dac2720acf3 100644 --- a/CRM/Contact/BAO/Query.php +++ b/CRM/Contact/BAO/Query.php @@ -4788,11 +4788,9 @@ public function getCachedContacts($cacheKey, $offset, $rowCount, $includeContact $onlyDeleted = in_array(array('deleted_contacts', '=', '1', '0', '0'), $this->_params); list($select, $from, $where) = $this->query(FALSE, FALSE, FALSE, $onlyDeleted); $from = " FROM civicrm_prevnext_cache pnc INNER JOIN civicrm_contact contact_a ON contact_a.id = pnc.entity_id1 AND pnc.cacheKey = '$cacheKey' " . substr($from, 31); - $order = " ORDER BY pnc.id"; - $groupByCol = array('contact_a.id', 'pnc.id'); - $groupBy = self::getGroupByFromSelectColumns($this->_select, $groupByCol); + $groupBy = self::getGroupByFromSelectColumns($this->_select, array('contact_a.id')); $limit = " LIMIT $offset, $rowCount"; - $query = "$select $from $where $groupBy $order $limit"; + $query = "$select $from $where $groupBy $limit"; return CRM_Core_DAO::executeQuery($query); } diff --git a/CRM/Contact/Form/Search/Custom/FullText.php b/CRM/Contact/Form/Search/Custom/FullText.php index 8fa70eead1bc..f52c1bde2734 100644 --- a/CRM/Contact/Form/Search/Custom/FullText.php +++ b/CRM/Contact/Form/Search/Custom/FullText.php @@ -224,6 +224,10 @@ public function buildTempTable() { ) ENGINE=HEAP DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci "; CRM_Core_DAO::executeQuery($sql); + + if (!empty($this->_formValues['is_unit_test'])) { + $this->_tableNameForTest = $this->_tableName; + } } public function fillTable() { @@ -259,7 +263,7 @@ public function filterACLContacts() { $sql = " DELETE t.* FROM {$this->_tableName} t -WHERE NOT EXISTS ( SELECT c.id +WHERE NOT EXISTS ( SELECT c.contact_id FROM civicrm_acl_contact_cache c WHERE c.user_id = %1 AND t.contact_id = c.contact_id ) "; @@ -269,7 +273,7 @@ public function filterACLContacts() { DELETE t.* FROM {$this->_tableName} t WHERE t.table_name = 'Activity' AND - NOT EXISTS ( SELECT c.id + NOT EXISTS ( SELECT c.contact_id FROM civicrm_acl_contact_cache c WHERE c.user_id = %1 AND ( t.target_contact_id = c.contact_id OR t.target_contact_id IS NULL ) ) "; @@ -279,7 +283,7 @@ public function filterACLContacts() { DELETE t.* FROM {$this->_tableName} t WHERE t.table_name = 'Activity' AND - NOT EXISTS ( SELECT c.id + NOT EXISTS ( SELECT c.contact_id FROM civicrm_acl_contact_cache c WHERE c.user_id = %1 AND ( t.assignee_contact_id = c.contact_id OR t.assignee_contact_id IS NULL ) ) "; diff --git a/CRM/Core/BAO/Cache.php b/CRM/Core/BAO/Cache.php index 1f295535a6f2..31c4355b6bea 100644 --- a/CRM/Core/BAO/Cache.php +++ b/CRM/Core/BAO/Cache.php @@ -146,18 +146,17 @@ public static function setItem(&$data, $group, $path, $componentID = NULL) { $table = self::getTableName(); $where = self::whereCache($group, $path, $componentID); - $id = CRM_Core_DAO::singleValueQuery("SELECT id FROM $table WHERE $where"); + $dataExists = CRM_Core_DAO::singleValueQuery("SELECT COUNT(*) FROM $table WHERE {$where}"); $now = date('Y-m-d H:i:s'); // FIXME - Use SQL NOW() or CRM_Utils_Time? $dataSerialized = serialize($data); // This table has a wonky index, so we cannot use REPLACE or // "INSERT ... ON DUPE". Instead, use SELECT+(INSERT|UPDATE). - if ($id) { - $sql = "UPDATE $table SET data = %1, created_date = %2 WHERE id = %3"; + if ($dataExists) { + $sql = "UPDATE $table SET data = %1, created_date = %2 WHERE {$where}"; $args = array( 1 => array($dataSerialized, 'String'), 2 => array($now, 'String'), - 3 => array($id, 'Int'), ); $dao = CRM_Core_DAO::executeQuery($sql, $args, TRUE, NULL, FALSE, FALSE); } diff --git a/CRM/Core/BAO/PrevNextCache.php b/CRM/Core/BAO/PrevNextCache.php index 17f4251e2924..271673d03073 100644 --- a/CRM/Core/BAO/PrevNextCache.php +++ b/CRM/Core/BAO/PrevNextCache.php @@ -506,7 +506,6 @@ public static function getSelection($cacheKey, $action = 'get', $entity_table = WHERE cacheKey LIKE %1 $actionGet $entity_whereClause -ORDER BY id "; $params[1] = array("{$cacheKey}%", 'String'); @@ -566,7 +565,7 @@ public static function buildSelectedContactPager(&$form, &$params) { $cacheKey = "civicrm search {$qfKey}"; $query = " -SELECT count(id) +SELECT count(*) FROM civicrm_prevnext_cache WHERE cacheKey LIKE %1 AND is_selected = 1 diff --git a/CRM/Core/CodeGen/Specification.php b/CRM/Core/CodeGen/Specification.php index 2a94606859b0..be5b19b48a58 100644 --- a/CRM/Core/CodeGen/Specification.php +++ b/CRM/Core/CodeGen/Specification.php @@ -463,16 +463,16 @@ public function composeTitle($name) { } /** - * @param $primaryXML - * @param $fields - * @param $table + * @param object $primaryXML + * @param array $fields + * @param array $table */ public function getPrimaryKey(&$primaryXML, &$fields, &$table) { $name = trim((string ) $primaryXML->name); /** need to make sure there is a field of type name */ if (!array_key_exists($name, $fields)) { - echo "primary key $name in $table->name does not have a field definition, ignoring\n"; + echo "primary key $name in {$table['name']} does not have a field definition, ignoring\n"; return; } diff --git a/CRM/Core/Reference/Basic.php b/CRM/Core/Reference/Basic.php index 411d30627bdf..797d5832ab9d 100644 --- a/CRM/Core/Reference/Basic.php +++ b/CRM/Core/Reference/Basic.php @@ -78,11 +78,16 @@ public function matchesTargetTable($tableName) { */ public function findReferences($targetDao) { $targetColumn = $this->getTargetKey(); + $select = 'id'; + // CRM-19385: Since id is removed, return all rows for cache tables. + if (!CRM_Core_BAO_SchemaHandler::checkIfFieldExists($this->getReferenceTable(), 'id')) { + $select = '*'; + } $params = array( 1 => array($targetDao->$targetColumn, 'String'), ); $sql = <<getReferenceTable()} WHERE {$this->getReferenceKey()} = %1 EOS; @@ -103,7 +108,7 @@ public function getReferenceCount($targetDao) { 1 => array($targetDao->$targetColumn, 'String'), ); $sql = <<getReferenceTable()} WHERE {$this->getReferenceKey()} = %1 EOS; diff --git a/CRM/Group/Page/Group.php b/CRM/Group/Page/Group.php index d78bec5cfcee..c371fe56bc76 100644 --- a/CRM/Group/Page/Group.php +++ b/CRM/Group/Page/Group.php @@ -128,7 +128,7 @@ public function browse($action = NULL) { if (!empty($_GET['update_smart_groups'])) { CRM_Contact_BAO_GroupContactCache::loadAll(); } - elseif (!CRM_Core_DAO::singleValueQuery("SELECT id FROM civicrm_group_contact_cache LIMIT 1")) { + elseif (!CRM_Core_DAO::singleValueQuery("SELECT contact_id FROM civicrm_group_contact_cache LIMIT 1")) { CRM_Core_Session::setStatus(ts('Count data for smart groups is not currently calculated. You may click Update Smart Groups to generate it. Be aware this can cause significant server load')); } diff --git a/CRM/Upgrade/Incremental/Base.php b/CRM/Upgrade/Incremental/Base.php index fcf11235f8e7..a0a49c7ffdd9 100644 --- a/CRM/Upgrade/Incremental/Base.php +++ b/CRM/Upgrade/Incremental/Base.php @@ -149,4 +149,20 @@ public static function addColumn($ctx, $table, $column, $properties) { return TRUE; } + /** + * Drop a column from a table if it exist. + * + * @param CRM_Queue_TaskContext $ctx + * @param string $table + * @param string $column + * @return bool + */ + public static function dropColumn($ctx, $table, $column) { + if (CRM_Core_BAO_SchemaHandler::checkIfFieldExists($table, $column)) { + CRM_Core_DAO::executeQuery("ALTER TABLE `$table` DROP COLUMN `$column`", + array(), TRUE, NULL, FALSE, FALSE); + } + return TRUE; + } + } diff --git a/tests/phpunit/CRM/Contact/Form/Search/Custom/FullTextTest.php b/tests/phpunit/CRM/Contact/Form/Search/Custom/FullTextTest.php new file mode 100644 index 000000000000..a935c8e2dcb7 --- /dev/null +++ b/tests/phpunit/CRM/Contact/Form/Search/Custom/FullTextTest.php @@ -0,0 +1,74 @@ +quickCleanup($this->_tablesToTruncate); + + $userId = $this->createLoggedInUser(); + // remove all permissions + $config = CRM_Core_Config::singleton(); + $config->userPermissionClass->permissions = array(); + + for ($i = 1; $i <= 10; $i++) { + $contactId = $this->individualCreate(array(), $i); + if ($i <= 5) { + $queryParams = array( + 1 => array($userId, 'Integer'), + 2 => array($contactId, 'Integer'), + ); + CRM_Core_DAO::executeQuery("INSERT INTO civicrm_acl_contact_cache ( user_id, contact_id, operation ) VALUES(%1, %2, 'View')", $queryParams); + } + $contactIDs[$i] = $contactId; + } + + $formValues = array('component_mode' => 1, 'operator' => 1, 'is_unit_test' => 1); + $fullText = new CRM_Contact_Form_Search_Custom_FullText($formValues); + $fullText->initialize(); + + //Assert that ACL contacts are filtered. + $queryParams = array(1 => array($userId, 'Integer')); + $whereClause = "WHERE NOT EXISTS (SELECT c.contact_id + FROM civicrm_acl_contact_cache c + WHERE c.user_id = %1 AND t.contact_id = c.contact_id )"; + + $count = CRM_Core_DAO::singleValueQuery("SELECT COUNT(*) FROM {$fullText->_tableNameForTest} t {$whereClause}", $queryParams); + $this->assertEmpty($count, 'ACL contacts are not removed.'); + } + +}