(WIP) (dev/core#217) Allow replacement of PrevNextCache implementation for search screens

CRM/Campaign/Form/Task.php

@@ -65,7 +65,7 @@ public function preProcess() {
     else {
       $qfKey = CRM_Utils_Request::retrieve('qfKey', 'String', $this);
       $cacheKey = "civicrm search {$qfKey}";
-      $allCids = CRM_Core_BAO_PrevNextCache::getSelection($cacheKey, "getall");
+      $allCids = Civi::service('prevnext')->getSelection($cacheKey, "getall");
       $ids = array_keys($allCids[$cacheKey]);
       $this->assign('totalSelectedVoters', count($ids));
     }

CRM/Campaign/Selector/Search.php

@@ -271,7 +271,7 @@ public function buildPrevNextCache($sort) {
 
     if (!$crmPID) {
       $cacheKey = "civicrm search {$this->_key}";
-      CRM_Core_BAO_PrevNextCache::deleteItem(NULL, $cacheKey, 'civicrm_contact');
+      Civi::service('prevnext')->deleteItem(NULL, $cacheKey, 'civicrm_contact');
 
       $sql = $this->_query->searchQuery(0, 0, $sort,
         FALSE, FALSE,
@@ -281,18 +281,19 @@ public function buildPrevNextCache($sort) {
         $this->_campaignFromClause
       );
       list($select, $from) = explode(' FROM ', $sql);
-      $insertSQL = "
-INSERT INTO civicrm_prevnext_cache ( entity_table, entity_id1, entity_id2, cacheKey, data )
-SELECT 'civicrm_contact', contact_a.id, contact_a.id, '$cacheKey', contact_a.display_name
+      $selectSQL = "
+      SELECT '$cacheKey', contact_a.id, contact_a.display_name
 FROM {$from}
 ";
-      $errorScope = CRM_Core_TemporaryErrorScope::ignoreException();
-      $result = CRM_Core_DAO::executeQuery($insertSQL);
-      unset($errorScope);
 
-      if (is_a($result, 'DB_Error')) {
+      try {
+        Civi::service('prevnext')->fillWithSql($cacheKey, $selectSQL);
+      }
+      catch (CRM_Core_Exception $e) {
+        // Heavy handed, no? Seems like this merits an explanation.
         return;
       }
+
       // also record an entry in the cache key table, so we can delete it periodically
       CRM_Core_BAO_Cache::setItem($cacheKey, 'CiviCRM Search PrevNextCache', $cacheKey);
     }

CRM/Contact/BAO/Contact.php

@@ -1013,7 +1013,10 @@ public static function deleteContact($id, $restore = FALSE, $skipUndelete = FALS
     CRM_Utils_Recent::delContact($id);
     self::updateContactCache($id, empty($restore));
 
-    // delete any dupe cache entry
+    // delete any prevnext/dupe cache entry
+    // These two calls are redundant in default deployments, but they're
+    // meaningful if "prevnext" is memory-backed.
+    Civi::service('prevnext')->deleteItem($id);
     CRM_Core_BAO_PrevNextCache::deleteItem($id);
 
     $transaction->commit();

CRM/Contact/BAO/Contact/Utils.php

@@ -917,6 +917,9 @@ public static function clearContactCaches($isEmptyPrevNextTable = FALSE) {
       return;
     }
     if ($isEmptyPrevNextTable) {
+      // These two calls are redundant in default deployments, but they're
+      // meaningful if "prevnext" is memory-backed.
+      Civi::service('prevnext')->deleteItem();
       CRM_Core_BAO_PrevNextCache::deleteItem();
     }
     // clear acl cache if any.

CRM/Contact/BAO/Query.php

@@ -4965,29 +4965,47 @@ public function searchQuery(
   }
 
   /**
-   * Fetch a list of contacts from the prev/next cache for displaying a search results page
+   * Fetch a list of contacts for displaying a search results page
    *
-   * @param string $cacheKey
-   * @param int $offset
-   * @param int $rowCount
+   * @param array $cids
+   *   List of contact IDs
    * @param bool $includeContactIds
    * @return CRM_Core_DAO
    */
-  public function getCachedContacts($cacheKey, $offset, $rowCount, $includeContactIds) {
+  public function getCachedContacts($cids, $includeContactIds) {
+    CRM_Utils_Type::validateAll($cids, 'Positive');
     $this->_includeContactIds = $includeContactIds;
     $onlyDeleted = in_array(array('deleted_contacts', '=', '1', '0', '0'), $this->_params);
     list($select, $from, $where) = $this->query(FALSE, FALSE, FALSE, $onlyDeleted);
-    $from = " FROM civicrm_prevnext_cache pnc INNER JOIN civicrm_contact contact_a ON contact_a.id = pnc.entity_id1 AND pnc.cacheKey = '$cacheKey' " . substr($from, 31);
-    $order = " ORDER BY pnc.id";
-    $groupByCol = array('contact_a.id', 'pnc.id');
-    $select = self::appendAnyValueToSelect($this->_select, $groupByCol, 'GROUP_CONCAT');
-    $groupBy = " GROUP BY " . implode(', ', $groupByCol);
-    $limit = " LIMIT $offset, $rowCount";
+    $select .= sprintf(", (%s) AS _wgt", $this->createSqlCase('contact_a.id', $cids));
+    $where .= sprintf(' AND contact_a.id IN (%s)', implode(',', $cids));
+    $order = 'ORDER BY _wgt';
+    $groupBy = '';
+    $limit = '';
     $query = "$select $from $where $groupBy $order $limit";
 
     return CRM_Core_DAO::executeQuery($query);
   }
 
+  /**
+   * Construct a SQL CASE expression.
+   *
+   * @param string $idCol
+   *   The name of a column with ID's (eg 'contact_a.id').
+   * @param array $cids
+   *   Array(int $weight => int $id).
+   * @return string
+   *   CASE WHEN id=123 THEN 1 WHEN id=456 THEN 2 END
+   */
+  private function createSqlCase($idCol, $cids) {
+    $buf = "CASE\n";
+    foreach ($cids as $weight => $cid) {
+      $buf .= " WHEN $idCol = $cid THEN $weight \n";
+    }
+    $buf .= "END\n";
+    return $buf;
+  }
+
   /**
    * Populate $this->_permissionWhereClause with permission related clause and update other
    * query related properties.

CRM/Contact/Form/Search.php

@@ -497,7 +497,7 @@ public function buildQuickForm() {
     if ($qfKeyParam && ($this->get('component_mode') <= CRM_Contact_BAO_Query::MODE_CONTACTS || $this->get('component_mode') == CRM_Contact_BAO_Query::MODE_CONTACTSRELATED)) {
       $this->addClass('crm-ajax-selection-form');
       $qfKeyParam = "civicrm search {$qfKeyParam}";
-      $selectedContactIdsArr = CRM_Core_BAO_PrevNextCache::getSelection($qfKeyParam);
+      $selectedContactIdsArr = Civi::service('prevnext')->getSelection($qfKeyParam);
       $selectedContactIds = array_keys($selectedContactIdsArr[$qfKeyParam]);
     }
 
@@ -782,7 +782,7 @@ public function postProcess() {
     ) {
       //reset the cache table for new search
       $cacheKey = "civicrm search {$this->controller->_key}";
-      CRM_Core_BAO_PrevNextCache::deleteItem(NULL, $cacheKey);
+      Civi::service('prevnext')->deleteItem(NULL, $cacheKey);
     }
 
     //get the button name

CRM/Contact/Form/Task.php

@@ -165,14 +165,11 @@ public static function preProcessCommon(&$form) {
     if ((CRM_Utils_Array::value('radio_ts', self::$_searchFormValues) == 'ts_all') ||
       ($form->_task == CRM_Contact_Task::SAVE_SEARCH)
     ) {
-      $sortByCharacter = $form->get('sortByCharacter');
-      $cacheKey = ($sortByCharacter && $sortByCharacter != 'all') ? "{$cacheKey}_alphabet" : $cacheKey;
-
       // since we don't store all contacts in prevnextcache, when user selects "all" use query to retrieve contacts
       // rather than prevnext cache table for most of the task actions except export where we rebuild query to fetch
       // final result set
       if ($useTable) {
-        $allCids = CRM_Core_BAO_PrevNextCache::getSelection($cacheKey, "getall");
+        $allCids = Civi::service('prevnext')->getSelection($cacheKey, "getall");
       }
       else {
         $allCids[$cacheKey] = self::getContactIds($form);
@@ -233,7 +230,7 @@ public static function preProcessCommon(&$form) {
       }
       else {
         // fetching selected contact ids of passed cache key
-        $selectedCids = CRM_Core_BAO_PrevNextCache::getSelection($cacheKey);
+        $selectedCids = Civi::service('prevnext')->getSelection($cacheKey);
         foreach ($selectedCids[$cacheKey] as $selectedCid => $ignore) {
           if ($useTable) {
             $insertString[] = " ( {$selectedCid} ) ";
@@ -273,7 +270,7 @@ public static function preProcessCommon(&$form) {
     ) {
       $sel = CRM_Utils_Array::value('radio_ts', self::$_searchFormValues);
       $form->assign('searchtype', $sel);
-      $result = CRM_Core_BAO_PrevNextCache::getSelectedContacts();
+      $result = self::getSelectedContactNames();
       $form->assign("value", $result);
     }
 
@@ -477,6 +474,29 @@ public function mergeContactIdsByHousehold() {
     }
   }
 
+  /**
+   * @return array
+   *   List of contact names.
+   *   NOTE: These are raw values from the DB. In current data-model, that means
+   *   they are pre-encoded HTML.
+   */
+  private static function getSelectedContactNames() {
+    $qfKey = CRM_Utils_Request::retrieve('qfKey', 'String');
+    $cacheKey = "civicrm search {$qfKey}";
+
+    $cids = array();
+    // Gymanstic time!
+    foreach (Civi::service('prevnext')->getSelection($cacheKey) as $cacheKey => $values) {
+      $cids = array_unique(array_merge($cids, array_keys($values)));
+    }
+
+    $result = CRM_Utils_SQL_Select::from('civicrm_contact')
+      ->where('id IN (#cids)', ['cids' => $cids])
+      ->execute()
+      ->fetchMap('id', 'sort_name');
+    return $result;
+  }
+
   /**
    * Given this task's list of targets, produce a hidden group.
    *

CRM/Contact/Page/AJAX.php

@@ -966,19 +966,19 @@ public static function selectUnselectContacts() {
           $elements[$key] = self::_convertToId($element);
         }
         CRM_Utils_Type::escapeAll($elements, 'Integer');
-        CRM_Core_BAO_PrevNextCache::markSelection($cacheKey, $actionToPerform, $elements);
+        Civi::service('prevnext')->markSelection($cacheKey, $actionToPerform, $elements);
       }
       else {
-        CRM_Core_BAO_PrevNextCache::markSelection($cacheKey, $actionToPerform);
+        Civi::service('prevnext')->markSelection($cacheKey, $actionToPerform);
       }
     }
     elseif ($variableType == 'single') {
       $cId = self::_convertToId($name);
       CRM_Utils_Type::escape($cId, 'Integer');
       $action = ($state == 'checked') ? 'select' : 'unselect';
-      CRM_Core_BAO_PrevNextCache::markSelection($cacheKey, $action, $cId);
+      Civi::service('prevnext')->markSelection($cacheKey, $action, $cId);
     }
-    $contactIds = CRM_Core_BAO_PrevNextCache::getSelection($cacheKey);
+    $contactIds = Civi::service('prevnext')->getSelection($cacheKey);
     $countSelectionCids = count($contactIds[$cacheKey]);
 
     $arrRet = array('getCount' => $countSelectionCids);

CRM/Contact/Page/View.php

@@ -117,7 +117,7 @@ public function preProcess() {
       'nextPrevError' => 0,
     );
     if ($qfKey) {
-      $pos = CRM_Core_BAO_PrevNextCache::getPositions("civicrm search $qfKey",
+      $pos = Civi::service('prevnext')->getPositions("civicrm search $qfKey",
         $this->_contactId,
         $this->_contactId
       );

CRM/Contact/Selector.php

@@ -578,8 +578,11 @@ public function &getRows($action, $offset, $rowCount, $sort, $output = NULL) {
     // and contain the search criteria (parameters)
     // note that the default action is basic
     if ($rowCount) {
+      /** @var CRM_Core_PrevNextCache_Interface $prevNext */
+      $prevNext = Civi::service('prevnext');
       $cacheKey = $this->buildPrevNextCache($sort);
-      $resultSet = $this->_query->getCachedContacts($cacheKey, $offset, $rowCount, $includeContactIds)->fetchGenerator();
+      $cids = $prevNext->fetch($cacheKey, $offset, $rowCount);
+      $resultSet = $this->_query->getCachedContacts($cids, $includeContactIds)->fetchGenerator();
     }
     else {
       $resultSet = $this->_query->searchQuery($offset, $rowCount, $sort, FALSE, $includeContactIds)->fetchGenerator();
@@ -881,7 +884,7 @@ public function buildPrevNextCache($sort) {
     // check for current != previous to ensure cache is not reset if paging is done without changing
     // sort criteria
     if (!$pageNum || (!empty($currentSortID) && $currentSortID != $previousSortID)) {
-      CRM_Core_BAO_PrevNextCache::deleteItem(NULL, $cacheKey, 'civicrm_contact');
+      Civi::service('prevnext')->deleteItem(NULL, $cacheKey, 'civicrm_contact');
       // this means it's fresh search, so set pageNum=1
       if (!$pageNum) {
         $pageNum = 1;
@@ -900,10 +903,9 @@ public function buildPrevNextCache($sort) {
     $sortByCharacter = CRM_Utils_Request::retrieve('sortByCharacter', 'String');
 
     //for text field pagination selection save
-    $countRow = CRM_Core_BAO_PrevNextCache::getCount($cacheKey, NULL, "entity_table = 'civicrm_contact'");
+    $countRow = Civi::service('prevnext')->getCount($cacheKey);
     // $sortByCharacter triggers a refresh in the prevNext cache
     if ($sortByCharacter && $sortByCharacter != 'all') {
-      $cacheKey .= "_alphabet";
       $this->fillupPrevNextCache($sort, $cacheKey, 0, max(self::CACHE_SIZE, $pageSize));
     }
     elseif (($firstRecord + $pageSize) >= $countRow) {
@@ -1039,17 +1041,11 @@ public function fillupPrevNextCache($sort, $cacheKey, $start = 0, $end = self::C
     // the other alternative of running the FULL query will just be incredibly inefficient
     // and slow things down way too much on large data sets / complex queries
 
-    $insertSQL = "
-INSERT INTO civicrm_prevnext_cache ( entity_table, entity_id1, entity_id2, cacheKey, data )
-SELECT DISTINCT 'civicrm_contact', contact_a.id, contact_a.id, '$cacheKey', contact_a.sort_name
-";
+    $selectSQL = "SELECT DISTINCT '$cacheKey', contact_a.id, contact_a.sort_name";
 
-    $sql = str_replace(array("SELECT contact_a.id as contact_id", "SELECT contact_a.id as id"), $insertSQL, $sql);
+    $sql = str_replace(array("SELECT contact_a.id as contact_id", "SELECT contact_a.id as id"), $selectSQL, $sql);
     try {
-      $result = CRM_Core_DAO::executeQuery($sql, [], FALSE, NULL, FALSE, TRUE, TRUE);
-      if (is_a($result, 'DB_Error')) {
-        throw new CRM_Core_Exception($result->message);
-      }
+      Civi::service('prevnext')->fillWithSql($cacheKey, $sql);
     }
     catch (CRM_Core_Exception $e) {
       if ($coreSearch) {
@@ -1089,18 +1085,17 @@ public function rebuildPreNextCache($start, $end, $sort, $cacheKey) {
     $dao = CRM_Core_DAO::executeQuery($sql);
 
     // build insert query, note that currently we build cache for 500 (self::CACHE_SIZE) contact records at a time, hence below approach
-    $insertValues = array();
+    $rows = [];
     while ($dao->fetch()) {
-      $insertValues[] = "('civicrm_contact', {$dao->contact_id}, {$dao->contact_id}, '{$cacheKey}', '" . CRM_Core_DAO::escapeString($dao->sort_name) . "')";
+      $rows[] = [
+        'entity_table' => 'civicrm_contact',
+        'entity_id1' => $dao->contact_id,
+        'entity_id2' => $dao->contact_id,
+        'data' => $dao->sort_name,
+      ];
     }
 
-    //update pre/next cache using single insert query
-    if (!empty($insertValues)) {
-      $sql = 'INSERT INTO civicrm_prevnext_cache ( entity_table, entity_id1, entity_id2, cacheKey, data ) VALUES
-' . implode(',', $insertValues);
-
-      $result = CRM_Core_DAO::executeQuery($sql);
-    }
+    Civi::service('prevnext')->fillWithArray($cacheKey, $rows);
   }
 
   /**