Skip to content

Commit

Permalink
(dev/core#217) Implement Redis driver for PrevNext handling
Browse files Browse the repository at this point in the history
  • Loading branch information
@totten
totten committed Jul 20, 2018
1 parent 6137c26 commit 718fa80
Show file tree
Hide file tree
Showing 2 changed files with 285 additions and 0 deletions.
270 changes: 270 additions & 0 deletions CRM/Core/PrevNextCache/Redis.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,270 @@
<?php
/*
+--------------------------------------------------------------------+
| CiviCRM version 5 |
+--------------------------------------------------------------------+
| Copyright CiviCRM LLC (c) 2004-2018 |
+--------------------------------------------------------------------+
| This file is a part of CiviCRM. |
| |
| CiviCRM is free software; you can copy, modify, and distribute it |
| under the terms of the GNU Affero General Public License |
| Version 3, 19 November 2007 and the CiviCRM Licensing Exception. |
| |
| CiviCRM is distributed in the hope that it will be useful, but |
| WITHOUT ANY WARRANTY; without even the implied warranty of |
| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
| See the GNU Affero General Public License for more details. |
| |
| You should have received a copy of the GNU Affero General Public |
| License and the CiviCRM Licensing Exception along |
| with this program; if not, contact CiviCRM LLC |
| at info[AT]civicrm[DOT]org. If you have questions about the |
| GNU Affero General Public License or the licensing of CiviCRM, |
| see the CiviCRM license FAQ at http://civicrm.org/licensing |
+--------------------------------------------------------------------+
*/

/**
* Class CRM_Core_PrevNextCache_Memory
*
* Store the previous/next cache in a Redis set.
*
* Each logical prev-next cache corresponds to three distinct items in Redis:
* - "{prefix}/{qfKey}/list" - Sorted set of `entity_id`, with all entities
* - "{prefix}/{qfkey}/sel" - Sorted set of `entity_id`, with only entities marked by user
* - "{prefix}/{qfkey}/data" - Hash mapping from `entity_id` to `data`
*
* @link https://github.com/phpredis/phpredis
*/
class CRM_Core_PrevNextCache_Redis implements CRM_Core_PrevNextCache_Interface {

const TTL = 21600;

/**
* @var Redis
*/
protected $redis;

/**
* @var string
*/
protected $prefix;

/**
* CRM_Core_PrevNextCache_Redis constructor.
* @param array $settings
*/
public function __construct($settings) {
$this->redis = CRM_Utils_Cache_Redis::connect($settings);
$this->prefix = isset($settings['prefix']) ? $settings['prefix'] : '';
$this->prefix .= \CRM_Utils_Cache::DELIMITER . 'prevnext' . \CRM_Utils_Cache::DELIMITER;
}

public function fillWithSql($cacheKey, $sql) {
$dao = CRM_Core_DAO::executeQuery($sql, [], FALSE, NULL, FALSE, TRUE, TRUE);
if (is_a($dao, 'DB_Error')) {
throw new CRM_Core_Exception($dao->message);
}

list($allKey, $dataKey, , $maxScore) = $this->initCacheKey($cacheKey);

while ($dao->fetch()) {
list (, $entity_id, $data) = array_values($dao->toArray());
$maxScore++;
$this->redis->zAdd($allKey, $maxScore, $entity_id);
$this->redis->hSet($dataKey, $entity_id, $data);
}

$dao->free();
return TRUE;
}

public function fillWithArray($cacheKey, $rows) {
list($allKey, $dataKey, , $maxScore) = $this->initCacheKey($cacheKey);

foreach ($rows as $row) {
$maxScore++;
$this->redis->zAdd($allKey, $maxScore, $row['entity_id1']);
$this->redis->hSet($dataKey, $row['entity_id1'], $row['data']);
}

return TRUE;
}

public function fetch($cacheKey, $offset, $rowCount, $includeContactIds, $queryBao) {
// Get the list of contact ID's from Redis, then grab full details about
// each contact using $queryBao.

$allKey = $this->key($cacheKey, 'all');
$cids = $this->redis->zRange($allKey, $offset, $offset + $rowCount - 1);
CRM_Utils_Type::validateAll($cids, 'Positive');

$queryBao->_includeContactIds = $includeContactIds;
$onlyDeleted = in_array(array('deleted_contacts', '=', '1', '0', '0'), $queryBao->_params);
list($select, $from, $where) = $queryBao->query(FALSE, FALSE, FALSE, $onlyDeleted);
$select .= sprintf(", (%s) AS _wgt", $this->createCase('contact_a.id', $cids));
$where .= sprintf(' AND contact_a.id IN (%s)', implode(',', $cids));
$order = 'ORDER BY _wgt';
$groupBy = '';
$limit = '';
$query = "$select $from $where $groupBy $order $limit";

return CRM_Core_DAO::executeQuery($query)->fetchGenerator();

}

/**
* Construct a SQL CASE expression.
*
* @param string $idCol
* The name of a column with ID's (eg 'contact_a.id').
* @param array $cids
* Array(int $weight => int $id).
* @return string
* CASE WHEN id=123 THEN 1 WHEN id=456 THEN 2 END
*/
private function createCase($idCol, $cids) {
$buf = "CASE\n";
foreach ($cids as $weight => $cid) {
$buf .= " WHEN $idCol = $cid THEN $weight \n";
}
$buf .= "END\n";
return $buf;
}

public function markSelection($cacheKey, $action, $ids = NULL) {
$allKey = $this->key($cacheKey, 'all');
$selKey = $this->key($cacheKey, 'sel');

if ($action === 'select') {
foreach ((array) $ids as $id) {
$score = $this->redis->zScore($allKey, $id);
$this->redis->zAdd($selKey, $score, $id);
}
}
elseif ($action === 'unselect' && $ids === NULL) {
$this->redis->delete($selKey);
$this->redis->setTimeout($selKey, self::TTL);
}
elseif ($action === 'unselect' && $ids !== NULL) {
foreach ((array) $ids as $id) {
$this->redis->zDelete($selKey, $id);
}
}
}

public function getSelection($cacheKey, $action = 'get') {
$allKey = $this->key($cacheKey, 'all');
$selKey = $this->key($cacheKey, 'sel');

if ($action === 'get') {
$result = [];
foreach ($this->redis->zRange($selKey, 0, -1) as $entity_id) {
$result[$entity_id] = 1;
}
return [$cacheKey => $result];
}
elseif ($action === 'getall') {
$result = [];
foreach ($this->redis->zRange($allKey, 0, -1) as $entity_id) {
$result[$entity_id] = 1;
}
return [$cacheKey => $result];
}
else {
throw new \CRM_Core_Exception("Unrecognized action: $action");
}
}

public function getPositions($cacheKey, $id1) {
$allKey = $this->key($cacheKey, 'all');
$dataKey = $this->key($cacheKey, 'data');

$rank = $this->redis->zRank($allKey, $id1);
if (!is_int($rank) || $rank < 0) {
return ['foundEntry' => 0];
}

$pos = ['foundEntry' => 1];

if ($rank > 0) {
$pos['prev'] = [];
foreach ($this->redis->zRange($allKey, $rank - 1, $rank - 1) as $value) {
$pos['prev']['id1'] = $value;
}
$pos['prev']['data'] = $this->redis->hGet($dataKey, $pos['prev']['id1']);
}

$count = $this->getCount($cacheKey);
if ($count > $rank + 1) {
$pos['next'] = [];
foreach ($this->redis->zRange($allKey, $rank + 1, $rank + 1) as $value) {
$pos['next']['id1'] = $value;
}
$pos['next']['data'] = $this->redis->hGet($dataKey, $pos['next']['id1']);
}

return $pos;
}

public function deleteItem($id = NULL, $cacheKey = NULL) {
if ($id === NULL && $cacheKey !== NULL) {
$allKey = $this->key($cacheKey, 'all');
$selKey = $this->key($cacheKey, 'sel');
$dataKey = $this->key($cacheKey, 'data');
$this->redis->delete($allKey, $selKey, $dataKey);
}
else {
throw new CRM_Core_Exception("Not implemented: Redis::deleteItem");
}
}

public function getCount($cacheKey) {
$allKey = $this->key($cacheKey, 'all');
return $this->redis->zSize($allKey);
}

/**
* Construct the full path to a cache item.
*
* @param string $cacheKey
* Identifier for this saved search.
* Ex: 'abcd1234abcd1234'.
* @param string $item
* Ex: 'list', 'rel', 'data'.
* @return string
* Ex: 'dmaster/prevnext/abcd1234abcd1234/list'
*/
private function key($cacheKey, $item) {
return $this->prefix . $cacheKey . '/' . $item;
}

/**
* Initialize any data-structures or timeouts for the cache-key.
*
* This is non-destructive -- if data already exists, it's preserved.
*
* @return array
* 0 => string $allItemsCacheKey,
* 1 => string $dataItemsCacheKey,
* 2 => string $selectedItemsCacheKey,
* 3 => int $maxExistingScore
*/
private function initCacheKey($cacheKey) {
$allKey = $this->key($cacheKey, 'all');
$selKey = $this->key($cacheKey, 'sel');
$dataKey = $this->key($cacheKey, 'data');

$this->redis->setTimeout($allKey, self::TTL);
$this->redis->setTimeout($dataKey, self::TTL);
$this->redis->setTimeout($selKey, self::TTL);

$maxScore = 0;
foreach ($this->redis->zRange($allKey, -1, -1, TRUE) as $lastElem => $lastScore) {
$maxScore = $lastScore;
}
return array($allKey, $dataKey, $selKey, $maxScore);
}

}
15 changes: 15 additions & 0 deletions Civi/Core/Container.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,14 @@ public function createContainer() {
[]
));

$container->setDefinition('prevnext.driver.redis', new Definition(
'CRM_Core_PrevNextCache_Redis',
[new Reference('cache_config')]
));

$container->setDefinition('cache_config', new Definition('ArrayObject'))
->setFactory(array(new Reference(self::SELF), 'createCacheConfig'));

$container->setDefinition('civi.mailing.triggers', new Definition(
'Civi\Core\SqlTrigger\TimestampTriggers',
array('civicrm_mailing', 'Mailing')
Expand Down Expand Up @@ -421,6 +429,13 @@ public static function createPrevNextCache($container) {
: $container->get('prevnext.driver.sql');
}

public static function createCacheConfig() {
$driver = \CRM_Utils_Cache::getCacheDriver();
$settings = \CRM_Utils_Cache::getCacheSettings($driver);
$settings['driver'] = $driver;
return new \ArrayObject($settings);
}

/**
* Get a list of boot services.
*
Expand Down

0 comments on commit 718fa80

Please sign in to comment.