Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement pdo: getForUrl(), getPercentileForUrl(), getAll() #436

Merged
merged 1 commit into from
Nov 10, 2021
Merged

Implement pdo: getForUrl(), getPercentileForUrl(), getAll() #436

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
188 changes: 177 additions & 11 deletions src/Db/PdoRepository.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,12 @@

namespace XHGui\Db;

use DateInterval;
use DateTime;
use Generator;
use PDO;
use RuntimeException;
use XHGui\Searcher\SearcherInterface;

class PdoRepository
{
Expand Down Expand Up @@ -90,20 +93,22 @@ public function getById(string $id): array
return $row;
}

public function countByUrl(string $url): int
public function countByUrl(array $options): int
{
$query = sprintf('
SELECT COUNT(*) AS count
FROM %s
WHERE "simple_url" LIKE :url
', $this->table);
WHERE %s',
$this->table,
$options['where']['conditions']
);
$stmt = $this->pdo->prepare($query);
$stmt->execute(['url' => '%' . $url . '%']);
$stmt->execute($options['where']['params']);

return (int)$stmt->fetchColumn();
}

public function findByUrl(string $url, string $direction, int $skip, int $perPage): Generator
public function findByUrl(array $options = []): Generator
{
$query = sprintf('
SELECT
Expand All @@ -122,16 +127,18 @@ public function findByUrl(string $url, string $direction, int $skip, int $perPag
"main_mu",
"main_pmu"
FROM %s
WHERE "simple_url" LIKE :url
ORDER BY "request_ts" %s
WHERE %s
ORDER BY %s %s
LIMIT %d OFFSET %d',
$this->table,
$direction,
$perPage,
$skip
$options['where']['conditions'],
$options['sort'],
$options['direction'],
$options['perPage'],
$options['skip']
);
$stmt = $this->pdo->prepare($query);
$stmt->execute(['url' => '%' . $url . '%']);
$stmt->execute($options['where']['params']);

while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) {
yield $row;
Expand Down Expand Up @@ -306,4 +313,163 @@ public function truncateWatches()
$this->pdo->exec(sprintf('DELETE FROM %s', $this->tableWatches))
);
}

public function aggregate(array $options)
{
$query = sprintf(
'SELECT
"id",
"request_ts",
"main_wt",
"main_ct",
"main_cpu",
"main_mu",
"main_pmu"
FROM %s
WHERE %s
ORDER BY %s %s',
$this->table,
$options['where']['conditions'],
$options['sort'],
$options['direction']
);
$stmt = $this->pdo->prepare($query);
$stmt->execute($options['where']['params']);

while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) {
yield $row;
}
}

/**
* Convert request data keys into pdo query.
*/
public function buildQuery(array $options): array
{
return [
'where' => $this->buildWhere($options['conditions']),
'sort' => $this->buildSort($options),
'direction' => $this->buildDirection($options),
'perPage' => $options['perPage'] ?? SearcherInterface::DEFAULT_PER_PAGE,
];
}

/**
* build pdo where
*/
public function buildWhere(array $search): array
{
$where = ['conditions' => '1=1', 'params' => []];

if (empty($search)) {
return $where;
}

if (isset($search['limit_custom']) && $search['limit_custom'][0] === 'P') {
$search['limit'] = $search['limit_custom'];
}
$hasLimit = (isset($search['limit']) && $search['limit'] !== -1);

// simple_url equals match
if (isset($search['simple_url'])) {
$where['conditions'] .= ' and simple_url = :simple_url';
$where['params']['simple_url'] = $search['simple_url'];
}

if (isset($search['date_start']) && !$hasLimit) {
$where['conditions'] .= ' and request_date >= :date_start';
$where['params']['date_start'] = $search['date_start'];
}

if (isset($search['date_end']) && !$hasLimit) {
$where['conditions'] .= ' and request_date <= :date_end';
$where['params']['date_end'] = $search['date_end'];
}

if (isset($search['request_start'])) {
$where['conditions'] .= ' and request_ts >= :request_start';
$where['params']['request_start'] = strtotime($search['request_start']);
}

if (isset($search['request_end'])) {
$where['conditions'] .= ' and request_ts <= :request_end';
$where['params']['request_end'] = strtotime($search['request_end']);
}

// TODO need JSON support
if (isset($search['remote_addr'])) {
$where['conditions'] .= ' and SERVER like :remote_addr';
glensc marked this conversation as resolved.
Show resolved Hide resolved
$where['params']['remote_addr'] = '%' . $search['remote_addr'] . '%';
}

if (isset($search['cookie'])) {
$where['conditions'] .= ' and SERVER like :cookie';
$where['params']['cookie'] = '%' . $search['cookie'] . '%';
}

if (isset($search['server_name'])) {
$where['conditions'] .= ' and SERVER like :server_name';
$where['params']['server_name'] = '%' . $search['server_name'] . '%';
}

if ($hasLimit && $search['limit'][0] === 'P') {
$date = new DateTime();
try {
$date->sub(new DateInterval($search['limit']));
$where['conditions'] .= ' and request_ts >= :limit_start';
$where['params']['limit_start'] = $date->getTimestamp();
} catch (\Exception $e) {
fengqi marked this conversation as resolved.
Show resolved Hide resolved
$where['conditions'] .= ' and request_ts >= :limit_start';
$where['params']['limit_start'] = time() + 86400;
}
}

// fuzzy match
if (isset($search['url'])) {
$where['conditions'] .= ' and url like :url';
$where['params']['url'] = '%' . $search['url'] . '%';
}

return $where;
}

/**
* build pdo order sort
*/
private function buildSort(array $options): string
{
if (!isset($options['sort'])) {
return 'request_ts';
}

$valid = ['time', 'wt', 'mu', 'cpu', 'pmu'];
if (isset($options['sort'])) {
if ($options['sort'] === 'time') {
return 'request_ts';
}

if (in_array($options['sort'], $valid, true)) {
return 'main_' . $options['sort'];
}
}

return $options['sort'];
}

/**
* build pdo order direction
*/
private function buildDirection(array $options): string
{
if (!isset($options['direction'])) {
return SearcherInterface::DEFAULT_DIRECTION;
}

$valid = ['desc', 'asc'];
if (in_array($options['direction'], $valid, true)) {
return $options['direction'];
}

return 'desc';
}
}
147 changes: 97 additions & 50 deletions src/Searcher/PdoSearcher.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,14 +69,57 @@ public function get($id): Profile
]);
}

public function getForUrl($url, $options, $conditions = []): void
public function getForUrl($url, $options, $conditions = []): array
{
throw NotImplementedException::notImplementedPdo(__METHOD__);
$conditions = array_merge(
(array)$conditions,
['simple_url' => $url]
);

$options = array_merge($options, [
'conditions' => $conditions,
]);

return $this->paginate($options);
}

public function getPercentileForUrl($percentile, $url, $search = []): void
public function getPercentileForUrl($percentile, $url, $search = []): array
{
throw NotImplementedException::notImplementedPdo(__METHOD__);
$search = array_merge((array)$search, ['simple_url' => $url]);
$option = $this->db->buildQuery(['conditions' => $search]);

$results = [];
foreach ($this->db->aggregate($option) as $row) {
$timestamp = $row['request_ts'];
$rowCount = $results[$timestamp]['row_count'] ?? 0;

$results[$timestamp]['_id'] = $timestamp;
$results[$timestamp]['row_count'] = $rowCount + 1;
$results[$timestamp]['raw_index'] = $percentile / 100;
$results[$timestamp]['wall_times'][] = intval($row['main_wt']);
$results[$timestamp]['cpu_times'][] = intval($row['main_cpu']);
$results[$timestamp]['mu_times'][] = intval($row['main_mu']);
$results[$timestamp]['pmu_times'][] = intval($row['main_pmu']);
}

$keys = [
'wall_times' => 'wt',
'cpu_times' => 'cpu',
'mu_times' => 'mu',
'pmu_times' => 'pmu',
];
foreach ($results as &$result) {
$result['date'] = date('Y-m-d H:i:s', $result['_id']);
unset($result['_id']);
$index = max(round($result['raw_index']) - 1, 0);
foreach ($keys as $key => $out) {
sort($result[$key]);
$result[$out] = $result[$key][$index] ?? null;
unset($result[$key]);
}
}

return array_values($results);
}

/**
Expand All @@ -92,52 +135,7 @@ public function getAvgsForUrl($url, $search = []): void
*/
public function getAll(SearchOptions $options): array
{
$page = $options['page'];
$direction = $options['direction'];
$perPage = $options['perPage'];
$url = $options['conditions']['url'] ?? '';

$totalRows = $this->db->countByUrl($url);
$totalPages = max(ceil($totalRows / $perPage), 1);
if ($page > $totalPages) {
$page = $totalPages;
}
$skip = ($page - 1) * $perPage;

$results = [];
foreach ($this->db->findByUrl($url, $direction, $skip, $perPage) as $row) {
$results[] = new Profile([
'_id' => $row['id'],
'meta' => [
'url' => $row['url'],
'SERVER' => json_decode($row['SERVER'], true),
'get' => json_decode($row['GET'], true),
'env' => json_decode($row['ENV'], true),
'simple_url' => $row['simple_url'],
'request_ts' => $row['request_ts'],
'request_ts_micro' => $row['request_ts_micro'],
'request_date' => $row['request_date'],
],
'profile' => [
'main()' => [
'wt' => (int) $row['main_wt'],
'ct' => (int) $row['main_ct'],
'cpu' => (int) $row['main_cpu'],
'mu' => (int) $row['main_mu'],
'pmu' => (int) $row['main_pmu'],
],
],
]);
}

return [
'results' => $results,
'sort' => 'meta.request_ts',
'direction' => $direction,
'page' => $page,
'perPage' => $perPage,
'totalPages' => $totalPages,
];
return $this->paginate($options->toArray());
}

/**
Expand Down Expand Up @@ -227,4 +225,53 @@ public function stats()

return $row;
}

/**
* {@inheritdoc}
*/
private function paginate(array $options): array
{
$opt = $this->db->buildQuery($options);

$totalRows = $this->db->countByUrl($opt);
$totalPages = max(ceil($totalRows / $opt['perPage']), 1);

$page = min(max($options['page'] ?? 1, 1), $totalPages);
$opt['skip'] = ($page - 1) * $opt['perPage'];

$results = [];
foreach ($this->db->findByUrl($opt) as $row) {
$results[] = new Profile([
'_id' => $row['id'],
'meta' => [
'url' => $row['url'],
'SERVER' => json_decode($row['SERVER'], true),
'get' => json_decode($row['GET'], true),
'env' => json_decode($row['ENV'], true),
'simple_url' => $row['simple_url'],
'request_ts' => $row['request_ts'],
'request_ts_micro' => $row['request_ts_micro'],
'request_date' => $row['request_date'],
],
'profile' => [
'main()' => [
'wt' => (int) $row['main_wt'],
'ct' => (int) $row['main_ct'],
'cpu' => (int) $row['main_cpu'],
'mu' => (int) $row['main_mu'],
'pmu' => (int) $row['main_pmu'],
],
],
]);
}

return [
'results' => $results,
'sort' => 'meta.request_ts',
'direction' => $opt['direction'],
'page' => $page,
'perPage' => $opt['perPage'],
'totalPages' => $totalPages,
];
}
}