From 32563bd40284dcc1532a8ac27d3172a670ef9db0 Mon Sep 17 00:00:00 2001 From: fengqi Date: Fri, 5 Nov 2021 18:52:57 +0800 Subject: [PATCH] Implement profile: getForUrl(), getPercentileForUrl(), getAll() --- src/Db/PdoRepository.php | 188 +++++++++++++++++++++++++++++++++-- src/Searcher/PdoSearcher.php | 147 +++++++++++++++++---------- 2 files changed, 274 insertions(+), 61 deletions(-) diff --git a/src/Db/PdoRepository.php b/src/Db/PdoRepository.php index d4016bca..7e680c7c 100644 --- a/src/Db/PdoRepository.php +++ b/src/Db/PdoRepository.php @@ -2,9 +2,12 @@ namespace XHGui\Db; +use DateInterval; +use DateTime; use Generator; use PDO; use RuntimeException; +use XHGui\Searcher\SearcherInterface; class PdoRepository { @@ -90,20 +93,22 @@ public function getById(string $id): array return $row; } - public function countByUrl(string $url): int + public function countByUrl(array $options): int { $query = sprintf(' SELECT COUNT(*) AS count FROM %s - WHERE "simple_url" LIKE :url - ', $this->table); + WHERE %s', + $this->table, + $options['where']['conditions'] + ); $stmt = $this->pdo->prepare($query); - $stmt->execute(['url' => '%' . $url . '%']); + $stmt->execute($options['where']['params']); return (int)$stmt->fetchColumn(); } - public function findByUrl(string $url, string $direction, int $skip, int $perPage): Generator + public function findByUrl(array $options = []): Generator { $query = sprintf(' SELECT @@ -122,16 +127,18 @@ public function findByUrl(string $url, string $direction, int $skip, int $perPag "main_mu", "main_pmu" FROM %s - WHERE "simple_url" LIKE :url - ORDER BY "request_ts" %s + WHERE %s + ORDER BY %s %s LIMIT %d OFFSET %d', $this->table, - $direction, - $perPage, - $skip + $options['where']['conditions'], + $options['sort'], + $options['direction'], + $options['perPage'], + $options['skip'] ); $stmt = $this->pdo->prepare($query); - $stmt->execute(['url' => '%' . $url . '%']); + $stmt->execute($options['where']['params']); while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) { yield $row; @@ -306,4 +313,163 @@ public function truncateWatches() $this->pdo->exec(sprintf('DELETE FROM %s', $this->tableWatches)) ); } + + public function aggregate(array $options) + { + $query = sprintf( + 'SELECT + "id", + "request_ts", + "main_wt", + "main_ct", + "main_cpu", + "main_mu", + "main_pmu" + FROM %s + WHERE %s + ORDER BY %s %s', + $this->table, + $options['where']['conditions'], + $options['sort'], + $options['direction'] + ); + $stmt = $this->pdo->prepare($query); + $stmt->execute($options['where']['params']); + + while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) { + yield $row; + } + } + + /** + * Convert request data keys into pdo query. + */ + public function buildQuery(array $options): array + { + return [ + 'where' => $this->buildWhere($options['conditions']), + 'sort' => $this->buildSort($options), + 'direction' => $this->buildDirection($options), + 'perPage' => $options['perPage'] ?? SearcherInterface::DEFAULT_PER_PAGE, + ]; + } + + /** + * build pdo where + */ + public function buildWhere(array $search): array + { + $where = ['conditions' => '1=1', 'params' => []]; + + if (empty($search)) { + return $where; + } + + if (isset($search['limit_custom']) && $search['limit_custom'][0] === 'P') { + $search['limit'] = $search['limit_custom']; + } + $hasLimit = (isset($search['limit']) && $search['limit'] !== -1); + + // simple_url equals match + if (isset($search['simple_url'])) { + $where['conditions'] .= ' and simple_url = :simple_url'; + $where['params']['simple_url'] = $search['simple_url']; + } + + if (isset($search['date_start']) && !$hasLimit) { + $where['conditions'] .= ' and request_date >= :date_start'; + $where['params']['date_start'] = $search['date_start']; + } + + if (isset($search['date_end']) && !$hasLimit) { + $where['conditions'] .= ' and request_date <= :date_end'; + $where['params']['date_end'] = $search['date_end']; + } + + if (isset($search['request_start'])) { + $where['conditions'] .= ' and request_ts >= :request_start'; + $where['params']['request_start'] = strtotime($search['request_start']); + } + + if (isset($search['request_end'])) { + $where['conditions'] .= ' and request_ts <= :request_end'; + $where['params']['request_end'] = strtotime($search['request_end']); + } + + // TODO need JSON support + if (isset($search['remote_addr'])) { + $where['conditions'] .= ' and SERVER like :remote_addr'; + $where['params']['remote_addr'] = '%' . $search['remote_addr'] . '%'; + } + + if (isset($search['cookie'])) { + $where['conditions'] .= ' and SERVER like :cookie'; + $where['params']['cookie'] = '%' . $search['cookie'] . '%'; + } + + if (isset($search['server_name'])) { + $where['conditions'] .= ' and SERVER like :server_name'; + $where['params']['server_name'] = '%' . $search['server_name'] . '%'; + } + + if ($hasLimit && $search['limit'][0] === 'P') { + $date = new DateTime(); + try { + $date->sub(new DateInterval($search['limit'])); + $where['conditions'] .= ' and request_ts >= :limit_start'; + $where['params']['limit_start'] = $date->getTimestamp(); + } catch (\Exception $e) { + $where['conditions'] .= ' and request_ts >= :limit_start'; + $where['params']['limit_start'] = time() + 86400; + } + } + + // fuzzy match + if (isset($search['url'])) { + $where['conditions'] .= ' and url like :url'; + $where['params']['url'] = '%' . $search['url'] . '%'; + } + + return $where; + } + + /** + * build pdo order sort + */ + private function buildSort(array $options): string + { + if (!isset($options['sort'])) { + return 'request_ts'; + } + + $valid = ['time', 'wt', 'mu', 'cpu', 'pmu']; + if (isset($options['sort'])) { + if ($options['sort'] === 'time') { + return 'request_ts'; + } + + if (in_array($options['sort'], $valid, true)) { + return 'main_' . $options['sort']; + } + } + + return $options['sort']; + } + + /** + * build pdo order direction + */ + private function buildDirection(array $options): string + { + if (!isset($options['direction'])) { + return SearcherInterface::DEFAULT_DIRECTION; + } + + $valid = ['desc', 'asc']; + if (in_array($options['direction'], $valid, true)) { + return $options['direction']; + } + + return 'desc'; + } } diff --git a/src/Searcher/PdoSearcher.php b/src/Searcher/PdoSearcher.php index ccb867f9..836208a4 100644 --- a/src/Searcher/PdoSearcher.php +++ b/src/Searcher/PdoSearcher.php @@ -69,14 +69,57 @@ public function get($id): Profile ]); } - public function getForUrl($url, $options, $conditions = []): void + public function getForUrl($url, $options, $conditions = []): array { - throw NotImplementedException::notImplementedPdo(__METHOD__); + $conditions = array_merge( + (array)$conditions, + ['simple_url' => $url] + ); + + $options = array_merge($options, [ + 'conditions' => $conditions, + ]); + + return $this->paginate($options); } - public function getPercentileForUrl($percentile, $url, $search = []): void + public function getPercentileForUrl($percentile, $url, $search = []): array { - throw NotImplementedException::notImplementedPdo(__METHOD__); + $search = array_merge((array)$search, ['simple_url' => $url]); + $option = $this->db->buildQuery(['conditions' => $search]); + + $results = []; + foreach ($this->db->aggregate($option) as $row) { + $timestamp = $row['request_ts']; + $rowCount = $results[$timestamp]['row_count'] ?? 0; + + $results[$timestamp]['_id'] = $timestamp; + $results[$timestamp]['row_count'] = $rowCount + 1; + $results[$timestamp]['raw_index'] = $percentile / 100; + $results[$timestamp]['wall_times'][] = intval($row['main_wt']); + $results[$timestamp]['cpu_times'][] = intval($row['main_cpu']); + $results[$timestamp]['mu_times'][] = intval($row['main_mu']); + $results[$timestamp]['pmu_times'][] = intval($row['main_pmu']); + } + + $keys = [ + 'wall_times' => 'wt', + 'cpu_times' => 'cpu', + 'mu_times' => 'mu', + 'pmu_times' => 'pmu', + ]; + foreach ($results as &$result) { + $result['date'] = date('Y-m-d H:i:s', $result['_id']); + unset($result['_id']); + $index = max(round($result['raw_index']) - 1, 0); + foreach ($keys as $key => $out) { + sort($result[$key]); + $result[$out] = $result[$key][$index] ?? null; + unset($result[$key]); + } + } + + return array_values($results); } /** @@ -92,52 +135,7 @@ public function getAvgsForUrl($url, $search = []): void */ public function getAll(SearchOptions $options): array { - $page = $options['page']; - $direction = $options['direction']; - $perPage = $options['perPage']; - $url = $options['conditions']['url'] ?? ''; - - $totalRows = $this->db->countByUrl($url); - $totalPages = max(ceil($totalRows / $perPage), 1); - if ($page > $totalPages) { - $page = $totalPages; - } - $skip = ($page - 1) * $perPage; - - $results = []; - foreach ($this->db->findByUrl($url, $direction, $skip, $perPage) as $row) { - $results[] = new Profile([ - '_id' => $row['id'], - 'meta' => [ - 'url' => $row['url'], - 'SERVER' => json_decode($row['SERVER'], true), - 'get' => json_decode($row['GET'], true), - 'env' => json_decode($row['ENV'], true), - 'simple_url' => $row['simple_url'], - 'request_ts' => $row['request_ts'], - 'request_ts_micro' => $row['request_ts_micro'], - 'request_date' => $row['request_date'], - ], - 'profile' => [ - 'main()' => [ - 'wt' => (int) $row['main_wt'], - 'ct' => (int) $row['main_ct'], - 'cpu' => (int) $row['main_cpu'], - 'mu' => (int) $row['main_mu'], - 'pmu' => (int) $row['main_pmu'], - ], - ], - ]); - } - - return [ - 'results' => $results, - 'sort' => 'meta.request_ts', - 'direction' => $direction, - 'page' => $page, - 'perPage' => $perPage, - 'totalPages' => $totalPages, - ]; + return $this->paginate($options->toArray()); } /** @@ -227,4 +225,53 @@ public function stats() return $row; } + + /** + * {@inheritdoc} + */ + private function paginate(array $options): array + { + $opt = $this->db->buildQuery($options); + + $totalRows = $this->db->countByUrl($opt); + $totalPages = max(ceil($totalRows / $opt['perPage']), 1); + + $page = min(max($options['page'] ?? 1, 1), $totalPages); + $opt['skip'] = ($page - 1) * $opt['perPage']; + + $results = []; + foreach ($this->db->findByUrl($opt) as $row) { + $results[] = new Profile([ + '_id' => $row['id'], + 'meta' => [ + 'url' => $row['url'], + 'SERVER' => json_decode($row['SERVER'], true), + 'get' => json_decode($row['GET'], true), + 'env' => json_decode($row['ENV'], true), + 'simple_url' => $row['simple_url'], + 'request_ts' => $row['request_ts'], + 'request_ts_micro' => $row['request_ts_micro'], + 'request_date' => $row['request_date'], + ], + 'profile' => [ + 'main()' => [ + 'wt' => (int) $row['main_wt'], + 'ct' => (int) $row['main_ct'], + 'cpu' => (int) $row['main_cpu'], + 'mu' => (int) $row['main_mu'], + 'pmu' => (int) $row['main_pmu'], + ], + ], + ]); + } + + return [ + 'results' => $results, + 'sort' => 'meta.request_ts', + 'direction' => $opt['direction'], + 'page' => $page, + 'perPage' => $opt['perPage'], + 'totalPages' => $totalPages, + ]; + } }