From a7a48ff4367a4aafd0beeb2ec508e52a77bcc34e Mon Sep 17 00:00:00 2001 From: Fanis Tharropoulos Date: Fri, 20 Sep 2024 16:55:14 +0300 Subject: [PATCH 1/8] fix(typesense): fix per_page error when paginating with query callbacks - Implement performPaginatedSearch for large result sets - Add maxTotalResults property to limit total fetched results - Adjust search method to use maxPerPage as default limit - Ensure search respects both maxPerPage and maxTotalResults limits This ensures that when the `getTotalCount` builder function is called with a query callback, the resulting request to the server will be broken down batches of 250 results per page, Typesense's maximum. --- src/Engines/TypesenseEngine.php | 65 ++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/src/Engines/TypesenseEngine.php b/src/Engines/TypesenseEngine.php index 18146989..f9de9504 100644 --- a/src/Engines/TypesenseEngine.php +++ b/src/Engines/TypesenseEngine.php @@ -15,6 +15,12 @@ class TypesenseEngine extends Engine { + /** + * The maximum amount of results that can be fetched per page. + * @var int + */ + private int $maxPerPage = 250; + /** * The Typesense client instance. * @@ -29,6 +35,14 @@ class TypesenseEngine extends Engine */ protected array $searchParameters = []; + + /** + * + * The maximum amount of results that can be fetched for pagination. + * @var int + */ + protected int $maxTotalResults; + /** * Create new Typesense engine instance. * @@ -186,12 +200,61 @@ protected function deleteDocument(TypesenseCollection $collectionIndex, $modelId */ public function search(Builder $builder) { + // If the limit exceeds Typesense's capabilities, perform a paginated search + if ($builder->limit >= $this->maxPerPage) { + return $this->performPaginatedSearch($builder); + } + return $this->performSearch( $builder, - $this->buildSearchParameters($builder, 1, $builder->limit) + $this->buildSearchParameters($builder, 1, $builder->limit ?? $this->maxPerPage) ); } + /** + * Perform a paginated search on the engine. + * @param \Laravel\Scout\Builder $builder + * @return mixed + * + * @throws \Http\Client\Exception + * @throws \Typesense\Exceptions\TypesenseClientError + */ + protected function performPaginatedSearch(Builder $builder) + { + $page = 1; + $limit = min($builder->limit ?? $this->maxPerPage, $this->maxPerPage, $this->maxTotalResults); + $remainingResults = min($builder->limit ?? $this->maxTotalResults, $this->maxTotalResults); + $results = new Collection; + + while ($remainingResults > 0) { + $search_res = $this->performSearch( + $builder, + $this->buildSearchParameters($builder, $page, $limit) + ); + $results = $results->concat($search_res['hits'] ?? []); + + if ($page === 1) { + $total_found = $search_res['found'] ?? 0; + } + + $remainingResults -= $limit; + $page++; + + if (count($search_res['hits'] ?? []) < $limit) { + break; + } + } + + return [ + 'hits' => $results->all(), + 'found' => $results->count(), + 'out_of' => $total_found, + 'page' => 1, + 'request_params' => $this->buildSearchParameters($builder, 1, $builder->limit ?? $this->maxPerPage), + ]; + } + + /** * Perform the given search on the engine with pagination. * From d209bfcbf2d84f9c6875e811b32a77d619e037a1 Mon Sep 17 00:00:00 2001 From: Fanis Tharropoulos Date: Fri, 20 Sep 2024 16:58:50 +0300 Subject: [PATCH 2/8] feat(typesense): addmax total results configuration for typesense - Introduce a new 'max_total_results' configuration option for Typesense in scout.php. - Update TypesenseEngine constructor and EngineManager to utilize this new parameter, providing better control over the maximum number of searchable results. This ensures that the resulting requests to the server when broken up to batches are handled by the user, not defaulting to Typesense's total records, as that can introduce perfomance issues for larger datasets. --- config/scout.php | 1 + src/EngineManager.php | 3 ++- src/Engines/TypesenseEngine.php | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/config/scout.php b/config/scout.php index ac1a88ea..68218c2f 100644 --- a/config/scout.php +++ b/config/scout.php @@ -173,6 +173,7 @@ 'num_retries' => env('TYPESENSE_NUM_RETRIES', 3), 'retry_interval_seconds' => env('TYPESENSE_RETRY_INTERVAL_SECONDS', 1), ], + // 'max_total_results' => env('TYPESENSE_MAX_TOTAL_RESULTS', 1000), 'model-settings' => [ // User::class => [ // 'collection-schema' => [ diff --git a/src/EngineManager.php b/src/EngineManager.php index 08d73f92..10b1be8e 100644 --- a/src/EngineManager.php +++ b/src/EngineManager.php @@ -153,9 +153,10 @@ protected function ensureMeilisearchClientIsInstalled() */ public function createTypesenseDriver() { + $config = config('scout.typesense'); $this->ensureTypesenseClientIsInstalled(); - return new TypesenseEngine(new Typesense(config('scout.typesense.client-settings'))); + return new TypesenseEngine(new Typesense($config['client-settings']), $config['max_total_results'] ?? 1000); } /** diff --git a/src/Engines/TypesenseEngine.php b/src/Engines/TypesenseEngine.php index f9de9504..a0d3158f 100644 --- a/src/Engines/TypesenseEngine.php +++ b/src/Engines/TypesenseEngine.php @@ -48,9 +48,10 @@ class TypesenseEngine extends Engine * * @param Typesense $typesense */ - public function __construct(Typesense $typesense) + public function __construct(Typesense $typesense, int $maxTotalResults) { $this->typesense = $typesense; + $this->maxTotalResults = $maxTotalResults; } /** From 91bde2d695613cf8efaebb7946af97d1334183f4 Mon Sep 17 00:00:00 2001 From: Fanis Tharropoulos Date: Fri, 20 Sep 2024 17:02:23 +0300 Subject: [PATCH 3/8] test(typesense): add test for paginated search with empty query callback - Implement new test method in TypesenseSearchableTest to verify pagination behavior with an empty query callback. This ensures the search functionality works correctly when no additional query constraints are applied. --- tests/Integration/SearchableTests.php | 7 +++++++ tests/Integration/TypesenseSearchableTest.php | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/tests/Integration/SearchableTests.php b/tests/Integration/SearchableTests.php index 9856f719..3bf9e366 100644 --- a/tests/Integration/SearchableTests.php +++ b/tests/Integration/SearchableTests.php @@ -104,4 +104,11 @@ protected function itCanUsePaginatedSearchWithQueryCallback() User::search('lar')->take(10)->query($queryCallback)->paginate(5, 'page', 2), ]; } + + protected function itCanUsePaginatedSearchWithEmptyQueryCallback() + { + $queryCallback = function ($query) {}; + + return User::search('*')->query($queryCallback)->paginate(); + } } diff --git a/tests/Integration/TypesenseSearchableTest.php b/tests/Integration/TypesenseSearchableTest.php index 30da065e..c6405f25 100644 --- a/tests/Integration/TypesenseSearchableTest.php +++ b/tests/Integration/TypesenseSearchableTest.php @@ -164,6 +164,14 @@ public function test_it_can_use_paginated_search_with_query_callback() ], $page2->pluck('name', 'id')->all()); } + public function test_it_can_usePaginatedSearchWithEmptyQueryCallback() + { + $res = $this->itCanUsePaginatedSearchWithEmptyQueryCallback(); + + $this->assertSame($res->total(), 44); + $this->assertSame($res->lastPage(), 3); + } + protected static function scoutDriver(): string { return 'typesense'; From 58445d8c13111712a54398200d9c293463daae33 Mon Sep 17 00:00:00 2001 From: Fanis Tharropoulos Date: Fri, 20 Sep 2024 17:17:42 +0300 Subject: [PATCH 4/8] style: ci linting errors --- src/Engines/TypesenseEngine.php | 4 +++- tests/Integration/SearchableTests.php | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Engines/TypesenseEngine.php b/src/Engines/TypesenseEngine.php index a0d3158f..db5374cb 100644 --- a/src/Engines/TypesenseEngine.php +++ b/src/Engines/TypesenseEngine.php @@ -17,6 +17,7 @@ class TypesenseEngine extends Engine { /** * The maximum amount of results that can be fetched per page. + * * @var int */ private int $maxPerPage = 250; @@ -37,8 +38,8 @@ class TypesenseEngine extends Engine /** - * * The maximum amount of results that can be fetched for pagination. + * * @var int */ protected int $maxTotalResults; @@ -214,6 +215,7 @@ public function search(Builder $builder) /** * Perform a paginated search on the engine. + * * @param \Laravel\Scout\Builder $builder * @return mixed * diff --git a/tests/Integration/SearchableTests.php b/tests/Integration/SearchableTests.php index 3bf9e366..ef25c85f 100644 --- a/tests/Integration/SearchableTests.php +++ b/tests/Integration/SearchableTests.php @@ -107,7 +107,8 @@ protected function itCanUsePaginatedSearchWithQueryCallback() protected function itCanUsePaginatedSearchWithEmptyQueryCallback() { - $queryCallback = function ($query) {}; + $queryCallback = function ($query) { + }; return User::search('*')->query($queryCallback)->paginate(); } From 63b5eb36506cf43c53d94082be0935b11d6758ba Mon Sep 17 00:00:00 2001 From: Fanis Tharropoulos Date: Fri, 20 Sep 2024 17:21:20 +0300 Subject: [PATCH 5/8] fix(test): fix missing constructor args in typesense unit test --- tests/Unit/TypesenseEngineTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Unit/TypesenseEngineTest.php b/tests/Unit/TypesenseEngineTest.php index 9f360a9b..93f3a5dc 100644 --- a/tests/Unit/TypesenseEngineTest.php +++ b/tests/Unit/TypesenseEngineTest.php @@ -27,7 +27,7 @@ protected function setUp(): void // Mock the Typesense client and pass it to the engine constructor $typesenseClient = $this->createMock(TypesenseClient::class); $this->engine = $this->getMockBuilder(TypesenseEngine::class) - ->setConstructorArgs([$typesenseClient]) + ->setConstructorArgs([$typesenseClient, 1000]) ->onlyMethods(['getOrCreateCollectionFromModel', 'buildSearchParameters']) ->getMock(); } From db1b4edc67be3f936fb0b14981bfe8fca7e6c59b Mon Sep 17 00:00:00 2001 From: Fanis Tharropoulos Date: Fri, 20 Sep 2024 17:27:58 +0300 Subject: [PATCH 6/8] style: more ci linting errors --- src/Engines/TypesenseEngine.php | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Engines/TypesenseEngine.php b/src/Engines/TypesenseEngine.php index db5374cb..a344a621 100644 --- a/src/Engines/TypesenseEngine.php +++ b/src/Engines/TypesenseEngine.php @@ -36,7 +36,6 @@ class TypesenseEngine extends Engine */ protected array $searchParameters = []; - /** * The maximum amount of results that can be fetched for pagination. * @@ -216,7 +215,7 @@ public function search(Builder $builder) /** * Perform a paginated search on the engine. * - * @param \Laravel\Scout\Builder $builder + * @param \Laravel\Scout\Builder $builder * @return mixed * * @throws \Http\Client\Exception @@ -257,7 +256,6 @@ protected function performPaginatedSearch(Builder $builder) ]; } - /** * Perform the given search on the engine with pagination. * From af4f342140c42c91a21bdb66dc729f1e919c7558 Mon Sep 17 00:00:00 2001 From: Taylor Otwell Date: Wed, 25 Sep 2024 12:09:02 -0400 Subject: [PATCH 7/8] formatting --- src/Engines/TypesenseEngine.php | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/Engines/TypesenseEngine.php b/src/Engines/TypesenseEngine.php index a344a621..7e532b22 100644 --- a/src/Engines/TypesenseEngine.php +++ b/src/Engines/TypesenseEngine.php @@ -15,13 +15,6 @@ class TypesenseEngine extends Engine { - /** - * The maximum amount of results that can be fetched per page. - * - * @var int - */ - private int $maxPerPage = 250; - /** * The Typesense client instance. * @@ -37,7 +30,14 @@ class TypesenseEngine extends Engine protected array $searchParameters = []; /** - * The maximum amount of results that can be fetched for pagination. + * The maximum number of results that can be fetched per page. + * + * @var int + */ + private int $maxPerPage = 250; + + /** + * The maximum number of results that can be fetched during pagination. * * @var int */ @@ -201,7 +201,7 @@ protected function deleteDocument(TypesenseCollection $collectionIndex, $modelId */ public function search(Builder $builder) { - // If the limit exceeds Typesense's capabilities, perform a paginated search + // If the limit exceeds Typesense's capabilities, perform a paginated search... if ($builder->limit >= $this->maxPerPage) { return $this->performPaginatedSearch($builder); } @@ -226,6 +226,7 @@ protected function performPaginatedSearch(Builder $builder) $page = 1; $limit = min($builder->limit ?? $this->maxPerPage, $this->maxPerPage, $this->maxTotalResults); $remainingResults = min($builder->limit ?? $this->maxTotalResults, $this->maxTotalResults); + $results = new Collection; while ($remainingResults > 0) { @@ -233,10 +234,11 @@ protected function performPaginatedSearch(Builder $builder) $builder, $this->buildSearchParameters($builder, $page, $limit) ); + $results = $results->concat($search_res['hits'] ?? []); if ($page === 1) { - $total_found = $search_res['found'] ?? 0; + $totalFound = $search_res['found'] ?? 0; } $remainingResults -= $limit; @@ -250,7 +252,7 @@ protected function performPaginatedSearch(Builder $builder) return [ 'hits' => $results->all(), 'found' => $results->count(), - 'out_of' => $total_found, + 'out_of' => $totalFound, 'page' => 1, 'request_params' => $this->buildSearchParameters($builder, 1, $builder->limit ?? $this->maxPerPage), ]; From 6e05b7fef4732dea24e3f4e0142ea9448501a217 Mon Sep 17 00:00:00 2001 From: Taylor Otwell Date: Wed, 25 Sep 2024 12:11:10 -0400 Subject: [PATCH 8/8] formatting --- src/Engines/TypesenseEngine.php | 88 ++++++++++++++++----------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/src/Engines/TypesenseEngine.php b/src/Engines/TypesenseEngine.php index 7e532b22..72861390 100644 --- a/src/Engines/TypesenseEngine.php +++ b/src/Engines/TypesenseEngine.php @@ -212,6 +212,46 @@ public function search(Builder $builder) ); } + /** + * Perform the given search on the engine with pagination. + * + * @param \Laravel\Scout\Builder $builder + * @param int $perPage + * @param int $page + * @return mixed + * + * @throws \Http\Client\Exception + * @throws \Typesense\Exceptions\TypesenseClientError + */ + public function paginate(Builder $builder, $perPage, $page) + { + return $this->performSearch( + $builder, + $this->buildSearchParameters($builder, $page, $perPage) + ); + } + + /** + * Perform the given search on the engine. + * + * @param \Laravel\Scout\Builder $builder + * @param array $options + * @return mixed + * + * @throws \Http\Client\Exception + * @throws \Typesense\Exceptions\TypesenseClientError + */ + protected function performSearch(Builder $builder, array $options = []): mixed + { + $documents = $this->getOrCreateCollectionFromModel($builder->model, false)->getDocuments(); + + if ($builder->callback) { + return call_user_func($builder->callback, $documents, $builder->query, $options); + } + + return $documents->search($options); + } + /** * Perform a paginated search on the engine. * @@ -230,21 +270,21 @@ protected function performPaginatedSearch(Builder $builder) $results = new Collection; while ($remainingResults > 0) { - $search_res = $this->performSearch( + $searchResults = $this->performSearch( $builder, $this->buildSearchParameters($builder, $page, $limit) ); - $results = $results->concat($search_res['hits'] ?? []); + $results = $results->concat($searchResults['hits'] ?? []); if ($page === 1) { - $totalFound = $search_res['found'] ?? 0; + $totalFound = $searchResults['found'] ?? 0; } $remainingResults -= $limit; $page++; - if (count($search_res['hits'] ?? []) < $limit) { + if (count($searchResults['hits'] ?? []) < $limit) { break; } } @@ -258,46 +298,6 @@ protected function performPaginatedSearch(Builder $builder) ]; } - /** - * Perform the given search on the engine with pagination. - * - * @param \Laravel\Scout\Builder $builder - * @param int $perPage - * @param int $page - * @return mixed - * - * @throws \Http\Client\Exception - * @throws \Typesense\Exceptions\TypesenseClientError - */ - public function paginate(Builder $builder, $perPage, $page) - { - return $this->performSearch( - $builder, - $this->buildSearchParameters($builder, $page, $perPage) - ); - } - - /** - * Perform the given search on the engine. - * - * @param \Laravel\Scout\Builder $builder - * @param array $options - * @return mixed - * - * @throws \Http\Client\Exception - * @throws \Typesense\Exceptions\TypesenseClientError - */ - protected function performSearch(Builder $builder, array $options = []): mixed - { - $documents = $this->getOrCreateCollectionFromModel($builder->model, false)->getDocuments(); - - if ($builder->callback) { - return call_user_func($builder->callback, $documents, $builder->query, $options); - } - - return $documents->search($options); - } - /** * Build the search parameters for a given Scout query builder. *