Skip to content

Commit

Permalink
Command Import Arguments (#75)
Browse files Browse the repository at this point in the history 
* Add arguments to command in order to allow page size of choice of collection to import

* Import Command UX and tests
  • Loading branch information
@npotier
npotier authored Feb 12, 2023
1 parent de2ba37 commit 00a67b2
Show file tree
Hide file tree
Showing 7 changed files with 205 additions and 47 deletions.
1 change: 1 addition & 0 deletions composer.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
"symfony/phpunit-bridge": "^5.0|^6.0",
"phpunit/phpunit": "^9.5",
"symfony/yaml": "^3.4 || ^4.4 || ^5.4 || ^6.0",
"dg/bypass-finals": "^1.4"
"phpspec/prophecy-phpunit": "^2.0"
},
"autoload": {
Expand Down
3 changes: 3 additions & 0 deletions phpunit.xml.dist
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,7 @@
<directory>./tests</directory>
</testsuite>
</testsuites>
<extensions>
<extension class="ACSEO\TypesenseBundle\Tests\Hook\BypassFinalHook" />
</extensions>
</phpunit>
109 changes: 88 additions & 21 deletions src/Command/ImportCommand.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@ protected function configure()
->setName(self::$defaultName)
->setDescription('Import collections from Database')
->addOption('action', null, InputOption::VALUE_OPTIONAL, 'Action modes for typesense import ("create", "upsert" or "update")', 'upsert')
->addOption('indexes', null, InputOption::VALUE_OPTIONAL, 'The index(es) to repopulate. Comma separated values')
->addOption('first-page', null, InputOption::VALUE_REQUIRED, 'The pager\'s page to start population from. Including the given page.', 1)
->addOption('last-page', null, InputOption::VALUE_REQUIRED, 'The pager\'s page to end population on. Including the given page.', null)
->addOption('max-per-page', null, InputOption::VALUE_REQUIRED, 'The pager\'s page size', 100)
;
}

Expand All @@ -63,41 +67,35 @@ protected function execute(InputInterface $input, OutputInterface $output): int

$action = $input->getOption('action');

$this->em->getConnection()->getConfiguration()->setSQLLogger(null);

$this->em->getConnection()->getConfiguration()->setMiddlewares(
[new \Doctrine\DBAL\Logging\Middleware(new \Psr\Log\NullLogger())]
);

$execStart = microtime(true);
$populated = 0;

$io->newLine();

$collectionDefinitions = $this->collectionManager->getCollectionDefinitions();
foreach ($collectionDefinitions as $collectionDefinition) {
$collectionName = $collectionDefinition['typesense_name'];
$class = $collectionDefinition['entity'];

$q = $this->em->createQuery('select e from '.$class.' e');
$entities = $q->toIterable();

$nbEntities = (int) $this->em->createQuery('select COUNT(u.id) from '.$class.' u')->getSingleScalarResult();
$populated += $nbEntities;
$indexes = (null !== $indexes = $input->getOption('indexes')) ? explode(',', $indexes) : \array_keys($collectionDefinitions);
foreach ($indexes as $index) {
if (!isset($collectionDefinitions[$index])) {
$io->error('Unable to find index "'.$index.'" in collection definition (available : '.implode(', ', array_keys($collectionDefinitions)).')');

$data = [];
foreach ($entities as $entity) {
$data[] = $this->transformer->convert($entity);
return 2;
}
}

$io->text('Import <info>['.$collectionName.'] '.$class.'</info>');

$result = $this->documentManager->import($collectionName, $data, $action);

if ($this->printErrors($io, $result)) {
foreach ($indexes as $index) {
try {
$populated += $this->populateIndex($input, $output, $index);
} catch (\Throwable $e) {
$this->isError = true;
$io->error('Error happened during the import of the collection : '.$collectionName.' (you can see them with the option -v)');
$io->error($e->getMessage());

return 2;
}

$io->newLine();
}

$io->newLine();
Expand All @@ -113,6 +111,75 @@ protected function execute(InputInterface $input, OutputInterface $output): int
return 0;
}

private function populateIndex(InputInterface $input, OutputInterface $output, string $index)
{
$populated = 0;
$io = new SymfonyStyle($input, $output);

$collectionDefinitions = $this->collectionManager->getCollectionDefinitions();
$collectionDefinition = $collectionDefinitions[$index];
$action = $input->getOption('action');

$firstPage = $input->getOption('first-page');
$maxPerPage = $input->getOption('max-per-page');

$collectionName = $collectionDefinition['typesense_name'];
$class = $collectionDefinition['entity'];

$nbEntities = (int) $this->em->createQuery('select COUNT(u.id) from '.$class.' u')->getSingleScalarResult();

$nbPages = ceil($nbEntities / $maxPerPage);

if ($input->getOption('last-page')) {
$lastPage = $input->getOption('last-page');
if ($lastPage > $nbPages) {
throw new \Exception('The last-page option ('.$lastPage.') is bigger than the number of pages ('.$nbPages.')');
}
} else {
$lastPage = $nbPages;
}

if ($lastPage < $firstPage) {
throw new \Exception('The first-page option ('.$firstPage.') is bigger than the last-page option ('.$lastPage.')');
}

$io->text('<info>['.$collectionName.'] '.$class.'</info> '.$nbEntities.' entries to insert splited into '.$nbPages.' pages of '.$maxPerPage.' elements. Insertion from page '.$firstPage.' to '.$lastPage.'.');

for ($i = $firstPage; $i <= $lastPage; ++$i) {
$q = $this->em->createQuery('select e from '.$class.' e')
->setFirstResult(($i - 1) * $maxPerPage)
->setMaxResults($maxPerPage)
;

if ($io->isDebug()) {
$io->text('<info>Running request : </info>'.$q->getSQL());
}

$entities = $q->toIterable();

$data = [];
foreach ($entities as $entity) {
$data[] = $this->transformer->convert($entity);
}

$io->text('Import <info>['.$collectionName.'] '.$class.'</info> Page '.$i.' of '.$lastPage.' ('.count($data).' items)');

$result = $this->documentManager->import($collectionName, $data, $action);

if ($this->printErrors($io, $result)) {
$this->isError = true;

throw new \Exception('Error happened during the import of the collection : '.$collectionName.' (you can see them with the option -v)');
}

$populated += count($data);
}

$io->newLine();

return $populated;
}

private function printErrors(SymfonyStyle $io, array $result): bool
{
$isError = false;
Expand Down
9 changes: 6 additions & 3 deletions tests/Functional/AllowNullConnexionTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
use ACSEO\TypesenseBundle\Tests\Functional\Entity\Book;
use ACSEO\TypesenseBundle\Transformer\DoctrineToTypesenseTransformer;
use Doctrine\DBAL\Connection;
use Doctrine\ORM\AbstractQuery;
use Doctrine\ORM\Query;
use Doctrine\ORM\Configuration;
use Doctrine\ORM\EntityManager;
use Symfony\Bundle\FrameworkBundle\Test\KernelTestCase;
Expand Down Expand Up @@ -60,7 +60,7 @@ public function testImportCommand()

// the output of the command in the console
$output = $commandTester->getDisplay();
self::assertStringContainsString('Import [books]', $output);
self::assertStringContainsString('[books] ACSEO\TypesenseBundle\Tests\Functional\Entity\Book', $output);
self::assertStringContainsString('[OK] '.self::NB_BOOKS.' elements populated', $output);
}

Expand Down Expand Up @@ -196,9 +196,12 @@ private function getMockedEntityManager($books)
$configuration = $this->createMock(Configuration::class);
$connection->method('getConfiguration')->willReturn($configuration);

$query = $this->createMock(AbstractQuery::class);
$query = $this->createMock(Query::class);
$em->method('createQuery')->willReturn($query);

$query->method('setFirstResult')->willReturn($query);
$query->method('setMaxResults')->willReturn($query);

$query->method('getSingleScalarResult')->willReturn(self::NB_BOOKS);

$query->method('toIterable')->willReturn($books);
Expand Down
110 changes: 90 additions & 20 deletions tests/Functional/TypesenseInteractionsTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
use ACSEO\TypesenseBundle\Tests\Functional\Entity\Book;
use ACSEO\TypesenseBundle\Transformer\DoctrineToTypesenseTransformer;
use Doctrine\DBAL\Connection;
use Doctrine\ORM\AbstractQuery;
use Doctrine\ORM\Query;
use Doctrine\ORM\Configuration;
use Doctrine\ORM\EntityManager;
use Doctrine\ORM\Event\LifecycleEventArgs;
Expand Down Expand Up @@ -54,22 +54,67 @@ public function testCreateCommand()

/**
* @depends testCreateCommand
* @dataProvider importCommandProvider
*/
public function testImportCommand()
public function testImportCommand($nbBooks, $maxPerPage = null, $firstPage = null, $lastPage = null, $expectedCount = null)
{
$commandTester = $this->importCommandTester();
$commandTester->execute([]);
$commandTester = $this->importCommandTester([
'nbBooks' => $nbBooks,
'maxPerPage' => $maxPerPage,
'firstPage' => $firstPage,
'lastPage' => $lastPage
]);

$commandOptions = ['-vvv'];

if ($maxPerPage != null) {
$commandOptions['--max-per-page'] = $maxPerPage;
}
if ($firstPage != null) {
$commandOptions['--first-page'] = $firstPage;
}
if ($lastPage != null) {
$commandOptions['--last-page'] = $lastPage;
}

$commandTester->execute($commandOptions);

// the output of the command in the console
$output = $commandTester->getDisplay();
self::assertStringContainsString('Import [books]', $output);
self::assertStringContainsString('[OK] '.self::NB_BOOKS.' elements populated', $output);

self::assertStringContainsString(
sprintf('[books] ACSEO\TypesenseBundle\Tests\Functional\Entity\Book %s entries to insert', $nbBooks),
$output
);
self::assertStringContainsString(
sprintf('[OK] %s elements populated', $expectedCount ?? $nbBooks),
$output
);
}

public function importCommandProvider()
{
return [
"insert 10 books one by one" => [
10, 1
],
"insert 42 books 10 by 10" => [
42, 10
],
"insert 130 books 100 per 100" => [
130, null //100 is by defaut
],
"insert 498 books 50 per 50, from page 8 to 10 and expect 148 inserted" => [
498, 50, 8, 10, 148
]
];
}

/**
* @depends testImportCommand
* @dataProvider importCommandProvider
*/
public function testSearchByAuthor()
public function testSearchByAuthor($nbBooks)
{
$typeSenseClient = new TypesenseClient($_ENV['TYPESENSE_URL'], $_ENV['TYPESENSE_KEY']);
$collectionClient = new CollectionClient($typeSenseClient);
Expand All @@ -79,17 +124,24 @@ public function testSearchByAuthor()
$bookDefinition = $collectionDefinitions['books'];

$bookFinder = new CollectionFinder($collectionClient, $em, $bookDefinition);
$results = $bookFinder->rawQuery(new TypesenseQuery('Nicolas', 'author'))->getResults();
self::assertCount(self::NB_BOOKS, $results, "result doesn't contains ".self::NB_BOOKS.' elements');
$query = new TypesenseQuery('Nicolas', 'author');

$query->maxHits($nbBooks < 250 ? $nbBooks : 250);
$query->perPage($nbBooks < 250 ? $nbBooks : 250);

$results = $bookFinder->rawQuery($query)->getResults();

self::assertCount(($nbBooks < 250 ? $nbBooks : 250), $results, "result doesn't contains ".$nbBooks.' elements');
self::assertArrayHasKey('document', $results[0], "First item does not have the key 'document'");
self::assertArrayHasKey('highlights', $results[0], "First item does not have the key 'highlights'");
self::assertArrayHasKey('text_match', $results[0], "First item does not have the key 'text_match'");
}

/**
* @depends testImportCommand
* @dataProvider importCommandProvider
*/
public function testSearchByTitle()
public function testSearchByTitle($nbBooks)
{
$typeSenseClient = new TypesenseClient($_ENV['TYPESENSE_URL'], $_ENV['TYPESENSE_KEY']);
$collectionClient = new CollectionClient($typeSenseClient);
Expand Down Expand Up @@ -202,22 +254,22 @@ private function createCommandTester(): CommandTester
return new CommandTester($application->find('typesense:create'));
}

private function importCommandTester(): CommandTester
private function importCommandTester($options): CommandTester
{
$application = new Application();

$application->setAutoExit(false);

// Prepare all mocked objects required to run the command
$books = $this->getMockedBooks();
$books = $this->getMockedBooks($options);
$collectionDefinitions = $this->getCollectionDefinitions(Book::class);
$typeSenseClient = new TypesenseClient($_ENV['TYPESENSE_URL'], $_ENV['TYPESENSE_KEY']);
$propertyAccessor = PropertyAccess::createPropertyAccessor();
$collectionClient = new CollectionClient($typeSenseClient);
$transformer = new DoctrineToTypesenseTransformer($collectionDefinitions, $propertyAccessor);
$documentManager = new DocumentManager($typeSenseClient);
$collectionManager = new CollectionManager($collectionClient, $transformer, $collectionDefinitions);
$em = $this->getMockedEntityManager($books);
$em = $this->getMockedEntityManager($books, $options);

$command = new ImportCommand($em, $collectionManager, $documentManager, $transformer);

Expand Down Expand Up @@ -271,19 +323,20 @@ private function getCollectionDefinitions($entityClass)
];
}

private function getMockedBooks()
private function getMockedBooks($options)
{
$author = new Author('Nicolas Potier', 'France');
$books = [];

for ($i = 0; $i < self::NB_BOOKS; ++$i) {
$books[] = new Book($i, self::BOOK_TITLES[$i], $author, new \DateTime());
$nbBooks = $options['nbBooks'] ?? self::NB_BOOKS;
for ($i = 0; $i < $nbBooks; ++$i) {
$books[] = new Book($i, self::BOOK_TITLES[$i] ?? 'Book '.$i, $author, new \DateTime());
}

return $books;
}

private function getMockedEntityManager($books)
private function getMockedEntityManager($books, array $options = [])
{
$em = $this->createMock(EntityManager::class);

Expand All @@ -293,12 +346,29 @@ private function getMockedEntityManager($books)
$configuration = $this->createMock(Configuration::class);
$connection->method('getConfiguration')->willReturn($configuration);

$query = $this->createMock(AbstractQuery::class);
$query = $this->createMock(Query::class);
$em->method('createQuery')->willReturn($query);

$query->method('getSingleScalarResult')->willReturn(self::NB_BOOKS);
$query->method('getSingleScalarResult')->willReturn(count($books));

$query->method('setFirstResult')->willReturn($query);
$query->method('setMaxResults')->willReturn($query);

// Dirty Method to count number of call to the method toIterable in order to return
// the good results
$this->cptToIterableCall = isset($options['firstPage']) ? ($options['firstPage']-1) : 0;

$maxPerPage = $options['maxPerPage'] ?? 100;

$query->method('toIterable')->will($this->returnCallback(function() use ($books, $maxPerPage){
$result = array_slice($books,
$this->cptToIterableCall * $maxPerPage,
$maxPerPage
);
$this->cptToIterableCall++;

$query->method('toIterable')->willReturn($books);
return $result;
}));

return $em;
}
Expand Down
Loading

0 comments on commit 00a67b2

Please sign in to comment.