From 88574ffa0ff3e9db0f0dbb2b21ced7c2e5c63026 Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Fri, 19 Jan 2024 19:16:05 +0100 Subject: [PATCH 1/4] FEATURE: Content Repository status Resolves: #4468 --- .../DoctrineDbalContentGraphProjection.php | 41 ++++++++++-- .../DoctrineDbalContentGraphSchemaBuilder.php | 2 +- .../Projection/HypergraphProjection.php | 61 ++++++++++++----- .../Classes/ContentRepository.php | 14 ++++ .../Infrastructure/DbalCheckpointStorage.php | 67 +++++++++++++++---- .../Classes/Infrastructure/DbalSchemaDiff.php | 52 ++++++++++++++ .../Infrastructure/DbalSchemaFactory.php | 10 ++- .../Projection/CheckpointStorageInterface.php | 5 ++ .../Projection/CheckpointStorageStatus.php | 30 +++++++++ .../CheckpointStorageStatusType.php | 12 ++++ .../ContentGraph/ContentGraphProjection.php | 6 ++ .../ContentStream/ContentStreamProjection.php | 59 +++++++++++----- .../NodeHiddenStateProjection.php | 44 +++++++++--- .../Projection/ProjectionInterface.php | 5 ++ .../Classes/Projection/ProjectionStatus.php | 35 ++++++++++ .../Projection/ProjectionStatusType.php | 14 ++++ .../Classes/Projection/ProjectionStatuses.php | 39 +++++++++++ .../Workspace/WorkspaceProjection.php | 45 ++++++++++--- .../ContentRepositoryStatus.php | 30 +++++++++ .../Infrastructure/DbalSchemaDiffTest.php | 50 ++++++++++++++ .../Classes/Command/CrCommandController.php | 48 +++++++++++++ .../Projection/AssetUsageProjection.php | 23 +++++++ .../Projection/AssetUsageRepository.php | 34 ++++++---- .../Projection/DocumentUriPathProjection.php | 47 ++++++++++--- .../ChangeProjection.php | 59 +++++++++------- 25 files changed, 716 insertions(+), 116 deletions(-) create mode 100644 Neos.ContentRepository.Core/Classes/Infrastructure/DbalSchemaDiff.php create mode 100644 Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageStatus.php create mode 100644 Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageStatusType.php create mode 100644 Neos.ContentRepository.Core/Classes/Projection/ProjectionStatus.php create mode 100644 Neos.ContentRepository.Core/Classes/Projection/ProjectionStatusType.php create mode 100644 Neos.ContentRepository.Core/Classes/Projection/ProjectionStatuses.php create mode 100644 Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php create mode 100644 Neos.ContentRepository.Core/Tests/Unit/Infrastructure/DbalSchemaDiffTest.php diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php index 0502fa6b8fa..31b47e66a3c 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php @@ -6,7 +6,6 @@ use Doctrine\DBAL\Connection; use Doctrine\DBAL\Schema\AbstractSchemaManager; -use Doctrine\DBAL\Schema\Comparator; use Doctrine\DBAL\Types\Types; use Neos\ContentGraph\DoctrineDbalAdapter\Domain\Projection\Feature\NodeDisabling; use Neos\ContentGraph\DoctrineDbalAdapter\Domain\Projection\Feature\NodeMove; @@ -46,10 +45,13 @@ use Neos\ContentRepository\Core\Feature\RootNodeCreation\Event\RootNodeAggregateWithNodeWasCreated; use Neos\ContentRepository\Core\Infrastructure\DbalCheckpointStorage; use Neos\ContentRepository\Core\Infrastructure\DbalClientInterface; +use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; use Neos\ContentRepository\Core\NodeType\NodeTypeName; +use Neos\ContentRepository\Core\Projection\CheckpointStorageStatusType; use Neos\ContentRepository\Core\Projection\ContentGraph\Timestamps; use Neos\ContentRepository\Core\Projection\ProjectionInterface; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\Projection\WithMarkStaleInterface; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateClassification; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; @@ -108,24 +110,49 @@ protected function getTableNamePrefix(): string public function setUp(): void { - $this->setupTables(); + foreach ($this->determineRequiredSqlStatements() as $statement) { + $this->getDatabaseConnection()->executeStatement($statement); + } $this->checkpointStorage->setUp(); } - private function setupTables(): void + /** + * @return array + */ + private function determineRequiredSqlStatements(): array { $connection = $this->dbalClient->getConnection(); $schemaManager = $connection->getSchemaManager(); if (!$schemaManager instanceof AbstractSchemaManager) { throw new \RuntimeException('Failed to retrieve Schema Manager', 1625653914); } - $schema = (new DoctrineDbalContentGraphSchemaBuilder($this->tableNamePrefix))->buildSchema($schemaManager); + return DbalSchemaDiff::determineRequiredSqlStatements($connection, $schema); + } - $schemaDiff = (new Comparator())->compare($schemaManager->createSchema(), $schema); - foreach ($schemaDiff->toSaveSql($connection->getDatabasePlatform()) as $statement) { - $connection->executeStatement($statement); + public function status(): ProjectionStatus + { + $checkpointStorageStatus = $this->checkpointStorage->status(); + if ($checkpointStorageStatus->type === CheckpointStorageStatusType::ERROR) { + return ProjectionStatus::error($checkpointStorageStatus->details); + } + if ($checkpointStorageStatus->type === CheckpointStorageStatusType::SETUP_REQUIRED) { + return ProjectionStatus::setupRequired($checkpointStorageStatus->details); + } + try { + $this->getDatabaseConnection()->connect(); + } catch (\Throwable $e) { + return ProjectionStatus::error(sprintf('Failed to connect to database: %s', $e->getMessage())); + } + try { + $requiredSqlStatements = $this->determineRequiredSqlStatements(); + } catch (\Throwable $e) { + return ProjectionStatus::error(sprintf('Failed to determine required SQL statements: %s', $e->getMessage())); + } + if ($requiredSqlStatements !== []) { + return ProjectionStatus::setupRequired(sprintf('The following SQL statement%s required: %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', implode(chr(10), $requiredSqlStatements))); } + return ProjectionStatus::ok(); } public function reset(): void diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphSchemaBuilder.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphSchemaBuilder.php index 93bbb781c1b..8b4ac041215 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphSchemaBuilder.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphSchemaBuilder.php @@ -42,7 +42,7 @@ private function createNodeTable(): Table DbalSchemaFactory::columnForDimensionSpacePointHash('origindimensionspacepointhash')->setNotnull(false), DbalSchemaFactory::columnForNodeTypeName('nodetypename'), (new Column('properties', Type::getType(Types::TEXT)))->setNotnull(true)->setCustomSchemaOption('collation', self::DEFAULT_TEXT_COLLATION), - (new Column('classification', Type::getType(Types::STRING)))->setLength(20)->setNotnull(true)->setCustomSchemaOption('charset', 'binary'), + (new Column('classification', Type::getType(Types::BINARY)))->setLength(20)->setNotnull(true), (new Column('created', Type::getType(Types::DATETIME_IMMUTABLE)))->setDefault('CURRENT_TIMESTAMP')->setNotnull(true), (new Column('originalcreated', Type::getType(Types::DATETIME_IMMUTABLE)))->setDefault('CURRENT_TIMESTAMP')->setNotnull(true), (new Column('lastmodified', Type::getType(Types::DATETIME_IMMUTABLE)))->setNotnull(false)->setDefault(null), diff --git a/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php b/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php index 5932e236157..735fe6df329 100644 --- a/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php +++ b/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php @@ -16,7 +16,6 @@ use Doctrine\DBAL\Connection; use Doctrine\DBAL\Schema\AbstractSchemaManager; -use Doctrine\DBAL\Schema\Comparator; use Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection\Feature\ContentStreamForking; use Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection\Feature\NodeCreation; use Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection\Feature\NodeDisabling; @@ -46,8 +45,11 @@ use Neos\ContentRepository\Core\Feature\NodeVariation\Event\NodeSpecializationVariantWasCreated; use Neos\ContentRepository\Core\Feature\RootNodeCreation\Event\RootNodeAggregateWithNodeWasCreated; use Neos\ContentRepository\Core\Infrastructure\DbalCheckpointStorage; +use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; +use Neos\ContentRepository\Core\Projection\CheckpointStorageStatusType; use Neos\ContentRepository\Core\Projection\ProjectionInterface; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventEnvelope; @@ -95,11 +97,50 @@ public function __construct( public function setUp(): void { - $this->setupTables(); + foreach ($this->determineRequiredSqlStatements() as $statement) { + $this->getDatabaseConnection()->executeStatement($statement); + } + $this->getDatabaseConnection()->executeStatement(' + CREATE INDEX IF NOT EXISTS node_properties ON ' . $this->tableNamePrefix . '_node USING GIN(properties); + + create index if not exists hierarchy_children + on ' . $this->tableNamePrefix . '_hierarchyhyperrelation using gin (childnodeanchors); + + create index if not exists restriction_affected + on ' . $this->tableNamePrefix . '_restrictionhyperrelation using gin (affectednodeaggregateids); + '); $this->checkpointStorage->setUp(); } - private function setupTables(): void + public function status(): ProjectionStatus + { + $checkpointStorageStatus = $this->checkpointStorage->status(); + if ($checkpointStorageStatus->type === CheckpointStorageStatusType::ERROR) { + return ProjectionStatus::error($checkpointStorageStatus->details); + } + if ($checkpointStorageStatus->type === CheckpointStorageStatusType::SETUP_REQUIRED) { + return ProjectionStatus::setupRequired($checkpointStorageStatus->details); + } + try { + $this->getDatabaseConnection()->connect(); + } catch (\Throwable $e) { + return ProjectionStatus::error(sprintf('Failed to connect to database: %s', $e->getMessage())); + } + try { + $requiredSqlStatements = $this->determineRequiredSqlStatements(); + } catch (\Throwable $e) { + return ProjectionStatus::error(sprintf('Failed to determine required SQL statements: %s', $e->getMessage())); + } + if ($requiredSqlStatements !== []) { + return ProjectionStatus::setupRequired(sprintf('The following SQL statement%s required: %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', implode(chr(10), $requiredSqlStatements))); + } + return ProjectionStatus::ok(); + } + + /** + * @return array + */ + private function determineRequiredSqlStatements(): array { $connection = $this->databaseClient->getConnection(); HypergraphSchemaBuilder::registerTypes($connection->getDatabasePlatform()); @@ -109,19 +150,7 @@ private function setupTables(): void } $schema = (new HypergraphSchemaBuilder($this->tableNamePrefix))->buildSchema(); - $schemaDiff = (new Comparator())->compare($schemaManager->createSchema(), $schema); - foreach ($schemaDiff->toSaveSql($connection->getDatabasePlatform()) as $statement) { - $connection->executeStatement($statement); - } - $connection->executeStatement(' - CREATE INDEX IF NOT EXISTS node_properties ON ' . $this->tableNamePrefix . '_node USING GIN(properties); - - create index if not exists hierarchy_children - on ' . $this->tableNamePrefix . '_hierarchyhyperrelation using gin (childnodeanchors); - - create index if not exists restriction_affected - on ' . $this->tableNamePrefix . '_restrictionhyperrelation using gin (affectednodeaggregateids); - '); + return DbalSchemaDiff::determineRequiredSqlStatements($connection, $schema); } public function reset(): void diff --git a/Neos.ContentRepository.Core/Classes/ContentRepository.php b/Neos.ContentRepository.Core/Classes/ContentRepository.php index 61e6ff1533d..1ef14fb2c37 100644 --- a/Neos.ContentRepository.Core/Classes/ContentRepository.php +++ b/Neos.ContentRepository.Core/Classes/ContentRepository.php @@ -35,7 +35,9 @@ use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionsAndCatchUpHooks; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; +use Neos\ContentRepository\Core\Projection\ProjectionStatuses; use Neos\ContentRepository\Core\Projection\Workspace\WorkspaceFinder; +use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryStatus; use Neos\ContentRepository\Core\SharedModel\User\UserIdProviderInterface; use Neos\EventStore\EventStoreInterface; use Neos\EventStore\Model\Event\EventMetadata; @@ -199,6 +201,18 @@ public function setUp(): void } } + public function status(): ContentRepositoryStatus + { + $projectionStatuses = ProjectionStatuses::create(); + foreach ($this->projectionsAndCatchUpHooks->projections as $projectionClassName => $projection) { + $projectionStatuses = $projectionStatuses->with($projectionClassName, $projection->status()); + } + return new ContentRepositoryStatus( + $this->eventStore->status(), + $projectionStatuses, + ); + } + public function resetProjectionStates(): void { foreach ($this->projectionsAndCatchUpHooks->projections as $projection) { diff --git a/Neos.ContentRepository.Core/Classes/Infrastructure/DbalCheckpointStorage.php b/Neos.ContentRepository.Core/Classes/Infrastructure/DbalCheckpointStorage.php index d79396cc7c0..f21e0bad735 100644 --- a/Neos.ContentRepository.Core/Classes/Infrastructure/DbalCheckpointStorage.php +++ b/Neos.ContentRepository.Core/Classes/Infrastructure/DbalCheckpointStorage.php @@ -16,6 +16,12 @@ use Doctrine\DBAL\Schema\Schema; use Doctrine\DBAL\Types\Types; use Neos\ContentRepository\Core\Projection\CheckpointStorageInterface; +use Doctrine\DBAL\Schema\Column; +use Doctrine\DBAL\Schema\Table; +use Doctrine\DBAL\Types\Type; +use Doctrine\DBAL\Types\Types; +use Neos\ContentRepository\Core\Projection\CheckpointStorageInterface; +use Neos\ContentRepository\Core\Projection\CheckpointStorageStatus; use Neos\EventStore\Model\Event\SequenceNumber; /** @@ -44,18 +50,7 @@ public function __construct( public function setUp(): void { - $schemaManager = $this->connection->getSchemaManager(); - if (!$schemaManager instanceof AbstractSchemaManager) { - throw new \RuntimeException('Failed to retrieve Schema Manager', 1652269057); - } - $schema = new Schema(); - $table = $schema->createTable($this->tableName); - $table->addColumn('subscriberid', Types::STRING, ['length' => 255]); - $table->addColumn('appliedsequencenumber', Types::INTEGER); - $table->setPrimaryKey(['subscriberid']); - - $schemaDiff = (new Comparator())->compare($schemaManager->createSchema(), $schema); - foreach ($schemaDiff->toSaveSql($this->platform) as $statement) { + foreach ($this->determineRequiredSqlStatements() as $statement) { $this->connection->executeStatement($statement); } try { @@ -65,6 +60,32 @@ public function setUp(): void } } + public function status(): CheckpointStorageStatus + { + try { + $this->connection->connect(); + } catch (\Throwable $e) { + return CheckpointStorageStatus::error(sprintf('Failed to connect to database for subscriber "%s": %s', $this->subscriberId, $e->getMessage())); + } + try { + $requiredSqlStatements = $this->determineRequiredSqlStatements(); + } catch (\Throwable $e) { + return CheckpointStorageStatus::error(sprintf('Failed to compare database schema for subscriber "%s": %s', $this->subscriberId, $e->getMessage())); + } + if ($requiredSqlStatements !== []) { + return CheckpointStorageStatus::setupRequired(sprintf('The following SQL statement%s required for subscriber "%s": %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', $this->subscriberId, implode(chr(10), $requiredSqlStatements))); + } + try { + $appliedSequenceNumber = $this->connection->fetchOne('SELECT appliedsequencenumber FROM ' . $this->tableName . ' WHERE subscriberid = :subscriberId', ['subscriberId' => $this->subscriberId]); + } catch (\Throwable $e) { + return CheckpointStorageStatus::error(sprintf('Failed to determine initial applied sequence number for subscriber "%s": %s', $this->subscriberId, $e->getMessage())); + } + if ($appliedSequenceNumber === false) { + return CheckpointStorageStatus::setupRequired(sprintf('Initial initial applied sequence number not set for subscriber "%s"', $this->subscriberId)); + } + return CheckpointStorageStatus::ok(); + } + public function acquireLock(): SequenceNumber { if ($this->connection->isTransactionActive()) { @@ -122,4 +143,26 @@ public function getHighestAppliedSequenceNumber(): SequenceNumber return SequenceNumber::fromInteger((int)$highestAppliedSequenceNumber); } + // -------------- + + /** + * @return array + */ + private function determineRequiredSqlStatements(): array + { + $schemaManager = $this->connection->getSchemaManager(); + if (!$schemaManager instanceof AbstractSchemaManager) { + throw new \RuntimeException('Failed to retrieve Schema Manager', 1705681161); + } + $tableSchema = new Table( + $this->tableName, + [ + (new Column('subscriberid', Type::getType(Types::STRING)))->setLength(255), + (new Column('appliedsequencenumber', Type::getType(Types::INTEGER))) + ] + ); + $tableSchema->setPrimaryKey(['subscriberid']); + $schema = DbalSchemaFactory::createSchemaWithTables($schemaManager, [$tableSchema]); + return DbalSchemaDiff::determineRequiredSqlStatements($this->connection, $schema); + } } diff --git a/Neos.ContentRepository.Core/Classes/Infrastructure/DbalSchemaDiff.php b/Neos.ContentRepository.Core/Classes/Infrastructure/DbalSchemaDiff.php new file mode 100644 index 00000000000..1af625d9def --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Infrastructure/DbalSchemaDiff.php @@ -0,0 +1,52 @@ + Array of SQL statements that have to be executed in order to create/adjust the tables + */ + public static function determineRequiredSqlStatements(Connection $connection, Schema $schema): array + { + $schemaManager = $connection->getSchemaManager(); + if (!$schemaManager instanceof AbstractSchemaManager) { + throw new \RuntimeException('Failed to retrieve Schema Manager', 1705679142); + } + try { + $platform = $connection->getDatabasePlatform(); + } catch (Exception $e) { + throw new \RuntimeException(sprintf('Failed to retrieve Database platform: %s', $e->getMessage()), 1705679144, $e); + } + if ($platform === null) { // @phpstan-ignore-line This is not possible according to doc types, but there is no corresponding type hint in DBAL 2.x + throw new \RuntimeException('Failed to retrieve Database platform', 1705679147); + } + $fromTableSchemas = []; + foreach ($schema->getTables() as $tableSchema) { + if ($schemaManager->tablesExist([$tableSchema->getName()])) { + $fromTableSchemas[] = $schemaManager->listTableDetails($tableSchema->getName()); + } + } + $fromSchema = new Schema($fromTableSchemas, [], $schemaManager->createSchemaConfig()); + return (new Comparator())->compare($fromSchema, $schema)->toSql($platform); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Infrastructure/DbalSchemaFactory.php b/Neos.ContentRepository.Core/Classes/Infrastructure/DbalSchemaFactory.php index 458b4eeada1..7f4613e6600 100644 --- a/Neos.ContentRepository.Core/Classes/Infrastructure/DbalSchemaFactory.php +++ b/Neos.ContentRepository.Core/Classes/Infrastructure/DbalSchemaFactory.php @@ -23,6 +23,11 @@ */ final class DbalSchemaFactory { + // This class only contains static members and should not be constructed + private function __construct() + { + } + /** * The NodeAggregateId is limited to 64 ascii characters and therefore we should do the same in the database. * @@ -48,9 +53,8 @@ public static function columnForNodeAggregateId(string $columnName): Column */ public static function columnForContentStreamId(string $columnName): Column { - return (new Column($columnName, Type::getType(Types::STRING))) - ->setLength(36) - ->setCustomSchemaOption('charset', 'binary'); + return (new Column($columnName, Type::getType(Types::BINARY))) + ->setLength(36); } /** diff --git a/Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageInterface.php b/Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageInterface.php index 4ca7f5b4606..43dc37f8ab7 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageInterface.php @@ -32,6 +32,11 @@ interface CheckpointStorageInterface */ public function setUp(): void; + /** + * Retrieve the status of this checkpoint storage instance + */ + public function status(): CheckpointStorageStatus; + /** * Obtain an exclusive lock (to prevent multiple instances from being executed simultaneously) * and return the highest {@see SequenceNumber} that was processed by this checkpoint storage. diff --git a/Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageStatus.php b/Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageStatus.php new file mode 100644 index 00000000000..ab96579a447 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageStatus.php @@ -0,0 +1,30 @@ +projectionImplementation->setUp(); } + public function status(): ProjectionStatus + { + return $this->projectionImplementation->status(); + } + public function reset(): void { $this->projectionImplementation->reset(); diff --git a/Neos.ContentRepository.Core/Classes/Projection/ContentStream/ContentStreamProjection.php b/Neos.ContentRepository.Core/Classes/Projection/ContentStream/ContentStreamProjection.php index 54b806975e0..d6310dff799 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/ContentStream/ContentStreamProjection.php +++ b/Neos.ContentRepository.Core/Classes/Projection/ContentStream/ContentStreamProjection.php @@ -17,7 +17,6 @@ use Doctrine\DBAL\Connection; use Doctrine\DBAL\Schema\AbstractSchemaManager; use Doctrine\DBAL\Schema\Column; -use Doctrine\DBAL\Schema\Comparator; use Doctrine\DBAL\Schema\Table; use Doctrine\DBAL\Types\Type; use Doctrine\DBAL\Types\Types; @@ -35,16 +34,18 @@ use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasPublished; use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceRebaseFailed; use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceWasRebased; +use Neos\ContentRepository\Core\Infrastructure\DbalCheckpointStorage; use Neos\ContentRepository\Core\Infrastructure\DbalClientInterface; +use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaFactory; use Neos\ContentRepository\Core\Projection\CheckpointStorageInterface; -use Neos\ContentRepository\Core\Infrastructure\DbalCheckpointStorage; +use Neos\ContentRepository\Core\Projection\CheckpointStorageStatusType; use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventEnvelope; -use Neos\Neos\AssetUsage\Dto\AssetUsageFilter; /** * See {@see ContentStreamFinder} for explanation. @@ -74,11 +75,47 @@ public function __construct( public function setUp(): void { - $this->setupTables(); + $statements = $this->determineRequiredSqlStatements(); + // MIGRATIONS + if ($this->getDatabaseConnection()->getSchemaManager()->tablesExist([$this->tableName])) { + // added 2023-04-01 + $statements[] = sprintf("UPDATE %s SET state='FORKED' WHERE state='REBASING'; ", $this->tableName); + } + foreach ($statements as $statement) { + $this->getDatabaseConnection()->executeStatement($statement); + } $this->checkpointStorage->setUp(); } - private function setupTables(): void + public function status(): ProjectionStatus + { + $checkpointStorageStatus = $this->checkpointStorage->status(); + if ($checkpointStorageStatus->type === CheckpointStorageStatusType::ERROR) { + return ProjectionStatus::error($checkpointStorageStatus->details); + } + if ($checkpointStorageStatus->type === CheckpointStorageStatusType::SETUP_REQUIRED) { + return ProjectionStatus::setupRequired($checkpointStorageStatus->details); + } + try { + $this->getDatabaseConnection()->connect(); + } catch (\Throwable $e) { + return ProjectionStatus::error(sprintf('Failed to connect to database: %s', $e->getMessage())); + } + try { + $requiredSqlStatements = $this->determineRequiredSqlStatements(); + } catch (\Throwable $e) { + return ProjectionStatus::error(sprintf('Failed to determine required SQL statements: %s', $e->getMessage())); + } + if ($requiredSqlStatements !== []) { + return ProjectionStatus::setupRequired(sprintf('The following SQL statement%s required: %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', implode(chr(10), $requiredSqlStatements))); + } + return ProjectionStatus::ok(); + } + + /** + * @return array + */ + private function determineRequiredSqlStatements(): array { $connection = $this->dbalClient->getConnection(); $schemaManager = $connection->getSchemaManager(); @@ -86,13 +123,6 @@ private function setupTables(): void throw new \RuntimeException('Failed to retrieve Schema Manager', 1625653914); } - // MIGRATIONS - $currentSchema = $schemaManager->createSchema(); - if ($currentSchema->hasTable($this->tableName)) { - // added 2023-04-01 - $connection->executeStatement(sprintf("UPDATE %s SET state='FORKED' WHERE state='REBASING'; ", $this->tableName)); - } - $schema = DbalSchemaFactory::createSchemaWithTables($schemaManager, [ (new Table($this->tableName, [ DbalSchemaFactory::columnForContentStreamId('contentStreamId')->setNotnull(true), @@ -104,10 +134,7 @@ private function setupTables(): void ])) ]); - $schemaDiff = (new Comparator())->compare($schemaManager->createSchema(), $schema); - foreach ($schemaDiff->toSaveSql($connection->getDatabasePlatform()) as $statement) { - $connection->executeStatement($statement); - } + return DbalSchemaDiff::determineRequiredSqlStatements($connection, $schema); } public function reset(): void diff --git a/Neos.ContentRepository.Core/Classes/Projection/NodeHiddenState/NodeHiddenStateProjection.php b/Neos.ContentRepository.Core/Classes/Projection/NodeHiddenState/NodeHiddenStateProjection.php index 2e5a17ea9bb..350c88fcd65 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/NodeHiddenState/NodeHiddenStateProjection.php +++ b/Neos.ContentRepository.Core/Classes/Projection/NodeHiddenState/NodeHiddenStateProjection.php @@ -17,7 +17,6 @@ use Doctrine\DBAL\Connection; use Doctrine\DBAL\Schema\AbstractSchemaManager; use Doctrine\DBAL\Schema\Column; -use Doctrine\DBAL\Schema\Comparator; use Doctrine\DBAL\Schema\Table; use Doctrine\DBAL\Types\Type; use Doctrine\DBAL\Types\Types; @@ -28,8 +27,11 @@ use Neos\ContentRepository\Core\Feature\NodeDisabling\Event\NodeAggregateWasEnabled; use Neos\ContentRepository\Core\Infrastructure\DbalCheckpointStorage; use Neos\ContentRepository\Core\Infrastructure\DbalClientInterface; +use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaFactory; +use Neos\ContentRepository\Core\Projection\CheckpointStorageStatusType; use Neos\ContentRepository\Core\Projection\ProjectionInterface; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventEnvelope; @@ -56,11 +58,41 @@ public function __construct( public function setUp(): void { - $this->setupTables(); + foreach ($this->determineRequiredSqlStatements() as $statement) { + $this->getDatabaseConnection()->executeStatement($statement); + } $this->checkpointStorage->setUp(); } - private function setupTables(): void + public function status(): ProjectionStatus + { + $checkpointStorageStatus = $this->checkpointStorage->status(); + if ($checkpointStorageStatus->type === CheckpointStorageStatusType::ERROR) { + return ProjectionStatus::error($checkpointStorageStatus->details); + } + if ($checkpointStorageStatus->type === CheckpointStorageStatusType::SETUP_REQUIRED) { + return ProjectionStatus::setupRequired($checkpointStorageStatus->details); + } + try { + $this->getDatabaseConnection()->connect(); + } catch (\Throwable $e) { + return ProjectionStatus::error(sprintf('Failed to connect to database: %s', $e->getMessage())); + } + try { + $requiredSqlStatements = $this->determineRequiredSqlStatements(); + } catch (\Throwable $e) { + return ProjectionStatus::error(sprintf('Failed to determine required SQL statements: %s', $e->getMessage())); + } + if ($requiredSqlStatements !== []) { + return ProjectionStatus::setupRequired(sprintf('The following SQL statement%s required: %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', implode(chr(10), $requiredSqlStatements))); + } + return ProjectionStatus::ok(); + } + + /** + * @return array + */ + private function determineRequiredSqlStatements(): array { $connection = $this->dbalClient->getConnection(); $schemaManager = $connection->getSchemaManager(); @@ -80,11 +112,7 @@ private function setupTables(): void ); $schema = DbalSchemaFactory::createSchemaWithTables($schemaManager, [$nodeHiddenStateTable]); - - $schemaDiff = (new Comparator())->compare($schemaManager->createSchema(), $schema); - foreach ($schemaDiff->toSaveSql($connection->getDatabasePlatform()) as $statement) { - $connection->executeStatement($statement); - } + return DbalSchemaDiff::determineRequiredSqlStatements($connection, $schema); } public function reset(): void diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php index 8c722561d83..22c65ced9f7 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php @@ -25,6 +25,11 @@ interface ProjectionInterface */ public function setUp(): void; + /** + * Determines the status of the projection (not to confuse with {@see getState()}) + */ + public function status(): ProjectionStatus; + public function canHandle(EventInterface $event): bool; public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void; diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatus.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatus.php new file mode 100644 index 00000000000..00163041945 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatus.php @@ -0,0 +1,35 @@ + + */ +final readonly class ProjectionStatuses implements \IteratorAggregate +{ + /** + * @param array>, ProjectionStatus> $statuses + */ + private function __construct( + public array $statuses, + ) { + } + + public static function create(): self + { + return new self([]); + } + + /** + * @param class-string> $projectionClassName + */ + public function with(string $projectionClassName, ProjectionStatus $projectionStatus): self + { + $statuses = $this->statuses; + $statuses[$projectionClassName] = $projectionStatus; + return new self($statuses); + } + + + public function getIterator(): \Traversable + { + return new \ArrayIterator($this->statuses); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Projection/Workspace/WorkspaceProjection.php b/Neos.ContentRepository.Core/Classes/Projection/Workspace/WorkspaceProjection.php index 52c792466e7..8296e74a57e 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/Workspace/WorkspaceProjection.php +++ b/Neos.ContentRepository.Core/Classes/Projection/Workspace/WorkspaceProjection.php @@ -17,7 +17,6 @@ use Doctrine\DBAL\Connection; use Doctrine\DBAL\Schema\AbstractSchemaManager; use Doctrine\DBAL\Schema\Column; -use Doctrine\DBAL\Schema\Comparator; use Doctrine\DBAL\Schema\Table; use Doctrine\DBAL\Types\Type; use Doctrine\DBAL\Types\Types; @@ -36,8 +35,11 @@ use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceWasRebased; use Neos\ContentRepository\Core\Infrastructure\DbalCheckpointStorage; use Neos\ContentRepository\Core\Infrastructure\DbalClientInterface; +use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaFactory; +use Neos\ContentRepository\Core\Projection\CheckpointStorageStatusType; use Neos\ContentRepository\Core\Projection\ProjectionInterface; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\Projection\WithMarkStaleInterface; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; @@ -74,11 +76,41 @@ public function __construct( public function setUp(): void { - $this->setupTables(); + foreach ($this->determineRequiredSqlStatements() as $statement) { + $this->getDatabaseConnection()->executeStatement($statement); + } $this->checkpointStorage->setUp(); } - private function setupTables(): void + public function status(): ProjectionStatus + { + $checkpointStorageStatus = $this->checkpointStorage->status(); + if ($checkpointStorageStatus->type === CheckpointStorageStatusType::ERROR) { + return ProjectionStatus::error($checkpointStorageStatus->details); + } + if ($checkpointStorageStatus->type === CheckpointStorageStatusType::SETUP_REQUIRED) { + return ProjectionStatus::setupRequired($checkpointStorageStatus->details); + } + try { + $this->getDatabaseConnection()->connect(); + } catch (\Throwable $e) { + return ProjectionStatus::error(sprintf('Failed to connect to database: %s', $e->getMessage())); + } + try { + $requiredSqlStatements = $this->determineRequiredSqlStatements(); + } catch (\Throwable $e) { + return ProjectionStatus::error(sprintf('Failed to determine required SQL statements: %s', $e->getMessage())); + } + if ($requiredSqlStatements !== []) { + return ProjectionStatus::setupRequired(sprintf('The following SQL statement%s required: %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', implode(chr(10), $requiredSqlStatements))); + } + return ProjectionStatus::ok(); + } + + /** + * @return array + */ + private function determineRequiredSqlStatements(): array { $connection = $this->dbalClient->getConnection(); $schemaManager = $connection->getSchemaManager(); @@ -93,15 +125,12 @@ private function setupTables(): void (new Column('workspacedescription', Type::getType(Types::STRING)))->setLength(255)->setNotnull(true)->setCustomSchemaOption('collation', self::DEFAULT_TEXT_COLLATION), (new Column('workspaceowner', Type::getType(Types::STRING)))->setLength(255)->setNotnull(false)->setCustomSchemaOption('collation', self::DEFAULT_TEXT_COLLATION), DbalSchemaFactory::columnForContentStreamId('currentcontentstreamid')->setNotNull(true), - (new Column('status', Type::getType(Types::STRING)))->setLength(20)->setNotnull(false)->setCustomSchemaOption('charset', 'binary') + (new Column('status', Type::getType(Types::BINARY)))->setLength(20)->setNotnull(false) ]); $workspaceTable->setPrimaryKey(['workspacename']); $schema = DbalSchemaFactory::createSchemaWithTables($schemaManager, [$workspaceTable]); - $schemaDiff = (new Comparator())->compare($schemaManager->createSchema(), $schema); - foreach ($schemaDiff->toSaveSql($connection->getDatabasePlatform()) as $statement) { - $connection->executeStatement($statement); - } + return DbalSchemaDiff::determineRequiredSqlStatements($connection, $schema); } public function reset(): void diff --git a/Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php b/Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php new file mode 100644 index 00000000000..10b5a0f2d03 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php @@ -0,0 +1,30 @@ +connection = DriverManager::getConnection(['url' => 'sqlite:///:memory:']); + } + + public static function determineRequiredSqlStatements_dataProvider(): iterable + { + $tableSchema1 = new Table('some_table', [new Column('id', Type::getType(Types::INTEGER)), new Column('name', Type::getType(Types::STRING))]); + $tableSchema2 = new Table('some_other_table', [new Column('id', Type::getType(Types::STRING)), new Column('name', Type::getType(Types::STRING))]); + yield 'empty schema' => [new Schema(), [], 'expectedSqlStatements' => []]; + yield 'no existing tables' => [new Schema([$tableSchema1, $tableSchema2]), [], ['CREATE TABLE some_table (id INTEGER NOT NULL, name VARCHAR(255) NOT NULL)', 'CREATE TABLE some_other_table (id VARCHAR(255) NOT NULL, name VARCHAR(255) NOT NULL)']]; + yield 'one existing' => [new Schema([$tableSchema1, $tableSchema2]), ['CREATE TABLE some_table (id INTEGER NOT NULL, name VARCHAR(255) NOT NULL)'], ['CREATE TABLE some_other_table (id VARCHAR(255) NOT NULL, name VARCHAR(255) NOT NULL)']]; + yield 'one altered' => [new Schema([$tableSchema1]), ['CREATE TABLE some_table (id INTEGER NOT NULL)'], ['ALTER TABLE some_table ADD COLUMN name VARCHAR(255) NOT NULL']]; + yield 'one altered, one missing' => [new Schema([$tableSchema1, $tableSchema2]), ['CREATE TABLE some_table (id INTEGER NOT NULL)'], ['CREATE TABLE some_other_table (id VARCHAR(255) NOT NULL, name VARCHAR(255) NOT NULL)', 'ALTER TABLE some_table ADD COLUMN name VARCHAR(255) NOT NULL']]; + yield 'one altered, one up to date' => [new Schema([$tableSchema1, $tableSchema2]), ['CREATE TABLE some_table (id INTEGER NOT NULL)', 'CREATE TABLE some_other_table (id VARCHAR(255) NOT NULL, name VARCHAR(255) NOT NULL)'], ['ALTER TABLE some_table ADD COLUMN name VARCHAR(255) NOT NULL']]; + yield 'two altered' => [new Schema([$tableSchema1, $tableSchema2]), ['CREATE TABLE some_table (id INTEGER NOT NULL)', 'CREATE TABLE some_other_table (id VARCHAR(255) NOT NULL)'], ['ALTER TABLE some_table ADD COLUMN name VARCHAR(255) NOT NULL', 'ALTER TABLE some_other_table ADD COLUMN name VARCHAR(255) NOT NULL']]; + } + + /** + * @test + * @dataProvider determineRequiredSqlStatements_dataProvider + */ + public function determineRequiredSqlStatements_tests(Schema $schema, array $preTestSqlStatements, array $expectedSqlStatements): void + { + foreach ($preTestSqlStatements as $statement) { + $this->connection->executeStatement($statement); + } + $actualSqlStatements = DbalSchemaDiff::determineRequiredSqlStatements($this->connection, $schema); + self::assertSame($expectedSqlStatements, $actualSqlStatements); + } +} diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index 593904438c7..3c5b873cacc 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -5,9 +5,12 @@ use Neos\ContentRepository\Core\Factory\ContentRepositoryId; use Neos\ContentRepository\Core\Projection\CatchUpOptions; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; +use Neos\ContentRepository\Core\Projection\ProjectionStatusType; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; use Neos\ContentRepositoryRegistry\Service\ProjectionReplayServiceFactory; use Neos\EventStore\Model\Event\SequenceNumber; +use Neos\EventStore\Model\EventStore\StatusType; use Neos\Flow\Cli\CommandController; use Neos\ContentRepository\Core\Service\ContentStreamPrunerFactory; use Neos\ContentRepository\Core\Service\WorkspaceMaintenanceServiceFactory; @@ -38,6 +41,51 @@ public function setupCommand(string $contentRepository = 'default'): void $this->outputLine('Content Repository "%s" was set up', [$contentRepositoryId->value]); } + /** + * Determine and output the status of the event store and all registered projections for a given Content Repository + * + * @param string $contentRepository Identifier of the Content Repository to determine the status for + */ + public function statusCommand(string $contentRepository = 'default'): void + { + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + $status = $this->contentRepositoryRegistry->get($contentRepositoryId)->status(); + + $hasErrorsOrWarnings = false; + + $this->output('Event Store: '); + $this->outputLine(match ($status->eventStoreStatus->type) { + StatusType::OK => 'OK', + StatusType::SETUP_REQUIRED => 'Setup required!', + StatusType::ERROR => 'ERROR', + }); + if ($status->eventStoreStatus->details !== '') { + $this->outputFormatted($status->eventStoreStatus->details, [], 2); + } + if ($status->eventStoreStatus->type !== StatusType::OK) { + $hasErrorsOrWarnings = true; + } + $this->outputLine(); + foreach ($status->projectionStatuses as $projectionName => $projectionStatus) { + $this->output('Projection "%s": ', [$projectionName]); + $this->outputLine(match ($projectionStatus->type) { + ProjectionStatusType::OK => 'OK', + ProjectionStatusType::SETUP_REQUIRED => 'Setup required!', + ProjectionStatusType::REPLAY_REQUIRED => 'Replay required!', + ProjectionStatusType::ERROR => 'ERROR', + }); + if ($projectionStatus->type !== ProjectionStatusType::OK) { + $hasErrorsOrWarnings = true; + } + if ($projectionStatus->details !== '') { + $this->outputFormatted($projectionStatus->details, [], 2); + } + } + if ($hasErrorsOrWarnings) { + $this->quit(1); + } + } + /** * Replays the specified projection of a Content Repository by resetting its state and performing a full catchup * diff --git a/Neos.Neos/Classes/AssetUsage/Projection/AssetUsageProjection.php b/Neos.Neos/Classes/AssetUsage/Projection/AssetUsageProjection.php index 0238af9be13..b0be9199383 100644 --- a/Neos.Neos/Classes/AssetUsage/Projection/AssetUsageProjection.php +++ b/Neos.Neos/Classes/AssetUsage/Projection/AssetUsageProjection.php @@ -21,7 +21,9 @@ use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasPublished; use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceWasRebased; use Neos\ContentRepository\Core\Infrastructure\DbalCheckpointStorage; +use Neos\ContentRepository\Core\Projection\CheckpointStorageStatusType; use Neos\ContentRepository\Core\Projection\ProjectionInterface; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventEnvelope; use Neos\Media\Domain\Model\AssetInterface; @@ -30,6 +32,7 @@ use Neos\Media\Domain\Repository\AssetRepository; use Neos\Neos\AssetUsage\Dto\AssetIdAndOriginalAssetId; use Neos\Neos\AssetUsage\Dto\AssetIdsByProperty; +use Neos\Neos\AssetUsage\Dto\AssetUsageFilter; use Neos\Neos\AssetUsage\Dto\AssetUsageNodeAddress; use Neos\Utility\Exception\InvalidTypeException; use Neos\Utility\TypeHandling; @@ -235,6 +238,26 @@ public function setUp(): void $this->checkpointStorage->setUp(); } + public function status(): ProjectionStatus + { + $checkpointStorageStatus = $this->checkpointStorage->status(); + if ($checkpointStorageStatus->type === CheckpointStorageStatusType::ERROR) { + return ProjectionStatus::error($checkpointStorageStatus->details); + } + if ($checkpointStorageStatus->type === CheckpointStorageStatusType::SETUP_REQUIRED) { + return ProjectionStatus::setupRequired($checkpointStorageStatus->details); + } + try { + $this->repository->findUsages(AssetUsageFilter::create()); + } catch (\Throwable $e) { + return ProjectionStatus::error($e->getMessage()); + } + if ($this->repository->isSetupRequired()) { + return ProjectionStatus::setupRequired(); + } + return ProjectionStatus::ok(); + } + public function canHandle(EventInterface $event): bool { return in_array($event::class, [ diff --git a/Neos.Neos/Classes/AssetUsage/Projection/AssetUsageRepository.php b/Neos.Neos/Classes/AssetUsage/Projection/AssetUsageRepository.php index 12b2bcca192..b62e8b4f237 100644 --- a/Neos.Neos/Classes/AssetUsage/Projection/AssetUsageRepository.php +++ b/Neos.Neos/Classes/AssetUsage/Projection/AssetUsageRepository.php @@ -16,6 +16,11 @@ use Doctrine\DBAL\Types\Type; use Doctrine\DBAL\Types\Types; use Neos\ContentRepository\Core\DimensionSpace\DimensionSpacePointSet; +use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; +use Neos\ContentRepository\Core\Infrastructure\DbalSchemaFactory; +use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; +use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; +use Neos\Neos\AssetUsage\Dto\AssetUsageNodeAddress; use Neos\ContentRepository\Core\DimensionSpace\OriginDimensionSpacePoint; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaFactory; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; @@ -40,21 +45,23 @@ public function __construct( public function setUp(): void { - $schemaManager = $this->dbal->getSchemaManager(); - if (!$schemaManager instanceof AbstractSchemaManager) { - throw new \RuntimeException('Failed to retrieve Schema Manager', 1625653914); - } - - $schema = DbalSchemaFactory::createSchemaWithTables($schemaManager, [self::databaseSchema($this->tableNamePrefix)]); - $schemaDiff = (new Comparator())->compare($schemaManager->createSchema(), $schema); - foreach ($schemaDiff->toSaveSql($this->dbal->getDatabasePlatform()) as $statement) { + foreach (DbalSchemaDiff::determineRequiredSqlStatements($this->dbal, $this->databaseSchema()) as $statement) { $this->dbal->executeStatement($statement); } } - private static function databaseSchema(string $tablePrefix): Table + public function isSetupRequired(): bool { - $table = new Table($tablePrefix, [ + return DbalSchemaDiff::determineRequiredSqlStatements($this->dbal, $this->databaseSchema()) !== []; + } + + private function databaseSchema(): Schema + { + $schemaManager = $this->dbal->getSchemaManager(); + if (!$schemaManager instanceof AbstractSchemaManager) { + throw new \RuntimeException('Failed to retrieve Schema Manager', 1625653914); + } + $table = new Table($this->tableNamePrefix, [ (new Column('assetid', Type::getType(Types::STRING)))->setLength(40)->setNotnull(true)->setDefault(''), (new Column('originalassetid', Type::getType(Types::STRING)))->setLength(40)->setNotnull(false)->setDefault(null), DbalSchemaFactory::columnForContentStreamId('contentstreamid')->setNotNull(true), @@ -64,13 +71,16 @@ private static function databaseSchema(string $tablePrefix): Table (new Column('propertyname', Type::getType(Types::STRING)))->setLength(255)->setNotnull(true)->setDefault('') ]); - return $table + $table ->addUniqueIndex(['assetid', 'originalassetid', 'contentstreamid', 'nodeaggregateid', 'origindimensionspacepointhash', 'propertyname'], 'assetperproperty') ->addIndex(['assetid']) ->addIndex(['originalassetid']) ->addIndex(['contentstreamid']) ->addIndex(['nodeaggregateid']) - ->addIndex(['origindimensionspacepointhash']); + ->addIndex(['origindimensionspacepointhash'], options: ['lengths' => [768]]); + ; + + return DbalSchemaFactory::createSchemaWithTables($schemaManager, [$table]); } public function findUsages(AssetUsageFilter $filter): AssetUsages diff --git a/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php b/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php index 9185975b296..75b91a8e521 100644 --- a/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php +++ b/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php @@ -7,7 +7,6 @@ use Doctrine\DBAL\Connection; use Doctrine\DBAL\DBALException; use Doctrine\DBAL\Schema\AbstractSchemaManager; -use Doctrine\DBAL\Schema\Comparator; use Doctrine\DBAL\Types\Types; use Neos\ContentRepository\Core\DimensionSpace\DimensionSpacePointSet; use Neos\ContentRepository\Core\DimensionSpace\OriginDimensionSpacePoint; @@ -31,9 +30,12 @@ use Neos\ContentRepository\Core\Feature\RootNodeCreation\Event\RootNodeAggregateWithNodeWasCreated; use Neos\ContentRepository\Core\Feature\WorkspaceCreation\Event\RootWorkspaceWasCreated; use Neos\ContentRepository\Core\Infrastructure\DbalCheckpointStorage; +use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; use Neos\ContentRepository\Core\NodeType\NodeTypeName; +use Neos\ContentRepository\Core\Projection\CheckpointStorageStatusType; use Neos\ContentRepository\Core\Projection\ProjectionInterface; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\Projection\WithMarkStaleInterface; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\EventStore\Model\Event\SequenceNumber; @@ -73,23 +75,48 @@ public function __construct( public function setUp(): void { - $this->setupTables(); + foreach ($this->determineRequiredSqlStatements() as $statement) { + $this->dbal->executeStatement($statement); + } $this->checkpointStorage->setUp(); } - private function setupTables(): void + public function status(): ProjectionStatus { - $connection = $this->dbal; - $schemaManager = $connection->getSchemaManager(); + $checkpointStorageStatus = $this->checkpointStorage->status(); + if ($checkpointStorageStatus->type === CheckpointStorageStatusType::ERROR) { + return ProjectionStatus::error($checkpointStorageStatus->details); + } + if ($checkpointStorageStatus->type === CheckpointStorageStatusType::SETUP_REQUIRED) { + return ProjectionStatus::setupRequired($checkpointStorageStatus->details); + } + try { + $this->dbal->connect(); + } catch (\Throwable $e) { + return ProjectionStatus::error(sprintf('Failed to connect to database: %s', $e->getMessage())); + } + try { + $requiredSqlStatements = $this->determineRequiredSqlStatements(); + } catch (\Throwable $e) { + return ProjectionStatus::error(sprintf('Failed to determine required SQL statements: %s', $e->getMessage())); + } + if ($requiredSqlStatements !== []) { + return ProjectionStatus::setupRequired(sprintf('The following SQL statement%s required: %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', implode(chr(10), $requiredSqlStatements))); + } + return ProjectionStatus::ok(); + } + + /** + * @return array + */ + private function determineRequiredSqlStatements(): array + { + $schemaManager = $this->dbal->getSchemaManager(); if (!$schemaManager instanceof AbstractSchemaManager) { throw new \RuntimeException('Failed to retrieve Schema Manager', 1625653914); } $schema = (new DocumentUriPathSchemaBuilder($this->tableNamePrefix))->buildSchema($schemaManager); - - $schemaDiff = (new Comparator())->compare($schemaManager->createSchema(), $schema); - foreach ($schemaDiff->toSaveSql($connection->getDatabasePlatform()) as $statement) { - $connection->executeStatement($statement); - } + return DbalSchemaDiff::determineRequiredSqlStatements($this->dbal, $schema); } diff --git a/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php b/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php index 64218c28e85..702e54d9116 100644 --- a/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php +++ b/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php @@ -18,7 +18,6 @@ use Doctrine\DBAL\DBALException; use Doctrine\DBAL\Schema\AbstractSchemaManager; use Doctrine\DBAL\Schema\Column; -use Doctrine\DBAL\Schema\Comparator; use Doctrine\DBAL\Schema\Table; use Doctrine\DBAL\Types\Type; use Doctrine\DBAL\Types\Types; @@ -38,8 +37,11 @@ use Neos\ContentRepository\Core\Feature\WorkspaceCreation\Event\RootWorkspaceWasCreated; use Neos\ContentRepository\Core\Infrastructure\DbalCheckpointStorage; use Neos\ContentRepository\Core\Infrastructure\DbalClientInterface; +use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaFactory; +use Neos\ContentRepository\Core\Projection\CheckpointStorageStatusType; use Neos\ContentRepository\Core\Projection\ProjectionInterface; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; use Neos\EventStore\Model\Event\SequenceNumber; @@ -80,11 +82,41 @@ public function __construct( public function setUp(): void { - $this->setupTables(); + foreach ($this->determineRequiredSqlStatements() as $statement) { + $this->getDatabaseConnection()->executeStatement($statement); + } $this->checkpointStorage->setUp(); } - private function setupTables(): void + public function status(): ProjectionStatus + { + $checkpointStorageStatus = $this->checkpointStorage->status(); + if ($checkpointStorageStatus->type === CheckpointStorageStatusType::ERROR) { + return ProjectionStatus::error($checkpointStorageStatus->details); + } + if ($checkpointStorageStatus->type === CheckpointStorageStatusType::SETUP_REQUIRED) { + return ProjectionStatus::setupRequired($checkpointStorageStatus->details); + } + try { + $this->getDatabaseConnection()->connect(); + } catch (\Throwable $e) { + return ProjectionStatus::error(sprintf('Failed to connect to database: %s', $e->getMessage())); + } + try { + $requiredSqlStatements = $this->determineRequiredSqlStatements(); + } catch (\Throwable $e) { + return ProjectionStatus::error(sprintf('Failed to determine required SQL statements: %s', $e->getMessage())); + } + if ($requiredSqlStatements !== []) { + return ProjectionStatus::setupRequired(sprintf('The following SQL statement%s required: %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', implode(chr(10), $requiredSqlStatements))); + } + return ProjectionStatus::ok(); + } + + /** + * @return array + */ + private function determineRequiredSqlStatements(): array { $connection = $this->dbalClient->getConnection(); $schemaManager = $connection->getSchemaManager(); @@ -92,21 +124,6 @@ private function setupTables(): void throw new \RuntimeException('Failed to retrieve Schema Manager', 1625653914); } - // MIGRATIONS - $currentSchema = $schemaManager->createSchema(); - if ($currentSchema->hasTable($this->tableNamePrefix)) { - $tableSchema = $currentSchema->getTable($this->tableNamePrefix); - // added 2023-03-18 - if ($tableSchema->hasColumn('nodeAggregateIdentifier')) { - // table in old format -> we migrate to new. - $connection->executeStatement(sprintf('ALTER TABLE %s CHANGE nodeAggregateIdentifier nodeAggregateId VARCHAR(255); ', $this->tableNamePrefix)); - } - // added 2023-03-18 - if ($tableSchema->hasColumn('contentStreamIdentifier')) { - $connection->executeStatement(sprintf('ALTER TABLE %s CHANGE contentStreamIdentifier contentStreamId VARCHAR(255); ', $this->tableNamePrefix)); - } - } - $changeTable = new Table($this->tableNamePrefix, [ DbalSchemaFactory::columnForContentStreamId('contentStreamId')->setNotNull(true), (new Column('created', Type::getType(Types::BOOLEAN)))->setNotnull(true), @@ -133,11 +150,7 @@ private function setupTables(): void $liveContentStreamsTable->setPrimaryKey(['contentstreamid']); $schema = DbalSchemaFactory::createSchemaWithTables($schemaManager, [$changeTable, $liveContentStreamsTable]); - - $schemaDiff = (new Comparator())->compare($schemaManager->createSchema(), $schema); - foreach ($schemaDiff->toSaveSql($connection->getDatabasePlatform()) as $statement) { - $connection->executeStatement($statement); - } + return DbalSchemaDiff::determineRequiredSqlStatements($connection, $schema); } public function reset(): void From cd893adebcc44031da2f3769c04387ebf3d8ada4 Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Sun, 21 Jan 2024 15:30:33 +0100 Subject: [PATCH 2/4] Omit db schema changes Those are extracted to #4847 --- .../src/DoctrineDbalContentGraphSchemaBuilder.php | 2 +- .../Classes/Infrastructure/DbalSchemaFactory.php | 10 +++------- .../Projection/Workspace/WorkspaceProjection.php | 2 +- .../AssetUsage/Projection/AssetUsageRepository.php | 10 +++------- 4 files changed, 8 insertions(+), 16 deletions(-) diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphSchemaBuilder.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphSchemaBuilder.php index 8b4ac041215..93bbb781c1b 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphSchemaBuilder.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphSchemaBuilder.php @@ -42,7 +42,7 @@ private function createNodeTable(): Table DbalSchemaFactory::columnForDimensionSpacePointHash('origindimensionspacepointhash')->setNotnull(false), DbalSchemaFactory::columnForNodeTypeName('nodetypename'), (new Column('properties', Type::getType(Types::TEXT)))->setNotnull(true)->setCustomSchemaOption('collation', self::DEFAULT_TEXT_COLLATION), - (new Column('classification', Type::getType(Types::BINARY)))->setLength(20)->setNotnull(true), + (new Column('classification', Type::getType(Types::STRING)))->setLength(20)->setNotnull(true)->setCustomSchemaOption('charset', 'binary'), (new Column('created', Type::getType(Types::DATETIME_IMMUTABLE)))->setDefault('CURRENT_TIMESTAMP')->setNotnull(true), (new Column('originalcreated', Type::getType(Types::DATETIME_IMMUTABLE)))->setDefault('CURRENT_TIMESTAMP')->setNotnull(true), (new Column('lastmodified', Type::getType(Types::DATETIME_IMMUTABLE)))->setNotnull(false)->setDefault(null), diff --git a/Neos.ContentRepository.Core/Classes/Infrastructure/DbalSchemaFactory.php b/Neos.ContentRepository.Core/Classes/Infrastructure/DbalSchemaFactory.php index 7f4613e6600..458b4eeada1 100644 --- a/Neos.ContentRepository.Core/Classes/Infrastructure/DbalSchemaFactory.php +++ b/Neos.ContentRepository.Core/Classes/Infrastructure/DbalSchemaFactory.php @@ -23,11 +23,6 @@ */ final class DbalSchemaFactory { - // This class only contains static members and should not be constructed - private function __construct() - { - } - /** * The NodeAggregateId is limited to 64 ascii characters and therefore we should do the same in the database. * @@ -53,8 +48,9 @@ public static function columnForNodeAggregateId(string $columnName): Column */ public static function columnForContentStreamId(string $columnName): Column { - return (new Column($columnName, Type::getType(Types::BINARY))) - ->setLength(36); + return (new Column($columnName, Type::getType(Types::STRING))) + ->setLength(36) + ->setCustomSchemaOption('charset', 'binary'); } /** diff --git a/Neos.ContentRepository.Core/Classes/Projection/Workspace/WorkspaceProjection.php b/Neos.ContentRepository.Core/Classes/Projection/Workspace/WorkspaceProjection.php index 8296e74a57e..d818d3ef935 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/Workspace/WorkspaceProjection.php +++ b/Neos.ContentRepository.Core/Classes/Projection/Workspace/WorkspaceProjection.php @@ -125,7 +125,7 @@ private function determineRequiredSqlStatements(): array (new Column('workspacedescription', Type::getType(Types::STRING)))->setLength(255)->setNotnull(true)->setCustomSchemaOption('collation', self::DEFAULT_TEXT_COLLATION), (new Column('workspaceowner', Type::getType(Types::STRING)))->setLength(255)->setNotnull(false)->setCustomSchemaOption('collation', self::DEFAULT_TEXT_COLLATION), DbalSchemaFactory::columnForContentStreamId('currentcontentstreamid')->setNotNull(true), - (new Column('status', Type::getType(Types::BINARY)))->setLength(20)->setNotnull(false) + (new Column('status', Type::getType(Types::STRING)))->setLength(20)->setNotnull(false)->setCustomSchemaOption('charset', 'binary') ]); $workspaceTable->setPrimaryKey(['workspacename']); diff --git a/Neos.Neos/Classes/AssetUsage/Projection/AssetUsageRepository.php b/Neos.Neos/Classes/AssetUsage/Projection/AssetUsageRepository.php index b62e8b4f237..df326cace18 100644 --- a/Neos.Neos/Classes/AssetUsage/Projection/AssetUsageRepository.php +++ b/Neos.Neos/Classes/AssetUsage/Projection/AssetUsageRepository.php @@ -11,17 +11,13 @@ use Doctrine\DBAL\Platforms\AbstractPlatform; use Doctrine\DBAL\Schema\AbstractSchemaManager; use Doctrine\DBAL\Schema\Column; -use Doctrine\DBAL\Schema\Comparator; +use Doctrine\DBAL\Schema\Schema; use Doctrine\DBAL\Schema\Table; use Doctrine\DBAL\Types\Type; use Doctrine\DBAL\Types\Types; use Neos\ContentRepository\Core\DimensionSpace\DimensionSpacePointSet; -use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; -use Neos\ContentRepository\Core\Infrastructure\DbalSchemaFactory; -use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; -use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; -use Neos\Neos\AssetUsage\Dto\AssetUsageNodeAddress; use Neos\ContentRepository\Core\DimensionSpace\OriginDimensionSpacePoint; +use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaFactory; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; @@ -77,7 +73,7 @@ private function databaseSchema(): Schema ->addIndex(['originalassetid']) ->addIndex(['contentstreamid']) ->addIndex(['nodeaggregateid']) - ->addIndex(['origindimensionspacepointhash'], options: ['lengths' => [768]]); + ->addIndex(['origindimensionspacepointhash']); ; return DbalSchemaFactory::createSchemaWithTables($schemaManager, [$table]); From ff9bea94bd492c9609680b6c03fae92c9e38a007 Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Sun, 21 Jan 2024 15:33:10 +0100 Subject: [PATCH 3/4] Fix namespace imports in DbalCheckpointStorage --- .../Classes/Infrastructure/DbalCheckpointStorage.php | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Infrastructure/DbalCheckpointStorage.php b/Neos.ContentRepository.Core/Classes/Infrastructure/DbalCheckpointStorage.php index f21e0bad735..d9d00aaccf7 100644 --- a/Neos.ContentRepository.Core/Classes/Infrastructure/DbalCheckpointStorage.php +++ b/Neos.ContentRepository.Core/Classes/Infrastructure/DbalCheckpointStorage.php @@ -12,10 +12,6 @@ use Doctrine\DBAL\Platforms\MySqlPlatform; use Doctrine\DBAL\Platforms\PostgreSqlPlatform; use Doctrine\DBAL\Schema\AbstractSchemaManager; -use Doctrine\DBAL\Schema\Comparator; -use Doctrine\DBAL\Schema\Schema; -use Doctrine\DBAL\Types\Types; -use Neos\ContentRepository\Core\Projection\CheckpointStorageInterface; use Doctrine\DBAL\Schema\Column; use Doctrine\DBAL\Schema\Table; use Doctrine\DBAL\Types\Type; From 543afbf1449816dda17536e3c46bbed2c443519d Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Sun, 21 Jan 2024 15:38:50 +0100 Subject: [PATCH 4/4] Add versbose and quiet flags to cr:setup CLI command --- .../Classes/Command/CrCommandController.php | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index 3c5b873cacc..4f45a632e49 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -14,6 +14,7 @@ use Neos\Flow\Cli\CommandController; use Neos\ContentRepository\Core\Service\ContentStreamPrunerFactory; use Neos\ContentRepository\Core\Service\WorkspaceMaintenanceServiceFactory; +use Symfony\Component\Console\Output\Output; final class CrCommandController extends CommandController { @@ -45,9 +46,14 @@ public function setupCommand(string $contentRepository = 'default'): void * Determine and output the status of the event store and all registered projections for a given Content Repository * * @param string $contentRepository Identifier of the Content Repository to determine the status for + * @param bool $verbose If set, more details will be shown + * @param bool $quiet If set, no output is generated. This is useful if only the exit code (0 = all OK, 1 = errors or warnings) is of interest */ - public function statusCommand(string $contentRepository = 'default'): void + public function statusCommand(string $contentRepository = 'default', bool $verbose = false, bool $quiet = false): void { + if ($quiet) { + $this->output->getOutput()->setVerbosity(Output::VERBOSITY_QUIET); + } $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); $status = $this->contentRepositoryRegistry->get($contentRepositoryId)->status(); @@ -59,7 +65,7 @@ public function statusCommand(string $contentRepository = 'default'): void StatusType::SETUP_REQUIRED => 'Setup required!', StatusType::ERROR => 'ERROR', }); - if ($status->eventStoreStatus->details !== '') { + if ($verbose && $status->eventStoreStatus->details !== '') { $this->outputFormatted($status->eventStoreStatus->details, [], 2); } if ($status->eventStoreStatus->type !== StatusType::OK) { @@ -77,7 +83,7 @@ public function statusCommand(string $contentRepository = 'default'): void if ($projectionStatus->type !== ProjectionStatusType::OK) { $hasErrorsOrWarnings = true; } - if ($projectionStatus->details !== '') { + if ($verbose && $projectionStatus->details !== '') { $this->outputFormatted($projectionStatus->details, [], 2); } }