Merge branch 'release/v2.1.1'

roadiz · Mar 13, 2023 · b35e2e6 · b35e2e6
2 parents 4881d22 + 4b03c69
commit b35e2e6
Show file tree

Hide file tree

Showing 19 changed files with 133 additions and 59 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,16 @@
+## [v2.1.1](https://github.com/roadiz/core-bundle-dev-app/compare/v2.1.0...v2.1.1) (2023-03-13)
+
+
+### ⚠ BREAKING CHANGES
+
+* All Solr documents MUST have a idempotent ID field to ease up Solr indexing (no more need to delete previous index entry before indexing an entity).
+
+### Features
+
+* Added `NodesSourcesHeadFactoryInterface` for better WebResponse and CommonContent responses maintainability. ([ed05a24](https://github.com/roadiz/core-bundle-dev-app/commit/ed05a24947da4caa5533b37190c480b0b5358bd5))
+* Rename `getCompositeIdentifier` to `getIdempotentIdentifier` and provided a fallback. ([e8c895a](https://github.com/roadiz/core-bundle-dev-app/commit/e8c895a056dee0f2668ed8d081a2021a45490174))
+* Updated Solr indexing tags fields for multivalued strings and use composite ID for easy overriding ([50a04af](https://github.com/roadiz/core-bundle-dev-app/commit/50a04afc913eb1a7b67cd550fc39305598c4db19))
+
 ## v2.1.0 (2023-03-06)
 
 ### ⚠ BREAKING CHANGES

diff --git a/lib/RoadizCoreBundle/config/services.yaml b/lib/RoadizCoreBundle/config/services.yaml
@@ -22,9 +22,9 @@ services:
         autowire: true      # Automatically injects dependencies in your services.
         autoconfigure: true # Automatically registers your services as commands, event subscribers, etc.
         bind:
-            $cmsVersion: '2.1.0'
+            $cmsVersion: '2.1.1'
             $appVersion: '%roadiz_core.app_version%'
-            $cmsVersionPrefix: 'develop'
+            $cmsVersionPrefix: 'main'
             $staticDomain: '%roadiz_core.static_domain_name%'
             $hideRoadizVersion: '%roadiz_core.hide_roadiz_version%'
             $inheritanceType: '%roadiz_core.inheritance_type%'

diff --git a/lib/RoadizCoreBundle/src/Api/DataTransformer/WebResponseOutputDataTransformer.php b/lib/RoadizCoreBundle/src/Api/DataTransformer/WebResponseOutputDataTransformer.php
@@ -8,7 +8,7 @@
 use RZ\Roadiz\Core\AbstractEntities\PersistableInterface;
 use RZ\Roadiz\Core\AbstractEntities\TranslationInterface;
 use RZ\Roadiz\CoreBundle\Api\Breadcrumbs\BreadcrumbsFactoryInterface;
-use RZ\Roadiz\CoreBundle\Api\Model\NodesSourcesHeadFactory;
+use RZ\Roadiz\CoreBundle\Api\Model\NodesSourcesHeadFactoryInterface;
 use RZ\Roadiz\CoreBundle\Api\Model\WebResponse;
 use RZ\Roadiz\CoreBundle\Api\Model\WebResponseInterface;
 use RZ\Roadiz\CoreBundle\Api\TreeWalker\AutoChildrenNodeSourceWalker;
@@ -23,15 +23,15 @@ final class WebResponseOutputDataTransformer implements WebResponseDataTransform
     use BlocksAwareWebResponseOutputDataTransformerTrait;
     use RealmsAwareWebResponseOutputDataTransformerTrait;
 
-    private NodesSourcesHeadFactory $nodesSourcesHeadFactory;
+    private NodesSourcesHeadFactoryInterface $nodesSourcesHeadFactory;
     private BreadcrumbsFactoryInterface $breadcrumbsFactory;
     private WalkerContextInterface $walkerContext;
     private CacheItemPoolInterface $cacheItemPool;
     private UrlGeneratorInterface $urlGenerator;
     private RealmResolverInterface $realmResolver;
 
     public function __construct(
-        NodesSourcesHeadFactory $nodesSourcesHeadFactory,
+        NodesSourcesHeadFactoryInterface $nodesSourcesHeadFactory,
         BreadcrumbsFactoryInterface $breadcrumbsFactory,
         WalkerContextInterface $walkerContext,
         CacheItemPoolInterface $cacheItemPool,

diff --git a/lib/RoadizCoreBundle/src/Api/Model/NodesSourcesHead.php b/lib/RoadizCoreBundle/src/Api/Model/NodesSourcesHead.php
@@ -80,6 +80,15 @@ public function getGoogleTagManager(): ?string
         return $this->settingsBag->get('google_tag_manager_id', null) ?? null;
     }
 
+    /**
+     * @return string|null
+     */
+    #[Serializer\Groups(["web_response", "nodes_sources_single", "walker"])]
+    public function getMatomoTagManager(): ?string
+    {
+        return $this->settingsBag->get('matomo_tag_manager_id', null) ?? null;
+    }
+
     /**
      * @return string|null
      */

diff --git a/lib/RoadizCoreBundle/src/Api/Model/NodesSourcesHeadFactory.php b/lib/RoadizCoreBundle/src/Api/Model/NodesSourcesHeadFactory.php
@@ -11,7 +11,7 @@
 use RZ\Roadiz\CoreBundle\EntityApi\NodeSourceApi;
 use Symfony\Component\Routing\Generator\UrlGeneratorInterface;
 
-final class NodesSourcesHeadFactory
+final class NodesSourcesHeadFactory implements NodesSourcesHeadFactoryInterface
 {
     private Settings $settingsBag;
     private UrlGeneratorInterface $urlGenerator;

diff --git a/lib/RoadizCoreBundle/src/Api/Model/NodesSourcesHeadFactoryInterface.php b/lib/RoadizCoreBundle/src/Api/Model/NodesSourcesHeadFactoryInterface.php
@@ -0,0 +1,14 @@
+<?php
+
+declare(strict_types=1);
+
+namespace RZ\Roadiz\CoreBundle\Api\Model;
+
+use RZ\Roadiz\Core\AbstractEntities\TranslationInterface;
+use RZ\Roadiz\CoreBundle\Entity\NodesSources;
+
+interface NodesSourcesHeadFactoryInterface
+{
+    public function createForTranslation(TranslationInterface $translation): NodesSourcesHeadInterface;
+    public function createForNodeSource(NodesSources $nodesSources): NodesSourcesHeadInterface;
+}
diff --git a/lib/RoadizCoreBundle/src/Api/Model/NodesSourcesHeadInterface.php b/lib/RoadizCoreBundle/src/Api/Model/NodesSourcesHeadInterface.php
@@ -15,6 +15,7 @@ public function getMetaTitle(): ?string;
     public function getMetaDescription(): ?string;
     public function getPolicyUrl(): ?string;
     public function getHomePageUrl(): ?string;
+    public function isNoIndex(): bool;
     public function getHomePage(): ?NodesSources;
     public function getShareImage(): ?DocumentInterface;
     public function getTranslation(): TranslationInterface;

diff --git a/lib/RoadizCoreBundle/src/SearchEngine/AbstractSearchHandler.php b/lib/RoadizCoreBundle/src/SearchEngine/AbstractSearchHandler.php
@@ -303,6 +303,11 @@ protected function isQuerySingleWord(string $q): bool
         return preg_match('#[\s\-\'\"\–\—\’\”\‘\“\/\+\.\,]#', $q) !== 1;
     }
 
+    protected function formatDateTimeToUTC(\DateTimeInterface $dateTime): string
+    {
+        return gmdate('Y-m-d\TH:i:s\Z', $dateTime->getTimestamp());
+    }
+
     /**
      * @param array $args
      *

diff --git a/lib/RoadizCoreBundle/src/SearchEngine/AbstractSolarium.php b/lib/RoadizCoreBundle/src/SearchEngine/AbstractSolarium.php
@@ -9,7 +9,6 @@
 use RZ\Roadiz\Markdown\MarkdownInterface;
 use Solarium\Core\Client\Client;
 use Solarium\Core\Query\DocumentInterface;
-use Solarium\Core\Query\Result\Result;
 use Solarium\Core\Query\Result\ResultInterface;
 use Solarium\QueryType\Update\Query\Document;
 use Solarium\QueryType\Update\Query\Query;
@@ -134,7 +133,7 @@ public function updateAndCommit(): ?ResultInterface
      */
     public function update(Query $update): void
     {
-        $this->clean($update);
+        // Since Solr ID are now deterministic and composite, we don't need to remove document, just update it.
         $this->createEmptyDocument($update);
         $this->index();
         // add the document to the update query
@@ -197,7 +196,7 @@ public function cleanAndCommit(): void
     public function index(): bool
     {
         if ($this->document instanceof Document) {
-            $this->document->setKey('id', uniqid('', true));
+            $this->document->setKey('id', $this->getIdempotentIdentifier());
 
             try {
                 foreach ($this->getFieldsAssoc() as $key => $value) {
@@ -310,4 +309,16 @@ public function cleanTextContent(?string $content, bool $stripMarkdown = true):
         $content = preg_replace('/[\x00-\x1F]/', '', $content);
         return $content;
     }
+
+    /**
+     * You MUST override this method to provide an idempotent identifier.
+     * This identifier MUST be the same for the same entity.
+     *
+     * @return string
+     */
+    protected function getIdempotentIdentifier(): string
+    {
+        // This is a fallback for backward compatibility.
+        return uniqid('', true);
+    }
 }
diff --git a/lib/RoadizCoreBundle/src/SearchEngine/DocumentSearchHandler.php b/lib/RoadizCoreBundle/src/SearchEngine/DocumentSearchHandler.php
@@ -72,15 +72,15 @@ protected function argFqProcess(array &$args): array
         }
 
         /*
-         * `all_tags_txt` can store all folders, even technical ones, this fields should not user-searchable.
+         * `all_tags_slugs_ss` can store all folders, even technical ones, this fields should not user-searchable.
          */
         if (!empty($args['folders'])) {
             if ($args['folders'] instanceof Folder) {
-                $args["fq"][] = sprintf('all_tags_txt:"%s"', $args['folders']->getFolderName());
+                $args["fq"][] = sprintf('all_tags_slugs_ss:"%s"', $args['folders']->getFolderName());
             } elseif (is_array($args['folders'])) {
                 foreach ($args['folders'] as $folder) {
                     if ($folder instanceof Folder) {
-                        $args["fq"][] = sprintf('all_tags_txt:"%s"', $folder->getFolderName());
+                        $args["fq"][] = sprintf('all_tags_slugs_ss:"%s"', $folder->getFolderName());
                     }
                 }
             }

diff --git a/lib/RoadizCoreBundle/src/SearchEngine/GlobalNodeSourceSearchHandler.php b/lib/RoadizCoreBundle/src/SearchEngine/GlobalNodeSourceSearchHandler.php
@@ -9,7 +9,6 @@
 use Doctrine\Persistence\ObjectManager;
 use RZ\Roadiz\CoreBundle\Entity\NodesSources;
 use RZ\Roadiz\CoreBundle\Entity\Translation;
-use RZ\Roadiz\CoreBundle\Repository\NodesSourcesRepository;
 
 /**
  * @package RZ\Roadiz\CoreBundle\SearchEngine

diff --git a/lib/RoadizCoreBundle/src/SearchEngine/NodeSourceSearchHandler.php b/lib/RoadizCoreBundle/src/SearchEngine/NodeSourceSearchHandler.php
@@ -102,15 +102,15 @@ protected function argFqProcess(array &$args): array
 
         /*
          * filter by tag or tags
-         * `all_tags_txt` can store all tags, even technical ones, this fields should not user-searchable.
+         * `all_tags_slugs_ss` can store all tags, even technical ones, this fields should not user-searchable.
          */
         if (!empty($args['tags'])) {
             if ($args['tags'] instanceof Tag) {
-                $args["fq"][] = sprintf('all_tags_txt:"%s"', $args['tags']->getTagName());
+                $args["fq"][] = sprintf('all_tags_slugs_ss:"%s"', $args['tags']->getTagName());
             } elseif (is_array($args['tags'])) {
                 foreach ($args['tags'] as $tag) {
                     if ($tag instanceof Tag) {
-                        $args["fq"][] = sprintf('all_tags_txt:"%s"', $tag->getTagName());
+                        $args["fq"][] = sprintf('all_tags_slugs_ss:"%s"', $tag->getTagName());
                     }
                 }
             }
@@ -163,7 +163,7 @@ protected function argFqProcess(array &$args): array
         if (isset($args['publishedAt'])) {
             $tmp = "published_at_dt:";
             if (!is_array($args['publishedAt']) && $args['publishedAt'] instanceof \DateTime) {
-                $tmp .= $args['publishedAt']->setTimezone(new \DateTimeZone('UTC'))->format('Y-m-d\TH:i:s\Z');
+                $tmp .= $this->formatDateTimeToUTC($args['publishedAt']);
             } elseif (
                 isset($args['publishedAt'][0]) &&
                 $args['publishedAt'][0] === "BETWEEN" &&
@@ -173,23 +173,23 @@ protected function argFqProcess(array &$args): array
                 $args['publishedAt'][2] instanceof \DateTime
             ) {
                 $tmp .= "[" .
-                    $args['publishedAt'][1]->setTimezone(new \DateTimeZone('UTC'))->format('Y-m-d\TH:i:s\Z') .
+                    $this->formatDateTimeToUTC($args['publishedAt'][1]) .
                     " TO " .
-                    $args['publishedAt'][2]->setTimezone(new \DateTimeZone('UTC'))->format('Y-m-d\TH:i:s\Z') . "]";
+                    $this->formatDateTimeToUTC($args['publishedAt'][2]) . "]";
             } elseif (
                 isset($args['publishedAt'][0]) &&
                 $args['publishedAt'][0] === "<=" &&
                 isset($args['publishedAt'][1]) &&
                 $args['publishedAt'][1] instanceof \DateTime
             ) {
-                $tmp .= "[* TO " . $args['publishedAt'][1]->setTimezone(new \DateTimeZone('UTC'))->format('Y-m-d\TH:i:s\Z') . "]";
+                $tmp .= "[* TO " . $this->formatDateTimeToUTC($args['publishedAt'][1]) . "]";
             } elseif (
                 isset($args['publishedAt'][0]) &&
                 $args['publishedAt'][0] === ">=" &&
                 isset($args['publishedAt'][1]) &&
                 $args['publishedAt'][1] instanceof \DateTime
             ) {
-                $tmp .= "[" . $args['publishedAt'][1]->setTimezone(new \DateTimeZone('UTC'))->format('Y-m-d\TH:i:s\Z') . " TO *]";
+                $tmp .= "[" . $this->formatDateTimeToUTC($args['publishedAt'][1]) . " TO *]";
             }
             unset($args['publishedAt']);
             $args["fq"][] = $tmp;

diff --git a/lib/RoadizCoreBundle/src/SearchEngine/SolariumDocument.php b/lib/RoadizCoreBundle/src/SearchEngine/SolariumDocument.php
@@ -5,11 +5,13 @@
 namespace RZ\Roadiz\CoreBundle\SearchEngine;
 
 use Psr\Log\LoggerInterface;
+use RZ\Roadiz\Core\AbstractEntities\PersistableInterface;
 use RZ\Roadiz\CoreBundle\Entity\Document;
 use RZ\Roadiz\Markdown\MarkdownInterface;
 use Solarium\Core\Query\DocumentInterface;
 use Solarium\Core\Query\Result\ResultInterface;
 use Solarium\QueryType\Update\Query\Query;
+use Symfony\Component\String\Slugger\AsciiSlugger;
 
 /**
  * Wrap a Solarium and a Document’ translations together to ease indexing.
@@ -203,4 +205,9 @@ public function index(): bool
 
         return true;
     }
+
+    protected function getIdempotentIdentifier(): string
+    {
+        throw new \InvalidArgumentException('SolariumDocument should not provide any ID');
+    }
 }
diff --git a/lib/RoadizCoreBundle/src/SearchEngine/SolariumDocumentTranslation.php b/lib/RoadizCoreBundle/src/SearchEngine/SolariumDocumentTranslation.php
@@ -10,6 +10,7 @@
 use RZ\Roadiz\Markdown\MarkdownInterface;
 use Solarium\QueryType\Update\Query\Query;
 use Symfony\Component\EventDispatcher\EventDispatcherInterface;
+use Symfony\Component\String\Slugger\AsciiSlugger;
 
 /**
  * Wrap a Solarium and a DocumentTranslation together to ease indexing.
@@ -65,4 +66,12 @@ public function clean(Query $update): bool
 
         return true;
     }
+
+    protected function getIdempotentIdentifier(): string
+    {
+        $namespace = explode('\\', get_class($this->documentTranslation));
+        // get last 3 parts of namespace
+        $namespace = array_slice($namespace, -3);
+        return (new AsciiSlugger())->slug(implode(' ', $namespace))->lower()->snake() . '.' . $this->documentTranslation->getId();
+    }
 }
diff --git a/lib/RoadizCoreBundle/src/SearchEngine/SolariumNodeSource.php b/lib/RoadizCoreBundle/src/SearchEngine/SolariumNodeSource.php
@@ -10,6 +10,7 @@
 use RZ\Roadiz\Markdown\MarkdownInterface;
 use Solarium\QueryType\Update\Query\Query;
 use Symfony\Component\EventDispatcher\EventDispatcherInterface;
+use Symfony\Component\String\Slugger\AsciiSlugger;
 
 /**
  * Wrap a Solarium and a NodeSource together to ease indexing.
@@ -70,4 +71,12 @@ public function clean(Query $update): bool
 
         return true;
     }
+
+    protected function getIdempotentIdentifier(): string
+    {
+        $namespace = explode('\\', get_class($this->nodeSource));
+        // get last 3 parts of namespace
+        $namespace = array_slice($namespace, -3);
+        return (new AsciiSlugger())->slug(implode(' ', $namespace))->lower()->snake() . '.' . $this->nodeSource->getId();
+    }
 }
diff --git a/lib/RoadizCoreBundle/src/SearchEngine/Subscriber/AbstractIndexingSubscriber.php b/lib/RoadizCoreBundle/src/SearchEngine/Subscriber/AbstractIndexingSubscriber.php
@@ -0,0 +1,17 @@
+<?php
+
+declare(strict_types=1);
+
+namespace RZ\Roadiz\CoreBundle\SearchEngine\Subscriber;
+
+use RZ\Roadiz\Core\AbstractEntities\PersistableInterface;
+use Symfony\Component\EventDispatcher\EventSubscriberInterface;
+use Symfony\Component\String\Slugger\AsciiSlugger;
+
+abstract class AbstractIndexingSubscriber implements EventSubscriberInterface
+{
+    protected function formatDateTimeToUTC(\DateTimeInterface $dateTime): string
+    {
+        return gmdate('Y-m-d\TH:i:s\Z', $dateTime->getTimestamp());
+    }
+}
diff --git a/...izCoreBundle/src/SearchEngine/Subscriber/DefaultDocumentTranslationIndexingSubscriber.php b/...izCoreBundle/src/SearchEngine/Subscriber/DefaultDocumentTranslationIndexingSubscriber.php
@@ -9,9 +9,8 @@
 use RZ\Roadiz\CoreBundle\Event\Document\DocumentTranslationIndexingEvent;
 use RZ\Roadiz\CoreBundle\SearchEngine\AbstractSolarium;
 use RZ\Roadiz\CoreBundle\SearchEngine\SolariumDocumentTranslation;
-use Symfony\Component\EventDispatcher\EventSubscriberInterface;
 
-final class DefaultDocumentTranslationIndexingSubscriber implements EventSubscriberInterface
+final class DefaultDocumentTranslationIndexingSubscriber extends AbstractIndexingSubscriber
 {
     /**
      * @inheritDoc
@@ -34,25 +33,13 @@ public function onIndexing(DocumentTranslationIndexingEvent $event): void
         $assoc[SolariumDocumentTranslation::IDENTIFIER_KEY] = $documentTranslation->getId();
         if ($document instanceof Document) {
             $assoc['document_id_i'] = $document->getId();
-            $assoc['created_at_dt'] = $document->getCreatedAt()
-                ->setTimezone(new \DateTimeZone('UTC'))
-                ->format('Y-m-d\TH:i:s\Z');
-            ;
-            $assoc['updated_at_dt'] = $document->getUpdatedAt()
-                ->setTimezone(new \DateTimeZone('UTC'))
-                ->format('Y-m-d\TH:i:s\Z');
-            ;
+            $assoc['created_at_dt'] = $this->formatDateTimeToUTC($document->getCreatedAt());
+            $assoc['updated_at_dt'] = $this->formatDateTimeToUTC($document->getUpdatedAt());
 
             $copyrightValidSince = $document->getCopyrightValidSince() ?? new \DateTime('1970-01-01 00:00:00');
             $copyrightValidUntil = $document->getCopyrightValidUntil() ?? new \DateTime('9999-12-31 23:59:59');
-            $assoc['copyright_valid_since_dt'] = $copyrightValidSince
-                ->setTimezone(new \DateTimeZone('UTC'))
-                ->format('Y-m-d\TH:i:s\Z');
-            ;
-            $assoc['copyright_valid_until_dt'] = $copyrightValidUntil
-                ->setTimezone(new \DateTimeZone('UTC'))
-                ->format('Y-m-d\TH:i:s\Z');
-            ;
+            $assoc['copyright_valid_since_dt'] = $this->formatDateTimeToUTC($copyrightValidSince);
+            $assoc['copyright_valid_until_dt'] = $this->formatDateTimeToUTC($copyrightValidUntil);
         }
         $assoc['filename_s'] = $document->getFilename();
         $assoc['mime_type_s'] = $document->getMimeType();
@@ -107,16 +94,16 @@ public function onIndexing(DocumentTranslationIndexingEvent $event): void
         $assoc['tags_txt_' . $lang] = implode(' ', $visibleFolderNames);
 
         /*
-         * `all_tags_txt` can store all folders, even technical one, this fields should not user searchable.
+         * `all_tags_slugs_ss` can store all folders, even technical one, this fields should not user searchable.
          */
         $allFolders = $document->getFolders();
         $allFolderNames = [];
         /** @var Folder $folder */
         foreach ($allFolders as $folder) {
             $allFolderNames[] = $folder->getFolderName();
         }
-        // Use all_tags_txt to be compatible with other data types
-        $assoc['all_tags_txt'] = array_filter(array_unique($allFolderNames));
+        // Use all_tags_slugs_ss to be compatible with other data types
+        $assoc['all_tags_slugs_ss'] = array_filter(array_unique($allFolderNames));
 
         /*
          * Collect data in a single field