Keep SQLite AI incremented after rollback (#1167)

.github/workflows/test-unit.yml

@@ -162,6 +162,12 @@ jobs:
           php -d opcache.enable_cli=1 vendor/bin/phpunit --exclude-group none $(if [ -n "$LOG_COVERAGE" ]; then echo --coverage-text; else echo --no-coverage; fi) --fail-on-warning --fail-on-risky $(if vendor/bin/phpunit --version | grep -q '^PHPUnit 9\.'; then echo -v; else echo --fail-on-notice --fail-on-deprecation --display-notices --display-deprecations --display-warnings --display-errors --display-incomplete --display-skipped; fi)
           if [ -n "$LOG_COVERAGE" ]; then mv coverage/phpunit.cov coverage/phpunit-sqlite.cov; fi
 
+      - name: "Run tests: SQLite 3.25.3"
+        run: |
+          apk add sqlite-dev=3.25.3-r0 --repository=https://dl-cdn.alpinelinux.org/alpine/v3.6/main
+          php -d opcache.enable_cli=1 vendor/bin/phpunit --exclude-group none $(if [ -n "$LOG_COVERAGE" ]; then echo --coverage-text; else echo --no-coverage; fi) --fail-on-warning --fail-on-risky $(if vendor/bin/phpunit --version | grep -q '^PHPUnit 9\.'; then echo -v; else echo --fail-on-notice --fail-on-deprecation --display-notices --display-deprecations --display-warnings --display-errors --display-incomplete --display-skipped; fi)
+          if [ -n "$LOG_COVERAGE" ]; then mv coverage/phpunit.cov coverage/phpunit-sqlite325.cov; fi
+
       - name: "Run tests: MySQL - PDO"
         if: success() || failure()
         env:

phpstan.neon.dist

@@ -42,7 +42,7 @@ parameters:
         -
             message: '~^Class Doctrine\\DBAL\\Platforms\\SqlitePlatform referenced with incorrect case: Doctrine\\DBAL\\Platforms\\SQLitePlatform\.$~'
             path: '*'
-            count: 25
+            count: 24
 
         # TODO these rules are generated, this ignores should be fixed in the code
         # for src/Schema/TestCase.php

src/Persistence/Sql/Sqlite/Connection.php

@@ -21,6 +21,7 @@ protected static function createDbalConfiguration(): Configuration
         $configuration->setMiddlewares([
             ...$configuration->getMiddlewares(),
             new EnableForeignKeys(),
+            new PreserveAutoincrementOnRollbackMiddleware(),
         ]);
 
         return $configuration;

src/Persistence/Sql/Sqlite/PreserveAutoincrementOnRollbackConnectionMiddleware.php

@@ -0,0 +1,154 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Atk4\Data\Persistence\Sql\Sqlite;
+
+use Atk4\Data\Persistence\Sql\Exception;
+use Doctrine\DBAL\Driver\Middleware\AbstractConnectionMiddleware;
+
+class PreserveAutoincrementOnRollbackConnectionMiddleware extends AbstractConnectionMiddleware
+{
+    private static string $libraryVersion;
+
+    private function createExpressionFromStringLiteral(string $value): Expression
+    {
+        return new Expression('\'' . str_replace('\'', '\'\'', $value) . '\'');
+    }
+
+    /**
+     * @return array<string, array<string, int>>
+     */
+    protected function listSequences(): array
+    {
+        if ((self::$libraryVersion ?? null) === null) {
+            $getLibraryVersionSql = (new Query())
+                ->field('sqlite_version()')
+                ->render()[0];
+            self::$libraryVersion = $this->query($getLibraryVersionSql)->fetchOne();
+        }
+
+        if (version_compare(self::$libraryVersion, '3.37') < 0) {
+            $listAllSchemasSql = (new Query())
+                ->table('pragma_database_list')
+                ->field('name')
+                ->render()[0];
+            $allSchemas = $this->query($listAllSchemasSql)->fetchFirstColumn();
+
+            $schemas = [];
+            foreach ($allSchemas as $schema) {
+                $dummySelectFromSqliteSequenceTableSql = (new Query())
+                    ->table($schema . '.sqlite_sequence')
+                    ->field('name')
+                    ->render()[0];
+                try {
+                    $this->query($dummySelectFromSqliteSequenceTableSql)->fetchFirstColumn();
+                    $schemas[] = $schema;
+                } catch (\Exception $e) {
+                    while ($e->getPrevious() !== null) {
+                        $e = $e->getPrevious();
+                    }
+
+                    if (!str_contains($e->getMessage(), 'HY000')
+                        || !str_contains($e->getMessage(), 'no such table: ' . $schema . '.sqlite_sequence')
+                    ) {
+                        throw $e;
+                    }
+                }
+            }
+        } else {
+            $listSchemasSql = (new Query())
+                ->table('pragma_table_list')
+                ->field('schema')
+                ->where('name', $this->createExpressionFromStringLiteral('sqlite_sequence'))
+                ->render()[0];
+            $schemas = $this->query($listSchemasSql)->fetchFirstColumn();
+        }
+
+        $res = [];
+        if ($schemas !== []) {
+            $listSequencesSql = implode("\nUNION ALL\n", array_map(function (string $schema) {
+                return (new Query())
+                    ->table($schema . '.sqlite_sequence')
+                    ->field($this->createExpressionFromStringLiteral($schema), 'schema')
+                    ->field('name')
+                    ->field('seq', 'value')
+                    ->render()[0];
+            }, $schemas));
+
+            $res = [];
+            foreach ($this->query($listSequencesSql)->fetchAllAssociative() as $row) {
+                $value = (int) $row['value'];
+                if (!is_int($row['value']) && (string) $value !== $row['value']) {
+                    throw (new Exception('Unexpected SQLite sequence value'))
+                        ->addMoreInfo('value', $row['value']);
+                }
+
+                $res[$row['schema']][$row['name']] = $value;
+            }
+        }
+
+        return $res;
+    }
+
+    /**
+     * @param array<string, array<string, int>> $beforeRollbackSequences
+     */
+    protected function restoreSequencesIfDecremented(array $beforeRollbackSequences): void
+    {
+        $afterRollbackSequences = $this->listSequences();
+
+        foreach ($beforeRollbackSequences as $schema => $beforeRollbackSequences2) {
+            foreach ($beforeRollbackSequences2 as $table => $beforeRollbackValue) {
+                $afterRollbackValue = $afterRollbackSequences[$schema][$table] ?? null;
+                if ($afterRollbackValue >= $beforeRollbackValue) {
+                    continue;
+                }
+
+                if ($afterRollbackValue === null) { // https://sqlite.org/forum/info/3e7cc380f0a159c6
+                    $query = (new Query())
+                        ->mode('insert')
+                        ->set('name', $this->createExpressionFromStringLiteral($table));
+                } else {
+                    $query = (new Query())
+                        ->mode('update')
+                        ->where('name', $this->createExpressionFromStringLiteral($table));
+                }
+                $query->table($schema . '.sqlite_sequence');
+                $query->set('seq', $this->createExpressionFromStringLiteral((string) $beforeRollbackValue));
+
+                $this->exec($query->render()[0]);
+            }
+        }
+    }
+
+    #[\Override]
+    public function exec(string $sql): int
+    {
+        $isRollback = str_starts_with(strtoupper(ltrim($sql)), 'ROLLBACK ');
+
+        if ($isRollback) {
+            $beforeRollbackSequences = $this->listSequences();
+        }
+
+        $res = parent::exec($sql);
+
+        if ($isRollback) {
+            $this->restoreSequencesIfDecremented($beforeRollbackSequences);
+        }
+
+        return $res;
+    }
+
+    #[\Override]
+    public function rollBack()
+    {
+        $beforeRollbackSequences = $this->listSequences();
+
+        $res = parent::rollBack();
+
+        $this->restoreSequencesIfDecremented($beforeRollbackSequences);
+
+        return $res;
+    }
+}

src/Persistence/Sql/Sqlite/PreserveAutoincrementOnRollbackMiddleware.php

@@ -0,0 +1,27 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Atk4\Data\Persistence\Sql\Sqlite;
+
+use Doctrine\DBAL\Driver;
+use Doctrine\DBAL\Driver\Connection;
+use Doctrine\DBAL\Driver\Middleware;
+use Doctrine\DBAL\Driver\Middleware\AbstractDriverMiddleware;
+
+class PreserveAutoincrementOnRollbackMiddleware implements Middleware
+{
+    #[\Override]
+    public function wrap(Driver $driver): Driver
+    {
+        return new class($driver) extends AbstractDriverMiddleware {
+            #[\Override]
+            public function connect(
+                #[\SensitiveParameter]
+                array $params
+            ): Connection {
+                return new PreserveAutoincrementOnRollbackConnectionMiddleware(parent::connect($params));
+            }
+        };
+    }
+}

tests/Persistence/Sql/WithDb/SelectTest.php

@@ -620,13 +620,6 @@ public function testImportAndAutoincrement(): void
             self::assertSame(103, $m->insert(['f1' => 'N']));
         });
 
-        // TODO workaround SQLite to be consistent with other databases
-        // https://stackoverflow.com/questions/27947712/sqlite-repeats-primary-key-autoincrement-value-after-rollback
-        // https://github.com/atk4/data/issues/1162
-        if ($this->getDatabasePlatform() instanceof SQLitePlatform) {
-            return;
-        }
-
         $invokeInAtomicAndThrowFx(static function () use ($invokeInAtomicAndThrowFx, $m) {
             self::assertSame(104, $m->insert(['f1' => 'O1']));
             $invokeInAtomicAndThrowFx(static function () use ($invokeInAtomicAndThrowFx, $m) {