feat: switch to using php for database import/export

ymirapp · Dec 6, 2024 · dd6bb08 · dd6bb08
1 parent 4914a8f
commit dd6bb08
Show file tree

Hide file tree

Showing 8 changed files with 312 additions and 164 deletions.
diff --git a/composer.json b/composer.json
@@ -16,8 +16,11 @@
     "require": {
         "php": ">=7.2.5",
         "ext-json": "*",
+        "ext-pdo": "*",
         "ext-zip": "*",
+        "ext-zlib": "*",
         "guzzlehttp/guzzle": "^7.0",
+        "ifsnop/mysqldump-php": "^2.12",
         "illuminate/collections": "^8.0|^9.0|^10.0|^11.0",
         "league/flysystem": "^2.1.1|^3.0",
         "league/flysystem-ftp": "^2.0|^3.0",

diff --git a/src/Command/Database/AbstractDatabaseCommand.php b/src/Command/Database/AbstractDatabaseCommand.php
@@ -85,8 +85,10 @@ protected function determineUser(Input $input, Output $output): string
     /**
      * Start a SSH tunnel to a private database server.
      */
-    protected function startSshTunnel(array $databaseServer): Process
+    protected function startSshTunnel(array $databaseServer, Output $output): Process
     {
+        $output->info(sprintf('Opening SSH tunnel to "<comment>%s</comment>" private database server', $databaseServer['name']));
+
         $network = $this->apiClient->getNetwork(Arr::get($databaseServer, 'network.id'));
 
         if (!is_array($network->get('bastion_host'))) {

diff --git a/src/Command/Database/ExportDatabaseCommand.php b/src/Command/Database/ExportDatabaseCommand.php
@@ -22,9 +22,11 @@
 use Ymir\Cli\CliConfiguration;
 use Ymir\Cli\Console\Input;
 use Ymir\Cli\Console\Output;
+use Ymir\Cli\Database\Connection;
+use Ymir\Cli\Database\Mysqldump;
+use Ymir\Cli\Exception\InvalidInputException;
 use Ymir\Cli\Process\Process;
 use Ymir\Cli\ProjectConfiguration\ProjectConfiguration;
-use Ymir\Cli\Tool\Mysql;
 
 class ExportDatabaseCommand extends AbstractDatabaseCommand
 {
@@ -59,65 +61,70 @@ protected function configure()
     {
         $this
             ->setName(self::NAME)
-            ->setDescription('Export a database to a local .sql.gz file')
-            ->addArgument('name', InputArgument::OPTIONAL, 'The name of the database to export')
+            ->setDescription('Export a database to a local SQL file')
+            ->addArgument('database', InputArgument::OPTIONAL, 'The database name to export from')
             ->addOption('server', null, InputOption::VALUE_REQUIRED, 'The ID or name of the database server to export a database from')
             ->addOption('user', null, InputOption::VALUE_REQUIRED, 'The user used to connect to the database server')
-            ->addOption('password', null, InputOption::VALUE_REQUIRED, 'The password of the user connecting to the database server');
+            ->addOption('password', null, InputOption::VALUE_REQUIRED, 'The password of the user connecting to the database server')
+            ->addOption('compression', null, InputOption::VALUE_REQUIRED, 'The compression method to use when exporting the database (gzip or none)', 'gzip');
     }
 
     /**
      * {@inheritdoc}
      */
     protected function perform(Input $input, Output $output)
     {
-        $databaseServer = $this->determineDatabaseServer('Which database server would you like to export a database from?', $input, $output);
-        $name = $this->determineDatabaseName($databaseServer, $input, $output);
-        $user = $this->determineUser($input, $output);
-        $password = $this->determinePassword($input, $output, $user);
+        $compression = $input->getStringOption('compression');
 
-        $filename = sprintf('%s_%s.sql.gz', $name, Carbon::now()->toDateString());
+        if (!in_array($compression, ['gzip', 'none'])) {
+            throw new InvalidInputException('The compression method must be either "gzip" or "none"');
+        }
+
+        $connection = $this->getConnection($input, $output);
+        $filename = sprintf('%s_%s.%s', $connection->getDatabase(), Carbon::now()->toDateString(), 'gzip' === $compression ? 'sql.gz' : 'sql');
 
         if ($this->filesystem->exists($filename) && !$output->confirm(sprintf('The "<comment>%s</comment>" backup file already exists. Do you want to overwrite it?', $filename), false)) {
             return;
         }
 
-        $host = $databaseServer['endpoint'];
-        $port = '3306';
         $tunnel = null;
 
-        if (!$databaseServer['publicly_accessible']) {
-            $output->info(sprintf('Opening SSH tunnel to "<comment>%s</comment>" database server', $databaseServer['name']));
-
-            $tunnel = $this->startSshTunnel($databaseServer);
-            $host = '127.0.0.1';
-            $port = '3305';
+        if (!$connection->needsSshTunnel()) {
+            $tunnel = $this->startSshTunnel($connection->getDatabaseServer(), $output);
         }
 
-        $output->infoWithDelayWarning(sprintf('Exporting "<comment>%s</comment>" database', $name));
+        $output->infoWithDelayWarning(sprintf('Exporting "<comment>%s</comment>" database', $connection->getDatabase()));
 
-        Mysql::export($filename, $host, $port, $user, $password, $name);
-
-        if ($tunnel instanceof Process) {
-            $tunnel->stop();
+        try {
+            Mysqldump::fromConnection($connection, ['compress' => $compression])->start($filename);
+        } catch (\Throwable $exception) {
+            throw new RuntimeException('Failed to export database: '.$exception->getMessage());
+        } finally {
+            if ($tunnel instanceof Process) {
+                $tunnel->stop();
+            }
         }
 
         $output->infoWithValue('Database exported successfully to', $filename);
     }
 
     /**
-     * Determine the name of the database to export.
+     * Get the connection to the database by prompting for any missing information.
      */
-    private function determineDatabaseName(array $databaseServer, Input $input, Output $output): string
+    private function getConnection(Input $input, Output $output): Connection
     {
-        $name = $input->getStringArgument('name');
+        $databaseServer = $this->determineDatabaseServer('Which database server would you like to export a database from?', $input, $output);
+        $database = $input->getStringArgument('database');
 
-        if (!empty($name)) {
-            return $name;
-        } elseif (!$databaseServer['publicly_accessible']) {
-            throw new RuntimeException('You must specify the name of the database to export for a private database server');
+        if (empty($database) && !$databaseServer['publicly_accessible']) {
+            throw new RuntimeException('You must specify the database name to export for a private database server');
+        } elseif (empty($database)) {
+            $database = $output->choice('Which database would you like to export?', $this->apiClient->getDatabases($databaseServer['id']));
         }
 
-        return $output->choice('Which database would you like to export?', $this->apiClient->getDatabases($databaseServer['id']));
+        $user = $this->determineUser($input, $output);
+        $password = $this->determinePassword($input, $output, $user);
+
+        return new Connection($database, $databaseServer, $user, $password);
     }
 }
diff --git a/src/Command/Database/ImportDatabaseCommand.php b/src/Command/Database/ImportDatabaseCommand.php
@@ -13,6 +13,7 @@
 
 namespace Ymir\Cli\Command\Database;
 
+use Illuminate\Support\LazyCollection;
 use Symfony\Component\Console\Exception\RuntimeException;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputOption;
@@ -21,10 +22,11 @@
 use Ymir\Cli\CliConfiguration;
 use Ymir\Cli\Console\Input;
 use Ymir\Cli\Console\Output;
+use Ymir\Cli\Database\Connection;
+use Ymir\Cli\Database\PDO;
 use Ymir\Cli\Exception\InvalidInputException;
 use Ymir\Cli\Process\Process;
 use Ymir\Cli\ProjectConfiguration\ProjectConfiguration;
-use Ymir\Cli\Tool\Mysql;
 
 class ImportDatabaseCommand extends AbstractDatabaseCommand
 {
@@ -59,49 +61,30 @@ protected function configure()
     {
         $this
             ->setName(self::NAME)
-            ->setDescription('Import a local .sql or .sql.gz file to a database')
-            ->addArgument('file', InputArgument::REQUIRED, 'The path to the local .sql or .sql.gz file')
-            ->addArgument('name', InputArgument::OPTIONAL, 'The name of the database to import')
+            ->setDescription('Import a local SQL backup to a database')
+            ->addArgument('filename', InputArgument::REQUIRED, 'The path to the local .sql or .sql.gz file')
+            ->addArgument('database', InputArgument::OPTIONAL, 'The database name to import into')
             ->addOption('server', null, InputOption::VALUE_REQUIRED, 'The ID or name of the database server to import a database to')
             ->addOption('user', null, InputOption::VALUE_REQUIRED, 'The user used to connect to the database server')
-            ->addOption('password', null, InputOption::VALUE_REQUIRED, 'The password of the user connecting to the database server')
-            ->addOption('force', 'f', InputOption::VALUE_NONE, 'Force the import even if there are SQL errors')
-            ->addOption('skip-ssl', null, InputOption::VALUE_NONE, 'Disable SSL for the connection to the database server');
+            ->addOption('password', null, InputOption::VALUE_REQUIRED, 'The password of the user connecting to the database server');
     }
 
     /**
      * {@inheritdoc}
      */
     protected function perform(Input $input, Output $output)
     {
-        $file = $input->getStringArgument('file');
-
-        if (!str_ends_with($file, '.sql') && !str_ends_with($file, '.sql.gz')) {
-            throw new InvalidInputException('You may only import .sql or .sql.gz files');
-        } elseif (!$this->filesystem->exists($file)) {
-            throw new InvalidInputException(sprintf('File "%s" doesn\'t exist', $file));
-        }
-
-        $databaseServer = $this->determineDatabaseServer('Which database server would you like to import a database to?', $input, $output);
-        $host = $databaseServer['endpoint'];
-        $name = $this->determineDatabaseName($databaseServer, $input, $output);
-        $port = '3306';
+        $filename = $this->getFilename($input);
+        $connection = $this->getConnection($input, $output);
         $tunnel = null;
 
-        $user = $this->determineUser($input, $output);
-        $password = $this->determinePassword($input, $output, $user);
-
-        if (!$databaseServer['publicly_accessible']) {
-            $output->info(sprintf('Opening SSH tunnel to "<comment>%s</comment>" database server', $databaseServer['name']));
-
-            $tunnel = $this->startSshTunnel($databaseServer);
-            $host = '127.0.0.1';
-            $port = '3305';
+        if ($connection->needsSshTunnel()) {
+            $tunnel = $this->startSshTunnel($connection->getDatabaseServer(), $output);
         }
 
-        $output->infoWithDelayWarning(sprintf('Importing "<comment>%s</comment>" to the "<comment>%s</comment>" database', $file, $name));
+        $output->infoWithDelayWarning(sprintf('Importing "<comment>%s</comment>" to the "<comment>%s</comment>" database', $filename, $connection->getDatabase()));
 
-        Mysql::import($file, $host, $port, $user, $password, $name, $input->getBooleanOption('force'), $input->getBooleanOption('skip-ssl'));
+        $this->importBackup($connection, $filename);
 
         if ($tunnel instanceof Process) {
             $tunnel->stop();
@@ -111,18 +94,98 @@ protected function perform(Input $input, Output $output)
     }
 
     /**
-     * Determine the name of the database to export.
+     * Get the connection to the database by prompting for any missing information.
+     */
+    private function getConnection(Input $input, Output $output): Connection
+    {
+        $databaseServer = $this->determineDatabaseServer('Which database server would you like to import a database to?', $input, $output);
+        $database = $input->getStringArgument('database');
+
+        if (empty($database) && !$databaseServer['publicly_accessible']) {
+            throw new RuntimeException('You must specify the database name to import the SQL backup to for a private database server');
+        } elseif (empty($database)) {
+            $database = $output->choice('Which database would you like to import the SQL backup to?', $this->apiClient->getDatabases($databaseServer['id']));
+        }
+
+        $user = $this->determineUser($input, $output);
+        $password = $this->determinePassword($input, $output, $user);
+
+        return new Connection($database, $databaseServer, $user, $password);
+    }
+
+    /**
+     * Get the filename of the SQL backup to import.
      */
-    private function determineDatabaseName(array $databaseServer, Input $input, Output $output): string
+    private function getFilename(Input $input): string
     {
-        $name = $input->getStringArgument('name');
+        $filename = $input->getStringArgument('filename');
 
-        if (!empty($name)) {
-            return $name;
-        } elseif (!$databaseServer['publicly_accessible']) {
-            throw new RuntimeException('You must specify the name of the database to import the SQL file to for a private database server');
+        if (!str_ends_with($filename, '.sql') && !str_ends_with($filename, '.sql.gz')) {
+            throw new InvalidInputException('You may only import .sql or .sql.gz files');
+        } elseif (!$this->filesystem->exists($filename)) {
+            throw new InvalidInputException(sprintf('File "%s" doesn\'t exist', $filename));
         }
 
-        return $output->choice('Which database would you like to import the SQL file to?', $this->apiClient->getDatabases($databaseServer['id']));
+        return $filename;
+    }
+
+    /**
+     * Imports the given SQL backup file using the given database connection.
+     */
+    private function importBackup(Connection $connection, string $filename)
+    {
+        try {
+            $isCompressed = str_ends_with($filename, '.gz');
+
+            $fclose = $isCompressed ? 'gzclose' : 'fclose';
+            $feof = $isCompressed ? 'gzeof' : 'feof';
+            $fgets = $isCompressed ? 'gzgets' : 'fgets';
+            $fopen = $isCompressed ? 'gzopen' : 'fopen';
+
+            $file = $fopen($filename, 'r');
+
+            if (!is_resource($file)) {
+                throw new RuntimeException('Failed to open file: '.$filename);
+            }
+
+            $lines = LazyCollection::make(function () use (&$file, $feof, $fgets) {
+                while (!$feof($file)) {
+                    yield $fgets($file);
+                }
+            });
+            $pdo = PDO::fromConnection($connection);
+            $query = '';
+
+            $lines->each(function ($line) use ($pdo, &$query) {
+                $line = trim($line);
+
+                if (str_starts_with($line, '--') || empty($line)) {
+                    return;
+                }
+
+                $query .= $line;
+
+                if (str_ends_with(trim($line), ';')) {
+                    $pdo->exec($query);
+                    $query = '';
+                }
+            });
+        } catch (\Throwable $exception) {
+            if ($exception instanceof RuntimeException) {
+                throw $exception;
+            }
+
+            $message = $exception->getMessage();
+
+            if ($exception instanceof \PDOException) {
+                $message = 'Failed to import database: '.$message;
+            }
+
+            throw new RuntimeException($message);
+        } finally {
+            if (isset($file, $fclose) && is_resource($file)) {
+                $fclose($file);
+            }
+        }
     }
 }