diff --git a/app/client/packages/rts/src/ctl/backup/BackupState.ts b/app/client/packages/rts/src/ctl/backup/BackupState.ts index 9192f654d51e..5f56bd13ef98 100644 --- a/app/client/packages/rts/src/ctl/backup/BackupState.ts +++ b/app/client/packages/rts/src/ctl/backup/BackupState.ts @@ -1,5 +1,7 @@ export class BackupState { readonly args: readonly string[]; + readonly dbUrl: string; + readonly initAt: string = new Date().toISOString().replace(/:/g, "-"); readonly errors: string[] = []; @@ -8,8 +10,9 @@ export class BackupState { isEncryptionEnabled: boolean = false; - constructor(args: string[]) { + constructor(args: string[], url: string) { this.args = Object.freeze([...args]); + this.dbUrl = url; // We seal `this` so that no link in the chain can "add" new properties to the state. This is intentional. If any // link wants to save data in the `BackupState`, which shouldn't even be needed in most cases, it should do so by diff --git a/app/client/packages/rts/src/ctl/backup/backup.test.ts b/app/client/packages/rts/src/ctl/backup/backup.test.ts index 7fa9ce48d9d3..2ecfc5e9b5e2 100644 --- a/app/client/packages/rts/src/ctl/backup/backup.test.ts +++ b/app/client/packages/rts/src/ctl/backup/backup.test.ts @@ -8,6 +8,7 @@ import { encryptBackupArchive, executeCopyCMD, executeMongoDumpCMD, + executePostgresDumpCMD, getAvailableBackupSpaceInBytes, getEncryptionPasswordFromUser, getGitRoot, @@ -66,6 +67,34 @@ describe("Backup Tests", () => { console.log(res); }); + test("Test postgres dump CMD generation", async () => { + const dest = "/dest"; + const url = "postgresql://username:password@host/appsmith"; + const cmd = [ + "pg_dump --host=host", + "--port=5432", + "--username=username", + "--dbname=appsmith", + "--schema=appsmith", + "--schema=public", + "--schema=temporal", + "--file=/dest/pg-data.sql", + "--verbose", + "--serializable-deferrable", + ].join(" "); + + const res = await executePostgresDumpCMD(dest, { + host: "host", + port: 5432, + username: "username", + password: "password", + database: "appsmith", + }); + + expect(res).toBe(cmd); + console.log(res); + }); + test("Test get gitRoot path when APPSMITH_GIT_ROOT is '' ", () => { expect(getGitRoot("")).toBe("/appsmith-stacks/git-storage"); }); @@ -246,7 +275,7 @@ test("Get DB name from Mongo URI 1", async () => { const mongodb_uri = "mongodb+srv://admin:password@test.cluster.mongodb.net/my_db_name?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin"; const expectedDBName = "my_db_name"; - const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri); + const dbName = utils.getDatabaseNameFromUrl(mongodb_uri); expect(dbName).toEqual(expectedDBName); }); @@ -255,7 +284,7 @@ test("Get DB name from Mongo URI 2", async () => { const mongodb_uri = "mongodb+srv://admin:password@test.cluster.mongodb.net/test123?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin"; const expectedDBName = "test123"; - const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri); + const dbName = utils.getDatabaseNameFromUrl(mongodb_uri); expect(dbName).toEqual(expectedDBName); }); @@ -264,7 +293,7 @@ test("Get DB name from Mongo URI 3", async () => { const mongodb_uri = "mongodb+srv://admin:password@test.cluster.mongodb.net/test123"; const expectedDBName = "test123"; - const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri); + const dbName = utils.getDatabaseNameFromUrl(mongodb_uri); expect(dbName).toEqual(expectedDBName); }); @@ -272,7 +301,23 @@ test("Get DB name from Mongo URI 3", async () => { test("Get DB name from Mongo URI 4", async () => { const mongodb_uri = "mongodb://appsmith:pAssW0rd!@localhost:27017/appsmith"; const expectedDBName = "appsmith"; - const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri); + const dbName = utils.getDatabaseNameFromUrl(mongodb_uri); expect(dbName).toEqual(expectedDBName); }); + +test("Get DB name from Postgres URL", async () => { + const dbName = utils.getDatabaseNameFromUrl( + "postgresql://user:password@host:5432/postgres_db", + ); + + expect(dbName).toEqual("postgres_db"); +}); + +test("Get DB name from Postgres URL with query params", async () => { + const dbName = utils.getDatabaseNameFromUrl( + "postgresql://user:password@host:5432/postgres_db?sslmode=disable", + ); + + expect(dbName).toEqual("postgres_db"); +}); diff --git a/app/client/packages/rts/src/ctl/backup/index.ts b/app/client/packages/rts/src/ctl/backup/index.ts index 0e2d70c30868..cf133f910bdd 100644 --- a/app/client/packages/rts/src/ctl/backup/index.ts +++ b/app/client/packages/rts/src/ctl/backup/index.ts @@ -8,15 +8,25 @@ import * as linkClasses from "./links"; import { BackupState } from "./BackupState"; export async function run(args: string[]) { + const url = utils.getDburl(); + + if (!url.startsWith("mongodb") && !url.startsWith("postgresql")) { + console.error("Only MongoDB and Postgres databases are supported."); + process.exitCode = 1; + + return; + } + await utils.ensureSupervisorIsRunning(); - const state: BackupState = new BackupState(args); + const state: BackupState = new BackupState(args, url); const chain: Link[] = [ new linkClasses.BackupFolderLink(state), new linkClasses.DiskSpaceLink(), new linkClasses.ManifestLink(state), new linkClasses.MongoDumpLink(state), + new linkClasses.PostgresDumpLink(state), new linkClasses.GitStorageLink(state), new linkClasses.EnvFileLink(state), diff --git a/app/client/packages/rts/src/ctl/backup/links/ManifestLink.ts b/app/client/packages/rts/src/ctl/backup/links/ManifestLink.ts index c9958aa52c9c..eddf1db96f32 100644 --- a/app/client/packages/rts/src/ctl/backup/links/ManifestLink.ts +++ b/app/client/packages/rts/src/ctl/backup/links/ManifestLink.ts @@ -14,7 +14,7 @@ export class ManifestLink implements Link { const version = await utils.getCurrentAppsmithVersion(); const manifestData = { appsmithVersion: version, - dbName: utils.getDatabaseNameFromMongoURI(utils.getDburl()), + dbName: utils.getDatabaseNameFromUrl(utils.getDburl()), }; await fsPromises.writeFile( diff --git a/app/client/packages/rts/src/ctl/backup/links/MongoDumpLink.ts b/app/client/packages/rts/src/ctl/backup/links/MongoDumpLink.ts index 13400928bb11..b689be8264a7 100644 --- a/app/client/packages/rts/src/ctl/backup/links/MongoDumpLink.ts +++ b/app/client/packages/rts/src/ctl/backup/links/MongoDumpLink.ts @@ -9,9 +9,13 @@ export class MongoDumpLink implements Link { constructor(private readonly state: BackupState) {} async doBackup() { - console.log("Exporting database"); - await executeMongoDumpCMD(this.state.backupRootPath, utils.getDburl()); - console.log("Exporting database done."); + const url = this.state.dbUrl; + + if (url.startsWith("mongodb")) { + console.log("Exporting database"); + await executeMongoDumpCMD(this.state.backupRootPath, url); + console.log("Exporting database done."); + } } } diff --git a/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts b/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts new file mode 100644 index 000000000000..81202e8954a9 --- /dev/null +++ b/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts @@ -0,0 +1,150 @@ +import type { Link } from "."; +import type { BackupState } from "../BackupState"; +import * as utils from "../../utils"; + +interface ConnectionDetails { + host: string; + port: number; + username: string; + password: string; + database: string; +} + +/** + * Backup & restore for Postgres database data using `pg_dump` and `psql`. + */ +export class PostgresDumpLink implements Link { + private postgresUrl: null | ConnectionDetails = null; + + constructor(private readonly state: BackupState) {} + + async preBackup() { + const url = this.state.dbUrl; + + if (url.startsWith("postgresql")) { + this.postgresUrl = parsePostgresUrl(url); + + return; + } + + if (process.env.APPSMITH_KEYCLOAK_DB_URL) { + const dbUrlFromEnv = process.env.APPSMITH_KEYCLOAK_DB_URL; + + if (dbUrlFromEnv.startsWith("postgresql://")) { + this.postgresUrl = parsePostgresUrl(dbUrlFromEnv); + } else if (dbUrlFromEnv.includes("/")) { + // then it's just the hostname and database in there + const [host, database] = dbUrlFromEnv.split("/"); + this.postgresUrl = { + host, + port: 5432, + username: process.env.APPSMITH_KEYCLOAK_DB_USERNAME, + password: process.env.APPSMITH_KEYCLOAK_DB_PASSWORD, + database, + }; + } else { + // Identify this as an invalid value for this env variable. + // But we ignore this fact for now, since Postgres itself is not a critical component yet. + console.warn( + "APPSMITH_KEYCLOAK_DB_URL is set, but it doesn't start with postgresql://. This is not a valid value for this env variable. But we ignore this fact for now, since Postgres itself is not a critical component yet.", + ); + } + } else if (process.env.APPSMITH_ENABLE_EMBEDDED_DB !== "0") { + this.postgresUrl = { + // Get unix_socket_directories from postgresql.conf, like in pg-utils.sh/get_unix_socket_directory. + // Unix socket directory + host: "/var/run/postgresql", + port: 5432, + username: "postgres", + password: process.env.APPSMITH_TEMPORAL_PASSWORD, + database: "appsmith", + }; + } else { + throw new Error("No Postgres DB URL found"); + } + } + + async doBackup() { + if (this.postgresUrl) { + await executePostgresDumpCMD(this.state.backupRootPath, this.postgresUrl); + } + } + + async doRestore(restoreContentsPath: string) { + const env = { + ...process.env, + }; + + const cmd = ["psql", "-v", "ON_ERROR_STOP=1"]; + + const isLocalhost = ["localhost", "127.0.0.1"].includes( + this.postgresUrl.host, + ); + + if (isLocalhost) { + env.PGHOST = "/var/run/postgresql"; + env.PGPORT = "5432"; + env.PGUSER = "postgres"; + env.PGPASSWORD = process.env.APPSMITH_TEMPORAL_PASSWORD; + env.PGDATABASE = this.postgresUrl.database; + } else { + env.PGHOST = this.postgresUrl.host; + env.PGPORT = this.postgresUrl.port.toString(); + env.PGUSER = this.postgresUrl.username; + env.PGPASSWORD = this.postgresUrl.password; + env.PGDATABASE = this.postgresUrl.database; + } + + await utils.execCommand( + [ + ...cmd, + "--command=DROP SCHEMA IF EXISTS public CASCADE; DROP SCHEMA IF EXISTS appsmith CASCADE; DROP SCHEMA IF EXISTS temporal CASCADE;", + ], + { env }, + ); + + await utils.execCommand( + [...cmd, `--file=${restoreContentsPath}/pg-data.sql`], + { env }, + ); + console.log("Restoring Postgres database completed"); + } +} + +function parsePostgresUrl(url: string): ConnectionDetails { + const parsed = new URL(url); + return { + host: parsed.hostname, + port: parseInt(parsed.port || "5432"), + username: parsed.username, + password: decodeURIComponent(parsed.password), + database: parsed.pathname.substring(1), + }; +} + +export async function executePostgresDumpCMD( + destFolder: string, + details: ConnectionDetails, +) { + const args = [ + "pg_dump", + `--host=${details.host}`, + `--port=${details.port || "5432"}`, + `--username=${details.username}`, + `--dbname=${details.database}`, + "--schema=appsmith", + "--schema=public", // Keycloak + "--schema=temporal", + `--file=${destFolder}/pg-data.sql`, + "--verbose", + "--serializable-deferrable", + ]; + + // Set password in environment since it's not allowed in the CLI + const env = { + ...process.env, + PGPASSWORD: details.password, + }; + + return await utils.execCommand(args, { env }); +} diff --git a/app/client/packages/rts/src/ctl/backup/links/index.ts b/app/client/packages/rts/src/ctl/backup/links/index.ts index da48f43c0531..1638fb96dd68 100644 --- a/app/client/packages/rts/src/ctl/backup/links/index.ts +++ b/app/client/packages/rts/src/ctl/backup/links/index.ts @@ -16,3 +16,4 @@ export * from "./EnvFileLink"; export * from "./GitStorageLink"; export * from "./ManifestLink"; export * from "./MongoDumpLink"; +export * from "./PostgresDumpLink"; diff --git a/app/client/packages/rts/src/ctl/restore.ts b/app/client/packages/rts/src/ctl/restore.ts index a32d27564c30..5457ad79ae4a 100644 --- a/app/client/packages/rts/src/ctl/restore.ts +++ b/app/client/packages/rts/src/ctl/restore.ts @@ -4,6 +4,8 @@ import os from "os"; import readlineSync from "readline-sync"; import * as utils from "./utils"; import * as Constants from "./constants"; +import { PostgresDumpLink } from "./backup/links/PostgresDumpLink"; +import { BackupState } from "./backup/BackupState"; const command_args = process.argv.slice(3); @@ -109,8 +111,26 @@ async function extractArchive(backupFilePath: string, restoreRootPath: string) { console.log("Extracting the backup archive completed"); } -async function restoreDatabase(restoreContentsPath: string, dbUrl: string) { +async function restoreDatabases(restoreContentsPath: string, dbUrl: string) { console.log("Restoring database..."); + + if (dbUrl.startsWith("mongodb")) { + await restoreMongoDB(restoreContentsPath, dbUrl); + } else { + throw new Error( + "Unsupported database type, only MongoDB and Postgres are supported", + ); + } + + // TODO: Get all link classes equipped with `doRestore` and refactor this to be like backup. + const link = new PostgresDumpLink(new BackupState([], "")); + await link.preBackup(); + await link.doRestore(restoreContentsPath); + + console.log("Restoring database completed"); +} + +async function restoreMongoDB(restoreContentsPath: string, dbUrl: string) { const cmd = [ "mongorestore", `--uri=${dbUrl}`, @@ -121,7 +141,7 @@ async function restoreDatabase(restoreContentsPath: string, dbUrl: string) { try { const fromDbName = await getBackupDatabaseName(restoreContentsPath); - const toDbName = utils.getDatabaseNameFromMongoURI(dbUrl); + const toDbName = utils.getDatabaseNameFromUrl(dbUrl); console.log("Restoring database from " + fromDbName + " to " + toDbName); cmd.push( @@ -139,6 +159,42 @@ async function restoreDatabase(restoreContentsPath: string, dbUrl: string) { console.log("Restoring database completed"); } +async function restorePostgres(restoreContentsPath: string, dbUrl: string) { + const cmd = [ + "pg_restore", + "--verbose", + "--clean", + `${restoreContentsPath}/pg-data`, + ]; + const url = new URL(dbUrl); + const isLocalhost = ["localhost", "127.0.0.1"].includes(url.hostname); + + if (isLocalhost) { + let dbName: string; + + try { + dbName = utils.getDatabaseNameFromUrl(dbUrl); + console.log("Restoring database to", dbName); + } catch (error) { + console.warn( + "Error reading manifest file. Assuming same database name as appsmith.", + error, + ); + dbName = "appsmith"; + } + cmd.push( + "-d", + "postgresql://localhost:5432/" + dbName, + // Use default user for local postgres + "--username=postgres", + ); + } else { + cmd.push("-d", dbUrl); + } + + await utils.execCommand(cmd); +} + async function restoreDockerEnvFile( restoreContentsPath: string, backupName: string, @@ -311,6 +367,24 @@ async function getBackupDatabaseName(restoreContentsPath: string) { } export async function run() { + const processesToPause = ["backend", "rts"]; + if ( + await fsPromises + .access(process.env.TMP + "/supervisor-conf.d/keycloak.conf") + .then(() => true) + .catch(() => false) + ) { + processesToPause.push("keycloak"); + } + if ( + await fsPromises + .access(process.env.TMP + "/supervisor-conf.d/temporal.conf") + .then(() => true) + .catch(() => false) + ) { + processesToPause.push("temporal"); + } + let cleanupArchive = false; let overwriteEncryptionKeys = true; let backupFilePath: string; @@ -364,8 +438,8 @@ export async function run() { console.log( "Restoring Appsmith instance from the backup at " + backupFilePath, ); - await utils.stop(["backend", "rts"]); - await restoreDatabase(restoreContentsPath, utils.getDburl()); + await utils.stop(processesToPause); + await restoreDatabases(restoreContentsPath, utils.getDburl()); await restoreDockerEnvFile( restoreContentsPath, backupName, @@ -383,7 +457,7 @@ export async function run() { await fsPromises.rm(backupFilePath, { force: true }); } - await utils.start(["backend", "rts"]); + await utils.start(processesToPause); process.exit(); } } diff --git a/app/client/packages/rts/src/ctl/utils.ts b/app/client/packages/rts/src/ctl/utils.ts index 62d0ec878871..acd29c5671a7 100644 --- a/app/client/packages/rts/src/ctl/utils.ts +++ b/app/client/packages/rts/src/ctl/utils.ts @@ -249,8 +249,13 @@ export async function execCommandSilent(cmd, options?) { }); } -export function getDatabaseNameFromMongoURI(uri) { - const uriParts = uri.split("/"); - - return uriParts[uriParts.length - 1].split("?")[0]; +/** + * Extracts database name from MongoDB or Postgres connection URL + * @param url - Database connection URL + * @returns Database name + */ +export function getDatabaseNameFromUrl(url: string) { + const parts = url.split("/"); + + return parts[parts.length - 1].split("?")[0]; }