UBERF-6313: Improve upgrade of workspace (hcengineering#5178)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

dev/tool/src/index.ts

@@ -48,20 +48,22 @@ import serverToken, { decodeToken, generateToken } from '@hcengineering/server-t
 import toolPlugin, { FileModelLogger } from '@hcengineering/server-tool'
 
 import { program, type Command } from 'commander'
-import { MongoClient, type Db } from 'mongodb'
+import { type Db, type MongoClient } from 'mongodb'
 import { clearTelegramHistory } from './telegram'
 import { diffWorkspace, updateField } from './workspace'
 
 import {
   getWorkspaceId,
   MeasureMetricsContext,
+  metricsToString,
   RateLimiter,
   type AccountRole,
   type Data,
   type Tx,
   type Version
 } from '@hcengineering/core'
 import { consoleModelLogger, type MigrateOperation } from '@hcengineering/model'
+import { getMongoClient } from '@hcengineering/mongo'
 import { openAIConfigDefaults } from '@hcengineering/openai'
 import { type StorageAdapter } from '@hcengineering/server-core'
 import path from 'path'
@@ -129,9 +131,10 @@ export function devTool (
   async function withDatabase (uri: string, f: (db: Db, client: MongoClient) => Promise<any>): Promise<void> {
     console.log(`connecting to database '${uri}'...`)
 
-    const client = await MongoClient.connect(uri)
-    await f(client.db(ACCOUNT_DB), client)
-    await client.close()
+    const client = getMongoClient(uri)
+    const _client = await client.getClient()
+    await f(_client.db(ACCOUNT_DB), _client)
+    client.close()
   }
 
   program.version('0.0.1')
@@ -285,14 +288,29 @@ export function devTool (
   program
     .command('upgrade-workspace <name>')
     .description('upgrade workspace')
-    .action(async (workspace, cmd) => {
+    .option('-f|--force [force]', 'Force update', true)
+    .action(async (workspace, cmd: { force: boolean }) => {
       const { mongodbUri, version, txes, migrateOperations } = prepareTools()
       await withDatabase(mongodbUri, async (db) => {
         const info = await getWorkspaceById(db, productId, workspace)
         if (info === null) {
           throw new Error(`workspace ${workspace} not found`)
         }
-        await upgradeWorkspace(version, txes, migrateOperations, productId, db, info.workspaceUrl ?? info.workspace)
+
+        const measureCtx = new MeasureMetricsContext('upgrade', {})
+
+        await upgradeWorkspace(
+          measureCtx,
+          version,
+          txes,
+          migrateOperations,
+          productId,
+          db,
+          info.workspaceUrl ?? info.workspace,
+          consoleModelLogger,
+          cmd.force
+        )
+        console.log(metricsToString(measureCtx.metrics, 'upgrade', 60), {})
       })
     })
 
@@ -312,6 +330,7 @@ export function devTool (
       const { mongodbUri, version, txes, migrateOperations } = prepareTools()
       await withDatabase(mongodbUri, async (db) => {
         const workspaces = await listWorkspacesRaw(db, productId)
+        workspaces.sort((a, b) => b.lastVisit - a.lastVisit)
 
         // We need to update workspaces with missing workspaceUrl
         for (const ws of workspaces) {
@@ -343,15 +362,16 @@ export function devTool (
           console.log(
             '---UPGRADING----',
             ws.workspace,
-            !cmd.console ? (logger as FileModelLogger).file : '',
+            ws.workspaceUrl,
             'pending: ',
             toProcess,
             'ETA:',
-            avgTime * toProcess
+            Math.floor(avgTime * toProcess * 100) / 100
           )
           toProcess--
           try {
             await upgradeWorkspace(
+              toolCtx,
               version,
               txes,
               migrateOperations,
@@ -383,6 +403,10 @@ export function devTool (
               })
             )
           )
+          console.log('Upgrade done')
+          // console.log((process as any)._getActiveHandles())
+          // console.log((process as any)._getActiveRequests())
+          process.exit()
         } else {
           console.log('UPGRADE write logs at:', cmd.logs)
           for (const ws of workspaces) {

models/chunter/src/migration.ts

@@ -13,14 +13,14 @@
 // limitations under the License.
 //
 
+import { chunterId } from '@hcengineering/chunter'
 import core, { type Class, type Doc, type Domain, type Ref, TxOperations } from '@hcengineering/core'
 import {
   type MigrateOperation,
   type MigrationClient,
   type MigrationUpgradeClient,
   tryMigrate
 } from '@hcengineering/model'
-import { chunterId } from '@hcengineering/chunter'
 import activity, { DOMAIN_ACTIVITY } from '@hcengineering/model-activity'
 import notification from '@hcengineering/notification'
 
@@ -35,15 +35,16 @@ export async function createDocNotifyContexts (
   attachedToClass: Ref<Class<Doc>>
 ): Promise<void> {
   const users = await client.findAll(core.class.Account, {})
+  const docNotifyContexts = await client.findAll(notification.class.DocNotifyContext, {
+    user: { $in: users.map((it) => it._id) },
+    attachedTo,
+    attachedToClass
+  })
   for (const user of users) {
     if (user._id === core.account.System) {
       continue
     }
-    const docNotifyContext = await client.findOne(notification.class.DocNotifyContext, {
-      user: user._id,
-      attachedTo,
-      attachedToClass
-    })
+    const docNotifyContext = docNotifyContexts.find((it) => it.user === user._id)
 
     if (docNotifyContext === undefined) {
       await tx.createDoc(notification.class.DocNotifyContext, core.space.Space, {

plugins/client-resources/src/index.ts

@@ -85,6 +85,11 @@ export default async () => {
   }
 }
 function createModelPersistence (workspace: string): TxPersistenceStore | undefined {
+  const overrideStore = getMetadata(clientPlugin.metadata.OverridePersistenceStore)
+  if (overrideStore !== undefined) {
+    return overrideStore
+  }
+
   let dbRequest: IDBOpenDBRequest | undefined
   let dbPromise: Promise<IDBDatabase | undefined> = Promise.resolve(undefined)
 

plugins/client/src/index.ts

@@ -13,7 +13,7 @@
 // limitations under the License.
 //
 
-import type { AccountClient, ClientConnectEvent } from '@hcengineering/core'
+import type { AccountClient, ClientConnectEvent, TxPersistenceStore } from '@hcengineering/core'
 import type { Plugin, Resource } from '@hcengineering/platform'
 import { Metadata, plugin } from '@hcengineering/platform'
 
@@ -76,7 +76,8 @@ export default plugin(clientId, {
     FilterModel: '' as Metadata<boolean>,
     ExtraPlugins: '' as Metadata<Plugin[]>,
     UseBinaryProtocol: '' as Metadata<boolean>,
-    UseProtocolCompression: '' as Metadata<boolean>
+    UseProtocolCompression: '' as Metadata<boolean>,
+    OverridePersistenceStore: '' as Metadata<TxPersistenceStore>
   },
   function: {
     GetClient: '' as Resource<ClientFactory>

server/account/src/index.ts

@@ -840,12 +840,12 @@ export async function createWorkspace (
           getWorkspaceId(initWS, productId),
           getWorkspaceId(workspaceInfo.workspace, productId)
         )
-        client = await upgradeModel(getTransactor(), wsId, txes, migrationOperation, ctxModellogger)
+        client = await upgradeModel(ctx, getTransactor(), wsId, txes, migrationOperation, ctxModellogger)
       } else {
         client = await initModel(ctx, getTransactor(), wsId, txes, migrationOperation, ctxModellogger)
       }
     } catch (err: any) {
-      return { workspaceInfo, err, client: {} as any }
+      return { workspaceInfo, err, client: null as any }
     }
     // Workspace is created, we need to clear disabled flag.
     await db
@@ -859,6 +859,7 @@ export async function createWorkspace (
  * @public
  */
 export async function upgradeWorkspace (
+  ctx: MeasureContext,
   version: Data<Version>,
   txes: Tx[],
   migrationOperation: [string, MigrateOperation][],
@@ -895,7 +896,7 @@ export async function upgradeWorkspace (
     }
   )
   await (
-    await upgradeModel(getTransactor(), getWorkspaceId(ws.workspace, productId), txes, migrationOperation, logger)
+    await upgradeModel(ctx, getTransactor(), getWorkspaceId(ws.workspace, productId), txes, migrationOperation, logger)
   ).close()
   return versionStr
 }

server/server/src/starter.ts

@@ -8,7 +8,7 @@ export function storageConfigFromEnv (): StorageConfiguration {
     process.env.STORAGE_CONFIG ?? '{ "default": "", "storages": []}'
   )
   if (storageConfig.storages.length === 0 || storageConfig.default === '') {
-    console.info('STORAGE_CONFIG is required for complex configuration, fallback to minio config')
+    // 'STORAGE_CONFIG is required for complex configuration, fallback to minio config'
 
     let minioEndpoint = process.env.MINIO_ENDPOINT
     if (minioEndpoint === undefined) {

server/tool/src/connect.ts

@@ -15,9 +15,10 @@
 //
 
 import client, { clientId } from '@hcengineering/client'
-import { Client, systemAccountEmail, WorkspaceId } from '@hcengineering/core'
+import { Client, LoadModelResponse, systemAccountEmail, Tx, WorkspaceId } from '@hcengineering/core'
 import { addLocation, getMetadata, getResource, setMetadata } from '@hcengineering/platform'
 import { generateToken } from '@hcengineering/server-token'
+import crypto from 'node:crypto'
 import plugin from './plugin'
 
 /**
@@ -27,7 +28,8 @@ export async function connect (
   transactorUrl: string,
   workspace: WorkspaceId,
   email?: string,
-  extra?: Record<string, string>
+  extra?: Record<string, string>,
+  model?: Tx[]
 ): Promise<Client> {
   const token = generateToken(email ?? systemAccountEmail, workspace, extra)
 
@@ -48,7 +50,25 @@ export async function connect (
   })
   addLocation(clientId, () => import('@hcengineering/client-resources'))
 
-  return await (
-    await getResource(client.function.GetClient)
-  )(token, transactorUrl)
+  if (model !== undefined) {
+    let prev = ''
+    const hashes = model.map((it) => {
+      const h = crypto.createHash('sha1')
+      h.update(prev)
+      h.update(JSON.stringify(it))
+      prev = h.digest('hex')
+      return prev
+    })
+    setMetadata(client.metadata.OverridePersistenceStore, {
+      load: async () => ({
+        hash: hashes[hashes.length - 1],
+        transactions: model,
+        full: true
+      }),
+      store: async (model: LoadModelResponse) => {}
+    })
+  }
+
+  const clientFactory = await getResource(client.function.GetClient)
+  return await clientFactory(token, transactorUrl)
 }