UBERF-6469: Fix slow index creation

dev/client-resources/src/index.ts

@@ -14,7 +14,7 @@
 //
 
 import clientPlugin from '@hcengineering/client'
-import { AccountClient, createClient } from '@hcengineering/core'
+import core, { AccountClient, createClient, groupByArray, MigrationState } from '@hcengineering/core'
 import { migrateOperations } from '@hcengineering/model-all'
 import { getMetadata, getResource } from '@hcengineering/platform'
 import { connect } from './connection'
@@ -27,9 +27,18 @@ export default async () => {
       GetClient: async (): Promise<AccountClient> => {
         if (client === undefined) {
           client = await createClient(connect)
+
+          const states = await client.findAll<MigrationState>(core.class.MigrationState, {})
+          const migrateState = new Map(
+            Array.from(groupByArray(states, (it) => it.plugin).entries()).map((it) => [
+              it[0],
+              new Set(it[1].map((q) => q.state))
+            ])
+          )
+          ;(client as any).migrateState = migrateState
           for (const op of migrateOperations) {
             console.log('Migrate', op[0])
-            await op[1].upgrade(client, {
+            await op[1].upgrade(client as any, {
               log (msg, data) {
                 console.log(msg, data)
               },

dev/tool/package.json

@@ -131,7 +131,7 @@
     "commander": "^8.1.0",
     "csv-parse": "~5.1.0",
     "email-addresses": "^5.0.0",
-    "fast-equals": "^2.0.3",
+    "fast-equals": "^5.0.1",
     "got": "^11.8.3",
     "libphonenumber-js": "^1.9.46",
     "mime-types": "~2.1.34",

dev/tool/src/index.ts

@@ -138,6 +138,7 @@ export function devTool (
     const _client = await client.getClient()
     await f(_client.db(ACCOUNT_DB), _client)
     client.close()
+    console.log(`closing database connection to '${uri}'...`)
   }
 
   program.version('0.0.1')
@@ -363,13 +364,12 @@ export function devTool (
 
           const avgTime = (Date.now() - st) / (workspaces.length - toProcess + 1)
           console.log(
-            '---UPGRADING----',
-            ws.workspace,
-            ws.workspaceUrl,
+            '----------------------------------------------------------\n---UPGRADING----',
             'pending: ',
             toProcess,
-            'ETA:',
-            Math.floor(avgTime * toProcess * 100) / 100
+            'ETA: ',
+            Math.floor(avgTime * toProcess),
+            ws.workspace
           )
           toProcess--
           try {
@@ -384,11 +384,11 @@ export function devTool (
               logger,
               cmd.force
             )
-            console.log('---UPGRADING-DONE----', ws.workspace, Date.now() - t)
+            console.log('---done---------', 'pending: ', toProcess, 'TIME:', Date.now() - t, ws.workspace)
           } catch (err: any) {
             withError.push(ws.workspace)
             logger.log('error', JSON.stringify(err))
-            console.log('---UPGRADING-FAILED----', ws.workspace, Date.now() - t)
+            console.log('   FAILED-------', 'pending: ', toProcess, 'TIME:', Date.now() - t, ws.workspace)
           } finally {
             if (!cmd.console) {
               ;(logger as FileModelLogger).close()
@@ -409,7 +409,7 @@ export function devTool (
           console.log('Upgrade done')
           // console.log((process as any)._getActiveHandles())
           // console.log((process as any)._getActiveRequests())
-          process.exit()
+          // process.exit()
         } else {
           console.log('UPGRADE write logs at:', cmd.logs)
           for (const ws of workspaces) {

models/activity/src/index.ts

@@ -117,7 +117,7 @@ export class TActivityMessage extends TAttachedDoc implements ActivityMessage {
     repliedPersons?: Ref<Person>[]
 
   @Prop(TypeTimestamp(), activity.string.LastReply)
-  @Index(IndexKind.Indexed)
+  // @Index(IndexKind.Indexed)
     lastReply?: Timestamp
 
   @Prop(Collection(activity.class.Reaction), activity.string.Reactions)
@@ -134,15 +134,15 @@ export class TDocUpdateMessage extends TActivityMessage implements DocUpdateMess
     objectId!: Ref<Doc>
 
   @Prop(TypeRef(core.class.Class), core.string.Class)
-  @Index(IndexKind.Indexed)
+  // @Index(IndexKind.Indexed)
     objectClass!: Ref<Class<Doc>>
 
   @Prop(TypeRef(core.class.TxCUD), core.string.Object)
-  @Index(IndexKind.Indexed)
+  // @Index(IndexKind.Indexed)
     txId!: Ref<TxCUD<Doc>>
 
   @Prop(TypeString(), core.string.Object)
-  @Index(IndexKind.Indexed)
+  // @Index(IndexKind.Indexed)
     action!: DocUpdateAction
 
   updateCollection?: string
@@ -153,11 +153,11 @@ export class TDocUpdateMessage extends TActivityMessage implements DocUpdateMess
 export class TActivityReference extends TActivityMessage implements ActivityReference {
   // Source document we have reference from, it should be parent document for Comment/Message.
   @Prop(TypeRef(core.class.Doc), core.string.Object)
-  @Index(IndexKind.Indexed)
+  // @Index(IndexKind.Indexed)
     srcDocId!: Ref<Doc>
 
   @Prop(TypeRef(core.class.Class), core.string.Class)
-  @Index(IndexKind.Indexed)
+  // @Index(IndexKind.Indexed)
     srcDocClass!: Ref<Class<Doc>>
 
   // Reference to comment/message in source doc
@@ -386,6 +386,22 @@ export function createModel (builder: Builder): void {
     },
     activity.ids.AddReactionNotification
   )
+
+  builder.createDoc(core.class.DomainIndexConfiguration, core.space.Model, {
+    domain: DOMAIN_ACTIVITY,
+    indexes: [
+      { attachedTo: 1, createdOn: 1 },
+      { attachedTo: 1, createdOn: -1 }
+    ],
+    disabled: [
+      { modifiedOn: 1 },
+      { createdOn: -1 },
+      { space: 1 },
+      { modifiedBy: 1 },
+      { createdBy: 1 },
+      { attachedToClass: 1 }
+    ]
+  })
 }
 
 export default activity

models/attachment/src/index.ts

@@ -272,6 +272,17 @@ export function createModel (builder: Builder): void {
       group: 'edit'
     }
   })
+  builder.createDoc(core.class.DomainIndexConfiguration, core.space.Model, {
+    domain: DOMAIN_ATTACHMENT,
+    disabled: [
+      { attachedToClass: 1 },
+      { modifiedOn: 1 },
+      { modifiedBy: 1 },
+      { createdBy: 1 },
+      { createdOn: -1 },
+      { state: 1 }
+    ]
+  })
 }
 
 export default attachment

models/bitrix/src/index.ts

@@ -133,4 +133,8 @@ export function createModel (builder: Builder): void {
     },
     bitrix.action.BitrixImport
   )
+  builder.createDoc(core.class.DomainIndexConfiguration, core.space.Model, {
+    domain: DOMAIN_BITRIX,
+    disabled: [{ _id: 1 }, { _class: 1 }, { space: 1 }, { modifiedBy: 1 }, { createdBy: 1 }, { createdOn: -1 }]
+  })
 }

models/bitrix/src/migration.ts

@@ -14,8 +14,14 @@
 //
 
 import core, { TxOperations } from '@hcengineering/core'
-import { type MigrateOperation, type MigrationClient, type MigrationUpgradeClient } from '@hcengineering/model'
+import {
+  tryUpgrade,
+  type MigrateOperation,
+  type MigrationClient,
+  type MigrationUpgradeClient
+} from '@hcengineering/model'
 import bitrix from './plugin'
+import { bitrixId } from '@hcengineering/bitrix'
 
 async function createSpace (tx: TxOperations): Promise<void> {
   const current = await tx.findOne(core.class.Space, {
@@ -40,7 +46,14 @@ async function createSpace (tx: TxOperations): Promise<void> {
 export const bitrixOperation: MigrateOperation = {
   async migrate (client: MigrationClient): Promise<void> {},
   async upgrade (client: MigrationUpgradeClient): Promise<void> {
-    const tx = new TxOperations(client, core.account.System)
-    await createSpace(tx)
+    await tryUpgrade(client, bitrixId, [
+      {
+        state: 'create-defaults',
+        func: async (client) => {
+          const tx = new TxOperations(client, core.account.System)
+          await createSpace(tx)
+        }
+      }
+    ])
   }
 }

models/board/src/migration.ts

@@ -20,7 +20,8 @@ import {
   type MigrationClient,
   type MigrationUpgradeClient,
   createOrUpdate,
-  tryMigrate
+  tryMigrate,
+  tryUpgrade
 } from '@hcengineering/model'
 import core, { DOMAIN_SPACE } from '@hcengineering/model-core'
 import { DOMAIN_TASK, createProjectType, createSequence, fixTaskTypes } from '@hcengineering/model-task'
@@ -184,7 +185,14 @@ export const boardOperation: MigrateOperation = {
     ])
   },
   async upgrade (client: MigrationUpgradeClient): Promise<void> {
-    const ops = new TxOperations(client, core.account.System)
-    await createDefaults(ops)
+    await tryUpgrade(client, boardId, [
+      {
+        state: 'board0001',
+        func: async (client) => {
+          const ops = new TxOperations(client, core.account.System)
+          await createDefaults(ops)
+        }
+      }
+    ])
   }
 }

models/calendar/src/index.ts

@@ -313,6 +313,18 @@ export function createModel (builder: Builder): void {
   builder.mixin(calendar.class.Event, core.class.Class, view.mixin.ObjectPresenter, {
     presenter: calendar.component.EventPresenter
   })
+
+  builder.createDoc(core.class.DomainIndexConfiguration, core.space.Model, {
+    domain: DOMAIN_CALENDAR,
+    disabled: [
+      { attachedToClass: 1 },
+      { modifiedOn: 1 },
+      { modifiedBy: 1 },
+      { createdBy: 1 },
+      { createdOn: -1 },
+      { state: 1 }
+    ]
+  })
 }
 
 export default calendar

models/calendar/src/migration.ts

@@ -18,6 +18,7 @@ import contact from '@hcengineering/contact'
 import core, { TxOperations, type Ref } from '@hcengineering/core'
 import {
   tryMigrate,
+  tryUpgrade,
   type MigrateOperation,
   type MigrationClient,
   type MigrationUpgradeClient
@@ -139,20 +140,32 @@ async function migrateTimezone (client: MigrationClient): Promise<void> {
 
 export const calendarOperation: MigrateOperation = {
   async migrate (client: MigrationClient): Promise<void> {
-    await fixEventDueDate(client)
-    await migrateReminders(client)
-    await fillOriginalStartTime(client)
-    await migrateSync(client)
-    await migrateExternalCalendars(client)
     await tryMigrate(client, calendarId, [
+      {
+        state: 'calendar001',
+        func: async (client) => {
+          await fixEventDueDate(client)
+          await migrateReminders(client)
+          await fillOriginalStartTime(client)
+          await migrateSync(client)
+          await migrateExternalCalendars(client)
+        }
+      },
       {
         state: 'timezone',
         func: migrateTimezone
       }
     ])
   },
   async upgrade (client: MigrationUpgradeClient): Promise<void> {
-    const tx = new TxOperations(client, core.account.System)
-    await migrateCalendars(tx)
+    await tryUpgrade(client, calendarId, [
+      {
+        state: 'u-calendar0002',
+        func: async (client) => {
+          const tx = new TxOperations(client, core.account.System)
+          await migrateCalendars(tx)
+        }
+      }
+    ])
   }
 }

models/chunter/src/index.ts

@@ -744,6 +744,11 @@ export function createModel (builder: Builder, options = { addApplication: true
   builder.mixin(chunter.class.ThreadMessage, core.class.Class, activity.mixin.ActivityMessagePreview, {
     presenter: chunter.component.ThreadMessagePreview
   })
+
+  builder.createDoc(core.class.DomainIndexConfiguration, core.space.Model, {
+    domain: DOMAIN_CHUNTER,
+    disabled: [{ _class: 1 }, { space: 1 }, { modifiedBy: 1 }, { createdBy: 1 }, { createdOn: -1 }]
+  })
 }
 
 export default chunter

models/chunter/src/migration.ts

@@ -19,14 +19,15 @@ import {
   type MigrateOperation,
   type MigrationClient,
   type MigrationUpgradeClient,
-  tryMigrate
+  tryMigrate,
+  tryUpgrade
 } from '@hcengineering/model'
 import activity, { DOMAIN_ACTIVITY } from '@hcengineering/model-activity'
 import notification from '@hcengineering/notification'
 
 import chunter from './plugin'
 
-const DOMAIN_COMMENT = 'comment' as Domain
+export const DOMAIN_COMMENT = 'comment' as Domain
 
 export async function createDocNotifyContexts (
   client: MigrationUpgradeClient,
@@ -128,18 +129,23 @@ export const chunterOperation: MigrateOperation = {
       {
         state: 'create-chat-messages',
         func: convertCommentsToChatMessages
-      }
-    ])
-    await tryMigrate(client, chunterId, [
+      },
       {
         state: 'remove-backlinks',
         func: removeBacklinks
       }
     ])
   },
   async upgrade (client: MigrationUpgradeClient): Promise<void> {
-    const tx = new TxOperations(client, core.account.System)
-    await createGeneral(client, tx)
-    await createRandom(client, tx)
+    await tryUpgrade(client, chunterId, [
+      {
+        state: 'create-defaults',
+        func: async (client) => {
+          const tx = new TxOperations(client, core.account.System)
+          await createGeneral(client, tx)
+          await createRandom(client, tx)
+        }
+      }
+    ])
   }
 }

models/contact/src/index.ts

@@ -1133,4 +1133,22 @@ export function createModel (builder: Builder): void {
   )
 
   createPublicLinkAction(builder, contact.class.Contact, contact.action.PublicLink)
+
+  builder.createDoc(core.class.DomainIndexConfiguration, core.space.Model, {
+    domain: DOMAIN_CHANNEL,
+    disabled: [
+      { attachedToClass: 1 },
+      { provider: 1 },
+      { space: 1 },
+      { modifiedBy: 1 },
+      { modifiedOn: 1 },
+      { createdBy: 1 },
+      { createdBy: -1 },
+      { createdOn: -1 }
+    ]
+  })
+  builder.createDoc(core.class.DomainIndexConfiguration, core.space.Model, {
+    domain: DOMAIN_CONTACT,
+    disabled: [{ attachedToClass: 1 }, { modifiedBy: 1 }, { createdBy: 1 }, { createdOn: -1 }, { attachedTo: 1 }]
+  })
 }