diff --git a/packages/transformer/src/IModelCloneContext.ts b/packages/transformer/src/IModelCloneContext.ts index 88dbbf73..271fb431 100644 --- a/packages/transformer/src/IModelCloneContext.ts +++ b/packages/transformer/src/IModelCloneContext.ts @@ -280,51 +280,4 @@ export class IModelCloneContext extends IModelElementCloneContext { }); return targetElementAspectProps; } - - private static aspectRemapTableName = "AspectIdRemaps"; - - public override saveStateToDb(db: SQLiteDb): void { - super.saveStateToDb(db); - if ( - DbResult.BE_SQLITE_DONE !== - db.executeSQL( - `CREATE TABLE ${IModelCloneContext.aspectRemapTableName} (Source INTEGER, Target INTEGER)` - ) - ) - throw Error( - "Failed to create the aspect remap table in the state database" - ); - db.saveChanges(); - db.withPreparedSqliteStatement( - `INSERT INTO ${IModelCloneContext.aspectRemapTableName} (Source, Target) VALUES (?, ?)`, - (stmt) => { - for (const [source, target] of this._aspectRemapTable) { - stmt.reset(); - stmt.bindId(1, source); - stmt.bindId(2, target); - if (DbResult.BE_SQLITE_DONE !== stmt.step()) - throw Error( - "Failed to insert aspect remapping into the state database" - ); - } - } - ); - } - - public override loadStateFromDb(db: SQLiteDb): void { - super.loadStateFromDb(db); - // FIXME: test this - db.withSqliteStatement( - `SELECT Source, Target FROM ${IModelCloneContext.aspectRemapTableName}`, - (stmt) => { - let status = DbResult.BE_SQLITE_ERROR; - while ((status = stmt.step()) === DbResult.BE_SQLITE_ROW) { - const source = stmt.getValue(0).getId(); - const target = stmt.getValue(1).getId(); - this._aspectRemapTable.set(source, target); - } - assert(status === DbResult.BE_SQLITE_DONE); - } - ); - } } diff --git a/packages/transformer/src/IModelExporter.ts b/packages/transformer/src/IModelExporter.ts index 789ac23b..2b5d2865 100644 --- a/packages/transformer/src/IModelExporter.ts +++ b/packages/transformer/src/IModelExporter.ts @@ -85,11 +85,11 @@ export type ExporterInitOptions = ExportChangesOptions; */ export type ExportChangesOptions = Omit & { skipPropagateChangesToRootElements?: boolean; -} & /** - * an array of ChangesetFileProps which are used to read the changesets and populate the ChangedInstanceIds using [[ChangedInstanceIds.initialize]] in [[IModelExporter.exportChanges]] - * @note mutually exclusive with @see changesetRanges, @see startChangeset and @see changedInstanceIds, so define one of the four, never more - */ - (| { csFileProps: ChangesetFileProps[] } +} /** + * an array of ChangesetFileProps which are used to read the changesets and populate the ChangedInstanceIds using [[ChangedInstanceIds.initialize]] in [[IModelExporter.exportChanges]] + * @note mutually exclusive with @see changesetRanges, @see startChangeset and @see changedInstanceIds, so define one of the four, never more + */ & ( + | { csFileProps: ChangesetFileProps[] } /** * Class instance that contains modified elements between 2 versions of an iModel. * If this parameter is not provided, then [[ChangedInstanceIds.initialize]] in [[IModelExporter.exportChanges]] @@ -998,118 +998,6 @@ export class IModelExporter { return this.handler.onProgress(); } } - - /** - * You may override this to store arbitrary json state in a exporter state dump, useful for some resumptions - * @see [[IModelTransformer.saveStateToFile]] - */ - protected getAdditionalStateJson(): any { - return {}; - } - - /** - * You may override this to load arbitrary json state in a transformer state dump, useful for some resumptions - * @see [[IModelTransformer.loadStateFromFile]] - */ - protected loadAdditionalStateJson(_additionalState: any): void {} - - /** - * Reload our state from a JSON object - * Intended for [[IModelTransformer.resumeTransformation]] - * @internal - * You can load custom json from the exporter save state for custom exporters by overriding [[IModelExporter.loadAdditionalStateJson]] - */ - public loadStateFromJson(state: IModelExporterState): void { - if (state.exporterClass !== this.constructor.name) - throw Error( - "resuming from a differently named exporter class, it is not necessarily valid to resume with a different exporter class" - ); - this.wantGeometry = state.wantGeometry; - this.wantTemplateModels = state.wantTemplateModels; - this.wantSystemSchemas = state.wantSystemSchemas; - this.visitElements = state.visitElements; - this.visitRelationships = state.visitRelationships; - this._excludedCodeSpecNames = new Set(state.excludedCodeSpecNames); - (this._excludedElementIds = CompressedId64Set.decompressSet( - state.excludedElementIds - )), - (this._excludedElementCategoryIds = CompressedId64Set.decompressSet( - state.excludedElementCategoryIds - )), - (this._excludedElementClasses = new Set( - state.excludedElementClassNames.map((c) => this.sourceDb.getJsClass(c)) - )); - this._exportElementAspectsStrategy.loadExcludedElementAspectClasses( - state.excludedElementAspectClassFullNames - ); - this._excludedRelationshipClasses = new Set( - state.excludedRelationshipClassNames.map((c) => - this.sourceDb.getJsClass(c) - ) - ); - this.loadAdditionalStateJson(state.additionalState); - } - - /** - * Serialize state to a JSON object - * Intended for [[IModelTransformer.resumeTransformation]] - * @internal - * You can add custom json to the exporter save state for custom exporters by overriding [[IModelExporter.getAdditionalStateJson]] - */ - public saveStateToJson(): IModelExporterState { - return { - exporterClass: this.constructor.name, - wantGeometry: this.wantGeometry, - wantTemplateModels: this.wantTemplateModels, - wantSystemSchemas: this.wantSystemSchemas, - visitElements: this.visitElements, - visitRelationships: this.visitRelationships, - excludedCodeSpecNames: [...this._excludedCodeSpecNames], - excludedElementIds: CompressedId64Set.compressSet( - this._excludedElementIds - ), - excludedElementCategoryIds: CompressedId64Set.compressSet( - this._excludedElementCategoryIds - ), - excludedElementClassNames: Array.from( - this._excludedElementClasses, - (cls) => cls.classFullName - ), - excludedElementAspectClassFullNames: [ - ...this._exportElementAspectsStrategy - .excludedElementAspectClassFullNames, - ], - excludedRelationshipClassNames: Array.from( - this._excludedRelationshipClasses, - (cls) => cls.classFullName - ), - additionalState: this.getAdditionalStateJson(), - }; - } -} - -/** - * The JSON format of a serialized IModelExporter instance - * Used for starting an exporter in the middle of an export operation, - * such as resuming a crashed transformation - * - * @note Must be kept synchronized with IModelExporter - * @internal - */ -export interface IModelExporterState { - exporterClass: string; - wantGeometry: boolean; - wantTemplateModels: boolean; - wantSystemSchemas: boolean; - visitElements: boolean; - visitRelationships: boolean; - excludedCodeSpecNames: string[]; - excludedElementIds: CompressedId64Set; - excludedElementCategoryIds: CompressedId64Set; - excludedElementClassNames: string[]; - excludedElementAspectClassFullNames: string[]; - excludedRelationshipClassNames: string[]; - additionalState?: any; } /** diff --git a/packages/transformer/src/IModelImporter.ts b/packages/transformer/src/IModelImporter.ts index 6109ed54..448d9204 100644 --- a/packages/transformer/src/IModelImporter.ts +++ b/packages/transformer/src/IModelImporter.ts @@ -38,7 +38,10 @@ import { SourceAndTarget, SubCategory, } from "@itwin/core-backend"; -import type { IModelTransformOptions } from "./IModelTransformer"; +import type { + IModelTransformOptions, + RelationshipPropsForDelete, +} from "./IModelTransformer"; import * as assert from "assert"; import { deleteElementTreeCascade } from "./ElementCascadingDeleter"; @@ -636,7 +639,9 @@ export class IModelImporter { } /** Delete the specified Relationship from the target iModel. */ - protected onDeleteRelationship(relationshipProps: RelationshipProps): void { + protected onDeleteRelationship( + relationshipProps: RelationshipPropsForDelete + ): void { // Only passing in what deleteInstance actually uses, full relationshipProps is not necessary. this.targetDb.relationships.deleteInstance({ id: relationshipProps.id, @@ -644,15 +649,15 @@ export class IModelImporter { } as RelationshipProps); Logger.logInfo( loggerCategory, - `Deleted relationship ${this.formatRelationshipForLogger( - relationshipProps - )}` + `Deleted relationship ${relationshipProps.classFullName} id=${relationshipProps.id}` ); this.trackProgress(); } /** Delete the specified Relationship from the target iModel. */ - public deleteRelationship(relationshipProps: RelationshipProps): void { + public deleteRelationship( + relationshipProps: RelationshipPropsForDelete + ): void { this.onDeleteRelationship(relationshipProps); } @@ -763,67 +768,6 @@ export class IModelImporter { } } - /** - * You may override this to store arbitrary json state in a exporter state dump, useful for some resumptions - * @see [[IModelTransformer.saveStateToFile]] - */ - protected getAdditionalStateJson(): any { - return {}; - } - - /** - * You may override this to load arbitrary json state in a transformer state dump, useful for some resumptions - * @see [[IModelTransformer.loadStateFromFile]] - */ - protected loadAdditionalStateJson(_additionalState: any): void {} - - /** - * Reload our state from a JSON object - * Intended for [[IModelTransformer.resumeTransformation]] - * @internal - * You can load custom json from the importer save state for custom importers by overriding [[IModelImporter.loadAdditionalStateJson]] - */ - public loadStateFromJson(state: IModelImporterState): void { - if (state.importerClass !== this.constructor.name) - throw Error( - "resuming from a differently named importer class, it is not necessarily valid to resume with a different importer class" - ); - // ignore readonly since this runs right after construction in [[IModelTransformer.resumeTransformation]] - (this.options as IModelTransformOptions) = state.options; - if (this.targetDb.iModelId !== state.targetDbId) - throw Error( - "can only load importer state when the same target is reused" - ); - // TODO: fix upstream, looks like a bad case for the linter rule when casting away readonly for this generic - // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion - (this.doNotUpdateElementIds as Set) = - CompressedId64Set.decompressSet(state.doNotUpdateElementIds); - this._duplicateCodeValueMap = new Map( - Object.entries(state.duplicateCodeValueMap) - ); - this.loadAdditionalStateJson(state.additionalState); - } - - /** - * Serialize state to a JSON object - * Intended for [[IModelTransformer.resumeTransformation]] - * @internal - * You can add custom json to the importer save state for custom importers by overriding [[IModelImporter.getAdditionalStateJson]] - */ - public saveStateToJson(): IModelImporterState { - return { - importerClass: this.constructor.name, - options: this.options, - targetDbId: - this.targetDb.iModelId || this.targetDb.nativeDb.getFilePath(), - doNotUpdateElementIds: CompressedId64Set.compressSet( - this.doNotUpdateElementIds - ), - duplicateCodeValueMap: Object.fromEntries(this._duplicateCodeValueMap), - additionalState: this.getAdditionalStateJson(), - }; - } - private resolveDuplicateCodeValues(): void { for (const [elementId, codeValue] of this._duplicateCodeValueMap) { const element = this.targetDb.elements.getElement(elementId); @@ -844,23 +788,6 @@ export class IModelImporter { } } -/** - * The JSON format of a serialized IModelimporter instance - * Used for starting an importer in the middle of an imxport operation, - * such as resuming a crashed transformation - * - * @note Must be kept synchronized with IModelImxporter - * @internal - */ -export interface IModelImporterState { - importerClass: string; - options: IModelImportOptions; - targetDbId: string; - doNotUpdateElementIds: CompressedId64Set; - duplicateCodeValueMap: Record; - additionalState?: any; -} - /** Returns true if a change within an Entity is detected. * @param entity The current persistent Entity. * @param entityProps The new EntityProps to compare against diff --git a/packages/transformer/src/IModelTransformer.ts b/packages/transformer/src/IModelTransformer.ts index c811814b..55922598 100644 --- a/packages/transformer/src/IModelTransformer.ts +++ b/packages/transformer/src/IModelTransformer.ts @@ -98,14 +98,9 @@ import { ExporterInitOptions, ExportSchemaResult, IModelExporter, - IModelExporterState, IModelExportHandler, } from "./IModelExporter"; -import { - IModelImporter, - IModelImporterState, - OptimizeGeometryOptions, -} from "./IModelImporter"; +import { IModelImporter, OptimizeGeometryOptions } from "./IModelImporter"; import { TransformerLoggerCategory } from "./TransformerLoggerCategory"; import { PendingReference, PendingReferenceMap } from "./PendingReferenceMap"; import { EntityKey, EntityMap } from "./EntityMap"; @@ -421,6 +416,11 @@ type ChangeDataState = */ export type InitFromExternalSourceAspectsArgs = InitOptions; +export interface RelationshipPropsForDelete { + id: Id64String; + classFullName: string; +} + type SyncType = "not-sync" | "forward" | "reverse"; /** Base class used to transform a source iModel into a different target iModel. @@ -2414,21 +2414,17 @@ export class IModelTransformer extends IModelExportHandler { return; } - const relArg = + const id = deletedRelData.relId ?? - ({ + this.targetDb.relationships.tryGetInstance(deletedRelData.classFullName, { sourceId: deletedRelData.sourceIdInTarget, targetId: deletedRelData.targetIdInTarget, - } as SourceAndTarget); - - // FIXME: make importer.deleteRelationship not need full props - const targetRelationship = this.targetDb.relationships.tryGetInstance( - deletedRelData.classFullName, - relArg - ); - - if (targetRelationship) { - this.importer.deleteRelationship(targetRelationship.toJSON()); + } as SourceAndTarget)?.id; + if (id) { + this.importer.deleteRelationship({ + id, + classFullName: deletedRelData.classFullName, + }); } if (deletedRelData.provenanceAspectId) { @@ -2489,12 +2485,10 @@ export class IModelTransformer extends IModelExportHandler { const targetRelInstanceId = json.targetRelInstanceId ?? json.provenanceRelInstanceId; if (targetRelInstanceId) { - const targetRelationship: Relationship = - this.targetDb.relationships.getInstance( - ElementRefersToElements.classFullName, - targetRelInstanceId - ); - this.importer.deleteRelationship(targetRelationship.toJSON()); + this.importer.deleteRelationship({ + id: targetRelInstanceId, + classFullName: ElementRefersToElements.classFullName, + }); } aspectDeleteIds.push(statement.getValue(0).getId()); } @@ -3160,7 +3154,6 @@ export class IModelTransformer extends IModelExportHandler { await this.exporter.exportFonts(); if (this._options.skipPropagateChangesToRootElements) { - // FIXME: This option in exportAll was a maybe. // The RepositoryModel and root Subject of the target iModel should not be transformed. await this.exporter.exportChildElements(IModel.rootSubjectId); // start below the root Subject await this.exporter.exportModelContents( @@ -3213,269 +3206,6 @@ export class IModelTransformer extends IModelExportHandler { }; } - /** @internal the name of the table where javascript state of the transformer is serialized in transformer state dumps */ - public static readonly jsStateTable = "TransformerJsState"; - - /** @internal the name of the table where the target state heuristics is serialized in transformer state dumps */ - public static readonly lastProvenanceEntityInfoTable = - "LastProvenanceEntityInfo"; - - /** - * Load the state of the active transformation from an open SQLiteDb - * You can override this if you'd like to load from custom tables in the resumable dump state, but you should call - * this super implementation - * @note the SQLiteDb must be open - */ - protected loadStateFromDb(db: SQLiteDb): void { - const lastProvenanceEntityInfo: IModelTransformer["_lastProvenanceEntityInfo"] = - db.withSqliteStatement( - `SELECT entityId, aspectId, aspectVersion, aspectKind FROM ${IModelTransformer.lastProvenanceEntityInfoTable}`, - (stmt) => { - if (DbResult.BE_SQLITE_ROW !== stmt.step()) - throw Error( - "expected row when getting lastProvenanceEntityId from target state table" - ); - const entityId = stmt.getValueString(0); - const isGuidOrGuidPair = entityId.includes("-"); - return isGuidOrGuidPair - ? entityId - : { - entityId, - aspectId: stmt.getValueString(1), - aspectVersion: stmt.getValueString(2), - aspectKind: stmt.getValueString(3) as ExternalSourceAspect.Kind, - }; - } - ); - - /* - // TODO: maybe save transformer state resumption state based on target changset and require calls - // to saveChanges - if () { - const [sourceFedGuid, targetFedGuid, relClassFullName] = lastProvenanceEntityInfo.split("/"); - const isRelProvenance = targetFedGuid !== undefined; - const instanceId = isRelProvenance - ? this.targetDb.elements.getElement({federationGuid: sourceFedGuid}) - : ""; - //const classId = - if (isRelProvenance) { - } - } - */ - - const targetHasCorrectLastProvenance = - typeof lastProvenanceEntityInfo === "string" || - // ignore provenance check if it's null since we can't bind those ids - !Id64.isValidId64(lastProvenanceEntityInfo.entityId) || - !Id64.isValidId64(lastProvenanceEntityInfo.aspectId) || - this.provenanceDb.withPreparedStatement( - ` - SELECT Version FROM ${ExternalSourceAspect.classFullName} - WHERE Scope.Id=:scopeId - AND ECInstanceId=:aspectId - AND Kind=:kind - AND Element.Id=:entityId - `, - (statement: ECSqlStatement): boolean => { - statement.bindId("scopeId", this.targetScopeElementId); - statement.bindId("aspectId", lastProvenanceEntityInfo.aspectId); - statement.bindString("kind", lastProvenanceEntityInfo.aspectKind); - statement.bindId("entityId", lastProvenanceEntityInfo.entityId); - const stepResult = statement.step(); - switch (stepResult) { - case DbResult.BE_SQLITE_ROW: - const version = statement.getValue(0).getString(); - return version === lastProvenanceEntityInfo.aspectVersion; - case DbResult.BE_SQLITE_DONE: - return false; - default: - throw new IModelError( - IModelStatus.SQLiteError, - `got sql error ${stepResult}` - ); - } - } - ); - - if (!targetHasCorrectLastProvenance) - throw Error( - [ - "Target for resuming from does not have the expected provenance ", - "from the target that the resume state was made with", - ].join("\n") - ); - this._lastProvenanceEntityInfo = lastProvenanceEntityInfo; - - const state = db.withSqliteStatement( - `SELECT data FROM ${IModelTransformer.jsStateTable}`, - (stmt) => { - if (DbResult.BE_SQLITE_ROW !== stmt.step()) - throw Error("expected row when getting data from js state table"); - return JSON.parse(stmt.getValueString(0)) as TransformationJsonState; - } - ); - if (state.transformerClass !== this.constructor.name) - throw Error( - "resuming from a differently named transformer class, it is not necessarily valid to resume with a different transformer class" - ); - // force assign to readonly options since we do not know how the transformer subclass takes options to pass to the superclass - (this as any)._options = state.options; - this.context.loadStateFromDb(db); - this.importer.loadStateFromJson(state.importerState); - this.exporter.loadStateFromJson(state.exporterState); - this._elementsWithExplicitlyTrackedProvenance = - CompressedId64Set.decompressSet(state.explicitlyTrackedElements); - this.loadAdditionalStateJson(state.additionalState); - } - - /** - * @deprecated in 0.1.x, this is buggy, and it is now equivalently efficient to simply restart the transformation - * from the original changeset - * - * Return a new transformer instance with the same remappings state as saved from a previous [[IModelTransformer.saveStateToFile]] call. - * This allows you to "resume" an iModel transformation, you will have to call [[IModelTransformer.processChanges]]/[[IModelTransformer.processAll]] - * again but the remapping state will cause already mapped elements to be skipped. - * To "resume" an iModel Transformation you need: - * - the sourceDb at the same changeset - * - the same targetDb in the state in which it was before - * @param statePath the path to the serialized state of the transformer, use [[IModelTransformer.saveStateToFile]] to get this from an existing transformer instance - * @param constructorArgs remaining arguments that you would normally pass to the Transformer subclass you are using, usually (sourceDb, targetDb) - * @note custom transformers with custom state may need to override this method in order to handle loading their own custom state somewhere - */ - public static resumeTransformation< - SubClass extends new ( - ...a: any[] - ) => IModelTransformer = typeof IModelTransformer, - >( - this: SubClass, - statePath: string, - ...constructorArgs: ConstructorParameters - ): InstanceType { - const transformer = new this(...constructorArgs); - const db = new SQLiteDb(); - db.openDb(statePath, OpenMode.Readonly); - try { - transformer.loadStateFromDb(db); - } finally { - db.closeDb(); - } - return transformer as InstanceType; - } - - /** - * You may override this to store arbitrary json state in a transformer state dump, useful for some resumptions - * @see [[IModelTransformer.saveStateToFile]] - */ - protected getAdditionalStateJson(): any { - return {}; - } - - /** - * You may override this to load arbitrary json state in a transformer state dump, useful for some resumptions - * @see [[IModelTransformer.loadStateFromFile]] - */ - protected loadAdditionalStateJson(_additionalState: any): void {} - - /** - * Save the state of the active transformation to an open SQLiteDb - * You can override this if you'd like to write custom tables to the resumable dump state, but you should call - * this super implementation - * @note the SQLiteDb must be open - */ - protected saveStateToDb(db: SQLiteDb): void { - const jsonState: TransformationJsonState = { - transformerClass: this.constructor.name, - options: this._options, - explicitlyTrackedElements: CompressedId64Set.compressSet( - this._elementsWithExplicitlyTrackedProvenance - ), - importerState: this.importer.saveStateToJson(), - exporterState: this.exporter.saveStateToJson(), - additionalState: this.getAdditionalStateJson(), - }; - - this.context.saveStateToDb(db); - if ( - DbResult.BE_SQLITE_DONE !== - db.executeSQL( - `CREATE TABLE ${IModelTransformer.jsStateTable} (data TEXT)` - ) - ) - throw Error("Failed to create the js state table in the state database"); - - if ( - DbResult.BE_SQLITE_DONE !== - db.executeSQL(` - CREATE TABLE ${IModelTransformer.lastProvenanceEntityInfoTable} ( - -- either the invalid id for null provenance state, federation guid (or pair for rels) of the entity, or a hex element id - entityId TEXT, - -- the following are only valid if the above entityId is a hex id representation - aspectId TEXT, - aspectVersion TEXT, - aspectKind TEXT - ) - `) - ) - throw Error( - "Failed to create the target state table in the state database" - ); - - db.saveChanges(); - db.withSqliteStatement( - `INSERT INTO ${IModelTransformer.jsStateTable} (data) VALUES (?)`, - (stmt) => { - stmt.bindString(1, JSON.stringify(jsonState)); - if (DbResult.BE_SQLITE_DONE !== stmt.step()) - throw Error("Failed to insert options into the state database"); - } - ); - - db.withSqliteStatement( - `INSERT INTO ${IModelTransformer.lastProvenanceEntityInfoTable} (entityId, aspectId, aspectVersion, aspectKind) VALUES (?,?,?,?)`, - (stmt) => { - const lastProvenanceEntityInfo = this - ._lastProvenanceEntityInfo as LastProvenanceEntityInfo; - stmt.bindString( - 1, - lastProvenanceEntityInfo?.entityId ?? - (this._lastProvenanceEntityInfo as string) - ); - stmt.bindString(2, lastProvenanceEntityInfo?.aspectId ?? ""); - stmt.bindString(3, lastProvenanceEntityInfo?.aspectVersion ?? ""); - stmt.bindString(4, lastProvenanceEntityInfo?.aspectKind ?? ""); - if (DbResult.BE_SQLITE_DONE !== stmt.step()) - throw Error("Failed to insert options into the state database"); - } - ); - - db.saveChanges(); - } - - /** - * @deprecated in 0.1.x, this is buggy, and it is now equivalently efficient to simply restart the transformation - * from the original changeset - * - * Save the state of the active transformation to a file path, if a file at the path already exists, it will be overwritten - * This state can be used by [[IModelTransformer.resumeTransformation]] to resume a transformation from this point. - * The serialization format is a custom sqlite database. - * @note custom transformers with custom state may override [[IModelTransformer.saveStateToDb]] or [[IModelTransformer.getAdditionalStateJson]] - * and [[IModelTransformer.loadStateFromDb]] (with a super call) or [[IModelTransformer.loadAdditionalStateJson]] - * if they have custom state that needs to be stored with - * potentially inside the same sqlite file in separate tables - */ - public saveStateToFile(nativeStatePath: string): void { - const db = new SQLiteDb(); - if (IModelJsFs.existsSync(nativeStatePath)) - IModelJsFs.unlinkSync(nativeStatePath); - db.createDb(nativeStatePath); - try { - this.saveStateToDb(db); - db.saveChanges(); - } finally { - db.closeDb(); - } - } - /** Export changes from the source iModel and import the transformed entities into the target iModel. * Inserts, updates, and deletes are determined by inspecting the changeset(s). * @note the transformer saves and pushes changes when its work is complete. @@ -3541,16 +3271,6 @@ export class IModelTransformer extends IModelExportHandler { } } -/** @internal the json part of a transformation's state */ -interface TransformationJsonState { - transformerClass: string; - options: IModelTransformOptions; - importerState: IModelImporterState; - exporterState: IModelExporterState; - explicitlyTrackedElements: CompressedId64Set; - additionalState?: any; -} - /** IModelTransformer that clones the contents of a template model. * @beta */ diff --git a/packages/transformer/src/test/IModelTransformerUtils.ts b/packages/transformer/src/test/IModelTransformerUtils.ts index f7f33d6d..1109217a 100644 --- a/packages/transformer/src/test/IModelTransformerUtils.ts +++ b/packages/transformer/src/test/IModelTransformerUtils.ts @@ -89,10 +89,11 @@ import { ViewDetails3dProps, } from "@itwin/core-common"; import { IModelExporter, IModelExportHandler } from "../IModelExporter"; -import { IModelImportOptions, IModelImporter } from "../IModelImporter"; +import { IModelImporter, IModelImportOptions } from "../IModelImporter"; import { IModelTransformer, IModelTransformOptions, + RelationshipPropsForDelete, } from "../IModelTransformer"; import { KnownTestLocations } from "./TestUtils/KnownTestLocations"; @@ -1967,7 +1968,7 @@ export class CountingIModelImporter extends IModelImporter { super.onUpdateRelationship(relationshipProps); } protected override onDeleteRelationship( - relationshipProps: RelationshipProps + relationshipProps: RelationshipPropsForDelete ): void { this.numRelationshipsDeleted++; super.onDeleteRelationship(relationshipProps); diff --git a/packages/transformer/src/test/standalone/IModelTransformerHub.test.ts b/packages/transformer/src/test/standalone/IModelTransformerHub.test.ts index 475d28e1..7cf2f315 100644 --- a/packages/transformer/src/test/standalone/IModelTransformerHub.test.ts +++ b/packages/transformer/src/test/standalone/IModelTransformerHub.test.ts @@ -886,7 +886,6 @@ describe("IModelTransformerHub", () => { }); it("should be able to handle relationship delete using fedguids", async () => { - // FIXME: This test should be removed once we have more structured testing with a case matrix const masterIModelName = "MasterNewRelProvenanceFedGuids"; const masterSeedFileName = path.join(outputDir, `${masterIModelName}.bim`); if (IModelJsFs.existsSync(masterSeedFileName)) @@ -973,7 +972,6 @@ describe("IModelTransformerHub", () => { }); it("should be able to handle relationship delete using new relationship provenance method with no fedguids", async () => { - // FIXME: This test should be removed once we have more structured testing with a case matrix // SEE: https://github.com/iTwin/imodel-transformer/issues/54 for the scenario this test exercises /** This test does the following: * sync master to branch with two elements, x and y, with NULL fed guid to force ESAs to be generated (For future relationship) @@ -1090,7 +1088,6 @@ describe("IModelTransformerHub", () => { }); it("should be able to handle relationship delete using old relationship provenance method with no fedguids", async () => { - // FIXME: This test should be removed once we have more structured testing with a case matrix // SEE: https://github.com/iTwin/imodel-transformer/issues/54 for the scenario this test exercises /** This test does the following: * sync master to branch with two elements, x and y, with NULL fed guid to force ESAs to be generated (For future relationship) @@ -3724,9 +3721,6 @@ describe("IModelTransformerHub", () => { }); } - // FIXME: As a side effect of fixing a bug in findRangeContaining, we error out with no changesummary data because we now properly skip changesetindices - // i.e. a range [4,4] with skip 4 now properly gets skipped. so we have no changesummary data. We need to revisit this after switching to affan's new API - // to read changesets directly. it("should skip provenance changesets made to branch during reverse sync", async () => { const timeline: Timeline = [ { master: { 1: 1 } }, diff --git a/packages/transformer/src/test/standalone/IModelTransformerResumption.test.ts b/packages/transformer/src/test/standalone/IModelTransformerResumption.test.ts deleted file mode 100644 index 5b334996..00000000 --- a/packages/transformer/src/test/standalone/IModelTransformerResumption.test.ts +++ /dev/null @@ -1,1208 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Bentley Systems, Incorporated. All rights reserved. - * See LICENSE.md in the project root for license terms and full copyright notice. - *--------------------------------------------------------------------------------------------*/ - -// this API is deprecated so no point in maintaining the tests -/* eslint-disable deprecation/deprecation */ - -import { - BriefcaseDb, - Element, - HubMock, - IModelDb, - IModelHost, - IModelJsNative, - Relationship, - SnapshotDb, - SQLiteDb, -} from "@itwin/core-backend"; -import * as TestUtils from "../TestUtils"; -import { - AccessToken, - DbResult, - GuidString, - Id64, - Id64String, - StopWatch, -} from "@itwin/core-bentley"; -import { ChangesetId, ElementProps } from "@itwin/core-common"; -import { assert, expect } from "chai"; -import * as sinon from "sinon"; -import { IModelImporter } from "../../IModelImporter"; -import { IModelExporter } from "../../IModelExporter"; -import { - IModelTransformer, - IModelTransformOptions, -} from "../../IModelTransformer"; -import { - assertIdentityTransformation, - HubWrappers, - IModelTransformerTestUtils, -} from "../IModelTransformerUtils"; -import { KnownTestLocations } from "../TestUtils/KnownTestLocations"; - -import "./TransformerTestStartup"; // calls startup/shutdown IModelHost before/after all tests - -const formatter = new Intl.NumberFormat("en-US", { - maximumFractionDigits: 2, - minimumFractionDigits: 2, -}); - -/** format a proportion (number between 0 and 1) to a percent */ -const percent = (n: number) => `${formatter.format(100 * n)}%`; - -class CountingImporter extends IModelImporter { - public owningTransformer: CountingTransformer | undefined; - public override importElement(elementProps: ElementProps): Id64String { - if (this.owningTransformer === undefined) - throw Error( - "uninitialized, '_owningTransformer' must have been set before transformations" - ); - ++this.owningTransformer.importedEntities; - return super.importElement(elementProps); - } -} - -/** this class services two functions, - * 1. make sure transformations can be resumed by subclasses with different constructor argument types - * 2. count operations that should not be reperformed by a resumed transformation - */ -class CountingTransformer extends IModelTransformer { - public importedEntities = 0; - public exportedEntities = 0; - constructor(opts: { - source: IModelDb; - target: IModelDb; - options?: IModelTransformOptions; - }) { - super(opts.source, new CountingImporter(opts.target), opts.options); - (this.importer as CountingImporter).owningTransformer = this; - } - public override onExportElement(sourceElement: Element) { - ++this.exportedEntities; - return super.onExportElement(sourceElement); - } - public override onExportRelationship(sourceRelationship: Relationship) { - ++this.exportedEntities; - return super.onExportRelationship(sourceRelationship); - } -} - -/** a transformer that executes a callback after X element exports */ -class CountdownTransformer extends IModelTransformer { - public relationshipExportsUntilCall: number | undefined; - public elementExportsUntilCall: number | undefined; - public callback: (() => Promise) | undefined; - public constructor(...args: ConstructorParameters) { - super(...args); - const _this = this; // eslint-disable-line @typescript-eslint/no-this-alias - const oldExportElem = this.exporter.exportElement; // eslint-disable-line @typescript-eslint/unbound-method - this.exporter.exportElement = async function (...args) { - if (_this.elementExportsUntilCall === 0) await _this.callback?.(); - - if (_this.elementExportsUntilCall !== undefined) - _this.elementExportsUntilCall--; - - return oldExportElem.call(this, ...args); - }; - const oldExportRel = this.exporter.exportRelationship; // eslint-disable-line @typescript-eslint/unbound-method - this.exporter.exportRelationship = async function (...args) { - if (_this.relationshipExportsUntilCall === 0) await _this.callback?.(); - - if (_this.relationshipExportsUntilCall !== undefined) - _this.relationshipExportsUntilCall--; - - return oldExportRel.call(this, ...args); - }; - } -} - -/** a transformer that crashes on the nth element export, set `elementExportsUntilCall` to control the count */ -class CountdownToCrashTransformer extends CountdownTransformer { - public constructor( - ...args: ConstructorParameters - ) { - super(...args); - this.callback = () => { - throw Error("crash"); - }; - } -} - -/** - * Wraps all IModel native addon functions and constructors in a randomly throwing wrapper, - * as well as all IModelTransformer functions - * @note you must call sinon.restore at the end of any test that uses this - */ -function setupCrashingNativeAndTransformer({ - methodCrashProbability = 1 / 800, - onCrashableCallMade, -}: { - methodCrashProbability?: number; - onCrashableCallMade?(): void; -} = {}) { - let crashingEnabled = false; - for (const [key, descriptor] of Object.entries( - Object.getOwnPropertyDescriptors(IModelHost.platform) - ) as [keyof (typeof IModelHost)["platform"], PropertyDescriptor][]) { - const superValue: unknown = descriptor.value; - if (typeof superValue === "function" && descriptor.writable) { - sinon.replace( - IModelHost.platform, - key, - function (this: IModelJsNative.DgnDb, ...args: any[]) { - onCrashableCallMade?.(); - if (crashingEnabled) { - // this does not at all test mid-method crashes... that might be doable by racing timeouts on async functions... - if (crashingEnabled && Math.random() <= methodCrashProbability) - throw Error("fake native crash"); - } - const isConstructor = (o: Function): o is new (...a: any[]) => any => - "prototype" in o; - if (isConstructor(superValue)) return new superValue(...args); - else return superValue.call(this, ...args); - } - ); - } - } - - for (const [key, descriptor] of Object.entries( - Object.getOwnPropertyDescriptors(IModelTransformer.prototype) - ) as [keyof IModelTransformer, PropertyDescriptor][]) { - const superValue: unknown = descriptor.value; - if (typeof superValue === "function" && descriptor.writable) { - sinon.replace( - IModelTransformer.prototype, - key, - function (this: IModelTransformer, ...args: any[]) { - onCrashableCallMade?.(); - if (crashingEnabled) { - // this does not at all test mid-method crashes... that might be doable by racing timeouts on async functions... - if (crashingEnabled && Math.random() <= methodCrashProbability) - throw Error("fake crash"); - } - return superValue.call(this, ...args); - } - ); - } - } - - return { - enableCrashes: (val = true) => { - crashingEnabled = val; - }, - }; -} - -async function transformWithCrashAndRecover< - Transformer extends IModelTransformer, - DbType extends IModelDb, ->({ - sourceDb, - targetDb, - transformer, - disableCrashing, - transformerProcessing = async (t, time = 0) => { - if (time === 0) await t.processSchemas(); - - await t.processAll(); - }, -}: { - sourceDb: DbType; - targetDb: DbType; - transformer: Transformer; - /* what processing to do for the transform; default impl is above */ - transformerProcessing?: ( - transformer: Transformer, - time?: number - ) => Promise; - disableCrashing?: (transformer: Transformer) => void; -}) { - let crashed = false; - try { - await transformerProcessing(transformer); - } catch (transformerErr) { - expect((transformerErr as Error).message).to.equal("crash"); - crashed = true; - const dumpPath = IModelTransformerTestUtils.prepareOutputFile( - "IModelTransformerResumption", - "transformer-state.db" - ); - transformer.saveStateToFile(dumpPath); - // eslint-disable-next-line @typescript-eslint/naming-convention - const TransformerClass = - transformer.constructor as typeof IModelTransformer; - transformer = TransformerClass.resumeTransformation( - dumpPath, - sourceDb, - targetDb - ) as Transformer; - disableCrashing?.(transformer); - await transformerProcessing(transformer); - } - - expect(crashed).to.be.true; - return targetDb; -} - -/** this utility is for consistency and readability, it doesn't provide any actual abstraction */ -async function transformNoCrash({ - targetDb, - transformer, - transformerProcessing = async (t) => { - await t.processSchemas(); - await t.processAll(); - }, -}: { - sourceDb: IModelDb; - targetDb: IModelDb; - transformer: Transformer; - transformerProcessing?: (transformer: Transformer) => Promise; -}): Promise { - await transformerProcessing(transformer); - return targetDb; -} - -describe.skip("test resuming transformations", () => { - let iTwinId: GuidString; - let accessToken: AccessToken; - let seedDbId: GuidString; - let seedDb: BriefcaseDb; - - before(async () => { - HubMock.startup( - "IModelTransformerResumption", - KnownTestLocations.outputDir - ); - iTwinId = HubMock.iTwinId; - accessToken = await HubWrappers.getAccessToken( - TestUtils.TestUserType.Regular - ); - const seedPath = IModelTransformerTestUtils.prepareOutputFile( - "IModelTransformerResumption", - "seed.bim" - ); - SnapshotDb.createEmpty(seedPath, { - rootSubject: { name: "resumption-tests-seed" }, - }).close(); - seedDbId = await IModelHost.hubAccess.createNewIModel({ - iTwinId, - iModelName: "ResumeTestsSeed", - description: "seed for resumption tests", - version0: seedPath, - noLocks: true, - }); - seedDb = await HubWrappers.downloadAndOpenBriefcase({ - accessToken, - iTwinId, - iModelId: seedDbId, - }); - await TestUtils.ExtensiveTestScenario.prepareDb(seedDb); - TestUtils.ExtensiveTestScenario.populateDb(seedDb); - seedDb.performCheckpoint(); - await seedDb.pushChanges({ accessToken, description: "populated seed db" }); - }); - - after(async () => { - await HubWrappers.closeAndDeleteBriefcaseDb(accessToken, seedDb); - HubMock.shutdown(); - }); - - it("resume old state after partially committed changes", async () => { - const sourceDb = seedDb; - - const [regularTransformer, regularTarget] = await (async () => { - const targetDbId = await IModelHost.hubAccess.createNewIModel({ - iTwinId, - iModelName: "targetDb2", - description: "non crashing target", - noLocks: true, - }); - const targetDb = await HubWrappers.downloadAndOpenBriefcase({ - accessToken, - iTwinId, - iModelId: targetDbId, - }); - const transformer = new IModelTransformer(sourceDb, targetDb); - await transformNoCrash({ sourceDb, targetDb, transformer }); - targetDb.saveChanges(); - return [transformer, targetDb] as const; - })(); - - const [resumedTransformer, resumedTarget] = await (async () => { - const targetDbId = await IModelHost.hubAccess.createNewIModel({ - iTwinId, - iModelName: "targetDb1", - description: "crashingTarget", - noLocks: true, - }); - let targetDb = await HubWrappers.downloadAndOpenBriefcase({ - accessToken, - iTwinId, - iModelId: targetDbId, - }); - let changesetId: ChangesetId; - const dumpPath = IModelTransformerTestUtils.prepareOutputFile( - "IModelTransformerResumption", - "transformer-state.db" - ); - let transformer = new CountdownTransformer(sourceDb, targetDb); - // after exporting 10 elements, save and push changes - transformer.elementExportsUntilCall = 10; - transformer.callback = async () => { - targetDb.saveChanges(); - await targetDb.pushChanges({ - accessToken, - description: "early state save", - }); - transformer.saveStateToFile(dumpPath); - changesetId = targetDb.changeset.id; - // now after another 10 exported elements, interrupt for resumption - transformer.elementExportsUntilCall = 10; - transformer.callback = () => { - throw Error("interrupt"); - }; - }; - let interrupted = false; - try { - await transformer.processSchemas(); - // will trigger the callback after 10 exported elements, which triggers another - // callback to be installed for after 20 elements, to throw an error to interrupt the transformation - await transformer.processAll(); - } catch (transformerErr) { - expect((transformerErr as Error).message).to.equal("interrupt"); - interrupted = true; - // redownload to simulate restarting without any JS state - expect(targetDb.nativeDb.hasUnsavedChanges()).to.be.true; - targetDb.close(); - targetDb = await HubWrappers.downloadAndOpenBriefcase({ - accessToken, - iTwinId, - iModelId: targetDbId, - }); - expect(targetDb.nativeDb.hasUnsavedChanges()).to.be.false; - expect(targetDb.changeset.id).to.equal(changesetId!); - // eslint-disable-next-line @typescript-eslint/naming-convention - const TransformerClass = - transformer.constructor as typeof IModelTransformer; - transformer = TransformerClass.resumeTransformation( - dumpPath, - sourceDb, - targetDb - ) as CountdownTransformer; - await transformer.processAll(); - targetDb.saveChanges(); - return [transformer, targetDb] as const; - } - expect(interrupted, "should have interrupted rather than completing").to - .be.true; - throw Error("unreachable"); - })(); - - const [elemMap, codeSpecMap, aspectMap] = new Array(3) - .fill(undefined) - .map(() => new Map()); - for (const [className, findMethod, map] of [ - ["bis.Element", "findTargetElementId", elemMap], - ["bis.CodeSpec", "findTargetCodeSpecId", codeSpecMap], - ["bis.ElementAspect", "findTargetAspectId", aspectMap], - ] as const) { - // eslint-disable-next-line deprecation/deprecation - for await (const [sourceElemId] of sourceDb.query( - `SELECT ECInstanceId from ${className}` - )) { - const idInRegular = - regularTransformer.context[findMethod](sourceElemId); - const idInResumed = - resumedTransformer.context[findMethod](sourceElemId); - map.set(idInRegular, idInResumed); - } - } - - await assertIdentityTransformation(regularTarget, resumedTarget, { - findTargetElementId: (id) => elemMap.get(id) ?? Id64.invalid, - findTargetCodeSpecId: (id) => codeSpecMap.get(id) ?? Id64.invalid, - findTargetAspectId: (id) => aspectMap.get(id) ?? Id64.invalid, - }); - await HubWrappers.closeAndDeleteBriefcaseDb(accessToken, resumedTarget); - await HubWrappers.closeAndDeleteBriefcaseDb(accessToken, regularTarget); - }); - - it("simple single crash transform resumption", async () => { - const sourceDbId = await IModelHost.hubAccess.createNewIModel({ - iTwinId, - iModelName: "sourceDb1", - description: "a db called sourceDb1", - noLocks: true, - version0: seedDb.pathName, - }); - const sourceDb = await HubWrappers.downloadAndOpenBriefcase({ - accessToken, - iTwinId, - iModelId: sourceDbId, - }); - - const crashingTarget = await (async () => { - const targetDbId = await IModelHost.hubAccess.createNewIModel({ - iTwinId, - iModelName: "targetDb1", - description: "crashingTarget", - noLocks: true, - }); - const targetDb = await HubWrappers.downloadAndOpenBriefcase({ - accessToken, - iTwinId, - iModelId: targetDbId, - }); - const transformer = new CountdownToCrashTransformer(sourceDb, targetDb); - transformer.elementExportsUntilCall = 10; - await transformWithCrashAndRecover({ - sourceDb, - targetDb, - transformer, - disableCrashing(t) { - t.elementExportsUntilCall = undefined; - }, - }); - targetDb.saveChanges(); - transformer.dispose(); - return targetDb; - })(); - - const regularTarget = await (async () => { - const targetDbId = await IModelHost.hubAccess.createNewIModel({ - iTwinId, - iModelName: "targetDb2", - description: "non crashing target", - noLocks: true, - }); - const targetDb = await HubWrappers.downloadAndOpenBriefcase({ - accessToken, - iTwinId, - iModelId: targetDbId, - }); - const transformer = new IModelTransformer(sourceDb, targetDb); - await transformNoCrash({ sourceDb, targetDb, transformer }); - targetDb.saveChanges(); - transformer.dispose(); - return targetDb; - })(); - - await assertIdentityTransformation(regularTarget, crashingTarget); - await HubWrappers.closeAndDeleteBriefcaseDb(accessToken, sourceDb); - await HubWrappers.closeAndDeleteBriefcaseDb(accessToken, crashingTarget); - await HubWrappers.closeAndDeleteBriefcaseDb(accessToken, regularTarget); - }); - - // FIXME: not (yet?) implemented for federation guid optimization branch - it.skip("should fail to resume from an old target", async () => { - const sourceDbId = await IModelHost.hubAccess.createNewIModel({ - iTwinId, - iModelName: "sourceDb1", - description: "a db called sourceDb1", - noLocks: true, - version0: seedDb.pathName, - }); - const sourceDb = await HubWrappers.downloadAndOpenBriefcase({ - accessToken, - iTwinId, - iModelId: sourceDbId, - }); - - const targetDbId = await IModelHost.hubAccess.createNewIModel({ - iTwinId, - iModelName: "targetDb1", - description: "crashingTarget", - noLocks: true, - }); - let targetDb = await HubWrappers.downloadAndOpenBriefcase({ - accessToken, - iTwinId, - iModelId: targetDbId, - }); - const transformer = new CountdownToCrashTransformer(sourceDb, targetDb); - transformer.elementExportsUntilCall = 10; - let crashed = false; - try { - await transformer.processSchemas(); - await transformer.processAll(); - } catch (transformerErr) { - expect((transformerErr as Error).message).to.equal("crash"); - crashed = true; - const dumpPath = IModelTransformerTestUtils.prepareOutputFile( - "IModelTransformerResumption", - "transformer-state.db" - ); - transformer.saveStateToFile(dumpPath); - // eslint-disable-next-line @typescript-eslint/naming-convention - const TransformerClass = - transformer.constructor as typeof IModelTransformer; - // redownload targetDb so that it is reset to the old state - targetDb.close(); - targetDb = await HubWrappers.downloadAndOpenBriefcase({ - accessToken, - iTwinId, - iModelId: targetDbId, - }); - expect(() => - TransformerClass.resumeTransformation(dumpPath, sourceDb, targetDb) - ).to.throw(/does not have the expected provenance/); - } - - expect(crashed).to.be.true; - transformer.dispose(); - await HubWrappers.closeAndDeleteBriefcaseDb(accessToken, sourceDb); - await HubWrappers.closeAndDeleteBriefcaseDb(accessToken, targetDb); - return targetDb; - }); - - /* eslint-disable @typescript-eslint/naming-convention */ - it("should fail to resume from a different(ly named) transformer classes", async () => { - async function testResumeCrashTransformerWithClasses({ - StartTransformerClass = IModelTransformer, - StartImporterClass = IModelImporter, - StartExporterClass = IModelExporter, - ResumeTransformerClass = StartTransformerClass, - ResumeImporterClass = StartImporterClass, - ResumeExporterClass = StartExporterClass, - }: { - StartTransformerClass?: typeof IModelTransformer; - StartImporterClass?: typeof IModelImporter; - StartExporterClass?: typeof IModelExporter; - ResumeTransformerClass?: typeof IModelTransformer; - ResumeImporterClass?: typeof IModelImporter; - ResumeExporterClass?: typeof IModelExporter; - } = {}) { - const sourceDb = seedDb; - const targetDb = SnapshotDb.createFrom( - seedDb, - IModelTransformerTestUtils.prepareOutputFile( - "IModelTransformerResumption", - "ResumeDifferentClass.bim" - ) - ); - - const transformer = new StartTransformerClass( - new StartExporterClass(sourceDb), - new StartImporterClass(targetDb) - ); - let crashed = false; - try { - await transformer.processSchemas(); - await transformer.processAll(); - } catch (transformerErr) { - expect((transformerErr as Error).message).to.equal("crash"); - crashed = true; - const dumpPath = IModelTransformerTestUtils.prepareOutputFile( - "IModelTransformerResumption", - "transformer-state.db" - ); - transformer.saveStateToFile(dumpPath); - expect(() => - ResumeTransformerClass.resumeTransformation( - dumpPath, - new ResumeExporterClass(sourceDb), - new ResumeImporterClass(targetDb) - ) - ).to.throw(/it is not.*valid to resume with a different.*class/); - } - - expect(crashed).to.be.true; - transformer.dispose(); - await HubWrappers.closeAndDeleteBriefcaseDb(accessToken, targetDb); - } - - class CrashOn2Transformer extends CountdownToCrashTransformer { - public constructor( - ...args: ConstructorParameters - ) { - super(...args); - this.elementExportsUntilCall = 2; - } - } - - class DifferentTransformerClass extends CrashOn2Transformer {} - class DifferentImporterClass extends IModelImporter {} - class DifferentExporterClass extends IModelExporter {} - - await testResumeCrashTransformerWithClasses({ - StartTransformerClass: CrashOn2Transformer, - ResumeTransformerClass: DifferentTransformerClass, - }); - await testResumeCrashTransformerWithClasses({ - StartTransformerClass: CrashOn2Transformer, - ResumeImporterClass: DifferentImporterClass, - }); - await testResumeCrashTransformerWithClasses({ - StartTransformerClass: CrashOn2Transformer, - ResumeExporterClass: DifferentExporterClass, - }); - }); - /* eslint-enable @typescript-eslint/naming-convention */ - - it("should save custom additional state", async () => { - class AdditionalStateImporter extends IModelImporter { - public state1 = "importer"; - protected override getAdditionalStateJson() { - return { state1: this.state1 }; - } - protected override loadAdditionalStateJson(additionalState: any) { - this.state1 = additionalState.state1; - } - } - class AdditionalStateExporter extends IModelExporter { - public state1 = "exporter"; - protected override getAdditionalStateJson() { - return { state1: this.state1 }; - } - protected override loadAdditionalStateJson(additionalState: any) { - this.state1 = additionalState.state1; - } - } - class AdditionalStateTransformer extends IModelTransformer { - public state1 = "default"; - public state2 = 42; - protected override saveStateToDb(db: SQLiteDb): void { - super.saveStateToDb(db); - db.executeSQL( - "CREATE TABLE additionalState (state1 TEXT, state2 INTEGER)" - ); - db.saveChanges(); - db.withSqliteStatement( - "INSERT INTO additionalState (state1) VALUES (?)", - (stmt) => { - stmt.bindString(1, this.state1); - expect(stmt.step()).to.equal(DbResult.BE_SQLITE_DONE); - } - ); - } - protected override loadStateFromDb(db: SQLiteDb): void { - super.loadStateFromDb(db); - db.withSqliteStatement("SELECT state1 FROM additionalState", (stmt) => { - expect(stmt.step()).to.equal(DbResult.BE_SQLITE_ROW); - this.state1 = stmt.getValueString(0); - }); - } - protected override getAdditionalStateJson() { - return { state2: this.state2 }; - } - protected override loadAdditionalStateJson(additionalState: any) { - this.state2 = additionalState.state2; - } - } - - const sourceDb = seedDb; - const targetDb = SnapshotDb.createEmpty( - IModelTransformerTestUtils.prepareOutputFile( - "IModelTransformerResumption", - "CustomAdditionalState.bim" - ), - { rootSubject: { name: "CustomAdditionalStateTest" } } - ); - - const transformer = new AdditionalStateTransformer( - new AdditionalStateExporter(sourceDb), - new AdditionalStateImporter(targetDb) - ); - transformer.state1 = "transformer-state-1"; - transformer.state2 = 43; - (transformer.importer as AdditionalStateImporter).state1 = - "importer-state-1"; - (transformer.exporter as AdditionalStateExporter).state1 = - "exporter-state-1"; - - const dumpPath = IModelTransformerTestUtils.prepareOutputFile( - "IModelTransformerResumption", - "transformer-state.db" - ); - transformer.saveStateToFile(dumpPath); - // eslint-disable-next-line @typescript-eslint/naming-convention - const TransformerClass = - transformer.constructor as typeof AdditionalStateTransformer; - transformer.dispose(); - const resumedTransformer = TransformerClass.resumeTransformation( - dumpPath, - new AdditionalStateExporter(sourceDb), - new AdditionalStateImporter(targetDb) - ); - expect(resumedTransformer).not.to.equal(transformer); - expect(resumedTransformer.state1).to.equal(transformer.state1); - expect(resumedTransformer.state2).to.equal(transformer.state2); - expect( - (resumedTransformer.importer as AdditionalStateImporter).state1 - ).to.equal((transformer.importer as AdditionalStateImporter).state1); - expect( - (resumedTransformer.exporter as AdditionalStateExporter).state1 - ).to.equal((transformer.exporter as AdditionalStateExporter).state1); - - resumedTransformer.dispose(); - targetDb.close(); - }); - - // FIXME: not (yet?) implemented for federation guid optimization branch - it.skip("should fail to resume from an old target while processing relationships", async () => { - const sourceDb = seedDb; - - const targetDbId = await IModelHost.hubAccess.createNewIModel({ - iTwinId, - iModelName: "targetDb1", - description: "crashingTarget", - noLocks: true, - }); - let targetDb = await HubWrappers.downloadAndOpenBriefcase({ - accessToken, - iTwinId, - iModelId: targetDbId, - }); - const transformer = new CountdownToCrashTransformer(sourceDb, targetDb); - transformer.relationshipExportsUntilCall = 2; - let crashed = false; - try { - await transformer.processSchemas(); - await transformer.processAll(); - } catch (transformerErr) { - expect((transformerErr as Error).message).to.equal("crash"); - crashed = true; - const dumpPath = IModelTransformerTestUtils.prepareOutputFile( - "IModelTransformerResumption", - "transformer-state.db" - ); - transformer.saveStateToFile(dumpPath); - // eslint-disable-next-line @typescript-eslint/naming-convention - const TransformerClass = - transformer.constructor as typeof IModelTransformer; - transformer.dispose(); - // redownload targetDb so that it is reset to the old state - targetDb.close(); - targetDb = await HubWrappers.downloadAndOpenBriefcase({ - accessToken, - iTwinId, - iModelId: targetDbId, - }); - expect(() => - TransformerClass.resumeTransformation(dumpPath, sourceDb, targetDb) - ).to.throw(/does not have the expected provenance/); - } - - expect(crashed).to.be.true; - await HubWrappers.closeAndDeleteBriefcaseDb(accessToken, targetDb); - return targetDb; - }); - - it("should succeed to resume from an up-to-date target while processing relationships", async () => { - const sourceDb = seedDb; - - const crashingTarget = await (async () => { - const targetDbId = await IModelHost.hubAccess.createNewIModel({ - iTwinId, - iModelName: "targetDb1", - description: "crashingTarget", - noLocks: true, - }); - const targetDb = await HubWrappers.downloadAndOpenBriefcase({ - accessToken, - iTwinId, - iModelId: targetDbId, - }); - const transformer = new CountdownToCrashTransformer(sourceDb, targetDb); - transformer.relationshipExportsUntilCall = 2; - let crashed = false; - try { - await transformer.processSchemas(); - await transformer.processAll(); - } catch (transformerErr) { - expect((transformerErr as Error).message).to.equal("crash"); - crashed = true; - const dumpPath = IModelTransformerTestUtils.prepareOutputFile( - "IModelTransformerResumption", - "transformer-state.db" - ); - transformer.saveStateToFile(dumpPath); - // eslint-disable-next-line @typescript-eslint/naming-convention - const TransformerClass = - transformer.constructor as typeof CountdownToCrashTransformer; - TransformerClass.resumeTransformation(dumpPath, sourceDb, targetDb); - transformer.relationshipExportsUntilCall = undefined; - await transformer.processAll(); - } - - expect(crashed).to.be.true; - targetDb.saveChanges(); - transformer.dispose(); - return targetDb; - })(); - - const regularTarget = await (async () => { - const targetDbId = await IModelHost.hubAccess.createNewIModel({ - iTwinId, - iModelName: "targetDb2", - description: "non crashing target", - noLocks: true, - }); - const targetDb = await HubWrappers.downloadAndOpenBriefcase({ - accessToken, - iTwinId, - iModelId: targetDbId, - }); - const transformer = new IModelTransformer(sourceDb, targetDb); - await transformNoCrash({ sourceDb, targetDb, transformer }); - targetDb.saveChanges(); - transformer.dispose(); - return targetDb; - })(); - - await assertIdentityTransformation(regularTarget, crashingTarget); - await HubWrappers.closeAndDeleteBriefcaseDb(accessToken, crashingTarget); - await HubWrappers.closeAndDeleteBriefcaseDb(accessToken, regularTarget); - }); - - it("processChanges crash and resume", async () => { - const sourceDbId = await IModelHost.hubAccess.createNewIModel({ - iTwinId, - iModelName: "sourceDb1", - description: "a db called sourceDb1", - noLocks: true, - }); - const sourceDb = await HubWrappers.downloadAndOpenBriefcase({ - accessToken, - iTwinId, - iModelId: sourceDbId, - }); - await TestUtils.ExtensiveTestScenario.prepareDb(sourceDb); - TestUtils.ExtensiveTestScenario.populateDb(sourceDb); - sourceDb.performCheckpoint(); - await sourceDb.pushChanges({ - accessToken, - description: "populated source db", - }); - - const targetDbRev0Path = IModelTransformerTestUtils.prepareOutputFile( - "IModelTransformerResumption", - "processChanges-targetDbRev0.bim" - ); - const targetDbRev0 = SnapshotDb.createFrom(sourceDb, targetDbRev0Path); - const provenanceTransformer = new IModelTransformer( - sourceDb, - targetDbRev0, - { wasSourceIModelCopiedToTarget: true } - ); - await provenanceTransformer.processAll(); - provenanceTransformer.dispose(); - targetDbRev0.performCheckpoint(); - - TestUtils.ExtensiveTestScenario.updateDb(sourceDb); - sourceDb.saveChanges(); - await sourceDb.pushChanges({ - accessToken, - description: "updated source db", - }); - - const regularTarget = await (async () => { - const targetDbId = await IModelHost.hubAccess.createNewIModel({ - iTwinId, - iModelName: "targetDb1", - description: "non crashing target", - noLocks: true, - version0: targetDbRev0Path, - }); - const targetDb = await HubWrappers.downloadAndOpenBriefcase({ - accessToken, - iTwinId, - iModelId: targetDbId, - }); - const transformer = new IModelTransformer(sourceDb, targetDb); - await transformNoCrash({ - sourceDb, - targetDb, - transformer, - async transformerProcessing(t) { - await t.processChanges({ accessToken }); - }, - }); - targetDb.saveChanges(); - transformer.dispose(); - return targetDb; - })(); - - const crashingTarget = await (async () => { - const targetDbId = await IModelHost.hubAccess.createNewIModel({ - iTwinId, - iModelName: "targetDb2", - description: "crashing target", - noLocks: true, - version0: targetDbRev0Path, - }); - const targetDb = await HubWrappers.downloadAndOpenBriefcase({ - accessToken, - iTwinId, - iModelId: targetDbId, - }); - const transformer = new CountdownToCrashTransformer(sourceDb, targetDb); - transformer.elementExportsUntilCall = 10; - await transformWithCrashAndRecover({ - sourceDb, - targetDb, - transformer, - disableCrashing(t) { - t.elementExportsUntilCall = undefined; - }, - async transformerProcessing(t) { - await t.processChanges({ accessToken }); - }, - }); - targetDb.saveChanges(); - await targetDb.pushChanges({ - accessToken, - description: "completed transformation that crashed", - }); - transformer.dispose(); - return targetDb; - })(); - - await assertIdentityTransformation(regularTarget, crashingTarget); - await HubWrappers.closeAndDeleteBriefcaseDb(accessToken, sourceDb); - await HubWrappers.closeAndDeleteBriefcaseDb(accessToken, crashingTarget); - await HubWrappers.closeAndDeleteBriefcaseDb(accessToken, regularTarget); - }); - - // env variables: - // TRANSFORMER_RESUMPTION_TEST_SINGLE_MODEL_ELEMENTS_BEFORE_CRASH (defaults to 2_500_000) - // TRANSFORMER_RESUMPTION_TEST_SINGLE_MODEL_PATH (defaults to the likely invalid "huge-model.bim") - // change "skip" to "only" to test local models - it.skip("local test single model", async () => { - const sourceDb = SnapshotDb.openFile("./huge-model.bim"); - - const crashingTarget = await (async () => { - const targetDbPath = "/tmp/huge-model-out.bim"; - const targetDb = SnapshotDb.createEmpty(targetDbPath, sourceDb); - const transformer = new CountdownToCrashTransformer(sourceDb, targetDb); - transformer.elementExportsUntilCall = - Number( - process.env - .TRANSFORMER_RESUMPTION_TEST_SINGLE_MODEL_ELEMENTS_BEFORE_CRASH - ) || 2_500_000; - await transformWithCrashAndRecover({ - sourceDb, - targetDb, - transformer, - disableCrashing(t) { - t.elementExportsUntilCall = undefined; - }, - }); - targetDb.saveChanges(); - transformer.dispose(); - return targetDb; - })(); - - const regularTarget = await (async () => { - const targetDbPath = "/tmp/huge-model-out.bim"; - const targetDb = SnapshotDb.createEmpty(targetDbPath, sourceDb); - const transformer = new IModelTransformer(sourceDb, targetDb); - await transformNoCrash({ sourceDb, targetDb, transformer }); - targetDb.saveChanges(); - transformer.dispose(); - return targetDb; - })(); - - await assertIdentityTransformation(regularTarget, crashingTarget); - await HubWrappers.closeAndDeleteBriefcaseDb(accessToken, crashingTarget); - await HubWrappers.closeAndDeleteBriefcaseDb(accessToken, regularTarget); - }); - - // replace "skip" with "only" to run several transformations with random native platform and transformer api method errors thrown - // you may control the amount of tests ran with the following environment variables - // TRANSFORMER_RESUMPTION_TEST_TOTAL_NON_CRASHING_TRANSFORMATIONS (defaults to 5) - // TRANSFORMER_RESUMPTION_TEST_TARGET_TOTAL_CRASHING_TRANSFORMATIONS (defaults to 50) - // TRANSFORMER_RESUMPTION_TEST_MAX_CRASHING_TRANSFORMATIONS (defaults to 200) - it.skip("crashing transforms stats gauntlet", async () => { - let crashableCallsMade = 0; - const { enableCrashes } = setupCrashingNativeAndTransformer({ - onCrashableCallMade() { - ++crashableCallsMade; - }, - }); - - // TODO: don't run a new control test to compare with every crash test, - // right now trying to run assertIdentityTransform against the control transform target dbs in the control loop yields - // BE_SQLITE_ERROR: Failed to prepare 'select * from (SELECT ECInstanceId FROM bis.Element) limit :sys_ecdb_count offset :sys_ecdb_offset'. The data source ECDb (parameter 'dataSourceECDb') must be a connection to the same ECDb file as the ECSQL parsing ECDb connection (parameter 'ecdb'). - // until that is investigated/fixed, the slow method here is used - async function runAndCompareWithControl( - crashingEnabledForThisTest: boolean - ) { - const sourceFileName = IModelTransformerTestUtils.resolveAssetFile( - "CompatibilityTestSeed.bim" - ); - const sourceDb = SnapshotDb.openFile(sourceFileName); - - async function transformWithMultipleCrashesAndRecover() { - const targetDbPath = IModelTransformerTestUtils.prepareOutputFile( - "IModelTransformerResumption", - "ResumeTransformationCrash.bim" - ); - const targetDb = SnapshotDb.createEmpty(targetDbPath, sourceDb); - let transformer = new CountingTransformer({ - source: sourceDb, - target: targetDb, - }); - const MAX_ITERS = 100; - let crashCount = 0; - let timer: StopWatch; - - enableCrashes(crashingEnabledForThisTest); - - for (let i = 0; i <= MAX_ITERS; ++i) { - timer = new StopWatch(); - timer.start(); - try { - await transformer.processSchemas(); - await transformer.processAll(); - break; - } catch (transformerErr) { - crashCount++; - const dumpPath = IModelTransformerTestUtils.prepareOutputFile( - "IModelTransformerResumption", - "transformer-state.db" - ); - enableCrashes(false); - transformer.saveStateToFile(dumpPath); - transformer = CountingTransformer.resumeTransformation(dumpPath, { - source: sourceDb, - target: targetDb, - }); - enableCrashes(true); - crashableCallsMade = 0; - console.log(`crashed after ${timer.elapsed.seconds} seconds`); // eslint-disable-line no-console - } - if (i === MAX_ITERS) assert.fail("crashed too many times"); - } - - console.log(`completed after ${crashCount} crashes`); // eslint-disable-line no-console - const result = { - resultDb: targetDb, - finalTransformationTime: timer!.elapsedSeconds, - finalTransformationCallsMade: crashableCallsMade, - crashCount, - importedEntityCount: transformer.importedEntities, - exportedEntityCount: transformer.exportedEntities, - }; - crashableCallsMade = 0; - return result; - } - - const { resultDb: crashingTarget, ...crashingTransformResult } = - await transformWithMultipleCrashesAndRecover(); - - const regularTarget = await (async () => { - const targetDbPath = IModelTransformerTestUtils.prepareOutputFile( - "IModelTransformerResumption", - "ResumeTransformationNoCrash.bim" - ); - const targetDb = SnapshotDb.createEmpty(targetDbPath, sourceDb); - const transformer = new IModelTransformer(sourceDb, targetDb); - enableCrashes(false); - return transformNoCrash({ sourceDb, targetDb, transformer }); - })(); - - await assertIdentityTransformation(regularTarget, crashingTarget); - await HubWrappers.closeAndDeleteBriefcaseDb(accessToken, crashingTarget); - await HubWrappers.closeAndDeleteBriefcaseDb(accessToken, regularTarget); - return crashingTransformResult; - } - - let totalCrashableCallsMade = 0; - let totalNonCrashingTransformationsTime = 0.0; - let totalImportedEntities = 0; - let totalExportedEntities = 0; - const totalNonCrashingTransformations = - Number( - process.env - .TRANSFORMER_RESUMPTION_TEST_TOTAL_NON_CRASHING_TRANSFORMATIONS - ) || 5; - for (let i = 0; i < totalNonCrashingTransformations; ++i) { - // eslint-disable-next-line @typescript-eslint/no-shadow - const result = await runAndCompareWithControl(false); - totalCrashableCallsMade += result.finalTransformationCallsMade; - totalNonCrashingTransformationsTime += result.finalTransformationTime; - totalImportedEntities += result.importedEntityCount; - totalExportedEntities += result.exportedEntityCount; - } - const avgNonCrashingTransformationsTime = - totalNonCrashingTransformationsTime / totalNonCrashingTransformations; - const avgCrashableCallsMade = - totalCrashableCallsMade / totalNonCrashingTransformations; - const avgImportedEntityCount = - totalImportedEntities / totalNonCrashingTransformations; - const avgExportedEntityCount = - totalExportedEntities / totalNonCrashingTransformations; - - // eslint-disable-next-line no-console - console.log( - `the average non crashing transformation took ${formatter.format( - avgNonCrashingTransformationsTime - )} and made ${formatter.format(avgCrashableCallsMade)} native calls.` - ); - - let totalCrashingTransformationsTime = 0.0; - const targetTotalCrashingTransformations = - Number( - process.env - .TRANSFORMER_RESUMPTION_TEST_TARGET_TOTAL_CRASHING_TRANSFORMATIONS - ) || 50; - const MAX_CRASHING_TRANSFORMS = - Number( - process.env.TRANSFORMER_RESUMPTION_TEST_MAX_CRASHING_TRANSFORMATIONS - ) || 200; - let totalCrashingTransformations = 0; - for ( - let i = 0; - i < MAX_CRASHING_TRANSFORMS && - totalCrashingTransformations < targetTotalCrashingTransformations; - ++i - ) { - // eslint-disable-next-line @typescript-eslint/no-shadow - const result = await runAndCompareWithControl(true); - if (result.crashCount === 0) continue; - - totalCrashingTransformations++; - const proportionOfNonCrashingTransformTime = - result.finalTransformationTime / avgNonCrashingTransformationsTime; - const proportionOfNonCrashingTransformCalls = - result.finalTransformationCallsMade / avgCrashableCallsMade; - const proportionOfNonCrashingEntityImports = - result.importedEntityCount / avgImportedEntityCount; - expect(result.exportedEntityCount).to.equal(avgExportedEntityCount); - const _ratioOfCallsToTime = - proportionOfNonCrashingTransformCalls / - proportionOfNonCrashingTransformTime; - const _ratioOfImportsToTime = - proportionOfNonCrashingEntityImports / - proportionOfNonCrashingTransformTime; - /* eslint-disable no-console */ - console.log( - `final resuming transformation took | ${percent( - proportionOfNonCrashingTransformTime - )} time | ${percent( - proportionOfNonCrashingTransformCalls - )} calls | ${percent( - proportionOfNonCrashingEntityImports - )} element imports |` - ); - /* eslint-enable no-console */ - totalCrashingTransformationsTime += result.finalTransformationTime; - } - - const avgCrashingTransformationsTime = - totalCrashingTransformationsTime / totalCrashingTransformations; - /* eslint-disable no-console */ - console.log(`avg crashable calls made: ${avgCrashableCallsMade}`); - console.log( - `avg non-crashing transformations time: ${avgNonCrashingTransformationsTime}` - ); - console.log( - `avg crash-resuming+completing transformations time: ${avgCrashingTransformationsTime}` - ); - /* eslint-enable no-console */ - sinon.restore(); - }); -});