Test native fix for transformer texture remapping (#5059)

Co-authored-by: Michael Belousov <MichaelBelousov@users.noreply.github.com>

common/changes/@itwin/core-backend/2023-02-10-17-45.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@itwin/core-backend",
+      "comment": "",
+      "type": "none"
+    }
+  ],
+  "packageName": "@itwin/core-backend"
+}

common/changes/@itwin/core-transformer/2023-02-10-17-45.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@itwin/core-transformer",
+      "comment": "add tests for texture remap handling, better prevent target id overlap",
+      "type": "none"
+    }
+  ],
+  "packageName": "@itwin/core-transformer"
+}

core/backend/src/test/imageData.ts

@@ -0,0 +1,14 @@
+
+import { Base64 } from "js-base64";
+
+const samplePngTextureData = [137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 3, 0, 0, 0, 3, 8, 2, 0, 0, 0, 217, 74, 34, 232, 0, 0, 0, 1, 115, 82, 71, 66, 0, 174, 206, 28, 233, 0, 0, 0, 4, 103, 65, 77, 65, 0, 0, 177, 143, 11, 252, 97, 5, 0, 0, 0, 9, 112, 72, 89, 115, 0, 0, 14, 195, 0, 0, 14, 195, 1, 199, 111, 168, 100, 0, 0, 0, 24, 73, 68, 65, 84, 24, 87, 99, 248, 15, 4, 12, 12, 64, 4, 198, 64, 46, 132, 5, 162, 254, 51, 0, 0, 195, 90, 10, 246, 127, 175, 154, 145, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130];
+const samplePngTextureDataBase64 = Base64.btoa(String.fromCharCode(...samplePngTextureData));
+
+/**
+ * This is an encoded png containing a 3x3 square with white in top left pixel, blue in middle pixel, and green in
+ * bottom right pixel. The rest of the square is red.
+ */
+export const samplePngTexture = {
+  data: samplePngTextureData,
+  base64: samplePngTextureDataBase64,
+};

core/backend/src/test/imodel/IModel.test.ts

@@ -3,7 +3,6 @@
 * See LICENSE.md in the project root for license terms and full copyright notice.
 *--------------------------------------------------------------------------------------------*/
 import { assert, expect } from "chai";
-import { Base64 } from "js-base64";
 import * as path from "path";
 import * as semver from "semver";
 import * as sinon from "sinon";
@@ -34,6 +33,7 @@ import { HubMock } from "../../HubMock";
 import { KnownTestLocations } from "../KnownTestLocations";
 import { IModelTestUtils } from "../IModelTestUtils";
 import { DisableNativeAssertions } from "../TestUtils";
+import { samplePngTexture } from "../imageData";
 
 // spell-checker: disable
 
@@ -379,16 +379,11 @@ describe("iModel", () => {
   });
 
   it("attempt to apply material to new element in imodel5", () => {
-    // This is an encoded png containing a 3x3 square with white in top left pixel, blue in middle pixel, and green in
-    // bottom right pixel.  The rest of the square is red.
-    const pngData = [137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 3, 0, 0, 0, 3, 8, 2, 0, 0, 0, 217, 74, 34, 232, 0, 0, 0, 1, 115, 82, 71, 66, 0, 174, 206, 28, 233, 0, 0, 0, 4, 103, 65, 77, 65, 0, 0, 177, 143, 11, 252, 97, 5, 0, 0, 0, 9, 112, 72, 89, 115, 0, 0, 14, 195, 0, 0, 14, 195, 1, 199, 111, 168, 100, 0, 0, 0, 24, 73, 68, 65, 84, 24, 87, 99, 248, 15, 4, 12, 12, 64, 4, 198, 64, 46, 132, 5, 162, 254, 51, 0, 0, 195, 90, 10, 246, 127, 175, 154, 145, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130];
-
     const testTextureName = "fake texture name";
     const testTextureFormat = ImageSourceFormat.Png;
-    const testTextureData = Base64.btoa(String.fromCharCode(...pngData));
     const testTextureDescription = "empty description";
 
-    const texId = Texture.insertTexture(imodel5, IModel.dictionaryId, testTextureName, testTextureFormat, testTextureData, testTextureDescription);
+    const texId = Texture.insertTexture(imodel5, IModel.dictionaryId, testTextureName, testTextureFormat, samplePngTexture.base64, testTextureDescription);
 
     /* eslint-disable @typescript-eslint/naming-convention */
     const matId = RenderMaterialElement.insert(imodel5, IModel.dictionaryId, "test material name",

core/backend/src/test/index.ts

@@ -7,3 +7,4 @@ export * from "./IModelTestUtils";
 export * from "./KnownTestLocations";
 export * from "./RevisionUtility";
 export * from "./TestChangeSetUtility";
+export * from "./imageData";

core/transformer/src/test/standalone/IModelTransformer.test.ts

@@ -13,15 +13,15 @@ import {
   ElementMultiAspect, ElementOwnsChildElements, ElementOwnsExternalSourceAspects, ElementOwnsMultiAspects, ElementOwnsUniqueAspect, ElementRefersToElements,
   ElementUniqueAspect, ExternalSourceAspect, GenericPhysicalMaterial, GeometricElement, IModelDb, IModelElementCloneContext, IModelHost, IModelJsFs,
   InformationRecordModel, InformationRecordPartition, LinkElement, Model, ModelSelector, OrthographicViewDefinition,
-  PhysicalModel, PhysicalObject, PhysicalPartition, PhysicalType, Relationship, RepositoryLink, Schema, SnapshotDb, SpatialCategory, StandaloneDb,
-  SubCategory, Subject,
+  PhysicalModel, PhysicalObject, PhysicalPartition, PhysicalType, Relationship, RenderMaterialElement, RepositoryLink, Schema, SnapshotDb, SpatialCategory, StandaloneDb,
+  SubCategory, Subject, Texture,
 } from "@itwin/core-backend";
 import * as ECSchemaMetaData from "@itwin/ecschema-metadata";
 import * as BackendTestUtils from "@itwin/core-backend/lib/cjs/test";
 import { DbResult, Guid, Id64, Id64String, Logger, LogLevel, OpenMode } from "@itwin/core-bentley";
 import {
   AxisAlignedBox3d, BriefcaseIdValue, Code, CodeScopeSpec, CodeSpec, ColorDef, CreateIModelProps, DefinitionElementProps, ElementAspectProps, ElementProps,
-  ExternalSourceAspectProps, IModel, IModelError, PhysicalElementProps, Placement3d, ProfileOptions, QueryRowFormat, RelatedElement, RelationshipProps,
+  ExternalSourceAspectProps, ImageSourceFormat, IModel, IModelError, PhysicalElementProps, Placement3d, ProfileOptions, QueryRowFormat, RelatedElement, RelationshipProps,
 } from "@itwin/core-common";
 import { Point3d, Range3d, StandardViewIndex, Transform, YawPitchRollAngles } from "@itwin/core-geometry";
 import { IModelExporter, IModelExportHandler, IModelTransformer, IModelTransformOptions, TransformerLoggerCategory } from "../../core-transformer";
@@ -1391,20 +1391,36 @@ describe("IModelTransformer", () => {
     ] as const;
   }
 
+  function createEmptyTargetWithIdsStartingAfterSource(sourceDb: IModelDb, createTarget: () => StandaloneDb): StandaloneDb {
+    const nextId = (db: IModelDb) => db.withSqliteStatement("SELECT Val FROM be_Local WHERE Name='bis_elementidsequence'", (s)=>[...s])[0].val;
+    sourceDb.saveChanges(); // save to make sure we get the latest id value
+    const sourceNextId = nextId(sourceDb);
+    const targetDb = createTarget();
+    const pathName = targetDb.pathName;
+    targetDb.withSqliteStatement("UPDATE be_Local SET Val=? WHERE Name='bis_elementidsequence'", (s)=>{
+      s.bindInteger(1, sourceNextId + 1);
+      assert(s.step() === DbResult.BE_SQLITE_DONE);
+    });
+    targetDb.saveChanges();
+    targetDb.close();
+    return StandaloneDb.openFile(pathName);
+  }
+
   /**
-   * A transformer that inserts an element at the beginning to ensure the target doesn't end up with the same ids as the source.
-   * Useful if you need to check that some source/target element references match and want to be sure it isn't a coincidence,
-   * which can happen deterministically in several cases, as well as just copy-paste errors where you accidentally test a
-   * source or target db against itself
+   * A transformer that resets the target's id sequence to ensure the target doesn't end up with the same ids as the source.
+   * Useful if you need to check that some source/target element references match and want to be sure it isn't a coincidence.
    * @note it modifies the target so there are side effects
    */
-  class ShiftElemIdsTransformer extends IModelTransformer {
-    constructor(...args: ConstructorParameters<typeof IModelTransformer>) {
-      super(...args);
-      try {
-        // the choice of element to insert is arbitrary, anything easy works
-        PhysicalModel.insert(this.targetDb, IModel.rootSubjectId, "MyShiftElemIdsPhysicalModel");
-      } catch (_err) { } // ignore error in case someone tries to transform the same target multiple times with this
+  class ShiftedIdsEmptyTargetTransformer extends IModelTransformer {
+    constructor(source: IModelDb, createTarget: () => StandaloneDb, options?: IModelTransformOptions) {
+      super(source, createEmptyTargetWithIdsStartingAfterSource(source, createTarget), options);
+    }
+  }
+
+  /** combination of @see AssertOrderTransformer and @see ShiftedIdsEmptyTargetTransformer */
+  class AssertOrderAndShiftIdsTransformer extends AssertOrderTransformer {
+    constructor(order: Id64String[], source: IModelDb, createTarget: () => StandaloneDb, options?: IModelTransformOptions) {
+      super(order, source, createEmptyTargetWithIdsStartingAfterSource(source, createTarget), options);
     }
   }
 
@@ -1416,31 +1432,35 @@ describe("IModelTransformer", () => {
     ] = createIModelWithDanglingReference({ name: "DanglingReferences", path: sourceDbPath });
 
     const targetDbPath = IModelTransformerTestUtils.prepareOutputFile("IModelTransformer", "DanglingReferenceTarget-reject.bim");
-    const targetDbForRejected = SnapshotDb.createEmpty(targetDbPath, { rootSubject: sourceDb.rootSubject });
+    const targetDbForRejected = StandaloneDb.createEmpty(targetDbPath, { rootSubject: sourceDb.rootSubject });
+    const targetDbForRejectedPath = targetDbForRejected.pathName;
+    targetDbForRejected.close();
 
-    const defaultTransformer = new ShiftElemIdsTransformer(sourceDb, targetDbForRejected);
+    const defaultTransformer = new ShiftedIdsEmptyTargetTransformer(sourceDb, () => StandaloneDb.openFile(targetDbForRejectedPath));
     await expect(defaultTransformer.processAll()).to.be.rejectedWith(
       /Found a reference to an element "[^"]*" that doesn't exist/
     );
+    defaultTransformer.targetDb.close();
 
-    const rejectDanglingReferencesTransformer = new ShiftElemIdsTransformer(sourceDb, targetDbForRejected, { danglingReferencesBehavior: "reject" });
+    const rejectDanglingReferencesTransformer = new ShiftedIdsEmptyTargetTransformer(sourceDb, () => StandaloneDb.openFile(targetDbForRejectedPath), { danglingReferencesBehavior: "reject" });
     await expect(rejectDanglingReferencesTransformer.processAll()).to.be.rejectedWith(
       /Found a reference to an element "[^"]*" that doesn't exist/
     );
+    defaultTransformer.targetDb.close();
 
     const runTransform = async (opts: Pick<IModelTransformOptions, "danglingReferencesBehavior">) => {
       const thisTransformTargetPath = IModelTransformerTestUtils.prepareOutputFile("IModelTransformer", `DanglingReferenceTarget-${opts.danglingReferencesBehavior}.bim`);
-      const targetDb = SnapshotDb.createEmpty(thisTransformTargetPath, { rootSubject: sourceDb.rootSubject });
-      const transformer = new ShiftElemIdsTransformer(sourceDb, targetDb, opts);
+      const createTargetDb = () => StandaloneDb.createEmpty(thisTransformTargetPath, { rootSubject: sourceDb.rootSubject });
+      const transformer = new ShiftedIdsEmptyTargetTransformer(sourceDb, createTargetDb, opts);
       await expect(transformer.processAll()).not.to.be.rejected;
-      targetDb.saveChanges();
+      transformer.targetDb.saveChanges();
 
       expect(sourceDb.elements.tryGetElement(physicalObjects[1].id)).to.be.undefined;
       const displayStyleInSource = sourceDb.elements.getElement<DisplayStyle3d>(displayStyleId);
       expect([...displayStyleInSource.settings.excludedElementIds]).to.include(physicalObjects[1].id);
 
       const displayStyleInTargetId = transformer.context.findTargetElementId(displayStyleId);
-      const displayStyleInTarget = targetDb.elements.getElement<DisplayStyle3d>(displayStyleInTargetId);
+      const displayStyleInTarget = transformer.targetDb.elements.getElement<DisplayStyle3d>(displayStyleInTargetId);
 
       const physObjsInTarget = physicalObjects.map((physObjInSource) => {
         const physObjInTargetId = transformer.context.findTargetElementId(physObjInSource.id);
@@ -2111,6 +2131,94 @@ describe("IModelTransformer", () => {
     }
   });
 
+  it("should remap textures in target iModel", async () => {
+    // create source iModel
+    const sourceDbFile: string = IModelTransformerTestUtils.prepareOutputFile("IModelTransformer", "Transform3d-Source.bim");
+    const sourceDb = SnapshotDb.createEmpty(sourceDbFile, { rootSubject: { name: "Transform3d-Source" } });
+    const categoryId = SpatialCategory.insert(sourceDb, IModel.dictionaryId, "SpatialCategory", { color: ColorDef.green.toJSON() });
+    const category = sourceDb.elements.getElement<SpatialCategory>(categoryId);
+    const sourceModelId = PhysicalModel.insert(sourceDb, IModel.rootSubjectId, "Physical");
+
+    const renderMaterialBothImgsId = RenderMaterialElement.insert(sourceDb, IModel.dictionaryId, "TextureMaterialBothImgs", {
+      paletteName: "something",
+    });
+
+    const texture1Id = Texture.insertTexture(sourceDb, IModel.dictionaryId, "Texture1", ImageSourceFormat.Png, BackendTestUtils.samplePngTexture.base64, "texture 1");
+    const texture2Id = Texture.insertTexture(sourceDb, IModel.dictionaryId, "Texture2", ImageSourceFormat.Png, BackendTestUtils.samplePngTexture.base64, "texture 2");
+
+    const renderMaterialBothImgs = sourceDb.elements.getElement<RenderMaterialElement>(renderMaterialBothImgsId);
+    // update the texture id into the model so that they are processed out of order (material exported before texture)
+    if (renderMaterialBothImgs.jsonProperties.materialAssets.renderMaterial.Map === undefined)
+      renderMaterialBothImgs.jsonProperties.materialAssets.renderMaterial.Map = {};
+    if (renderMaterialBothImgs.jsonProperties.materialAssets.renderMaterial.Map.Pattern === undefined)
+      renderMaterialBothImgs.jsonProperties.materialAssets.renderMaterial.Map.Pattern = {};
+    if (renderMaterialBothImgs.jsonProperties.materialAssets.renderMaterial.Map.Normal === undefined)
+      renderMaterialBothImgs.jsonProperties.materialAssets.renderMaterial.Map.Normal = {};
+    renderMaterialBothImgs.jsonProperties.materialAssets.renderMaterial.Map.TextureId = texture1Id;
+    renderMaterialBothImgs.jsonProperties.materialAssets.renderMaterial.Map.Pattern.TextureId = texture1Id;
+    renderMaterialBothImgs.jsonProperties.materialAssets.renderMaterial.Map.Normal.TextureId = texture2Id;
+    renderMaterialBothImgs.update();
+
+    const renderMaterialOnlyPatternId = RenderMaterialElement.insert(sourceDb, IModel.dictionaryId, "TextureMaterialOnlyPattern", {
+      paletteName: "something",
+      patternMap: {
+        TextureId: texture1Id, // eslint-disable-line @typescript-eslint/naming-convention
+      },
+    });
+
+    const renderMaterialOnlyNormalId  = RenderMaterialElement.insert(sourceDb, IModel.dictionaryId, "TextureMaterialOnlyNormal", {
+      paletteName: "something",
+      normalMap: {
+        TextureId: texture2Id, // eslint-disable-line @typescript-eslint/naming-convention
+      },
+    });
+
+    const physObjs = [renderMaterialBothImgsId, renderMaterialOnlyNormalId, renderMaterialOnlyPatternId].map((renderMaterialId) => {
+      const physicalObjectProps1: PhysicalElementProps = {
+        classFullName: PhysicalObject.classFullName,
+        model: sourceModelId,
+        category: categoryId,
+        code: Code.createEmpty(),
+        userLabel: `PhysicalObject`,
+        geom: IModelTransformerTestUtils.createBox(Point3d.create(1, 1, 1), categoryId, category.myDefaultSubCategoryId(), renderMaterialId),
+        placement: Placement3d.fromJSON({ origin: { x: 0, y: 0 }, angles: {} }),
+      };
+      return sourceDb.elements.insertElement(physicalObjectProps1);
+    });
+
+    // create target iModel
+    const targetDbFile: string = IModelTransformerTestUtils.prepareOutputFile("IModelTransformer", "Transform3d-Target.bim");
+    const createTargetDb = () => StandaloneDb.createEmpty(targetDbFile, { rootSubject: { name: "Transform3d-Target" } });
+
+    // transform
+    const transformer = new AssertOrderAndShiftIdsTransformer([renderMaterialBothImgsId, texture1Id], sourceDb, createTargetDb);
+    await transformer.processAll();
+
+    const texture1IdInTarget = transformer.context.findTargetElementId(texture1Id);
+    const texture2IdInTarget = transformer.context.findTargetElementId(texture2Id);
+    assert(Id64.isValidId64(texture1IdInTarget));
+    assert(Id64.isValidId64(texture2IdInTarget));
+
+    for (const objId of physObjs) {
+      const objInTargetId = transformer.context.findTargetElementId(objId);
+      const objInTarget = transformer.targetDb.elements.getElement<PhysicalObject>({ id: objInTargetId, wantGeometry: true });
+      assert(objInTarget.geom);
+      const materialOfObjInTargetId = objInTarget.geom.find((g) => g.material?.materialId)?.material?.materialId;
+      assert(materialOfObjInTargetId);
+
+      const materialOfObjInTarget = transformer.targetDb.elements.getElement<RenderMaterialElement>(materialOfObjInTargetId);
+      if (materialOfObjInTarget.jsonProperties.materialAssets.renderMaterial.Map.Pattern)
+        expect(materialOfObjInTarget.jsonProperties.materialAssets.renderMaterial.Map.Pattern.TextureId).to.equal(texture1IdInTarget);
+      if (materialOfObjInTarget.jsonProperties.materialAssets.renderMaterial.Map.Normal)
+        expect(materialOfObjInTarget.jsonProperties.materialAssets.renderMaterial.Map.Normal.TextureId).to.equal(texture2IdInTarget);
+    }
+
+    // clean up
+    transformer.dispose();
+    sourceDb.close();
+    transformer.targetDb.close();
+  });
+
   /** unskip to generate a javascript CPU profile on just the processAll portion of an iModel */
   it.skip("should profile an IModel transformation", async function () {
     const sourceDbFile = IModelTransformerTestUtils.prepareOutputFile("IModelTransformer", "ProfileTransformation.bim");