From eb8312f1b1d04a223244b856a07f1a7b4569e65a Mon Sep 17 00:00:00 2001 From: TheCodeTherapy Date: Wed, 8 Nov 2023 15:00:24 +0000 Subject: [PATCH 1/2] avatar interfaces refactor --- .../web-avatar-client/src/AvatarEditor.tsx | 50 ++++++++--- example/web-avatar-client/src/index.tsx | 1 - .../src/CharacterPartsSelector.tsx | 85 ++++++++++--------- packages/3d-web-avatar-editor-ui/src/index.ts | 7 +- packages/3d-web-avatar-editor-ui/src/types.ts | 14 ++- .../3d-web-avatar/src/character/Character.ts | 78 +++++------------ .../src/camera/CameraManager.ts | 10 +-- .../src/AvatarRenderer.ts | 74 +++------------- .../src/AvatarVisualizer.tsx | 26 ++++-- .../src/index.ts | 1 - 10 files changed, 152 insertions(+), 194 deletions(-) diff --git a/example/web-avatar-client/src/AvatarEditor.tsx b/example/web-avatar-client/src/AvatarEditor.tsx index 98f83629..9c0ae920 100644 --- a/example/web-avatar-client/src/AvatarEditor.tsx +++ b/example/web-avatar-client/src/AvatarEditor.tsx @@ -2,10 +2,9 @@ import { AvatarVisualizer, CharacterPartsSelector, CollectionDataType, - CharacterComposition, Character, ModelLoader, - BodyPartTypes, + CharacterComposition, } from "@mml-io/3d-web-standalone-avatar-editor"; import React from "react"; import { Object3D } from "three"; @@ -13,21 +12,44 @@ import { Object3D } from "three"; import idleAnimationURL from "../../assets/avatar/anims/AS_Andor_Stand_Idle.glb"; import hdrURL from "../../assets/hdr/industrial_sunset_2k.hdr"; -export function AvatarEditor(props: { collectionData: CollectionDataType }) { +type BodyPartTypes = "fullBody" | "head" | "upperBody" | "lowerBody" | "feet"; + +export type MMLCharacterDescriptionPart = { + url: string; +}; + +export type MMLCharacterDescription = { + base: MMLCharacterDescriptionPart; + parts: MMLCharacterDescriptionPart[]; +}; + +const partToCameraOffset = new Map< + BodyPartTypes, + { + offset: { x: number; y: number; z: number }; + targetDistance: number; + } +>([ + ["head", { offset: { x: 0, y: 1.616079270843859, z: 0 }, targetDistance: 0.8 }], + ["lowerBody", { offset: { x: 0, y: 0.6694667063220178, z: 0 }, targetDistance: 1.3 }], + ["feet", { offset: { x: 0, y: 0.2194667063220177, z: 0 }, targetDistance: 0.9 }], + ["fullBody", { offset: { x: 0, y: 1.079590141424593, z: 0 }, targetDistance: 2.5 }], + ["upperBody", { offset: { x: 0, y: 1.199837285184325, z: 0 }, targetDistance: 1.2 }], +]); + +export function AvatarEditor(props: { collectionData: C }) { const [characterMesh, setCharacterMesh] = React.useState(null); const [character] = React.useState(new Character(new ModelLoader())); const [selectedPart, setSelectedPart] = React.useState("fullBody"); const onComposedCharacter = React.useCallback( - async (characterParts: CharacterComposition) => { - const { fullBody, head, upperBody, lowerBody, feet } = characterParts; + async (characterParts: CharacterComposition) => { + const { fullBody, parts } = characterParts; + // The character parts picker provides the full body separately from the parts that are then layered onto it const obj3d = await character.mergeBodyParts( - fullBody.asset, - head.asset, - upperBody.asset, - lowerBody.asset, - feet.asset, + fullBody.url, + Object.values(parts).map((part) => part.url), ); setCharacterMesh(obj3d); setSelectedPart("fullBody"); @@ -41,19 +63,23 @@ export function AvatarEditor(props: { collectionData: CollectionDataType }) { } }; + const partEntry = partToCameraOffset.get(selectedPart)!; + return ( <> {characterMesh && ( )} diff --git a/example/web-avatar-client/src/index.tsx b/example/web-avatar-client/src/index.tsx index de4991f8..adeae729 100644 --- a/example/web-avatar-client/src/index.tsx +++ b/example/web-avatar-client/src/index.tsx @@ -7,7 +7,6 @@ import collectionData from "./collection.json"; class App { root: Root; - constructor() { this.init(); } diff --git a/packages/3d-web-avatar-editor-ui/src/CharacterPartsSelector.tsx b/packages/3d-web-avatar-editor-ui/src/CharacterPartsSelector.tsx index 36d573ce..32b19ddb 100644 --- a/packages/3d-web-avatar-editor-ui/src/CharacterPartsSelector.tsx +++ b/packages/3d-web-avatar-editor-ui/src/CharacterPartsSelector.tsx @@ -1,63 +1,66 @@ import React, { useCallback, useEffect, useState } from "react"; -import { AssetDescription, BodyPartTypes, CharacterComposition, CollectionDataType } from "./types"; +import { AssetDescription, CharacterComposition, CollectionDataType } from "./types"; -type CharacterPartsSelectorProps = { - collectionData: CollectionDataType; - onSelectingPart: (part: BodyPartTypes) => void; - onComposedCharacter: (characterParts: CharacterComposition) => void; +type CharacterPartsSelectorProps = { + collectionData: C; + fullBodyKey: keyof C; + onSelectingPart: (part: keyof C) => void; + onComposedCharacter: (characterComposition: CharacterComposition) => void; }; -export const CharacterPartsSelector: React.FC = ({ +export function CharacterPartsSelector({ collectionData, + fullBodyKey, onSelectingPart, onComposedCharacter, -}) => { - const [selectedPart, setSelectedPart] = useState(null); - const [currentSelection, setCurrentSelection] = useState({ - fullBody: collectionData.fullBody[0], - head: collectionData.head[0], - upperBody: collectionData.upperBody[0], - lowerBody: collectionData.lowerBody[0], - feet: collectionData.feet[0], - }); +}: CharacterPartsSelectorProps) { + const [selectedPart, setSelectedPart] = useState(null); + const [currentSelection, setCurrentSelection] = useState>(() => + Object.entries(collectionData).reduce>( + ( + acc: Record, + [key, asset]: [keyof C, Array], + ) => { + acc[key] = asset[0]; + return acc; + }, + {} as Record, + ), + ); const createMMLDescription = useCallback(() => { - const description = ` - - - - + const fullBody = currentSelection[fullBodyKey]; + const remainingParts = Object.entries(currentSelection).filter(([key]) => key !== fullBodyKey); + const description = ` +${remainingParts.map(([key, asset]) => ``).join("\n")} `; console.log(description); - }, [currentSelection]); + }, [currentSelection, fullBodyKey]); useEffect(() => { + const fullBody = currentSelection[fullBodyKey]; + const remainingParts = Object.entries(currentSelection).filter(([key]) => key !== fullBodyKey); onComposedCharacter({ - fullBody: currentSelection.fullBody, - head: currentSelection.head, - upperBody: currentSelection.upperBody, - lowerBody: currentSelection.lowerBody, - feet: currentSelection.feet, + fullBody: { url: fullBody.asset }, + parts: remainingParts.reduce( + (accParts: Record, [key, asset]: [keyof C, AssetDescription]) => { + accParts[key] = { url: asset.asset }; + return accParts; + }, + {} as Record, + ), }); createMMLDescription(); - }, [ - onComposedCharacter, - currentSelection.fullBody, - currentSelection.head, - currentSelection.upperBody, - currentSelection.lowerBody, - currentSelection.feet, - createMMLDescription, - ]); + }, [onComposedCharacter, fullBodyKey, currentSelection, createMMLDescription]); - const handleThumbnailClick = (part: BodyPartTypes) => { + const handleThumbnailClick = (part: string) => { onSelectingPart(part); setSelectedPart(part); }; - const handleModalThumbnailClick = (part: BodyPartTypes, item: AssetDescription) => { + const handleModalThumbnailClick = (part: string, item: AssetDescription) => { const selectedData = item; setCurrentSelection((prev) => ({ ...prev, [part]: selectedData })); createMMLDescription(); @@ -67,12 +70,12 @@ export const CharacterPartsSelector: React.FC = ({ const renderThumbnails = () => { return (
- {["fullBody", "head", "upperBody", "lowerBody", "feet"].map((part) => ( + {Object.keys(collectionData).map((part) => ( {part} handleThumbnailClick(part as BodyPartTypes)} + onClick={() => handleThumbnailClick(part as string)} /> ))}
@@ -103,6 +106,6 @@ export const CharacterPartsSelector: React.FC = ({ {renderModal()} ); -}; +} export default CharacterPartsSelector; diff --git a/packages/3d-web-avatar-editor-ui/src/index.ts b/packages/3d-web-avatar-editor-ui/src/index.ts index d6976a6b..b1e912fc 100644 --- a/packages/3d-web-avatar-editor-ui/src/index.ts +++ b/packages/3d-web-avatar-editor-ui/src/index.ts @@ -1,7 +1,2 @@ export { CharacterPartsSelector } from "./CharacterPartsSelector"; -export type { - AssetDescription, - CollectionDataType, - BodyPartTypes, - CharacterComposition, -} from "./types"; +export type { AssetDescription, CollectionDataType, CharacterComposition } from "./types"; diff --git a/packages/3d-web-avatar-editor-ui/src/types.ts b/packages/3d-web-avatar-editor-ui/src/types.ts index 4fc06d19..aa78a07d 100644 --- a/packages/3d-web-avatar-editor-ui/src/types.ts +++ b/packages/3d-web-avatar-editor-ui/src/types.ts @@ -1,11 +1,17 @@ -export type BodyPartTypes = "fullBody" | "head" | "upperBody" | "lowerBody" | "feet"; - export type AssetDescription = { name: string; asset: string; thumb: string; }; -export type CollectionDataType = Record>; +export type CollectionDataType = Record>; -export type CharacterComposition = Record; +export type CharacterComposition = { + fullBody: { url: string }; + parts: Record< + keyof C, + { + url: string; + } + >; +}; diff --git a/packages/3d-web-avatar/src/character/Character.ts b/packages/3d-web-avatar/src/character/Character.ts index 94982489..b87023c5 100644 --- a/packages/3d-web-avatar/src/character/Character.ts +++ b/packages/3d-web-avatar/src/character/Character.ts @@ -13,22 +13,20 @@ export class Character { constructor(private modelLoader: ModelLoader) {} - public async mergeBodyParts( - fullBodyURL: string, - headURL: string, - upperBodyURL: string, - lowerBodyURL: string, - feetURL: string, - ): Promise { + public async mergeBodyParts(fullBodyURL: string, bodyParts: Array): Promise { const fullBodyAsset = await this.modelLoader.load(fullBodyURL); const fullBodyGLTF = this.skeletonHelpers.cloneGLTF(fullBodyAsset as GLTF, "fullBody"); - const headAsset = await this.modelLoader.load(headURL); - const upperBodyAsset = await this.modelLoader.load(upperBodyURL); - const lowerBodyAsset = await this.modelLoader.load(lowerBodyURL); - const feetAsset = await this.modelLoader.load(feetURL); - - const skinnedMeshesToRemove: SkinnedMesh[] = []; + const assetPromises: Array> = bodyParts.map( + (partUrl) => { + return new Promise((resolve) => { + this.modelLoader.load(partUrl).then((asset) => { + resolve({ asset: asset!, partUrl }); + }); + }); + }, + ); + const assets = await Promise.all(assetPromises); const fullBodyModelGroup = fullBodyGLTF.gltf.scene; @@ -45,49 +43,17 @@ export class Character { this.sharedSkeleton = fullBodyGLTF.sharedSkeleton; this.sharedMatrixWorld = fullBodyGLTF.matrixWorld; - skinnedMeshesToRemove.forEach((child) => { - child.removeFromParent(); - }); - - const headGLTF = this.skeletonHelpers.cloneGLTF(headAsset as GLTF, "headGLTF"); - const headModelGroup = headGLTF.gltf.scene; - headModelGroup.traverse((child) => { - if (child.type === "SkinnedMesh") { - (child as SkinnedMesh).castShadow = true; - (child as SkinnedMesh).bind(this.sharedSkeleton!, this.sharedMatrixWorld!); - this.skinnedMeshesParent?.children.splice(0, 0, child as SkinnedMesh); - } - }); - - const upperBodyGLTF = this.skeletonHelpers.cloneGLTF(upperBodyAsset as GLTF, "upperBodyGLTF"); - const upperBodyModelGroup = upperBodyGLTF.gltf.scene; - upperBodyModelGroup.traverse((child) => { - if (child.type === "SkinnedMesh") { - (child as SkinnedMesh).castShadow = true; - (child as SkinnedMesh).bind(this.sharedSkeleton!, this.sharedMatrixWorld!); - this.skinnedMeshesParent?.children.splice(1, 0, child as SkinnedMesh); - } - }); - - const lowerBodyGLTF = this.skeletonHelpers.cloneGLTF(lowerBodyAsset as GLTF, "lowerBodyGLTF"); - const lowerBodyModelGroup = lowerBodyGLTF.gltf.scene; - lowerBodyModelGroup.traverse((child) => { - if (child.type === "SkinnedMesh") { - (child as SkinnedMesh).castShadow = true; - (child as SkinnedMesh).bind(this.sharedSkeleton!, this.sharedMatrixWorld!); - this.skinnedMeshesParent?.children.splice(2, 0, child as SkinnedMesh); - } - }); - - const feetGLTF = this.skeletonHelpers.cloneGLTF(feetAsset as GLTF, "feetGLTF"); - const feetModelGroup = feetGLTF.gltf.scene; - feetModelGroup.traverse((child) => { - if (child.type === "SkinnedMesh") { - (child as SkinnedMesh).castShadow = true; - (child as SkinnedMesh).bind(this.sharedSkeleton!, this.sharedMatrixWorld!); - this.skinnedMeshesParent?.children.splice(3, 0, child as SkinnedMesh); - } - }); + for (const loadingAsset of assets) { + const gltf = this.skeletonHelpers.cloneGLTF(loadingAsset.asset, loadingAsset.partUrl); + const modelGroup = gltf.gltf.scene; + modelGroup.traverse((child) => { + if (child.type === "SkinnedMesh") { + (child as SkinnedMesh).castShadow = true; + (child as SkinnedMesh).bind(this.sharedSkeleton!, this.sharedMatrixWorld!); + this.skinnedMeshesParent?.children.splice(3, 0, child as SkinnedMesh); + } + }); + } return fullBodyGLTF!.gltf.scene as Object3D; } diff --git a/packages/3d-web-client-core/src/camera/CameraManager.ts b/packages/3d-web-client-core/src/camera/CameraManager.ts index 59145aa2..48a8e971 100644 --- a/packages/3d-web-client-core/src/camera/CameraManager.ts +++ b/packages/3d-web-client-core/src/camera/CameraManager.ts @@ -47,7 +47,6 @@ export class CameraManager { private lerpFactor: number = 0; private lerpDuration: number = 2.1; - private fixedOnTarget: boolean = false; constructor( targetElement: HTMLElement, @@ -80,7 +79,6 @@ export class CameraManager { private onMouseMove(event: MouseEvent): void { if (!this.dragging || getTweakpaneActive()) return; - this.fixedOnTarget = false; if (this.targetTheta === null || this.targetPhi === null) return; this.targetTheta += event.movementX * 0.01; this.targetPhi -= event.movementY * 0.01; @@ -89,7 +87,6 @@ export class CameraManager { } private onMouseWheel(event: WheelEvent): void { - this.fixedOnTarget = false; const scrollAmount = event.deltaY * 0.001; this.targetDistance += scrollAmount; this.targetDistance = Math.max( @@ -115,9 +112,10 @@ export class CameraManager { } } - public setLerpedTarget(target: Vector3): void { + public setLerpedTarget(target: Vector3, targetDistance: number): void { this.isLerping = true; - this.fixedOnTarget = true; + this.targetDistance = targetDistance; + this.desiredDistance = targetDistance; this.setTarget(target); } @@ -167,7 +165,7 @@ export class CameraManager { this.lerpFactor += 0.01 / this.lerpDuration; this.lerpFactor = Math.min(1, this.lerpFactor); this.target.lerpVectors(this.lerpTarget, this.finalTarget, this.easeOutExpo(this.lerpFactor)); - } else if (!this.fixedOnTarget) { + } else { this.adjustCameraPosition(); } diff --git a/packages/3d-web-standalone-avatar-editor/src/AvatarRenderer.ts b/packages/3d-web-standalone-avatar-editor/src/AvatarRenderer.ts index e3129298..dd3f7af1 100644 --- a/packages/3d-web-standalone-avatar-editor/src/AvatarRenderer.ts +++ b/packages/3d-web-standalone-avatar-editor/src/AvatarRenderer.ts @@ -1,5 +1,4 @@ import { ModelLoader } from "@mml-io/3d-web-avatar"; -import { BodyPartTypes } from "@mml-io/3d-web-avatar-editor-ui"; import { TimeManager, CameraManager, CollisionsManager } from "@mml-io/3d-web-client-core"; import { AnimationMixer, @@ -40,11 +39,10 @@ export class AvatarRenderer { private animationAsset: GLTF | null | undefined = null; private lights: Lights; - private lookAt: Vector3; private floor: Mesh | null = null; - public selectedPart: BodyPartTypes = "fullBody"; - private cameraFocusMap: Map = new Map(); + public cameraTargetOffset: { x?: number; y?: number; z?: number } = {}; + public cameraTargetDistance: number = 0; constructor( private hdrURL: string, @@ -59,8 +57,6 @@ export class AvatarRenderer { this.useHDRI(this.hdrURL); - this.lookAt = new Vector3().copy(this.scene.position).add(this.camOffset); - // Floor this.floor = new Floor(this.floorSize).mesh; this.scene.add(this.floor); @@ -107,63 +103,20 @@ export class AvatarRenderer { ); } - public setSelectedPart(part: BodyPartTypes) { + public setDistanceAndOffset( + cameraTargetOffset: { x?: number; y?: number; z?: number }, + cameraTargetDistance: number, + ) { if (!this.cameraManager) return; - this.selectedPart = part; - if (this.cameraFocusMap.has(part)) { - this.cameraManager.setLerpedTarget(this.cameraFocusMap.get(part)!); - switch (part) { - case "fullBody": { - this.cameraManager.targetDistance = 2.5; - break; - } - case "head": { - this.cameraManager.targetDistance = 0.8; - break; - } - case "upperBody": { - this.cameraManager.targetDistance = 1.2; - break; - } - case "lowerBody": { - this.cameraManager.targetDistance = 1.3; - break; - } - case "feet": { - this.cameraManager.targetDistance = 0.9; - break; - } - default: { - break; - } - } - } + this.cameraTargetOffset = cameraTargetOffset; + this.cameraTargetDistance = cameraTargetDistance; + this.cameraManager.setLerpedTarget( + new Vector3(this.cameraTargetOffset.x, this.cameraTargetOffset.y, this.cameraTargetOffset.z), + this.cameraTargetDistance, + ); } public async animateCharacter(model: Object3D) { - model.traverse((child) => { - if (child.type === "Bone") { - if (child.name === "head") { - this.cameraFocusMap.set("head", child.getWorldPosition(new Vector3())); - } - if (child.name === "spine_01") { - this.cameraFocusMap.set("fullBody", child.getWorldPosition(new Vector3())); - } - if (child.name === "spine_03") { - this.cameraFocusMap.set("upperBody", child.getWorldPosition(new Vector3())); - } - if (child.name === "pelvis") { - this.cameraFocusMap.set( - "lowerBody", - child.getWorldPosition(new Vector3()).sub(new Vector3(0.0, 0.35, 0.0)), - ); - this.cameraFocusMap.set( - "feet", - child.getWorldPosition(new Vector3()).sub(new Vector3(0.0, 0.8, 0.0)), - ); - } - } - }); this.mixer = new AnimationMixer(model); if (this.animationAsset === null) { this.animationAsset = await this.modelLoader.load(this.idleAnimationURL); @@ -192,8 +145,7 @@ export class AvatarRenderer { Math.PI / 2.3, Math.PI / 2, ); - this.cameraManager.setLerpedTarget(new Vector3(0, 0.9, 0)); - this.cameraManager.targetDistance = 2.1; + this.cameraManager.setLerpedTarget(new Vector3(0, 0.9, 0), 2.1); } if (this.cameraManager?.camera) { this.cameraManager.update(); diff --git a/packages/3d-web-standalone-avatar-editor/src/AvatarVisualizer.tsx b/packages/3d-web-standalone-avatar-editor/src/AvatarVisualizer.tsx index 75605f06..6862e905 100644 --- a/packages/3d-web-standalone-avatar-editor/src/AvatarVisualizer.tsx +++ b/packages/3d-web-standalone-avatar-editor/src/AvatarVisualizer.tsx @@ -1,4 +1,3 @@ -import { BodyPartTypes } from "@mml-io/3d-web-avatar-editor-ui"; import React, { useCallback, useEffect, useRef } from "react"; import { Object3D } from "three"; @@ -8,14 +7,26 @@ type AvatarVisualizerProps = { characterMesh: Object3D; hdrURL: string; idleAnimationURL: string; - selectedPart: BodyPartTypes; + cameraTargetOffset: { + x?: number; + y?: number; + z?: number; + }; + cameraTargetDistance: number; }; +export type XYZ = { x?: number; y?: number; z?: number }; + +function isXYZEqual(a: XYZ, b: XYZ) { + return a.x === b.x && a.y === b.y && a.z === b.z; +} + export const AvatarVisualizer: React.FC = ({ characterMesh, hdrURL, idleAnimationURL, - selectedPart, + cameraTargetOffset, + cameraTargetDistance, }) => { const containerRef = useRef(null); const visualizerRef = useRef(null); @@ -34,11 +45,14 @@ export const AvatarVisualizer: React.FC = ({ useEffect(() => { if (visualizerRef.current) { - if (visualizerRef.current.selectedPart !== selectedPart) { - visualizerRef.current.setSelectedPart(selectedPart); + if ( + isXYZEqual(visualizerRef.current.cameraTargetOffset, cameraTargetOffset) || + visualizerRef.current.cameraTargetDistance !== cameraTargetDistance + ) { + visualizerRef.current.setDistanceAndOffset(cameraTargetOffset, cameraTargetDistance); } } - }, [selectedPart]); + }, [cameraTargetOffset, cameraTargetDistance]); useEffect(() => { const visualizer = visualizerRef.current; diff --git a/packages/3d-web-standalone-avatar-editor/src/index.ts b/packages/3d-web-standalone-avatar-editor/src/index.ts index 18bb3ede..e69cad8a 100644 --- a/packages/3d-web-standalone-avatar-editor/src/index.ts +++ b/packages/3d-web-standalone-avatar-editor/src/index.ts @@ -1,7 +1,6 @@ export type { AssetDescription, CollectionDataType, - BodyPartTypes, CharacterComposition, } from "@mml-io/3d-web-avatar-editor-ui"; export { CharacterPartsSelector } from "@mml-io/3d-web-avatar-editor-ui"; From 6ec2bcf718685f8b4a8c19463c36add90e918701 Mon Sep 17 00:00:00 2001 From: TheCodeTherapy Date: Wed, 8 Nov 2023 15:50:16 +0000 Subject: [PATCH 2/2] removes unecessary type declaration --- example/web-avatar-client/src/AvatarEditor.tsx | 9 --------- 1 file changed, 9 deletions(-) diff --git a/example/web-avatar-client/src/AvatarEditor.tsx b/example/web-avatar-client/src/AvatarEditor.tsx index 9c0ae920..291f5263 100644 --- a/example/web-avatar-client/src/AvatarEditor.tsx +++ b/example/web-avatar-client/src/AvatarEditor.tsx @@ -14,15 +14,6 @@ import hdrURL from "../../assets/hdr/industrial_sunset_2k.hdr"; type BodyPartTypes = "fullBody" | "head" | "upperBody" | "lowerBody" | "feet"; -export type MMLCharacterDescriptionPart = { - url: string; -}; - -export type MMLCharacterDescription = { - base: MMLCharacterDescriptionPart; - parts: MMLCharacterDescriptionPart[]; -}; - const partToCameraOffset = new Map< BodyPartTypes, {