From a3ca8cfbda89d808c9def4635f155e98a0da2706 Mon Sep 17 00:00:00 2001 From: m000z0rz Date: Fri, 2 Aug 2024 10:27:15 -0500 Subject: [PATCH] [OoT/MM] Support ZAPD bone enums for skeleton import (#396) * Ignore IntelliJ editor data * [OoT] Add skeleton import test scripts In preparation for fixing skeleton imports from the MM decomp, make a script for testing skeleton imports. Running `python3 scripts/make_all_skeletons.py ` will attempt to import all skeletons from every file in the decomp that appears to contain a skeleton, and report on how many files raised exceptions during the import. The generated .blend files are stored in the output folder. Including a third argument of ` will attempt to import all animations for files that contain a single skeleton as well. Running this on the mm decomp at commit 803ff1fb1593cdc0c62d14882973af04dc0f988e (from 2024-07-13) results in only 3/174 (1.7%) of files with skeletons importing them without exceptions. * [OoT] Strip comments in limb list The MM decomp specifies `EnumName` values for limbs in the asset XML, which generates an enum naming bone indicies. These enum values are also generated in comments in the limb list: void* gDekuButlerSkelLimbs[] = { &gDekuButlerRootLimb, /* DEKU_BUTLER_LIMB_ROOT */ Fast64's current limb list parsing does not expect comments here, which causes the limb list parse to fail. Add a function to strip comments of this style, and strip them before parsing limb list entires. * [OoT] Identify and parse enums when importing skeletnos The MM decomp defines EnumNames for limbs in asset XML, which causes ZAPD to generate limb definitions that use these enum values for the next child and next sibling (with an offset of 1). StandardLimb gDekuButlerRootLimb = { { 0, 2775, 0 }, DEKU_BUTLER_LIMB_PELVIS - 1, LIMB_DONE, ootGetLimb currently only supports int values, hex values, or LIMB_DONE here. In preparation for supporting limb enum values of this form, add the object's header file to skeletonData and parse all enums found during skeleton import. The next patch in this series will use the parsed enums to handle limb definitions of this form. * [OoT] Support limb enums for limb nextChild / nextSibling Use the parsed enums to support next child and next sibling definitions of the form ` - 1`. With this change and the others from this patch series, `make_all_skeletons.py` goes from just 3/174 (1.1%) of MM decomp files with successful skeleton imports to 169/174 (97.1%). --- .gitignore | 2 +- fast64_internal/oot/oot_utility.py | 57 ++++++++++- .../oot/skeleton/importer/functions.py | 63 ++++++++++-- fast64_internal/oot/skeleton/utility.py | 6 +- scripts/oot/make_all_skeletons.py | 94 ++++++++++++++++++ scripts/oot/make_skeletons.py | 97 +++++++++++++++++++ 6 files changed, 304 insertions(+), 15 deletions(-) create mode 100644 scripts/oot/make_all_skeletons.py create mode 100644 scripts/oot/make_skeletons.py diff --git a/.gitignore b/.gitignore index 28693ec7e..0cb2f6e73 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,4 @@ __pycache__/ /.venv fast64_updater/ .python-version - +.idea diff --git a/fast64_internal/oot/oot_utility.py b/fast64_internal/oot/oot_utility.py index f8febd130..7c9fe103c 100644 --- a/fast64_internal/oot/oot_utility.py +++ b/fast64_internal/oot/oot_utility.py @@ -8,7 +8,7 @@ from bpy.types import Object from bpy.utils import register_class, unregister_class from bpy.types import Object -from typing import Callable, Optional, TYPE_CHECKING +from typing import Callable, Optional, TYPE_CHECKING, List from .oot_constants import ootSceneIDToName from dataclasses import dataclass @@ -183,6 +183,43 @@ def getOOTScale(actorScale: float) -> float: return bpy.context.scene.ootBlenderScale * actorScale +@dataclass +class OOTEnum: + """ + Represents a enum parsed from C code + """ + + name: str + vals: List[str] + + @staticmethod + def fromMatch(m: re.Match): + return OOTEnum(m.group("name"), OOTEnum.parseVals(m.group("vals"))) + + @staticmethod + def parseVals(valsCode: str) -> List[str]: + return [entry.strip() for entry in ootStripComments(valsCode).split(",")] + + def indexOrNone(self, valueorNone: str): + return self.vals.index(valueorNone) if valueorNone in self.vals else None + + +def ootGetEnums(code: str) -> List["OOTEnum"]: + return [ + OOTEnum.fromMatch(m) + for m in re.finditer( + r"(?[A-Za-z0-9\_]+)" # doesn't start with extern (is defined here) + + r"\s*\{" # typedef enum gDekukButlerLimb + + r"(?P[^\}]*)" # opening curly brace + + r"\s*\}" # values + + r"\s*\1" # closing curly brace + + r"\s*;", # name again # end statement + code, + ) + ] + + def replaceMatchContent(data: str, newContent: str, match: re.Match, index: int) -> str: return data[: match.start(index)] + newContent + data[match.end(index) :] @@ -213,6 +250,12 @@ def getSceneDirFromLevelName(name): return None +def ootStripComments(code: str) -> str: + code = re.sub(r"\/\*[^*]*\*+(?:[^/*][^*]*\*+)*\/", "", code) # replace /* ... */ comments + # TODO: replace end of line (// ...) comments + return code + + @dataclass class ExportInfo: """Contains all parameters used for a scene export. Any new parameters for scene export should be added here.""" @@ -415,7 +458,7 @@ def checkEmptyName(name): raise PluginError("No name entered for the exporter.") -def ootGetObjectPath(isCustomExport, exportPath, folderName): +def ootGetObjectPath(isCustomExport: bool, exportPath: str, folderName: str) -> str: if isCustomExport: filepath = exportPath else: @@ -425,6 +468,16 @@ def ootGetObjectPath(isCustomExport, exportPath, folderName): return filepath +def ootGetObjectHeaderPath(isCustomExport: bool, exportPath: str, folderName: str) -> str: + if isCustomExport: + filepath = exportPath + else: + filepath = os.path.join( + ootGetPath(exportPath, isCustomExport, "assets/objects/", folderName, False, False), folderName + ".h" + ) + return filepath + + def ootGetPath(exportPath, isCustomExport, subPath, folderName, makeIfNotExists, useFolderForCustom): if isCustomExport: path = bpy.path.abspath(os.path.join(exportPath, (folderName if useFolderForCustom else ""))) diff --git a/fast64_internal/oot/skeleton/importer/functions.py b/fast64_internal/oot/skeleton/importer/functions.py index 04f25d088..d2687582b 100644 --- a/fast64_internal/oot/skeleton/importer/functions.py +++ b/fast64_internal/oot/skeleton/importer/functions.py @@ -1,10 +1,12 @@ +import re +from typing import List import mathutils, bpy, math from ....f3d.f3d_gbi import F3D, get_F3D_GBI from ....f3d.f3d_parser import getImportData, parseF3D -from ....utility import hexOrDecInt, applyRotation +from ....utility import hexOrDecInt, applyRotation, PluginError from ...oot_f3d_writer import ootReadActorScale from ...oot_model_classes import OOTF3DContext, ootGetIncludedAssetData -from ...oot_utility import ootGetObjectPath, getOOTScale +from ...oot_utility import ootGetObjectPath, getOOTScale, ootGetObjectHeaderPath, ootGetEnums, ootStripComments from ...oot_texture_array import ootReadTextureArrays from ..constants import ootSkeletonImportDict from ..properties import OOTSkeletonImportSettings @@ -58,6 +60,7 @@ def ootAddLimbRecursively( parentBoneName: str, f3dContext: OOTF3DContext, useFarLOD: bool, + enums: List["OOTEnum"], ): limbName = f3dContext.getLimbName(limbIndex) boneName = f3dContext.getBoneName(limbIndex) @@ -82,9 +85,9 @@ def ootAddLimbRecursively( LIMB_DONE = 0xFF nextChildIndexStr = matchResult.group(4) - nextChildIndex = LIMB_DONE if nextChildIndexStr == "LIMB_DONE" else hexOrDecInt(nextChildIndexStr) + nextChildIndex = ootEvaluateLimbExpression(nextChildIndexStr, enums) nextSiblingIndexStr = matchResult.group(5) - nextSiblingIndex = LIMB_DONE if nextSiblingIndexStr == "LIMB_DONE" else hexOrDecInt(nextSiblingIndexStr) + nextSiblingIndex = ootEvaluateLimbExpression(nextSiblingIndexStr, enums) # str(limbIndex) + " " + str(translation) + " " + str(nextChildIndex) + " " + \ # str(nextSiblingIndex) + " " + str(dlName)) @@ -102,17 +105,51 @@ def ootAddLimbRecursively( if nextChildIndex != LIMB_DONE: isLOD |= ootAddLimbRecursively( - nextChildIndex, skeletonData, obj, armatureObj, currentTransform, boneName, f3dContext, useFarLOD + nextChildIndex, skeletonData, obj, armatureObj, currentTransform, boneName, f3dContext, useFarLOD, enums ) if nextSiblingIndex != LIMB_DONE: isLOD |= ootAddLimbRecursively( - nextSiblingIndex, skeletonData, obj, armatureObj, parentTransform, parentBoneName, f3dContext, useFarLOD + nextSiblingIndex, + skeletonData, + obj, + armatureObj, + parentTransform, + parentBoneName, + f3dContext, + useFarLOD, + enums, ) return isLOD +def ootEvaluateLimbExpression(expr: str, enums: List["OOTEnum"]) -> int: + """ + Evaluate an expression used to define a limb index. + Limited support for expected expression values: + - "LIMB_DONE" + - int value + - hex value + - " - 1" + """ + LIMB_DONE = 0xFF + + if expr == "LIMB_DONE": + return LIMB_DONE + + m = re.search(r"(?P[A-Za-z0-9\_]+)\s*-\s*1", expr) + if m is not None: + val = m.group("val") + index = next((enum.indexOrNone(val) for enum in enums if enum.indexOrNone(val) is not None), None) + if index is None: + raise PluginError(f"Couldn't find index for enum value {val}") + + return index - 1 + + return hexOrDecInt(expr) + + def ootBuildSkeleton( skeletonName, overlayName, @@ -147,11 +184,16 @@ def ootBuildSkeleton( f3dContext.mat().draw_layer.oot = armatureObj.ootDrawLayer + # Parse enums, which may be used to link bones by index + enums = ootGetEnums(skeletonData) + if overlayName is not None: ootReadTextureArrays(basePath, overlayName, skeletonName, f3dContext, isLink, flipbookArrayIndex2D) transformMatrix = mathutils.Matrix.Scale(1 / actorScale, 4) - isLOD = ootAddLimbRecursively(0, skeletonData, obj, armatureObj, transformMatrix, None, f3dContext, useFarLOD) + isLOD = ootAddLimbRecursively( + 0, skeletonData, obj, armatureObj, transformMatrix, None, f3dContext, useFarLOD, enums + ) for dlEntry in f3dContext.dlList: limbName = f3dContext.getLimbName(dlEntry.limbIndex) boneName = f3dContext.getBoneName(dlEntry.limbIndex) @@ -216,7 +258,10 @@ def ootImportSkeletonC(basePath: str, importSettings: OOTSkeletonImportSettings) isLink = False restPoseData = None - filepaths = [ootGetObjectPath(isCustomImport, importPath, folderName)] + filepaths = [ + ootGetObjectPath(isCustomImport, importPath, folderName), + ootGetObjectHeaderPath(isCustomImport, importPath, folderName), + ] removeDoubles = importSettings.removeDoubles importNormals = importSettings.importNormals @@ -231,7 +276,7 @@ def ootImportSkeletonC(basePath: str, importSettings: OOTSkeletonImportSettings) matchResult = ootGetLimbs(skeletonData, limbsName, False) limbsData = matchResult.group(2) - limbList = [entry.strip()[1:] for entry in limbsData.split(",") if entry.strip() != ""] + limbList = [entry.strip()[1:] for entry in ootStripComments(limbsData).split(",") if entry.strip() != ""] f3dContext = OOTF3DContext(get_F3D_GBI(), limbList, basePath) f3dContext.mat().draw_layer.oot = drawLayer diff --git a/fast64_internal/oot/skeleton/utility.py b/fast64_internal/oot/skeleton/utility.py index a7897bb53..0a565c355 100644 --- a/fast64_internal/oot/skeleton/utility.py +++ b/fast64_internal/oot/skeleton/utility.py @@ -1,7 +1,7 @@ import mathutils, bpy, os, re from ...utility_anim import armatureApplyWithMesh from ..oot_model_classes import OOTVertexGroupInfo -from ..oot_utility import checkForStartBone, getStartBone, getNextBone +from ..oot_utility import checkForStartBone, getStartBone, getNextBone, ootStripComments from ...utility import ( PluginError, @@ -65,7 +65,7 @@ def ootGetLimb(skeletonData, limbName, continueOnError): matchResult = re.search( "[A-Za-z0-9\_]*Limb\s*" + re.escape(limbName) - + "\s*=\s*\{\s*\{\s*([^,\s]*)\s*,\s*([^,\s]*)\s*,\s*([^,\s]*)\s*\},\s*([^, ]*)\s*,\s*([^, ]*)\s*,\s*" + + "\s*=\s*\{\s*\{\s*([^,\s]*)\s*,\s*([^,\s]*)\s*,\s*([^,\s]*)\s*\},\s*([^,]*)\s*,\s*([^,]*)\s*,\s*" + dlRegex + "\s*\}\s*;\s*", skeletonData, @@ -147,7 +147,7 @@ def ootRemoveSkeleton(filepath, objectName, skeletonName): return skeletonDataC = skeletonDataC[: matchResult.start(0)] + skeletonDataC[matchResult.end(0) :] limbsData = matchResult.group(2) - limbList = [entry.strip()[1:] for entry in limbsData.split(",") if entry.strip() != ""] + limbList = [entry.strip()[1:] for entry in ootStripComments(limbsData).split(",") if entry.strip() != ""] headerMatch = getDeclaration(skeletonDataH, limbsName) if headerMatch is not None: diff --git a/scripts/oot/make_all_skeletons.py b/scripts/oot/make_all_skeletons.py new file mode 100644 index 000000000..99199eb82 --- /dev/null +++ b/scripts/oot/make_all_skeletons.py @@ -0,0 +1,94 @@ +import re +import subprocess +import sys +import shutil + +from pathlib import Path +from typing import List + +""" +A test script to try import skeletons (and optionally animations) in a decomp +folder, generating .blend files for file, and report on successes & failures. + +WARNING: the specified output folder is unconditionally deleted before generating +new output! + +Usage: +python3 make_all_skeletons.py ["1" to import animations too] + +Example: +python3 make_all_skeletons.py ~/git/mm blend-files 1 +""" + + +def main(): + filePaths: List[Path] = [] + failFiles: List[Path] = [] + successFiles: List[Path] = [] + + print(f"args {sys.argv}") + decompPath = Path(sys.argv[1]) + outputPath = Path(sys.argv[2]) + importAnimations = len(sys.argv) > 3 and sys.argv[3] == "1" + + # Delete the output folder if it already exists + if outputPath.exists(): + shutil.rmtree(outputPath) + + # populate filePaths with paths to all files in the decomp + # that appear to contain a skeleton + for inPath in (Path(decompPath) / "assets" / "objects").rglob("*.c"): + with open(inPath, "r") as file: + contents = file.read() + if re.search(r"(Flex)?SkeletonHeader\s*(?P[A-Za-z0-9\_]+)\s*=", contents) is not None: + filePaths.append(inPath) + + for i, inPath in enumerate(filePaths): + # Generate the output path as a subdir in the output folder with the same structure + # as the file's location in the decomp + outPath: Path = outputPath.joinpath(*inPath.parts[len(decompPath.parts) :]).with_suffix(".blend") + objectName = inPath.parts[-2] + + # Make sure all the subdirs exist + outPath.parent.mkdir(parents=True, exist_ok=True) + + # Run make_skeletons.py in blender to build the .blend file + args = [ + "blender", + "--background", + "--python-exit-code", # note: python-exit-code MUST come before python, or you'll always get 0! + "1", + "--python", + "make_skeletons.py", + "--", + decompPath, + inPath, + outPath, + objectName, + ] + + if importAnimations: + args.append("1") + + res = subprocess.run(args) + + if res.returncode == 0: + successFiles.append(inPath) + else: + failFiles.append(inPath) + print("! Failed") + + # Report progress + print(f"Progress: {i + 1}/{len(filePaths)} done") + percentSuccessful = round(len(successFiles) / len(filePaths) * 100, 1) + percentFailed = round(len(failFiles) / len(filePaths) * 100, 1) + print(f"\tSuccessful: {len(successFiles)} {percentSuccessful:.1f}%") + print(f"\tFailed: {len(failFiles)} {percentFailed:.1f}%", flush=True) + + # After all imports have been tried, list all the files with any failures + print("Files with failures:") + print("\n".join(str(f) for f in failFiles)) + + +if __name__ == "__main__": + main() diff --git a/scripts/oot/make_skeletons.py b/scripts/oot/make_skeletons.py new file mode 100644 index 000000000..089377de1 --- /dev/null +++ b/scripts/oot/make_skeletons.py @@ -0,0 +1,97 @@ +import re + +import bpy +import sys + +# import path +from bpy.path import abspath + +""" +A script that can be run in blender to import all skeletons and animations +(if there's only one skeleton) in a file from OOT or MM + +Usage: +blender --background --python-exit-code 1 --python make_skeletons.py -- ["1" to import animations too] + +Example: +blender --background --python-exit-code 1 --python make_skeletons.py -- ~/git/mm ~/git/mm/assets/objects/object_dnj/object_dnj.c deku_butler.blend object_dnj 1 +""" +args = sys.argv[(sys.argv.index("--") + 1) :] + +decompPath = args[0] +inFile = args[1] +outFile = args[2] +objectName = args[3] +importAnimations = len(args) > 4 and args[4] == "1" + +# objectName = path.basename(path.dirname(inFile)) +print(f"decomp path {decompPath}") +print(f"inFile {inFile}") +print(f"outFile {outFile}") +print(f"object name {objectName}") + +# delete the default cube +if bpy.context.view_layer.objects.active.name == "Cube": + bpy.ops.object.delete() + + +with open(inFile, "r") as file: + code = file.read() + +# Identify all skeleton headers in the input file +skeletonNames = list( + m.group("name") for m in re.finditer(r"(Flex)?SkeletonHeader\s*(?P[A-Za-z0-9\_]+)\s*=", code) +) + +# Setup Fast64 settings +bpy.context.scene.gameEditorMode = "OOT" +bpy.context.scene.ootDecompPath = abspath(decompPath) +bpy.context.scene.fast64.oot.animImportSettings.folderName = objectName + +# These aren't used by the script, but we may as well set them to reasonable values +# in case someone is going to work out of the output blend file +bpy.context.scene.fast64.oot.skeletonExportSettings.folder = objectName +bpy.context.scene.fast64.oot.animExportSettings.folderName = objectName +bpy.context.scene.fast64.oot.DLExportSettings.folder = objectName +bpy.context.scene.fast64.oot.collisionExportSettings.folder = objectName + +# Import all skeletons from the file +errs = [] +for skeletonName in skeletonNames: + imp = bpy.context.scene.fast64.oot.skeletonImportSettings + imp.name = skeletonName # e.g. gDekuButlerSkel + imp.folder = objectName # e.g. object_dnj + # TODO: maybe try to identify an appropriate overlay, or allow it as an argument + imp.actorOverlayName = "" # e.g. ovl_En_Dno + + res = bpy.ops.object.oot_import_skeleton() + if "CANCELLED" in res: + errs.append(f"Failed to import skeleton {skeletonName}") + +# Import animations if there's only one skeleton and animation import was anbled +if len(skeletonNames) == 1 and importAnimations: + animationNames = list(m.group("name") for m in re.finditer(r"AnimationHeader\s*(?P[A-Za-z0-9\_]+)\s*=", code)) + + # select the armature + bpy.context.view_layer.objects.active = bpy.context.view_layer.objects[skeletonNames[0]] + + # import each animation + for animationName in animationNames: + bpy.context.scene.fast64.oot.animImportSettings.animName = animationName + res = bpy.ops.object.oot_import_anim() + if "CANCELLED" in res: + errs.append(f"Failed to import animation {animationName}") + +if len(errs) > 0: + raise RuntimeError(f"Errors running skeleton import: {errs}") + +# Set viewport shading to show textures +for area in bpy.context.screen.areas: + if area.type == "VIEW_3D": + for space in area.spaces: + if space.type == "VIEW_3D": + space.shading.type = "MATERIAL" + +# Save the file if anything was imported +if len(skeletonNames) > 0: + bpy.ops.wm.save_mainfile(filepath=outFile)