[OoT/MM] Support ZAPD bone enums for skeleton import (#396)

* Ignore IntelliJ editor data

* [OoT] Add skeleton import test scripts

In preparation for fixing skeleton imports from the MM decomp, make a
script for testing skeleton imports.

Running `python3 scripts/make_all_skeletons.py <decomp path> <output
folder>` will attempt to import all skeletons from every file in the
decomp that appears to contain a skeleton, and report on how many
files raised exceptions during the import. The generated .blend files
are stored in the output folder. Including a third argument of ` will
attempt to import all animations for files that contain a single
skeleton as well.

Running this on the mm decomp at commit
803ff1fb1593cdc0c62d14882973af04dc0f988e (from 2024-07-13) results in
only 3/174 (1.7%) of files with skeletons importing them without exceptions.

* [OoT] Strip comments in limb list

The MM decomp specifies `EnumName` values for limbs in the asset XML,
which generates an enum naming bone indicies. These enum values are
also generated in comments in the limb list:

  void* gDekuButlerSkelLimbs[] = {
    &gDekuButlerRootLimb, /* DEKU_BUTLER_LIMB_ROOT */

Fast64's current limb list parsing does not expect comments here,
which causes the limb list parse to fail.

Add a function to strip comments of this style, and strip them before
parsing limb list entires.

* [OoT] Identify and parse enums when importing skeletnos

The MM decomp defines EnumNames for limbs in asset XML, which causes
ZAPD to generate limb definitions that use these enum values for the
next child and next sibling (with an offset of 1).

  StandardLimb gDekuButlerRootLimb = {
    { 0, 2775, 0 }, DEKU_BUTLER_LIMB_PELVIS - 1, LIMB_DONE,

ootGetLimb currently only supports int values, hex values, or
LIMB_DONE here.

In preparation for supporting limb enum values of this form, add the
object's header file to skeletonData and parse all enums found during
skeleton import. The next patch in this series will use the parsed
enums to handle limb definitions of this form.

* [OoT] Support limb enums for limb nextChild / nextSibling

Use the parsed enums to support next child and next sibling
definitions of the form `<limb enum value> - 1`.

With this change and the others from this patch series,
`make_all_skeletons.py` goes from just 3/174 (1.1%) of MM decomp files
with successful skeleton imports to 169/174 (97.1%).

.gitignore

@@ -4,4 +4,4 @@ __pycache__/
 /.venv
 fast64_updater/
 .python-version
-
+.idea

fast64_internal/oot/oot_utility.py

@@ -8,7 +8,7 @@
 from bpy.types import Object
 from bpy.utils import register_class, unregister_class
 from bpy.types import Object
-from typing import Callable, Optional, TYPE_CHECKING
+from typing import Callable, Optional, TYPE_CHECKING, List
 from .oot_constants import ootSceneIDToName
 from dataclasses import dataclass
 
@@ -183,6 +183,43 @@ def getOOTScale(actorScale: float) -> float:
     return bpy.context.scene.ootBlenderScale * actorScale
 
 
+@dataclass
+class OOTEnum:
+    """
+    Represents a enum parsed from C code
+    """
+
+    name: str
+    vals: List[str]
+
+    @staticmethod
+    def fromMatch(m: re.Match):
+        return OOTEnum(m.group("name"), OOTEnum.parseVals(m.group("vals")))
+
+    @staticmethod
+    def parseVals(valsCode: str) -> List[str]:
+        return [entry.strip() for entry in ootStripComments(valsCode).split(",")]
+
+    def indexOrNone(self, valueorNone: str):
+        return self.vals.index(valueorNone) if valueorNone in self.vals else None
+
+
+def ootGetEnums(code: str) -> List["OOTEnum"]:
+    return [
+        OOTEnum.fromMatch(m)
+        for m in re.finditer(
+            r"(?<!extern)\s*"
+            + r"typedef\s*enum\s*(?P<name>[A-Za-z0-9\_]+)"  # doesn't start with extern (is defined here)
+            + r"\s*\{"  # typedef enum gDekukButlerLimb
+            + r"(?P<vals>[^\}]*)"  # opening curly brace
+            + r"\s*\}"  # values
+            + r"\s*\1"  # closing curly brace
+            + r"\s*;",  # name again  # end statement
+            code,
+        )
+    ]
+
+
 def replaceMatchContent(data: str, newContent: str, match: re.Match, index: int) -> str:
     return data[: match.start(index)] + newContent + data[match.end(index) :]
 
@@ -213,6 +250,12 @@ def getSceneDirFromLevelName(name):
     return None
 
 
+def ootStripComments(code: str) -> str:
+    code = re.sub(r"\/\*[^*]*\*+(?:[^/*][^*]*\*+)*\/", "", code)  # replace /* ... */ comments
+    # TODO: replace end of line (// ...) comments
+    return code
+
+
 @dataclass
 class ExportInfo:
     """Contains all parameters used for a scene export. Any new parameters for scene export should be added here."""
@@ -415,7 +458,7 @@ def checkEmptyName(name):
         raise PluginError("No name entered for the exporter.")
 
 
-def ootGetObjectPath(isCustomExport, exportPath, folderName):
+def ootGetObjectPath(isCustomExport: bool, exportPath: str, folderName: str) -> str:
     if isCustomExport:
         filepath = exportPath
     else:
@@ -425,6 +468,16 @@ def ootGetObjectPath(isCustomExport, exportPath, folderName):
     return filepath
 
 
+def ootGetObjectHeaderPath(isCustomExport: bool, exportPath: str, folderName: str) -> str:
+    if isCustomExport:
+        filepath = exportPath
+    else:
+        filepath = os.path.join(
+            ootGetPath(exportPath, isCustomExport, "assets/objects/", folderName, False, False), folderName + ".h"
+        )
+    return filepath
+
+
 def ootGetPath(exportPath, isCustomExport, subPath, folderName, makeIfNotExists, useFolderForCustom):
     if isCustomExport:
         path = bpy.path.abspath(os.path.join(exportPath, (folderName if useFolderForCustom else "")))

fast64_internal/oot/skeleton/importer/functions.py

@@ -1,10 +1,12 @@
+import re
+from typing import List
 import mathutils, bpy, math
 from ....f3d.f3d_gbi import F3D, get_F3D_GBI
 from ....f3d.f3d_parser import getImportData, parseF3D
-from ....utility import hexOrDecInt, applyRotation
+from ....utility import hexOrDecInt, applyRotation, PluginError
 from ...oot_f3d_writer import ootReadActorScale
 from ...oot_model_classes import OOTF3DContext, ootGetIncludedAssetData
-from ...oot_utility import ootGetObjectPath, getOOTScale
+from ...oot_utility import ootGetObjectPath, getOOTScale, ootGetObjectHeaderPath, ootGetEnums, ootStripComments
 from ...oot_texture_array import ootReadTextureArrays
 from ..constants import ootSkeletonImportDict
 from ..properties import OOTSkeletonImportSettings
@@ -58,6 +60,7 @@ def ootAddLimbRecursively(
     parentBoneName: str,
     f3dContext: OOTF3DContext,
     useFarLOD: bool,
+    enums: List["OOTEnum"],
 ):
     limbName = f3dContext.getLimbName(limbIndex)
     boneName = f3dContext.getBoneName(limbIndex)
@@ -82,9 +85,9 @@ def ootAddLimbRecursively(
 
     LIMB_DONE = 0xFF
     nextChildIndexStr = matchResult.group(4)
-    nextChildIndex = LIMB_DONE if nextChildIndexStr == "LIMB_DONE" else hexOrDecInt(nextChildIndexStr)
+    nextChildIndex = ootEvaluateLimbExpression(nextChildIndexStr, enums)
     nextSiblingIndexStr = matchResult.group(5)
-    nextSiblingIndex = LIMB_DONE if nextSiblingIndexStr == "LIMB_DONE" else hexOrDecInt(nextSiblingIndexStr)
+    nextSiblingIndex = ootEvaluateLimbExpression(nextSiblingIndexStr, enums)
 
     # str(limbIndex) + " " + str(translation) + " " + str(nextChildIndex) + " " + \
     # 	str(nextSiblingIndex) + " " + str(dlName))
@@ -102,17 +105,51 @@ def ootAddLimbRecursively(
 
     if nextChildIndex != LIMB_DONE:
         isLOD |= ootAddLimbRecursively(
-            nextChildIndex, skeletonData, obj, armatureObj, currentTransform, boneName, f3dContext, useFarLOD
+            nextChildIndex, skeletonData, obj, armatureObj, currentTransform, boneName, f3dContext, useFarLOD, enums
         )
 
     if nextSiblingIndex != LIMB_DONE:
         isLOD |= ootAddLimbRecursively(
-            nextSiblingIndex, skeletonData, obj, armatureObj, parentTransform, parentBoneName, f3dContext, useFarLOD
+            nextSiblingIndex,
+            skeletonData,
+            obj,
+            armatureObj,
+            parentTransform,
+            parentBoneName,
+            f3dContext,
+            useFarLOD,
+            enums,
         )
 
     return isLOD
 
 
+def ootEvaluateLimbExpression(expr: str, enums: List["OOTEnum"]) -> int:
+    """
+    Evaluate an expression used to define a limb index.
+    Limited support for expected expression values:
+    - "LIMB_DONE"
+    - int value
+    - hex value
+    - "<ENUM_VALUE> - 1"
+    """
+    LIMB_DONE = 0xFF
+
+    if expr == "LIMB_DONE":
+        return LIMB_DONE
+
+    m = re.search(r"(?P<val>[A-Za-z0-9\_]+)\s*-\s*1", expr)
+    if m is not None:
+        val = m.group("val")
+        index = next((enum.indexOrNone(val) for enum in enums if enum.indexOrNone(val) is not None), None)
+        if index is None:
+            raise PluginError(f"Couldn't find index for enum value {val}")
+
+        return index - 1
+
+    return hexOrDecInt(expr)
+
+
 def ootBuildSkeleton(
     skeletonName,
     overlayName,
@@ -147,11 +184,16 @@ def ootBuildSkeleton(
 
     f3dContext.mat().draw_layer.oot = armatureObj.ootDrawLayer
 
+    # Parse enums, which may be used to link bones by index
+    enums = ootGetEnums(skeletonData)
+
     if overlayName is not None:
         ootReadTextureArrays(basePath, overlayName, skeletonName, f3dContext, isLink, flipbookArrayIndex2D)
 
     transformMatrix = mathutils.Matrix.Scale(1 / actorScale, 4)
-    isLOD = ootAddLimbRecursively(0, skeletonData, obj, armatureObj, transformMatrix, None, f3dContext, useFarLOD)
+    isLOD = ootAddLimbRecursively(
+        0, skeletonData, obj, armatureObj, transformMatrix, None, f3dContext, useFarLOD, enums
+    )
     for dlEntry in f3dContext.dlList:
         limbName = f3dContext.getLimbName(dlEntry.limbIndex)
         boneName = f3dContext.getBoneName(dlEntry.limbIndex)
@@ -216,7 +258,10 @@ def ootImportSkeletonC(basePath: str, importSettings: OOTSkeletonImportSettings)
         isLink = False
         restPoseData = None
 
-    filepaths = [ootGetObjectPath(isCustomImport, importPath, folderName)]
+    filepaths = [
+        ootGetObjectPath(isCustomImport, importPath, folderName),
+        ootGetObjectHeaderPath(isCustomImport, importPath, folderName),
+    ]
 
     removeDoubles = importSettings.removeDoubles
     importNormals = importSettings.importNormals
@@ -231,7 +276,7 @@ def ootImportSkeletonC(basePath: str, importSettings: OOTSkeletonImportSettings)
 
     matchResult = ootGetLimbs(skeletonData, limbsName, False)
     limbsData = matchResult.group(2)
-    limbList = [entry.strip()[1:] for entry in limbsData.split(",") if entry.strip() != ""]
+    limbList = [entry.strip()[1:] for entry in ootStripComments(limbsData).split(",") if entry.strip() != ""]
 
     f3dContext = OOTF3DContext(get_F3D_GBI(), limbList, basePath)
     f3dContext.mat().draw_layer.oot = drawLayer

fast64_internal/oot/skeleton/utility.py

@@ -1,7 +1,7 @@
 import mathutils, bpy, os, re
 from ...utility_anim import armatureApplyWithMesh
 from ..oot_model_classes import OOTVertexGroupInfo
-from ..oot_utility import checkForStartBone, getStartBone, getNextBone
+from ..oot_utility import checkForStartBone, getStartBone, getNextBone, ootStripComments
 
 from ...utility import (
     PluginError,
@@ -65,7 +65,7 @@ def ootGetLimb(skeletonData, limbName, continueOnError):
     matchResult = re.search(
         "[A-Za-z0-9\_]*Limb\s*"
         + re.escape(limbName)
-        + "\s*=\s*\{\s*\{\s*([^,\s]*)\s*,\s*([^,\s]*)\s*,\s*([^,\s]*)\s*\},\s*([^, ]*)\s*,\s*([^, ]*)\s*,\s*"
+        + "\s*=\s*\{\s*\{\s*([^,\s]*)\s*,\s*([^,\s]*)\s*,\s*([^,\s]*)\s*\},\s*([^,]*)\s*,\s*([^,]*)\s*,\s*"
         + dlRegex
         + "\s*\}\s*;\s*",
         skeletonData,
@@ -147,7 +147,7 @@ def ootRemoveSkeleton(filepath, objectName, skeletonName):
         return
     skeletonDataC = skeletonDataC[: matchResult.start(0)] + skeletonDataC[matchResult.end(0) :]
     limbsData = matchResult.group(2)
-    limbList = [entry.strip()[1:] for entry in limbsData.split(",") if entry.strip() != ""]
+    limbList = [entry.strip()[1:] for entry in ootStripComments(limbsData).split(",") if entry.strip() != ""]
 
     headerMatch = getDeclaration(skeletonDataH, limbsName)
     if headerMatch is not None:

scripts/oot/make_all_skeletons.py

@@ -0,0 +1,94 @@
+import re
+import subprocess
+import sys
+import shutil
+
+from pathlib import Path
+from typing import List
+
+"""
+A test script to try import skeletons (and optionally animations) in a decomp
+folder, generating .blend files for file, and report on successes & failures.
+
+WARNING: the specified output folder is unconditionally deleted before generating 
+new output!
+
+Usage:
+python3 make_all_skeletons.py <path to decomp> <output folder>  ["1" to import animations too]
+
+Example:
+python3 make_all_skeletons.py ~/git/mm blend-files 1 
+"""
+
+
+def main():
+    filePaths: List[Path] = []
+    failFiles: List[Path] = []
+    successFiles: List[Path] = []
+
+    print(f"args {sys.argv}")
+    decompPath = Path(sys.argv[1])
+    outputPath = Path(sys.argv[2])
+    importAnimations = len(sys.argv) > 3 and sys.argv[3] == "1"
+
+    # Delete the output folder if it already exists
+    if outputPath.exists():
+        shutil.rmtree(outputPath)
+
+    # populate filePaths with paths to all files in the decomp
+    # that appear to contain a skeleton
+    for inPath in (Path(decompPath) / "assets" / "objects").rglob("*.c"):
+        with open(inPath, "r") as file:
+            contents = file.read()
+            if re.search(r"(Flex)?SkeletonHeader\s*(?P<name>[A-Za-z0-9\_]+)\s*=", contents) is not None:
+                filePaths.append(inPath)
+
+    for i, inPath in enumerate(filePaths):
+        # Generate the output path as a subdir in the output folder with the same structure
+        #  as the file's location in the decomp
+        outPath: Path = outputPath.joinpath(*inPath.parts[len(decompPath.parts) :]).with_suffix(".blend")
+        objectName = inPath.parts[-2]
+
+        # Make sure all the subdirs exist
+        outPath.parent.mkdir(parents=True, exist_ok=True)
+
+        # Run make_skeletons.py in blender to build the .blend file
+        args = [
+            "blender",
+            "--background",
+            "--python-exit-code",  # note: python-exit-code MUST come before python, or you'll always get 0!
+            "1",
+            "--python",
+            "make_skeletons.py",
+            "--",
+            decompPath,
+            inPath,
+            outPath,
+            objectName,
+        ]
+
+        if importAnimations:
+            args.append("1")
+
+        res = subprocess.run(args)
+
+        if res.returncode == 0:
+            successFiles.append(inPath)
+        else:
+            failFiles.append(inPath)
+            print("! Failed")
+
+        # Report progress
+        print(f"Progress: {i + 1}/{len(filePaths)} done")
+        percentSuccessful = round(len(successFiles) / len(filePaths) * 100, 1)
+        percentFailed = round(len(failFiles) / len(filePaths) * 100, 1)
+        print(f"\tSuccessful:  {len(successFiles)} {percentSuccessful:.1f}%")
+        print(f"\tFailed:  {len(failFiles)} {percentFailed:.1f}%", flush=True)
+
+    # After all imports have been tried, list all the files with any failures
+    print("Files with failures:")
+    print("\n".join(str(f) for f in failFiles))
+
+
+if __name__ == "__main__":
+    main()