From 3376f2fff5e4e3abadde763521a60b8d64a51b64 Mon Sep 17 00:00:00 2001 From: snake-biscuits <36507175+snake-biscuits@users.noreply.github.com> Date: Tue, 28 Sep 2021 23:02:43 +1000 Subject: [PATCH] Initial Release --- .github/workflows/python-package.yml | 39 + .gitignore | 169 ++++ CHANGELOG.md | 4 + LICENSE.txt | 21 + README.md | 95 +++ io_import_rbsp/TODOS.txt | 7 + io_import_rbsp/__init__.py | 87 ++ io_import_rbsp/bsp_tool/CHANGELOG.md | 47 ++ io_import_rbsp/bsp_tool/LICENSE | 21 + io_import_rbsp/bsp_tool/README.md | 157 ++++ io_import_rbsp/bsp_tool/__init__.py | 87 ++ io_import_rbsp/bsp_tool/base.py | 169 ++++ io_import_rbsp/bsp_tool/branches/README.md | 203 +++++ io_import_rbsp/bsp_tool/branches/__init__.py | 179 ++++ .../bsp_tool/branches/arkane/__init__.py | 7 + .../bsp_tool/branches/arkane/dark_messiah.py | 40 + io_import_rbsp/bsp_tool/branches/base.py | 143 ++++ .../bsp_tool/branches/gearbox/__init__.py | 5 + .../bsp_tool/branches/gearbox/bshift.py | 24 + .../bsp_tool/branches/id_software/__init__.py | 14 + .../bsp_tool/branches/id_software/quake.py | 280 ++++++ .../bsp_tool/branches/id_software/quake2.py | 124 +++ .../bsp_tool/branches/id_software/quake3.py | 224 +++++ .../branches/infinity_ward/__init__.py | 11 + .../branches/infinity_ward/call_of_duty1.py | 234 +++++ .../bsp_tool/branches/nexon/__init__.py | 26 + .../bsp_tool/branches/nexon/cso2.py | 132 +++ .../bsp_tool/branches/nexon/cso2_2018.py | 40 + .../bsp_tool/branches/nexon/vindictus.py | 220 +++++ .../bsp_tool/branches/py_struct_as_cpp.py | 293 +++++++ .../bsp_tool/branches/respawn/__init__.py | 30 + .../bsp_tool/branches/respawn/apex_legends.py | 403 +++++++++ .../bsp_tool/branches/respawn/titanfall.py | 712 ++++++++++++++++ .../bsp_tool/branches/respawn/titanfall2.py | 270 ++++++ .../bsp_tool/branches/ritual/__init__.py | 9 + io_import_rbsp/bsp_tool/branches/shared.py | 283 +++++++ .../bsp_tool/branches/valve/__init__.py | 19 + .../bsp_tool/branches/valve/alien_swarm.py | 114 +++ .../bsp_tool/branches/valve/goldsrc.py | 116 +++ .../bsp_tool/branches/valve/left4dead.py | 122 +++ .../bsp_tool/branches/valve/left4dead2.py | 126 +++ .../bsp_tool/branches/valve/orange_box.py | 203 +++++ .../bsp_tool/branches/valve/sdk_2013.py | 39 + .../bsp_tool/branches/valve/source.py | 799 ++++++++++++++++++ io_import_rbsp/bsp_tool/branches/vector.py | 257 ++++++ io_import_rbsp/bsp_tool/gearbox.py | 9 + io_import_rbsp/bsp_tool/id_software.py | 120 +++ io_import_rbsp/bsp_tool/infinity_ward.py | 69 ++ io_import_rbsp/bsp_tool/lumps.py | 381 +++++++++ io_import_rbsp/bsp_tool/respawn.py | 196 +++++ io_import_rbsp/bsp_tool/valve.py | 84 ++ io_import_rbsp/rbsp/__init__.py | 4 + io_import_rbsp/rbsp/apex_legends/__init__.py | 58 ++ io_import_rbsp/rbsp/apex_legends/materials.py | 24 + io_import_rbsp/rbsp/rpak_materials.py | 82 ++ io_import_rbsp/rbsp/titanfall/__init__.py | 59 ++ io_import_rbsp/rbsp/titanfall/append_geo.py | 118 +++ io_import_rbsp/rbsp/titanfall/entities.py | 109 +++ io_import_rbsp/rbsp/titanfall/materials.py | 16 + io_import_rbsp/rbsp/titanfall/props.py | 33 + .../rbsp/titanfall/r1_mesh_compiler_notes.txt | 49 ++ io_import_rbsp/rbsp/titanfall2/__init__.py | 3 + io_import_rbsp/rbsp/titanfall2/materials.py | 24 + 63 files changed, 8042 insertions(+) create mode 100644 .github/workflows/python-package.yml create mode 100644 .gitignore create mode 100644 CHANGELOG.md create mode 100644 LICENSE.txt create mode 100644 README.md create mode 100644 io_import_rbsp/TODOS.txt create mode 100644 io_import_rbsp/__init__.py create mode 100644 io_import_rbsp/bsp_tool/CHANGELOG.md create mode 100644 io_import_rbsp/bsp_tool/LICENSE create mode 100644 io_import_rbsp/bsp_tool/README.md create mode 100644 io_import_rbsp/bsp_tool/__init__.py create mode 100644 io_import_rbsp/bsp_tool/base.py create mode 100644 io_import_rbsp/bsp_tool/branches/README.md create mode 100644 io_import_rbsp/bsp_tool/branches/__init__.py create mode 100644 io_import_rbsp/bsp_tool/branches/arkane/__init__.py create mode 100644 io_import_rbsp/bsp_tool/branches/arkane/dark_messiah.py create mode 100644 io_import_rbsp/bsp_tool/branches/base.py create mode 100644 io_import_rbsp/bsp_tool/branches/gearbox/__init__.py create mode 100644 io_import_rbsp/bsp_tool/branches/gearbox/bshift.py create mode 100644 io_import_rbsp/bsp_tool/branches/id_software/__init__.py create mode 100644 io_import_rbsp/bsp_tool/branches/id_software/quake.py create mode 100644 io_import_rbsp/bsp_tool/branches/id_software/quake2.py create mode 100644 io_import_rbsp/bsp_tool/branches/id_software/quake3.py create mode 100644 io_import_rbsp/bsp_tool/branches/infinity_ward/__init__.py create mode 100644 io_import_rbsp/bsp_tool/branches/infinity_ward/call_of_duty1.py create mode 100644 io_import_rbsp/bsp_tool/branches/nexon/__init__.py create mode 100644 io_import_rbsp/bsp_tool/branches/nexon/cso2.py create mode 100644 io_import_rbsp/bsp_tool/branches/nexon/cso2_2018.py create mode 100644 io_import_rbsp/bsp_tool/branches/nexon/vindictus.py create mode 100644 io_import_rbsp/bsp_tool/branches/py_struct_as_cpp.py create mode 100644 io_import_rbsp/bsp_tool/branches/respawn/__init__.py create mode 100644 io_import_rbsp/bsp_tool/branches/respawn/apex_legends.py create mode 100644 io_import_rbsp/bsp_tool/branches/respawn/titanfall.py create mode 100644 io_import_rbsp/bsp_tool/branches/respawn/titanfall2.py create mode 100644 io_import_rbsp/bsp_tool/branches/ritual/__init__.py create mode 100644 io_import_rbsp/bsp_tool/branches/shared.py create mode 100644 io_import_rbsp/bsp_tool/branches/valve/__init__.py create mode 100644 io_import_rbsp/bsp_tool/branches/valve/alien_swarm.py create mode 100644 io_import_rbsp/bsp_tool/branches/valve/goldsrc.py create mode 100644 io_import_rbsp/bsp_tool/branches/valve/left4dead.py create mode 100644 io_import_rbsp/bsp_tool/branches/valve/left4dead2.py create mode 100644 io_import_rbsp/bsp_tool/branches/valve/orange_box.py create mode 100644 io_import_rbsp/bsp_tool/branches/valve/sdk_2013.py create mode 100644 io_import_rbsp/bsp_tool/branches/valve/source.py create mode 100644 io_import_rbsp/bsp_tool/branches/vector.py create mode 100644 io_import_rbsp/bsp_tool/gearbox.py create mode 100644 io_import_rbsp/bsp_tool/id_software.py create mode 100644 io_import_rbsp/bsp_tool/infinity_ward.py create mode 100644 io_import_rbsp/bsp_tool/lumps.py create mode 100644 io_import_rbsp/bsp_tool/respawn.py create mode 100644 io_import_rbsp/bsp_tool/valve.py create mode 100644 io_import_rbsp/rbsp/__init__.py create mode 100644 io_import_rbsp/rbsp/apex_legends/__init__.py create mode 100644 io_import_rbsp/rbsp/apex_legends/materials.py create mode 100644 io_import_rbsp/rbsp/rpak_materials.py create mode 100644 io_import_rbsp/rbsp/titanfall/__init__.py create mode 100644 io_import_rbsp/rbsp/titanfall/append_geo.py create mode 100644 io_import_rbsp/rbsp/titanfall/entities.py create mode 100644 io_import_rbsp/rbsp/titanfall/materials.py create mode 100644 io_import_rbsp/rbsp/titanfall/props.py create mode 100644 io_import_rbsp/rbsp/titanfall/r1_mesh_compiler_notes.txt create mode 100644 io_import_rbsp/rbsp/titanfall2/__init__.py create mode 100644 io_import_rbsp/rbsp/titanfall2/materials.py diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 0000000..86ffa6d --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,39 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Python tests - core + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.7, 3.8, 3.9] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest pytest-cov fake-bpy-module-2.93 + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: | + pytest -vv --cov=bsp_tool diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d53c839 --- /dev/null +++ b/.gitignore @@ -0,0 +1,169 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# Visual Studio Code +.vscode/ + +# Blender +.blend1 +.blend2 + +# C++ +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +# Other +*.blend[12] diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..fd62717 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,4 @@ +# Changelog + +# v1.0.0_b2.93 (~2021) +Initial Release diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..9833469 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Jared Ketterer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..e0edbfe --- /dev/null +++ b/README.md @@ -0,0 +1,95 @@ +# io_import_rbsp +Blender 2.93 importer for `rBSP` files (Titanfall Engine `.bsp`) + + + +## Installation + * Get Blender 2.93+ from [Steam](https://store.steampowered.com/app/365670/Blender/) / [Blender.org](https://www.blender.org/download/) + * Head over to [releases](https://github.com/snake-biscuits/io_import_rbsp/releases/) & download `io_import_rbsp_v1.0.0_b2.93.zip` +Then, in Blender: + * `Edit > Preferences > Add-ons > Install` + * Find `io_import_rbsp_v1.0.0_b2.93.zip` + * Click `Install Addon` + * Check the box to enable `Import-Export: io_import_rbsp` + + +## Usage + +> WARNING: Titanfall Engine maps are huge +> Imports can take multiple minutes and a few GB of RAM for the geometry alone +> Test a small map before loading Olympus and setting your PC on fire + +### Extracting `.bsp`s +You will need to extract `.bsp`, `.bsp_lump` & `.ent` files for any map you want to extract. To do this: + * Grab a [Respawn VPK extractor](#Respawn-VPK) + * Locate the `.vpk`s for the game you want to work with (game must be installed) + - `Titanfall/vpk/` + - `Titanfall2/vpk/` + - `Apex Legends/vpk/` + * Open the `*.bsp.pak000_dir.vpk` for the map you want to load + - Titanfall 2 map names can be found here: [NoSkill Modding Wiki](https://noskill.gitbook.io/titanfall2/documentation/file-location/vpk-file-names) + - Lobbies are stored in `mp_common.bsp.pak000_dir.vpk` + * Extract the `.bsp`, `.ent`s & `.bsp_lumps` from the `maps/` folder to someplace you'll remember + - each `.vpk` holds assets for one `.bsp` (textures and models are stored elsewhere) + + +### Blender Importer +Once you've extracted the files you need: + * `File > Import > Titanfall Engine .bsp` + * Select the `.bsp` (`.bsp_lump` & `.ent` files need to be in the same folder) + * Choose your settings + * Click Import + * Wait a few minutes (Can easily take 1hr+ on Apex Legends maps) + + + + +## Related Tools + +### Respawn VPK + * [TitanfallVPKTool](https://github.com/p0358/TitanfallVPKTool) + - by `P0358` + * [UnoVPKTool](https://github.com/Unordinal/UnoVPKTool) + - by `Unordinal` + * [RSPNVPK](https://github.com/squidgyberries/RSPNVPK) + - Fork of `MrSteyk`'s Tool + * [Titanfall_VPKTool3.4_Portable](https://github.com/Wanty5883/Titanfall2/blob/master/tools/Titanfall_VPKTool3.4_Portable.zip) + - by `Cra0kalo` (currently Closed Source) + +### Other + * [SourceIO](https://github.com/REDxEYE/SourceIO) + - GoldSrc & Source Engine importer (`.bsp`, `.vmt`, `.vtf`, `.mdl`) + * [SourceOps](https://github.com/bonjorno7/SourceOps) + - Source Engine model exporter + * [PyD3DBSP](https://github.com/mauserzjeh/PyD3DBSP) (Archived) + - Call of Duty 2 `.bsp` importer + * [blender_io_mesh_bsp](https://github.com/andyp123/blender_io_mesh_bsp) + - Quake 1 `.bsp` importer + * [Blender_BSP_Importer](https://github.com/QuakeTools/Blender_BSP_Importer) + - Quake 3 `.bsp` importer + + +## FAQs + * Why can't I see anything? + - Titanfall Engine maps are huge, you need to increase your view distance + - `3D View > N > View > Clip Start: 16, End: 51200` + - You will also need to increase the clipping distance for all cameras + * It broke? Help? + - Ask around on Discord, you might've missed a step someplace + - If you're loading a brand new Apex map, it might not be supported yet + * Can I use this to make custom maps? + - No, we don't know enough about Respawn's `.bsp` format to make compilers + - As easy as it might sound on paper, editing a `.bsp` directly is no small task + * Can I use this for animations? + - Sure! but be sure to credit the tool someplace + - And credit Respawn too! they made the maps in the first place + +### Further Questions +I can be found on the following Titanfall related Discord Servers as `b!scuit#3659`: + * Titanfall 1: [TF Remnant Fleet](https://discord.gg/hKpQeJqdZR) + * Titanfall 2: [NoSkill Community](https://discord.gg/sEgmTKg) + * Apex Legends: [R5Reloaded](https://discord.com/invite/jqMkUdXrBr) + + +> NOTE: I am a fully time Uni Student in an Australian Timezone +> Don't go expecting an immediate response diff --git a/io_import_rbsp/TODOS.txt b/io_import_rbsp/TODOS.txt new file mode 100644 index 0000000..fa8306d --- /dev/null +++ b/io_import_rbsp/TODOS.txt @@ -0,0 +1,7 @@ +build script: + pack latest bsp_tool + strip out extensions + build blender addon .zip: + exclude build files & tests + +disclaimers for steyk / dogecore involvement in apex material importing? \ No newline at end of file diff --git a/io_import_rbsp/__init__.py b/io_import_rbsp/__init__.py new file mode 100644 index 0000000..fc14f34 --- /dev/null +++ b/io_import_rbsp/__init__.py @@ -0,0 +1,87 @@ +import bpy +from bpy_extras.io_utils import ImportHelper +from bpy.props import StringProperty, BoolProperty +from bpy.types import Operator + +from . import bsp_tool +from . import rbsp + + +bl_info = { + "name": "io_import_rbsp", + "author": "Jared Ketterer (Bikkie)", + "version": (1, 00, 0), + "blender": (2, 93, 0), + "location": "File > Import > Titanfall Engine .bsp", + "description": "Import maps from Titanfall, Titanfall 2 & Apex Legends", + "doc_url": "https://github.com/snake-biscuits/io_import_rbsp", + "tracker_url": "https://github.com/snake-biscuits/io_import_rbsp/issues", + "category": "Import-Export" +} + + +class ImportRBSP(Operator, ImportHelper): + """Load Titanfall Engine rBSP""" + bl_idname = "io_import_rbsp.rbsp_import" + bl_label = "Titanfall Engine .bsp" + filename_ext = ".bsp" + filter_glob: StringProperty(default="*.bsp", options={"HIDDEN"}, maxlen=255) # noqa F722 + # TODO: load_materials EnumProperty: None, Names, Base Colours, Nodes + load_geometry: BoolProperty(name="Geometry", description="Load .bsp Geometry", default=True) # noqa F722 + load_entities: BoolProperty(name="Entities", description="Load .bsp Entities", default=True) # noqa F722 + # TODO: cubemap volumes? + # TODO: load_lighting EnumProperty: None, Empties, All, PortalLights + # TODO: load_prop_dynamic EnumProperty: None, Empties, Low-Poly, High-Poly + # TODO: load_prop_static EnumProperty: None, Empties, Low-Poly, High-Poly, Skybox Only + # TODO: Lock out some options if SourceIO is not installed, alert the user + # TODO: Warnings for missing files + # TODO: Lightmaps with Pillow (PIL) + + def execute(self, context): + bsp = bsp_tool.load_bsp(self.filepath) + import_script = {bsp_tool.branches.respawn.titanfall: rbsp.titanfall, + bsp_tool.branches.respawn.titanfall2: rbsp.titanfall2, + bsp_tool.branches.respawn.apex_legends: rbsp.apex_legends} + importer = import_script[bsp.branch] + # master_collection + if bsp.filename not in bpy.data.collections: + master_collection = bpy.data.collections.new(bsp.filename) + bpy.context.scene.collection.children.link(master_collection) + else: + master_collection = bpy.data.collections[bsp.filename] + # materials + materials = importer.materials.base_colours(bsp) + # geometry + if self.load_geometry: + # TODO: rename Model[0] "worldspawn" + # TODO: skybox collection + # TODO: load specific model / mesh (e.g. worldspawn only, skip tool brushes etc.) + importer.geometry(bsp, master_collection, materials) + # entities + if self.load_entities: + # TODO: link worldspawn to Model[0] + importer.entities.as_empties(bsp, master_collection) + # NOTE: Eevee has limited lighting, try Cycles + # props + # TODO: import scale (Engine Units -> Inches) + return {"FINISHED"} + + +# Only needed if you want to add into a dynamic menu +def menu_func_import(self, context): + self.layout.operator(ImportRBSP.bl_idname, text=ImportRBSP.bl_label) + + +def register(): + bpy.utils.register_submodule_factory(__name__, ("bsp_tool", "rbsp")) + bpy.utils.register_class(ImportRBSP) + bpy.types.TOPBAR_MT_file_import.append(menu_func_import) + + +def unregister(): + bpy.utils.unregister_class(ImportRBSP) + bpy.types.TOPBAR_MT_file_import.remove(menu_func_import) + + +if __name__ == "__main__": + register() diff --git a/io_import_rbsp/bsp_tool/CHANGELOG.md b/io_import_rbsp/bsp_tool/CHANGELOG.md new file mode 100644 index 0000000..96f29de --- /dev/null +++ b/io_import_rbsp/bsp_tool/CHANGELOG.md @@ -0,0 +1,47 @@ +# Changelog + +# v0.3.0 (~2021) + +## New + * Added `load_bsp` function to identify bsp type + * Added `D3DBsp`, `IdTechBsp`, `RespawnBsp` & `ValveBsp` classes + * Added general support for the PakFile lump + * Added general support for the GameLump lump + * Extension scripts + * `archive.py` extractor for CoD `.iwd` / Quake `.pk3` + * `diff.py` compare bsps for changelogs / study + * `lightmaps.py` bsp lightmap -> `.png` + * `lump_analysis.py` determine lump sizes with stats + * Prototype Blender 2.92 RespawnBsp editor + * Made a basic C++ 17 implementation in `src/` + +## Changed + * `Bsp` lumps are loaded dynamically, reducing memory usage + * New wrapper classes can be found in `bsp_tool/lumps.py` + * `mods/` changed to `branches/` + * added subfolders for developers + * helpful lists for auto-detecting a .bsp's origin + * renamed `team_fortress2` to `valve/orange_box` + * `LumpClasses` now end up in 3 dictionaries per branch script + * `BASIC_LUMP_CLASSES` for types like `short int` + * `LUMP_CLASSES` for standard `LumpClasses` + * `SPECIAL_LUMP_CLASSES` for irregular types (e.g. PakFile) + * `GAME_LUMP_CLASSES` for game lump SpecialLumpClasses + * `Bsp`s no longer print to console once loaded + * `Base.Bsp` & subclasses have reserved ALL CAPS member names for lumps only + * BSP_VERSION, FILE_MAGIC, HEADERS, REVISION -> bsp_version, file_magic, headers, revision + * TODO: load external lumps and internal lumps at the same time + +## New Supported Games + * Call of Duty 1 + * Counter-Strike: Global Offensive + * Counter-Strike: Online 2 + * Counter-Strike: Source + * Quake + * Quake 3 Arena + +## Updated Game Support + * Apex Legends + * Orange Box + * Titanfall + * Titanfall 2 diff --git a/io_import_rbsp/bsp_tool/LICENSE b/io_import_rbsp/bsp_tool/LICENSE new file mode 100644 index 0000000..9833469 --- /dev/null +++ b/io_import_rbsp/bsp_tool/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Jared Ketterer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/io_import_rbsp/bsp_tool/README.md b/io_import_rbsp/bsp_tool/README.md new file mode 100644 index 0000000..1ec18ec --- /dev/null +++ b/io_import_rbsp/bsp_tool/README.md @@ -0,0 +1,157 @@ +# bsp_tool + A library for .bsp file analysis & modification + +`bsp_tool` provides a Command Line Interface for exploring & editing .bsp files +Current development is focused on bringing new maps to Counter-Strike: Online 2 & the Titanfall Engine + + +## Installation +To use the latest version, clone from git: +``` +$ git clone git@github.com:snake-biscuits/bsp_tool.git +``` + +Or to use the latest stable release, install via [pip](https://pypi.org/project/bsp-tool/) (Python 3.7+): +``` +pip install bsp_tool +``` + +> NOTE: The last PyPi release (v0.2.2) is close to a year old +> v0.3.0 has made many [changes](./CHANGELOG.md) and is the recommended version + + +## Fair Use +**Please do not use `bsp_tool` to copy or steal another creator's work** +The primary goal of `bsp_tool` is to extend community mapping tools + + +### Always + - **Ask** the creator's permission before touching their work + - **Understand** that by default creator's works are under copyright + - [US Law Copyright FAQ](https://www.copyright.gov/help/faq/faq-general.html#mywork) + - [US Copyright Duration](https://www.copyright.gov/help/faq/faq-duration.html) + - [Circular 15a](https://www.copyright.gov/circs/circ15a.pdf) + - **Contact** the original creator to get their permission + - This can get complicated + - Some creators don't hold the copyright on their works + - often because of Company / Publisher contracts + - **Credit** the original creator; once you have permission to share a derivative work + - **Support** the official release + +**DO NOT** use this tool to steal another creator's work +**DO** use this tool to understand the .bsp format(s) and create more specific tools + +> Be aware that this gets even more complicated with commercial projects + + +## Usage + +To load a .bsp file in python: + +```python +>>> import bsp_tool +>>> bsp_tool.load_bsp("map_folder/filename.bsp") + +``` + +Full documentation: [snake-biscuits.github.io/bsp_tool/](https://snake-biscuits.github.io/bsp_tool/) + + +## Supported Games + +> The :x: emoji indicates tests are failing +> The :o: emoji indicates a lack of .bsps to test + + * [Arkane Studios](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/arkane) + - [Dark Messiah of Might & Magic](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/arkane/dark_messiah.py) :x: + * [Gearbox Software](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/gearbox) + - [Half-Life: Blue Shift](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/gearbox/bshift.py) :x: + - [Half-Life: Opposing Force](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/goldsrc.py) + * [Id Software](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/id_software) + - [Quake](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/id_software/quake.py) :x: + - [Quake II](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/id_software/quake2.py) + - [Quake III Arena](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/id_software/quake3.py) + - Quake 4 :o: + - Quake Champions :o: + - [Quake Live](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/id_software/quake3.py) + * [Infinity Ward](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/infinity_ward) + - [Call of Duty](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/infinity_ward/call_of_duty1.py) :x: + - Call of Duty 2 :x: + - Call of Duty 4: Modern Warfare :x: + * [Nexon](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/nexon) + - [Counter-Strike: Online 2](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/nexon/cso2.py) :x: + - [Vindictus](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/nexon/vindictus.py) :o: + * [Respawn Entertainment](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/respawn) + - [Apex Legends](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/respawn/apex_legends.py) + - [Titanfall 2](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/respawn/titanfall2.py) + - [Titanfall](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/respawn/titanfall.py) + - [Titanfall: Online](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/respawn/titanfall.py) + * Ritual Entertainment + - American McGee's Alice :o: + - Heavy Metal F.A.K.K. 2 :o: + - Medal of Honor: Allied Assault :o: + - SiN :o: + - SiN: Gold :o: + - SiN Episodes: Emergence :o: + - Star Trek: Elite Force II :o: + * [Valve Software](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve) + - [Alien Swarm](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/alien_swarm.py) + - [Alien Swarm: Reactive Drop](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/alien_swarm.py) + - [Counter-Strike: Condition Zero](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/goldsrc.py) + - [Counter-Strike: Condition Zero - Deleted Scenes](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/goldsrc.py) + - [Counter-Strike: Global Offensive](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/sdk_2013.py) + - [Counter-Strike: Source](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/source.py) + - [Counter-Strike](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/goldsrc.py) + - [Day of Defeat](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/goldsrc.py) + - [Day of Defeat: Source](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/source.py) + - [Deathmatch Classic](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/goldsrc.py) + - [Half-Life](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/goldsrc.py) + - [Half-Life 2](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/source.py) + - [Half-Life 2: Deathmatch](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/source.py) + - [Half-Life 2: Episode 1](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/source.py) + - [Half-Life 2: Episode 2](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/orange_box.py) + - [Half-Life 2: Lost Coast](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/orange_box.py) + - [Half-Life Deathmatch: Source](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/source.py) + - [Half-Life: Source](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/source.py) + - [Left 4 Dead](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/left4dead.py) + - [Left 4 Dead 2](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/left4dead2.py) + - [Portal](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/orange_box.py) + - [Portal 2](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/sdk_2013.py) + - [Richochet](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/goldsrc.py) + - [Source Filmmaker](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/sdk_2013.py) + - [Source SDK 2013](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/orange_box.py) + - [Team Fortress 2](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/orange_box.py) + - [Team Fortress Classic](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/goldsrc.py) + * Other + - [Hexen 2](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/id_software/quake.py) :x: + - [Black Mesa](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/sdk_2013.py) + - [Blade Symphony](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/sdk_2013.py) + - Brink :o: + - Daikatana :o: + - [Fortress Forever](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/orange_box.py) + - [G-String](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/orange_box.py) + - [Garry's Mod](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/orange_box.py) + - [Halfquake Trilogy](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/goldsrc.py) + - [NEOTOKYO](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/orange_box.py) + - [Sven Co-op](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/goldsrc.py) + - [Synergy](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/source.py) + - Tactical Intervention :x: + - [Team Fortress Quake](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/id_software/quake.py) :x: + - Vampire: The Masquerade - Bloodlines :o: + + diff --git a/io_import_rbsp/bsp_tool/__init__.py b/io_import_rbsp/bsp_tool/__init__.py new file mode 100644 index 0000000..09559f3 --- /dev/null +++ b/io_import_rbsp/bsp_tool/__init__.py @@ -0,0 +1,87 @@ +"""A library for .bsp file analysis & modification""" +__all__ = ["base", "branches", "load_bsp", "lumps", "tools", + "GoldSrcBsp", "ValveBsp", "QuakeBsp", "IdTechBsp", "D3DBsp", "RespawnBsp"] + +import difflib +import os +from types import ModuleType +from typing import Union + +from . import base # base.Bsp base class +from . import branches # all known .bsp variant definitions +from . import lumps # handles loading data dynamically +from .id_software import QuakeBsp, IdTechBsp +from .infinity_ward import D3DBsp +from .respawn import RespawnBsp +from .valve import GoldSrcBsp, ValveBsp + + +# NOTE: Quake Live branch_script should be quake3, but auto-detect defaults to quake2 on BSP_VERSION +# NOTE: CoD1 auto-detect by version defaults to ApexLegends + + +developers_by_file_magic = {b"IBSP": IdTechBsp, # or D3DBsp + b"rBSP": RespawnBsp, + b"VBSP": ValveBsp} +# HACK: GoldSrcBsp has no file-magic, substituting BSP_VERSION +goldsrc_versions = [branches.valve.goldsrc.BSP_VERSION, branches.gearbox.bshift.BSP_VERSION] +developers_by_file_magic.update({v.to_bytes(4, "little"): GoldSrcBsp for v in goldsrc_versions}) + +developers_by_file_magic.update({branches.id_software.quake.BSP_VERSION.to_bytes(4, "little"): QuakeBsp}) + +cod_ibsp_versions = [getattr(branches.infinity_ward, b).BSP_VERSION for b in branches.infinity_ward.__all__] + + +def guess_by_file_magic(filename: str) -> (base.Bsp, int): + """returns BspVariant & version""" + if os.path.getsize(filename) == 0: # HL2/ d2_coast_02.bsp + raise RuntimeError(f"{filename} is an empty file") + BspVariant = None + if filename.endswith(".d3dbsp"): + BspVariant = D3DBsp + elif filename.endswith(".bsp"): + with open(filename, "rb") as bsp_file: + file_magic = bsp_file.read(4) + if file_magic not in developers_by_file_magic: + raise RuntimeError(f"'{filename}' does not resemble a .bsp file") + bsp_version = int.from_bytes(bsp_file.read(4), "little") + BspVariant = developers_by_file_magic[file_magic] + if BspVariant == GoldSrcBsp: + bsp_version = int.from_bytes(file_magic, "little") + # D3DBsp has b"IBSP" file_magic + if file_magic == b"IBSP" and bsp_version in cod_ibsp_versions: + BspVariant = D3DBsp + else: # invalid extension + raise RuntimeError(f"{filename} is not a .bsp file!") + return BspVariant, bsp_version + + +def load_bsp(filename: str, branch: Union[str, ModuleType] = "Unknown"): + """Calculate and return the correct base.Bsp sub-class for the given .bsp""" + if not os.path.exists(filename): + raise FileNotFoundError(f".bsp file '{filename}' does not exist.") + BspVariant, bsp_version = guess_by_file_magic(filename) + if isinstance(branch, ModuleType): + return BspVariant(branch, filename, autoload=True) + elif isinstance(branch, str): + # TODO: default to other methods on fail + branch: str = branches.simplify_name(branch) + if branch != "unknown": # not default + if branch not in branches.by_name: + close_matches = difflib.get_close_matches(branch, branches.by_name) + if len(close_matches) == 0: + raise NotImplementedError(f"'{branch}' .bsp format is not supported, yet.") + else: + print(f"'{branch}'.bsp format is not supported. Assumed branches:", + "\n".join(close_matches), + f"Trying '{close_matches[0]}'...", sep="\n") + branch: str = close_matches[0] + branch: ModuleType = branches.by_name[branch] # "name" -> branch_script + # guess branch by format version + else: + if bsp_version not in branches.by_version: + raise NotImplementedError(f"{BspVariant} v{bsp_version} is not supported!") + branch: ModuleType = branches.by_version[bsp_version] + return BspVariant(branch, filename, autoload=True) + else: + raise TypeError(f"Cannot use branch of type `{branch.__class__.__name__}`") diff --git a/io_import_rbsp/bsp_tool/base.py b/io_import_rbsp/bsp_tool/base.py new file mode 100644 index 0000000..b66b4ec --- /dev/null +++ b/io_import_rbsp/bsp_tool/base.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import collections +import enum # for type hints +import os +import struct +from types import MethodType, ModuleType +from typing import Dict, List + +from . import lumps + + +# NOTE: LumpHeaders must have these attrs, but how they are read / order will vary +LumpHeader = collections.namedtuple("LumpHeader", ["offset", "length", "version", "fourCC"]) +ExternalLumpHeader = collections.namedtuple("ExternalLumpHeader", ["offset", "length", "version", "fourCC", + "filename", "filesize"]) +# NOTE: if fourCC != 0: lump is compressed (fourCC value == uncompressed size) + + +class Bsp: + """Bsp base class""" + bsp_version: int = 0 # .bsp format version + associated_files: List[str] # files in the folder of loaded file with similar names + branch: ModuleType # soft copy of "branch script" + bsp_file_size: int = 0 # size of .bsp in bytes + file_magic: bytes = b"XBSP" + filename: str + folder: str + headers: Dict[str, LumpHeader] + # ^ {"LUMP_NAME": LumpHeader} + loading_errors: Dict[str, Exception] + # ^ {"LUMP_NAME": Exception encountered} + + def __init__(self, branch: ModuleType, filename: str = "untitled.bsp", autoload: bool = True): + if not filename.endswith(".bsp"): + raise RuntimeError("Not a .bsp") + filename = os.path.realpath(filename) + self.folder, self.filename = os.path.split(filename) + self.set_branch(branch) + self.headers = dict() + if autoload: + if os.path.exists(filename): + self._preload() + else: + print(f"{filename} not found, creating a new .bsp") + self.headers = {L.name: LumpHeader(0, 0, 0, 0) for L in self.branch.LUMP} + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): + self.file.close() + + def __repr__(self): + version = f"({self.file_magic.decode('ascii', 'ignore')} version {self.bsp_version})" + game = self.branch.__name__[len(self.branch.__package__) + 1:] + return f"<{self.__class__.__name__} '{self.filename}' {game} {version} at 0x{id(self):016X}>" + + def _read_header(self, LUMP: enum.Enum) -> LumpHeader: + """Reads bytes of lump""" + self.file.seek(self.branch.lump_header_address[LUMP]) + offset, length, version, fourCC = struct.unpack("4I", self.file.read(16)) + # TODO: use a read & write function / struct.iter_unpack + # -- this could potentially allow for simplified subclasses + # -- e.g. LumpHeader(*struct.unpack("4I", self.file.read(16))) -> self.LumpHeader(self.file) + header = LumpHeader(offset, length, version, fourCC) + return header + + def _preload(self): + """Loads filename using the format outlined in this .bsp's branch defintion script""" + local_files = os.listdir(self.folder) + def is_related(f): return f.startswith(os.path.splitext(self.filename)[0]) + self.associated_files = [f for f in local_files if is_related(f)] + # open .bsp + self.file = open(os.path.join(self.folder, self.filename), "rb") + file_magic = self.file.read(4) + if file_magic != self.file_magic: + raise RuntimeError(f"{self.file} is not a valid .bsp!") + self.bsp_version = int.from_bytes(self.file.read(4), "little") + self.file.seek(0, 2) # move cursor to end of file + self.bsp_file_size = self.file.tell() + + self.headers = dict() + self.loading_errors: Dict[str, Exception] = dict() + for LUMP_enum in self.branch.LUMP: + # CHECK: is lump external? (are associated_files overriding) + lump_header = self._read_header(LUMP_enum) + LUMP_NAME = LUMP_enum.name + self.headers[LUMP_NAME] = lump_header + if lump_header.length == 0: + continue + try: + if LUMP_NAME == "GAME_LUMP": + GameLumpClasses = getattr(self.branch, "GAME_LUMP_CLASSES", dict()) + BspLump = lumps.GameLump(self.file, lump_header, GameLumpClasses) + elif LUMP_NAME in self.branch.LUMP_CLASSES: + LumpClass = self.branch.LUMP_CLASSES[LUMP_NAME][lump_header.version] + BspLump = lumps.create_BspLump(self.file, lump_header, LumpClass) + elif LUMP_NAME in self.branch.SPECIAL_LUMP_CLASSES: + SpecialLumpClass = self.branch.SPECIAL_LUMP_CLASSES[LUMP_NAME][lump_header.version] + d_file, d_header = lumps.decompressed(self.file, lump_header) + d_file.seek(d_header.offset) + lump_data = d_file.read(d_header.length) + BspLump = SpecialLumpClass(lump_data) + elif LUMP_NAME in self.branch.BASIC_LUMP_CLASSES: + LumpClass = self.branch.BASIC_LUMP_CLASSES[LUMP_NAME][lump_header.version] + BspLump = lumps.create_BasicBspLump(self.file, lump_header, LumpClass) + else: + BspLump = lumps.create_RawBspLump(self.file, lump_header) + except KeyError: # lump VERSION not supported + self.loading_errors[LUMP_NAME] = KeyError(f"{LUMP_NAME} v{lump_header.version} is not supported") + BspLump = lumps.create_RawBspLump(self.file, lump_header) + except Exception as exc: + self.loading_errors[LUMP_NAME] = exc + BspLump = lumps.create_RawBspLump(self.file, lump_header) + setattr(self, LUMP_NAME, BspLump) + + def save_as(self, filename: str): + """Expects outfile to be a file with write bytes capability""" + raise NotImplementedError() + # os.makedirs(os.path.dirname(os.path.realpath(filename)), exist_ok=True) + # outfile = open(filename, "wb") + # # try to preserve the original order of lumps + # outfile.write(self.file_magic) + # outfile.write(self.version.to_bytes(4, "little")) # .bsp format version + # for LUMP in self.branch.LUMP: + # pass # calculate and write each header + # # adapting each header to bytes could be hard + # # write the contents of each lump + # outfile.write(b"0001") # map revision + # # write contents of lumps + + def save(self): + self.save_as(os.path.join(self.folder, self.filename)) + + def set_branch(self, branch: ModuleType): + """Calling .set_branch(...) on a loaded .bsp will not convert it!""" + # branch is a "branch script" that has been imported into python + # if writing your own "branch script", see branches/README.md for a guide + self.branch = branch + # attach methods + for method in getattr(branch, "methods", list()): + method = MethodType(method, self) + setattr(self, method.__name__, method) + # NOTE: does not remove methods from former branch + # could we also attach static methods? + + def lump_as_bytes(self, lump_name: str) -> bytes: + # NOTE: if a lump failed to read correctly, converting to bytes will fail + # -- this is because LumpClasses are expected + # -- even though the bytes are saved directly to a RawBspLump... FIXME + if not hasattr(self, lump_name): + return b"" # lump is empty / deleted + lump_entries = getattr(self, lump_name) + lump_version = self.headers[lump_name].version + # NOTE: IBSP & GoldSrcBsp don't have lump versions + if lump_name in self.branch.BASIC_LUMP_CLASSES: + _format = self.branch.BASIC_LUMP_CLASSES[lump_name][lump_version]._format + raw_lump = struct.pack(f"{len(lump_entries)}{_format}", *lump_entries) + elif lump_name in self.branch.LUMP_CLASSES: + _format = self.branch.LUMP_CLASSES[lump_name][lump_version]._format + raw_lump = b"".join([struct.pack(_format, *x.flat()) for x in lump_entries]) + elif lump_name in self.branch.SPECIAL_LUMP_CLASSES: + raw_lump = lump_entries.as_bytes() + elif lump_name == "GAME_LUMP": + raw_lump = lump_entries.as_bytes() + else: # assume lump_entries is RawBspLump + raw_lump = bytes(lump_entries) + return raw_lump diff --git a/io_import_rbsp/bsp_tool/branches/README.md b/io_import_rbsp/bsp_tool/branches/README.md new file mode 100644 index 0000000..bba0ac6 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/README.md @@ -0,0 +1,203 @@ +# How bsp_tool loads .bsps +If you're using `bsp_tool.load_bsp("reallycool.bsp")` to load a `.bsp` a few things happen behind the scenes to figure out the format +Since bsp_tool supports a range of `.bsp` variants, a single script to handle the rough format wasn't going to cut it +To narrow down the exact format of a bsp file `load_bsp` looks at some key information in each file: + + +### Developer variants +First, `load_bsp` tries to determine the developer behind the chosen .bsp +If the file extension is `.d3dbsp`, it's a Call of Duty 2 or 4 `D3DBsp` +Other bsps use the `.bsp` extension (Call of Duty 1 included) +The developer is identified from the "file-magic", the first four bytes of any .bsp are: + - `b"IBSP"` for `IdTechBsp` Id Software + - `b"IBSP"` for `D3DBsp` Infinity Ward + - `b"rBSP"` for `RespawnBsp` Respawn + - `b"VBSP"` for `ValveBsp` Valve + +> This rule isn't perfect! most mods out there are Source Engine forks with b"VBSP" + +> GoldSrc .bsp files don't have any file magic! + +Most of the major differences between each developer's format are the number of lumps & bsp header +They also use some lumps which are unique to each developer's Quake based engine +More on those differences in an upcoming wiki page... + + +### Game variants +Once `load_bsp` knows the developer, it has to work out which game a `.bsp` comes from + +In the `.bsp` header there will always be a 4 byte int for the `.bsp` format version +Unfortunately this isn't totally unique from game to game, [most Source Engine titles use version 20](https://developer.valvesoftware.com/wiki/Source_BSP_File_Format#Versions) +This is where `load_bsp`'s second (optional!) argument comes in, `branch` + +`branch` can be either a string or a python script + +```python +>>> import bsp_tool + +>>> bsp_tool.load_bsp("tests/maps/pl_upward.bsp", branch=bsp_tool.branches.valve.orange_box) +Loading pl_upward.bsp (VBSP version 20)... +Loaded pl_upward.bsp + + +>>> bsp_tool.load_bsp("tests/maps/test_bigbox.bsp", branch="Quake3") +Loading test_bigbox.bsp (IBSP version 46)... +Loaded test_bigbox.bsp + + +>>> bsp_tool.load_bsp("tests/maps/test2.bsp") +Loading test2.bsp (VBSP version 20)... +Loaded test2.bsp + +``` +In the above example `bsp_tool.branches.valve.orange_box` points to [`bsp_tool/branches/valve/orange_box.py`](https://github.com/snake-biscuits/bsp_tool/blob/master/bsp_tool/branches/valve/orange_box.py) +This branch script is used to intialise the `Bsp` subclass chosen when `load_bsp` works out the developer + +When `branch` is a string, `load_bsp` uses `branch` as a key in the `bsp_tool.branches.by_name` dictionary to choose a script +Bsp classes take the branch script as their first argument and **do not have defaults** (except `ValveBsp`) + +When `branch` is `"unknown"` (default) the bsp format version is used as a key in the `bsp_tool.branches.by_version` dictionary + + + +# Branch scripts +Now that we know a branch script is needed to load a specific .bsp variant, why might we need to make one? +Well, bsp_tool doesn't cover a lot of formats, and those it does aren't mapped completely either! + +But with branch scripts you can develop a rough map of a particular format while copying definitions from other scripts +[`nexon/vindictus.py`](https://github.com/snake-biscuits/bsp_tool/blob/master/bsp_tool/branches/nexon/vindictus.py) for example, imports `valve.orange_box` and copies most of the format +This saves a lot of code! Especially since they only differ on the format of a handful of lumps and share a .bsp version + + +## Overall structure +The branch scripts that come with bsp_tool have a common format to make reading them as consistent as possible + +```python +import enum +from .. import base +from .. import shared # special lumps + +BSP_VERSION = 20 + +class LUMP(enum.Enum): + ENTITIES = 0 + AREAS = 20 + PAKFILE = 40 + +lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} + +# classes for each lump, in alphabetical order: [1 / 64] + shared.Entities & PakFile +class Area(base.Struct): # LUMP 20 + num_area_portals: int # number of AreaPortals after first_area_portal in this Area + first_area_portal: int # index of first AreaPortal + __slots__ = ["num_area_portals", "first_area_portal"] + _format = "2i" + +LUMP_CLASSES = {"AREAS": {0: Area}} + +SPECIAL_LUMP_CLASSES = {"ENTITIES": {0: shared.Entities}, + "PAKFILE": {0: shared.PakFile}} +``` + +If you compare [`bsp_tool/branches/valve/orange_box.py`](https://github.com/snake-biscuits/bsp_tool/blob/master/bsp_tool/branches/valve/orange_box.py) you'll see I've left a lot out here, but this basic batch script is a great start for translating any .bsp variant + +At the top we have the bsp format version, mostly as a note +Next comes the `LUMP` enum, this lists each lump in the order they appear in the bsp header + +> Lumps without LumpClasses are loaded as raw bytes +> Invalid LumpClasses will generate an error (silent, saved to bsp.loading_errors) +> Invalid lumps will still be loaded as raw bytes + +Attached to this we have `lump_header_address`, this connects each LUMP entry to the offset .bsp where it's header begins +Then comes the lump classes, these translate most lumps into python objects (more on them later) +We also have some special lump classes, these are loaded in a different way to other lumps, and some are shared across **almost all** bsp variants + +The Bsp class reads the headers for each lump and holds the contents in `Bsp.headers` +This dictionary of headers takes the name given in the branch scripts' `LUMP` class +Lump names are tied to a dictionary, which ties lump version (`int`) to LumpClass +From there, a lump is either saved as `Bsp.RAW_LUMPNAME` (bytes) or `Bsp.LUMPNAME` (List[LumpClass]) if it the lump is listed in `LUMP_CLASSES` + + +## Lump classes +A majority of lumps are very simple, being a list of fixed length structs +bsp_tool loads these lumps with python's built in `struct` module +`struct.iter_unpack` takes a format specifier string and a stream of bytes +This stream of bytes must contain a whole number of these structures or an error will be raised + +The lump class in the example is a subclass of [`bsp_tool.branches.base.Struct`](https://github.com/snake-biscuits/bsp_tool/blob/master/bsp_tool/branches/base.py#L5) +`base.Struct` exists to make defining a lump's specific format quick using very little code + +The definition usually has 3 parts: +```python +class LumpClass(base.struct): + __slots__ = ["unknown", "flags"] + _format = "5i" + _arrays = {"unknown": [*"abcd"]} +``` + +`__slots__` names the main attributes of the LumpClass +`_format` holds the format string for `struct.iter_unpack` +(I recommend also giving type hints for each attribute, so others don't have to work them out from `_format`) +`_arrays` is optional, it holds a dictionary for generating a `base.MappedArray` to help group attributes +For the most complex use of arrays (so far), see: [`branches.id_software.quake3.Face`](https://github.com/snake-biscuits/bsp_tool/blob/06f184b0cdf5133ea12ce8e0f5442398d6310d2a/bsp_tool/branches/id_software/quake3.py#L60) + +So the above example would turn the C struct: +```C +struct LumpClass { + int unknown_a; + int unknown_b; + int unknown_c; + int unknown_d; + int flags; +} +``` +into: +```python +LumpClass.unknown.a +LumpClass.unknown.b +LumpClass.unknown.c +LumpClass.unknown.d +LumpClass.flags +``` +Lump classes don't have to be subclasses of `base.Struct` though, the only requirement is the `_format` attribute +This is essential because each lump class is initialised with the tuple `struct.iter_unpack` returns for each struct +And to read these raw bytes `Bsp.load_lumps` uses something similar to `struct.iter_unpack(LumpClass._format, RAW_LUMP)` +If the tuple returned has a length of 0 `bsp.LUMP = list(map(LumpClass, [t[0] for t in tuples]))` +Else: `Bsp.LUMP = list(map(LumpClass, tuples))` +To support re-saving LumpClasses, a `.flat()` method is required, which must return a tuple near identical to the one it was made from (same types) + + +## Special lump classes +Not all lumps are as simple as a list of structs, and this is where special lump classes come in +Special lump classes are initialised from the raw bytes of a lump, turning them into python objects that are easier to work with +All that's really required is an `__init__` method and an `.as_bytes()` method for re-saving + +Here's `branches.shared.TextureDataStringData` as an example of how basic a special lump class can be: +```python +class TextureDataStringData(list): + def __init__(self, raw_texture_data_string_data): + super().__init__([t.decode("ascii", errors="ignore") for t in raw_texture_data_string_data.split(b"\0")]) + + def as_bytes(self): + return b"\0".join([t.encode("ascii") for t in self]) + b"\0" +``` +By inheriting `list` you can use all the features of python lists while still importing the data with `__init__` & saving it back with `.as_bytes()` +You can of course make more complex classes, like adding methods (though they won't be connected to their parent `Bsp`) +Speaking of methods + + +## Methods +While not listed in the example branch scripts, you can add methods to a `Bsp` with a branch script! +The only requirements are that you have a list of functions in `methods` somewhere in the script + +```python +def areaportals_of_area(bsp, area_index): + area = bsp.AREAS[area_index] + return bsp.AREA_PORTALS[area.first_areaportal:area.first_areaportal + area.num_areaportals] + + +methods = [areaportals_of_area] +``` + +These methods are attached when the `Bsp` is initialised +The only requirements for these functions is that the first argument be `bsp`, since as a method the `Bsp` will pass itself to the function diff --git a/io_import_rbsp/bsp_tool/branches/__init__.py b/io_import_rbsp/bsp_tool/branches/__init__.py new file mode 100644 index 0000000..1947a4f --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/__init__.py @@ -0,0 +1,179 @@ +__all__ = ["arkane", "gearbox", "id_software", "infinity_ward", "nexon", "respawn", "valve", + "by_magic", "by_name", "by_version"] + +from . import arkane +from . import gearbox +from . import id_software +from . import infinity_ward +from . import nexon +from . import respawn +from . import valve + + +__doc__ = """Index of developers of bsp format variants""" + +FILE_MAGIC_developer = {b"IBSP": [id_software, infinity_ward], + b"rBSP": [respawn], + b"VBSP": [arkane, nexon, valve]} +# id_software.quake, valve.goldsrc & gearbox.bshift have no file_magic + +# NOTE: bsp_tool/__init__.py: load_bsp can be fed a string to select the relevant branch script +# by_name is searched with a lowercase, numbers & letters only version of that string +# NOTE: some (but not all!) games are listed here have multiple valid names (including internal mod names) +# TODO: generate from branch.GAMES (folder names) +# TODO: handle branchscripts with multiple bsp_versions elegantly + + +def simplify_name(name: str) -> str: + """'Counter-Strike: Online 2' -> 'counterstrikeonline2'""" + return "".join(filter(str.isalnum, name.lower())) + + +by_name = { + # Id Software - Id Tech + "fortress": id_software.quake, + "hexenii": id_software.quake, + "quake": id_software.quake, + "quakeii": id_software.quake2, + "quake2": id_software.quake2, + "quake3": id_software.quake3, + "quakeiii": id_software.quake3, + # TODO: Quake 4 + # TODO: Quake Champions + "quakelive": id_software.quake3, + # Infinity Ward - IW Engine(s) + "callofduty": infinity_ward.call_of_duty1, + "cod": infinity_ward.call_of_duty1, + # TODO: Call of Duty 2 + # TODO: Call of Duty 4 + # NEXON - Source Engine + "counterstrikeonline2": nexon.cso2, + "cso2": nexon.cso2, + "csonline2": nexon.cso2, + "mabinogi": nexon.vindictus, + "mabinogiheroes": nexon.vindictus, + "vindictus": nexon.vindictus, + # Respawn Entertainment - Titanfall Engine + "titanfall": respawn.titanfall, + "titanfall2": respawn.titanfall2, + "apexlegends": respawn.apex_legends, + "apex": respawn.apex_legends, + "r1": respawn.titanfall, # internal names + "r2": respawn.titanfall2, + "r5": respawn.apex_legends, + # Valve Software - Source Engine + "source": valve.source, + "orangebox": valve.orange_box, + "2013sdk": valve.sdk_2013, + "sourcemods": valve.sdk_2013, + "alienswarm": valve.alien_swarm, + "alienswarmreactivedrop": valve.alien_swarm, + "blackmesa": valve.sdk_2013, + "bladesymphony": valve.sdk_2013, + "counterstrikeglobaloffensive": valve.sdk_2013, + "counterstrikesource": valve.source, + "csgo": valve.sdk_2013, + "css": valve.source, + "cssource": valve.source, + "dayofdefeatsource": valve.orange_box, + "dods": valve.orange_box, + "episode1": valve.orange_box, + "episode2": valve.orange_box, + "episodic": valve.orange_box, + "fortressforever": valve.orange_box, + "garrysmod": valve.orange_box, + "globaloffensive": valve.sdk_2013, + "gmod": valve.orange_box, + "gstring": valve.orange_box, # awesome sourcemod + "halflife1sourcedeathmatch": valve.source, + "halflife2": valve.source, + "halflife2ep1": valve.source, + "halflife2ep2": valve.orange_box, + "halflife2episode1": valve.source, + "halflife2episode2": valve.orange_box, + "halflife2episodic": valve.source, + "halflife2hl1": valve.source, + "halflifesource": valve.source, + "hl2": valve.source, + "hl2ep1": valve.source, + "hl2ep2": valve.orange_box, + "hls": valve.source, + "hlsource": valve.source, + "l4d": valve.left4dead, + "l4d2": valve.left4dead2, + "left4dead": valve.left4dead, + "left4dead2": valve.left4dead2, + "neotokyo": valve.orange_box, + "portal": valve.orange_box, + "portal2": valve.sdk_2013, + "portalreloaded": valve.sdk_2013, + "sourcefilmmaker": valve.sdk_2013, + "synergy": valve.source, + "teamfortress2": valve.orange_box, + "tf2": valve.orange_box, + # Valve Software - GoldSrc Engine (more mods @ https://half-life.fandom.com/wiki/Mods) + "goldsrc": valve.goldsrc, + "007nightfire": valve.goldsrc, # untested + "blueshift": gearbox.bshift, + "bshift": gearbox.bshift, + "counterstrike": valve.goldsrc, # CS 1.6 + "counterstrikeconditionzero": valve.goldsrc, + "counterstrikeneo": valve.goldsrc, # obscure & untested + "counterstrikeonline": valve.goldsrc, # obscure & untested + "cs": valve.goldsrc, # CS 1.6 + "cscz": valve.goldsrc, + "csn": valve.goldsrc, # obscure & untested + "cso": valve.goldsrc, # obscure & untested + "dayofdefeat": valve.goldsrc, + "deathmatchclassic": valve.goldsrc, + "dmc": valve.goldsrc, + "dod": valve.goldsrc, + "gunmanchronicles": valve.goldsrc, + "halflife": valve.goldsrc, + "halflifeblueshift": gearbox.bshift, + "halflifebshift": gearbox.bshift, + "halflifericochet": valve.goldsrc, + "halflifecstrike": valve.goldsrc, + "halflifeopposingforce": valve.goldsrc, + "halfquaketrilogy": valve.goldsrc, + "hlblueshift": gearbox.bshift, + "hlopposingforce": valve.goldsrc, + "jamesbond007nightfire": valve.goldsrc, # untested + "nightfire": valve.goldsrc, + "opposingforce": valve.goldsrc, + "ricochet": valve.goldsrc, + "svencoop": valve.goldsrc, + "teamfortressclassic": valve.goldsrc, + "tfc": valve.goldsrc + } + +# NOTE: limiting because many games share version numbers +# could also be generated from loaded scripts +by_version = { + # Id Software + id_software.quake.BSP_VERSION: id_software.quake, # 23 + id_software.quake2.BSP_VERSION: id_software.quake2, # 38 + id_software.quake3.BSP_VERSION: id_software.quake3, # 46 + # Infinity Ward + infinity_ward.call_of_duty1.BSP_VERSION: infinity_ward.call_of_duty1, # 59 + # Nexon + nexon.cso2.BSP_VERSION: nexon.cso2, # 1.00? + # Respawn Entertainment + respawn.titanfall.BSP_VERSION: respawn.titanfall, # 29 + respawn.titanfall2.BSP_VERSION: respawn.titanfall2, # 37 + respawn.apex_legends.BSP_VERSION: respawn.apex_legends, # 47 + 48: respawn.apex_legends, # Introduced in Season 7 with Olympus + 49: respawn.apex_legends, # Introduced in Season 8 with Canyonlands Staging + 50: respawn.apex_legends, # Introduced in Season 10 with Arena Skygarden + # Valve Software + valve.goldsrc.BSP_VERSION: valve.goldsrc, # can't really use this system for GoldSrc + valve.source.BSP_VERSION: valve.source, # 19 (and sometimes 20?) + valve.orange_box.BSP_VERSION: valve.orange_box, # 20 (many sub-variants) + valve.sdk_2013.BSP_VERSION: valve.sdk_2013, # 21 + # Other + # arkane.dark_messiah.BSP_VERSION: arkane.dark_messiah, # 20.4 ? + gearbox.bshift.BSP_VERSION: gearbox.bshift # 30 + } + +# NOTE: ata4's bspsrc uses unique entity classnames to identify branches +# -- might be an idea to copy this for an autodetect refactor diff --git a/io_import_rbsp/bsp_tool/branches/arkane/__init__.py b/io_import_rbsp/bsp_tool/branches/arkane/__init__.py new file mode 100644 index 0000000..0cb2457 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/arkane/__init__.py @@ -0,0 +1,7 @@ +__all__ = ["dark_messiah"] + +from . import dark_messiah + +__doc__ = """Arkane Studios made a number of Source Engine powered projects. Few released.""" + +FILE_MAGIC = "VBSP" diff --git a/io_import_rbsp/bsp_tool/branches/arkane/dark_messiah.py b/io_import_rbsp/bsp_tool/branches/arkane/dark_messiah.py new file mode 100644 index 0000000..23681e7 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/arkane/dark_messiah.py @@ -0,0 +1,40 @@ +# https://developer.valvesoftware.com/wiki/Source_BSP_File_Format/Game-Specific#Dark_Messiah_of_Might_and_Magic +import enum +import struct + +from ..valve import orange_box, source + + +BSP_VERSION = 20 +# NOTE: BSP_VERSION is stored as 2 shorts? + +GAMES = ["Dark Messiah of Might and Magic"] + +LUMP = orange_box.LUMP +lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} + + +def read_lump_header(file, LUMP: enum.Enum) -> source.SourceLumpHeader: + file.seek(lump_header_address[LUMP]) + offset, length, version, fourCC = struct.unpack("4I", file.read(16)) + header = source.SourceLumpHeader(offset, length, version, fourCC) + return header + +# classes for lumps, in alphabetical order: +# TODO: dheader_t, texinfo_t, dgamelump_t, dmodel_t + +# classes for special lumps, in alphabetical order: +# TODO: StaticPropLumpv6 + + +# {"LUMP_NAME": {version: LumpClass}} +BASIC_LUMP_CLASSES = orange_box.BASIC_LUMP_CLASSES.copy() + +LUMP_CLASSES = orange_box.LUMP_CLASSES.copy() + +SPECIAL_LUMP_CLASSES = orange_box.SPECIAL_LUMP_CLASSES.copy() + +# GAME_LUMP_CLASSES = {"sprp": {6: lambda raw_lump: shared.GameLump_SPRP(raw_lump, StaticPropLumpv6)}} + + +methods = [*orange_box.methods] diff --git a/io_import_rbsp/bsp_tool/branches/base.py b/io_import_rbsp/bsp_tool/branches/base.py new file mode 100644 index 0000000..d3185c8 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/base.py @@ -0,0 +1,143 @@ +"""Base classes for defining .bsp lump structs""" +from typing import Any, Dict, Iterable, List, Union + + +class Struct: + """base class for tuple <-> class conversion + bytes <-> tuple conversion is handled by the struct module""" + __slots__: List[str] = list() # names of atributes, in order + _format: str = str() # struct module format string + _arrays: Dict[str, Any] = dict() # slots to be mapped into MappedArrays + # each value in _arrays is a mapping to generate a MappedArray from + + def __init__(self, _tuple: Iterable): + # _tuple comes from: struct.unpack(self._format, bytes) + _tuple_index = 0 + for attr in self.__slots__: + if attr not in self._arrays: + value = _tuple[_tuple_index] + length = 1 + else: + array_map = self._arrays[attr] + if isinstance(array_map, list): # array_map: List[str] + length = len(array_map) + array = _tuple[_tuple_index:_tuple_index + length] + value = MappedArray(array, mapping=array_map) + elif isinstance(array_map, dict): # array_map: Dict[str, List[str]] + length = mapping_length(array_map) + array = _tuple[_tuple_index:_tuple_index + length] + value = MappedArray(array, mapping=array_map) # nested + elif isinstance(array_map, int): + length = array_map + value = _tuple[_tuple_index:_tuple_index + length] + else: + raise RuntimeError(f"{type(array_map)} {array_map} in {self.__class__.__name__}._arrays") + setattr(self, attr, value) + _tuple_index += length + # TODO: throw a warning if the whole tuple won't fit + # report len(_tuple) & struct.calcsize(self._format) + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + else: + return self.flat() == other.flat() + + def __hash__(self): + return hash(tuple(self.flat())) + + def __iter__(self) -> Iterable: + return iter([getattr(self, attr) for attr in self.__slots__]) + + def __repr__(self) -> str: + components = {s: getattr(self, s) for s in self.__slots__} + return f"<{self.__class__.__name__} {components}>" + + def flat(self) -> list: + """recreates the _tuple this instance was initialised from""" + _tuple = [] + for slot in self.__slots__: + value = getattr(self, slot) + if isinstance(value, MappedArray): + _tuple.extend(value.flat()) + elif isinstance(value, Iterable): + _tuple.extend(value) + else: + _tuple.append(value) + return _tuple + + +def mapping_length(mapping: Dict[str, Any]) -> int: + length = 0 + for sub_mapping in mapping.values(): + if isinstance(sub_mapping, list): + length += len(sub_mapping) + elif isinstance(sub_mapping, int): + length += sub_mapping + elif isinstance(sub_mapping, dict): + length += mapping_length(sub_mapping) + elif sub_mapping is None: + length += 1 + else: + raise RuntimeError(f"Unexpexted Mapping! ({mapping}, {sub_mapping})") + return length + + +class MappedArray: + """Maps a given iterable to a series of names, can even be a nested mapping""" + _mapping: Union[List[str], Dict[str, Any]] = [*"xyz"] + # _mapping cane be either a list of attr names to map a given array to, + # or, a dict containing a list of attr names, or another dict + # this second form is difficult to express as a type hint + + def __init__(self, array: Iterable, mapping: Any = None): + if mapping is None: + mapping = self._mapping # hack to use a default + if isinstance(mapping, dict): + self._mapping = list(mapping.keys()) + array_index = 0 + for attr, child_mapping in mapping.items(): + # TODO: child_mapping of type int takes a slice, storing a mutable list + if child_mapping is not None: + segment = array[array_index:array_index + mapping_length({None: child_mapping})] + array_index += len(child_mapping) + child = MappedArray(segment, mapping=child_mapping) # will recurse again if child_mapping is a dict + else: # if {"attr": None} + array_index += 1 + child = array[array_index] # take a single item, not a slice + setattr(self, attr, child) + elif isinstance(mapping, list): # List[str] + for attr, value in zip(mapping, array): + setattr(self, attr, value) + self._mapping = mapping + else: + raise RuntimeError(f"Unexpected mapping: {type(mapping)}") + + def __eq__(self, other: Iterable) -> bool: + return all([(a == b) for a, b in zip(self, other)]) + + def __getitem__(self, index: str) -> Any: + return getattr(self, self._mapping[index]) + + def __hash__(self): + return hash(tuple(self.flat())) + + def __iter__(self) -> Iterable: + return iter([getattr(self, attr) for attr in self._mapping]) + + def __repr__(self) -> str: + attrs = [] + for attr, value in zip(self._mapping, self): + attrs.append(f"{attr}: {value}") + return f"<{self.__class__.__name__} ({', '.join(attrs)})>" + + def flat(self) -> list: + """recreates the array this instance was generated from""" + array = [] + for attr in self._mapping: + value = getattr(self, attr) + if isinstance(value, MappedArray): + array.extend(value.flat()) # recursive call + else: + array.append(value) + return array diff --git a/io_import_rbsp/bsp_tool/branches/gearbox/__init__.py b/io_import_rbsp/bsp_tool/branches/gearbox/__init__.py new file mode 100644 index 0000000..c7572d9 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/gearbox/__init__.py @@ -0,0 +1,5 @@ +__all__ = ["bshift"] + +from . import bshift + +__doc__ = """Gearbox's second Half-Life expansion: Blue Shift made a few changes to the GoldSrc engine.""" diff --git a/io_import_rbsp/bsp_tool/branches/gearbox/bshift.py b/io_import_rbsp/bsp_tool/branches/gearbox/bshift.py new file mode 100644 index 0000000..51fcc86 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/gearbox/bshift.py @@ -0,0 +1,24 @@ +# https://valvedev.info/tools/bspfix/ +# https://developer.valvesoftware.com/wiki/Hl_bs.fgd +from ..valve import goldsrc + + +BSP_VERSION = 30 + +GAMES = ["Half-Life/bshift"] # Half-Life: Blue Shift + +LUMP = goldsrc.LUMP +# NOTE: different headers? +lump_header_address = {LUMP_ID: (4 + i * 8) for i, LUMP_ID in enumerate(LUMP)} + + +BASIC_LUMP_CLASSES = goldsrc.BASIC_LUMP_CLASSES.copy() + +LUMP_CLASSES = goldsrc.LUMP_CLASSES.copy() +# PLANES lump failing + +SPECIAL_LUMP_CLASSES = goldsrc.SPECIAL_LUMP_CLASSES.copy() +# ENTITIES lump failing + + +methods = [*goldsrc.methods] diff --git a/io_import_rbsp/bsp_tool/branches/id_software/__init__.py b/io_import_rbsp/bsp_tool/branches/id_software/__init__.py new file mode 100644 index 0000000..2bcc2f3 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/id_software/__init__.py @@ -0,0 +1,14 @@ +__all__ = ["quake", "quake2", "quake3"] + +from . import quake +from . import quake2 +from . import quake3 +# TODO: Quake 4? +# TODO: Quake Champions? +# TODO: Hexen 2 (no file-magic) + +__doc__ = """Id Software's Quake Engine and it's predecessors have formed the basis for many modern engines.""" + +FILE_MAGIC = b"IBSP" + +branches = [quake, quake3] diff --git a/io_import_rbsp/bsp_tool/branches/id_software/quake.py b/io_import_rbsp/bsp_tool/branches/id_software/quake.py new file mode 100644 index 0000000..6516497 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/id_software/quake.py @@ -0,0 +1,280 @@ +# https://github.com/id-Software/Quake/blob/master/WinQuake/bspfile.h (v29) +# https://www.gamers.org/dEngine/quake/spec/quake-spec34/qkspec_4.htm (v23) +import enum +from typing import Dict, List +import struct + +from .. import base +from .. import shared # special lumps + + +BSP_VERSION = 29 + +GAMES = ["Quake"] + + +# lump names & indices: +class LUMP(enum.Enum): + ENTITIES = 0 # one long string + PLANES = 1 + MIP_TEXTURES = 2 + VERTICES = 3 + VISIBILITY = 4 # appears to be same as in Source Engine + NODES = 5 + TEXTURE_INFO = 6 + FACES = 7 + LIGHTMAPS = 8 # 8bpp 0x00-0xFF black-white + CLIP_NODES = 9 + LEAVES = 10 # indexed by NODES, index ranges of LEAF_FACES + LEAF_FACES = 11 # indices into FACES, sorted for (start, len) lookups + EDGES = 12 + SURFEDGES = 13 # indices into EDGES (-ve indices reverse edge direction) + MODELS = 14 + +# A rough map of the relationships between lumps: +# /-> LEAF_BRUSHES +# ENTITIES -> MODELS -> NODES -> LEAVES -> LEAF_FACES -> FACES +# \-> CLIP_NODES -> PLANES + +# FACES -> SURFEDGES -> EDGES -> VERTICES +# |-> TEXTURE_INFO -> MIP_TEXTURES +# |-> LIGHTMAPS +# \-> PLANES + + +lump_header_address = {LUMP_ID: (4 + i * 8) for i, LUMP_ID in enumerate(LUMP)} + + +# engine limits: +class MAX(enum.Enum): + ENTITIES = 1024 + PLANES = 32767 + MIP_TEXTURES = 512 + MIP_TEXTURES_SIZE = 0x200000 # in bytes + VERTICES = 65535 + VISIBILITY_SIZE = 0x100000 # in bytes + NODES = 32767 # "because negative shorts are contents" + TEXTURE_INFO = 4096 + FACES = 65535 + LIGHTING_SIZE = 0x100000 # in bytes + CLIP_NODES = 32767 + LEAVES = 8192 + MARK_SURFACES = 65535 + EDGES = 256000 + MODELS = 256 + BRUSHES = 4096 # for radiant / q2map ? + ENTITY_KEY = 32 + ENTITY_STRING = 65536 + ENTITY_VALUE = 1024 + PORTALS = 65536 # related to leaves + SURFEDGES = 512000 + + +# flag enums +class Contents(enum.IntFlag): + """Brush flags""" + # src/public/bspflags.h + # NOTE: compiler gets these flags from a combination of all textures on the brush + # e.g. any non opaque face means this brush is non-opaque, and will not block vis + # visible + EMPTY = -1 + SOLID = -2 + WATER = -3 + SLIME = -4 + LAVA = -5 + SKY = -6 + ORIGIN = -7 # remove when compiling from .map to .bsp + CLIP = -8 # "changed to contents_solid" + CURRENT_0 = -9 + CURRENT_90 = -10 + CURRENT_180 = -11 + CURRENT_270 = -12 + CURRENT_UP = -13 + CURRENT_DOWN = -14 + + +# classes for lumps, in alphabetical order: +class ClipNode(base.Struct): # LUMP 9 + # https://en.wikipedia.org/wiki/Half-space_(geometry) + # NOTE: bounded by associated model + # basic convex solid stuff + plane: int + children: List[int] # +ve ClipNode, -1 outside model, -2 inside model + __slots__ = ["plane", "children"] + _format = "I2h" + _arrays = {"children": ["front", "back"]} + + +class Edge(list): # LUMP 12 + _format = "2H" # List[int] + + def flat(self): + return self # HACK + + +class Face(base.Struct): # LUMP 7 + plane: int + side: int # 0 or 1 for side of plane + first_edge: int + num_edges: int + texture_info_index: int # index of this face's TextureInfo + lighting_type: int # 0x00=lightmap, 0xFF=no-lightmap, 0x01=fast-pulse, 0x02=slow-pulse, 0x03-0x10 other + base_light: int # 0x00 bright - 0xFF dark (lowest possible light level) + light: int + lightmap: int # index into lightmap lump, or -1 + __slots__ = ["plane", "side", "first_edge", "num_edges", "texture_info_index", + "lighting_type", "base_light", "light", "lightmap"] + _format = "2HI2H4Bi" + _arrays = {"light": 2} + + +class Leaf(base.Struct): # LUMP 10 + type: int # see LeafType enum + cluster: int # index into the VISIBILITY lump + bounds: List[int] + first_leaf_face: int + num_leaf_faces: int + sound: List[int] # ambient master of all 4 elements (0x00 - 0xFF) + __slots__ = ["type", "cluster", "bounds", "first_leaf_face", + "num_leaf_faces", "sound"] + _format = "2i6h2H4B" + _arrays = {"bounds": {"mins": [*"xyz"], "maxs": [*"xyz"]}, + "sound": ["water", "sky", "slime", "lava"]} + + +class LeafType(enum.Enum): + # NOTE: other types exist, but are unknown + NORMAL = -1 + SOLID = -2 + WATER = -3 + SLIME = -4 + LAVA = -5 + SKY = -6 + # NOTE: types below 6 may not be implemented, but act like water + + +class Model(base.Struct): # LUMP 14 + bounds: List[float] # mins & maxs + origin: List[float] + first_node: int # first node in NODES lumps + clip_nodes: int # 1st & second CLIP_NODES indices + node_id3: int # usually 0, unsure of lump + num_leaves: int + first_leaf_face: int + num_leaf_faces: int + __slots__ = ["bounds", "origin", "first_node", "clip_nodes", "node_id3", + "num_leaves", "first_face", "num_faces"] + _format = "9f7i" + _arrays = {"bounds": {"mins": [*"xyz"], "maxs": [*"xyz"]}, "origin": [*"xyz"], + "clip_nodes": 2} + + +class MipTexture(base.Struct): # LUMP 2 (used in MipTextureLump) + name: str # texture name + # if name starts with "*" it scrolls + # if name starts with "+" it ... + size: List[int] # width & height + offsets: List[int] # offset from entry start to texture + __slots__ = ["name", "size", "offsets"] + _format = "16s6I" + _arrays = {"size": ["width", "height"], + "offsets": ["full", "half", "quarter", "eighth"]} + # TODO: transparently access texture data with offsets + # -- this may require a lumps.BspLump subclass, like GameLump + + +class Node(base.Struct): # LUMP 5 + plane_index: int + children: List[int] # +ve Node, -ve Leaf + # NOTE: -1 (leaf 0) is a dummy leaf & terminates tree searches + bounds: List[int] + # NOTE: bounds are generous, rounding up to the nearest 16 units + first_face: int + num_faces: int + _format = "I8h" + _arrays = {"children": ["front", "back"], + "bounds": {"mins": [*"xyz"], "maxs": [*"xyz"]}} + + +class Plane(base.Struct): # LUMP 1 + normal: List[float] + distance: float + type: int # 0-5 (Axial X-Z, Non-Axial X-Z) + __slots__ = ["normal", "distance", "type"] + _format = "4fI" + _arrays = {"normal": [*"xyz"]} + + +class TextureInfo(base.Struct): # LUMP 6 + U: List[float] + V: List[float] + mip_texture_index: int + animated: int # 0 or 1 + __slots__ = ["U", "V", "mip_texture_index", "animated"] + _format = "8f2I" + _arrays = {"U": [*"xyzw"], "V": [*"xyzw"]} + + +class Vertex(base.MappedArray): # LUMP 3 + """a point in 3D space""" + x: float + y: float + z: float + _mapping = [*"xyz"] + _format = "3f" + + +# special lump classes, in alphabetical order: +class MipTextureLump: # LUMP 2 + """Lists MipTextures and handles lookups""" + _bytes: bytes + _changes: Dict[int, MipTexture] + + def __init__(self, raw_lump: bytes): + self._bytes = raw_lump + mip_texture_count = int.from_bytes(raw_lump[:4], "little") + self.offsets = struct.iter_unpack(f"{mip_texture_count}I") + + def __getitem__(self, index): + if index in self._changes: + entry = self._changes[index] + else: + start = self.offsets[index] + entry_bytes = self._bytes[start: start + struct.calcsize(MipTexture._format)] + entry = MipTexture(struct.unpack(MipTexture._format, entry_bytes)) + # TODO: map MipTexture offsets to texture data in lump + return entry + + def as_bytes(self): + # NOTE: this lump seems really complex, need to test on actual .bsps + raise NotImplementedError("Haven't tested to locate texture data yet") + + +# {"LUMP": LumpClass} +BASIC_LUMP_CLASSES = {"LEAF_FACES": shared.Shorts, + "SURFEDGES": shared.Shorts} + +LUMP_CLASSES = {"CLIP_NODES": ClipNode, + "EDGES": Edge, + "FACES": Face, + "LEAVES": Leaf, + "MODELS": Model, + "NODES": Node, + "PLANES": Plane, + "TEXTURE_INFO": TextureInfo, + "VERTICES": Vertex} + +SPECIAL_LUMP_CLASSES = {"ENTITIES": shared.Entities, + "MIP_TEXTURES": MipTextureLump} + + +# branch exclusive methods, in alphabetical order: +def vertices_of_face(bsp, face_index: int) -> List[float]: + raise NotImplementedError() + + +def vertices_of_model(bsp, model_index: int) -> List[float]: + raise NotImplementedError() + + +methods = [shared.worldspawn_volume] diff --git a/io_import_rbsp/bsp_tool/branches/id_software/quake2.py b/io_import_rbsp/bsp_tool/branches/id_software/quake2.py new file mode 100644 index 0000000..1ba05ed --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/id_software/quake2.py @@ -0,0 +1,124 @@ +# https://www.flipcode.com/archives/Quake_2_BSP_File_Format.shtml +# https://github.com/id-Software/Quake-2/blob/master/qcommon/qfiles.h#L214 +import enum +from typing import List + +from . import quake +from .. import base +from .. import shared + + +BSP_VERSION = 38 + +GAMES = ["Quake II", "Heretic II", "SiN", "Daikatana"] + + +class LUMP(enum.Enum): + ENTITIES = 0 + PLANES = 1 + VERTICES = 2 + VISIBILITY = 3 + NODES = 4 + TEXTURE_INFO = 5 + FACES = 6 + LIGHTMAPS = 7 + LEAVES = 8 + LEAF_FACES = 9 + LEAF_BRUSHES = 10 + EDGES = 11 + SURFEDGES = 12 + MODELS = 13 + BRUSHES = 14 + BRUSHSIDES = 15 + POP = 16 # ? + AREAS = 17 + AREA_PORTALS = 18 + +# TODO: new MAX & Contets enums + +# A rough map of the relationships between lumps: +# ENTITIES -> MODELS -> NODES -> LEAVES -> LEAF_FACES -> FACES +# \-> LEAF_BRUSHES + +# FACES -> SURFEDGES -> EDGES -> VERTICES +# |-> TEXTURE_INFO -> MIP_TEXTURES +# |-> LIGHTMAPS +# \-> PLANES + +# LEAF_FACES -> FACES +# LEAF_BRUSHES -> BRUSHES + + +lump_header_address = {LUMP_ID: (8 + i * 8) for i, LUMP_ID in enumerate(LUMP)} + + +# classes for lumps, in alphabetical order: +class Leaf(base.Struct): # LUMP 10 + type: int # see LeafType enum + cluster: int # index into the VISIBILITY lump + area: int + bounds: List[int] + first_leaf_face: int # index to this Leaf's first LeafFace + num_leaf_faces: int # the number of LeafFaces after first_face in this Leaf + first_leaf_brush: int # index to this Leaf's first LeafBrush + num_leaf_brushes: int # the number of LeafBrushes after first_brush in this Leaf + __slots__ = ["type", "cluster", "area", "bounds", "first_leaf_face", + "num_leaf_faces", "first_leaf_brush", "num_leaf_brushes"] + _format = "I12H" + _arrays = {"bounds": {"mins": [*"xyz"], "maxs": [*"xyz"]}} + + +class Model(base.Struct): # LUMP 13 + bounds: List[float] # mins & maxs + origin: List[float] + first_node: int # first node in NODES lumps + first_face: int + num_faces: int + __slots__ = ["bounds", "origin", "first_node", "first_face", "num_faces"] + _format = "9f3i" + _arrays = {"bounds": {"mins": [*"xyz"], "maxs": [*"xyz"]}, "origin": [*"xyz"]} + + +class Node(base.Struct): # LUMP 4 + plane_index: int + children: List[int] # +ve Node, -ve Leaf + # NOTE: -1 (leaf 0) is a dummy leaf & terminates tree searches + bounds: List[int] + # NOTE: bounds are generous, rounding up to the nearest 16 units + first_face: int + num_faces: int + _format = "I2i8h" + _arrays = {"children": ["front", "back"], + "bounds": {"mins": [*"xyz"], "maxs": [*"xyz"]}} + + +class TextureInfo(base.Struct): # LUMP 5 + U: List[float] + V: List[float] + flags: int # "miptex flags & overrides" + value: int # "light emission etc." + name: bytes # texture name + next: int # index into TextureInfo lump for animations (-1 = last frame) + __slots__ = ["U", "V", "flags", "value", "name", "next"] + _format = "8f2I32sI" + _arrays = {"U": [*"xyzw"], "V": [*"xyzw"]} + + +# {"LUMP": LumpClass} +BASIC_LUMP_CLASSES = {"LEAF_FACES": shared.Shorts, + "SURFEDGES": shared.Ints} + +LUMP_CLASSES = {"EDGES": quake.Edge, + "FACES": quake.Face, + "LEAVES": Leaf, + "MODELS": Model, + "NODES": Node, + "PLANES": quake.Plane, + "TEXTURE_INFO": TextureInfo, + "VERTICES": quake.Vertex} + +SPECIAL_LUMP_CLASSES = {"ENTITIES": shared.Entities} +# TODO: Visibility + + +methods = [shared.worldspawn_volume] diff --git a/io_import_rbsp/bsp_tool/branches/id_software/quake3.py b/io_import_rbsp/bsp_tool/branches/id_software/quake3.py new file mode 100644 index 0000000..7aa3f5a --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/id_software/quake3.py @@ -0,0 +1,224 @@ +# https://www.mralligator.com/q3/ +import enum +from typing import List +import struct + +from .. import base +from .. import shared + + +BSP_VERSION = 46 + +GAMES = ["Quake 3 Arena", "Quake Live"] + + +class LUMP(enum.Enum): + ENTITIES = 0 # one long string + TEXTURES = 1 + PLANES = 2 + NODES = 3 + LEAVES = 4 + LEAF_FACES = 5 + LEAF_BRUSHES = 6 + MODELS = 7 + BRUSHES = 8 + BRUSH_SIDES = 9 + VERTICES = 10 + MESH_VERTICES = 11 + EFFECTS = 12 + FACES = 13 + LIGHTMAPS = 14 # 3 128x128 RGB888 images + LIGHT_VOLUMES = 15 + VISIBILITY = 16 + + +# a rough map of the relationships between lumps +# Model -> Brush -> BrushSide +# | |-> Texture +# |-> Face -> MeshVertex +# |-> Texture +# |-> Vertex + + +lump_header_address = {LUMP_ID: (8 + i * 8) for i, LUMP_ID in enumerate(LUMP)} + + +# classes for lumps, in alphabetical order: +class Brush(base.Struct): # LUMP 8 + first_side: int # index into BrushSide lump + num_sides: int # number of BrushSides after first_side in this Brush + texture: int # index into Texture lump + __slots__ = ["first_side", "num_sides", "texture"] + _format = "3i" + + +class BrushSide(base.Struct): # LUMP 9 + plane: int # index into Plane lump + texture: int # index into Texture lump + __slots__ = ["plane", "texture"] + _format = "2i" + + +class Effect(base.Struct): # LUMP 12 + name: str + brush: int # index into Brush lump + unknown: int # Always 5, except in q3dm8, which has one effect with -1 + __slots__ = ["name", "brush", "unknown"] + _format = "64s2i" + + +class Face(base.Struct): # LUMP 13 + texture: int # index into Texture lump + effect: int # index into Effect lump; -1 for no effect + type: int # polygon, patch, mesh, billboard (env_sprite) + first_vertex: int # index into Vertex lump + num_vertices: int # number of Vertices after first_vertex in this face + first_mesh_vertex: int # index into MeshVertex lump + num_mesh_vertices: int # number of MeshVertices after first_mesh_vertex in this face + # lightmap.index: int # which of the 3 lightmap textures to use + # lightmap.top_left: List[int] # approximate top-left corner of visible lightmap segment + # lightmap.size: List[int] # size of visible lightmap segment + # lightmap.origin: List[float] # world space lightmap origin + # lightmap.vector: List[List[float]] # lightmap texture projection vectors + normal: List[float] + size: List[float] # texture patch dimensions + __slots__ = ["texture", "effect", "type", "first_vertex", "num_vertices", + "first_mesh_vertex", "num_mesh_vertices", "lightmap", "normal", "size"] + _format = "12i12f2i" + _arrays = {"lightmap": {"index": None, "top_left": [*"xy"], "size": ["width", "height"], + "origin": [*"xyz"], "vector": {"s": [*"xyz"], "t": [*"xyz"]}}, + "normal": [*"xyz"], "size": ["width", "height"]} + + +class Leaf(base.Struct): # LUMP 4 + cluster: int # index into VisData + area: int + mins: List[float] # Bounding box + maxs: List[float] + first_leaf_face: int # index into LeafFace lump + num_leaf_faces: int # number of LeafFaces in this Leaf + __slots__ = ["cluster", "area", "mins", "maxs", "first_leaf_face", "num_leaf_faces"] + _format = "12i" + _arrays = {"mins": [*"xyz"], "maxs": [*"xyz"]} + + +class Lightmap(list): # LUMP 14 + """Raw pixel bytes, 128x128 RGB_888 image""" + _format = "3s" * 128 * 128 # 128x128 RGB_888 + + def __init__(self, _tuple): + self._pixels: List[bytes] = _tuple # RGB_888 + + def __getitem__(self, row) -> List[bytes]: # returns 3 bytes: b"\xRR\xGG\xBB" + # Lightmap[row][column] returns self.__getitem__(row)[column] + # to get a specific pixel: self._pixels[index] + row_start = row * 512 + return self._pixels[row_start:row_start + 512] # TEST: does it work with negative indices? + + def flat(self) -> bytes: + return b"".join(self._pixels) + + +class LightVolume(base.Struct): # LUMP 15 + # LightVolumess make up a 3D grid whose dimensions are: + # x = floor(MODELS[0].maxs.x / 64) - ceil(MODELS[0].mins.x / 64) + 1 + # y = floor(MODELS[0].maxs.y / 64) - ceil(MODELS[0].mins.y / 64) + 1 + # z = floor(MODELS[0].maxs.z / 128) - ceil(MODELS[0].mins.z / 128) + 1 + _format = "8B" + __slots__ = ["ambient", "directional", "direction"] + _arrays = {"ambient": [*"rgb"], "directional": [*"rgb"], "direction": ["phi", "theta"]} + + +class Model(base.Struct): # LUMP 7 + mins: List[float] # Bounding box + maxs: List[float] + first_face: int # index into Face lump + num_faces: int # number of Faces after first_face included in this Model + first_brush: int # index into Brush lump + num_brushes: int # number of Brushes after first_brush included in this Model + __slots__ = ["mins", "maxs", "first_face", "num_faces", "first_brush", "num_brushes"] + _format = "6f4i" + _arrays = {"mins": [*"xyz"], "maxs": [*"xyz"]} + + +class Node(base.Struct): # LUMP 3 + plane: int # index into Plane lump; the plane this node was split from the bsp tree with + child: List[int] # two indices; into the Node lump if positive, the Leaf lump if negative + __slots__ = ["plane", "child", "mins", "maxs"] + _format = "9i" + _arrays = {"child": [*"ab"], "mins": [*"xyz"], "maxs": [*"xyz"]} + + +class Plane(base.Struct): # LUMP 2 + normal: List[float] + distance: float + __slots__ = ["normal", "distance"] + _format = "4f" + _arrays = {"normal": [*"xyz"]} + + +class Texture(base.Struct): # LUMP 1 + name: str # 64 char texture name; stored in WAD (Where's All the Data)? + flags: int # rendering bit flags? + contents: int # SOLID, AIR etc. + __slots__ = ["name", "flags", "contents"] + _format = "64s2i" + + +class Vertex(base.Struct): # LUMP 10 + position: List[float] + # uv.texture: List[float] + # uv.lightmap: List[float] + normal: List[float] + colour: bytes # 1 RGBA32 pixel / texel + __slots__ = ["position", "uv", "normal", "colour"] + _format = "10f4B" + _arrays = {"position": [*"xyz"], "uv": {"texture": [*"uv"], "lightmap": [*"uv"]}, + "normal": [*"xyz"]} + + +# special lump classes, in alphabetical order: +class Visibility: + """Cluster X is visible from Cluster Y if: + bit (1 << Y % 8) of vecs[X * vector_size + Y // 8] is set + NOTE: Clusters are associated with Leaves""" + def __init__(self, raw_visiblity: bytes): + self.vector_count, self.vector_size = struct.unpack("2i", raw_visiblity[:8]) + self.vectors = struct.unpack(f"{self.vector_count * self.vector_size}B", raw_visiblity[8:]) + + def as_bytes(self): + vectors_bytes = f"{self.vector_count * self.vector_size}B" + return struct.pack(f"2i{vectors_bytes}", (self.vector_count, self.vector_size, *self.vectors)) + + +BASIC_LUMP_CLASSES = {"LEAF_BRUSHES": shared.Ints, + "LEAF_FACES": shared.Ints, + "MESH_VERTICES": shared.Ints} + +LUMP_CLASSES = {"BRUSHES": Brush, + "BRUSH_SIDES": BrushSide, + "EFFECTS": Effect, + "FACES": Face, + "LEAVES": Leaf, + "LIGHTMAPS": Lightmap, + "LIGHT_VOLUMES": LightVolume, + "MODELS": Model, + "NODES": Node, + "PLANES": Plane, + "TEXTURES": Texture, + "VERTICES": Vertex} + +SPECIAL_LUMP_CLASSES = {"ENTITIES": shared.Entities, + "VISIBILITY": Visibility} + + +# branch exclusive methods, in alphabetical order: +def vertices_of_face(bsp, face_index: int) -> List[float]: + raise NotImplementedError() + + +def vertices_of_model(bsp, model_index: int) -> List[float]: + raise NotImplementedError() + + +methods = [shared.worldspawn_volume] diff --git a/io_import_rbsp/bsp_tool/branches/infinity_ward/__init__.py b/io_import_rbsp/bsp_tool/branches/infinity_ward/__init__.py new file mode 100644 index 0000000..c853fca --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/infinity_ward/__init__.py @@ -0,0 +1,11 @@ +__all__ = ["call_of_duty1"] + +from . import call_of_duty1 +# TODO: CoD2 & 4 + +__doc__ = """Infinity Ward created the Call of Duty Franchise, built on the idTech3 (Q3A) engine.""" + +# NOTE: Infinity Ward .bsps share Id Software's FILE_MAGIC +FILE_MAGIC = b"IBSP" + +branches = [call_of_duty1] diff --git a/io_import_rbsp/bsp_tool/branches/infinity_ward/call_of_duty1.py b/io_import_rbsp/bsp_tool/branches/infinity_ward/call_of_duty1.py new file mode 100644 index 0000000..9a020e3 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/infinity_ward/call_of_duty1.py @@ -0,0 +1,234 @@ +# https://wiki.zeroy.com/index.php?title=Call_of_Duty_1:_d3dbsp +# NOTE: Call of Duty 1 has .bsp files in .pk3 archives +# -- later games instead use .d3dbsp in .iwd archives +import enum +from typing import List + +from .. import base +from .. import shared # special lumps + + +BSP_VERSION = 59 + +GAMES = ["Call of Duty"] + + +class LUMP(enum.Enum): + SHADERS = 0 + LIGHTMAPS = 1 + PLANES = 2 + BRUSH_SIDES = 3 + BRUSHES = 4 + TRIANGLE_SOUPS = 6 + DRAW_VERTICES = 7 + DRAW_INDICES = 8 + CULL_GROUPS = 9 # visibility + CULL_GROUP_INDICES = 10 + PORTAL_VERTICES = 11 # areaportals; doors & windows + OCCLUDERS = 12 + OCCLUDER_PLANES = 13 + OCCLUDER_EDGES = 14 + OCCLUDER_INDICES = 15 + AABB_TREES = 16 # Physics? or Vis Nodes? + CELLS = 17 + PORTALS = 18 + LIGHT_INDICES = 19 + NODES = 20 + LEAVES = 21 + LEAF_BRUSHES = 22 + LEAF_SURFACES = 23 + PATCH_COLLISION = 24 # decal clipping? reference for painting bullet holes? + COLLISION_VERTICES = 25 + COLLISION_INDICES = 26 + MODELS = 27 + VISIBILITY = 28 # SPECIAL: Binary Partition tree (read bit by bit, with masks?) + LIGHTS = 29 # SPECIAL: string (typically ENTITIES would be #0) + ENTITIES = 30 + UNKNOWN_31 = 31 # FOGS ? + # big 32nd lump at ed of file, not in header? + # likely a zip / pakfile + # checking for file-magic would confirm this + + +lump_header_address = {LUMP_ID: (8 + i * 8) for i, LUMP_ID in enumerate(LUMP)} + + +# classes for lumps, in alphabetical order: +# all are incomplete guesses; only lump sizes are known +class AxisAlignedBoundingBox(base.Struct): # LUMP 16 + """AABB tree""" + # too small to be mins & maxs of an AABB; probably indices (hence: AABB_TREE) + data: bytes + __slots__ = ["data"] + _format = "12s" # size may be incorrect + + +class Brush(base.Struct): # LUMP 4 + first_side: int # index into the BrushSide lump + num_sides: int # number of sides after first_side in this Brush + __slots__ = ["first_side", "num_sides"] + _format = "2i" + + +class BrushSide(base.Struct): # LUMP 3 + plane: int # index into Plane lump + shader: int # index into Texture lump + __slots__ = ["plane", "shader"] + _format = "2i" + + +class Cell(base.Struct): # LUMP 17 + """No idea what this is / does""" + data: bytes + __slots__ = ["data"] + _format = "52s" + + +class CollisionVertex(base.MappedArray): # LUMP 25 + x: float + y: float + z: float + _mapping = [*"xyz"] + _format = "3f" + + +class CullGroup(base.Struct): # LUMP 9 + data: bytes + __slots__ = ["data"] + _format = "32s" + + +class DrawVertex(base.Struct): # LUMP 7 + data: bytes + __slots__ = ["data"] + _format = "44s" + + +class Leaf(base.Struct): # LUMP 21 + # first_leaf_brush + # num_leaf_brushes + data: bytes + __slots__ = ["data"] + _format = "36s" + + +class Light(base.Struct): # LUMP 30 + # attenuations, colours, strengths + data: bytes + __slots__ = ["data"] + _format = "72s" + + +class Lightmap(list): # LUMP 1 + """Raw pixel bytes, 512x512 RGB_888 image""" + _format = "3s" * 512 * 512 # 512x512 RGB_888 + + def __init__(self, _tuple): + self._pixels: List[bytes] = _tuple # RGB_888 + + def __getitem__(self, row) -> List[bytes]: # returns 3 bytes: b"\xRR\xGG\xBB" + # Lightmap[row][column] returns self.__getitem__(row)[column] + # to get a specific pixel: self._pixels[index] + row_start = row * 512 + return self._pixels[row_start:row_start + 512] # TEST: does it work with negative indices? + + def flat(self) -> bytes: + return b"".join(self._pixels) + + +class Model(base.Struct): # LUMP 27 + mins: List[float] # Bounding box + maxs: List[float] + first_face: int # index into Face lump + num_faces: int # number of Faces after first_face included in this Model + first_brush: int # index into Brush lump + num_brushes: int # number of Brushes after first_brush included in this Model + unknown: List[bytes] + __slots__ = ["mins", "maxs", "first_face", "num_faces", "first_brush", "num_brushes", "unknown"] + _format = "6f4i4c4c" + _arrays = {"mins": [*"xyz"], "maxs": [*"xyz"], "unknown": 2} + + +class Node(base.Struct): # LUMP 20 + data: bytes + __slots__ = ["data"] + _format = "36c" + + +class Occluder(base.Struct): # LUMP 12 + first_occluder_plane: int # index into the OccluderPlane lump + num_occluder_planes: int # number of OccluderPlanes after first_occluder_plane in this Occluder + first_occluder_edges: int # index into the OccluderEdge lump + num_occluder_edges: int # number of OccluderEdges after first_occluder_edge in this Occluder + __slots__ = ["first_occluder_plane", "num_occluder_planes", "first_occluder_edge", "num_occluder_edges"] + _format = "4i" + + +class PatchCollision(base.Struct): # LUMP 24 + data: bytes + __slots__ = ["data"] + _format = "16s" + + +class Plane(base.Struct): # LUMP 2 + normal: List[float] + distance: float + __slots__ = ["normal", "distance"] + _format = "4f" + _arrays = {"normal": [*"xyz"]} + + +class Portal(base.Struct): # LUMP 18 + data: bytes + __slots__ = ["data"] + _format = "16s" + + +class Shader(base.Struct): # LUMP 0 + # assuming the same as Quake3 TEXTURE + texture: str + flags: int + contents: int + __slots__ = ["texture", "flags", "contents"] + _format = "64s2i" + + +class TriangleSoup(base.Struct): # LUMP 5 + data: bytes + __slots__ = ["data"] + _format = "16s" + + +# {"LUMP_NAME": {version: LumpClass}} +BASIC_LUMP_CLASSES = {"COLLISION_INDICES": shared.UnsignedShorts, + "CULL_GROUP_INDICES": shared.UnsignedInts, + "DRAW_INDICES": shared.UnsignedShorts, + "LEAF_BRUSHES": shared.UnsignedInts, + "LEAF_SURFACES": shared.UnsignedInts, + "LIGHT_INDICES": shared.UnsignedShorts, + "OCCLUDER_EDGES": shared.UnsignedShorts, + "OCCLUDER_INDICES": shared.UnsignedShorts, + "OCCLUDER_PLANES": shared.UnsignedInts} + +LUMP_CLASSES = {"AABB_TREES": AxisAlignedBoundingBox, + "BRUSHES": Brush, + "BRUSH_SIDES": BrushSide, + "CELLS": Cell, + "COLLISION_VERTICES": CollisionVertex, + "CULL_GROUPS": CullGroup, + "DRAW_VERTICES": DrawVertex, + "LEAVES": Leaf, + "LIGHTS": Light, + "LIGHTMAPS": Lightmap, + "MODELS": Model, + "NODES": Node, + "OCCLUDERS": Occluder, + "PATCH_COLLISION": PatchCollision, + "PLANES": Plane, + "PORTALS": Portal, + "SHADERS": Shader, + "TRIANGLE_SOUPS": TriangleSoup} + +SPECIAL_LUMP_CLASSES = {"ENTITIES": shared.Entities} + +methods = [] diff --git a/io_import_rbsp/bsp_tool/branches/nexon/__init__.py b/io_import_rbsp/bsp_tool/branches/nexon/__init__.py new file mode 100644 index 0000000..32b10c9 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/nexon/__init__.py @@ -0,0 +1,26 @@ +__all__ = ["FILE_MAGIC", "cso2", "cso2_2018", "vindictus"] + +from . import cso2 # CounterStrike Online 2 +from . import cso2_2018 +from . import vindictus + +__doc__ = """Nexon is a South Korean / Japanese developer +They have worked with both Valve & Respawn's Source Engine (CS:O & TF:O [cancelled])""" + +# NOTE: Nexon games are build on source, and use it's FILE_MAGIC +FILE_MAGIC = b"VBSP" + +branches = [cso2, vindictus] + + +titanfall_online = """Titanfall: Online was a planned F2P Titanfall spin-off developed by Respawn Entertainment & Nexon. +An open beta was held in August & September of 2017[1][2], the game was later cancelled in 2018[3]. +Sources: +[1] https://titanfall.fandom.com/wiki/Titanfall_Online +[2] https://twitter.com/ZhugeEX/status/893143346021105665 +[3] https://kotaku.com/titanfall-online-cancelled-in-south-korea-1827440902 + +Titanfall: Online's .bsp format is near identical to titanfall's +the only known difference is no .bsp_lump files are used & files are stored in Nexon's proprietary .pkg archives +Files from the Titanfall:Online closed beta can be found in the internet archive @: +https://archive.org/details/titanfall-online_202107""" diff --git a/io_import_rbsp/bsp_tool/branches/nexon/cso2.py b/io_import_rbsp/bsp_tool/branches/nexon/cso2.py new file mode 100644 index 0000000..b27c7a4 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/nexon/cso2.py @@ -0,0 +1,132 @@ +# https://git.sr.ht/~leite/cso2-bsp-converter/tree/master/item/src/bsptypes.hpp +import collections +import enum +import io +import struct +import zipfile + +from . import vindictus + + +# NOTE: there are two variants with identical version numbers +# -- 2013-2017 & 2017-present +BSP_VERSION = 100 # 1.00? + + +class LUMP(enum.Enum): + ENTITIES = 0 + PLANES = 1 + TEXTURE_DATA = 2 + VERTICES = 3 + VISIBILITY = 4 + NODES = 5 + TEXTURE_INFO = 6 + FACES = 7 # version 1 + LIGHTING = 8 # version 1 + OCCLUSION = 9 # version 2 + LEAVES = 10 # version 1 + FACEIDS = 11 + EDGES = 12 + SURFEDGES = 13 + MODELS = 14 + WORLD_LIGHTS = 15 + LEAF_FACES = 16 + LEAF_BRUSHES = 17 + BRUSHES = 18 + BRUSH_SIDES = 19 + AREAS = 20 + AREA_PORTALS = 21 + UNUSED_22 = 22 + UNUSED_23 = 23 + UNUSED_24 = 24 + UNUSED_25 = 25 + DISPLACEMENT_INFO = 26 + ORIGINAL_FACES = 27 + PHYSICS_DISPLACEMENT = 28 + PHYSICS_COLLIDE = 29 + VERTEX_NORMALS = 30 + VERTEX_NORMAL_INDICES = 31 + DISPLACEMENT_LIGHTMAP_ALPHAS = 32 + DISPLACEMENT_VERTICES = 33 + DISPLACEMENT_LIGHTMAP_SAMPLE_POSITIONS = 34 + GAME_LUMP = 35 + LEAF_WATER_DATA = 36 + PRIMITIVES = 37 + PRIMITIVE_VERTICES = 38 + PRIMITIVE_INDICES = 39 + PAKFILE = 40 + CLIP_PORTAL_VERTICES = 41 + CUBEMAPS = 42 + TEXTURE_DATA_STRING_DATA = 43 + TEXTURE_DATA_STRING_TABLE = 44 + OVERLAYS = 45 + LEAF_MIN_DIST_TO_WATER = 46 + FACE_MARCO_TEXTURE_INFO = 47 + DISPLACEMENT_TRIS = 48 + PHYSICS_COLLIDE_SURFACE = 49 + WATER_OVERLAYS = 50 + LEAF_AMBIENT_INDEX_HDR = 51 + LEAF_AMBIENT_INDEX = 52 + LIGHTING_HDR = 53 # version 1 + WORLD_LIGHTS_HDR = 54 + LEAF_AMBIENT_LIGHTING_HDR = 55 # version 1 + LEAF_AMBIENT_LIGHTING = 56 # version 1 + XZIP_PAKFILE = 57 + FACES_HDR = 58 + MAP_FLAGS = 59 + OVERLAY_FADES = 60 + UNKNOWN_61 = 61 # version 1 + UNUSED_62 = 62 + UNUSED_63 = 63 + + +lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} +CSO2LumpHeader = collections.namedtuple("CSO2LumpHeader", ["offset", "length", "version", "compressed", "fourCC"]) +# NOTE: looking at headers, a half int of value 0x0001 seemed attached to version seemed to make sense + + +def read_lump_header(file, LUMP: enum.Enum) -> CSO2LumpHeader: + file.seek(lump_header_address[LUMP]) + offset, length, version, compressed = struct.unpack("2I2H", file.read(12)) + fourCC = int.from_bytes(file.read(4), "big") # fourCC is big endian for some reason + header = CSO2LumpHeader(offset, length, version, bool(compressed), fourCC) + return header +# NOTE: lump header formats could easily be a: LumpClass(base.Struct) + + +# classes for each lump, in alphabetical order: +# NOTE: dcubemap_t: 160 bytes + + +# special lump classes, in alphabetical order: +class PakFile(zipfile.ZipFile): # WIP + """CSO2 PakFiles have a custom .zip format""" + # b"CS" file magic & different header format? + def __init__(self, raw_zip: bytes): + # TODO: translate header to b"PK\x03\x04..." + raw_zip = b"".join([b"PK", raw_zip[2:]]) + self._buffer = io.BytesIO(raw_zip) + super(PakFile, self).__init__(self._buffer) + + def as_bytes(self) -> bytes: + # TODO: translate header to b"CS\x03\x04..." + raw_zip = self._buffer.getvalue() + raw_zip = b"".join([b"CS", raw_zip[2:]]) + return raw_zip + + +# {"LUMP_NAME": {version: LumpClass}} +BASIC_LUMP_CLASSES = vindictus.BASIC_LUMP_CLASSES.copy() + +LUMP_CLASSES = vindictus.LUMP_CLASSES.copy() + +SPECIAL_LUMP_CLASSES = vindictus.SPECIAL_LUMP_CLASSES.copy() +SPECIAL_LUMP_CLASSES.update({"PAKFILE": {0: PakFile}}) # WIP + +GAME_LUMP_CLASSES = vindictus.GAME_LUMP_CLASSES.copy() + + +# branch exclusive methods, in alphabetical order: + + +methods = [*vindictus.methods] diff --git a/io_import_rbsp/bsp_tool/branches/nexon/cso2_2018.py b/io_import_rbsp/bsp_tool/branches/nexon/cso2_2018.py new file mode 100644 index 0000000..33c9189 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/nexon/cso2_2018.py @@ -0,0 +1,40 @@ +# https://git.sr.ht/~leite/cso2-bsp-converter/tree/master/item/src/bsptypes.hpp +from .. import base +from . import cso2 + + +BSP_VERSION = 100 +# NOTE: almost all version numbers match 2013 era maps, this makes detection a pain + +LUMP = cso2.LUMP +lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} +read_lump_header = cso2.read_lump_header + + +# classes for each lump, in alphabetical order: +class DisplacementInfo(base.Struct): # LUMP 26 + # NOTE: 10 bytes more than Vindictus + __slots__ = ["unknown"] # not yet used + _format = "242B" + _arrays = {"unknown": 242} + +# TODO: dcubemap_t: 164 bytes +# TODO: Facev1 + +# special lump classes, in alphabetical order: + + +BASIC_LUMP_CLASSES = cso2.BASIC_LUMP_CLASSES.copy() + +LUMP_CLASSES = cso2.LUMP_CLASSES.copy() +LUMP_CLASSES.update({"DISPLACEMENT_INFO": {0: DisplacementInfo}}) + +SPECIAL_LUMP_CLASSES = cso2.SPECIAL_LUMP_CLASSES.copy() + +# {"lump": {version: SpecialLumpClass}} +GAME_LUMP_CLASSES = cso2.GAME_LUMP_CLASSES.copy() + + +# branch exclusive methods, in alphabetical order: + +methods = [*cso2.methods] diff --git a/io_import_rbsp/bsp_tool/branches/nexon/vindictus.py b/io_import_rbsp/bsp_tool/branches/nexon/vindictus.py new file mode 100644 index 0000000..70dd73e --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/nexon/vindictus.py @@ -0,0 +1,220 @@ +"""Vindictus. A MMO-RPG build in the Source Engine. Also known as Mabinogi Heroes""" +import collections +import enum +import struct +from typing import List + +from .. import base +from .. import shared +from ..valve import orange_box, source + +BSP_VERSION = 20 + + +class LUMP(enum.Enum): + ENTITIES = 0 + PLANES = 1 + TEXTURE_DATA = 2 + VERTICES = 3 + VISIBILITY = 4 + NODES = 5 + TEXTURE_INFO = 6 + FACES = 7 + LIGHTING = 8 + OCCLUSION = 9 + LEAVES = 10 + FACEIDS = 11 + EDGES = 12 + SURFEDGES = 13 + MODELS = 14 + WORLD_LIGHTS = 15 + LEAF_FACES = 16 + LEAF_BRUSHES = 17 + BRUSHES = 18 + BRUSH_SIDES = 19 + AREAS = 20 + AREA_PORTALS = 21 + UNUSED_22 = 22 + UNUSED_23 = 23 + UNUSED_24 = 24 + UNUSED_25 = 25 + DISPLACEMENT_INFO = 26 + ORIGINAL_FACES = 27 + PHYSICS_DISPLACEMENT = 28 + PHYSICS_COLLIDE = 29 + VERTEX_NORMALS = 30 + VERTEX_NORMAL_INDICES = 31 + DISPLACEMENT_LIGHTMAP_ALPHAS = 32 + DISPLACEMENT_VERTICES = 33 + DISPLACEMENT_LIGHTMAP_SAMPLE_POSITIONS = 34 + GAME_LUMP = 35 + LEAF_WATER_DATA = 36 + PRIMITIVES = 37 + PRIMITIVE_VERTICES = 38 + PRIMITIVE_INDICES = 39 + PAKFILE = 40 + CLIP_PORTAL_VERTICES = 41 + CUBEMAPS = 42 + TEXTURE_DATA_STRING_DATA = 43 + TEXTURE_DATA_STRING_TABLE = 44 + OVERLAYS = 45 + LEAF_MIN_DIST_TO_WATER = 46 + FACE_MARCO_TEXTURE_INFO = 47 + DISPLACEMENT_TRIS = 48 + PHYSICS_COLLIDE_SURFACE = 49 + WATER_OVERLAYS = 50 + LEAF_AMBIENT_INDEX_HDR = 51 + LEAF_AMBIENT_INDEX = 52 + LIGHTING_HDR = 53 + WORLD_LIGHTS_HDR = 54 + LEAF_AMBIENT_LIGHTING_HDR = 55 + LEAF_AMBIENT_LIGHTING = 56 + XZIP_PAKFILE = 57 + FACES_HDR = 58 + MAP_FLAGS = 59 + OVERLAY_FADES = 60 + UNUSED_61 = 61 + UNUSED_62 = 62 + UNUSED_63 = 63 + + +lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} +VindictusLumpHeader = collections.namedtuple("VindictusLumpHeader", ["id", "flags", "version", "offset", "length"]) +# since vindictus has a unique header format, valve .bsp have a header reading function in here + + +def read_lump_header(file, LUMP_ID: enum.Enum) -> VindictusLumpHeader: + file.seek(lump_header_address[LUMP_ID]) + id, flags, version, offset, length = struct.unpack("5i", file.read(20)) + header = VindictusLumpHeader(id, flags, version, offset, length) + return header + + +# class for each lump in alphabetical order: [10 / 64] + orange_box.LUMP_CLASSES +class Area(base.Struct): # LUMP 20 + num_area_portals: int # number or AreaPortals after first_area_portal in this Area + first_area_portal: int # index into AreaPortal lump + __slots__ = ["num_area_portals", "first_area_portal"] + _format = "2i" + + +class AreaPortal(base.Struct): # LUMP 21 + portal_key: int # unique ID? + other_area: int # ??? + first_clip_portal_vertex: int # index into the ClipPortalVertex lump + num_clip_portal_vertices: int # number of ClipPortalVertices after first_clip_portal_vertex in this AreaPortal + __slots__ = ["portal_key", "other_area", "first_clip_portal_vertex", + "num_clip_portal_vertices", "plane_num"] + _format = "4Ii" + + +class BrushSide(base.Struct): # LUMP 19 + plane: int # index into Plane lump + texture_info: int # index into TextureInfo lump + displacement_info: int # index into DisplacementInfo lump + bevel: int # smoothing group? + __slots__ = ["plane_num", "texture_info", "displacement_info", "bevel"] + _format = "I3i" + + +class DisplacementInfo(source.DisplacementInfo): # LUMP 26 + start_position: List[float] # approximate XYZ of first point in face this DisplacementInfo is rotated around + displacement_vertex_start: int # index into DisplacementVertex lump + displacement_triangle_start: int # index into DisplacementTriangle lump + power: int # 2, 3 or 4; indicates subdivision level + smoothing_angle: float + unknown: int # don't know what this does in Vindictus' format + contents: int # contents flags + face: int # index into Face lump + __slots__ = ["start_position", "displacement_vertex_start", "displacement_triangle_start", + "power", "smoothing_angle", "unknown", "contents", "face", + "lightmap_alpha_start", "lightmap_sample_position_start", + "edge_neighbours", "corner_neighbours", "allowed_verts"] + _format = "3f3if2iI2i144c10I" # Neighbours are also different + _arrays = {"start_position": [*"xyz"], "edge_neighbours": 72, + "corner_neighbours": 72, "allowed_verts": 10} + + +class Edge(list): # LUMP 12 + _format = "2I" + + def flat(self): + return self # HACK + + +class Face(base.Struct): # LUMP 7 + plane: int # index into Plane lump + __slots__ = ["plane", "side", "on_node", "unknown", "first_edge", + "num_edges", "texture_info", "displacement_info", "surface_fog_volume_id", + "styles", "light_offset", "area", + "lightmap_texture_mins_in_luxels", + "lightmap_texture_size_in_luxels", + "original_face", "num_primitives", "first_primitive_id", + "smoothing_groups"] + _format = "I2bh5i4bif4i4I" + _arrays = {"styles": 4, "lightmap_texture_mins_in_luxels": [*"st"], + "lightmap_texture_size_in_luxels": [*"st"]} + + +class Leaf(base.Struct): # LUMP 10 + __slots__ = ["contents", "cluster", "flags", "mins", "maxs", + "firstleafface", "numleaffaces", "firstleafbrush", + "numleafbrushes", "leafWaterDataID"] + _format = "9i4Ii" + _arrays = {"mins": [*"xyz"], "maxs": [*"xyz"]} + + +class Node(base.Struct): # LUMP 5 + __slots__ = ["planenum", "children", "mins", "maxs", "firstface", + "numfaces", "padding"] + _format = "12i" + _arrays = {"children": 2, "mins": [*"xyz"], "maxs": [*"xyz"]} + + +class Overlay(base.Struct): # LUMP 45 + __slots__ = ["id", "texture_info", "face_count_and_render_order", + "faces", "u", "v", "uv_points", "origin", "normal"] + _format = "2iIi4f18f" + _arrays = {"faces": 64, # OVERLAY_BSP_FACE_COUNT (bspfile.h:998) + "u": 2, "v": 2, + "uv_points": {P: [*"xyz"] for P in "ABCD"}} + + +# classes for special lumps, in alphabetical order: +# TODO: StaticPropv5, StaticPropv6 +# TODO: GameLump (alternate header format) +# old: struct GameLumpHeader { char id[4]; unsigned short flags, version; int offset, length; }; +# new: struct GameLumpHeader { char id[4]; int flags, version, offset, length; }; +# also the sprp lump includes a "scales" sub-lump after leaves +# struct StaticPropScale { int static_prop; Vector scales; }; // Different scalar for each axis +# struct StaticPropScalesLump { int count; StaticPropScales scales[count]; }; + + +# {"LUMP_NAME": {version: LumpClass}} +BASIC_LUMP_CLASSES = orange_box.BASIC_LUMP_CLASSES.copy() +BASIC_LUMP_CLASSES["LEAF_FACES"] = {0: shared.UnsignedInts} + +LUMP_CLASSES = orange_box.LUMP_CLASSES.copy() +LUMP_CLASSES.update({"AREAS": {0: Area}, + "AREA_PORTALS": {0: AreaPortal}, + "BRUSH_SIDES": {0: BrushSide}, + "DISPLACEMENT_INFO": {0: DisplacementInfo}, + "EDGES": {0: Edge}, + "FACES": {0: Face}, + # NOTE: LeafBrush also differs from orange_box (not implemented) + "LEAVES": {0: Leaf}, + "NODES": {0: Node}, + "ORIGINAL_FACES": {0: Face}}) + +SPECIAL_LUMP_CLASSES = orange_box.SPECIAL_LUMP_CLASSES.copy() + +# {"lump": {version: SpecialLumpClass}} +GAME_LUMP_CLASSES = orange_box.GAME_LUMP_CLASSES.copy() +# GAME_LUMP_CLASSES = {"sprp": {5: lambda raw_lump: GameLump_SPRP(raw_lump, StaticPropv5), +# 6: lambda raw_lump: GameLump_SPRP(raw_lump, StaticPropv6)}} + + +# branch exclusive methods, in alphabetical order: + + +methods = [*orange_box.methods] diff --git a/io_import_rbsp/bsp_tool/branches/py_struct_as_cpp.py b/io_import_rbsp/bsp_tool/branches/py_struct_as_cpp.py new file mode 100644 index 0000000..3cabe7f --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/py_struct_as_cpp.py @@ -0,0 +1,293 @@ +from __future__ import annotations +import enum +import inspect +import itertools +import re +from typing import Any, Dict, List, Union + +from .base import mapping_length, MappedArray, Struct +# class base.Struct: +# __slots__: List[str] # top-level attr name +# _format: str # types +# _arrays: Dict[str, any] # child mapping +# # ^ {"attr": _mapping} + +# class base.MappedArray: +# _mapping: int # list of len _mapping +# _mapping: List[str] # list of named children +# _mapping: Dict[str, Any] # children with their own mappings +# # ^ {"child": _mapping} +# _mapping: None # plain, siblings have mappings +# # e.g. {"id": None, "position": [*"xy"]} + +# type aliases +StructMapping = Union[Dict[str, Any], List[str], int, None] # _arrays or _mapping +CType = str +TypeMap = Dict[CType, str] +# ^ {"type": "member"} + + +def lump_class_as_c(lump_class: Union[MappedArray, Struct]) -> str: + name = lump_class.__name__ + formats = split_format(lump_class._format) + if issubclass(lump_class, Struct): + attrs = lump_class.__slots__ + mappings = getattr(lump_class, "_arrays", dict()) + elif issubclass(lump_class, MappedArray): + special_mapping = lump_class._mapping + if isinstance(special_mapping, list): + attrs, mappings = special_mapping, None + elif isinstance(special_mapping, dict): + attrs = special_mapping.keys() + mappings = special_mapping + else: + raise TypeError(f"Cannot convert {type(lump_class)} to C") + return make_c_struct(name, attrs, formats, mappings) + # ^ ("name", {"type": "member"}) + + +# TODO: revise +def make_c_struct(name: str, attrs: List[str], formats: List[str], mappings: StructMapping = dict()) -> Dict[str, TypeMap]: + members = list() + i = 0 + for attr in attrs: + member_name = attr + sub_mapping = mappings.get(attr) + if isinstance(sub_mapping, int): + format_size = sub_mapping + elif isinstance(sub_mapping, list): + format_size = len(sub_mapping) + elif isinstance(sub_mapping, dict): + format_size = mapping_length(sub_mapping) + else: + raise TypeError(f"Invalid sub_mapping: {sub_mapping}") + attr_format = formats[i:i+format_size] + c_type, member_name = c_type_of(member_name, attr_format, sub_mapping) + # TODO: convert c_type to one-liner + members.append((c_type, member_name)) + return {name: {a: t for t, a in members}} + # ^ {"name": {"type": "member"}} + + +def split_format(_format: str) -> List[str]: + _format = re.findall(r"[0-9]*[xcbB\?hHiIlLqQnNefdspP]", _format.replace(" ", "")) + out = list() + for f in _format: + match_numbered = re.match(r"([0-9]+)([xcbB\?hHiIlLqQnNefdpP])", f) + # NOTE: does not decompress strings + if match_numbered is not None: + count, f = match_numbered.groups() + out.extend(f * int(count)) + else: + out.append(f) + return out + + +type_LUT = {"c": "char", "?": "bool", + "b": "char", "B": "unsigned char", + "h": "short", "H": "unsigned short", + "i": "int", "I": "unsigned int", + "f": "float", "g": "double"} + + +def c_type_of(attr: str, formats: List[str], mapping: StructMapping) -> (CType, str): + if not isinstance(mapping, (dict, list, int, None)): + raise TypeError(f"Invalid mapping: {type(mapping)}") + if isinstance(mapping, None): # one object + c_type = type_LUT[formats[0]] + return c_type, attr + if isinstance(mapping, int): # C list type + if len(set(formats)) == 1: + c_type = type_LUT[formats[0]] + return c_type, f"{attr}[{mapping}]" + # ^ ("type", "name[count]") + else: # mixed types + if mapping > 26: # ("attr", "2ih", 4) -> ("struct { int A[2]; short B; }", "attr") + raise NotImplementedError("Cannot convert list of mixed type") + mapping = [chr(97 + i) for i in range(mapping)] # i=0: a, i=25: z + return c_type_of(attr, formats, mapping) # use list mapping instead + # ^ {"attr": {"child_A": Type, "child_B": Type}} + elif isinstance(mapping, (list, dict)): # basic / nested struct + return make_c_struct(attr, formats, mapping) + # ^ {"attr": {"child_1": Type, "child_2": Type}} + else: + raise RuntimeError("How did this happen?") + + +def apply_typing(members: Dict[str, CType]) -> Dict[str, CType]: + """{'name', 'char[256]'} -> {'name[256]', 'char'}""" + # NOTE: type may be a struct + out = dict() + # ^ {"member_1": "type", "member_2": "type[]"} + for member, _type in members.items(): + count_search = re.search(r"([a-z]+)\[([0-9]+)\]", _type) # type[count] + if count_search is not None: + _type, count = count_search.groups() + member = f"{member}[{count}]" + out[member] = _type + return out + + +def compact_members(members: Dict[str, str]) -> List[str]: + """run after apply_typing""" + # NOTE: type may be a struct, cannot chain inner structs! + # - in this case, the inner must be declared externally + members = list(members.items()) + # ^ [("member_1", "type"), ("member_2", "type")] + type_groups = [list(members[0])] + # ^ [["type", "member_1", "member_2"], ["type", "member_1"]] + for member, _type in members[1:]: + if _type == type_groups[-1][0]: + type_groups[-1].append(member) + else: + type_groups.append([_type, member]) + out = list() + # NOTE: this automatically creates "lines" & does not allow for aligning variables + for _type, *members in type_groups: + out.append(f"{_type} {', '.join(map(str, members))};") + return out + + +pattern_thc = re.compile(r"([\w\.]+):\s[\w\[\]]+ # ([\w ]+)") +# NOTE: also catches commented type hints to allow labelling of inner members + + +def get_type_hint_comments(cls: object) -> Dict[str, str]: + out = dict() # {"member": "comment"} + for line in inspect.getsource(cls): + match = pattern_thc.seach(line) + if match is None: + continue + member, comment = match.groups() + out[member] = comment + return out + + +class Style(enum.IntFlag): + # masks + TYPE_MASK = 0b11 + ONER = 0b01 + INNER = 0b10 + # major styles + OUTER_FULL = 0b00 + # struct OuterFull { + # type member; + # }; + OUTER_ONER = 0b01 # struct OuterOner { type a, b; type c; }; + INNER_FULL = 0b10 + # struct { // inner_full + # type member; + # } inner_full; + INNER_ONER = 0b11 # struct { type d, e, f; } inner_oner; + # _FULL bonus flags + ALIGN_MEMBERS = 0x04 + ALIGN_COMMENTS = 0x08 + # TODO: align with inner structs? + # TODO: COMPACT = 0x10 # use compact members + + +def definition_as_str(name: str, members: Dict[str, Any], mode: int = 0x04, comments: Dict[str, str] = dict()) -> str: + # members = {"member_1": "type", "member_2": "type[]"} + # comments = {"member_1": "comment"} + if not mode & Style.INNER: # snake_case -> CamelCase + name = "".join([word.title() for word in name.split("_")]) + # NOTE: this should be redundant, but strict styles make reading Cpp easier + # generate output + output_type = mode & Style.TYPE_MASK + if output_type & Style.INNER: + opener, closer = "struct {", "}" + f" {name};\n" + else: # OUTER + opener, closer = f"struct {name} " + "{", "};\n" + if output_type & Style.ONER: + joiner = " " + definitions = compact_members(apply_typing(members)) + else: # FULL (multi-line) + joiner = "\n" + alignment = 1 + if mode & Style.ALIGN_MEMBERS: + alignment = max([len(t) for t in members.keys() if not t.startswith("struct")]) + 2 + half_definitions = [f" {t.ljust(alignment)} {n};" for n, t in apply_typing(members).items()] + alignment = max([len(d) for d in half_definitions]) if mode & Style.ALIGN_COMMENTS else 1 + definitions = [] + for member, definition in zip(members, half_definitions): + if member in comments: + definitions.append(f"{definition.ljust(alignment)} \\\\ {comments[member]}") + else: + definitions.append(definition) + if output_type & Style.INNER: + opener += f" // {name}" + return joiner.join([opener, *definitions, closer]) + + +def branch_script_as_cpp(branch_script): + raise NotImplementedError() + bs = branch_script + out = [f"const int BSP_VERSION = {bs.BSP_VERSION};\n"] + lumps = {L.value: L.name for L in bs.LUMP} # will error if bs.LUMP is incomplete + half = (len(lumps) + 1) // 2 + 1 + + def justify(x: Any) -> str: + return str(x).rjust(len(str(len(lumps)))) + + decls = [] + for i, j in itertools.zip_longest(lumps[1:half], lumps[half + 1:]): + declarations = [f" {lumps[i]} = {justify(i)}"] + if j is not None: + declarations.append(f"{lumps[j]} = {justify(j)},") + decls.append(", ".join(declarations)) + decls[-1] = decls[-1][:-1] + ";" # replace final comma with a semi-colon + # TODO: align all the `=` signs + lump_decls = "\n".join(["namespace LUMP {", + f" const int {lumps[0]} = {justify(0)}, {lumps[half]} = {justify(half)},", + decls, "}\n\n"]) + out.extend(lump_decls) + # TODO: BasicLumpClasses -> comments? + # TODO: LumpClasses -> lump_class_as_c + # TODO: SpecialLumpClasses -> inspect.getsource(SpecialLumpClass) in Cpp TODO + # TODO: methods -> [inspect.getsource(m) for m in methods] in Cpp TODO + +# Nice to haves: +# Give common inner structs their own types (this would require the user to name each type) +# struct Vector { float x, y, z; }; +# struct LumpClass { +# int id; +# Vector origin; +# char name[128]; +# unsigned short unknown[4]; +# }; + +# LumpClass.__doc__ in a comment above it's Cpp definition +# // for one liner, /* */ with padded spaces for multi-line + + +if __name__ == "__main__": + pass + + # print(split_format("128s4I2bhHhh")) # OK + members = {"id": "int", "name": "char[256]", "inner": "struct { float a, b; }", + "skin": "short", "flags": "short"} + comments = {"id": "GUID", "inner": "inner struct"} + # print(apply_typing(members)) # OK + # print(compact_members(members)) # OK + print("=== Outer Full ===") + print(definition_as_str("Test", members, comments=comments, mode=Style.OUTER_FULL)) # OK + print("=== Outer Full + Aligned Members ===") + print(definition_as_str("Test", members, comments=comments, mode=0 | 4)) # OK + print("=== Outer Full + Aligned Members & Comments ===") + print(definition_as_str("Test", members, comments=comments, mode=0 | 4 | 8)) # OK + print("=== Outer Oner ===") + print(definition_as_str("Test", members, comments=comments, mode=Style.OUTER_ONER)) # OK + print("=== Inner Full ===") + print(definition_as_str("test", members, comments=comments, mode=Style.INNER_FULL)) # OK + print("=== Inner Full + Aligned Members ===") + print(definition_as_str("test", members, comments=comments, mode=2 | 4)) # OK + print("=== Outer Full + Aligned Members & Comments ===") + print(definition_as_str("Test", members, comments=comments, mode=2 | 4 | 8)) # OK + print("=== Inner Oner ===") + print(definition_as_str("test", members, comments=comments, mode=Style.INNER_ONER)) # OK + + # TODO: test branch_script_as_cpp + # from bsp_tool.branches.id_software.quake3 import Face # noqa F401 + # TODO: test lump_class_as_c(Face) + # from bsp_tool.branches.valve.orange_box import Plane + # print(lump_class_as_c(Plane)) diff --git a/io_import_rbsp/bsp_tool/branches/respawn/__init__.py b/io_import_rbsp/bsp_tool/branches/respawn/__init__.py new file mode 100644 index 0000000..a71afe7 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/respawn/__init__.py @@ -0,0 +1,30 @@ +__all__ = ["FILE_MAGIC", "apex_legends", "titanfall", "titanfall2"] + +from . import apex_legends +from . import titanfall +from . import titanfall2 + +__doc__ = """Respawn Entertainment was founded by former Infinity Ward members. +Their version of the Source Engine was forked around 2010 and is heavily modified + +The Titanfall Engine has b"rBSP" file-magic and 128 lumps +~72 of the 128 lumps appear in .bsp_lump files +the naming convention for these files is: "..bsp_lump" +where is a lowercase four digit hexadecimal string +e.g. mp_rr_canyonlands.004a.bsp_lump -> 0x4A -> 74 -> VertexUnlitTS + +entities are stored across 5 different .ent files per .bsp +the 5 files are: env, fx, script, snd, spawn +NOTE: the ENTITY_PARTITIONS lump may define which of these a .bsp is to use +e.g. mp_rr_canyonlands_env.ent # kings canyon lighting, fog etc. +each .ent file has a header similar to: ENTITIES02 model_count=28 +model_count appears to be the same across all .ent files for a given .bsp + +presumably all this file splitting has to do with streaming data into memory""" + +FILE_MAGIC = b"rBSP" + +# Trivia: +# All Respawn's games give the error "Not an IBSP file" when FILE_MAGIC is incorrect +# - this text refers to Quake FILE_MAGIC, a game which released in 1996. +# - does this mean Apex Legends contains code from the Quake engine? Yeah, probably. diff --git a/io_import_rbsp/bsp_tool/branches/respawn/apex_legends.py b/io_import_rbsp/bsp_tool/branches/respawn/apex_legends.py new file mode 100644 index 0000000..fada157 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/respawn/apex_legends.py @@ -0,0 +1,403 @@ +import enum +from typing import List + +from .. import base +from .. import shared +from ..valve import source +from . import titanfall, titanfall2 + + +BSP_VERSION = 47 + +GAMES = ["Apex Legends"] +GAME_VERSIONS = {"Apex Legends": 47, + "Apex Legends: Season 7 - Ascension": 48, # Olympus + "Apex Legends: Season 8 - Mayhem": 49, # King's Canyon map update 3 + "Apex Legends: Season 10 - Emergence": 50} # Arenas: Encore / SkyGarden + + +class LUMP(enum.Enum): + ENTITIES = 0x0000 + PLANES = 0x0001 + TEXTURE_DATA = 0x0002 + VERTICES = 0x0003 + LIGHTPROBE_PARENT_INFOS = 0x0004 + SHADOW_ENVIRONMENTS = 0x0005 + UNUSED_6 = 0x0006 + UNUSED_7 = 0x0007 + UNUSED_8 = 0x0008 + UNUSED_9 = 0x0009 + UNUSED_10 = 0x000A + UNUSED_11 = 0x000B + UNUSED_12 = 0x000C + UNUSED_13 = 0x000D + MODELS = 0x000E + SURFACE_NAMES = 0x000F + CONTENT_MASKS = 0x0010 + SURFACE_PROPERTIES = 0x0011 + BVH_NODES = 0x0012 + BVH_LEAF_DATA = 0x0013 + PACKED_VERTICES = 0x0014 + UNUSED_21 = 0x0015 + UNUSED_22 = 0x0016 + UNUSED_23 = 0x0017 + ENTITY_PARTITIONS = 0x0018 + UNUSED_25 = 0x0019 + UNUSED_26 = 0x001A + UNUSED_27 = 0x001B + UNUSED_28 = 0x001C + UNUSED_29 = 0x001D + VERTEX_NORMALS = 0x001E + UNUSED_31 = 0x001F + UNUSED_32 = 0x0020 + UNUSED_33 = 0x0021 + UNUSED_34 = 0x0022 + GAME_LUMP = 0x0023 + UNUSED_36 = 0x0024 + UNKNOWN_37 = 0x0025 # connected to VIS lumps + UNKNOWN_38 = 0x0026 + UNKNOWN_39 = 0x0027 # connected to VIS lumps + PAKFILE = 0x0028 # zip file, contains cubemaps + UNUSED_41 = 0x0029 + CUBEMAPS = 0x002A + UNKNOWN_43 = 0x002B + UNUSED_44 = 0x002C + UNUSED_45 = 0x002D + UNUSED_46 = 0x002E + UNUSED_47 = 0x002F + UNUSED_48 = 0x0030 + UNUSED_49 = 0x0031 + UNUSED_50 = 0x0032 + UNUSED_51 = 0x0033 + UNUSED_52 = 0x0034 + UNUSED_53 = 0x0035 + WORLDLIGHTS = 0x0036 + WORLDLIGHTS_PARENT_INFO = 0x0037 + UNUSED_56 = 0x0038 + UNUSED_57 = 0x0039 + UNUSED_58 = 0x003A + UNUSED_59 = 0x003B + UNUSED_60 = 0x003C + UNUSED_61 = 0x003D + UNUSED_62 = 0x003E + UNUSED_63 = 0x003F + UNUSED_64 = 0x0040 + UNUSED_65 = 0x0041 + UNUSED_66 = 0x0042 + UNUSED_67 = 0x0043 + UNUSED_68 = 0x0044 + UNUSED_69 = 0x0045 + UNUSED_70 = 0x0046 + VERTEX_UNLIT = 0x0047 # VERTEX_RESERVED_0 + VERTEX_LIT_FLAT = 0x0048 # VERTEX_RESERVED_1 + VERTEX_LIT_BUMP = 0x0049 # VERTEX_RESERVED_2 + VERTEX_UNLIT_TS = 0x004A # VERTEX_RESERVED_3 + VERTEX_BLINN_PHONG = 0x004B # VERTEX_RESERVED_4 + VERTEX_RESERVED_5 = 0x004C + VERTEX_RESERVED_6 = 0x004D + VERTEX_RESERVED_7 = 0x004E + MESH_INDICES = 0x004F + MESHES = 0x0050 + MESH_BOUNDS = 0x0051 + MATERIAL_SORT = 0x0052 + LIGHTMAP_HEADERS = 0x0053 + UNUSED_84 = 0x0054 + CM_GRID = 0x0055 + UNUSED_86 = 0x0056 + UNUSED_87 = 0x0057 + UNUSED_88 = 0x0058 + UNUSED_89 = 0x0059 + UNUSED_90 = 0x005A + UNUSED_91 = 0x005B + UNUSED_92 = 0x005C + UNUSED_93 = 0x005D + UNUSED_94 = 0x005E + UNUSED_95 = 0x005F + UNUSED_96 = 0x0060 + UNKNOWN_97 = 0x0061 + LIGHTMAP_DATA_SKY = 0x0062 + CSM_AABB_NODES = 0x0063 + CSM_OBJ_REFERENCES = 0x0064 + LIGHTPROBES = 0x0065 + STATIC_PROP_LIGHTPROBE_INDEX = 0x0066 + LIGHTPROBE_TREE = 0x0067 + LIGHTPROBE_REFERENCES = 0x0068 + LIGHTMAP_DATA_REAL_TIME_LIGHTS = 0x0069 + CELL_BSP_NODES = 0x006A + CELLS = 0x006B + PORTALS = 0x006C + PORTAL_VERTICES = 0x006D + PORTAL_EDGES = 0x006E + PORTAL_VERTEX_EDGES = 0x006F + PORTAL_VERTEX_REFERENCES = 0x0070 + PORTAL_EDGE_REFERENCES = 0x0071 + PORTAL_EDGE_INTERSECT_EDGE = 0x0072 + PORTAL_EDGE_INTERSECT_AT_VERTEX = 0x0073 + PORTAL_EDGE_INTERSECT_HEADER = 0x0074 + OCCLUSION_MESH_VERTICES = 0x0075 + OCCLUSION_MESH_INDICES = 0x0076 + CELL_AABB_NODES = 0x0077 + OBJ_REFERENCES = 0x0078 + OBJ_REFERENCE_BOUNDS = 0x0079 + LIGHTMAP_DATA_RTL_PAGE = 0x007A + LEVEL_INFO = 0x007B + SHADOW_MESH_OPAQUE_VERTICES = 0x007C + SHADOW_MESH_ALPHA_VERTICES = 0x007D + SHADOW_MESH_INDICES = 0x007E + SHADOW_MESH_MESHES = 0x007F + +# Known lump changes from Titanfall 2 -> Apex Legends: +# New: +# UNUSED_15 -> SURFACE_NAMES +# UNUSED_16 -> CONTENT_MASKS +# UNUSED_17 -> SURFACE_PROPERTIES +# UNUSED_18 -> BVH_NODES +# UNUSED_19 -> BVH_LEAF_DATA +# UNUSED_20 -> PACKED_VERTICES +# UNUSED_37 -> UNKNOWN_37 +# UNUSED_38 -> UNKNOWN_38 +# UNUSED_39 -> UNKNOWN_39 +# TEXTURE_DATA_STRING_DATA -> UNKNOWN_43 +# TRICOLL_BEVEL_INDICES -> UNKNOWN_97 +# Deprecated: +# LIGHTPROBE_BSP_NODES +# LIGHTPROBE_BSP_REF_IDS +# PHYSICS_COLLIDE +# LEAF_WATER_DATA +# TEXTURE_DATA_STRING_TABLE +# PHYSICS_LEVEL +# TRICOLL_TRIS +# TRICOLL_NODES +# TRICOLL_HEADERS +# CM_GRID_CELLS +# CM_GEO_SETS +# CM_GEO_SET_BOUNDS +# CM_PRIMITIVES +# CM_PRIMITIVE_BOUNDS +# CM_UNIQUE_CONTENTS +# CM_BRUSHES +# CM_BRUSH_SIDE_PLANE_OFFSETS +# CM_BRUSH_SIDE_PROPS +# CM_BRUSH_TEX_VECS +# TRICOLL_BEVEL_STARTS + +# Rough map of the relationships between lumps: +# Model -> Mesh -> MaterialSort -> TextureData -> SurfaceName +# |-> VertexReservedX +# |-> MeshIndex? +# +# MeshBounds & Mesh (must have equal number of each) +# +# VertexReservedX -> Vertex +# |-> VertexNormal +# +# ??? -> ShadowMeshIndices -?> ShadowMesh -> ??? +# ??? -> Brush -?> Plane +# +# LightmapHeader -> LIGHTMAP_DATA_SKY +# |-> LIGHTMAP_DATA_REAL_TIME_LIGHTS +# +# Portal -?> PortalEdge -> PortalVertex +# PortalEdgeRef -> PortalEdge +# PortalVertRef -> PortalVertex +# PortalEdgeIntersect -> PortalEdge? +# |-> PortalVertex +# +# PortalEdgeIntersectHeader -> ??? +# NOTE: there are always as many intersect headers as edges +# NOTE: there are also always as many vert refs as edge refs +# +# Grid probably defines the bounds of CM_GRID_CELLS, with CM_GRID_CELLS indexing other objects? + + +lump_header_address = {LUMP_ID: (16 + i * 16) for i, LUMP_ID in enumerate(LUMP)} + + +# # classes for lumps, in alphabetical order: +# NOTE: LightmapHeader.count doesn't look like a count, quite off in general + +class MaterialSort(base.Struct): # LUMP 82 (0052) + texture_data: int # index of this MaterialSort's TextureData + lightmap_index: int # index of this MaterialSort's LightmapHeader (can be -1) + unknown: List[int] # ({0?}, {??..??}) + vertex_offset: int # offset into appropriate VERTEX_RESERVED_X lump + __slots__ = ["texture_data", "lightmap_index", "unknown", "vertex_offset"] + _format = "4hI" # 12 bytes + _arrays = {"unknown": 2} + + +class Mesh(base.Struct): # LUMP 80 (0050) + first_mesh_index: int # index into this Mesh's VertexReservedX + num_triangles: int # number of triangles in VertexReservedX after first_mesh_index + # start_vertices: int # index to this Mesh's first VertexReservedX + # num_vertices: int + unknown: List[int] + material_sort: int # index of this Mesh's MaterialSort + flags: int # Flags(mesh.flags & Flags.MASK_VERTEX).name == "VERTEX_RESERVED_X" + __slots__ = ["first_mesh_index", "num_triangles", "unknown", "material_sort", "flags"] + _format = "IH3ihHI" # 28 bytes + _arrays = {"unknown": 4} + + +class Model(base.Struct): # LUMP 14 (000E) + mins: List[float] # AABB mins + maxs: List[float] # AABB maxs + first_mesh: int + num_meshes: int + unknown: List[int] # \_(;/)_/ + __slots__ = ["mins", "maxs", "first_mesh", "num_meshes", "unknown"] + _format = "6f2I8i" + _arrays = {"mins": [*"xyz"], "maxs": [*"xyz"], "unknown": 8} + + +class PackedVertex(base.MappedArray): # LUMP 20 (0014) + """a point in 3D space""" + x: int + y: int + z: int + _mapping = [*"xyz"] + _format = "3h" + + +class ShadowMesh(base.Struct): # LUMP 7F (0127) + start_index: int # assumed + num_triangles: int # assumed + unknown: List[int] # usually (1, -1) + __slots__ = ["start_index", "num_triangles", "unknown"] + _format = "2I2h" # assuming 12 bytes + _arrays = {"unknown": 2} + + +class TextureData(base.Struct): # LUMP 2 (0002) + """Name indices get out of range errors?""" + name_index: int # index of this TextureData's SurfaceName + # NOTE: indexes the starting char of the SurfaceName, skipping TextureDataStringTable + size: List[int] # texture dimensions + flags: int + __slots__ = ["name_index", "size", "flags"] + _format = "4i" # 16 bytes? + _arrays = {"size": ["width", "height"]} + + +# special vertices +class VertexBlinnPhong(base.Struct): # LUMP 75 (004B) + __slots__ = ["position_index", "normal_index", "uv", "uv2"] + _format = "2I4f" # 24 bytes + _arrays = {"uv": [*"uv"], "uv2": [*"uv"]} + + +class VertexLitBump(base.Struct): # LUMP 73 (0049) + position_index: int # index into Vertex lump + normal_index: int # index into VertexNormal lump + uv: List[float] # texture coordindates + unused: int # -1 + unknown: List[float] # vertex colour for _bm materials? + __slots__ = ["position_index", "normal_index", "uv", "unused", "unknown"] + _format = "2I2fi3f" # 32 bytes + _arrays = {"uv": [*"uv"], "unknown": 3} + + +class VertexLitFlat(base.Struct): # LUMP 72 (0048) + position_index: int # index into Vertex lump + normal_index: int # index into VertexNormal lump + uv: List[float] # texture coordindates + __slots__ = ["position_index", "normal_index", "uv", "unknown"] + _format = "2I2fi" # 20 bytes + _arrays = {"uv": [*"uv"]} + + +class VertexUnlit(base.Struct): # LUMP 71 (0047) + position_index: int # index into Vertex lump + normal_index: int # index into VertexNormal lump + uv: List[float] # texture coordindates + __slots__ = ["position_index", "normal_index", "uv", "unknown"] + _format = "2i2fi" # 20 bytes + _arrays = {"uv": [*"uv"]} + + +class VertexUnlitTS(base.Struct): # LUMP 74 (004A) + position_index: int # index into VERTICES + normal_index: int # index into VERTEX_NORMALS + uv: List[float] # texture coordinates + unknown: List[int] # 8 bytes + __slots__ = ["position_index", "normal_index", "uv", "unknown"] + _format = "2I2f2i" # 24 bytes + _arrays = {"uv": [*"uv"], "unknown": 2} + + +# special lump classes, in alphabetical order: +def ApexSPRP(raw_lump): + return titanfall2.GameLump_SPRP(raw_lump, titanfall2.StaticPropv13) + + +# NOTE: all Apex lumps are version 0, except GAME_LUMP +# {"LUMP_NAME": {version: LumpClass}} +BASIC_LUMP_CLASSES = titanfall2.BASIC_LUMP_CLASSES.copy() + +LUMP_CLASSES = titanfall2.LUMP_CLASSES.copy() +LUMP_CLASSES.update({"LIGHTMAP_HEADERS": {0: titanfall.LightmapHeader}, + "MATERIAL_SORT": {0: MaterialSort}, + "MESHES": {0: Mesh}, + "MODELS": {0: Model}, + "PACKED_VERTICES": {0: PackedVertex}, + "PLANES": {0: titanfall.Plane}, + "TEXTURE_DATA": {0: TextureData}, + "VERTEX_BLINN_PHONG": {0: VertexBlinnPhong}, + "VERTEX_LIT_BUMP": {0: VertexLitBump}, + "VERTEX_LIT_FLAT": {0: VertexLitFlat}, + "VERTEX_UNLIT": {0: VertexUnlit}, + "VERTEX_UNLIT_TS": {0: VertexUnlitTS}}) +LUMP_CLASSES.pop("CM_GRID") + +SPECIAL_LUMP_CLASSES = titanfall2.SPECIAL_LUMP_CLASSES.copy() +SPECIAL_LUMP_CLASSES.pop("TEXTURE_DATA_STRING_DATA") +SPECIAL_LUMP_CLASSES.update({"SURFACE_NAMES": {0: shared.TextureDataStringData}}) + + +GAME_LUMP_CLASSES = {"sprp": {bsp_version: ApexSPRP for bsp_version in (47, 48, 49, 50)}} + + +# branch exclusive methods, in alphabetical order: +def get_TextureData_SurfaceName(bsp, texture_data_index: int) -> str: + texture_data = bsp.TEXTURE_DATA[texture_data_index] + return bsp.SURFACE_NAMES.as_bytes()[texture_data.name_index:].lstrip(b"\0").partition(b"\0")[0].decode() + + +def get_Mesh_SurfaceName(bsp, mesh_index: int) -> str: + """Returns the name of the .vmt applied to bsp.MESHES[mesh_index]""" + mesh = bsp.MESHES[mesh_index] + material_sort = bsp.MATERIAL_SORT[mesh.material_sort] + return bsp.get_TextureData_SurfaceName(material_sort.texture_data) + + +# "debug" methods for investigating the compile process +def debug_TextureData(bsp): + print("# TextureData_index TextureData.name_index SURFACE_NAMES[name_index] TextureData.flags") + for i, td in enumerate(bsp.TEXTURE_DATA): + texture_name = bsp.get_TextureData_SurfaceName(i) + print(f"{i:02d} {td.name_index:03d} {texture_name:<48s} {source.Surface(td.flags)!r}") + + +def debug_unused_SurfaceNames(bsp): + return set(bsp.SURFACE_NAMES).difference({bsp.get_TextureData_SurfaceName(i) for i in range(len(bsp.TEXTURE_DATA))}) + + +def debug_Mesh_stats(bsp): + print("# index VERTEX_LUMP texture_data_index texture mesh_indices_range") + for i, model in enumerate(bsp.MODELS): + print(f"# MODELS[{i}]") + for j in range(model.first_mesh, model.first_mesh + model.num_meshes): + mesh = bsp.MESHES[j] + material_sort = bsp.MATERIAL_SORT[mesh.material_sort] + texture_name = bsp.get_TextureData_SurfaceName(material_sort.texture_data) + vertex_lump = (titanfall.Flags(mesh.flags) & titanfall.Flags.MASK_VERTEX).name + indices = set(bsp.MESH_INDICES[mesh.first_mesh_index:mesh.first_mesh_index + mesh.num_triangles * 3]) + _min, _max = min(indices), max(indices) + _range = f"({_min}->{_max})" if indices == {*range(_min, _max + 1)} else indices + print(f"{j:02d} {vertex_lump:<15s} {material_sort.texture_data:02d} {texture_name:<48s} {_range}") + + +methods = [titanfall.vertices_of_mesh, titanfall.vertices_of_model, + titanfall.search_all_entities, shared.worldspawn_volume, + get_TextureData_SurfaceName, get_Mesh_SurfaceName, + debug_TextureData, debug_unused_SurfaceNames, debug_Mesh_stats] diff --git a/io_import_rbsp/bsp_tool/branches/respawn/titanfall.py b/io_import_rbsp/bsp_tool/branches/respawn/titanfall.py new file mode 100644 index 0000000..c3ae22c --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/respawn/titanfall.py @@ -0,0 +1,712 @@ +# https://developer.valvesoftware.com/wiki/Source_BSP_File_Format/Game-Specific#Titanfall +import enum +import io +import struct +from typing import Dict, List, Union + +from .. import base +from .. import shared +from ..id_software import quake +from ..valve import source + + +BSP_VERSION = 29 + +GAMES = ["Titanfall", "Titanfall: Online"] +GAME_VERSIONS = {"Titanfall": 29, "Titanfall: Online": 29} + + +class LUMP(enum.Enum): + ENTITIES = 0x0000 + PLANES = 0x0001 + TEXTURE_DATA = 0x0002 + VERTICES = 0x0003 + UNUSED_4 = 0x0004 + UNUSED_5 = 0x0005 + UNUSED_6 = 0x0006 + UNUSED_7 = 0x0007 + UNUSED_8 = 0x0008 + UNUSED_9 = 0x0009 + UNUSED_10 = 0x000A + UNUSED_11 = 0x000B + UNUSED_12 = 0x000C + UNUSED_13 = 0x000D + MODELS = 0x000E + UNUSED_15 = 0x000F + UNUSED_16 = 0x0010 + UNUSED_17 = 0x0011 + UNUSED_18 = 0x0012 + UNUSED_19 = 0x0013 + UNUSED_20 = 0x0014 + UNUSED_21 = 0x0015 + UNUSED_22 = 0x0016 + UNUSED_23 = 0x0017 + ENTITY_PARTITIONS = 0x0018 + UNUSED_25 = 0x0019 + UNUSED_26 = 0x001A + UNUSED_27 = 0x001B + UNUSED_28 = 0x001C + PHYSICS_COLLIDE = 0x001D + VERTEX_NORMALS = 0x001E + UNUSED_31 = 0x001F + UNUSED_32 = 0x0020 + UNUSED_33 = 0x0021 + UNUSED_34 = 0x0022 + GAME_LUMP = 0x0023 + LEAF_WATER_DATA = 0x0024 + UNUSED_37 = 0x0025 + UNUSED_38 = 0x0026 + UNUSED_39 = 0x0027 + PAKFILE = 0x0028 # zip file, contains cubemaps + UNUSED_41 = 0x0029 + CUBEMAPS = 0x002A + TEXTURE_DATA_STRING_DATA = 0x002B + TEXTURE_DATA_STRING_TABLE = 0x002C + UNUSED_45 = 0x002D + UNUSED_46 = 0x002E + UNUSED_47 = 0x002F + UNUSED_48 = 0x0030 + UNUSED_49 = 0x0031 + UNUSED_50 = 0x0032 + UNUSED_51 = 0x0033 + UNUSED_52 = 0x0034 + UNUSED_53 = 0x0035 + WORLDLIGHTS = 0x0036 + UNUSED_55 = 0x0037 + UNUSED_56 = 0x0038 + UNUSED_57 = 0x0039 + UNUSED_58 = 0x003A + UNUSED_59 = 0x003B + UNUSED_60 = 0x003C + UNUSED_61 = 0x003D + PHYSICS_LEVEL = 0x003E + UNUSED_63 = 0x003F + UNUSED_64 = 0x0040 + UNUSED_65 = 0x0041 + TRICOLL_TRIS = 0x0042 + UNUSED_67 = 0x0043 + TRICOLL_NODES = 0x0044 + TRICOLL_HEADERS = 0x0045 + PHYSICS_TRIANGLES = 0x0046 + VERTEX_UNLIT = 0x0047 # VERTEX_RESERVED_0 + VERTEX_LIT_FLAT = 0x0048 # VERTEX_RESERVED_1 + VERTEX_LIT_BUMP = 0x0049 # VERTEX_RESERVED_2 + VERTEX_UNLIT_TS = 0x004A # VERTEX_RESERVED_3 + VERTEX_BLINN_PHONG = 0x004B # VERTEX_RESERVED_4 + VERTEX_RESERVED_5 = 0x004C + VERTEX_RESERVED_6 = 0x004D + VERTEX_RESERVED_7 = 0x004E + MESH_INDICES = 0x004F + MESHES = 0x0050 + MESH_BOUNDS = 0x0051 + MATERIAL_SORT = 0x0052 + LIGHTMAP_HEADERS = 0x0053 + UNUSED_84 = 0x0054 + CM_GRID = 0x0055 + CM_GRID_CELLS = 0x0056 + CM_GEO_SETS = 0x0057 + CM_GEO_SET_BOUNDS = 0x0058 + CM_PRIMITIVES = 0x0059 + CM_PRIMITIVE_BOUNDS = 0x005A + CM_UNIQUE_CONTENTS = 0x005B + CM_BRUSHES = 0x005C + CM_BRUSH_SIDE_PLANE_OFFSETS = 0x005D + CM_BRUSH_SIDE_PROPS = 0x005E + CM_BRUSH_TEX_VECS = 0x005F + TRICOLL_BEVEL_STARTS = 0x0060 + TRICOLL_BEVEL_INDICES = 0x0061 + LIGHTMAP_DATA_SKY = 0x0062 + CSM_AABB_NODES = 0x0063 + CSM_OBJ_REFERENCES = 0x0064 + LIGHTPROBES = 0x0065 + STATIC_PROP_LIGHTPROBE_INDEX = 0x0066 + LIGHTPROBE_TREE = 0x0067 + LIGHTPROBE_REFERENCES = 0x0068 + LIGHTMAP_DATA_REAL_TIME_LIGHTS = 0x0069 + CELL_BSP_NODES = 0x006A + CELLS = 0x006B + PORTALS = 0x006C + PORTAL_VERTICES = 0x006D + PORTAL_EDGES = 0x006E + PORTAL_VERTEX_EDGES = 0x006F + PORTAL_VERTEX_REFERENCES = 0x0070 + PORTAL_EDGE_REFERENCES = 0x0071 + PORTAL_EDGE_INTERSECT_AT_EDGE = 0x0072 + PORTAL_EDGE_INTERSECT_AT_VERTEX = 0x0073 + PORTAL_EDGE_INTERSECT_HEADER = 0x0074 + OCCLUSION_MESH_VERTICES = 0x0075 + OCCLUSION_MESH_INDICES = 0x0076 + CELL_AABB_NODES = 0x0077 + OBJ_REFERENCES = 0x0078 + OBJ_REFERENCE_BOUNDS = 0x0079 + UNUSED_122 = 0x007A + LEVEL_INFO = 0x007B + SHADOW_MESH_OPAQUE_VERTICES = 0x007C + SHADOW_MESH_ALPHA_VERTICES = 0x007D + SHADOW_MESH_INDICES = 0x007E + SHADOW_MESH_MESHES = 0x007F + +# Rough map of the relationships between lumps: + +# /-> MaterialSort -> TextureData -> TextureDataStringTable -> TextureDataStringData +# Model -> Mesh -> MeshIndices -\-> VertexReservedX -> Vertex +# \-> .flags (VertexReservedX) \--> VertexNormal +# \-> .uv + +# MeshBounds & Mesh are indexed in paralell? +# +# TextureData -> TextureDataStringTable -> TextureDataStringTable +# VertexReservedX -> Vertex +# |-> VertexNormal +# +# LeafWaterData -> TextureData -> water material +# NOTE: LeafWaterData is also used in calculating VPhysics / PHYSICS_COLLIDE +# +# ??? -> ShadowMeshIndices -?> ShadowMesh -> ??? +# ??? -> Brush -?> Plane +# +# LightmapHeader -> LIGHTMAP_DATA_SKY +# |-> LIGHTMAP_DATA_REAL_TIME_LIGHTS +# +# Portal -?> PortalEdge -> PortalVertex +# PortalEdgeRef -> PortalEdge +# PortalVertRef -> PortalVertex +# PortalEdgeIntersect -> PortalEdge? +# |-> PortalVertex +# +# PortalEdgeIntersectHeader -> ??? +# NOTE: there are always as many intersect headers as edges +# NOTE: there are also always as many vert refs as edge refs +# +# Grid probably defines the bounds of CM_GRID_CELLS, with CM_GRID_CELLS indexing other objects? + + +lump_header_address = {LUMP_ID: (16 + i * 16) for i, LUMP_ID in enumerate(LUMP)} + + +# flag enums +class Flags(enum.IntFlag): + # source.Surface (source.TextureInfo / titanfall.TextureData ?) + SKY_2D = 0x0002 # TODO: test overriding sky with this in-game + SKY = 0x0004 + WARP = 0x0008 # water surface? + TRANSLUCENT = 0x0010 # VERTEX_UNLIT_TS ? + # titanfall.Mesh.flags + VERTEX_LIT_FLAT = 0x000 # VERTEX_RESERVED_1 + VERTEX_LIT_BUMP = 0x200 # VERTEX_RESERVED_2 + VERTEX_UNLIT = 0x400 # VERTEX_RESERVED_0 + VERTEX_UNLIT_TS = 0x600 # VERTEX_RESERVED_3 + # VERTEX_BLINN_PHONG = 0x??? # VERTEX_RESERVED_4 + # guesses + TRIGGER = 0x40000 + # masks + MASK_VERTEX = 0x600 + + +# # classes for lumps, in alphabetical order: +class Bounds(base.Struct): # LUMP 88 & 90 (0058 & 005A) + unknown: List[int] # shorts seem to work best? doesn't look like AABB bounds? + __slots__ = ["unknown"] + _format = "8h" + _arrays = {"unknown": 8} + + +class Brush(base.Struct): # LUMP 92 (005C) + mins: List[float] + flags: int + maxs: List[float] + unknown: int # almost always 0 + __slots__ = ["mins", "flags", "maxs", "unknown"] + _format = "3fi3fi" + _arrays = {"mins": [*"xyz"], "maxs": [*"xyz"]} + + +class Cell(base.Struct): # LUMP 107 (006B) + """BVH4? (GDC 2018 - Extreme SIMD: Optimized Collision Detection in Titanfall) +https://www.youtube.com/watch?v=6BIfqfC1i7U +https://gdcvault.com/play/1025126/Extreme-SIMD-Optimized-Collision-Detection""" + a: int + b: int + c: int + d: int # always -1? + _format = "4h" + __slots__ = [*"abcd"] + + +class Cubemap(base.Struct): # LUMP 42 (002A) + origin: List[int] + unknown: int # index? flags? + __slots__ = ["origin", "unknown"] + _format = "3iI" + _arrays = {"origin": [*"xyz"]} + + +# NOTE: only one 28 byte entry per file +class Grid(base.Struct): # LUMP 85 (0055) + scale: float # scaled against some global vector in engine, I think + unknown: List[int] + __slots__ = ["scale", "unknown"] + _format = "f6i" + _arrays = {"unknown": 6} + + +class LeafWaterData(base.Struct): + surface_z: float # global Z height of the water's surface + min_z: float # bottom of the water volume? + texture_data: int # index to this LeafWaterData's TextureData + _mapping = ["surface_z", "min_z", "texture_data"] + _format = "2fI" + + +class LightmapHeader(base.Struct): # LUMP 83 (0053) + count: int # assuming this counts the number of lightmaps this size + # NOTE: there's actually 2 identically sized lightmaps for each header (for titanfall2) + width: int + height: int + __slots__ = ["count", "width", "height"] + _format = "I2H" + + +class LightProbeRef(base.Struct): # LUMP 104 (0068) + origin: List[float] # coords of LightProbe + lightprobe: int # index of this LightProbeRef's LightProbe + __slots__ = ["origin", "lightprobe"] + _format = "3fI" + _arrays = {"origin": [*"xyz"]} + + +class MaterialSort(base.MappedArray): # LUMP 82 (0052) + texture_data: int # index of this MaterialSort's TextureData + lightmap_header: int # index of this MaterialSort's LightmapHeader + cubemap: int # index of this MaterialSort's Cubemap + unknown: int + vertex_offset: int # offset into appropriate VERTEX_RESERVED_X lump + _mapping = ["texture_data", "lightmap_header", "cubemap", "unknown", "vertex_offset"] + _format = "4hi" # 12 bytes + + +class Mesh(base.Struct): # LUMP 80 (0050) + first_mesh_index: int # index into this Mesh's VertexReservedX + num_triangles: int # number of triangles in VertexReservedX after first_mesh_index + start_vertices: int # index to this Mesh's first VertexReservedX + num_vertices: int + unknown: List[int] + # for mp_box.VERTEX_LIT_BUMP: (2, -256, -1, ?, ?, ?) + # for mp_box.VERTEX_UNLIT: (0, -1, -1, -1, -1, -1) + material_sort: int # index of this Mesh's MaterialSort + flags: int # Flags(mesh.flags & Flags.MASK_VERTEX).name == "VERTEX_RESERVED_X" + __slots__ = ["first_mesh_index", "num_triangles", "start_vertices", + "num_vertices", "unknown", "material_sort", "flags"] + _format = "IH8hHI" # 28 Bytes + _arrays = {"unknown": 6} + + +class MeshBounds(base.Struct): # LUMP 81 (0051) + # NOTE: these are all guesses based on GDC 2018 - Extreme SIMD + mins: List[float] # TODO: verify + flags_1: int # unsure + maxs: List[float] + flags_2: int + __slots__ = ["mins", "flags_1", "maxs", "flags_2"] + _format = "3fI3fI" + _arrays = {"mins": [*"xyz"], "maxs": [*"xyz"]} + + +class Model(base.Struct): # LUMP 14 (000E) + """bsp.MODELS[0] is always worldspawn""" + mins: List[float] # bounding box mins + maxs: List[float] # bounding box maxs + first_mesh: int # index of first Mesh + num_meshes: int # number of Meshes after first_mesh in this model + __slots__ = ["mins", "maxs", "first_mesh", "num_meshes"] + _format = "6f2I" + _arrays = {"mins": [*"xyz"], "maxs": [*"xyz"]} + + +class Node(base.Struct): # LUMP 99, 106 & 119 (0063, 006A & 0077) + mins: List[float] + unknown_1: int + maxs: List[float] + unknown_2: int + __slots__ = ["mins", "unknown_1", "maxs", "unknown_2"] + _format = "3fi3fi" + _arrays = {"mins": [*"xyz"], "maxs": [*"xyz"]} + + +class ObjRefBounds(base.Struct): # LUMP 121 (0079) + # NOTE: w is always 0, could be a copy of the Node class + # - CM_BRUSHES Brush may also use this class + # NOTE: introduced in v29, not present in v25 + mins: List[float] + maxs: List[float] + _format = "8f" + __slots__ = ["mins", "maxs"] + _arrays = {"mins": [*"xyzw"], "maxs": [*"xyzw"]} + + +class Plane(base.Struct): # LUMP 1 (0001) + normal: List[float] # normal unit vector + distance: float + __slots__ = ["normal", "distance"] + _format = "4f" + _arrays = {"normal": [*"xyz"]} + + +class Portal(base.Struct): # LUMP 108 (006C) + unknown: List[int] + index: int # looks like an index + __slots__ = ["unknown", "index"] + _format = "3I" + _arrays = {"unknown": 2} + + +class PortalEdgeIntersect(base.Struct): # LUMP 114 & 115 (0072 & 0073) + unknown: List[int] # oftens ends with a few -1, allows for variable length? + __slots__ = ["unknown"] + _format = "4i" + _arrays = {"unknown": 4} + + +class PortalEdgeIntersectHeader(base.MappedArray): # LUMP 116 (0074) + start: int # 0 - 3170 + count: int # 1 - 6 + _mapping = ["start", "count"] # assumed + _format = "2i" + + +class ShadowMesh(base.Struct): # LUMP 127 (007F) + start_index: int # assuming to be like Mesh; unsure what lump is indexed + num_triangles: int + # unknown.one: int # usually one + # unknown.negative_one: int # usually negative one + __slots__ = ["start_index", "num_triangles", "unknown"] + _format = "2I2h" # assuming 12 bytes + _arrays = {"unknown": ["one", "negative_one"]} + + +class ShadowMeshAlphaVertex(base.Struct): # LUMP 125 (007D) + origin: List[float] + unknown: List[int] # unknown[1] might be a float + _format = "3f2i" + __slots__ = ["origin", "unknown"] + _arrays = {"origin": [*"xyz"], "unknown": 2} + + +class StaticPropv12(base.Struct): # sprp GAME_LUMP (0023) + origin: List[float] # x, y, z + angles: List[float] # pitch, yaw, roll + model_name: int # index into GAME_LUMP.sprp.model_names + first_leaf: int + num_leaves: int # NOTE: Titanfall doesn't have visleaves? + solid_mode: int # bitflags + flags: int + skin: int + cubemap: int # index of this StaticProp's Cubemap + unknown: int + fade_distance: float + cpu_level: List[int] # min, max (-1 = any) + gpu_level: List[int] # min, max (-1 = any) + diffuse_modulation: List[int] # RGBA 32-bit colour + scale: float + disable_x360: int + collision_flags: List[int] # add, remove + __slots__ = ["origin", "angles", "model_name", "first_leaf", "num_leaves", + "solid_mode", "flags", "skin", "cubemap", "unknown", + "forced_fade_scale", "cpu_level", "gpu_level", + "diffuse_modulation", "scale", "disable_x360", "collision_flags"] + _format = "6f3H2Bi2h4i2f8bfi2H" + _arrays = {"origin": [*"xyz"], "angles": [*"yzx"], "unknown": 6, "fade_distance": ["min", "max"], + "cpu_level": ["min", "max"], "gpu_level": ["min", "max"], + "diffuse_modulation": [*"rgba"], "collision_flags": ["add", "remove"]} + + +class TextureData(base.Struct): # LUMP 2 (0002) + """Hybrid of Source TextureData & TextureInfo""" + reflectivity: List[float] # matches .vtf reflectivity.rgb (always? black in r2) + name_index: int # index of material name in TEXTURE_DATA_STRING_DATA / TABLE + size: List[int] # dimensions of full texture + view: List[int] # dimensions of visible section of texture + flags: int # matches Mesh's .flags; probably from source.TextureInfo + __slots__ = ["reflectivity", "name_index", "width", "height", + "view_width", "view_height", "flags"] + _format = "3f6i" + _arrays = {"reflectivity": [*"rgb"], "size": ["width", "height"], "view": ["width", "height"]} + + +class TextureVector(base.Struct): # LUMP 95 (005F) + __slots__ = ["s", "t"] + __format = "8f" + _arrays = {"s": [*"xyzw"], "t": [*"xyzw"]} + + +# special vertices +class VertexBlinnPhong(base.Struct): # LUMP 75 (004B) + """Not used?""" + position_index: int # index into Vertex lump + normal_index: int # index into VertexNormal lump + __slots__ = ["position_index", "normal_index", "unknown"] + _format = "4I" # 16 bytes + _arrays = {"unknown": 2} + + +class VertexLitBump(base.Struct): # LUMP 73 (0049) + """Common Worldspawn Geometry""" + position_index: int # index into Vertex lump + normal_index: int # index into VertexNormal lump + uv: List[float] # albedo / normal / gloss / specular uv + unused: int # -1 + uv2: List[float] # small 0-1 floats, lightmap uv? + unknown: List[int] # (0, 0, ?, ?) + # {v[-2:] for v in mp_box.VERTEX_LIT_BUMP}} + # {x[0] for x in _}.union({x[1] for x in _}) # all numbers + # for "mp_box": {*range(27)} - {0, 1, 6, 17, 19, 22, 25} + __slots__ = ["position_index", "normal_index", "uv", "unknown"] + _format = "2I2fi2f4i" # 44 bytes + _arrays = {"uv": [*"uv"], "unknown": 7} + + +class VertexLitFlat(base.Struct): # LUMP 72 (0048) + """Uncommon Worldspawn Geometry""" + position_index: int # index into Vertex lump + normal_index: int # index into VertexNormal lump + uv: List[float] # uv coords + unknown: List[int] + __slots__ = ["position_index", "normal_index", "uv", "unknown"] + _format = "2I2f5I" + _arrays = {"uv": [*"uv"], "unknown": 5} + + +class VertexUnlit(base.Struct): # LUMP 71 (0047) + """Tool Brushes""" + position_index: int # index into Vertex lump + normal_index: int # index into VertexNormal lump + uv: List[float] # uv coords + unknown: int # usually -1 + __slots__ = ["position_index", "normal_index", "uv", "unknown"] + _format = "2I2fi" # 20 bytes + _arrays = {"uv": [*"uv"]} + + +class VertexUnlitTS(base.Struct): # LUMP 74 (004A) + """Glass""" + position_index: int # index into Vertex lump + normal_index: int # index into VertexNormal lump + uv: List[float] # uv coords + unknown: List[int] + __slots__ = ["position_index", "normal_index", "uv", "unknown"] + _format = "2I2f3I" # 28 bytes + _arrays = {"uv": [*"uv"], "unknown": 3} + + +VertexReservedX = Union[VertexBlinnPhong, VertexLitBump, VertexLitFlat, VertexUnlit, VertexUnlitTS] # type hint + + +# classes for special lumps, in alphabetical order: +class EntityPartitions(list): + """name of each used .ent file""" + def __init__(self, raw_lump: bytes): + super().__init__(raw_lump.decode("ascii")[:-1].split(" ")) + + def as_bytes(self) -> bytes: + return " ".join(self).encode("ascii") + b"\0" + + +class GameLump_SPRP: + """unique to TitanFall""" + _static_prop_format: str # StaticPropClass._format + model_names: List[str] + leaves: List[int] + unknown_1: int + unknown_2: int + props: List[object] # List[StaticPropClass] + + def __init__(self, raw_sprp_lump: bytes, StaticPropClass: object): + self._static_prop_format = StaticPropClass._format + sprp_lump = io.BytesIO(raw_sprp_lump) + model_name_count = int.from_bytes(sprp_lump.read(4), "little") + model_names = struct.iter_unpack("128s", sprp_lump.read(128 * model_name_count)) + setattr(self, "model_names", [t[0].replace(b"\0", b"").decode() for t in model_names]) + leaf_count = int.from_bytes(sprp_lump.read(4), "little") # usually 0 + leaves = list(struct.iter_unpack("H", sprp_lump.read(2 * leaf_count))) + setattr(self, "leaves", leaves) + prop_count, unknown_1, unknown_2 = struct.unpack("3i", sprp_lump.read(12)) + self.unknown_1, self.unknown_2 = unknown_1, unknown_2 + prop_size = struct.calcsize(StaticPropClass._format) + props = struct.iter_unpack(StaticPropClass._format, sprp_lump.read(prop_count * prop_size)) + setattr(self, "props", list(map(StaticPropClass, props))) + + def as_bytes(self) -> bytes: + return b"".join([len(self.model_names).to_bytes(4, "little"), + *[struct.pack("128s", n.encode("ascii")) for n in self.model_names], + len(self.leaves).to_bytes(4, "little"), + *[struct.pack("H", L) for L in self.leaves], + struct.pack("3I", len(self.props), self.unknown_1, self.unknown_2), + *[struct.pack(self._static_prop_format, *p.flat()) for p in self.props]]) + + +# {"LUMP_NAME": {version: LumpClass}} +BASIC_LUMP_CLASSES = {"CM_BRUSH_SIDE_PLANE_OFFSETS": {0: shared.UnsignedShorts}, + "CM_BRUSH_SIDE_PROPS": {0: shared.UnsignedShorts}, + "CM_GRID_CELLS": {0: shared.UnsignedInts}, + "CM_PRIMITIVES": {0: shared.UnsignedInts}, + "CM_UNIQUE_CONTENTS": {0: shared.UnsignedInts}, # flags? + "CSM_OBJ_REFERENCES": {0: shared.UnsignedShorts}, + "MESH_INDICES": {0: shared.UnsignedShorts}, + "OBJ_REFERENCES": {0: shared.UnsignedShorts}, + "OCCLUSION_MESH_INDICES": {0: shared.Shorts}, + "PORTAL_EDGE_REFERENCES": {0: shared.UnsignedShorts}, + "PORTAL_VERTEX_REFERENCES": {0: shared.UnsignedShorts}, + "SHADOW_MESH_INDICES": {0: shared.UnsignedShorts}, + "TEXTURE_DATA_STRING_TABLE": {0: shared.UnsignedShorts}, + "TRICOLL_BEVEL_STARTS": {0: shared.UnsignedShorts}, + "TRICOLL_BEVEL_INDICES": {0: shared.UnsignedInts}} + +LUMP_CLASSES = {"CELLS": {0: Cell}, + "CELL_AABB_NODES": {0: Node}, + # "CELL_BSP_NODES": {0: Node}, + "CM_BRUSHES": {0: Brush}, + # "CM_BRUSH_TEX_VECS": {0: TextureVector}, + "CM_GEO_SET_BOUNDS": {0: Bounds}, + "CM_GRID": {0: Grid}, + "CM_PRIMITIVE_BOUNDS": {0: Bounds}, + "CSM_AABB_NODES": {0: Node}, + "CUBEMAPS": {0: Cubemap}, + "LEAF_WATER_DATA": {0: LeafWaterData}, + "LIGHTMAP_HEADERS": {1: LightmapHeader}, + "LIGHTPROBE_REFERENCES": {0: LightProbeRef}, + "MATERIAL_SORT": {0: MaterialSort}, + "MESHES": {0: Mesh}, + "MESH_BOUNDS": {0: MeshBounds}, + "MODELS": {0: Model}, + "OBJ_REFERENCE_BOUNDS": {0: ObjRefBounds}, + "OCCLUSION_MESH_VERTICES": {0: quake.Vertex}, + "PLANES": {1: Plane}, + "PORTALS": {0: Portal}, + "PORTAL_EDGES": {0: quake.Edge}, + "PORTAL_EDGE_INTERSECT_AT_VERTEX": {0: PortalEdgeIntersect}, + "PORTAL_EDGE_INTERSECT_AT_EDGE": {0: PortalEdgeIntersect}, + "PORTAL_EDGE_INTERSECT_HEADER": {0: PortalEdgeIntersectHeader}, + "PORTAL_VERTICES": {0: quake.Vertex}, + "PORTAL_VERTEX_EDGES": {0: PortalEdgeIntersect}, + "SHADOW_MESH_MESHES": {0: ShadowMesh}, + "SHADOW_MESH_ALPHA_VERTICES": {0: ShadowMeshAlphaVertex}, + "SHADOW_MESH_OPAQUE_VERTICES": {0: quake.Vertex}, + "TEXTURE_DATA": {1: TextureData}, + "VERTEX_NORMALS": {0: quake.Vertex}, + "VERTICES": {0: quake.Vertex}, + "VERTEX_BLINN_PHONG": {0: VertexBlinnPhong}, + "VERTEX_LIT_BUMP": {1: VertexLitBump}, + "VERTEX_LIT_FLAT": {1: VertexLitFlat}, + "VERTEX_UNLIT": {0: VertexUnlit}, + "VERTEX_UNLIT_TS": {0: VertexUnlitTS}} + +SPECIAL_LUMP_CLASSES = {"ENTITY_PARTITIONS": {0: EntityPartitions}, + "ENTITIES": {0: shared.Entities}, + # NOTE: .ent files are handled directly by the RespawnBsp class + "PAKFILE": {0: shared.PakFile}, + # "PHYSICS_COLLIDE": {0: shared.PhysicsCollide}, + "TEXTURE_DATA_STRING_DATA": {0: shared.TextureDataStringData}} + +GAME_LUMP_CLASSES = {"sprp": {12: lambda raw_lump: GameLump_SPRP(raw_lump, StaticPropv12)}} + + +# branch exclusive methods, in alphabetical order: +def vertices_of_mesh(bsp, mesh_index: int) -> List[VertexReservedX]: + """gets the VertexReservedX linked to bsp.MESHES[mesh_index]""" + # https://raw.githubusercontent.com/Wanty5883/Titanfall2/master/tools/TitanfallMapExporter.py (McSimp) + mesh = bsp.MESHES[mesh_index] + material_sort = bsp.MATERIAL_SORT[mesh.material_sort] + start = mesh.first_mesh_index + finish = start + mesh.num_triangles * 3 + indices = [material_sort.vertex_offset + i for i in bsp.MESH_INDICES[start:finish]] + VERTEX_LUMP = getattr(bsp, (Flags(mesh.flags) & Flags.MASK_VERTEX).name) + return [VERTEX_LUMP[i] for i in indices] + + +def vertices_of_model(bsp, model_index: int) -> List[VertexReservedX]: + """gets the VertexReservedX linked to every Mesh in bsp.MODELS[model_index]""" + # NOTE: model 0 is worldspawn, other models are referenced by entities + out = list() + model = bsp.MODELS[model_index] + for i in range(model.first_mesh, model.num_meshes): + out.extend(bsp.vertices_of_mesh(i)) + return out + + +def replace_texture(bsp, texture: str, replacement: str): + """Substitutes a texture name in the .bsp (if it is present)""" + texture_index = bsp.TEXTURE_DATA_STRING_DATA.index(texture) # fails if texture is not in bsp + bsp.TEXTURE_DATA_STRING_DATA.insert(texture_index, replacement) + bsp.TEXTURE_DATA_STRING_DATA.pop(texture_index + 1) + bsp.TEXTURE_DATA_STRING_TABLE = list() + offset = 0 # starting index of texture name in raw TEXTURE_DATA_STRING_DATA + for texture_name in bsp.TEXTURE_DATA_STRING_DATA: + bsp.TEXTURE_DATA_STRING_TABLE.append(offset) + offset += len(texture_name) + 1 # +1 for null byte + + +def find_mesh_by_texture(bsp, texture: str) -> Mesh: + """This is a generator, will yeild one Mesh at a time. Very innefficient!""" + texture_index = bsp.TEXTURE_DATA_STRING_DATA.index(texture) # fails if texture is not in bsp + for texture_data_index, texture_data in enumerate(bsp.TEXTURE_DATA): + if texture_data.name_index != texture_index: + continue + for material_sort_index, material_sort in enumerate(bsp.MATERIAL_SORT): + if material_sort.texture_data != texture_data_index: + continue + for mesh in bsp.MESHES: + if mesh.material_sort == material_sort_index: + yield mesh + + +def get_mesh_texture(bsp, mesh_index: int) -> str: + """Returns the name of the .vmt applied to bsp.MESHES[mesh_index]""" + mesh = bsp.MESHES[mesh_index] + material_sort = bsp.MATERIAL_SORT[mesh.material_sort] + texture_data = bsp.TEXTURE_DATA[material_sort.texture_data] + return bsp.TEXTURE_DATA_STRING_DATA[texture_data.name_index] + + +def search_all_entities(bsp, **search: Dict[str, str]) -> Dict[str, List[Dict[str, str]]]: + """search_all_entities(key="value") -> {"LUMP": [{"key": "value", ...}]}""" + out = dict() + for LUMP_name in ("ENTITIES", *(f"ENTITIES_{s}" for s in ("env", "fx", "script", "snd", "spawn"))): + entity_lump = getattr(bsp, LUMP_name, shared.Entities(b"")) + results = entity_lump.search(**search) + if len(results) != 0: + out[LUMP_name] = results + return out + + +# "debug" methods for investigating the compile process +def debug_TextureData(bsp): + print("# TD_index TD.name TextureData.flags") + for i, td in enumerate(bsp.TEXTURE_DATA): + print(f"{i:02d} {bsp.TEXTURE_DATA_STRING_DATA[td.name_index]:<48s} {source.Surface(td.flags)!r}") + + +def debug_unused_TextureData(bsp): + used_texture_datas = {bsp.MATERIAL_SORT[m.material_sort].texture_data for m in bsp.MESHES} + return used_texture_datas.difference({*range(len(bsp.TEXTURE_DATA))}) + + +def debug_Mesh_stats(bsp): + print("# index vertex_lump texture_data_index texture mesh_indices_range") + for i, model in enumerate(bsp.MODELS): + print(f"# MODELS[{i}]") + for j in range(model.first_mesh, model.first_mesh + model.num_meshes): + mesh = bsp.MESHES[j] + material_sort = bsp.MATERIAL_SORT[mesh.material_sort] + texture_data = bsp.TEXTURE_DATA[material_sort.texture_data] + texture_name = bsp.TEXTURE_DATA_STRING_DATA[texture_data.name_index] + vertex_lump = (Flags(mesh.flags) & Flags.MASK_VERTEX).name + indices = set(bsp.MESH_INDICES[mesh.first_mesh_index:mesh.first_mesh_index + mesh.num_triangles * 3]) + _min, _max = min(indices), max(indices) + _range = f"({_min}->{_max})" if indices == {*range(_min, _max + 1)} else indices + print(f"{j:02d} {vertex_lump:<15s} {material_sort.texture_data:02d} {texture_name:<48s} {_range}") + + +methods = [vertices_of_mesh, vertices_of_model, + replace_texture, find_mesh_by_texture, get_mesh_texture, + search_all_entities, shared.worldspawn_volume, + debug_TextureData, debug_unused_TextureData, debug_Mesh_stats] diff --git a/io_import_rbsp/bsp_tool/branches/respawn/titanfall2.py b/io_import_rbsp/bsp_tool/branches/respawn/titanfall2.py new file mode 100644 index 0000000..a2ead12 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/respawn/titanfall2.py @@ -0,0 +1,270 @@ +import enum +import io +import struct +from typing import List + +from .. import base +from . import titanfall + + +BSP_VERSION = 37 + +GAMES = ["Titanfall 2"] +GAME_VERSIONS = {"Titanfall 2": 37} + + +class LUMP(enum.Enum): + ENTITIES = 0x0000 + PLANES = 0x0001 + TEXTURE_DATA = 0x0002 + VERTICES = 0x0003 + LIGHTPROBE_PARENT_INFOS = 0x0004 + SHADOW_ENVIRONMENTS = 0x0005 + LIGHTPROBE_BSP_NODES = 0x0006 + LIGHTPROBE_BSP_REF_IDS = 0x0007 + UNUSED_8 = 0x0008 + UNUSED_9 = 0x0009 + UNUSED_10 = 0x000A + UNUSED_11 = 0x000B + UNUSED_12 = 0x000C + UNUSED_13 = 0x000D + MODELS = 0x000E + UNUSED_15 = 0x000F + UNUSED_16 = 0x0010 + UNUSED_17 = 0x0011 + UNUSED_18 = 0x0012 + UNUSED_19 = 0x0013 + UNUSED_20 = 0x0014 + UNUSED_21 = 0x0015 + UNUSED_22 = 0x0016 + UNUSED_23 = 0x0017 + ENTITY_PARTITIONS = 0x0018 + UNUSED_25 = 0x0019 + UNUSED_26 = 0x001A + UNUSED_27 = 0x001B + UNUSED_28 = 0x001C + PHYSICS_COLLIDE = 0x001D + VERTEX_NORMALS = 0x001E + UNUSED_31 = 0x001F + UNUSED_32 = 0x0020 + UNUSED_33 = 0x0021 + UNUSED_34 = 0x0022 + GAME_LUMP = 0x0023 + UNUSED_36 = 0x0024 + UNUSED_37 = 0x0025 + UNUSED_38 = 0x0026 + UNUSED_39 = 0x0027 + PAKFILE = 0x0028 # zip file, contains cubemaps + UNUSED_41 = 0x0029 + CUBEMAPS = 0x002A + TEXTURE_DATA_STRING_DATA = 0x002B + TEXTURE_DATA_STRING_TABLE = 0x002C + UNUSED_45 = 0x002D + UNUSED_46 = 0x002E + UNUSED_47 = 0x002F + UNUSED_48 = 0x0030 + UNUSED_49 = 0x0031 + UNUSED_50 = 0x0032 + UNUSED_51 = 0x0033 + UNUSED_52 = 0x0034 + UNUSED_53 = 0x0035 + WORLDLIGHTS = 0x0036 + WORLDLIGHTS_PARENT_INFO = 0x0037 + UNUSED_56 = 0x0038 + UNUSED_57 = 0x0039 + UNUSED_58 = 0x003A + UNUSED_59 = 0x003B + UNUSED_60 = 0x003C + UNUSED_61 = 0x003D + UNUSED_62 = 0x003E + UNUSED_63 = 0x003F + UNUSED_64 = 0x0040 + UNUSED_65 = 0x0041 + TRICOLL_TRIS = 0x0042 + UNUSED_67 = 0x0043 + TRICOLL_NODES = 0x0044 + TRICOLL_HEADERS = 0x0045 + UNUSED_70 = 0x0046 + VERTEX_UNLIT = 0x0047 # VERTEX_RESERVED_0 + VERTEX_LIT_FLAT = 0x0048 # VERTEX_RESERVED_1 + VERTEX_LIT_BUMP = 0x0049 # VERTEX_RESERVED_2 + VERTEX_UNLIT_TS = 0x004A # VERTEX_RESERVED_3 + VERTEX_BLINN_PHONG = 0x004B # VERTEX_RESERVED_4 + VERTEX_RESERVED_5 = 0x004C + VERTEX_RESERVED_6 = 0x004D + VERTEX_RESERVED_7 = 0x004E + MESH_INDICES = 0x004F + MESHES = 0x0050 + MESH_BOUNDS = 0x0051 + MATERIAL_SORT = 0x0052 + LIGHTMAP_HEADERS = 0x0053 + UNUSED_84 = 0x0054 + CM_GRID = 0x0055 + CM_GRID_CELLS = 0x0056 + CM_GEO_SETS = 0x0057 + CM_GEO_SET_BOUNDS = 0x0058 + CM_PRIMITIVES = 0x0059 + CM_PRIMITIVE_BOUNDS = 0x005A + CM_UNIQUE_CONTENTS = 0x005B + CM_BRUSHES = 0x005C + CM_BRUSH_SIDE_PLANE_OFFSETS = 0x005D + CM_BRUSH_SIDE_PROPS = 0x005E + CM_BRUSH_TEX_VECS = 0x005F + TRICOLL_BEVEL_STARTS = 0x0060 + TRICOLL_BEVEL_INDICES = 0x0061 + LIGHTMAP_DATA_SKY = 0x0062 + CSM_AABB_NODES = 0x0063 + CSM_OBJ_REFERENCES = 0x0064 + LIGHTPROBES = 0x0065 + STATIC_PROP_LIGHTPROBE_INDEX = 0x0066 + LIGHTPROBE_TREE = 0x0067 + LIGHTPROBE_REFERENCES = 0x0068 + LIGHTMAP_DATA_REAL_TIME_LIGHTS = 0x0069 + CELL_BSP_NODES = 0x006A + CELLS = 0x006B + PORTALS = 0x006C + PORTAL_VERTICES = 0x006D + PORTAL_EDGES = 0x006E + PORTAL_VERTEX_EDGES = 0x006F + PORTAL_VERTEX_REFERENCES = 0x0070 + PORTAL_EDGE_REFERENCES = 0x0071 + PORTAL_EDGE_INTERSECT_EDGE = 0x0072 + PORTAL_EDGE_INTERSECT_AT_VERTEX = 0x0073 + PORTAL_EDGE_INTERSECT_HEADER = 0x0074 + OCCLUSION_MESH_VERTICES = 0x0075 + OCCLUSION_MESH_INDICES = 0x0076 + CELL_AABB_NODES = 0x0077 + OBJ_REFERENCES = 0x0078 + OBJ_REFERENCE_BOUNDS = 0x0079 + LIGHTMAP_DATA_RTL_PAGE = 0x007A + LEVEL_INFO = 0x007B + SHADOW_MESH_OPAQUE_VERTICES = 0x007C + SHADOW_MESH_ALPHA_VERTICES = 0x007D + SHADOW_MESH_INDICES = 0x007E + SHADOW_MESH_MESHES = 0x007F + +# Known lump changes from Titanfall -> Titanfall 2: +# New: +# UNUSED_4 -> LIGHTPROBE_PARENT_INFOS +# UNUSED_5 -> SHADOW_ENVIRONMENTS +# UNUSED_6 -> LIGHTPROBE_BSP_NODES +# UNUSED_7 -> LIGHTPROBE_BSP_REF_IDS +# UNUSED_55 -> WORLDLIGHTS_PARENT_INFO +# UNUSED_122 -> LIGHTMAP_DATA_RTL_PAGE +# Deprecated: +# LEAF_WATER_DATA +# PHYSICS_LEVEL +# PHYSICS_TRIANGLES + +# Rough map of the relationships between lumps: +# Model -> Mesh -> MaterialSort -> TextureData +# |-> VertexReservedX +# |-> MeshIndex +# +# MeshBounds & Mesh (must have equal number of each) +# +# TextureData -> TextureDataStringTable -> TextureDataStringTable +# VertexReservedX -> Vertex +# |-> VertexNormal +# +# ??? -> ShadowMeshIndices -?> ShadowMesh -> ??? +# ??? -> Brush -?> Plane +# +# LightmapHeader -> LIGHTMAP_DATA_SKY +# |-> LIGHTMAP_DATA_REAL_TIME_LIGHTS +# +# Portal -?> PortalEdge -> PortalVertex +# PortalEdgeRef -> PortalEdge +# PortalVertRef -> PortalVertex +# PortalEdgeIntersect -> PortalEdge? +# |-> PortalVertex +# +# PortalEdgeIntersectHeader -> ??? +# NOTE: there are always as many intersect headers as edges +# NOTE: there are also always as many vert refs as edge refs +# +# Grid probably defines the bounds of CM_GRID_CELLS, with CM_GRID_CELLS indexing other objects? + + +lump_header_address = {LUMP_ID: (16 + i * 16) for i, LUMP_ID in enumerate(LUMP)} + + +# # classes for lumps, in alphabetical order:: +class LightmapPage(base.Struct): + data: bytes + _format = "128s" + __slots__ = ["data"] + + +# TODO: LightProbeRef + + +class StaticPropv13(base.Struct): # sprp GAME_LUMP (0023) + origin: List[float] # x, y, z + angles: List[float] # pitch, yaw, roll + unknown_1: List[int] + model_name: int # index into GAME_LUMP.sprp.model_names + solid_mode: int # bitflags + flags: int + unknown_2: List[int] + forced_fade_scale: float + lighting_origin: List[float] # x, y, z + cpu_level: List[int] # min, max (-1 = any) + gpu_level: List[int] # min, max (-1 = any) + diffuse_modulation: List[int] # RGBA 32-bit colour + collision_flags: List[int] # add, remove + # NOTE: no skin or cubemap + __slots__ = ["origin", "angles", "unknown_1", "model_name", "solid_mode", "flags", + "unknown_2", "forced_fade_scale", "lighting_origin", "cpu_level", + "gpu_level", "diffuse_modulation", "collision_flags"] + _format = "6f4bH2B4b4f8b2H" # 64 bytes + _arrays = {"origin": [*"xyz"], "angles": [*"yzx"], "unknown_1": 4, "unknown_2": 4, + "lighting_origin": [*"xyz"], "cpu_level": ["min", "max"], + "gpu_level": ["min", "max"], "diffuse_modulation": [*"rgba"], + "collision_flags": ["add", "remove"]} + + +# classes for special lumps, in alphabetical order: +class GameLump_SPRP: + """New in Titanfall 2""" + _static_prop_format: str # StaticPropClass._format + model_names: List[str] + unknown_1: int + unknown_2: int # indices? + props: List[object] # List[StaticPropClass] + + def __init__(self, raw_sprp_lump: bytes, StaticPropClass: object): + self._static_prop_format = StaticPropClass._format + sprp_lump = io.BytesIO(raw_sprp_lump) + model_names_count = int.from_bytes(sprp_lump.read(4), "little") + model_names = struct.iter_unpack("128s", sprp_lump.read(128 * model_names_count)) + setattr(self, "model_names", [t[0].replace(b"\0", b"").decode() for t in model_names]) + prop_count, unknown_1, unknown_2 = struct.unpack("3i", sprp_lump.read(12)) + self.unknown_1, self.unknown_2 = unknown_1, unknown_2 + prop_size = struct.calcsize(StaticPropClass._format) + props = struct.iter_unpack(StaticPropClass._format, sprp_lump.read(prop_count * prop_size)) + setattr(self, "props", list(map(StaticPropClass, props))) + # TODO: check if are there any leftover bytes at the end? + + def as_bytes(self) -> bytes: + # NOTE: additions to .props must be of the correct type, + # GameLump_SPRP does not perform conversions of any kind! + return b"".join([int.to_bytes(len(self.model_names), 4, "little"), + *[struct.pack("128s", n) for n in self.model_names], + *struct.pack("3I", len(self.props), self.unknown_1, self.unknown_2), + *[struct.pack(self._static_prop_format, *p.flat()) for p in self.props]]) + + +# {"LUMP_NAME": {version: LumpClass}} +BASIC_LUMP_CLASSES = titanfall.BASIC_LUMP_CLASSES.copy() + +LUMP_CLASSES = titanfall.LUMP_CLASSES.copy() +LUMP_CLASSES["LIGHTMAP_DATA_REAL_TIME_LIGHTS_PAGE"] = {0: LightmapPage} +LUMP_CLASSES.pop("LIGHTPROBE_REFERENCES") # size doesn't match + +SPECIAL_LUMP_CLASSES = titanfall.SPECIAL_LUMP_CLASSES.copy() + +GAME_LUMP_CLASSES = {"sprp": {13: lambda raw_lump: GameLump_SPRP(raw_lump, StaticPropv13)}} + +# branch exclusive methods, in alphabetical order: +methods = [*titanfall.methods] diff --git a/io_import_rbsp/bsp_tool/branches/ritual/__init__.py b/io_import_rbsp/bsp_tool/branches/ritual/__init__.py new file mode 100644 index 0000000..388c598 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/ritual/__init__.py @@ -0,0 +1,9 @@ +# https://web.archive.org/web/20070611074310/http://www.ritual.com/index.php?section=games/overview +# http://ritualistic.chrissstrahl.de/games/ef2/gdkdocs/ +__all__ = [] +# TODO: ubertools +# TODO: mcgee +# TODO: star_trek + + +__doc__ = """Ritual Entertainment developed Übertools for Quake III""" diff --git a/io_import_rbsp/bsp_tool/branches/shared.py b/io_import_rbsp/bsp_tool/branches/shared.py new file mode 100644 index 0000000..9359615 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/shared.py @@ -0,0 +1,283 @@ +import collections +import enum +import io +import itertools +import math +import re +import struct +import zipfile +from typing import Dict, List + + +# TODO: adapt SpecialLumpClasses to be more in-line with lumps.BspLump subclasses +# TODO: make special classes __init__ method create an empty mutable object +# TODO: move current special class __init__ to a .from_bytes() method +# TODO: prototype the system for saving game lumps to file +# -- need to know filesize, but modify the headers of each lump to have file relative offsets +# -- NOTE: fully implemented in RespawnBsp.save_as +# TODO: make a base class for SpecialLumpClasses the facilitates dynamic indexing +# -- use the lumps system to dynamically index a file: +# -- do an initial scan for where each entry begins & have a .read_entry() method +# TODO: consider using __repr__ methods, as SpecialLumpClasses can get large + + +# flag enums +class SPRP_flags(enum.IntFlag): + FADES = 0x1 # use fade distances + USE_LIGHTING_ORIGIN = 0x2 + NO_DRAW = 0x4 # computed at run time based on dx level + # the following are set in a level editor: + IGNORE_NORMALS = 0x8 + NO_SHADOW = 0x10 + SCREEN_SPACE_FADE = 0x20 + # next 3 are for lighting compiler + NO_PER_VERTEX_LIGHTING = 0x40 + NO_SELF_SHADOWING = 0x80 + NO_PER_TEXEL_LIGHTING = 0x100 + EDITOR_MASK = 0x1D8 + + +# Basic Lump Classes +class Ints(int): + _format = "i" + + +class Shorts(int): + _format = "h" + + +class UnsignedInts(int): + _format = "I" + + +class UnsignedShorts(int): + _format = "H" + + +# Special Lump Classes +class Entities(list): + # TODO: match "classname" to python classes (optional) + # -- use fgd-tools? + # TODO: use a true __init__ & .from_bytes() @staticmethod + def __init__(self, raw_entities: bytes): + entities: List[Dict[str, str]] = list() + # ^ [{"key": "value"}] + # TODO: handle newlines in keys / values + for line_no, line in enumerate(raw_entities.decode(errors="ignore").splitlines()): + if re.match(r"^\s*$", line): # line is blank / whitespace + continue + if "{" in line: # new entity + ent = dict() + elif '"' in line: + key_value_pair = re.search(r'"([^"]*)"\s"([^"]*)"', line) + if not key_value_pair: + print(f"ERROR LOADING ENTITIES: Line {line_no:05d}: {line}") + continue + key, value = key_value_pair.groups() + if key not in ent: + ent[key] = value + else: # don't override duplicate keys, share a list instead + # generally duplicate keys are ouputs + if isinstance(ent[key], list): # more than 2 of this key + ent[key].append(value) + else: # second occurance of key + ent[key] = [ent[key], value] + elif "}" in line: # close entity + entities.append(ent) + elif line == b"\x00".decode(): # ignore null bytes + continue + elif line.startswith("//"): # ignore comments + continue + else: + raise RuntimeError(f"Unexpected line in entities: L{line_no}: {line.encode()}") + super().__init__(entities) + + def search(self, **search: Dict[str, str]) -> List[Dict[str, str]]: + """.search(classname="light_environment") -> [{"classname": "light_environment", ...}]""" + # NOTE: exact matches only! + return [e for e in self if all([e.get(k, "") == v for k, v in search.items()])] + + # TODO: find_regex + + # TODO: find_any (any k == v, not all) + + # TODO: find_any_regex + + def as_bytes(self) -> bytes: + entities = [] + for entity_dict in self: # Dict[str, Union[str, List[str]]] + entity = ["{"] + for key, value in entity_dict.items(): + if isinstance(value, str): + entity.append(f'"{key}" "{value}"') + elif isinstance(value, list): # multiple entries + entity.extend([f'"{key}" "{v}"' for v in value]) + else: + raise RuntimeError("Entity values must be") + entity.append("}") + entities.append("\n".join(entity)) + return b"\n".join(map(lambda e: e.encode("ascii"), entities)) + b"\n\x00" + + +class GameLump_SPRP: # Mostly for Source + def __init__(self, raw_sprp_lump: bytes, StaticPropClass: object): + """Get StaticPropClass from GameLump version""" + # # lambda raw_lump: GameLump_SPRP(raw_lump, StaticPropvXX) + sprp_lump = io.BytesIO(raw_sprp_lump) + model_name_count = int.from_bytes(sprp_lump.read(4), "little") + model_names = struct.iter_unpack("128s", sprp_lump.read(128 * model_name_count)) + setattr(self, "model_names", [t[0].replace(b"\0", b"").decode() for t in model_names]) + leaf_count = int.from_bytes(sprp_lump.read(4), "little") + leaves = itertools.chain(*struct.iter_unpack("H", sprp_lump.read(2 * leaf_count))) + setattr(self, "leaves", list(leaves)) + prop_count = int.from_bytes(sprp_lump.read(4), "little") + read_size = struct.calcsize(StaticPropClass._format) * prop_count + props = struct.iter_unpack(StaticPropClass._format, sprp_lump.read(read_size)) + setattr(self, "props", list(map(StaticPropClass, props))) + here = sprp_lump.tell() + end = sprp_lump.seek(0, 2) + assert here == end, "Had some leftover bytes, bad format" + + def as_bytes(self) -> bytes: + if len(self.props) > 0: + prop_format = self.props[0]._format + else: + prop_format = "" + return b"".join([int.to_bytes(len(self.model_names), 4, "little"), + *[struct.pack("128s", n) for n in self.model_names], + int.to_bytes(len(self.leaves), 4, "little"), + *[struct.pack("H", L) for L in self.leaves], + int.to_bytes(len(self.props), 4, "little"), + *[struct.pack(prop_format, *p.flat()) for p in self.props]]) + + +class PakFile(zipfile.ZipFile): + def __init__(self, raw_zip: bytes): + self._buffer = io.BytesIO(raw_zip) + super(PakFile, self).__init__(self._buffer) + + def as_bytes(self) -> bytes: + return self._buffer.getvalue() + + +# PhysicsBlock headers +CollideHeader = collections.namedtuple("swapcollideheader_t", ["id", "version", "model_type"]) +# struct swapcollideheader_t { int size, vphysicsID; short version, model_type; }; +SurfaceHeader = collections.namedtuple("swapcompactsurfaceheader_t", ["size", "drag_axis_areas", "axis_map_size"]) +# struct swapcompactsurfaceheader_t { int surfaceSize; Vector dragAxisAreas; int axisMapSize; }; +MoppHeader = collections.namedtuple("swapmoppsurfaceheader_t", ["size"]) +# struct swapmoppsurfaceheader_t { int moppSize; }; + + +class PhysicsBlock: # TODO: actually process this data + # byte swapper: https://github.com/ValveSoftware/source-sdk-2013/blob/master/sp/src/utils/common/bsplib.cpp#L1677 + def __init__(self, raw_lump: bytes): + """Editting not yet supported""" + self._raw = raw_lump + lump = io.BytesIO(raw_lump) + header = CollideHeader(*struct.unpack("4s2h", lump.read(8))) + assert header.id == b"VPHY", "if 'YHPV' byte order is flipped" + # version isn't checked by the byteswap, probably important for VPHYSICS + if header.model_type == 0: + size, *drag_axis_areas, axis_map_size = struct.unpack("i3fi", lump.read(20)) + surface_header = SurfaceHeader(size, drag_axis_areas, axis_map_size) + elif header.model_type == 1: + surface_header = MoppHeader(*struct.unpack("i", lump.read(4))) + else: + raise RuntimeError("Invalid model type") + self.header = (header, surface_header) + self.data = lump.read(surface_header.size) + assert lump.tell() == len(raw_lump) + + def as_bytes(self) -> bytes: + header, surface_header = self.header + if header.model_type == 0: + size, drag_axis_areas, axis_map_size = surface_header + surface_header = struct.pack("i3fi", len(self.data), *drag_axis_areas, axis_map_size) + elif header.model_type == 1: + surface_header = struct.pack("i", len(self.data)) + else: + raise RuntimeError("Invalid model type") + header = struct.pack("4s2h", *header) + return b"".join([header, surface_header, self.data]) + + +# PhysicsCollide headers +PhysicsHeader = collections.namedtuple("dphysmodel_t", ["model", "data_size", "script_size", "solid_count"]) +# struct dphysmodel_t { int model_index, data_size, keydata_size, solid_count; }; + + +class PhysicsCollide(list): + """[model_index: int, solids: List[bytes], script: bytes]""" + # passed to VCollideLoad in vphysics.dll + def __init__(self, raw_lump: bytes): + collision_models = list() + lump = io.BytesIO(raw_lump) + header = PhysicsHeader(*struct.unpack("4i", lump.read(16))) + while header != PhysicsHeader(-1, -1, 0, 0) and lump.tell() != len(raw_lump): + solids = list() + for i in range(header.solid_count): + # CPhysCollisionEntry->WriteCollisionBinary + cb_size = int.from_bytes(lump.read(4), "little") + solids.append(PhysicsBlock(lump.read(cb_size))) + # NOTE: should have read as many bytes as header.data_size + script = lump.read(header.script_size) # ascii + collision_models.append([header.model, solids, script]) + header = PhysicsHeader(*struct.unpack("4i", lump.read(16))) + assert header == PhysicsHeader(-1, -1, 0, 0), "PhysicsCollide ended incorrectly" + super().__init__(collision_models) + + def as_bytes(self) -> bytes: + # NOTE: consistently missing 16 bytes + def phy_bytes(collision_model): + model, solids, script = collision_model + solid_count = len(solids) + data_size = len([s for s in solids]) + solid_count * 4 + header = struct.pack("4i", model, data_size, len(script), solid_count) + solid_binaries = list() + for phy_block in solids: + collision_data = phy_block.as_bytes() + solid_binaries.append(len(collision_data).to_bytes(4, "little")) + solid_binaries.append(collision_data) + return b"".join([header, *solid_binaries, script]) + return b"".join(map(phy_bytes, self)) + + +class TextureDataStringData(list): + def __init__(self, raw_texture_data_string_data: bytes): + super().__init__([t.decode("ascii", errors="ignore") for t in raw_texture_data_string_data[:-1].split(b"\0")]) + + # TODO: use regex to search + # def find(self, pattern: str) -> List[str]: + # pattern = pattern.lower() + # return fnmatch.filter(map(str.lower, self), f"*{pattern}*") + + def as_bytes(self) -> bytes: + return b"\0".join([t.encode("ascii") for t in self]) + b"\0" + + +class Visiblity: + # seems to be the same across Source & Quake engines + # is Titanfall (v29) the same? + def __init__(self, raw_visibility: bytes): + visibility_data = [v[0] for v in struct.iter_unpack("i", raw_visibility)] + num_clusters = visibility_data + for i in range(num_clusters): + i = (2 * i) + 1 + pvs_offset = visibility_data[i] # noqa: F841 + pas_offset = visibility_data[i + 1] # noqa: F841 + # ^ pointers into RLE encoded bits mapping the PVS tree + # from bytes inside the .bsp file? + raise NotImplementedError("Understanding of Visibility lump is incomplete") + + def as_bytes(self) -> bytes: + raise NotImplementedError("Visibility lump hard") + + +# methods +def worldspawn_volume(bsp): + """allows for sorting maps by size""" + worldspawn = bsp.ENTITIES[0] + maxs = map(float, worldspawn["world_maxs"].split()) + mins = map(float, worldspawn["world_mins"].split()) + return math.sqrt(sum([(b - a) ** 2 for a, b in zip(mins, maxs)])) diff --git a/io_import_rbsp/bsp_tool/branches/valve/__init__.py b/io_import_rbsp/bsp_tool/branches/valve/__init__.py new file mode 100644 index 0000000..fa810dc --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/valve/__init__.py @@ -0,0 +1,19 @@ +__all__ = ["FILE_MAGIC", "alien_swarm", "branches", "sdk_2013", "source", + "goldsrc", "left4dead", "left4dead2", "orange_box"] + +from . import alien_swarm +from . import sdk_2013 +from . import source +from . import goldsrc # Most GoldSrc Games +from . import left4dead +from . import left4dead2 +from . import orange_box # Most Source Engine Games +# TODO: Portal 2 + +__doc__ = """Valve Software developed the GoldSrc Engine, building on idTech 2. +This variant powered Half-Life & CS:1.6. Valve went on to develop the Source Engine for Half-Life 2.""" + +# TRIVIA: when FILE_MAGIC is incorrect Source engine games give the error message: "Not an IBSP file" +# - this refers to the IdTech 2 FILE_MAGIC, pre-dating Half-Life 1! +FILE_MAGIC = b"VBSP" +# NOTE: GoldSrcBsp has no FILE_MAGIC diff --git a/io_import_rbsp/bsp_tool/branches/valve/alien_swarm.py b/io_import_rbsp/bsp_tool/branches/valve/alien_swarm.py new file mode 100644 index 0000000..27652aa --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/valve/alien_swarm.py @@ -0,0 +1,114 @@ +# https://developer.valvesoftware.com/wiki/Alien_Swarm_(engine_branch) +import enum +import struct + +from . import orange_box +from . import source + + +BSP_VERSION = 21 + +GAMES = ["Alien Swarm", "Alien Swarm Reactive Drop"] + + +class LUMP(enum.Enum): + ENTITIES = 0 + PLANES = 1 + TEXTURE_DATA = 2 + VERTICES = 3 + VISIBILITY = 4 + NODES = 5 + TEXTURE_INFO = 6 + FACES = 7 + LIGHTING = 8 + OCCLUSION = 9 + LEAVES = 10 + FACE_IDS = 11 + EDGES = 12 + SURFEDGES = 13 + MODELS = 14 + WORLD_LIGHTS = 15 + LEAF_FACES = 16 + LEAF_BRUSHES = 17 + BRUSHES = 18 + BRUSH_SIDES = 19 + AREAS = 20 + AREA_PORTALS = 21 + UNUSED_22 = 22 + UNUSED_23 = 23 + UNUSED_24 = 24 + UNUSED_25 = 25 + DISPLACEMENT_INFO = 26 + ORIGINAL_FACES = 27 + PHYSICS_DISPLACEMENT = 28 + PHYSICS_COLLIDE = 29 + VERTEX_NORMALS = 30 + VERTEX_NORMAL_INDICES = 31 + DISPLACEMENT_LIGHTMAP_ALPHAS = 32 + DISPLACEMENT_VERTICES = 33 + DISPLACEMENT_LIGHTMAP_SAMPLE_POSITIONS = 34 + GAME_LUMP = 35 + LEAF_WATER_DATA = 36 + PRIMITIVES = 37 + PRIMITIVE_VERTICES = 38 + PRIMITIVE_INDICES = 39 + PAKFILE = 40 + CLIP_PORTAL_VERTICES = 41 + CUBEMAPS = 42 + TEXTURE_DATA_STRING_DATA = 43 + TEXTURE_DATA_STRING_TABLE = 44 + OVERLAYS = 45 + LEAF_MIN_DIST_TO_WATER = 46 + FACE_MACRO_TEXTURE_INFO = 47 + DISPLACEMENT_TRIS = 48 + PHYSICS_COLLIDE_SURFACE = 49 + WATER_OVERLAYS = 50 + LEAF_AMBIENT_INDEX_HDR = 51 + LEAF_AMBIENT_INDEX = 52 + LIGHTING_HDR = 53 + WORLD_LIGHTS_HDR = 54 + LEAF_AMBIENT_LIGHTING_HDR = 55 + LEAF_AMBIENT_LIGHTING = 56 + XZIP_PAKFILE = 57 + FACES_HDR = 58 + MAP_FLAGS = 59 + OVERLAY_FADES = 60 + UNUSED_61 = 61 + UNUSED_62 = 62 + DISPLACEMENT_MULTIBLEND = 63 + +# Known lump changes from Orange Box -> Alien Swarm: +# New: +# UNUSED_63 -> DISPLACEMENT_MULTIBLEND +# Deprecated: +# ??? + + +lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} + + +def read_lump_header(file, LUMP: enum.Enum) -> source.SourceLumpHeader: + file.seek(lump_header_address[LUMP]) + offset, length, version, fourCC = struct.unpack("4I", file.read(16)) + header = source.SourceLumpHeader(offset, length, version, fourCC) + return header + + +# classes for lumps, in alphabetical order: +# TODO: WorldLightHDR + +# classes for special lumps, in alphabetical order: + +# {"LUMP_NAME": {version: LumpClass}} +BASIC_LUMP_CLASSES = orange_box.BASIC_LUMP_CLASSES.copy() + +LUMP_CLASSES = orange_box.LUMP_CLASSES.copy() +LUMP_CLASSES.pop("WORLD_LIGHTS") +LUMP_CLASSES.pop("WORLD_LIGHTS_HDR") + +SPECIAL_LUMP_CLASSES = orange_box.SPECIAL_LUMP_CLASSES.copy() + +GAME_LUMP_CLASSES = orange_box.GAME_LUMP_CLASSES.copy() + +# branch exclusive methods, in alphabetical order: +methods = [*orange_box.methods] diff --git a/io_import_rbsp/bsp_tool/branches/valve/goldsrc.py b/io_import_rbsp/bsp_tool/branches/valve/goldsrc.py new file mode 100644 index 0000000..a766910 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/valve/goldsrc.py @@ -0,0 +1,116 @@ +# https://github.com/ValveSoftware/halflife/blob/master/utils/common/bspfile.h +# http://hlbsp.sourceforge.net/index.php?content=bspdef +# https://valvedev.info/tools/bsptwomap/ +import enum + +from ..id_software import quake # GoldSrc was forked from IdTech 2 during Quake II development + +BSP_VERSION = 30 + +GAMES = [*[f"Half-Life/{mod}" for mod in [ + "cstrike", # Counter-Strike + "czero", # Counter-Strike: Condition Zero + "czeror", # Counter-Strike: Condition Zero - Deleted Scenes + "dmc", # Deathmatch Classic + "dod", # Day of Defeat + "gearbox", # Half-Life: Opposing Force + "ricochet", # Ricochet + "tfc", # Team Fortress Classic + "valve"]], # Half-Life + "Halfquake Trilogy", "Sven Co-op"] + + +# lump names & indices: +class LUMP(enum.Enum): + ENTITIES = 0 + PLANES = 1 + MIP_TEXTURES = 2 + VERTICES = 3 + VISIBILITY = 4 + NODES = 5 + TEXTURE_INFO = 6 + FACES = 7 + LIGHTING = 8 + CLIP_NODES = 9 + LEAVES = 10 + MARK_SURFACES = 11 + EDGES = 12 + MODELS = 14 + +# Known lump changes from Quake II -> GoldSrc: +# New: +# MARK_SURFACES + + +lump_header_address = {LUMP_ID: (4 + i * 8) for i, LUMP_ID in enumerate(LUMP)} + + +# Engine limits: +class MAX(enum.Enum): + ENTITIES = 1024 + PLANES = 32767 + MIP_TEXTURES = 512 + MIP_TEXTURES_SIZE = 0x200000 # in bytes + VERTICES = 65535 + VISIBILITY_SIZE = 0x200000 # in bytes + NODES = 32767 # "because negative shorts are contents" + TEXTURE_INFO = 8192 + FACES = 65535 + LIGHTING_SIZE = 0x200000 # in bytes + CLIP_NODES = 32767 + LEAVES = 8192 + MARK_SURFACES = 65535 + EDGES = 256000 + MODELS = 400 + BRUSHES = 4096 # for radiant / q2map ? + ENTITY_KEY = 32 + ENTITY_STRING = 128 * 1024 + ENTITY_VALUE = 1024 + PORTALS = 65536 # related to leaves + SURFEDGES = 512000 + + +# flag enums +class Contents(enum.IntFlag): # src/public/bspflags.h + """Brush flags""" + # NOTE: compiler gets these flags from a combination of all textures on the brush + # e.g. any non opaque face means this brush is non-opaque, and will not block vis + # visible + EMPTY = -1 + SOLID = -2 + WATER = -3 + SLIME = -4 + LAVA = -5 + SKY = -6 + ORIGIN = -7 # removed when compiling from .map / .vmf to .bsp + CLIP = -8 # "changed to contents_solid" + CURRENT_0 = -9 + CURRENT_90 = -10 + CURRENT_180 = -11 + CURRENT_270 = -12 + CURRENT_UP = -13 + CURRENT_DOWN = -14 + TRANSLUCENT = -15 + + +# classes for lumps, in alphabetical order:: +# TODO: Model, Node + +# classes for special lumps, in alphabetical order: +# TODO: make a special LumpCLass for MipTextures +# -- any lump containing offsets needs it's own BspLump subclass +# {"TEXTURES": lambda raw_lump: lump.MipTextures(quake.MipTexture, raw_lump)} + +# {"LUMP_NAME": {version: LumpClass}} +BASIC_LUMP_CLASSES = quake.BASIC_LUMP_CLASSES.copy() + +LUMP_CLASSES = quake.LUMP_CLASSES.copy() +LUMP_CLASSES.pop("MODELS") +LUMP_CLASSES.pop("NODES") + +SPECIAL_LUMP_CLASSES = quake.SPECIAL_LUMP_CLASSES.copy() +SPECIAL_LUMP_CLASSES.pop("MIP_TEXTURES") + + +# branch exclusive methods, in alphabetical order: +methods = [*quake.methods] diff --git a/io_import_rbsp/bsp_tool/branches/valve/left4dead.py b/io_import_rbsp/bsp_tool/branches/valve/left4dead.py new file mode 100644 index 0000000..9b64973 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/valve/left4dead.py @@ -0,0 +1,122 @@ +# https://developer.valvesoftware.com/wiki/Left_4_Dead_(engine_branch) +import collections +import enum +import struct + +from . import orange_box +from . import source + + +BSP_VERSION = 20 + +GAMES = ["Left 4 Dead"] + + +class LUMP(enum.Enum): + ENTITIES = 0 + PLANES = 1 + TEXTURE_DATA = 2 + VERTICES = 3 + VISIBILITY = 4 + NODES = 5 + TEXTURE_INFO = 6 + FACES = 7 + LIGHTING = 8 + OCCLUSION = 9 + LEAVES = 10 + FACE_IDS = 11 + EDGES = 12 + SURFEDGES = 13 + MODELS = 14 + WORLD_LIGHTS = 15 + LEAF_FACES = 16 + LEAF_BRUSHES = 17 + BRUSHES = 18 + BRUSH_SIDES = 19 + AREAS = 20 + AREA_PORTALS = 21 + UNUSED_22 = 22 + UNUSED_23 = 23 + UNUSED_24 = 24 + UNUSED_25 = 25 + DISPLACEMENT_INFO = 26 + ORIGINAL_FACES = 27 + PHYSICS_DISPLACEMENT = 28 + PHYSICS_COLLIDE = 29 + VERTEX_NORMALS = 30 + VERTEX_NORMAL_INDICES = 31 + DISPLACEMENT_LIGHTMAP_ALPHAS = 32 + DISPLACEMENT_VERTICES = 33 + DISPLACEMENT_LIGHTMAP_SAMPLE_POSITIONS = 34 + GAME_LUMP = 35 + LEAF_WATER_DATA = 36 + PRIMITIVES = 37 + PRIMITIVE_VERTICES = 38 + PRIMITIVE_INDICES = 39 + PAKFILE = 40 + CLIP_PORTAL_VERTICES = 41 + CUBEMAPS = 42 + TEXTURE_DATA_STRING_DATA = 43 + TEXTURE_DATA_STRING_TABLE = 44 + OVERLAYS = 45 + LEAF_MIN_DIST_TO_WATER = 46 + FACE_MACRO_TEXTURE_INFO = 47 + DISPLACEMENT_TRIS = 48 + PHYSICS_COLLIDE_SURFACE = 49 + WATER_OVERLAYS = 50 + LEAF_AMBIENT_INDEX_HDR = 51 + LEAF_AMBIENT_INDEX = 52 + LIGHTING_HDR = 53 + WORLD_LIGHTS_HDR = 54 + LEAF_AMBIENT_LIGHTING_HDR = 55 + LEAF_AMBIENT_LIGHTING = 56 + XZIP_PAKFILE = 57 + FACES_HDR = 58 + MAP_FLAGS = 59 + OVERLAY_FADES = 60 + LUMP_OVERLAY_SYSTEM_LEVELS = 61 # overlay CPU & GPU limits + UNUSED_62 = 62 + UNUSED_63 = 63 + +# Known lump changes from Orange Box -> Left 4 Dead: +# New: +# UNUSED_61 -> LUMP_OVERLAY_SYSTEM_LEVELS + + +lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} +Left4Dead2LumpHeader = collections.namedtuple("Left4DeadLumpHeader", ["length", "offset", "version", "fourCC"]) +# length and offset are swapped for L4D2 + + +def read_lump_header(file, LUMP: enum.Enum) -> source.SourceLumpHeader: + file.seek(lump_header_address[LUMP]) + offset, length, version, fourCC = struct.unpack("4I", file.read(16)) + header = source.SourceLumpHeader(offset, length, version, fourCC) + return header + + +# classes for lumps, in alphabetical order: + + +# classes for special lumps, in alphabetical order: +# TODO: StaticPropv8 + + +# {"LUMP_NAME": {version: LumpClass}} +BASIC_LUMP_CLASSES = orange_box.BASIC_LUMP_CLASSES.copy() + +LUMP_CLASSES = orange_box.LUMP_CLASSES.copy() +LUMP_CLASSES.pop("WORLD_LIGHTS") +LUMP_CLASSES.pop("WORLD_LIGHTS_HDR") + +SPECIAL_LUMP_CLASSES = orange_box.SPECIAL_LUMP_CLASSES.copy() + +# {"lump": {version: SpecialLumpClass}} +GAME_LUMP_CLASSES = orange_box.GAME_LUMP_CLASSES.copy() +# TODO: GAME_LUMP_CLASSES["sprp"].update({8: lambda raw_lump: shared.GameLump_SPRP(raw_lump, StaticPropv8)}) + + +# branch exclusive methods, in alphabetical order: + + +methods = [*orange_box.methods] diff --git a/io_import_rbsp/bsp_tool/branches/valve/left4dead2.py b/io_import_rbsp/bsp_tool/branches/valve/left4dead2.py new file mode 100644 index 0000000..7ec3bdc --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/valve/left4dead2.py @@ -0,0 +1,126 @@ +# https://developer.valvesoftware.com/wiki/Left_4_Dead_(engine_branch) +# https://developer.valvesoftware.com/wiki/Source_BSP_File_Format/Game-Specific#Left_4_Dead_2_.2F_Contagion +import collections +import enum +import struct + +from .. id_software import quake +from . import left4dead + + +BSP_VERSION = 21 + +GAMES = ["Left 4 Dead 2"] + + +class LUMP(enum.Enum): + ENTITIES = 0 + PLANES = 1 + TEXTURE_DATA = 2 + VERTICES = 3 + VISIBILITY = 4 + NODES = 5 + TEXTURE_INFO = 6 + FACES = 7 + LIGHTING = 8 + OCCLUSION = 9 + LEAVES = 10 + FACE_IDS = 11 + EDGES = 12 + SURFEDGES = 13 + MODELS = 14 + WORLD_LIGHTS = 15 + LEAF_FACES = 16 + LEAF_BRUSHES = 17 + BRUSHES = 18 + BRUSH_SIDES = 19 + AREAS = 20 + AREA_PORTALS = 21 + PROP_COLLISION = 22 + PROP_HULLS = 23 + PROP_HULL_VERTS = 24 + PROP_HULL_TRIS = 25 + DISPLACEMENT_INFO = 26 + ORIGINAL_FACES = 27 + PHYSICS_DISPLACEMENT = 28 + PHYSICS_COLLIDE = 29 + VERTEX_NORMALS = 30 + VERTEX_NORMAL_INDICES = 31 + DISPLACEMENT_LIGHTMAP_ALPHAS = 32 + DISPLACEMENT_VERTICES = 33 + DISPLACEMENT_LIGHTMAP_SAMPLE_POSITIONS = 34 + GAME_LUMP = 35 + LEAF_WATER_DATA = 36 + PRIMITIVES = 37 + PRIMITIVE_VERTICES = 38 + PRIMITIVE_INDICES = 39 + PAKFILE = 40 + CLIP_PORTAL_VERTICES = 41 + CUBEMAPS = 42 + TEXTURE_DATA_STRING_DATA = 43 + TEXTURE_DATA_STRING_TABLE = 44 + OVERLAYS = 45 + LEAF_MIN_DIST_TO_WATER = 46 + FACE_MACRO_TEXTURE_INFO = 47 + DISPLACEMENT_TRIS = 48 + PROP_BLOB = 49 + WATER_OVERLAYS = 50 + LEAF_AMBIENT_INDEX_HDR = 51 + LEAF_AMBIENT_INDEX = 52 + LIGHTING_HDR = 53 + WORLD_LIGHTS_HDR = 54 + LEAF_AMBIENT_LIGHTING_HDR = 55 + LEAF_AMBIENT_LIGHTING = 56 + XZIP_PAKFILE = 57 + FACES_HDR = 58 + MAP_FLAGS = 59 + OVERLAY_FADES = 60 + LUMP_OVERLAY_SYSTEM_LEVELS = 61 # overlay CPU & GPU limits + LUMP_PHYSLEVEL = 62 + UNUSED_63 = 63 + +# Known lump changes from Left 4 Dead -> Left 4 Dead 2: +# New: +# UNUSED_22 -> PROP_COLLISION +# UNUSED_23 -> PROP_HULLS +# UNUSED_24 -> PROP_HULL_VERTS +# UNUSED_25 -> PROP_HULL_TRIS +# PHYSICS_COLLIDE_SURFACE -> PROP_BLOB +# UNUSED_62 -> LUMP_PHYSLEVEL + + +lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} +Left4Dead2LumpHeader = collections.namedtuple("Left4DeadLumpHeader", ["version", "offset", "length", "fourCC"]) + + +def read_lump_header(file, LUMP: enum.Enum) -> Left4Dead2LumpHeader: + file.seek(lump_header_address[LUMP]) + version, offset, length, fourCC = struct.unpack("4I", file.read(16)) + header = Left4Dead2LumpHeader(version, offset, length, fourCC) + return header + + +# classes for lumps, in alphabetical order: +# TODO: PropHull, PropHullTri + + +# classes for special lumps, in alphabetical order: +# TODO: PropCollision, PropBlob + + +# {"LUMP_NAME": {version: LumpClass}} +BASIC_LUMP_CLASSES = left4dead.BASIC_LUMP_CLASSES.copy() + +LUMP_CLASSES = left4dead.LUMP_CLASSES.copy() +LUMP_CLASSES.update({"PROP_HULL_VERTS": quake.Vertex}) + +SPECIAL_LUMP_CLASSES = left4dead.SPECIAL_LUMP_CLASSES.copy() + +# {"lump": {version: SpecialLumpClass}} +GAME_LUMP_CLASSES = left4dead.GAME_LUMP_CLASSES.copy() +# TODO: GAME_LUMP_CLASSES["sprp"].update({8: lambda raw_lump: shared.GameLump_SPRP(raw_lump, StaticPropv8)}) + + +# branch exclusive methods, in alphabetical order: + +methods = [*left4dead.methods] diff --git a/io_import_rbsp/bsp_tool/branches/valve/orange_box.py b/io_import_rbsp/bsp_tool/branches/valve/orange_box.py new file mode 100644 index 0000000..a50f3c0 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/valve/orange_box.py @@ -0,0 +1,203 @@ +# https://github.com/ValveSoftware/source-sdk-2013/blob/master/sp/src/public/bspfile.h +import enum +import io +import struct +from typing import List + +from .. import base +from .. import shared +from . import source + + +BSP_VERSION = 20 +# NOTE: v20 Source BSPs differ widely, since many forks are of this version + +GAMES = ["Day of Defeat: Source", + "G String", + "Garry's Mod", + "Half-Life 2: Episode 2", + "Half-Life 2 Update", + "NEOTOKYO", + "Portal", + "Team Fortress 2"] + + +class LUMP(enum.Enum): + ENTITIES = 0 + PLANES = 1 + TEXTURE_DATA = 2 + VERTICES = 3 + VISIBILITY = 4 + NODES = 5 + TEXTURE_INFO = 6 + FACES = 7 # version 1 + LIGHTING = 8 # version 1 + OCCLUSION = 9 # version 2 + LEAVES = 10 # version 1 + FACE_IDS = 11 # TF2 branch, for mapping debug & detail prop seed + EDGES = 12 + SURFEDGES = 13 + MODELS = 14 + WORLD_LIGHTS = 15 + LEAF_FACES = 16 + LEAF_BRUSHES = 17 + BRUSHES = 18 + BRUSH_SIDES = 19 + AREAS = 20 + AREA_PORTALS = 21 + UNUSED_22 = 22 + UNUSED_23 = 23 + UNUSED_24 = 24 + UNUSED_25 = 25 + DISPLACEMENT_INFO = 26 + ORIGINAL_FACES = 27 + PHYSICS_DISPLACEMENT = 28 + PHYSICS_COLLIDE = 29 + VERTEX_NORMALS = 30 + VERTEX_NORMAL_INDICES = 31 + DISPLACEMENT_LIGHTMAP_ALPHAS = 32 # deprecated / X360 ? + DISPLACEMENT_VERTICES = 33 + DISPLACEMENT_LIGHTMAP_SAMPLE_POSITIONS = 34 + GAME_LUMP = 35 + LEAF_WATER_DATA = 36 + PRIMITIVES = 37 + PRIMITIVE_VERTICES = 38 # deprecated / X360 ? + PRIMITIVE_INDICES = 39 + PAKFILE = 40 + CLIP_PORTAL_VERTICES = 41 + CUBEMAPS = 42 + TEXTURE_DATA_STRING_DATA = 43 + TEXTURE_DATA_STRING_TABLE = 44 + OVERLAYS = 45 + LEAF_MIN_DIST_TO_WATER = 46 + FACE_MACRO_TEXTURE_INFO = 47 + DISPLACEMENT_TRIS = 48 + PHYSICS_COLLIDE_SURFACE = 49 # deprecated / X360 ? + WATER_OVERLAYS = 50 # deprecated / X360 ? + LEAF_AMBIENT_INDEX_HDR = 51 + LEAF_AMBIENT_INDEX = 52 + LIGHTING_HDR = 53 # version 1 + WORLD_LIGHTS_HDR = 54 + LEAF_AMBIENT_LIGHTING_HDR = 55 # version 1 + LEAF_AMBIENT_LIGHTING = 56 # version 1 + XZIP_PAKFILE = 57 # deprecated / X360 ? + FACES_HDR = 58 # version 1 + MAP_FLAGS = 59 + OVERLAY_FADES = 60 + UNUSED_61 = 61 + UNUSED_62 = 62 + UNUSED_63 = 63 + + +lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} + + +def read_lump_header(file, LUMP: enum.Enum) -> source.SourceLumpHeader: + file.seek(lump_header_address[LUMP]) + offset, length, version, fourCC = struct.unpack("4I", file.read(16)) + header = source.SourceLumpHeader(offset, length, version, fourCC) + return header + +# a rough map of the relationships between lumps: +# Node -> Face -> Plane +# |-> DisplacementInfo -> DisplacementVertex +# |-> SurfEdge -> Edge -> Vertex +# +# PRIMITIVES or "water indices" are a leftover from Quake. +# In the Source Engine they are used to correct for "t-junctions". +# "t-junctions" are a type of innacuracy which arises in BSP construction. +# In brush-based .bsp, Constructive Solid Geometry (CSG) operations occur. +# CSG "slices" & can potentially merges brushes, this also helps define visleaves +# (CSG operations are the same as the Boolen Modifier in Blender). +# These "slices" must be applied to brush faces, +# which are stored as a clockwise series of 3D points. +# Some slices create erroneous edges, especially where func_detail meets world. +# The PRIMITIVES lump forces a specific shape to compensate for these errors. +# +# ClipPortalVertices are AreaPortal geometry + + +# classes for each lump, in alphabetical order: +class Leaf(base.Struct): # LUMP 10 + """Endpoint of a vis tree branch, a pocket of Faces""" + contents: int # contents bitflags + cluster: int # index of this Leaf's cluster (parent node?) (visibility?) + area_flags: int # area + flags (short area:9; short flags:7;) + # area and flags are held in the same float + # area = leaf[2] & 0xFF80 >> 7 # 9 bits + # flags = leaf[2] & 0x007F # 7 bits + # TODO: automatically split area & flags, merging back for flat() + # why was this done when the struct is padded by one short anyway? + mins: List[float] # bounding box minimums along XYZ axes + maxs: List[float] # bounding box maximums along XYZ axes + first_leaf_face: int # index of first LeafFace + num_leaf_faces: int # number of LeafFaces + first_leaf_brush: int # index of first LeafBrush + num_leaf_brushes: int # number of LeafBrushes + leaf_water_data_id: int # -1 if this leaf isn't submerged + padding: int # should be empty + __slots__ = ["contents", "cluster", "area_flags", "mins", "maxs", + "first_leaf_face", "num_leaf_faces", "first_leaf_brush", + "num_leaf_brushes", "leaf_water_data_id", "padding"] + _format = "i8h4H2h" + _arrays = {"mins": [*"xyz"], "maxs": [*"xyz"]} + + +# classes for special lumps, in alphabetical order: +class PhysicsDisplacement(list): # LUMP 28 + def __init__(self, raw_lump: bytes): + lump = io.BytesIO(raw_lump) + count = int.from_bytes(lump.read(2), "little") + data_sizes = list(*struct.unpack(f"{count}H", lump.read(count * 2))) + physics_data = list() + for size in data_sizes: + physics_data.append(lump.read(size)) + super().__init__(physics_data) + + def as_bytes(self) -> bytes: + count = len(self).to_bytes(2, "little") + sizes = map(lambda s: s.to_bytes(2, "little"), [len(d) for d in self]) + return b"".join(count, *sizes, *self) + + +class StaticPropv10(base.Struct): # sprp GAME LUMP (LUMP 35) + origin: List[float] # origin.xyz + angles: List[float] # origin.yzx QAngle; Z0 = East + name_index: int # index into AME_LUMP.sprp.model_names + first_leaf: int # index into Leaf lump + num_leafs: int # number of Leafs after first_leaf this StaticPropv10 is in + solid_mode: int # collision flags enum + skin: int # index of this StaticProp's skin in the .mdl + fade_distance: List[float] # min & max distances to fade out + lighting_origin: List[float] # xyz position to sample lighting from + forced_fade_scale: float # relative to pixels used to render on-screen? + dx_level: List[int] # supported directX level, will not render depending on settings + flags: int # other flags + lightmap: List[int] # dimensions of this StaticProp's lightmap (GAME_LUMP.static prop lighting?) + __slots__ = ["origin", "angles", "name_index", "first_leaf", "num_leafs", + "solid_mode", "skin", "fade_distance", "lighting_origin", + "forced_fade_scale", "dx_level", "flags", "lightmap"] + _format = "6f3HBi6f2Hi2H" + _arrays = {"origin": [*"xyz"], "angles": [*"yzx"], "fade_distance": ["min", "max"], + "lighting_origin": [*"xyz"], "dx_level": ["min", "max"], + "lightmap": ["width", "height"]} + + +# {"LUMP_NAME": {version: LumpClass}} +BASIC_LUMP_CLASSES = source.BASIC_LUMP_CLASSES.copy() + +LUMP_CLASSES = source.LUMP_CLASSES.copy() +LUMP_CLASSES.update({"LEAVES": {1: Leaf}}) + +SPECIAL_LUMP_CLASSES = source.SPECIAL_LUMP_CLASSES.copy() + +# {"lump": {version: SpecialLumpClass}} +GAME_LUMP_CLASSES = source.GAME_LUMP_CLASSES.copy() +GAME_LUMP_CLASSES["sprp"].update({7: lambda raw_lump: shared.GameLump_SPRP(raw_lump, StaticPropv10), # 7* + 10: lambda raw_lump: shared.GameLump_SPRP(raw_lump, StaticPropv10)}) + + +# branch exclusive methods, in alphabetical order: + + +methods = [*source.methods] diff --git a/io_import_rbsp/bsp_tool/branches/valve/sdk_2013.py b/io_import_rbsp/bsp_tool/branches/valve/sdk_2013.py new file mode 100644 index 0000000..a512912 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/valve/sdk_2013.py @@ -0,0 +1,39 @@ +# https://github.com/ValveSoftware/source-sdk-2013/ +import enum +import struct + +from . import orange_box +from . import source + + +BSP_VERSION = 21 + +GAMES = ["Counter-Strike: Global Offensive", "Blade Symphony", "Portal 2", + "Source Filmmaker"] +GAME_VERSIONS = {game: BSP_VERSION for game in GAMES} + +LUMP = orange_box.LUMP +lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} + + +def read_lump_header(file, LUMP: enum.Enum) -> source.SourceLumpHeader: + file.seek(lump_header_address[LUMP]) + offset, length, version, fourCC = struct.unpack("4I", file.read(16)) + header = source.SourceLumpHeader(offset, length, version, fourCC) + return header + + +# classes for each lump, in alphabetical order: + +# {"LUMP_NAME": {version: LumpClass}} +BASIC_LUMP_CLASSES = orange_box.BASIC_LUMP_CLASSES.copy() + +LUMP_CLASSES = orange_box.LUMP_CLASSES.copy() +LUMP_CLASSES.pop("WORLD_LIGHTS") +LUMP_CLASSES.pop("WORLD_LIGHTS_HDR") + +SPECIAL_LUMP_CLASSES = orange_box.SPECIAL_LUMP_CLASSES.copy() + +GAME_LUMP_CLASSES = orange_box.GAME_LUMP_CLASSES.copy() + +methods = [*orange_box.methods] diff --git a/io_import_rbsp/bsp_tool/branches/valve/source.py b/io_import_rbsp/bsp_tool/branches/valve/source.py new file mode 100644 index 0000000..5730b6c --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/valve/source.py @@ -0,0 +1,799 @@ +import collections +import enum +import struct +from typing import List + +from .. import base +from .. import shared +from .. import vector +from ..id_software import quake + + +BSP_VERSION = 19 # & 20 + +GAMES = ["Counter-Strike: Source", # counter-strike source/cstrike + "Half-Life 1: Source - Deathmatch", # Half-Life 1 Source Deathmatch/hl1mp + "Half-Life 2", # Half-Life 2/hl2 + "Half-Life 2: Episode 1"] # Half-Life 2/episodic +GAME_VERSIONS = {game: BSP_VERSION for game in GAMES} + + +class LUMP(enum.Enum): + ENTITIES = 0 + PLANES = 1 + TEXTURE_DATA = 2 + VERTICES = 3 + VISIBILITY = 4 + NODES = 5 + TEXTURE_INFO = 6 + FACES = 7 # version 1 + LIGHTING = 8 # version 1 + OCCLUSION = 9 # version 2 + LEAVES = 10 # version 1 + FACE_IDS = 11 + EDGES = 12 + SURFEDGES = 13 + MODELS = 14 + WORLD_LIGHTS = 15 + LEAF_FACES = 16 + LEAF_BRUSHES = 17 + BRUSHES = 18 + BRUSH_SIDES = 19 + AREAS = 20 + AREA_PORTALS = 21 + PORTALS = 22 + CLUSTERS = 23 + PORTAL_VERTICES = 24 + CLUSTER_PORTALS = 25 + DISPLACEMENT_INFO = 26 + ORIGINAL_FACES = 27 + PHYSICS_DISPLACEMENT = 28 + PHYSICS_COLLIDE = 29 + VERTEX_NORMALS = 30 + VERTEX_NORMAL_INDICES = 31 + DISPLACEMENT_LIGHTMAP_ALPHAS = 32 # deprecated / X360 ? + DISPLACEMENT_VERTICES = 33 + DISPLACEMENT_LIGHTMAP_SAMPLE_POSITIONS = 34 + GAME_LUMP = 35 + LEAF_WATER_DATA = 36 + PRIMITIVES = 37 + PRIMITIVE_VERTICES = 38 # deprecated / X360 ? + PRIMITIVE_INDICES = 39 + PAKFILE = 40 + CLIP_PORTAL_VERTICES = 41 + CUBEMAPS = 42 + TEXTURE_DATA_STRING_DATA = 43 + TEXTURE_DATA_STRING_TABLE = 44 + OVERLAYS = 45 + LEAF_MIN_DIST_TO_WATER = 46 + FACE_MACRO_TEXTURE_INFO = 47 + DISPLACEMENT_TRIS = 48 + PHYSICS_COLLIDE_SURFACE = 49 # deprecated / X360 ? + WATER_OVERLAYS = 50 # deprecated / X360 ? + LIGHTMAP_PAGES = 51 + LIGHTMAP_PAGE_INFOS = 52 + LIGHTING_HDR = 53 # version 1 + WORLD_LIGHTS_HDR = 54 + LEAF_AMBIENT_LIGHTING_HDR = 55 # version 1 + LEAF_AMBIENT_LIGHTING = 56 # version 1 + XZIP_PAKFILE = 57 # deprecated / X360 ? + FACES_HDR = 58 # version 1 + MAP_FLAGS = 59 + OVERLAY_FADES = 60 + UNUSED_61 = 61 + UNUSED_62 = 62 + UNUSED_63 = 63 + + +lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} +SourceLumpHeader = collections.namedtuple("SourceLumpHeader", ["offset", "length", "version", "fourCC"]) + + +def read_lump_header(file, LUMP: enum.Enum) -> SourceLumpHeader: + file.seek(lump_header_address[LUMP]) + offset, length, version, fourCC = struct.unpack("4I", file.read(16)) + header = SourceLumpHeader(offset, length, version, fourCC) + return header + +# TODO: changes from GoldSrc -> Source +# MipTexture.flags -> TextureInfo.flags (Surface enum) + +# a rough map of the relationships between lumps: +# Node -> Face -> Plane +# |-> DisplacementInfo -> DisplacementVertex +# |-> SurfEdge -> Edge -> Vertex +# +# PRIMITIVES or "water indices" are a leftover from Quake. +# In the Source Engine they are used to correct for "t-junctions". +# "t-junctions" are a type of innacuracy which arises in BSP construction. +# In brush-based .bsp, Constructive Solid Geometry (CSG) operations occur. +# CSG "slices" & can potentially merges brushes, this also helps define visleaves +# (CSG operations are the same as the Boolen Modifier in Blender). +# These "slices" must be applied to brush faces, +# which are stored as a clockwise series of 3D points. +# Some slices create erroneous edges, especially where func_detail meets world. +# The PRIMITIVES lump forces a specific shape to compensate for these errors. +# +# ClipPortalVertices are AreaPortal geometry + + +# engine limits: (2013 SDK bspfile.h) +class MIN(enum.Enum): + DISPLACEMENT_POWER = 2 + + +class MAX(enum.Enum): + # misc: + CUBEMAP_SAMPLES = 1024 + DISPLACEMENT_POWER = 4 + DISPLACEMENT_CORNER_NEIGHBORS = 4 + ENTITY_KEY, ENTITY_VALUE = 32, 1024 # key value pair sizes + LIGHTMAPS = 4 # ? left over from Quake / GoldSrc lightmap format ? + LIGHTMAP_DIMENSION_WITH_BORDER_BRUSH = 35 # "vbsp cut limit" +1 (to account for rounding errors) + LIGHTMAP_DIMENSION_WITHOUT_BORDER_BRUSH = 32 + LIGHTMAP_DIMENSION_WITH_BORDER_DISPLACEMENT = 128 + LIGHTMAP_DIMENSION_WITHOUT_BORDER_DISPLACEMENT = 125 + # absolute maximum, based on previous values + LIGHTMAP_DIMENSION_WITH_BORDER = LIGHTMAP_DIMENSION_WITH_BORDER_DISPLACEMENT + LIGHTMAP_DIMENSION_WITHOUT_BORDER = LIGHTMAP_DIMENSION_WITHOUT_BORDER_DISPLACEMENT + LIGHTING_STYLES = 64 + PORTAL_VERTICES = 128000 + # lumps: + ENTITIES = 8192 + PLANES = 65536 + TEXTURE_DATA = 2048 + VERTICES = 65536 + VISIBILITY_CLUSTERS = 65536 + VISIBILITY_SIZE = 0x1000000 # "increased in BSPVERSION 7" + NODES = 65536 + TEXTURE_INFO = 12288 + FACES = 65536 + LIGHTING_SIZE = 0x1000000 + LEAVES = 65536 + EDGES = 256000 + SURFEDGES = 512000 + MODELS = 1024 + WORLD_LIGHTS = 8192 + LEAF_FACES = 65536 + LEAF_BRUSHES = 65536 + BRUSHES = 8192 + BRUSH_SIDES = 65536 + AREAS = 256 + AREA_BYTES = AREAS // 8 + AREA_PORTALS = 1024 + # UNUSED_24 [PORTALVERTS] = 128000 + DISPLACEMENT_INFO = 2048 + ORIGINAL_FACES = FACES + VERTEX_NORMALS = 256000 + VERTEX_NORMAL_INDICES = 256000 + DISPLACEMENT_VERTICES_FOR_ONE = (2 ** DISPLACEMENT_POWER + 1) ** 2 + DISPLACEMENT_VERTICES = DISPLACEMENT_INFO * DISPLACEMENT_VERTICES_FOR_ONE + LEAF_WATER_DATA = 32768 + PRIMITIVES = 32768 + PRIMITIVE_VERTICES = 65536 + PRIMITIVE_INDICES = 65536 + TEXDATA_STRING_DATA = 256000 + TEXDATA_STRING_TABLE = 65536 + OVERLAYS = 512 + DISPLACEMENT_TRIANGLES_FOR_ONE = 2 ** DISPLACEMENT_POWER * 3 + DISPLACEMENT_TRIANGLES = DISPLACEMENT_INFO * DISPLACEMENT_TRIANGLES_FOR_ONE + WATER_OVERLAYS = 16384 + LIGHTING_HDR_SIZE = LIGHTING_SIZE + WORLD_LIGHTS_HDR = WORLD_LIGHTS + FACES_HDR = FACES + + +class MAX_X360(enum.Enum): # "force static arrays to be very small" + """#if !defined( BSP_USE_LESS_MEMORY )""" + ENTITIES = 2 + PLANES = 2 + TEXTURE_DATA = 2 + VERTICES = 2 + VISIBILITY_CLUSTERS = 2 + NODES = 2 + TEXTURE_INFO = 2 + FACES = 2 + LIGHTING_SIZE = 2 + LEAVES = 2 + EDGES = 2 + SURFEDGES = 2 + MODELS = 2 + WORLD_LIGHTS = 2 + LEAF_FACES = 2 + LEAF_BRUSHES = 2 + BRUSHES = 2 + BRUSH_SIDES = 2 + AREAS = 2 + AREA_BYTES = 2 + AREA_PORTALS = 2 + # UNUSED_24 [PORTALVERTS] = 2 + DISPLACEMENT_INFO = 2 + ORIGINAL_FACES = FACES + VERTEX_NORMALS = 2 + VERTEX_NORMAL_INDICES = 2 + DISPLACEMENT_VERTICES_FOR_ONE = (2 ** MAX.DISPLACEMENT_POWER.value + 1) ** 2 + DISPLACEMENT_VERTICES = DISPLACEMENT_INFO * DISPLACEMENT_VERTICES_FOR_ONE + LEAF_WATER_DATA = 2 + PRIMITIVES = 2 + PRIMITIVE_VERTICES = 2 + PRIMITIVE_INDICES = 2 + TEXDATA_STRING_DATA = 2 + TEXDATA_STRING_TABLE = 2 + OVERLAYS = 2 + DISPLACEMENT_TRIANGLES_FOR_ONE = 2 ** MAX.DISPLACEMENT_POWER.value * 3 + DISPLACEMENT_TRIANGLES = DISPLACEMENT_INFO * DISPLACEMENT_TRIANGLES_FOR_ONE + WATER_OVERLAYS = 2 + LIGHTING_HDR_SIZE = LIGHTING_SIZE + WORLD_LIGHTS_HDR = WORLD_LIGHTS + FACES_HDR = FACES + + +# flag enums +class Contents(enum.IntFlag): # src/public/bspflags.h + """Brush flags""" # NOTE: vbsp sets these in src/utils/vbsp/textures.cpp + # visible + EMPTY = 0x00 + SOLID = 0x01 + WINDOW = 0x02 + AUX = 0x04 # unused? + GRATE = 0x08 # allows bullets & vis + SLIME = 0x10 + WATER = 0x20 + MIST = 0x40 # BLOCK_LOS, blocks AI line of sight + OPAQUE = 0x80 # blocks NPC line of sight, may be non-solid + TEST_FOG_VOLUME = 0x100 # cannot be seen through, but may be non-solid + UNUSED_1 = 0x200 + UNUSED_2 = 0x400 # titanfall vertex lump flags? + TEAM1 = 0x0800 + TEAM2 = 0x1000 + IGNORE_NO_DRAW_OPAQUE = 0x2000 # ignore opaque if Surface.NO_DRAW + MOVEABLE = 0x4000 # pushables + # not visible + AREAPORTAL = 0x8000 + PLAYER_CLIP = 0x10000 + MONSTER_CLIP = 0x20000 + # CURRENT_ flags are for moving water + CURRENT_0 = 0x40000 + CURRENT_90 = 0x80000 + CURRENT_180 = 0x100000 + CURRENT_270 = 0x200000 + CURRENT_UP = 0x400000 + CURRENT_DOWN = 0x800000 + ORIGIN = 0x1000000 # "removed before bsping an entity" + MONSTER = 0x2000000 # in-game only, shouldn't be in a .bsp + DEBRIS = 0x4000000 + DETAIL = 0x8000000 # func_detail; for VVIS (visleaf assembly from Brushes) + TRANSLUCENT = 0x10000000 + LADDER = 0x20000000 + HITBOX = 0x40000000 # requests hit tracing use hitboxes + + +class DisplacementFlags(enum.IntFlag): + """DisplacementInfo collision flags""" + UNUSED = 1 + NO_PHYS = 2 + NO_HULL = 4 + NO_RAY = 8 + + +class DispTris(enum.IntFlag): + """DisplacementTriangle flags""" + SURFACE = 0x01 + WALKABLE = 0x02 + BUILDABLE = 0x04 + SURFPROP1 = 0x08 # ? + SURFPROP2 = 0x10 # ? + + +class Surface(enum.IntFlag): # src/public/bspflags.h + """TextureInfo flags""" # NOTE: vbsp sets these in src/utils/vbsp/textures.cpp + LIGHT = 0x0001 # "value will hold the light strength" + SKY_2D = 0x0002 # don't draw, indicates we should skylight + draw 2d sky but not draw the 3D skybox + SKY = 0x0004 # don't draw, but add to skybox + WARP = 0x0008 # turbulent water warp + TRANSLUCENT = 0x0010 + NO_PORTAL = 0x0020 # the surface can not have a portal placed on it + TRIGGER = 0x0040 # xbox hack to work around elimination of trigger surfaces, which breaks occluders + NO_DRAW = 0x0080 + HINT = 0x0100 # make a bsp split on this face + SKIP = 0x0200 # don't split on this face, allows for non-closed brushes + NO_LIGHT = 0x0400 # don't calculate light + BUMPLIGHT = 0x0800 # calculate three lightmaps for the surface for bumpmapping (ssbump?) + NO_SHADOWS = 0x1000 + NO_DECALS = 0x2000 + NO_CHOP = 0x4000 # don't subdivide patches on this surface + HITBOX = 0x8000 # surface is part of a hitbox + + +# classes for each lump, in alphabetical order: +class Area(base.MappedArray): # LUMP 20 + num_area_portals: int # number of AreaPortals after first_area_portal in this Area + first_area_portal: int # index of first AreaPortal + _mapping = ["num_area_portals", "first_area_portal"] + _format = "2i" + + +class AreaPortal(base.MappedArray): # LUMP 21 + # public/bspfile.h dareaportal_t & utils/vbsp/portals.cpp EmitAreaPortals + portal_key: int # for tying to entities + first_clip_portal_vert: int # index into the ClipPortalVertex lump + num_clip_portal_vertices: int # number of ClipPortalVertices after first_clip_portal_vertex in this AreaPortal + plane: int # index of into the Plane lump + _mapping = ["portal_key", "other_area", "first_clip_portal_vertex", + "num_clip_portal_vertices", "plane"] + _format = "4Hi" + + +class Brush(base.Struct): # LUMP 18 + """Assumed to carry over from .vmf""" + first_side: int # index into BrushSide lump + num_sides: int # number of BrushSides after first_side in this Brush + contents: int # contents bitflags + __slots__ = ["first_side", "num_sides", "contents"] + _format = "3i" + + +class BrushSide(base.Struct): # LUMP 19 + plane: int # index into Plane lump + texture_info: int # index into TextureInfo lump + displacement_info: int # index into DisplacementInfo lump + bevel: int # smoothing group? + __slots__ = ["plane", "texture_info", "displacement_info", "bevel"] + _format = "H3h" + + +class Cubemap(base.Struct): # LUMP 42 + """Location (origin) & resolution (size)""" + origin: List[float] # origin.xyz + size: int # texture dimension (each face of a cubemap is square) + __slots__ = ["origin", "size"] + _format = "4i" + _arrays = {"origin": [*"xyz"]} + + +class DisplacementInfo(base.Struct): # LUMP 26 + """Holds the information defining a displacement""" + start_position: List[float] # rough XYZ of the vertex to orient around + displacement_vert_start: int # index of first DisplacementVertex + displacement_tri_start: int # index of first DisplacementTriangle + # ^ length of sequence for each varies depending on power + power: int # level of subdivision + flags: int # see DisplacementFlags + min_tesselation: int # for tesselation shaders / triangle assembley? + smoothing_angle: float # ? + contents: int # contents bitflags + map_face: int # index of Face? + __slots__ = ["start_position", "displacement_vert_start", "displacement_tri_start", + "power", "flags", "min_tesselation", "smoothing_angle", "contents", + "map_face", "lightmap_alpha_start", "lightmap_sample_position_start", + "edge_neighbours", "corner_neighbours", "allowed_vertices"] + _format = "3f3iHhfiH2i88c10i" + _arrays = {"start_position": [*"xyz"], "edge_neighbours": 44, + "corner_neighbours": 44, "allowed_vertices": 10} + # TODO: map neighbours with base.Struct subclasses, rather than MappedArrays + # both the __init__ & flat methods may need some changes to accommodate this + + # def __init__(self, _tuple): + # super(base.Struct, self).__init__(_tuple) + # self.edge_neighbours = ... + # self.corner_neighbours = ... + + +class DisplacementVertex(base.Struct): # LUMP 33 + """The positional deformation & blend value of a point in a displacement""" + vector: List[float] # direction of vertex offset from barymetric base + distance: float # length to scale deformation vector by + alpha: float # [0-1] material blend factor + __slots__ = ["vector", "distance", "alpha"] + _format = "5f" + _arrays = {"vector": [*"xyz"]} + + +class Face(base.Struct): # LUMP 7 + """makes up Models (including worldspawn), also referenced by LeafFaces""" + plane: int # index into Plane lump + side: int # "faces opposite to the node's plane direction" + on_node: bool # if False, face is in a leaf + first_edge: int # index into the SurfEdge lump + num_edges: int # number of SurfEdges after first_edge in this Face + texture_info: int # index into the TextureInfo lump + displacement_info: int # index into the DisplacementInfo lump (None if -1) + surface_fog_volume_id: int # t-junctions? QuakeIII vertex-lit fog? + styles: int # 4 different lighting states? "switchable lighting info" + light_offset: int # index of first pixel in LIGHTING / LIGHTING_HDR + area: float # surface area of this face + lightmap: List[float] + # lightmap.mins # dimensions of lightmap segment + # lightmap.size # scalars for lightmap segment + original_face: int # ORIGINAL_FACES index, -1 if this is an original face + num_primitives: int # non-zero if t-juncts are present? number of Primitives + first_primitive_id: int # index of Primitive + smoothing_groups: int # lightmap smoothing group + __slots__ = ["plane", "side", "on_node", "first_edge", "num_edges", + "texture_info", "displacement_info", "surface_fog_volume_id", "styles", + "light_offset", "area", "lightmap", "original_face", + "num_primitives", "first_primitive_id", "smoothing_groups"] + _format = "Hb?i4h4bif5i2HI" + _arrays = {"styles": 4, "lightmap": {"mins": [*"xy"], "size": ["width", "height"]}} + + +class LeafWaterData(base.Struct): + surface_z: float # global Z height of the water's surface + min_z: float # bottom of the water volume? + texture_data: int # index to this LeafWaterData's TextureData + _format = "2fI" + _mapping = ["surface_z", "min_z", "texture_data"] + + +class Model(base.Struct): # LUMP 14 + """Brush based entities; Index 0 is worldspawn""" + mins: List[float] # bounding box minimums along XYZ axes + maxs: List[float] # bounding box maximums along XYZ axes + origin: List[float] # center of model, worldspawn is always at 0 0 0 + head_node: int # index into Node lump + first_face: int # index into Face lump + num_faces: int # number of Faces after first_face in this Model + __slots__ = ["mins", "maxs", "origin", "head_node", "first_face", "num_faces"] + _format = "9f3i" + _arrays = {"mins": [*"xyz"], "maxs": [*"xyz"], "origin": [*"xyz"]} + + +class Node(base.Struct): # LUMP 5 + plane: int # index into Plane lump + children: List[int] # 2 indices; Node if positive, Leaf if negative + mins: List[float] # bounding box minimums along XYZ axes + maxs: List[float] # bounding box maximums along XYZ axes + first_face: int # index into Face lump + num_faces: int # number of Faces after first_face in this Node + area: int # index into Area lump, if all children are in the same area, else -1 + padding: int # should be empty + __slots__ = ["plane", "children", "mins", "maxs", "first_face", "num_faces", + "area", "padding"] + # area is appears to always be 0 + # however leaves correctly connect to all areas + _format = "3i6h2H2h" + _arrays = {"children": 2, "mins": [*"xyz"], "maxs": [*"xyz"]} + + +class OverlayFade(base.MappedArray): # LUMP 60 + """Holds fade distances for the overlay of the same index""" + _mapping = ["min", "max"] + _format = "2f" + + +class Plane(base.Struct): # LUMP 1 + """3D Plane defining shape, used for physics & BSP/CSG calculations?""" + normal: List[float] + distance: float + type: int # flags for axis alignment, appears to be unused + __slots__ = ["normal", "distance", "type"] + _format = "4fi" + _arrays = {"normal": [*"xyz"]} + + +class TextureData(base.Struct): # LUMP 2 + """Data on this view of a texture (.vmt), indexed by TextureInfo""" + reflectivity: List[float] + name_index: int # index of texture name in TEXTURE_DATA_STRING_TABLE / TABLE + size: List[int] # dimensions of full texture + view: List[int] # dimensions of visible section of texture + __slots__ = ["reflectivity", "name_index", "size", "view"] + _format = "3f5i" + _arrays = {"reflectivity": [*"rgb"], "size": ["width", "height"], "view": ["width", "height"]} + + +class TextureInfo(base.Struct): # LUMP 6 + """Texture projection info & index into TEXTURE_DATA""" + texture: List[List[float]] # 2 texture projection vectors + lightmap: List[List[float]] # 2 lightmap projection vectors + flags: int # Surface flags + texture_data: int # index of TextureData + __slots__ = ["texture", "lightmap", "flags", "texture_data"] + _format = "16f2i" + _arrays = {"texture": {"s": [*"xyz", "offset"], "t": [*"xyz", "offset"]}, + "lightmap": {"s": [*"xyz", "offset"], "t": [*"xyz", "offset"]}} + # ^ nested MappedArrays; texture.s.x, texture.t.x + + +class WorldLight(base.Struct): # LUMP 15 + """A static light""" + origin: List[float] # origin point of this light source + intensity: float # light strength scalar + normal: List[float] # light direction + cluster: int # ? + type: int # some enum? + style: int # related to face styles? + # see base.fgd: + stop_dot: float # ? + stop_dot2: float # ? + exponent: float # falloff? + radius: float + # attenuations: + constant: float + linear: float + quadratic: float + # ^ these factor into some equation... + flags: int # bitflags? + texture_info: int # index of TextureInfo + owner: int # parent entity ID? + __slots__ = ["origin", "intensity", "normal", "cluster", "type", "style", + "stop_dot", "stop_dot2", "exponent", "radius", + "constant", "linear", "quadratic", # attenuation + "flags", "texture_info", "owner"] + _format = "9f3i7f3i" + _arrays = {"origin": [*"xyz"], "intensity": [*"xyz"], "normal": [*"xyz"]} + + +# classes for special lumps, in alphabetical order: +class StaticPropv4(base.Struct): # sprp GAME LUMP (LUMP 35) + """https://github.com/ValveSoftware/source-sdk-2013/blob/master/sp/src/public/gamebspfile.h#L151""" + origin: List[float] # origin.xyz + angles: List[float] # origin.yzx QAngle; Z0 = East + name_index: int # index into AME_LUMP.sprp.model_names + first_leaf: int # index into Leaf lump + num_leafs: int # number of Leafs after first_leaf this StaticPropv10 is in + solid_mode: int # collision flags enum + flags: int # other flags + skin: int # index of this StaticProp's skin in the .mdl + fade_distance: List[float] # min & max distances to fade out + lighting_origin: List[float] # xyz position to sample lighting from + __slots__ = ["origin", "angles", "name_index", "first_leaf", "num_leafs", + "solid_mode", "flags", "skin", "fade_distance", "lighting_origin"] + _format = "6f3H2Bi5f" + _arrays = {"origin": [*"xyz"], "angles": [*"yzx"], "fade_distance": ["min", "max"], + "lighting_origin": [*"xyz"]} + + +class StaticPropv5(base.Struct): # sprp GAME LUMP (LUMP 35) + """https://github.com/ValveSoftware/source-sdk-2013/blob/master/sp/src/public/gamebspfile.h#L168""" + origin: List[float] # origin.xyz + angles: List[float] # origin.yzx QAngle; Z0 = East + name_index: int # index into AME_LUMP.sprp.model_names + first_leaf: int # index into Leaf lump + num_leafs: int # number of Leafs after first_leaf this StaticPropv10 is in + solid_mode: int # collision flags enum + flags: int # other flags + skin: int # index of this StaticProp's skin in the .mdl + fade_distance: List[float] # min & max distances to fade out + lighting_origin: List[float] # xyz position to sample lighting from + forced_fade_scale: float # relative to pixels used to render on-screen? + __slots__ = ["origin", "angles", "name_index", "first_leaf", "num_leafs", + "solid_mode", "skin", "fade_distance", "lighting_origin", + "forced_fade_scale"] + _format = "6f3HBi6f2Hi2H" + _arrays = {"origin": [*"xyz"], "angles": [*"yzx"], "fade_distance": ["min", "max"], + "lighting_origin": [*"xyz"]} + + +class StaticPropv6(base.Struct): # sprp GAME LUMP (LUMP 35) + origin: List[float] # origin.xyz + angles: List[float] # origin.yzx QAngle; Z0 = East + name_index: int # index into AME_LUMP.sprp.model_names + first_leaf: int # index into Leaf lump + num_leafs: int # number of Leafs after first_leaf this StaticPropv10 is in + solid_mode: int # collision flags enum + skin: int # index of this StaticProp's skin in the .mdl + fade_distance: List[float] # min & max distances to fade out + lighting_origin: List[float] # xyz position to sample lighting from + forced_fade_scale: float # relative to pixels used to render on-screen? + dx_level: List[int] # supported directX level, will not render depending on settings + __slots__ = ["origin", "angles", "name_index", "first_leaf", "num_leafs", + "solid_mode", "skin", "fade_distance", "lighting_origin", + "forced_fade_scale", "dx_level"] + _format = "6f3HBi6f2Hi2H" + _arrays = {"origin": [*"xyz"], "angles": [*"yzx"], "fade_distance": ["min", "max"], + "lighting_origin": [*"xyz"], "dx_level": ["min", "max"]} + + +# {"LUMP_NAME": {version: LumpClass}} +BASIC_LUMP_CLASSES = {"DISPLACEMENT_TRIS": {0: shared.UnsignedShorts}, + "LEAF_FACES": {0: shared.UnsignedShorts}, + "SURFEDGES": {0: shared.Ints}, + "TEXTURE_DATA_STRING_TABLE": {0: shared.UnsignedShorts}} + +LUMP_CLASSES = {"AREAS": {0: Area}, + "AREA_PORTALS": {0: AreaPortal}, + "BRUSHES": {0: Brush}, + "BRUSH_SIDES": {0: BrushSide}, + "CUBEMAPS": {0: Cubemap}, + "DISPLACEMENT_INFO": {0: DisplacementInfo}, + "DISPLACEMENT_VERTICES": {0: DisplacementVertex}, + "EDGES": {0: quake.Edge}, + "FACES": {1: Face}, + "LEAF_WATER_DATA": {0: LeafWaterData}, + "MODELS": {0: Model}, + "NODES": {0: Node}, + "OVERLAY_FADES": {0: OverlayFade}, + "ORIGINAL_FACES": {0: Face}, + "PLANES": {0: Plane}, + "TEXTURE_DATA": {0: TextureData}, + "TEXTURE_INFO": {0: TextureInfo}, + "VERTICES": {0: quake.Vertex}, + "VERTEX_NORMALS": {0: quake.Vertex}, + "WORLD_LIGHTS": {0: WorldLight}, + "WORLD_LIGHTS_HDR": {0: WorldLight}} + +SPECIAL_LUMP_CLASSES = {"ENTITIES": {0: shared.Entities}, + "TEXTURE_DATA_STRING_DATA": {0: shared.TextureDataStringData}, + "PAKFILE": {0: shared.PakFile}, + # "PHYSICS_COLLIDE": {0: shared.PhysicsCollide} + } + +# {"lump": {version: SpecialLumpClass}} +GAME_LUMP_CLASSES = {"sprp": {4: lambda raw_lump: shared.GameLump_SPRP(raw_lump, StaticPropv4), + 5: lambda raw_lump: shared.GameLump_SPRP(raw_lump, StaticPropv5), + 6: lambda raw_lump: shared.GameLump_SPRP(raw_lump, StaticPropv6)}} +# NOTE: having some errors with CS:S + + +# branch exclusive methods, in alphabetical order: +def vertices_of_face(bsp, face_index: int) -> List[float]: + """Format: [Position, Normal, TexCoord, LightCoord, Colour]""" + face = bsp.FACES[face_index] + uvs, uv2s = [], [] + first_edge = face.first_edge + edges = [] + positions = [] + for surfedge in bsp.SURFEDGES[first_edge:(first_edge + face.num_edges)]: + if surfedge >= 0: # index is positive + edge = bsp.EDGES[surfedge] + positions.append(bsp.VERTICES[bsp.EDGES[surfedge][0]]) + # ^ utils/vrad/trace.cpp:637 + else: # index is negatice + edge = bsp.EDGES[-surfedge][::-1] # reverse + positions.append(bsp.VERTICES[bsp.EDGES[-surfedge][1]]) + # ^ utils/vrad/trace.cpp:635 + edges.append(edge) + positions = t_junction_fixer(bsp, face, positions, edges) + texture_info = bsp.TEXTURE_INFO[face.texture_info] + texture_data = bsp.TEXTURE_DATA[texture_info.texture_data] + texture = texture_info.texture + lightmap = texture_info.lightmap + + def vector_of(P): + """returns the normal of plane (P)""" + return (P.x, P.y, P.z) + + # texture vector -> uv calculation discovered in: + # github.com/VSES/SourceEngine2007/blob/master/src_main/engine/matsys_interface.cpp + # SurfComputeTextureCoordinate & SurfComputeLightmapCoordinate + for P in positions: + # texture UV + uv = [vector.dot(P, vector_of(texture.s)) + texture.s.offset, + vector.dot(P, vector_of(texture.t)) + texture.t.offset] + uv[0] /= texture_data.view.width if texture_data.view.width != 0 else 1 + uv[1] /= texture_data.view.height if texture_data.view.height != 0 else 1 + uvs.append(vector.vec2(*uv)) + # lightmap UV + uv2 = [vector.dot(P, vector_of(lightmap.s)) + lightmap.s.offset, + vector.dot(P, vector_of(lightmap.t)) + lightmap.t.offset] + if any([(face.lightmap.mins.x == 0), (face.lightmap.mins.y == 0)]): + uv2 = [0, 0] # invalid / no lighting + else: + uv2[0] -= face.lightmap.mins.x + uv2[1] -= face.lightmap.mins.y + uv2[0] /= face.lightmap.size.x + uv2[1] /= face.lightmap.size.y + uv2s.append(uv2) + normal = [bsp.PLANES[face.plane].normal] * len(positions) # X Y Z + colour = [texture_data.reflectivity] * len(positions) # R G B + return list(zip(positions, normal, uvs, uv2s, colour)) + + +def t_junction_fixer(bsp, face: int, positions: List[List[float]], edges: List[List[float]]) -> List[List[float]]: + # report to bsp.log rather than printing + # bsp may need a method wrapper to give a warning to check the logs + # face_index = bsp.FACES.index(face) + # first_edge = face.first_edge + if {positions.count(P) for P in positions} != {1}: + # print(f"Face #{face_index} has interesting edges (t-junction?):") + # print("\tAREA:", f"{face.area:.3f}") + # center = sum(map(vector.vec3, positions), start=vector.vec3()) / len(positions) + # print("\tCENTER:", f"({center:.3f})") + # print("\tSURFEDGES:", bsp.SURFEDGES[first_edge:first_edge + face.num_edges]) + # print("\tEDGES:", edges) + # loops = [(e[0] == edges[i-1][1]) for i, e in enumerate(edges)] + # if not all(loops): + # print("\tWARINNG! EDGES do not loop!") + # print("\tLOOPS:", loops) + # print("\tPOSITIONS:", [bsp.VERTICES.index(P) for P in positions]) + + # PATCH + # -- if you see 1 index between 2 indentical indicies: + # -- compress the 3 indices down to just the first + repeats = [i for i, P in enumerate(positions) if positions.count(P) != 1] + # if len(repeats) > 0: + # print("\tREPEATS:", repeats) + # print([bsp.VERTICES.index(P) for P in positions], "-->") + if len(repeats) == 2: + index_a, index_b = repeats + if index_b - index_a == 2: + # edge goes out to one point and doubles back; delete it + positions.pop(index_a + 1) + positions.pop(index_a + 1) + # what about Ts around the ends? + print([bsp.VERTICES.index(P) for P in positions]) + else: + if repeats[1] == repeats[0] + 1 and repeats[1] == repeats[2] - 1: + positions.pop(repeats[1]) + positions.pop(repeats[1]) + print([bsp.VERTICES.index(P) for P in positions]) + return positions + + +def displacement_indices(power: int) -> List[List[int]]: # not directly a method + """returns an array of indices ((2 ** power) + 1) ** 2 long""" + power2 = 2 ** power + power2A = power2 + 1 + power2B = power2 + 2 + power2C = power2 + 3 + tris = [] + for line in range(power2): + line_offset = power2A * line + for block in range(2 ** (power - 1)): + offset = line_offset + 2 * block + if line % 2 == 0: # |\|/| + tris.extend([offset + 0, offset + power2A, offset + 1]) + tris.extend([offset + power2A, offset + power2B, offset + 1]) + tris.extend([offset + power2B, offset + power2C, offset + 1]) + tris.extend([offset + power2C, offset + 2, offset + 1]) + else: # |/|\| + tris.extend([offset + 0, offset + power2A, offset + power2B]) + tris.extend([offset + 1, offset + 0, offset + power2B]) + tris.extend([offset + 2, offset + 1, offset + power2B]) + tris.extend([offset + power2C, offset + 2, offset + power2B]) + return tris + + +def vertices_of_displacement(bsp, face_index: int) -> List[List[float]]: + """Format: [Position, Normal, TexCoord, LightCoord, Colour]""" + face = bsp.FACES[face_index] + if face.displacement_info == -1: + raise RuntimeError(f"Face #{face_index} is not a displacement!") + base_vertices = bsp.vertices_of_face(face_index) + if len(base_vertices) != 4: + raise RuntimeError(f"Face #{face_index} does not have 4 corners (probably t-junctions)") + disp_info = bsp.DISPLACEMENT_INFO[face.displacement_info] + start = vector.vec3(*disp_info.start_position) + base_quad = [vector.vec3(*P) for P, N, uv, uv2, rgb in base_vertices] + # rotate so the point closest to start on the quad is index 0 + if start not in base_quad: + start = sorted(base_quad, key=lambda P: (start - P).magnitude())[0] + starting_index = base_quad.index(start) + + def rotated(q): + return q[starting_index:] + q[:starting_index] + + A, B, C, D = rotated(base_quad) + AD = D - A + BC = C - B + quad = rotated(base_vertices) + uvA, uvB, uvC, uvD = [vector.vec2(*uv) for P, N, uv, uv2, rgb in quad] + uvAD = uvD - uvA + uvBC = uvC - uvB + uv2A, uv2B, uv2C, uv2D = [vector.vec2(*uv2) for P, N, uv, uv2, rgb in quad] + uv2AD = uv2D - uv2A + uv2BC = uv2C - uv2B + power2 = 2 ** disp_info.power + disp_verts = bsp.DISPLACEMENT_VERTICES[disp_info.displacement_vert_start:] + disp_verts = disp_verts[:(power2 + 1) ** 2] + vertices = [] + for index, disp_vertex in enumerate(disp_verts): + t1 = index % (power2 + 1) / power2 + t2 = index // (power2 + 1) / power2 + bary_vert = vector.lerp(A + (AD * t1), B + (BC * t1), t2) + # ^ interpolates across the base_quad to find the barymetric point + displacement_vert = [x * disp_vertex.distance for x in disp_vertex.vector] + true_vertex = [a + b for a, b in zip(bary_vert, displacement_vert)] + texture_uv = vector.lerp(uvA + (uvAD * t1), uvB + (uvBC * t1), t2) + lightmap_uv = vector.lerp(uv2A + (uv2AD * t1), uv2B + (uv2BC * t1), t2) + normal = base_vertices[0][1] + colour = base_vertices[0][4] + vertices.append((true_vertex, normal, texture_uv, lightmap_uv, colour)) + return vertices + + +# TODO: vertices_of_model: walk the node tree +# TODO: vertices_of_node + +methods = [vertices_of_face, vertices_of_displacement, shared.worldspawn_volume] diff --git a/io_import_rbsp/bsp_tool/branches/vector.py b/io_import_rbsp/bsp_tool/branches/vector.py new file mode 100644 index 0000000..cb5eff0 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/vector.py @@ -0,0 +1,257 @@ +"""2D & 3D vector classes""" +from __future__ import annotations + +import itertools +import math +from typing import Iterable, Union + + +class vec2: + """2D vector class""" + __slots__ = ["x", "y"] + x: float + y: float + + def __init__(self, x: float = 0, y: float = 0): + self.x, self.y = x, y + + def __abs__(self) -> float: + return self.magnitude() + + def __add__(self, other: Iterable) -> vec2: + return vec2(*map(math.fsum, itertools.zip_longest(self, other, fillvalue=0))) + + def __eq__(self, other: Union[float, Iterable]) -> bool: + if isinstance(other, (vec2, Iterable)): + if [*self] == [*other]: + return True + elif isinstance(other, vec3): + if [*self, 0] == [*other]: + return True + elif isinstance(other, (int, float)): + if self.magnitude() == other: + return True + return False + + def __format__(self, format_spec: str = "") -> str: + return " ".join([format(i, format_spec) for i in self]) + + def __floordiv__(self, other: float) -> vec2: + return vec2(self.x // other, self.y // other) + + def __getitem__(self, key: int) -> float: + return [self.x, self.y][key] + + def __iter__(self) -> Iterable: + return iter((self.x, self.y)) + + def __len__(self) -> int: + return 2 + + def __mul__(self, other: float) -> vec2: + return vec2(self.x * other, self.y * other) + + def __neg__(self) -> vec2: + return vec2(-self.x, -self.y) + + def __repr__(self) -> str: + return str([self.x, self.y]) + + def __rmul__(self, other: float) -> vec2: + return self.__mul__(other) + + def __setitem__(self, key: Union[int, slice], value: float): + if isinstance(key, slice): + for k, v in zip(["x", "y", "z"][key], value): + self.__setattr__(k, v) + elif key == 0: + self.x = value + elif key == 1: + self.y = value + + def __sub__(self, other: Iterable) -> vec2: + return vec2(*map(math.fsum, itertools.zip_longest(self, -other, fillvalue=0))) + + def __truediv__(self, other: float) -> vec2: + return vec2(self.x / other, self.y / other) + + def magnitude(self) -> float: + """length of vector""" + return math.sqrt(self.sqrmagnitude()) + + def normalised(self) -> vec2: + """returns this vector if length was 1 (unless length is 0), does not mutate""" + m = self.sqrmagnitude() + if m != 0: + return vec2(self.x/m, self.y/m) + else: + return self + + def rotated(self, degrees: float) -> vec2: + """returns this vector rotated clockwise on Z-axis""" + theta = math.radians(degrees) + cos_theta = math.cos(theta) + sin_theta = math.sin(theta) + x = round(math.fsum([self[0] * cos_theta, self[1] * sin_theta]), 6) + y = round(math.fsum([self[1] * cos_theta, -self[0] * sin_theta]), 6) + return vec2(x, y) + + def sqrmagnitude(self) -> float: + """vec2.magnitude but without math.sqrt + handy for comparing length quickly""" + return math.fsum([i ** 2 for i in self]) + + +class vec3: + """3D vector class""" + __slots__ = ["x", "y", "z"] + + def __init__(self, x: float = 0, y: float = 0, z: float = 0): + self.x, self.y, self.z = x, y, z + + def __abs__(self) -> float: + return self.magnitude() + + def __add__(self, other: Iterable) -> vec3: + return vec3(*map(math.fsum, zip(self, other))) + + def __eq__(self, other: Union[float, Iterable]) -> bool: + if isinstance(other, vec2): + if [*self] == [*other, 0]: + return True + elif isinstance(other, (vec3, Iterable)): + if [*self] == [*other]: + return True + elif isinstance(other, (int, float)): + if self.magnitude() == other: + return True + return False + + def __format__(self, format_spec: str = "") -> str: + return " ".join([format(i, format_spec) for i in self]) + + def __floordiv__(self, other: Union[float, Iterable]) -> vec3: + return vec3(self.x // other, self.y // other, self.z // other) + + def __getitem__(self, key: int) -> float: + return [self.x, self.y, self.z][key] + + def __iter__(self) -> Iterable: + return iter((self.x, self.y, self.z)) + + def __len__(self) -> int: + return 3 + + def __mul__(self, other: Union[float, Iterable]) -> vec3: + if isinstance(other, (int, float)): + return vec3(*[i * other for i in self]) + elif isinstance(other, Iterable): + return vec3(math.fsum([self[1] * other[2], -self[2] * other[1]]), + math.fsum([self[2] * other[0], -self[0] * other[2]]), + math.fsum([self[0] * other[1], -self[1] * other[0]])) + + def __neg__(self) -> vec3: + return vec3(-self.x, -self.y, -self.z) + + def __repr__(self) -> str: + return str([self.x, self.y, self.z]) + + def __rmul__(self, other: Union[float, Iterable]) -> vec3: + return self.__mul__(other) + + def __setitem__(self, key: Union[int, slice], value: float): + if isinstance(key, slice): + for k, v in zip(["x", "y", "z"][key], value): + self.__setattr__(k, v) + elif key == 0: + self.x = value + elif key == 1: + self.y = value + elif key == 2: + self.z = value + + def __sub__(self, other: Iterable) -> vec3: + return vec3(*map(math.fsum, zip(self, -other))) + + def __truediv__(self, other: float) -> vec3: + return vec3(self.x / other, self.y / other, self.z / other) + + def magnitude(self) -> float: + """length of vector""" + return math.sqrt(self.sqrmagnitude()) + + def normalise(self) -> vec3: + """returns unit vector without mutating""" + m = self.magnitude() + if m != 0: + return vec3(self.x/m, self.y/m, self.z/m) + else: + return self + + def rotate(self, x: float = 0, y: float = 0, z: float = 0) -> vec3: + """This method can be used on any iterable, inputs are degrees rotated around axis""" + angles = [math.radians(i) for i in (x, y, z)] + cos_x, sin_x = math.cos(angles[0]), math.sin(angles[0]) + cos_y, sin_y = math.cos(angles[1]), math.sin(angles[1]) + cos_z, sin_z = math.cos(angles[2]), math.sin(angles[2]) + out = vec3(self[0], + math.fsum([self[1] * cos_x, -self[2] * sin_x]), + math.fsum([self[1] * sin_x, self[2] * cos_x])) + out = vec3(math.fsum([out.x * cos_y, out.z * sin_y]), + out.y, + math.fsum([out.z * cos_y, out.x * sin_y])) + out = vec3(math.fsum([out.x * cos_z, -out.y * sin_z]), + math.fsum([out.x * sin_z, out.y * cos_z]), + out.z) + out = vec3(*[round(i, 6) for i in out]) + return out + + def sqrmagnitude(self) -> float: + """vec3.magnitude but without math.sqrt + handy for comparing length quickly""" + return math.fsum([i ** 2 for i in self]) + + +def dot(a: Iterable, b: Iterable) -> float: + """Returns the dot product of two vectors""" + return math.fsum([i * j for i, j in itertools.zip_longest(a, b, fillvalue=0)]) + + +def lerp(a: Union[float, Iterable], b: Union[float, Iterable], t: float) -> Union[float, list]: + """Interpolates between two given points by t [0-1]""" + if isinstance(a, Iterable) and isinstance(b, Iterable): + r = [lerp(i, j, t) for i, j in itertools.zip_longest(a, b, fillvalue=0)] + return r + else: + return math.fsum([a, t * math.fsum([b, -a])]) + + +def angle_between(a: vec3, b: vec3) -> float: + return dot(a, b) / (a.magnitude() * b.magnitude()) + + +def sort_clockwise(points: vec3, normal: Iterable) -> list: + """Sorts a series of point into a clockwise order (first point in sequence remains the same)""" + center = sum(points, vec3()) / len(points) + def score(a, b): return dot(normal, (a - center) * (b - center)) + left = [] + right = [] + for i, point in enumerate(points[1:]): + i += 1 + (left if score(points[0], point) >= 0 else right).append(i) + # TODO: simplify the proximity calculations + proximity = dict() # number of points between self and start + for i, point in enumerate(points[1:]): + i += 1 + if i in left: + proximity[i] = len(right) + for j in left: + if score(point, points[j]) >= 0: + proximity[i] += 1 + else: + proximity[i] = 0 + for j in right: + if score(point, points[j]) >= 0: + proximity[i] += 1 + sorted_vec3s = [points[0]] + [points[i] for i in sorted(proximity.keys(), key=lambda k: proximity[k])] + return sorted_vec3s diff --git a/io_import_rbsp/bsp_tool/gearbox.py b/io_import_rbsp/bsp_tool/gearbox.py new file mode 100644 index 0000000..2ee62ef --- /dev/null +++ b/io_import_rbsp/bsp_tool/gearbox.py @@ -0,0 +1,9 @@ +# https://valvedev.info/tools/bspfix/ +# https://developer.valvesoftware.com/wiki/Hl_bs.fgd +from . import valve + + +class GoldSrcBsp(valve.GoldSrcBsp): # WIP + # https://developer.valvesoftware.com/wiki/Half-Life:_Blue_Shift#Mapping + # TODO: figure out what gearbox changed + pass diff --git a/io_import_rbsp/bsp_tool/id_software.py b/io_import_rbsp/bsp_tool/id_software.py new file mode 100644 index 0000000..243913d --- /dev/null +++ b/io_import_rbsp/bsp_tool/id_software.py @@ -0,0 +1,120 @@ +import collections +import enum # for type hints +import os +import struct +from types import ModuleType +from typing import Dict + +from . import base +from . import lumps + + +IdTechLumpHeader = collections.namedtuple("IdTechLumpHeader", ["offset", "length"]) + + +class QuakeBsp(base.Bsp): # Quake 1 only? + # NOTE: QuakeBsp has no file_magic? + + def __init__(self, branch: ModuleType, filename: str = "untitled.bsp", autoload: bool = True): + super(QuakeBsp, self).__init__(branch, filename, autoload) + + def __repr__(self): + version = f"(version {self.bsp_version})" # no file_magic + game = self.branch.__name__[len(self.branch.__package__) + 1:] + return f"<{self.__class__.__name__} '{self.filename}' {game} {version} at 0x{id(self):016X}>" + + def _preload(self): + self.file = open(os.path.join(self.folder, self.filename), "rb") + self.bsp_version = int.from_bytes(self.file.read(4), "little") + self.file.seek(0, 2) # move cursor to end of file + self.bsp_file_size = self.file.tell() + + self.headers = dict() + self.loading_errors: Dict[str, Exception] = dict() + for LUMP_enum in self.branch.LUMP: + LUMP_NAME = LUMP_enum.name + self.file.seek(self.branch.lump_header_address[LUMP_enum]) + offset, length = struct.unpack("2I", self.file.read(8)) + lump_header = IdTechLumpHeader(offset, length) + self.headers[LUMP_NAME] = lump_header + if length == 0: + continue # empty lump + try: + if LUMP_NAME in self.branch.LUMP_CLASSES: + LumpClass = self.branch.LUMP_CLASSES[LUMP_NAME] + BspLump = lumps.create_BspLump(self.file, lump_header, LumpClass) + elif LUMP_NAME in self.branch.SPECIAL_LUMP_CLASSES: + SpecialLumpClass = self.branch.SPECIAL_LUMP_CLASSES[LUMP_NAME] + self.file.seek(offset) + BspLump = SpecialLumpClass(self.file.read(length)) + elif LUMP_NAME in self.branch.BASIC_LUMP_CLASSES: + LumpClass = self.branch.BASIC_LUMP_CLASSES[LUMP_NAME] + BspLump = lumps.create_BasicBspLump(self.file, lump_header, LumpClass) + else: + BspLump = lumps.create_RawBspLump(self.file, lump_header) + except Exception as exc: + self.loading_errors[LUMP_NAME] = exc + BspLump = lumps.create_RawBspLump(self.file, lump_header) + # NOTE: doesn't decompress LZMA, fix that + setattr(self, LUMP_NAME, BspLump) + + def _read_header(self, LUMP: enum.Enum) -> IdTechLumpHeader: + """Reads bytes of lump""" + self.file.seek(self.branch.lump_header_address[LUMP]) + offset, length = struct.unpack("2I", self.file.read(8)) + header = IdTechLumpHeader(offset, length) + return header + + +class IdTechBsp(base.Bsp): + file_magic = b"IBSP" + # https://www.mralligator.com/q3/ + # NOTE: Quake 3 .bsp are usually stored in .pk3 files + + def _preload(self): + """Loads filename using the format outlined in this .bsp's branch defintion script""" + local_files = os.listdir(self.folder) + def is_related(f): return f.startswith(os.path.splitext(self.filename)[0]) + self.associated_files = [f for f in local_files if is_related(f)] + # open .bsp + self.file = open(os.path.join(self.folder, self.filename), "rb") + file_magic = self.file.read(4) + if file_magic != self.file_magic: + raise RuntimeError(f"{self.file} is not a valid .bsp!") + self.bsp_version = int.from_bytes(self.file.read(4), "little") + self.file.seek(0, 2) # move cursor to end of file + self.bsp_file_size = self.file.tell() + + self.headers = dict() + self.loading_errors: Dict[str, Exception] = dict() + for LUMP_enum in self.branch.LUMP: + # CHECK: is lump external? (are associated_files overriding) + lump_header = self._read_header(LUMP_enum) + LUMP_name = LUMP_enum.name + self.headers[LUMP_name] = lump_header + if lump_header.length == 0: + continue + try: + if LUMP_name in self.branch.LUMP_CLASSES: + LumpClass = self.branch.LUMP_CLASSES[LUMP_name] + BspLump = lumps.create_BspLump(self.file, lump_header, LumpClass) + elif LUMP_name in self.branch.SPECIAL_LUMP_CLASSES: + SpecialLumpClass = self.branch.SPECIAL_LUMP_CLASSES[LUMP_name] + self.file.seek(lump_header.offset) + lump_data = self.file.read(lump_header.length) + BspLump = SpecialLumpClass(lump_data) + elif LUMP_name in self.branch.BASIC_LUMP_CLASSES: + LumpClass = self.branch.BASIC_LUMP_CLASSES[LUMP_name] + BspLump = lumps.create_BasicBspLump(self.file, lump_header, LumpClass) + else: + BspLump = lumps.create_RawBspLump(self.file, lump_header) + except Exception as exc: + self.loading_errors[LUMP_name] = exc + BspLump = lumps.create_RawBspLump(self.file, lump_header) + setattr(self, LUMP_name, BspLump) + + def _read_header(self, LUMP: enum.Enum) -> (IdTechLumpHeader, bytes): + self.file.seek(self.branch.lump_header_address[LUMP]) + offset, length = struct.unpack("2i", self.file.read(8)) + header = IdTechLumpHeader(offset, length) + return header diff --git a/io_import_rbsp/bsp_tool/infinity_ward.py b/io_import_rbsp/bsp_tool/infinity_ward.py new file mode 100644 index 0000000..7f97429 --- /dev/null +++ b/io_import_rbsp/bsp_tool/infinity_ward.py @@ -0,0 +1,69 @@ +import collections +import enum +import os +import struct +from typing import Dict + +from . import base +from . import lumps + + +LumpHeader = collections.namedtuple("LumpHeader", ["length", "offset"]) + + +class D3DBsp(base.Bsp): + file_magic = b"IBSP" + # https://wiki.zeroy.com/index.php?title=Call_of_Duty_1:_d3dbsp + # https://wiki.zeroy.com/index.php?title=Call_of_Duty_2:_d3dbsp + # https://wiki.zeroy.com/index.php?title=Call_of_Duty_4:_d3dbsp + # NOTE: Call of Duty 1 has .bsp files in .pk3 archives + # NOTE: Call of Duty 2 has .d3dbsp in .iwd archives + # NOTE: Call of Duty 4 has .d3dbsp in .ff archives + + def _preload(self): + """Loads filename using the format outlined in this .bsp's branch defintion script""" + local_files = os.listdir(self.folder) + def is_related(f): return f.startswith(os.path.splitext(self.filename)[0]) + self.associated_files = [f for f in local_files if is_related(f)] + # open .bsp + self.file = open(os.path.join(self.folder, self.filename), "rb") + file_magic = self.file.read(4) + if file_magic != self.file_magic: + raise RuntimeError(f"{self.file} is not a valid .bsp!") + self.bsp_version = int.from_bytes(self.file.read(4), "little") + self.file.seek(0, 2) # move cursor to end of file + self.bsp_file_size = self.file.tell() + + self.headers = dict() + self.loading_errors: Dict[str, Exception] = dict() + for LUMP_enum in self.branch.LUMP: + # CHECK: is lump external? (are associated_files overriding) + lump_header = self._read_header(LUMP_enum) + LUMP_NAME = LUMP_enum.name + self.headers[LUMP_NAME] = lump_header + if lump_header.length == 0: + continue + try: + if LUMP_NAME in self.branch.LUMP_CLASSES: + LumpClass = self.branch.LUMP_CLASSES[LUMP_NAME] + BspLump = lumps.create_BspLump(self.file, lump_header, LumpClass) + elif LUMP_NAME in self.branch.SPECIAL_LUMP_CLASSES: + SpecialLumpClass = self.branch.SPECIAL_LUMP_CLASSES[LUMP_NAME] + self.file.seek(lump_header.offset) + lump_data = self.file.read(lump_header.length) + BspLump = SpecialLumpClass(lump_data) + elif LUMP_NAME in self.branch.BASIC_LUMP_CLASSES: + LumpClass = self.branch.BASIC_LUMP_CLASSES[LUMP_NAME] + BspLump = lumps.create_BasicBspLump(self.file, lump_header, LumpClass) + else: + BspLump = lumps.create_RawBspLump(self.file, lump_header) + except Exception as exc: + self.loading_errors[LUMP_NAME] = exc + BspLump = lumps.create_RawBspLump(self.file, lump_header) + setattr(self, LUMP_NAME, BspLump) + + def _read_header(self, LUMP: enum.Enum) -> (LumpHeader, bytes): + self.file.seek(self.branch.lump_header_address[LUMP]) + length, offset = struct.unpack("2i", self.file.read(8)) + header = LumpHeader(length, offset) + return header diff --git a/io_import_rbsp/bsp_tool/lumps.py b/io_import_rbsp/bsp_tool/lumps.py new file mode 100644 index 0000000..a04cd25 --- /dev/null +++ b/io_import_rbsp/bsp_tool/lumps.py @@ -0,0 +1,381 @@ +from __future__ import annotations + +import collections +import io +import lzma +import struct +from typing import Any, Dict, Union + + +def _remap_negative_index(index: int, length: int) -> int: + "simplify to positive integer" + if index < 0: + index = length + index + if index >= length or index < 0: + raise IndexError("list index out of range") + return index + + +def _remap_slice(_slice: slice, length: int) -> slice: + "simplify to positive start & stop within range(0, length)" + start, stop, step = _slice.start, _slice.stop, _slice.step + if start is None: + start = 0 + elif start < 0: + start = max(length + start, 0) + if start > length: + start = length + if stop is None or stop > length: + stop = length + elif stop < 0: + stop = max(length + stop, 0) + if step is None: + step = 1 + return slice(start, stop, step) + + +def decompressed(file: io.BufferedReader, lump_header: collections.namedtuple) -> io.BytesIO: + """Takes a lump and decompresses it if nessecary. Also corrects lump_header offset & length""" + if getattr(lump_header, "fourCC", 0) != 0: + if not hasattr(lump_header, "filename"): # internal compressed lump + file.seek(lump_header.offset) + data = file.read(lump_header.length) + else: # external compressed lump is unlikely, but possible + data = open(lump_header.filename, "rb").read() + # have to remap lzma header format slightly + lzma_header = struct.unpack("4s2I5c", data[:17]) + # b"LZMA" = lzma_header[0] + actual_size = lzma_header[1] + assert actual_size == lump_header.fourCC + # compressed_size = lzma_header[2] + properties = b"".join(lzma_header[3:]) + _filter = lzma._decode_filter_properties(lzma.FILTER_LZMA1, properties) + decompressor = lzma.LZMADecompressor(lzma.FORMAT_RAW, None, [_filter]) + decoded_data = decompressor.decompress(data[17:]) + decoded_data = decoded_data[:actual_size] # trim any excess bytes + assert len(decoded_data) == actual_size + file = io.BytesIO(decoded_data) + # HACK: trick BspLump into recognisind the decompressed lump sze + LumpHeader = lump_header.__class__ # how2 edit a tuple + lump_header_dict = dict(zip(LumpHeader._fields, lump_header)) + lump_header_dict["offset"] = 0 + lump_header_dict["length"] = lump_header.fourCC + lump_header = LumpHeader(*[lump_header_dict[f] for f in LumpHeader._fields]) + return file, lump_header + + +def create_BspLump(file: io.BufferedReader, lump_header: collections.namedtuple, LumpClass: object = None) -> BspLump: + if hasattr(lump_header, "fourCC"): + file, lump_header = decompressed(file, lump_header) + if not hasattr(lump_header, "filename"): + if LumpClass is not None: + return BspLump(file, lump_header, LumpClass) + else: + return RawBspLump(file, lump_header) + else: + if LumpClass is not None: + return ExternalBspLump(lump_header, LumpClass) + else: + return ExternalRawBspLump(lump_header) + + +def create_RawBspLump(file: io.BufferedReader, lump_header: collections.namedtuple) -> RawBspLump: + if not hasattr(lump_header, "filename"): + return RawBspLump(file, lump_header) + else: + return ExternalRawBspLump(lump_header) + + +def create_BasicBspLump(file: io.BufferedReader, lump_header: collections.namedtuple, LumpClass: object) -> BasicBspLump: # noqa E502 + if hasattr(lump_header, "fourCC"): + file, lump_header = decompressed(file, lump_header) + if not hasattr(lump_header, "filename"): + return BasicBspLump(file, lump_header, LumpClass) + else: + return ExternalBasicBspLump(lump_header, LumpClass) + + +class RawBspLump: + """Maps an open binary file to a list-like object""" + file: io.BufferedReader # file opened in "rb" (read-bytes) mode + offset: int # position in file where lump begins + _changes: Dict[int, bytes] + # ^ {index: new_byte} + _length: int # number of indexable entries + + def __init__(self, file: io.BufferedReader, lump_header: collections.namedtuple): + self.file = file + self.offset = lump_header.offset + self._changes = dict() + # ^ {index: new_value} + self._length = lump_header.length + + def __repr__(self): + return f"<{self.__class__.__name__}; {len(self)} bytes at 0x{id(self):016X}>" + + def __getitem__(self, index: Union[int, slice]) -> bytes: + """Reads bytes from the start of the lump""" + if isinstance(index, int): + index = _remap_negative_index(index, self._length) + if index in self._changes: + return self._changes[index] + else: + self.file.seek(self.offset + index) + return self.file.read(1)[0] # return 1 0-255 integer, matching bytes behaviour + elif isinstance(index, slice): + _slice = _remap_slice(index, self._length) + return bytes([self[i] for i in range(_slice.start, _slice.stop, _slice.step)]) + else: + raise TypeError(f"list indices must be integers or slices, not {type(index)}") + + def __iadd__(self, other_bytes: bytes): + if not isinstance(other_bytes, bytes): + raise TypeError(f"can't concat {other_bytes.__class__.__name__} to bytes") + start = self._length + self._length += len(other_bytes) + self[start:] = other_bytes + + def __setitem__(self, index: Union[int, slice], value: Any): + # TODO: allow slice assignment to act like insert/extend + if isinstance(index, int): + index = _remap_negative_index(index, self._length) + self._changes[index] = value + elif isinstance(index, slice): + _slice = _remap_slice(index, self._length) + for i, v in zip(range(_slice.start, _slice.stop, _slice.step), value): + self[i] = v + else: + raise TypeError(f"list indices must be integers or slices, not {type(index)}") + + def __iter__(self): + return iter([self[i] for i in range(self._length)]) + + def __len__(self): + return self._length + + +class BspLump(RawBspLump): + """Dynamically reads LumpClasses from a binary file""" + file: io.BufferedReader # file opened in "rb" (read-bytes) mode + offset: int # position in file where lump begins + LumpClass: object + _changes: Dict[int, Any] + # ^ {index: LumpClass(new_entry)} + # NOTE: there are no checks to ensure changes are the correct type or size + _entry_size: int # sizeof(LumpClass) + _length: int # number of indexable entries + + def __init__(self, file: io.BufferedReader, lump_header: collections.namedtuple, LumpClass: object): + self.file = file + self.offset = lump_header.offset + self._changes = dict() # changes must be applied externally + self._entry_size = struct.calcsize(LumpClass._format) + if lump_header.length % self._entry_size != 0: + raise RuntimeError(f"LumpClass does not divide lump evenly! ({lump_header.length} / {self._entry_size})") + self._length = lump_header.length // self._entry_size + self.LumpClass = LumpClass + + def __repr__(self): + return f"<{self.__class__.__name__}: {self.LumpClass.__name__}[{len(self)}] at 0x{id(self):016X}>" + + def __delitem__(self, index: Union[int, slice]): + if isinstance(index, int): + index = _remap_negative_index(index, self._length) + self[index:] = self[index + 1:] + self._length -= 1 + elif isinstance(index, slice): + _slice = _remap_slice(index, self._length) + for i in range(_slice.start, _slice.stop, _slice.step): + del self[i] + else: + raise TypeError(f"list indices must be integers or slices, not {type(index)}") + + def __getitem__(self, index: Union[int, slice]): + """Reads bytes from self.file & returns LumpClass(es)""" + # read bytes -> struct.unpack tuples -> LumpClass + # NOTE: BspLump[index] = LumpClass(entry) + if isinstance(index, int): + index = _remap_negative_index(index, self._length) + if index in self._changes: + return self._changes[index] + else: + self.file.seek(self.offset + (index * self._entry_size)) + raw_entry = struct.unpack(self.LumpClass._format, self.file.read(self._entry_size)) + return self.LumpClass(raw_entry) + elif isinstance(index, slice): + _slice = _remap_slice(index, self._length) + out = list() + for i in range(_slice.start, _slice.stop, _slice.step): + out.append(self[i]) + return out + else: + raise TypeError(f"list indices must be integers or slices, not {type(index)}") + + def append(self, entry): + self._length += 1 + self[-1] = entry + + def extend(self, entries: bytes): + for entry in entries: + self.append(entry) + + def find(self, **kwargs): + """Returns all lump entries which have the queried values [e.g. find(x=0)]""" + out = list() + entire_lump = self[::] # load all LumpClasses + for entry in entire_lump: + if all([getattr(entry, attr) == value for attr, value in kwargs.items()]): + out.append(entry) + return out + + def insert(self, index: int, entry: Any): + self._length += 1 + self[index + 1:] = self[index:] + self[index] = entry + + def pop(self, index: Union[int, slice]) -> Union[int, bytes]: + out = self[index] + del self[index] + return out + + +class BasicBspLump(BspLump): + """Dynamically reads LumpClasses from a binary file""" + file: io.BufferedReader # file opened in "rb" (read-bytes) mode + offset: int # position in file where lump begins + LumpClass: object + _changes: Dict[int, Any] + # ^ {index: LumpClass(new_entry)} + # NOTE: there are no checks to ensure changes are the correct type or size + _entry_size: int # sizeof(LumpClass) + _length: int # number of indexable entries + + def __getitem__(self, index: Union[int, slice]): + """Reads bytes from self.file & returns LumpClass(es)""" + # read bytes -> struct.unpack tuples -> LumpClass + # NOTE: BspLump[index] = LumpClass(entry) + if isinstance(index, int): + index = _remap_negative_index(index, self._length) + self.file.seek(self.offset + (index * self._entry_size)) + raw_entry = struct.unpack(self.LumpClass._format, self.file.read(self._entry_size)) + # NOTE: only the following line has changed + return self.LumpClass(raw_entry[0]) + elif isinstance(index, slice): + _slice = _remap_slice(index, self._length) + out = list() + for i in range(_slice.start, _slice.stop, _slice.step): + out.append(self[i]) + return out + else: + raise TypeError(f"list indices must be integers or slices, not {type(index)}") + + +class ExternalRawBspLump(RawBspLump): + """Maps an open binary file to a list-like object""" + file: io.BufferedReader # file opened in "rb" (read-bytes) mode + offset: int # position in file where lump begins + _changes: Dict[int, bytes] + # ^ {index: new_byte} + _length: int # number of indexable entries + # -- should also override any returned entries with _changes + + def __init__(self, lump_header: collections.namedtuple): + self.file = open(lump_header.filename, "rb") + self.offset = 0 + self._changes = dict() # changes must be applied externally + self._length = lump_header.filesize + + +class ExternalBspLump(BspLump): + """Dynamically reads LumpClasses from a binary file""" + # NOTE: this class does not handle compressed lumps + file: io.BufferedReader # file opened in "rb" (read-bytes) mode + offset: int # position in file where lump begins + LumpClass: object + _changes: Dict[int, Any] + # ^ {index: LumpClass(new_entry)} + # NOTE: there are no checks to ensure changes are the correct type or size + _entry_size: int # sizeof(LumpClass) + _length: int # number of indexable entries + + def __init__(self, lump_header: collections.namedtuple, LumpClass: object): + super(ExternalBspLump, self).__init__(None, lump_header, LumpClass) + self.file = open(lump_header.filename, "rb") + self.offset = 0 # NOTE: 16 if ValveBsp + self._changes = dict() # changes must be applied externally + + +class ExternalBasicBspLump(BasicBspLump): + """Dynamically reads LumpClasses from a binary file""" + # NOTE: this class does not handle compressed lumps + file: io.BufferedReader # file opened in "rb" (read-bytes) mode + offset: int # position in file where lump begins + _changes: Dict[int, Any] + # ^ {index: LumpClass(new_entry)} + _length: int # number of indexable entries + + def __init__(self, lump_header: collections.namedtuple, LumpClass: object): + super(ExternalBasicBspLump, self).__init__(None, lump_header, LumpClass) + self.file = open(lump_header.filename, "rb") + self.offset = 0 + self._changes = dict() # changes must be applied externally + + +GameLumpHeader = collections.namedtuple("GameLumpHeader", ["id", "flags", "version", "offset", "length"]) + + +class GameLump: + is_external = False + loading_errors: Dict[str, Any] + # ^ {"child_lump": Exception} + + def __init__(self, file: io.BufferedReader, lump_header: collections.namedtuple, LumpClasses: Dict[str, object]): + self.loading_errors = dict() + if not hasattr(lump_header, "filename"): + file.seek(lump_header.offset) + else: + self.is_external = True + file = open(lump_header.filename, "rb") + game_lumps_count = int.from_bytes(file.read(4), "little") + self.headers = dict() + for i in range(game_lumps_count): + _id, flags, version, offset, length = struct.unpack("4s2H2i", file.read(16)) + _id = _id.decode("ascii")[::-1] # b"prps" -> "sprp" + if self.is_external: + offset = offset - lump_header.offset + child_header = GameLumpHeader(_id, flags, version, offset, length) + self.headers[_id] = child_header + for child_name, child_header in self.headers.items(): + child_LumpClass = LumpClasses.get(child_name, dict()).get(child_header.version, None) + if child_LumpClass is None: + setattr(self, child_name, create_RawBspLump(file, child_header)) + else: + file.seek(child_header.offset) + try: + child_lump = child_LumpClass(file.read(child_header.length)) + except Exception as exc: + self.loading_errors[child_name] = exc + child_lump = create_RawBspLump(file, child_header) + setattr(self, child_name, child_lump) + + def as_bytes(self, lump_offset=0): + """lump_offset makes headers relative to the file""" + out = [] + out.append(len(self.headers).to_bytes(4, "little")) + headers = [] + cursor_offset = lump_offset + 4 + len(self.headers) * 16 + for child_name, child_header in self.headers.items(): + child_lump = getattr(self, child_name) + if isinstance(child_lump, RawBspLump): + child_lump_bytes = child_lump[::] + else: + child_lump_bytes = child_lump.as_bytes() # SpecialLumpClass method + out.append(child_lump_bytes) + # calculate header + _id, flags, version, offset, length = child_header + _id = _id.encode("ascii")[::-1] # "sprp" -> b"prps" + offset, length = cursor_offset, len(child_lump_bytes) + cursor_offset += length + headers.append(struct.pack("4s2H2i", _id, flags, version, offset, length)) + out[1:1] = headers # insert headers after calculating + return b"".join(out) diff --git a/io_import_rbsp/bsp_tool/respawn.py b/io_import_rbsp/bsp_tool/respawn.py new file mode 100644 index 0000000..9598e7c --- /dev/null +++ b/io_import_rbsp/bsp_tool/respawn.py @@ -0,0 +1,196 @@ +from collections import namedtuple +import enum +import os +import struct +from types import ModuleType +from typing import Dict + +from . import base +from . import lumps +from .base import LumpHeader +from .branches import shared + + +ExternalLumpHeader = namedtuple("ExternalLumpHeader", ["offset", "length", "version", "fourCC", "filename", "filesize"]) + + +class RespawnBsp(base.Bsp): + # https://developer.valvesoftware.com/wiki/Source_BSP_File_Format/Game-Specific#Titanfall + # https://raw.githubusercontent.com/Wanty5883/Titanfall2/master/tools/TitanfallMapExporter.py + file_magic = b"rBSP" + # NOTE: respawn .bsp files are usually stored in .vpk files + # -- Respawn's .vpk format is different to Valve's + # -- You'll need the Titanfall specific .vpk tool to extract maps + # -- https://noskill.gitbook.io/titanfall2/how-to-start-modding/modding-introduction/modding-tools + + def __init__(self, branch: ModuleType, filename: str = "untitled.bsp", autoload: bool = True): + super(RespawnBsp, self).__init__(branch, filename, autoload) + # NOTE: bsp revision appears before headers, not after (as in Valve's variant) + + def _read_header(self, LUMP: enum.Enum) -> (LumpHeader, bytes): + """Read a lump from self.file, while it is open (during __init__ only)""" + self.file.seek(self.branch.lump_header_address[LUMP]) + offset, length, version, fourCC = struct.unpack("4I", self.file.read(16)) + # TODO: use a read & write function / struct.iter_unpack + # -- this could potentially allow for simplified subclasses + # -- e.g. LumpHeader(*struct.unpack("4I", self.file.read(16))) -> self.LumpHeader(self.file) + header = LumpHeader(offset, length, version, fourCC) + return header + + def _preload(self): + """Loads filename using the format outlined in this .bsp's branch defintion script""" + local_files = os.listdir(self.folder) + def is_related(f): return f.startswith(os.path.splitext(self.filename)[0]) + self.associated_files = [f for f in local_files if is_related(f)] + self.file = open(os.path.join(self.folder, self.filename), "rb") + file_magic = self.file.read(4) + if file_magic != self.file_magic: + raise RuntimeError(f"{self.file} is not a valid .bsp!") + self.bsp_version = int.from_bytes(self.file.read(4), "little") + self.revision = int.from_bytes(self.file.read(4), "little") # just for rBSP + assert int.from_bytes(self.file.read(4), "little") == 127 + self.file.seek(0, 2) # move cursor to end of file + self.bsp_file_size = self.file.tell() + + self.loading_errors: Dict[str, Exception] = dict() + # internal & external lumps + # TODO: store both internal & external lumps + # TODO: break down into a _load_lump method, allowing reloading per lump + for LUMP in self.branch.LUMP: # external .bsp.00XX.bsp_lump lump + external = False + lump_filename = f"{self.filename}.{LUMP.value:04x}.bsp_lump" + lump_header = self._read_header(LUMP) + if lump_filename in self.associated_files: # .bsp_lump file exists + external = True + lump_filename = os.path.join(self.folder, lump_filename) + lump_filesize = os.path.getsize(os.path.join(self.folder, lump_filename)) + lump_header = ExternalLumpHeader(*lump_header, lump_filename, lump_filesize) + self.headers[LUMP.name] = lump_header + if lump_header.length == 0: + continue # skip empty lumps + try: + if LUMP.name == "GAME_LUMP": + GameLumpClasses = getattr(self.branch, "GAME_LUMP_CLASSES", dict()) + BspLump = lumps.GameLump(self.file, lump_header, GameLumpClasses) + elif LUMP.name in self.branch.LUMP_CLASSES: + LumpClass = self.branch.LUMP_CLASSES[LUMP.name][lump_header.version] + BspLump = lumps.create_BspLump(self.file, lump_header, LumpClass) + elif LUMP.name in self.branch.BASIC_LUMP_CLASSES: + LumpClass = self.branch.BASIC_LUMP_CLASSES[LUMP.name][lump_header.version] + BspLump = lumps.create_BasicBspLump(self.file, lump_header, LumpClass) + elif LUMP.name in self.branch.SPECIAL_LUMP_CLASSES: + SpecialLumpClass = self.branch.SPECIAL_LUMP_CLASSES[LUMP.name][lump_header.version] + if not external: + self.file.seek(lump_header.offset) + lump_data = self.file.read(lump_header.length) + else: + lump_data = open(lump_header.filename, "rb").read() + BspLump = SpecialLumpClass(lump_data) + else: + BspLump = lumps.create_RawBspLump(self.file, lump_header) + except KeyError: # lump version not supported + BspLump = lumps.create_RawBspLump(self.file, lump_header) + except Exception as exc: + self.loading_errors[LUMP.name] = exc + BspLump = lumps.create_RawBspLump(self.file, lump_header) + setattr(self, LUMP.name, BspLump) + + # .ent files + for ent_filetype in ("env", "fx", "script", "snd", "spawn"): + entity_file = f"{self.filename[:-4]}_{ent_filetype}.ent" # e.g. "mp_glitch_env.ent" + if entity_file in self.associated_files: + with open(os.path.join(self.folder, entity_file), "rb") as ent_file: + LUMP_name = f"ENTITIES_{ent_filetype}" + self.headers[LUMP_name] = ent_file.readline().decode().rstrip("\n") + # Titanfall: ENTITIES01 + # Apex Legends: ENTITIES02 model_count=0 + setattr(self, LUMP_name, shared.Entities(ent_file.read())) + # each .ent file also has a null byte at the very end + + def save_as(self, filename: str, single_file: bool = False): + # NOTE: this method is innacurate and inconvenient + lump_order = sorted([L for L in self.branch.LUMP], + key=lambda L: (self.headers[L.name].offset, self.headers[L.name].length)) + # ^ {"lump.name": LumpHeader / ExternalLumpHeader} + # NOTE: messes up on empty lumps, so we can't get an exact 1:1 copy /; + external_lumps = {L.name for L in self.branch.LUMP if isinstance(self.headers[L.name], ExternalLumpHeader)} + if single_file: + external_lumps = set() + raw_lumps: Dict[str, bytes] = dict() + # ^ {"LUMP.name": b"raw lump data]"} + for LUMP in self.branch.LUMP: + lump_bytes = self.lump_as_bytes(LUMP.name) + if lump_bytes != b"": # don't write empty lumps + raw_lumps[LUMP.name] = lump_bytes + # recalculate headers + current_offset = 0 + headers = dict() + for LUMP in lump_order: + if LUMP.name not in raw_lumps: # lump is not present + version = self.headers[LUMP.name].version # PHYSICS_LEVEL needs version preserved + headers[LUMP.name] = LumpHeader(current_offset, 0, version, 0) + continue + # wierd hack to align unused lump offsets correctly + if current_offset == 0: + current_offset = 16 + (16 * 128) # first byte after headers + offset = current_offset + length = len(raw_lumps[LUMP.name]) + version = self.headers[LUMP.name].version + fourCC = 0 # fourCC is always 0 because we aren't encoding + if LUMP.name in external_lumps: + external_lump_filename = f"{os.path.basename(filename)}.{LUMP.value:04x}.bsp_lump" + header = ExternalLumpHeader(offset, 0, version, fourCC, external_lump_filename, length) + # ^ offset, length, version, fourCC + else: + header = LumpHeader(offset, length, version, fourCC) + headers[LUMP.name] = header # recorded for noting padding + current_offset += length + # pad to start at the next multiple of 4 bytes + if current_offset % 4 != 0: + current_offset += 4 - current_offset % 4 + del current_offset + if "GAME_LUMP" in raw_lumps and "GAME_LUMP" not in external_lumps: + raw_lumps["GAME_LUMP"] = self.GAME_LUMP.as_bytes(headers["GAME_LUMP"].offset) + # make file + os.makedirs(os.path.dirname(os.path.realpath(filename)), exist_ok=True) + outfile = open(filename, "wb") + outfile.write(struct.pack("4s3I", self.file_magic, self.bsp_version, self.revision, 127)) + # write headers + for LUMP in self.branch.LUMP: + header = headers[LUMP.name] + outfile.write(struct.pack("4I", header.offset, header.length, header.version, header.fourCC)) + # write lump contents (cannot be done until headers allocate padding) + for LUMP in lump_order: + if LUMP.name not in raw_lumps: + continue + # write external lump + if LUMP.name in external_lumps: + external_lump = f"{filename}.{LUMP.value:04x}.bsp_lump" + with open(external_lump, "wb") as out_lumpfile: + out_lumpfile.write(raw_lumps[LUMP.name]) + else: # write lump to file + padding_length = headers[LUMP.name].offset - outfile.tell() + if padding_length > 0: # NOTE: padding_length should not exceed 3 + outfile.write(b"\0" * padding_length) + outfile.write(raw_lumps[LUMP.name]) + # final padding + end = outfile.tell() + padding_length = 0 + if end % 4 != 0: + padding_length = 4 - end % 4 + outfile.write(b"\0" * padding_length) + outfile.close() # main .bsp is written + # write .ent lumps + for ent_variant in ("env", "fx", "script", "snd", "spawn"): + if not hasattr(self, f"ENTITIES_{ent_variant}"): + continue + ent_filename = f"{os.path.splitext(filename)[0]}_{ent_variant}.ent" + with open(ent_filename, "wb") as ent_file: + # TODO: generate header if none exists + ent_file.write(self.headers[f"ENTITIES_{ent_variant}"].encode("ascii")) + ent_file.write(b"\n") + ent_file.write(getattr(self, f"ENTITIES_{ent_variant}").as_bytes()) + + def save(self, single_file: bool = False): + self.save_as(os.path.join(self.folder, self.filename), single_file) + self._preload() # reload lumps, clearing all BspLump._changes diff --git a/io_import_rbsp/bsp_tool/valve.py b/io_import_rbsp/bsp_tool/valve.py new file mode 100644 index 0000000..6370687 --- /dev/null +++ b/io_import_rbsp/bsp_tool/valve.py @@ -0,0 +1,84 @@ +from collections import namedtuple # for type hints +import enum # for type hints +import os +import struct +from types import ModuleType +from typing import Dict + +from . import base +from . import lumps +from .id_software import IdTechBsp + + +GoldSrcLumpHeader = namedtuple("GoldSrcLumpHeader", ["offset", "length"]) + + +class GoldSrcBsp(IdTechBsp): # TODO: subclass QuakeBsp? + # https://github.com/ValveSoftware/halflife/blob/master/utils/common/bspfile.h + # http://hlbsp.sourceforge.net/index.php?content=bspdef + # NOTE: GoldSrcBsp has no file_magic! + + def __init__(self, branch: ModuleType, filename: str = "untitled.bsp", autoload: bool = True): + super(GoldSrcBsp, self).__init__(branch, filename, autoload) + + def __repr__(self): + version = f"(version {self.bsp_version})" # no file_magic + game = self.branch.__name__[len(self.branch.__package__) + 1:] + return f"<{self.__class__.__name__} '{self.filename}' {game} {version} at 0x{id(self):016X}>" + + def _preload(self): + self.file = open(os.path.join(self.folder, self.filename), "rb") + self.bsp_version = int.from_bytes(self.file.read(4), "little") + self.file.seek(0, 2) # move cursor to end of file + self.bsp_file_size = self.file.tell() + + self.headers = dict() + self.loading_errors: Dict[str, Exception] = dict() + for LUMP_enum in self.branch.LUMP: + LUMP_NAME = LUMP_enum.name + self.file.seek(self.branch.lump_header_address[LUMP_enum]) + offset, length = struct.unpack("2I", self.file.read(8)) + lump_header = GoldSrcLumpHeader(offset, length) + self.headers[LUMP_NAME] = lump_header + if length == 0: + continue # empty lump + try: + if LUMP_NAME in self.branch.LUMP_CLASSES: + LumpClass = self.branch.LUMP_CLASSES[LUMP_NAME] + BspLump = lumps.create_BspLump(self.file, lump_header, LumpClass) + elif LUMP_NAME in self.branch.SPECIAL_LUMP_CLASSES: + SpecialLumpClass = self.branch.SPECIAL_LUMP_CLASSES[LUMP_NAME] + self.file.seek(offset) + BspLump = SpecialLumpClass(self.file.read(length)) + elif LUMP_NAME in self.branch.BASIC_LUMP_CLASSES: + LumpClass = self.branch.BASIC_LUMP_CLASSES[LUMP_NAME] + BspLump = lumps.create_BasicBspLump(self.file, lump_header, LumpClass) + else: + BspLump = lumps.create_RawBspLump(self.file, lump_header) + except Exception as exc: + self.loading_errors[LUMP_NAME] = exc + BspLump = lumps.create_RawBspLump(self.file, lump_header) + # NOTE: doesn't decompress LZMA, fix that + setattr(self, LUMP_NAME, BspLump) + + def _read_header(self, LUMP: enum.Enum) -> GoldSrcLumpHeader: + """Reads bytes of lump""" + self.file.seek(self.branch.lump_header_address[LUMP]) + offset, length = struct.unpack("2I", self.file.read(8)) + header = GoldSrcLumpHeader(offset, length) + return header + + +class ValveBsp(base.Bsp): + # https://developer.valvesoftware.com/wiki/Source_BSP_File_Format + file_magic = b"VBSP" + + def __init__(self, branch: ModuleType, filename: str = "untitled.bsp", autoload: bool = True): + super(ValveBsp, self).__init__(branch, filename, autoload) + + def _read_header(self, LUMP: enum.Enum) -> namedtuple: # LumpHeader + """Get LUMP from self.branch.LUMP; e.g. self.branch.LUMP.ENTITIES""" + # NOTE: each branch of VBSP has unique headers, + # -- so branch.read_lump_header function is used + # TODO: move to a system of using header LumpClasses instead of the above + return self.branch.read_lump_header(self.file, LUMP) diff --git a/io_import_rbsp/rbsp/__init__.py b/io_import_rbsp/rbsp/__init__.py new file mode 100644 index 0000000..b698827 --- /dev/null +++ b/io_import_rbsp/rbsp/__init__.py @@ -0,0 +1,4 @@ +__all__ = ["titanfall", "titanfall2", "apex_legends"] +from . import titanfall +from . import titanfall2 +from . import apex_legends diff --git a/io_import_rbsp/rbsp/apex_legends/__init__.py b/io_import_rbsp/rbsp/apex_legends/__init__.py new file mode 100644 index 0000000..4895cd0 --- /dev/null +++ b/io_import_rbsp/rbsp/apex_legends/__init__.py @@ -0,0 +1,58 @@ +__all__ = ["entities", "geometry", "materials", "props"] +import bmesh +import bpy + +from ..titanfall import entities, props +from . import materials + + +def geometry(bsp, master_collection, materials): + geometry_collection = bpy.data.collections.new("geometry") + master_collection.children.link(geometry_collection) + # load_model + for model_index, model in enumerate(bsp.MODELS): + model_collection = bpy.data.collections.new(f"model #{model_index}") + geometry_collection.children.link(model_collection) + # load_mesh + for mesh_index in range(model.first_mesh, model.first_mesh + model.num_meshes): + blender_mesh = bpy.data.meshes.new(f"mesh #{mesh_index}") # mesh object + blender_bmesh = bmesh.new() # mesh data + mesh_vertices = bsp.vertices_of_mesh(mesh_index) + bmesh_vertices = dict() + # ^ {bsp_vertex.position_index: BMVert} + face_uvs = list() + # ^ [{vertex_position_index: (u, v)}] + for triangle_index in range(0, len(mesh_vertices), 3): + face_indices = list() + uvs = dict() + for vert_index in reversed(range(3)): # inverted winding order + bsp_vertex = mesh_vertices[triangle_index + vert_index] + vertex = bsp.VERTICES[bsp_vertex.position_index] + if bsp_vertex.position_index not in bmesh_vertices: + bmesh_vertices[bsp_vertex.position_index] = blender_bmesh.verts.new(vertex) + face_indices.append(bsp_vertex.position_index) + uvs[tuple(vertex)] = (bsp_vertex.uv.u, -bsp_vertex.uv.v) # inverted V-axis + try: + blender_bmesh.faces.new([bmesh_vertices[vpi] for vpi in face_indices]) + face_uvs.append(uvs) # index must match bmesh.faces index + # HACKY BUGFIX + except ValueError: + pass # "face already exists", idk why this happens + del bmesh_vertices + # apply uv + uv_layer = blender_bmesh.loops.layers.uv.new() + blender_bmesh.faces.ensure_lookup_table() + for face, uv_dict in zip(blender_bmesh.faces, face_uvs): + for loop in face.loops: # loops correspond to verts + loop[uv_layer].uv = uv_dict[tuple(loop.vert.co)] + blender_bmesh.to_mesh(blender_mesh) + blender_bmesh.free() + # apply material + material_index = bsp.SURFACE_NAMES.index(bsp.get_Mesh_SurfaceName(mesh_index)) + blender_mesh.materials.append(materials[material_index]) + # push to collection + blender_mesh.update() + blender_object = bpy.data.objects.new(blender_mesh.name, blender_mesh) + model_collection.objects.link(blender_object) + if len(model_collection.objects) == 0: + bpy.data.collections.remove(model_collection) diff --git a/io_import_rbsp/rbsp/apex_legends/materials.py b/io_import_rbsp/rbsp/apex_legends/materials.py new file mode 100644 index 0000000..a5613cb --- /dev/null +++ b/io_import_rbsp/rbsp/apex_legends/materials.py @@ -0,0 +1,24 @@ +import bpy + +from typing import List + + +def base_colours(bsp) -> List[bpy.types.Material]: + materials = list() + tool_texture_colours = {"TOOLS\\TOOLSBLACK": (0, 0, 0, 1), + "TOOLS\\TOOLSENVMAPVOLUME": (0.752, 0.0, 0.972, .25), + "TOOLS\\TOOLSFOGVOLUME": (0.752, 0.0, 0.972, .25), + "TOOLS\\TOOLSLIGHTPROBEVOLUME": (0.752, 0.0, 0.972, .25), + "TOOLS\\TOOLSOUT_OF_BOUNDS": (0.913, 0.39, 0.003, .25), + "TOOLS\\TOOLSSKYBOX": (0.441, 0.742, 0.967, .25), + "TOOLS\\TOOLSTRIGGER": (0.944, 0.048, 0.004, .25), + "TOOLS\\TOOLSTRIGGER_CAPTUREPOINT": (0.273, 0.104, 0.409, .25)} + for i, vmt_name in enumerate(bsp.SURFACE_NAMES): + material = bpy.data.materials.new(vmt_name) + *colour, alpha = tool_texture_colours.get(vmt_name, (0.8, 0.8, 0.8, 1.0)) + alpha = 0.25 if vmt_name.startswith("world\\atmosphere") else alpha + material.diffuse_color = (*colour, alpha) + if alpha != 1: + material.blend_method = "BLEND" + materials.append(material) + return materials diff --git a/io_import_rbsp/rbsp/rpak_materials.py b/io_import_rbsp/rbsp/rpak_materials.py new file mode 100644 index 0000000..d5703af --- /dev/null +++ b/io_import_rbsp/rbsp/rpak_materials.py @@ -0,0 +1,82 @@ +# by MySteyk & Dogecore +# TODO: extraction instructions & testing +import json +import os.path +from typing import List + +import bpy + + +loaded_materials = {} + +MATERIAL_LOAD_PATH = "" # put your path here + +# normal has special logic +MATERIAL_INPUT_LINKING = { + "color": "Base Color", + "rough": "Roughness", + "spec": "Specular", + "illumm": "Emission", +} + + +def load_material_data_from_name(subpath): + full_path = MATERIAL_LOAD_PATH + subpath + ".json" + if not os.path.isfile(full_path): + return False + return json.load(open(full_path, "rb")) + + +def load_image_from_subpath(subpath): + full_path = MATERIAL_LOAD_PATH + subpath + if not os.path.isfile(full_path): + return False + return bpy.data.images.load(full_path) + + +def load_materials(bsp) -> List[bpy.types.Material]: + materials = [] + for material_name in bsp.TEXTURE_DATA_STRING_DATA: + if material_name in loaded_materials: + materials.append(loaded_materials[material_name]) + continue + mat_data = load_material_data_from_name(material_name) + material = bpy.data.materials.new("materials/" + material_name) + if not mat_data: + loaded_materials[material_name] = material + materials.append(material) + # raise ValueError(f"Material data for material {material_name} does not exist!") + continue + # print(material_name, mat_data) + material.use_nodes = True + bsdf = material.node_tree.nodes["Principled BSDF"] + # data link + for mat_data_entry in MATERIAL_INPUT_LINKING.keys(): + texture_file = mat_data[mat_data_entry] + if texture_file == "": + print(f"Texture type {mat_data_entry} doesn't exist in {material_name}'s material data, skipping.") + continue + img = load_image_from_subpath(texture_file) + if not img: + raise ValueError(f"{material_name}'s texture {texture_file} ({mat_data_entry}) doesn't exist!") + continue + tex = material.node_tree.nodes.new("ShaderNodeTexImage") + tex.image = img + material.node_tree.links.new(bsdf.inputs[MATERIAL_INPUT_LINKING[mat_data_entry]], tex.outputs["Color"]) + if mat_data_entry == "color": + material.node_tree.links.new(bsdf.inputs["Alpha"], tex.outputs["Alpha"]) + # normal link + if mat_data["normal"] != "": + texture_file = mat_data["normal"] + normalmap = material.node_tree.nodes.new("ShaderNodeNormalMap") + img = load_image_from_subpath(texture_file) + if not img: + raise ValueError(f"Texture {texture_file} for material {material_name} (normal) doesn't exist!") + continue + tex = material.node_tree.nodes.new("ShaderNodeTexImage") + tex.image = img + material.node_tree.links.new(normalmap.inputs["Color"], tex.outputs["Color"]) + material.node_tree.links.new(bsdf.inputs["Normal"], normalmap.outputs["Normal"]) + loaded_materials[material_name] = material + materials.append(material) + return materials diff --git a/io_import_rbsp/rbsp/titanfall/__init__.py b/io_import_rbsp/rbsp/titanfall/__init__.py new file mode 100644 index 0000000..ea824cd --- /dev/null +++ b/io_import_rbsp/rbsp/titanfall/__init__.py @@ -0,0 +1,59 @@ +__all__ = ["entities", "geometry", "materials", "props"] +import bmesh +import bpy + +from . import entities +from . import materials +from . import props + + +def geometry(bsp, master_collection, materials): + geometry_collection = bpy.data.collections.new("geometry") + master_collection.children.link(geometry_collection) + # load_model + for model_index, model in enumerate(bsp.MODELS): + model_collection = bpy.data.collections.new(f"model #{model_index}") + geometry_collection.children.link(model_collection) + # load_mesh + for mesh_index in range(model.first_mesh, model.first_mesh + model.num_meshes): + mesh = bsp.MESHES[mesh_index] # to look up TextureData + # blender mesh assembly + blender_mesh = bpy.data.meshes.new(f"mesh #{mesh_index}") # mesh object + blender_bmesh = bmesh.new() # mesh data + mesh_vertices = bsp.vertices_of_mesh(mesh_index) + bmesh_vertices = dict() + # ^ {bsp_vertex.position_index: BMVert} + face_uvs = list() + # ^ [{vertex_position_index: (u, v)}] + for triangle_index in range(0, len(mesh_vertices), 3): + face_indices = list() + uvs = dict() + for vert_index in reversed(range(3)): # inverted winding order + bsp_vertex = mesh_vertices[triangle_index + vert_index] + vertex = bsp.VERTICES[bsp_vertex.position_index] + if bsp_vertex.position_index not in bmesh_vertices: + bmesh_vertices[bsp_vertex.position_index] = blender_bmesh.verts.new(vertex) + face_indices.append(bsp_vertex.position_index) + uvs[tuple(vertex)] = (bsp_vertex.uv.u, -bsp_vertex.uv.v) # inverted V-axis + try: + blender_bmesh.faces.new([bmesh_vertices[vpi] for vpi in face_indices]) + face_uvs.append(uvs) + # HACKY BUGFIX + except ValueError: + pass # "face already exists", idk why this happens + del bmesh_vertices + # apply uv + uv_layer = blender_bmesh.loops.layers.uv.new() + blender_bmesh.faces.ensure_lookup_table() + for face, uv_dict in zip(blender_bmesh.faces, face_uvs): + for loop in face.loops: # loops correspond to verts + loop[uv_layer].uv = uv_dict[tuple(loop.vert.co)] + blender_bmesh.to_mesh(blender_mesh) + blender_bmesh.free() + texture_data = bsp.TEXTURE_DATA[bsp.MATERIAL_SORT[mesh.material_sort].texture_data] + blender_mesh.materials.append(materials[texture_data.name_index]) + blender_mesh.update() + blender_object = bpy.data.objects.new(blender_mesh.name, blender_mesh) + model_collection.objects.link(blender_object) + if len(model_collection.objects) == 0: + bpy.data.collections.remove(model_collection) diff --git a/io_import_rbsp/rbsp/titanfall/append_geo.py b/io_import_rbsp/rbsp/titanfall/append_geo.py new file mode 100644 index 0000000..26ca678 --- /dev/null +++ b/io_import_rbsp/rbsp/titanfall/append_geo.py @@ -0,0 +1,118 @@ +import bpy +import importlib +import itertools +from typing import List + +from ... import bsp_tool + + +importlib.reload(bsp_tool) +importlib.reload(bsp_tool.base) +importlib.reload(bsp_tool.lumps) +importlib.reload(bsp_tool.branches.base) +importlib.reload(bsp_tool.branches.shared) +importlib.reload(bsp_tool.branches.id_software.quake) +importlib.reload(bsp_tool.branches.respawn.titanfall) + +titanfall = bsp_tool.branches.respawn.titanfall + +TITANFALL = "E:/Mod/Titanfall/maps/" +TITANFALL_ONLINE = "E:/Mod/TitanfallOnline/maps/" +bsp = bsp_tool.load_bsp(TITANFALL_ONLINE + "mp_box.bsp") + +# /-> MaterialSort -> TextureData -> TextureDataStringTable -> TextureDataStringData +# Model -> Mesh -> MeshIndices -\-> VertexReservedX -> Vertex +# \-> .flags (VertexReservedX) \--> VertexNormal +# \-> .uv + +obj = bpy.context.selected_objects[0] +entity = dict(classname="func_brush", + model=f"*{len(bsp.MODELS)}", + targetname="orange_tower", + origin=f"{obj.location.x} {obj.location.y} {obj.location.z}", + # NOTE: selected_objects may not all have the same center + # loose defaults: + solidbsp="0", shadowdepthnocache="0", + invert_exclusion="0", drawinfastreflection="0", + disableshadows="0", disableshadowdepth="0", + disableflashlight="0", startdisabled="0", + spawnflags="2", solidity="0", vrad_brush_cast_shadows="0") +bsp.ENTITIES.append(entity) + +# NOTE: obj. bound_box +bound_points = list(itertools.chain(*[o.bound_box for o in bpy.context.selected_objects])) +xs = {P[0] for P in bound_points} +ys = {P[1] for P in bound_points} +zs = {P[2] for P in bound_points} +# TODO: MESH_BOUNDS (mins, flags_1?, maxs, flags_2?) +model = titanfall.Model((min(xs), min(ys), min(zs), + max(xs), max(ys), max(zs), + len(bsp.MESHES), len(bpy.context.selected_objects))) +bsp.MODELS.append(model) + +for obj in bpy.context.selected_objects: + texture_name = obj.material_slots[0].name + bsp.TEXTURE_DATA_STRING_TABLE.append(len(bsp.TEXTURE_DATA_STRING_DATA.as_bytes())) + bsp.TEXTURE_DATA_STRING_DATA.append(texture_name) + + texture_data = titanfall.TextureData((*obj.material_slots[0].material.diffuse_color[:3], + len(bsp.TEXTURE_DATA_STRING_DATA) - 1, + *[128, 128], *[128, 128], + titanfall.Flags.VERTEX_UNLIT)) + bsp.TEXTURE_DATA.append(texture_data) + + material_sort = titanfall.MaterialSort((len(bsp.TEXTURE_DATA) - 1, 0, -1, 0, 0)) + bsp.MATERIAL_SORT.append(material_sort) + # NOTE: expecting one mesh per material, but we could split it ourselves + + # NOTE: not checking bsp.VERTICES for reusable positions + raw_vertices = [tuple(v.co) for v in obj.data.vertices] + vertices = list(set(raw_vertices)) + remapped_vertices = {i: vertices.index(v) + len(bsp.VERTICES) for i, v in enumerate(raw_vertices)} + Vertex = titanfall.LUMP_CLASSES["VERTICES"][0] + bsp.VERTICES.extend([Vertex(v) for v in vertices]) + del raw_vertices, vertices, Vertex + + # NOTE: not checking bsp.VERTEX_NORMALS for reusable normals + raw_normals = [tuple(v.normal)for v in obj.data.vertices] + normals = list(set(raw_normals)) + remapped_normals = {i: normals.index(n) + len(bsp.VERTEX_NORMALS) for i, n in enumerate(raw_normals)} + VertexNormal = titanfall.LUMP_CLASSES["VERTEX_NORMALS"][0] + bsp.VERTEX_NORMALS.extend([VertexNormal(n) for n in normals]) + del raw_normals, normals, VertexNormal + + special_vertices: List[titanfall.VertexReservedX] = list() + # [VertexReservedX(position_index, normal_index, *uv, ...)] + mesh_indices: List[int] = list() + + # https://docs.blender.org/api/current/bpy.types.Mesh.html + for poly in obj.data.polygons: + for i in poly.loop_indices: + # NOTE: could triangulate here + vertex_index = obj.data.loops[i].vertex_index + position_index = remapped_vertices[vertex_index] + normal_index = remapped_normals[vertex_index] + uv = tuple(obj.data.uv_layers.active.data[i].uv) + vertex = (position_index, normal_index, *uv) + if vertex not in special_vertices: + mesh_indices.append(len(special_vertices)) + special_vertices.append(vertex) + else: + mesh_indices.append(special_vertices.index(vertex)) + + mesh = titanfall.Mesh((len(bsp.MESH_INDICES), len(mesh_indices) // 3, # first_mesh_index, num_triangles + len(bsp.VERTEX_UNLIT), len(special_vertices), # first_vertex, num_vertices + 0, -1, -1, -1, -1, -1, # unknown + len(bsp.MATERIAL_SORT) - 1, titanfall.Flags.VERTEX_UNLIT)) # material_sort, flags + bsp.MESHES.append(mesh) + + bsp.MESH_INDICES.extend(mesh_indices) + + # bsp.VERTEX_LIT_BUMP.extend([r1.VertexLitBump((*v, -1, 0.0, 0.0, 0, 0, 2, 9)) for v in special_vertices]) + # NOTE: crunching all uv2 to 1 point could be bad, + # -- unknown=(0, 0, 2, 9) is mp_box.VERTEX_LIT_BUMP[0].unknown + bsp.VERTEX_UNLIT.extend([titanfall.VertexUnlit((*v, -1)) for v in special_vertices]) + + +bsp.save_as("E:/Mod/TitanfallOnline/TitanFallOnline/Data/r1/maps/mp_box.bsp") +print("Write complete") diff --git a/io_import_rbsp/rbsp/titanfall/entities.py b/io_import_rbsp/rbsp/titanfall/entities.py new file mode 100644 index 0000000..d26520a --- /dev/null +++ b/io_import_rbsp/rbsp/titanfall/entities.py @@ -0,0 +1,109 @@ +import math +from typing import Dict, List + +import bpy +import mathutils + + +def srgb_to_linear(*srgb: List[float]) -> List[float]: + """colourspace translation for lights""" + linear = list() + for s in srgb: + if s <= 0.0404482362771082: + lin = s / 12.92 + else: + lin = ((s + 0.055) / 1.055) ** 2.4 + linear.append(lin) + return linear + + +def linear_to_srgb(*linear: List[float]) -> List[float]: + """colourspace translation for lights""" + srgb = list() + for lin in linear: + if lin > 0.00313066844250063: + s = 1.055 * (lin ** (1.0 / 2.4)) - 0.055 + else: + s = 12.92 * lin + srgb.append(s) + return srgb + + +def ent_to_light(entity: Dict[str, str]) -> bpy.types.PointLight: + """Reference objects for entities""" + light_name = entity.get("targetname", entity["classname"]) + # IdTech / IW / Source / Titanfall Engine light type + if entity["classname"] == "light": + light = bpy.data.lights.new(light_name, "POINT") + elif entity["classname"] == "light_spot": + light = bpy.data.lights.new(light_name, "SPOT") + outer_angle, inner_angle = map(lambda x: math.radians(float(entity[x])), ("_cone", "_inner_cone")) + light.spot_size = math.radians(outer_angle) + light.spot_blend = 1 - inner_angle / outer_angle + elif entity["classname"] == "light_environment": + light = bpy.data.lights.new(light_name, "SUN") + light.angle = math.radians(float(entity.get("SunSpreadAngle", "0"))) + # rough light values conversion + light.cycles.use_multiple_importance_sampling = False + if entity.get("_lightHDR", "-1 -1 -1 1") == "-1 -1 -1 1": + r, g, b, brightness = map(float, entity["_light"].split()) + light.color = srgb_to_linear(r, g, b) + light.energy = brightness + else: + r, g, b, brightness = map(float, entity["_lightHDR"].split()) + light.color = srgb_to_linear(r, g, b) + light.energy = brightness * float(entity.get("_lightscaleHDR", "1")) + # TODO: use vector math nodes to mimic light curves + if "_zero_percent_distance" in entity: + light.use_custom_distance = True + light.cutoff_distance = float(entity["_zero_percent_distance"]) + light.energy = light.energy / 100 + return light + + +ent_object_data = {"light": ent_to_light, "light_spot": ent_to_light, "light_environment": ent_to_light, + "ambient_generic": lambda e: bpy.data.speakers.new(e.get("targetname", e["classname"]))} +# ^ {"classname": new_object_data_func} +# TODO: cubemaps, lightprobes, props, areaportals +# NOTE: in the Titanfall Engine info_target has a "editorclass" key +# -- this is likely used for script based object classes (weapon pickups, cameras etc.) + + +def as_empties(bsp, master_collection): + all_entities = (bsp.ENTITIES, bsp.ENTITIES_env, bsp.ENTITIES_fx, + bsp.ENTITIES_script, bsp.ENTITIES_snd, bsp.ENTITIES_spawn) + block_names = ("bsp", "env", "fx", "script", "sound", "spawn") + entities_collection = bpy.data.collections.new("entities") + master_collection.children.link(entities_collection) + for entity_block, block_name in zip(all_entities, block_names): + entity_collection = bpy.data.collections.new(block_name) + entities_collection.children.link(entity_collection) + for entity in entity_block: + object_data = ent_object_data.get(entity["classname"], lambda e: None)(entity) + name = entity.get("targetname", entity["classname"]) + entity_object = bpy.data.objects.new(name, object_data) + if object_data is None: + entity_object.empty_display_type = "SPHERE" + entity_object.empty_display_size = 64 + if entity["classname"].startswith("info_node"): + entity_object.empty_display_type = "CUBE" + entity_collection.objects.link(entity_object) + # location + position = [*map(float, entity.get("origin", "0 0 0").split())] + entity_object.location = position + if entity.get("model", "").startswith("*"): + model_collection = bpy.data.collections.get(f"model #{entity['model'][1:]}") + if model_collection is not None: + model_collection.name = entity.get("targetname", model_collection.name) + for mesh_object in model_collection.objects: + mesh_object.location = position + # rotation + angles = [*map(lambda x: math.radians(float(x)), entity.get("angles", "0 0 0").split())] + angles[0] = math.radians(-float(entity.get("pitch", -math.degrees(angles[0])))) + entity_object.rotation_euler = mathutils.Euler(angles, "YZX") + # NOTE: default orientation is facing east (+X), props may appear rotated? + # TODO: further optimisation (props with shared worldmodel share mesh data) [ent_object_data] + for field in entity: + entity_object[field] = entity[field] + # TODO: once all ents are loaded, connect paths for keyframe_rope / path_track etc. + # TODO: do a second pass of entities to apply parental relationships (based on targetnames) diff --git a/io_import_rbsp/rbsp/titanfall/materials.py b/io_import_rbsp/rbsp/titanfall/materials.py new file mode 100644 index 0000000..b5effe3 --- /dev/null +++ b/io_import_rbsp/rbsp/titanfall/materials.py @@ -0,0 +1,16 @@ +from typing import List + +import bpy + + +def base_colours(bsp) -> List[bpy.types.Material]: + materials = list() + for i, vmt_name in enumerate(bsp.TEXTURE_DATA_STRING_DATA): + material = bpy.data.materials.new(vmt_name) + colour = [td.reflectivity for td in bsp.TEXTURE_DATA if td.name_index == i][0] + alpha = 1.0 if not vmt_name.startswith("TOOLS") else 0.25 + material.diffuse_color = (*colour, alpha) + if alpha != 1: + material.blend_method = "BLEND" + materials.append(material) + return materials diff --git a/io_import_rbsp/rbsp/titanfall/props.py b/io_import_rbsp/rbsp/titanfall/props.py new file mode 100644 index 0000000..5e96f4e --- /dev/null +++ b/io_import_rbsp/rbsp/titanfall/props.py @@ -0,0 +1,33 @@ +import bpy +import mathutils + + +def as_empties(bsp, master_collection): + """Requires all models to be extracted beforehand""" + prop_collection = bpy.data.collections.new("static props") + master_collection.children.link(prop_collection) + for prop in bsp.GAME_LUMP.sprp.props: + prop_object = bpy.data.objects.new(bsp.GAME_LUMP.sprp.model_names[prop.model_name], None) + # TODO: link mesh data by model_name + prop_object.empty_display_type = "SPHERE" + prop_object.empty_display_size = 64 + prop_object.location = [*prop.origin] + prop_object.rotation_euler = mathutils.Euler((prop.angles[2], prop.angles[0], 90 + prop.angles[1])) + prop_collection.objects.link(prop_object) + + +def as_models(bsp, master_collection): + raise NotImplementedError() + # model_dir = os.path.join(game_dir, "models") + # TODO: hook into SourceIO to import .mdl files + # TODO: make a collection for static props + # try: + # bpy.ops.source_io.mdl(filepath=model_dir, files=[{"name": "error.mdl"}]) + # except Exception as exc: + # print("Source IO not installed!", exc) + # else: + # for mdl_name in bsp.GAME_LUMP.sprp.mdl_names: + # bpy.ops.source_io.mdl(filepath=model_dir, files=[{"name": mdl_name}]) + # now find it..., each model creates a collection... + # this is gonna be real memory intensive... + # TODO: instance each prop at listed location & rotation etc. (preserve object data) diff --git a/io_import_rbsp/rbsp/titanfall/r1_mesh_compiler_notes.txt b/io_import_rbsp/rbsp/titanfall/r1_mesh_compiler_notes.txt new file mode 100644 index 0000000..a847dad --- /dev/null +++ b/io_import_rbsp/rbsp/titanfall/r1_mesh_compiler_notes.txt @@ -0,0 +1,49 @@ +# from .map / .vmf / .blend / .bsp (VBSP): +new_vertices: List[Tuple[float, float, float]] # x, y, z +new_indices: List[int] # must be triangles +material_name: str # must split geo by material + + +# to .bsp (rBSP) +bsp = bsp_tool.load_bsp(...) +r1 = bsp_tool.branches.respawn.titafall + +vertex_index_offset = len(bsp.VERTICES) +bsp.VERTICES.extend(new_vertices) +mesh_index_offset = len(bsp.VERTEX_UNLIT) +special_vertices = [r1.VertexUnlit(position=vertex_index_offset + i, ...) for i, v in new_vertices] +bsp.VERTEX_UNLIT.extend(special_vertices) + +mesh = r1.Mesh() +mesh.start_index = len(bsp.MESH_INDICES) +mesh.num_triangles = len(new_indices) // 3 +mesh.unknown = (0,) * 8 # try copying if that doesn't work +mesh.material_sort = len(bsp.MATERIAL_SORT) +mesh.flags |= Flags.VERTEX_UNLIT +bsp.MESHES.append(mesh) + +material_sort = r1.MaterialSort() +material_sort.texture_data = len(bsp.TEXTURE_DATA) +material_sort.lightmap_header = 0 # borrow some of the lightmap? +material_sort.cubemap = -1 +material_sort.vertex_offset = 0 +bsp.MATERIAL_SORT.append(material_sort) + +texture_data = r1.TextureData() +texture_data.name_index = len(bsp.TEXTURE_DATA) +# size.width, size.height +# view.width, view.height +texture_data.flags = source.Surface.SKIP +bsp.TEXTURE_DATA.append(texture_data) + +bsp.TEXTURE_DATA_STRING_DATA.apppend(material_name) + +bsp.MESH_INDICES.extend([i + mesh_index_offset for i in mesh_indices]) + +# TODO: new model targeting the new mesh(es) +# TODO: func_brush using the model in bsp.ENTITIES + +# not even complete but I think I know enough to put in some dummy values for the rest + +# NOTE: trigger brushes are defined with "colN" keys in entities? +# -- looks like 64bit text encoded physics bytes? diff --git a/io_import_rbsp/rbsp/titanfall2/__init__.py b/io_import_rbsp/rbsp/titanfall2/__init__.py new file mode 100644 index 0000000..f0cf277 --- /dev/null +++ b/io_import_rbsp/rbsp/titanfall2/__init__.py @@ -0,0 +1,3 @@ +__all__ = ["entities", "geometry", "materials", "props"] +from ..titanfall import geometry, entities, props +from . import materials diff --git a/io_import_rbsp/rbsp/titanfall2/materials.py b/io_import_rbsp/rbsp/titanfall2/materials.py new file mode 100644 index 0000000..9bd96b0 --- /dev/null +++ b/io_import_rbsp/rbsp/titanfall2/materials.py @@ -0,0 +1,24 @@ +from typing import List + +import bpy + + +def base_colours(bsp) -> List[bpy.types.Material]: + materials = list() + tool_texture_colours = {"TOOLS\\TOOLSBLACK": (0, 0, 0, 1), + "TOOLS\\TOOLSENVMAPVOLUME": (0.752, 0.0, 0.972, .25), + "TOOLS\\TOOLSFOGVOLUME": (0.752, 0.0, 0.972, .25), + "TOOLS\\TOOLSLIGHTPROBEVOLUME": (0.752, 0.0, 0.972, .25), + "TOOLS\\TOOLSOUT_OF_BOUNDS": (0.913, 0.39, 0.003, .25), + "TOOLS\\TOOLSSKYBOX": (0.441, 0.742, 0.967, .25), + "TOOLS\\TOOLSTRIGGER": (0.944, 0.048, 0.004, .25), + "TOOLS\\TOOLSTRIGGER_CAPTUREPOINT": (0.273, 0.104, 0.409, .25)} + for i, vmt_name in enumerate(bsp.TEXTURE_DATA_STRING_DATA): + material = bpy.data.materials.new(vmt_name) + *colour, alpha = tool_texture_colours.get(vmt_name, (0.8, 0.8, 0.8, 1.0)) + alpha = 0.25 if vmt_name.startswith("world\\atmosphere") else alpha + material.diffuse_color = (*colour, alpha) + if alpha != 1: + material.blend_method = "BLEND" + materials.append(material) + return materials