diff --git a/CHANGELOG.md b/CHANGELOG.md index ef4cbb8..1cc8373 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## v1.2.1_b2.93 (8th December 2021) + +### Updates + * Updated internal `bsp_tool` to [latest version](https://github.com/snake-biscuits/bsp_tool/commit/13836462855b4cbd8049098a1df350b71eb1094f) + +### Fixed + * Apex Legends Season 11 (post 19th November path) maps are now correctly detected + + ## v1.2.0_b2.93 (1st October 2021) ### Added @@ -9,6 +18,7 @@ ### Fixed * Generated trigger brush geo is no longer inside-out + ## v1.1.0_b2.93 (1st October 2021) ### Added diff --git a/io_import_rbsp/__init__.py b/io_import_rbsp/__init__.py index a36224b..c311ee1 100644 --- a/io_import_rbsp/__init__.py +++ b/io_import_rbsp/__init__.py @@ -10,8 +10,8 @@ bl_info = { "name": "io_import_rbsp", "author": "Jared Ketterer (snake-biscuits / Bikkie)", - "version": (1, 2, 0), - "blender": (2, 93, 0), + "version": (1, 2, 1), + "blender": (3, 0, 0), "location": "File > Import > Titanfall Engine .bsp", "description": "Import maps from Titanfall, Titanfall 2 & Apex Legends", "doc_url": "https://github.com/snake-biscuits/io_import_rbsp", diff --git a/io_import_rbsp/bsp_tool/CHANGELOG.md b/io_import_rbsp/bsp_tool/CHANGELOG.md deleted file mode 100644 index 96f29de..0000000 --- a/io_import_rbsp/bsp_tool/CHANGELOG.md +++ /dev/null @@ -1,47 +0,0 @@ -# Changelog - -# v0.3.0 (~2021) - -## New - * Added `load_bsp` function to identify bsp type - * Added `D3DBsp`, `IdTechBsp`, `RespawnBsp` & `ValveBsp` classes - * Added general support for the PakFile lump - * Added general support for the GameLump lump - * Extension scripts - * `archive.py` extractor for CoD `.iwd` / Quake `.pk3` - * `diff.py` compare bsps for changelogs / study - * `lightmaps.py` bsp lightmap -> `.png` - * `lump_analysis.py` determine lump sizes with stats - * Prototype Blender 2.92 RespawnBsp editor - * Made a basic C++ 17 implementation in `src/` - -## Changed - * `Bsp` lumps are loaded dynamically, reducing memory usage - * New wrapper classes can be found in `bsp_tool/lumps.py` - * `mods/` changed to `branches/` - * added subfolders for developers - * helpful lists for auto-detecting a .bsp's origin - * renamed `team_fortress2` to `valve/orange_box` - * `LumpClasses` now end up in 3 dictionaries per branch script - * `BASIC_LUMP_CLASSES` for types like `short int` - * `LUMP_CLASSES` for standard `LumpClasses` - * `SPECIAL_LUMP_CLASSES` for irregular types (e.g. PakFile) - * `GAME_LUMP_CLASSES` for game lump SpecialLumpClasses - * `Bsp`s no longer print to console once loaded - * `Base.Bsp` & subclasses have reserved ALL CAPS member names for lumps only - * BSP_VERSION, FILE_MAGIC, HEADERS, REVISION -> bsp_version, file_magic, headers, revision - * TODO: load external lumps and internal lumps at the same time - -## New Supported Games - * Call of Duty 1 - * Counter-Strike: Global Offensive - * Counter-Strike: Online 2 - * Counter-Strike: Source - * Quake - * Quake 3 Arena - -## Updated Game Support - * Apex Legends - * Orange Box - * Titanfall - * Titanfall 2 diff --git a/io_import_rbsp/bsp_tool/LICENSE b/io_import_rbsp/bsp_tool/LICENSE deleted file mode 100644 index 9833469..0000000 --- a/io_import_rbsp/bsp_tool/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2020 Jared Ketterer - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/io_import_rbsp/bsp_tool/README.md b/io_import_rbsp/bsp_tool/README.md deleted file mode 100644 index 1ec18ec..0000000 --- a/io_import_rbsp/bsp_tool/README.md +++ /dev/null @@ -1,157 +0,0 @@ -# bsp_tool - A library for .bsp file analysis & modification - -`bsp_tool` provides a Command Line Interface for exploring & editing .bsp files -Current development is focused on bringing new maps to Counter-Strike: Online 2 & the Titanfall Engine - - -## Installation -To use the latest version, clone from git: -``` -$ git clone git@github.com:snake-biscuits/bsp_tool.git -``` - -Or to use the latest stable release, install via [pip](https://pypi.org/project/bsp-tool/) (Python 3.7+): -``` -pip install bsp_tool -``` - -> NOTE: The last PyPi release (v0.2.2) is close to a year old -> v0.3.0 has made many [changes](./CHANGELOG.md) and is the recommended version - - -## Fair Use -**Please do not use `bsp_tool` to copy or steal another creator's work** -The primary goal of `bsp_tool` is to extend community mapping tools - - -### Always - - **Ask** the creator's permission before touching their work - - **Understand** that by default creator's works are under copyright - - [US Law Copyright FAQ](https://www.copyright.gov/help/faq/faq-general.html#mywork) - - [US Copyright Duration](https://www.copyright.gov/help/faq/faq-duration.html) - - [Circular 15a](https://www.copyright.gov/circs/circ15a.pdf) - - **Contact** the original creator to get their permission - - This can get complicated - - Some creators don't hold the copyright on their works - - often because of Company / Publisher contracts - - **Credit** the original creator; once you have permission to share a derivative work - - **Support** the official release - -**DO NOT** use this tool to steal another creator's work -**DO** use this tool to understand the .bsp format(s) and create more specific tools - -> Be aware that this gets even more complicated with commercial projects - - -## Usage - -To load a .bsp file in python: - -```python ->>> import bsp_tool ->>> bsp_tool.load_bsp("map_folder/filename.bsp") - -``` - -Full documentation: [snake-biscuits.github.io/bsp_tool/](https://snake-biscuits.github.io/bsp_tool/) - - -## Supported Games - -> The :x: emoji indicates tests are failing -> The :o: emoji indicates a lack of .bsps to test - - * [Arkane Studios](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/arkane) - - [Dark Messiah of Might & Magic](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/arkane/dark_messiah.py) :x: - * [Gearbox Software](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/gearbox) - - [Half-Life: Blue Shift](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/gearbox/bshift.py) :x: - - [Half-Life: Opposing Force](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/goldsrc.py) - * [Id Software](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/id_software) - - [Quake](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/id_software/quake.py) :x: - - [Quake II](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/id_software/quake2.py) - - [Quake III Arena](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/id_software/quake3.py) - - Quake 4 :o: - - Quake Champions :o: - - [Quake Live](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/id_software/quake3.py) - * [Infinity Ward](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/infinity_ward) - - [Call of Duty](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/infinity_ward/call_of_duty1.py) :x: - - Call of Duty 2 :x: - - Call of Duty 4: Modern Warfare :x: - * [Nexon](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/nexon) - - [Counter-Strike: Online 2](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/nexon/cso2.py) :x: - - [Vindictus](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/nexon/vindictus.py) :o: - * [Respawn Entertainment](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/respawn) - - [Apex Legends](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/respawn/apex_legends.py) - - [Titanfall 2](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/respawn/titanfall2.py) - - [Titanfall](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/respawn/titanfall.py) - - [Titanfall: Online](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/respawn/titanfall.py) - * Ritual Entertainment - - American McGee's Alice :o: - - Heavy Metal F.A.K.K. 2 :o: - - Medal of Honor: Allied Assault :o: - - SiN :o: - - SiN: Gold :o: - - SiN Episodes: Emergence :o: - - Star Trek: Elite Force II :o: - * [Valve Software](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve) - - [Alien Swarm](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/alien_swarm.py) - - [Alien Swarm: Reactive Drop](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/alien_swarm.py) - - [Counter-Strike: Condition Zero](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/goldsrc.py) - - [Counter-Strike: Condition Zero - Deleted Scenes](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/goldsrc.py) - - [Counter-Strike: Global Offensive](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/sdk_2013.py) - - [Counter-Strike: Source](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/source.py) - - [Counter-Strike](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/goldsrc.py) - - [Day of Defeat](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/goldsrc.py) - - [Day of Defeat: Source](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/source.py) - - [Deathmatch Classic](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/goldsrc.py) - - [Half-Life](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/goldsrc.py) - - [Half-Life 2](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/source.py) - - [Half-Life 2: Deathmatch](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/source.py) - - [Half-Life 2: Episode 1](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/source.py) - - [Half-Life 2: Episode 2](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/orange_box.py) - - [Half-Life 2: Lost Coast](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/orange_box.py) - - [Half-Life Deathmatch: Source](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/source.py) - - [Half-Life: Source](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/source.py) - - [Left 4 Dead](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/left4dead.py) - - [Left 4 Dead 2](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/left4dead2.py) - - [Portal](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/orange_box.py) - - [Portal 2](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/sdk_2013.py) - - [Richochet](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/goldsrc.py) - - [Source Filmmaker](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/sdk_2013.py) - - [Source SDK 2013](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/orange_box.py) - - [Team Fortress 2](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/orange_box.py) - - [Team Fortress Classic](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/goldsrc.py) - * Other - - [Hexen 2](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/id_software/quake.py) :x: - - [Black Mesa](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/sdk_2013.py) - - [Blade Symphony](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/sdk_2013.py) - - Brink :o: - - Daikatana :o: - - [Fortress Forever](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/orange_box.py) - - [G-String](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/orange_box.py) - - [Garry's Mod](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/orange_box.py) - - [Halfquake Trilogy](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/goldsrc.py) - - [NEOTOKYO](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/orange_box.py) - - [Sven Co-op](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/goldsrc.py) - - [Synergy](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/valve/source.py) - - Tactical Intervention :x: - - [Team Fortress Quake](https://github.com/snake-biscuits/bsp_tool/tree/master/bsp_tool/branches/id_software/quake.py) :x: - - Vampire: The Masquerade - Bloodlines :o: - - diff --git a/io_import_rbsp/bsp_tool/__init__.py b/io_import_rbsp/bsp_tool/__init__.py index 09559f3..f983a18 100644 --- a/io_import_rbsp/bsp_tool/__init__.py +++ b/io_import_rbsp/bsp_tool/__init__.py @@ -1,87 +1,89 @@ """A library for .bsp file analysis & modification""" __all__ = ["base", "branches", "load_bsp", "lumps", "tools", - "GoldSrcBsp", "ValveBsp", "QuakeBsp", "IdTechBsp", "D3DBsp", "RespawnBsp"] + "ArkaneBsp", "D3DBsp", "GoldSrcBsp", "IdTechBsp", "InfinityWardBsp", + "QuakeBsp", "RavenBsp", "RespawnBsp", "RitualBsp", "ValveBsp"] -import difflib import os from types import ModuleType -from typing import Union from . import base # base.Bsp base class from . import branches # all known .bsp variant definitions -from . import lumps # handles loading data dynamically +from . import lumps +from .arkane import ArkaneBsp from .id_software import QuakeBsp, IdTechBsp -from .infinity_ward import D3DBsp +from .infinity_ward import InfinityWardBsp, D3DBsp +from .raven import RavenBsp from .respawn import RespawnBsp +from .ritual import RitualBsp from .valve import GoldSrcBsp, ValveBsp -# NOTE: Quake Live branch_script should be quake3, but auto-detect defaults to quake2 on BSP_VERSION -# NOTE: CoD1 auto-detect by version defaults to ApexLegends +BspVariant_from_file_magic = {b"2015": RitualBsp, + b"EF2!": RitualBsp, + b"FAKK": RitualBsp, + b"IBSP": IdTechBsp, # + InfinityWardBsp + D3DBsp + b"rBSP": RespawnBsp, + b"RBSP": RavenBsp, + b"VBSP": ValveBsp} # + ArkaneBsp +# NOTE: if no file_magic is present: +# - QuakeBsp +# - GoldSrcBsp +# - 256-bit XOR encoded Tactical Intervention .bsp +# detect GoldSrcBsp +GoldSrc_versions = {*branches.valve.goldsrc.GAME_VERSIONS.values(), + *branches.gearbox.blue_shift.GAME_VERSIONS.values(), + *branches.gearbox.nightfire.GAME_VERSIONS.values()} +# detect InfinityWardBsp / D3DBsp +InfinityWard_versions = {v for s in branches.infinity_ward.scripts for v in s.GAME_VERSIONS.values()} +# detect QuakeBsp +Quake_versions = {*branches.id_software.quake.GAME_VERSIONS.values()} -developers_by_file_magic = {b"IBSP": IdTechBsp, # or D3DBsp - b"rBSP": RespawnBsp, - b"VBSP": ValveBsp} -# HACK: GoldSrcBsp has no file-magic, substituting BSP_VERSION -goldsrc_versions = [branches.valve.goldsrc.BSP_VERSION, branches.gearbox.bshift.BSP_VERSION] -developers_by_file_magic.update({v.to_bytes(4, "little"): GoldSrcBsp for v in goldsrc_versions}) -developers_by_file_magic.update({branches.id_software.quake.BSP_VERSION.to_bytes(4, "little"): QuakeBsp}) - -cod_ibsp_versions = [getattr(branches.infinity_ward, b).BSP_VERSION for b in branches.infinity_ward.__all__] - - -def guess_by_file_magic(filename: str) -> (base.Bsp, int): - """returns BspVariant & version""" - if os.path.getsize(filename) == 0: # HL2/ d2_coast_02.bsp - raise RuntimeError(f"{filename} is an empty file") - BspVariant = None - if filename.endswith(".d3dbsp"): - BspVariant = D3DBsp - elif filename.endswith(".bsp"): - with open(filename, "rb") as bsp_file: - file_magic = bsp_file.read(4) - if file_magic not in developers_by_file_magic: - raise RuntimeError(f"'{filename}' does not resemble a .bsp file") - bsp_version = int.from_bytes(bsp_file.read(4), "little") - BspVariant = developers_by_file_magic[file_magic] - if BspVariant == GoldSrcBsp: - bsp_version = int.from_bytes(file_magic, "little") - # D3DBsp has b"IBSP" file_magic - if file_magic == b"IBSP" and bsp_version in cod_ibsp_versions: - BspVariant = D3DBsp - else: # invalid extension - raise RuntimeError(f"{filename} is not a .bsp file!") - return BspVariant, bsp_version - - -def load_bsp(filename: str, branch: Union[str, ModuleType] = "Unknown"): +def load_bsp(filename: str, branch_script: ModuleType = None) -> base.Bsp: """Calculate and return the correct base.Bsp sub-class for the given .bsp""" + # TODO: OPTION: use filepath to guess game / branch + # verify path if not os.path.exists(filename): raise FileNotFoundError(f".bsp file '{filename}' does not exist.") - BspVariant, bsp_version = guess_by_file_magic(filename) - if isinstance(branch, ModuleType): - return BspVariant(branch, filename, autoload=True) - elif isinstance(branch, str): - # TODO: default to other methods on fail - branch: str = branches.simplify_name(branch) - if branch != "unknown": # not default - if branch not in branches.by_name: - close_matches = difflib.get_close_matches(branch, branches.by_name) - if len(close_matches) == 0: - raise NotImplementedError(f"'{branch}' .bsp format is not supported, yet.") - else: - print(f"'{branch}'.bsp format is not supported. Assumed branches:", - "\n".join(close_matches), - f"Trying '{close_matches[0]}'...", sep="\n") - branch: str = close_matches[0] - branch: ModuleType = branches.by_name[branch] # "name" -> branch_script - # guess branch by format version + elif os.path.getsize(filename) == 0: # HL2/ d2_coast_02.bsp + raise RuntimeError(f"{filename} is an empty file") + # parse header + with open(filename, "rb") as bsp_file: + file_magic = bsp_file.read(4) + version = int.from_bytes(bsp_file.read(4), "little") + # identify BspVariant + if filename.lower().endswith(".d3dbsp"): # CoD2 & CoD4 + assert file_magic == b"IBSP", "Mystery .d3dbsp!" + assert version in InfinityWard_versions, "Unexpected .d3dbsp format version!" + if version >= branches.infinity_ward.call_of_duty4.BSP_VERSION: + BspVariant = D3DBsp else: - if bsp_version not in branches.by_version: - raise NotImplementedError(f"{BspVariant} v{bsp_version} is not supported!") - branch: ModuleType = branches.by_version[bsp_version] - return BspVariant(branch, filename, autoload=True) - else: - raise TypeError(f"Cannot use branch of type `{branch.__class__.__name__}`") + BspVariant = InfinityWardBsp + elif filename.lower().endswith(".bsp"): + if file_magic not in BspVariant_from_file_magic: # Quake / GoldSrc + version = int.from_bytes(file_magic, "little") + file_magic = None + if version in Quake_versions: + BspVariant = QuakeBsp + elif version in GoldSrc_versions: + BspVariant = GoldSrcBsp + else: + raise NotImplementedError("TODO: Check if encrypted Tactical Intervention .bsp") + else: + if file_magic == b"IBSP" and version in InfinityWard_versions: # CoD + BspVariant = InfinityWardBsp + elif file_magic == b"VBSP" and version > 0xFFFF: # Dark Messiah + version = (version & 0xFFFF, version >> 16) # major, minor + BspVariant = ArkaneBsp + else: + BspVariant = BspVariant_from_file_magic[file_magic] + else: # invalid extension + raise RuntimeError(f"{filename} is not a .bsp file!") + # identify branch script + # TODO: ata4's bspsrc uses unique entity classnames to identify branches + # -- need this for identifying variants with overlapping versions + # -- e.g. (b"VBSP", 20) & (b"VBSP", 21) + if branch_script is None: + branch_script = branches.script_from_file_magic_and_version[(file_magic, version)] + return BspVariant(branch_script, filename, autoload=True) # might raise errors diff --git a/io_import_rbsp/bsp_tool/arkane.py b/io_import_rbsp/bsp_tool/arkane.py new file mode 100644 index 0000000..f120921 --- /dev/null +++ b/io_import_rbsp/bsp_tool/arkane.py @@ -0,0 +1,62 @@ +import os +import struct +from typing import Dict + +from . import lumps +from . import valve + + +class ArkaneBsp(valve.ValveBsp): + def __repr__(self): + major, minor = self.bsp_version + version = f"({self.file_magic.decode('ascii', 'ignore')} version {major}.{minor})" + game = self.branch.__name__[len(self.branch.__package__) + 1:] + return f"<{self.__class__.__name__} '{self.filename}' {game} {version} at 0x{id(self):016X}>" + + def _preload(self): + """Loads filename using the format outlined in this .bsp's branch defintion script""" + local_files = os.listdir(self.folder) + def is_related(f): return f.startswith(os.path.splitext(self.filename)[0]) + self.associated_files = [f for f in local_files if is_related(f)] + # open .bsp + self.file = open(os.path.join(self.folder, self.filename), "rb") + file_magic = self.file.read(4) + assert file_magic == self.file_magic, f"{self.file} is not a valid .bsp!" + self.bsp_version = tuple(*struct.iter_unpack("2h", self.file.read(4))) + self.file.seek(0, 2) # move cursor to end of file + self.bsp_file_size = self.file.tell() + + self.headers = dict() + self.loading_errors: Dict[str, Exception] = dict() + for LUMP_enum in self.branch.LUMP: + # CHECK: is lump external? (are associated_files overriding) + lump_header = self._read_header(LUMP_enum) + LUMP_NAME = LUMP_enum.name + self.headers[LUMP_NAME] = lump_header + if lump_header.length == 0: + continue + try: + if LUMP_NAME == "GAME_LUMP": + GameLumpClasses = getattr(self.branch, "GAME_LUMP_CLASSES", dict()) + BspLump = lumps.GameLump(self.file, lump_header, GameLumpClasses, self.branch.GAME_LUMP_HEADER) + elif LUMP_NAME in self.branch.LUMP_CLASSES: + LumpClass = self.branch.LUMP_CLASSES[LUMP_NAME][lump_header.version] + BspLump = lumps.create_BspLump(self.file, lump_header, LumpClass) + elif LUMP_NAME in self.branch.SPECIAL_LUMP_CLASSES: + SpecialLumpClass = self.branch.SPECIAL_LUMP_CLASSES[LUMP_NAME][lump_header.version] + decompressed_file, decompressed_header = lumps.decompressed(self.file, lump_header) + decompressed_file.seek(decompressed_header.offset) + lump_data = decompressed_file.read(decompressed_header.length) + BspLump = SpecialLumpClass(lump_data) + elif LUMP_NAME in self.branch.BASIC_LUMP_CLASSES: + LumpClass = self.branch.BASIC_LUMP_CLASSES[LUMP_NAME][lump_header.version] + BspLump = lumps.create_BasicBspLump(self.file, lump_header, LumpClass) + else: + BspLump = lumps.create_RawBspLump(self.file, lump_header) + except KeyError: # lump VERSION not supported + self.loading_errors[LUMP_NAME] = KeyError(f"{LUMP_NAME} v{lump_header.version} is not supported") + BspLump = lumps.create_RawBspLump(self.file, lump_header) + except Exception as exc: + self.loading_errors[LUMP_NAME] = exc + BspLump = lumps.create_RawBspLump(self.file, lump_header) + setattr(self, LUMP_NAME, BspLump) diff --git a/io_import_rbsp/bsp_tool/base.py b/io_import_rbsp/bsp_tool/base.py index b66b4ec..ee53174 100644 --- a/io_import_rbsp/bsp_tool/base.py +++ b/io_import_rbsp/bsp_tool/base.py @@ -6,11 +6,14 @@ import struct from types import MethodType, ModuleType from typing import Dict, List +import warnings from . import lumps -# NOTE: LumpHeaders must have these attrs, but how they are read / order will vary +# TODO: align base.Bsp closer to Quake, rather than Source + +# NOTE: these LumpHeader defintions are not universal! many branches differ LumpHeader = collections.namedtuple("LumpHeader", ["offset", "length", "version", "fourCC"]) ExternalLumpHeader = collections.namedtuple("ExternalLumpHeader", ["offset", "length", "version", "fourCC", "filename", "filesize"]) @@ -32,7 +35,7 @@ class Bsp: # ^ {"LUMP_NAME": Exception encountered} def __init__(self, branch: ModuleType, filename: str = "untitled.bsp", autoload: bool = True): - if not filename.endswith(".bsp"): + if not filename.lower().endswith(".bsp"): raise RuntimeError("Not a .bsp") filename = os.path.realpath(filename) self.folder, self.filename = os.path.split(filename) @@ -42,8 +45,9 @@ def __init__(self, branch: ModuleType, filename: str = "untitled.bsp", autoload: if os.path.exists(filename): self._preload() else: - print(f"{filename} not found, creating a new .bsp") + warnings.warn(UserWarning(f"{filename} not found, creating a new .bsp")) self.headers = {L.name: LumpHeader(0, 0, 0, 0) for L in self.branch.LUMP} + # NOTE: ^ this doesn't acount for some branches' alternate LumpHeader structs def __enter__(self): return self @@ -62,7 +66,7 @@ def _read_header(self, LUMP: enum.Enum) -> LumpHeader: offset, length, version, fourCC = struct.unpack("4I", self.file.read(16)) # TODO: use a read & write function / struct.iter_unpack # -- this could potentially allow for simplified subclasses - # -- e.g. LumpHeader(*struct.unpack("4I", self.file.read(16))) -> self.LumpHeader(self.file) + # -- e.g. LumpHeader(*struct.unpack("4I", self.file.read(16))) -> self.LumpHeader.from_file(self.file) header = LumpHeader(offset, length, version, fourCC) return header @@ -74,8 +78,7 @@ def is_related(f): return f.startswith(os.path.splitext(self.filename)[0]) # open .bsp self.file = open(os.path.join(self.folder, self.filename), "rb") file_magic = self.file.read(4) - if file_magic != self.file_magic: - raise RuntimeError(f"{self.file} is not a valid .bsp!") + assert file_magic == self.file_magic, f"{self.file} is not a valid .bsp!" self.bsp_version = int.from_bytes(self.file.read(4), "little") self.file.seek(0, 2) # move cursor to end of file self.bsp_file_size = self.file.tell() @@ -92,15 +95,15 @@ def is_related(f): return f.startswith(os.path.splitext(self.filename)[0]) try: if LUMP_NAME == "GAME_LUMP": GameLumpClasses = getattr(self.branch, "GAME_LUMP_CLASSES", dict()) - BspLump = lumps.GameLump(self.file, lump_header, GameLumpClasses) + BspLump = lumps.GameLump(self.file, lump_header, GameLumpClasses, self.branch.GAME_LUMP_HEADER) elif LUMP_NAME in self.branch.LUMP_CLASSES: LumpClass = self.branch.LUMP_CLASSES[LUMP_NAME][lump_header.version] BspLump = lumps.create_BspLump(self.file, lump_header, LumpClass) elif LUMP_NAME in self.branch.SPECIAL_LUMP_CLASSES: SpecialLumpClass = self.branch.SPECIAL_LUMP_CLASSES[LUMP_NAME][lump_header.version] - d_file, d_header = lumps.decompressed(self.file, lump_header) - d_file.seek(d_header.offset) - lump_data = d_file.read(d_header.length) + decompressed_file, decompressed_header = lumps.decompressed(self.file, lump_header) + decompressed_file.seek(decompressed_header.offset) + lump_data = decompressed_file.read(decompressed_header.length) BspLump = SpecialLumpClass(lump_data) elif LUMP_NAME in self.branch.BASIC_LUMP_CLASSES: LumpClass = self.branch.BASIC_LUMP_CLASSES[LUMP_NAME][lump_header.version] diff --git a/io_import_rbsp/bsp_tool/branches/README.md b/io_import_rbsp/bsp_tool/branches/README.md deleted file mode 100644 index bba0ac6..0000000 --- a/io_import_rbsp/bsp_tool/branches/README.md +++ /dev/null @@ -1,203 +0,0 @@ -# How bsp_tool loads .bsps -If you're using `bsp_tool.load_bsp("reallycool.bsp")` to load a `.bsp` a few things happen behind the scenes to figure out the format -Since bsp_tool supports a range of `.bsp` variants, a single script to handle the rough format wasn't going to cut it -To narrow down the exact format of a bsp file `load_bsp` looks at some key information in each file: - - -### Developer variants -First, `load_bsp` tries to determine the developer behind the chosen .bsp -If the file extension is `.d3dbsp`, it's a Call of Duty 2 or 4 `D3DBsp` -Other bsps use the `.bsp` extension (Call of Duty 1 included) -The developer is identified from the "file-magic", the first four bytes of any .bsp are: - - `b"IBSP"` for `IdTechBsp` Id Software - - `b"IBSP"` for `D3DBsp` Infinity Ward - - `b"rBSP"` for `RespawnBsp` Respawn - - `b"VBSP"` for `ValveBsp` Valve - -> This rule isn't perfect! most mods out there are Source Engine forks with b"VBSP" - -> GoldSrc .bsp files don't have any file magic! - -Most of the major differences between each developer's format are the number of lumps & bsp header -They also use some lumps which are unique to each developer's Quake based engine -More on those differences in an upcoming wiki page... - - -### Game variants -Once `load_bsp` knows the developer, it has to work out which game a `.bsp` comes from - -In the `.bsp` header there will always be a 4 byte int for the `.bsp` format version -Unfortunately this isn't totally unique from game to game, [most Source Engine titles use version 20](https://developer.valvesoftware.com/wiki/Source_BSP_File_Format#Versions) -This is where `load_bsp`'s second (optional!) argument comes in, `branch` - -`branch` can be either a string or a python script - -```python ->>> import bsp_tool - ->>> bsp_tool.load_bsp("tests/maps/pl_upward.bsp", branch=bsp_tool.branches.valve.orange_box) -Loading pl_upward.bsp (VBSP version 20)... -Loaded pl_upward.bsp - - ->>> bsp_tool.load_bsp("tests/maps/test_bigbox.bsp", branch="Quake3") -Loading test_bigbox.bsp (IBSP version 46)... -Loaded test_bigbox.bsp - - ->>> bsp_tool.load_bsp("tests/maps/test2.bsp") -Loading test2.bsp (VBSP version 20)... -Loaded test2.bsp - -``` -In the above example `bsp_tool.branches.valve.orange_box` points to [`bsp_tool/branches/valve/orange_box.py`](https://github.com/snake-biscuits/bsp_tool/blob/master/bsp_tool/branches/valve/orange_box.py) -This branch script is used to intialise the `Bsp` subclass chosen when `load_bsp` works out the developer - -When `branch` is a string, `load_bsp` uses `branch` as a key in the `bsp_tool.branches.by_name` dictionary to choose a script -Bsp classes take the branch script as their first argument and **do not have defaults** (except `ValveBsp`) - -When `branch` is `"unknown"` (default) the bsp format version is used as a key in the `bsp_tool.branches.by_version` dictionary - - - -# Branch scripts -Now that we know a branch script is needed to load a specific .bsp variant, why might we need to make one? -Well, bsp_tool doesn't cover a lot of formats, and those it does aren't mapped completely either! - -But with branch scripts you can develop a rough map of a particular format while copying definitions from other scripts -[`nexon/vindictus.py`](https://github.com/snake-biscuits/bsp_tool/blob/master/bsp_tool/branches/nexon/vindictus.py) for example, imports `valve.orange_box` and copies most of the format -This saves a lot of code! Especially since they only differ on the format of a handful of lumps and share a .bsp version - - -## Overall structure -The branch scripts that come with bsp_tool have a common format to make reading them as consistent as possible - -```python -import enum -from .. import base -from .. import shared # special lumps - -BSP_VERSION = 20 - -class LUMP(enum.Enum): - ENTITIES = 0 - AREAS = 20 - PAKFILE = 40 - -lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} - -# classes for each lump, in alphabetical order: [1 / 64] + shared.Entities & PakFile -class Area(base.Struct): # LUMP 20 - num_area_portals: int # number of AreaPortals after first_area_portal in this Area - first_area_portal: int # index of first AreaPortal - __slots__ = ["num_area_portals", "first_area_portal"] - _format = "2i" - -LUMP_CLASSES = {"AREAS": {0: Area}} - -SPECIAL_LUMP_CLASSES = {"ENTITIES": {0: shared.Entities}, - "PAKFILE": {0: shared.PakFile}} -``` - -If you compare [`bsp_tool/branches/valve/orange_box.py`](https://github.com/snake-biscuits/bsp_tool/blob/master/bsp_tool/branches/valve/orange_box.py) you'll see I've left a lot out here, but this basic batch script is a great start for translating any .bsp variant - -At the top we have the bsp format version, mostly as a note -Next comes the `LUMP` enum, this lists each lump in the order they appear in the bsp header - -> Lumps without LumpClasses are loaded as raw bytes -> Invalid LumpClasses will generate an error (silent, saved to bsp.loading_errors) -> Invalid lumps will still be loaded as raw bytes - -Attached to this we have `lump_header_address`, this connects each LUMP entry to the offset .bsp where it's header begins -Then comes the lump classes, these translate most lumps into python objects (more on them later) -We also have some special lump classes, these are loaded in a different way to other lumps, and some are shared across **almost all** bsp variants - -The Bsp class reads the headers for each lump and holds the contents in `Bsp.headers` -This dictionary of headers takes the name given in the branch scripts' `LUMP` class -Lump names are tied to a dictionary, which ties lump version (`int`) to LumpClass -From there, a lump is either saved as `Bsp.RAW_LUMPNAME` (bytes) or `Bsp.LUMPNAME` (List[LumpClass]) if it the lump is listed in `LUMP_CLASSES` - - -## Lump classes -A majority of lumps are very simple, being a list of fixed length structs -bsp_tool loads these lumps with python's built in `struct` module -`struct.iter_unpack` takes a format specifier string and a stream of bytes -This stream of bytes must contain a whole number of these structures or an error will be raised - -The lump class in the example is a subclass of [`bsp_tool.branches.base.Struct`](https://github.com/snake-biscuits/bsp_tool/blob/master/bsp_tool/branches/base.py#L5) -`base.Struct` exists to make defining a lump's specific format quick using very little code - -The definition usually has 3 parts: -```python -class LumpClass(base.struct): - __slots__ = ["unknown", "flags"] - _format = "5i" - _arrays = {"unknown": [*"abcd"]} -``` - -`__slots__` names the main attributes of the LumpClass -`_format` holds the format string for `struct.iter_unpack` -(I recommend also giving type hints for each attribute, so others don't have to work them out from `_format`) -`_arrays` is optional, it holds a dictionary for generating a `base.MappedArray` to help group attributes -For the most complex use of arrays (so far), see: [`branches.id_software.quake3.Face`](https://github.com/snake-biscuits/bsp_tool/blob/06f184b0cdf5133ea12ce8e0f5442398d6310d2a/bsp_tool/branches/id_software/quake3.py#L60) - -So the above example would turn the C struct: -```C -struct LumpClass { - int unknown_a; - int unknown_b; - int unknown_c; - int unknown_d; - int flags; -} -``` -into: -```python -LumpClass.unknown.a -LumpClass.unknown.b -LumpClass.unknown.c -LumpClass.unknown.d -LumpClass.flags -``` -Lump classes don't have to be subclasses of `base.Struct` though, the only requirement is the `_format` attribute -This is essential because each lump class is initialised with the tuple `struct.iter_unpack` returns for each struct -And to read these raw bytes `Bsp.load_lumps` uses something similar to `struct.iter_unpack(LumpClass._format, RAW_LUMP)` -If the tuple returned has a length of 0 `bsp.LUMP = list(map(LumpClass, [t[0] for t in tuples]))` -Else: `Bsp.LUMP = list(map(LumpClass, tuples))` -To support re-saving LumpClasses, a `.flat()` method is required, which must return a tuple near identical to the one it was made from (same types) - - -## Special lump classes -Not all lumps are as simple as a list of structs, and this is where special lump classes come in -Special lump classes are initialised from the raw bytes of a lump, turning them into python objects that are easier to work with -All that's really required is an `__init__` method and an `.as_bytes()` method for re-saving - -Here's `branches.shared.TextureDataStringData` as an example of how basic a special lump class can be: -```python -class TextureDataStringData(list): - def __init__(self, raw_texture_data_string_data): - super().__init__([t.decode("ascii", errors="ignore") for t in raw_texture_data_string_data.split(b"\0")]) - - def as_bytes(self): - return b"\0".join([t.encode("ascii") for t in self]) + b"\0" -``` -By inheriting `list` you can use all the features of python lists while still importing the data with `__init__` & saving it back with `.as_bytes()` -You can of course make more complex classes, like adding methods (though they won't be connected to their parent `Bsp`) -Speaking of methods - - -## Methods -While not listed in the example branch scripts, you can add methods to a `Bsp` with a branch script! -The only requirements are that you have a list of functions in `methods` somewhere in the script - -```python -def areaportals_of_area(bsp, area_index): - area = bsp.AREAS[area_index] - return bsp.AREA_PORTALS[area.first_areaportal:area.first_areaportal + area.num_areaportals] - - -methods = [areaportals_of_area] -``` - -These methods are attached when the `Bsp` is initialised -The only requirements for these functions is that the first argument be `bsp`, since as a method the `Bsp` will pass itself to the function diff --git a/io_import_rbsp/bsp_tool/branches/__init__.py b/io_import_rbsp/bsp_tool/branches/__init__.py index 1947a4f..5327db0 100644 --- a/io_import_rbsp/bsp_tool/branches/__init__.py +++ b/io_import_rbsp/bsp_tool/branches/__init__.py @@ -1,179 +1,70 @@ -__all__ = ["arkane", "gearbox", "id_software", "infinity_ward", "nexon", "respawn", "valve", - "by_magic", "by_name", "by_version"] +"""Index of all known .bsp format variants""" +__all__ = ["arkane", "gearbox", "id_software", "infinity_ward", "nexon", + "raven", "respawn", "ritual", "scripts_from_file_magic", "game_path_table"] from . import arkane from . import gearbox from . import id_software from . import infinity_ward +from . import ion_storm from . import nexon +from . import raven from . import respawn +from . import ritual +from . import troika from . import valve +# TODO: xatrix.kingpin +# ^ https://github.com/QuakeTools/Kingpin-SDK-v1.21 +# (Kingpin allegedly has it's own KRadiant "on the CD") -__doc__ = """Index of developers of bsp format variants""" +# NOTE: this dict can be generated from branch_scripts, but listing it here is more convenient +scripts_from_file_magic = {None: [id_software.quake, + *gearbox.scripts, + raven.hexen2, + valve.goldsrc], + b"2015": [ritual.moh_allied_assault], + b"EF2!": [ritual.star_trek_elite_force2], + b"FAKK": [ritual.fakk2], + b"IBSP": [id_software.quake2, + id_software.quake3, + *infinity_ward.scripts, + # NOTE: most of infinity_ward.scripts will be *.d3dbsp + ion_storm.daikatana, + raven.soldier_of_fortune, + ritual.sin], + b"rBSP": [*respawn.scripts], + b"RBSP": [raven.soldier_of_fortune2, + ritual.sin], + b"VBSP": [*arkane.scripts, + *nexon.scripts, + *troika.scripts, + *[s for s in valve.scripts if (s is not valve.goldsrc)]]} -FILE_MAGIC_developer = {b"IBSP": [id_software, infinity_ward], - b"rBSP": [respawn], - b"VBSP": [arkane, nexon, valve]} -# id_software.quake, valve.goldsrc & gearbox.bshift have no file_magic -# NOTE: bsp_tool/__init__.py: load_bsp can be fed a string to select the relevant branch script -# by_name is searched with a lowercase, numbers & letters only version of that string -# NOTE: some (but not all!) games are listed here have multiple valid names (including internal mod names) -# TODO: generate from branch.GAMES (folder names) -# TODO: handle branchscripts with multiple bsp_versions elegantly +script_from_file_magic_and_version = dict() +# ^ {(file_magic, version): branch_script} +for file_magic, branch_scripts in scripts_from_file_magic.items(): + for branch_script in branch_scripts: + for version in branch_script.GAME_VERSIONS.values(): + script_from_file_magic_and_version[(file_magic, version)] = branch_script +# FORCED DEFAULTS: +script_from_file_magic_and_version[(b"IBSP", 46)] = id_software.quake3 +# ^ NOT raven.soldier_of_fortune +script_from_file_magic_and_version[(b"VBSP", 20)] = valve.orange_box +# ^ NOT nexon.vindictus OR valve.left4dead +script_from_file_magic_and_version[(b"VBSP", 21)] = valve.sdk_2013 +# ^ NOT valve.alien_swarm OR valve.left4dead2 +script_from_file_magic_and_version[(b"VBSP", 100)] = nexon.cso2 +# ^ NOT nexon.cso2_2018 +script_from_file_magic_and_version[(b"RBSP", 1)] = raven.soldier_of_fortune2 +# ^ NOT ritual.sin -def simplify_name(name: str) -> str: - """'Counter-Strike: Online 2' -> 'counterstrikeonline2'""" - return "".join(filter(str.isalnum, name.lower())) - -by_name = { - # Id Software - Id Tech - "fortress": id_software.quake, - "hexenii": id_software.quake, - "quake": id_software.quake, - "quakeii": id_software.quake2, - "quake2": id_software.quake2, - "quake3": id_software.quake3, - "quakeiii": id_software.quake3, - # TODO: Quake 4 - # TODO: Quake Champions - "quakelive": id_software.quake3, - # Infinity Ward - IW Engine(s) - "callofduty": infinity_ward.call_of_duty1, - "cod": infinity_ward.call_of_duty1, - # TODO: Call of Duty 2 - # TODO: Call of Duty 4 - # NEXON - Source Engine - "counterstrikeonline2": nexon.cso2, - "cso2": nexon.cso2, - "csonline2": nexon.cso2, - "mabinogi": nexon.vindictus, - "mabinogiheroes": nexon.vindictus, - "vindictus": nexon.vindictus, - # Respawn Entertainment - Titanfall Engine - "titanfall": respawn.titanfall, - "titanfall2": respawn.titanfall2, - "apexlegends": respawn.apex_legends, - "apex": respawn.apex_legends, - "r1": respawn.titanfall, # internal names - "r2": respawn.titanfall2, - "r5": respawn.apex_legends, - # Valve Software - Source Engine - "source": valve.source, - "orangebox": valve.orange_box, - "2013sdk": valve.sdk_2013, - "sourcemods": valve.sdk_2013, - "alienswarm": valve.alien_swarm, - "alienswarmreactivedrop": valve.alien_swarm, - "blackmesa": valve.sdk_2013, - "bladesymphony": valve.sdk_2013, - "counterstrikeglobaloffensive": valve.sdk_2013, - "counterstrikesource": valve.source, - "csgo": valve.sdk_2013, - "css": valve.source, - "cssource": valve.source, - "dayofdefeatsource": valve.orange_box, - "dods": valve.orange_box, - "episode1": valve.orange_box, - "episode2": valve.orange_box, - "episodic": valve.orange_box, - "fortressforever": valve.orange_box, - "garrysmod": valve.orange_box, - "globaloffensive": valve.sdk_2013, - "gmod": valve.orange_box, - "gstring": valve.orange_box, # awesome sourcemod - "halflife1sourcedeathmatch": valve.source, - "halflife2": valve.source, - "halflife2ep1": valve.source, - "halflife2ep2": valve.orange_box, - "halflife2episode1": valve.source, - "halflife2episode2": valve.orange_box, - "halflife2episodic": valve.source, - "halflife2hl1": valve.source, - "halflifesource": valve.source, - "hl2": valve.source, - "hl2ep1": valve.source, - "hl2ep2": valve.orange_box, - "hls": valve.source, - "hlsource": valve.source, - "l4d": valve.left4dead, - "l4d2": valve.left4dead2, - "left4dead": valve.left4dead, - "left4dead2": valve.left4dead2, - "neotokyo": valve.orange_box, - "portal": valve.orange_box, - "portal2": valve.sdk_2013, - "portalreloaded": valve.sdk_2013, - "sourcefilmmaker": valve.sdk_2013, - "synergy": valve.source, - "teamfortress2": valve.orange_box, - "tf2": valve.orange_box, - # Valve Software - GoldSrc Engine (more mods @ https://half-life.fandom.com/wiki/Mods) - "goldsrc": valve.goldsrc, - "007nightfire": valve.goldsrc, # untested - "blueshift": gearbox.bshift, - "bshift": gearbox.bshift, - "counterstrike": valve.goldsrc, # CS 1.6 - "counterstrikeconditionzero": valve.goldsrc, - "counterstrikeneo": valve.goldsrc, # obscure & untested - "counterstrikeonline": valve.goldsrc, # obscure & untested - "cs": valve.goldsrc, # CS 1.6 - "cscz": valve.goldsrc, - "csn": valve.goldsrc, # obscure & untested - "cso": valve.goldsrc, # obscure & untested - "dayofdefeat": valve.goldsrc, - "deathmatchclassic": valve.goldsrc, - "dmc": valve.goldsrc, - "dod": valve.goldsrc, - "gunmanchronicles": valve.goldsrc, - "halflife": valve.goldsrc, - "halflifeblueshift": gearbox.bshift, - "halflifebshift": gearbox.bshift, - "halflifericochet": valve.goldsrc, - "halflifecstrike": valve.goldsrc, - "halflifeopposingforce": valve.goldsrc, - "halfquaketrilogy": valve.goldsrc, - "hlblueshift": gearbox.bshift, - "hlopposingforce": valve.goldsrc, - "jamesbond007nightfire": valve.goldsrc, # untested - "nightfire": valve.goldsrc, - "opposingforce": valve.goldsrc, - "ricochet": valve.goldsrc, - "svencoop": valve.goldsrc, - "teamfortressclassic": valve.goldsrc, - "tfc": valve.goldsrc - } - -# NOTE: limiting because many games share version numbers -# could also be generated from loaded scripts -by_version = { - # Id Software - id_software.quake.BSP_VERSION: id_software.quake, # 23 - id_software.quake2.BSP_VERSION: id_software.quake2, # 38 - id_software.quake3.BSP_VERSION: id_software.quake3, # 46 - # Infinity Ward - infinity_ward.call_of_duty1.BSP_VERSION: infinity_ward.call_of_duty1, # 59 - # Nexon - nexon.cso2.BSP_VERSION: nexon.cso2, # 1.00? - # Respawn Entertainment - respawn.titanfall.BSP_VERSION: respawn.titanfall, # 29 - respawn.titanfall2.BSP_VERSION: respawn.titanfall2, # 37 - respawn.apex_legends.BSP_VERSION: respawn.apex_legends, # 47 - 48: respawn.apex_legends, # Introduced in Season 7 with Olympus - 49: respawn.apex_legends, # Introduced in Season 8 with Canyonlands Staging - 50: respawn.apex_legends, # Introduced in Season 10 with Arena Skygarden - # Valve Software - valve.goldsrc.BSP_VERSION: valve.goldsrc, # can't really use this system for GoldSrc - valve.source.BSP_VERSION: valve.source, # 19 (and sometimes 20?) - valve.orange_box.BSP_VERSION: valve.orange_box, # 20 (many sub-variants) - valve.sdk_2013.BSP_VERSION: valve.sdk_2013, # 21 - # Other - # arkane.dark_messiah.BSP_VERSION: arkane.dark_messiah, # 20.4 ? - gearbox.bshift.BSP_VERSION: gearbox.bshift # 30 - } - -# NOTE: ata4's bspsrc uses unique entity classnames to identify branches -# -- might be an idea to copy this for an autodetect refactor +game_path_table = dict() +# ^ {"game": (script, version)} +for developer in (arkane, gearbox, id_software, infinity_ward, nexon, raven, respawn, ritual, valve): + for script in developer.scripts: + for game_path in script.GAME_PATHS: + game_path_table[game_path] = (script, script.GAME_VERSIONS[game_path]) diff --git a/io_import_rbsp/bsp_tool/branches/arkane/__init__.py b/io_import_rbsp/bsp_tool/branches/arkane/__init__.py index 0cb2457..1a599ed 100644 --- a/io_import_rbsp/bsp_tool/branches/arkane/__init__.py +++ b/io_import_rbsp/bsp_tool/branches/arkane/__init__.py @@ -1,7 +1,7 @@ -__all__ = ["dark_messiah"] +"""Arkane Studios made a number of Source Engine powered projects. +Few made it to release.""" +from . import dark_messiah_multiplayer +from . import dark_messiah_singleplayer -from . import dark_messiah -__doc__ = """Arkane Studios made a number of Source Engine powered projects. Few released.""" - -FILE_MAGIC = "VBSP" +scripts = [dark_messiah_multiplayer, dark_messiah_singleplayer] diff --git a/io_import_rbsp/bsp_tool/branches/arkane/dark_messiah.py b/io_import_rbsp/bsp_tool/branches/arkane/dark_messiah_multiplayer.py similarity index 55% rename from io_import_rbsp/bsp_tool/branches/arkane/dark_messiah.py rename to io_import_rbsp/bsp_tool/branches/arkane/dark_messiah_multiplayer.py index 23681e7..509d2e5 100644 --- a/io_import_rbsp/bsp_tool/branches/arkane/dark_messiah.py +++ b/io_import_rbsp/bsp_tool/branches/arkane/dark_messiah_multiplayer.py @@ -2,15 +2,22 @@ import enum import struct -from ..valve import orange_box, source +from ..valve import orange_box +from ..valve import source -BSP_VERSION = 20 -# NOTE: BSP_VERSION is stored as 2 shorts? +FILE_MAGIC = b"VBSP" + +BSP_VERSION = (20, 4) # int.from_bytes(struct.pack("2H", 20, 4), "little") + +GAME_PATHS = ["Dark Messiah of Might and Magic Multi-Player"] + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} -GAMES = ["Dark Messiah of Might and Magic"] LUMP = orange_box.LUMP + +# struct DarkMessiahBspHeader { char file_magic[4]; short version[2]; SourceLumpHeader headers[64]; int revision;}; lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} @@ -20,10 +27,7 @@ def read_lump_header(file, LUMP: enum.Enum) -> source.SourceLumpHeader: header = source.SourceLumpHeader(offset, length, version, fourCC) return header -# classes for lumps, in alphabetical order: -# TODO: dheader_t, texinfo_t, dgamelump_t, dmodel_t -# classes for special lumps, in alphabetical order: # TODO: StaticPropLumpv6 @@ -31,10 +35,14 @@ def read_lump_header(file, LUMP: enum.Enum) -> source.SourceLumpHeader: BASIC_LUMP_CLASSES = orange_box.BASIC_LUMP_CLASSES.copy() LUMP_CLASSES = orange_box.LUMP_CLASSES.copy() +LUMP_CLASSES.pop("WORLD_LIGHTS") # sdk_2013? +LUMP_CLASSES.pop("WORLD_LIGHTS_HDR") SPECIAL_LUMP_CLASSES = orange_box.SPECIAL_LUMP_CLASSES.copy() -# GAME_LUMP_CLASSES = {"sprp": {6: lambda raw_lump: shared.GameLump_SPRP(raw_lump, StaticPropLumpv6)}} +GAME_LUMP_HEADER = orange_box.GAME_LUMP_HEADER + +GAME_LUMP_CLASSES = {"sprp": {6: lambda raw_lump: source.GameLump_SPRP(raw_lump, None)}} methods = [*orange_box.methods] diff --git a/io_import_rbsp/bsp_tool/branches/arkane/dark_messiah_singleplayer.py b/io_import_rbsp/bsp_tool/branches/arkane/dark_messiah_singleplayer.py new file mode 100644 index 0000000..c31a4a9 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/arkane/dark_messiah_singleplayer.py @@ -0,0 +1,95 @@ +# https://developer.valvesoftware.com/wiki/Source_BSP_File_Format/Game-Specific#Dark_Messiah_of_Might_and_Magic +import enum +import struct +from typing import List + +from .. import base +from ..valve import orange_box +from ..valve import source + + +FILE_MAGIC = b"VBSP" + +BSP_VERSION = (20, 4) # int.from_bytes(struct.pack("2H", 20, 4), "little") + +GAME_PATHS = ["Dark Messiah of Might and Magic Single Player"] + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} + + +LUMP = orange_box.LUMP + +# struct DarkMessiahBspHeader { char file_magic[4]; short version[2]; SourceLumpHeader headers[64]; int revision;}; +lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} + + +def read_lump_header(file, LUMP: enum.Enum) -> source.SourceLumpHeader: + file.seek(lump_header_address[LUMP]) + offset, length, version, fourCC = struct.unpack("4I", file.read(16)) + header = source.SourceLumpHeader(offset, length, version, fourCC) + return header + + +# classes for lumps, in alphabetical order: +class Model(base.Struct): # LUMP 14 + bounds: List[float] + # bounds.mins: List[float] # xyz + # bounds.maxs: List[float] # xyz + origin: List[float] + unknown: int + head_node: int # index into Node lump + first_face: int # index into Face lump + num_faces: int # number of Faces after first_face in this Model + __slots__ = ["bounds", "origin", "unknown", "head_node", "first_face", "num_faces"] + _format = "9f4i" + _arrays = {"bounds": {"mins": [*"xyz"], "maxs": [*"xyz"]}} + + +class TextureInfo(base.Struct): # LUMP 6 + """Texture projection info & index into TEXTURE_DATA""" + texture: List[List[float]] # 2 texture projection vectors + lightmap: List[List[float]] # 2 lightmap projection vectors + unknown: bytes + flags: int # Surface flags + texture_data: int # index of TextureData + __slots__ = ["texture", "lightmap", "flags", "texture_data"] + _format = "16f24s2i" + _arrays = {"texture": {"s": [*"xyz", "offset"], "t": [*"xyz", "offset"]}, + "lightmap": {"s": [*"xyz", "offset"], "t": [*"xyz", "offset"]}} + # ^ nested MappedArrays; texture.s.x, texture.t.x + + +# classes for special lumps, in alphabetical order: +class GameLumpHeader(base.MappedArray): + id: str + flags: int + version: int + offset: int + length: int + unknown: int + _mapping = ["id", "flags", "version", "offset", "length", "unknown"] + _format = "4s2H3i" + + +# TODO: StaticPropLumpv6 + + +# {"LUMP_NAME": {version: LumpClass}} +BASIC_LUMP_CLASSES = orange_box.BASIC_LUMP_CLASSES.copy() + +LUMP_CLASSES = orange_box.LUMP_CLASSES.copy() +LUMP_CLASSES.pop("MODELS") +LUMP_CLASSES.pop("TEXTURE_INFO") +LUMP_CLASSES.pop("WORLD_LIGHTS") +LUMP_CLASSES.pop("WORLD_LIGHTS_HDR") +# LUMP_CLASSES.update({"MODELS": {0: Model}, +# "TEXTURE_INFO": {0: TextureInfo}}) + +SPECIAL_LUMP_CLASSES = orange_box.SPECIAL_LUMP_CLASSES.copy() + +GAME_LUMP_HEADER = GameLumpHeader + +GAME_LUMP_CLASSES = {"sprp": {6: lambda raw_lump: source.GameLump_SPRP(raw_lump, None)}} + + +methods = [*orange_box.methods] diff --git a/io_import_rbsp/bsp_tool/branches/base.py b/io_import_rbsp/bsp_tool/branches/base.py index d3185c8..ff9cf24 100644 --- a/io_import_rbsp/bsp_tool/branches/base.py +++ b/io_import_rbsp/bsp_tool/branches/base.py @@ -1,41 +1,74 @@ """Base classes for defining .bsp lump structs""" +from __future__ import annotations +import re +import struct from typing import Any, Dict, Iterable, List, Union +# TODO: _decode: Dict[str, BytesDecodeArgs] class variable for both Struct & MappedArray +# ^ BytesDecodeArgs = Dict[str, str] +# ^^ {"encoding": "utf-8", "errors": "strict"} -> bytes(...).decode(encoding="utf-8", errors="strict") +# TODO: LumpClass(**{"attr.sub": value}) & MappedArray(**{"attr.sub": value}) +# TODO: _subclass: Dict[str, Any] class variable for both Struct & MappedArray +# ^ {"attr": SubClass, "attr2.sub": SubClass} +# child_MappedArray = ...; SubClass.__init__(child_MappedArray) +# allows for nesting vector.Vec3 in Structs + class Struct: """base class for tuple <-> class conversion bytes <-> tuple conversion is handled by the struct module""" __slots__: List[str] = list() # names of atributes, in order + # NOTE: since we are using __slots__, defaults cannot be set as class variables + # -- override the _defaults() classmethod instead (use super & modify the returned dict) _format: str = str() # struct module format string _arrays: Dict[str, Any] = dict() # slots to be mapped into MappedArrays # each value in _arrays is a mapping to generate a MappedArray from + # TODO: _child_subclasses: dict[str, Any] + # e.g. {"plane.normal": vector.Vec3} - def __init__(self, _tuple: Iterable): - # _tuple comes from: struct.unpack(self._format, bytes) - _tuple_index = 0 - for attr in self.__slots__: + def __init__(self, *args, **kwargs): + # LumpClass(attr1, [attr2_1, attr2_2]) + # LumpClass(attr1, attr2=[attr2_1, attr2_2]) + # NOTE: can only set top-level value, no partial init of nested attr via kwargs (yet) + # UNLESS: LumpClass(attr3=MappedArray(attr3_x=value, _mapping=LumpClass._arrays["attr3"]) + # BETTER: LumpClass(attr3_x=value) # parse kwarg as attr3.x & generate attr3 MappedArray (expensive!) + # BEST: LumpClass(**{"attr3.x": value}) # no chance of overlapping attr names + assert len(args) <= len(self.__slots__), "Too many arguments! Should match top level attributes!" + invalid_kwargs = set(kwargs).difference(set(self.__slots__)) + # TODO: could branch here and check for subattr kwargs + assert len(invalid_kwargs) == 0, f"Invalid kwargs: {invalid_kwargs}" + if len(args) == len(self.__slots__): # cheeky recursion avoider + default_values = dict() + # NOTE: could also skip generating defaults if arg + kwargs defines the whole struct + # -- however that's probably more work to detect than could be saved so \_(0.0)_/ + else: + default_values = self._defaults() + # ^ {"attr": value} + default_values.update(dict(zip(self.__slots__, args))) + default_values.update(kwargs) + parsed = dict() + types = split_format(self._format) + for attr, value in default_values.items(): if attr not in self._arrays: - value = _tuple[_tuple_index] - length = 1 + setattr(self, attr, value) + continue + mapping = self._arrays[attr] + if isinstance(value, MappedArray): + # TODO: the mappings don't quite match? + # value._mapping seems to only be the top level? so == wont work on nested MappedArrays... + assert value._mapping == mapping + setattr(self, attr, value) + elif isinstance(mapping, int): + assert len(value) == mapping + setattr(self, attr, value) + elif isinstance(mapping, (list, dict)): + start = mapping_length(parsed) + length = mapping_length({None: self._arrays.get(attr)}) + child_format = "".join(types[start:start + length]) + setattr(self, attr, MappedArray.from_tuple(value, _format=child_format, _mapping=mapping)) else: - array_map = self._arrays[attr] - if isinstance(array_map, list): # array_map: List[str] - length = len(array_map) - array = _tuple[_tuple_index:_tuple_index + length] - value = MappedArray(array, mapping=array_map) - elif isinstance(array_map, dict): # array_map: Dict[str, List[str]] - length = mapping_length(array_map) - array = _tuple[_tuple_index:_tuple_index + length] - value = MappedArray(array, mapping=array_map) # nested - elif isinstance(array_map, int): - length = array_map - value = _tuple[_tuple_index:_tuple_index + length] - else: - raise RuntimeError(f"{type(array_map)} {array_map} in {self.__class__.__name__}._arrays") - setattr(self, attr, value) - _tuple_index += length - # TODO: throw a warning if the whole tuple won't fit - # report len(_tuple) & struct.calcsize(self._format) + raise RuntimeError(f"{self.__class__.__name__} has bad _arrays!") + parsed[attr] = self._arrays.get(attr) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -66,56 +99,135 @@ def flat(self) -> list: _tuple.append(value) return _tuple + @classmethod + def _defaults(cls) -> Dict[str, Any]: + types = split_format(cls._format) + global type_defaults + defaults = cls.from_tuple([type_defaults[t] if not t.endswith("s") else "" for t in types]) + return dict(zip(cls.__slots__, defaults)) -def mapping_length(mapping: Dict[str, Any]) -> int: + # convertors + @classmethod + def from_bytes(cls, _bytes: bytes) -> Struct: + assert len(_bytes) == struct.calcsize(cls.format) + _tuple = struct.unpack(cls._format, _bytes) + expected_length = len(cls.__slots__) + mapping_length(cls._arrays) - len(cls._arrays) + assert len(_tuple) == expected_length + # TODO: ^ test + return cls.from_tuple(_tuple) + + @classmethod + def from_tuple(cls, _tuple: Iterable) -> Struct: + """_tuple comes from: struct.unpack(self._format, bytes)""" + out_args = list() + types = split_format(cls._format) + _tuple_index = 0 + for attr in cls.__slots__: + if attr not in cls._arrays: + value = _tuple[_tuple_index] + length = 1 + else: + # partition up children + child_mapping = cls._arrays[attr] + if isinstance(child_mapping, (list, dict)): # child_mapping: List[str] + length = len(child_mapping) if isinstance(child_mapping, list) else mapping_length(child_mapping) + array = _tuple[_tuple_index:_tuple_index + length] + child_format = "".join(types[_tuple_index:_tuple_index + length]) + value = MappedArray.from_tuple(array, _mapping=child_mapping, _format=child_format) + elif isinstance(child_mapping, int): + length = child_mapping + value = _tuple[_tuple_index:_tuple_index + length] + else: + raise RuntimeError(f"Invalid type: {type(child_mapping)} in {cls.__class__.__name__}._arrays") + out_args.append(value) + _tuple_index += length + return cls(*out_args) + + def as_bytes(self) -> bytes: + return struct.pack(self._format, *self.flat()) + + @classmethod + def as_cpp(self) -> str: # C++ struct definition + # TODO: move py_struct_as_cpp here, or import & map + raise NotImplementedError() + + +# mapping_length of Struct = mapping)length({s: Struct._arrays.get(s) for s in Struct.__slots__}) +def mapping_length(mapping: Union[List[str], Dict[str, Any], None]) -> int: + if isinstance(mapping, list): + return len(mapping) length = 0 - for sub_mapping in mapping.values(): - if isinstance(sub_mapping, list): - length += len(sub_mapping) - elif isinstance(sub_mapping, int): - length += sub_mapping - elif isinstance(sub_mapping, dict): - length += mapping_length(sub_mapping) - elif sub_mapping is None: + for child_mapping in mapping.values(): + if isinstance(child_mapping, list): + length += len(child_mapping) + elif isinstance(child_mapping, int): + length += child_mapping + elif isinstance(child_mapping, dict): + length += mapping_length(child_mapping) + elif child_mapping is None: length += 1 else: - raise RuntimeError(f"Unexpexted Mapping! ({mapping}, {sub_mapping})") + raise RuntimeError(f"Unexpexted Mapping! ({mapping}, {child_mapping})") return length class MappedArray: """Maps a given iterable to a series of names, can even be a nested mapping""" - _mapping: Union[List[str], Dict[str, Any]] = [*"xyz"] + _format: str = "" + _mapping: Union[List[str], Dict[str, Any]] = [] # _mapping cane be either a list of attr names to map a given array to, # or, a dict containing a list of attr names, or another dict # this second form is difficult to express as a type hint - def __init__(self, array: Iterable, mapping: Any = None): - if mapping is None: - mapping = self._mapping # hack to use a default - if isinstance(mapping, dict): - self._mapping = list(mapping.keys()) - array_index = 0 - for attr, child_mapping in mapping.items(): - # TODO: child_mapping of type int takes a slice, storing a mutable list - if child_mapping is not None: - segment = array[array_index:array_index + mapping_length({None: child_mapping})] - array_index += len(child_mapping) - child = MappedArray(segment, mapping=child_mapping) # will recurse again if child_mapping is a dict - else: # if {"attr": None} - array_index += 1 - child = array[array_index] # take a single item, not a slice - setattr(self, attr, child) - elif isinstance(mapping, list): # List[str] - for attr, value in zip(mapping, array): - setattr(self, attr, value) - self._mapping = mapping + # TODO: test subclass definitions (MappedArray, vector.Vec3) + + def __init__(self, *args, _mapping: Any = None, _format: str = None, **kwargs): + if _format is None: + _format = self._format + self._format = _format + if _mapping is None: + _mapping = self._mapping + self._mapping = _mapping + assert len(args) <= len(_mapping), "Too many arguments! Should match top level attributes!" + # NOTE: _mapping & _format might be passed in as regular args + # TODO: check to see if _mapping or _mapping, _format are on the tail of args + invalid_kwargs = set(kwargs).difference(set(_mapping)) + # TODO: could branch here and check for subattr kwargs; e.g. "x.i" + assert len(invalid_kwargs) == 0, f"Invalid kwargs: {invalid_kwargs}" + if len(args) == len(_mapping): # cheeky recursion avoider + default_values = dict() + # NOTE: could also skip generating defaults if arg + kwargs defines the whole struct + # -- however that's probably more work to detect than could be saved so \_(0.0)_/ else: - raise RuntimeError(f"Unexpected mapping: {type(mapping)}") + default_values = self._defaults(_mapping=_mapping, _format=_format) + # ^ {"attr": value} + default_values.update(dict(zip(_mapping, args))) + default_values.update(kwargs) + if isinstance(_mapping, list): + for attr, value in default_values.items(): + setattr(self, attr, value) + return + for attr, value in default_values.items(): + child_mapping = _mapping[attr] + if isinstance(value, MappedArray): + assert isinstance(child_mapping, (list, dict)), f"Invalid child_mapping for {attr}: {child_mapping}" + assert value._mapping == child_mapping # depth doesn't match? + setattr(self, attr, value) + elif isinstance(child_mapping, int): + assert len(value) == child_mapping + setattr(self, attr, value) + elif isinstance(child_mapping, (list, dict)): + setattr(self, attr, MappedArray.from_tuple(value, _mapping=child_mapping)) + elif child_mapping is None: + setattr(self, attr, value) + else: + raise RuntimeError(f"{self.__class__.__name__} has bad _mapping") def __eq__(self, other: Iterable) -> bool: return all([(a == b) for a, b in zip(self, other)]) + # TODO: __getattr__ swizzle detection (if and only if mapping is a list of single chars) + def __getitem__(self, index: str) -> Any: return getattr(self, self._mapping[index]) @@ -141,3 +253,123 @@ def flat(self) -> list: else: array.append(value) return array + + @classmethod + def _defaults(cls, _mapping: Any = None, _format: str = None) -> Dict[str, Any]: + if _format is None: + _format = cls._format + if _mapping is None: + _mapping = cls._mapping + types = split_format(_format) + assert mapping_length(_mapping) == len(types), "Invalid mapping for format!" + global type_defaults + # TODO: allow defautlt strings (requires a type_defaults function (see below)) + # -- pass down type_defaults _string_mode (warn / trim / fail) ? + defaults = cls.from_tuple([type_defaults[t] if not t.endswith("s") else "" for t in types], + _mapping=_mapping, _format=_format) + return dict(zip(list(_mapping), defaults)) + + # convertors + @classmethod + def from_bytes(cls, _bytes: bytes, _mapping: Any = None, _format: str = None) -> MappedArray: + if _format is None: + _format = cls._format + if _mapping is None: + _mapping = cls._mapping + assert len(_bytes) == struct.calcsize(_format) + _tuple = struct.unpack(_format, _bytes) + assert len(_tuple) == mapping_length(_mapping), f"{_tuple}" + return cls.from_tuple(_tuple, _mapping=_mapping, _format=_format) + + @classmethod + def from_tuple(cls, array: Iterable, _mapping: Any = None, _format: str = None) -> MappedArray: + if _format is None: + _format = cls._format + if _mapping is None: + _mapping = cls._mapping + assert len(array) == mapping_length(_mapping), f"{cls.__name__}({array}, _mapping={_mapping})" + out_args = list() + if not isinstance(_mapping, (dict, list)): + raise RuntimeError(f"Unexpected mapping: {type(_mapping)}") + elif isinstance(_mapping, dict): + types = split_format(_format) + array_index = 0 + for child_mapping in _mapping.values(): + # TODO: child_mapping of type int takes a slice, storing a mutable list + if child_mapping is not None: + length = mapping_length({None: child_mapping}) + segment = array[array_index:array_index + length] + child_format = "".join(types[array_index:array_index + length]) + array_index += len(child_mapping) + child = MappedArray.from_tuple(segment, _mapping=child_mapping, _format=child_format) + # NOTE: ^ will recurse again if child_mapping is a dict + else: # if {"attr": None} + array_index += 1 + child = array[array_index] # take a single item, not a slice + out_args.append(child) + elif isinstance(_mapping, list): # List[str] + out_args = array + out = cls(*out_args, _mapping=_mapping, _format=_format) + return out + + def as_bytes(self) -> bytes: + return struct.pack(self._format, *self.flat()) + + @classmethod + def as_cpp(cls, _mapping: Any = None, _format: str = None) -> str: # C++ struct definition + if _format is None: + _format = cls._format + if _mapping is None: + _mapping = cls._mapping + types = split_format(_format) + assert mapping_length(_mapping) == len(types), "Invalid mapping for format!" + raise NotImplementedError() + # out = list() + # out.append("struct {cls.__name__}" + "{\n") + # if isinstance(cls._mapping, dict): + # i = 0 + # for attr, attr_mapping in cls._mapping.item(): + # if child_mapping is None: + # type_char = types[i] + # i += 1 + # elif isinstance(cls._mapping, list): + # for type_char, attr in zip(split_format(_format), cls._mapping): + # out.append(f"\t{type_LUT[type_char]} {attr}\n") + # else: + # raise RuntimeError(f"Invalid _mapping type: {type(cls._mapping)}") + # out.append("};\n") + + +def split_format(_format: str) -> List[str]: + """split a struct format string to zip with tuple""" + # NOTE: strings returned as r"/d+s" + # FIXME: does not check to see if format is valid! invalid chars are thrown out silently + _format = re.findall(r"[0-9]*[xcbB\?hHiIlLqQnNefgdspP]", _format.replace(" ", "")) + out = list() + for f in _format: + match_numbered = re.match(r"([0-9]+)([xcbB\?hHiIlLqQnNefgdpP])", f) + # NOTE: does not decompress strings + if match_numbered is not None: + count, f = match_numbered.groups() + out.extend(f * int(count)) + else: + out.append(f) + return out + + +type_LUT = {"c": "char", "?": "bool", + "b": "char", "B": "unsigned char", + "h": "short", "H": "unsigned short", + "i": "int", "I": "unsigned int", + "f": "float", "g": "double"} +# NOTE: can't detect strings with a dict +# -- to catch strings: type_defaults[t] if not t.endswith("s") else "" +# TODO: make a function to lookup type and check / trim string sizes +# -- a trim / warn / fail setting would be ideal + +type_defaults = {"c": b"", "?": False, + "b": 0, "B": 0, + "h": 0, "H": 0, + "i": 0, "I": 0, + "f": 0.0, "g": 0.0, + "s": ""} diff --git a/io_import_rbsp/bsp_tool/branches/gearbox/__init__.py b/io_import_rbsp/bsp_tool/branches/gearbox/__init__.py index c7572d9..d41a5f3 100644 --- a/io_import_rbsp/bsp_tool/branches/gearbox/__init__.py +++ b/io_import_rbsp/bsp_tool/branches/gearbox/__init__.py @@ -1,5 +1,6 @@ -__all__ = ["bshift"] +"""Gearbox's second Half-Life expansion: Blue Shift made a few changes to the GoldSrc engine.""" +from . import blue_shift +from . import nightfire -from . import bshift -__doc__ = """Gearbox's second Half-Life expansion: Blue Shift made a few changes to the GoldSrc engine.""" +scripts = [blue_shift, nightfire] diff --git a/io_import_rbsp/bsp_tool/branches/gearbox/blue_shift.py b/io_import_rbsp/bsp_tool/branches/gearbox/blue_shift.py new file mode 100644 index 0000000..613c041 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/gearbox/blue_shift.py @@ -0,0 +1,49 @@ +# https://valvedev.info/tools/bspfix/ +import enum + +from ..valve import goldsrc + + +FILE_MAGIC = None + +BSP_VERSION = 30 + +GAME_PATHS = ["Half-Life/blue_shift"] # Half-Life: Blue Shift + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} + + +class LUMP(enum.Enum): + PLANES = 0 + ENTITIES = 1 + MIP_TEXTURES = 2 + VERTICES = 3 + VISIBILITY = 4 + NODES = 5 + TEXTURE_INFO = 6 + FACES = 7 + LIGHTING = 8 + CLIP_NODES = 9 + LEAVES = 10 + MARK_SURFACES = 11 + EDGES = 12 + SURFEDGES = 13 + MODELS = 14 + + +# struct QuakeBspHeader { int version; QuakeLumpHeader headers[15]; }; +lump_header_address = {LUMP_ID: (4 + i * 8) for i, LUMP_ID in enumerate(LUMP)} + +# Known lump changes from GoldSrc -> Blue Shift: +# ENTITIES -> PLANES +# PLANES -> ENTITIES + + +BASIC_LUMP_CLASSES = goldsrc.BASIC_LUMP_CLASSES.copy() + +LUMP_CLASSES = goldsrc.LUMP_CLASSES.copy() + +SPECIAL_LUMP_CLASSES = goldsrc.SPECIAL_LUMP_CLASSES.copy() + + +methods = [*goldsrc.methods] diff --git a/io_import_rbsp/bsp_tool/branches/gearbox/bshift.py b/io_import_rbsp/bsp_tool/branches/gearbox/bshift.py deleted file mode 100644 index 51fcc86..0000000 --- a/io_import_rbsp/bsp_tool/branches/gearbox/bshift.py +++ /dev/null @@ -1,24 +0,0 @@ -# https://valvedev.info/tools/bspfix/ -# https://developer.valvesoftware.com/wiki/Hl_bs.fgd -from ..valve import goldsrc - - -BSP_VERSION = 30 - -GAMES = ["Half-Life/bshift"] # Half-Life: Blue Shift - -LUMP = goldsrc.LUMP -# NOTE: different headers? -lump_header_address = {LUMP_ID: (4 + i * 8) for i, LUMP_ID in enumerate(LUMP)} - - -BASIC_LUMP_CLASSES = goldsrc.BASIC_LUMP_CLASSES.copy() - -LUMP_CLASSES = goldsrc.LUMP_CLASSES.copy() -# PLANES lump failing - -SPECIAL_LUMP_CLASSES = goldsrc.SPECIAL_LUMP_CLASSES.copy() -# ENTITIES lump failing - - -methods = [*goldsrc.methods] diff --git a/io_import_rbsp/bsp_tool/branches/gearbox/nightfire.py b/io_import_rbsp/bsp_tool/branches/gearbox/nightfire.py new file mode 100644 index 0000000..e90a04f --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/gearbox/nightfire.py @@ -0,0 +1,104 @@ +# https://code.google.com/archive/p/jbn-bsp-lump-tools +import enum +from typing import List + +from .. import base +from .. import shared +from ..valve import goldsrc + + +FILE_MAGIC = None + +BSP_VERSION = 42 + +GAME_PATHS = ["James Bond 007: Nightfire"] + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} + + +class LUMP(enum.Enum): + ENTITIES = 0 + PLANES = 1 + TEXTURES = 2 + MATERIALS = 3 + VERTICES = 4 + NORMALS = 5 + INDICES = 6 + VISIBILITY = 7 + NODES = 8 + FACES = 9 + LIGHTMAPS = 10 + LEAVES = 11 + MARK_SURFACES = 12 + MARK_BRUSHES = 13 + MODELS = 14 + BRUSHES = 15 + BRUSH_SIDES = 16 + TEXTURE_INFO = 17 + + +# struct QuakeBspHeader { int version; QuakeLumpHeader headers[15]; }; +lump_header_address = {LUMP_ID: (4 + i * 8) for i, LUMP_ID in enumerate(LUMP)} + + +# classes for lumps, in alphabetical order: +class BrushSide(base.MappedArray): # LUMP 16 + plane: int + face: int # changed from TextureInfo in Quake 2 + _mapping = ["plane", "face"] + _format = "2i" + + +class Face(base.MappedArray): # LUMP 9 + plane: int + first_vertex: int + num_vertices: int + first_index: int + num_indices: int + flags: int + texture: int + material: int + texture_scale: int + unknown: int + light_styles: int + lightmap: int + _mapping = ["plane", "first_vertex", "num_vertices", "first_index", + "num_indices", "flags", "texture", "material", "texture_scale", + "unknown", "light_styles", "lightmap"] + _format = "5iI6i" + + +class Leaf(base.Struct): # LUMP 11 + contents: int + cluster: int + mins: List[float] + maxs: List[float] + first_mark_brush: int + num_mark_brushes: int + first_mark_face: int + num_mark_faces: int + _format = "2i6f4i" + _arrays = {"mins": [*"xyz"], "maxs": [*"xyz"]} + + +class TextureInfo(base.Struct): # LUMP 17 + u: List[float] + v: List[float] + __slots__ = ["u", "v"] + _format = "8f" + _arrays = {"u": [*"xyzw"], "v": [*"xyzw"]} + + +BASIC_LUMP_CLASSES = goldsrc.BASIC_LUMP_CLASSES.copy() +BASIC_LUMP_CLASSES.update({"INDICES": shared.UnsignedInts}) + +LUMP_CLASSES = goldsrc.LUMP_CLASSES.copy() +LUMP_CLASSES.update({"BRUSH_SIDES": BrushSide, + "FACES": Face, + "LEAVES": Leaf, + "TEXTURE_INFO": TextureInfo}) + +SPECIAL_LUMP_CLASSES = goldsrc.SPECIAL_LUMP_CLASSES.copy() + + +methods = [*goldsrc.methods] diff --git a/io_import_rbsp/bsp_tool/branches/id_software/__init__.py b/io_import_rbsp/bsp_tool/branches/id_software/__init__.py index 2bcc2f3..f99243c 100644 --- a/io_import_rbsp/bsp_tool/branches/id_software/__init__.py +++ b/io_import_rbsp/bsp_tool/branches/id_software/__init__.py @@ -1,14 +1,10 @@ -__all__ = ["quake", "quake2", "quake3"] - +"""Id Software's Quake Engine and it's predecessors have formed the basis for many modern engines.""" from . import quake from . import quake2 from . import quake3 -# TODO: Quake 4? -# TODO: Quake Champions? -# TODO: Hexen 2 (no file-magic) - -__doc__ = """Id Software's Quake Engine and it's predecessors have formed the basis for many modern engines.""" +# TODO: quake4 (IdTech4 == no .bsp?) +# TODO: quake_champions (proprietary archives) +# TODO: hexen2 (extends quake) -FILE_MAGIC = b"IBSP" -branches = [quake, quake3] +scripts = [quake, quake2, quake3] diff --git a/io_import_rbsp/bsp_tool/branches/id_software/quake.py b/io_import_rbsp/bsp_tool/branches/id_software/quake.py index 6516497..39bfa7b 100644 --- a/io_import_rbsp/bsp_tool/branches/id_software/quake.py +++ b/io_import_rbsp/bsp_tool/branches/id_software/quake.py @@ -8,9 +8,13 @@ from .. import shared # special lumps +FILE_MAGIC = None + BSP_VERSION = 29 -GAMES = ["Quake"] +GAME_PATHS = ["Quake", "Team Fortress Quake"] + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} # lump names & indices: @@ -31,21 +35,22 @@ class LUMP(enum.Enum): SURFEDGES = 13 # indices into EDGES (-ve indices reverse edge direction) MODELS = 14 + +# struct QuakeBspHeader { int version; QuakeLumpHeader headers[15]; }; +lump_header_address = {LUMP_ID: (4 + i * 8) for i, LUMP_ID in enumerate(LUMP)} + + # A rough map of the relationships between lumps: -# /-> LEAF_BRUSHES # ENTITIES -> MODELS -> NODES -> LEAVES -> LEAF_FACES -> FACES # \-> CLIP_NODES -> PLANES +# /-> TEXTURE_INFO -> MIP_TEXTURES # FACES -> SURFEDGES -> EDGES -> VERTICES -# |-> TEXTURE_INFO -> MIP_TEXTURES -# |-> LIGHTMAPS +# \--> LIGHTMAPS # \-> PLANES -lump_header_address = {LUMP_ID: (4 + i * 8) for i, LUMP_ID in enumerate(LUMP)} - - -# engine limits: +# Engine limits: class MAX(enum.Enum): ENTITIES = 1024 PLANES = 32767 @@ -191,7 +196,7 @@ class Node(base.Struct): # LUMP 5 # NOTE: bounds are generous, rounding up to the nearest 16 units first_face: int num_faces: int - _format = "I8h" + _format = "I10h" _arrays = {"children": ["front", "back"], "bounds": {"mins": [*"xyz"], "maxs": [*"xyz"]}} @@ -206,13 +211,13 @@ class Plane(base.Struct): # LUMP 1 class TextureInfo(base.Struct): # LUMP 6 - U: List[float] - V: List[float] + u: List[float] + v: List[float] mip_texture_index: int animated: int # 0 or 1 - __slots__ = ["U", "V", "mip_texture_index", "animated"] + __slots__ = ["u", "v", "mip_texture_index", "animated"] _format = "8f2I" - _arrays = {"U": [*"xyzw"], "V": [*"xyzw"]} + _arrays = {"u": [*"xyzw"], "v": [*"xyzw"]} class Vertex(base.MappedArray): # LUMP 3 @@ -264,8 +269,8 @@ def as_bytes(self): "TEXTURE_INFO": TextureInfo, "VERTICES": Vertex} -SPECIAL_LUMP_CLASSES = {"ENTITIES": shared.Entities, - "MIP_TEXTURES": MipTextureLump} +SPECIAL_LUMP_CLASSES = {"ENTITIES": shared.Entities} +# TODO: "MIP_TEXTURES": MipTextureLump # branch exclusive methods, in alphabetical order: diff --git a/io_import_rbsp/bsp_tool/branches/id_software/quake2.py b/io_import_rbsp/bsp_tool/branches/id_software/quake2.py index 1ba05ed..35b558b 100644 --- a/io_import_rbsp/bsp_tool/branches/id_software/quake2.py +++ b/io_import_rbsp/bsp_tool/branches/id_software/quake2.py @@ -8,9 +8,13 @@ from .. import shared +FILE_MAGIC = b"IBSP" + BSP_VERSION = 38 -GAMES = ["Quake II", "Heretic II", "SiN", "Daikatana"] +GAME_PATHS = ["Anachronox", "Quake II", "Heretic II"] + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} class LUMP(enum.Enum): @@ -29,30 +33,45 @@ class LUMP(enum.Enum): SURFEDGES = 12 MODELS = 13 BRUSHES = 14 - BRUSHSIDES = 15 + BRUSH_SIDES = 15 POP = 16 # ? AREAS = 17 AREA_PORTALS = 18 -# TODO: new MAX & Contets enums + +# struct Quake2BspHeader { char file_magic[4]; int version; QuakeLumpHeader headers[19]; }; +lump_header_address = {LUMP_ID: (8 + i * 8) for i, LUMP_ID in enumerate(LUMP)} + +# TODO: MAX & Contents enums # A rough map of the relationships between lumps: # ENTITIES -> MODELS -> NODES -> LEAVES -> LEAF_FACES -> FACES # \-> LEAF_BRUSHES # FACES -> SURFEDGES -> EDGES -> VERTICES -# |-> TEXTURE_INFO -> MIP_TEXTURES -# |-> LIGHTMAPS +# \--> TEXTURE_INFO -> MIP_TEXTURES +# \--> LIGHTMAPS # \-> PLANES # LEAF_FACES -> FACES # LEAF_BRUSHES -> BRUSHES -lump_header_address = {LUMP_ID: (8 + i * 8) for i, LUMP_ID in enumerate(LUMP)} +# classes for lumps, in alphabetical order: +class Brush(base.MappedArray): # LUMP 14 + first_side: int + num_sides: int + contents: int + _mapping = ["first_side", "num_sides", "contents"] + _format = "3i" + + +class BrushSide(base.MappedArray): # LUMP 15 + plane: int + texture_info: int + _format = "Hh" -# classes for lumps, in alphabetical order: class Leaf(base.Struct): # LUMP 10 type: int # see LeafType enum cluster: int # index into the VISIBILITY lump @@ -108,7 +127,9 @@ class TextureInfo(base.Struct): # LUMP 5 BASIC_LUMP_CLASSES = {"LEAF_FACES": shared.Shorts, "SURFEDGES": shared.Ints} -LUMP_CLASSES = {"EDGES": quake.Edge, +LUMP_CLASSES = {"BRUSHES": Brush, + "BRUSH_SIDES": BrushSide, + "EDGES": quake.Edge, "FACES": quake.Face, "LEAVES": Leaf, "MODELS": Model, diff --git a/io_import_rbsp/bsp_tool/branches/id_software/quake3.py b/io_import_rbsp/bsp_tool/branches/id_software/quake3.py index 7aa3f5a..8e0fc8e 100644 --- a/io_import_rbsp/bsp_tool/branches/id_software/quake3.py +++ b/io_import_rbsp/bsp_tool/branches/id_software/quake3.py @@ -1,4 +1,5 @@ # https://www.mralligator.com/q3/ +# https://github.com/zturtleman/spearmint/blob/master/code/qcommon/bsp_q3.c import enum from typing import List import struct @@ -7,9 +8,17 @@ from .. import shared +FILE_MAGIC = b"IBSP" + BSP_VERSION = 46 -GAMES = ["Quake 3 Arena", "Quake Live"] +GAME_PATHS = ["Quake 3 Arena", "Quake Live", "Return to Castle Wolfenstein", + "Wolfenstein Enemy Territory", "Dark Salvation"] # https://mangledeyestudios.itch.io/dark-salvation + +GAME_VERSIONS = {"Quake 3 Arena": 46, "Quake Live": 46, + "Return to Castle Wolfenstein": 47, + "Wolfenstein Enemy Territory": 47, + "Dark Salvation": 666} class LUMP(enum.Enum): @@ -32,15 +41,26 @@ class LUMP(enum.Enum): VISIBILITY = 16 -# a rough map of the relationships between lumps +# struct Quake3BspHeader { char file_magic[4]; int version; QuakeLumpHeader headers[17]; }; +lump_header_address = {LUMP_ID: (8 + i * 8) for i, LUMP_ID in enumerate(LUMP)} + +# a rough map of the relationships between lumps: +# +# /-> Texture # Model -> Brush -> BrushSide -# | |-> Texture -# |-> Face -> MeshVertex -# |-> Texture -# |-> Vertex +# \-> Face -> MeshVertex +# \--> Texture +# \-> Vertex -lump_header_address = {LUMP_ID: (8 + i * 8) for i, LUMP_ID in enumerate(LUMP)} +# flag enums +class SurfaceType(enum.Enum): + BAD = 0 + PLANAR = 1 + PATCH = 2 # displacement-like + TRIANGLE_SOUP = 3 # mesh (dynamic LoD?) + FLARE = 4 # billboard sprite? + FOLIAGE = 5 # classes for lumps, in alphabetical order: @@ -70,7 +90,7 @@ class Effect(base.Struct): # LUMP 12 class Face(base.Struct): # LUMP 13 texture: int # index into Texture lump effect: int # index into Effect lump; -1 for no effect - type: int # polygon, patch, mesh, billboard (env_sprite) + surface_type: int # see SurfaceType enum first_vertex: int # index into Vertex lump num_vertices: int # number of Vertices after first_vertex in this face first_mesh_vertex: int # index into MeshVertex lump @@ -81,13 +101,13 @@ class Face(base.Struct): # LUMP 13 # lightmap.origin: List[float] # world space lightmap origin # lightmap.vector: List[List[float]] # lightmap texture projection vectors normal: List[float] - size: List[float] # texture patch dimensions - __slots__ = ["texture", "effect", "type", "first_vertex", "num_vertices", + patch: List[float] # for patches (displacement-like) + __slots__ = ["texture", "effect", "surface_type", "first_vertex", "num_vertices", "first_mesh_vertex", "num_mesh_vertices", "lightmap", "normal", "size"] _format = "12i12f2i" _arrays = {"lightmap": {"index": None, "top_left": [*"xy"], "size": ["width", "height"], "origin": [*"xyz"], "vector": {"s": [*"xyz"], "t": [*"xyz"]}}, - "normal": [*"xyz"], "size": ["width", "height"]} + "normal": [*"xyz"], "patch": ["width", "height"]} class Leaf(base.Struct): # LUMP 4 @@ -97,7 +117,10 @@ class Leaf(base.Struct): # LUMP 4 maxs: List[float] first_leaf_face: int # index into LeafFace lump num_leaf_faces: int # number of LeafFaces in this Leaf - __slots__ = ["cluster", "area", "mins", "maxs", "first_leaf_face", "num_leaf_faces"] + first_leaf_brush: int # index into LeafBrush lump + num_leaf_brushes: int # number of LeafBrushes in this Leaf + __slots__ = ["cluster", "area", "mins", "maxs", "first_leaf_face", + "num_leaf_faces", "first_leaf_brush", "num_leaf_brushes"] _format = "12i" _arrays = {"mins": [*"xyz"], "maxs": [*"xyz"]} @@ -120,7 +143,7 @@ def flat(self) -> bytes: class LightVolume(base.Struct): # LUMP 15 - # LightVolumess make up a 3D grid whose dimensions are: + # LightVolumes make up a 3D grid whose dimensions are: # x = floor(MODELS[0].maxs.x / 64) - ceil(MODELS[0].mins.x / 64) + 1 # y = floor(MODELS[0].maxs.y / 64) - ceil(MODELS[0].mins.y / 64) + 1 # z = floor(MODELS[0].maxs.z / 128) - ceil(MODELS[0].mins.z / 128) + 1 @@ -159,10 +182,10 @@ class Plane(base.Struct): # LUMP 2 class Texture(base.Struct): # LUMP 1 name: str # 64 char texture name; stored in WAD (Where's All the Data)? - flags: int # rendering bit flags? - contents: int # SOLID, AIR etc. - __slots__ = ["name", "flags", "contents"] + flags: List[int] + __slots__ = ["name", "flags"] _format = "64s2i" + _arrays = {"flags": ["surface", "contents"]} class Vertex(base.Struct): # LUMP 10 @@ -174,11 +197,11 @@ class Vertex(base.Struct): # LUMP 10 __slots__ = ["position", "uv", "normal", "colour"] _format = "10f4B" _arrays = {"position": [*"xyz"], "uv": {"texture": [*"uv"], "lightmap": [*"uv"]}, - "normal": [*"xyz"]} + "normal": [*"xyz"], "colour": [*"rgba"]} # special lump classes, in alphabetical order: -class Visibility: +class Visibility: # same as Quake / QuakeII? """Cluster X is visible from Cluster Y if: bit (1 << Y % 8) of vecs[X * vector_size + Y // 8] is set NOTE: Clusters are associated with Leaves""" @@ -208,7 +231,7 @@ def as_bytes(self): "TEXTURES": Texture, "VERTICES": Vertex} -SPECIAL_LUMP_CLASSES = {"ENTITIES": shared.Entities, +SPECIAL_LUMP_CLASSES = {"ENTITIES": shared.Entities, "VISIBILITY": Visibility} diff --git a/io_import_rbsp/bsp_tool/branches/infinity_ward/__init__.py b/io_import_rbsp/bsp_tool/branches/infinity_ward/__init__.py index c853fca..cefa75d 100644 --- a/io_import_rbsp/bsp_tool/branches/infinity_ward/__init__.py +++ b/io_import_rbsp/bsp_tool/branches/infinity_ward/__init__.py @@ -1,11 +1,11 @@ -__all__ = ["call_of_duty1"] +"""Infinity Ward created the Call of Duty Franchise, built on the idTech3 (RTCW) engine. +.bsp format shares IdTech's b'IBSP' FILE_MAGIC""" +from . import call_of_duty1 # (.bsp in .pk3) +from . import call_of_duty2 # (.d3dbsp in .iwd) +from . import call_of_duty4 # (.d3dbsp in .ff) +# TODO: blops3 -from . import call_of_duty1 -# TODO: CoD2 & 4 +# NOTE: I'm not buying any CoDs until Kotick is gone -__doc__ = """Infinity Ward created the Call of Duty Franchise, built on the idTech3 (Q3A) engine.""" -# NOTE: Infinity Ward .bsps share Id Software's FILE_MAGIC -FILE_MAGIC = b"IBSP" - -branches = [call_of_duty1] +scripts = [call_of_duty1, call_of_duty2, call_of_duty4] diff --git a/io_import_rbsp/bsp_tool/branches/infinity_ward/call_of_duty1.py b/io_import_rbsp/bsp_tool/branches/infinity_ward/call_of_duty1.py index 9a020e3..efe7e96 100644 --- a/io_import_rbsp/bsp_tool/branches/infinity_ward/call_of_duty1.py +++ b/io_import_rbsp/bsp_tool/branches/infinity_ward/call_of_duty1.py @@ -1,16 +1,20 @@ # https://wiki.zeroy.com/index.php?title=Call_of_Duty_1:_d3dbsp -# NOTE: Call of Duty 1 has .bsp files in .pk3 archives -# -- later games instead use .d3dbsp in .iwd archives import enum from typing import List from .. import base -from .. import shared # special lumps +from .. import shared +from ..id_software import quake +# from ..id_software import quake3 +FILE_MAGIC = b"IBSP" + BSP_VERSION = 59 -GAMES = ["Call of Duty"] +GAME_PATHS = ["Call of Duty", "Call of Duty: United Offensive"] + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} class LUMP(enum.Enum): @@ -22,14 +26,14 @@ class LUMP(enum.Enum): TRIANGLE_SOUPS = 6 DRAW_VERTICES = 7 DRAW_INDICES = 8 - CULL_GROUPS = 9 # visibility + CULL_GROUPS = 9 CULL_GROUP_INDICES = 10 - PORTAL_VERTICES = 11 # areaportals; doors & windows + PORTAL_VERTICES = 11 OCCLUDERS = 12 OCCLUDER_PLANES = 13 OCCLUDER_EDGES = 14 OCCLUDER_INDICES = 15 - AABB_TREES = 16 # Physics? or Vis Nodes? + AABB_TREES = 16 CELLS = 17 PORTALS = 18 LIGHT_INDICES = 19 @@ -37,44 +41,47 @@ class LUMP(enum.Enum): LEAVES = 21 LEAF_BRUSHES = 22 LEAF_SURFACES = 23 - PATCH_COLLISION = 24 # decal clipping? reference for painting bullet holes? + PATCH_COLLISION = 24 COLLISION_VERTICES = 25 COLLISION_INDICES = 26 MODELS = 27 - VISIBILITY = 28 # SPECIAL: Binary Partition tree (read bit by bit, with masks?) - LIGHTS = 29 # SPECIAL: string (typically ENTITIES would be #0) + VISIBILITY = 28 + LIGHTS = 29 ENTITIES = 30 UNKNOWN_31 = 31 # FOGS ? - # big 32nd lump at ed of file, not in header? - # likely a zip / pakfile - # checking for file-magic would confirm this + # big "32nd lump" at end of file, not in header? +# struct InfinityWardBspHeader { char file_magic[4]; int version; QuakeLumpHeader headers[32]; }; lump_header_address = {LUMP_ID: (8 + i * 8) for i, LUMP_ID in enumerate(LUMP)} # classes for lumps, in alphabetical order: -# all are incomplete guesses; only lump sizes are known +# NOTE: all are incomplete guesses class AxisAlignedBoundingBox(base.Struct): # LUMP 16 """AABB tree""" - # too small to be mins & maxs of an AABB; probably indices (hence: AABB_TREE) - data: bytes - __slots__ = ["data"] - _format = "12s" # size may be incorrect + # not floats. some kind of node indices? + unknown: List[int] + __slots__ = ["unknown"] + _format = "3I" + _arrays = {"unknown": 3} -class Brush(base.Struct): # LUMP 4 - first_side: int # index into the BrushSide lump - num_sides: int # number of sides after first_side in this Brush - __slots__ = ["first_side", "num_sides"] - _format = "2i" +class Brush(base.MappedArray): # LUMP 6 + # NOTE: first side is calculated via: sum([b.num_sides for b in bsp.BRUSHES[-i]]) - 1 + num_sides: int + material_id: int # Brush's overall contents flag? + _mapping = ["num_sides", "material_id"] + _format = "2H" class BrushSide(base.Struct): # LUMP 3 plane: int # index into Plane lump + # NOTE: in some cases the plane index is a distance instead (float) + # "first 6 entries indicated by an entry in lump 6 [brushes] are distances (float), rest is plane ID's" shader: int # index into Texture lump __slots__ = ["plane", "shader"] - _format = "2i" + _format = "2I" class Cell(base.Struct): # LUMP 17 @@ -84,14 +91,6 @@ class Cell(base.Struct): # LUMP 17 _format = "52s" -class CollisionVertex(base.MappedArray): # LUMP 25 - x: float - y: float - z: float - _mapping = [*"xyz"] - _format = "3f" - - class CullGroup(base.Struct): # LUMP 9 data: bytes __slots__ = ["data"] @@ -145,14 +144,14 @@ class Model(base.Struct): # LUMP 27 num_brushes: int # number of Brushes after first_brush included in this Model unknown: List[bytes] __slots__ = ["mins", "maxs", "first_face", "num_faces", "first_brush", "num_brushes", "unknown"] - _format = "6f4i4c4c" + _format = "6f6i" _arrays = {"mins": [*"xyz"], "maxs": [*"xyz"], "unknown": 2} class Node(base.Struct): # LUMP 20 data: bytes __slots__ = ["data"] - _format = "36c" + _format = "36s" class Occluder(base.Struct): # LUMP 12 @@ -160,11 +159,13 @@ class Occluder(base.Struct): # LUMP 12 num_occluder_planes: int # number of OccluderPlanes after first_occluder_plane in this Occluder first_occluder_edges: int # index into the OccluderEdge lump num_occluder_edges: int # number of OccluderEdges after first_occluder_edge in this Occluder + # first, num? isn't it usually the opposite? interesting __slots__ = ["first_occluder_plane", "num_occluder_planes", "first_occluder_edge", "num_occluder_edges"] _format = "4i" class PatchCollision(base.Struct): # LUMP 24 + """'Patches' are the CoD version of Source's Displacements (think of a fabric patch on torn clothes)""" data: bytes __slots__ = ["data"] _format = "16s" @@ -185,50 +186,58 @@ class Portal(base.Struct): # LUMP 18 class Shader(base.Struct): # LUMP 0 - # assuming the same as Quake3 TEXTURE + """possibly based on Quake3 Texture LumpClass""" texture: str - flags: int - contents: int - __slots__ = ["texture", "flags", "contents"] + flags: List[int] + __slots__ = ["texture", "flags"] _format = "64s2i" + _arrays = {"flags": ["surface", "contents"]} -class TriangleSoup(base.Struct): # LUMP 5 - data: bytes - __slots__ = ["data"] - _format = "16s" +class TriangleSoup(base.MappedArray): # LUMP 5 + material: int + draw_order: int # ? + first_vertex: int + num_vertices: int + first_triangle: int + num_triangles: int + _mapping = ["material", "draw_order", "first_vertex", "num_vertices", + "first_triangle", "num_triangles"] + _format = "2HI2HI" -# {"LUMP_NAME": {version: LumpClass}} +# {"LUMP_NAME": LumpClass} BASIC_LUMP_CLASSES = {"COLLISION_INDICES": shared.UnsignedShorts, "CULL_GROUP_INDICES": shared.UnsignedInts, "DRAW_INDICES": shared.UnsignedShorts, "LEAF_BRUSHES": shared.UnsignedInts, "LEAF_SURFACES": shared.UnsignedInts, "LIGHT_INDICES": shared.UnsignedShorts, - "OCCLUDER_EDGES": shared.UnsignedShorts, "OCCLUDER_INDICES": shared.UnsignedShorts, "OCCLUDER_PLANES": shared.UnsignedInts} -LUMP_CLASSES = {"AABB_TREES": AxisAlignedBoundingBox, - "BRUSHES": Brush, +LUMP_CLASSES = { + # "AABB_TREES": AxisAlignedBoundingBox, + # "BRUSHES": Brush, "BRUSH_SIDES": BrushSide, - "CELLS": Cell, - "COLLISION_VERTICES": CollisionVertex, - "CULL_GROUPS": CullGroup, - "DRAW_VERTICES": DrawVertex, + # "CELLS": Cell, + # "COLLISION_VERTICES": quake.Vertex, + # "CULL_GROUPS": CullGroup, + # "DRAW_VERTICES": DrawVertex, "LEAVES": Leaf, - "LIGHTS": Light, + # "LIGHTS": Light, "LIGHTMAPS": Lightmap, - "MODELS": Model, - "NODES": Node, - "OCCLUDERS": Occluder, - "PATCH_COLLISION": PatchCollision, + # "MODELS": Model, + # "NODES": Node, + # "OCCLUDERS": Occluder, + "OCCLUDER_EDGES": quake.Edge, + # "PATCH_COLLISION": PatchCollision, "PLANES": Plane, - "PORTALS": Portal, + # "PORTALS": Portal, "SHADERS": Shader, "TRIANGLE_SOUPS": TriangleSoup} SPECIAL_LUMP_CLASSES = {"ENTITIES": shared.Entities} -methods = [] + +methods = [shared.worldspawn_volume] diff --git a/io_import_rbsp/bsp_tool/branches/infinity_ward/call_of_duty2.py b/io_import_rbsp/bsp_tool/branches/infinity_ward/call_of_duty2.py new file mode 100644 index 0000000..f68f5a1 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/infinity_ward/call_of_duty2.py @@ -0,0 +1,207 @@ +# https://wiki.zeroy.com/index.php?title=Call_of_Duty_2:_d3dbsp +# https://github.com/mauserzjeh/PyD3DBSP/blob/master/pyd3dbsp/read_d3dbsp.py +# TODO: see modding tools: cod2map.exe -info +import enum +from typing import List + +from .. import base +from ..id_software import quake +from ..id_software import quake3 +from . import call_of_duty1 + + +FILE_MAGIC = b"IBSP" + +BSP_VERSION = 4 + +GAME_PATHS = ["Call of Duty 2"] + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} + + +class LUMP(enum.Enum): + SHADERS = 0 + LIGHTMAPS = 1 + LIGHT_GRID_HASHES = 2 + LIGHT_GRID_VALUES = 3 # MODEL_LIGHTING + PLANES = 4 + BRUSH_SIDES = 5 + BRUSHES = 6 + TRIANGLE_SOUPS = 7 + VERTICES = 8 + TRIANGLES = 9 + CULL_GROUPS = 10 + CULL_GROUP_INDICES = 11 + SHADOW_VERTICES = 12 + SHADOW_INDICES = 13 + SHADOW_CLUSTERS = 14 + SHADOW_AABB_TREES = 15 + SHADOW_SOURCES = 16 + PORTAL_VERTICES = 17 + OCCLUDERS = 18 + OCCLUDER_PLANES = 19 + OCCLUDER_EDGES = 20 + OCCLUDER_INDICES = 21 + AABB_TREE = 22 + CELLS = 23 + PORTALS = 24 + NODES = 25 + LEAVES = 26 + LEAF_BRUSHES = 27 + LEAF_SURFACES = 28 + COLLISION_VERTICES = 29 + COLLISION_EDGES = 30 + COLLISION_TRIANGLES = 31 + COLLISION_BORDERS = 32 + COLLISION_PARTS = 33 + COLLISION_AABBS = 34 + MODELS = 35 + VISIBILITY = 36 + ENTITIES = 37 + PATHS = 38 # singleplayer only, for AI? + LIGHTS = 39 + + +# struct InfinityWardBspHeader { char file_magic[4]; int version; QuakeLumpHeader headers[32]; }; +lump_header_address = {LUMP_ID: (8 + i * 8) for i, LUMP_ID in enumerate(LUMP)} + +# Known lump changes from Call of Duty -> Call of Duty 2: +# New: +# LIGHT_GRID_HASH +# LIGHT_GRID_VALUES +# DRAW_VERTICES -> VERTICES +# DRAW_INDICES -> TRIANGLES ? +# SHADOW_VERTICES +# SHADOW_INDICES +# SHADOW_CLUSTERS +# SHADOW_AABB_TREES +# SHADOW_SOURCES +# COLLISION_EDGES +# COLLISION_TRIANGLES +# COLLISION_BORDERS +# COLLISION_PARTS +# COLLISION_AABBS +# PATHS +# Deprecated: +# LIGHT_INDICES +# COLLISION_INDICES + +# TODO: a rough map of the relationships between lumps: + + +# flag enums +class Surface(enum.IntFlag): + """wiki.zeroy.com/index.php?title=Call_of_Duty_2:_d3dbsp#Lump.5B0.5D_-_Materials""" + # NOTE: zeroy's detailing of Contents is confusing, but most of these are paired with: Contents = 0x01; + BARK = 0x001 << 20 + BRICK = 0x002 << 20 + CARPET = 0x003 << 20 + CLOTH = 0x004 << 20 + CONCRETE = 0x005 << 20 + DIRT = 0x006 << 20 + FLESH = 0x007 << 20 + FOLIAGE = 0x008 << 20 # Contents = 0x02 + GLASS = 0x009 << 20 # Contents = 0x10 + GRASS = 0x00A << 20 + GRAVEL = 0x00B << 20 + ICE = 0x00C << 20 + METAL = 0x00D << 20 + MUD = 0x00E << 20 + PAPER = 0x00F << 20 + PLASTER = 0x01 << 200 + ROCK = 0x011 << 20 + SAND = 0x012 << 20 + SNOW = 0x013 << 20 + WATER = 0x014 << 20 # Contents = 0x20 + WOOD = 0x015 << 20 + ASPHALT = 0x016 << 20 + PORTAL = 0x8 << 28 # Contents = 0x00 + + +# TODO: Contents enum.IntFlag + +# classes for lumps, in alphabetical order: +class CollisionEdge(base.Struct): # LUMP 30 + unknown: int # an index? + position: List[float] + normal: List[List[float]] + distance: float + __slots__ = ["unknown", "position", "normal", "distance"] + _format = "I13f" + _arrays = {"position": [*"xyz"], + "normal": {"A": [*"xyz"], "B": [*"xyz"], "C": [*"xyz"]}} + # NOTE: {"normal": {3: [*"xyz"]}} doesn't work /; (yet) + + +class CollisionTriangle(base.Struct): # LUMP 31 + normal: List[float] + distance: float + unknown_1: List[float] + unknown_2: List[int] + __slots__ = ["normal", "distance", "unknown_1", "unknown_2"] + _format = "12f6" + _arrays = {"normal": [*"xyz"], "unknown": 8, "id": 6} + + +class Model(base.Struct): # LUMP 35 + mins: List[float] + maxs: List[float] + first_triangle_soup: int + num_triangle_soups: int + first_mesh: int # "Patches" (Quake3+ displacements / q3map2 geo) + num_meshes: int # indexes Collision* lumps? brush geo physics is brush based + first_brush: int + num_brushes: int + __slots__ = ["mins", "maxs", "first_triangle_soup", "num_triangle_soups", + "first_mesh", "num_meshes", "first_brush", "num_brushes"] + _format = "6f6i" + _arrays = {"mins": [*"xyz"], "maxs": [*"xyz"]} + + +class Triangle(list): # LUMP 9 + _format = "3H" # List[int] + + def flat(self): + return self # HACK + + +class TriangleSoup(base.MappedArray): # LUMP 7 + material: int + draw_order: int # ? + first_vertex: int + num_vertices: int + first_triangle: int + num_triangles: int + _mapping = ["material", "draw_order", "first_vertex", "num_vertices", + "first_triangle", "num_triangles"] + _format = "2HI2HI" + + +class Vertex(base.Struct): # LUMP 8 + position: List[float] + normal: List[float] + colour: List[int] # RGBA32 + uv0: List[float] # albedo / normal ? + uv1: List[float] # lightmap ? + unknown: List[float] # texture vectors? too short... additional uvs? + __slots__ = ["position", "normal", "colour", "uv0", "uv1", "unknown"] + _format = "6f4B10f" + _arrays = {"position": [*"xyz"], "normal": [*"xyz"], "colour": [*"rgba"], + "uv0": [*"uv"], "uv1": [*"uv"], "unknown": 6} + + +# {"LUMP_NAME": LumpClass} +BASIC_LUMP_CLASSES = call_of_duty1.BASIC_LUMP_CLASSES.copy() + +LUMP_CLASSES = call_of_duty1.LUMP_CLASSES.copy() +LUMP_CLASSES.pop("LIGHTMAPS") # 4 MB per lightmap? +LUMP_CLASSES.update({"LIGHT_GRID_HASH": quake3.LightVolume, + "PORTAL_VERTICES": quake.Vertex, + "TRIANGLES": Triangle, + "TRIANGLE_SOUPS": TriangleSoup, + "VERTICES": Vertex}) + +SPECIAL_LUMP_CLASSES = call_of_duty1.SPECIAL_LUMP_CLASSES.copy() + + +methods = [*call_of_duty1.methods] diff --git a/io_import_rbsp/bsp_tool/branches/infinity_ward/call_of_duty4.py b/io_import_rbsp/bsp_tool/branches/infinity_ward/call_of_duty4.py new file mode 100644 index 0000000..06ed8f3 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/infinity_ward/call_of_duty4.py @@ -0,0 +1,107 @@ +# https://wiki.zeroy.com/index.php?title=Call_of_Duty_4:_d3dbsp +import enum + +from .. import shared +from ..id_software import quake +from . import call_of_duty1 +from . import call_of_duty2 + + +FILE_MAGIC = b"IBSP" + +BSP_VERSION = 22 + +GAME_PATHS = ["Call of Duty 4: Modern Warfare"] + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} + + +# NOTE: lumps are given ids and headers reference these ids in order +class LUMP(enum.Enum): + SHADERS = 0x00 + LIGHTMAPS = 0x01 + LIGHT_GRID_POINTS = 0x02 + LIGHT_GRID_COLOURS = 0x03 + PLANES = 0x04 + BRUSH_SIDES = 0x05 + UNKNOWN_6 = 0x06 + UNKNOWN_7 = 0x07 + BRUSHES = 0x08 + LAYERED_TRIANGLE_SOUPS = 0x09 + LAYERED_VERTICES = 0x0A + LAYERED_INDICES = 0x0B + PORTAL_VERTICES = 0x13 + LAYERED_AABB_TREE = 0x18 + CELLS = 0x19 + PORTALS = 0x1A + NODES = 0x1B + LEAVES = 0x1C + LEAF_BRUSHES = 0x1D + LEAF_SURFACES = 0x1E + COLLISION_VERTICES = 0x1F + COLLISION_TRIANGLES = 0x20 + COLLISION_EDGE_WALK = 0x21 + COLLISION_BORDERS = 0x22 + COLLISION_PARTS = 0x23 + COLLISION_AABBS = 0x24 + MODELS = 0x25 + ENTITIES = 0x27 + PATHS = 0x28 + REFLECTION_PROBES = 0x29 # textures? pretty huge + LAYERED_DATA = 0x2A + PRIMARY_LIGHTS = 0x2B + LIGHT_GRID_HEADER = 0x2C + LIGHT_GRID_ROWS = 0x2D + SIMPLE_TRIANGLE_SOUPS = 0x2F + SIMPLE_VERTICES = 0x30 + SIMPLE_INDICES = 0x31 + SIMPLE_AABB_TREE = 0x33 + LIGHT_REGIONS = 0x34 + LIGHT_REGION_HULLS = 0x35 + LIGHT_REGION_AXES = 0x36 + + +# Known lump changes from Call of Duty 2 -> Call of Duty 4: +# New: +# LIGHT_GRID_HASHES -> LIGHT_GRID_POINTS +# LIGHT_GRID_VALUES -> LIGHT_GRID_COLOURS +# UNKNOWN_6 +# UNKNOWN_7 +# TRIANGLE_SOUPS -> LAYERED_TRIANGLE_SOUPS & SIMPLE_TRIANGLE_SOUPS +# VERTICES -> LAYERED_VERTICES & SIMPLE_VERTICES +# TRIANGLES -> LAYERED_INDICES & SIMPLE_INDICES ? +# AABB_TREE -> LAYERED_AABB_TREE & SIMPLE_AABB_TREE +# COLLISION_EDGES -> COLLISION_EDGE_WALK ? +# REFLECTION_PROBES +# LAYERED_DATA +# PRIMARY_LIGHTS +# LIGHT_GRID_HEADER +# LIGHT_GRID_ROWS +# LIGHT_REGIONS +# LIGHT_REGION_HULLS +# LIGHT_REGION_AXES +# Deprecated: +# CULL_GROUPS +# CULL_GROUP_INDICES +# NOTE: func_cull_group is still present in CoD4Radiant + +# TODO: a rough map of the relationships between lumps: + + +# {"LUMP_NAME": LumpClass} +BASIC_LUMP_CLASSES = {"LAYERED_INDICES": shared.UnsignedShorts, + "LIGHT_GRID_POINTS": shared.UnsignedInts, + "LIGHT_REGIONS": shared.UnsignedBytes, + "SIMPLE_INDICES": shared.UnsignedShorts} + +LUMP_CLASSES = {"COLLISION_TRIANGLES": call_of_duty2.Triangle, + "COLLISION_VERTICES": quake.Vertex, + "LAYERED_VERTICES": call_of_duty2.Vertex, + "SHADERS": call_of_duty1.Shader, + "SIMPLE_VERTICES": call_of_duty2.Vertex} + +SPECIAL_LUMP_CLASSES = {"ENTITIES": shared.Entities} + + +# NOTE: no mins & maxs in worldspawn? +methods = [] diff --git a/io_import_rbsp/bsp_tool/branches/ion_storm/__init__.py b/io_import_rbsp/bsp_tool/branches/ion_storm/__init__.py new file mode 100644 index 0000000..08469bd --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/ion_storm/__init__.py @@ -0,0 +1,5 @@ +"""John Romero & others broke away from Id Software early in Quake 2 development""" +from . import daikatana + + +scripts = [daikatana] diff --git a/io_import_rbsp/bsp_tool/branches/ion_storm/daikatana.py b/io_import_rbsp/bsp_tool/branches/ion_storm/daikatana.py new file mode 100644 index 0000000..e42178f --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/ion_storm/daikatana.py @@ -0,0 +1,40 @@ +# https://bitbucket.org/daikatana13/daikatana +from ..id_software import quake2 + + +FILE_MAGIC = b"IBSP" + +BSP_VERSION = 41 + +GAME_PATHS = ["Daikatana"] + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} + + +LUMP = quake2.LUMP # NOTE: ASSUMED + +# struct Quake2BspHeader { char file_magic[4]; int version; QuakeLumpHeader headers[19]; }; +lump_header_address = {LUMP_ID: (8 + i * 8) for i, LUMP_ID in enumerate(LUMP)} + +# A rough map of the relationships between lumps: +# ENTITIES -> MODELS -> NODES -> LEAVES -> LEAF_FACES -> FACES +# \-> LEAF_BRUSHES + +# FACES -> SURFEDGES -> EDGES -> VERTICES +# \--> TEXTURE_INFO -> MIP_TEXTURES +# \--> LIGHTMAPS +# \-> PLANES + +# LEAF_FACES -> FACES +# LEAF_BRUSHES -> BRUSHES + +# {"LUMP_NAME": {version: LumpClass}} +BASIC_LUMP_CLASSES = quake2.BASIC_LUMP_CLASSES.copy() + +LUMP_CLASSES = quake2.LUMP_CLASSES.copy() +LUMP_CLASSES.pop("LEAVES") + +SPECIAL_LUMP_CLASSES = quake2.SPECIAL_LUMP_CLASSES.copy() + + +methods = [*quake2.methods] diff --git a/io_import_rbsp/bsp_tool/branches/nexon/__init__.py b/io_import_rbsp/bsp_tool/branches/nexon/__init__.py index 32b10c9..0f5829c 100644 --- a/io_import_rbsp/bsp_tool/branches/nexon/__init__.py +++ b/io_import_rbsp/bsp_tool/branches/nexon/__init__.py @@ -1,17 +1,11 @@ -__all__ = ["FILE_MAGIC", "cso2", "cso2_2018", "vindictus"] - +"""Nexon is a South Korean / Japanese developer +They have worked with both Valve & Respawn's Source Engine (CS:O & TF:O [cancelled])""" from . import cso2 # CounterStrike Online 2 from . import cso2_2018 from . import vindictus -__doc__ = """Nexon is a South Korean / Japanese developer -They have worked with both Valve & Respawn's Source Engine (CS:O & TF:O [cancelled])""" - -# NOTE: Nexon games are build on source, and use it's FILE_MAGIC -FILE_MAGIC = b"VBSP" - -branches = [cso2, vindictus] +scripts = [cso2, cso2_2018, vindictus] titanfall_online = """Titanfall: Online was a planned F2P Titanfall spin-off developed by Respawn Entertainment & Nexon. An open beta was held in August & September of 2017[1][2], the game was later cancelled in 2018[3]. diff --git a/io_import_rbsp/bsp_tool/branches/nexon/cso2.py b/io_import_rbsp/bsp_tool/branches/nexon/cso2.py index b27c7a4..5824a8e 100644 --- a/io_import_rbsp/bsp_tool/branches/nexon/cso2.py +++ b/io_import_rbsp/bsp_tool/branches/nexon/cso2.py @@ -1,3 +1,4 @@ +"""2013-2017 format""" # https://git.sr.ht/~leite/cso2-bsp-converter/tree/master/item/src/bsptypes.hpp import collections import enum @@ -8,10 +9,14 @@ from . import vindictus -# NOTE: there are two variants with identical version numbers -# -- 2013-2017 & 2017-present +FILE_MAGIC = b"VBSP" + BSP_VERSION = 100 # 1.00? +GAME_PATHS = ["Counter-Strike: Online 2"] + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} + class LUMP(enum.Enum): ENTITIES = 0 @@ -80,7 +85,9 @@ class LUMP(enum.Enum): UNUSED_63 = 63 +# struct CSO2BspHeader { char file_magic[4]; int version; CSO2LumpHeader headers[64]; int revision; }; lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} + CSO2LumpHeader = collections.namedtuple("CSO2LumpHeader", ["offset", "length", "version", "compressed", "fourCC"]) # NOTE: looking at headers, a half int of value 0x0001 seemed attached to version seemed to make sense @@ -101,17 +108,19 @@ def read_lump_header(file, LUMP: enum.Enum) -> CSO2LumpHeader: # special lump classes, in alphabetical order: class PakFile(zipfile.ZipFile): # WIP """CSO2 PakFiles have a custom .zip format""" + # NOTE: it's not as simple as changing the FILE_MAGIC + # -- this appears to be a unique implementation of .zip # b"CS" file magic & different header format? def __init__(self, raw_zip: bytes): # TODO: translate header to b"PK\x03\x04..." - raw_zip = b"".join([b"PK", raw_zip[2:]]) + raw_zip = b"".join([b"PK", raw_zip[2:]]) # not that easy self._buffer = io.BytesIO(raw_zip) super(PakFile, self).__init__(self._buffer) def as_bytes(self) -> bytes: # TODO: translate header to b"CS\x03\x04..." raw_zip = self._buffer.getvalue() - raw_zip = b"".join([b"CS", raw_zip[2:]]) + raw_zip = b"".join([b"CS", raw_zip[2:]]) # not that easy return raw_zip @@ -119,14 +128,19 @@ def as_bytes(self) -> bytes: BASIC_LUMP_CLASSES = vindictus.BASIC_LUMP_CLASSES.copy() LUMP_CLASSES = vindictus.LUMP_CLASSES.copy() +LUMP_CLASSES.pop("BRUSH_SIDES") +LUMP_CLASSES.pop("CUBEMAPS") +LUMP_CLASSES.pop("DISPLACEMENT_INFO") +LUMP_CLASSES.pop("LEAVES") +LUMP_CLASSES.pop("ORIGINAL_FACES") +LUMP_CLASSES.pop("OVERLAYS") SPECIAL_LUMP_CLASSES = vindictus.SPECIAL_LUMP_CLASSES.copy() -SPECIAL_LUMP_CLASSES.update({"PAKFILE": {0: PakFile}}) # WIP - -GAME_LUMP_CLASSES = vindictus.GAME_LUMP_CLASSES.copy() +SPECIAL_LUMP_CLASSES.pop("PAKFILE") -# branch exclusive methods, in alphabetical order: +# NOTE: GameLump is busted atm? +GAME_LUMP_CLASSES = vindictus.GAME_LUMP_CLASSES.copy() methods = [*vindictus.methods] diff --git a/io_import_rbsp/bsp_tool/branches/nexon/cso2_2018.py b/io_import_rbsp/bsp_tool/branches/nexon/cso2_2018.py index 33c9189..231c6ec 100644 --- a/io_import_rbsp/bsp_tool/branches/nexon/cso2_2018.py +++ b/io_import_rbsp/bsp_tool/branches/nexon/cso2_2018.py @@ -1,13 +1,24 @@ +"""2018-onwards format""" # https://git.sr.ht/~leite/cso2-bsp-converter/tree/master/item/src/bsptypes.hpp from .. import base from . import cso2 -BSP_VERSION = 100 -# NOTE: almost all version numbers match 2013 era maps, this makes detection a pain +FILE_MAGIC = b"VBSP" + +BSP_VERSION = 100 # 1.00? + +GAME_PATHS = ["Counter-Strike: Online 2"] + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} + LUMP = cso2.LUMP + + +# struct CSO2BspHeader { char file_magic[4]; int version; CSO2LumpHeader headers[64]; int revision; }; lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} + read_lump_header = cso2.read_lump_header @@ -21,9 +32,8 @@ class DisplacementInfo(base.Struct): # LUMP 26 # TODO: dcubemap_t: 164 bytes # TODO: Facev1 -# special lump classes, in alphabetical order: - +# {"LUMP_NAME": {version: LumpClass}} BASIC_LUMP_CLASSES = cso2.BASIC_LUMP_CLASSES.copy() LUMP_CLASSES = cso2.LUMP_CLASSES.copy() @@ -31,10 +41,8 @@ class DisplacementInfo(base.Struct): # LUMP 26 SPECIAL_LUMP_CLASSES = cso2.SPECIAL_LUMP_CLASSES.copy() -# {"lump": {version: SpecialLumpClass}} GAME_LUMP_CLASSES = cso2.GAME_LUMP_CLASSES.copy() +# ^ {"lump": {version: SpecialLumpClass}} -# branch exclusive methods, in alphabetical order: - methods = [*cso2.methods] diff --git a/io_import_rbsp/bsp_tool/branches/nexon/vindictus.py b/io_import_rbsp/bsp_tool/branches/nexon/vindictus.py index 70dd73e..a87a987 100644 --- a/io_import_rbsp/bsp_tool/branches/nexon/vindictus.py +++ b/io_import_rbsp/bsp_tool/branches/nexon/vindictus.py @@ -1,6 +1,9 @@ +# https://developer.valvesoftware.com/wiki/Source_BSP_File_Format/Game-Specific#Vindictus """Vindictus. A MMO-RPG build in the Source Engine. Also known as Mabinogi Heroes""" import collections import enum +import io +import itertools import struct from typing import List @@ -8,8 +11,15 @@ from .. import shared from ..valve import orange_box, source + +FILE_MAGIC = b"VBSP" + BSP_VERSION = 20 +GAME_PATHS = ["Vindictus"] + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} + class LUMP(enum.Enum): ENTITIES = 0 @@ -78,9 +88,10 @@ class LUMP(enum.Enum): UNUSED_63 = 63 +# struct VindictusBspHeader { char file_magic[4]; int version; VindictusLumpHeader headers[64]; int revision; }; lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} + VindictusLumpHeader = collections.namedtuple("VindictusLumpHeader", ["id", "flags", "version", "offset", "length"]) -# since vindictus has a unique header format, valve .bsp have a header reading function in here def read_lump_header(file, LUMP_ID: enum.Enum) -> VindictusLumpHeader: @@ -143,17 +154,62 @@ def flat(self): class Face(base.Struct): # LUMP 7 - plane: int # index into Plane lump + plane: int # index into Plane lump + side: int # "faces opposite to the node's plane direction" + on_node: bool # if False, face is in a leaf + unknown: int + first_edge: int # index into the SurfEdge lump + num_edges: int # number of SurfEdges after first_edge in this Face + texture_info: int # index into the TextureInfo lump + displacement_info: int # index into the DisplacementInfo lump (None if -1) + surface_fog_volume_id: int # t-junctions? QuakeIII vertex-lit fog? + styles: int # 4 different lighting states? "switchable lighting info" + light_offset: int # index of first pixel in LIGHTING / LIGHTING_HDR + area: float # surface area of this face + lightmap: List[float] + # lightmap.mins # dimensions of lightmap segment + # lightmap.size # scalars for lightmap segment + original_face: int # ORIGINAL_FACES index, -1 if this is an original face + num_primitives: int # non-zero if t-juncts are present? number of Primitives + first_primitive_id: int # index of Primitive + smoothing_groups: int # lightmap smoothing group __slots__ = ["plane", "side", "on_node", "unknown", "first_edge", - "num_edges", "texture_info", "displacement_info", "surface_fog_volume_id", - "styles", "light_offset", "area", - "lightmap_texture_mins_in_luxels", - "lightmap_texture_size_in_luxels", - "original_face", "num_primitives", "first_primitive_id", - "smoothing_groups"] + "num_edges", "texture_info", "displacement_info", + "surface_fog_volume_id", "styles", "light_offset", "area", + "lightmap", "original_face", "num_primitives", + "first_primitive_id", "smoothing_groups"] _format = "I2bh5i4bif4i4I" - _arrays = {"styles": 4, "lightmap_texture_mins_in_luxels": [*"st"], - "lightmap_texture_size_in_luxels": [*"st"]} + _arrays = {"styles": 4, "lightmap": {"mins": [*"xy"], "size": ["width", "height"]}} + + +class Facev2(base.Struct): # LUMP 7 (v2) + plane: int # index into Plane lump + side: int # "faces opposite to the node's plane direction" + on_node: bool # if False, face is in a leaf + unknown_1: int + first_edge: int # index into the SurfEdge lump + num_edges: int # number of SurfEdges after first_edge in this Face + texture_info: int # index into the TextureInfo lump + displacement_info: int # index into the DisplacementInfo lump (None if -1) + surface_fog_volume_id: int # t-junctions? QuakeIII vertex-lit fog? + unknown_2: int + styles: List[int] # 4 different lighting states? "switchable lighting info" + light_offset: int # index of first pixel in LIGHTING / LIGHTING_HDR + area: float # surface area of this face + lightmap: List[float] + # lightmap.mins # dimensions of lightmap segment + # lightmap.size # scalars for lightmap segment + original_face: int # ORIGINAL_FACES index, -1 if this is an original face + num_primitives: int # non-zero if t-juncts are present? number of Primitives + first_primitive_id: int # index of Primitive + smoothing_groups: int # lightmap smoothing group + __slots__ = ["plane", "side", "on_node", "unknown_1", "first_edge", + "num_edges", "texture_info", "displacement_info", + "surface_fog_volume_id", "unknown_2", "styles", + "light_offset", "area", "lightmap", "original_face", + "num_primitives", "first_primitive_id", "smoothing_groups"] + _format = "I2bh6i4bif4i4I" + _arrays = {"styles": 4, "lightmap": {"mins": [*"xy"], "size": ["width", "height"]}} class Leaf(base.Struct): # LUMP 10 @@ -181,18 +237,68 @@ class Overlay(base.Struct): # LUMP 45 # classes for special lumps, in alphabetical order: -# TODO: StaticPropv5, StaticPropv6 -# TODO: GameLump (alternate header format) -# old: struct GameLumpHeader { char id[4]; unsigned short flags, version; int offset, length; }; -# new: struct GameLumpHeader { char id[4]; int flags, version, offset, length; }; -# also the sprp lump includes a "scales" sub-lump after leaves -# struct StaticPropScale { int static_prop; Vector scales; }; // Different scalar for each axis -# struct StaticPropScalesLump { int count; StaticPropScales scales[count]; }; +class GameLumpHeader(base.MappedArray): + id: str + flags: int + version: int + offset: int + length: int + _mapping = ["id", "flags", "version", "offset", "length"] + _format = "4s4i" + + +class GameLump_SPRP: + def __init__(self, raw_sprp_lump: bytes, StaticPropClass: object): + """Get StaticPropClass from GameLump version""" + # lambda raw_lump: GameLump_SPRP(raw_lump, StaticPropvXX) + sprp_lump = io.BytesIO(raw_sprp_lump) + model_name_count = int.from_bytes(sprp_lump.read(4), "little") + model_names = struct.iter_unpack("128s", sprp_lump.read(128 * model_name_count)) + setattr(self, "model_names", [t[0].replace(b"\0", b"").decode() for t in model_names]) + leaf_count = int.from_bytes(sprp_lump.read(4), "little") + leaves = itertools.chain(*struct.iter_unpack("H", sprp_lump.read(2 * leaf_count))) + setattr(self, "leaves", list(leaves)) + prop_count = int.from_bytes(sprp_lump.read(4), "little") + scale_count = int.from_bytes(sprp_lump.read(4), "little") + read_size = struct.calcsize(StaticPropScale._format) * scale_count + scales = struct.iter_unpack(StaticPropScale._format, sprp_lump.read(read_size)) + setattr(self, "scales", list(scales)) + if StaticPropClass is not None: + read_size = struct.calcsize(StaticPropClass._format) * prop_count + props = struct.iter_unpack(StaticPropClass._format, sprp_lump.read(read_size)) + setattr(self, "props", list(map(StaticPropClass.from_tuple, props))) + else: + prop_bytes = sprp_lump.read() + prop_size = len(prop_bytes) // prop_count + setattr(self, "props", list(struct.iter_unpack(f"{prop_size}s", prop_bytes))) + here = sprp_lump.tell() + end = sprp_lump.seek(0, 2) + assert here == end, "Had some leftover bytes, bad format" + + def as_bytes(self) -> bytes: + if len(self.props) > 0: + prop_format = self.props[0]._format + else: + prop_format = "" + return b"".join([int.to_bytes(len(self.model_names), 4, "little"), + *[struct.pack("128s", n) for n in self.model_names], + int.to_bytes(len(self.leaves), 4, "little"), + *[struct.pack("H", L) for L in self.leaves], + int.to_bytes(len(self.scales), 4, "little"), + *[struct.pack(StaticPropScale._format, s) for s in self.scales], + int.to_bytes(len(self.props), 4, "little"), + *[struct.pack(prop_format, *p.flat()) for p in self.props]]) + + +class StaticPropScale(base.MappedArray): + _mapping = ["index", *"xyz"] + _format = "i3f" # {"LUMP_NAME": {version: LumpClass}} BASIC_LUMP_CLASSES = orange_box.BASIC_LUMP_CLASSES.copy() -BASIC_LUMP_CLASSES["LEAF_FACES"] = {0: shared.UnsignedInts} +BASIC_LUMP_CLASSES.update({"LEAF_BRUSHES": {0: shared.UnsignedInts}, + "LEAF_FACES": {0: shared.UnsignedInts}}) LUMP_CLASSES = orange_box.LUMP_CLASSES.copy() LUMP_CLASSES.update({"AREAS": {0: Area}, @@ -200,21 +306,21 @@ class Overlay(base.Struct): # LUMP 45 "BRUSH_SIDES": {0: BrushSide}, "DISPLACEMENT_INFO": {0: DisplacementInfo}, "EDGES": {0: Edge}, - "FACES": {0: Face}, - # NOTE: LeafBrush also differs from orange_box (not implemented) + "FACES": {1: Face, + 2: Facev2}, "LEAVES": {0: Leaf}, "NODES": {0: Node}, - "ORIGINAL_FACES": {0: Face}}) + "ORIGINAL_FACES": {1: Face, + 2: Facev2}, + "OVERLAYS": {0: Overlay}}) SPECIAL_LUMP_CLASSES = orange_box.SPECIAL_LUMP_CLASSES.copy() +GAME_LUMP_HEADER = source.GameLumpHeader + # {"lump": {version: SpecialLumpClass}} GAME_LUMP_CLASSES = orange_box.GAME_LUMP_CLASSES.copy() -# GAME_LUMP_CLASSES = {"sprp": {5: lambda raw_lump: GameLump_SPRP(raw_lump, StaticPropv5), -# 6: lambda raw_lump: GameLump_SPRP(raw_lump, StaticPropv6)}} - - -# branch exclusive methods, in alphabetical order: +GAME_LUMP_CLASSES.update({"sprp": {6: lambda raw_lump: GameLump_SPRP(raw_lump, source.StaticPropv6)}}) methods = [*orange_box.methods] diff --git a/io_import_rbsp/bsp_tool/branches/physics.py b/io_import_rbsp/bsp_tool/branches/physics.py new file mode 100644 index 0000000..5480c89 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/physics.py @@ -0,0 +1,165 @@ +"""PhysicsCollide SpecialLumpClasses""" +from __future__ import annotations +import collections +import io +import struct +from typing import List + +from . import base + + +# PhysicsCollide headers +ModelHeader = collections.namedtuple("dphysmodel_t", ["model", "data_size", "script_size", "solid_count"]) +# struct dphysmodel_t { int model_index, data_size, keydata_size, solid_count; }; +PhysicsObject = collections.namedtuple("PhysicsObject", ["header", "solids", "script"]) + + +class CollideLump(list): + """[model_index: int, solids: List[bytes], script: bytes]""" + # passed to VCollideLoad in vphysics.dll + def __init__(self, raw_lump: bytes) -> List[PhysicsObject]: + collision_models = list() + lump = io.BytesIO(raw_lump) + header = ModelHeader(*struct.unpack("4i", lump.read(16))) + while header != ModelHeader(-1, -1, 0, 0) and lump.tell() != len(raw_lump): + start = lump.tell() + solids = list() + for i in range(header.solid_count): + # CPhysCollisionEntry->WriteCollisionBinary + cb_size = int.from_bytes(lump.read(4), "little") + solids.append(Block(lump.read(cb_size))) + assert lump.tell() - start == header.data_size + script = lump.read(header.script_size) # ascii + assert len(script) == header.script_size + collision_models.append(PhysicsObject(header, solids, script)) + # read next header (sets conditions for the while loop) + header = ModelHeader(*struct.unpack("4i", lump.read(16))) + assert header == ModelHeader(-1, -1, 0, 0), "PhysicsCollide ended incorrectly" + super().__init__(collision_models) # TODO: this is a terrible interface + + def as_bytes(self) -> bytes: + def phy_bytes(collision_model): + header, solids, script = collision_model + phy_blocks = list() + for phy_block in solids: + collision_data = phy_block.as_bytes() + phy_blocks.append(len(collision_data).to_bytes(4, "little")) + phy_blocks.append(collision_data) + phy_block_bytes = b"".join(phy_blocks) + header = struct.pack("4i", header.model, len(phy_block_bytes), len(script), len(solids)) + return b"".join([header, phy_block_bytes, script]) + tail = struct.pack("4i", -1, -1, 0, 0) # null header + return b"".join([*map(phy_bytes, self), tail]) + + +# PhysicsBlock headers +CollideHeader = collections.namedtuple("swapcollideheader_t", ["id", "version", "model_type"]) +# struct swapcollideheader_t { int size, vphysicsID; short version, model_type; }; +SurfaceHeader = collections.namedtuple("swapcompactsurfaceheader_t", ["size", "drag_axis_areas", "axis_map_size"]) +# struct swapcompactsurfaceheader_t { int surfaceSize; Vector dragAxisAreas; int axisMapSize; }; +MoppHeader = collections.namedtuple("swapmoppsurfaceheader_t", ["size"]) +# struct swapmoppsurfaceheader_t { int moppSize; }; +# NOTE: the "swap" in the names refers to the format differing across byte-orders +# -- Source swaps byte order for selected fields when compiled for consoles +# -- PC is primarily little-endian, while the Xbox 360 has more big-endian fields + + +class Block: # TODO: actually process this data + # byte swapper: https://github.com/ValveSoftware/source-sdk-2013/blob/master/sp/src/utils/common/bsplib.cpp#L1677 + def __init__(self, raw_lump: bytes): + """Editting not yet supported""" + self._raw = raw_lump + lump = io.BytesIO(raw_lump) + header = CollideHeader(*struct.unpack("4s2h", lump.read(8))) + assert header.id == b"VPHY", "if 'YHPV' byte order is flipped" + # version isn't checked by the byteswap, probably important for VPHYSICS + if header.model_type == 0: + size, *drag_axis_areas, axis_map_size = struct.unpack("i3fi", lump.read(20)) + surface_header = SurfaceHeader(size, drag_axis_areas, axis_map_size) + # self.data = CollisionModel(lump) + # - CollisionModel + # - TreeNode (recursive) <- CollisionModel.tree_offset + # - TreeNode (left) <- TreeNode._offset + sizeof(TreeNode) + # - TreeNode (right) <- TreeNode.right_node_offset + # if (TreeNode.right_node_offset == 0) + # - ConvexLeaf <- TreeNode.convex_offset + # - ConvexTriangle[ConvexLeaf.triangle_count] + # - Vertex[max(*ConvexTriangle.edges[::])] + elif header.model_type == 1: + surface_header = MoppHeader(*struct.unpack("i", lump.read(4))) + # yeah, no idea what this does + else: + raise RuntimeError("Invalid model type") + self.header = (header, surface_header) + self.data = lump.read(surface_header.size) + assert lump.tell() == len(raw_lump) + + def as_bytes(self) -> bytes: + header, surface_header = self.header + if header.model_type == 0: # SurfaceHeader (swapcompactsurfaceheader_t) + size, drag_axis_areas, axis_map_size = surface_header + surface_header = struct.pack("i3fi", len(self.data), *drag_axis_areas, axis_map_size) + elif header.model_type == 1: # MoppHeader (swapmoppsurfaceheader_t) + surface_header = struct.pack("i", len(self.data)) + else: + raise RuntimeError("Invalid model type") + header = struct.pack("4s2h", *header) + return b"".join([header, surface_header, self.data]) + + +# TODO: child / recursive __init__(s) +# -- ... +# TODO: reverting nested structure to bytes +# TODO: create a structure that makes navigating the tree optional +# -- ideally just get all the triangles and make a mesh +# -- however using the tree could be useful for reverse engineering + +class CollisionModel(base.Struct): + """struct CollisionModel { float unknown[7]; int surface, tree_offset, padding[2]; };""" + unknown: List[float] + __slots__ = ["unknown", "surface", "tree_offset", "padding"] + _format = "7f4i" + _arrays = {"unknown": 7, "padding": 2} + + @classmethod + def from_stream(cls, bytestream: io.RawIOBase) -> CollisionModel: + # TODO: load "header" w/ cls.from_bytes, then allocate children + raise NotImplementedError() + + def as_bytes(self) -> bytes: + # header = super(self, CollisionModel).as_bytes() + # TODO: handle children + raise NotImplementedError() + + +class TreeNode(base.Struct): + """struct TreeNode { int node_size[2]; float unknown[5]; };""" + __slots__ = ["node_size", "unknown"] + _format = "2i5f" + _arrays = {"node_size": 2, "unknown": 5} + + # TODO: children + # 2x TreeNode if node_size[0] != 0 + # else b"IDST" (IDSTUDIOHEADER), ConvexLeaf + # src/utils/motionmapper/motionmapper.h + # src/utils/vbsp/staticprop.cpp + + +class ConvexLeaf(base.Struct): + """struct ConvexLeaf { int vertex_offset, padding[2]; short triangle_count, unused; };""" + __slots__ = ["vertex_offset", "padding", "triangle_count", "unused"] + _format = "3i2h" + _arrays = {"padding": 2} + + +class ConvexTriangle(base.Struct): + """struct ConvexTriange { int padding; short edges[3][2]; };""" + __slots__ = ["padding", "edge"] + _format = "i6h" + _arrays = {"edge": {"AB": 2, "BC": 2, "CA": 2}} + + +class Vertex(base.MappedArray): + """struct Vertex { float x, y, z, w };""" + _mapping = [*"xyzw"] + _format = "4f" diff --git a/io_import_rbsp/bsp_tool/branches/py_struct_as_cpp.py b/io_import_rbsp/bsp_tool/branches/py_struct_as_cpp.py deleted file mode 100644 index 3cabe7f..0000000 --- a/io_import_rbsp/bsp_tool/branches/py_struct_as_cpp.py +++ /dev/null @@ -1,293 +0,0 @@ -from __future__ import annotations -import enum -import inspect -import itertools -import re -from typing import Any, Dict, List, Union - -from .base import mapping_length, MappedArray, Struct -# class base.Struct: -# __slots__: List[str] # top-level attr name -# _format: str # types -# _arrays: Dict[str, any] # child mapping -# # ^ {"attr": _mapping} - -# class base.MappedArray: -# _mapping: int # list of len _mapping -# _mapping: List[str] # list of named children -# _mapping: Dict[str, Any] # children with their own mappings -# # ^ {"child": _mapping} -# _mapping: None # plain, siblings have mappings -# # e.g. {"id": None, "position": [*"xy"]} - -# type aliases -StructMapping = Union[Dict[str, Any], List[str], int, None] # _arrays or _mapping -CType = str -TypeMap = Dict[CType, str] -# ^ {"type": "member"} - - -def lump_class_as_c(lump_class: Union[MappedArray, Struct]) -> str: - name = lump_class.__name__ - formats = split_format(lump_class._format) - if issubclass(lump_class, Struct): - attrs = lump_class.__slots__ - mappings = getattr(lump_class, "_arrays", dict()) - elif issubclass(lump_class, MappedArray): - special_mapping = lump_class._mapping - if isinstance(special_mapping, list): - attrs, mappings = special_mapping, None - elif isinstance(special_mapping, dict): - attrs = special_mapping.keys() - mappings = special_mapping - else: - raise TypeError(f"Cannot convert {type(lump_class)} to C") - return make_c_struct(name, attrs, formats, mappings) - # ^ ("name", {"type": "member"}) - - -# TODO: revise -def make_c_struct(name: str, attrs: List[str], formats: List[str], mappings: StructMapping = dict()) -> Dict[str, TypeMap]: - members = list() - i = 0 - for attr in attrs: - member_name = attr - sub_mapping = mappings.get(attr) - if isinstance(sub_mapping, int): - format_size = sub_mapping - elif isinstance(sub_mapping, list): - format_size = len(sub_mapping) - elif isinstance(sub_mapping, dict): - format_size = mapping_length(sub_mapping) - else: - raise TypeError(f"Invalid sub_mapping: {sub_mapping}") - attr_format = formats[i:i+format_size] - c_type, member_name = c_type_of(member_name, attr_format, sub_mapping) - # TODO: convert c_type to one-liner - members.append((c_type, member_name)) - return {name: {a: t for t, a in members}} - # ^ {"name": {"type": "member"}} - - -def split_format(_format: str) -> List[str]: - _format = re.findall(r"[0-9]*[xcbB\?hHiIlLqQnNefdspP]", _format.replace(" ", "")) - out = list() - for f in _format: - match_numbered = re.match(r"([0-9]+)([xcbB\?hHiIlLqQnNefdpP])", f) - # NOTE: does not decompress strings - if match_numbered is not None: - count, f = match_numbered.groups() - out.extend(f * int(count)) - else: - out.append(f) - return out - - -type_LUT = {"c": "char", "?": "bool", - "b": "char", "B": "unsigned char", - "h": "short", "H": "unsigned short", - "i": "int", "I": "unsigned int", - "f": "float", "g": "double"} - - -def c_type_of(attr: str, formats: List[str], mapping: StructMapping) -> (CType, str): - if not isinstance(mapping, (dict, list, int, None)): - raise TypeError(f"Invalid mapping: {type(mapping)}") - if isinstance(mapping, None): # one object - c_type = type_LUT[formats[0]] - return c_type, attr - if isinstance(mapping, int): # C list type - if len(set(formats)) == 1: - c_type = type_LUT[formats[0]] - return c_type, f"{attr}[{mapping}]" - # ^ ("type", "name[count]") - else: # mixed types - if mapping > 26: # ("attr", "2ih", 4) -> ("struct { int A[2]; short B; }", "attr") - raise NotImplementedError("Cannot convert list of mixed type") - mapping = [chr(97 + i) for i in range(mapping)] # i=0: a, i=25: z - return c_type_of(attr, formats, mapping) # use list mapping instead - # ^ {"attr": {"child_A": Type, "child_B": Type}} - elif isinstance(mapping, (list, dict)): # basic / nested struct - return make_c_struct(attr, formats, mapping) - # ^ {"attr": {"child_1": Type, "child_2": Type}} - else: - raise RuntimeError("How did this happen?") - - -def apply_typing(members: Dict[str, CType]) -> Dict[str, CType]: - """{'name', 'char[256]'} -> {'name[256]', 'char'}""" - # NOTE: type may be a struct - out = dict() - # ^ {"member_1": "type", "member_2": "type[]"} - for member, _type in members.items(): - count_search = re.search(r"([a-z]+)\[([0-9]+)\]", _type) # type[count] - if count_search is not None: - _type, count = count_search.groups() - member = f"{member}[{count}]" - out[member] = _type - return out - - -def compact_members(members: Dict[str, str]) -> List[str]: - """run after apply_typing""" - # NOTE: type may be a struct, cannot chain inner structs! - # - in this case, the inner must be declared externally - members = list(members.items()) - # ^ [("member_1", "type"), ("member_2", "type")] - type_groups = [list(members[0])] - # ^ [["type", "member_1", "member_2"], ["type", "member_1"]] - for member, _type in members[1:]: - if _type == type_groups[-1][0]: - type_groups[-1].append(member) - else: - type_groups.append([_type, member]) - out = list() - # NOTE: this automatically creates "lines" & does not allow for aligning variables - for _type, *members in type_groups: - out.append(f"{_type} {', '.join(map(str, members))};") - return out - - -pattern_thc = re.compile(r"([\w\.]+):\s[\w\[\]]+ # ([\w ]+)") -# NOTE: also catches commented type hints to allow labelling of inner members - - -def get_type_hint_comments(cls: object) -> Dict[str, str]: - out = dict() # {"member": "comment"} - for line in inspect.getsource(cls): - match = pattern_thc.seach(line) - if match is None: - continue - member, comment = match.groups() - out[member] = comment - return out - - -class Style(enum.IntFlag): - # masks - TYPE_MASK = 0b11 - ONER = 0b01 - INNER = 0b10 - # major styles - OUTER_FULL = 0b00 - # struct OuterFull { - # type member; - # }; - OUTER_ONER = 0b01 # struct OuterOner { type a, b; type c; }; - INNER_FULL = 0b10 - # struct { // inner_full - # type member; - # } inner_full; - INNER_ONER = 0b11 # struct { type d, e, f; } inner_oner; - # _FULL bonus flags - ALIGN_MEMBERS = 0x04 - ALIGN_COMMENTS = 0x08 - # TODO: align with inner structs? - # TODO: COMPACT = 0x10 # use compact members - - -def definition_as_str(name: str, members: Dict[str, Any], mode: int = 0x04, comments: Dict[str, str] = dict()) -> str: - # members = {"member_1": "type", "member_2": "type[]"} - # comments = {"member_1": "comment"} - if not mode & Style.INNER: # snake_case -> CamelCase - name = "".join([word.title() for word in name.split("_")]) - # NOTE: this should be redundant, but strict styles make reading Cpp easier - # generate output - output_type = mode & Style.TYPE_MASK - if output_type & Style.INNER: - opener, closer = "struct {", "}" + f" {name};\n" - else: # OUTER - opener, closer = f"struct {name} " + "{", "};\n" - if output_type & Style.ONER: - joiner = " " - definitions = compact_members(apply_typing(members)) - else: # FULL (multi-line) - joiner = "\n" - alignment = 1 - if mode & Style.ALIGN_MEMBERS: - alignment = max([len(t) for t in members.keys() if not t.startswith("struct")]) + 2 - half_definitions = [f" {t.ljust(alignment)} {n};" for n, t in apply_typing(members).items()] - alignment = max([len(d) for d in half_definitions]) if mode & Style.ALIGN_COMMENTS else 1 - definitions = [] - for member, definition in zip(members, half_definitions): - if member in comments: - definitions.append(f"{definition.ljust(alignment)} \\\\ {comments[member]}") - else: - definitions.append(definition) - if output_type & Style.INNER: - opener += f" // {name}" - return joiner.join([opener, *definitions, closer]) - - -def branch_script_as_cpp(branch_script): - raise NotImplementedError() - bs = branch_script - out = [f"const int BSP_VERSION = {bs.BSP_VERSION};\n"] - lumps = {L.value: L.name for L in bs.LUMP} # will error if bs.LUMP is incomplete - half = (len(lumps) + 1) // 2 + 1 - - def justify(x: Any) -> str: - return str(x).rjust(len(str(len(lumps)))) - - decls = [] - for i, j in itertools.zip_longest(lumps[1:half], lumps[half + 1:]): - declarations = [f" {lumps[i]} = {justify(i)}"] - if j is not None: - declarations.append(f"{lumps[j]} = {justify(j)},") - decls.append(", ".join(declarations)) - decls[-1] = decls[-1][:-1] + ";" # replace final comma with a semi-colon - # TODO: align all the `=` signs - lump_decls = "\n".join(["namespace LUMP {", - f" const int {lumps[0]} = {justify(0)}, {lumps[half]} = {justify(half)},", - decls, "}\n\n"]) - out.extend(lump_decls) - # TODO: BasicLumpClasses -> comments? - # TODO: LumpClasses -> lump_class_as_c - # TODO: SpecialLumpClasses -> inspect.getsource(SpecialLumpClass) in Cpp TODO - # TODO: methods -> [inspect.getsource(m) for m in methods] in Cpp TODO - -# Nice to haves: -# Give common inner structs their own types (this would require the user to name each type) -# struct Vector { float x, y, z; }; -# struct LumpClass { -# int id; -# Vector origin; -# char name[128]; -# unsigned short unknown[4]; -# }; - -# LumpClass.__doc__ in a comment above it's Cpp definition -# // for one liner, /* */ with padded spaces for multi-line - - -if __name__ == "__main__": - pass - - # print(split_format("128s4I2bhHhh")) # OK - members = {"id": "int", "name": "char[256]", "inner": "struct { float a, b; }", - "skin": "short", "flags": "short"} - comments = {"id": "GUID", "inner": "inner struct"} - # print(apply_typing(members)) # OK - # print(compact_members(members)) # OK - print("=== Outer Full ===") - print(definition_as_str("Test", members, comments=comments, mode=Style.OUTER_FULL)) # OK - print("=== Outer Full + Aligned Members ===") - print(definition_as_str("Test", members, comments=comments, mode=0 | 4)) # OK - print("=== Outer Full + Aligned Members & Comments ===") - print(definition_as_str("Test", members, comments=comments, mode=0 | 4 | 8)) # OK - print("=== Outer Oner ===") - print(definition_as_str("Test", members, comments=comments, mode=Style.OUTER_ONER)) # OK - print("=== Inner Full ===") - print(definition_as_str("test", members, comments=comments, mode=Style.INNER_FULL)) # OK - print("=== Inner Full + Aligned Members ===") - print(definition_as_str("test", members, comments=comments, mode=2 | 4)) # OK - print("=== Outer Full + Aligned Members & Comments ===") - print(definition_as_str("Test", members, comments=comments, mode=2 | 4 | 8)) # OK - print("=== Inner Oner ===") - print(definition_as_str("test", members, comments=comments, mode=Style.INNER_ONER)) # OK - - # TODO: test branch_script_as_cpp - # from bsp_tool.branches.id_software.quake3 import Face # noqa F401 - # TODO: test lump_class_as_c(Face) - # from bsp_tool.branches.valve.orange_box import Plane - # print(lump_class_as_c(Plane)) diff --git a/io_import_rbsp/bsp_tool/branches/raven/__init__.py b/io_import_rbsp/bsp_tool/branches/raven/__init__.py new file mode 100644 index 0000000..fd8e228 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/raven/__init__.py @@ -0,0 +1,10 @@ +"""Raven Software worked closely with Id Software +They were later acquired by Activision +After this acquisition, many devs left to found Human Head Studios +Raven has also worked on a number of recent Call of Duty titles (~2010-present)""" +from . import hexen2 +from . import soldier_of_fortune +from . import soldier_of_fortune2 + + +scripts = [hexen2, soldier_of_fortune, soldier_of_fortune2] diff --git a/io_import_rbsp/bsp_tool/branches/raven/hexen2.py b/io_import_rbsp/bsp_tool/branches/raven/hexen2.py new file mode 100644 index 0000000..87d4734 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/raven/hexen2.py @@ -0,0 +1,28 @@ +from ..id_software import quake + + +FILE_MAGIC = None + +BSP_VERSION = 29 + +GAME_PATHS = ["Hexen 2"] + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} + + +LUMP = quake.LUMP + + +# struct QuakeBspHeader { int version; QuakeLumpHeader headers[15]; }; +lump_header_address = {LUMP_ID: (4 + i * 8) for i, LUMP_ID in enumerate(LUMP)} + + +BASIC_LUMP_CLASSES = quake.BASIC_LUMP_CLASSES.copy() + +LUMP_CLASSES = quake.LUMP_CLASSES.copy() +LUMP_CLASSES.pop("MODELS") + +SPECIAL_LUMP_CLASSES = quake.SPECIAL_LUMP_CLASSES.copy() + + +methods = [*quake.methods] diff --git a/io_import_rbsp/bsp_tool/branches/raven/soldier_of_fortune.py b/io_import_rbsp/bsp_tool/branches/raven/soldier_of_fortune.py new file mode 100644 index 0000000..9d04e7d --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/raven/soldier_of_fortune.py @@ -0,0 +1,29 @@ +from ..id_software import quake2 + + +FILE_MAGIC = b"IBSP" + +BSP_VERSION = 46 + +GAME_PATHS = ["Soldier of Fortune"] + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} + + +LUMP = quake2.LUMP + + +# struct Quake2BspHeader { char file_magic[4]; int version; QuakeLumpHeader headers[19]; }; +lump_header_address = {LUMP_ID: (8 + i * 8) for i, LUMP_ID in enumerate(LUMP)} + + +BASIC_LUMP_CLASSES = quake2.BASIC_LUMP_CLASSES.copy() + +LUMP_CLASSES = quake2.LUMP_CLASSES.copy() +LUMP_CLASSES.pop("FACES") +LUMP_CLASSES.pop("LEAVES") + +SPECIAL_LUMP_CLASSES = quake2.SPECIAL_LUMP_CLASSES.copy() + + +methods = [*quake2.methods] diff --git a/io_import_rbsp/bsp_tool/branches/raven/soldier_of_fortune2.py b/io_import_rbsp/bsp_tool/branches/raven/soldier_of_fortune2.py new file mode 100644 index 0000000..e6a3953 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/raven/soldier_of_fortune2.py @@ -0,0 +1,82 @@ +# https://github.com/TTimo/GtkRadiant/blob/master/tools/quake3/q3map2/game_sof2.h +# https://github.com/TTimo/GtkRadiant/blob/master/tools/urt/tools/quake3/q3map2/bspfile_rbsp.c +import enum + +from .. import shared +from ..id_software import quake3 + + +FILE_MAGIC = b"RBSP" + +BSP_VERSION = 1 + +GAME_PATHS = ["Soldier of Fortune 2", + "Star Wars Jedi Knight - Jedi Academy", + "Star Wars Jedi Knight II - Jedi Outcast"] + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} + + +class LUMP(enum.Enum): + ENTITIES = 0 + SHADERS = 1 + PLANES = 2 + NODES = 3 + LEAVES = 4 + LEAF_SURFACES = 5 + LEAF_BRUSHES = 6 + MODELS = 7 + BRUSHES = 8 + BRUSH_SIDES = 9 + DRAW_VERTICES = 10 + DRAW_INDICES = 11 + FOGS = 12 + SURFACES = 13 + LIGHTMAPS = 14 + LIGHT_GRID = 15 + VISIBILITY = 16 + LIGHT_ARRAY = 17 + + +# struct RavenBspHeader { char file_magic[4]; int version; QuakeLumpHeader headers[18]; }; +lump_header_address = {LUMP_ID: (8 + i * 8) for i, LUMP_ID in enumerate(LUMP)} + +# Known lump changes from Quake 3 -> Raven: +# New: +# FACES -> SURFACES +# LEAF_FACES -> LEAF_SURFACES +# TEXTURES -> SHADERS +# VERTICES -> DRAW_VERTICES +# MESH_VERTICES -> DRAW_INDICES + +# a rough map of the relationships between lumps: +# +# /-> Texture +# Model -> Brush -> BrushSide +# \-> Face -> MeshVertex +# \--> Texture +# \-> Vertex + + +# LIGHT_GRID / LIGHT_ARRAY +# https://github.com/TTimo/GtkRadiant/blob/master/tools/urt/tools/quake3/q3map2/bspfile_rbsp.c#L89 +# https://github.com/TTimo/GtkRadiant/blob/master/tools/urt/tools/quake3/q3map2/bspfile_rbsp.c#L115 + + +BASIC_LUMP_CLASSES = {"LEAF_BRUSHES": shared.Ints, + "LEAF_SURFACES": shared.Ints, + "DRAW_INDICES": shared.Ints} + +LUMP_CLASSES = {"BRUSHES": quake3.Brush, + "LEAVES": quake3.Leaf, + "LIGHTMAPS": quake3.Lightmap, + "MODELS": quake3.Model, + "NODES": quake3.Node, + "PLANES": quake3.Plane} +# NOTE: BRUSH_SIDES, DRAW_VERTICES & SURFACES differ; related to RitualBsp? + +SPECIAL_LUMP_CLASSES = {"ENTITIES": shared.Entities, + "VISIBILITY": quake3.Visibility} + + +methods = [shared.worldspawn_volume] diff --git a/io_import_rbsp/bsp_tool/branches/respawn/__init__.py b/io_import_rbsp/bsp_tool/branches/respawn/__init__.py index a71afe7..e7433cd 100644 --- a/io_import_rbsp/bsp_tool/branches/respawn/__init__.py +++ b/io_import_rbsp/bsp_tool/branches/respawn/__init__.py @@ -1,15 +1,11 @@ -__all__ = ["FILE_MAGIC", "apex_legends", "titanfall", "titanfall2"] - -from . import apex_legends -from . import titanfall -from . import titanfall2 - -__doc__ = """Respawn Entertainment was founded by former Infinity Ward members. -Their version of the Source Engine was forked around 2010 and is heavily modified +"""Respawn Entertainment was founded by former Infinity Ward members. +Their version of the Source Engine was forked around 2011 from Portal 2. +While some remnants of the 2013 Source SDK remain, much is brand new +(though similarities to CoD 2 & CoD 4's formats exist in Titanfall). The Titanfall Engine has b"rBSP" file-magic and 128 lumps ~72 of the 128 lumps appear in .bsp_lump files -the naming convention for these files is: "..bsp_lump" +the naming convention for .bsp_lump files is: "..bsp_lump" where is a lowercase four digit hexadecimal string e.g. mp_rr_canyonlands.004a.bsp_lump -> 0x4A -> 74 -> VertexUnlitTS @@ -21,10 +17,19 @@ model_count appears to be the same across all .ent files for a given .bsp presumably all this file splitting has to do with streaming data into memory""" +# NOTE: CoD 4 FastFiles (*.ff) also decimated .bsps +# NOTE: .ent files for entities was introduced in Quake 3 +# NOTE: Level scripting was introduced with QuakeC scripts +# NOTE: Respawn uses Valve's VScript + custom Squirrel .nut scripts in a VM +# -- Likely forked from Left 4 Dead / Left 4 Dead 2 +from . import apex_legends +from . import titanfall +from . import titanfall2 + -FILE_MAGIC = b"rBSP" +scripts = [apex_legends, titanfall, titanfall2] # Trivia: # All Respawn's games give the error "Not an IBSP file" when FILE_MAGIC is incorrect -# - this text refers to Quake FILE_MAGIC, a game which released in 1996. +# - this text refers to QuakeII FILE_MAGIC, a game which released in 1997. # - does this mean Apex Legends contains code from the Quake engine? Yeah, probably. diff --git a/io_import_rbsp/bsp_tool/branches/respawn/apex_legends.py b/io_import_rbsp/bsp_tool/branches/respawn/apex_legends.py index a4f2c67..a961306 100644 --- a/io_import_rbsp/bsp_tool/branches/respawn/apex_legends.py +++ b/io_import_rbsp/bsp_tool/branches/respawn/apex_legends.py @@ -4,16 +4,22 @@ from .. import base from .. import shared from ..valve import source -from . import titanfall, titanfall2 +from . import titanfall +from . import titanfall2 +FILE_MAGIC = b"rBSP" + BSP_VERSION = 47 -GAMES = ["Apex Legends"] +GAME_PATHS = ["Apex Legends"] + GAME_VERSIONS = {"Apex Legends": 47, "Apex Legends: Season 7 - Ascension": 48, # Olympus "Apex Legends: Season 8 - Mayhem": 49, # King's Canyon map update 3 - "Apex Legends: Season 10 - Emergence": 50} # Arenas: Encore / SkyGarden + "Apex Legends: Season 10 - Emergence": 50, # Arenas: Encore / SkyGarden + "Apex Legends: Season 11 - Escape": 65586} # Nov19th patch version(50, 1) & all data in .bsp_lump +# NOTE: ^ (maps in depot/ still contain many lumps in the .bsp, as before) class LUMP(enum.Enum): @@ -146,6 +152,10 @@ class LUMP(enum.Enum): SHADOW_MESH_INDICES = 0x007E SHADOW_MESH_MESHES = 0x007F + +# struct RespawnBspHeader { char file_magic[4]; int version, revision, lump_count; SourceLumpHeader headers[128]; }; +lump_header_address = {LUMP_ID: (16 + i * 16) for i, LUMP_ID in enumerate(LUMP)} + # Known lump changes from Titanfall 2 -> Apex Legends: # New: # UNUSED_15 -> SURFACE_NAMES @@ -181,7 +191,7 @@ class LUMP(enum.Enum): # CM_BRUSH_TEX_VECS # TRICOLL_BEVEL_STARTS -# Rough map of the relationships between lumps: +# a rough map of the relationships between lumps: # Model -> Mesh -> MaterialSort -> TextureData -> SurfaceName # \--> VertexReservedX # \-> MeshIndex? @@ -208,10 +218,7 @@ class LUMP(enum.Enum): # NOTE: there are also always as many vert refs as edge refs -lump_header_address = {LUMP_ID: (16 + i * 16) for i, LUMP_ID in enumerate(LUMP)} - - -# # classes for lumps, in alphabetical order: +# classes for lumps, in alphabetical order: # NOTE: LightmapHeader.count doesn't look like a count, quite off in general class MaterialSort(base.Struct): # LUMP 82 (0052) @@ -305,6 +312,7 @@ class VertexLitFlat(base.Struct): # LUMP 72 (0048) class VertexUnlit(base.Struct): # LUMP 71 (0047) + # NOTE: identical to VertexLitFlat? position_index: int # index into Vertex lump normal_index: int # index into VertexNormal lump uv: List[float] # texture coordindates @@ -333,6 +341,7 @@ def ApexSPRP(raw_lump): BASIC_LUMP_CLASSES = titanfall2.BASIC_LUMP_CLASSES.copy() LUMP_CLASSES = titanfall2.LUMP_CLASSES.copy() +LUMP_CLASSES.pop("CM_GRID") LUMP_CLASSES.update({"LIGHTMAP_HEADERS": {0: titanfall.LightmapHeader}, "MATERIAL_SORT": {0: MaterialSort}, "MESHES": {0: Mesh}, @@ -346,13 +355,14 @@ def ApexSPRP(raw_lump): "VERTEX_LIT_FLAT": {0: VertexLitFlat}, "VERTEX_UNLIT": {0: VertexUnlit}, "VERTEX_UNLIT_TS": {0: VertexUnlitTS}}) -LUMP_CLASSES.pop("CM_GRID") SPECIAL_LUMP_CLASSES = titanfall2.SPECIAL_LUMP_CLASSES.copy() SPECIAL_LUMP_CLASSES.pop("TEXTURE_DATA_STRING_DATA") SPECIAL_LUMP_CLASSES.update({"SURFACE_NAMES": {0: shared.TextureDataStringData}}) +GAME_LUMP_HEADER = source.GameLumpHeader + GAME_LUMP_CLASSES = {"sprp": {bsp_version: ApexSPRP for bsp_version in (47, 48, 49, 50)}} diff --git a/io_import_rbsp/bsp_tool/branches/respawn/titanfall.py b/io_import_rbsp/bsp_tool/branches/respawn/titanfall.py index b77b303..85ff9a7 100644 --- a/io_import_rbsp/bsp_tool/branches/respawn/titanfall.py +++ b/io_import_rbsp/bsp_tool/branches/respawn/titanfall.py @@ -10,9 +10,12 @@ from ..valve import source +FILE_MAGIC = b"rBSP" + BSP_VERSION = 29 -GAMES = ["Titanfall", "Titanfall: Online"] +GAME_PATHS = ["Titanfall", "Titanfall: Online"] + GAME_VERSIONS = {"Titanfall": 29, "Titanfall: Online": 29} @@ -79,7 +82,7 @@ class LUMP(enum.Enum): UNUSED_59 = 0x003B UNUSED_60 = 0x003C UNUSED_61 = 0x003D - PHYSICS_LEVEL = 0x003E + PHYSICS_LEVEL = 0x003E # from L4D2 / INFRA ? UNUSED_63 = 0x003F UNUSED_64 = 0x0040 UNUSED_65 = 0x0041 @@ -146,19 +149,19 @@ class LUMP(enum.Enum): SHADOW_MESH_INDICES = 0x007E SHADOW_MESH_MESHES = 0x007F + +# struct RespawnBspHeader { char file_magic[4]; int version, revision, lump_count; SourceLumpHeader headers[128]; }; +lump_header_address = {LUMP_ID: (16 + i * 16) for i, LUMP_ID in enumerate(LUMP)} + # Rough map of the relationships between lumps: # /-> MaterialSort -> TextureData -> TextureDataStringTable -> TextureDataStringData -# Model -> Mesh -> MeshIndices -\-> VertexReservedX -> Vertex -# \-> .flags (VertexReservedX) \--> VertexNormal -# \-> .uv +# Model -> Mesh -> MeshIndex -\-> VertexReservedX -> Vertex +# \-> .flags (VertexReservedX) \--> VertexNormal +# \-> .uv # MeshBounds & Mesh are indexed in paralell? # -# TextureData -> TextureDataStringTable -> TextureDataStringTable -# VertexReservedX -> Vertex -# \-> VertexNormal -# # LeafWaterData -> TextureData -> water material # NOTE: LeafWaterData is also used in calculating VPhysics / PHYSICS_COLLIDE # @@ -184,24 +187,21 @@ class LUMP(enum.Enum): # (? * ? + ?) * 4 -> GridCell -lump_header_address = {LUMP_ID: (16 + i * 16) for i, LUMP_ID in enumerate(LUMP)} - - # flag enums class Flags(enum.IntFlag): - # source.Surface (source.TextureInfo / titanfall.TextureData ?) + # source.Surface (source.TextureInfo rolled into titanfall.TextureData ?) SKY_2D = 0x0002 # TODO: test overriding sky with this in-game SKY = 0x0004 - WARP = 0x0008 # water surface? - TRANSLUCENT = 0x0010 # VERTEX_UNLIT_TS ? + WARP = 0x0008 # Quake water surface? + TRANSLUCENT = 0x0010 # decals & atmo? # titanfall.Mesh.flags VERTEX_LIT_FLAT = 0x000 # VERTEX_RESERVED_1 VERTEX_LIT_BUMP = 0x200 # VERTEX_RESERVED_2 VERTEX_UNLIT = 0x400 # VERTEX_RESERVED_0 VERTEX_UNLIT_TS = 0x600 # VERTEX_RESERVED_3 # VERTEX_BLINN_PHONG = 0x??? # VERTEX_RESERVED_4 - # guesses - TRIGGER = 0x40000 + SKIP = 0x20000 # 0x200 in valve.source.Surface (<< 8?) + TRIGGER = 0x40000 # guessing # masks MASK_VERTEX = 0x600 @@ -289,18 +289,18 @@ class MaterialSort(base.MappedArray): # LUMP 82 (0052) class Mesh(base.Struct): # LUMP 80 (0050) - first_mesh_index: int # index into this Mesh's VertexReservedX - num_triangles: int # number of triangles in VertexReservedX after first_mesh_index - start_vertices: int # index to this Mesh's first VertexReservedX + first_mesh_index: int # index into MeshIndices + num_triangles: int # number of triangles in MeshIndices after first_mesh_index + first_vertex: int # index to this Mesh's first VertexReservedX num_vertices: int unknown: List[int] # for mp_box.VERTEX_LIT_BUMP: (2, -256, -1, ?, ?, ?) # for mp_box.VERTEX_UNLIT: (0, -1, -1, -1, -1, -1) material_sort: int # index of this Mesh's MaterialSort flags: int # Flags(mesh.flags & Flags.MASK_VERTEX).name == "VERTEX_RESERVED_X" - __slots__ = ["first_mesh_index", "num_triangles", "start_vertices", + __slots__ = ["first_mesh_index", "num_triangles", "first_vertex", "num_vertices", "unknown", "material_sort", "flags"] - _format = "IH8hHI" # 28 Bytes + _format = "I3H6hHI" # 28 Bytes _arrays = {"unknown": 6} @@ -438,7 +438,7 @@ class TextureData(base.Struct): # LUMP 2 (0002) class TextureVector(base.Struct): # LUMP 95 (005F) __slots__ = ["s", "t"] - __format = "8f" + _format = "8f" _arrays = {"s": [*"xyzw"], "t": [*"xyzw"]} @@ -534,9 +534,10 @@ def __init__(self, raw_sprp_lump: bytes, StaticPropClass: object): setattr(self, "leaves", leaves) prop_count, unknown_1, unknown_2 = struct.unpack("3i", sprp_lump.read(12)) self.unknown_1, self.unknown_2 = unknown_1, unknown_2 - prop_size = struct.calcsize(StaticPropClass._format) - props = struct.iter_unpack(StaticPropClass._format, sprp_lump.read(prop_count * prop_size)) - setattr(self, "props", list(map(StaticPropClass, props))) + # TODO: if StaticPropClass is None: split into appropriate groups of bytes + read_size = struct.calcsize(StaticPropClass._format) * prop_count + props = struct.iter_unpack(StaticPropClass._format, sprp_lump.read(read_size)) + setattr(self, "props", list(map(StaticPropClass.from_tuple, props))) def as_bytes(self) -> bytes: return b"".join([len(self.model_names).to_bytes(4, "little"), @@ -566,9 +567,9 @@ def as_bytes(self) -> bytes: LUMP_CLASSES = {"CELLS": {0: Cell}, "CELL_AABB_NODES": {0: Node}, - # "CELL_BSP_NODES": {0: Node}, + # "CELL_BSP_NODES": {0: Node}, "CM_BRUSHES": {0: Brush}, - # "CM_BRUSH_TEX_VECS": {0: TextureVector}, + "CM_BRUSH_TEX_VECS": {0: TextureVector}, "CM_GEO_SET_BOUNDS": {0: Bounds}, "CM_GRID": {0: Grid}, "CM_PRIMITIVE_BOUNDS": {0: Bounds}, @@ -607,9 +608,11 @@ def as_bytes(self) -> bytes: "ENTITIES": {0: shared.Entities}, # NOTE: .ent files are handled directly by the RespawnBsp class "PAKFILE": {0: shared.PakFile}, - "PHYSICS_COLLIDE": {0: shared.PhysicsCollide}, + "PHYSICS_COLLIDE": {0: shared.physics.CollideLump}, "TEXTURE_DATA_STRING_DATA": {0: shared.TextureDataStringData}} +GAME_LUMP_HEADER = source.GameLumpHeader + GAME_LUMP_CLASSES = {"sprp": {12: lambda raw_lump: GameLump_SPRP(raw_lump, StaticPropv12)}} diff --git a/io_import_rbsp/bsp_tool/branches/respawn/titanfall2.py b/io_import_rbsp/bsp_tool/branches/respawn/titanfall2.py index 326d2f6..11c1fc7 100644 --- a/io_import_rbsp/bsp_tool/branches/respawn/titanfall2.py +++ b/io_import_rbsp/bsp_tool/branches/respawn/titanfall2.py @@ -4,12 +4,16 @@ from typing import List from .. import base +from ..valve import source from . import titanfall +FILE_MAGIC = b"rBSP" + BSP_VERSION = 37 -GAMES = ["Titanfall 2"] +GAME_PATHS = ["Titanfall 2"] + GAME_VERSIONS = {"Titanfall 2": 37} @@ -143,6 +147,10 @@ class LUMP(enum.Enum): SHADOW_MESH_INDICES = 0x007E SHADOW_MESH_MESHES = 0x007F + +# struct RespawnBspHeader { char file_magic[4]; int version, revision, lump_count; SourceLumpHeader headers[128]; }; +lump_header_address = {LUMP_ID: (16 + i * 16) for i, LUMP_ID in enumerate(LUMP)} + # Known lump changes from Titanfall -> Titanfall 2: # New: # UNUSED_4 -> LIGHTPROBE_PARENT_INFOS @@ -184,9 +192,6 @@ class LUMP(enum.Enum): # NOTE: there are also always as many vert refs as edge refs -lump_header_address = {LUMP_ID: (16 + i * 16) for i, LUMP_ID in enumerate(LUMP)} - - # # classes for lumps, in alphabetical order:: class LightmapPage(base.Struct): data: bytes @@ -239,9 +244,10 @@ def __init__(self, raw_sprp_lump: bytes, StaticPropClass: object): setattr(self, "model_names", [t[0].replace(b"\0", b"").decode() for t in model_names]) prop_count, unknown_1, unknown_2 = struct.unpack("3i", sprp_lump.read(12)) self.unknown_1, self.unknown_2 = unknown_1, unknown_2 - prop_size = struct.calcsize(StaticPropClass._format) - props = struct.iter_unpack(StaticPropClass._format, sprp_lump.read(prop_count * prop_size)) - setattr(self, "props", list(map(StaticPropClass, props))) + # TODO: if StaticPropClass is None: split into appropriate groups of bytes + read_size = struct.calcsize(StaticPropClass._format) * prop_count + props = struct.iter_unpack(StaticPropClass._format, sprp_lump.read(read_size)) + setattr(self, "props", list(map(StaticPropClass.from_tuple, props))) # TODO: check if are there any leftover bytes at the end? def as_bytes(self) -> bytes: @@ -262,6 +268,8 @@ def as_bytes(self) -> bytes: SPECIAL_LUMP_CLASSES = titanfall.SPECIAL_LUMP_CLASSES.copy() +GAME_LUMP_HEADER = source.GameLumpHeader + GAME_LUMP_CLASSES = {"sprp": {13: lambda raw_lump: GameLump_SPRP(raw_lump, StaticPropv13)}} # branch exclusive methods, in alphabetical order: diff --git a/io_import_rbsp/bsp_tool/branches/ritual/__init__.py b/io_import_rbsp/bsp_tool/branches/ritual/__init__.py index 388c598..11e3b5f 100644 --- a/io_import_rbsp/bsp_tool/branches/ritual/__init__.py +++ b/io_import_rbsp/bsp_tool/branches/ritual/__init__.py @@ -1,9 +1,10 @@ +"""Ritual Entertainment developed Übertools from Quake III Arena (extends RavenBsp?)""" # https://web.archive.org/web/20070611074310/http://www.ritual.com/index.php?section=games/overview # http://ritualistic.chrissstrahl.de/games/ef2/gdkdocs/ -__all__ = [] -# TODO: ubertools -# TODO: mcgee -# TODO: star_trek +from . import fakk2 +from . import moh_allied_assault +from . import sin +from . import star_trek_elite_force2 -__doc__ = """Ritual Entertainment developed Übertools for Quake III""" +scripts = [fakk2, moh_allied_assault, sin, star_trek_elite_force2] diff --git a/io_import_rbsp/bsp_tool/branches/ritual/fakk2.py b/io_import_rbsp/bsp_tool/branches/ritual/fakk2.py new file mode 100644 index 0000000..148ad8f --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/ritual/fakk2.py @@ -0,0 +1,112 @@ +# https://github.com/zturtleman/spearmint/blob/master/code/qcommon/bsp_fakk.c +import enum +from typing import List + +from .. import base +from .. import shared +from ..id_software import quake3 + +FILE_MAGIC = b"FAKK" + +BSP_VERSION = 12 + +GAME_PATHS = ["Heavy Metal: F.A.K.K. 2", "American McGee's Alice"] + +GAME_VERSIONS = {"Heavy Metal: F.A.K.K. 2": 12, "American McGee's Alice": 42} + + +class LUMP(enum.Enum): + SHADERS = 0 + PLANES = 1 + LIGHTMAPS = 2 + SURFACES = 3 + DRAW_VERTICES = 4 + DRAW_INDICES = 5 + LEAF_BRUSHES = 6 + LEAF_SURFACES = 7 + LEAVES = 8 + NODES = 9 + BRUSH_SIDES = 10 + BRUSHES = 11 + FOGS = 12 + MODELS = 13 + ENTITIES = 14 + VISIBILITY = 15 + LIGHT_GRID = 16 + ENTITY_LIGHTS = 17 + ENTITY_LIGHTS_VISIBILITY = 18 + LIGHT_DEFINITIONS = 19 + + +# RitualBspHeader { char file_magic[4]; int version, checksum; QuakeLumpHeader headers[20]; }; +lump_header_address = {LUMP_ID: (12 + i * 8) for i, LUMP_ID in enumerate(LUMP)} + +# Known lump changes from Quake 3 -> Ubertools: +# New: +# FACES -> SURFACES +# LEAF_FACES -> LEAF_SURFACES +# TEXTURES -> SHADERS +# VERTICES -> DRAW_VERTICES +# MESH_VERTICES -> DRAW_INDICES + +# a rough map of the relationships between lumps: +# +# /-> Shader +# Model -> Brush -> BrushSide +# \-> Face -> MeshVertex +# \--> Texture +# \-> Vertex + + +# classes for lumps, in alphabetical order: +class Shader(base.Struct): # LUMP 0 + name: str + flags: List[int] + subdivisions: int # ??? new + __slots__ = ["name", "flags", "subdivisions"] + _format = "64s3i" + _arrays = {"flags": ["surface", "contents"]} + + +class Surface(base.Struct): # LUMP 3 + shader: int # index into Shader lump + fog: int # index into Fog lump + surface_type: int # see SurfaceType enum + first_vertex: int # index into DrawVertex lump + num_vertices: int # number of DrawVertices after first_vertex in this face + first_index: int # index into DrawIndices lump + num_indices: int # number of DrawIndices after first_index in this face + # lightmap.index: int # which of the 3 lightmap textures to use + # lightmap.top_left: List[int] # approximate top-left corner of visible lightmap segment + # lightmap.size: List[int] # size of visible lightmap segment + # lightmap.origin: List[float] # world space lightmap origin + # lightmap.vector: List[List[float]] # lightmap texture projection vectors + # NOTE: lightmap.vector is used for patches; first 2 indices are LoD bounds? + normal: List[float] + patch: List[float] # for patches (displacement-like) + subdivisions: float # ??? new + __slots__ = ["texture", "fog", "surface_type", "first_vertex", "num_vertices", + "first_index", "num_indices", "lightmap", "normal", "size", "subdivisions"] + _format = "12i12f2if" + _arrays = {"lightmap": {"index": None, "top_left": [*"xy"], "size": ["width", "height"], + "origin": [*"xyz"], "vector": {"s": [*"xyz"], "t": [*"xyz"]}}, + "normal": [*"xyz"], "patch": ["width", "height"]} + + +BASIC_LUMP_CLASSES = {"LEAF_BRUSHES": shared.Ints, + "LEAF_SURFACES": shared.Ints, + "DRAW_INDICES": shared.Ints} + +LUMP_CLASSES = {"BRUSH_SIDES": quake3.BrushSide, + "DRAW_VERTICES": quake3.Vertex, + "LEAVES": quake3.Leaf, + "MODELS": quake3.Model, + "NODES": quake3.Node, + "PLANES": quake3.Plane, + "SHADERS": Shader, + "SURFACES": Surface} + +SPECIAL_LUMP_CLASSES = quake3.SPECIAL_LUMP_CLASSES.copy() + +# branch exclusive methods, in alphabetical order: +methods = [shared.worldspawn_volume] diff --git a/io_import_rbsp/bsp_tool/branches/ritual/moh_allied_assault.py b/io_import_rbsp/bsp_tool/branches/ritual/moh_allied_assault.py new file mode 100644 index 0000000..87db31f --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/ritual/moh_allied_assault.py @@ -0,0 +1,129 @@ +# https://github.com/zturtleman/spearmint/blob/master/code/qcommon/bsp_mohaa.c +# TODO: finish copying implementation +import enum +from typing import List + +from .. import base +from .. import shared +from . import fakk2 + + +FILE_MAGIC = b"2015" + +BSP_VERSION = 19 + +GAME_PATHS = ["Medal of Honor: Allied Assault", + "Medal of Honor: Allied Assault - Breakthrough", + "Medal of Honor: Allied Assault - Spearhead"] + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} + + +class LUMP(enum.Enum): + SHADERS = 0 + PLANES = 1 + LIGHTMAPS = 2 + SURFACES = 3 + DRAW_VERTICES = 4 + DRAW_INDICES = 5 + LEAF_BRUSHES = 6 + LEAF_SURFACES = 7 + LEAVES = 8 + NODES = 9 + SIDE_EQUATIONS = 10 + BRUSH_SIDES = 11 + BRUSHES = 12 + MODELS = 13 + ENTITIES = 14 + VISIBILITY = 15 + LIGHT_GRID_PALETTE = 16 + LIGHT_GRID_OFFSETS = 17 + LIGHT_GRID_DATA = 18 + SPHERE_LIGHTS = 19 + SPHERE_LIGHT_VIS = 20 + LIGHT_DEFINITIONS = 21 + TERRAIN = 22 + TERRAIN_INDICES = 23 + STATIC_MODEL_DATA = 24 + STATIC_MODEL_DEFINITIONS = 25 + STATIC_MODEL_INDICES = 26 + UNKNOWN_27 = 27 + + +# RitualBspHeader { char file_magic[4]; int version, checksum; QuakeLumpHeader headers[20]; }; +lump_header_address = {LUMP_ID: (12 + i * 8) for i, LUMP_ID in enumerate(LUMP)} + +# Known lump changes from Ubertools -> Allied Assault: +# New: +# SIDE_EQUATIONS +# LIGHT_GRID_PALETTE +# LIGHT_GRID_OFFSETS +# LIGHT_GRID -> LIGHT_GRID_DATA? +# ENT_LIGHTS -> SPHERE_LIGHTS +# ENT_LIGHTS_VIS -> SPHERE_LIGHTS_VIS +# TERRAIN +# TERRAIN_INDICES +# STATIC_MODEL_DATA +# STATIC_MODEL_DEFINITIONS +# STATIC_MODEL_INDICES +# UNKNOWN_27 +# Deprecated: +# FOGS + +# a rough map of the relationships between lumps: +# +# /-> Shader +# Model -> Brush -> BrushSide +# \-> Face -> MeshVertex +# \--> Texture +# \-> Vertex + + +# classes for lumps, in alphabetical order: +class BrushSide(base.MappedArray): + plane: int # index into Plane lump + shader: int # index into Shader lump + equation: int # index into SideEquation lump + _mapping = ["plane", "shader", "equation"] + _format = "3i" + + +class Leaf(base.Struct): # LUMP 4 + cluster: int # index into VisData + area: int + mins: List[float] # Bounding box + maxs: List[float] + first_leaf_face: int # index into LeafFace lump + num_leaf_faces: int # number of LeafFaces in this Leaf + first_leaf_brush: int # index into LeafBrush lump + num_leaf_brushes: int # number of LeafBrushes in this Leaf + padding: List[int] + first_static_model: int # index into StaticModel lump + num_static_models: int # number of StaticModels in this Leaf + __slots__ = ["cluster", "area", "mins", "maxs", "first_leaf_face", + "num_leaf_faces", "first_leaf_brush", "num_leaf_brushes", + "padding", "first_static_model", "num_static_models"] # new + _format = "16i" + _arrays = {"mins": [*"xyz"], "maxs": [*"xyz"], "padding": 2} + + +class Shader(base.Struct): # LUMP 0 + name: str + flags: List[int] + subdivisions: int + fence_mask: str # new + __slots__ = ["name", "flags", "subdivisions", "fence_mask"] + _format = "64s3i64s" + _arrays = {"flags": ["surface", "contents"]} + + +BASIC_LUMP_CLASSES = fakk2.BASIC_LUMP_CLASSES.copy() + +LUMP_CLASSES = fakk2.LUMP_CLASSES.copy() +LUMP_CLASSES.update({"BRUSH_SIDES": BrushSide, + "LEAVES": Leaf, + "SHADERS": Shader}) + +SPECIAL_LUMP_CLASSES = fakk2.SPECIAL_LUMP_CLASSES.copy() + +methods = [shared.worldspawn_volume] diff --git a/io_import_rbsp/bsp_tool/branches/ritual/sin.py b/io_import_rbsp/bsp_tool/branches/ritual/sin.py new file mode 100644 index 0000000..5e34f81 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/ritual/sin.py @@ -0,0 +1,33 @@ +"""All three creditted programmers worked on Heavy Metal F.A.K.K. 2 +Some also went on to work on MoH:AA expansions & some Valve titles""" +from ..id_software import quake2 + + +FILE_MAGIC = b"RBSP" # Ubertools? +# NOTE: 1 b"IBSP" map exists, will it play? + +BSP_VERSION = 1 + +GAME_PATHS = ["SiN", "SiN Gold"] + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} + + +LUMP = quake2.LUMP + + +# struct Quake2BspHeader { char file_magic[4]; int version; QuakeLumpHeader headers[19]; }; +lump_header_address = {LUMP_ID: (8 + i * 8) for i, LUMP_ID in enumerate(LUMP)} + + +# {"LUMP": LumpClass} +BASIC_LUMP_CLASSES = quake2.BASIC_LUMP_CLASSES.copy() + +LUMP_CLASSES = quake2.LUMP_CLASSES.copy() +LUMP_CLASSES.pop("FACES") +LUMP_CLASSES.pop("TEXTURE_INFO") + +SPECIAL_LUMP_CLASSES = quake2.SPECIAL_LUMP_CLASSES.copy() + + +methods = [*quake2.methods] diff --git a/io_import_rbsp/bsp_tool/branches/ritual/star_trek_elite_force2.py b/io_import_rbsp/bsp_tool/branches/ritual/star_trek_elite_force2.py new file mode 100644 index 0000000..53e4d66 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/ritual/star_trek_elite_force2.py @@ -0,0 +1,112 @@ +# ef2GameSource3/Shared/qcommon/qfiles.h +# https://github.com/zturtleman/spearmint/blob/master/code/qcommon/bsp_ef2.c +# TODO: finish copying implementation +import enum +from typing import List + +from .. import base +from .. import shared +from . import fakk2 + + +FILE_MAGIC = b"EF2!" + +BSP_VERSION = 20 + +GAME_PATHS = ["Star Trek: Elite Force II"] + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} + + +class LUMP(enum.Enum): + SHADERS = 0 + PLANES = 1 + LIGHTMAPS = 2 + BASE_LIGHTMAPS = 3 + CONT_LIGHTMAPS = 4 + SURFACES = 5 + DRAW_VERTICES = 6 + DRAW_INDICES = 7 + LEAF_BRUSHES = 8 + LEAF_SURFACES = 9 + LEAFS = 10 + NODES = 11 + BRUSH_SIDES = 12 + BRUSHES = 13 + FOGS = 14 + MODELS = 15 + ENTITIES = 16 + VISIBILITY = 17 + LIGHT_GRID = 18 + ENTITY_LIGHTS = 19 + ENTITY_LIGHTS_VISIBILITY = 20 + LIGHT_DEFINITIONS = 21 + BASE_LIGHTING_VERTICES = 22 + CONT_LIGHTING_VERTICES = 23 + BASE_LIGHTING_SURFACES = 24 + LIGHTING_SURFACES = 25 + LIGHTING_VERTEX_SURFACES = 26 + LIGHTING_GROUPS = 27 + STATIC_LOD_MODELS = 28 + BSP_INFO = 29 + + +# RitualBspHeader { char file_magic[4]; int version, checksum; QuakeLumpHeader headers[20]; }; +lump_header_address = {LUMP_ID: (12 + i * 8) for i, LUMP_ID in enumerate(LUMP)} + + +# classes for lumps, in alphabetical order: +class DrawVertex(base.Struct): # LUMP 10 + position: List[float] + # uv.texture: List[float] + # uv.lightmap: List[float] + normal: List[float] + colour: bytes # 1 RGBA32 pixel / texel + lod_extra: float # ??? + lightmap: List[float] # union { float lightmap[2]; int collapse_map;} + __slots__ = ["position", "uv", "normal", "colour", "lod_extra", "lightmap"] + _format = "8f4B3f" + _arrays = {"position": [*"xyz"], "uv": [*"uv"], "normal": [*"xyz"], + "colour": [*"rgba"], "lightmap": [*"uv"]} + + +class Surface(base.Struct): # LUMP 3 + shader: int # index into Shader lump + fog: int # index into Fog lump + surface_type: int # see SurfaceType enum + first_vertex: int # index into DrawVertex lump + num_vertices: int # number of DrawVertices after first_vertex in this face + first_index: int # index into DrawIndices lump + num_indices: int # number of DrawIndices after first_index in this face + # lightmap.index: int # which of the 3 lightmap textures to use + # lightmap.top_left: List[int] # approximate top-left corner of visible lightmap segment + # lightmap.size: List[int] # size of visible lightmap segment + # lightmap.origin: List[float] # world space lightmap origin + # lightmap.vector: List[List[float]] # lightmap texture projection vectors + # NOTE: lightmap.vector is used for patches; first 2 indices are LoD bounds? + normal: List[float] + patch: List[float] # for patches (displacement-like) + subdivisions: float # ??? new + base_lighting_surface: int # index into BaseLightingSurface lump + terrain: List[int] + __slots__ = ["texture", "fog", "surface_type", "first_vertex", "num_vertices", + "first_index", "num_indices", "lightmap", "normal", "size", + "subdivisions", "base_lighting_surface", "terrain"] + _format = "12i12f2if6i" + _arrays = {"lightmap": {"index": None, "top_left": [*"xy"], + "size": ["width", "height"], "origin": [*"xyz"], + "vector": {"s": [*"xyz"], "t": [*"xyz"]}}, + "normal": [*"xyz"], "patch": ["width", "height"], + "terrain": {"inverted": None, "face_flags": 4}} + + +BASIC_LUMP_CLASSES = fakk2.BASIC_LUMP_CLASSES.copy() + +LUMP_CLASSES = fakk2.LUMP_CLASSES.copy() +LUMP_CLASSES.update({"DRAW_VERTICES": DrawVertex, + "SURFACES": Surface}) + +SPECIAL_LUMP_CLASSES = fakk2.SPECIAL_LUMP_CLASSES.copy() + + +methods = [shared.worldspawn_volume] diff --git a/io_import_rbsp/bsp_tool/branches/shared.py b/io_import_rbsp/bsp_tool/branches/shared.py index 9d4ca9f..8093dc2 100644 --- a/io_import_rbsp/bsp_tool/branches/shared.py +++ b/io_import_rbsp/bsp_tool/branches/shared.py @@ -1,13 +1,12 @@ -import collections -import enum import io -import itertools import math import re import struct import zipfile from typing import Dict, List +from . import physics # noqa F401 + # TODO: adapt SpecialLumpClasses to be more in-line with lumps.BspLump subclasses # TODO: make special classes __init__ method create an empty mutable object @@ -21,23 +20,11 @@ # TODO: consider using __repr__ methods, as SpecialLumpClasses can get large -# flag enums -class SPRP_flags(enum.IntFlag): - FADES = 0x1 # use fade distances - USE_LIGHTING_ORIGIN = 0x2 - NO_DRAW = 0x4 # computed at run time based on dx level - # the following are set in a level editor: - IGNORE_NORMALS = 0x8 - NO_SHADOW = 0x10 - SCREEN_SPACE_FADE = 0x20 - # next 3 are for lighting compiler - NO_PER_VERTEX_LIGHTING = 0x40 - NO_SELF_SHADOWING = 0x80 - NO_PER_TEXEL_LIGHTING = 0x100 - EDITOR_MASK = 0x1D8 +# Basic Lump Classes +class Bytes(int): + _format = "b" -# Basic Lump Classes class Ints(int): _format = "i" @@ -46,6 +33,10 @@ class Shorts(int): _format = "h" +class UnsignedBytes(int): + _format = "b" + + class UnsignedInts(int): _format = "I" @@ -63,7 +54,8 @@ def __init__(self, raw_entities: bytes): entities: List[Dict[str, str]] = list() # ^ [{"key": "value"}] # TODO: handle newlines in keys / values - for line_no, line in enumerate(raw_entities.decode(errors="ignore").splitlines()): + enumerated_lines = enumerate(raw_entities.decode(errors="ignore").splitlines()) + for line_no, line in enumerated_lines: if re.match(r"^\s*$", line): # line is blank / whitespace continue if "{" in line: # new entity @@ -71,7 +63,20 @@ def __init__(self, raw_entities: bytes): elif '"' in line: key_value_pair = re.search(r'"([^"]*)"\s"([^"]*)"', line) if not key_value_pair: - print(f"ERROR LOADING ENTITIES: Line {line_no:05d}: {line}") + open_key_value_pair = re.search(r'"([^"]*)"\s"([^"]*)', line) + if not open_key_value_pair: + RuntimeError(f"Unexpected line in entities: L{line_no}: {line.encode()}") + key, value = open_key_value_pair.groups() + # TODO: use regex to catch CRLF line endings & unexpected whitespace + tail = re.search(r'([^"]*)"\s*$', line) + while not tail: + if "{" in line or "}" in line: + RuntimeError(f"Unexpected line in entities: L{line_no}: {line.encode()}") + line_no, line = next(enumerated_lines) + # NOTE: ^ might've broken line numbers? + value += line + tail = re.search(r'([^"]*)"\s*$', line) + value += tail.groups()[0] continue key, value = key_value_pair.groups() if key not in ent: @@ -119,38 +124,6 @@ def as_bytes(self) -> bytes: return b"\n".join(map(lambda e: e.encode("ascii"), entities)) + b"\n\x00" -class GameLump_SPRP: # Mostly for Source - def __init__(self, raw_sprp_lump: bytes, StaticPropClass: object): - """Get StaticPropClass from GameLump version""" - # # lambda raw_lump: GameLump_SPRP(raw_lump, StaticPropvXX) - sprp_lump = io.BytesIO(raw_sprp_lump) - model_name_count = int.from_bytes(sprp_lump.read(4), "little") - model_names = struct.iter_unpack("128s", sprp_lump.read(128 * model_name_count)) - setattr(self, "model_names", [t[0].replace(b"\0", b"").decode() for t in model_names]) - leaf_count = int.from_bytes(sprp_lump.read(4), "little") - leaves = itertools.chain(*struct.iter_unpack("H", sprp_lump.read(2 * leaf_count))) - setattr(self, "leaves", list(leaves)) - prop_count = int.from_bytes(sprp_lump.read(4), "little") - read_size = struct.calcsize(StaticPropClass._format) * prop_count - props = struct.iter_unpack(StaticPropClass._format, sprp_lump.read(read_size)) - setattr(self, "props", list(map(StaticPropClass, props))) - here = sprp_lump.tell() - end = sprp_lump.seek(0, 2) - assert here == end, "Had some leftover bytes, bad format" - - def as_bytes(self) -> bytes: - if len(self.props) > 0: - prop_format = self.props[0]._format - else: - prop_format = "" - return b"".join([int.to_bytes(len(self.model_names), 4, "little"), - *[struct.pack("128s", n) for n in self.model_names], - int.to_bytes(len(self.leaves), 4, "little"), - *[struct.pack("H", L) for L in self.leaves], - int.to_bytes(len(self.props), 4, "little"), - *[struct.pack(prop_format, *p.flat()) for p in self.props]]) - - class PakFile(zipfile.ZipFile): def __init__(self, raw_zip: bytes): self._buffer = io.BytesIO(raw_zip) @@ -160,88 +133,6 @@ def as_bytes(self) -> bytes: return self._buffer.getvalue() -# PhysicsBlock headers -CollideHeader = collections.namedtuple("swapcollideheader_t", ["id", "version", "model_type"]) -# struct swapcollideheader_t { int size, vphysicsID; short version, model_type; }; -SurfaceHeader = collections.namedtuple("swapcompactsurfaceheader_t", ["size", "drag_axis_areas", "axis_map_size"]) -# struct swapcompactsurfaceheader_t { int surfaceSize; Vector dragAxisAreas; int axisMapSize; }; -MoppHeader = collections.namedtuple("swapmoppsurfaceheader_t", ["size"]) -# struct swapmoppsurfaceheader_t { int moppSize; }; - - -class PhysicsBlock: # TODO: actually process this data - # byte swapper: https://github.com/ValveSoftware/source-sdk-2013/blob/master/sp/src/utils/common/bsplib.cpp#L1677 - def __init__(self, raw_lump: bytes): - """Editting not yet supported""" - self._raw = raw_lump - lump = io.BytesIO(raw_lump) - header = CollideHeader(*struct.unpack("4s2h", lump.read(8))) - assert header.id == b"VPHY", "if 'YHPV' byte order is flipped" - # version isn't checked by the byteswap, probably important for VPHYSICS - if header.model_type == 0: - size, *drag_axis_areas, axis_map_size = struct.unpack("i3fi", lump.read(20)) - surface_header = SurfaceHeader(size, drag_axis_areas, axis_map_size) - elif header.model_type == 1: - surface_header = MoppHeader(*struct.unpack("i", lump.read(4))) - else: - raise RuntimeError("Invalid model type") - self.header = (header, surface_header) - self.data = lump.read(surface_header.size) - assert lump.tell() == len(raw_lump) - - def as_bytes(self) -> bytes: - header, surface_header = self.header - if header.model_type == 0: - size, drag_axis_areas, axis_map_size = surface_header - surface_header = struct.pack("i3fi", len(self.data), *drag_axis_areas, axis_map_size) - elif header.model_type == 1: - surface_header = struct.pack("i", len(self.data)) - else: - raise RuntimeError("Invalid model type") - header = struct.pack("4s2h", *header) - return b"".join([header, surface_header, self.data]) - - -# PhysicsCollide headers -PhysicsHeader = collections.namedtuple("dphysmodel_t", ["model", "data_size", "script_size", "solid_count"]) -# struct dphysmodel_t { int model_index, data_size, keydata_size, solid_count; }; - - -class PhysicsCollide(list): - """[model_index: int, solids: List[bytes], script: bytes]""" - # passed to VCollideLoad in vphysics.dll - def __init__(self, raw_lump: bytes): - collision_models = list() - lump = io.BytesIO(raw_lump) - header = PhysicsHeader(*struct.unpack("4i", lump.read(16))) - while header != PhysicsHeader(-1, -1, 0, 0) and lump.tell() != len(raw_lump): - solids = list() - for i in range(header.solid_count): - # CPhysCollisionEntry->WriteCollisionBinary - cb_size = int.from_bytes(lump.read(4), "little") - solids.append(PhysicsBlock(lump.read(cb_size))) - # TODO: assert header.data_size bytes were read - script = lump.read(header.script_size) # ascii - collision_models.append([header.model, solids, script]) - header = PhysicsHeader(*struct.unpack("4i", lump.read(16))) - assert header == PhysicsHeader(-1, -1, 0, 0), "PhysicsCollide ended incorrectly" - super().__init__(collision_models) - - def as_bytes(self) -> bytes: - def phy_bytes(collision_model): - model_index, solids, script = collision_model - phy_blocks = list() - for phy_block in solids: - collision_data = phy_block.as_bytes() - phy_blocks.append(len(collision_data).to_bytes(4, "little")) - phy_blocks.append(collision_data) - phy_block_bytes = b"".join(phy_blocks) - header = struct.pack("4i", model_index, len(phy_block_bytes), len(script), len(solids)) - return b"".join([header, phy_block_bytes, script]) - tail = struct.pack("4i", -1, -1, 0, 0) - return b"".join([*map(phy_bytes, self), tail]) - - class TextureDataStringData(list): def __init__(self, raw_texture_data_string_data: bytes): super().__init__([t.decode("ascii", errors="ignore") for t in raw_texture_data_string_data[:-1].split(b"\0")]) diff --git a/io_import_rbsp/bsp_tool/branches/troika/__init__.py b/io_import_rbsp/bsp_tool/branches/troika/__init__.py new file mode 100644 index 0000000..b16ab93 --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/troika/__init__.py @@ -0,0 +1,5 @@ +"""Creators of Vampire: The Masquerade - Bloodlines""" +from . import vampire + + +scripts = [vampire] diff --git a/io_import_rbsp/bsp_tool/branches/troika/vampire.py b/io_import_rbsp/bsp_tool/branches/troika/vampire.py new file mode 100644 index 0000000..884734f --- /dev/null +++ b/io_import_rbsp/bsp_tool/branches/troika/vampire.py @@ -0,0 +1,70 @@ +# https://developer.valvesoftware.com/wiki/Source_BSP_File_Format/Game-Specific#Vampire_The_Masquerade_-_Bloodlines +from typing import List + +from .. import base +from ..valve import source + + +FILE_MAGIC = b"VBSP" + +BSP_VERSION = 17 # technically older than HL2's Source Engine branch + +GAME_PATHS = ["Vampire The Masquerade - Bloodlines"] + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} + + +LUMP = source.LUMP + +# struct SourceBspHeader { char file_magic[4]; int version; SourceLumpHeader headers[64]; int revision; }; +lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} + + +read_lump_header = source.read_lump_header + + +# classes for lumps, in alphabetical order: +class Face(base.Struct): # LUMP 7 + """makes up Models (including worldspawn), also referenced by LeafFaces""" + light_colours: List[List[int]] # 8x RGBExp32 + plane: int # index into Plane lump + side: int # "faces opposite to the node's plane direction" + on_node: bool # if False, face is in a leaf + first_edge: int # index into the SurfEdge lump + num_edges: int # number of SurfEdges after first_edge in this Face + texture_info: int # index into the TextureInfo lump + displacement_info: int # index into the DisplacementInfo lump (None if -1) + surface_fog_volume_id: int # t-junctions? QuakeIII vertex-lit fog? + styles: List[List[int]] # "switchable lighting info"; selects an additional lightmap + light_offset: int # index of first pixel in LIGHTING / LIGHTING_HDR + area: float # surface area of this face + lightmap: List[float] + # lightmap.mins # dimensions of lightmap segment + # lightmap.size # scalars for lightmap segment + original_face: int # ORIGINAL_FACES index, -1 if this is an original face + smoothing_groups: int # lightmap smoothing group + __slots__ = ["light_colours", "plane", "side", "on_node", "first_edge", "num_edges", + "texture_info", "displacement_info", "surface_fog_volume_id", "styles", + "light_offset", "area", "lightmap", "original_face", "smoothing_groups"] + _format = "32BHb?i4h8b8b8bif5iI" + _arrays = {"light_colours": {i: [*"rgbe"] for i in range(8)}, + "styles": {"base": 8, "day": 8, "night": 8}, + "lightmap": {"mins": [*"xy"], "size": ["width", "height"]}} + + +# {"LUMP_NAME": {version: LumpClass}} +BASIC_LUMP_CLASSES = source.BASIC_LUMP_CLASSES.copy() + +LUMP_CLASSES = source.LUMP_CLASSES.copy() +LUMP_CLASSES.update({"FACES": {0: Face}, + "ORIGINAL_FACES": {0: Face}}) + +SPECIAL_LUMP_CLASSES = source.SPECIAL_LUMP_CLASSES.copy() +SPECIAL_LUMP_CLASSES.pop("PHYSICS_COLLIDE") + +GAME_LUMP_HEADER = source.GameLumpHeader + +GAME_LUMP_CLASSES = source.GAME_LUMP_CLASSES.copy() + + +methods = [*source.methods] diff --git a/io_import_rbsp/bsp_tool/branches/valve/__init__.py b/io_import_rbsp/bsp_tool/branches/valve/__init__.py index fa810dc..75cf5dd 100644 --- a/io_import_rbsp/bsp_tool/branches/valve/__init__.py +++ b/io_import_rbsp/bsp_tool/branches/valve/__init__.py @@ -1,6 +1,5 @@ -__all__ = ["FILE_MAGIC", "alien_swarm", "branches", "sdk_2013", "source", - "goldsrc", "left4dead", "left4dead2", "orange_box"] - +"""Valve Software developed the GoldSrc Engine, building on idTech 2. +This variant powered Half-Life & CS:1.6. Valve went on to develop the Source Engine for Half-Life 2.""" from . import alien_swarm from . import sdk_2013 from . import source @@ -8,12 +7,11 @@ from . import left4dead from . import left4dead2 from . import orange_box # Most Source Engine Games -# TODO: Portal 2 +# TODO: portal2 +# TODO: vampire -__doc__ = """Valve Software developed the GoldSrc Engine, building on idTech 2. -This variant powered Half-Life & CS:1.6. Valve went on to develop the Source Engine for Half-Life 2.""" + +scripts = [alien_swarm, goldsrc, left4dead, left4dead2, orange_box, sdk_2013, source] # TRIVIA: when FILE_MAGIC is incorrect Source engine games give the error message: "Not an IBSP file" # - this refers to the IdTech 2 FILE_MAGIC, pre-dating Half-Life 1! -FILE_MAGIC = b"VBSP" -# NOTE: GoldSrcBsp has no FILE_MAGIC diff --git a/io_import_rbsp/bsp_tool/branches/valve/alien_swarm.py b/io_import_rbsp/bsp_tool/branches/valve/alien_swarm.py index 27652aa..a867254 100644 --- a/io_import_rbsp/bsp_tool/branches/valve/alien_swarm.py +++ b/io_import_rbsp/bsp_tool/branches/valve/alien_swarm.py @@ -6,9 +6,13 @@ from . import source +FILE_MAGIC = b"VBSP" + BSP_VERSION = 21 -GAMES = ["Alien Swarm", "Alien Swarm Reactive Drop"] +GAME_PATHS = ["Alien Swarm", "Alien Swarm Reactive Drop"] + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} class LUMP(enum.Enum): @@ -77,6 +81,10 @@ class LUMP(enum.Enum): UNUSED_62 = 62 DISPLACEMENT_MULTIBLEND = 63 + +# struct SourceBspHeader { char file_magic[4]; int version; SourceLumpHeader headers[64]; int revision; }; +lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} + # Known lump changes from Orange Box -> Alien Swarm: # New: # UNUSED_63 -> DISPLACEMENT_MULTIBLEND @@ -84,9 +92,6 @@ class LUMP(enum.Enum): # ??? -lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} - - def read_lump_header(file, LUMP: enum.Enum) -> source.SourceLumpHeader: file.seek(lump_header_address[LUMP]) offset, length, version, fourCC = struct.unpack("4I", file.read(16)) @@ -108,7 +113,9 @@ def read_lump_header(file, LUMP: enum.Enum) -> source.SourceLumpHeader: SPECIAL_LUMP_CLASSES = orange_box.SPECIAL_LUMP_CLASSES.copy() +GAME_LUMP_HEADER = orange_box.GAME_LUMP_HEADER + GAME_LUMP_CLASSES = orange_box.GAME_LUMP_CLASSES.copy() -# branch exclusive methods, in alphabetical order: + methods = [*orange_box.methods] diff --git a/io_import_rbsp/bsp_tool/branches/valve/goldsrc.py b/io_import_rbsp/bsp_tool/branches/valve/goldsrc.py index 18a1ee1..6a7444c 100644 --- a/io_import_rbsp/bsp_tool/branches/valve/goldsrc.py +++ b/io_import_rbsp/bsp_tool/branches/valve/goldsrc.py @@ -3,21 +3,28 @@ # https://valvedev.info/tools/bsptwomap/ import enum -from ..id_software import quake # GoldSrc was forked from IdTech 2 during Quake II development +from ..id_software import quake +# NOTE: GoldSrc was forked from IdTech 2 during Quake II development +# -- so some elements of both quake & quake2 formats are present + + +FILE_MAGIC = None BSP_VERSION = 30 -GAMES = [*[f"Half-Life/{mod}" for mod in [ - "cstrike", # Counter-Strike - "czero", # Counter-Strike: Condition Zero - "czeror", # Counter-Strike: Condition Zero - Deleted Scenes - "dmc", # Deathmatch Classic - "dod", # Day of Defeat - "gearbox", # Half-Life: Opposing Force - "ricochet", # Ricochet - "tfc", # Team Fortress Classic - "valve"]], # Half-Life - "Halfquake Trilogy", "Sven Co-op"] +GAME_PATHS = [*[f"Half-Life/{mod}" for mod in [ + "cstrike", # Counter-Strike + "czero", # Counter-Strike: Condition Zero + "czeror", # Counter-Strike: Condition Zero - Deleted Scenes + "dmc", # Deathmatch Classic + "dod", # Day of Defeat + "gearbox", # Half-Life: Opposing Force + "ricochet", # Ricochet + "tfc", # Team Fortress Classic + "valve"]], # Half-Life + "Halfquake Trilogy", "Sven Co-op"] + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} class LUMP(enum.Enum): @@ -34,6 +41,7 @@ class LUMP(enum.Enum): LEAVES = 10 MARK_SURFACES = 11 EDGES = 12 + SURFEDGES = 13 MODELS = 14 # Known lump changes from Quake II -> GoldSrc: @@ -41,6 +49,7 @@ class LUMP(enum.Enum): # MARK_SURFACES +# struct QuakeBspHeader { int version; QuakeLumpHeader headers[15]; }; lump_header_address = {LUMP_ID: (4 + i * 8) for i, LUMP_ID in enumerate(LUMP)} @@ -92,8 +101,9 @@ class Contents(enum.IntFlag): # src/public/bspflags.h TRANSLUCENT = -15 -# classes for lumps, in alphabetical order:: -# TODO: Model, Node +# classes for lumps, in alphabetical order: +# TODO: Model +# TODO: Node # classes for special lumps, in alphabetical order: # TODO: make a special LumpCLass for MipTextures @@ -108,8 +118,6 @@ class Contents(enum.IntFlag): # src/public/bspflags.h LUMP_CLASSES.pop("NODES") SPECIAL_LUMP_CLASSES = quake.SPECIAL_LUMP_CLASSES.copy() -SPECIAL_LUMP_CLASSES.pop("MIP_TEXTURES") -# branch exclusive methods, in alphabetical order: methods = [*quake.methods] diff --git a/io_import_rbsp/bsp_tool/branches/valve/left4dead.py b/io_import_rbsp/bsp_tool/branches/valve/left4dead.py index 9b64973..8bef5b8 100644 --- a/io_import_rbsp/bsp_tool/branches/valve/left4dead.py +++ b/io_import_rbsp/bsp_tool/branches/valve/left4dead.py @@ -7,9 +7,13 @@ from . import source +FILE_MAGIC = b"VBSP" + BSP_VERSION = 20 -GAMES = ["Left 4 Dead"] +GAME_PATHS = ["Left 4 Dead"] + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} class LUMP(enum.Enum): @@ -78,12 +82,15 @@ class LUMP(enum.Enum): UNUSED_62 = 62 UNUSED_63 = 63 + +# struct SourceBspHeader { char file_magic[4]; int version; SourceLumpHeader headers[64]; int revision; }; +lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} + # Known lump changes from Orange Box -> Left 4 Dead: # New: # UNUSED_61 -> LUMP_OVERLAY_SYSTEM_LEVELS -lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} Left4Dead2LumpHeader = collections.namedtuple("Left4DeadLumpHeader", ["length", "offset", "version", "fourCC"]) # length and offset are swapped for L4D2 @@ -111,9 +118,12 @@ def read_lump_header(file, LUMP: enum.Enum) -> source.SourceLumpHeader: SPECIAL_LUMP_CLASSES = orange_box.SPECIAL_LUMP_CLASSES.copy() +GAME_LUMP_HEADER = orange_box.GAME_LUMP_HEADER + # {"lump": {version: SpecialLumpClass}} GAME_LUMP_CLASSES = orange_box.GAME_LUMP_CLASSES.copy() -# TODO: GAME_LUMP_CLASSES["sprp"].update({8: lambda raw_lump: shared.GameLump_SPRP(raw_lump, StaticPropv8)}) +GAME_LUMP_CLASSES["sprp"].pop(7) +# TODO: GAME_LUMP_CLASSES["sprp"].update({8: lambda raw_lump: source.GameLump_SPRP(raw_lump, StaticPropv8)}) # branch exclusive methods, in alphabetical order: diff --git a/io_import_rbsp/bsp_tool/branches/valve/left4dead2.py b/io_import_rbsp/bsp_tool/branches/valve/left4dead2.py index 7ec3bdc..34d8e3d 100644 --- a/io_import_rbsp/bsp_tool/branches/valve/left4dead2.py +++ b/io_import_rbsp/bsp_tool/branches/valve/left4dead2.py @@ -4,13 +4,17 @@ import enum import struct -from .. id_software import quake +from ..id_software import quake from . import left4dead +FILE_MAGIC = b"VBSP" + BSP_VERSION = 21 -GAMES = ["Left 4 Dead 2"] +GAME_PATHS = ["Left 4 Dead 2"] + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} class LUMP(enum.Enum): @@ -76,9 +80,13 @@ class LUMP(enum.Enum): MAP_FLAGS = 59 OVERLAY_FADES = 60 LUMP_OVERLAY_SYSTEM_LEVELS = 61 # overlay CPU & GPU limits - LUMP_PHYSLEVEL = 62 + LUMP_PHYSICS_LEVEL = 62 UNUSED_63 = 63 + +# struct SourceBspHeader { char file_magic[4]; int version; SourceLumpHeader headers[64]; int revision; }; +lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} + # Known lump changes from Left 4 Dead -> Left 4 Dead 2: # New: # UNUSED_22 -> PROP_COLLISION @@ -86,10 +94,8 @@ class LUMP(enum.Enum): # UNUSED_24 -> PROP_HULL_VERTS # UNUSED_25 -> PROP_HULL_TRIS # PHYSICS_COLLIDE_SURFACE -> PROP_BLOB -# UNUSED_62 -> LUMP_PHYSLEVEL +# UNUSED_62 -> LUMP_PHYSICS_LEVEL - -lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} Left4Dead2LumpHeader = collections.namedtuple("Left4DeadLumpHeader", ["version", "offset", "length", "fourCC"]) @@ -101,26 +107,29 @@ def read_lump_header(file, LUMP: enum.Enum) -> Left4Dead2LumpHeader: # classes for lumps, in alphabetical order: -# TODO: PropHull, PropHullTri +# TODO: PropHull +# TODO: PropHullTri # classes for special lumps, in alphabetical order: -# TODO: PropCollision, PropBlob +# TODO: PropCollision +# TODO: PropBlob +# TODO: StaticPropv8 # {"LUMP_NAME": {version: LumpClass}} BASIC_LUMP_CLASSES = left4dead.BASIC_LUMP_CLASSES.copy() LUMP_CLASSES = left4dead.LUMP_CLASSES.copy() -LUMP_CLASSES.update({"PROP_HULL_VERTS": quake.Vertex}) +LUMP_CLASSES.update({"PROP_HULL_VERTS": {0: quake.Vertex}}) SPECIAL_LUMP_CLASSES = left4dead.SPECIAL_LUMP_CLASSES.copy() +GAME_LUMP_HEADER = left4dead.GAME_LUMP_HEADER + # {"lump": {version: SpecialLumpClass}} GAME_LUMP_CLASSES = left4dead.GAME_LUMP_CLASSES.copy() -# TODO: GAME_LUMP_CLASSES["sprp"].update({8: lambda raw_lump: shared.GameLump_SPRP(raw_lump, StaticPropv8)}) - +# TODO: GAME_LUMP_CLASSES["sprp"].update({8: lambda raw_lump: source.GameLump_SPRP(raw_lump, StaticPropv8)}) -# branch exclusive methods, in alphabetical order: methods = [*left4dead.methods] diff --git a/io_import_rbsp/bsp_tool/branches/valve/orange_box.py b/io_import_rbsp/bsp_tool/branches/valve/orange_box.py index 1d85ab4..c03b29d 100644 --- a/io_import_rbsp/bsp_tool/branches/valve/orange_box.py +++ b/io_import_rbsp/bsp_tool/branches/valve/orange_box.py @@ -5,21 +5,23 @@ from typing import List from .. import base -from .. import shared from . import source -BSP_VERSION = 20 -# NOTE: v20 Source BSPs differ widely, since many forks are of this version +FILE_MAGIC = b"VBSP" -GAMES = ["Day of Defeat: Source", - "G String", - "Garry's Mod", - "Half-Life 2: Episode 2", - "Half-Life 2 Update", - "NEOTOKYO", - "Portal", - "Team Fortress 2"] +BSP_VERSION = 20 # NOTE: v20 Source BSPs differ widely, since many forks are of this version + +GAME_PATHS = ["Day of Defeat: Source", # TODO: full paths + "G String", + "Garry's Mod", + "Half-Life 2: Episode 2", + "Half-Life 2 Update", + "NEOTOKYO", + "Portal", + "Team Fortress 2"] + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} class LUMP(enum.Enum): @@ -89,6 +91,7 @@ class LUMP(enum.Enum): UNUSED_63 = 63 +# struct SourceBspHeader { char file_magic[4]; int version; SourceLumpHeader headers[64]; int revision; }; lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} @@ -181,13 +184,13 @@ class StaticPropv10(base.Struct): # sprp GAME LUMP (LUMP 35) SPECIAL_LUMP_CLASSES = source.SPECIAL_LUMP_CLASSES.copy() -# {"lump": {version: SpecialLumpClass}} -GAME_LUMP_CLASSES = source.GAME_LUMP_CLASSES.copy() -GAME_LUMP_CLASSES["sprp"].update({7: lambda raw_lump: shared.GameLump_SPRP(raw_lump, StaticPropv10), # 7* - 10: lambda raw_lump: shared.GameLump_SPRP(raw_lump, StaticPropv10)}) +GAME_LUMP_HEADER = source.GAME_LUMP_HEADER -# branch exclusive methods, in alphabetical order: +# {"lump": {version: SpecialLumpClass}} +GAME_LUMP_CLASSES = source.GAME_LUMP_CLASSES.copy() +GAME_LUMP_CLASSES["sprp"].update({7: lambda raw_lump: source.GameLump_SPRP(raw_lump, StaticPropv10), # 7* + 10: lambda raw_lump: source.GameLump_SPRP(raw_lump, StaticPropv10)}) methods = [*source.methods] diff --git a/io_import_rbsp/bsp_tool/branches/valve/sdk_2013.py b/io_import_rbsp/bsp_tool/branches/valve/sdk_2013.py index a512912..e1658ed 100644 --- a/io_import_rbsp/bsp_tool/branches/valve/sdk_2013.py +++ b/io_import_rbsp/bsp_tool/branches/valve/sdk_2013.py @@ -2,17 +2,99 @@ import enum import struct +# from . import alien_swarm +# from . import left4dead +# from . import left4dead2 from . import orange_box from . import source +FILE_MAGIC = b"VBSP" + BSP_VERSION = 21 -GAMES = ["Counter-Strike: Global Offensive", "Blade Symphony", "Portal 2", - "Source Filmmaker"] -GAME_VERSIONS = {game: BSP_VERSION for game in GAMES} +GAME_PATHS = ["Blade Symphony/berimbau", + "Counter-Strike Global Offensive/csgo", + "infra/infra", + "Portal 2/portal2", + "Source Filmmaker/game/tf"] +# also sourcemods / mapbase + +GAME_VERSIONS = {"infra/infra": 22, + **{GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS if GAME_PATH != "infra/infra"}} + + +# Counter-Strike Global Offensive/bin/bsppack.dll +class LUMP(enum.Enum): + ENTITIES = 0 + PLANES = 1 + TEXTURE_DATA = 2 + VERTICES = 3 + VISIBILITY = 4 + NODES = 5 + TEXTURE_INFO = 6 + FACES = 7 + LIGHTING = 8 + OCCLUSION = 9 + LEAVES = 10 + FACE_IDS = 11 + EDGES = 12 + SURFEDGES = 13 + MODELS = 14 + WORLD_LIGHTS = 15 + LEAF_FACES = 16 + LEAF_BRUSHES = 17 + BRUSHES = 18 + BRUSH_SIDES = 19 + AREAS = 20 + AREA_PORTALS = 21 + FACE_BRUSHES = 22 # infra + FACE_BRUSH_LIST = 23 # infra + UNUSED_24 = 24 + UNUSED_25 = 25 + DISPLACEMENT_INFO = 26 + ORIGINAL_FACES = 27 + PHYSICS_DISPLACEMENT = 28 + PHYSICS_COLLIDE = 29 + VERTEX_NORMALS = 30 + VERTEX_NORMAL_INDICES = 31 + DISPLACEMENT_LIGHTMAP_ALPHAS = 32 # deprecated / X360 ? + DISPLACEMENT_VERTICES = 33 + DISPLACEMENT_LIGHTMAP_SAMPLE_POSITIONS = 34 + GAME_LUMP = 35 + LEAF_WATER_DATA = 36 + PRIMITIVES = 37 + PRIMITIVE_VERTICES = 38 # deprecated / X360 ? + PRIMITIVE_INDICES = 39 + PAKFILE = 40 + CLIP_PORTAL_VERTICES = 41 + CUBEMAPS = 42 + TEXTURE_DATA_STRING_DATA = 43 + TEXTURE_DATA_STRING_TABLE = 44 + OVERLAYS = 45 + LEAF_MIN_DIST_TO_WATER = 46 + FACE_MACRO_TEXTURE_INFO = 47 + DISPLACEMENT_TRIS = 48 + PROP_BLOB = 49 # left4dead + WATER_OVERLAYS = 50 # deprecated / X360 ? + LEAF_AMBIENT_INDEX_HDR = 51 + LEAF_AMBIENT_INDEX = 52 + LIGHTING_HDR = 53 + WORLD_LIGHTS_HDR = 54 + LEAF_AMBIENT_LIGHTING_HDR = 55 + LEAF_AMBIENT_LIGHTING = 56 + XZIP_PAKFILE = 57 # deprecated / X360 ? + FACES_HDR = 58 + MAP_FLAGS = 59 + OVERLAY_FADES = 60 + OVERLAY_SYSTEM_LEVELS = 61 # left4dead + PHYSICS_LEVEL = 62 # left4dead2 + DISPLACEMENT_MULTIBLEND = 63 # alienswarm -LUMP = orange_box.LUMP +# TODO: Known lump changes from Orange Box -> Source SDK 2013: + + +# struct SourceBspHeader { char file_magic[4]; int version; SourceLumpHeader headers[64]; int revision; }; lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} @@ -23,8 +105,6 @@ def read_lump_header(file, LUMP: enum.Enum) -> source.SourceLumpHeader: return header -# classes for each lump, in alphabetical order: - # {"LUMP_NAME": {version: LumpClass}} BASIC_LUMP_CLASSES = orange_box.BASIC_LUMP_CLASSES.copy() @@ -34,6 +114,9 @@ def read_lump_header(file, LUMP: enum.Enum) -> source.SourceLumpHeader: SPECIAL_LUMP_CLASSES = orange_box.SPECIAL_LUMP_CLASSES.copy() +GAME_LUMP_HEADER = orange_box.GAME_LUMP_HEADER + GAME_LUMP_CLASSES = orange_box.GAME_LUMP_CLASSES.copy() +GAME_LUMP_CLASSES["sprp"].pop(10) methods = [*orange_box.methods] diff --git a/io_import_rbsp/bsp_tool/branches/valve/source.py b/io_import_rbsp/bsp_tool/branches/valve/source.py index ad22640..9fcefd2 100644 --- a/io_import_rbsp/bsp_tool/branches/valve/source.py +++ b/io_import_rbsp/bsp_tool/branches/valve/source.py @@ -1,5 +1,7 @@ import collections import enum +import io +import itertools import struct from typing import List @@ -9,13 +11,16 @@ from ..id_software import quake +FILE_MAGIC = b"VBSP" + BSP_VERSION = 19 # & 20 -GAMES = ["Counter-Strike: Source", # counter-strike source/cstrike - "Half-Life 1: Source - Deathmatch", # Half-Life 1 Source Deathmatch/hl1mp - "Half-Life 2", # Half-Life 2/hl2 - "Half-Life 2: Episode 1"] # Half-Life 2/episodic -GAME_VERSIONS = {game: BSP_VERSION for game in GAMES} +GAME_PATHS = ["counter-strike source/cstrike", # Counter-Strike: Source + "Half-Life 1 Source Deathmatch/hl1mp", # Half-Life 1: Source - Deathmatch + "Half-Life 2/hl2", # Half-Life 2 + "Half-Life 2/episodic"] # Half-Life 2: Episode 1 + +GAME_VERSIONS = {GAME_PATH: BSP_VERSION for GAME_PATH in GAME_PATHS} class LUMP(enum.Enum): @@ -85,7 +90,9 @@ class LUMP(enum.Enum): UNUSED_63 = 63 +# struct SourceBspHeader { char file_magic[4]; int version; SourceLumpHeader headers[64]; int revision; }; lump_header_address = {LUMP_ID: (8 + i * 16) for i, LUMP_ID in enumerate(LUMP)} + SourceLumpHeader = collections.namedtuple("SourceLumpHeader", ["offset", "length", "version", "fourCC"]) @@ -275,6 +282,21 @@ class DispTris(enum.IntFlag): SURFPROP2 = 0x10 # ? +class SPRP_flags(enum.IntFlag): + FADES = 0x1 # use fade distances + USE_LIGHTING_ORIGIN = 0x2 + NO_DRAW = 0x4 # computed at run time based on dx level + # the following are set in a level editor: + IGNORE_NORMALS = 0x8 + NO_SHADOW = 0x10 + SCREEN_SPACE_FADE = 0x20 + # next 3 are for lighting compiler + NO_PER_VERTEX_LIGHTING = 0x40 + NO_SELF_SHADOWING = 0x80 + NO_PER_TEXEL_LIGHTING = 0x100 + EDITOR_MASK = 0x1D8 + + class Surface(enum.IntFlag): # src/public/bspflags.h """TextureInfo flags""" # NOTE: vbsp sets these in src/utils/vbsp/textures.cpp LIGHT = 0x0001 # "value will hold the light strength" @@ -504,12 +526,55 @@ class WorldLight(base.Struct): # LUMP 15 _arrays = {"origin": [*"xyz"], "intensity": [*"xyz"], "normal": [*"xyz"]} -# classes for special lumps, in alphabetical order: +# special lump classes, in alphabetical order: +class GameLumpHeader(base.MappedArray): + id: str + flags: int + version: int + offset: int + length: int + _mapping = ["id", "flags", "version", "offset", "length"] + _format = "4s2H2i" + + +class GameLump_SPRP: + def __init__(self, raw_sprp_lump: bytes, StaticPropClass: object): + """Get StaticPropClass from GameLump version""" + # lambda raw_lump: GameLump_SPRP(raw_lump, StaticPropvXX) + sprp_lump = io.BytesIO(raw_sprp_lump) + model_name_count = int.from_bytes(sprp_lump.read(4), "little") + model_names = struct.iter_unpack("128s", sprp_lump.read(128 * model_name_count)) + setattr(self, "model_names", [t[0].replace(b"\0", b"").decode() for t in model_names]) + leaf_count = int.from_bytes(sprp_lump.read(4), "little") + leaves = itertools.chain(*struct.iter_unpack("H", sprp_lump.read(2 * leaf_count))) + setattr(self, "leaves", list(leaves)) + prop_count = int.from_bytes(sprp_lump.read(4), "little") + # TODO: if StaticPropClass is None: split into appropriate groups of bytes + read_size = struct.calcsize(StaticPropClass._format) * prop_count + props = struct.iter_unpack(StaticPropClass._format, sprp_lump.read(read_size)) + setattr(self, "props", list(map(StaticPropClass.from_tuple, props))) + here = sprp_lump.tell() + end = sprp_lump.seek(0, 2) + assert here == end, "Had some leftover bytes, bad format" + + def as_bytes(self) -> bytes: + if len(self.props) > 0: + prop_format = self.props[0]._format + else: + prop_format = "" + return b"".join([int.to_bytes(len(self.model_names), 4, "little"), + *[struct.pack("128s", n) for n in self.model_names], + int.to_bytes(len(self.leaves), 4, "little"), + *[struct.pack("H", L) for L in self.leaves], + int.to_bytes(len(self.props), 4, "little"), + *[struct.pack(prop_format, *p.flat()) for p in self.props]]) + + class StaticPropv4(base.Struct): # sprp GAME LUMP (LUMP 35) """https://github.com/ValveSoftware/source-sdk-2013/blob/master/sp/src/public/gamebspfile.h#L151""" origin: List[float] # origin.xyz angles: List[float] # origin.yzx QAngle; Z0 = East - name_index: int # index into AME_LUMP.sprp.model_names + name_index: int # index into GAME_LUMP.sprp.model_names first_leaf: int # index into Leaf lump num_leafs: int # number of Leafs after first_leaf this StaticPropv10 is in solid_mode: int # collision flags enum @@ -524,11 +589,11 @@ class StaticPropv4(base.Struct): # sprp GAME LUMP (LUMP 35) "lighting_origin": [*"xyz"]} -class StaticPropv5(base.Struct): # sprp GAME LUMP (LUMP 35) +class StaticPropv5(base.Struct): # sprp GAME LUMP (LUMP 35) [version 5] """https://github.com/ValveSoftware/source-sdk-2013/blob/master/sp/src/public/gamebspfile.h#L168""" origin: List[float] # origin.xyz angles: List[float] # origin.yzx QAngle; Z0 = East - name_index: int # index into AME_LUMP.sprp.model_names + name_index: int # index into GAME_LUMP.sprp.model_names first_leaf: int # index into Leaf lump num_leafs: int # number of Leafs after first_leaf this StaticPropv10 is in solid_mode: int # collision flags enum @@ -538,35 +603,38 @@ class StaticPropv5(base.Struct): # sprp GAME LUMP (LUMP 35) lighting_origin: List[float] # xyz position to sample lighting from forced_fade_scale: float # relative to pixels used to render on-screen? __slots__ = ["origin", "angles", "name_index", "first_leaf", "num_leafs", - "solid_mode", "skin", "fade_distance", "lighting_origin", + "solid_mode", "flags", "skin", "fade_distance", "lighting_origin", "forced_fade_scale"] - _format = "6f3HBi6f2Hi2H" + _format = "6f3H2Bi6f" _arrays = {"origin": [*"xyz"], "angles": [*"yzx"], "fade_distance": ["min", "max"], "lighting_origin": [*"xyz"]} -class StaticPropv6(base.Struct): # sprp GAME LUMP (LUMP 35) +class StaticPropv6(base.Struct): # sprp GAME LUMP (LUMP 35) [version 6] + """https://github.com/ValveSoftware/source-sdk-2013/blob/master/sp/src/public/gamebspfile.h#L186""" origin: List[float] # origin.xyz angles: List[float] # origin.yzx QAngle; Z0 = East - name_index: int # index into AME_LUMP.sprp.model_names + name_index: int # index into GAME_LUMP.sprp.model_names first_leaf: int # index into Leaf lump num_leafs: int # number of Leafs after first_leaf this StaticPropv10 is in solid_mode: int # collision flags enum + flags: int # other flags skin: int # index of this StaticProp's skin in the .mdl fade_distance: List[float] # min & max distances to fade out lighting_origin: List[float] # xyz position to sample lighting from forced_fade_scale: float # relative to pixels used to render on-screen? dx_level: List[int] # supported directX level, will not render depending on settings __slots__ = ["origin", "angles", "name_index", "first_leaf", "num_leafs", - "solid_mode", "skin", "fade_distance", "lighting_origin", + "solid_mode", "flags", "skin", "fade_distance", "lighting_origin", "forced_fade_scale", "dx_level"] - _format = "6f3HBi6f2Hi2H" + _format = "6f3H2Bi6f2H" _arrays = {"origin": [*"xyz"], "angles": [*"yzx"], "fade_distance": ["min", "max"], "lighting_origin": [*"xyz"], "dx_level": ["min", "max"]} # {"LUMP_NAME": {version: LumpClass}} BASIC_LUMP_CLASSES = {"DISPLACEMENT_TRIS": {0: shared.UnsignedShorts}, + "LEAF_BRUSHES": {0: shared.UnsignedShorts}, "LEAF_FACES": {0: shared.UnsignedShorts}, "SURFEDGES": {0: shared.Ints}, "TEXTURE_DATA_STRING_TABLE": {0: shared.UnsignedShorts}} @@ -596,14 +664,15 @@ class StaticPropv6(base.Struct): # sprp GAME LUMP (LUMP 35) SPECIAL_LUMP_CLASSES = {"ENTITIES": {0: shared.Entities}, "TEXTURE_DATA_STRING_DATA": {0: shared.TextureDataStringData}, "PAKFILE": {0: shared.PakFile}, - "PHYSICS_COLLIDE": {0: shared.PhysicsCollide} - } + "PHYSICS_COLLIDE": {0: shared.physics.CollideLump}} + + +GAME_LUMP_HEADER = GameLumpHeader # {"lump": {version: SpecialLumpClass}} -GAME_LUMP_CLASSES = {"sprp": {4: lambda raw_lump: shared.GameLump_SPRP(raw_lump, StaticPropv4), - 5: lambda raw_lump: shared.GameLump_SPRP(raw_lump, StaticPropv5), - 6: lambda raw_lump: shared.GameLump_SPRP(raw_lump, StaticPropv6)}} -# NOTE: having some errors with CS:S +GAME_LUMP_CLASSES = {"sprp": {4: lambda raw_lump: GameLump_SPRP(raw_lump, StaticPropv4), + 5: lambda raw_lump: GameLump_SPRP(raw_lump, StaticPropv5), + 6: lambda raw_lump: GameLump_SPRP(raw_lump, StaticPropv6)}} # branch exclusive methods, in alphabetical order: @@ -776,7 +845,7 @@ def rotated(q): return vertices -# TODO: vertices_of_model: walk the node tree +# TODO: vertices_of_model (walk the node tree) # TODO: vertices_of_node methods = [vertices_of_face, vertices_of_displacement, shared.worldspawn_volume] diff --git a/io_import_rbsp/bsp_tool/gearbox.py b/io_import_rbsp/bsp_tool/gearbox.py deleted file mode 100644 index 2ee62ef..0000000 --- a/io_import_rbsp/bsp_tool/gearbox.py +++ /dev/null @@ -1,9 +0,0 @@ -# https://valvedev.info/tools/bspfix/ -# https://developer.valvesoftware.com/wiki/Hl_bs.fgd -from . import valve - - -class GoldSrcBsp(valve.GoldSrcBsp): # WIP - # https://developer.valvesoftware.com/wiki/Half-Life:_Blue_Shift#Mapping - # TODO: figure out what gearbox changed - pass diff --git a/io_import_rbsp/bsp_tool/id_software.py b/io_import_rbsp/bsp_tool/id_software.py index 243913d..d5dfbc6 100644 --- a/io_import_rbsp/bsp_tool/id_software.py +++ b/io_import_rbsp/bsp_tool/id_software.py @@ -13,7 +13,7 @@ class QuakeBsp(base.Bsp): # Quake 1 only? - # NOTE: QuakeBsp has no file_magic? + file_magic = None def __init__(self, branch: ModuleType, filename: str = "untitled.bsp", autoload: bool = True): super(QuakeBsp, self).__init__(branch, filename, autoload) @@ -79,8 +79,7 @@ def is_related(f): return f.startswith(os.path.splitext(self.filename)[0]) # open .bsp self.file = open(os.path.join(self.folder, self.filename), "rb") file_magic = self.file.read(4) - if file_magic != self.file_magic: - raise RuntimeError(f"{self.file} is not a valid .bsp!") + assert file_magic == self.file_magic, f"{self.file} is not a valid .bsp!" self.bsp_version = int.from_bytes(self.file.read(4), "little") self.file.seek(0, 2) # move cursor to end of file self.bsp_file_size = self.file.tell() @@ -113,7 +112,7 @@ def is_related(f): return f.startswith(os.path.splitext(self.filename)[0]) BspLump = lumps.create_RawBspLump(self.file, lump_header) setattr(self, LUMP_name, BspLump) - def _read_header(self, LUMP: enum.Enum) -> (IdTechLumpHeader, bytes): + def _read_header(self, LUMP: enum.Enum) -> IdTechLumpHeader: self.file.seek(self.branch.lump_header_address[LUMP]) offset, length = struct.unpack("2i", self.file.read(8)) header = IdTechLumpHeader(offset, length) diff --git a/io_import_rbsp/bsp_tool/infinity_ward.py b/io_import_rbsp/bsp_tool/infinity_ward.py index 7f97429..78ff190 100644 --- a/io_import_rbsp/bsp_tool/infinity_ward.py +++ b/io_import_rbsp/bsp_tool/infinity_ward.py @@ -2,7 +2,9 @@ import enum import os import struct +from types import ModuleType from typing import Dict +import warnings from . import base from . import lumps @@ -11,14 +13,27 @@ LumpHeader = collections.namedtuple("LumpHeader", ["length", "offset"]) -class D3DBsp(base.Bsp): - file_magic = b"IBSP" +class InfinityWardBsp(base.Bsp): # https://wiki.zeroy.com/index.php?title=Call_of_Duty_1:_d3dbsp # https://wiki.zeroy.com/index.php?title=Call_of_Duty_2:_d3dbsp - # https://wiki.zeroy.com/index.php?title=Call_of_Duty_4:_d3dbsp - # NOTE: Call of Duty 1 has .bsp files in .pk3 archives - # NOTE: Call of Duty 2 has .d3dbsp in .iwd archives - # NOTE: Call of Duty 4 has .d3dbsp in .ff archives + file_magic = b"IBSP" + # NOTE: Call of Duty 1 .bsp are stored in .pk3 (.zip) archives + # NOTE: Call of Duty 2 .d3dbsp are stored in .iwd (.zip) archives + + def __init__(self, branch: ModuleType, filename: str = "untitled.bsp", autoload: bool = True): + if not (filename.lower().endswith(".bsp") or filename.lower().endswith(".d3dbsp")): + # ^ slight alteration to allow .d3dbsp extension + raise RuntimeError("Not a .bsp") + filename = os.path.realpath(filename) + self.folder, self.filename = os.path.split(filename) + self.set_branch(branch) + self.headers = dict() + if autoload: + if os.path.exists(filename): + self._preload() + else: + warnings.warn(UserWarning(f"{filename} not found, creating a new .bsp")) + self.headers = {L.name: LumpHeader(0, 0) for L in self.branch.LUMP} def _preload(self): """Loads filename using the format outlined in this .bsp's branch defintion script""" @@ -28,8 +43,7 @@ def is_related(f): return f.startswith(os.path.splitext(self.filename)[0]) # open .bsp self.file = open(os.path.join(self.folder, self.filename), "rb") file_magic = self.file.read(4) - if file_magic != self.file_magic: - raise RuntimeError(f"{self.file} is not a valid .bsp!") + assert file_magic == self.file_magic, f"{self.file} is not a valid .bsp!" self.bsp_version = int.from_bytes(self.file.read(4), "little") self.file.seek(0, 2) # move cursor to end of file self.bsp_file_size = self.file.tell() @@ -62,8 +76,101 @@ def is_related(f): return f.startswith(os.path.splitext(self.filename)[0]) BspLump = lumps.create_RawBspLump(self.file, lump_header) setattr(self, LUMP_NAME, BspLump) - def _read_header(self, LUMP: enum.Enum) -> (LumpHeader, bytes): + def _read_header(self, LUMP: enum.Enum) -> LumpHeader: self.file.seek(self.branch.lump_header_address[LUMP]) length, offset = struct.unpack("2i", self.file.read(8)) header = LumpHeader(length, offset) return header + + +CoD4LumpHeader = collections.namedtuple("LumpHeader", ["id", "length", "offset", "name"]) +# NOTE: offset is calculated from the sum of preceding lump's lengths (+ padding) +# NOTE: name is calculated from id, just for human-readability + + +class D3DBsp(base.Bsp): + # https://wiki.zeroy.com/index.php?title=Call_of_Duty_4:_d3dbsp + # https://github.com/SE2Dev/D3DBSP_Converter/blob/master/D3DBSP_Lib/D3DBSP.cpp + file_magic = b"IBSP" + lump_count: int + # NOTE: Call of Duty 2 [InfinityWardBsp] uses the .d3dbsp extension + # NOTE: Call of Duty 4 .d3dbsp are stored in .ff archives (see extensions.archive.FastFile) + # -- lumps are possibly divided into multiple files, quake3 map compilation generates many files + + def __init__(self, branch: ModuleType, filename: str = "untitled.bsp", autoload: bool = True): + if not filename.lower().endswith(".d3dbsp"): + # ^ slight alteration to allow .d3dbsp extension + raise RuntimeError("Not a .d3dbsp") + filename = os.path.realpath(filename) + self.folder, self.filename = os.path.split(filename) + self.set_branch(branch) + self.headers = dict() + if autoload: + if os.path.exists(filename): + self._preload() + else: + warnings.warn(UserWarning(f"{filename} not found, creating a new .bsp")) + self.headers = {L.name: LumpHeader(0, 0) for L in self.branch.LUMP} + + def _preload(self): + """Loads filename using the format outlined in this .bsp's branch defintion script""" + local_files = os.listdir(self.folder) + def is_related(f): return f.startswith(os.path.splitext(self.filename)[0]) + self.associated_files = [f for f in local_files if is_related(f)] + # open .bsp + self.file = open(os.path.join(self.folder, self.filename), "rb") + file_magic = self.file.read(4) + assert file_magic == self.file_magic, f"{self.file} is not a valid .bsp!" + self.bsp_version = int.from_bytes(self.file.read(4), "little") + self.lump_count = int.from_bytes(self.file.read(4), "little") + self.file.seek(0, 2) # move cursor to end of file + self.bsp_file_size = self.file.tell() + # load headers & lumps + self.headers = list() # order matters + self.loading_errors: Dict[str, Exception] = dict() + cursor = 12 + (self.lump_count * 8) # end of headers; for "reading" lumps + for i in range(self.lump_count): + # read header + self.file.seek(12 + 8 * i) + _id, length = struct.unpack("2i", self.file.read(8)) + assert length != 0, "cursed, idk how you got this error" + if _id != 0x07: # UNKNOWN_7 is padded to every 2nd byte? + cursor = cursor + (4 - cursor & 3) + offset = cursor + # NOTE: could be wrong + cursor += length + LUMP_enum = self.branch.LUMP(_id) + LUMP_NAME = LUMP_enum.name + lump_header = CoD4LumpHeader(_id, length, offset, LUMP_NAME) + self.headers.append(lump_header) + try: + if LUMP_NAME in self.branch.LUMP_CLASSES: + LumpClass = self.branch.LUMP_CLASSES[LUMP_NAME] + BspLump = lumps.create_BspLump(self.file, lump_header, LumpClass) + elif LUMP_NAME in self.branch.SPECIAL_LUMP_CLASSES: + SpecialLumpClass = self.branch.SPECIAL_LUMP_CLASSES[LUMP_NAME] + self.file.seek(lump_header.offset) + lump_data = self.file.read(lump_header.length) + BspLump = SpecialLumpClass(lump_data) + elif LUMP_NAME in self.branch.BASIC_LUMP_CLASSES: + LumpClass = self.branch.BASIC_LUMP_CLASSES[LUMP_NAME] + BspLump = lumps.create_BasicBspLump(self.file, lump_header, LumpClass) + else: + BspLump = lumps.create_RawBspLump(self.file, lump_header) + except Exception as exc: + self.loading_errors[LUMP_NAME] = exc + BspLump = lumps.create_RawBspLump(self.file, lump_header) + setattr(self, LUMP_NAME, BspLump) + + def _read_header(self, LUMP: enum.Enum) -> CoD4LumpHeader: + raise NotImplementedError("CoD4LumpHeaders aren't ordered") + + def print_headers(self): + print("LUMP_NAME", " " * 14, "OFFSET", "LENGTH") + print("-" * 38) + for header in self.headers: + print(f"{header.name:<24} {header.offset:06X} {header.length:06X}") + + +# NOTE: XenonBsp also exists (named after the XBox360 processor) +# -- however we aren't supporting console *.bsp diff --git a/io_import_rbsp/bsp_tool/lumps.py b/io_import_rbsp/bsp_tool/lumps/__init__.py similarity index 90% rename from io_import_rbsp/bsp_tool/lumps.py rename to io_import_rbsp/bsp_tool/lumps/__init__.py index a04cd25..4fc9789 100644 --- a/io_import_rbsp/bsp_tool/lumps.py +++ b/io_import_rbsp/bsp_tool/lumps/__init__.py @@ -1,3 +1,4 @@ +"""handles dynamically loading entries from lumps of all kinds""" from __future__ import annotations import collections @@ -176,7 +177,7 @@ def __init__(self, file: io.BufferedReader, lump_header: collections.namedtuple, self.LumpClass = LumpClass def __repr__(self): - return f"<{self.__class__.__name__}: {self.LumpClass.__name__}[{len(self)}] at 0x{id(self):016X}>" + return f"<{self.__class__.__name__}({len(self)} {self.LumpClass.__name__}) at 0x{id(self):016X}>" def __delitem__(self, index: Union[int, slice]): if isinstance(index, int): @@ -200,9 +201,9 @@ def __getitem__(self, index: Union[int, slice]): return self._changes[index] else: self.file.seek(self.offset + (index * self._entry_size)) - raw_entry = struct.unpack(self.LumpClass._format, self.file.read(self._entry_size)) - return self.LumpClass(raw_entry) - elif isinstance(index, slice): + _tuple = struct.unpack(self.LumpClass._format, self.file.read(self._entry_size)) + return self.LumpClass.from_tuple(_tuple) + elif isinstance(index, slice): # LAZY HACK _slice = _remap_slice(index, self._length) out = list() for i in range(_slice.start, _slice.stop, _slice.step): @@ -321,15 +322,14 @@ def __init__(self, lump_header: collections.namedtuple, LumpClass: object): self._changes = dict() # changes must be applied externally -GameLumpHeader = collections.namedtuple("GameLumpHeader", ["id", "flags", "version", "offset", "length"]) - - class GameLump: is_external = False loading_errors: Dict[str, Any] # ^ {"child_lump": Exception} - def __init__(self, file: io.BufferedReader, lump_header: collections.namedtuple, LumpClasses: Dict[str, object]): + def __init__(self, file: io.BufferedReader, lump_header: collections.namedtuple, + LumpClasses: Dict[str, object], GameLumpHeaderClass: object): + self.GameLumpHeaderClass = GameLumpHeaderClass self.loading_errors = dict() if not hasattr(lump_header, "filename"): file.seek(lump_header.offset) @@ -338,13 +338,13 @@ def __init__(self, file: io.BufferedReader, lump_header: collections.namedtuple, file = open(lump_header.filename, "rb") game_lumps_count = int.from_bytes(file.read(4), "little") self.headers = dict() + # {"child_name": child_header} for i in range(game_lumps_count): - _id, flags, version, offset, length = struct.unpack("4s2H2i", file.read(16)) - _id = _id.decode("ascii")[::-1] # b"prps" -> "sprp" - if self.is_external: - offset = offset - lump_header.offset - child_header = GameLumpHeader(_id, flags, version, offset, length) - self.headers[_id] = child_header + child_header = GameLumpHeaderClass.from_bytes(file.read(struct.calcsize(GameLumpHeaderClass._format))) + # ^ this is why we need a .from_stream() method for SpecialLumpClasses + if self.is_external: # HACK (does this ever happen?) + child_header.offset = child_header.offset - lump_header.offset + self.headers[child_header.id.decode("ascii")[::-1]] = child_header # b"prps" -> "sprp" for child_name, child_header in self.headers.items(): child_LumpClass = LumpClasses.get(child_name, dict()).get(child_header.version, None) if child_LumpClass is None: @@ -363,7 +363,9 @@ def as_bytes(self, lump_offset=0): out = [] out.append(len(self.headers).to_bytes(4, "little")) headers = [] - cursor_offset = lump_offset + 4 + len(self.headers) * 16 + # skip the headers + cursor_offset = lump_offset + 4 + len(self.headers) * struct.calcsize(self.GameLumpHeaderClass._format) + # write child lumps for child_name, child_header in self.headers.items(): child_lump = getattr(self, child_name) if isinstance(child_lump, RawBspLump): @@ -371,11 +373,11 @@ def as_bytes(self, lump_offset=0): else: child_lump_bytes = child_lump.as_bytes() # SpecialLumpClass method out.append(child_lump_bytes) - # calculate header - _id, flags, version, offset, length = child_header - _id = _id.encode("ascii")[::-1] # "sprp" -> b"prps" - offset, length = cursor_offset, len(child_lump_bytes) - cursor_offset += length - headers.append(struct.pack("4s2H2i", _id, flags, version, offset, length)) - out[1:1] = headers # insert headers after calculating + # recalculate header + child_header.offset = cursor_offset + child_header.length = len(child_lump_bytes) + cursor_offset += child_header.length + headers.append(child_header) + # and finally inject the headers back in before "writing" + out[1:1] = headers return b"".join(out) diff --git a/io_import_rbsp/bsp_tool/raven.py b/io_import_rbsp/bsp_tool/raven.py new file mode 100644 index 0000000..082cf37 --- /dev/null +++ b/io_import_rbsp/bsp_tool/raven.py @@ -0,0 +1,9 @@ +from . import id_software + + +class RavenBsp(id_software.IdTechBsp): + file_magic = b"RBSP" + + # includes marker lump: + # https://github.com/TTimo/GtkRadiant/blob/master/tools/urt/tools/quake3/q3map2/bspfile_rbsp.c#L308 + # sprintf( marker, "I LOVE MY Q3MAP2 %s on %s)", Q3MAP_VERSION, asctime( localtime( &t ) ) ); diff --git a/io_import_rbsp/bsp_tool/respawn.py b/io_import_rbsp/bsp_tool/respawn.py index 9598e7c..3286d1c 100644 --- a/io_import_rbsp/bsp_tool/respawn.py +++ b/io_import_rbsp/bsp_tool/respawn.py @@ -18,6 +18,7 @@ class RespawnBsp(base.Bsp): # https://developer.valvesoftware.com/wiki/Source_BSP_File_Format/Game-Specific#Titanfall # https://raw.githubusercontent.com/Wanty5883/Titanfall2/master/tools/TitanfallMapExporter.py file_magic = b"rBSP" + lump_count: int # NOTE: respawn .bsp files are usually stored in .vpk files # -- Respawn's .vpk format is different to Valve's # -- You'll need the Titanfall specific .vpk tool to extract maps @@ -27,7 +28,7 @@ def __init__(self, branch: ModuleType, filename: str = "untitled.bsp", autoload: super(RespawnBsp, self).__init__(branch, filename, autoload) # NOTE: bsp revision appears before headers, not after (as in Valve's variant) - def _read_header(self, LUMP: enum.Enum) -> (LumpHeader, bytes): + def _read_header(self, LUMP: enum.Enum) -> LumpHeader: """Read a lump from self.file, while it is open (during __init__ only)""" self.file.seek(self.branch.lump_header_address[LUMP]) offset, length, version, fourCC = struct.unpack("4I", self.file.read(16)) @@ -44,11 +45,11 @@ def is_related(f): return f.startswith(os.path.splitext(self.filename)[0]) self.associated_files = [f for f in local_files if is_related(f)] self.file = open(os.path.join(self.folder, self.filename), "rb") file_magic = self.file.read(4) - if file_magic != self.file_magic: - raise RuntimeError(f"{self.file} is not a valid .bsp!") + assert file_magic == self.file_magic, f"{self.file} is not a valid .bsp!" self.bsp_version = int.from_bytes(self.file.read(4), "little") - self.revision = int.from_bytes(self.file.read(4), "little") # just for rBSP - assert int.from_bytes(self.file.read(4), "little") == 127 + self.revision = int.from_bytes(self.file.read(4), "little") + self.lump_count = int.from_bytes(self.file.read(4), "little") + assert self.lump_count == 127, "irregular RespawnBsp lump_count" self.file.seek(0, 2) # move cursor to end of file self.bsp_file_size = self.file.tell() @@ -71,7 +72,7 @@ def is_related(f): return f.startswith(os.path.splitext(self.filename)[0]) try: if LUMP.name == "GAME_LUMP": GameLumpClasses = getattr(self.branch, "GAME_LUMP_CLASSES", dict()) - BspLump = lumps.GameLump(self.file, lump_header, GameLumpClasses) + BspLump = lumps.GameLump(self.file, lump_header, GameLumpClasses, self.branch.GAME_LUMP_HEADER) elif LUMP.name in self.branch.LUMP_CLASSES: LumpClass = self.branch.LUMP_CLASSES[LUMP.name][lump_header.version] BspLump = lumps.create_BspLump(self.file, lump_header, LumpClass) diff --git a/io_import_rbsp/bsp_tool/ritual.py b/io_import_rbsp/bsp_tool/ritual.py new file mode 100644 index 0000000..ea1144d --- /dev/null +++ b/io_import_rbsp/bsp_tool/ritual.py @@ -0,0 +1,55 @@ +import os +from typing import Dict + +from . import id_software +from . import lumps + + +class RitualBsp(id_software.IdTechBsp): + _file_magics = (b"RBSP", b"FAKK", b"2015", b"EF2!") + checksum: int # how is this calculated / checked? + + def _preload(self): # big copy-paste, should use super + dheader_t + """Loads filename using the format outlined in this .bsp's branch defintion script""" + local_files = os.listdir(self.folder) + def is_related(f): return f.startswith(os.path.splitext(self.filename)[0]) + self.associated_files = [f for f in local_files if is_related(f)] + # open .bsp + self.file = open(os.path.join(self.folder, self.filename), "rb") + # struct { int file_magic, bsp_version, checksum; lump_t lumps[20] }; + self.file_magic = self.file.read(4) + assert self.file_magic in self._file_magics, f"{self.file} is not a valid .bsp!" + assert self.file_magic == self.branch.FILE_MAGIC, f"{self.file} is not from {self.branch.GAME_PATHS[0]}!" + self.bsp_version = int.from_bytes(self.file.read(4), "little") + self.checksum = int.from_bytes(self.file.read(4), "little") + self.file.seek(0, 2) # move cursor to end of file + self.bsp_file_size = self.file.tell() + + # NOTE: this section should be it's own method + self.headers = dict() + self.loading_errors: Dict[str, Exception] = dict() + for LUMP_enum in self.branch.LUMP: + # CHECK: is lump external? (are associated_files overriding) + lump_header = self._read_header(LUMP_enum) + LUMP_name = LUMP_enum.name + self.headers[LUMP_name] = lump_header + if lump_header.length == 0: + continue + try: + if LUMP_name in self.branch.LUMP_CLASSES: + LumpClass = self.branch.LUMP_CLASSES[LUMP_name] + BspLump = lumps.create_BspLump(self.file, lump_header, LumpClass) + elif LUMP_name in self.branch.SPECIAL_LUMP_CLASSES: + SpecialLumpClass = self.branch.SPECIAL_LUMP_CLASSES[LUMP_name] + self.file.seek(lump_header.offset) + lump_data = self.file.read(lump_header.length) + BspLump = SpecialLumpClass(lump_data) + elif LUMP_name in self.branch.BASIC_LUMP_CLASSES: + LumpClass = self.branch.BASIC_LUMP_CLASSES[LUMP_name] + BspLump = lumps.create_BasicBspLump(self.file, lump_header, LumpClass) + else: + BspLump = lumps.create_RawBspLump(self.file, lump_header) + except Exception as exc: + self.loading_errors[LUMP_name] = exc + BspLump = lumps.create_RawBspLump(self.file, lump_header) + setattr(self, LUMP_name, BspLump) diff --git a/io_import_rbsp/bsp_tool/valve.py b/io_import_rbsp/bsp_tool/valve.py index 6370687..fda0e74 100644 --- a/io_import_rbsp/bsp_tool/valve.py +++ b/io_import_rbsp/bsp_tool/valve.py @@ -6,20 +6,17 @@ from typing import Dict from . import base +from . import id_software from . import lumps -from .id_software import IdTechBsp GoldSrcLumpHeader = namedtuple("GoldSrcLumpHeader", ["offset", "length"]) -class GoldSrcBsp(IdTechBsp): # TODO: subclass QuakeBsp? +class GoldSrcBsp(id_software.IdTechBsp): # TODO: QuakeBsp subclass? + file_magic = None # https://github.com/ValveSoftware/halflife/blob/master/utils/common/bspfile.h # http://hlbsp.sourceforge.net/index.php?content=bspdef - # NOTE: GoldSrcBsp has no file_magic! - - def __init__(self, branch: ModuleType, filename: str = "untitled.bsp", autoload: bool = True): - super(GoldSrcBsp, self).__init__(branch, filename, autoload) def __repr__(self): version = f"(version {self.bsp_version})" # no file_magic @@ -76,7 +73,9 @@ class ValveBsp(base.Bsp): def __init__(self, branch: ModuleType, filename: str = "untitled.bsp", autoload: bool = True): super(ValveBsp, self).__init__(branch, filename, autoload) - def _read_header(self, LUMP: enum.Enum) -> namedtuple: # LumpHeader + # TODO: migrate Source specific functionality from base.Bsp to ValveBsp + + def _read_header(self, LUMP: enum.Enum) -> namedtuple: # any LumpHeader """Get LUMP from self.branch.LUMP; e.g. self.branch.LUMP.ENTITIES""" # NOTE: each branch of VBSP has unique headers, # -- so branch.read_lump_header function is used