significant speedup in table read, starting work on movable buffer of…

…fsets

src/pyd2s/character.py

@@ -4,7 +4,6 @@
 '''
 
 import re
-import math
 from enum import Enum
 from os.path import dirname, join, isfile
 
@@ -35,15 +34,7 @@ def code(self):
         '''
         produce the short code of the character class
         '''
-        return {
-            self.AMAZON: 'ama',
-            self.SORCERESS: 'sor',
-            self.NECROMANCER: 'nec',
-            self.PALADIN: 'pal',
-            self.BARBARIAN: 'bar',
-            self.DRUID: 'dru',
-            self.ASSASSIN: 'ass',
-        }[self]
+        return GameData.playerclass[self.value]['Code']
 
     def __str__(self):
         '''
@@ -83,7 +74,7 @@ def __str__(self):
         '''
         produce the title of the skill
         '''
-        return GameData.skills[self.value]['skill']
+        return GameData.get_string(GameData.skills[self.value]['skill'])
 
 
 class AmazonSkillTree(SkillTree):
@@ -388,24 +379,7 @@ def bits(self):
                 '''
                 the bit width of the stat value
                 '''
-                return {
-                    self.STRENGTH: 10,
-                    self.ENERGY: 10,
-                    self.DEXTERITY: 10,
-                    self.VITALITY: 10,
-                    self.STATPTS: 10,
-                    self.NEWSKILLS: 8,
-                    self.HITPOINTS: 21,
-                    self.MAXHP: 21,
-                    self.MANA: 21,
-                    self.MAXMANA: 21,
-                    self.STAMINA: 21,
-                    self.MAXSTAMINA: 21,
-                    self.LEVEL: 7,
-                    self.EXPERIENCE: 32,
-                    self.GOLD: 25,
-                    self.GOLDBANK: 25,
-                }[self]
+                return int(GameData.itemstatcost[self.value]['CSvBits'])
 
         def __init__(self, buffer):
             '''
@@ -424,15 +398,15 @@ def __init__(self, buffer):
             if self._buffer.sparse:
                 return
 
-            position = 767 * 8
+            ptr = self._buffer.BitReadPointer(self._buffer, 767 * 8)
             while True:
-                statid = self._buffer.getbits(position, 9)
+                statid = ptr.read_bits(9)
                 if statid == 0x1FF:
                     break
                 stat = self.CharacterStat(statid)
-                self._positions[stat] = position + 9
-                position += 9 + stat.bits
-            self._end = position
+                self._positions[stat] = ptr.value
+                ptr.read_bits(stat.bits)
+            self._end = ptr.value
 
         @property
         def _header(self):
@@ -448,7 +422,7 @@ def length(self):
             '''
             the length of the stat section in bytes
             '''
-            return math.ceil((self._end + 9) / 8) - 765
+            return (self._end - 1) // 8 + 1 - 765
 
         def __getitem__(self, statid):
             '''
@@ -493,7 +467,7 @@ def __init__(self, buffer, offset):
             constructor - propagate buffer and parse stats section
             '''
             self._buffer = buffer
-            self._offset = offset
+            self._offset = buffer.dynamic_offset(offset)
 
             if self._header != 'if':
                 raise ValueError('invalid save: mismatched stat data section header')

src/pyd2s/gamedata.py

@@ -28,10 +28,10 @@ class _GameData:
         'gems': 'code',
         'properties': 'code',
         'runes': 'Name',
-        'playerclass': 'Code',
     }
     _TABLE_INDICES = {
         'itemstatcost': 'Stat',
+        'playerclass': 'Code',
     }
     _TABLE_INDEX_OFFSETS = {
         'magicprefix': 1,
@@ -175,42 +175,29 @@ def _load_string_tbl(self, filename):
         '''
         load one string table file into memory
         '''
-        def read_null_terminated_string(file):
-            '''
-            read a null-terminated string from a given file-like object
-            '''
-            res = bytearray()
-            while True:
-                next_byte = file.read(1)[0]
-                if next_byte == 0:
-                    break
-                res.append(next_byte)
-            return res.decode('cp1252')
-
         with open(filename, 'rb') as tbl:
-            header = tbl.read(21)
-            num_entries = struct.unpack_from('<H', header, 2)[0]
+            data = tbl.read()
 
-            entries = []
-            for _ in range(num_entries):
-                entries.append(struct.unpack('<H', tbl.read(2))[0])
+        pos = 0
+        num_entries = struct.unpack_from('<H', data, pos + 2)[0]
 
-            node_start = 21 + num_entries * 2
+        pos = 21
+        entries = []
+        for i in range(num_entries):
+            entries.append(struct.unpack_from('<H', data, pos + 2 * i)[0])
 
-            entry_dict = {}
-            for entry in entries:
-                tbl.seek(node_start + entry * 17)
-                hash_entry = tbl.read(17)
-                key_offset = struct.unpack_from('<L', hash_entry, 7)[0]
-                val_offset = struct.unpack_from('<L', hash_entry, 11)[0]
-
-                tbl.seek(key_offset)
-                key = read_null_terminated_string(tbl).lower()
-                tbl.seek(val_offset)
-                val = read_null_terminated_string(tbl)
-
-                if key not in entry_dict:
-                    entry_dict[key] = val
+        pos = 21 + num_entries * 2
+        entry_dict = {}
+        for entry in entries:
+            _pos = pos + entry * 17
+            key_offset = struct.unpack_from('<L', data, _pos + 7)[0]
+            val_offset = struct.unpack_from('<L', data, _pos + 11)[0]
+
+            key = data[key_offset:data.index(0, key_offset)].decode('cp1252').lower()
+            val = data[val_offset:data.index(0, val_offset)].decode('cp1252')
+
+            if key not in entry_dict:
+                entry_dict[key] = val
 
         return entry_dict
 

src/pyd2s/item.py

@@ -78,7 +78,7 @@ def __init__(self, buffer, offset):
         constructor
         '''
         self._buffer = buffer
-        self._offset = offset
+        self._offset = buffer.dynamic_offset(offset)
 
         self._itemdata = GameData.itemdata[self.type]
 

src/pyd2s/itemdata.py

@@ -18,7 +18,7 @@ def __init__(self, buffer, offset):
         constructor - propagate buffer
         '''
         self._buffer = buffer
-        self._offset = offset
+        self._offset = buffer.dynamic_offset(offset)
 
         # items on player / belt / cursor / in stash
         self._pdata = []
@@ -101,6 +101,10 @@ def _read_mdata(self, ptr):
             raise ValueError('invalid save: mismatched mercenary item data section header')
         ptr += 2
 
+        if self._buffer[ptr:ptr+2].decode('ascii') == 'kf':
+            # no mercenary data
+            return ptr + 2
+
         if self._buffer[ptr:ptr+2].decode('ascii') != 'JM':
             raise ValueError('invalid save: mismatched mercenary item data section header')
         ptr += 2

src/pyd2s/itemstat.py

@@ -480,7 +480,7 @@ def _str_formatter_27(self, *_):
         skill_desc = GameData.skilldesc[skill_data['skilldesc']]
         skill_name = GameData.get_string(skill_desc['str name'])
         class_code = skill_data['charclass']
-        class_name = GameData.playerclass[class_code]['Player Class']
+        class_name = GameData.playerclass_index[class_code]['Player Class']
         charstats = next(charstat for charstat in GameData.charstats
                          if charstat['class'] == class_name)
         str2 = GameData.get_string(charstats['StrClassOnly'])

src/pyd2s/savebuffer.py

@@ -46,19 +46,55 @@ def read_string(self):
                 res.append(next_char)
             return res.decode('ascii')
 
+        @property
+        def value(self):
+            '''
+            the current value of the pointer
+            '''
+            return self._pos
+
         @property
         def distance(self):
             '''
             the distance the pointer has travelled in bits
             '''
             return self._pos - self._start
 
+    class DynamicOffset:
+        '''
+        an offset into the save buffer that is kept around and updated when necessary
+        '''
+        def __init__(self, offset):
+            '''
+            constructor
+            '''
+            self._offset = offset
+
+        def __add__(self, other):
+            '''
+            add to the offset
+            '''
+            return self._offset + other
+
+        def __mul__(self, other):
+            '''
+            multiply the offset
+            '''
+            return self._offset * other
+
+        def __index__(self):
+            '''
+            use the offset as an index
+            '''
+            return self._offset
+
     def __init__(self, path):
         '''
         constructor - read the input file into memory
         '''
         self._path = path
         self._newpath = None
+        self._dynamic_offsets = []
 
         with open(path, 'rb') as save:
             super().__init__(save.read())
@@ -139,6 +175,14 @@ def addbits(self, start, length, align_at):
         self[byte:byte] = fill
         self.setbits(start + length, self.getbits(start, align_at - start), align_at - start)
 
+    def dynamic_offset(self, offset):
+        '''
+        produce a dynamic reference to an offset into the buffer
+        '''
+        res = self.DynamicOffset(offset)
+        self._dynamic_offsets.append(res)
+        return res
+
     def flush(self):
         '''
         write the changed data back to disk