Skip to content

Commit

Permalink
Load datafile into memory instead of keeping file open
Browse files Browse the repository at this point in the history 
When loading a datafile (map) with `CDataFileReader`, immediately read the entire map file into memory and perform all further operations on the memory buffer. This allows the file to be closed immediately after reading it into memory, so map files can be written in the editor while the respective map is loaded on a server or by another client or tool.

Remove `CDataFileReader::File` and `CMap::File`, as the file is immediately closed after reading and also not exposed to the datafile reader anymore.

Rename `CDataFileReader::MapSize` and `CMap::MapSize` to `FileSize` and make them return the actual size of the map that was read into memory instead of the size that is declared in the datafile header.

Add `CDataFileReader::FileData` and `CMap::FileData` to get a `const` pointer to the entire file memory buffer.

Internally add `CDataFileMemoryReader` to handle reading operations on the memory buffer as replacement for `io_read` and `io_seek` reading from the file.
  • Loading branch information
@Robyt3
Robyt3 committed Jul 26, 2023
1 parent 22bd19b commit c740a97
Show file tree
Hide file tree
Showing 8 changed files with 113 additions and 93 deletions.
4 changes: 2 additions & 2 deletions src/engine/client/client.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3967,7 +3967,7 @@ void CClient::DemoRecorder_Start(const char *pFilename, bool WithTimestamp, int
str_format(aFilename, sizeof(aFilename), "demos/%s.demo", pFilename);

SHA256_DIGEST Sha256 = m_pMap->Sha256();
m_aDemoRecorder[Recorder].Start(Storage(), m_pConsole, aFilename, GameClient()->NetVersion(), m_aCurrentMap, &Sha256, m_pMap->Crc(), "client", m_pMap->MapSize(), 0, m_pMap->File());
m_aDemoRecorder[Recorder].Start(Storage(), m_pConsole, aFilename, GameClient()->NetVersion(), m_aCurrentMap, &Sha256, m_pMap->Crc(), "client", m_pMap->FileSize(), m_pMap->FileData());
}
}

Expand Down Expand Up @@ -4885,7 +4885,7 @@ void CClient::RaceRecord_Start(const char *pFilename)
else
{
SHA256_DIGEST Sha256 = m_pMap->Sha256();
m_aDemoRecorder[RECORDER_RACE].Start(Storage(), m_pConsole, pFilename, GameClient()->NetVersion(), m_aCurrentMap, &Sha256, m_pMap->Crc(), "client", m_pMap->MapSize(), 0, m_pMap->File());
m_aDemoRecorder[RECORDER_RACE].Start(Storage(), m_pConsole, pFilename, GameClient()->NetVersion(), m_aCurrentMap, &Sha256, m_pMap->Crc(), "client", m_pMap->FileSize(), m_pMap->FileData());
}
}

Expand Down
4 changes: 2 additions & 2 deletions src/engine/map.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,11 @@ class IEngineMap : public IMap
virtual bool Load(const char *pMapName) = 0;
virtual void Unload() = 0;
virtual bool IsLoaded() const = 0;
virtual IOHANDLE File() const = 0;

virtual SHA256_DIGEST Sha256() const = 0;
virtual unsigned Crc() const = 0;
virtual int MapSize() const = 0;
virtual const unsigned char *FileData() const = 0;
virtual unsigned FileSize() const = 0;
};

extern IEngineMap *CreateEngineMap();
Expand Down
157 changes: 88 additions & 69 deletions src/engine/shared/datafile.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,9 +80,9 @@ struct CDatafileInfo

struct CDatafile
{
IOHANDLE m_File;
SHA256_DIGEST m_Sha256;
unsigned m_Crc;

CDatafileInfo m_Info;
CDatafileHeader m_Header;
int m_DataStartOffset;
Expand All @@ -91,48 +91,70 @@ struct CDatafile
char *m_pData;
};

bool CDataFileReader::Open(class IStorage *pStorage, const char *pFilename, int StorageType)
class CDataFileMemoryReader
{
log_trace("datafile", "loading. filename='%s'", pFilename);
private:
unsigned char *m_pFileData;
unsigned m_FileSize;
unsigned m_Position = 0;

IOHANDLE File = pStorage->OpenFile(pFilename, IOFLAG_READ, StorageType);
if(!File)
public:
CDataFileMemoryReader(unsigned char *pFileData, unsigned FileSize) :
m_pFileData(pFileData), m_FileSize(FileSize)
{
dbg_msg("datafile", "could not open '%s'", pFilename);
return false;
}

// take the CRC of the file and store it
unsigned Crc = 0;
SHA256_DIGEST Sha256;
~CDataFileMemoryReader()
{
enum
{
BUFFER_SIZE = 64 * 1024
};
free(m_pFileData);
}

SHA256_CTX Sha256Ctxt;
sha256_init(&Sha256Ctxt);
unsigned char aBuffer[BUFFER_SIZE];
void SetPosition(unsigned Position)
{
m_Position = Position;
}

while(true)
{
unsigned Bytes = io_read(File, aBuffer, BUFFER_SIZE);
if(Bytes == 0)
break;
Crc = crc32(Crc, aBuffer, Bytes);
sha256_update(&Sha256Ctxt, aBuffer, Bytes);
}
Sha256 = sha256_finish(&Sha256Ctxt);
unsigned Read(void *pBuffer, unsigned Size)
{
dbg_assert(Size > 0, "Size cannot be zero");
if(m_Position >= m_FileSize)
return 0;
const unsigned ActualSize = minimum(m_FileSize - m_Position, Size);
mem_copy(pBuffer, m_pFileData + m_Position, ActualSize);
m_Position += ActualSize;
return ActualSize;
}

const unsigned char *FileData() const
{
return m_pFileData;
}

unsigned FileSize() const
{
return m_FileSize;
}
};

bool CDataFileReader::Open(class IStorage *pStorage, const char *pFilename, int StorageType)
{
log_trace("datafile", "loading. filename='%s'", pFilename);

io_seek(File, 0, IOSEEK_START);
void *pFileDataTmp;
unsigned ReadFileSize;
if(!pStorage->ReadFile(pFilename, StorageType, &pFileDataTmp, &ReadFileSize))
{
dbg_msg("datafile", "could not open/read '%s'", pFilename);
return false;
}
// The memory reader owns the file data now and fill free it
std::shared_ptr<CDataFileMemoryReader> pTmpMemoryReader = std::make_shared<CDataFileMemoryReader>(static_cast<unsigned char *>(pFileDataTmp), ReadFileSize);

// TODO: change this header
CDatafileHeader Header;
if(sizeof(Header) != io_read(File, &Header, sizeof(Header)))
if(sizeof(Header) != pTmpMemoryReader->Read(&Header, sizeof(Header)))
{
dbg_msg("datafile", "couldn't load header");
dbg_msg("datafile", "could not read datafile header");
return false;
}
if(Header.m_aID[0] != 'A' || Header.m_aID[1] != 'T' || Header.m_aID[2] != 'A' || Header.m_aID[3] != 'D')
Expand Down Expand Up @@ -167,7 +189,6 @@ bool CDataFileReader::Open(class IStorage *pStorage, const char *pFilename, int
AllocSize += Header.m_NumRawData * sizeof(int); // add space for data sizes
if(Size > (((int64_t)1) << 31) || Header.m_NumItemTypes < 0 || Header.m_NumItems < 0 || Header.m_NumRawData < 0 || Header.m_ItemSize < 0)
{
io_close(File);
dbg_msg("datafile", "unable to load file, invalid file information");
return false;
}
Expand All @@ -178,59 +199,59 @@ bool CDataFileReader::Open(class IStorage *pStorage, const char *pFilename, int
pTmpDataFile->m_ppDataPtrs = (char **)(pTmpDataFile + 1);
pTmpDataFile->m_pDataSizes = (int *)(pTmpDataFile->m_ppDataPtrs + Header.m_NumRawData);
pTmpDataFile->m_pData = (char *)(pTmpDataFile->m_pDataSizes + Header.m_NumRawData);
pTmpDataFile->m_File = File;
pTmpDataFile->m_Sha256 = Sha256;
pTmpDataFile->m_Crc = Crc;
pTmpDataFile->m_Sha256 = sha256(pTmpMemoryReader->FileData(), pTmpMemoryReader->FileSize());
pTmpDataFile->m_Crc = crc32(0, pTmpMemoryReader->FileData(), pTmpMemoryReader->FileSize());

// clear the data pointers and sizes
mem_zero(pTmpDataFile->m_ppDataPtrs, Header.m_NumRawData * sizeof(void *));
mem_zero(pTmpDataFile->m_pDataSizes, Header.m_NumRawData * sizeof(int));

// read types, offsets, sizes and item data
unsigned ReadSize = io_read(File, pTmpDataFile->m_pData, Size);
unsigned ReadSize = pTmpMemoryReader->Read(pTmpDataFile->m_pData, Size);
if(ReadSize != Size)
{
io_close(pTmpDataFile->m_File);
free(pTmpDataFile);
dbg_msg("datafile", "couldn't load the whole thing, wanted=%d got=%d", Size, ReadSize);
return false;
}

Close();
m_pDataFile = pTmpDataFile;

#if defined(CONF_ARCH_ENDIAN_BIG)
swap_endian(m_pDataFile->m_pData, sizeof(int), minimum(static_cast<unsigned>(Header.m_Swaplen), Size) / sizeof(int));
swap_endian(pTmpDataFile->m_pData, sizeof(int), minimum(static_cast<unsigned>(Header.m_Swaplen), Size) / sizeof(int));
#endif

if(DEBUG)
{
dbg_msg("datafile", "allocsize=%d", AllocSize);
dbg_msg("datafile", "readsize=%d", ReadSize);
dbg_msg("datafile", "swaplen=%d", Header.m_Swaplen);
dbg_msg("datafile", "item_size=%d", m_pDataFile->m_Header.m_ItemSize);
dbg_msg("datafile", "item_size=%d", pTmpDataFile->m_Header.m_ItemSize);
}

m_pDataFile->m_Info.m_pItemTypes = (CDatafileItemType *)m_pDataFile->m_pData;
m_pDataFile->m_Info.m_pItemOffsets = (int *)&m_pDataFile->m_Info.m_pItemTypes[m_pDataFile->m_Header.m_NumItemTypes];
m_pDataFile->m_Info.m_pDataOffsets = &m_pDataFile->m_Info.m_pItemOffsets[m_pDataFile->m_Header.m_NumItems];
m_pDataFile->m_Info.m_pDataSizes = &m_pDataFile->m_Info.m_pDataOffsets[m_pDataFile->m_Header.m_NumRawData];
pTmpDataFile->m_Info.m_pItemTypes = (CDatafileItemType *)pTmpDataFile->m_pData;
pTmpDataFile->m_Info.m_pItemOffsets = (int *)&pTmpDataFile->m_Info.m_pItemTypes[pTmpDataFile->m_Header.m_NumItemTypes];
pTmpDataFile->m_Info.m_pDataOffsets = &pTmpDataFile->m_Info.m_pItemOffsets[pTmpDataFile->m_Header.m_NumItems];
pTmpDataFile->m_Info.m_pDataSizes = &pTmpDataFile->m_Info.m_pDataOffsets[pTmpDataFile->m_Header.m_NumRawData];

if(Header.m_Version == 4)
m_pDataFile->m_Info.m_pItemStart = (char *)&m_pDataFile->m_Info.m_pDataSizes[m_pDataFile->m_Header.m_NumRawData];
pTmpDataFile->m_Info.m_pItemStart = (char *)&pTmpDataFile->m_Info.m_pDataSizes[pTmpDataFile->m_Header.m_NumRawData];
else
m_pDataFile->m_Info.m_pItemStart = (char *)&m_pDataFile->m_Info.m_pDataOffsets[m_pDataFile->m_Header.m_NumRawData];
m_pDataFile->m_Info.m_pDataStart = m_pDataFile->m_Info.m_pItemStart + m_pDataFile->m_Header.m_ItemSize;
pTmpDataFile->m_Info.m_pItemStart = (char *)&pTmpDataFile->m_Info.m_pDataOffsets[pTmpDataFile->m_Header.m_NumRawData];
pTmpDataFile->m_Info.m_pDataStart = pTmpDataFile->m_Info.m_pItemStart + pTmpDataFile->m_Header.m_ItemSize;

log_trace("datafile", "loading done. datafile='%s'", pFilename);

// success! replace old with new data.
Close();
m_pMemoryReader = pTmpMemoryReader;
m_pDataFile = pTmpDataFile;

return true;
}

bool CDataFileReader::Close()
void CDataFileReader::Close()
{
if(!m_pDataFile)
return true;
return;

// free the data that is loaded
for(int i = 0; i < m_pDataFile->m_Header.m_NumRawData; i++)
Expand All @@ -240,17 +261,10 @@ bool CDataFileReader::Close()
m_pDataFile->m_pDataSizes[i] = 0;
}

io_close(m_pDataFile->m_File);
free(m_pDataFile);
m_pDataFile = nullptr;
return true;
}

IOHANDLE CDataFileReader::File() const
{
if(!m_pDataFile)
return 0;
return m_pDataFile->m_File;
m_pMemoryReader.reset();
}

int CDataFileReader::NumData() const
Expand Down Expand Up @@ -298,12 +312,10 @@ int CDataFileReader::GetDataSize(int Index) const
return m_pDataFile->m_pDataSizes[Index];
}

void *CDataFileReader::GetDataImpl(int Index, int Swap)
void *CDataFileReader::GetDataImpl(int Index, bool Swap)
{
if(!m_pDataFile)
{
return nullptr;
}

if(Index < 0 || Index >= m_pDataFile->m_Header.m_NumRawData)
return nullptr;
Expand All @@ -329,8 +341,8 @@ void *CDataFileReader::GetDataImpl(int Index, int Swap)
m_pDataFile->m_pDataSizes[Index] = UncompressedSize;

// read the compressed data
io_seek(m_pDataFile->m_File, m_pDataFile->m_DataStartOffset + m_pDataFile->m_Info.m_pDataOffsets[Index], IOSEEK_START);
io_read(m_pDataFile->m_File, pTemp, DataSize);
m_pMemoryReader->SetPosition(m_pDataFile->m_DataStartOffset + m_pDataFile->m_Info.m_pDataOffsets[Index]);
m_pMemoryReader->Read(pTemp, DataSize);

// decompress the data, TODO: check for errors
s = UncompressedSize;
Expand All @@ -348,8 +360,8 @@ void *CDataFileReader::GetDataImpl(int Index, int Swap)
log_trace("datafile", "loading data index=%d size=%d", Index, DataSize);
m_pDataFile->m_ppDataPtrs[Index] = (char *)malloc(DataSize);
m_pDataFile->m_pDataSizes[Index] = DataSize;
io_seek(m_pDataFile->m_File, m_pDataFile->m_DataStartOffset + m_pDataFile->m_Info.m_pDataOffsets[Index], IOSEEK_START);
io_read(m_pDataFile->m_File, m_pDataFile->m_ppDataPtrs[Index], DataSize);
m_pMemoryReader->SetPosition(m_pDataFile->m_DataStartOffset + m_pDataFile->m_Info.m_pDataOffsets[Index]);
m_pMemoryReader->Read(m_pDataFile->m_ppDataPtrs[Index], DataSize);
}

#if defined(CONF_ARCH_ENDIAN_BIG)
Expand All @@ -363,12 +375,12 @@ void *CDataFileReader::GetDataImpl(int Index, int Swap)

void *CDataFileReader::GetData(int Index)
{
return GetDataImpl(Index, 0);
return GetDataImpl(Index, false);
}

void *CDataFileReader::GetDataSwapped(int Index)
{
return GetDataImpl(Index, 1);
return GetDataImpl(Index, true);
}

void CDataFileReader::ReplaceData(int Index, char *pData, size_t Size)
Expand Down Expand Up @@ -543,11 +555,18 @@ unsigned CDataFileReader::Crc() const
return m_pDataFile->m_Crc;
}

int CDataFileReader::MapSize() const
const unsigned char *CDataFileReader::FileData() const
{
if(!m_pDataFile)
if(!m_pMemoryReader)
return nullptr;
return m_pMemoryReader->FileData();
}

unsigned CDataFileReader::FileSize() const
{
if(!m_pMemoryReader)
return 0;
return m_pDataFile->m_Header.m_Size + 16;
return m_pMemoryReader->FileSize();
}

CDataFileWriter::CDataFileWriter()
Expand Down
17 changes: 10 additions & 7 deletions src/engine/shared/datafile.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@

#include <zlib.h>

#include <memory>

enum
{
ITEMTYPE_EX = 0xffff,
Expand All @@ -18,22 +20,22 @@ enum
// raw datafile access
class CDataFileReader
{
struct CDatafile *m_pDataFile;
void *GetDataImpl(int Index, int Swap);
std::shared_ptr<class CDataFileMemoryReader> m_pMemoryReader = nullptr;
struct CDatafile *m_pDataFile = nullptr;

void *GetDataImpl(int Index, bool Swap);
int GetFileDataSize(int Index) const;

int GetExternalItemType(int InternalType);
int GetInternalItemType(int ExternalType);

public:
CDataFileReader() :
m_pDataFile(nullptr) {}
CDataFileReader() {}
~CDataFileReader() { Close(); }

bool Open(class IStorage *pStorage, const char *pFilename, int StorageType);
bool Close();
void Close();
bool IsOpen() const { return m_pDataFile != nullptr; }
IOHANDLE File() const;

void *GetData(int Index);
void *GetDataSwapped(int Index); // makes sure that the data is 32bit LE ints when saved
Expand All @@ -51,7 +53,8 @@ class CDataFileReader

SHA256_DIGEST Sha256() const;
unsigned Crc() const;
int MapSize() const;
const unsigned char *FileData() const;
unsigned FileSize() const;
};

// write access
Expand Down
3 changes: 1 addition & 2 deletions src/engine/shared/demo.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,12 +45,11 @@ CDemoRecorder::CDemoRecorder(class CSnapshotDelta *pSnapshotDelta, bool NoMapDat
}

// Record
int CDemoRecorder::Start(class IStorage *pStorage, class IConsole *pConsole, const char *pFilename, const char *pNetVersion, const char *pMap, SHA256_DIGEST *pSha256, unsigned Crc, const char *pType, unsigned MapSize, unsigned char *pMapData, IOHANDLE MapFile, DEMOFUNC_FILTER pfnFilter, void *pUser)
int CDemoRecorder::Start(class IStorage *pStorage, class IConsole *pConsole, const char *pFilename, const char *pNetVersion, const char *pMap, SHA256_DIGEST *pSha256, unsigned Crc, const char *pType, unsigned MapSize, const unsigned char *pMapData, IOHANDLE MapFile, DEMOFUNC_FILTER pfnFilter, void *pUser)
{
m_pfnFilter = pfnFilter;
m_pUser = pUser;

m_pMapData = pMapData;
m_pConsole = pConsole;

IOHANDLE DemoFile = pStorage->OpenFile(pFilename, IOFLAG_WRITE, IStorage::TYPE_SAVE);
Expand Down
3 changes: 1 addition & 2 deletions src/engine/shared/demo.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ class CDemoRecorder : public IDemoRecorder
int m_NumTimelineMarkers;
int m_aTimelineMarkers[MAX_TIMELINE_MARKERS];
bool m_NoMapData;
unsigned char *m_pMapData;

DEMOFUNC_FILTER m_pfnFilter;
void *m_pUser;
Expand All @@ -38,7 +37,7 @@ class CDemoRecorder : public IDemoRecorder
CDemoRecorder(class CSnapshotDelta *pSnapshotDelta, bool NoMapData = false);
CDemoRecorder() {}

int Start(class IStorage *pStorage, class IConsole *pConsole, const char *pFilename, const char *pNetversion, const char *pMap, SHA256_DIGEST *pSha256, unsigned MapCrc, const char *pType, unsigned MapSize, unsigned char *pMapData, IOHANDLE MapFile = nullptr, DEMOFUNC_FILTER pfnFilter = nullptr, void *pUser = nullptr);
int Start(class IStorage *pStorage, class IConsole *pConsole, const char *pFilename, const char *pNetversion, const char *pMap, SHA256_DIGEST *pSha256, unsigned MapCrc, const char *pType, unsigned MapSize, const unsigned char *pMapData, IOHANDLE MapFile = nullptr, DEMOFUNC_FILTER pfnFilter = nullptr, void *pUser = nullptr);
int Stop() override;

void AddDemoMarker();
Expand Down
Loading

0 comments on commit c740a97

Please sign in to comment.