diff --git a/autotest/.gitignore b/autotest/.gitignore index d11a5f4d5742..c210b383a2e4 100644 --- a/autotest/.gitignore +++ b/autotest/.gitignore @@ -1,4 +1,5 @@ *.pyc +*.o ogr/tmp ogr/data/delete_nas.gfs ogr/data/replace_nas.gfs @@ -15,3 +16,9 @@ gdrivers/data/byte.jp2.gz.properties gdrivers/data/utm.json gdrivers/tmp osr/tmp +cpp/gdal_unit_test +cpp/test_virtualmem +cpp/testclosedondestroydm +cpp/testcopywords +cpp/testperfcopywords +cpp/testthreadcond diff --git a/gdal/alg/gdalrasterize.cpp b/gdal/alg/gdalrasterize.cpp index e5f679cab668..147061c2f1a6 100644 --- a/gdal/alg/gdalrasterize.cpp +++ b/gdal/alg/gdalrasterize.cpp @@ -844,7 +844,7 @@ CPLErr GDALRasterizeLayers( GDALDatasetH hDS, ; else { - GIntBig nYChunkSize64 = GDALGetCacheMax64() / nScanlineBytes; + GIntBig nYChunkSize64 = poDS->GetRasterBlockManager()->GetCacheMax() / nScanlineBytes; if (nYChunkSize64 > INT_MAX) nYChunkSize = INT_MAX; else diff --git a/gdal/frmts/epsilon/epsilondataset.cpp b/gdal/frmts/epsilon/epsilondataset.cpp index 7eb0267ad756..b4aa4ed087e6 100644 --- a/gdal/frmts/epsilon/epsilondataset.cpp +++ b/gdal/frmts/epsilon/epsilondataset.cpp @@ -29,6 +29,7 @@ #include "epsilon.h" #include "gdal_pam.h" +#include "cpl_multiproc.h" CPL_CVSID("$Id$"); @@ -337,11 +338,12 @@ CPLErr EpsilonRasterBand::IReadBlock( int nBlockXOff, poBlock->DropLock(); break; } - - memcpy(pabySrcBlock, - poGDS->pabyRGBData + (iOtherBand - 1) * nBlockXSize * nBlockYSize, - nBlockXSize * nBlockYSize); - + { + CPLMutexHolderD( poGDS->GetRasterBand(iOtherBand)->GetRWMutex() ); + memcpy(pabySrcBlock, + poGDS->pabyRGBData + (iOtherBand - 1) * nBlockXSize * nBlockYSize, + nBlockXSize * nBlockYSize); + } poBlock->DropLock(); } } diff --git a/gdal/frmts/gtiff/geotiff.cpp b/gdal/frmts/gtiff/geotiff.cpp index 55966aab95ab..cddd4e641e0c 100644 --- a/gdal/frmts/gtiff/geotiff.cpp +++ b/gdal/frmts/gtiff/geotiff.cpp @@ -1679,7 +1679,7 @@ CPLErr GTiffRasterBand::IRasterIO( GDALRWFlag eRWFlag, GIntBig nRequiredMem = (GIntBig)poGDS->nBands * nXBlocks * nYBlocks * nBlockXSize * nBlockYSize * (GDALGetDataTypeSize(eDataType) / 8); - if (nRequiredMem > GDALGetCacheMax64()) + if (nRequiredMem > poDS->GetRasterBlockManager()->GetCacheMax()) { if (!poGDS->bHasWarnedDisableAggressiveBandCaching) { @@ -1961,7 +1961,7 @@ CPLErr GTiffRasterBand::FillCacheForOtherBands( int nBlockXOff, int nBlockYOff ) /* enough to accomodate the size of all the blocks, don't enter */ /* -------------------------------------------------------------------- */ if( poGDS->nBands != 1 && !poGDS->bLoadingOtherBands && - nBlockXSize * nBlockYSize * (GDALGetDataTypeSize(eDataType) / 8) < GDALGetCacheMax64() / poGDS->nBands) + nBlockXSize * nBlockYSize * (GDALGetDataTypeSize(eDataType) / 8) < poDS->GetRasterBlockManager()->GetCacheMax() / poGDS->nBands) { int iOtherBand; diff --git a/gdal/frmts/jp2kak/jp2kakdataset.cpp b/gdal/frmts/jp2kak/jp2kakdataset.cpp index 38c6a9706aff..b12b2351b0b3 100644 --- a/gdal/frmts/jp2kak/jp2kakdataset.cpp +++ b/gdal/frmts/jp2kak/jp2kakdataset.cpp @@ -618,8 +618,11 @@ CPLErr JP2KAKRasterBand::IReadBlock( int nBlockXOff, int nBlockYOff, if( poBlock ) { - memcpy( poBlock->GetDataRef(), pabyWrkBuffer + nBandStart, - nWordSize * nBlockXSize * nBlockYSize ); + { + CPLMutexHolderD( poBand->GetRWMutex() ); + memcpy( poBlock->GetDataRef(), pabyWrkBuffer + nBandStart, + nWordSize * nBlockXSize * nBlockYSize ); + } poBlock->DropLock(); } } diff --git a/gdal/frmts/mbtiles/mbtilesdataset.cpp b/gdal/frmts/mbtiles/mbtilesdataset.cpp index d21d5ddc411d..46b653ed6fbe 100644 --- a/gdal/frmts/mbtiles/mbtilesdataset.cpp +++ b/gdal/frmts/mbtiles/mbtilesdataset.cpp @@ -31,6 +31,7 @@ #include "gdal_pam.h" #include "ogr_api.h" #include "cpl_vsil_curl_priv.h" +#include "cpl_multiproc.h" #include "zlib.h" #include "json.h" @@ -293,11 +294,14 @@ CPLErr MBTilesBand::IReadBlock( int nBlockXOff, int nBlockYOff, void * pImage) poBlock->DropLock(); break; } - if (nTileBands == 3 && poGDS->nBands == 4 && iOtherBand == 4) + { + CPLMutexHolderD( poGDS->GetRasterBand(iOtherBand)->GetRWMutex() ); memset(pabySrcBlock, 255, nBlockXSize * nBlockYSize); + } else if (nTileBands == 1 && (poGDS->nBands == 3 || poGDS->nBands == 4)) { + CPLMutexHolderD( poGDS->GetRasterBand(iOtherBand)->GetRWMutex() ); int i; if (pSrcImage) { @@ -381,10 +385,11 @@ CPLErr MBTilesBand::IReadBlock( int nBlockXOff, int nBlockYOff, void * pImage) poBlock->DropLock(); break; } - - memset(pabySrcBlock, (iOtherBand == 4) ? 0 : 255, - nBlockXSize * nBlockYSize); - + { + CPLMutexHolderD( poGDS->GetRasterBand(iOtherBand)->GetRWMutex() ); + memset(pabySrcBlock, (iOtherBand == 4) ? 0 : 255, + nBlockXSize * nBlockYSize); + } poBlock->DropLock(); } } diff --git a/gdal/frmts/openjpeg/openjpegdataset.cpp b/gdal/frmts/openjpeg/openjpegdataset.cpp index 9aad065ee74d..9670d7ae21f9 100644 --- a/gdal/frmts/openjpeg/openjpegdataset.cpp +++ b/gdal/frmts/openjpeg/openjpegdataset.cpp @@ -439,10 +439,9 @@ static void JP2OpenJPEGReadBlockInThread(void* userdata) poBlock->DropLock(); break; } - + poGDS->ReadBlock(nBand, fp, nBlockXOff, nBlockYOff, pDstBuffer, nBandCount, panBandMap); - poBlock->DropLock(); } @@ -468,7 +467,7 @@ int JP2OpenJPEGDataset::PreloadBlocks(JP2OpenJPEGRasterBand* poBand, int nMaxThreads = GetNumThreads(); if( !bUseSetDecodeArea && nMaxThreads > 1 ) { - if( nReqMem > GDALGetCacheMax64() / (nBandCount == 0 ? 1 : nBandCount) ) + if( nReqMem > poRasterBlockManager->GetCacheMax() / (nBandCount == 0 ? 1 : nBandCount) ) return FALSE; int nBlocksToLoad = 0; @@ -508,7 +507,7 @@ int JP2OpenJPEGDataset::PreloadBlocks(JP2OpenJPEGRasterBand* poBand, } else { - if( nReqMem <= GDALGetCacheMax64() / nBands ) + if( nReqMem <= poRasterBlockManager->GetCacheMax() / nBands ) { oJob.nBandCount = nBands; oJob.panBandMap = NULL; @@ -716,7 +715,6 @@ CPLErr JP2OpenJPEGDataset::ReadBlock( int nBand, VSILFILE* fp, continue; } } - if (bIs420) { CPLAssert((int)psImage->comps[0].w >= nWidthToRead); @@ -729,6 +727,7 @@ CPLErr JP2OpenJPEGDataset::ReadBlock( int nBand, VSILFILE* fp, OPJ_INT32* pSrcY = psImage->comps[0].data; OPJ_INT32* pSrcCb = psImage->comps[1].data; OPJ_INT32* pSrcCr = psImage->comps[2].data; + CPLMutexHolderD( &hRWMutex ); GByte* pDst = (GByte*)pDstBuffer; for(int j=0;jcomps[iBand-1].w == nBlockXSize && (int)psImage->comps[iBand-1].h == nBlockYSize) diff --git a/gdal/frmts/postgisraster/postgisrasterdataset.cpp b/gdal/frmts/postgisraster/postgisrasterdataset.cpp index 39a8740f8aaf..1519cc06db8b 100644 --- a/gdal/frmts/postgisraster/postgisrasterdataset.cpp +++ b/gdal/frmts/postgisraster/postgisrasterdataset.cpp @@ -34,6 +34,7 @@ * SOFTWARE. **********************************************************************/ #include "postgisraster.h" +#include "cpl_multiproc.h" #include #ifdef _WIN32 @@ -973,11 +974,13 @@ void PostGISRasterDataset::CacheTile(const char* pszMetadata, { GDALRasterBlock* poBlock = poRTB->GetLockedBlockRef(0, 0, TRUE); if( poBlock != NULL ) - { - // Point block data ref to fetched data - memcpy(poBlock->GetDataRef(), (void *)pbyDataToRead, - nExpectedBandDataSize); - + { + { + CPLMutexHolderD( poRTB->GetRWMutex() ); + // Point block data ref to fetched data + memcpy(poBlock->GetDataRef(), (void *)pbyDataToRead, + nExpectedBandDataSize); + } poBlock->DropLock(); } } @@ -1065,7 +1068,7 @@ GBool PostGISRasterDataset::LoadSources(int nXOff, int nYOff, int nXSize, int nY { GIntBig nMemoryRequiredForTiles = PQntuples(poResult) * nTileWidth * nTileHeight * GDALGetDataTypeSize(GetRasterBand(nBand)->GetRasterDataType()) / 8; - GIntBig nCacheMax = (GIntBig) GDALGetCacheMax64(); + GIntBig nCacheMax = (GIntBig) poRasterBlockManager->GetCacheMax(); if( nBands * nMemoryRequiredForTiles <= nCacheMax ) { bLoadRasters = TRUE; diff --git a/gdal/frmts/postgisraster/postgisrasterrasterband.cpp b/gdal/frmts/postgisraster/postgisrasterrasterband.cpp index 419f1e623547..1269fd7eed65 100644 --- a/gdal/frmts/postgisraster/postgisrasterrasterband.cpp +++ b/gdal/frmts/postgisraster/postgisrasterrasterband.cpp @@ -565,7 +565,7 @@ CPLErr PostGISRasterRasterBand::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int bAllBandCaching = FALSE; if (nTilesToFetch > 0) { - GIntBig nCacheMax = (GIntBig) GDALGetCacheMax64(); + GIntBig nCacheMax = (GIntBig) GetRasterBlockManager()->GetCacheMax(); if( nMemoryRequiredForTiles > nCacheMax ) { CPLDebug("PostGIS_Raster", diff --git a/gdal/frmts/vrt/vrtwarped.cpp b/gdal/frmts/vrt/vrtwarped.cpp index b93db67775fd..7f718be3a97b 100644 --- a/gdal/frmts/vrt/vrtwarped.cpp +++ b/gdal/frmts/vrt/vrtwarped.cpp @@ -31,6 +31,7 @@ #include "vrtdataset.h" #include "cpl_minixml.h" #include "cpl_string.h" +#include "cpl_multiproc.h" #include "gdalwarper.h" #include "gdal_alg_priv.h" #include @@ -1200,6 +1201,7 @@ CPLErr VRTWarpedDataset::ProcessBlock( int iBlockX, int iBlockY ) { if ( poBlock->GetDataRef() != NULL ) { + CPLMutexHolderD( poBand->GetRWMutex() ); GDALCopyWords( pabyDstBuffer + iBand*nBlockXSize*nBlockYSize*nWordSize, psWO->eWorkingDataType, nWordSize, poBlock->GetDataRef(), @@ -1292,6 +1294,7 @@ CPLErr VRTWarpedRasterBand::IReadBlock( int nBlockXOff, int nBlockYOff, int nDataBytes; nDataBytes = (GDALGetDataTypeSize(poBlock->GetDataType()) / 8) * poBlock->GetXSize() * poBlock->GetYSize(); + CPLMutexHolderD( GetRWMutex() ); memcpy( pImage, poBlock->GetDataRef(), nDataBytes ); } diff --git a/gdal/frmts/wms/gdalwmsrasterband.cpp b/gdal/frmts/wms/gdalwmsrasterband.cpp index 6e01767096f3..9a96d53a4710 100644 --- a/gdal/frmts/wms/gdalwmsrasterband.cpp +++ b/gdal/frmts/wms/gdalwmsrasterband.cpp @@ -29,6 +29,7 @@ ****************************************************************************/ #include "wmsdriver.h" +#include "cpl_multiproc.h" GDALWMSRasterBand::GDALWMSRasterBand(GDALWMSDataset *parent_dataset, int band, double scale) { // printf("[%p] GDALWMSRasterBand::GDALWMSRasterBand(%p, %d, %f)\n", this, parent_dataset, band, scale); @@ -623,11 +624,14 @@ CPLErr GDALWMSRasterBand::ReadBlockFromFile(int x, int y, const char *file_name, for (int ib = 1; ib <= m_parent_dataset->nBands; ++ib) { if (ret == CE_None) { void *p = NULL; + void ** hThisMutex = NULL; GDALRasterBlock *b = NULL; if ((buffer != NULL) && (ib == to_buffer_band)) { p = buffer; + hThisMutex = NULL; } else { GDALWMSRasterBand *band = static_cast(m_parent_dataset->GetRasterBand(ib)); + hThisMutex = band->GetRWMutex(); if (m_overview >= 0) band = static_cast(band->GetOverview(m_overview)); if (!band->IsBlockInCache(x, y)) { b = band->GetLockedBlockRef(x, y, true); @@ -664,6 +668,7 @@ CPLErr GDALWMSRasterBand::ReadBlockFromFile(int x, int y, const char *file_name, if (accepted_as_no_alpha) { // the file had 3 bands and we are reading band 4 (Alpha) so fill with 255 (no alpha) + CPLMutexHolderD( hThisMutex ); GByte *byte_buffer = reinterpret_cast(p); for (int y = 0; y < sy; ++y) { for (int x = 0; x < sx; ++x) { @@ -686,6 +691,7 @@ CPLErr GDALWMSRasterBand::ReadBlockFromFile(int x, int y, const char *file_name, } if (ret == CE_None) { GByte *band_color_table = color_table + 256 * (ib - 1); + CPLMutexHolderD( hThisMutex ); GByte *byte_buffer = reinterpret_cast(p); for (int y = 0; y < sy; ++y) { for (int x = 0; x < sx; ++x) { @@ -724,11 +730,14 @@ CPLErr GDALWMSRasterBand::ZeroBlock(int x, int y, int to_buffer_band, void *buff for (int ib = 1; ib <= m_parent_dataset->nBands; ++ib) { if (ret == CE_None) { void *p = NULL; + void **hThisMutex = NULL; GDALRasterBlock *b = NULL; if ((buffer != NULL) && (ib == to_buffer_band)) { p = buffer; + hThisMutex = NULL; } else { GDALWMSRasterBand *band = static_cast(m_parent_dataset->GetRasterBand(ib)); + hThisMutex = band->GetRWMutex(); if (m_overview >= 0) band = static_cast(band->GetOverview(m_overview)); if (!band->IsBlockInCache(x, y)) { b = band->GetLockedBlockRef(x, y, true); @@ -742,6 +751,7 @@ CPLErr GDALWMSRasterBand::ZeroBlock(int x, int y, int to_buffer_band, void *buff } } if (p != NULL) { + CPLMutexHolderD( hThisMutex ); unsigned char *b = reinterpret_cast(p); int block_size = nBlockXSize * nBlockYSize * (GDALGetDataTypeSize(eDataType) / 8); for (int i = 0; i < block_size; ++i) b[i] = 0; diff --git a/gdal/gcore/GNUmakefile b/gdal/gcore/GNUmakefile index e4de40f03c47..d5a4d11a4714 100644 --- a/gdal/gcore/GNUmakefile +++ b/gdal/gcore/GNUmakefile @@ -10,7 +10,8 @@ OBJ = gdalopeninfo.o gdaldrivermanager.o gdaldriver.o gdaldataset.o \ gdalallvalidmaskband.o gdalnodatamaskband.o gdal_rpcimdio.o \ gdalproxydataset.o gdalproxypool.o gdaldefaultasync.o \ gdalnodatavaluesmaskband.o gdaldllmain.o gdalexif.o gdalclientserver.o \ - gdalgeorefpamdataset.o gdaljp2abstractdataset.o gdalvirtualmem.o + gdalgeorefpamdataset.o gdaljp2abstractdataset.o gdalvirtualmem.o \ + gdalrasterblockmanager.o # Enable the following if you want to use MITAB's code to convert # .tab coordinate systems into well known text. But beware that linking diff --git a/gdal/gcore/gdal_priv.h b/gdal/gcore/gdal_priv.h index 7470bcca0202..c23226fb3036 100644 --- a/gdal/gcore/gdal_priv.h +++ b/gdal/gcore/gdal_priv.h @@ -44,6 +44,8 @@ class GDALRasterAttributeTable; class GDALProxyDataset; class GDALProxyRasterBand; class GDALAsyncReader; +class GDALRasterBlock; +class GDALRasterBlockManager; /* -------------------------------------------------------------------- */ /* Pull in the public declarations. This gets the C apis, and */ @@ -275,14 +277,17 @@ class CPL_DLL GDALDataset : public GDALMajorObject protected: GDALDriver *poDriver; GDALAccess eAccess; + GDALRasterBlockManager *poRasterBlockManager; // Stored raster information. int nRasterXSize; int nRasterYSize; int nBands; GDALRasterBand **papoBands; + void *hRWMutex; int bForceCachedIO; + int bDatasetCache; int nRefCount; int bShared; @@ -315,7 +320,7 @@ class CPL_DLL GDALDataset : public GDALMajorObject virtual int CloseDependentDatasets(); int ValidateLayerCreationOptions( const char* const* papszLCO ); - + friend class GDALRasterBand; public: @@ -326,6 +331,9 @@ class CPL_DLL GDALDataset : public GDALMajorObject int GetRasterCount( void ); GDALRasterBand *GetRasterBand( int ); + GDALRasterBlockManager *GetRasterBlockManager() { return poRasterBlockManager; } + + virtual void SetCacheMax( GIntBig ); virtual void FlushCache(void); virtual const char *GetProjectionRef(void); @@ -443,6 +451,45 @@ class CPL_DLL GDALDataset : public GDALMajorObject OGRStyleTable *m_poStyleTable; }; +/* ******************************************************************** */ +/* GDALRasterBlockManager */ +/* ******************************************************************** */ + +//! The cache management for the rasterblock system + +class CPL_DLL GDALRasterBlockManager +{ + friend class GDALRasterBlock; + + int bCacheMaxInitialized; + GIntBig nCacheMax; + volatile GIntBig nCacheUsed; + + volatile GDALRasterBlock *poOldest; /* tail */ + volatile GDALRasterBlock *poNewest; /* head */ + + void *hRBMMutex; + + public: + GDALRasterBlockManager(); + virtual ~GDALRasterBlockManager(); + + void SetCacheMax( GIntBig nBytes ); + GIntBig GetCacheMax(void); + GIntBig GetCacheUsed(void); + int FlushCacheBlock(void); + void FlushTillBelow(); + void Verify(); + int SafeLockBlock( GDALRasterBlock ** ); + + void DestroyRBMMutex(); +}; + +CPL_C_START +GDALRasterBlockManager CPL_DLL * GetGDALRasterBlockManager( void ); +void CPL_DLL CPL_STDCALL GDALDestroyRasterBlockManager( void ); +CPL_C_END + /* ******************************************************************** */ /* GDALRasterBlock */ /* ******************************************************************** */ @@ -452,7 +499,10 @@ class CPL_DLL GDALDataset : public GDALMajorObject class CPL_DLL GDALRasterBlock { GDALDataType eType; - + + int bAttachedToCache; + int bAttachedToBand; + int bDelete; int bDirty; int nLockCount; @@ -465,22 +515,29 @@ class CPL_DLL GDALRasterBlock void *pData; GDALRasterBand *poBand; + + GDALRasterBlockManager *poManager; GDALRasterBlock *poNext; GDALRasterBlock *poPrevious; + + friend class GDALRasterBlockManager; public: - GDALRasterBlock( GDALRasterBand *, int, int ); + GDALRasterBlock( GDALRasterBand *, int, int, GDALRasterBlockManager * ); virtual ~GDALRasterBlock(); CPLErr Internalize( void ); void Touch( void ); void MarkDirty( void ); void MarkClean( void ); - void AddLock( void ) { nLockCount++; } - void DropLock( void ) { nLockCount--; } + void AddLock( void ); + void DropLock( void ); + void MarkForDeletion( void ) { bDelete = TRUE; } + void AttachToBand( void ) { bAttachedToBand = TRUE; } + void UnattachFromBand( void ) { bAttachedToBand = FALSE; } void Detach(); - + CPLErr Write(); GDALDataType GetDataType() { return eType; } @@ -497,13 +554,6 @@ class CPL_DLL GDALRasterBlock /// @return source raster band of the raster block. GDALRasterBand *GetBand() { return poBand; } - static int FlushCacheBlock(); - static void Verify(); - - static int SafeLockBlock( GDALRasterBlock ** ); - - /* Should only be called by GDALDestroyDriverManager() */ - static void DestroyRBMutex(); }; /* ******************************************************************** */ @@ -548,6 +598,7 @@ class CPL_DLL GDALRasterBand : public GDALMajorObject void SetFlushBlockErr( CPLErr eErr ); friend class GDALRasterBlock; + friend class GDALRasterBlockManager; protected: GDALDataset *poDS; @@ -560,6 +611,8 @@ class CPL_DLL GDALRasterBand : public GDALMajorObject GDALAccess eAccess; /* stuff related to blocking, and raster cache */ + void *hBandMutex; + void *hRWMutex; int nBlockXSize; int nBlockYSize; int nBlocksPerRow; @@ -592,9 +645,9 @@ class CPL_DLL GDALRasterBand : public GDALMajorObject int InitBlockInfo(); - CPLErr AdoptBlock( int, int, GDALRasterBlock * ); + GDALRasterBlock *AdoptBlock( int, int, GDALRasterBlock * ); GDALRasterBlock *TryGetLockedBlockRef( int nXBlockOff, int nYBlockYOff ); - + public: GDALRasterBand(); @@ -605,6 +658,7 @@ class CPL_DLL GDALRasterBand : public GDALMajorObject int GetBand(); GDALDataset*GetDataset(); + void **GetRWMutex(); GDALDataType GetRasterDataType( void ); void GetBlockSize( int *, int * ); GDALAccess GetAccess(); @@ -618,7 +672,11 @@ class CPL_DLL GDALRasterBand : public GDALMajorObject GDALRasterBlock *GetLockedBlockRef( int nXBlockOff, int nYBlockOff, int bJustInitialize = FALSE ); - CPLErr FlushBlock( int = -1, int = -1, int bWriteDirtyBlock = TRUE ); + + GDALRasterBlockManager *GetRasterBlockManager(); + + GDALRasterBlock *UnadoptBlock( int = -1, int = -1 ); + CPLErr FlushBlock( int = -1, int = -1 ); unsigned char* GetIndexColorTranslationTo(/* const */ GDALRasterBand* poReferenceBand, unsigned char* pTranslationTable = NULL, diff --git a/gdal/gcore/gdaldataset.cpp b/gdal/gcore/gdaldataset.cpp index ea1990a749a5..a410460ae48c 100644 --- a/gdal/gcore/gdaldataset.cpp +++ b/gdal/gcore/gdaldataset.cpp @@ -193,9 +193,21 @@ GDALDataset::GDALDataset() /* -------------------------------------------------------------------- */ bForceCachedIO = CSLTestBoolean( CPLGetConfigOption( "GDAL_FORCE_CACHING", "NO") ); + bDatasetCache = CSLTestBoolean( + CPLGetConfigOption( "GDAL_DATASET_CACHING", "NO") ); + + if ( bDatasetCache ) + { + poRasterBlockManager = new GDALRasterBlockManager(); + } + else + { + poRasterBlockManager = GetGDALRasterBlockManager(); + } m_poStyleTable = NULL; m_hMutex = NULL; + hRWMutex = NULL; } @@ -301,8 +313,43 @@ GDALDataset::~GDALDataset() if( m_hMutex != NULL ) CPLDestroyMutex( m_hMutex ); + if( hRWMutex != NULL ) + CPLDestroyMutex( hRWMutex ); + +/* -------------------------------------------------------------------- */ +/* Destroy the raster block manager if it is not global */ +/* -------------------------------------------------------------------- */ + + if ( bDatasetCache ) + { + delete poRasterBlockManager; + } + poRasterBlockManager = NULL; + } +/************************************************************************/ +/* SetCacheMax() */ +/************************************************************************/ + +/** + * \brief Set the cache size for a dataset + * + * Sets the cache size for a raster dataset if GDAL_DATASET_CACHING is + * enabled. If this is called with GDAL_DATASET_CACHING set to NO, then + * this method will do nothing. + * + */ + +void GDALDataset::SetCacheMax(GIntBig nCacheMax) +{ + if ( bDatasetCache ) + { + poRasterBlockManager->SetCacheMax(nCacheMax); + } +} + + /************************************************************************/ /* FlushCache() */ /************************************************************************/ diff --git a/gdal/gcore/gdaldrivermanager.cpp b/gdal/gcore/gdaldrivermanager.cpp index ca2eca7e5bdf..9d249585382d 100644 --- a/gdal/gcore/gdaldrivermanager.cpp +++ b/gdal/gcore/gdaldrivermanager.cpp @@ -264,9 +264,9 @@ GDALDriverManager::~GDALDriverManager() } /* -------------------------------------------------------------------- */ -/* Cleanup raster block mutex */ +/* Cleanup raster block manager and its mutex */ /* -------------------------------------------------------------------- */ - GDALRasterBlock::DestroyRBMutex(); + GDALDestroyRasterBlockManager(); /* -------------------------------------------------------------------- */ /* Cleanup gdaltransformer.cpp mutex */ diff --git a/gdal/gcore/gdalrasterband.cpp b/gdal/gcore/gdalrasterband.cpp index 2344e8e9f3d4..0cab72b38f6c 100644 --- a/gdal/gcore/gdalrasterband.cpp +++ b/gdal/gcore/gdalrasterband.cpp @@ -32,6 +32,7 @@ #include "gdal_priv.h" #include "gdal_rat.h" #include "cpl_string.h" +#include "cpl_multiproc.h" #define SUBBLOCK_SIZE 64 #define TO_SUBBLOCK(x) ((x) >> 6) @@ -49,6 +50,8 @@ GDALRasterBand::GDALRasterBand() { poDS = NULL; + hBandMutex = NULL; + hRWMutex = NULL; nBand = 0; nRasterXSize = nRasterYSize = 0; @@ -102,6 +105,12 @@ GDALRasterBand::~GDALRasterBand() nMaskFlags = 0; bOwnMask = false; } + if( hBandMutex != NULL ) + CPLDestroyMutex(hBandMutex); + hBandMutex = NULL; + if( hRWMutex != NULL ) + CPLDestroyMutex(hRWMutex); + hRWMutex = NULL; } /************************************************************************/ @@ -415,6 +424,7 @@ CPLErr GDALRasterBand::ReadBlock( int nXBlockOff, int nYBlockOff, /* -------------------------------------------------------------------- */ /* Invoke underlying implementation method. */ /* -------------------------------------------------------------------- */ + CPLMutexHolderD( GetRWMutex() ); return( IReadBlock( nXBlockOff, nYBlockOff, pImage ) ); } @@ -536,6 +546,7 @@ CPLErr GDALRasterBand::WriteBlock( int nXBlockOff, int nYBlockOff, /* -------------------------------------------------------------------- */ /* Invoke underlying implementation method. */ /* -------------------------------------------------------------------- */ + CPLMutexHolderD( GetRWMutex() ); return( IWriteBlock( nXBlockOff, nYBlockOff, pImage ) ); } @@ -675,6 +686,10 @@ int GDALRasterBand::InitBlockInfo() if( papoBlocks != NULL ) return TRUE; + CPLMutexHolderD( &hBandMutex ); + + if( papoBlocks != NULL ) + return TRUE; /* Do some validation of raster and block dimensions in case the driver */ /* would have neglected to do it itself */ if( nBlockXSize <= 0 || nBlockYSize <= 0 ) @@ -762,21 +777,23 @@ int GDALRasterBand::InitBlockInfo() /* AdoptBlock() */ /* */ /* Add a block to the raster band's block matrix. If this */ -/* exceeds our maximum blocks for this layer, flush the oldest */ -/* block out. */ +/* block already exists then return that block, otherwise */ +/* return the block passed in */ /* */ /* This method is protected. */ /************************************************************************/ -CPLErr GDALRasterBand::AdoptBlock( int nXBlockOff, int nYBlockOff, +GDALRasterBlock *GDALRasterBand::AdoptBlock( int nXBlockOff, int nYBlockOff, GDALRasterBlock * poBlock ) { int nBlockIndex; if( !InitBlockInfo() ) - return CE_Failure; + return NULL; + // Now aquire the band mutex + CPLMutexHolderD( &hBandMutex ); /* -------------------------------------------------------------------- */ /* Simple case without subblocking. */ /* -------------------------------------------------------------------- */ @@ -785,15 +802,21 @@ CPLErr GDALRasterBand::AdoptBlock( int nXBlockOff, int nYBlockOff, nBlockIndex = nXBlockOff + nYBlockOff * nBlocksPerRow; if( papoBlocks[nBlockIndex] == poBlock ) - return( CE_None ); + return poBlock; + // Something else has already assigned it to the block. Lets get + // that block instead. if( papoBlocks[nBlockIndex] != NULL ) - FlushBlock( nXBlockOff, nYBlockOff ); + { + GDALRasterBlock *poExistingBlock = papoBlocks[nBlockIndex]; + poExistingBlock->AddLock(); + return poExistingBlock; + } papoBlocks[nBlockIndex] = poBlock; - poBlock->Touch(); + poBlock->AttachToBand(); - return( CE_None ); + return poBlock; } /* -------------------------------------------------------------------- */ @@ -813,7 +836,7 @@ CPLErr GDALRasterBand::AdoptBlock( int nXBlockOff, int nYBlockOff, { ReportError( CE_Failure, CPLE_OutOfMemory, "Out of memory in AdoptBlock()." ); - return CE_Failure; + return NULL; } } @@ -827,15 +850,20 @@ CPLErr GDALRasterBand::AdoptBlock( int nXBlockOff, int nYBlockOff, + WITHIN_SUBBLOCK(nYBlockOff) * SUBBLOCK_SIZE; if( papoSubBlockGrid[nBlockInSubBlock] == poBlock ) - return CE_None; - + return NULL; + + // Something assigned something to this block already, lets use it. if( papoSubBlockGrid[nBlockInSubBlock] != NULL ) - FlushBlock( nXBlockOff, nYBlockOff ); + { + GDALRasterBlock *poExistingBlock = papoSubBlockGrid[nBlockInSubBlock]; + poExistingBlock->AddLock(); + return poExistingBlock; + } papoSubBlockGrid[nBlockInSubBlock] = poBlock; - poBlock->Touch(); + poBlock->AttachToBand(); - return CE_None; + return poBlock; } /************************************************************************/ @@ -865,9 +893,15 @@ CPLErr GDALRasterBand::FlushCache() eFlushBlockErr = CE_None; } + if (papoBlocks == NULL) return eGlobalErr; + GDALRasterBlock *poBlock = NULL; + // We must aquire both the RW Mutex first, followed by the band mutex + // order is very important here in order to avoid any deadlock situations. + CPLMutexHolder oHolderRW(GetRWMutex(),1000.0,__FILE__,__LINE__); + CPLMutexHolderD( &hBandMutex ); /* -------------------------------------------------------------------- */ /* Flush all blocks in memory ... this case is without subblocking.*/ /* -------------------------------------------------------------------- */ @@ -877,14 +911,24 @@ CPLErr GDALRasterBand::FlushCache() { for( int iX = 0; iX < nBlocksPerRow; iX++ ) { - if( papoBlocks[iX + iY*nBlocksPerRow] != NULL ) + poBlock = papoBlocks[iX + iY*nBlocksPerRow]; + if ( poBlock != NULL ) { - CPLErr eErr; - - eErr = FlushBlock( iX, iY, eGlobalErr == CE_None ); - - if( eErr != CE_None ) - eGlobalErr = eErr; + poBlock->AddLock(); + UnadoptBlock( iX, iY ); + if (eGlobalErr == CE_None) + { + CPLErr eErr; + eErr = poBlock->Write(); + + if( eErr != CE_None ) + eGlobalErr = eErr; + } + else + { + poBlock->MarkClean(); + } + poBlock->DropLock(); } } } @@ -912,15 +956,25 @@ CPLErr GDALRasterBand::FlushCache() { for( int iX = 0; iX < SUBBLOCK_SIZE; iX++ ) { - if( papoSubBlockGrid[iX + iY * SUBBLOCK_SIZE] != NULL ) + poBlock = papoSubBlockGrid[iX + iY * SUBBLOCK_SIZE]; + if ( poBlock != NULL) { - CPLErr eErr; - - eErr = FlushBlock( iX + iSBX * SUBBLOCK_SIZE, - iY + iSBY * SUBBLOCK_SIZE, - eGlobalErr == CE_None ); - if( eErr != CE_None ) - eGlobalErr = eErr; + poBlock->AddLock(); + UnadoptBlock( iX + iSBX * SUBBLOCK_SIZE, + iY + iSBY * SUBBLOCK_SIZE); + if (eGlobalErr == CE_None) + { + CPLErr eErr; + eErr = poBlock->Write(); + + if( eErr != CE_None ) + eGlobalErr = eErr; + } + else + { + poBlock->MarkClean(); + } + poBlock->DropLock(); } } } @@ -954,7 +1008,7 @@ CPLErr CPL_STDCALL GDALFlushRasterCache( GDALRasterBandH hBand ) } /************************************************************************/ -/* FlushBlock() */ +/* FlushBlock() */ /* */ /* Flush a block out of the block cache. If it has been */ /* modified write it to disk. If no specific tile is */ @@ -963,14 +1017,38 @@ CPLErr CPL_STDCALL GDALFlushRasterCache( GDALRasterBandH hBand ) /* Protected method. */ /************************************************************************/ -CPLErr GDALRasterBand::FlushBlock( int nXBlockOff, int nYBlockOff, int bWriteDirtyBlock ) +CPLErr GDALRasterBand::FlushBlock( int nXBlockOff, int nYBlockOff ) + +{ + GDALRasterBlock *poBlock = NULL; + { + // Now aquire the band mutex + CPLMutexHolderD( &hBandMutex ); + poBlock = UnadoptBlock( nXBlockOff, nYBlockOff ); + poBlock->AddLock(); + } + CPLErr eErr = poBlock->Write(); + poBlock->DropLock(); + + return eErr; +} + +/************************************************************************/ +/* UnadoptBlock() */ +/* */ +/* Remove a block out of the block array. */ +/* */ +/* Protected method. */ +/************************************************************************/ + +GDALRasterBlock *GDALRasterBand::UnadoptBlock( int nXBlockOff, int nYBlockOff ) { int nBlockIndex; GDALRasterBlock *poBlock = NULL; if( !papoBlocks ) - return CE_None; + return NULL; /* -------------------------------------------------------------------- */ /* Validate the request */ @@ -979,82 +1057,97 @@ CPLErr GDALRasterBand::FlushBlock( int nXBlockOff, int nYBlockOff, int bWriteDir { ReportError( CE_Failure, CPLE_IllegalArg, "Illegal nBlockXOff value (%d) in " - "GDALRasterBand::FlushBlock()\n", + "GDALRasterBand::UnadoptBlock()\n", nXBlockOff ); - return( CE_Failure ); + return( NULL ); } if( nYBlockOff < 0 || nYBlockOff >= nBlocksPerColumn ) { ReportError( CE_Failure, CPLE_IllegalArg, "Illegal nBlockYOff value (%d) in " - "GDALRasterBand::FlushBlock()\n", + "GDALRasterBand::UnadoptBlock()\n", nYBlockOff ); - return( CE_Failure ); + return( NULL ); } + { + // Now aquire the band mutex + CPLMutexHolderD( &hBandMutex ); + /* -------------------------------------------------------------------- */ /* Simple case for single level caches. */ /* -------------------------------------------------------------------- */ - if( !bSubBlockingActive ) - { - nBlockIndex = nXBlockOff + nYBlockOff * nBlocksPerRow; + if( !bSubBlockingActive ) + { + nBlockIndex = nXBlockOff + nYBlockOff * nBlocksPerRow; - GDALRasterBlock::SafeLockBlock( papoBlocks + nBlockIndex ); - poBlock = papoBlocks[nBlockIndex]; - papoBlocks[nBlockIndex] = NULL; - } + poBlock = papoBlocks[nBlockIndex]; + papoBlocks[nBlockIndex] = NULL; + } /* -------------------------------------------------------------------- */ /* Identify our subblock. */ /* -------------------------------------------------------------------- */ - else - { - int nSubBlock = TO_SUBBLOCK(nXBlockOff) - + TO_SUBBLOCK(nYBlockOff) * nSubBlocksPerRow; - - if( papoBlocks[nSubBlock] == NULL ) - return CE_None; + else + { + int nSubBlock = TO_SUBBLOCK(nXBlockOff) + + TO_SUBBLOCK(nYBlockOff) * nSubBlocksPerRow; + + if( papoBlocks[nSubBlock] == NULL ) + return NULL; /* -------------------------------------------------------------------- */ /* Check within subblock. */ /* -------------------------------------------------------------------- */ - GDALRasterBlock **papoSubBlockGrid = - (GDALRasterBlock **) papoBlocks[nSubBlock]; - - int nBlockInSubBlock = WITHIN_SUBBLOCK(nXBlockOff) - + WITHIN_SUBBLOCK(nYBlockOff) * SUBBLOCK_SIZE; - - GDALRasterBlock::SafeLockBlock( papoSubBlockGrid + nBlockInSubBlock ); + GDALRasterBlock **papoSubBlockGrid = + (GDALRasterBlock **) papoBlocks[nSubBlock]; + + int nBlockInSubBlock = WITHIN_SUBBLOCK(nXBlockOff) + + WITHIN_SUBBLOCK(nYBlockOff) * SUBBLOCK_SIZE; + + poBlock = papoSubBlockGrid[nBlockInSubBlock]; + papoSubBlockGrid[nBlockInSubBlock] = NULL; + } + + if( poBlock == NULL ) + return NULL; + poBlock->UnattachFromBand(); - poBlock = papoSubBlockGrid[nBlockInSubBlock]; - papoSubBlockGrid[nBlockInSubBlock] = NULL; + poBlock->MarkForDeletion(); } + + return poBlock; +} -/* -------------------------------------------------------------------- */ -/* Is the target block dirty? If so we need to write it. */ -/* -------------------------------------------------------------------- */ - CPLErr eErr = CE_None; - if( poBlock == NULL ) - return CE_None; +/************************************************************************/ +/* GetRasterBlockManager() */ +/************************************************************************/ - poBlock->Detach(); +/** + * \brief Get the raster block manager for the band. + * + * @return Return the raster block manager used for the band. + */ + +GDALRasterBlockManager *GDALRasterBand::GetRasterBlockManager() +{ + if ( poDS == NULL ) + { + return GetGDALRasterBlockManager(); + } + else + { + return poDS->poRasterBlockManager; + } +} - if( bWriteDirtyBlock && poBlock->GetDirty() ) - eErr = poBlock->Write(); -/* -------------------------------------------------------------------- */ -/* Deallocate the block; */ -/* -------------------------------------------------------------------- */ - poBlock->DropLock(); - delete poBlock; - return eErr; -} /************************************************************************/ /* TryGetLockedBlockRef() */ @@ -1086,6 +1179,8 @@ GDALRasterBlock *GDALRasterBand::TryGetLockedBlockRef( int nXBlockOff, { int nBlockIndex = 0; + GDALRasterBlockManager *poRBM = GetRasterBlockManager(); + GDALRasterBlock *poBlock; if( !InitBlockInfo() ) return( NULL ); @@ -1112,39 +1207,50 @@ GDALRasterBlock *GDALRasterBand::TryGetLockedBlockRef( int nXBlockOff, return( NULL ); } + { + // Now aquire the band mutex + CPLMutexHolderD( &hBandMutex ); + /* -------------------------------------------------------------------- */ /* Simple case for single level caches. */ /* -------------------------------------------------------------------- */ - if( !bSubBlockingActive ) - { - nBlockIndex = nXBlockOff + nYBlockOff * nBlocksPerRow; - - GDALRasterBlock::SafeLockBlock( papoBlocks + nBlockIndex ); + if( !bSubBlockingActive ) + { + nBlockIndex = nXBlockOff + nYBlockOff * nBlocksPerRow; + + poRBM->SafeLockBlock( papoBlocks + nBlockIndex ); - return papoBlocks[nBlockIndex]; - } + poBlock = papoBlocks[nBlockIndex]; + } + else + { /* -------------------------------------------------------------------- */ /* Identify our subblock. */ /* -------------------------------------------------------------------- */ - int nSubBlock = TO_SUBBLOCK(nXBlockOff) - + TO_SUBBLOCK(nYBlockOff) * nSubBlocksPerRow; + int nSubBlock = TO_SUBBLOCK(nXBlockOff) + + TO_SUBBLOCK(nYBlockOff) * nSubBlocksPerRow; - if( papoBlocks[nSubBlock] == NULL ) - return NULL; + if( papoBlocks[nSubBlock] == NULL ) + return NULL; /* -------------------------------------------------------------------- */ /* Check within subblock. */ /* -------------------------------------------------------------------- */ - GDALRasterBlock **papoSubBlockGrid = - (GDALRasterBlock **) papoBlocks[nSubBlock]; + GDALRasterBlock **papoSubBlockGrid = + (GDALRasterBlock **) papoBlocks[nSubBlock]; - int nBlockInSubBlock = WITHIN_SUBBLOCK(nXBlockOff) - + WITHIN_SUBBLOCK(nYBlockOff) * SUBBLOCK_SIZE; + int nBlockInSubBlock = WITHIN_SUBBLOCK(nXBlockOff) + + WITHIN_SUBBLOCK(nYBlockOff) * SUBBLOCK_SIZE; - GDALRasterBlock::SafeLockBlock( papoSubBlockGrid + nBlockInSubBlock ); + poRBM->SafeLockBlock( papoSubBlockGrid + nBlockInSubBlock ); - return papoSubBlockGrid[nBlockInSubBlock]; + poBlock = papoSubBlockGrid[nBlockInSubBlock]; + } + } + if ( NULL != poBlock ) + poBlock->Touch(); + return poBlock; } /************************************************************************/ @@ -1187,6 +1293,7 @@ GDALRasterBlock * GDALRasterBand::GetLockedBlockRef( int nXBlockOff, { GDALRasterBlock *poBlock = NULL; + GDALRasterBlockManager *poRBM = GetRasterBlockManager(); /* -------------------------------------------------------------------- */ /* Try and fetch from cache. */ @@ -1203,9 +1310,9 @@ GDALRasterBlock * GDALRasterBand::GetLockedBlockRef( int nXBlockOff, if( !InitBlockInfo() ) return( NULL ); - /* -------------------------------------------------------------------- */ - /* Validate the request */ - /* -------------------------------------------------------------------- */ +/* -------------------------------------------------------------------- */ +/* Validate the request */ +/* -------------------------------------------------------------------- */ if( nXBlockOff < 0 || nXBlockOff >= nBlocksPerRow ) { ReportError( CE_Failure, CPLE_IllegalArg, @@ -1225,39 +1332,60 @@ GDALRasterBlock * GDALRasterBand::GetLockedBlockRef( int nXBlockOff, return( NULL ); } - - poBlock = new GDALRasterBlock( this, nXBlockOff, nYBlockOff ); + + + poBlock = new GDALRasterBlock( this, nXBlockOff, nYBlockOff, poRBM ); poBlock->AddLock(); /* allocate data space */ if( poBlock->Internalize() != CE_None ) { + poBlock->MarkForDeletion(); poBlock->DropLock(); - delete poBlock; + CPLError( CE_Failure, CPLE_AppDefined, + "Internalize Failure!"); return( NULL ); } - - if ( AdoptBlock( nXBlockOff, nYBlockOff, poBlock ) != CE_None ) + GDALRasterBlock *poAdopt = AdoptBlock( nXBlockOff, nYBlockOff, poBlock ); + if ( poAdopt == NULL ) { + poBlock->MarkForDeletion(); poBlock->DropLock(); - delete poBlock; + CPLError( CE_Failure, CPLE_AppDefined, + "AdoptBlock Failure!"); return( NULL ); } - - if( !bJustInitialize - && IReadBlock(nXBlockOff,nYBlockOff,poBlock->GetDataRef()) != CE_None) + + if ( poAdopt != poBlock ) { + // There was already an existing block, delete one we just created. + poBlock->MarkForDeletion(); poBlock->DropLock(); - FlushBlock( nXBlockOff, nYBlockOff ); - ReportError( CE_Failure, CPLE_AppDefined, - "IReadBlock failed at X offset %d, Y offset %d", - nXBlockOff, nYBlockOff ); - return( NULL ); + poBlock = poAdopt; + poBlock->Touch(); } - - if( !bJustInitialize ) - { + else + { + poBlock->Touch(); + poRBM->FlushTillBelow(); + } + + if ( !bJustInitialize ) + { + // Set the RW Mutex + CPLMutexHolderD( GetRWMutex() ); + + if( IReadBlock(nXBlockOff,nYBlockOff,poBlock->GetDataRef()) != CE_None) + { + UnadoptBlock( nXBlockOff, nYBlockOff ); + poBlock->DropLock(); + ReportError( CE_Failure, CPLE_AppDefined, + "IReadBlock failed at X offset %d, Y offset %d", + nXBlockOff, nYBlockOff ); + return( NULL ); + } + nBlockReads++; if( nBlockReads == nBlocksPerRow * nBlocksPerColumn + 1 && nBand == 1 && poDS != NULL ) @@ -1267,7 +1395,6 @@ GDALRasterBlock * GDALRasterBand::GetLockedBlockRef( int nXBlockOff, } } } - return poBlock; } @@ -1334,31 +1461,36 @@ CPLErr GDALRasterBand::Fill(double dfRealValue, double dfImaginaryValue) { // Copy first element to the rest of the block for (unsigned char* blockPtr = srcBlock + elementSize; - blockPtr < srcBlock + blockByteSize; blockPtr += elementSize) { - memcpy(blockPtr, srcBlock, elementSize); + blockPtr < srcBlock + blockByteSize; blockPtr += elementSize) { + memcpy(blockPtr, srcBlock, elementSize); } - // Write block to block cache - for (int j = 0; j < nBlocksPerColumn; ++j) { - for (int i = 0; i < nBlocksPerRow; ++i) { - GDALRasterBlock* destBlock = GetLockedBlockRef(i, j, TRUE); - if (destBlock == NULL) { - ReportError(CE_Failure, CPLE_OutOfMemory, - "GDALRasterBand::Fill(): Error " - "while retrieving cache block.\n"); - VSIFree(srcBlock); - return CE_Failure; - } - if (destBlock->GetDataRef() == NULL) - { + // Obtain RW Mutex + { + CPLMutexHolderD( GetRWMutex() ); + + // Write block to block cache + for (int j = 0; j < nBlocksPerColumn; ++j) { + for (int i = 0; i < nBlocksPerRow; ++i) { + GDALRasterBlock* destBlock = GetLockedBlockRef(i, j, TRUE); + if (destBlock == NULL) { + ReportError(CE_Failure, CPLE_OutOfMemory, + "GDALRasterBand::Fill(): Error " + "while retrieving cache block.\n"); + VSIFree(srcBlock); + return CE_Failure; + } + if (destBlock->GetDataRef() == NULL) + { + destBlock->DropLock(); + VSIFree(srcBlock); + return CE_Failure; + } + memcpy(destBlock->GetDataRef(), srcBlock, blockByteSize); + destBlock->MarkDirty(); destBlock->DropLock(); - VSIFree(srcBlock); - return CE_Failure; } - memcpy(destBlock->GetDataRef(), srcBlock, blockByteSize); - destBlock->MarkDirty(); - destBlock->DropLock(); - } + } } // Free up the source block @@ -2647,6 +2779,25 @@ GDALDataset *GDALRasterBand::GetDataset() return poDS; } +/************************************************************************/ +/* GetRWMutex() */ +/************************************************************************/ + +/** + * \brief Fetch the mutex to protect against multiple readwrites against + * the band at once. + * + * @return the pointer to the Mutex + */ + +void **GDALRasterBand::GetRWMutex() +{ + if (NULL == poDS) + return &hRWMutex; + else + return &(poDS->hRWMutex); +} + /************************************************************************/ /* GDALGetBandDataset() */ /************************************************************************/ @@ -2765,6 +2916,9 @@ CPLErr GDALRasterBand::GetHistogram( double dfMin, double dfMax, const char* pszPixelType = GetMetadataItem("PIXELTYPE", "IMAGE_STRUCTURE"); int bSignedByte = (pszPixelType != NULL && EQUAL(pszPixelType, "SIGNEDBYTE")); + // Set the RW Mutex + CPLMutexHolderD( GetRWMutex() ); + if ( bApproxOK && HasArbitraryOverviews() ) { /* -------------------------------------------------------------------- */ @@ -3532,6 +3686,9 @@ GDALRasterBand::ComputeStatistics( int bApproxOK, const char* pszPixelType = GetMetadataItem("PIXELTYPE", "IMAGE_STRUCTURE"); int bSignedByte = (pszPixelType != NULL && EQUAL(pszPixelType, "SIGNEDBYTE")); + // Set the RW Mutex + CPLMutexHolderD( GetRWMutex() ); + if ( bApproxOK && HasArbitraryOverviews() ) { /* -------------------------------------------------------------------- */ @@ -4007,6 +4164,9 @@ CPLErr GDALRasterBand::ComputeRasterMinMax( int bApproxOK, const char* pszPixelType = GetMetadataItem("PIXELTYPE", "IMAGE_STRUCTURE"); int bSignedByte = (pszPixelType != NULL && EQUAL(pszPixelType, "SIGNEDBYTE")); + // Set the RW Mutex + CPLMutexHolderD( GetRWMutex() ); + if ( bApproxOK && HasArbitraryOverviews() ) { /* -------------------------------------------------------------------- */ diff --git a/gdal/gcore/gdalrasterblock.cpp b/gdal/gcore/gdalrasterblock.cpp index 6314f47db29f..456c3033f2b6 100644 --- a/gdal/gcore/gdalrasterblock.cpp +++ b/gdal/gcore/gdalrasterblock.cpp @@ -34,15 +34,6 @@ CPL_CVSID("$Id$"); -static int bCacheMaxInitialized = FALSE; -static GIntBig nCacheMax = 40 * 1024*1024; -static volatile GIntBig nCacheUsed = 0; - -static volatile GDALRasterBlock *poOldest = NULL; /* tail */ -static volatile GDALRasterBlock *poNewest = NULL; /* head */ - -static void *hRBMutex = NULL; - /************************************************************************/ /* GDALSetCacheMax() */ /************************************************************************/ @@ -62,7 +53,7 @@ static void *hRBMutex = NULL; void CPL_STDCALL GDALSetCacheMax( int nNewSizeInBytes ) { - GDALSetCacheMax64(nNewSizeInBytes); + GetGDALRasterBlockManager()->SetCacheMax(nNewSizeInBytes); } @@ -89,22 +80,7 @@ void CPL_STDCALL GDALSetCacheMax( int nNewSizeInBytes ) void CPL_STDCALL GDALSetCacheMax64( GIntBig nNewSizeInBytes ) { - bCacheMaxInitialized = TRUE; - nCacheMax = nNewSizeInBytes; - -/* -------------------------------------------------------------------- */ -/* Flush blocks till we are under the new limit or till we */ -/* can't seem to flush anymore. */ -/* -------------------------------------------------------------------- */ - while( nCacheUsed > nCacheMax ) - { - GIntBig nOldCacheUsed = nCacheUsed; - - GDALFlushCacheBlock(); - - if( nCacheUsed == nOldCacheUsed ) - break; - } + GetGDALRasterBlockManager()->SetCacheMax(nNewSizeInBytes); } /************************************************************************/ @@ -128,7 +104,7 @@ void CPL_STDCALL GDALSetCacheMax64( GIntBig nNewSizeInBytes ) int CPL_STDCALL GDALGetCacheMax() { - GIntBig nRes = GDALGetCacheMax64(); + GIntBig nRes = GetGDALRasterBlockManager()->GetCacheMax(); if (nRes > INT_MAX) { static int bHasWarned = FALSE; @@ -164,28 +140,7 @@ int CPL_STDCALL GDALGetCacheMax() GIntBig CPL_STDCALL GDALGetCacheMax64() { - if( !bCacheMaxInitialized ) - { - const char* pszCacheMax = CPLGetConfigOption("GDAL_CACHEMAX",NULL); - bCacheMaxInitialized = TRUE; - if( pszCacheMax != NULL ) - { - GIntBig nNewCacheMax = (GIntBig)CPLScanUIntBig(pszCacheMax, strlen(pszCacheMax)); - if( nNewCacheMax < 100000 ) - { - if (nNewCacheMax < 0) - { - CPLError(CE_Failure, CPLE_NotSupported, - "Invalid value for GDAL_CACHEMAX. Using default value."); - return nCacheMax; - } - nNewCacheMax *= 1024 * 1024; - } - nCacheMax = nNewCacheMax; - } - } - - return nCacheMax; + return GetGDALRasterBlockManager()->GetCacheMax(); } /************************************************************************/ @@ -201,6 +156,7 @@ GIntBig CPL_STDCALL GDALGetCacheMax64() int CPL_STDCALL GDALGetCacheUsed() { + GIntBig nCacheUsed = GetGDALRasterBlockManager()->GetCacheUsed(); if (nCacheUsed > INT_MAX) { static int bHasWarned = FALSE; @@ -231,7 +187,7 @@ int CPL_STDCALL GDALGetCacheUsed() GIntBig CPL_STDCALL GDALGetCacheUsed64() { - return nCacheUsed; + return GetGDALRasterBlockManager()->GetCacheUsed(); } /************************************************************************/ @@ -250,9 +206,8 @@ GIntBig CPL_STDCALL GDALGetCacheUsed64() * or if they are currently locked. */ int CPL_STDCALL GDALFlushCacheBlock() - { - return GDALRasterBlock::FlushCacheBlock(); + return GetGDALRasterBlockManager()->FlushCacheBlock(); } /************************************************************************/ @@ -285,61 +240,6 @@ int CPL_STDCALL GDALFlushCacheBlock() * common. */ -/************************************************************************/ -/* FlushCacheBlock() */ -/* */ -/* Note, if we have alot of blocks locked for a long time, this */ -/* method is going to get slow because it will have to traverse */ -/* the linked list a long ways looking for a flushing */ -/* candidate. It might help to re-touch locked blocks to push */ -/* them to the top of the list. */ -/************************************************************************/ - -/** - * \brief Attempt to flush at least one block from the cache. - * - * This static method is normally used to recover memory when a request - * for a new cache block would put cache memory use over the established - * limit. - * - * C++ analog to the C function GDALFlushCacheBlock(). - * - * @return TRUE if successful or FALSE if no flushable block is found. - */ - -int GDALRasterBlock::FlushCacheBlock() - -{ - int nXOff, nYOff; - GDALRasterBand *poBand; - - { - CPLMutexHolderD( &hRBMutex ); - GDALRasterBlock *poTarget = (GDALRasterBlock *) poOldest; - - while( poTarget != NULL && poTarget->GetLockCount() > 0 ) - poTarget = poTarget->poPrevious; - - if( poTarget == NULL ) - return FALSE; - - poTarget->Detach(); - - nXOff = poTarget->GetXOff(); - nYOff = poTarget->GetYOff(); - poBand = poTarget->GetBand(); - } - - CPLErr eErr = poBand->FlushBlock( nXOff, nYOff ); - if (eErr != CE_None) - { - /* Save the error for later reporting */ - poBand->SetFlushBlockErr(eErr); - } - - return TRUE; -} - /************************************************************************/ /* GDALRasterBlock() */ /************************************************************************/ @@ -360,17 +260,22 @@ int GDALRasterBlock::FlushCacheBlock() */ GDALRasterBlock::GDALRasterBlock( GDALRasterBand *poBandIn, - int nXOffIn, int nYOffIn ) + int nXOffIn, int nYOffIn, + GDALRasterBlockManager *poManagerIn ) { CPLAssert( NULL != poBandIn ); poBand = poBandIn; + poManager = poManagerIn; poBand->GetBlockSize( &nXSize, &nYSize ); eType = poBand->GetRasterDataType(); pData = NULL; bDirty = FALSE; + bAttachedToCache = FALSE; + bAttachedToBand = FALSE; + bDelete = FALSE; nLockCount = 0; poNext = poPrevious = NULL; @@ -392,27 +297,62 @@ GDALRasterBlock::GDALRasterBlock( GDALRasterBand *poBandIn, GDALRasterBlock::~GDALRasterBlock() { - Detach(); - if( pData != NULL ) { - int nSizeInBytes; - VSIFree( pData ); + } + + CPLAssert( nLockCount == 0 ); + +} + +/************************************************************************/ +/* AddLock() */ +/************************************************************************/ + +/** + * Add a usage lock to the block. + */ + +void GDALRasterBlock::AddLock() + +{ + CPLMutexHolderD( &(poBand->hBandMutex) ); + nLockCount++; +} + +/************************************************************************/ +/* DropLock() */ +/************************************************************************/ + +/** + * Remove a lock from the block. + */ - nSizeInBytes = nXSize * nYSize * (GDALGetDataTypeSize(eType) / 8); +void GDALRasterBlock::DropLock() +{ + int bDeleteNow = FALSE; + { + CPLMutexHolderD( &(poBand->hBandMutex) ); + nLockCount--; + CPLAssert( nLockCount >= 0 ); + if ( nLockCount == 0 && bDelete ) { - CPLMutexHolderD( &hRBMutex ); - nCacheUsed -= nSizeInBytes; + if ( bAttachedToCache ) + Detach(); + if ( bAttachedToBand ) + poBand->UnadoptBlock(nXOff, nYOff); + bDeleteNow = TRUE; } } + if ( bDeleteNow ) + { + if ( bDirty ) + Write(); + delete this; + } - CPLAssert( nLockCount == 0 ); - -#ifdef ENABLE_DEBUG - Verify(); -#endif } /************************************************************************/ @@ -431,14 +371,26 @@ GDALRasterBlock::~GDALRasterBlock() void GDALRasterBlock::Detach() { - CPLMutexHolderD( &hRBMutex ); + CPLMutexHolderD( &(poManager->hRBMMutex) ); + int nSizeInBytes; + if ( bAttachedToCache ) + { + nSizeInBytes = nXSize * nYSize * (GDALGetDataTypeSize(eType)/8); + poManager->nCacheUsed -= nSizeInBytes; + } + else + { + return; + } - if( poOldest == this ) - poOldest = poPrevious; + bAttachedToCache = FALSE; - if( poNewest == this ) + if( poManager->poOldest == this ) + poManager->poOldest = poPrevious; + + if( poManager->poNewest == this ) { - poNewest = poNext; + poManager->poNewest = poNext; } if( poPrevious != NULL ) @@ -451,45 +403,6 @@ void GDALRasterBlock::Detach() poNext = NULL; } -/************************************************************************/ -/* Verify() */ -/************************************************************************/ - -/** - * Confirms (via assertions) that the block cache linked list is in a - * consistent state. - */ - -void GDALRasterBlock::Verify() - -{ - CPLMutexHolderD( &hRBMutex ); - - CPLAssert( (poNewest == NULL && poOldest == NULL) - || (poNewest != NULL && poOldest != NULL) ); - - if( poNewest != NULL ) - { - CPLAssert( poNewest->poPrevious == NULL ); - CPLAssert( poOldest->poNext == NULL ); - - for( GDALRasterBlock *poBlock = (GDALRasterBlock *) poNewest; - poBlock != NULL; - poBlock = poBlock->poNext ) - { - if( poBlock->poPrevious ) - { - CPLAssert( poBlock->poPrevious->poNext == poBlock ); - } - - if( poBlock->poNext ) - { - CPLAssert( poBlock->poNext->poPrevious == poBlock ); - } - } - } -} - /************************************************************************/ /* Write() */ /************************************************************************/ @@ -507,6 +420,11 @@ void GDALRasterBlock::Verify() CPLErr GDALRasterBlock::Write() { + if ( !GetDirty() ) + return CE_None; + + CPLMutexHolderD( poBand->GetRWMutex() ); + if( !GetDirty() ) return CE_None; @@ -535,13 +453,23 @@ CPLErr GDALRasterBlock::Write() void GDALRasterBlock::Touch() { - CPLMutexHolderD( &hRBMutex ); + int nSizeInBytes; + CPLMutexHolderD( &(poManager->hRBMMutex) ); + if ( bDelete ) + return; + + if ( !bAttachedToCache ) + { + nSizeInBytes = nXSize * nYSize * (GDALGetDataTypeSize(eType)/8); + poManager->nCacheUsed += nSizeInBytes; + bAttachedToCache = TRUE; + } - if( poNewest == this ) + if( poManager->poNewest == this ) return; - if( poOldest == this ) - poOldest = this->poPrevious; + if( poManager->poOldest == this ) + poManager->poOldest = this->poPrevious; if( poPrevious != NULL ) poPrevious->poNext = poNext; @@ -550,22 +478,23 @@ void GDALRasterBlock::Touch() poNext->poPrevious = poPrevious; poPrevious = NULL; - poNext = (GDALRasterBlock *) poNewest; + poNext = (GDALRasterBlock *) poManager->poNewest; - if( poNewest != NULL ) + if( poManager->poNewest != NULL ) { - CPLAssert( poNewest->poPrevious == NULL ); - poNewest->poPrevious = this; + CPLAssert( poManager->poNewest->poPrevious == NULL ); + poManager->poNewest->poPrevious = this; } - poNewest = this; + poManager->poNewest = this; - if( poOldest == NULL ) + if( poManager->poOldest == NULL ) { CPLAssert( poPrevious == NULL && poNext == NULL ); - poOldest = this; + poManager->poOldest = this; } + #ifdef ENABLE_DEBUG - Verify(); + poManager->Verify(); #endif } @@ -578,8 +507,6 @@ void GDALRasterBlock::Touch() * * This method allocates memory for the block, and attempts to flush other * blocks, if necessary, to bring the total cache size back within the limits. - * The newly allocated block is touched and will be considered most recently - * used in the LRU list. * * @return CE_None on success or CE_Failure if memory allocation fails. */ @@ -587,11 +514,8 @@ void GDALRasterBlock::Touch() CPLErr GDALRasterBlock::Internalize() { - CPLMutexHolderD( &hRBMutex ); void *pNewData; int nSizeInBytes; - GIntBig nCurCacheMax = GDALGetCacheMax64(); - /* No risk of overflow as it is checked in GDALRasterBand::InitBlockInfo() */ nSizeInBytes = nXSize * nYSize * (GDALGetDataTypeSize(eType) / 8); @@ -609,27 +533,6 @@ CPLErr GDALRasterBlock::Internalize() pData = pNewData; -/* -------------------------------------------------------------------- */ -/* Flush old blocks if we are nearing our memory limit. */ -/* -------------------------------------------------------------------- */ - AddLock(); /* don't flush this block! */ - - nCacheUsed += nSizeInBytes; - while( nCacheUsed > nCurCacheMax ) - { - GIntBig nOldCacheUsed = nCacheUsed; - - GDALFlushCacheBlock(); - - if( nCacheUsed == nOldCacheUsed ) - break; - } - -/* -------------------------------------------------------------------- */ -/* Add this block to the list. */ -/* -------------------------------------------------------------------- */ - Touch(); - DropLock(); return( CE_None ); } @@ -648,6 +551,7 @@ CPLErr GDALRasterBlock::Internalize() void GDALRasterBlock::MarkDirty() { + //CPLMutexHolderD( &(poBand->hBandMutex) ); bDirty = TRUE; } @@ -666,50 +570,7 @@ void GDALRasterBlock::MarkDirty() void GDALRasterBlock::MarkClean() { + //CPLMutexHolderD( &(poBand->hBandMutex) ); bDirty = FALSE; } -/************************************************************************/ -/* SafeLockBlock() */ -/************************************************************************/ - -/** - * \brief Safely lock block. - * - * This method locks a GDALRasterBlock (and touches it) in a thread-safe - * manner. The global block cache mutex is held while locking the block, - * in order to avoid race conditions with other threads that might be - * trying to expire the block at the same time. The block pointer may be - * safely NULL, in which case this method does nothing. - * - * @param ppBlock Pointer to the block pointer to try and lock/touch. - */ - -int GDALRasterBlock::SafeLockBlock( GDALRasterBlock ** ppBlock ) - -{ - CPLAssert( NULL != ppBlock ); - - CPLMutexHolderD( &hRBMutex ); - - if( *ppBlock != NULL ) - { - (*ppBlock)->AddLock(); - (*ppBlock)->Touch(); - - return TRUE; - } - else - return FALSE; -} - -/************************************************************************/ -/* DestroyRBMutex() */ -/************************************************************************/ - -void GDALRasterBlock::DestroyRBMutex() -{ - if( hRBMutex != NULL ) - CPLDestroyMutex(hRBMutex); - hRBMutex = NULL; -} diff --git a/gdal/gcore/gdalrasterblockmanager.cpp b/gdal/gcore/gdalrasterblockmanager.cpp new file mode 100644 index 000000000000..08f7158c4533 --- /dev/null +++ b/gdal/gcore/gdalrasterblockmanager.cpp @@ -0,0 +1,402 @@ +/****************************************************************************** + * $Id$ + * + * Project: GDAL Core + * Purpose: Implementation of GDALRasterBlockManager class. + * Author: Blake Thompson, flippmoke@gmail.com + * + ****************************************************************************** + * Copyright (c) 1998, Frank Warmerdam + * Copyright (c) 2009-2013, Even Rouault + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#include "gdal_priv.h" +#include "cpl_string.h" +#include "cpl_multiproc.h" +#include "ogr_srs_api.h" +#include "cpl_multiproc.h" +#include "gdal_pam.h" +#include "gdal_alg_priv.h" + +#ifdef _MSC_VER +# ifdef MSVC_USE_VLD +# include +# include +# endif +#endif + +CPL_CVSID("$Id$"); + + +/************************************************************************/ +/* ==================================================================== */ +/* GDALRasterBlockManager */ +/* ==================================================================== */ +/************************************************************************/ + +static volatile GDALRasterBlockManager *poRBM = NULL; +static void *hRBMGlobalMutex = NULL; + +void** GDALGetphRBMMutex() { return &hRBMGlobalMutex; } + +/************************************************************************/ +/* GetGDALRasterBlockManager() */ +/* */ +/* A freestanding function to get the global instance of the */ +/* GDALRasterBlockManager. */ +/************************************************************************/ + +/** + * \brief Fetch the global GDAL raster block manager. + * + * This function fetches the pointer to the singleton global raster block manager. + * If the driver manager doesn't exist it is automatically created. + * + * @return pointer to the global raster block manager. This should not be able + * to fail. + */ + +GDALRasterBlockManager * GetGDALRasterBlockManager() + +{ + if( poRBM == NULL ) + { + CPLMutexHolderD( &hRBMGlobalMutex ); + + if( poRBM == NULL ) + poRBM = new GDALRasterBlockManager(); + } + + CPLAssert( NULL != poRBM ); + + return const_cast( poRBM ); +} + +/************************************************************************/ +/* GDALRasterBlockManager() */ +/************************************************************************/ + +GDALRasterBlockManager::GDALRasterBlockManager() + +{ + + bCacheMaxInitialized = FALSE; + nCacheMax = 40 * 1024*1024; + nCacheUsed = 0; + + poOldest = NULL; /* tail */ + poNewest = NULL; /* head */ + + hRBMMutex = NULL; + +} + +/************************************************************************/ +/* ~GDALRasterBlockManager() */ +/************************************************************************/ + +GDALRasterBlockManager::~GDALRasterBlockManager() + +{ + DestroyRBMMutex(); + poOldest = NULL; + poNewest = NULL; +} + +/************************************************************************/ +/* SetCacheMax() */ +/************************************************************************/ + +/** + * \brief Set maximum cache memory. + * + * This function sets the maximum amount of memory that Manager is permitted + * to use for GDALRasterBlock caching. The unit of the value is bytes. + * + * Note: On 32 bit platforms, the maximum amount of memory that can be addressed + * by a process might be 2 GB or 3 GB, depending on the operating system + * capabilities. This function will not make any attempt to check the + * consistency of the passed value with the effective capabilities of the OS. + * + * @param nNewSizeInBytes the maximum number of bytes for caching. + * + * @since GDAL 1.8.0 + */ + +void GDALRasterBlockManager::SetCacheMax( GIntBig nNewSizeInBytes ) + +{ + bCacheMaxInitialized = TRUE; + nCacheMax = nNewSizeInBytes; + +/* -------------------------------------------------------------------- */ +/* Flush blocks till we are under the new limit or till we */ +/* can't seem to flush anymore. */ +/* -------------------------------------------------------------------- */ + FlushTillBelow(); +} + +/************************************************************************/ +/* GDALGetCacheMax() */ +/************************************************************************/ + +/** + * \brief Get maximum cache memory. + * + * Gets the maximum amount of memory available to the GDALRasterBlock + * caching system for caching GDAL read/write imagery. + * + * The first type this function is called, it will read the GDAL_CACHEMAX + * configuation option to initialize the maximum cache memory. + * + * @return maximum in bytes. + * + * @since GDAL 1.8.0 + */ + +GIntBig GDALRasterBlockManager::GetCacheMax() +{ + if( !bCacheMaxInitialized ) + { + const char* pszCacheMax = CPLGetConfigOption("GDAL_CACHEMAX",NULL); + bCacheMaxInitialized = TRUE; + if( pszCacheMax != NULL ) + { + GIntBig nNewCacheMax = (GIntBig)CPLScanUIntBig(pszCacheMax, strlen(pszCacheMax)); + if( nNewCacheMax < 100000 ) + { + if (nNewCacheMax < 0) + { + CPLError(CE_Failure, CPLE_NotSupported, + "Invalid value for GDAL_CACHEMAX. Using default value."); + return nCacheMax; + } + nNewCacheMax *= 1024 * 1024; + } + nCacheMax = nNewCacheMax; + } + } + + return nCacheMax; +} + +/************************************************************************/ +/* GDALGetCacheUsed() */ +/************************************************************************/ + +/** + * \brief Get cache memory used. + * + * @return the number of bytes of memory currently in use by the + * GDALRasterBlock memory caching. + * + * @since GDAL 1.8.0 + */ + +GIntBig GDALRasterBlockManager::GetCacheUsed() +{ + return nCacheUsed; +} + +/************************************************************************/ +/* FlushCacheBlock() */ +/* */ +/* Note, if we have alot of blocks locked for a long time, this */ +/* method is going to get slow because it will have to traverse */ +/* the linked list a long ways looking for a flushing */ +/* candidate. */ +/************************************************************************/ + +/** + * \brief Attempt to flush at least one block from the cache. + * + * This method is normally used to recover memory when a request + * for a new cache block would put cache memory use over the established + * limit. + * + * C++ analog to the C function GDALFlushCacheBlock(). + * + * @return TRUE if successful or FALSE if no flushable block is found. + */ + +int GDALRasterBlockManager::FlushCacheBlock() +{ + int nXOff, nYOff; + GDALRasterBand *poBand; + GDALRasterBlock *poTarget = NULL;; + + { + CPLMutexHolderD( &hRBMMutex ); + poTarget = (GDALRasterBlock *) poOldest; + while( poTarget != NULL && poTarget->GetLockCount() > 0 ) + poTarget = poTarget->poPrevious; + + if( poTarget == NULL ) + return FALSE; + + nXOff = poTarget->GetXOff(); + nYOff = poTarget->GetYOff(); + poBand = poTarget->GetBand(); + poTarget->Detach(); + } + + CPLErr eErr = poBand->FlushBlock( nXOff, nYOff ); + + if (eErr != CE_None) + { + /* Save the error for later reporting */ + poBand->SetFlushBlockErr(eErr); + } + + return TRUE; +} + +/************************************************************************/ +/* FlushTillBelow() */ +/************************************************************************/ + +/** + * \brief Attempt to get below the maximum cache size. + */ + +void GDALRasterBlockManager::FlushTillBelow() +{ + while( nCacheUsed > nCacheMax ) + { + GIntBig nOldCacheUsed = nCacheUsed; + + FlushCacheBlock(); + + if( nCacheUsed == nOldCacheUsed ) + break; + } +} + +/************************************************************************/ +/* Verify() */ +/************************************************************************/ + +/** + * Confirms (via assertions) that the block cache linked list is in a + * consistent state. + */ + +void GDALRasterBlockManager::Verify() + +{ + CPLMutexHolderD( &hRBMMutex ); + + CPLAssert( (poNewest == NULL && poOldest == NULL) + || (poNewest != NULL && poOldest != NULL) ); + + if( poNewest != NULL ) + { + CPLAssert( poNewest->poPrevious == NULL ); + CPLAssert( poOldest->poNext == NULL ); + + for( GDALRasterBlock *poBlock = (GDALRasterBlock *) poNewest; + poBlock != NULL; + poBlock = poBlock->poNext ) + { + if( poBlock->poPrevious ) + { + CPLAssert( poBlock->poPrevious->poNext == poBlock ); + } + + if( poBlock->poNext ) + { + CPLAssert( poBlock->poNext->poPrevious == poBlock ); + } + } + } +} + +/************************************************************************/ +/* SafeLockBlock() */ +/************************************************************************/ + +/** + * \brief Safely lock block. + * + * This method locks a GDALRasterBlock in a thread-safe + * manner. The block cache mutex is held while locking the block, + * in order to avoid race conditions with other threads that might be + * trying to expire the block at the same time. The block pointer may be + * safely NULL, in which case this method does nothing. + * + * @param ppBlock Pointer to the block pointer to try and lock. + */ + +int GDALRasterBlockManager::SafeLockBlock( GDALRasterBlock ** ppBlock ) + +{ + CPLAssert( NULL != ppBlock ); + + if( *ppBlock != NULL ) + { + (*ppBlock)->AddLock(); + return TRUE; + } + else + return FALSE; +} + +/************************************************************************/ +/* DestroyRBMMutex() */ +/************************************************************************/ + +void GDALRasterBlockManager::DestroyRBMMutex() +{ + if( hRBMMutex != NULL ) + CPLDestroyMutex(hRBMMutex); + hRBMMutex = NULL; +} + +/************************************************************************/ +/* GDALDestroyRasterBlockManager() */ +/************************************************************************/ + +/** + * \brief Destroy the global raster block manager. + * + * NOTE: This function is not thread safe. It should not be called while + * other threads are actively using GDAL. + */ + +void CPL_STDCALL GDALDestroyRasterBlockManager( void ) + +{ + // THREADSAFETY: We would like to lock the mutex here, but it + // needs to be reacquired within the destructor during driver + // deregistration. + if( poRBM != NULL ) + { + CPLMutexHolderD( &hRBMGlobalMutex ); + if (poRBM != NULL ) { + delete poRBM; + } + } + if ( hRBMGlobalMutex != NULL ) + CPLDestroyMutex(hRBMGlobalMutex); + hRBMGlobalMutex = NULL; +} + + diff --git a/gdal/gcore/makefile.vc b/gdal/gcore/makefile.vc index 61835b9ab48a..9b8ea8d1cccf 100644 --- a/gdal/gcore/makefile.vc +++ b/gdal/gcore/makefile.vc @@ -11,7 +11,7 @@ OBJ = gdalopeninfo.obj gdaldrivermanager.obj gdaldriver.obj \ gdalnodatavaluesmaskband.obj gdaldefaultasync.obj \ gdaldllmain.obj gdalexif.obj gdalclientserver.obj \ gdalgeorefpamdataset.obj gdaljp2abstractdataset.obj \ - gdalvirtualmem.obj + gdalvirtualmem.obj gdalrasterblockmanager.obj RES = Version.res diff --git a/gdal/gcore/rasterio.cpp b/gdal/gcore/rasterio.cpp index c017f7e17e12..a1a725e4c8c2 100644 --- a/gdal/gcore/rasterio.cpp +++ b/gdal/gcore/rasterio.cpp @@ -30,6 +30,7 @@ ****************************************************************************/ #include "gdal_priv.h" +#include "cpl_multiproc.h" // Define a list of "C++" compilers that have broken template support or // broken scoping so we can fall back on the legacy implementation of @@ -117,13 +118,10 @@ CPLErr GDALRasterBand::IRasterIO( GDALRWFlag eRWFlag, if( poBlock == NULL ) { CPLError( CE_Failure, CPLE_AppDefined, - "GetBlockRef failed at X block offset %d, " + "GetBlockRef failed at X block offset %d, " "Y block offset %d", 0, nLBlockY ); - return( CE_Failure ); + return( CE_Failure ); } - - if( eRWFlag == GF_Write ) - poBlock->MarkDirty(); pabySrcBlock = (GByte *) poBlock->GetDataRef(); if( pabySrcBlock == NULL ) @@ -135,32 +133,37 @@ CPLErr GDALRasterBand::IRasterIO( GDALRWFlag eRWFlag, nSrcByteOffset = ((iSrcY-nLBlockY*nBlockYSize)*nBlockXSize + nXOff) * nBandDataSize; - - if( eDataType == eBufType ) - { - if( eRWFlag == GF_Read ) - memcpy( ((GByte *) pData) + (size_t)iBufYOff * nLineSpace, - pabySrcBlock + nSrcByteOffset, - nLineSpace ); - else - memcpy( pabySrcBlock + nSrcByteOffset, - ((GByte *) pData) + (size_t)iBufYOff * nLineSpace, - nLineSpace ); - } - else { - /* type to type conversion */ + CPLMutexHolderD( GetRWMutex() ); + if( eRWFlag == GF_Write ) + poBlock->MarkDirty(); - if( eRWFlag == GF_Read ) - GDALCopyWords( pabySrcBlock + nSrcByteOffset, - eDataType, nBandDataSize, - ((GByte *) pData) + (size_t)iBufYOff * nLineSpace, - eBufType, nPixelSpace, nBufXSize ); + if( eDataType == eBufType ) + { + if( eRWFlag == GF_Read ) + memcpy( ((GByte *) pData) + (size_t)iBufYOff * nLineSpace, + pabySrcBlock + nSrcByteOffset, + nLineSpace ); + else + memcpy( pabySrcBlock + nSrcByteOffset, + ((GByte *) pData) + (size_t)iBufYOff * nLineSpace, + nLineSpace ); + } else - GDALCopyWords( ((GByte *) pData) + (size_t)iBufYOff * nLineSpace, - eBufType, nPixelSpace, - pabySrcBlock + nSrcByteOffset, - eDataType, nBandDataSize, nBufXSize ); + { + /* type to type conversion */ + + if( eRWFlag == GF_Read ) + GDALCopyWords( pabySrcBlock + nSrcByteOffset, + eDataType, nBandDataSize, + ((GByte *) pData) + (size_t)iBufYOff * nLineSpace, + eBufType, nPixelSpace, nBufXSize ); + else + GDALCopyWords( ((GByte *) pData) + (size_t)iBufYOff * nLineSpace, + eBufType, nPixelSpace, + pabySrcBlock + nSrcByteOffset, + eDataType, nBandDataSize, nBufXSize ); + } } } @@ -268,14 +271,11 @@ CPLErr GDALRasterBand::IRasterIO( GDALRWFlag eRWFlag, if( !poBlock ) { CPLError( CE_Failure, CPLE_AppDefined, - "GetBlockRef failed at X block offset %d, " + "GetBlockRef failed at X block offset %d, " "Y block offset %d", nLBlockX, nLBlockY ); return( CE_Failure ); } - if( eRWFlag == GF_Write ) - poBlock->MarkDirty(); - pabySrcBlock = (GByte *) poBlock->GetDataRef(); if( pabySrcBlock == NULL ) { @@ -286,39 +286,45 @@ CPLErr GDALRasterBand::IRasterIO( GDALRWFlag eRWFlag, /* -------------------------------------------------------------------- */ /* Copy over this chunk of data. */ /* -------------------------------------------------------------------- */ - iSrcOffset = ((size_t)iSrcX - (size_t)nLBlockX*nBlockXSize - + ((size_t)(iSrcY) - (size_t)nLBlockY*nBlockYSize) * nBlockXSize)*nBandDataSize; - /* Fill up as many rows as possible for the loaded block */ - int kmax = MIN(nBlockYSize - (iSrcY % nBlockYSize), nBufYSize - iBufYOff); - for(int k=0; kMarkDirty(); + + iSrcOffset = ((size_t)iSrcX - (size_t)nLBlockX*nBlockXSize + + ((size_t)(iSrcY) - (size_t)nLBlockY*nBlockYSize) * nBlockXSize)*nBandDataSize; + /* Fill up as many rows as possible for the loaded block */ + int kmax = MIN(nBlockYSize - (iSrcY % nBlockYSize), nBufYSize - iBufYOff); + for(int k=0; kMarkDirty(); - pabyDstBlock = (GByte *) poBlock->GetDataRef(); if( pabyDstBlock == NULL ) { @@ -411,22 +415,26 @@ CPLErr GDALRasterBand::IRasterIO( GDALRWFlag eRWFlag, /* -------------------------------------------------------------------- */ /* Copy over this pixel of data. */ /* -------------------------------------------------------------------- */ - iDstOffset = ((size_t)iDstX - (size_t)nLBlockX*nBlockXSize - + ((size_t)iDstY - (size_t)nLBlockY*nBlockYSize) * nBlockXSize)*nBandDataSize; - - if( eDataType == eBufType ) { - memcpy( pabyDstBlock + iDstOffset, - ((GByte *) pData) + iBufOffset, nBandDataSize ); - } - else - { - /* type to type conversion ... ouch, this is expensive way - of handling single words */ + CPLMutexHolderD( GetRWMutex() ); + poBlock->MarkDirty(); + iDstOffset = ((size_t)iDstX - (size_t)nLBlockX*nBlockXSize + + ((size_t)iDstY - (size_t)nLBlockY*nBlockYSize) * nBlockXSize)*nBandDataSize; + + if( eDataType == eBufType ) + { + memcpy( pabyDstBlock + iDstOffset, + ((GByte *) pData) + iBufOffset, nBandDataSize ); + } + else + { + /* type to type conversion ... ouch, this is expensive way + of handling single words */ - GDALCopyWords( ((GByte *) pData) + iBufOffset, eBufType, 0, - pabyDstBlock + iDstOffset, eDataType, 0, - 1 ); + GDALCopyWords( ((GByte *) pData) + iBufOffset, eBufType, 0, + pabyDstBlock + iDstOffset, eDataType, 0, + 1 ); + } } } } @@ -499,23 +507,26 @@ CPLErr GDALRasterBand::IRasterIO( GDALRWFlag eRWFlag, /* Copy over this pixel of data. */ /* -------------------------------------------------------------------- */ iSrcOffset = ((size_t)nDiffX + iSrcOffsetCst)*nBandDataSize; - - if( bByteCopy ) - { - ((GByte *) pData)[iBufOffset] = pabySrcBlock[iSrcOffset]; - } - else if( eDataType == eBufType ) { - memcpy( ((GByte *) pData) + iBufOffset, - pabySrcBlock + iSrcOffset, nBandDataSize ); - } - else - { - /* type to type conversion ... ouch, this is expensive way - of handling single words */ - GDALCopyWords( pabySrcBlock + iSrcOffset, eDataType, 0, - ((GByte *) pData) + iBufOffset, eBufType, 0, - 1 ); + CPLMutexHolderD( GetRWMutex() ); + + if( bByteCopy ) + { + ((GByte *) pData)[iBufOffset] = pabySrcBlock[iSrcOffset]; + } + else if( eDataType == eBufType ) + { + memcpy( ((GByte *) pData) + iBufOffset, + pabySrcBlock + iSrcOffset, nBandDataSize ); + } + else + { + /* type to type conversion ... ouch, this is expensive way + of handling single words */ + GDALCopyWords( pabySrcBlock + iSrcOffset, eDataType, 0, + ((GByte *) pData) + iBufOffset, eBufType, 0, + 1 ); + } } iBufOffset += nPixelSpace; @@ -2576,9 +2587,6 @@ GDALDataset::BlockBasedRasterIO( GDALRWFlag eRWFlag, goto CleanupAndReturn; } - if( eRWFlag == GF_Write ) - poBlock->MarkDirty(); - if( papoBlocks[iBand] != NULL ) papoBlocks[iBand]->DropLock(); @@ -2598,35 +2606,40 @@ GDALDataset::BlockBasedRasterIO( GDALRWFlag eRWFlag, /* -------------------------------------------------------------------- */ iSrcOffset = (iSrcX - nLBlockX*nBlockXSize + (iSrcY - nLBlockY*nBlockYSize) * nBlockXSize)*nBandDataSize; - - for( iBand = 0; iBand < nBandCount; iBand++ ) { - GByte *pabySrcBlock = papabySrcBlock[iBand]; - int iBandBufOffset = iBufOffset + iBand * nBandSpace; - - if( eDataType == eBufType ) - { - if( eRWFlag == GF_Read ) - memcpy( ((GByte *) pData) + iBandBufOffset, - pabySrcBlock + iSrcOffset, nBandDataSize ); - else - memcpy( pabySrcBlock + iSrcOffset, - ((GByte *)pData) + iBandBufOffset, nBandDataSize ); - } - else + CPLMutexHolderD( &hRWMutex ); + for( iBand = 0; iBand < nBandCount; iBand++ ) { - /* type to type conversion ... ouch, this is expensive way - of handling single words */ + GByte *pabySrcBlock = papabySrcBlock[iBand]; + int iBandBufOffset = iBufOffset + iBand * nBandSpace; - if( eRWFlag == GF_Read ) - GDALCopyWords( pabySrcBlock + iSrcOffset, eDataType, 0, - ((GByte *) pData) + iBandBufOffset, - eBufType, 0, 1 ); + if( eRWFlag == GF_Write ) + papoBlocks[iBand]->MarkDirty(); + + if( eDataType == eBufType ) + { + if( eRWFlag == GF_Read ) + memcpy( ((GByte *) pData) + iBandBufOffset, + pabySrcBlock + iSrcOffset, nBandDataSize ); + else + memcpy( pabySrcBlock + iSrcOffset, + ((GByte *)pData) + iBandBufOffset, nBandDataSize ); + } else - GDALCopyWords( ((GByte *) pData) + iBandBufOffset, - eBufType, 0, - pabySrcBlock + iSrcOffset, eDataType, 0, - 1 ); + { + /* type to type conversion ... ouch, this is expensive way + of handling single words */ + + if( eRWFlag == GF_Read ) + GDALCopyWords( pabySrcBlock + iSrcOffset, eDataType, 0, + ((GByte *) pData) + iBandBufOffset, + eBufType, 0, 1 ); + else + GDALCopyWords( ((GByte *) pData) + iBandBufOffset, + eBufType, 0, + pabySrcBlock + iSrcOffset, eDataType, 0, + 1 ); + } } } @@ -2691,12 +2704,15 @@ static void GDALCopyWholeRasterGetSwathSize(GDALRasterBand *poSrcPrototypeBand, /* greater than the block cache. */ const char* pszSwathSize = CPLGetConfigOption("GDAL_SWATH_SIZE", NULL); int nTargetSwathSize; + GDALRasterBlockManager *poSrcRBM = poSrcPrototypeBand->GetRasterBlockManager(); + GDALRasterBlockManager *poDstRBM = poDstPrototypeBand->GetRasterBlockManager(); + GIntBig nCacheMaxMin = MIN(poSrcRBM->GetCacheMax(), poDstRBM->GetCacheMax()); if( pszSwathSize != NULL ) nTargetSwathSize = atoi(pszSwathSize); else { /* As a default, take one 1/4 of the cache size */ - nTargetSwathSize = MIN(INT_MAX, GDALGetCacheMax64() / 4); + nTargetSwathSize = MIN(INT_MAX, nCacheMaxMin / 4); /* but if the minimum idal swath buf size is less, then go for it to */ /* avoid unnecessarily abusing RAM usage */ @@ -2709,11 +2725,11 @@ static void GDALCopyWholeRasterGetSwathSize(GDALRasterBand *poSrcPrototypeBand, nTargetSwathSize = 1000000; /* But let's check that */ - if (bDstIsCompressed && bInterleave && nTargetSwathSize > GDALGetCacheMax64()) + if (bDstIsCompressed && bInterleave && nTargetSwathSize > nCacheMaxMin) { CPLError(CE_Warning, CPLE_AppDefined, "When translating into a compressed interleave format, the block cache size (" CPL_FRMT_GIB ") " - "should be at least the size of the swath (%d) (GDAL_SWATH_SIZE config. option)", GDALGetCacheMax64(), nTargetSwathSize); + "should be at least the size of the swath (%d) (GDAL_SWATH_SIZE config. option)", nCacheMaxMin, nTargetSwathSize); } #define IS_MULTIPLE_OF(x,y) ((y)%(x) == 0) diff --git a/gdal/port/cpl_multiproc.cpp b/gdal/port/cpl_multiproc.cpp index f6cba164ea6a..0f98030c27c1 100644 --- a/gdal/port/cpl_multiproc.cpp +++ b/gdal/port/cpl_multiproc.cpp @@ -74,8 +74,11 @@ CPLMutexHolder::CPLMutexHolder( void **phMutex, double dfWaitInSeconds, "CPLMutexHolder: Request %p for pid %ld at %d/%s.\n", *phMutex, (long) CPLGetPID(), nLine, pszFile ); #endif - - if( !CPLCreateOrAcquireMutex( phMutex, dfWaitInSeconds ) ) + if ( NULL == phMutex ) + { + hMutex = NULL; + } + else if( !CPLCreateOrAcquireMutex( phMutex, dfWaitInSeconds ) ) { fprintf( stderr, "CPLMutexHolder: Failed to acquire mutex!\n" ); hMutex = NULL; @@ -1127,6 +1130,11 @@ static void *CPLCreateMutexInternal(int bAlreadyInGlobalLock); int CPLCreateOrAcquireMutex( void **phMutex, double dfWaitInSeconds ) { + // Check before going into the global lock, otherwise + // we risk using the global lock too often. + if ( NULL != *phMutex ) + return CPLAcquireMutex( *phMutex, dfWaitInSeconds ); + int bSuccess = FALSE; pthread_mutex_lock(&global_mutex); @@ -1240,9 +1248,9 @@ int CPLAcquireMutex( void *hMutexIn, double dfWaitInSeconds ) if( err != 0 ) { if( err == EDEADLK ) - fprintf(stderr, "CPLAcquireMutex: Error = %d/EDEADLK", err ); + fprintf(stderr, "CPLAcquireMutex: Error = %d/EDEADLK\n", err ); else - fprintf(stderr, "CPLAcquireMutex: Error = %d", err ); + fprintf(stderr, "CPLAcquireMutex: Error = %d\n", err ); return FALSE; } diff --git a/gdal/wince/msvc80/gdalce_dll/gdalce_dll.vcproj b/gdal/wince/msvc80/gdalce_dll/gdalce_dll.vcproj index f40bc45bffe1..5e238867ad2d 100644 --- a/gdal/wince/msvc80/gdalce_dll/gdalce_dll.vcproj +++ b/gdal/wince/msvc80/gdalce_dll/gdalce_dll.vcproj @@ -1052,6 +1052,10 @@ RelativePath="..\..\..\gcore\gdalrasterblock.cpp" > + +