Per Dataset Caching

autotest/.gitignore

@@ -1,4 +1,5 @@
 *.pyc
+*.o
 ogr/tmp
 ogr/data/delete_nas.gfs
 ogr/data/replace_nas.gfs
@@ -15,3 +16,9 @@ gdrivers/data/byte.jp2.gz.properties
 gdrivers/data/utm.json
 gdrivers/tmp
 osr/tmp
+cpp/gdal_unit_test
+cpp/test_virtualmem
+cpp/testclosedondestroydm
+cpp/testcopywords
+cpp/testperfcopywords
+cpp/testthreadcond

gdal/alg/gdalrasterize.cpp

@@ -844,7 +844,7 @@ CPLErr GDALRasterizeLayers( GDALDatasetH hDS,
         ;
     else
     {
-        GIntBig nYChunkSize64 = GDALGetCacheMax64() / nScanlineBytes;
+        GIntBig nYChunkSize64 = poDS->GetRasterBlockManager()->GetCacheMax() / nScanlineBytes;
         if (nYChunkSize64 > INT_MAX)
             nYChunkSize = INT_MAX;
         else

gdal/frmts/epsilon/epsilondataset.cpp

@@ -29,6 +29,7 @@
 
 #include "epsilon.h"
 #include "gdal_pam.h"
+#include "cpl_multiproc.h"
 
 CPL_CVSID("$Id$");
 
@@ -337,11 +338,12 @@ CPLErr EpsilonRasterBand::IReadBlock( int nBlockXOff,
                     poBlock->DropLock();
                     break;
                 }
-
-                memcpy(pabySrcBlock,
-                       poGDS->pabyRGBData + (iOtherBand - 1) * nBlockXSize * nBlockYSize,
-                       nBlockXSize * nBlockYSize);
-
+                {
+                    CPLMutexHolderD( poGDS->GetRasterBand(iOtherBand)->GetRWMutex() );
+                    memcpy(pabySrcBlock,
+                           poGDS->pabyRGBData + (iOtherBand - 1) * nBlockXSize * nBlockYSize,
+                           nBlockXSize * nBlockYSize);
+                }
                 poBlock->DropLock();
             }
         }

gdal/frmts/gtiff/geotiff.cpp

@@ -1679,7 +1679,7 @@ CPLErr GTiffRasterBand::IRasterIO( GDALRWFlag eRWFlag,
         GIntBig nRequiredMem = (GIntBig)poGDS->nBands * nXBlocks * nYBlocks *
                                 nBlockXSize * nBlockYSize *
                                (GDALGetDataTypeSize(eDataType) / 8);
-        if (nRequiredMem > GDALGetCacheMax64())
+        if (nRequiredMem > poDS->GetRasterBlockManager()->GetCacheMax())
         {
             if (!poGDS->bHasWarnedDisableAggressiveBandCaching)
             {
@@ -1961,7 +1961,7 @@ CPLErr GTiffRasterBand::FillCacheForOtherBands( int nBlockXOff, int nBlockYOff )
 /*      enough to accomodate the size of all the blocks, don't enter    */
 /* -------------------------------------------------------------------- */
     if( poGDS->nBands != 1 && !poGDS->bLoadingOtherBands &&
-        nBlockXSize * nBlockYSize * (GDALGetDataTypeSize(eDataType) / 8) < GDALGetCacheMax64() / poGDS->nBands)
+        nBlockXSize * nBlockYSize * (GDALGetDataTypeSize(eDataType) / 8) < poDS->GetRasterBlockManager()->GetCacheMax() / poGDS->nBands)
     {
         int iOtherBand;
 

gdal/frmts/jp2kak/jp2kakdataset.cpp

@@ -618,8 +618,11 @@ CPLErr JP2KAKRasterBand::IReadBlock( int nBlockXOff, int nBlockYOff,
 
                 if( poBlock )
                 {
-                    memcpy( poBlock->GetDataRef(), pabyWrkBuffer + nBandStart, 
-                            nWordSize * nBlockXSize * nBlockYSize );
+                    {
+                        CPLMutexHolderD( poBand->GetRWMutex() );
+                        memcpy( poBlock->GetDataRef(), pabyWrkBuffer + nBandStart, 
+                                nWordSize * nBlockXSize * nBlockYSize );
+                    }
                     poBlock->DropLock();
                 }
             }

gdal/frmts/mbtiles/mbtilesdataset.cpp

@@ -31,6 +31,7 @@
 #include "gdal_pam.h"
 #include "ogr_api.h"
 #include "cpl_vsil_curl_priv.h"
+#include "cpl_multiproc.h"
 
 #include "zlib.h"
 #include "json.h"
@@ -293,11 +294,14 @@ CPLErr MBTilesBand::IReadBlock( int nBlockXOff, int nBlockYOff, void * pImage)
                         poBlock->DropLock();
                         break;
                     }
-
                     if (nTileBands == 3 && poGDS->nBands == 4 && iOtherBand == 4)
+                    {
+                        CPLMutexHolderD( poGDS->GetRasterBand(iOtherBand)->GetRWMutex() );
                         memset(pabySrcBlock, 255, nBlockXSize * nBlockYSize);
+                    }
                     else if (nTileBands == 1 && (poGDS->nBands == 3 || poGDS->nBands == 4))
                     {
+                        CPLMutexHolderD( poGDS->GetRasterBand(iOtherBand)->GetRWMutex() );
                         int i;
                         if (pSrcImage)
                         {
@@ -381,10 +385,11 @@ CPLErr MBTilesBand::IReadBlock( int nBlockXOff, int nBlockYOff, void * pImage)
                 poBlock->DropLock();
                 break;
             }
-
-            memset(pabySrcBlock, (iOtherBand == 4) ? 0 : 255,
-                   nBlockXSize * nBlockYSize);
-
+            {
+                CPLMutexHolderD( poGDS->GetRasterBand(iOtherBand)->GetRWMutex() );
+                memset(pabySrcBlock, (iOtherBand == 4) ? 0 : 255,
+                       nBlockXSize * nBlockYSize);
+            }
             poBlock->DropLock();
         }
     }

gdal/frmts/openjpeg/openjpegdataset.cpp

@@ -439,10 +439,9 @@ static void JP2OpenJPEGReadBlockInThread(void* userdata)
             poBlock->DropLock();
             break;
         }
-
+        
         poGDS->ReadBlock(nBand, fp, nBlockXOff, nBlockYOff, pDstBuffer,
                          nBandCount, panBandMap);
-
         poBlock->DropLock();
     }
 
@@ -468,7 +467,7 @@ int JP2OpenJPEGDataset::PreloadBlocks(JP2OpenJPEGRasterBand* poBand,
     int nMaxThreads = GetNumThreads();
     if( !bUseSetDecodeArea && nMaxThreads > 1 )
     {
-        if( nReqMem > GDALGetCacheMax64() / (nBandCount == 0 ? 1 : nBandCount) )
+        if( nReqMem > poRasterBlockManager->GetCacheMax() / (nBandCount == 0 ? 1 : nBandCount) )
             return FALSE;
 
         int nBlocksToLoad = 0;
@@ -508,7 +507,7 @@ int JP2OpenJPEGDataset::PreloadBlocks(JP2OpenJPEGRasterBand* poBand,
             }
             else
             {
-                if( nReqMem <= GDALGetCacheMax64() / nBands )
+                if( nReqMem <= poRasterBlockManager->GetCacheMax() / nBands )
                 {
                     oJob.nBandCount = nBands;
                     oJob.panBandMap = NULL;
@@ -716,7 +715,6 @@ CPLErr JP2OpenJPEGDataset::ReadBlock( int nBand, VSILFILE* fp,
                 continue;
             }
         }
-
         if (bIs420)
         {
             CPLAssert((int)psImage->comps[0].w >= nWidthToRead);
@@ -729,6 +727,7 @@ CPLErr JP2OpenJPEGDataset::ReadBlock( int nBand, VSILFILE* fp,
             OPJ_INT32* pSrcY = psImage->comps[0].data;
             OPJ_INT32* pSrcCb = psImage->comps[1].data;
             OPJ_INT32* pSrcCr = psImage->comps[2].data;
+            CPLMutexHolderD( &hRWMutex );
             GByte* pDst = (GByte*)pDstBuffer;
             for(int j=0;j<nHeightToRead;j++)
             {
@@ -761,6 +760,7 @@ CPLErr JP2OpenJPEGDataset::ReadBlock( int nBand, VSILFILE* fp,
                     }
                 }
             }
+            CPLMutexHolderD( &hRWMutex );
 
             if ((int)psImage->comps[iBand-1].w == nBlockXSize &&
                 (int)psImage->comps[iBand-1].h == nBlockYSize)

gdal/frmts/postgisraster/postgisrasterdataset.cpp

@@ -34,6 +34,7 @@
  * SOFTWARE.
  **********************************************************************/
 #include "postgisraster.h"
+#include "cpl_multiproc.h"
 #include <math.h>
 
 #ifdef _WIN32
@@ -973,11 +974,13 @@ void PostGISRasterDataset::CacheTile(const char* pszMetadata,
         {
             GDALRasterBlock* poBlock = poRTB->GetLockedBlockRef(0, 0, TRUE);
             if( poBlock != NULL )
-            {
-                // Point block data ref to fetched data
-                memcpy(poBlock->GetDataRef(), (void *)pbyDataToRead, 
-                    nExpectedBandDataSize);
-
+            {   
+                {
+                    CPLMutexHolderD( poRTB->GetRWMutex() );
+                    // Point block data ref to fetched data
+                    memcpy(poBlock->GetDataRef(), (void *)pbyDataToRead, 
+                        nExpectedBandDataSize);
+                }
                 poBlock->DropLock();
             }
         }
@@ -1065,7 +1068,7 @@ GBool PostGISRasterDataset::LoadSources(int nXOff, int nYOff, int nXSize, int nY
         {
             GIntBig nMemoryRequiredForTiles = PQntuples(poResult) * nTileWidth * nTileHeight *
                 GDALGetDataTypeSize(GetRasterBand(nBand)->GetRasterDataType()) / 8;
-            GIntBig nCacheMax = (GIntBig) GDALGetCacheMax64();
+            GIntBig nCacheMax = (GIntBig) poRasterBlockManager->GetCacheMax();
             if( nBands * nMemoryRequiredForTiles <= nCacheMax )
             {
                 bLoadRasters = TRUE;

gdal/frmts/postgisraster/postgisrasterrasterband.cpp

@@ -565,7 +565,7 @@ CPLErr PostGISRasterRasterBand::IRasterIO(GDALRWFlag eRWFlag, int nXOff,
     int bAllBandCaching = FALSE;
     if (nTilesToFetch > 0)
     {
-        GIntBig nCacheMax = (GIntBig) GDALGetCacheMax64();
+        GIntBig nCacheMax = (GIntBig) GetRasterBlockManager()->GetCacheMax();
         if( nMemoryRequiredForTiles > nCacheMax )
         {
             CPLDebug("PostGIS_Raster",

gdal/frmts/vrt/vrtwarped.cpp

@@ -31,6 +31,7 @@
 #include "vrtdataset.h"
 #include "cpl_minixml.h"
 #include "cpl_string.h"
+#include "cpl_multiproc.h"
 #include "gdalwarper.h"
 #include "gdal_alg_priv.h"
 #include <cassert>
@@ -1200,6 +1201,7 @@ CPLErr VRTWarpedDataset::ProcessBlock( int iBlockX, int iBlockY )
         {
             if ( poBlock->GetDataRef() != NULL )
             {
+                CPLMutexHolderD( poBand->GetRWMutex() );
                 GDALCopyWords( pabyDstBuffer + iBand*nBlockXSize*nBlockYSize*nWordSize,
                             psWO->eWorkingDataType, nWordSize, 
                             poBlock->GetDataRef(), 
@@ -1292,6 +1294,7 @@ CPLErr VRTWarpedRasterBand::IReadBlock( int nBlockXOff, int nBlockYOff,
         int nDataBytes;
         nDataBytes = (GDALGetDataTypeSize(poBlock->GetDataType()) / 8)
             * poBlock->GetXSize() * poBlock->GetYSize();
+        CPLMutexHolderD( GetRWMutex() );
         memcpy( pImage, poBlock->GetDataRef(), nDataBytes );
     }
 

gdal/frmts/wms/gdalwmsrasterband.cpp

@@ -29,6 +29,7 @@
  ****************************************************************************/
 
 #include "wmsdriver.h"
+#include "cpl_multiproc.h"
 
 GDALWMSRasterBand::GDALWMSRasterBand(GDALWMSDataset *parent_dataset, int band, double scale) {
     //	printf("[%p] GDALWMSRasterBand::GDALWMSRasterBand(%p, %d, %f)\n", this, parent_dataset, band, scale);
@@ -623,11 +624,14 @@ CPLErr GDALWMSRasterBand::ReadBlockFromFile(int x, int y, const char *file_name,
             for (int ib = 1; ib <= m_parent_dataset->nBands; ++ib) {
                 if (ret == CE_None) {
                     void *p = NULL;
+                    void ** hThisMutex = NULL;
                     GDALRasterBlock *b = NULL;
                     if ((buffer != NULL) && (ib == to_buffer_band)) {
                         p = buffer;
+                        hThisMutex = NULL;
                     } else {
                         GDALWMSRasterBand *band = static_cast<GDALWMSRasterBand *>(m_parent_dataset->GetRasterBand(ib));
+                        hThisMutex = band->GetRWMutex();
                         if (m_overview >= 0) band = static_cast<GDALWMSRasterBand *>(band->GetOverview(m_overview));
                         if (!band->IsBlockInCache(x, y)) {
                             b = band->GetLockedBlockRef(x, y, true);
@@ -664,6 +668,7 @@ CPLErr GDALWMSRasterBand::ReadBlockFromFile(int x, int y, const char *file_name,
                                if (accepted_as_no_alpha)
                                {
                                   // the file had 3 bands and we are reading band 4 (Alpha) so fill with 255 (no alpha)
+                                  CPLMutexHolderD( hThisMutex );
                                   GByte *byte_buffer = reinterpret_cast<GByte *>(p);
                                   for (int y = 0; y < sy; ++y) {
                                      for (int x = 0; x < sx; ++x) {
@@ -686,6 +691,7 @@ CPLErr GDALWMSRasterBand::ReadBlockFromFile(int x, int y, const char *file_name,
                             }
                             if (ret == CE_None) {
                                 GByte *band_color_table = color_table + 256 * (ib - 1);
+                                CPLMutexHolderD( hThisMutex );
                                 GByte *byte_buffer = reinterpret_cast<GByte *>(p);
                                 for (int y = 0; y < sy; ++y) {
                                     for (int x = 0; x < sx; ++x) {
@@ -724,11 +730,14 @@ CPLErr GDALWMSRasterBand::ZeroBlock(int x, int y, int to_buffer_band, void *buff
     for (int ib = 1; ib <= m_parent_dataset->nBands; ++ib) {
         if (ret == CE_None) {
             void *p = NULL;
+            void **hThisMutex = NULL;
             GDALRasterBlock *b = NULL;
             if ((buffer != NULL) && (ib == to_buffer_band)) {
                 p = buffer;
+                hThisMutex = NULL;
             } else {
                 GDALWMSRasterBand *band = static_cast<GDALWMSRasterBand *>(m_parent_dataset->GetRasterBand(ib));
+                hThisMutex = band->GetRWMutex();
                 if (m_overview >= 0) band = static_cast<GDALWMSRasterBand *>(band->GetOverview(m_overview));
                 if (!band->IsBlockInCache(x, y)) {
                     b = band->GetLockedBlockRef(x, y, true);
@@ -742,6 +751,7 @@ CPLErr GDALWMSRasterBand::ZeroBlock(int x, int y, int to_buffer_band, void *buff
                 }
             }
             if (p != NULL) {
+                CPLMutexHolderD( hThisMutex );
                 unsigned char *b = reinterpret_cast<unsigned char *>(p);
                 int block_size = nBlockXSize * nBlockYSize * (GDALGetDataTypeSize(eDataType) / 8);
                 for (int i = 0; i < block_size; ++i) b[i] = 0;

gdal/gcore/GNUmakefile

@@ -10,7 +10,8 @@ OBJ	=	gdalopeninfo.o gdaldrivermanager.o gdaldriver.o gdaldataset.o \
 		gdalallvalidmaskband.o gdalnodatamaskband.o gdal_rpcimdio.o \
  		gdalproxydataset.o gdalproxypool.o gdaldefaultasync.o \
 		gdalnodatavaluesmaskband.o gdaldllmain.o gdalexif.o gdalclientserver.o \
-		gdalgeorefpamdataset.o gdaljp2abstractdataset.o gdalvirtualmem.o
+		gdalgeorefpamdataset.o gdaljp2abstractdataset.o gdalvirtualmem.o \
+		gdalrasterblockmanager.o
 
 # Enable the following if you want to use MITAB's code to convert
 # .tab coordinate systems into well known text.  But beware that linking