From 888adf5e0458dee69043c00fe0265478d90721d1 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 12 Sep 2020 13:19:35 +0200 Subject: [PATCH 001/255] VRTDataset::IRasterIO(): allow source overviews to be used when non-nearest resampling is used, and the VRT bands don't expose overviews (fixes #2911) --- autotest/gcore/vrt_read.py | 53 +++++++++++++++++++++++++++++++++++ gdal/frmts/vrt/vrtdataset.cpp | 21 +++++++++++++- 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/autotest/gcore/vrt_read.py b/autotest/gcore/vrt_read.py index 0815270575ec..f11d1790b9a1 100755 --- a/autotest/gcore/vrt_read.py +++ b/autotest/gcore/vrt_read.py @@ -1410,3 +1410,56 @@ def test_vrt_dataset_rasterio_recursion_detection_does_not_trigger(): ds = gdal.Open('data/rgbsmall.tif') ref_data = ds.ReadRaster(0,0,50,50,25,25,resample_alg=gdal.GRIORA_Cubic) assert got_data == ref_data + + +def test_vrt_dataset_rasterio_non_nearest_resampling_source_with_ovr(): + + ds = gdal.GetDriverByName('GTiff').Create('/vsimem/src.tif', 10, 10, 3) + ds.GetRasterBand(1).Fill(255) + ds.BuildOverviews('NONE', [2]) + ds.GetRasterBand(1).GetOverview(0).Fill(10) + ds = None + + vrt_text = """ + + Red + + + /vsimem/src.tif + 1 + + + + + /vsimem/src.tif + 1 + + + + + + Green + + /vsimem/src.tif + 2 + + + + Blue + + /vsimem/src.tif + 3 + + +""" + ds = gdal.Open(vrt_text) + + got_data = ds.ReadRaster(0,0,10,10,4,4) + got_data = struct.unpack('B' * 4 * 4 * 3, got_data) + assert got_data[0] == 10 + + got_data = ds.ReadRaster(0,0,10,10,4,4,resample_alg=gdal.GRIORA_Cubic) + got_data = struct.unpack('B' * 4 * 4 * 3, got_data) + assert got_data[0] == 10 + + gdal.Unlink('/vsimem/src.tif') diff --git a/gdal/frmts/vrt/vrtdataset.cpp b/gdal/frmts/vrt/vrtdataset.cpp index bd38574b54f7..661e03df5049 100644 --- a/gdal/frmts/vrt/vrtdataset.cpp +++ b/gdal/frmts/vrt/vrtdataset.cpp @@ -1927,12 +1927,31 @@ CPLErr VRTDataset::IRasterIO( GDALRWFlag eRWFlag, return eErr; } - CPLErr eErr = GDALDataset::IRasterIO( eRWFlag, nXOff, nYOff, nXSize, nYSize, + CPLErr eErr; + if( eRWFlag == GF_Read && + psExtraArg->eResampleAlg != GRIORA_NearestNeighbour && + nBufXSize < nXSize && nBufYSize < nYSize && nBandCount > 1 ) + { + // Force going through VRTSourcedRasterBand::IRasterIO(), otherwise + // GDALDataset::IRasterIOResampled() would be used without source + // overviews being potentially used. + eErr = GDALDataset::BandBasedRasterIO( + eRWFlag, nXOff, nYOff, nXSize, nYSize, pData, nBufXSize, nBufYSize, eBufType, nBandCount, panBandMap, nPixelSpace, nLineSpace, nBandSpace, psExtraArg ); + } + else + { + eErr = GDALDataset::IRasterIO( eRWFlag, nXOff, nYOff, nXSize, nYSize, + pData, nBufXSize, nBufYSize, + eBufType, + nBandCount, panBandMap, + nPixelSpace, nLineSpace, nBandSpace, + psExtraArg ); + } m_nRecursionCounter --; return eErr; } From 1b6f78424da1784fcf6fcdabe3228600a0688eb2 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 12 Sep 2020 13:24:52 +0200 Subject: [PATCH 002/255] MEM: MEMDataset::IRasterIO(): make it fallback to GDALDataset::BandBasedRasterIO() to limit code duplication --- gdal/frmts/mem/memdataset.cpp | 44 ++++------------------------------- 1 file changed, 5 insertions(+), 39 deletions(-) diff --git a/gdal/frmts/mem/memdataset.cpp b/gdal/frmts/mem/memdataset.cpp index 4d01a63ea00b..754efed2c9ed 100644 --- a/gdal/frmts/mem/memdataset.cpp +++ b/gdal/frmts/mem/memdataset.cpp @@ -398,45 +398,11 @@ CPLErr MEMDataset::IRasterIO( GDALRWFlag eRWFlag, nPixelSpaceBuf, nLineSpaceBuf, nBandSpaceBuf, psExtraArg ); - GDALProgressFunc pfnProgressGlobal = psExtraArg->pfnProgress; - void *pProgressDataGlobal = psExtraArg->pProgressData; - - CPLErr eErr = CE_None; - for( int iBandIndex = 0; - iBandIndex < nBandCount && eErr == CE_None; - iBandIndex++ ) - { - GDALRasterBand *poBand = GetRasterBand(panBandMap[iBandIndex]); - - if (poBand == nullptr) - { - eErr = CE_Failure; - break; - } - - GByte *pabyBandData - = reinterpret_cast(pData) + iBandIndex * nBandSpaceBuf; - - psExtraArg->pfnProgress = GDALScaledProgress; - psExtraArg->pProgressData = - GDALCreateScaledProgress( 1.0 * iBandIndex / nBandCount, - 1.0 * (iBandIndex + 1) / nBandCount, - pfnProgressGlobal, - pProgressDataGlobal ); - - eErr = poBand->RasterIO( eRWFlag, nXOff, nYOff, nXSize, nYSize, - reinterpret_cast( pabyBandData ), - nBufXSize, nBufYSize, - eBufType, nPixelSpaceBuf, nLineSpaceBuf, - psExtraArg ); - - GDALDestroyScaledProgress( psExtraArg->pProgressData ); - } - - psExtraArg->pfnProgress = pfnProgressGlobal; - psExtraArg->pProgressData = pProgressDataGlobal; - - return eErr; + return GDALDataset::BandBasedRasterIO( eRWFlag, nXOff, nYOff, nXSize, nYSize, + pData, nBufXSize, nBufYSize, + eBufType, nBandCount, panBandMap, + nPixelSpaceBuf, nLineSpaceBuf, nBandSpaceBuf, + psExtraArg ); } /************************************************************************/ From 13e9e7585fc85fe4946447657e74f391e736ac4e Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 12 Sep 2020 13:25:30 +0200 Subject: [PATCH 003/255] GDALDataset::IRasterIO(): make it try overviews when non-nearest resampling is done before doing RasterIOResampled() on full resolution dataset --- gdal/gcore/gdaldataset.cpp | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/gdal/gcore/gdaldataset.cpp b/gdal/gcore/gdaldataset.cpp index 0d046188155f..0941b3f1d0e2 100644 --- a/gdal/gcore/gdaldataset.cpp +++ b/gdal/gcore/gdaldataset.cpp @@ -2032,7 +2032,6 @@ CPLErr GDALDataset::IRasterIO( GDALRWFlag eRWFlag, GDALRasterIOExtraArg* psExtraArg ) { - const char *pszInterleave = nullptr; CPLAssert(nullptr != pData); @@ -2061,6 +2060,23 @@ CPLErr GDALDataset::IRasterIO( GDALRWFlag eRWFlag, psExtraArg->eResampleAlg == GRIORA_Lanczos) && !(nXSize == nBufXSize && nYSize == nBufYSize) && nBandCount > 1 ) { + if( nBufXSize < nXSize && nBufYSize < nYSize ) + { + int bTried = FALSE; + const CPLErr eErr = + TryOverviewRasterIO( eRWFlag, + nXOff, nYOff, nXSize, nYSize, + pData, nBufXSize, nBufYSize, + eBufType, + nBandCount, panBandMap, + nPixelSpace, nLineSpace, + nBandSpace, + psExtraArg, + &bTried ); + if( bTried ) + return eErr; + } + GDALDataType eFirstBandDT = GDT_Unknown; int nFirstMaskFlags = 0; GDALRasterBand *poFirstMaskBand = nullptr; From b627cf4cda1170c4a1a29609b0531334bb0c55ea Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 12 Sep 2020 13:25:59 +0200 Subject: [PATCH 004/255] mem.py: test GDALDataset::IRasterIO() with non-nearest resampling --- autotest/gdrivers/mem.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/autotest/gdrivers/mem.py b/autotest/gdrivers/mem.py index 1768c0a3e5b6..bf13864bc96e 100755 --- a/autotest/gdrivers/mem.py +++ b/autotest/gdrivers/mem.py @@ -608,6 +608,24 @@ def test_mem_colortable(): assert ds.GetRasterBand(1).GetColorTable() is None +############################################################################### +# Test dataset RasterIO with non nearest resampling + +def test_mem_dataset_rasterio_non_nearest_resampling_source_with_ovr(): + + ds = gdal.GetDriverByName('MEM').Create('', 10, 10, 3) + ds.GetRasterBand(1).Fill(255) + ds.BuildOverviews('NONE', [2]) + ds.GetRasterBand(1).GetOverview(0).Fill(10) + + got_data = ds.ReadRaster(0,0,10,10,5,5) + got_data = struct.unpack('B' * 5 * 5 * 3, got_data) + assert got_data[0] == 10 + + got_data = ds.ReadRaster(0,0,10,10,5,5,resample_alg=gdal.GRIORA_Cubic) + got_data = struct.unpack('B' * 5 * 5 * 3, got_data) + assert got_data[0] == 10 + ############################################################################### # cleanup From b136d1886491c3b291622ec58cb97ac64d88fdf7 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 12 Sep 2020 15:51:39 +0200 Subject: [PATCH 005/255] VRT: fix VRTRasterBand nodata handling when creating implicit overviews (fixes #2920) --- autotest/gcore/vrt_read.py | 59 +++++++++++++++++++++++++++++++++++ gdal/frmts/vrt/vrtdataset.cpp | 24 +++++++++----- gdal/frmts/vrt/vrtdataset.h | 2 ++ 3 files changed, 77 insertions(+), 8 deletions(-) diff --git a/autotest/gcore/vrt_read.py b/autotest/gcore/vrt_read.py index f11d1790b9a1..add478388443 100755 --- a/autotest/gcore/vrt_read.py +++ b/autotest/gcore/vrt_read.py @@ -1463,3 +1463,62 @@ def test_vrt_dataset_rasterio_non_nearest_resampling_source_with_ovr(): assert got_data[0] == 10 gdal.Unlink('/vsimem/src.tif') + + +def test_vrt_implicit_ovr_with_hidenodatavalue(): + + ds = gdal.GetDriverByName('GTiff').Create('/vsimem/src.tif', 256, 256, 3) + ds.GetRasterBand(1).Fill(255) + ds.BuildOverviews('NONE', [2]) + ds.GetRasterBand(1).GetOverview(0).Fill(10) + ds = None + + vrt_text = """ + + Red + 5 + 1 + + /vsimem/src.tif + 1 + + + + + + Green + 5 + 1 + + /vsimem/src.tif + 2 + + + + + + Blue + 5 + 1 + + /vsimem/src.tif + 3 + + + + +""" + ds = gdal.Open(vrt_text) + assert ds.GetRasterBand(1).GetOverviewCount() == 1 + + got_data = ds.ReadRaster(0,0,256,256,64,64) + got_data = struct.unpack('B' * 64 * 64 * 3, got_data) + assert got_data[0] == 5 + assert got_data[32*64+32] == 10 + + got_data = ds.GetRasterBand(1).ReadRaster(0,0,256,256,64,64) + got_data = struct.unpack('B' * 64 * 64, got_data) + assert got_data[0] == 5 + assert got_data[32*64+32] == 10 + + gdal.Unlink('/vsimem/src.tif') diff --git a/gdal/frmts/vrt/vrtdataset.cpp b/gdal/frmts/vrt/vrtdataset.cpp index 661e03df5049..68785f774d05 100644 --- a/gdal/frmts/vrt/vrtdataset.cpp +++ b/gdal/frmts/vrt/vrtdataset.cpp @@ -2080,15 +2080,17 @@ void VRTDataset::BuildVirtualOverviews() const auto CreateOverviewBand = [&poOvrVDS, nOvrXSize, nOvrYSize, dfXRatio, dfYRatio] - (GDALRasterBand* poBand) + (VRTSourcedRasterBand* poVRTBand) { - VRTSourcedRasterBand* poVRTBand - = cpl::down_cast(poBand); VRTSourcedRasterBand* poOvrVRTBand = new VRTSourcedRasterBand( poOvrVDS, - poBand->GetBand(), + poVRTBand->GetBand(), poVRTBand->GetRasterDataType(), nOvrXSize, nOvrYSize); + poOvrVRTBand->CopyCommonInfoFrom(poVRTBand); + poOvrVRTBand->m_bNoDataValueSet = poVRTBand->m_bNoDataValueSet; + poOvrVRTBand->m_dfNoDataValue = poVRTBand->m_dfNoDataValue; + poOvrVRTBand->m_bHideNoDataValue = poVRTBand->m_bHideNoDataValue; VRTSimpleSource* poSrcSource = cpl::down_cast( poVRTBand->papoSources[0] ); @@ -2110,7 +2112,7 @@ void VRTDataset::BuildVirtualOverviews() } if( poNewSource ) { - auto poNewSourceBand = poBand->GetBand() == 0 ? + auto poNewSourceBand = poVRTBand->GetBand() == 0 ? poNewSource->GetMaskBandMainBand() : poNewSource->GetBand(); CPLAssert(poNewSourceBand); @@ -2125,13 +2127,19 @@ void VRTDataset::BuildVirtualOverviews() for( int i = 0; i < nBands; i++ ) { - poOvrVDS->SetBand( poOvrVDS->GetRasterCount() + 1, - CreateOverviewBand(GetRasterBand(i+1)) ); + VRTSourcedRasterBand* poSrcBand + = cpl::down_cast(GetRasterBand(i+1)); + auto poOvrVRTBand = CreateOverviewBand(poSrcBand); + poOvrVDS->SetBand( poOvrVDS->GetRasterCount() + 1, poOvrVRTBand ); + } if( m_poMaskBand ) { - poOvrVDS->SetMaskBand( CreateOverviewBand(m_poMaskBand) ); + VRTSourcedRasterBand* poSrcBand + = cpl::down_cast(m_poMaskBand); + auto poOvrVRTBand = CreateOverviewBand(poSrcBand); + poOvrVDS->SetMaskBand(poOvrVRTBand); } } } diff --git a/gdal/frmts/vrt/vrtdataset.h b/gdal/frmts/vrt/vrtdataset.h index 3c4e49a8dd2e..e0490184e9c1 100644 --- a/gdal/frmts/vrt/vrtdataset.h +++ b/gdal/frmts/vrt/vrtdataset.h @@ -451,6 +451,8 @@ class VRTPansharpenedDataset final: public VRTDataset class CPL_DLL VRTRasterBand CPL_NON_FINAL: public GDALRasterBand { protected: + friend class VRTDataset; + int m_bIsMaskBand; int m_bNoDataValueSet; From adf0cd29fb7e479c33474ed88072945ad8c8cc69 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 12 Sep 2020 20:10:58 +0200 Subject: [PATCH 006/255] Fix language in Doxygen doc [ci skip] --- gdal/apps/gdal_grid_lib.cpp | 2 +- gdal/apps/gdal_rasterize_lib.cpp | 2 +- gdal/apps/gdal_translate_lib.cpp | 2 +- gdal/apps/gdalbuildvrt_lib.cpp | 2 +- gdal/apps/gdaldem_lib.cpp | 2 +- gdal/apps/gdalmdimtranslate_lib.cpp | 2 +- gdal/apps/gdalwarp_lib.cpp | 2 +- gdal/apps/nearblack_lib.cpp | 2 +- gdal/apps/ogr2ogr_lib.cpp | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/gdal/apps/gdal_grid_lib.cpp b/gdal/apps/gdal_grid_lib.cpp index 5f0d076762ff..76f20ee274e3 100644 --- a/gdal/apps/gdal_grid_lib.cpp +++ b/gdal/apps/gdal_grid_lib.cpp @@ -724,7 +724,7 @@ static OGRGeometryCollection* LoadGeometry( const char* pszDS, * @param pszDest the destination dataset path. * @param hSrcDataset the source dataset handle. * @param psOptionsIn the options struct returned by GDALGridOptionsNew() or NULL. - * @param pbUsageError the pointer to int variable to determine any usage error has occurred or NULL. + * @param pbUsageError pointer to a integer output variable to store if any usage error has occurred or NULL. * @return the output dataset (new dataset that must be closed using GDALClose()) or NULL in case of error. * * @since GDAL 2.1 diff --git a/gdal/apps/gdal_rasterize_lib.cpp b/gdal/apps/gdal_rasterize_lib.cpp index d696ad063518..717d3ac0d1f6 100644 --- a/gdal/apps/gdal_rasterize_lib.cpp +++ b/gdal/apps/gdal_rasterize_lib.cpp @@ -567,7 +567,7 @@ struct GDALRasterizeOptions * @param hDstDS the destination dataset or NULL. * @param hSrcDataset the source dataset handle. * @param psOptionsIn the options struct returned by GDALRasterizeOptionsNew() or NULL. - * @param pbUsageError the pointer to int variable to determine any usage error has occurred or NULL. + * @param pbUsageError pointer to a integer output variable to store if any usage error has occurred or NULL. * @return the output dataset (new dataset that must be closed using GDALClose(), or hDstDS is not NULL) or NULL in case of error. * * @since GDAL 2.1 diff --git a/gdal/apps/gdal_translate_lib.cpp b/gdal/apps/gdal_translate_lib.cpp index 7e6f43a658e5..434fef8b73b7 100644 --- a/gdal/apps/gdal_translate_lib.cpp +++ b/gdal/apps/gdal_translate_lib.cpp @@ -664,7 +664,7 @@ static double AdjustNoDataValue( double dfInputNoDataValue, * @param pszDest the destination dataset path. * @param hSrcDataset the source dataset handle. * @param psOptionsIn the options struct returned by GDALTranslateOptionsNew() or NULL. - * @param pbUsageError the pointer to int variable to determine any usage error has occurred or NULL. + * @param pbUsageError pointer to a integer output variable to store if any usage error has occurred or NULL. * @return the output dataset (new dataset that must be closed using GDALClose()) or NULL in case of error. * * @since GDAL 2.1 diff --git a/gdal/apps/gdalbuildvrt_lib.cpp b/gdal/apps/gdalbuildvrt_lib.cpp index 6c6bd622cbe1..fdfb1d8dd48a 100644 --- a/gdal/apps/gdalbuildvrt_lib.cpp +++ b/gdal/apps/gdalbuildvrt_lib.cpp @@ -1552,7 +1552,7 @@ GDALBuildVRTOptions* GDALBuildVRTOptionsClone(const GDALBuildVRTOptions *psOptio * @param pahSrcDS the list of input datasets (or NULL, exclusive with papszSrcDSNames) * @param papszSrcDSNames the list of input dataset names (or NULL, exclusive with pahSrcDS) * @param psOptionsIn the options struct returned by GDALBuildVRTOptionsNew() or NULL. - * @param pbUsageError the pointer to int variable to determine any usage error has occurred. + * @param pbUsageError pointer to a integer output variable to store if any usage error has occurred. * @return the output dataset (new dataset that must be closed using GDALClose()) or NULL in case of error. * * @since GDAL 2.1 diff --git a/gdal/apps/gdaldem_lib.cpp b/gdal/apps/gdaldem_lib.cpp index 74936543e798..e8db0491e688 100644 --- a/gdal/apps/gdaldem_lib.cpp +++ b/gdal/apps/gdaldem_lib.cpp @@ -3330,7 +3330,7 @@ static Algorithm GetAlgorithm(const char* pszProcessing) * should be NULL otherwise) * @param psOptionsIn the options struct returned by * GDALDEMProcessingOptionsNew() or NULL. - * @param pbUsageError the pointer to int variable to determine any usage + * @param pbUsageError pointer to a integer output variable to store if any usage * error has occurred or NULL. * @return the output dataset (new dataset that must be closed using * GDALClose()) or NULL in case of error. diff --git a/gdal/apps/gdalmdimtranslate_lib.cpp b/gdal/apps/gdalmdimtranslate_lib.cpp index ab60890868b0..800bf49938c2 100644 --- a/gdal/apps/gdalmdimtranslate_lib.cpp +++ b/gdal/apps/gdalmdimtranslate_lib.cpp @@ -1629,7 +1629,7 @@ static GDALDatasetH CopyToNonMultiDimensionalDriver( * @param nSrcCount the number of input datasets. * @param pahSrcDS the list of input datasets. * @param psOptions the options struct returned by GDALMultiDimTranslateOptionsNew() or NULL. - * @param pbUsageError the pointer to int variable to determine any usage error has occurred or NULL. + * @param pbUsageError pointer to a integer output variable to store if any usage error has occurred or NULL. * @return the output dataset (new dataset that must be closed using GDALClose(), or hDstDS is not NULL) or NULL in case of error. * * @since GDAL 3.1 diff --git a/gdal/apps/gdalwarp_lib.cpp b/gdal/apps/gdalwarp_lib.cpp index c5b35709b278..a2767120e90f 100644 --- a/gdal/apps/gdalwarp_lib.cpp +++ b/gdal/apps/gdalwarp_lib.cpp @@ -1118,7 +1118,7 @@ GDALDatasetH GDALWarpIndirect( const char *pszDest, * @param nSrcCount the number of input datasets. * @param pahSrcDS the list of input datasets. * @param psOptionsIn the options struct returned by GDALWarpAppOptionsNew() or NULL. - * @param pbUsageError the pointer to int variable to determine any usage error has occurred. + * @param pbUsageError pointer to a integer output variable to store if any usage error has occurred, or NULL. * @return the output dataset (new dataset that must be closed using GDALClose(), or hDstDS if not NULL) or NULL in case of error. * * @since GDAL 2.1 diff --git a/gdal/apps/nearblack_lib.cpp b/gdal/apps/nearblack_lib.cpp index 7e08d0247352..73e5e7e3c9c7 100644 --- a/gdal/apps/nearblack_lib.cpp +++ b/gdal/apps/nearblack_lib.cpp @@ -99,7 +99,7 @@ static void ProcessLine( GByte *pabyLine, GByte *pabyMask, int iStart, * @param hDstDS the destination dataset or NULL. Might be equal to hSrcDataset. * @param hSrcDataset the source dataset handle. * @param psOptionsIn the options struct returned by GDALNearblackOptionsNew() or NULL. - * @param pbUsageError the pointer to int variable to determine any usage error has occurred or NULL. + * @param pbUsageError pointer to a integer output variable to store if any usage error has occurred or NULL. * @return the output dataset (new dataset that must be closed using GDALClose(), or hDstDS is not NULL) or NULL in case of error. * * @since GDAL 2.1 diff --git a/gdal/apps/ogr2ogr_lib.cpp b/gdal/apps/ogr2ogr_lib.cpp index 5b25e46034ee..5d51447ccaba 100644 --- a/gdal/apps/ogr2ogr_lib.cpp +++ b/gdal/apps/ogr2ogr_lib.cpp @@ -1978,7 +1978,7 @@ static GDALDataset* GDALVectorTranslateCreateCopy( * @param nSrcCount the number of input datasets (only 1 supported currently) * @param pahSrcDS the list of input datasets. * @param psOptionsIn the options struct returned by GDALVectorTranslateOptionsNew() or NULL. - * @param pbUsageError the pointer to int variable to determine any usage error has occurred + * @param pbUsageError pointer to a integer output variable to store if any usage error has occurred, or NULL. * @return the output dataset (new dataset that must be closed using GDALClose(), or hDstDS is not NULL) or NULL in case of error. * * @since GDAL 2.1 From 255f08f93b70e23643aca62af61448f9295c67a8 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 13 Sep 2020 00:01:20 +0200 Subject: [PATCH 007/255] ODS and XLSX: do not create files with Zip64 extension, to avoid compatibility issue with LibreOffice Calc --- gdal/ogr/ogrsf_frmts/ods/ogrodsdatasource.cpp | 2 ++ gdal/ogr/ogrsf_frmts/xlsx/ogrxlsxdatasource.cpp | 2 ++ 2 files changed, 4 insertions(+) diff --git a/gdal/ogr/ogrsf_frmts/ods/ogrodsdatasource.cpp b/gdal/ogr/ogrsf_frmts/ods/ogrodsdatasource.cpp index 8a5dfc0bc3d1..2e8f0175cd13 100644 --- a/gdal/ogr/ogrsf_frmts/ods/ogrodsdatasource.cpp +++ b/gdal/ogr/ogrsf_frmts/ods/ogrodsdatasource.cpp @@ -1821,6 +1821,8 @@ void OGRODSDataSource::FlushCache() } } + CPLConfigOptionSetter oZip64Disable("CPL_CREATE_ZIP64", "NO", false); + /* Maintain new ZIP files opened */ void *hZIP = CPLCreateZip(pszName, nullptr); if (hZIP == nullptr) diff --git a/gdal/ogr/ogrsf_frmts/xlsx/ogrxlsxdatasource.cpp b/gdal/ogr/ogrsf_frmts/xlsx/ogrxlsxdatasource.cpp index bddbe2cb5b5f..019f513b20fc 100644 --- a/gdal/ogr/ogrsf_frmts/xlsx/ogrxlsxdatasource.cpp +++ b/gdal/ogr/ogrsf_frmts/xlsx/ogrxlsxdatasource.cpp @@ -2297,6 +2297,8 @@ void OGRXLSXDataSource::FlushCache() } } + CPLConfigOptionSetter oZip64Disable("CPL_CREATE_ZIP64", "NO", false); + /* Maintain new ZIP files opened */ CPLString osTmpFilename(CPLSPrintf("/vsizip/%s", pszName)); VSILFILE* fpZIP = VSIFOpenExL(osTmpFilename, "wb", true); From 6f9a4c07145332e315babb3752423e9af1f18bb6 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 13 Sep 2020 13:48:25 +0200 Subject: [PATCH 008/255] /vsis3/: additional retry on failures (patch by Robin Princeley) --- gdal/port/cpl_vsil_s3.cpp | 94 +++++++++++++++++++++++++++++++-------- 1 file changed, 75 insertions(+), 19 deletions(-) diff --git a/gdal/port/cpl_vsil_s3.cpp b/gdal/port/cpl_vsil_s3.cpp index 690c396515e9..0ffc0d2e6110 100644 --- a/gdal/port/cpl_vsil_s3.cpp +++ b/gdal/port/cpl_vsil_s3.cpp @@ -936,6 +936,14 @@ CPLString IVSIS3LikeFSHandler::UploadPart(const CPLString& osFilename, nRetryCount++; bRetry = true; } + else if( requestHelper.sWriteFuncData.pBuffer != nullptr && + poS3HandleHelper->CanRestartOnError(requestHelper.sWriteFuncData.pBuffer, + requestHelper.sWriteFuncHeaderData.pBuffer, + false) ) + { + UpdateMapFromHandle(poS3HandleHelper); + bRetry = true; + } else { CPLDebug(GetDebugKey(), "%s", @@ -1014,6 +1022,7 @@ VSIS3WriteHandle::WriteChunked( const void *pBuffer, size_t nSize, size_t nMemb m_hCurlMulti = curl_multi_init(); } + WriteFuncStruct sWriteFuncData; double dfRetryDelay = m_dfRetryDelay; int nRetryCount = 0; // We can only easily retry at the first chunk of a transfer @@ -1031,6 +1040,11 @@ VSIS3WriteHandle::WriteChunked( const void *pBuffer, size_t nSize, size_t nMemb ReadCallBackBufferChunked); curl_easy_setopt(hCurlHandle, CURLOPT_READDATA, this); + VSICURLInitWriteFuncStruct(&sWriteFuncData, nullptr, nullptr, nullptr); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, + VSICurlHandleWriteFunc); + VSICURLInitWriteFuncStruct(&m_sWriteFuncHeaderData, nullptr, nullptr, nullptr); curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, &m_sWriteFuncHeaderData); curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, @@ -1092,13 +1106,6 @@ VSIS3WriteHandle::WriteChunked( const void *pBuffer, size_t nSize, size_t nMemb m_sWriteFuncHeaderData.pBuffer, m_osCurlErrBuf.c_str()) : 0.0; - curl_multi_remove_handle(m_hCurlMulti, m_hCurl); - curl_easy_cleanup(m_hCurl); - m_hCurl = nullptr; - - CPLFree(m_sWriteFuncHeaderData.pBuffer); - m_sWriteFuncHeaderData.pBuffer = nullptr; - if( dfNewRetryDelay > 0 && nRetryCount < m_nMaxRetry ) { @@ -1113,6 +1120,14 @@ VSIS3WriteHandle::WriteChunked( const void *pBuffer, size_t nSize, size_t nMemb nRetryCount++; bRetry = true; } + else if( sWriteFuncData.pBuffer != nullptr && + m_poS3HandleHelper->CanRestartOnError(sWriteFuncData.pBuffer, + m_sWriteFuncHeaderData.pBuffer, + false) ) + { + m_poFS->UpdateMapFromHandle(m_poS3HandleHelper); + bRetry = true; + } else { CPLError(CE_Failure, CPLE_AppDefined, @@ -1121,8 +1136,20 @@ VSIS3WriteHandle::WriteChunked( const void *pBuffer, size_t nSize, size_t nMemb m_osCurlErrBuf.c_str()); curl_slist_free_all(headers); - return 0; + bRetry = false; } + + curl_multi_remove_handle(m_hCurlMulti, m_hCurl); + curl_easy_cleanup(m_hCurl); + + CPLFree(sWriteFuncData.pBuffer); + CPLFree(m_sWriteFuncHeaderData.pBuffer); + + m_hCurl = nullptr; + sWriteFuncData.pBuffer = nullptr; + m_sWriteFuncHeaderData.pBuffer = nullptr; + if( !bRetry ) + return 0; } } } @@ -1148,12 +1175,6 @@ VSIS3WriteHandle::WriteChunked( const void *pBuffer, size_t nSize, size_t nMemb bCanRetry ? CPLHTTPGetNewRetryDelay( static_cast(response_code), dfRetryDelay, m_sWriteFuncHeaderData.pBuffer, m_osCurlErrBuf.c_str()) : 0.0; - curl_multi_remove_handle(m_hCurlMulti, m_hCurl); - curl_easy_cleanup(m_hCurl); - m_hCurl = nullptr; - - CPLFree(m_sWriteFuncHeaderData.pBuffer); - m_sWriteFuncHeaderData.pBuffer = nullptr; if( dfNewRetryDelay > 0 && nRetryCount < m_nMaxRetry ) @@ -1169,14 +1190,33 @@ VSIS3WriteHandle::WriteChunked( const void *pBuffer, size_t nSize, size_t nMemb nRetryCount++; bRetry = true; } + else if( sWriteFuncData.pBuffer != nullptr && + m_poS3HandleHelper->CanRestartOnError(sWriteFuncData.pBuffer, + m_sWriteFuncHeaderData.pBuffer, + false) ) + { + m_poFS->UpdateMapFromHandle(m_poS3HandleHelper); + bRetry = true; + } else { CPLError(CE_Failure, CPLE_AppDefined, "Error %d: %s", static_cast(response_code), m_osCurlErrBuf.c_str()); - return 0; + bRetry = false; + nMemb = 0; } + + curl_multi_remove_handle(m_hCurlMulti, m_hCurl); + curl_easy_cleanup(m_hCurl); + + CPLFree(sWriteFuncData.pBuffer); + CPLFree(m_sWriteFuncHeaderData.pBuffer); + + m_hCurl = nullptr; + sWriteFuncData.pBuffer = nullptr; + m_sWriteFuncHeaderData.pBuffer = nullptr; } } } @@ -1505,7 +1545,15 @@ bool IVSIS3LikeFSHandler::CompleteMultipart(const CPLString& osFilename, nRetryCount++; bRetry = true; } - else + else if( requestHelper.sWriteFuncData.pBuffer != nullptr && + poS3HandleHelper->CanRestartOnError(requestHelper.sWriteFuncData.pBuffer, + requestHelper.sWriteFuncHeaderData.pBuffer, + false) ) + { + UpdateMapFromHandle(poS3HandleHelper); + bRetry = true; + } + else { CPLDebug("S3", "%s", requestHelper.sWriteFuncData.pBuffer ? requestHelper.sWriteFuncData.pBuffer : "(null)"); @@ -1581,7 +1629,15 @@ bool IVSIS3LikeFSHandler::AbortMultipart(const CPLString& osFilename, nRetryCount++; bRetry = true; } - else + else if( requestHelper.sWriteFuncData.pBuffer != nullptr && + poS3HandleHelper->CanRestartOnError(requestHelper.sWriteFuncData.pBuffer, + requestHelper.sWriteFuncHeaderData.pBuffer, + false) ) + { + UpdateMapFromHandle(poS3HandleHelper); + bRetry = true; + } + else { CPLDebug("S3", "%s", requestHelper.sWriteFuncData.pBuffer ? requestHelper.sWriteFuncData.pBuffer : "(null)"); @@ -3812,7 +3868,7 @@ bool IVSIS3LikeFSHandler::Sync( const char* pszSource, const char* pszTarget, if( VSI_ISDIR(sTarget.st_mode) ) { osTarget = CPLFormFilename(osTarget, CPLGetFilename(pszSource), nullptr); - bTargetIsFile = VSIStatL(osTarget, &sTarget) == 0 && + bTargetIsFile = VSIStatL(osTarget, &sTarget) == 0 && !CPL_TO_BOOL(VSI_ISDIR(sTarget.st_mode)); } } @@ -3914,7 +3970,7 @@ bool IVSIS3LikeFSHandler::Sync( const char* pszSource, const char* pszTarget, if( poS3HandleHelper == nullptr ) return false; UpdateHandleFromMap(poS3HandleHelper.get()); - const auto osUploadID = + const auto osUploadID = InitiateMultipartUpload(osTarget, poS3HandleHelper.get(), nMaxRetry, From b78518d96d41e766136cdbfa95f9d03994d7969e Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 13 Sep 2020 15:44:23 +0200 Subject: [PATCH 009/255] ogrlvbaglayer.cpp: make cppcheck happy [ci skip] --- gdal/ogr/ogrsf_frmts/lvbag/ogrlvbaglayer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gdal/ogr/ogrsf_frmts/lvbag/ogrlvbaglayer.cpp b/gdal/ogr/ogrsf_frmts/lvbag/ogrlvbaglayer.cpp index 66b86488849b..1f4340cfb487 100644 --- a/gdal/ogr/ogrsf_frmts/lvbag/ogrlvbaglayer.cpp +++ b/gdal/ogr/ogrsf_frmts/lvbag/ogrlvbaglayer.cpp @@ -139,7 +139,7 @@ static inline const char* XMLTagSplit( const char *pszName ) void OGRLVBAGLayer::AddSpatialRef( OGRwkbGeometryType eTypeIn ) { OGRGeomFieldDefn *poGeomField = poFeatureDefn->GetGeomFieldDefn(0); - OGRSpatialReference* poSRS = new OGRSpatialReference{}; + OGRSpatialReference* poSRS = new OGRSpatialReference(); poSRS->importFromURN(pszSpecificationUrn); poGeomField->SetSpatialRef(poSRS); poGeomField->SetType(eTypeIn); @@ -466,7 +466,7 @@ void OGRLVBAGLayer::StartElementCbk( const char *pszName, const char **ppszAttr OGRGeomFieldDefn *poGeomField = poFeatureDefn->GetGeomFieldDefn(0); if( EQUAL("srsname", papszIter[0]) && poGeomField->GetSpatialRef() == nullptr ) { - OGRSpatialReference* poSRS = new OGRSpatialReference{}; + OGRSpatialReference* poSRS = new OGRSpatialReference(); poSRS->importFromURN(papszIter[1]); poGeomField->SetSpatialRef(poSRS); poSRS->Release(); From 68527c50dbe6401f394c8b4587f1efcfde2389ae Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 13 Sep 2020 15:46:09 +0200 Subject: [PATCH 010/255] ogrfeature.cpp: make cppcheck happy --- gdal/ogr/ogrfeature.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gdal/ogr/ogrfeature.cpp b/gdal/ogr/ogrfeature.cpp index 7c1429014f5d..d94ec50d81cb 100644 --- a/gdal/ogr/ogrfeature.cpp +++ b/gdal/ogr/ogrfeature.cpp @@ -3266,7 +3266,7 @@ char* OGRFeature::GetFieldAsSerializedJSon( int iField ) const if( eType == OFTStringList ) { json_object* poObj = json_object_new_array(); - auto&& papszValues = GetFieldAsStringList(iField); + char** papszValues = GetFieldAsStringList(iField); for( int i=0; papszValues[i] != nullptr; i++) { json_object_array_add( poObj, From 318adbf5e9a6f38700b465353decb691affb3a20 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 13 Sep 2020 15:47:50 +0200 Subject: [PATCH 011/255] ogrfielddefn.cpp: remove useless check --- gdal/ogr/ogrfielddefn.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gdal/ogr/ogrfielddefn.cpp b/gdal/ogr/ogrfielddefn.cpp index 64251b8a34c7..5bec375b18fd 100644 --- a/gdal/ogr/ogrfielddefn.cpp +++ b/gdal/ogr/ogrfielddefn.cpp @@ -1497,8 +1497,7 @@ void OGRUpdateFieldType( OGRFieldDefn* poFDefn, else if( eNewType == OFTInteger64List || eNewType == OFTRealList || eNewType == OFTStringList ) { - if( eNewType != OFTIntegerList ) - poFDefn->SetSubType(OFSTNone); + poFDefn->SetSubType(OFSTNone); poFDefn->SetType(eNewType); } else if( eNewType != OFTInteger && eNewType != OFTInteger64 ) From 887eef7c26a68a2d6bc530da4199974d2fae1365 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 13 Sep 2020 16:04:29 +0200 Subject: [PATCH 012/255] cpl_vsil_s3.cpp: make cppcheck happy --- gdal/port/cpl_vsil_s3.cpp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/gdal/port/cpl_vsil_s3.cpp b/gdal/port/cpl_vsil_s3.cpp index 0ffc0d2e6110..7796bb200890 100644 --- a/gdal/port/cpl_vsil_s3.cpp +++ b/gdal/port/cpl_vsil_s3.cpp @@ -436,13 +436,7 @@ bool VSIDIRS3::IssueListDir() NetworkStatisticsLogger::LogGET(requestHelper.sWriteFuncData.nSize); - if( requestHelper.sWriteFuncData.pBuffer == nullptr) - { - curl_easy_cleanup(hCurlHandle); - return false; - } - - if( response_code != 200 ) + if( response_code != 200 || requestHelper.sWriteFuncData.pBuffer == nullptr ) { bool bUpdateMap = true; if( requestHelper.sWriteFuncData.pBuffer != nullptr && From 518b025b4281f26277efa5221f326821eec8a8e2 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 13 Sep 2020 16:09:40 +0200 Subject: [PATCH 013/255] netcdfdataset.cpp: make cppcheck happy --- gdal/frmts/netcdf/netcdfdataset.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gdal/frmts/netcdf/netcdfdataset.cpp b/gdal/frmts/netcdf/netcdfdataset.cpp index 29c519104c57..f0de9b872303 100644 --- a/gdal/frmts/netcdf/netcdfdataset.cpp +++ b/gdal/frmts/netcdf/netcdfdataset.cpp @@ -8045,18 +8045,18 @@ static void CopyMetadata( GDALDataset* poSrcDS, char **papszFieldData = nullptr; // Remove the following band meta but set them later from band data. - const char *papszIgnoreBand[] = { CF_ADD_OFFSET, CF_SCALE_FACTOR, + const char * const papszIgnoreBand[] = { CF_ADD_OFFSET, CF_SCALE_FACTOR, "valid_range", "_Unsigned", _FillValue, "coordinates", nullptr }; - const char *papszIgnoreGlobal[] = { "NETCDF_DIM_EXTRA", nullptr }; + const char * const papszIgnoreGlobal[] = { "NETCDF_DIM_EXTRA", nullptr }; char **papszMetadata = nullptr; if( poSrcDS ) { papszMetadata = poSrcDS->GetMetadata(); } - else + else if( poSrcBand ) { papszMetadata = poSrcBand->GetMetadata(); } From 9814e085ef02d28b6253efa8d02c105f93cae9b7 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 13 Sep 2020 16:11:52 +0200 Subject: [PATCH 014/255] gdalwarpkernel.cpp: fix memleak in error code path. CID 1432694 --- gdal/alg/gdalwarpkernel.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gdal/alg/gdalwarpkernel.cpp b/gdal/alg/gdalwarpkernel.cpp index 26c191ba8d66..b5857953923a 100644 --- a/gdal/alg/gdalwarpkernel.cpp +++ b/gdal/alg/gdalwarpkernel.cpp @@ -342,6 +342,8 @@ void* GWKThreadsCreate( char** papszWarpOptions, psThreadData->poJobQueue = poThreadPool->CreateJobQueue(); psThreadData->pTransformerArgInput = pTransformerArg; } + else if( hCond ) + CPLDestroyCond(hCond); return psThreadData; } From 1f859e21fc357aade14433e7b1faef2ac9f0c259 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 13 Sep 2020 16:15:14 +0200 Subject: [PATCH 015/255] gdal_create: make Coverity happy. CID 1432686 --- gdal/apps/gdal_create.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/gdal/apps/gdal_create.cpp b/gdal/apps/gdal_create.cpp index fdb100c88f6e..07b6b6ec23c6 100644 --- a/gdal/apps/gdal_create.cpp +++ b/gdal/apps/gdal_create.cpp @@ -170,9 +170,13 @@ MAIN_START(argc, argv) else if( i+4 < argc && EQUAL(argv[i],"-a_ullr") ) { bGeoTransform = true; + // coverity[tainted_data] dfULX = CPLAtofM(argv[++i]); + // coverity[tainted_data] dfULY = CPLAtofM(argv[++i]); + // coverity[tainted_data] dfLRX = CPLAtofM(argv[++i]); + // coverity[tainted_data] dfLRY = CPLAtofM(argv[++i]); } else if( i < argc-1 && EQUAL(argv[i],"-a_srs") ) @@ -184,6 +188,7 @@ MAIN_START(argc, argv) { bSetNoData = true; ++i; + // coverity[tainted_data] dfNoDataValue = CPLAtofM(argv[i]); } @@ -200,9 +205,11 @@ MAIN_START(argc, argv) } else { + // coverity[tainted_data] while(i < argc-1 && ArgIsNumeric(argv[i+1])) { ++i; + // coverity[tainted_data] adfBurnValues.push_back(CPLAtof(argv[i])); } } From 5348c16e8af1e10312f96d256dd58e5834927f6d Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 13 Sep 2020 16:23:53 +0200 Subject: [PATCH 016/255] ogr2ogr_lib.cpp: avoid Coverity false positive. CID 1432684 --- gdal/apps/ogr2ogr_lib.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/gdal/apps/ogr2ogr_lib.cpp b/gdal/apps/ogr2ogr_lib.cpp index 5d51447ccaba..f130a86c64f9 100644 --- a/gdal/apps/ogr2ogr_lib.cpp +++ b/gdal/apps/ogr2ogr_lib.cpp @@ -4395,10 +4395,6 @@ static bool SetupCT( TargetLayerInfo* psInfo, return false; } poCT = new CompositeCT( poGCPCoordTrans, false, poCT, true ); - } - - if( poCT != psInfo->m_apoCT[iGeom].get() ) - { psInfo->m_apoCT[iGeom].reset(poCT); } } From e25abcd945ff5c11a4d7c7fa97cdf648a97a8681 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 13 Sep 2020 16:24:54 +0200 Subject: [PATCH 017/255] esric_dataset.cpp: silence Coverity warning. CID 1432676 --- gdal/frmts/esric/esric_dataset.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/gdal/frmts/esric/esric_dataset.cpp b/gdal/frmts/esric/esric_dataset.cpp index c17230cd06fb..c43ccea6c7ea 100644 --- a/gdal/frmts/esric/esric_dataset.cpp +++ b/gdal/frmts/esric/esric_dataset.cpp @@ -301,6 +301,7 @@ Bundle& ECDataset::GetBundle(const char* fname) { } } // No empties, eject one + // coverity[dont_call] Bundle& bundle = bundles[rand() % bundles.size()]; bundle.Init(fname); return bundle; From 806c52be89d281347440440e981922cc6758accd Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 13 Sep 2020 16:26:58 +0200 Subject: [PATCH 018/255] gt_citation.cpp: silence Coverity warning. CID 1432674 --- gdal/frmts/gtiff/gt_citation.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/gdal/frmts/gtiff/gt_citation.cpp b/gdal/frmts/gtiff/gt_citation.cpp index 3694dce701cb..f87c76c0c27c 100644 --- a/gdal/frmts/gtiff/gt_citation.cpp +++ b/gdal/frmts/gtiff/gt_citation.cpp @@ -493,8 +493,11 @@ OGRBoolean SetCitationToSRS( GTIF* hGTIF, char* szCTString, int nCTStringLen, } } if( unitSize == 0.0 ) - GDALGTIFKeyGetDOUBLE( hGTIF, ProjLinearUnitSizeGeoKey, - &unitSize, 0, 1 ); + { + CPL_IGNORE_RET_VAL( + GDALGTIFKeyGetDOUBLE( hGTIF, ProjLinearUnitSizeGeoKey, + &unitSize, 0, 1 )); + } poSRS->SetLinearUnits( ctNames[CitLUnitsName], unitSize); *linearUnitIsSet = TRUE; } From faff6f84417cceda98b06fd532fea6c45547fbf3 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 13 Sep 2020 16:29:43 +0200 Subject: [PATCH 019/255] Internal libgeotiff: resync with upstream (GTIFGetGCSInfoEx(): remove dead code (GDAL Coverity CID 1432690)) --- gdal/frmts/gtiff/libgeotiff/geo_normalize.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/gdal/frmts/gtiff/libgeotiff/geo_normalize.c b/gdal/frmts/gtiff/libgeotiff/geo_normalize.c index 1b09b5a6a4f9..0dbb9657a0ea 100644 --- a/gdal/frmts/gtiff/libgeotiff/geo_normalize.c +++ b/gdal/frmts/gtiff/libgeotiff/geo_normalize.c @@ -466,10 +466,6 @@ int GTIFGetGCSInfoEx( void* ctxIn, return TRUE; } - - if( nGCSCode == KvUserDefined ) - return FALSE; - /* -------------------------------------------------------------------- */ /* Search the database. */ /* -------------------------------------------------------------------- */ From 8a918ebcffe7788c6f1f9682414003d64e0b14d6 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 13 Sep 2020 16:32:06 +0200 Subject: [PATCH 020/255] BAG: make Coverity happy. CID 1432673 --- gdal/frmts/hdf5/bagdataset.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gdal/frmts/hdf5/bagdataset.cpp b/gdal/frmts/hdf5/bagdataset.cpp index ac33d614ff7d..56eb692e5371 100644 --- a/gdal/frmts/hdf5/bagdataset.cpp +++ b/gdal/frmts/hdf5/bagdataset.cpp @@ -1766,7 +1766,7 @@ CPLErr BAGGeorefMDSuperGridBand::IReadBlock( int nBlockXOff, int nBlockYOff, voi const GUInt64 arrayStartIdx[2] = { 0, poGDS->m_nSuperGridRefinementStartIndex + - (nRasterYSize - 1 - nBlockYOff) * nBlockXSize + static_cast(nRasterYSize - 1 - nBlockYOff) * nBlockXSize }; size_t count[2] = { 1, static_cast(nBlockXSize) }; const GInt64 arrayStep[2] = {1, 1}; From fab3246500a8fd0a11ed1f4ffe47c21dec1b8a2b Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 13 Sep 2020 16:32:52 +0200 Subject: [PATCH 021/255] KEA: fix memleak in error code path. CID 1400395 --- gdal/frmts/kea/keaband.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gdal/frmts/kea/keaband.cpp b/gdal/frmts/kea/keaband.cpp index 722be5ecb8ab..24ffaad55823 100644 --- a/gdal/frmts/kea/keaband.cpp +++ b/gdal/frmts/kea/keaband.cpp @@ -198,8 +198,11 @@ CPLErr KEARasterBand::SetHistogramFromString(const char *pszString) GDALRasterAttributeTable *pTable = this->GetDefaultRAT(); if( pTable == nullptr ) + { + CPLFree(pszBinValues); return CE_Failure; - + } + // find histogram column if it exists int nCol = pTable->GetColOfUsage(GFU_PixelCount); if( nCol == -1 ) From e657e887a65ae00e0999affe6b2426f88edec144 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 13 Sep 2020 16:40:57 +0200 Subject: [PATCH 022/255] TGA: try to silence CID 1432685 --- gdal/frmts/tga/tgadataset.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gdal/frmts/tga/tgadataset.cpp b/gdal/frmts/tga/tgadataset.cpp index 1c06087c7213..4dd03cf38fba 100644 --- a/gdal/frmts/tga/tgadataset.cpp +++ b/gdal/frmts/tga/tgadataset.cpp @@ -202,8 +202,9 @@ GDALTGARasterBand::GDALTGARasterBand(GDALTGADataset* poDSIn, int nBandIn, { VSIFSeekL(poDSIn->m_fpImage, 18 + poDSIn->m_sImageHeader.nIDLength, SEEK_SET); m_poColorTable.reset(new GDALColorTable()); - std::vector abyData(poDSIn->m_sImageHeader.nColorMapLength * - ((poDSIn->m_sImageHeader.nColorMapEntrySize + 7) / 8)); + const int nColorTableByteCount = poDSIn->m_sImageHeader.nColorMapLength * + ((poDSIn->m_sImageHeader.nColorMapEntrySize + 7) / 8); + std::vector abyData(nColorTableByteCount); VSIFReadL(&abyData[0], 1, abyData.size(), poDSIn->m_fpImage); if( poDSIn->m_sImageHeader.nColorMapEntrySize == 24 ) { From a722237e01939ecdf148073913e1b20701d18396 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 13 Sep 2020 16:44:05 +0200 Subject: [PATCH 023/255] gcore/overview.cpp: try to silence CID 1432680 --- gdal/gcore/overview.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gdal/gcore/overview.cpp b/gdal/gcore/overview.cpp index da06bcf08989..5e36db9a03bb 100644 --- a/gdal/gcore/overview.cpp +++ b/gdal/gcore/overview.cpp @@ -3125,7 +3125,10 @@ GDALRegenerateOverviews( GDALRasterBandH hSrcBand, // Only configurable for debug / testing const char* pszChunkYSize = CPLGetConfigOption("GDAL_OVR_CHUNKYSIZE", nullptr); if( pszChunkYSize ) + { + // coverity[tainted_data] nFullResYChunk = atoi(pszChunkYSize); + } const GDALDataType eSrcDataType = poSrcBand->GetRasterDataType(); const GDALDataType eWrkDataType = GDALDataTypeIsComplex( eSrcDataType )? From cc53f8b99ef06ce441731c96ad36227a6c6eb193 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 13 Sep 2020 16:47:07 +0200 Subject: [PATCH 024/255] LVBAG: silence CID 1432677 --- gdal/ogr/ogrsf_frmts/lvbag/ogrlvbaglayer.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gdal/ogr/ogrsf_frmts/lvbag/ogrlvbaglayer.cpp b/gdal/ogr/ogrsf_frmts/lvbag/ogrlvbaglayer.cpp index 1f4340cfb487..af99b979e916 100644 --- a/gdal/ogr/ogrsf_frmts/lvbag/ogrlvbaglayer.cpp +++ b/gdal/ogr/ogrsf_frmts/lvbag/ogrlvbaglayer.cpp @@ -647,11 +647,11 @@ void OGRLVBAGLayer::EndElementCbk( const char *pszName ) } else if( poGeomField->GetType() == wkbMultiPolygon && poGeom->getGeometryType() == wkbGeometryCollection - && dynamic_cast(poGeom.get())->getNumGeometries() > 0 - && dynamic_cast(poGeom.get())->getGeometryRef(0)->getGeometryType() == wkbPolygon ) + && poGeom->toGeometryCollection()->getNumGeometries() > 0 + && poGeom->toGeometryCollection()->getGeometryRef(0)->getGeometryType() == wkbPolygon ) { std::unique_ptr poMultiPolygon = std::unique_ptr{ new OGRMultiPolygon }; - for (auto &poChildGeom : dynamic_cast(poGeom.get())) + for (auto &poChildGeom : poGeom->toGeometryCollection()) poMultiPolygon->addGeometry(poChildGeom); poGeom.reset(poMultiPolygon.release()); } From e2aa1ec93fb7cde8b825197f86dad5f1130abd79 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 13 Sep 2020 16:51:45 +0200 Subject: [PATCH 025/255] MAPML: fix CID 1432682 --- gdal/ogr/ogrsf_frmts/mapml/ogrmapmldataset.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gdal/ogr/ogrsf_frmts/mapml/ogrmapmldataset.cpp b/gdal/ogr/ogrsf_frmts/mapml/ogrmapmldataset.cpp index 7b8b0ae17ba8..5a48167fc8a3 100644 --- a/gdal/ogr/ogrsf_frmts/mapml/ogrmapmldataset.cpp +++ b/gdal/ogr/ogrsf_frmts/mapml/ogrmapmldataset.cpp @@ -806,12 +806,14 @@ OGRMapMLWriterDataset::~OGRMapMLWriterDataset() if( psExtra ) { CPLXMLNode* psLastChild = m_psExtent->psChild; - while( psLastChild->psNext ) - psLastChild = psLastChild->psNext; if( psLastChild == nullptr ) m_psExtent->psChild = psExtra; else + { + while( psLastChild->psNext ) + psLastChild = psLastChild->psNext; psLastChild->psNext = psExtra; + } } } From 8f5bab77ee3fdd0bffe1c3cf24908fa0e8a148d7 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 13 Sep 2020 16:54:25 +0200 Subject: [PATCH 026/255] PGeo: fix libPath where /usr/local/lib64 or /usr/lib would not have been tried (Coverity CID 1432693) --- gdal/ogr/ogrsf_frmts/pgeo/ogrpgeodriver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gdal/ogr/ogrsf_frmts/pgeo/ogrpgeodriver.cpp b/gdal/ogr/ogrsf_frmts/pgeo/ogrpgeodriver.cpp index a2a55cfc65bb..63ef4f1838ce 100644 --- a/gdal/ogr/ogrsf_frmts/pgeo/ogrpgeodriver.cpp +++ b/gdal/ogr/ogrsf_frmts/pgeo/ogrpgeodriver.cpp @@ -207,7 +207,7 @@ bool OGRODBCMDBDriver::FindDriverLib() const int nLibNames = sizeof(aszDefaultLibName) / sizeof(aszDefaultLibName[0]); const char* libPath[] = { "/usr/lib64", - "/usr/local/lib64" + "/usr/local/lib64", "/usr/lib", "/usr/local/lib" }; From c5e79e5b123c19df7447b5ca00eb7f04e116d9f1 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 13 Sep 2020 22:10:42 +0200 Subject: [PATCH 027/255] docker/alpine-normal: update to spatialite 5.0.0 [ci skip] --- gdal/docker/alpine-normal/Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/gdal/docker/alpine-normal/Dockerfile b/gdal/docker/alpine-normal/Dockerfile index bfec57c78d7a..4d0c5feb4757 100644 --- a/gdal/docker/alpine-normal/Dockerfile +++ b/gdal/docker/alpine-normal/Dockerfile @@ -215,12 +215,13 @@ RUN mkdir proj \ && for i in /build_proj/usr/bin/*; do strip -s $i 2>/dev/null || /bin/true; done # Build spatialite -ARG SPATIALITE_VERSION=4.3.0a +ARG SPATIALITE_VERSION=5.0.0 RUN if test "${SPATIALITE_VERSION}" != ""; then ( \ wget -q http://www.gaia-gis.it/gaia-sins/libspatialite-${SPATIALITE_VERSION}.tar.gz \ && tar xzf libspatialite-${SPATIALITE_VERSION}.tar.gz \ && rm -f libspatialite-${SPATIALITE_VERSION}.tar.gz \ && cd libspatialite-${SPATIALITE_VERSION} \ + && apk add --no-cache minizip-dev \ && if test "${RSYNC_REMOTE}" != ""; then \ echo "Downloading cache..."; \ rsync -ra ${RSYNC_REMOTE}/spatialite/ $HOME/; \ @@ -229,7 +230,7 @@ RUN if test "${SPATIALITE_VERSION}" != ""; then ( \ export CXX="ccache g++"; \ ccache -M 100M; \ fi \ - && CFLAGS="-DACCEPT_USE_OF_DEPRECATED_PROJ_API_H -O2" ./configure --prefix=/usr --disable-static \ + && ./configure --prefix=/usr --disable-static \ && make -j$(nproc) \ && make install \ && if test "${RSYNC_REMOTE}" != ""; then \ @@ -357,7 +358,7 @@ RUN apk add --no-cache \ libjpeg-turbo libpng libwebp expat \ icu-libs \ python3 py3-numpy ${POPPLER} pcre libpq libxml2 portablexdr openjpeg \ - openexr libheif xerces-c geos cfitsio \ + openexr libheif xerces-c geos cfitsio minizip \ # Remove /usr/lib/libopenjp2.so.2.3.0 since we are building v2.3.1 manually # && rm -f /usr/lib/libopenjp2.so.2.3.0 \ # libturbojpeg.so is not used by GDAL. Only libjpeg.so* From 6416d089f89c4ae1f04f3a8a6f9706715e44fe31 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 14 Sep 2020 09:37:26 +0200 Subject: [PATCH 028/255] GMLAS: avoid running out of file descriptors in case of big number of layers --- gdal/ogr/ogrsf_frmts/gmlas/ogrgmlaslayer.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/gdal/ogr/ogrsf_frmts/gmlas/ogrgmlaslayer.cpp b/gdal/ogr/ogrsf_frmts/gmlas/ogrgmlaslayer.cpp index a55ee32409a3..108f5234359f 100644 --- a/gdal/ogr/ogrsf_frmts/gmlas/ogrgmlaslayer.cpp +++ b/gdal/ogr/ogrsf_frmts/gmlas/ogrgmlaslayer.cpp @@ -1461,7 +1461,15 @@ OGRFeatureDefn* OGRGMLASLayer::GetLayerDefn() !m_poDS->GetConf().m_oXLinkResolution.m_aoURLSpecificRules.empty() ) { if( m_poReader == nullptr ) + { InitReader(); + // Avoid keeping too many file descriptor opened + if( m_fpGML != nullptr ) + m_poDS->PushUnusedGMLFilePointer(m_fpGML); + m_fpGML = nullptr; + delete m_poReader; + m_poReader = nullptr; + } } } return m_poFeatureDefn; From 86d08a5e15b4cd22fb0a94482b0c8343db2d304f Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 12 Sep 2020 17:17:49 +0200 Subject: [PATCH 029/255] Workaround issue with UTF-8 precomposed vs decomposed encodings on MacOS filesystems that affect sidecar file discovery (fixes #2903) --- autotest/gcore/tiff_read.py | 19 ++++++++++ gdal/gcore/gdal_misc.cpp | 54 +++++++++++++++++++++++++-- gdal/gcore/gdal_priv.h | 3 ++ gdal/gcore/gdaljp2abstractdataset.cpp | 1 + gdal/gcore/gdalpamdataset.cpp | 11 ++++-- 5 files changed, 82 insertions(+), 6 deletions(-) diff --git a/autotest/gcore/tiff_read.py b/autotest/gcore/tiff_read.py index e95049e6a6af..ce930b8bae8a 100755 --- a/autotest/gcore/tiff_read.py +++ b/autotest/gcore/tiff_read.py @@ -3489,3 +3489,22 @@ def test_tiff_read_geodetic_tiff_grid(): ds = gdal.Open('data/test_hgrid_with_subgrid.tif') assert ds.GetSubDatasets()[0][1] == 'Page 1 (10P x 10L x 2B): CAwest' + + + +############################################################################### +# Test fix for https://github.com/OSGeo/gdal/issues/2903 +# related to precomposed vs decomposed UTF-8 filenames on MacOSX + +def test_tiff_read_utf8_encoding_issue_2903(): + + precomposed_utf8 = b'\xc3\xa4'.decode('utf-8') + tmp_tif_filename = 'tmp/%s.tif' % precomposed_utf8 + tmp_tfw_filename = 'tmp/%s.tfw' % precomposed_utf8 + open(tmp_tif_filename, 'wb').write(open('data/byte_nogeoref.tif', 'rb').read()) + open(tmp_tfw_filename, 'wb').write(open('data/byte_nogeoref.tfw', 'rb').read()) + ds = gdal.Open(tmp_tif_filename) + assert ds.GetGeoTransform()[0] != 0 + ds = None + os.unlink(tmp_tif_filename) + os.unlink(tmp_tfw_filename) diff --git a/gdal/gcore/gdal_misc.cpp b/gdal/gcore/gdal_misc.cpp index e13a28c31454..26d686032a8f 100644 --- a/gdal/gcore/gdal_misc.cpp +++ b/gdal/gcore/gdal_misc.cpp @@ -1308,7 +1308,8 @@ CPLString GDALFindAssociatedFile( const char *pszBaseFilename, { CPLString osTarget = CPLResetExtension( pszBaseFilename, pszExt ); - if( papszSiblingFiles == nullptr ) + if( papszSiblingFiles == nullptr || + !GDALCanReliablyUseSiblingFileList(osTarget.c_str()) ) { VSIStatBufL sStatBuf; @@ -1781,7 +1782,7 @@ int GDALReadTabFile2( const char * pszBaseFilename, const char *pszTAB = CPLResetExtension( pszBaseFilename, "tab" ); - if (papszSiblingFiles) + if (papszSiblingFiles && GDALCanReliablyUseSiblingFileList(pszTAB)) { int iSibling = CSLFindString(papszSiblingFiles, CPLGetFilename(pszTAB)); if (iSibling >= 0) @@ -2032,7 +2033,7 @@ int GDALReadWorldFile2( const char *pszBaseFilename, const char *pszExtension, const char *pszTFW = CPLResetExtension( pszBaseFilename, szExtLower ); - if (papszSiblingFiles) + if (papszSiblingFiles && GDALCanReliablyUseSiblingFileList(pszTFW)) { const int iSibling = CSLFindString(papszSiblingFiles, CPLGetFilename(pszTFW)); @@ -4070,6 +4071,53 @@ int GDALCanFileAcceptSidecarFile(const char* pszFilename) return TRUE; } +/************************************************************************/ +/* GDALCanReliablyUseSiblingFileList() */ +/************************************************************************/ + +/* Try to address https://github.com/OSGeo/gdal/issues/2903 */ +/* - On Apple HFS+ filesystem, filenames are stored in a variant of UTF-8 NFD */ +/* (normalization form decomposed). The filesystem takes care of converting */ +/* precomposed form as often coming from user interface to this NFD variant */ +/* See https://stackoverflow.com/questions/6153345/different-utf8-encoding-in-filenames-os-x */ +/* And readdir() will return such NFD variant encoding. Consequently comparing */ +/* the user filename with ones with readdir() is not reliable */ +/* - APFS preserves both case and normalization of the filename on disk in all */ +/* variants. In macOS High Sierra, APFS is normalization-insensitive in both */ +/* the case-insensitive and case-sensitive variants, using a hash-based native */ +/* normalization scheme. APFS preserves the normalization of the filename and */ +/* uses hashes of the normalized form of the filename to provide normalization */ +/* insensitivity. */ +/* From https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/APFS_Guide/FAQ/FAQ.html */ +/* Issues might still arise if the file has been created using one of the UTF-8 */ +/* encoding (likely the decomposed one if using MacOS specific API), but the */ +/* string passed to GDAL for opening would be with another one (likely the precomposed one) */ +bool GDALCanReliablyUseSiblingFileList(const char* pszFilename) +{ +#ifdef __APPLE__ + for( int i = 0; pszFilename[i] != 0; ++i ) + { + if( reinterpret_cast(pszFilename)[i] > 127 ) + { + // non-ASCII character found + + // if this is a network storage, assume no issue + if( strstr(pszFilename, "/vsicurl/") || + strstr(pszFilename, "/vsis3/") || + strstr(pszFilename, "/vsicurl/") ) + { + return true; + } + return false; + } + } + return true; +#else + (void) pszFilename; + return true; +#endif +} + /************************************************************************/ /* GDALAdjustNoDataCloseToFloatMax() */ /************************************************************************/ diff --git a/gdal/gcore/gdal_priv.h b/gdal/gcore/gdal_priv.h index a101bed5eed3..476ccbb943aa 100644 --- a/gdal/gcore/gdal_priv.h +++ b/gdal/gcore/gdal_priv.h @@ -2779,6 +2779,9 @@ void GDALSerializeOpenOptionsToXML( CPLXMLNode* psParentNode, char** papszOpenOp char** GDALDeserializeOpenOptionsFromXML( CPLXMLNode* psParentNode ); int GDALCanFileAcceptSidecarFile(const char* pszFilename); + +bool GDALCanReliablyUseSiblingFileList(const char* pszFilename); + //! @endcond #endif /* ndef GDAL_PRIV_H_INCLUDED */ diff --git a/gdal/gcore/gdaljp2abstractdataset.cpp b/gdal/gcore/gdaljp2abstractdataset.cpp index d9c5db95c831..cb3d780c5f52 100644 --- a/gdal/gcore/gdaljp2abstractdataset.cpp +++ b/gdal/gcore/gdaljp2abstractdataset.cpp @@ -324,6 +324,7 @@ char **GDALJP2AbstractDataset::GetFileList() if( pszWldFilename != nullptr && m_nGeoTransformGeorefSrcIndex == m_nWORLDFILEIndex && + GDALCanReliablyUseSiblingFileList(pszWldFilename) && CSLFindString( papszFileList, pszWldFilename ) == -1 ) { double l_adfGeoTransform[6]; diff --git a/gdal/gcore/gdalpamdataset.cpp b/gdal/gcore/gdalpamdataset.cpp index f4413d9e06bd..fb985a097f38 100644 --- a/gdal/gcore/gdalpamdataset.cpp +++ b/gdal/gcore/gdalpamdataset.cpp @@ -839,7 +839,8 @@ CPLErr GDALPamDataset::TryLoadXML(char **papszSiblingFiles) int nLastErrNo = CPLGetLastErrorNo(); CPLString osLastErrorMsg = CPLGetLastErrorMsg(); - if (papszSiblingFiles != nullptr && IsPamFilenameAPotentialSiblingFile()) + if( papszSiblingFiles != nullptr && IsPamFilenameAPotentialSiblingFile() && + GDALCanReliablyUseSiblingFileList(psPam->pszPamFilename) ) { const int iSibling = CSLFindString( papszSiblingFiles, @@ -1195,6 +1196,7 @@ char **GDALPamDataset::GetFileList() char **papszFileList = GDALDataset::GetFileList(); if( psPam && !psPam->osPhysicalFilename.empty() + && GDALCanReliablyUseSiblingFileList(psPam->osPhysicalFilename.c_str()) && CSLFindString( papszFileList, psPam->osPhysicalFilename ) == -1 ) { papszFileList = CSLInsertString( papszFileList, 0, @@ -1207,7 +1209,9 @@ char **GDALPamDataset::GetFileList() if (!bAddPamFile) { VSIStatBufL sStatBuf; - if (oOvManager.GetSiblingFiles() != nullptr && IsPamFilenameAPotentialSiblingFile()) + if (oOvManager.GetSiblingFiles() != nullptr && + IsPamFilenameAPotentialSiblingFile() && + GDALCanReliablyUseSiblingFileList(psPam->pszPamFilename) ) { bAddPamFile = CSLFindString(oOvManager.GetSiblingFiles(), CPLGetFilename(psPam->pszPamFilename)) >= 0; @@ -1225,6 +1229,7 @@ char **GDALPamDataset::GetFileList() } if( psPam && !psPam->osAuxFilename.empty() && + GDALCanReliablyUseSiblingFileList(psPam->osAuxFilename.c_str()) && CSLFindString( papszFileList, psPam->osAuxFilename ) == -1 ) { papszFileList = CSLAddString( papszFileList, psPam->osAuxFilename ); @@ -1556,7 +1561,7 @@ CPLErr GDALPamDataset::TryLoadAux(char **papszSiblingFiles) if( strlen(pszPhysicalFile) == 0 ) return CE_None; - if( papszSiblingFiles ) + if( papszSiblingFiles && GDALCanReliablyUseSiblingFileList(pszPhysicalFile) ) { CPLString osAuxFilename = CPLResetExtension( pszPhysicalFile, "aux"); int iSibling = CSLFindString( papszSiblingFiles, From 2c00fa17e79707e214417eea612f3bcb2f5b23c9 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 14 Sep 2020 12:42:54 +0200 Subject: [PATCH 030/255] FITS: do not emit error if no georeferencing is found And add related tests Funded by CNES --- autotest/gdrivers/data/fits/byte_merc.fits | Bin 0 -> 5760 bytes .../data/fits/offset_scale_no_georef.fits | Bin 0 -> 5760 bytes autotest/gdrivers/fits.py | 17 +++++++++ gdal/frmts/fits/fitsdataset.cpp | 34 +++++++++++------- 4 files changed, 39 insertions(+), 12 deletions(-) create mode 100644 autotest/gdrivers/data/fits/byte_merc.fits create mode 100644 autotest/gdrivers/data/fits/offset_scale_no_georef.fits diff --git a/autotest/gdrivers/data/fits/byte_merc.fits b/autotest/gdrivers/data/fits/byte_merc.fits new file mode 100644 index 0000000000000000000000000000000000000000..e607e709435d8f5b6fcc03315b2be55db67f1c28 GIT binary patch literal 5760 zcmeH|&2!pD7{>djyr;OGi4`F}4DI9~6D1+Rm;t5Za>gy)3mu!}(9W)1E;x~LvfnN9k!w*gq?T7k0-W*>nqQ~{U z3w!e2{@k*q+G|&BZUhh}Cz2SXoZqYAI&D-!k-HwN!qkDa&ddX^3jat22wyG6t zi+*YLz`NI1tyRi2YG9G+tjz)v^MF30b1qrhURV1x4%uk#gF;(B&XEfq_V1X403vBoli7>SugD2X^B0uiK2N^F(_S0K_rU{PfnCWMfw zTuK!LDwEfFo~9zjKA|$Ew|otL&(kDPpbLcSLvo1-9hHEkfM=3jG3ZgEkSL{MnA0aG z2}>A`RM3D$i5&FMT7wh|NscLb^Or!P7zBBMp-;Sod-a%B*>Fg=gmM@EP()o zL0gavfFD3Ea?!$=TBexMBdKEW1+LaXkbJe%Otj13L$H4(uG*Ik0nJ=fKW^|Fi>t0k$9G A*8l(j literal 0 HcmV?d00001 diff --git a/autotest/gdrivers/data/fits/offset_scale_no_georef.fits b/autotest/gdrivers/data/fits/offset_scale_no_georef.fits new file mode 100644 index 0000000000000000000000000000000000000000..87bf48b46930b934ae0a705a2a80a43761f18e81 GIT binary patch literal 5760 zcmeH{v2)r;6vppQc~zX7NeIS+b0m`j6OwRegU2W{9y+nCEzN?&=n31*wp(tgQZ1!Q zN|)E>-$|K#E1R*o!8cBqLQexiyKjH{z3;6U_D)WZyF?#oM|kvxF2bp#Ni3N}99_i8 zjMA7s^}Hc*>PHhlnQSdQ>UpQV^Cx(Jffp@iW0_EVLE|uGn&bDxPkow)H*&g#Dj)jo z^WG4@cIsuTckYDx=FZaQg0fArtbSijvp(MIQ2>itf4B?SI_O zBiE18dMAx#N;7}y+LHQVMDiw;5r=WKRexu2a?^sqTbSlv7`mdN$o+O7odi+=<*0Zwrc7)sK{}Y zTwcdG#8>`wfuoZKDBe%xM#BTo`5ZsBFi|# zIySj*OTMXpi_BUBIwRd}QfOqjEDM$aUTCov&||oisI?Iob30ortiYBA8ZfC*gA1os zWtgx9hV;Y0#$Z2>21?q82Aim{pc* zU>KVV;Y5x!-xk)c*o*{^Fa`q5$=6T-wfb)ewz*kBaPj@eH>0f1%xaavN`-5^&a*tv g*Ty;Sf`$5{ipqh?fy#l(fy#l(fy#l(f#2GJpS-rpk^lez literal 0 HcmV?d00001 diff --git a/autotest/gdrivers/fits.py b/autotest/gdrivers/fits.py index ff3f990cf818..633ec4a2d2e5 100755 --- a/autotest/gdrivers/fits.py +++ b/autotest/gdrivers/fits.py @@ -114,3 +114,20 @@ def test_fits_offscale(): assert offset == -0.0039525691699605 assert scale == 1.00395256917 + +def test_fits_read_offset_scale_no_georef(): + + ds = gdal.Open('data/fits/offset_scale_no_georef.fits') + assert gdal.GetLastErrorMsg() == '' + assert ds.GetRasterBand(1).GetOffset() != 0 + assert ds.GetRasterBand(1).GetScale() != 1 + + +def test_fits_read_georef_merc(): + + ds = gdal.Open('data/fits/byte_merc.fits') + assert gdal.GetLastErrorMsg() == '' + wkt = ds.GetProjectionRef() + assert wkt == 'PROJCS["Mercator_Earth",GEOGCS["GCS_Earth",DATUM["D_Earth",SPHEROID["Earth",6378206.4,294.978698213898]],PRIMEM["Reference_Meridian",0],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]]],PROJECTION["Mercator_1SP"],PARAMETER["central_meridian",0],PARAMETER["scale_factor",1],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]],AXIS["Easting",EAST],AXIS["Northing",NORTH]]' + gt = ds.GetGeoTransform() + assert gt == pytest.approx((-13095897.481058259, 72.23522015778646, 0.0, 3991653.2130816197, 0.0, -72.23522015778646), abs=1e-3) diff --git a/gdal/frmts/fits/fitsdataset.cpp b/gdal/frmts/fits/fitsdataset.cpp index 4c4fba518d49..82cfde71e418 100644 --- a/gdal/frmts/fits/fitsdataset.cpp +++ b/gdal/frmts/fits/fitsdataset.cpp @@ -80,6 +80,7 @@ class FITSDataset final : public GDALPamDataset { void WriteFITSInfo(); bool bFITSInfoChanged; + void LoadGeoreferencing(); void LoadFITSInfo(); public: @@ -1339,15 +1340,12 @@ CPLErr FITSRasterBand::DeleteNoDataValue() } /************************************************************************/ -/* LoadFITSInfo() */ +/* LoadGeoreferencing() */ /************************************************************************/ -void FITSDataset::LoadFITSInfo() - +void FITSDataset::LoadGeoreferencing() { int status = 0; - int bitpix; - double dfScale, dfOffset; double crpix1, crpix2, crval1, crval2, cdelt1, cdelt2, pc[4], cd[4]; double aRadius, cRadius, invFlattening = 0.0; double falseEast = 0.0, falseNorth = 0.0, scale = 1.0; @@ -1365,8 +1363,7 @@ void FITSDataset::LoadFITSInfo() if( status ) { strncpy(target, "Undefined", 10); - CPLError(CE_Failure, CPLE_AppDefined, - "OBJECT keyword is missing"); + CPLDebug("FITS", "OBJECT keyword is missing"); status = 0; } @@ -1378,14 +1375,14 @@ void FITSDataset::LoadFITSInfo() fits_read_key(hFITS, TDOUBLE, "A_RADIUS", &aRadius, nullptr, &status); if( status ) { - CPLError(CE_Failure, CPLE_AppDefined, + CPLDebug("FITS", "No Radii keyword available, metadata will not contain DATUM information."); - status = 0; + return; } else { fits_read_key(hFITS, TDOUBLE, "C_RADIUS", &cRadius, nullptr, &status); if( status ) { - CPLError(CE_Failure, CPLE_AppDefined, + CPLError(CE_Warning, CPLE_AppDefined, "No polar radius keyword available, setting C_RADIUS = A_RADIUS"); cRadius = aRadius; status = 0; @@ -1528,10 +1525,23 @@ void FITSDataset::LoadFITSInfo() } } } else { - CPLError(CE_Failure, CPLE_AppDefined, + CPLError(CE_Warning, CPLE_AppDefined, "No CTYPE keywords: no geospatial information available."); - status = 0; } +} + +/************************************************************************/ +/* LoadFITSInfo() */ +/************************************************************************/ + +void FITSDataset::LoadFITSInfo() + +{ + int status = 0; + int bitpix; + double dfScale, dfOffset; + + LoadGeoreferencing(); CPLAssert(!bMetadataChanged); CPLAssert(!bNoDataChanged); From 46a7ee216b9004404c8e63f842a4f73318fdf27a Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 14 Sep 2020 12:43:00 +0200 Subject: [PATCH 031/255] FITS: register .fits as an extension for the driver Funded by CNES --- gdal/frmts/fits/fitsdataset.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/gdal/frmts/fits/fitsdataset.cpp b/gdal/frmts/fits/fitsdataset.cpp index 82cfde71e418..21befebf6434 100644 --- a/gdal/frmts/fits/fitsdataset.cpp +++ b/gdal/frmts/fits/fitsdataset.cpp @@ -1598,6 +1598,7 @@ void GDALRegister_FITS() "drivers/raster/fits.html" ); poDriver->SetMetadataItem( GDAL_DMD_CREATIONDATATYPES, "Byte UInt16 Int16 UInt32 Int32 Float32 Float64" ); + poDriver->SetMetadataItem( GDAL_DMD_EXTENSIONS, "fits" ); poDriver->pfnOpen = FITSDataset::Open; poDriver->pfnCreate = FITSDataset::Create; From e98d810f94099ef12a12d3153c294f9917dff6b5 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 14 Sep 2020 12:43:02 +0200 Subject: [PATCH 032/255] FITS: adopt m_ naming convention for member variables No functional change Funded by CNES --- gdal/frmts/fits/fitsdataset.cpp | 432 +++++++++++++++----------------- 1 file changed, 207 insertions(+), 225 deletions(-) diff --git a/gdal/frmts/fits/fitsdataset.cpp b/gdal/frmts/fits/fitsdataset.cpp index 21befebf6434..1991113f9813 100644 --- a/gdal/frmts/fits/fitsdataset.cpp +++ b/gdal/frmts/fits/fitsdataset.cpp @@ -54,31 +54,31 @@ class FITSDataset final : public GDALPamDataset { friend class FITSRasterBand; - fitsfile* hFITS; + fitsfile* m_hFITS = nullptr; - GDALDataType gdalDataType; // GDAL code for the image type - int fitsDataType; // FITS code for the image type + GDALDataType m_gdalDataType = GDT_Unknown; // GDAL code for the image type + int m_fitsDataType = 0; // FITS code for the image type - bool isExistingFile; - long highestOffsetWritten; // How much of image has been written + bool m_isExistingFile = false; + long m_highestOffsetWritten = 0; // How much of image has been written - bool bNoDataChanged; - bool bNoDataSet; - double dfNoDataValue; + bool m_bNoDataChanged = false; + bool m_bNoDataSet = false; + double m_dfNoDataValue = -9999.0; - bool bMetadataChanged; + bool m_bMetadataChanged = false; FITSDataset(); // Others should not call this constructor explicitly - CPLErr Init(fitsfile* hFITS_, bool isExistingFile_); + CPLErr Init(fitsfile* hFITS, bool isExistingFile); - OGRSpatialReference oSRS{}; + OGRSpatialReference m_oSRS{}; - double adfGeoTransform[6]; - bool bGeoTransformValid; + double m_adfGeoTransform[6]; + bool m_bGeoTransformValid = false; void WriteFITSInfo(); - bool bFITSInfoChanged; + bool m_bFITSInfoChanged = false; void LoadGeoreferencing(); void LoadFITSInfo(); @@ -110,15 +110,15 @@ class FITSRasterBand final: public GDALPamRasterBand { friend class FITSDataset; - bool bHaveOffsetScale; - double dfOffset; - double dfScale; + bool m_bHaveOffsetScale = false; + double m_dfOffset = 0.0; + double m_dfScale = 1.0; protected: - FITSDataset *poFDS; + FITSDataset *m_poFDS = nullptr; - bool bNoDataSet; - double dfNoDataValue; + bool m_bNoDataSet = false; + double m_dfNoDataValue = -9999.0; public: @@ -144,16 +144,11 @@ class FITSRasterBand final: public GDALPamRasterBand { /************************************************************************/ FITSRasterBand::FITSRasterBand( FITSDataset *poDSIn, int nBandIn ) : - bHaveOffsetScale(false), - dfOffset(0.0), - dfScale(1.0), - poFDS(poDSIn), - bNoDataSet(false), - dfNoDataValue(-9999.0) + m_poFDS(poDSIn) { poDS = poDSIn; nBand = nBandIn; - eDataType = poDSIn->gdalDataType; + eDataType = poDSIn->m_gdalDataType; nBlockXSize = poDSIn->nRasterXSize; nBlockYSize = 1; } @@ -174,8 +169,8 @@ FITSRasterBand::~FITSRasterBand() CPLErr FITSRasterBand::IReadBlock(CPL_UNUSED int nBlockXOff, int nBlockYOff, void* pImage ) { // A FITS block is one row (we assume BSQ formatted data) - FITSDataset* dataset = (FITSDataset*) poDS; - fitsfile* hFITS = dataset->hFITS; + FITSDataset* dataset = m_poFDS; + fitsfile* hFITS = dataset->m_hFITS; int status = 0; // Since a FITS block is a whole row, nBlockXOff must be zero @@ -192,14 +187,14 @@ CPLErr FITSRasterBand::IReadBlock(CPL_UNUSED int nBlockXOff, int nBlockYOff, // If we haven't written this block to the file yet, then attempting // to read causes an error, so in this case, just return zeros. - if (!dataset->isExistingFile && offset > dataset->highestOffsetWritten) { + if (!dataset->m_isExistingFile && offset > dataset->m_highestOffsetWritten) { memset(pImage, 0, nBlockXSize * nBlockYSize * GDALGetDataTypeSize(eDataType) / 8); return CE_None; } // Otherwise read in the image data - fits_read_img(hFITS, dataset->fitsDataType, offset, nElements, + fits_read_img(hFITS, dataset->m_fitsDataType, offset, nElements, nullptr, pImage, nullptr, &status); // Capture special case of non-zero status due to data range @@ -226,8 +221,8 @@ CPLErr FITSRasterBand::IReadBlock(CPL_UNUSED int nBlockXOff, int nBlockYOff, CPLErr FITSRasterBand::IWriteBlock( CPL_UNUSED int nBlockXOff, int nBlockYOff, void* pImage ) { - FITSDataset* dataset = (FITSDataset*) poDS; - fitsfile* hFITS = dataset->hFITS; + FITSDataset* dataset = m_poFDS; + fitsfile* hFITS = dataset->m_hFITS; int status = 0; // Since a FITS block is a whole row, nBlockXOff must be zero @@ -239,7 +234,7 @@ CPLErr FITSRasterBand::IWriteBlock( CPL_UNUSED int nBlockXOff, int nBlockYOff, LONGLONG offset = static_cast(nBand - 1) * nRasterXSize * nRasterYSize + static_cast(nBlockYOff) * nRasterXSize + 1; long nElements = nRasterXSize; - fits_write_img(hFITS, dataset->fitsDataType, offset, nElements, + fits_write_img(hFITS, dataset->m_fitsDataType, offset, nElements, pImage, &status); // Capture special case of non-zero status due to data range @@ -257,8 +252,8 @@ CPLErr FITSRasterBand::IWriteBlock( CPL_UNUSED int nBlockXOff, int nBlockYOff, } // When we write a block, update the offset counter that we've written - if (offset > dataset->highestOffsetWritten) - dataset->highestOffsetWritten = offset; + if (offset > dataset->m_highestOffsetWritten) + dataset->m_highestOffsetWritten = offset; return CE_None; } @@ -291,26 +286,15 @@ static bool isIgnorableFITSHeader(const char* name) { /* FITSDataset() */ /************************************************************************/ -FITSDataset::FITSDataset(): - hFITS(nullptr), - gdalDataType(GDT_Unknown), - fitsDataType(0), - isExistingFile(false), - highestOffsetWritten(0), - bNoDataChanged(false), - bNoDataSet(false), - dfNoDataValue(-9999.0), - bMetadataChanged(false), - bGeoTransformValid(false), - bFITSInfoChanged(false) +FITSDataset::FITSDataset() { - oSRS.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); - adfGeoTransform[0] = 0; - adfGeoTransform[1] = 1; - adfGeoTransform[2] = 0; - adfGeoTransform[3] = 0; - adfGeoTransform[4] = 0; - adfGeoTransform[5] = 1; + m_oSRS.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); + m_adfGeoTransform[0] = 0; + m_adfGeoTransform[1] = 1; + m_adfGeoTransform[2] = 0; + m_adfGeoTransform[3] = 0; + m_adfGeoTransform[4] = 0; + m_adfGeoTransform[5] = 1; } /************************************************************************/ @@ -320,7 +304,7 @@ FITSDataset::FITSDataset(): FITSDataset::~FITSDataset() { int status; - if( hFITS ) + if( m_hFITS ) { if(eAccess == GA_Update) { @@ -328,8 +312,8 @@ FITSDataset::~FITSDataset() { // capability. Write any meta data to the file that's compatible with // FITS. status = 0; - fits_movabs_hdu(hFITS, 1, nullptr, &status); - fits_write_key_longwarn(hFITS, &status); + fits_movabs_hdu(m_hFITS, 1, nullptr, &status); + fits_write_key_longwarn(m_hFITS, &status); if (status) { CPLError(CE_Warning, CPLE_AppDefined, "Couldn't move to first HDU in FITS file %s (%d).\n", @@ -363,7 +347,7 @@ FITSDataset::~FITSDataset() { // handle. Note: to avoid a compiler warning we copy the // const value string to a non const one. char* valueCpy = CPLStrdup(value); - fits_update_key_longstr(hFITS, key, valueCpy, nullptr, &status); + fits_update_key_longstr(m_hFITS, key, valueCpy, nullptr, &status); CPLFree(valueCpy); // Check for errors. @@ -383,8 +367,8 @@ FITSDataset::~FITSDataset() { } // Writing nodata value - if (gdalDataType != GDT_Float32 && gdalDataType != GDT_Float64) { - fits_update_key( hFITS, TDOUBLE, "BLANK", &dfNoDataValue, nullptr, &status); + if (m_gdalDataType != GDT_Float32 && m_gdalDataType != GDT_Float64) { + fits_update_key( m_hFITS, TDOUBLE, "BLANK", &m_dfNoDataValue, nullptr, &status); if (status) { // Throw a warning with CFITSIO error status, then ignore status @@ -401,8 +385,8 @@ FITSDataset::~FITSDataset() { GDALRasterBand* poSrcBand = GDALPamDataset::GetRasterBand(1); double dfScale = poSrcBand->GetScale(&pbSuccess); double dfOffset = poSrcBand->GetOffset(&pbSuccess); - if (bMetadataChanged) { - fits_update_key( hFITS, TDOUBLE, "BSCALE", &dfScale, nullptr, &status); + if (m_bMetadataChanged) { + fits_update_key( m_hFITS, TDOUBLE, "BSCALE", &dfScale, nullptr, &status); if (status) { // Throw a warning with CFITSIO error status, then ignore status @@ -412,7 +396,7 @@ FITSDataset::~FITSDataset() { status = 0; return; } - fits_update_key( hFITS, TDOUBLE, "BZERO", &dfOffset, nullptr, &status); + fits_update_key( m_hFITS, TDOUBLE, "BZERO", &dfOffset, nullptr, &status); if (status) { // Throw a warning with CFITSIO error status, then ignore status @@ -428,7 +412,7 @@ FITSDataset::~FITSDataset() { GDALPamDataset::SetSpatialRef(GDALPamDataset::GetSpatialRef()); // Write geographic info - if (bFITSInfoChanged) { + if (m_bFITSInfoChanged) { WriteFITSInfo(); } @@ -437,7 +421,7 @@ FITSDataset::~FITSDataset() { } // Close the FITS handle - fits_close_file(hFITS, &status); + fits_close_file(m_hFITS, &status); } } @@ -449,7 +433,7 @@ FITSDataset::~FITSDataset() { bool FITSDataset::GetRawBinaryLayout(GDALDataset::RawBinaryLayout& sLayout) { int status = 0; - if( fits_is_compressed_image( hFITS, &status) ) + if( fits_is_compressed_image( m_hFITS, &status) ) return false; GDALDataType eDT = GetRasterBand(1)->GetRasterDataType(); if( eDT == GDT_UInt16 || eDT == GDT_UInt32 ) @@ -459,7 +443,7 @@ bool FITSDataset::GetRawBinaryLayout(GDALDataset::RawBinaryLayout& sLayout) OFF_T headerstart = 0; OFF_T datastart = 0; OFF_T dataend = 0; - fits_get_hduoff(hFITS, &headerstart, &datastart, &dataend, &status); + fits_get_hduoff(m_hFITS, &headerstart, &datastart, &dataend, &status); if( nBands > 1 ) sLayout.eInterleaving = RawBinaryLayout::Interleaving::BSQ; sLayout.eDataType = eDT; @@ -475,15 +459,15 @@ bool FITSDataset::GetRawBinaryLayout(GDALDataset::RawBinaryLayout& sLayout) /* Init() */ /************************************************************************/ -CPLErr FITSDataset::Init(fitsfile* hFITS_, bool isExistingFile_) { +CPLErr FITSDataset::Init(fitsfile* hFITS, bool isExistingFile) { - hFITS = hFITS_; - isExistingFile = isExistingFile_; - highestOffsetWritten = 0; + m_hFITS = hFITS; + m_isExistingFile = isExistingFile; + m_highestOffsetWritten = 0; int status = 0; // Move to the primary HDU - fits_movabs_hdu(hFITS, 1, nullptr, &status); + fits_movabs_hdu(m_hFITS, 1, nullptr, &status); if (status) { CPLError(CE_Failure, CPLE_AppDefined, "Couldn't move to first HDU in FITS file %s (%d).\n", @@ -514,44 +498,44 @@ CPLErr FITSDataset::Init(fitsfile* hFITS_, bool isExistingFile_) { offset = 0.; } - fits_read_key(hFITS, TDOUBLE, "BLANK", &dfNoDataValue, nullptr, &status); - bNoDataSet = !status; + fits_read_key(hFITS, TDOUBLE, "BLANK", &m_dfNoDataValue, nullptr, &status); + m_bNoDataSet = !status; status = 0; // Determine data type and nodata value if BLANK keyword is absent if (bitpix == BYTE_IMG) { - gdalDataType = GDT_Byte; - fitsDataType = TBYTE; + m_gdalDataType = GDT_Byte; + m_fitsDataType = TBYTE; } else if (bitpix == SHORT_IMG) { if (offset == 32768.) { - gdalDataType = GDT_UInt16; - fitsDataType = TUSHORT; + m_gdalDataType = GDT_UInt16; + m_fitsDataType = TUSHORT; } else { - gdalDataType = GDT_Int16; - fitsDataType = TSHORT; + m_gdalDataType = GDT_Int16; + m_fitsDataType = TSHORT; } } else if (bitpix == LONG_IMG) { if (offset == 2147483648.) { - gdalDataType = GDT_UInt32; - fitsDataType = TUINT; + m_gdalDataType = GDT_UInt32; + m_fitsDataType = TUINT; } else { - gdalDataType = GDT_Int32; - fitsDataType = TINT; + m_gdalDataType = GDT_Int32; + m_fitsDataType = TINT; } } else if (bitpix == FLOAT_IMG) { - gdalDataType = GDT_Float32; - fitsDataType = TFLOAT; + m_gdalDataType = GDT_Float32; + m_fitsDataType = TFLOAT; } else if (bitpix == DOUBLE_IMG) { - gdalDataType = GDT_Float64; - fitsDataType = TDOUBLE; + m_gdalDataType = GDT_Float64; + m_fitsDataType = TDOUBLE; } else { CPLError(CE_Failure, CPLE_AppDefined, @@ -592,10 +576,10 @@ CPLErr FITSDataset::Init(fitsfile* hFITS_, bool isExistingFile_) { int nKeys = 0; int nMoreKeys = 0; - fits_get_hdrspace(hFITS, &nKeys, &nMoreKeys, &status); + fits_get_hdrspace(m_hFITS, &nKeys, &nMoreKeys, &status); for(keyNum = 1; keyNum <= nKeys; keyNum++) { - fits_read_keyn(hFITS, keyNum, key, value, nullptr, &status); + fits_read_keyn(m_hFITS, keyNum, key, value, nullptr, &status); if (status) { CPLError(CE_Failure, CPLE_AppDefined, "Error while reading key %d from FITS file %s (%d)", @@ -678,8 +662,8 @@ GDALDataset* FITSDataset::Open(GDALOpenInfo* poOpenInfo) { FITSDataset* dataset = new FITSDataset(); dataset->eAccess = poOpenInfo->eAccess; - dataset->bMetadataChanged = false; - dataset->bNoDataChanged = false; + dataset->m_bMetadataChanged = false; + dataset->m_bNoDataChanged = false; // Set up the description and initialize the dataset dataset->SetDescription(poOpenInfo->pszFilename); @@ -819,7 +803,7 @@ void FITSDataset::WriteFITSInfo() /* -------------------------------------------------------------------- */ /* Write out projection definition. */ /* -------------------------------------------------------------------- */ - const bool bHasProjection = !oSRS.IsEmpty(); + const bool bHasProjection = !m_oSRS.IsEmpty(); if( bHasProjection ) { @@ -827,7 +811,7 @@ void FITSDataset::WriteFITSInfo() std::string object, ctype1, ctype2; - const char* target = oSRS.GetAttrValue("DATUM",0); + const char* target = m_oSRS.GetAttrValue("DATUM",0); if ( target ) { if ( strstr(target, "Moon") ) { object.assign("Moon"); @@ -867,18 +851,18 @@ void FITSDataset::WriteFITSInfo() ctype2.assign("EA"); } - fits_update_key( hFITS, TSTRING, "OBJECT", + fits_update_key( m_hFITS, TSTRING, "OBJECT", const_cast(static_cast(object.c_str())), nullptr, &status); } - double aradius = oSRS.GetSemiMajor(); + double aradius = m_oSRS.GetSemiMajor(); double bradius = aradius; - double cradius = oSRS.GetSemiMinor(); + double cradius = m_oSRS.GetSemiMinor(); cfactor = aradius * DEG2RAD; - fits_update_key( hFITS, TDOUBLE, "A_RADIUS", &aradius, nullptr, &status); + fits_update_key( m_hFITS, TDOUBLE, "A_RADIUS", &aradius, nullptr, &status); if (status) { // Throw a warning with CFITSIO error status, then ignore status @@ -888,7 +872,7 @@ void FITSDataset::WriteFITSInfo() status = 0; return; } - fits_update_key( hFITS, TDOUBLE, "B_RADIUS", &bradius, nullptr, &status); + fits_update_key( m_hFITS, TDOUBLE, "B_RADIUS", &bradius, nullptr, &status); if (status) { // Throw a warning with CFITSIO error status, then ignore status @@ -898,7 +882,7 @@ void FITSDataset::WriteFITSInfo() status = 0; return; } - fits_update_key( hFITS, TDOUBLE, "C_RADIUS", &cradius, nullptr, &status); + fits_update_key( m_hFITS, TDOUBLE, "C_RADIUS", &cradius, nullptr, &status); if (status) { // Throw a warning with CFITSIO error status, then ignore status @@ -909,7 +893,7 @@ void FITSDataset::WriteFITSInfo() return; } - const char* unit = oSRS.GetAttrValue("UNIT",0); + const char* unit = m_oSRS.GetAttrValue("UNIT",0); ctype1.append("LN-"); ctype2.append("LT-"); @@ -918,29 +902,29 @@ void FITSDataset::WriteFITSInfo() // strcat(ctype2a, "PY-"); std::string fitsproj; - const char* projection = oSRS.GetAttrValue("PROJECTION",0); + const char* projection = m_oSRS.GetAttrValue("PROJECTION",0); double centlon = 0, centlat = 0; if (projection) { if ( strstr(projection, "Sinusoidal") ) { fitsproj.assign("SFL"); - centlon = oSRS.GetProjParm("central_meridian", 0, nullptr); + centlon = m_oSRS.GetProjParm("central_meridian", 0, nullptr); } else if ( strstr(projection, "Equirectangular") ) { fitsproj.assign("CAR"); - centlat = oSRS.GetProjParm("standard_parallel_1", 0, nullptr); - centlon = oSRS.GetProjParm("central_meridian", 0, nullptr); + centlat = m_oSRS.GetProjParm("standard_parallel_1", 0, nullptr); + centlon = m_oSRS.GetProjParm("central_meridian", 0, nullptr); } else if ( strstr(projection, "Orthographic") ) { fitsproj.assign("SIN"); - centlat = oSRS.GetProjParm("standard_parallel_1", 0, nullptr); - centlon = oSRS.GetProjParm("central_meridian", 0, nullptr); + centlat = m_oSRS.GetProjParm("standard_parallel_1", 0, nullptr); + centlon = m_oSRS.GetProjParm("central_meridian", 0, nullptr); } else if ( strstr(projection, "Mercator_1SP") || strstr(projection, "Mercator") ) { fitsproj.assign("MER"); - centlat = oSRS.GetProjParm("standard_parallel_1", 0, nullptr); - centlon = oSRS.GetProjParm("central_meridian", 0, nullptr); + centlat = m_oSRS.GetProjParm("standard_parallel_1", 0, nullptr); + centlon = m_oSRS.GetProjParm("central_meridian", 0, nullptr); } else if ( strstr(projection, "Polar_Stereographic") || strstr(projection, "Stereographic_South_Pole") || strstr(projection, "Stereographic_North_Pole") ) { fitsproj.assign("STG"); - centlat = oSRS.GetProjParm("latitude_of_origin", 0, nullptr); - centlon = oSRS.GetProjParm("central_meridian", 0, nullptr); + centlat = m_oSRS.GetProjParm("latitude_of_origin", 0, nullptr); + centlon = m_oSRS.GetProjParm("central_meridian", 0, nullptr); } /* @@ -960,7 +944,7 @@ void FITSDataset::WriteFITSInfo() ctype1.append(fitsproj); ctype2.append(fitsproj); - fits_update_key( hFITS, TSTRING, "CTYPE1", + fits_update_key( m_hFITS, TSTRING, "CTYPE1", const_cast( static_cast(ctype1.c_str())), nullptr, &status); @@ -974,7 +958,7 @@ void FITSDataset::WriteFITSInfo() return; } - fits_update_key( hFITS, TSTRING, "CTYPE2", + fits_update_key( m_hFITS, TSTRING, "CTYPE2", const_cast( static_cast(ctype2.c_str())), nullptr, &status); @@ -990,16 +974,16 @@ void FITSDataset::WriteFITSInfo() } - UpperLeftCornerX = adfGeoTransform[0] - falseEast; - UpperLeftCornerY = adfGeoTransform[3] - falseNorth; + UpperLeftCornerX = m_adfGeoTransform[0] - falseEast; + UpperLeftCornerY = m_adfGeoTransform[3] - falseNorth; if ( centlon > 180. ) { centlon = centlon - 180.; } if ( strstr(unit, "metre") ) { // convert degrees/pixel to m/pixel - mapres = 1. / adfGeoTransform[1] ; // mapres is pixel/meters - mres = adfGeoTransform[1] / cfactor ; // mres is deg/pixel + mapres = 1. / m_adfGeoTransform[1] ; // mapres is pixel/meters + mres = m_adfGeoTransform[1] / cfactor ; // mres is deg/pixel crpix1 = - (UpperLeftCornerX * mapres) + centlon / mres + 0.5; // assuming that center latitude is also the origin of the coordinate // system: this is not always true. @@ -1007,8 +991,8 @@ void FITSDataset::WriteFITSInfo() crpix2 = (UpperLeftCornerY * mapres) + 0.5; // - (centlat / mres); } else if ( strstr(unit, "degree") ) { //convert m/pixel to pixel/degree - mapres = 1. / adfGeoTransform[1] / cfactor; // mapres is pixel/deg - mres = adfGeoTransform[1] ; // mres is meters/pixel + mapres = 1. / m_adfGeoTransform[1] / cfactor; // mapres is pixel/deg + mres = m_adfGeoTransform[1] ; // mres is meters/pixel crpix1 = - (UpperLeftCornerX * mres) + centlon / mapres + 0.5; // assuming that center latitude is also the origin of the coordinate // system: this is not always true. @@ -1018,7 +1002,7 @@ void FITSDataset::WriteFITSInfo() /// Write WCS CRPIXia CRVALia CTYPEia here - fits_update_key( hFITS, TDOUBLE, "CRVAL1", ¢lon, nullptr, &status); + fits_update_key( m_hFITS, TDOUBLE, "CRVAL1", ¢lon, nullptr, &status); if (status) { // Throw a warning with CFITSIO error status, then ignore status @@ -1028,7 +1012,7 @@ void FITSDataset::WriteFITSInfo() status = 0; return; } - fits_update_key( hFITS, TDOUBLE, "CRVAL2", ¢lat, nullptr, &status); + fits_update_key( m_hFITS, TDOUBLE, "CRVAL2", ¢lat, nullptr, &status); if (status) { // Throw a warning with CFITSIO error status, then ignore status @@ -1038,7 +1022,7 @@ void FITSDataset::WriteFITSInfo() status = 0; return; } - fits_update_key( hFITS, TDOUBLE, "CRPIX1", &crpix1, nullptr, &status); + fits_update_key( m_hFITS, TDOUBLE, "CRPIX1", &crpix1, nullptr, &status); if (status) { // Throw a warning with CFITSIO error status, then ignore status @@ -1048,7 +1032,7 @@ void FITSDataset::WriteFITSInfo() status = 0; return; } - fits_update_key( hFITS, TDOUBLE, "CRPIX2", &crpix2, nullptr, &status); + fits_update_key( m_hFITS, TDOUBLE, "CRPIX2", &crpix2, nullptr, &status); if (status) { // Throw a warning with CFITSIO error status, then ignore status @@ -1062,7 +1046,7 @@ void FITSDataset::WriteFITSInfo() /* -------------------------------------------------------------------- */ /* Write geotransform if valid. */ /* -------------------------------------------------------------------- */ - if( bGeoTransformValid ) + if( m_bGeoTransformValid ) { /* -------------------------------------------------------------------- */ @@ -1072,10 +1056,10 @@ void FITSDataset::WriteFITSInfo() /// Write WCS CDELTia and PCi_ja here double cd[4]; - cd[0] = adfGeoTransform[1] / cfactor; - cd[1] = adfGeoTransform[2] / cfactor; - cd[2] = adfGeoTransform[4] / cfactor; - cd[3] = adfGeoTransform[5] / cfactor; + cd[0] = m_adfGeoTransform[1] / cfactor; + cd[1] = m_adfGeoTransform[2] / cfactor; + cd[2] = m_adfGeoTransform[4] / cfactor; + cd[3] = m_adfGeoTransform[5] / cfactor; double pc[4]; pc[0] = 1.; @@ -1083,7 +1067,7 @@ void FITSDataset::WriteFITSInfo() pc[2] = cd[2] / cd[3]; pc[3] = - 1.; - fits_update_key( hFITS, TDOUBLE, "CDELT1", &cd[0], nullptr, &status); + fits_update_key( m_hFITS, TDOUBLE, "CDELT1", &cd[0], nullptr, &status); if (status) { // Throw a warning with CFITSIO error status, then ignore status @@ -1094,7 +1078,7 @@ void FITSDataset::WriteFITSInfo() return; } - fits_update_key( hFITS, TDOUBLE, "CDELT2", &cd[3], nullptr, &status); + fits_update_key( m_hFITS, TDOUBLE, "CDELT2", &cd[3], nullptr, &status); if (status) { // Throw a warning with CFITSIO error status, then ignore status @@ -1105,7 +1089,7 @@ void FITSDataset::WriteFITSInfo() return; } - fits_update_key( hFITS, TDOUBLE, "PC1_1", &pc[0], nullptr, &status); + fits_update_key( m_hFITS, TDOUBLE, "PC1_1", &pc[0], nullptr, &status); if (status) { // Throw a warning with CFITSIO error status, then ignore status @@ -1116,7 +1100,7 @@ void FITSDataset::WriteFITSInfo() return; } - fits_update_key( hFITS, TDOUBLE, "PC1_2", &pc[1], nullptr, &status); + fits_update_key( m_hFITS, TDOUBLE, "PC1_2", &pc[1], nullptr, &status); if (status) { // Throw a warning with CFITSIO error status, then ignore status @@ -1127,7 +1111,7 @@ void FITSDataset::WriteFITSInfo() return; } - fits_update_key( hFITS, TDOUBLE, "PC2_1", &pc[2], nullptr, &status); + fits_update_key( m_hFITS, TDOUBLE, "PC2_1", &pc[2], nullptr, &status); if (status) { // Throw a warning with CFITSIO error status, then ignore status @@ -1138,7 +1122,7 @@ void FITSDataset::WriteFITSInfo() return; } - fits_update_key( hFITS, TDOUBLE, "PC2_2", &pc[3], nullptr, &status); + fits_update_key( m_hFITS, TDOUBLE, "PC2_2", &pc[3], nullptr, &status); if (status) { // Throw a warning with CFITSIO error status, then ignore status @@ -1160,7 +1144,7 @@ void FITSDataset::WriteFITSInfo() const OGRSpatialReference* FITSDataset::GetSpatialRef() const { - return oSRS.IsEmpty() ? nullptr : &oSRS; + return m_oSRS.IsEmpty() ? nullptr : &m_oSRS; } /************************************************************************/ @@ -1172,15 +1156,15 @@ CPLErr FITSDataset::SetSpatialRef( const OGRSpatialReference * poSRS ) { if( poSRS == nullptr || poSRS->IsEmpty() ) { - oSRS.Clear(); + m_oSRS.Clear(); } else { - oSRS = *poSRS; - oSRS.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); + m_oSRS = *poSRS; + m_oSRS.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); } - bFITSInfoChanged = true; + m_bFITSInfoChanged = true; return CE_None; } @@ -1192,9 +1176,9 @@ CPLErr FITSDataset::SetSpatialRef( const OGRSpatialReference * poSRS ) CPLErr FITSDataset::GetGeoTransform( double * padfTransform ) { - memcpy( padfTransform, adfGeoTransform, sizeof(double) * 6 ); + memcpy( padfTransform, m_adfGeoTransform, sizeof(double) * 6 ); - if( !bGeoTransformValid ) + if( !m_bGeoTransformValid ) return CE_Failure; return CE_None; @@ -1207,10 +1191,8 @@ CPLErr FITSDataset::GetGeoTransform( double * padfTransform ) CPLErr FITSDataset::SetGeoTransform( double * padfTransform ) { - bGeoTransformValid = false; - - memcpy( adfGeoTransform, padfTransform, sizeof(double)*6 ); - bGeoTransformValid = true; + memcpy( m_adfGeoTransform, padfTransform, sizeof(double)*6 ); + m_bGeoTransformValid = true; return CE_None; } @@ -1223,8 +1205,8 @@ double FITSRasterBand::GetOffset( int *pbSuccess ) { if( pbSuccess ) - *pbSuccess = bHaveOffsetScale; - return dfOffset; + *pbSuccess = m_bHaveOffsetScale; + return m_dfOffset; } /************************************************************************/ @@ -1234,11 +1216,11 @@ double FITSRasterBand::GetOffset( int *pbSuccess ) CPLErr FITSRasterBand::SetOffset( double dfNewValue ) { - if( !bHaveOffsetScale || dfNewValue != dfOffset ) - poFDS->bMetadataChanged = true; + if( !m_bHaveOffsetScale || dfNewValue != m_dfOffset ) + m_poFDS->m_bMetadataChanged = true; - bHaveOffsetScale = true; - dfOffset = dfNewValue; + m_bHaveOffsetScale = true; + m_dfOffset = dfNewValue; return CE_None; } @@ -1250,8 +1232,8 @@ double FITSRasterBand::GetScale( int *pbSuccess ) { if( pbSuccess ) - *pbSuccess = bHaveOffsetScale; - return dfScale; + *pbSuccess = m_bHaveOffsetScale; + return m_dfScale; } /************************************************************************/ @@ -1261,11 +1243,11 @@ double FITSRasterBand::GetScale( int *pbSuccess ) CPLErr FITSRasterBand::SetScale( double dfNewValue ) { - if( !bHaveOffsetScale || dfNewValue != dfScale ) - poFDS->bMetadataChanged = true; + if( !m_bHaveOffsetScale || dfNewValue != m_dfScale ) + m_poFDS->m_bMetadataChanged = true; - bHaveOffsetScale = true; - dfScale = dfNewValue; + m_bHaveOffsetScale = true; + m_dfScale = dfNewValue; return CE_None; } @@ -1276,20 +1258,20 @@ CPLErr FITSRasterBand::SetScale( double dfNewValue ) double FITSRasterBand::GetNoDataValue( int * pbSuccess ) { - if( bNoDataSet ) + if( m_bNoDataSet ) { if( pbSuccess ) *pbSuccess = TRUE; - return dfNoDataValue; + return m_dfNoDataValue; } - if( poFDS->bNoDataSet ) + if( m_poFDS->m_bNoDataSet ) { if( pbSuccess ) *pbSuccess = TRUE; - return poFDS->dfNoDataValue; + return m_poFDS->m_dfNoDataValue; } return GDALPamRasterBand::GetNoDataValue( pbSuccess ); @@ -1302,20 +1284,20 @@ double FITSRasterBand::GetNoDataValue( int * pbSuccess ) CPLErr FITSRasterBand::SetNoDataValue( double dfNoData ) { - if( poFDS->bNoDataSet && poFDS->dfNoDataValue == dfNoData ) + if( m_poFDS->m_bNoDataSet && m_poFDS->m_dfNoDataValue == dfNoData ) { - bNoDataSet = true; - dfNoDataValue = dfNoData; + m_bNoDataSet = true; + m_dfNoDataValue = dfNoData; return CE_None; } - poFDS->bNoDataSet = true; - poFDS->dfNoDataValue = dfNoData; + m_poFDS->m_bNoDataSet = true; + m_poFDS->m_dfNoDataValue = dfNoData; - poFDS->bNoDataChanged = true; + m_poFDS->m_bNoDataChanged = true; - bNoDataSet = true; - dfNoDataValue = dfNoData; + m_bNoDataSet = true; + m_dfNoDataValue = dfNoData; return CE_None; } @@ -1326,16 +1308,16 @@ CPLErr FITSRasterBand::SetNoDataValue( double dfNoData ) CPLErr FITSRasterBand::DeleteNoDataValue() { - if( !poFDS->bNoDataSet ) + if( !m_poFDS->m_bNoDataSet ) return CE_None; - poFDS->bNoDataSet = false; - poFDS->dfNoDataValue = -9999.0; + m_poFDS->m_bNoDataSet = false; + m_poFDS->m_dfNoDataValue = -9999.0; - poFDS->bNoDataChanged = true; + m_poFDS->m_bNoDataChanged = true; - bNoDataSet = false; - dfNoDataValue = -9999.0; + m_bNoDataSet = false; + m_dfNoDataValue = -9999.0; return CE_None; } @@ -1359,7 +1341,7 @@ void FITSDataset::LoadGeoreferencing() /* Get the transform from the FITS file. */ /* -------------------------------------------------------------------- */ - fits_read_key(hFITS, TSTRING, "OBJECT", target, nullptr, &status); + fits_read_key(m_hFITS, TSTRING, "OBJECT", target, nullptr, &status); if( status ) { strncpy(target, "Undefined", 10); @@ -1372,14 +1354,14 @@ void FITSDataset::LoadGeoreferencing() DatumName.assign("D_"); DatumName.append(target); - fits_read_key(hFITS, TDOUBLE, "A_RADIUS", &aRadius, nullptr, &status); + fits_read_key(m_hFITS, TDOUBLE, "A_RADIUS", &aRadius, nullptr, &status); if( status ) { CPLDebug("FITS", "No Radii keyword available, metadata will not contain DATUM information."); return; } else { - fits_read_key(hFITS, TDOUBLE, "C_RADIUS", &cRadius, nullptr, &status); + fits_read_key(m_hFITS, TDOUBLE, "C_RADIUS", &cRadius, nullptr, &status); if( status ) { CPLError(CE_Warning, CPLE_AppDefined, @@ -1395,16 +1377,16 @@ void FITSDataset::LoadGeoreferencing() /* Waiting for linear keywords standardization only deg ctype are used */ /* Check if WCS are there */ - fits_read_key(hFITS, TSTRING, "CTYPE1", ctype, nullptr, &status); + fits_read_key(m_hFITS, TSTRING, "CTYPE1", ctype, nullptr, &status); if ( !status ) { /* Check if angular WCS are there */ if ( strstr(ctype, "LN") ) { /* Reading reference points */ - fits_read_key(hFITS, TDOUBLE, "CRPIX1", &crpix1, nullptr, &status); - fits_read_key(hFITS, TDOUBLE, "CRPIX2", &crpix2, nullptr, &status); - fits_read_key(hFITS, TDOUBLE, "CRVAL1", &crval1, nullptr, &status); - fits_read_key(hFITS, TDOUBLE, "CRVAL2", &crval2, nullptr, &status); + fits_read_key(m_hFITS, TDOUBLE, "CRPIX1", &crpix1, nullptr, &status); + fits_read_key(m_hFITS, TDOUBLE, "CRPIX2", &crpix2, nullptr, &status); + fits_read_key(m_hFITS, TDOUBLE, "CRVAL1", &crval1, nullptr, &status); + fits_read_key(m_hFITS, TDOUBLE, "CRVAL2", &crval2, nullptr, &status); if( status ) { CPLError(CE_Failure, CPLE_AppDefined, @@ -1412,13 +1394,13 @@ void FITSDataset::LoadGeoreferencing() status = 0; } else { /* Check for CDELT and PC matrix representation */ - fits_read_key(hFITS, TDOUBLE, "CDELT1", &cdelt1, nullptr, &status); + fits_read_key(m_hFITS, TDOUBLE, "CDELT1", &cdelt1, nullptr, &status); if ( ! status ) { - fits_read_key(hFITS, TDOUBLE, "CDELT2", &cdelt2, nullptr, &status); - fits_read_key(hFITS, TDOUBLE, "PC1_1", &pc[0], nullptr, &status); - fits_read_key(hFITS, TDOUBLE, "PC1_2", &pc[1], nullptr, &status); - fits_read_key(hFITS, TDOUBLE, "PC2_1", &pc[2], nullptr, &status); - fits_read_key(hFITS, TDOUBLE, "PC2_2", &pc[3], nullptr, &status); + fits_read_key(m_hFITS, TDOUBLE, "CDELT2", &cdelt2, nullptr, &status); + fits_read_key(m_hFITS, TDOUBLE, "PC1_1", &pc[0], nullptr, &status); + fits_read_key(m_hFITS, TDOUBLE, "PC1_2", &pc[1], nullptr, &status); + fits_read_key(m_hFITS, TDOUBLE, "PC2_1", &pc[2], nullptr, &status); + fits_read_key(m_hFITS, TDOUBLE, "PC2_2", &pc[3], nullptr, &status); cd[0] = cdelt1 * pc[0]; cd[1] = cdelt1 * pc[1]; cd[2] = cdelt2 * pc[2]; @@ -1426,31 +1408,31 @@ void FITSDataset::LoadGeoreferencing() status = 0; } else { /* Look for CD matrix representation */ - fits_read_key(hFITS, TDOUBLE, "CD1_1", &cd[0], nullptr, &status); - fits_read_key(hFITS, TDOUBLE, "CD1_2", &cd[1], nullptr, &status); - fits_read_key(hFITS, TDOUBLE, "CD2_1", &cd[2], nullptr, &status); - fits_read_key(hFITS, TDOUBLE, "CD2_2", &cd[3], nullptr, &status); + fits_read_key(m_hFITS, TDOUBLE, "CD1_1", &cd[0], nullptr, &status); + fits_read_key(m_hFITS, TDOUBLE, "CD1_2", &cd[1], nullptr, &status); + fits_read_key(m_hFITS, TDOUBLE, "CD2_1", &cd[2], nullptr, &status); + fits_read_key(m_hFITS, TDOUBLE, "CD2_2", &cd[3], nullptr, &status); } double radfac = DEG2RAD * aRadius; - adfGeoTransform[1] = cd[0] * radfac; - adfGeoTransform[2] = cd[1] * radfac; - adfGeoTransform[4] = cd[2] * radfac; - adfGeoTransform[5] = - cd[3] * radfac ; + m_adfGeoTransform[1] = cd[0] * radfac; + m_adfGeoTransform[2] = cd[1] * radfac; + m_adfGeoTransform[4] = cd[2] * radfac; + m_adfGeoTransform[5] = - cd[3] * radfac ; if ( crval1 > 180. ) { crval1 = crval1 - 180.; } /* NOTA BENE: FITS standard define pixel integers at the center of the pixel, 0.5 must be subtract to have UpperLeft corner */ - adfGeoTransform[0] = crval1 * radfac - adfGeoTransform[1] * (crpix1-0.5); + m_adfGeoTransform[0] = crval1 * radfac - m_adfGeoTransform[1] * (crpix1-0.5); // assuming that center latitude is also the origin of the coordinate // system: this is not always true. // More generic implementation coming soon - adfGeoTransform[3] = - adfGeoTransform[5] * (crpix2-0.5); + m_adfGeoTransform[3] = - m_adfGeoTransform[5] * (crpix2-0.5); //+ crval2 * radfac; - bGeoTransformValid = true; + m_bGeoTransformValid = true; } char* pstr = strrchr(ctype, '-'); @@ -1465,13 +1447,13 @@ void FITSDataset::LoadGeoreferencing() /* Sinusoidal / SFL projection */ if( strcmp(pstr,"SFL" ) == 0 ) { projName.assign("Sinusoidal_"); - oSRS.SetSinusoidal(crval1, falseEast, falseNorth); + m_oSRS.SetSinusoidal(crval1, falseEast, falseNorth); /* Mercator, Oblique (Hotine) Mercator, Transverse Mercator */ /* Mercator / MER projection */ } else if( strcmp(pstr,"MER" ) == 0 ) { projName.assign("Mercator_"); - oSRS.SetMercator(crval2, crval1, scale, falseEast, falseNorth); + m_oSRS.SetMercator(crval2, crval1, scale, falseEast, falseNorth); /* Equirectangular / CAR projection */ } else if( strcmp(pstr,"CAR" ) == 0 ) { @@ -1482,43 +1464,43 @@ void FITSDataset::LoadGeoreferencing() But FITS WCS only supports projections on the sphere we assume here that the local radius is the one computed at the projection center */ - oSRS.SetEquirectangular2(crval2, crval1, crval2, falseEast, falseNorth); + m_oSRS.SetEquirectangular2(crval2, crval1, crval2, falseEast, falseNorth); /* Lambert Azimuthal Equal Area / ZEA projection */ } else if( strcmp(pstr,"ZEA" ) == 0 ) { projName.assign("Lambert_Azimuthal_Equal_Area_"); - oSRS.SetLAEA(crval2, crval1, falseEast, falseNorth); + m_oSRS.SetLAEA(crval2, crval1, falseEast, falseNorth); /* Lambert Conformal Conic 1SP / COO projection */ } else if( strcmp(pstr,"COO" ) == 0 ) { projName.assign("Lambert_Conformal_Conic_1SP_"); - oSRS.SetLCC1SP (crval2, crval1, scale, falseEast, falseNorth); + m_oSRS.SetLCC1SP (crval2, crval1, scale, falseEast, falseNorth); /* Orthographic / SIN projection */ } else if( strcmp(pstr,"SIN" ) == 0 ) { projName.assign("Orthographic_"); - oSRS.SetOrthographic(crval2, crval1, falseEast, falseNorth); + m_oSRS.SetOrthographic(crval2, crval1, falseEast, falseNorth); /* Point Perspective / AZP projection */ } else if( strcmp(pstr,"AZP" ) == 0 ) { projName.assign("perspective_point_height_"); - oSRS.SetProjection(SRS_PP_PERSPECTIVE_POINT_HEIGHT); + m_oSRS.SetProjection(SRS_PP_PERSPECTIVE_POINT_HEIGHT); /* # appears to need height... maybe center lon/lat */ /* Polar Stereographic / STG projection */ } else if( strcmp(pstr,"STG" ) == 0 ) { projName.assign("Polar_Stereographic_"); - oSRS.SetStereographic(crval2, crval1, scale, falseEast, falseNorth); + m_oSRS.SetStereographic(crval2, crval1, scale, falseEast, falseNorth); } else { CPLError(CE_Failure, CPLE_AppDefined, "Unknown projection."); } projName.append(target); - oSRS.SetProjParm(SRS_PP_FALSE_EASTING,0.0); - oSRS.SetProjParm(SRS_PP_FALSE_NORTHING,0.0); + m_oSRS.SetProjParm(SRS_PP_FALSE_EASTING,0.0); + m_oSRS.SetProjParm(SRS_PP_FALSE_NORTHING,0.0); - oSRS.SetNode("PROJCS",projName.c_str()); + m_oSRS.SetNode("PROJCS",projName.c_str()); - oSRS.SetGeogCS(GeogName.c_str(), DatumName.c_str(), target, aRadius, invFlattening, + m_oSRS.SetGeogCS(GeogName.c_str(), DatumName.c_str(), target, aRadius, invFlattening, "Reference_Meridian", 0.0, "degree", 0.0174532925199433); } else { CPLError(CE_Failure, CPLE_AppDefined, "Unknown projection."); @@ -1543,24 +1525,24 @@ void FITSDataset::LoadFITSInfo() LoadGeoreferencing(); - CPLAssert(!bMetadataChanged); - CPLAssert(!bNoDataChanged); + CPLAssert(!m_bMetadataChanged); + CPLAssert(!m_bNoDataChanged); - bMetadataChanged = false; - bNoDataChanged = false; + m_bMetadataChanged = false; + m_bNoDataChanged = false; - bitpix = this->fitsDataType; + bitpix = this->m_fitsDataType; FITSRasterBand *poBand = cpl::down_cast(GetRasterBand(1)); if (bitpix != TUSHORT && bitpix != TUINT) { - fits_read_key(hFITS, TDOUBLE, "BSCALE", &dfScale, nullptr, &status); + fits_read_key(m_hFITS, TDOUBLE, "BSCALE", &dfScale, nullptr, &status); if( status ) { status = 0; dfScale = 1.; } - fits_read_key(hFITS, TDOUBLE, "BZERO", &dfOffset, nullptr, &status); + fits_read_key(m_hFITS, TDOUBLE, "BZERO", &dfOffset, nullptr, &status); if( status ) { status = 0; @@ -1568,14 +1550,14 @@ void FITSDataset::LoadFITSInfo() } if ( dfScale != 1. || dfOffset != 0. ) { - poBand->bHaveOffsetScale = true; - poBand->dfScale = dfScale; - poBand->dfOffset = dfOffset; + poBand->m_bHaveOffsetScale = true; + poBand->m_dfScale = dfScale; + poBand->m_dfOffset = dfOffset; } } - fits_read_key(hFITS, TDOUBLE, "BLANK", &dfNoDataValue, nullptr, &status); - bNoDataSet = !status; + fits_read_key(m_hFITS, TDOUBLE, "BLANK", &m_dfNoDataValue, nullptr, &status); + m_bNoDataSet = !status; } /************************************************************************/ From 1a13b3099a3ff0d17df8df85e14bb723dc09d251 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 14 Sep 2020 12:43:04 +0200 Subject: [PATCH 033/255] FITS: add support for reading multiple-extension FITS files through subdatasets Funded by CNES --- autotest/conftest.py | 3 +- .../gdrivers/data/fits/empty_primary_hdu.fits | 1 + .../fits/image_in_first_and_second_hdu.fits | Bin 0 -> 11520 bytes ..._second_and_fourth_hdu_table_in_third.fits | Bin 0 -> 20160 bytes .../data/fits/image_in_second_hdu.fits | Bin 0 -> 8640 bytes autotest/gdrivers/fits.py | 82 ++- autotest/gdrivers/generate_fits.py | 72 ++ gdal/doc/source/drivers/raster/fits.rst | 60 +- gdal/frmts/fits/fitsdataset.cpp | 626 ++++++++++++------ 9 files changed, 635 insertions(+), 209 deletions(-) create mode 100644 autotest/gdrivers/data/fits/empty_primary_hdu.fits create mode 100644 autotest/gdrivers/data/fits/image_in_first_and_second_hdu.fits create mode 100644 autotest/gdrivers/data/fits/image_in_second_and_fourth_hdu_table_in_third.fits create mode 100644 autotest/gdrivers/data/fits/image_in_second_hdu.fits create mode 100644 autotest/gdrivers/generate_fits.py diff --git a/autotest/conftest.py b/autotest/conftest.py index 5939f4e6f3fc..fd88131436f7 100755 --- a/autotest/conftest.py +++ b/autotest/conftest.py @@ -19,7 +19,8 @@ # So we skip searching them during test collection. collect_ignore = ["kml_generate_test_files.py", "gdrivers/netcdf_cfchecks.py", - "gdrivers/generate_bag.py"] + "gdrivers/generate_bag.py", + "gdrivers/generate_fits.py"] # we set ECW to not resolve projection and datum strings to get 3.x behavior. gdal.SetConfigOption("ECW_DO_NOT_RESOLVE_DATUM_PROJECTION", "YES") diff --git a/autotest/gdrivers/data/fits/empty_primary_hdu.fits b/autotest/gdrivers/data/fits/empty_primary_hdu.fits new file mode 100644 index 000000000000..0fbca37546cd --- /dev/null +++ b/autotest/gdrivers/data/fits/empty_primary_hdu.fits @@ -0,0 +1 @@ +SIMPLE = T / file does conform to FITS standard BITPIX = 16 / number of bits per data pixel NAXIS = 0 / number of data axes EXTEND = T / FITS dataset may contain extensions COMMENT FITS (Flexible Image Transport System) format is defined in 'AstronomyCOMMENT and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H END \ No newline at end of file diff --git a/autotest/gdrivers/data/fits/image_in_first_and_second_hdu.fits b/autotest/gdrivers/data/fits/image_in_first_and_second_hdu.fits new file mode 100644 index 0000000000000000000000000000000000000000..ee015a445154d7f13db67e8f4a86e7320fe93043 GIT binary patch literal 11520 zcmeI0U2oGc6oxx~$+@B+OTWOIeXw(^5H08=mLF3EHTgERImkOA;)ilxMB`8z1o+jM(u`9~ljA)WI7u zuaNrA-TVFh<|#{D-!&TExP#~S*T&=et`RP>F!|8I^ZjVl`>k2&s1&ttixWzd+&QVO zGWzKJH?uenlZ=1YeE4Eo$emhpOQTZUNy;jr?Z#A&(%L#%z9ipb;fPP2%eB%nCua8p z>nfv7S?_t=d}@|A_qA0k+droVQ*28~lh?1#Y2)#e%d2bd^KxZ!`Ig4R;V5_>3P2-%GK|gm^O>J*>GScyb%r9IX5V@C$l9W(=BuZsq*5*+WGgjo$wUPFQSvAiaa7XpkB38hd~zjp3~fqGn^kk6(RRu;Ie@t``0$wY=1i_qyGVvgyU( z$dA{Y3;Kx&5CI}U1c(3;*pI-$;Q=)wKm>>Y5g-CYfCvx)B0vQ8Bv8mdpF1PZJioTf z`MRlySh51iR8hk2H1y8=^Jx?F&M;lc`Z|jr?!UHvL*;kdO<#n0VFe4YmZ1>s4~^n? zUH-YH^d5wtr=Rl~V{t`otUL41n|hTben?fZMlM_os?gt=e@?}}TxG}8C7%hLah@el zI4g)0x;yjFafIbLs(@yZ#Mu&FKhHq-c-K+0yf3E9;|x7N+Q(55!H$DwYE1H-wt!Vzt)?|dU~$wA{xY~ zMbF@*sm?Os9bjMBd6`STw~kN0UugJortX1f`RxD3SMzp3@x%CXJ** z;|boN>;3c>)vFX~P@f180U|&Ih`_HQ@XTf(e*E@q3#}3XB0vO)01+SpM1Tko0V423 zppbv2`sbfr|J?bl>z`e_KOR{xEPyW5KU*C>xiDs~wO&+S_9g#E?DdS3Q`D%)@6!Jf vf6dc59fb%G0U|&Ih`{ej;NbAplb5Cr5g-CYfCvx)B0vO)01+Sp^#r~HZv5f# literal 0 HcmV?d00001 diff --git a/autotest/gdrivers/data/fits/image_in_second_hdu.fits b/autotest/gdrivers/data/fits/image_in_second_hdu.fits new file mode 100644 index 0000000000000000000000000000000000000000..392e7e1bf73956087ec66d96c6829b187404bcb5 GIT binary patch literal 8640 zcmeH~(Qeu>6o$JuIagzoT1cRobL zmJA2ud>ntt#gA)u`_FBFJIEOxlwcYz7zCUNnDBVYlL#b-2itQ&NIwqzB*82866?U1J}ko(TVzoBL(dx8E>^ky85y3&_$%!6$Y>9}cCU4S zr_QU7rci=Oi2SuWw)Ddo*h;ckggnmhn@+de?s@o&`ob>{3$_Z!sLYQ1R|a0<$70D7 z3GP}*7JUV!(3cPj2-q}?Spdi`SVAT|=Fxh`L%E=rm$S79C!%l(Z+wwP3@YDhm#|d$ zO0|BA9>)_Num+S3!?ZqI#bOb6;7hgMfd|LYHtwKsZ++Kg;fS6Ozd~MFZQAg3c^0qP zX*ZwTK^J+W#f$6DU)&!!M>rL?1fA9ZCNnmfi!_3CiFXE-m5s7-W!zkuhG$kAm0IJv zw(Y!Sc^2YSf7=xRV^@|o4_g|C@-(V`Sl&rgkO4A42FL&zAOmCoK*dSdc6yjM?5_2Q zNqR4MsQj{7eGiWxIXjl0;U<%J&(F{04DY1;tn;{eAU~VPTd??5&Q!kN@-XS*{zF6a zFh=I)d(~eaPsh%DV!!DOR2~P|K>?>3sEX0?l*FQkH>iGya##t zhtB5lpZtv_Nf@wQLe1^s@Z#T4uk-jw@B3$#x_U2<*Sn5Ye|o;gsopD!drls&RjU;h v)A#$MJg%hEAOmE843Ggb@ShpD_;f*o43GgbKnBPF86X2>fDDj<6Ak # ############################################################################### -# Copyright (c) 2008, Even Rouault +# Copyright (c) 2008,2020, Even Rouault # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -28,10 +28,11 @@ # DEALINGS IN THE SOFTWARE. ############################################################################### +import os import pytest from osgeo import gdal - +import gdaltest pytestmark = pytest.mark.require_driver('FITS') @@ -131,3 +132,80 @@ def test_fits_read_georef_merc(): assert wkt == 'PROJCS["Mercator_Earth",GEOGCS["GCS_Earth",DATUM["D_Earth",SPHEROID["Earth",6378206.4,294.978698213898]],PRIMEM["Reference_Meridian",0],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]]],PROJECTION["Mercator_1SP"],PARAMETER["central_meridian",0],PARAMETER["scale_factor",1],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]],AXIS["Easting",EAST],AXIS["Northing",NORTH]]' gt = ds.GetGeoTransform() assert gt == pytest.approx((-13095897.481058259, 72.23522015778646, 0.0, 3991653.2130816197, 0.0, -72.23522015778646), abs=1e-3) + + +def test_fits_read_empty_primary_hdu(): + + filename = 'data/fits/empty_primary_hdu.fits' + assert os.path.exists(filename) + with gdaltest.error_handler(): + assert gdal.Open(filename) is None + +def test_fits_read_image_in_second_hdu(): + + ds = gdal.Open('data/fits/image_in_second_hdu.fits') + assert gdal.GetLastErrorMsg() == '' + assert ds is not None + assert ds.RasterXSize == 1 + assert ds.RasterYSize == 2 + assert ds.RasterCount == 1 + assert ds.GetMetadata() == {'BAR': 'BAZ ', 'FOO': 'BAR_override', 'FOO2': 'BAR2 '} + + +def test_fits_read_image_in_first_and_second_hdu(): + + ds = gdal.Open('data/fits/image_in_first_and_second_hdu.fits') + assert gdal.GetLastErrorMsg() == '' + assert ds is not None + assert ds.RasterCount == 0 + assert ds.GetMetadata() == {'EXTNAME': 'FIRST_IMAGE'} + + sub_ds = ds.GetSubDatasets() + assert len(sub_ds) == 2 + assert sub_ds[0][0] == 'FITS:"data/fits/image_in_first_and_second_hdu.fits":1' + assert sub_ds[0][1] == 'HDU 1 (1x2, 1 band), FIRST_IMAGE' + assert sub_ds[1][0] == 'FITS:"data/fits/image_in_first_and_second_hdu.fits":2' + assert sub_ds[1][1] == 'HDU 2 (1x3, 1 band)' + + with gdaltest.error_handler(): + assert gdal.Open('FITS:') is None + assert gdal.Open('FITS:"data/fits/image_in_first_and_second_hdu.fits"') is None + assert gdal.Open('FITS:"data/fits/image_in_first_and_second_hdu.fits":0') is None + assert gdal.Open('FITS:"data/fits/image_in_first_and_second_hdu.fits":3') is None + assert gdal.Open('FITS:"data/fits/non_existing.fits":1') is None + + ds = gdal.Open('FITS:"data/fits/image_in_first_and_second_hdu.fits":1') + assert ds.GetSubDatasets() == [] + assert ds.RasterXSize == 1 + assert ds.RasterYSize == 2 + assert ds.GetMetadata() == {'EXTNAME': 'FIRST_IMAGE'} + + ds = gdal.Open('FITS:"data/fits/image_in_first_and_second_hdu.fits":2') + assert ds.GetSubDatasets() == [] + assert ds.RasterXSize == 1 + assert ds.RasterYSize == 3 + assert ds.GetMetadata() == {} + + +def test_fits_read_image_in_second_and_fourth_hdu_table_in_third(): + + ds = gdal.Open('data/fits/image_in_second_and_fourth_hdu_table_in_third.fits') + assert gdal.GetLastErrorMsg() == '' + assert ds is not None + assert ds.RasterCount == 0 + assert ds.GetMetadata() == {'FOO': 'BAR '} + + sub_ds = ds.GetSubDatasets() + assert len(sub_ds) == 2 + + ds = gdal.Open(sub_ds[0][0]) + assert ds.GetSubDatasets() == [] + assert ds.RasterXSize == 1 + assert ds.RasterYSize == 2 + assert ds.GetMetadata() == {'EXTNAME': 'FIRST_IMAGE', 'FOO': 'BAR '} + + ds = gdal.Open(sub_ds[1][0]) + assert ds.GetSubDatasets() == [] + assert ds.RasterXSize == 1 + assert ds.RasterYSize == 3 + assert ds.GetMetadata() == {'EXTNAME': 'SECOND_IMAGE', 'FOO': 'BAR '} diff --git a/autotest/gdrivers/generate_fits.py b/autotest/gdrivers/generate_fits.py new file mode 100644 index 000000000000..8f72719d1819 --- /dev/null +++ b/autotest/gdrivers/generate_fits.py @@ -0,0 +1,72 @@ +#!/usr/bin/env pytest +# -*- coding: utf-8 -*- +############################################################################### +# $Id$ +# +# Project: GDAL/OGR Test Suite +# Purpose: Generate FITS samples +# Author: Even Rouault +# +############################################################################### +# Copyright (c) 2020, Even Rouault +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +############################################################################### + +import fitsio +import os +import numpy as np + +data_dir = os.path.join(os.path.dirname(__file__), 'data', 'fits') + +fitsio.write(os.path.join(data_dir, 'empty_primary_hdu.fits'), data=None, clobber=True) + +filename = os.path.join(data_dir, 'image_in_second_hdu.fits') +with fitsio.FITS(filename,'rw',clobber=True) as fits: + fits.write(data=None, header={'FOO': 'BAR', 'FOO2': 'BAR2'}, clobber=True) + fits[-1].write_checksum() + img = np.arange(2,dtype='B').reshape(2,1) + fits.write(data=img, header={'FOO':'BAR_override', 'BAR':'BAZ'}) + fits[-1].write_checksum() + +filename = os.path.join(data_dir, 'image_in_first_and_second_hdu.fits') +with fitsio.FITS(filename,'rw',clobber=True) as fits: + img = np.arange(2,dtype='B').reshape(2,1) + fits.write(data=img, extname='FIRST_IMAGE') + img = np.arange(3,dtype='B').reshape(3,1) + fits.write(data=img) + +filename = os.path.join(data_dir, 'image_in_second_and_fourth_hdu_table_in_third.fits') +with fitsio.FITS(filename,'rw',clobber=True) as fits: + fits.write(data=None, header={'FOO': 'BAR'}) + fits[-1].write_checksum() + + img = np.arange(2,dtype='B').reshape(2,1) + fits.write(data=img, extname='FIRST_IMAGE') + fits[-1].write_checksum() + + nrows=2 + data = np.zeros(nrows, dtype=[('int','i4'),('double','f8')]) + data['int'] = np.arange(nrows,dtype='i4') + data['double'] = np.arange(nrows,dtype='f8') + fits.write_table(data) + + img = np.arange(3,dtype='B').reshape(3,1) + fits.write(data=img, extname='SECOND_IMAGE') + fits[-1].write_checksum() diff --git a/gdal/doc/source/drivers/raster/fits.rst b/gdal/doc/source/drivers/raster/fits.rst index 3084039ce2cc..68be9c38662a 100644 --- a/gdal/doc/source/drivers/raster/fits.rst +++ b/gdal/doc/source/drivers/raster/fits.rst @@ -43,6 +43,64 @@ are managed via standard :cpp:func:`GDALRasterBand::GetOffset` / :cpp:func:`GDAL and :cpp:func:`GDALRasterBand::GetScale` / :cpp:func:`GDALRasterBand::SetScale` GDAL functions and no more referred as metadata. +Multiple image support +---------------------- + +Starting with GDAL 3.2, Multi-Extension FITS (MEF) files that contain one or +more extensions following the primary HDU are supported. When more than 2 image +HDUs are found, they are reported as subdatasets. + +The connection string for a given subdataset/HDU is ``FITS:"filename.fits":hdu_number`` + +Examples +-------- + +* Listing subdatasets in a MEF .fits: + + :: + + $ gdalinfo ../autotest/gdrivers/data/fits/image_in_first_and_second_hdu.fits + + Driver: FITS/Flexible Image Transport System + Files: ../autotest/gdrivers/data/fits/image_in_first_and_second_hdu.fits + Size is 512, 512 + Metadata: + EXTNAME=FIRST_IMAGE + Subdatasets: + SUBDATASET_1_NAME=FITS:"../autotest/gdrivers/data/fits/image_in_first_and_second_hdu.fits":1 + SUBDATASET_1_DESC=HDU 1 (1x2, 1 band), FIRST_IMAGE + SUBDATASET_2_NAME=FITS:"../autotest/gdrivers/data/fits/image_in_first_and_second_hdu.fits":2 + SUBDATASET_2_DESC=HDU 2 (1x3, 1 band) + Corner Coordinates: + Upper Left ( 0.0, 0.0) + Lower Left ( 0.0, 512.0) + Upper Right ( 512.0, 0.0) + Lower Right ( 512.0, 512.0) + Center ( 256.0, 256.0) + +* Opening a given HDU: + + :: + + $ gdalinfo FITS:"../autotest/gdrivers/data/fits/image_in_first_and_second_hdu.fits":1 + + Driver: FITS/Flexible Image Transport System + Files: none associated + Size is 1, 2 + Metadata: + EXTNAME=FIRST_IMAGE + Corner Coordinates: + Upper Left ( 0.0, 0.0) + Lower Left ( 0.0, 2.0) + Upper Right ( 1.0, 0.0) + Lower Right ( 1.0, 2.0) + Center ( 0.5, 1.0) + Band 1 Block=1x1 Type=Byte, ColorInterp=Undefined + + +Other +----- + NOTE: Implemented as ``gdal/frmts/fits/fitsdataset.cpp``. .. _notes-on-cfitsio-linking: @@ -59,8 +117,6 @@ From distros """""""""""" On Fedora/CentOS install CFITSIO then GDAL with dnf (yum): cfitsio is automatically linked. -Starting from Debian 10, Ubuntu 18.04 GDAL is packaged disabling CFITSIO link (see https://bugs.debian.org/422537): having GDAL linked against CFITSIO asks for recompile from source. - MacOSX ^^^^^^ The last versions of the MacOSX packages are not linked against CFITSIO. diff --git a/gdal/frmts/fits/fitsdataset.cpp b/gdal/frmts/fits/fitsdataset.cpp index 1991113f9813..ed271b1ea529 100644 --- a/gdal/frmts/fits/fitsdataset.cpp +++ b/gdal/frmts/fits/fitsdataset.cpp @@ -6,7 +6,7 @@ * ****************************************************************************** * Copyright (c) 2001, Simon Perkins - * Copyright (c) 2008-2018, Even Rouault + * Copyright (c) 2008-2020, Even Rouault * Copyright (c) 2018, Chiara Marmo * * Permission is hereby granted, free of charge, to any person obtaining a @@ -56,6 +56,7 @@ class FITSDataset final : public GDALPamDataset { fitsfile* m_hFITS = nullptr; + int m_hduNum = 0; GDALDataType m_gdalDataType = GDT_Unknown; // GDAL code for the image type int m_fitsDataType = 0; // FITS code for the image type @@ -68,25 +69,30 @@ class FITSDataset final : public GDALPamDataset { bool m_bMetadataChanged = false; - FITSDataset(); // Others should not call this constructor explicitly - - CPLErr Init(fitsfile* hFITS, bool isExistingFile); + CPLStringList m_aosSubdatasets{}; OGRSpatialReference m_oSRS{}; double m_adfGeoTransform[6]; bool m_bGeoTransformValid = false; - void WriteFITSInfo(); bool m_bFITSInfoChanged = false; + + FITSDataset(); // Others should not call this constructor explicitly + + CPLErr Init(fitsfile* hFITS, bool isExistingFile, int hduNum); + void LoadGeoreferencing(); void LoadFITSInfo(); + void WriteFITSInfo(); + void LoadMetadata(); public: ~FITSDataset(); static GDALDataset* Open( GDALOpenInfo* ); + static int Identify( GDALOpenInfo* ); static GDALDataset* Create( const char* pszFilename, int nXSize, int nYSize, int nBands, GDALDataType eType, @@ -95,6 +101,7 @@ class FITSDataset final : public GDALPamDataset { CPLErr SetSpatialRef(const OGRSpatialReference* poSRS) override; virtual CPLErr GetGeoTransform( double * ) override; virtual CPLErr SetGeoTransform( double * ) override; + char** GetMetadata(const char* papszDomain = nullptr) override; bool GetRawBinaryLayout(GDALDataset::RawBinaryLayout&) override; @@ -268,15 +275,15 @@ CPLErr FITSRasterBand::IWriteBlock( CPL_UNUSED int nBlockXOff, int nBlockYOff, // Simple static function to determine if FITS header keyword should // be saved in meta data. -constexpr int ignorableHeaderCount = 18; -static const char* const ignorableFITSHeaders[ignorableHeaderCount] = { +static const char* const ignorableFITSHeaders[] = { "SIMPLE", "BITPIX", "NAXIS", "NAXIS1", "NAXIS2", "NAXIS3", "END", "XTENSION", "PCOUNT", "GCOUNT", "EXTEND", "CONTINUE", - "COMMENT", "", "LONGSTRN", "BZERO", "BSCALE", "BLANK" + "COMMENT", "", "LONGSTRN", "BZERO", "BSCALE", "BLANK", + "CHECKSUM", "DATASUM", }; static bool isIgnorableFITSHeader(const char* name) { - for (int i = 0; i < ignorableHeaderCount; ++i) { - if (strcmp(name, ignorableFITSHeaders[i]) == 0) + for (const char* keyword: ignorableFITSHeaders) { + if (strcmp(name, keyword) == 0) return true; } return false; @@ -306,13 +313,13 @@ FITSDataset::~FITSDataset() { int status; if( m_hFITS ) { - if(eAccess == GA_Update) + if(m_hduNum > 0 && eAccess == GA_Update) { // Only do this if we've successfully opened the file and update // capability. Write any meta data to the file that's compatible with // FITS. status = 0; - fits_movabs_hdu(m_hFITS, 1, nullptr, &status); + fits_movabs_hdu(m_hFITS, m_hduNum, nullptr, &status); fits_write_key_longwarn(m_hFITS, &status); if (status) { CPLError(CE_Warning, CPLE_AppDefined, @@ -432,6 +439,8 @@ FITSDataset::~FITSDataset() { bool FITSDataset::GetRawBinaryLayout(GDALDataset::RawBinaryLayout& sLayout) { + if( m_hduNum == 0 ) + return false; int status = 0; if( fits_is_compressed_image( m_hFITS, &status) ) return false; @@ -459,174 +468,227 @@ bool FITSDataset::GetRawBinaryLayout(GDALDataset::RawBinaryLayout& sLayout) /* Init() */ /************************************************************************/ -CPLErr FITSDataset::Init(fitsfile* hFITS, bool isExistingFile) { +CPLErr FITSDataset::Init(fitsfile* hFITS, bool isExistingFile, int hduNum) { - m_hFITS = hFITS; - m_isExistingFile = isExistingFile; - m_highestOffsetWritten = 0; - int status = 0; + m_hFITS = hFITS; + m_isExistingFile = isExistingFile; - // Move to the primary HDU - fits_movabs_hdu(m_hFITS, 1, nullptr, &status); - if (status) { - CPLError(CE_Failure, CPLE_AppDefined, - "Couldn't move to first HDU in FITS file %s (%d).\n", - GetDescription(), status); - return CE_Failure; - } - - // Get the image info for this dataset (note that all bands in a FITS dataset - // have the same type) - int bitpix; - int naxis; - const int maxdim = 3; - long naxes[maxdim]; - double offset; - - fits_get_img_param(hFITS, maxdim, &bitpix, &naxis, naxes, &status); - if (status) { - CPLError(CE_Failure, CPLE_AppDefined, - "Couldn't determine image parameters of FITS file %s (%d).", - GetDescription(), status); - return CE_Failure; - } - fits_read_key(hFITS, TDOUBLE, "BZERO", &offset, nullptr, &status); - if( status ) - { - // BZERO is not mandatory offset defaulted to 0 if BZERO is missing - status = 0; - offset = 0.; - } + int status = 0; + double offset; - fits_read_key(hFITS, TDOUBLE, "BLANK", &m_dfNoDataValue, nullptr, &status); - m_bNoDataSet = !status; - status = 0; + int hduType = 0; + fits_movabs_hdu(hFITS, hduNum, &hduType, &status); + if (status) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Couldn't move to HDU %d in FITS file %s (%d).", + hduNum, GetDescription(), status); + return CE_Failure; + } - // Determine data type and nodata value if BLANK keyword is absent - if (bitpix == BYTE_IMG) { - m_gdalDataType = GDT_Byte; - m_fitsDataType = TBYTE; - } - else if (bitpix == SHORT_IMG) { - if (offset == 32768.) + if( hduType != IMAGE_HDU ) { - m_gdalDataType = GDT_UInt16; - m_fitsDataType = TUSHORT; + CPLError(CE_Failure, CPLE_AppDefined, + "HDU %d is not an image.", hduNum); + return CE_Failure; } - else { - m_gdalDataType = GDT_Int16; - m_fitsDataType = TSHORT; + + // Get the image info for this dataset (note that all bands in a FITS dataset + // have the same type) + int bitpix = 0; + int naxis = 0; + const int maxdim = 3; + long naxes[maxdim] = {0,0,0}; + fits_get_img_param(hFITS, maxdim, &bitpix, &naxis, naxes, &status); + if (status) { + CPLError(CE_Failure, CPLE_AppDefined, + "Couldn't determine image parameters of FITS file %s (%d)", + GetDescription(), status); + return CE_Failure; } - } - else if (bitpix == LONG_IMG) { - if (offset == 2147483648.) + + m_hduNum = hduNum; + + fits_read_key(hFITS, TDOUBLE, "BZERO", &offset, nullptr, &status); + if( status ) { - m_gdalDataType = GDT_UInt32; - m_fitsDataType = TUINT; + // BZERO is not mandatory offset defaulted to 0 if BZERO is missing + status = 0; + offset = 0.; + } + + fits_read_key(hFITS, TDOUBLE, "BLANK", &m_dfNoDataValue, nullptr, &status); + m_bNoDataSet = !status; + status = 0; + + // Determine data type and nodata value if BLANK keyword is absent + if (bitpix == BYTE_IMG) { + m_gdalDataType = GDT_Byte; + m_fitsDataType = TBYTE; + } + else if (bitpix == SHORT_IMG) { + if (offset == 32768.) + { + m_gdalDataType = GDT_UInt16; + m_fitsDataType = TUSHORT; + } + else { + m_gdalDataType = GDT_Int16; + m_fitsDataType = TSHORT; + } + } + else if (bitpix == LONG_IMG) { + if (offset == 2147483648.) + { + m_gdalDataType = GDT_UInt32; + m_fitsDataType = TUINT; + } + else { + m_gdalDataType = GDT_Int32; + m_fitsDataType = TINT; + } + } + else if (bitpix == FLOAT_IMG) { + m_gdalDataType = GDT_Float32; + m_fitsDataType = TFLOAT; + } + else if (bitpix == DOUBLE_IMG) { + m_gdalDataType = GDT_Float64; + m_fitsDataType = TDOUBLE; } else { - m_gdalDataType = GDT_Int32; - m_fitsDataType = TINT; + CPLError(CE_Failure, CPLE_AppDefined, + "FITS file %s has unknown data type: %d.", GetDescription(), + bitpix); + return CE_Failure; } - } - else if (bitpix == FLOAT_IMG) { - m_gdalDataType = GDT_Float32; - m_fitsDataType = TFLOAT; - } - else if (bitpix == DOUBLE_IMG) { - m_gdalDataType = GDT_Float64; - m_fitsDataType = TDOUBLE; - } - else { - CPLError(CE_Failure, CPLE_AppDefined, - "FITS file %s has unknown data type: %d.", GetDescription(), - bitpix); - return CE_Failure; - } - // Determine image dimensions - we assume BSQ ordering - if (naxis == 2) { - nRasterXSize = static_cast(naxes[0]); - nRasterYSize = static_cast(naxes[1]); - nBands = 1; - } - else if (naxis == 3) { - nRasterXSize = static_cast(naxes[0]); - nRasterYSize = static_cast(naxes[1]); - nBands = static_cast(naxes[2]); - } - else { - CPLError(CE_Failure, CPLE_AppDefined, - "FITS file %s does not have 2 or 3 dimensions.", - GetDescription()); - return CE_Failure; - } - - // Create the bands - for (int i = 0; i < nBands; ++i) - SetBand(i+1, new FITSRasterBand(this, i+1)); - - // Read header information from file and use it to set metadata - // This process understands the CONTINUE standard for long strings. - // We don't bother to capture header names that duplicate information - // already captured elsewhere (e.g. image dimensions and type) - int keyNum; - char key[100]; - char value[100]; - - int nKeys = 0; - int nMoreKeys = 0; - fits_get_hdrspace(m_hFITS, &nKeys, &nMoreKeys, &status); - for(keyNum = 1; keyNum <= nKeys; keyNum++) - { - fits_read_keyn(m_hFITS, keyNum, key, value, nullptr, &status); - if (status) { - CPLError(CE_Failure, CPLE_AppDefined, - "Error while reading key %d from FITS file %s (%d)", - keyNum, GetDescription(), status); - return CE_Failure; + // Determine image dimensions - we assume BSQ ordering + if (naxis == 2) { + nRasterXSize = static_cast(naxes[0]); + nRasterYSize = static_cast(naxes[1]); + nBands = 1; } - if (strcmp(key, "END") == 0) { - // We should not get here in principle since the END - // keyword shouldn't be counted in nKeys, but who knows. - break; + else if (naxis == 3) { + nRasterXSize = static_cast(naxes[0]); + nRasterYSize = static_cast(naxes[1]); + nBands = static_cast(naxes[2]); } - else if (isIgnorableFITSHeader(key)) { - // Ignore it + else { + CPLError(CE_Failure, CPLE_AppDefined, + "FITS file %s does not have 2 or 3 dimensions.", + GetDescription()); + return CE_Failure; } - else { // Going to store something, but check for long strings etc - // Strip off leading and trailing quote if present - char* newValue = value; - if (value[0] == '\'' && value[strlen(value) - 1] == '\'') - { - newValue = value + 1; - value[strlen(value) - 1] = '\0'; - } - // Check for long string - if (strrchr(newValue, '&') == newValue + strlen(newValue) - 1) - { - // Value string ends in "&", so use long string conventions - char* longString = nullptr; - fits_read_key_longstr(hFITS, key, &longString, nullptr, &status); - // Note that read_key_longstr already strips quotes - if( status ) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Error while reading long string for key %s from " - "FITS file %s (%d)", key, GetDescription(), status); - return CE_Failure; - } - SetMetadataItem(key, longString); - free(longString); - } - else - { // Normal keyword - SetMetadataItem(key, newValue); - } + + // Create the bands + for (int i = 0; i < nBands; ++i) + SetBand(i+1, new FITSRasterBand(this, i+1)); + + return CE_None; +} + +/************************************************************************/ +/* LoadMetadata() */ +/************************************************************************/ + +void FITSDataset::LoadMetadata() +{ + // Read header information from file and use it to set metadata + // This process understands the CONTINUE standard for long strings. + // We don't bother to capture header names that duplicate information + // already captured elsewhere (e.g. image dimensions and type) + int keyNum; + char key[100]; + char value[100]; + CPLStringList aosMD; + + int nKeys = 0; + int nMoreKeys = 0; + int status = 0; + fits_get_hdrspace(m_hFITS, &nKeys, &nMoreKeys, &status); + for(keyNum = 1; keyNum <= nKeys; keyNum++) + { + fits_read_keyn(m_hFITS, keyNum, key, value, nullptr, &status); + if (status) { + CPLError(CE_Failure, CPLE_AppDefined, + "Error while reading key %d from FITS file %s (%d)", + keyNum, GetDescription(), status); + return; + } + if (strcmp(key, "END") == 0) { + // We should not get here in principle since the END + // keyword shouldn't be counted in nKeys, but who knows. + break; + } + else if (isIgnorableFITSHeader(key)) { + // Ignore it + } + else { // Going to store something, but check for long strings etc + // Strip off leading and trailing quote if present + char* newValue = value; + if (value[0] == '\'' && value[strlen(value) - 1] == '\'') + { + newValue = value + 1; + value[strlen(value) - 1] = '\0'; + } + // Check for long string + if (strrchr(newValue, '&') == newValue + strlen(newValue) - 1) + { + // Value string ends in "&", so use long string conventions + char* longString = nullptr; + fits_read_key_longstr(m_hFITS, key, &longString, nullptr, &status); + // Note that read_key_longstr already strips quotes + if( status ) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Error while reading long string for key %s from " + "FITS file %s (%d)", key, GetDescription(), status); + return; + } + SetMetadataItem(key, longString); + free(longString); + } + else + { // Normal keyword + SetMetadataItem(key, newValue); + } + } } - } +} - return CE_None; +/************************************************************************/ +/* Identify() */ +/************************************************************************/ + +int FITSDataset::Identify(GDALOpenInfo* poOpenInfo) +{ + if( STARTS_WITH(poOpenInfo->pszFilename, "FITS:") ) + return true; + + const char* fitsID = "SIMPLE = T"; // Spaces important! + const size_t fitsIDLen = strlen(fitsID); // Should be 30 chars long + + if (static_cast(poOpenInfo->nHeaderBytes) < fitsIDLen) + return false; + if (memcmp(poOpenInfo->pabyHeader, fitsID, fitsIDLen) != 0) + return false; + return true; +} + +/************************************************************************/ +/* GetMetadata() */ +/************************************************************************/ + +char **FITSDataset::GetMetadata( const char *pszDomain ) + +{ + if( pszDomain != nullptr && EQUAL(pszDomain,"SUBDATASETS") ) + { + return m_aosSubdatasets.List(); + } + + return GDALPamDataset::GetMetadata(pszDomain); } /************************************************************************/ @@ -635,58 +697,213 @@ CPLErr FITSDataset::Init(fitsfile* hFITS, bool isExistingFile) { GDALDataset* FITSDataset::Open(GDALOpenInfo* poOpenInfo) { - const char* fitsID = "SIMPLE = T"; // Spaces important! - size_t fitsIDLen = strlen(fitsID); // Should be 30 chars long + if( !Identify(poOpenInfo) ) + return nullptr; - if ((size_t)poOpenInfo->nHeaderBytes < fitsIDLen) - return nullptr; - if (memcmp(poOpenInfo->pabyHeader, fitsID, fitsIDLen)) - return nullptr; + CPLString osFilename(poOpenInfo->pszFilename); + int iSelectedHDU = 0; + if( STARTS_WITH(poOpenInfo->pszFilename, "FITS:") ) + { + const CPLStringList aosTokens( + CSLTokenizeString2(poOpenInfo->pszFilename, ":", + CSLT_HONOURSTRINGS | CSLT_PRESERVEESCAPES)); + if( aosTokens.size() != 3 ) + { + return nullptr; + } + osFilename = aosTokens[1]; + iSelectedHDU = atoi(aosTokens[2]); + if( iSelectedHDU <= 0 ) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid HDU number"); + return nullptr; + } + } - // Get access mode and attempt to open the file - int status = 0; - fitsfile* hFITS = nullptr; - if (poOpenInfo->eAccess == GA_ReadOnly) - fits_open_file(&hFITS, poOpenInfo->pszFilename, READONLY, &status); - else - fits_open_file(&hFITS, poOpenInfo->pszFilename, READWRITE, &status); - if (status) { - CPLError(CE_Failure, CPLE_AppDefined, - "Error while opening FITS file %s (%d).\n", - poOpenInfo->pszFilename, status); - fits_close_file(hFITS, &status); - return nullptr; - } + // Get access mode and attempt to open the file + int status = 0; + fitsfile* hFITS = nullptr; + if (poOpenInfo->eAccess == GA_ReadOnly) + fits_open_file(&hFITS, osFilename.c_str(), READONLY, &status); + else + fits_open_file(&hFITS, osFilename.c_str(), READWRITE, &status); + if (status) { + CPLError(CE_Failure, CPLE_AppDefined, + "Error while opening FITS file %s (%d).\n", + osFilename.c_str(), status); + fits_close_file(hFITS, &status); + return nullptr; + } - // Create a FITSDataset object and initialize it from the FITS handle - FITSDataset* dataset = new FITSDataset(); - dataset->eAccess = poOpenInfo->eAccess; +/* -------------------------------------------------------------------- */ +/* Iterate over HDUs */ +/* -------------------------------------------------------------------- */ + bool firstHDUIsDummy = false; + int firstValidHDU = 0; + CPLStringList aosSubdatasets; + if( iSelectedHDU == 0 ) + { + int numHDUs = 0; + fits_get_num_hdus(hFITS, &numHDUs, &status); + if( numHDUs <= 0 ) + { + fits_close_file(hFITS, &status); + return nullptr; + } - dataset->m_bMetadataChanged = false; - dataset->m_bNoDataChanged = false; + for( int iHDU = 1; iHDU <= numHDUs; iHDU++ ) + { + int hduType = 0; + fits_movabs_hdu(hFITS, iHDU, &hduType, &status); + if (status) + { + continue; + } + + if( hduType != IMAGE_HDU ) + { + continue; + } + + int bitpix = 0; + int naxis = 0; + const int maxdim = 3; + long naxes[maxdim] = {0,0,0}; + fits_get_img_param(hFITS, maxdim, &bitpix, &naxis, naxes, &status); + if (status) + { + continue; + } + + if( naxis != 2 && naxis != 3 ) + { + if( naxis == 0 && iHDU == 1 ) + { + firstHDUIsDummy = true; + } + continue; + } + + char szExtname[81] = { 0 }; + fits_read_key(hFITS, TSTRING, "EXTNAME", szExtname, nullptr, &status); + status = 0; + + const int nIdx = aosSubdatasets.size() / 2 + 1; + aosSubdatasets.AddNameValue( + CPLSPrintf("SUBDATASET_%d_NAME", nIdx), + CPLSPrintf("FITS:\"%s\":%d", poOpenInfo->pszFilename, iHDU)); + CPLString osDesc(CPLSPrintf("HDU %d (%dx%d, %d band%s)", iHDU, + static_cast(naxes[0]), + static_cast(naxes[1]), + naxis == 3 ? static_cast(naxes[2]) : 1, + (naxis == 3 && naxes[2] > 1) ? "s" : "")); + if( szExtname[0] ) + { + osDesc += ", "; + osDesc += szExtname; + } + aosSubdatasets.AddNameValue( + CPLSPrintf("SUBDATASET_%d_DESC", nIdx), + osDesc); + + if( firstValidHDU == 0 ) + { + firstValidHDU = iHDU; + } + } + if( aosSubdatasets.size() == 2 ) + { + aosSubdatasets.Clear(); + } + } + else + { + if( iSelectedHDU != 1 ) + { + int hduType = 0; + fits_movabs_hdu(hFITS, 1, &hduType, &status); + if( status == 0 ) + { + int bitpix = 0; + int naxis = 0; + const int maxdim = 3; + long naxes[maxdim] = {0,0,0}; + fits_get_img_param(hFITS, maxdim, &bitpix, &naxis, naxes, &status); + if( status == 0 && naxis == 0 ) + { + firstHDUIsDummy = true; + } + } + status = 0; + } + firstValidHDU = iSelectedHDU; + } + + if( firstValidHDU == 0 ) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Cannot find HDU of image type with 2 or 3 axes"); + fits_close_file(hFITS, &status); + return nullptr; + } + + // Create a FITSDataset object and initialize it from the FITS handle + FITSDataset* dataset = new FITSDataset(); + dataset->eAccess = poOpenInfo->eAccess; + dataset->m_aosSubdatasets = aosSubdatasets; + + // Set up the description and initialize the dataset + dataset->SetDescription(poOpenInfo->pszFilename); + if( aosSubdatasets.size() > 2 ) + { + firstValidHDU = 0; + dataset->m_hFITS = hFITS; + dataset->m_isExistingFile = true; + int hduType = 0; + fits_movabs_hdu(hFITS, 1, &hduType, &status); + } + else + { + if (dataset->Init(hFITS, true, firstValidHDU) != CE_None) { + delete dataset; + return nullptr; + } + } + + // If the first HDU is a dummy one, load its metadata first, and then + // add/override it by the one of the image HDU + if( firstHDUIsDummy && firstValidHDU > 1 ) + { + int hduType = 0; + status = 0; + fits_movabs_hdu(hFITS, 1, &hduType, &status); + if( status == 0 ) + { + dataset->LoadMetadata(); + } + status = 0; + fits_movabs_hdu(hFITS, firstValidHDU, &hduType, &status); + if( status ) { + delete dataset; + return nullptr; + } + } + dataset->LoadMetadata(); - // Set up the description and initialize the dataset - dataset->SetDescription(poOpenInfo->pszFilename); - if (dataset->Init(hFITS, true) != CE_None) { - delete dataset; - return nullptr; - } - else - { /* -------------------------------------------------------------------- */ -/* Initialize any information. */ +/* Initialize any information. */ /* -------------------------------------------------------------------- */ - dataset->SetDescription( poOpenInfo->pszFilename ); - dataset->LoadFITSInfo(); - dataset->TryLoadXML(); + dataset->SetDescription( poOpenInfo->pszFilename ); + dataset->LoadFITSInfo(); + dataset->TryLoadXML(); /* -------------------------------------------------------------------- */ /* Check for external overviews. */ /* -------------------------------------------------------------------- */ - dataset->oOvManager.Initialize( dataset, poOpenInfo->pszFilename, poOpenInfo->GetSiblingFiles() ); + dataset->oOvManager.Initialize( dataset, poOpenInfo->pszFilename, poOpenInfo->GetSiblingFiles() ); - return dataset; - } + return dataset; } /************************************************************************/ @@ -773,7 +990,7 @@ GDALDataset *FITSDataset::Create( const char* pszFilename, dataset->SetDescription(pszFilename); // Init recalculates a lot of stuff we already know, but... - if (dataset->Init(hFITS, false) != CE_None) { + if (dataset->Init(hFITS, false, 1) != CE_None) { delete dataset; return nullptr; } @@ -1583,6 +1800,7 @@ void GDALRegister_FITS() poDriver->SetMetadataItem( GDAL_DMD_EXTENSIONS, "fits" ); poDriver->pfnOpen = FITSDataset::Open; + poDriver->pfnIdentify = FITSDataset::Identify; poDriver->pfnCreate = FITSDataset::Create; poDriver->pfnCreateCopy = nullptr; From e23249284b249d9b200eb89632ee73ac2d89b084 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 14 Sep 2020 12:43:06 +0200 Subject: [PATCH 034/255] SEGY: avoid opening FITS files Funded by CNES --- gdal/ogr/ogrsf_frmts/segy/ogrsegydriver.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/gdal/ogr/ogrsf_frmts/segy/ogrsegydriver.cpp b/gdal/ogr/ogrsf_frmts/segy/ogrsegydriver.cpp index aa9fde3e812d..9645b0851719 100644 --- a/gdal/ogr/ogrsf_frmts/segy/ogrsegydriver.cpp +++ b/gdal/ogr/ogrsf_frmts/segy/ogrsegydriver.cpp @@ -78,6 +78,11 @@ static GDALDataset *OGRSEGYDriverOpen( GDALOpenInfo* poOpenInfo ) { return nullptr; } + const char* fitsID = "SIMPLE = T"; + if( STARTS_WITH_CI((const char*)poOpenInfo->pabyHeader, fitsID)) + { + return nullptr; + } // -------------------------------------------------------------------- // Try to decode the header encoded as EBCDIC and then ASCII @@ -120,8 +125,6 @@ static GDALDataset *OGRSEGYDriverOpen( GDALOpenInfo* poOpenInfo ) #if DEBUG_VERBOSE CPLDebug("SEGY", "Header = \n%s", pabyASCIITextHeader); #endif - CPLFree(pabyASCIITextHeader); - pabyASCIITextHeader = nullptr; // -------------------------------------------------------------------- // Read the next 400 bytes, where the Binary File Header is From ce76b70517d4785fdd87fa6fb71c631c21db6d8e Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 14 Sep 2020 12:43:11 +0200 Subject: [PATCH 035/255] FITS: add support for reading binary tables Funded by CNES --- autotest/gdrivers/data/fits/binary_table.fits | 3 + autotest/gdrivers/fits.py | 334 +++++++ autotest/gdrivers/generate_fits.py | 106 +++ gdal/doc/source/drivers/raster/fits.rst | 202 ++++- gdal/frmts/fits/fitsdataset.cpp | 835 ++++++++++++++++-- 5 files changed, 1428 insertions(+), 52 deletions(-) create mode 100644 autotest/gdrivers/data/fits/binary_table.fits diff --git a/autotest/gdrivers/data/fits/binary_table.fits b/autotest/gdrivers/data/fits/binary_table.fits new file mode 100644 index 000000000000..51727fd69619 --- /dev/null +++ b/autotest/gdrivers/data/fits/binary_table.fits @@ -0,0 +1,3 @@ +SIMPLE = T / conforms to FITS standard BITPIX = 8 / array data type NAXIS = 0 / number of array dimensions EXTEND = T END XTENSION= 'BINTABLE' / binary table extension BITPIX = 8 / array data type NAXIS = 2 / number of array dimensions NAXIS1 = 331 / length of dimension 1 NAXIS2 = 3 / length of dimension 2 PCOUNT = 273 / number of group parameters GCOUNT = 1 / number of groups TFIELDS = 46 / number of table fields EXTNAME = 'MyTable ' TTYPE1 = 'B_scaled_integer' TFORM1 = 'B ' TTYPE2 = 'B_scaled' TFORM2 = 'B ' TTYPE3 = 'I_scaled_integer' TFORM3 = 'I ' TTYPE4 = 'I_scaled' TFORM4 = 'I ' TTYPE5 = 'J_scaled_integer' TFORM5 = 'J ' TTYPE6 = 'J_scaled' TFORM6 = 'J ' TTYPE7 = 'K_scaled' TFORM7 = 'K ' TTYPE8 = 'E_scaled' TFORM8 = 'E ' TTYPE9 = 'D_scaled' TFORM9 = 'D ' TTYPE10 = 'C_scaled' TFORM10 = 'C ' TTYPE11 = 'M_scaled' TFORM11 = 'M ' TTYPE12 = 'L ' TFORM12 = 'L ' TTYPE13 = '2L ' TFORM13 = '2L ' TTYPE14 = 'PL ' TFORM14 = 'PL(3) ' TTYPE15 = 'QL ' TFORM15 = 'QL(3) ' TTYPE16 = 'X ' TFORM16 = 'X ' TTYPE17 = '33X ' TFORM17 = '33X ' TTYPE18 = 'B ' TFORM18 = 'B ' TNULL18 = 3 TTYPE19 = '2B ' TFORM19 = '2B ' TTYPE20 = 'PB ' TFORM20 = 'PB(3) ' TTYPE21 = 'BDIM ' TFORM21 = '6B ' TDIM21 = '(3,2) ' TTYPE22 = 'I ' TFORM22 = 'I ' TTYPE23 = '2I ' TFORM23 = '2I ' TTYPE24 = 'PI ' TFORM24 = 'PI(3) ' TTYPE25 = 'J ' TFORM25 = 'J ' TTYPE26 = '2J ' TFORM26 = '2J ' TTYPE27 = 'PJ ' TFORM27 = 'PJ(3) ' TTYPE28 = 'K ' TFORM28 = 'K ' TTYPE29 = '2K ' TFORM29 = '2K ' TTYPE30 = 'PK ' TFORM30 = 'PK(3) ' TTYPE31 = 'A ' TFORM31 = 'A ' TTYPE32 = 'A2 ' TFORM32 = '2A ' TTYPE33 = 'PA ' TFORM33 = 'PA(3) ' TTYPE34 = 'ADIM ' TFORM34 = '6A ' TDIM34 = '(2, 3) ' TTYPE35 = 'E ' TFORM35 = 'E ' TTYPE36 = '2E ' TFORM36 = '2E ' TTYPE37 = 'PE ' TFORM37 = 'PE(3) ' TTYPE38 = 'D ' TFORM38 = 'D ' TTYPE39 = '2D ' TFORM39 = '2D ' TTYPE40 = 'PD ' TFORM40 = 'PD(3) ' TTYPE41 = 'C ' TFORM41 = 'C ' TTYPE42 = '2C ' TFORM42 = '2C ' TTYPE43 = 'PC ' TFORM43 = 'PC(3) ' TTYPE44 = 'M ' TFORM44 = 'M ' TTYPE45 = '2M ' TFORM45 = '2M ' TTYPE46 = 'PM ' TFORM46 = 'PM(3) ' TZERO1 = -128 TSCAL2 = 1.5 TZERO2 = 2.5 TZERO3 = 32768 TSCAL4 = 1.5 TZERO4 = 2.5 TZERO5 = 2147483648 TSCAL6 = 1.5 TZERO6 = 2.5 TSCAL7 = 1.5 TZERO7 = 2.5 TSCAL8 = 1.5 TZERO8 = 2.5 TSCAL9 = 1.5 TZERO9 = 2.5 TSCAL10 = 1.5 TZERO10 = 2.5 TSCAL11 = 1.5 TZERO11 = 2.5 END €€€€€? ?ô? @?ô@TTF€úªÕU€ÿ +ÿÿÿ€€ÿ€€ÿÿÿ€€ÿÿÿÿÿÿÿ-AABUABabAb? ? @]?ô í(Œç?ô í(Œç@q? @? @@? ™?ô í(Œç@?ô í(Œç@@?ôÁÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ@@FFTúªÕU€ÿÿ ÿÿÿÿÿ€ÿÿÿÿÿÿ€!ÿÿÿÿÿÿÿÿÿÿÿÿÿÿ€=BCDWCDcdCd@@? e@@ö”Fs‚?ô@? @? ? @©@?ô@?ô?ô@áFFF +-UZq™Áÿÿ€ÿÿ€€ÿÿÿÿÿÿ€€ÿÿÿÿÿÿÿÿÿÿÿÿÿÿ€ABCDE ? @@? @?ô í(Œç@@ö”Fs‚?ô@? @@? @? ? @@? ?ô í(Œç@@?ô@?ô?ô@@?ô \ No newline at end of file diff --git a/autotest/gdrivers/fits.py b/autotest/gdrivers/fits.py index c7125aa01175..8e17d0fdd7a5 100755 --- a/autotest/gdrivers/fits.py +++ b/autotest/gdrivers/fits.py @@ -32,6 +32,7 @@ import pytest from osgeo import gdal +from osgeo import ogr import gdaltest pytestmark = pytest.mark.require_driver('FITS') @@ -209,3 +210,336 @@ def test_fits_read_image_in_second_and_fourth_hdu_table_in_third(): assert ds.RasterXSize == 1 assert ds.RasterYSize == 3 assert ds.GetMetadata() == {'EXTNAME': 'SECOND_IMAGE', 'FOO': 'BAR '} + + +# Test opening raster files in vector mode +def test_fits_open_raster_only_in_vector_mode(): + + filename = 'data/fits/empty_primary_hdu.fits' + assert os.path.exists(filename) + with gdaltest.error_handler(): + assert ogr.Open(filename) is None + + filename = 'data/fits/byte_merc.fits' + assert os.path.exists(filename) + with gdaltest.error_handler(): + assert ogr.Open(filename) is None + + filename = 'data/fits/image_in_first_and_second_hdu.fits' + assert os.path.exists(filename) + with gdaltest.error_handler(): + assert ogr.Open(filename) is None + + ds = gdal.OpenEx('data/fits/image_in_second_and_fourth_hdu_table_in_third.fits', gdal.OF_VECTOR) + assert ds.GetMetadata('SUBDATASETS') == {} + + +# Test opening vector files in raster mode +def test_fits_open_vector_only_in_raster_mode(): + + filename = 'data/fits/binary_table.fits' + assert os.path.exists(filename) + with gdaltest.error_handler(): + assert gdal.Open(filename) is None + + +# Test opening with gdal.OF_RASTER | gdal.OF_VECTOR +def test_fits_open_mix_mode(): + + # Raster only content + ds = gdal.OpenEx('data/fits/byte_merc.fits', gdal.OF_RASTER | gdal.OF_VECTOR) + assert ds.RasterCount != 0 + assert ds.GetLayerCount() == 0 + + # Vector only content + ds = gdal.OpenEx('data/fits/binary_table.fits', gdal.OF_RASTER | gdal.OF_VECTOR) + assert ds.RasterCount == 0 + assert ds.GetLayerCount() != 0 + + # Mix of raster and vector + ds = gdal.OpenEx('data/fits/image_in_second_and_fourth_hdu_table_in_third.fits', gdal.OF_RASTER | gdal.OF_VECTOR) + assert ds.GetMetadata('SUBDATASETS') != {} + assert ds.GetLayerCount() != 0 + + +def test_fits_vector(): + + ds = ogr.Open('data/fits/binary_table.fits') + assert ds.GetLayerCount() == 1 + assert ds.GetLayer(-1) is None + assert ds.GetLayer(1) is None + lyr = ds.GetLayer(0) + assert lyr.GetName() == 'MyTable' + assert lyr.GetGeomType() == ogr.wkbNone + assert lyr.TestCapability(ogr.OLCFastFeatureCount) == 1 + assert lyr.TestCapability(ogr.OLCRandomRead) == 1 + assert lyr.TestCapability("something_else") == 0 + assert lyr.GetFeatureCount() == 3 + lyr_defn = lyr.GetLayerDefn() + assert lyr_defn.GetFieldCount() == 78 + mapType = {} + for x in ["ogr.OFTInteger", "ogr.OFTInteger64", "ogr.OFTReal", + "ogr.OFTString", "ogr.OFTIntegerList", "ogr.OFTInteger64List", + "ogr.OFTRealList", "ogr.OFTStringList"]: + mapType[eval(x)] = x + mapSubType = {} + for x in ["ogr.OFSTNone", "ogr.OFSTBoolean", "ogr.OFSTFloat32", "ogr.OFSTInt16"]: + mapSubType[eval(x)] = x + got = [(lyr_defn.GetFieldDefn(i).GetNameRef(), + mapType[lyr_defn.GetFieldDefn(i).GetType()], + mapSubType[lyr_defn.GetFieldDefn(i).GetSubType()], + lyr_defn.GetFieldDefn(i).GetWidth()) for i in range(lyr_defn.GetFieldCount())] + expected = [('B_scaled_integer', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('B_scaled', 'ogr.OFTReal', 'ogr.OFSTNone', 0), + ('I_scaled_integer', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('I_scaled', 'ogr.OFTReal', 'ogr.OFSTNone', 0), + ('J_scaled_integer', 'ogr.OFTInteger64', 'ogr.OFSTNone', 0), + ('J_scaled', 'ogr.OFTReal', 'ogr.OFSTNone', 0), + ('K_scaled', 'ogr.OFTReal', 'ogr.OFSTNone', 0), + ('E_scaled', 'ogr.OFTReal', 'ogr.OFSTNone', 0), + ('D_scaled', 'ogr.OFTReal', 'ogr.OFSTNone', 0), + ('C_scaled', 'ogr.OFTString', 'ogr.OFSTNone', 0), + ('M_scaled', 'ogr.OFTString', 'ogr.OFSTNone', 0), + ('L', 'ogr.OFTInteger', 'ogr.OFSTBoolean', 0), + ('2L', 'ogr.OFTIntegerList', 'ogr.OFSTBoolean', 0), + ('PL', 'ogr.OFTIntegerList', 'ogr.OFSTBoolean', 0), + ('QL', 'ogr.OFTIntegerList', 'ogr.OFSTBoolean', 0), + ('X_bit1', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit1', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit2', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit3', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit4', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit5', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit6', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit7', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit8', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit9', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit10', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit11', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit12', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit13', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit14', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit15', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit16', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit17', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit18', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit19', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit20', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit21', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit22', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit23', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit24', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit25', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit26', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit27', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit28', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit29', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit30', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit31', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit32', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('33X_bit33', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('B', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('2B', 'ogr.OFTIntegerList', 'ogr.OFSTNone', 0), + ('PB', 'ogr.OFTIntegerList', 'ogr.OFSTNone', 0), + ('BDIM', 'ogr.OFTIntegerList', 'ogr.OFSTNone', 0), + ('I', 'ogr.OFTInteger', 'ogr.OFSTInt16', 0), + ('2I', 'ogr.OFTIntegerList', 'ogr.OFSTInt16', 0), + ('PI', 'ogr.OFTIntegerList', 'ogr.OFSTInt16', 0), + ('J', 'ogr.OFTInteger', 'ogr.OFSTNone', 0), + ('2J', 'ogr.OFTIntegerList', 'ogr.OFSTNone', 0), + ('PJ', 'ogr.OFTIntegerList', 'ogr.OFSTNone', 0), + ('K', 'ogr.OFTInteger64', 'ogr.OFSTNone', 0), + ('2K', 'ogr.OFTInteger64List', 'ogr.OFSTNone', 0), + ('PK', 'ogr.OFTInteger64List', 'ogr.OFSTNone', 0), + ('A', 'ogr.OFTString', 'ogr.OFSTNone', 1), + ('A2', 'ogr.OFTString', 'ogr.OFSTNone', 2), + ('PA', 'ogr.OFTString', 'ogr.OFSTNone', 0), + ('ADIM', 'ogr.OFTStringList', 'ogr.OFSTNone', 2), + ('E', 'ogr.OFTReal', 'ogr.OFSTFloat32', 0), + ('2E', 'ogr.OFTRealList', 'ogr.OFSTFloat32', 0), + ('PE', 'ogr.OFTRealList', 'ogr.OFSTFloat32', 0), + ('D', 'ogr.OFTReal', 'ogr.OFSTNone', 0), + ('2D', 'ogr.OFTRealList', 'ogr.OFSTNone', 0), + ('PD', 'ogr.OFTRealList', 'ogr.OFSTNone', 0), + ('C', 'ogr.OFTString', 'ogr.OFSTNone', 0), + ('2C', 'ogr.OFTStringList', 'ogr.OFSTNone', 0), + ('PC', 'ogr.OFTStringList', 'ogr.OFSTNone', 0), + ('M', 'ogr.OFTString', 'ogr.OFSTNone', 0), + ('2M', 'ogr.OFTStringList', 'ogr.OFSTNone', 0), + ('PM', 'ogr.OFTStringList', 'ogr.OFSTNone', 0)] + if got != expected: + import pprint + pp = pprint.PrettyPrinter() + pp.pprint(got) + assert False + + expected_f1 = [ + -128, + 5.5, + 0, + -49149.5, + 0, + -3221225469.5, + -1.3835058055282164e+19, + 4.375, + 4.375, + '4.375 + 5.875j', + '4.375 + 5.875j', + 0, + [0, 0], + [1, 0], + [1, 0], + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 1, + 0, + [255, 0], + [255, 0], + [0, 255, 0, 255, 0, 255], + -32768, + [-32768, 32767], + [-32768, 32767], + -2147483648, + [-2147483648, 2147483647], + [-2147483648, 2147483647], + -9223372036854775808, + [-9223372036854775808, 9223372036854775807], + [-9223372036854775808, 9223372036854775807], + 'A', + 'AB', + 'AB', + ['AB', 'ab', 'Ab'], + 1.25, + [1.25, 2.25], + [1.25, 2.25], + 1.2534, + [1.2534, 2.25], + [1.2534, 2.25], + '1.25 + 2.25j', + ['1.25 + 2.25j', '2.25 + 1.25j'], + ['1.25 + 2.25j', '2.25 + 1.25j'], + '1.25340000000000007 + 2.25j', + ['1.25340000000000007 + 2.25j', '2.25 + 1.25j'], + ['1.25340000000000007 + 2.25j', '2.25 + 1.25j']] + + f = lyr.GetNextFeature() + got = [f.GetField(i) for i in range(f.GetFieldCount())] + assert got == expected_f1 + + expected_f2 = [ + 127, + 385.0, + 65535, + 49153.0, + 4294967295, + 3221225473.0, + 1.3835058055282164e+19, + 5.875, + 5.875, + '2.5 + 2.5j', + '2.5 + 2.5j', + 0, + [0, 0], + [0, 1, 0], + [0, 1, 0], + 0, + 1, + 1, + 1, + 1, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 1, + 255, + [0, 255], + [0, 255, 0], + [255, 255, 0, 0, 0, 255], + 32767, + [32767, -32768], + [32767, 0, -32768], + 2147483647, + [2147483647, -2147483648], + [2147483647, 0, -2147483648], + 9223372036854775807, + [9223372036854775807, -9223372036854775808], + [9223372036854775807, 0, -9223372036854775808], + 'B', + 'CD', + 'CDE', + ['CD', 'cd', 'Cd'], + 2.25, + [2.25, 1.25], + [2.25, 1.25, 2.25], + 2.25, + [2.2534, 1.25], + [2.2534, 1.25, 2.25], + '2.25 + 1.25j', + ['2.25 + 1.25j', '1.25 + 2.25j'], + ['2.25 + 1.25j', '1.25 + 2.25j', '2.25 + 1.25j'], + '2.25 + 1.25j', + ['2.25 + 1.25j', '1.25 + 2.25j'], + ['2.25 + 1.25j', '1.25 + 2.25j', '2.25 + 1.25j']] + + f = lyr.GetNextFeature() + got = [f.GetField(i) for i in range(f.GetFieldCount())] + assert got == expected_f2 + + f = lyr.GetNextFeature() + assert f.GetField('B') is None diff --git a/autotest/gdrivers/generate_fits.py b/autotest/gdrivers/generate_fits.py index 8f72719d1819..28ba68a09ad3 100644 --- a/autotest/gdrivers/generate_fits.py +++ b/autotest/gdrivers/generate_fits.py @@ -70,3 +70,109 @@ img = np.arange(3,dtype='B').reshape(3,1) fits.write(data=img, extname='SECOND_IMAGE') fits[-1].write_checksum() + + +from astropy.io import fits + +filename = os.path.join(data_dir, 'binary_table.fits') +if os.path.exists(filename): + os.unlink(filename) + +hdr = fits.Header() +hdr['EXTNAME'] = 'MyTable' +hdu = fits.BinTableHDU.from_columns([ + fits.Column(name='B_scaled_integer', format='B',array=[0, 255, 3]), + fits.Column(name='B_scaled', format='B',array=[0, 255]), + fits.Column(name='I_scaled_integer', format='I',array=[-32768, 32767]), + fits.Column(name='I_scaled', format='I',array=[-32768, 32767]), + fits.Column(name='J_scaled_integer', format='J',array=[-2147483648, 2147483647]), + fits.Column(name='J_scaled', format='J',array=[-2147483648, 2147483647]), + fits.Column(name='K_scaled', format='K',array=[-9223372036854775808, 9223372036854775807]), + fits.Column(name='E_scaled', format='E',array=[1.25, 2.25]), + fits.Column(name='D_scaled', format='D',array=[1.25, 2.25]), + fits.Column(name='C_scaled', format='C',array=[1.25 + 2.25j]), + fits.Column(name='M_scaled', format='M',array=[1.25 + 2.25j]), + + fits.Column(name='L', format='L',array=[True, False]), + fits.Column(name='2L', format='2L',array=[[True, False], [False, True]]), + fits.Column(name='PL', format='PL()',array=[[True, False], [False, True, False], []]), + fits.Column(name='QL', format='QL()',array=[[True, False], [False, True, False], []]), + + fits.Column(name='X', format='X',array=np.array([[1], [0]], dtype=np.uint8)), + fits.Column(name='33X', format='33X',array=np.array([[1,1,1,1,1,0,1,0,1,0,1,0,1,0,1,0,1,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,1], + [1,1,1,1,1,0,1,0,1,0,1,0,1,0,1,0,1,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,1]], dtype=np.uint8)), + # PX doesn't seem to work + + fits.Column(name='B', format='B',array=[0, 255, 3],null=3), + fits.Column(name='2B', format='2B',array=[[255,0], [0,255]]), + fits.Column(name='PB', format='PB()',array=[[255,0], [0,255,0], []]), + fits.Column(name='BDIM', format='6B',dim='(3,2)',array=[[[0,255,0], [255,0,255]], [[255,255,0], [0,0,255]]]), + + fits.Column(name='I', format='I',array=[-32768, 32767]), + fits.Column(name='2I', format='2I',array=[[-32768, 32767], [32767, -32768]]), + fits.Column(name='PI', format='PI()',array=[[-32768, 32767], [32767, 0, -32768], []]), + + fits.Column(name='J', format='J',array=[-2147483648, 2147483647]), + fits.Column(name='2J', format='2J',array=[[-2147483648, 2147483647], [2147483647, -2147483648]]), + fits.Column(name='PJ', format='PJ()',array=[[-2147483648, 2147483647], [2147483647, 0, -2147483648], []]), + + fits.Column(name='K', format='K',array=[-9223372036854775808, 9223372036854775807]), + fits.Column(name='2K', format='2K',array=[[-9223372036854775808, 9223372036854775807], [9223372036854775807, -9223372036854775808]]), + fits.Column(name='PK', format='PK()',array=[[-9223372036854775808, 9223372036854775807], [9223372036854775807, 0, -9223372036854775808], []]), + + fits.Column(name='A', format='A',array=["A", "B"]), + fits.Column(name='A2', format='A2',array=["AB", "CD"]), + fits.Column(name='PA', format='PA()',array=["AB", "CDE"]), + fits.Column(name='ADIM', format='6A',dim='(2, 3)',array=[["AB", "ab", "Ab"], ["CD", "cd", "Cd"]]), + + fits.Column(name='E', format='E',array=[1.25, 2.25]), + fits.Column(name='2E', format='2E',array=[[1.25, 2.25], [2.25, 1.25]]), + fits.Column(name='PE', format='PE()',array=[[1.25, 2.25], [2.25, 1.25, 2.25],[]]), + + fits.Column(name='D', format='D',array=[1.2534, 2.25]), + fits.Column(name='2D', format='2D',array=[[1.2534, 2.25], [2.2534, 1.25]]), + fits.Column(name='PD', format='PD()',array=[[1.2534, 2.25], [2.2534, 1.25, 2.25],[]]), + + fits.Column(name='C', format='C',array=[1.25 + 2.25j, 2.25 + 1.25j]), + fits.Column(name='2C', format='2C',array=[[1.25 + 2.25j, 2.25 + 1.25j],[2.25 + 1.25j, 1.25 + 2.25j]]), + fits.Column(name='PC', format='PC',array=[[1.25 + 2.25j, 2.25 + 1.25j],[2.25 + 1.25j, 1.25 + 2.25j, 2.25 + 1.25j],[]]), + + fits.Column(name='M', format='M',array=[1.2534 + 2.25j, 2.25 + 1.25j]), + fits.Column(name='2M', format='2M',array=[[1.2534 + 2.25j, 2.25 + 1.25j],[2.25 + 1.25j, 1.25 + 2.25j]]), + fits.Column(name='PM', format='PM',array=[[1.2534 + 2.25j, 2.25 + 1.25j],[2.25 + 1.25j, 1.25 + 2.25j, 2.25 + 1.25j],[]]), +], header=hdr) +hdu.writeto(filename) + +# Add back zero & scal info with fitsio, since there are some issues with +# astropy for integer data types +with fitsio.FITS(filename,'rw') as f: + hdu = f[-1] + hdu.write_key('TZERO1', -128) + + hdu.write_key('TSCAL2', 1.5) + hdu.write_key('TZERO2', 2.5) + + hdu.write_key('TZERO3', 32768) + + hdu.write_key('TSCAL4', 1.5) + hdu.write_key('TZERO4', 2.5) + + hdu.write_key('TZERO5', 2147483648) + + hdu.write_key('TSCAL6', 1.5) + hdu.write_key('TZERO6', 2.5) + + hdu.write_key('TSCAL7', 1.5) + hdu.write_key('TZERO7', 2.5) + + hdu.write_key('TSCAL8', 1.5) + hdu.write_key('TZERO8', 2.5) + + hdu.write_key('TSCAL9', 1.5) + hdu.write_key('TZERO9', 2.5) + + hdu.write_key('TSCAL10', 1.5) + hdu.write_key('TZERO10', 2.5) + + hdu.write_key('TSCAL11', 1.5) + hdu.write_key('TZERO11', 2.5) diff --git a/gdal/doc/source/drivers/raster/fits.rst b/gdal/doc/source/drivers/raster/fits.rst index 68be9c38662a..ee2f53172cd1 100644 --- a/gdal/doc/source/drivers/raster/fits.rst +++ b/gdal/doc/source/drivers/raster/fits.rst @@ -52,6 +52,201 @@ HDUs are found, they are reported as subdatasets. The connection string for a given subdataset/HDU is ``FITS:"filename.fits":hdu_number`` +Binary table support +-------------------- + +Starting with GDAL 3.2, binary tables will be exposed as vector layers (read-only +support). + +The FITS data types are mapped to OGR data types as the following: + +.. list-table:: Data types + :header-rows: 1 + + * - TFORM value + - TSCAL, TOFFSET value + - Occurence count + - OGR field type + - OGR field subtype + * - 'L' (Logical) + - ignored + - 1 + - OFTInteger + - OFSTBoolean + * - 'L' (Logical) + - ignored + - > 1 + - OFTIntegerList + - OFSTBoolean + * - 'X' (bit) + - ignored + - each bit mapped to a OGR field + - OFTInteger + - OFSTNone + * - 'B' (unsigned byte) + - 1, 0 (unsigned byte) or 1, -128 (signed byte) + - 1 + - OFTInteger + - OFSTNone + * - 'B' (unsigned byte) + - 1, 0 (unsigned byte) or 1, -128 (signed byte) + - > 1 + - OFTIntegerList + - OFSTNone + * - 'I' (16 bit signed integer) + - 1, 0 + - 1 + - OFTInteger + - OFSTInt16 + * - 'I' (16 bit integer, interpreted as unsigned) + - 1, 32768 + - 1 + - OFTInteger + - OFSTNone + * - 'I' (16 bit signed integer) + - other than (1,0) and (1,32768) + - 1 + - OFTReal + - OFSTNone + * - 'I' (16 bit integer) + - 1, 0 + - >1 + - OFTIntegerList + - OFSTInt16 + * - 'I' (16 bit integer, interpreted as unsigned) + - 1, 32768 + - >1 + - OFTIntegerList + - OFSTNone + * - 'I' (16 bit signed integer) + - other than (1, 0) and (1, 32768) + - > 1 + - OFTRealList + - OFSTNone + * - 'J' (32 bit signed integer) + - 1, 0 + - 1 + - OFTInteger + - OFSTNone + * - 'J' (32 bit integer, interpreted as unsigned) + - 1, 2147483648 + - 1 + - OFTInteger + - OFSTNone + * - 'J' (32 bit signed integer) + - other than (1, 0) and (1, 2147483648) + - 1 + - OFTReal + - OFSTNone + * - 'J' (32 bit integer) + - 1, 0 + - >1 + - OFTIntegerList + - OFSTNone + * - 'J' (32 bit integer, interpreted as unsigned) + - 1, 2147483648 + - >1 + - OFTIntegerList + - OFSTNone + * - 'J' (32 bit signed integer) + - other than (1, 0) and (1, 2147483648) + - > 1 + - OFTRealList + - OFSTNone + * - 'K' (64 bit signed integer) + - 1, 0 + - 1 + - OFTInteger64 + - OFSTNone + * - 'K' (64 bit signed integer) + - other than (1, 0) + - 1 + - OFTRealList + - OFSTNone + * - 'K' (64 bit signed integer) + - 1, 0 + - > 1 + - OFTInteger64 + - OFSTNone + * - 'K' (64 bit signed integer) + - other than (1, 0) + - > 1 + - OFTRealList + - OFSTNone + * - 'A' (character) + - ignored + - if TFORM='Axxx' and no TDIM header + - OFTString + - OFSTNone + * - 'A' (character) + - ignored + - TDIM for 2D field, or variable length ('PA') + - OFTStringList + - OFSTNone + * - 'E' (single precision floating point) + - 1, 0 + - 1 + - OFTReal + - OFSTFloat32 + * - 'E' (single precision floating point) + - other than (1, 0) + - 1 + - OFTReal + - OFSTNone + * - 'E' (single precision floating point) + - 1, 0 + - > 1 + - OFTRealList + - OFSTFloat32 + * - 'E' (single precision floating point) + - other than (1, 0) + - > 1 + - OFTRealList + - OFSTNone + * - 'D' (double precision floating point) + - any + - 1 + - OFTReal + - OFSTNone + * - 'D' (double precision floating point) + - any + - > 1 + - OFTRealList + - OFSTNone + * - 'C' (single precision complex) + - any + - 1 + - OFTString whose value is of the form "x + yj" + - OFSTNone + * - 'C' (single precision complex) + - any + - > 1 + - OFTStringList whose values are of the form "x + yj" + - OFSTNone + * - 'M' (double precision complex) + - any + - 1 + - OFTString whose value is of the form "x + yj" + - OFSTNone + * - 'M' (double precision complex) + - any + - > 1 + - OFTStringList whose values are of the form "x + yj" + - OFSTNone + +Fields with a repeat count > 1 expressing fixed size arrays, or fields using +array descriptors 'P' and 'Q' for variable length arrays are mapped to OGR OFTxxxxxList +data types. The potential 2D structure of such field has no direct equivalence in +OGR, so OGR will expose a linear structure. For fixed size arrays, the user can retrieve +the value of the TDIMxx header in the layer metadata to recover the dimensionality +of the field. + +Fields that have TSCAL and/or TZERO headers are automatically scaled and offset +to the physical value (only applieds to numeric data types) + +TNULL headers are used for integer numeric data types and for a single-occurence +field to set a OGR field to NULL. + Examples -------- @@ -78,7 +273,7 @@ Examples Lower Right ( 512.0, 512.0) Center ( 256.0, 256.0) -* Opening a given HDU: +* Opening a given raster HDU: :: @@ -97,6 +292,11 @@ Examples Center ( 0.5, 1.0) Band 1 Block=1x1 Type=Byte, ColorInterp=Undefined +* Listing potential binary tables in a FITS file: + + :: + + $ ogrinfo my.fits Other ----- diff --git a/gdal/frmts/fits/fitsdataset.cpp b/gdal/frmts/fits/fitsdataset.cpp index ed271b1ea529..c0345e60f821 100644 --- a/gdal/frmts/fits/fitsdataset.cpp +++ b/gdal/frmts/fits/fitsdataset.cpp @@ -32,13 +32,14 @@ #include "gdal_frmts.h" #include "gdal_pam.h" #include "ogr_spatialref.h" +#include "ogrsf_frmts.h" #include #include #include #include - +#include CPL_CVSID("$Id$") @@ -49,10 +50,12 @@ CPL_CVSID("$Id$") /************************************************************************/ class FITSRasterBand; +class FITSLayer; class FITSDataset final : public GDALPamDataset { friend class FITSRasterBand; + friend class FITSLayer; fitsfile* m_hFITS = nullptr; @@ -78,6 +81,7 @@ class FITSDataset final : public GDALPamDataset { bool m_bFITSInfoChanged = false; + std::vector> m_apoLayers{}; FITSDataset(); // Others should not call this constructor explicitly @@ -86,7 +90,7 @@ class FITSDataset final : public GDALPamDataset { void LoadGeoreferencing(); void LoadFITSInfo(); void WriteFITSInfo(); - void LoadMetadata(); + void LoadMetadata(GDALMajorObject* poTarget); public: ~FITSDataset(); @@ -103,6 +107,9 @@ class FITSDataset final : public GDALPamDataset { virtual CPLErr SetGeoTransform( double * ) override; char** GetMetadata(const char* papszDomain = nullptr) override; + int GetLayerCount() override { return static_cast(m_apoLayers.size()); } + OGRLayer* GetLayer(int) override; + bool GetRawBinaryLayout(GDALDataset::RawBinaryLayout&) override; }; @@ -146,6 +153,687 @@ class FITSRasterBand final: public GDALPamRasterBand { }; +/************************************************************************/ +/* ==================================================================== */ +/* FITSLayer */ +/* ==================================================================== */ +/************************************************************************/ +namespace +{ + struct ColDesc + { + std::string typechar{}; + int iCol = 0; // numbering starting at 1 + int iBit = 0; // numbering starting at 1 + int nRepeat = 0; + int nItems = 1; + double dfOffset = 0; + double dfScale = 1; + bool bHasNull = false; + LONGLONG nNullValue = 0; + int nTypeCode = 0; // unset + }; +} + +class FITSLayer final: public OGRLayer, public OGRGetNextFeatureThroughRaw +{ + friend class FITSDataset; + FITSDataset *m_poDS = nullptr; + int m_hduNum = 0; + OGRFeatureDefn *m_poFeatureDefn = nullptr; + LONGLONG m_nCurRow = 1; + LONGLONG m_nRows = 0; + + std::vector m_aoColDescs; + + OGRFeature* GetNextRawFeature(); + +public: + + FITSLayer(FITSDataset* poDS, int hduNum, const char* pszExtName); + ~FITSLayer(); + + OGRFeatureDefn* GetLayerDefn() override { return m_poFeatureDefn; } + void ResetReading() override; + int TestCapability(const char*) override; + OGRFeature *GetFeature(GIntBig) override; + GIntBig GetFeatureCount(int bForce) override; + + DEFINE_GET_NEXT_FEATURE_THROUGH_RAW(FITSLayer) +}; + + +/************************************************************************/ +/* FITSLayer() */ +/************************************************************************/ + +FITSLayer::FITSLayer(FITSDataset* poDS, int hduNum, const char* pszExtName): + m_poDS(poDS), m_hduNum(hduNum) +{ + if( pszExtName[0] != 0 ) + m_poFeatureDefn = new OGRFeatureDefn(pszExtName); + else + m_poFeatureDefn = new OGRFeatureDefn(CPLSPrintf("Table HDU %d", hduNum)); + m_poFeatureDefn->Reference(); + m_poFeatureDefn->SetGeomType(wkbNone); + SetDescription(m_poFeatureDefn->GetName()); + + int status = 0; + fits_movabs_hdu(m_poDS->m_hFITS, hduNum, nullptr, &status); + + m_poDS->LoadMetadata(this); + + fits_get_num_rowsll(m_poDS->m_hFITS, &m_nRows, &status); + if( status ) + { + CPLError(CE_Failure, CPLE_AppDefined, + "fits_get_num_rowsll() failed"); + } + int nCols = 0; + status = 0; + fits_get_num_cols(m_poDS->m_hFITS, &nCols, &status); + if( status ) + { + CPLError(CE_Failure, CPLE_AppDefined, + "fits_get_num_cols() failed"); + } + + std::vector aosNames(nCols); + std::vector apszNames(nCols); + for( int i = 0; i < nCols; i++ ) + { + aosNames[i].resize(80); + apszNames[i] = &aosNames[i][0]; + } + + status = 0; + fits_read_btblhdrll(m_poDS->m_hFITS, nCols, nullptr, nullptr, + &apszNames[0], + nullptr, + nullptr, nullptr, nullptr, &status); + if( status ) + { + CPLError(CE_Failure, CPLE_AppDefined, + "fits_read_btblhdrll() failed"); + } + + for(int i = 0; i < nCols; i++ ) + { + aosNames[i].resize(strlen(aosNames[i].c_str())); + + char typechar[80]; + LONGLONG nRepeat = 0; + double dfScale = 0; + double dfOffset = 0; + status = 0; + fits_get_bcolparmsll(m_poDS->m_hFITS, i+1, + nullptr, // column name + nullptr, // unit + typechar, + &nRepeat, + &dfScale, + &dfOffset, + nullptr, // nulval + nullptr, // tdisp + &status); + if( status ) + { + CPLError(CE_Failure, CPLE_AppDefined, + "fits_get_bcolparmsll() failed"); + } + + ColDesc oCol; + + status = 0; + fits_read_key(m_poDS->m_hFITS, TLONGLONG, CPLSPrintf("TNULL%d", i+1), + &oCol.nNullValue, nullptr, &status); + oCol.bHasNull = status == 0; + + OGRFieldType eType = OFTString; + OGRFieldSubType eSubType = OFSTNone; + if( typechar[0] == 'L' ) // Logical + { + eType = OFTInteger; + eSubType = OFSTBoolean; + } + else if( typechar[0] == 'X' ) // Bit array + { + if( nRepeat > 128 ) + { + CPLDebug("FITS", "Too large repetition count for column %s", + aosNames[i].c_str()); + continue; + } + for( int j = 1; j <= nRepeat; j++ ) + { + OGRFieldDefn oFieldDefn( + (aosNames[i] + CPLSPrintf("_bit%d", j)).c_str(), + OFTInteger); + m_poFeatureDefn->AddFieldDefn(&oFieldDefn); + + ColDesc oColBit; + oColBit.typechar = typechar; + oColBit.iCol = i + 1; + oColBit.iBit = j; + m_aoColDescs.emplace_back(oColBit); + } + continue; + } + else if( typechar[0] == 'B' ) // Unsigned byte + { + if( dfOffset == -128 && dfScale == 1 ) + { + eType = OFTInteger; // signed byte + oCol.nTypeCode = TSBYTE; + // fits_read_col() automatically offsets numeric values + dfOffset = 0; + } + else if( dfOffset != 0 || dfScale != 1 ) + eType = OFTReal; + else + eType = OFTInteger; + } + else if( typechar[0] == 'I' ) // 16-bit signed integer + { + if( dfOffset == 32768.0 && dfScale == 1 ) + { + eType = OFTInteger; // unsigned 16-bit integer + oCol.nTypeCode = TUSHORT; + // fits_read_col() automatically offsets numeric values + dfOffset = 0; + } + else if( dfOffset != 0 || dfScale != 1 ) + eType = OFTReal; + else + { + eType = OFTInteger; + eSubType = OFSTInt16; + } + } + else if( typechar[0] == 'J' ) // 32-bit signed integer + { + if( dfOffset == 2147483648.0 && dfScale == 1 ) + { + eType = OFTInteger64; // unsigned 32-bit integer --> needs to promote to 64 bits + oCol.nTypeCode = TUINT; + // fits_read_col() automatically offsets numeric values + dfOffset = 0; + } + else if( dfOffset != 0 || dfScale != 1 ) + eType = OFTReal; + else + eType = OFTInteger; + } + else if( typechar[0] == 'K' ) // 64-bit signed integer + { + if( dfOffset != 0 || dfScale != 1 ) + eType = OFTReal; + else + eType = OFTInteger64; + } + else if( typechar[0] == 'A' ) // Character + { + + status = 0; + LONGLONG nWidth = 0; + fits_get_coltypell(m_poDS->m_hFITS, i+1, + nullptr, // typecode + nullptr, // repeat + &nWidth, + &status); + if( status ) + { + CPLError(CE_Failure, CPLE_AppDefined, + "fits_get_coltypell() failed"); + } + if( nRepeat >= 2 * nWidth && nWidth != 0) + { + oCol.nItems = static_cast(nRepeat / nWidth); + eType = OFTStringList; + nRepeat = nWidth; + } + else + { + eType = OFTString; + } + } + else if( typechar[0] == 'E' ) // IEEE754 32bit + { + eType = OFTReal; + if( dfOffset == 0 && dfScale == 1 ) + eSubType = OFSTFloat32; + // fits_read_col() automatically scales numeric values + dfOffset = 0; + dfScale = 1; + } + else if( typechar[0] == 'D' ) // IEEE754 64bit + { + eType = OFTReal; + // fits_read_col() automatically scales numeric values + dfOffset = 0; + dfScale = 1; + } + else if( typechar[0] == 'C' ) // IEEE754 32bit complex + { + eType = OFTString; + // fits_read_col() automatically scales numeric values + dfOffset = 0; + dfScale = 1; + } + else if( typechar[0] == 'M' ) // IEEE754 64bit complex + { + eType = OFTString; + // fits_read_col() automatically scales numeric values + dfOffset = 0; + dfScale = 1; + } + else if( typechar[0] == 'P' || typechar[0] == 'Q' ) // Array + { + if( typechar[1] == 'L' ) + { + eType = OFTIntegerList; + eSubType = OFSTBoolean; + } + else if( typechar[1] == 'B' ) + { + eType = OFTIntegerList; + } + else if( typechar[1] == 'I' ) + { + eType = OFTIntegerList; + eSubType = OFSTInt16; + } + else if( typechar[1] == 'J' ) + { + eType = OFTIntegerList; + } + else if( typechar[1] == 'K' ) + { + eType = OFTInteger64List; + } + else if( typechar[1] == 'A' ) + { + eType = OFTString; + } + else if( typechar[1] == 'E' ) + { + eType = OFTRealList; + if( dfOffset == 0 && dfScale == 1 ) + eSubType = OFSTFloat32; + // fits_read_col() automatically scales numeric values + dfOffset = 0; + dfScale = 1; + } + else if( typechar[1] == 'D' ) + { + eType = OFTRealList; + // fits_read_col() automatically scales numeric values + dfOffset = 0; + dfScale = 1; + } + else if( typechar[1] == 'C' ) + { + eType = OFTStringList; + // fits_read_col() automatically scales numeric values + dfOffset = 0; + dfScale = 1; + } + else if( typechar[1] == 'M' ) + { + eType = OFTStringList; + // fits_read_col() automatically scales numeric values + dfOffset = 0; + dfScale = 1; + } + else + { + CPLDebug("FITS", "Unhandled type %s", typechar); + continue; + } + } + else + { + CPLDebug("FITS", "Unhandled type %s", typechar); + continue; + } + + if( nRepeat > 1 && typechar[0] != 'A' ) + { + if( eType == OFTInteger ) + eType = OFTIntegerList; + else if( eType == OFTInteger64 ) + eType = OFTInteger64List; + else if( eType == OFTReal ) + eType = OFTRealList; + else if( eType == OFTString ) + eType = OFTStringList; + } + + OGRFieldDefn oFieldDefn( aosNames[i].c_str(), eType ); + oFieldDefn.SetSubType(eSubType); + if( typechar[0] == 'A' ) + oFieldDefn.SetWidth(static_cast(nRepeat)); + m_poFeatureDefn->AddFieldDefn(&oFieldDefn); + + oCol.typechar = typechar; + oCol.iCol = i + 1; + oCol.nRepeat = static_cast(nRepeat); + oCol.dfOffset = dfOffset; + oCol.dfScale = dfScale; + m_aoColDescs.emplace_back(oCol); + } +} + +/************************************************************************/ +/* ~FITSLayer() */ +/************************************************************************/ + +FITSLayer::~FITSLayer() +{ + m_poFeatureDefn->Release(); +} + +/************************************************************************/ +/* GetFeatureCount() */ +/************************************************************************/ + +GIntBig FITSLayer::GetFeatureCount(int bForce) +{ + if( m_poAttrQuery == nullptr && m_poFilterGeom == nullptr ) + return m_nRows; + return OGRLayer::GetFeatureCount(bForce); +} + +/************************************************************************/ +/* ResetReading() */ +/************************************************************************/ + +void FITSLayer::ResetReading() +{ + m_nCurRow = 1; +} + +/************************************************************************/ +/* ReadCol */ +/************************************************************************/ + +template struct ReadCol +{ + static void Read(fitsfile* hFITS, const ColDesc& colDesc, + int iField, LONGLONG irow, OGRFeature* poFeature, + int nRepeat) + { + int status = 0; + std::vector x(nRepeat); + fits_read_col(hFITS, TYPECODE, + colDesc.iCol, irow, 1, nRepeat, nullptr, + &x[0], nullptr, &status); + if( nRepeat == 1 && colDesc.bHasNull && + x[0] == static_cast(colDesc.nNullValue) ) + { + poFeature->SetFieldNull(iField); + } + else if( colDesc.dfScale != 1.0 || colDesc.dfOffset != 0.0 ) + { + std::vector scaled; + scaled.reserve(nRepeat); + for( int i = 0; i < nRepeat; ++i ) + { + scaled.push_back(static_cast(x[i]) * + colDesc.dfScale + colDesc.dfOffset); + } + poFeature->SetField(iField, nRepeat, &scaled[0]); + } + else if( nRepeat == 1 ) + { + poFeature->SetField(iField, static_cast(x[0])); + } + else + { + std::vector xGDAL; + xGDAL.reserve(nRepeat); + for( int i = 0; i < nRepeat; i++ ) + xGDAL.push_back(x[i]); + poFeature->SetField(iField, nRepeat, &xGDAL[0]); + } + } +}; + +/************************************************************************/ +/* GetNextRawFeature() */ +/************************************************************************/ + +OGRFeature* FITSLayer::GetNextRawFeature() +{ + if( m_nCurRow <= 0 || m_nCurRow > m_nRows ) + return nullptr; + OGRFeature* poFeature = new OGRFeature(m_poFeatureDefn); + + { + int status = 0; + fits_movabs_hdu(m_poDS->m_hFITS, m_hduNum, nullptr, &status); + } + + const auto ReadField = [this, &poFeature](const ColDesc& colDesc, + int iField, + char typechar, + int nRepeat) + { + int status = 0; + if( typechar == 'L' ) + { + std::vector x(nRepeat); + fits_read_col(m_poDS->m_hFITS, TLOGICAL, + colDesc.iCol, m_nCurRow, 1, nRepeat, nullptr, + &x[0], nullptr, &status); + if( nRepeat == 1 ) + { + poFeature->SetField(iField, x[0] == '1' ? 1 : 0); + } + else + { + std::vector intValues; + intValues.reserve(nRepeat); + for( int i = 0; i < nRepeat; ++i ) + { + intValues.push_back(x[i] == '1' ? 1 : 0); + } + poFeature->SetField(iField, nRepeat, &intValues[0]); + } + } + else if( typechar == 'X' ) + { + char x = 0; + fits_read_col_bit(m_poDS->m_hFITS, colDesc.iCol, m_nCurRow, + colDesc.iBit, 1, &x, &status); + poFeature->SetField(iField, x); + } + else if( typechar == 'B' ) + { + if( colDesc.nTypeCode == TSBYTE ) + { + ReadCol::Read( + m_poDS->m_hFITS, colDesc, iField, + m_nCurRow, poFeature, nRepeat); + } + else + { + ReadCol::Read( + m_poDS->m_hFITS, colDesc, iField, + m_nCurRow, poFeature, nRepeat); + } + } + else if( typechar == 'I' ) + { + if( colDesc.nTypeCode == TUSHORT ) + { + ReadCol::Read( + m_poDS->m_hFITS, colDesc, iField, + m_nCurRow, poFeature, nRepeat); + } + else + { + ReadCol::Read( + m_poDS->m_hFITS, colDesc, iField, + m_nCurRow, poFeature, nRepeat); + } + } + else if( typechar == 'J' ) + { + if( colDesc.nTypeCode == TUINT ) + { + ReadCol::Read( + m_poDS->m_hFITS, colDesc, iField, + m_nCurRow, poFeature, nRepeat); + } + else + { + ReadCol::Read( + m_poDS->m_hFITS, colDesc, iField, + m_nCurRow, poFeature, nRepeat); + } + } + else if( typechar == 'K' ) + { + ReadCol::Read( + m_poDS->m_hFITS, colDesc, iField, + m_nCurRow, poFeature, nRepeat); + } + else if( typechar == 'A' ) // Character + { + if( colDesc.nItems > 1 ) + { + CPLStringList aosList; + for(int iItem = 1; iItem <= colDesc.nItems; iItem++ ) + { + std::string osStr; + osStr.resize(nRepeat); + char* pszStr = &osStr[0]; + fits_read_col_str(m_poDS->m_hFITS, colDesc.iCol, m_nCurRow, iItem, 1, + nullptr, &pszStr, nullptr, &status); + aosList.AddString(pszStr); + } + poFeature->SetField(iField, aosList.List()); + } + else + { + std::string osStr; + osStr.resize(nRepeat); + char* pszStr = &osStr[0]; + fits_read_col_str(m_poDS->m_hFITS, colDesc.iCol, m_nCurRow, 1, 1, + nullptr, &pszStr, nullptr, &status); + poFeature->SetField(iField, osStr.c_str()); + } + } + else if( typechar == 'E' ) // IEEE754 32bit + { + ReadCol::Read(m_poDS->m_hFITS, colDesc, iField, + m_nCurRow, poFeature, nRepeat); + } + else if( typechar == 'D' ) // IEEE754 64bit + { + std::vector x(nRepeat); + fits_read_col(m_poDS->m_hFITS, TDOUBLE, + colDesc.iCol, m_nCurRow, 1, nRepeat, nullptr, + &x[0], nullptr, &status); + if( nRepeat == 1 ) + poFeature->SetField(iField, x[0]); + else + poFeature->SetField(iField, nRepeat, &x[0]); + } + else if( typechar == 'C' ) // IEEE754 32bit complex + { + std::vector x(2 * nRepeat); + fits_read_col(m_poDS->m_hFITS, TCOMPLEX, + colDesc.iCol, m_nCurRow, 1, nRepeat, nullptr, + &x[0], nullptr, &status); + CPLStringList aosList; + for( int i = 0; i < nRepeat; ++i ) + aosList.AddString(CPLSPrintf("%.18g + %.18gj", + x[2*i+0], x[2*i+1])); + if( nRepeat == 1 ) + poFeature->SetField(iField, aosList[0]); + else + poFeature->SetField(iField, aosList.List()); + } + else if( typechar == 'M' ) // IEEE754 64bit complex + { + std::vector x(2 * nRepeat); + fits_read_col(m_poDS->m_hFITS, TDBLCOMPLEX, + colDesc.iCol, m_nCurRow, 1, nRepeat, nullptr, + &x[0], nullptr, &status); + CPLStringList aosList; + for( int i = 0; i < nRepeat; ++i ) + { + aosList.AddString(CPLSPrintf("%.18g + %.18gj", + x[2*i+0], + x[2*i+1])); + } + if( nRepeat == 1 ) + poFeature->SetField(iField, aosList[0]); + else + poFeature->SetField(iField, aosList.List()); + } + else + { + CPLDebug("FITS", "Unhandled typechar %c", typechar); + } + if( status ) + { + CPLError(CE_Failure, CPLE_AppDefined, + "fits_read_col() failed"); + } + }; + + const int nFieldCount = poFeature->GetFieldCount(); + for( int iField = 0; iField < nFieldCount; iField++ ) + { + const auto& colDesc = m_aoColDescs[iField]; + if( colDesc.typechar[0] == 'P' || colDesc.typechar[0] == 'Q' ) + { + int status = 0; + LONGLONG nRepeat = 0; + fits_read_descriptll(m_poDS->m_hFITS,colDesc.iCol, m_nCurRow, + &nRepeat, nullptr, &status); + ReadField(colDesc, iField, colDesc.typechar[1], + static_cast(nRepeat)); + } + else + { + ReadField(colDesc, iField, colDesc.typechar[0], colDesc.nRepeat); + } + } + poFeature->SetFID(m_nCurRow); + m_nCurRow ++; + return poFeature; +} + +/************************************************************************/ +/* GetFeature() */ +/************************************************************************/ + +OGRFeature* FITSLayer::GetFeature(GIntBig nFID) +{ + m_nCurRow = nFID; + return GetNextRawFeature(); +} + +/************************************************************************/ +/* TestCapability() */ +/************************************************************************/ + +int FITSLayer::TestCapability(const char* pszCap) +{ + if( EQUAL(pszCap, OLCFastFeatureCount) ) + return m_poAttrQuery == nullptr && m_poFilterGeom == nullptr; + + if( EQUAL(pszCap, OLCRandomRead) ) + return true; + + return false; +} + + /************************************************************************/ /* FITSRasterBand() */ /************************************************************************/ @@ -592,7 +1280,7 @@ CPLErr FITSDataset::Init(fitsfile* hFITS, bool isExistingFile, int hduNum) { /* LoadMetadata() */ /************************************************************************/ -void FITSDataset::LoadMetadata() +void FITSDataset::LoadMetadata(GDALMajorObject* poTarget) { // Read header information from file and use it to set metadata // This process understands the CONTINUE standard for long strings. @@ -646,12 +1334,12 @@ void FITSDataset::LoadMetadata() "FITS file %s (%d)", key, GetDescription(), status); return; } - SetMetadataItem(key, longString); + poTarget->SetMetadataItem(key, longString); free(longString); } else { // Normal keyword - SetMetadataItem(key, newValue); + poTarget->SetMetadataItem(key, newValue); } } } @@ -691,6 +1379,17 @@ char **FITSDataset::GetMetadata( const char *pszDomain ) return GDALPamDataset::GetMetadata(pszDomain); } +/************************************************************************/ +/* GetLayer() */ +/************************************************************************/ + +OGRLayer* FITSDataset::GetLayer(int idx) +{ + if( idx < 0 || idx >= GetLayerCount() ) + return nullptr; + return m_apoLayers[idx].get(); +} + /************************************************************************/ /* Open() */ /************************************************************************/ @@ -735,6 +1434,11 @@ GDALDataset* FITSDataset::Open(GDALOpenInfo* poOpenInfo) { fits_close_file(hFITS, &status); return nullptr; } + // Create a FITSDataset object + auto dataset = std::unique_ptr(new FITSDataset()); + dataset->m_isExistingFile = true; + dataset->m_hFITS = hFITS; + dataset->eAccess = poOpenInfo->eAccess; /* -------------------------------------------------------------------- */ /* Iterate over HDUs */ @@ -748,7 +1452,6 @@ GDALDataset* FITSDataset::Open(GDALOpenInfo* poOpenInfo) { fits_get_num_hdus(hFITS, &numHDUs, &status); if( numHDUs <= 0 ) { - fits_close_file(hFITS, &status); return nullptr; } @@ -761,6 +1464,23 @@ GDALDataset* FITSDataset::Open(GDALOpenInfo* poOpenInfo) { continue; } + char szExtname[81] = { 0 }; + fits_read_key(hFITS, TSTRING, "EXTNAME", szExtname, nullptr, &status); + status = 0; + int nExtVer = 0; + fits_read_key(hFITS, TINT, "EXTVER", &nExtVer, nullptr, &status); + status = 0; + CPLString osExtname(szExtname); + if( nExtVer > 0 ) + osExtname += CPLSPrintf(" %d", nExtVer); + + if( hduType == BINARY_TBL && + (poOpenInfo->nOpenFlags & GDAL_OF_VECTOR) != 0 ) + { + dataset->m_apoLayers.push_back(std::unique_ptr( + new FITSLayer(dataset.get(), iHDU, osExtname.c_str()))); + } + if( hduType != IMAGE_HDU ) { continue; @@ -785,27 +1505,26 @@ GDALDataset* FITSDataset::Open(GDALOpenInfo* poOpenInfo) { continue; } - char szExtname[81] = { 0 }; - fits_read_key(hFITS, TSTRING, "EXTNAME", szExtname, nullptr, &status); - status = 0; - - const int nIdx = aosSubdatasets.size() / 2 + 1; - aosSubdatasets.AddNameValue( - CPLSPrintf("SUBDATASET_%d_NAME", nIdx), - CPLSPrintf("FITS:\"%s\":%d", poOpenInfo->pszFilename, iHDU)); - CPLString osDesc(CPLSPrintf("HDU %d (%dx%d, %d band%s)", iHDU, - static_cast(naxes[0]), - static_cast(naxes[1]), - naxis == 3 ? static_cast(naxes[2]) : 1, - (naxis == 3 && naxes[2] > 1) ? "s" : "")); - if( szExtname[0] ) + if( (poOpenInfo->nOpenFlags & GDAL_OF_RASTER) != 0 ) { - osDesc += ", "; - osDesc += szExtname; + const int nIdx = aosSubdatasets.size() / 2 + 1; + aosSubdatasets.AddNameValue( + CPLSPrintf("SUBDATASET_%d_NAME", nIdx), + CPLSPrintf("FITS:\"%s\":%d", poOpenInfo->pszFilename, iHDU)); + CPLString osDesc(CPLSPrintf("HDU %d (%dx%d, %d band%s)", iHDU, + static_cast(naxes[0]), + static_cast(naxes[1]), + naxis == 3 ? static_cast(naxes[2]) : 1, + (naxis == 3 && naxes[2] > 1) ? "s" : "")); + if( !osExtname.empty() ) + { + osDesc += ", "; + osDesc += osExtname; + } + aosSubdatasets.AddNameValue( + CPLSPrintf("SUBDATASET_%d_DESC", nIdx), + osDesc); } - aosSubdatasets.AddNameValue( - CPLSPrintf("SUBDATASET_%d_DESC", nIdx), - osDesc); if( firstValidHDU == 0 ) { @@ -839,35 +1558,46 @@ GDALDataset* FITSDataset::Open(GDALOpenInfo* poOpenInfo) { } firstValidHDU = iSelectedHDU; } + const bool rasterFound = + (poOpenInfo->nOpenFlags & GDAL_OF_RASTER) != 0 && firstValidHDU > 0; - if( firstValidHDU == 0 ) + if( !rasterFound ) { - CPLError(CE_Failure, CPLE_AppDefined, - "Cannot find HDU of image type with 2 or 3 axes"); - fits_close_file(hFITS, &status); - return nullptr; + if( (poOpenInfo->nOpenFlags & GDAL_OF_RASTER) != 0 && + (poOpenInfo->nOpenFlags & GDAL_OF_VECTOR) == 0 ) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Cannot find HDU of image type with 2 or 3 axes"); + return nullptr; + } + } + if( dataset->m_apoLayers.empty() ) + { + if( (poOpenInfo->nOpenFlags & GDAL_OF_RASTER) == 0 && + (poOpenInfo->nOpenFlags & GDAL_OF_VECTOR) != 0 ) + { + return nullptr; + } } - // Create a FITSDataset object and initialize it from the FITS handle - FITSDataset* dataset = new FITSDataset(); - dataset->eAccess = poOpenInfo->eAccess; dataset->m_aosSubdatasets = aosSubdatasets; // Set up the description and initialize the dataset dataset->SetDescription(poOpenInfo->pszFilename); - if( aosSubdatasets.size() > 2 ) - { - firstValidHDU = 0; - dataset->m_hFITS = hFITS; - dataset->m_isExistingFile = true; - int hduType = 0; - fits_movabs_hdu(hFITS, 1, &hduType, &status); - } - else + if( rasterFound ) { - if (dataset->Init(hFITS, true, firstValidHDU) != CE_None) { - delete dataset; - return nullptr; + if( aosSubdatasets.size() > 2 ) + { + firstValidHDU = 0; + int hduType = 0; + fits_movabs_hdu(hFITS, 1, &hduType, &status); + } + else + { + if( firstValidHDU != 0 && + dataset->Init(hFITS, true, firstValidHDU) != CE_None) { + return nullptr; + } } } @@ -880,30 +1610,32 @@ GDALDataset* FITSDataset::Open(GDALOpenInfo* poOpenInfo) { fits_movabs_hdu(hFITS, 1, &hduType, &status); if( status == 0 ) { - dataset->LoadMetadata(); + dataset->LoadMetadata(dataset.get()); } status = 0; fits_movabs_hdu(hFITS, firstValidHDU, &hduType, &status); if( status ) { - delete dataset; return nullptr; } } - dataset->LoadMetadata(); + if( rasterFound ) + { + dataset->LoadMetadata(dataset.get()); + dataset->LoadFITSInfo(); + } /* -------------------------------------------------------------------- */ /* Initialize any information. */ /* -------------------------------------------------------------------- */ dataset->SetDescription( poOpenInfo->pszFilename ); - dataset->LoadFITSInfo(); dataset->TryLoadXML(); /* -------------------------------------------------------------------- */ /* Check for external overviews. */ /* -------------------------------------------------------------------- */ - dataset->oOvManager.Initialize( dataset, poOpenInfo->pszFilename, poOpenInfo->GetSiblingFiles() ); + dataset->oOvManager.Initialize( dataset.get(), poOpenInfo->pszFilename, poOpenInfo->GetSiblingFiles() ); - return dataset; + return dataset.release(); } /************************************************************************/ @@ -1791,6 +2523,7 @@ void GDALRegister_FITS() poDriver->SetDescription( "FITS" ); poDriver->SetMetadataItem( GDAL_DCAP_RASTER, "YES" ); + poDriver->SetMetadataItem( GDAL_DCAP_VECTOR, "YES" ); poDriver->SetMetadataItem( GDAL_DMD_LONGNAME, "Flexible Image Transport System" ); poDriver->SetMetadataItem( GDAL_DMD_HELPTOPIC, From 71f41dc7b4840807cdf3c05ef435a4533c806898 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 15 Sep 2020 16:59:51 +0200 Subject: [PATCH 036/255] OCI: fix server 12.2 version detection --- gdal/ogr/ogrsf_frmts/oci/ogrocisession.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gdal/ogr/ogrsf_frmts/oci/ogrocisession.cpp b/gdal/ogr/ogrsf_frmts/oci/ogrocisession.cpp index 2a8fb92cb99b..7b11ba6e53fd 100644 --- a/gdal/ogr/ogrsf_frmts/oci/ogrocisession.cpp +++ b/gdal/ogr/ogrsf_frmts/oci/ogrocisession.cpp @@ -316,7 +316,7 @@ int OGROCISession::EstablishSession( const char *pszUseridIn, /* Set maximun name length (before 12.2 ? 30 : 128) */ /* -------------------------------------------------------------------- */ - if( nServerVersion >= 12 && nServerRelease >= 2 ) + if( nServerVersion > 12 || (nServerVersion == 12 && nServerRelease >= 2) ) { nMaxNameLength = 128; } From 1bd08bdc58612e6283132281f8040b599ce78500 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 16 Sep 2020 14:34:24 +0200 Subject: [PATCH 037/255] Doc: fix issues in gdalwarpkernel.cpp [skip travis] [skip appveyor] --- gdal/alg/gdalwarpkernel.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gdal/alg/gdalwarpkernel.cpp b/gdal/alg/gdalwarpkernel.cpp index b5857953923a..5b39d239c447 100644 --- a/gdal/alg/gdalwarpkernel.cpp +++ b/gdal/alg/gdalwarpkernel.cpp @@ -683,14 +683,14 @@ static CPLErr GWKRun( GDALWarpKernel *poWK, * * \code * float dfPixelValue; - * int nBand = 1; // Band indexes are zero based. + * int nBand = 2-1; // Band indexes are zero based. * int nPixel = 3; // Zero based. * int nLine = 4; // Zero based. * * assert( nPixel >= 0 && nPixel < poKern->nSrcXSize ); * assert( nLine >= 0 && nLine < poKern->nSrcYSize ); * assert( nBand >= 0 && nBand < poKern->nBands ); - * dfPixelValue = ((float *) poKern->papabySrcImage[nBand-1]) + * dfPixelValue = ((float *) poKern->papabySrcImage[nBand]) * [nPixel + nLine * poKern->nSrcXSize]; * \endcode * @@ -712,7 +712,7 @@ static CPLErr GWKRun( GDALWarpKernel *poWK, * * \code * int bIsValid = TRUE; - * int nBand = 1; // Band indexes are zero based. + * int nBand = 2-1; // Band indexes are zero based. * int nPixel = 3; // Zero based. * int nLine = 4; // Zero based. * @@ -806,14 +806,14 @@ static CPLErr GWKRun( GDALWarpKernel *poWK, * * \code * float dfPixelValue; - * int nBand = 1; // Band indexes are zero based. + * int nBand = 2-1; // Band indexes are zero based. * int nPixel = 3; // Zero based. * int nLine = 4; // Zero based. * * assert( nPixel >= 0 && nPixel < poKern->nDstXSize ); * assert( nLine >= 0 && nLine < poKern->nDstYSize ); * assert( nBand >= 0 && nBand < poKern->nBands ); - * dfPixelValue = ((float *) poKern->papabyDstImage[nBand-1]) + * dfPixelValue = ((float *) poKern->papabyDstImage[nBand]) * [nPixel + nLine * poKern->nSrcYSize]; * \endcode * From c7ce4c5ddacf1b6606590a27fe90b4dfc554b098 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 16 Sep 2020 21:39:56 +0200 Subject: [PATCH 038/255] Warp: fix computation of kernel resampling width when wraping accros the antimeridian Contributes to improving #2754, regarding not manually passing -wo XSCALE= Does not address correct guessing of the target dataset resolution --- autotest/utilities/test_gdalwarp_lib.py | 35 +++++++++++ gdal/alg/gdalwarpkernel.cpp | 83 ++++++++++++++++++++++++- 2 files changed, 117 insertions(+), 1 deletion(-) diff --git a/autotest/utilities/test_gdalwarp_lib.py b/autotest/utilities/test_gdalwarp_lib.py index c3dc60efeac4..e51916c369db 100755 --- a/autotest/utilities/test_gdalwarp_lib.py +++ b/autotest/utilities/test_gdalwarp_lib.py @@ -1988,6 +1988,41 @@ def test_gdalwarp_lib_no_crs(): out_ds = gdal.Warp('', src_ds, options = '-of MEM -ct "+proj=unitconvert +xy_in=1 +xy_out=2"') assert out_ds.GetGeoTransform() == (0.0, 5.0, 0.0, 0.0, 0.0, -5.0) + +############################################################################### +# Test that the warp kernel properly computes the resampling kernel xsize +# when wraping along the antimeridian (related to #2754) + +def test_gdalwarp_lib_xscale_antimeridian(): + + sr = osr.SpatialReference() + sr.SetFromUserInput("WGS84") + + src1_ds = gdal.GetDriverByName('GTiff').Create('/vsimem/src1.tif', 1000, 1000) + src1_ds.SetGeoTransform([179, 0.001, 0, 50, 0, -0.001]) + src1_ds.SetProjection(sr.ExportToWkt()) + src1_ds.GetRasterBand(1).Fill(100) + src1_ds = None + + src2_ds = gdal.GetDriverByName('GTiff').Create('/vsimem/src2.tif', 1000, 1000) + src2_ds.SetGeoTransform([-180, 0.001, 0, 50, 0, -0.001]) + src2_ds.SetProjection(sr.ExportToWkt()) + src2_ds.GetRasterBand(1).Fill(200) + src2_ds = None + + source = gdal.BuildVRT('', ['/vsimem/src1.tif', '/vsimem/src2.tif']) + # Wrap to UTM zone 1 accross the antimeridian + ds = gdal.Warp('', source, options="-of MEM -t_srs EPSG:32601 -te 276000 5464000 290000 5510000 -tr 1000 1000 -r cubic") + vals = struct.unpack('B' * ds.RasterXSize * ds.RasterYSize, ds.ReadRaster()) + assert vals[0] == 100 + assert vals[ds.RasterXSize - 1] == 200 + # Check that the set of values is jsut 100 and 200. If the xscale was wrong, + # we would take intou account 0 values outsize of the 2 tiles. + assert set(vals) == set([100, 200]) + + gdal.Unlink('/vsimem/src1.tif') + gdal.Unlink('/vsimem/src2.tif') + ############################################################################### # Cleanup diff --git a/gdal/alg/gdalwarpkernel.cpp b/gdal/alg/gdalwarpkernel.cpp index 5b39d239c447..e93d061b516a 100644 --- a/gdal/alg/gdalwarpkernel.cpp +++ b/gdal/alg/gdalwarpkernel.cpp @@ -1065,12 +1065,93 @@ CPLErr GDALWarpKernel::PerformWarp() // XSCALE and YSCALE undocumented for now. Can help in some cases. // Best would probably be a per-pixel scale computation. const char* pszXScale = CSLFetchNameValue(papszWarpOptions, "XSCALE"); - if( pszXScale != nullptr ) + if( pszXScale != nullptr && !EQUAL(pszXScale, "FROM_GRID_SAMPLING") ) dfXScale = CPLAtof(pszXScale); const char* pszYScale = CSLFetchNameValue(papszWarpOptions, "YSCALE"); if( pszYScale != nullptr ) dfYScale = CPLAtof(pszYScale); + // If the xscale is significantly lower than the yscale, this is highly + // suspicious of a situation of wrapping a very large virtual file in + // geographic coordinates with left and right parts being close to the + // antimeridian. In that situation, the xscale computed by the above method + // is completely wrong. Prefer doing an average of a few sample points + // instead + if( (dfYScale / dfXScale > 100 || + (pszXScale != nullptr && EQUAL(pszXScale, "FROM_GRID_SAMPLING"))) ) + { + // Sample points along a grid + const int nPointsX = std::min(10, nDstXSize); + const int nPointsY = std::min(10, nDstYSize); + const int nPoints = 3 * nPointsX * nPointsY; + std::vector padfX; + std::vector padfY; + std::vector padfZ(nPoints); + std::vector pabSuccess(nPoints); + for(int iY = 0; iY < nPointsY; iY++) + { + for(int iX = 0; iX < nPointsX; iX++) + { + const double dfX = nPointsX == 1 ? 0.0 : + static_cast(iX) * nDstXSize / (nPointsX - 1); + const double dfY = nPointsY == 1 ? 0.0 : + static_cast(iY) * nDstYSize / (nPointsY - 1); + + // Reproject each destination sample point and its neighbours + // at (x+1,y) and (x,y+1), so as to get the local scale. + padfX.push_back( dfX ); + padfY.push_back( dfY ); + + padfX.push_back( (iX == nPointsX - 1) ? dfX - 1 : dfX + 1 ); + padfY.push_back( dfY); + + padfX.push_back( dfX ); + padfY.push_back( (iY == nPointsY - 1) ? dfY - 1 : dfY + 1 ); + } + } + pfnTransformer(pTransformerArg, TRUE, nPoints, + &padfX[0], &padfY[0], &padfZ[0], &pabSuccess[0]); + + // Compute the xscale at each sampling point + std::vector adfXScales; + for( int i = 0; i < nPoints; i+=3 ) + { + if( pabSuccess[i] && pabSuccess[i+1] && pabSuccess[i+2] ) + { + const double dfPointXScale = 1.0 / std::max( + std::abs(padfX[i+1] - padfX[i]), std::abs(padfX[i+2] - padfX[i]) ); + adfXScales.push_back(dfPointXScale); + } + } + + // Sort by increasing xcale + std::sort(adfXScales.begin(), adfXScales.end()); + + if( !adfXScales.empty() ) + { + // Compute the average of scales, but eliminate outliers small + // scales, if some samples are just along the discontinuity. + const double dfMaxPointXScale = adfXScales.back(); + double dfSumPointXScale = 0; + int nCountPointScale = 0; + for( double dfPointXScale : adfXScales ) + { + if( dfPointXScale > dfMaxPointXScale / 10 ) + { + dfSumPointXScale += dfPointXScale; + nCountPointScale ++; + } + } + + const double dfXScaleFromSampling = dfSumPointXScale / nCountPointScale; +#if DEBUG_VERBOSE + CPLDebug("WARP", "Correcting dfXScale from %f to %f", + dfXScale, dfXScaleFromSampling); +#endif + dfXScale = dfXScaleFromSampling; + } + } + #if DEBUG_VERBOSE CPLDebug("WARP", "dfXScale = %f, dfYScale = %f", dfXScale, dfYScale); #endif From a819d5963a8d94f594ca495c28bc0facf503c34c Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 17 Sep 2020 12:46:14 +0200 Subject: [PATCH 039/255] autotest/cpp/Makefile: fix dependencies for quick_test target --- autotest/cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autotest/cpp/Makefile b/autotest/cpp/Makefile index 024ec170a035..f5cceb1f8dfe 100644 --- a/autotest/cpp/Makefile +++ b/autotest/cpp/Makefile @@ -14,7 +14,7 @@ test check: all make quick_test ./testperfcopywords -quick_test: gdal_unit_test testcopywords testclosedondestroydm testthreadcond testvirtualmem testblockcache testblockcachewrite testblockcachelimits testmultithreadedwriting testdestroy bug1488 +quick_test: gdal_unit_test testcopywords testclosedondestroydm testthreadcond testvirtualmem testblockcache testblockcachewrite testblockcachelimits testmultithreadedwriting testdestroy test_osr_set_proj_search_paths bug1488 proj_with_fork ./gdal_unit_test ./testcopywords ./testclosedondestroydm From 79ba54e1c67a602cd788eea814f4c9f9b0040e08 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 17 Sep 2020 12:48:23 +0200 Subject: [PATCH 040/255] Fixes related to GEOS 3.9 OverlayNG The only change in code is the TransformBeforeAntimeridianToWGS84() method. Instead of intersecting the geometry with a multipolygon made of two parts, one left of the antimeridian, one right of it, but which were at a very close distance, which resulted in a LINESTRING EMPTY intersection with OverlayNG, we compute the difference between our geometry and a thin polygon around the antimeridian, which works for older GEOS too. --- autotest/ogr/ogr_geojson.py | 58 +++++++++++++++-------- autotest/ogr/ogr_geom.py | 7 +-- autotest/ogr/ogr_geos.py | 2 +- autotest/ogr/ogr_layer_algebra.py | 8 ++-- autotest/ogr/ogr_mvt.py | 7 +-- autotest/ogr/ogr_sql_sqlite.py | 2 +- autotest/pymod/ogrtest.py | 6 +++ autotest/utilities/test_ogr2ogr.py | 11 ++--- gdal/ogr/ogrgeometryfactory.cpp | 75 +++++++++++------------------- 9 files changed, 89 insertions(+), 87 deletions(-) diff --git a/autotest/ogr/ogr_geojson.py b/autotest/ogr/ogr_geojson.py index 08963ef6c3a9..bdef2a29c28e 100755 --- a/autotest/ogr/ogr_geojson.py +++ b/autotest/ogr/ogr_geojson.py @@ -2001,7 +2001,16 @@ def test_ogr_geojson_56(): ] } """ - assert json.loads(got) == json.loads(expected) + + j_got = json.loads(got) + j_expected = json.loads(expected) + assert j_got["bbox"] == j_expected["bbox"] + assert len(j_expected["features"]) == 5 + assert ogrtest.check_feature_geometry(ogr.CreateGeometryFromJson(json.dumps(j_got["features"][0]["geometry"])), ogr.CreateGeometryFromJson(json.dumps(j_expected["features"][0]["geometry"]))) == 0 + assert ogrtest.check_feature_geometry(ogr.CreateGeometryFromJson(json.dumps(j_got["features"][1]["geometry"])), ogr.CreateGeometryFromJson(json.dumps(j_expected["features"][1]["geometry"]))) == 0 + assert ogrtest.check_feature_geometry(ogr.CreateGeometryFromJson(json.dumps(j_got["features"][2]["geometry"])), ogr.CreateGeometryFromJson(json.dumps(j_expected["features"][2]["geometry"]))) == 0 + assert ogrtest.check_feature_geometry(ogr.CreateGeometryFromJson(json.dumps(j_got["features"][3]["geometry"])), ogr.CreateGeometryFromJson(json.dumps(j_expected["features"][3]["geometry"]))) == 0 + assert ogrtest.check_feature_geometry(ogr.CreateGeometryFromJson(json.dumps(j_got["features"][4]["geometry"])), ogr.CreateGeometryFromJson(json.dumps(j_expected["features"][4]["geometry"]))) == 0 # Test polygon geometry that covers the whole world (#2833) @@ -2082,7 +2091,14 @@ def test_ogr_geojson_57(): ] } """ - assert json.loads(got) == json.loads(expected) + + j_got = json.loads(got) + j_expected = json.loads(expected) + assert j_got["bbox"] == j_expected["bbox"] + assert len(j_expected["features"]) == 2 + assert ogrtest.check_feature_geometry(ogr.CreateGeometryFromJson(json.dumps(j_got["features"][0]["geometry"])), ogr.CreateGeometryFromJson(json.dumps(j_expected["features"][0]["geometry"]))) == 0 + assert ogrtest.check_feature_geometry(ogr.CreateGeometryFromJson(json.dumps(j_got["features"][1]["geometry"])), ogr.CreateGeometryFromJson(json.dumps(j_expected["features"][1]["geometry"]))) == 0 + # Polar case: slice of spherical cap (not intersecting antimeridian, west hemisphere) src_ds = gdal.GetDriverByName('Memory').Create('', 0, 0, 0) @@ -2151,7 +2167,14 @@ def test_ogr_geojson_57(): ] } """ - assert json.loads(got) == json.loads(expected) + expected_geos_overlay_ng = """{ +"type": "FeatureCollection", +"bbox": [ 135.0000000, 88.6984598, -135.0000000, 90.0000000 ], +"features": [ +{ "type": "Feature", "properties": { }, "bbox": [ 135.0, 88.6984598, -135.0, 90.0 ], "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -135.0, 88.6984598 ], [ -180.0, 90.0 ], [ -180.0, 89.0796531 ], [ -135.0, 88.6984598 ] ] ], [ [ [ 180.0, 90.0 ], [ 135.0, 88.6984598 ], [ 180.0, 89.0796531 ], [ 180.0, 90.0 ] ] ] ] } } +] +}""" + assert json.loads(got) == json.loads(expected) or json.loads(got) == json.loads(expected_geos_overlay_ng), got # Polar case: EPSG:3031: WGS 84 / Antarctic Polar Stereographic src_ds = gdal.GetDriverByName('Memory').Create('', 0, 0, 0) @@ -2174,7 +2197,11 @@ def test_ogr_geojson_57(): ] } """ - assert json.loads(got) == json.loads(expected) + j_got = json.loads(got) + j_expected = json.loads(expected) + assert j_got["bbox"] == j_expected["bbox"] + assert len(j_expected["features"]) == 1 + assert ogrtest.check_feature_geometry(ogr.CreateGeometryFromJson(json.dumps(j_got["features"][0]["geometry"])), ogr.CreateGeometryFromJson(json.dumps(j_expected["features"][0]["geometry"]))) == 0 # Antimeridian case: EPSG:32660: WGS 84 / UTM zone 60N with polygon and line crossing src_ds = gdal.GetDriverByName('Memory').Create('', 0, 0, 0) @@ -2199,26 +2226,19 @@ def test_ogr_geojson_57(): "type": "FeatureCollection", "bbox": [ 178.5275649, 0.0000000, -179.0681936, 37.0308258 ], "features": [ -{ "type": "Feature", "properties": { }, "bbox": [ 178.8892102, 36.0816324, -179.0681936, 37.0308258 ], "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 180.0, 36.1071354 ], [ 180.0, 37.0082839 ], [ 178.9112998, 37.0308258 ], [ 178.8892102, 36.1298163 ], [ 180.0, 36.1071354 ] ] ], [ [ [ -179.0681936, 36.9810434 ], [ -180.0, 37.0082839 ], [ -180.0, 36.1071354 ], [ -179.1135277, 36.0816324 ], [ -179.0681936, 36.9810434 ] ] ] ] } }, +{ "type": "Feature", "properties": { }, "bbox": [ 178.8892102, 36.0816324, -179.0681936, 37.0308258 ], "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 180.0, 36.1071354 ], [ 180.0, 36.1071354 ], [ 180.0, 37.0082839 ], [ 180.0, 37.0082839 ], [ 178.9112998, 37.0308258 ], [ 178.8892102, 36.1298163 ], [ 180.0, 36.1071354 ] ] ], [ [ [ -180.0, 37.0082839 ], [ -180.0, 36.1071354 ], [ -180.0, 36.1071354 ], [ -179.1135277, 36.0816324 ], [ -179.0681936, 36.9810434 ], [ -180.0, 37.0082839 ] ] ] ] } }, { "type": "Feature", "properties": { }, "bbox": [ 178.8892102, 36.1298163, -179.0681936, 36.9810434 ], "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 178.8892102, 36.1298163 ], [ 180.0, 36.5995612 ] ], [ [ -180.0, 36.5995612 ], [ -179.0681936, 36.9810434 ] ] ] } }, { "type": "Feature", "properties": { }, "bbox": [ 178.5275649, 0.0, -179.8562277, 0.0 ], "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 178.5275649, 0.0 ], [ 180.0, 0.0 ] ], [ [ -180.0, 0.0 ], [ -179.8562277, 0.0 ] ] ] } } ] } """ - - # with proj 4.9.3 - expected2 = """{ -"type": "FeatureCollection", -"bbox": [ 178.5275649, 0.0000000, -179.0681936, 37.0308258 ], -"features": [ -{ "type": "Feature", "properties": { }, "bbox": [ 178.8892102, 36.0816324, -179.0681936, 37.0308258 ], "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -179.0681936, 36.9810434 ], [ -180.0, 37.0082839 ], [ -180.0, 36.1071354 ], [ -179.1135277, 36.0816324 ], [ -179.0681936, 36.9810434 ] ] ], [ [ [ 178.8892102, 36.1298163 ], [ 180.0, 36.1071354 ], [ 180.0, 37.0082839 ], [ 178.9112998, 37.0308258 ], [ 178.8892102, 36.1298163 ] ] ] ] } }, -{ "type": "Feature", "properties": { }, "bbox": [ 178.8892102, 36.1298163, -179.0681936, 36.9810434 ], "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 178.8892102, 36.1298163 ], [ 180.0, 36.5995612 ] ], [ [ -180.0, 36.5995612 ], [ -179.0681936, 36.9810434 ] ] ] } }, -{ "type": "Feature", "properties": { }, "bbox": [ 178.5275649, 0.0, -179.8562277, 0.0 ], "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 178.5275649, 0.0 ], [ 180.0, 0.0 ] ], [ [ -180.0, 0.0 ], [ -179.8562277, 0.0 ] ] ] } } -] -} -""" - - assert json.loads(got) == json.loads(expected) or json.loads(got) == json.loads(expected2) + j_got = json.loads(got) + j_expected = json.loads(expected) + assert j_got["bbox"] == j_expected["bbox"] + assert len(j_expected["features"]) == 3 + assert ogrtest.check_feature_geometry(ogr.CreateGeometryFromJson(json.dumps(j_got["features"][0]["geometry"])), ogr.CreateGeometryFromJson(json.dumps(j_expected["features"][0]["geometry"]))) == 0 + assert ogrtest.check_feature_geometry(ogr.CreateGeometryFromJson(json.dumps(j_got["features"][1]["geometry"])), ogr.CreateGeometryFromJson(json.dumps(j_expected["features"][1]["geometry"]))) == 0 + assert ogrtest.check_feature_geometry(ogr.CreateGeometryFromJson(json.dumps(j_got["features"][2]["geometry"])), ogr.CreateGeometryFromJson(json.dumps(j_expected["features"][2]["geometry"]))) == 0 # Antimeridian case: EPSG:32660: WGS 84 / UTM zone 60N wit polygon on west of antimeridian src_ds = gdal.GetDriverByName('Memory').Create('', 0, 0, 0) diff --git a/autotest/ogr/ogr_geom.py b/autotest/ogr/ogr_geom.py index 3d169b95f60c..3f1855de48b6 100755 --- a/autotest/ogr/ogr_geom.py +++ b/autotest/ogr/ogr_geom.py @@ -2617,7 +2617,8 @@ def test_ogr_geom_getcurvegeometry(): g1 = ogr.CreateGeometryFromWkt('CIRCULARSTRING (0 0,1 1,2 0))') g2 = ogr.CreateGeometryFromWkt('CIRCULARSTRING (2 0,1 -1,0 0))') g3 = g1.Union(g2) - assert g3.ExportToWkt() == 'MULTICURVE (CIRCULARSTRING (0 0,1 1,2 0),CIRCULARSTRING (2 0,1 -1,0 0))' + assert g3.ExportToWkt() == 'MULTICURVE (CIRCULARSTRING (0 0,1 1,2 0),CIRCULARSTRING (2 0,1 -1,0 0))' or \ + g3.ExportToWkt() == 'MULTICURVE (CIRCULARSTRING (2 0,1 -1,0 0),CIRCULARSTRING (0 0,1 1,2 0))' # GEOS OverlayNG g1 = ogr.CreateGeometryFromWkt('POINT(1 2)') g1 = g1.Buffer(0.5) @@ -3364,12 +3365,12 @@ def test_ogr_geom_makevalid(): # Invalid g = ogr.CreateGeometryFromWkt('POLYGON ((0 0,10 10,0 10,10 0,0 0))') g = g.MakeValid() - assert g is None or g.ExportToWkt() == 'MULTIPOLYGON (((0 0,5 5,10 0,0 0)),((5 5,0 10,10 10,5 5)))' + assert g is None or ogrtest.check_feature_geometry(g, 'MULTIPOLYGON (((0 0,5 5,10 0,0 0)),((5 5,0 10,10 10,5 5)))') == 0, g.ExportToWkt() # Invalid g = ogr.CreateGeometryFromWkt('CURVEPOLYGON ((0 0,10 10,0 10,10 0,0 0))') g = g.MakeValid() - assert g is None or g.ExportToWkt() == 'MULTIPOLYGON (((0 0,5 5,10 0,0 0)),((5 5,0 10,10 10,5 5)))' + assert g is None or ogrtest.check_feature_geometry(g, 'MULTIPOLYGON (((0 0,5 5,10 0,0 0)),((5 5,0 10,10 10,5 5)))') == 0, g.ExportToWkt() return 'success' diff --git a/autotest/ogr/ogr_geos.py b/autotest/ogr/ogr_geos.py index 443c6020ec17..498ca37274d5 100755 --- a/autotest/ogr/ogr_geos.py +++ b/autotest/ogr/ogr_geos.py @@ -340,7 +340,7 @@ def test_ogr_geos_unioncascaded(): gdal.ErrorReset() cascadedunion = g1.UnionCascaded() - assert cascadedunion.ExportToWkt() == 'POLYGON ((0 0,0 1,0.5 1.0,0.5 1.5,1.5 1.5,1.5 0.5,1.0 0.5,1 0,0 0))', \ + assert ogrtest.check_feature_geometry(cascadedunion, 'POLYGON ((0 0,0 1,0.5 1.0,0.5 1.5,1.5 1.5,1.5 0.5,1.0 0.5,1 0,0 0))') == 0, \ ('Got: %s' % cascadedunion.ExportToWkt()) ############################################################################### diff --git a/autotest/ogr/ogr_layer_algebra.py b/autotest/ogr/ogr_layer_algebra.py index fc81ef63b73a..43be7f890f8b 100755 --- a/autotest/ogr/ogr_layer_algebra.py +++ b/autotest/ogr/ogr_layer_algebra.py @@ -188,10 +188,10 @@ def test_algebra_intersection(): break g = feat.GetGeometryRef() - if g.Equals(f1[0]): + if ogrtest.check_feature_geometry(g, f1[0]) == 0: assert feat.GetField('A') == f1[1] and feat.GetField('B') == f1[2], \ 'Did not get expected field values.' - elif g.Equals(f2[0]): + elif ogrtest.check_feature_geometry(g, f2[0]) == 0: assert feat.GetField('A') == f2[1] and feat.GetField('B') == f2[2], \ 'Did not get expected field values.' else: @@ -220,10 +220,10 @@ def test_algebra_intersection(): break g = feat.GetGeometryRef() - if g.Equals(f1[0]): + if ogrtest.check_feature_geometry(g, f1[0]) == 0: assert feat.GetField('A') == f1[1] and feat.GetField('B') == f1[2], \ 'Did not get expected field values. (1)' - elif g.Equals(f2[0]): + elif ogrtest.check_feature_geometry(g, f2[0]) == 0: assert feat.GetField('A') == f2[1] and feat.GetField('B') == f2[2], \ 'Did not get expected field values. (2)' else: diff --git a/autotest/ogr/ogr_mvt.py b/autotest/ogr/ogr_mvt.py index 96cbc1eb6317..a651083f880f 100755 --- a/autotest/ogr/ogr_mvt.py +++ b/autotest/ogr/ogr_mvt.py @@ -291,9 +291,10 @@ def test_ogr_mvt_point_polygon_clip(): ds = ogr.Open('data/mvt/point_polygon/1') lyr = ds.GetLayer(1) f = lyr.GetNextFeature() - if ogrtest.check_feature_geometry(f, 'MULTIPOLYGON (((0.0 112515.30563578,0 0,-112515.30563578 0.0,-112515.30563578 112515.30563578,0.0 112515.30563578)))') != 0: - f.DumpReadable() - pytest.fail() + expected_wkt = 'MULTIPOLYGON (((0.0 112515.30563578,0 0,-112515.30563578 0.0,-112515.30563578 112515.30563578,0.0 112515.30563578)))' + expected_wkt2 = 'MULTIPOLYGON (((-112515.30563578 112515.30563578,0.0 112515.30563578,0 0,-112515.30563578 0.0,-112515.30563578 112515.30563578)))' + assert ogrtest.check_feature_geometry(f, expected_wkt) == 0 or ogrtest.check_feature_geometry(f, expected_wkt2) == 0, \ + f.GetGeometryRef().ExportToWkt() ############################################################################### diff --git a/autotest/ogr/ogr_sql_sqlite.py b/autotest/ogr/ogr_sql_sqlite.py index 3698785ba809..af2084687484 100755 --- a/autotest/ogr/ogr_sql_sqlite.py +++ b/autotest/ogr/ogr_sql_sqlite.py @@ -1888,7 +1888,7 @@ def test_ogr_sql_sqlite_st_makevalid(): ds.ReleaseResultSet(sql_lyr) if make_valid_available: - assert wkt == 'MULTIPOLYGON (((0.5 0.5,0 0,0 1,0.5 0.5)),((0.5 0.5,1 1,1 0,0.5 0.5)))' + assert ogrtest.check_feature_geometry(ogr.CreateGeometryFromWkt(wkt), 'MULTIPOLYGON (((0.5 0.5,0 0,0 1,0.5 0.5)),((0.5 0.5,1 1,1 0,0.5 0.5)))') == 0, wkt diff --git a/autotest/pymod/ogrtest.py b/autotest/pymod/ogrtest.py index 93a5613ee48b..be22310c078e 100755 --- a/autotest/pymod/ogrtest.py +++ b/autotest/pymod/ogrtest.py @@ -104,6 +104,12 @@ def check_feature_geometry(feat, geom, max_error=0.0001): gdaltest.post_reason('point counts do not match') return 1 + # ST_Equals(a,b) <==> ST_Within(a,b) && ST_Within(b,a) + # We can't use OGRGeometry::Equals() because it doesn't not test spatial + # equality, but structural one + if have_geos() and f_geom.Within(geom) and geom.Within(f_geom): + return 0 + if f_geom.GetGeometryCount() > 0: count = f_geom.GetGeometryCount() for i in range(count): diff --git a/autotest/utilities/test_ogr2ogr.py b/autotest/utilities/test_ogr2ogr.py index 8c6be916ca48..ca7441a75363 100755 --- a/autotest/utilities/test_ogr2ogr.py +++ b/autotest/utilities/test_ogr2ogr.py @@ -519,24 +519,21 @@ def test_ogr2ogr_18(): gdaltest.runexternal(test_cli_utilities.get_ogr2ogr_path() + ' -wrapdateline -t_srs EPSG:4326 tmp/wrapdateline_dst.shp tmp/wrapdateline_src.shp') - expected_wkt = 'MULTIPOLYGON (((-179.667822828781 36.0983491954137,-179.974688335419 27.0898861430767,-180.0 27.0904291236983,-180.0 36.1071354433546,-179.667822828781 36.0983491954137)),((180.0 27.0904291237411,179.017505655195 27.1079795236252,179.222391385437 36.1240958321293,180.0 36.1071354433546,180.0 27.0904291237411)))' + expected_wkt = 'MULTIPOLYGON (((179.222391385437 36.124095832137,180.0 36.1071354434926,180.0 36.107135443432,180.0 27.0904291237556,179.017505655194 27.1079795236266,179.222391385437 36.124095832137)),((-180 36.1071354434425,-179.667822828784 36.0983491954849,-179.974688335432 27.0898861430914,-180 27.0904291237129,-180 27.090429123727,-180 36.107135443432,-180 36.1071354434425)))' + expected_wkt2 = 'MULTIPOLYGON (((179.017505655194 27.1079795236266,179.222391385437 36.124095832137,180.0 36.1071354434926,180.0 36.107135443432,180.0 27.0904291237556,179.017505655194 27.1079795236266)),((-180 27.090429123727,-180 36.107135443432,-180 36.1071354434425,-179.667822828784 36.0983491954849,-179.974688335432 27.0898861430914,-180 27.0904291237129,-180 27.090429123727)))' # with geos OverlayNG - expected_geom = ogr.CreateGeometryFromWkt(expected_wkt) ds = ogr.Open('tmp/wrapdateline_dst.shp') lyr = ds.GetLayer(0) feat = lyr.GetNextFeature() got_wkt = feat.GetGeometryRef().ExportToWkt() - ret = ogrtest.check_feature_geometry(feat, expected_geom) + ok = ogrtest.check_feature_geometry(feat, expected_wkt) == 0 or ogrtest.check_feature_geometry(feat, expected_wkt2) == 0 feat.Destroy() - expected_geom.Destroy() ds.Destroy() ogr.GetDriverByName('ESRI Shapefile').DeleteDataSource('tmp/wrapdateline_src.shp') ogr.GetDriverByName('ESRI Shapefile').DeleteDataSource('tmp/wrapdateline_dst.shp') - if ret == 0: - return - pytest.fail(got_wkt) + assert ok, got_wkt ############################################################################### # Test -clipsrc diff --git a/gdal/ogr/ogrgeometryfactory.cpp b/gdal/ogr/ogrgeometryfactory.cpp index b5287494efd1..ef966edf4927 100644 --- a/gdal/ogr/ogrgeometryfactory.cpp +++ b/gdal/ogr/ogrgeometryfactory.cpp @@ -3603,92 +3603,69 @@ static OGRGeometry* TransformBeforeAntimeridianToWGS84( const double EPS = 1e-9; - // Build a multipolygon (in projected space) with 2 parts: one part left - // of the antimeridian, one part east - const OGRwkbGeometryType eType = wkbFlatten(poDstGeom->getGeometryType()); - - // If we have lines, then to get better accuracy of the intersection with - // the main geometry, we need to add extra points - const bool bHasLines = (eType == wkbLineString || - eType == wkbMultiLineString); - - OGRLinearRing* poLR1 = new OGRLinearRing(); - poLR1->addPoint( sEnvelope.MinX, sEnvelope.MinY ); - if( bHasLines ) + // Build a very thin polygon cutting the antimeridian at our points + OGRLinearRing* poLR = new OGRLinearRing; { - double x = 180.0 - EPS; + double x = 180.0-EPS; double y = aoPoints[0].y-EPS; poRevCT->Transform(1, &x, &y); - poLR1->addPoint( x, y ); + poLR->addPoint( x, y ); } for( const auto& oPoint: aoPoints ) { - double x = 180.0 - EPS; + double x = 180.0-EPS; double y = oPoint.y; poRevCT->Transform(1, &x, &y); - poLR1->addPoint( x, y ); + poLR->addPoint( x, y ); } - if( bHasLines ) { - double x = 180.0 - EPS; + double x = 180.0-EPS; double y = aoPoints.back().y+EPS; poRevCT->Transform(1, &x, &y); - poLR1->addPoint( x, y ); + poLR->addPoint( x, y ); } - poLR1->addPoint( sEnvelope.MinX, sEnvelope.MaxY ); - poLR1->addPoint( sEnvelope.MinX, sEnvelope.MinY ); - OGRPolygon* poPoly1 = new OGRPolygon(); - poPoly1->addRingDirectly( poLR1 ); - - - OGRLinearRing* poLR2 = new OGRLinearRing(); - poLR2->addPoint( sEnvelope.MaxX, sEnvelope.MinY ); - if( bHasLines ) { - double x = -180.0 + EPS; - double y = aoPoints[0].y-EPS; + double x = 180.0+EPS; + double y = aoPoints.back().y+EPS; poRevCT->Transform(1, &x, &y); - poLR2->addPoint( x, y ); + poLR->addPoint( x, y ); } - for( const auto& oPoint: aoPoints ) + for( size_t i = aoPoints.size(); i > 0; ) { - double x = -180.0 + EPS; + --i; + const OGRRawPoint& oPoint = aoPoints[i]; + double x = 180.0+EPS; double y = oPoint.y; poRevCT->Transform(1, &x, &y); - poLR2->addPoint( x, y ); + poLR->addPoint( x, y ); } - if( bHasLines ) { - double x = -180.0 + EPS; - double y = aoPoints.back().y+EPS; + double x = 180.0+EPS; + double y = aoPoints[0].y-EPS; poRevCT->Transform(1, &x, &y); - poLR2->addPoint( x, y ); + poLR->addPoint( x, y ); } - poLR2->addPoint( sEnvelope.MaxX, sEnvelope.MaxY ); - poLR2->addPoint( sEnvelope.MaxX, sEnvelope.MinY ); - OGRPolygon* poPoly2 = new OGRPolygon(); - poPoly2->addRingDirectly( poLR2 ); + poLR->closeRings(); - OGRMultiPolygon oMP; - oMP.addGeometryDirectly(poPoly1); - oMP.addGeometryDirectly(poPoly2); + OGRPolygon oPolyToCut; + oPolyToCut.addRingDirectly(poLR); #if DEBUG_VERBOSE char* pszWKT = NULL; - oMP.exportToWkt(&pszWKT); - CPLDebug("OGR", "MP without antimeridian: %s", pszWKT); + oPolyToCut.exportToWkt(&pszWKT); + CPLDebug("OGR", "Geometry to cut: %s", pszWKT); CPLFree(pszWKT); #endif // Get the geometry without the antimeridian - OGRGeometry* poInter = poDstGeom->Intersection(&oMP); + OGRGeometry* poInter = poDstGeom->Difference(&oPolyToCut); if( poInter != nullptr ) { delete poDstGeom; poDstGeom = poInter; + bNeedPostCorrectionOut = true; } - bNeedPostCorrectionOut = true; return poDstGeom; } From e382b0caf390aed9b0465e2cd2bcf02e26f82652 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 18 Sep 2020 11:31:16 +0200 Subject: [PATCH 041/255] cog.py: add missing unlink --- autotest/gcore/cog.py | 1 + 1 file changed, 1 insertion(+) diff --git a/autotest/gcore/cog.py b/autotest/gcore/cog.py index 124cc4bbda17..3b4ee2cf06e2 100755 --- a/autotest/gcore/cog.py +++ b/autotest/gcore/cog.py @@ -990,3 +990,4 @@ def test_cog_overview_size(): assert (ds.RasterXSize, ds.RasterYSize) == (20480 // 4, 40960 // 4) ovr_size = [ (ds.GetRasterBand(1).GetOverview(i).XSize, ds.GetRasterBand(1).GetOverview(i).YSize) for i in range(ds.GetRasterBand(1).GetOverviewCount()) ] assert ovr_size == [(2048, 4096), (1024, 2048), (512, 1024), (256, 512), (128, 256)] + gdal.Unlink(filename) From b14cf2525d3f73526cf2a1fdf7b5840f93902405 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 18 Sep 2020 11:35:31 +0200 Subject: [PATCH 042/255] COG: fix crash when source dataset is non-Byte/non-UInt16 with a color table (fixes #2946) --- autotest/gcore/cog.py | 22 ++++++++++++++++++++++ gdal/frmts/gtiff/gt_overview.cpp | 2 ++ 2 files changed, 24 insertions(+) diff --git a/autotest/gcore/cog.py b/autotest/gcore/cog.py index 3b4ee2cf06e2..1eee03202e54 100755 --- a/autotest/gcore/cog.py +++ b/autotest/gcore/cog.py @@ -30,6 +30,7 @@ ############################################################################### import pytest +import struct import sys from osgeo import gdal @@ -991,3 +992,24 @@ def test_cog_overview_size(): ovr_size = [ (ds.GetRasterBand(1).GetOverview(i).XSize, ds.GetRasterBand(1).GetOverview(i).YSize) for i in range(ds.GetRasterBand(1).GetOverviewCount()) ] assert ovr_size == [(2048, 4096), (1024, 2048), (512, 1024), (256, 512), (128, 256)] gdal.Unlink(filename) + + +############################################################################### +# Test bugfix for https://github.com/OSGeo/gdal/issues/2946 + + +def test_cog_float32_color_table(): + + src_ds = gdal.GetDriverByName('MEM').Create('', 1024, 1024, 1, gdal.GDT_Float32) + src_ds.GetRasterBand(1).Fill(1.0) + ct = gdal.ColorTable() + src_ds.GetRasterBand(1).SetColorTable(ct) + filename = '/vsimem/test_cog_float32_color_table.tif' + # Silence warning about color table not being copied + with gdaltest.error_handler(): + ds = gdal.GetDriverByName('COG').CreateCopy(filename, src_ds) # segfault + assert ds + assert ds.GetRasterBand(1).GetColorTable() is None + assert struct.unpack('f', ds.ReadRaster(0,0,1,1))[0] == 1.0 + assert struct.unpack('f', ds.GetRasterBand(1).GetOverview(0).ReadRaster(0,0,1,1))[0] == 1.0 + gdal.Unlink(filename) diff --git a/gdal/frmts/gtiff/gt_overview.cpp b/gdal/frmts/gtiff/gt_overview.cpp index e9fdeb57251e..05bbc7dbdd4d 100644 --- a/gdal/frmts/gtiff/gt_overview.cpp +++ b/gdal/frmts/gtiff/gt_overview.cpp @@ -546,6 +546,8 @@ GTIFFBuildOverviewsEx( const char * pszFilename, if( nBands == 3 ) nPhotometric = PHOTOMETRIC_RGB; else if( papoBandList[0]->GetColorTable() != nullptr + && (papoBandList[0]->GetRasterDataType() == GDT_Byte || + papoBandList[0]->GetRasterDataType() == GDT_UInt16) && !STARTS_WITH_CI(pszResampling, "AVERAGE_BIT2") ) { nPhotometric = PHOTOMETRIC_PALETTE; From 474414349ad14d918a620974947c5f061696bf0d Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 18 Sep 2020 12:57:06 +0200 Subject: [PATCH 043/255] gdalwarp: better guess of target resolution when target extent is specified (fixes #2754) --- autotest/utilities/test_gdalwarp.py | 8 +- gdal/apps/gdalwarp_lib.cpp | 133 +++++++++++++++++++++++++--- 2 files changed, 125 insertions(+), 16 deletions(-) diff --git a/autotest/utilities/test_gdalwarp.py b/autotest/utilities/test_gdalwarp.py index 825256c30cd8..3fd920f88489 100755 --- a/autotest/utilities/test_gdalwarp.py +++ b/autotest/utilities/test_gdalwarp.py @@ -1023,6 +1023,8 @@ def test_gdalwarp_40(): ############################################################################### # Test source fill ratio heuristics (#3120) +# Also check that we guess a reasonable resolution (#2754), from the source +# dataset and target extent def test_gdalwarp_41(): @@ -1055,14 +1057,16 @@ def test_gdalwarp_41(): gdaltest.runexternal(test_cli_utilities.get_gdalwarp_path() + ' tmp/test_gdalwarp_41_src.tif tmp/test_gdalwarp_41.tif -overwrite -t_srs EPSG:4326 -te -180 -90 180 90 -wo INIT_DEST=127 -wo SKIP_NOSOURCE=YES') ds = gdal.Open('tmp/test_gdalwarp_41.tif') - assert ds.GetRasterBand(1).Checksum() == 25945 + assert ds.RasterXSize == 2052 + assert ds.RasterYSize == 1026 + assert ds.GetRasterBand(1).Checksum() == 57091 ds = None # Check when source fill ratio heuristics is OFF gdaltest.runexternal(test_cli_utilities.get_gdalwarp_path() + ' tmp/test_gdalwarp_41_src.tif tmp/test_gdalwarp_41.tif -overwrite -t_srs EPSG:4326 -te -180 -90 180 90 -wo INIT_DEST=127 -wo SKIP_NOSOURCE=YES -wo SRC_FILL_RATIO_HEURISTICS=NO') ds = gdal.Open('tmp/test_gdalwarp_41.tif') - assert ds.GetRasterBand(1).Checksum() == 65068 + assert ds.GetRasterBand(1).Checksum() == 31890 ds = None ############################################################################### diff --git a/gdal/apps/gdalwarp_lib.cpp b/gdal/apps/gdalwarp_lib.cpp index a2767120e90f..a4c24b2129af 100644 --- a/gdal/apps/gdalwarp_lib.cpp +++ b/gdal/apps/gdalwarp_lib.cpp @@ -2958,6 +2958,13 @@ GDALWarpCreateOutput( int nSrcCount, GDALDatasetH *pahSrcDS, const char *pszFile !(psOptions->dfMinX == 0.0 && psOptions->dfMinY == 0.0 && psOptions->dfMaxX == 0.0 && psOptions->dfMaxY == 0.0) ); + // If -te is specified, not not -tr and -ts + const bool bKnownTargetExtentButNotResolution = + !(psOptions->dfMinX == 0.0 && psOptions->dfMinY == 0.0 && + psOptions->dfMaxX == 0.0 && psOptions->dfMaxY == 0.0) && + psOptions->nForcePixels == 0 && psOptions->nForceLines == 0 && + psOptions->dfXRes == 0 && psOptions->dfYRes == 0; + if( phTransformArg ) *phTransformArg = nullptr; @@ -3011,21 +3018,24 @@ GDALWarpCreateOutput( int nSrcCount, GDALDatasetH *pahSrcDS, const char *pszFile CPLStringList aoTOList(papszTO, FALSE); + double dfResFromSourceAndTargetExtent = std::numeric_limits::infinity(); + for( int iSrc = 0; iSrc < nSrcCount; iSrc++ ) { /* -------------------------------------------------------------------- */ /* Check that there's at least one raster band */ /* -------------------------------------------------------------------- */ - if ( GDALGetRasterCount(pahSrcDS[iSrc]) == 0 ) + GDALDatasetH hSrcDS = pahSrcDS[iSrc]; + if ( GDALGetRasterCount(hSrcDS) == 0 ) { - CPLError(CE_Failure, CPLE_AppDefined, "Input file %s has no raster bands.", GDALGetDescription(pahSrcDS[iSrc]) ); + CPLError(CE_Failure, CPLE_AppDefined, "Input file %s has no raster bands.", GDALGetDescription(hSrcDS) ); if( hCT != nullptr ) GDALDestroyColorTable( hCT ); return nullptr; } if( eDT == GDT_Unknown ) - eDT = GDALGetRasterDataType(GDALGetRasterBand(pahSrcDS[iSrc],1)); + eDT = GDALGetRasterDataType(GDALGetRasterBand(hSrcDS,1)); /* -------------------------------------------------------------------- */ /* If we are processing the first file, and it has a raster */ @@ -3033,7 +3043,7 @@ GDALWarpCreateOutput( int nSrcCount, GDALDatasetH *pahSrcDS, const char *pszFile /* -------------------------------------------------------------------- */ if( iSrc == 0 ) { - hRAT = GDALGetDefaultRAT( GDALGetRasterBand(pahSrcDS[iSrc],1) ); + hRAT = GDALGetDefaultRAT( GDALGetRasterBand(hSrcDS,1) ); if( hRAT != nullptr ) { if ( psOptions->eResampleAlg != GRA_NearestNeighbour && @@ -3044,7 +3054,7 @@ GDALWarpCreateOutput( int nSrcCount, GDALDatasetH *pahSrcDS, const char *pszFile { CPLError(CE_Warning, CPLE_AppDefined, "Warning: Input file %s has a thematic RAT, which will likely lead " "to bad results when using a resampling method other than nearest neighbour " - "or mode so we are discarding it.\n", GDALGetDescription(pahSrcDS[iSrc])); + "or mode so we are discarding it.\n", GDALGetDescription(hSrcDS)); } hRAT = nullptr; } @@ -3052,7 +3062,7 @@ GDALWarpCreateOutput( int nSrcCount, GDALDatasetH *pahSrcDS, const char *pszFile { if( !psOptions->bQuiet ) printf( "Copying raster attribute table from %s to new file.\n", - GDALGetDescription(pahSrcDS[iSrc])); + GDALGetDescription(hSrcDS)); } } } @@ -3063,20 +3073,20 @@ GDALWarpCreateOutput( int nSrcCount, GDALDatasetH *pahSrcDS, const char *pszFile /* -------------------------------------------------------------------- */ if( iSrc == 0 ) { - nDstBandCount = GDALGetRasterCount(pahSrcDS[iSrc]); - hCT = GDALGetRasterColorTable( GDALGetRasterBand(pahSrcDS[iSrc],1) ); + nDstBandCount = GDALGetRasterCount(hSrcDS); + hCT = GDALGetRasterColorTable( GDALGetRasterBand(hSrcDS,1) ); if( hCT != nullptr ) { hCT = GDALCloneColorTable( hCT ); if( !psOptions->bQuiet ) printf( "Copying color table from %s to new file.\n", - GDALGetDescription(pahSrcDS[iSrc]) ); + GDALGetDescription(hSrcDS) ); } for(int iBand = 0; iBand < nDstBandCount; iBand++) { GDALColorInterp eInterp = - GDALGetRasterColorInterpretation(GDALGetRasterBand(pahSrcDS[iSrc],iBand+1)); + GDALGetRasterColorInterpretation(GDALGetRasterBand(hSrcDS,iBand+1)); apeColorInterpretations.push_back( eInterp ); } } @@ -3087,7 +3097,7 @@ GDALWarpCreateOutput( int nSrcCount, GDALDatasetH *pahSrcDS, const char *pszFile /* -------------------------------------------------------------------- */ if( iSrc == 0 && osThisTargetSRS.empty() ) { - const auto osThisSourceSRS = GetSrcDSProjection( pahSrcDS[iSrc], papszTO ); + const auto osThisSourceSRS = GetSrcDSProjection( hSrcDS, papszTO ); if( !osThisSourceSRS.empty() ) { osThisTargetSRS = osThisSourceSRS; @@ -3100,7 +3110,7 @@ GDALWarpCreateOutput( int nSrcCount, GDALDatasetH *pahSrcDS, const char *pszFile /* destination coordinate system. */ /* -------------------------------------------------------------------- */ hTransformArg = - GDALCreateGenImgProjTransformer2( pahSrcDS[iSrc], nullptr, aoTOList.List() ); + GDALCreateGenImgProjTransformer2( hSrcDS, nullptr, aoTOList.List() ); if( hTransformArg == nullptr ) { @@ -3111,16 +3121,105 @@ GDALWarpCreateOutput( int nSrcCount, GDALDatasetH *pahSrcDS, const char *pszFile GDALTransformerInfo* psInfo = static_cast(hTransformArg); +/* -------------------------------------------------------------------- */ +/* Get approximate output resolution */ +/* -------------------------------------------------------------------- */ + + if( bKnownTargetExtentButNotResolution ) + { + // Sample points along a grid + constexpr int nPointsX = 10; + constexpr int nPointsY = 10; + constexpr int nPoints = 3 * nPointsX * nPointsY; + std::vector padfX; + std::vector padfY; + std::vector padfZ(nPoints); + std::vector pabSuccess(nPoints); + const double dfEps = std::min( + psOptions->dfMaxX - psOptions->dfMinX, + std::abs(psOptions->dfMaxY - psOptions->dfMinY)) / 1000; + for(int iY = 0; iY < nPointsY; iY++) + { + for(int iX = 0; iX < nPointsX; iX++) + { + const double dfX = psOptions->dfMinX + + static_cast(iX) * (psOptions->dfMaxX - psOptions->dfMinX) / (nPointsX - 1); + const double dfY = psOptions->dfMinY + + static_cast(iY) * (psOptions->dfMaxY - psOptions->dfMinY) / (nPointsY - 1); + + // Reproject each destination sample point and its neighbours + // at (x+1,y) and (x,y+1), so as to get the local scale. + padfX.push_back( dfX ); + padfY.push_back( dfY ); + + padfX.push_back( (iX == nPointsX - 1) ? dfX - dfEps : dfX + dfEps ); + padfY.push_back( dfY); + + padfX.push_back( dfX ); + padfY.push_back( (iY == nPointsY - 1) ? dfY - dfEps : dfY + dfEps ); + } + } + + psInfo->pfnTransform(hTransformArg, TRUE, nPoints, + &padfX[0], &padfY[0], &padfZ[0], + &pabSuccess[0]); + + // Compute the resolution at sampling points + std::vector adfRes; + const int nSrcXSize = GDALGetRasterXSize(hSrcDS); + const int nSrcYSize = GDALGetRasterYSize(hSrcDS); + + const auto Distance = [](double x, double y) + { + return sqrt(x*x + y*y); + }; + + for( int i = 0; i < nPoints; i+=3 ) + { + if( pabSuccess[i] && pabSuccess[i+1] && pabSuccess[i+2] && + padfX[i] >= 0 && padfX[i] <= nSrcXSize && + padfY[i] >= 0 && padfY[i] <= nSrcYSize ) + { + const double dfRes1 = std::abs(dfEps) / + Distance(padfX[i+1] - padfX[i], padfY[i+1] - padfY[i]); + const double dfRes2 = std::abs(dfEps) / + Distance(padfX[i+2] - padfX[i], padfY[i+2] - padfY[i]); + if( std::isfinite(dfRes1) && std::isfinite(dfRes2) ) + { + adfRes.push_back((dfRes1 + dfRes2) / 2); + } + } + } + + // Find the minimum resolution that is at least 10 times greater + // than te median, to remove outliers. + std::sort(adfRes.begin(), adfRes.end()); + if( !adfRes.empty() ) + { + const double dfMedian = adfRes[ adfRes.size() / 2 ]; + for( const double dfRes: adfRes ) + { + if( dfRes > dfMedian / 10 ) + { + dfResFromSourceAndTargetExtent = std::min( + dfResFromSourceAndTargetExtent, dfRes); + break; + } + } + } + } + /* -------------------------------------------------------------------- */ /* Get approximate output definition. */ /* -------------------------------------------------------------------- */ + if( bNeedsSuggestedWarpOutput ) { double adfThisGeoTransform[6]; double adfExtent[4]; int nThisPixels, nThisLines; - if ( GDALSuggestedWarpOutput2( pahSrcDS[iSrc], + if ( GDALSuggestedWarpOutput2( hSrcDS, psInfo->pfnTransform, hTransformArg, adfThisGeoTransform, &nThisPixels, &nThisLines, @@ -3177,7 +3276,7 @@ GDALWarpCreateOutput( int nSrcCount, GDALDatasetH *pahSrcDS, const char *pszFile CPLDebug("WARP", "Recompute out extent with CHECK_WITH_INVERT_PROJ=TRUE"); const CPLErr eErr = GDALSuggestedWarpOutput2( - pahSrcDS[iSrc], + hSrcDS, psInfo->pfnTransform, hTransformArg, adfThisGeoTransform, &nThisPixels, &nThisLines, @@ -3228,6 +3327,12 @@ GDALWarpCreateOutput( int nSrcCount, GDALDatasetH *pahSrcDS, const char *pszFile } } + if( std::isfinite(dfResFromSourceAndTargetExtent) ) + { + dfWrkResX = dfResFromSourceAndTargetExtent; + dfWrkResY = dfResFromSourceAndTargetExtent; + } + /* -------------------------------------------------------------------- */ /* Did we have any usable sources? */ /* -------------------------------------------------------------------- */ From dff5c06e0b0155f9bea6c7014cea130c7bc153c5 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 18 Sep 2020 14:48:51 +0200 Subject: [PATCH 044/255] OAPIF: fix memory leak when reading schema from .xsd --- gdal/ogr/ogrsf_frmts/wfs/ogroapifdriver.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/gdal/ogr/ogrsf_frmts/wfs/ogroapifdriver.cpp b/gdal/ogr/ogrsf_frmts/wfs/ogroapifdriver.cpp index 945fd47b89ef..b47d369c2f46 100644 --- a/gdal/ogr/ogrsf_frmts/wfs/ogroapifdriver.cpp +++ b/gdal/ogr/ogrsf_frmts/wfs/ogroapifdriver.cpp @@ -1043,14 +1043,14 @@ void OGROAPIFLayer::GetSchema() if( m_bDescribedByIsXML ) { - std::vector aosClasses; + std::vector apoClasses; bool bFullyUnderstood = false; - bool bHaveSchema = GMLParseXSD( m_osDescribedByURL, aosClasses, + bool bHaveSchema = GMLParseXSD( m_osDescribedByURL, apoClasses, bFullyUnderstood ); - if (bHaveSchema && aosClasses.size() == 1) + if (bHaveSchema && apoClasses.size() == 1) { CPLDebug("OAPIF", "Using XML schema"); - const auto poGMLFeatureClass = aosClasses[0]; + auto poGMLFeatureClass = apoClasses[0]; if( poGMLFeatureClass->GetGeometryPropertyCount() == 1 ) { // Force linear type as we work with GeoJSON data @@ -1085,6 +1085,9 @@ void OGROAPIFLayer::GetSchema() m_apoFieldsFromSchema.emplace_back(std::move(oField)); } } + + for( auto poFeatureClass: apoClasses ) + delete poFeatureClass; } else { From fa839b55f793867fe0bfe7239f18216793be4f7f Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 18 Sep 2020 16:59:11 +0200 Subject: [PATCH 045/255] PCRaster: fix Create() mode by propagating eAccess = GA_Update (fixes #2948) --- autotest/gdrivers/pcraster.py | 20 ++++++++++++-------- gdal/frmts/pcraster/pcrasterdataset.cpp | 5 +++-- gdal/frmts/pcraster/pcrasterdataset.h | 2 +- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/autotest/gdrivers/pcraster.py b/autotest/gdrivers/pcraster.py index 50712732ad0f..f69999feab13 100755 --- a/autotest/gdrivers/pcraster.py +++ b/autotest/gdrivers/pcraster.py @@ -34,17 +34,14 @@ import gdaltest import pytest +pytestmark = pytest.mark.require_driver('PCRaster') + ############################################################################### # Perform simple read test. def test_pcraster_1(): - gdaltest.pcraster_drv = gdal.GetDriverByName('PCRaster') - - if gdaltest.pcraster_drv is None: - pytest.skip() - tst = gdaltest.GDALTest('PCRaster', 'pcraster/ldd.map', 1, 4528) return tst.testOpen() @@ -54,9 +51,6 @@ def test_pcraster_1(): def test_pcraster_2(): - if gdaltest.pcraster_drv is None: - pytest.skip() - ds = gdal.Open('data/pcraster/ldd.map') gt = ds.GetGeoTransform() @@ -67,6 +61,16 @@ def test_pcraster_2(): band1 = ds.GetRasterBand(1) assert band1.GetNoDataValue() == 255, 'PCRaster NODATA value wrong or missing.' +############################################################################### +def test_pcraster_createcopy(): + + tst = gdaltest.GDALTest('PCRaster', 'pcraster/ldd.map', 1, 4528) + return tst.testCreateCopy(new_filename = 'tmp/ldd.map') + +############################################################################### +def test_pcraster_create(): + tst = gdaltest.GDALTest('PCRaster', 'float32.tif', 1, 4672, options=['PCRASTER_VALUESCALE=VS_SCALAR']) + return tst.testCreate(new_filename = 'tmp/float32.map') diff --git a/gdal/frmts/pcraster/pcrasterdataset.cpp b/gdal/frmts/pcraster/pcrasterdataset.cpp index 8d001940d76a..3aa9d70d3a31 100644 --- a/gdal/frmts/pcraster/pcrasterdataset.cpp +++ b/gdal/frmts/pcraster/pcrasterdataset.cpp @@ -66,7 +66,7 @@ GDALDataset* PCRasterDataset::open( if(map) { CPLErrorReset(); - dataset = new PCRasterDataset(map); + dataset = new PCRasterDataset(map, info->eAccess); if( CPLGetLastErrorType() != CE_None ) { delete dataset; @@ -286,7 +286,7 @@ GDALDataset* PCRasterDataset::createCopy( /*! \param mapIn PCRaster map handle. It is ours to close. */ -PCRasterDataset::PCRasterDataset( MAP* mapIn) : +PCRasterDataset::PCRasterDataset( MAP* mapIn, GDALAccess eAccessIn ) : GDALPamDataset(), d_map(mapIn), d_west(0.0), @@ -298,6 +298,7 @@ PCRasterDataset::PCRasterDataset( MAP* mapIn) : d_location_changed(false) { // Read header info. + eAccess = eAccessIn; nRasterXSize = static_cast(RgetNrCols(d_map)); nRasterYSize = static_cast(RgetNrRows(d_map)); if( !GDALCheckDatasetDimensions(nRasterXSize, nRasterYSize) ) diff --git a/gdal/frmts/pcraster/pcrasterdataset.h b/gdal/frmts/pcraster/pcrasterdataset.h index 5bb0c8d20a12..e285ab0461a0 100644 --- a/gdal/frmts/pcraster/pcrasterdataset.h +++ b/gdal/frmts/pcraster/pcrasterdataset.h @@ -113,7 +113,7 @@ class PCRasterDataset final: public GDALPamDataset // CREATORS //---------------------------------------------------------------------------- - explicit PCRasterDataset (MAP* map); + explicit PCRasterDataset (MAP* map, GDALAccess eAccess); /* virtual */ ~PCRasterDataset (); From c844925163e58df4b0faa2befa26434c19c04e07 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 19 Sep 2020 15:10:00 +0200 Subject: [PATCH 046/255] gdal2tiles.py: make sure configuration options specified with --config are passed to worker processes (fixes #2950) --- gdal/swig/python/scripts/gdal2tiles.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/gdal/swig/python/scripts/gdal2tiles.py b/gdal/swig/python/scripts/gdal2tiles.py index 278d4d8534c2..7bbcf190f76b 100755 --- a/gdal/swig/python/scripts/gdal2tiles.py +++ b/gdal/swig/python/scripts/gdal2tiles.py @@ -3195,6 +3195,13 @@ def main(): # TODO: gbataille - use mkdtemp to work in a temp directory # TODO: gbataille - debug intermediate tiles.vrt not produced anymore? # TODO: gbataille - Refactor generate overview tiles to not depend on self variables + + # For multiprocessing, we need to propagate the configuration options to + # the environment, so that forked processes can inherit them. + for i in range(len(sys.argv)): + if sys.argv[i] == '--config' and i + 2 < len(sys.argv): + os.environ[sys.argv[i+1]] = sys.argv[i+2] + argv = gdal.GeneralCmdLineProcessor(sys.argv) if argv is None: return From e27e47865c7b35ea02daf5db52f2392a15c0c946 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 19 Sep 2020 15:58:59 +0200 Subject: [PATCH 047/255] VRT: rename member variable --- gdal/frmts/vrt/vrtdataset.h | 2 +- gdal/frmts/vrt/vrtrasterband.cpp | 46 ++++++++++++++++---------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/gdal/frmts/vrt/vrtdataset.h b/gdal/frmts/vrt/vrtdataset.h index e0490184e9c1..0562d9219402 100644 --- a/gdal/frmts/vrt/vrtdataset.h +++ b/gdal/frmts/vrt/vrtdataset.h @@ -474,7 +474,7 @@ class CPL_DLL VRTRasterBand CPL_NON_FINAL: public GDALRasterBand void Initialize( int nXSize, int nYSize ); - std::vector m_apoOverviews{}; + std::vector m_aoOverviewInfos{}; VRTRasterBand *m_poMaskBand; diff --git a/gdal/frmts/vrt/vrtrasterband.cpp b/gdal/frmts/vrt/vrtrasterband.cpp index 59298dfe79fb..b70e6432be12 100644 --- a/gdal/frmts/vrt/vrtrasterband.cpp +++ b/gdal/frmts/vrt/vrtrasterband.cpp @@ -529,9 +529,9 @@ CPLErr VRTRasterBand::XMLInit( CPLXMLNode * psTree, /* -------------------------------------------------------------------- */ const int nSrcBand = atoi(CPLGetXMLValue( psNode, "SourceBand", "1" ) ); - m_apoOverviews.resize( m_apoOverviews.size() + 1 ); - m_apoOverviews.back().osFilename = pszSrcDSName; - m_apoOverviews.back().nBand = nSrcBand; + m_aoOverviewInfos.resize( m_aoOverviewInfos.size() + 1 ); + m_aoOverviewInfos.back().osFilename = pszSrcDSName; + m_aoOverviewInfos.back().nBand = nSrcBand; CPLFree( pszSrcDSName ); } @@ -747,7 +747,7 @@ CPLXMLNode *VRTRasterBand::SerializeToXML( const char *pszVRTPath ) /* Overviews */ /* ==================================================================== */ - for( int iOvr = 0; iOvr < static_cast( m_apoOverviews.size() ); iOvr ++ ) + for( int iOvr = 0; iOvr < static_cast( m_aoOverviewInfos.size() ); iOvr ++ ) { CPLXMLNode *psOVR_XML = CPLCreateXMLNode( psTree, CXT_Element, "Overview" ); @@ -756,15 +756,15 @@ CPLXMLNode *VRTRasterBand::SerializeToXML( const char *pszVRTPath ) const char *pszRelativePath = nullptr; VSIStatBufL sStat; - if( VSIStatExL( m_apoOverviews[iOvr].osFilename, &sStat, VSI_STAT_EXISTS_FLAG ) != 0 ) + if( VSIStatExL( m_aoOverviewInfos[iOvr].osFilename, &sStat, VSI_STAT_EXISTS_FLAG ) != 0 ) { - pszRelativePath = m_apoOverviews[iOvr].osFilename; + pszRelativePath = m_aoOverviewInfos[iOvr].osFilename; bRelativeToVRT = FALSE; } else { pszRelativePath = - CPLExtractRelativePath( pszVRTPath, m_apoOverviews[iOvr].osFilename, + CPLExtractRelativePath( pszVRTPath, m_aoOverviewInfos[iOvr].osFilename, &bRelativeToVRT ); } @@ -776,7 +776,7 @@ CPLXMLNode *VRTRasterBand::SerializeToXML( const char *pszVRTPath ) CXT_Text, bRelativeToVRT ? "1" : "0" ); CPLSetXMLValue( psOVR_XML, "SourceBand", - CPLSPrintf("%d",m_apoOverviews[iOvr].nBand) ); + CPLSPrintf("%d",m_aoOverviewInfos[iOvr].nBand) ); } /* ==================================================================== */ @@ -1085,9 +1085,9 @@ VRTRasterBand::GetDefaultHistogram( double *pdfMin, double *pdfMax, void VRTRasterBand::GetFileList(char*** ppapszFileList, int *pnSize, int *pnMaxSize, CPLHashSet* hSetFiles) { - for( unsigned int iOver = 0; iOver < m_apoOverviews.size(); iOver++ ) + for( unsigned int iOver = 0; iOver < m_aoOverviewInfos.size(); iOver++ ) { - const CPLString &osFilename = m_apoOverviews[iOver].osFilename; + const CPLString &osFilename = m_aoOverviewInfos[iOver].osFilename; /* -------------------------------------------------------------------- */ /* Is the filename even a real filesystem object? */ @@ -1131,8 +1131,8 @@ int VRTRasterBand::GetOverviewCount() { // First: overviews declared in element - if( !m_apoOverviews.empty() ) - return static_cast(m_apoOverviews.size()); + if( !m_aoOverviewInfos.empty() ) + return static_cast(m_aoOverviewInfos.size()); // If not found, external .ovr overviews const int nOverviewCount = GDALRasterBand::GetOverviewCount(); @@ -1156,19 +1156,19 @@ GDALRasterBand *VRTRasterBand::GetOverview( int iOverview ) { // First: overviews declared in element - if( !m_apoOverviews.empty() ) + if( !m_aoOverviewInfos.empty() ) { if( iOverview < 0 - || iOverview >= static_cast( m_apoOverviews.size() ) ) + || iOverview >= static_cast( m_aoOverviewInfos.size() ) ) return nullptr; - if( m_apoOverviews[iOverview].poBand == nullptr - && !m_apoOverviews[iOverview].bTriedToOpen ) + if( m_aoOverviewInfos[iOverview].poBand == nullptr + && !m_aoOverviewInfos[iOverview].bTriedToOpen ) { - m_apoOverviews[iOverview].bTriedToOpen = TRUE; + m_aoOverviewInfos[iOverview].bTriedToOpen = TRUE; CPLConfigOptionSetter oSetter("CPL_ALLOW_VSISTDIN", "NO", true); GDALDataset *poSrcDS = GDALDataset::FromHandle( - GDALOpenShared( m_apoOverviews[iOverview].osFilename, + GDALOpenShared( m_aoOverviewInfos[iOverview].osFilename, GA_ReadOnly ) ); if( poSrcDS == nullptr ) @@ -1181,16 +1181,16 @@ GDALRasterBand *VRTRasterBand::GetOverview( int iOverview ) return nullptr; } - m_apoOverviews[iOverview].poBand = poSrcDS->GetRasterBand( - m_apoOverviews[iOverview].nBand ); + m_aoOverviewInfos[iOverview].poBand = poSrcDS->GetRasterBand( + m_aoOverviewInfos[iOverview].nBand ); - if (m_apoOverviews[iOverview].poBand == nullptr) + if (m_aoOverviewInfos[iOverview].poBand == nullptr) { GDALClose( GDALDataset::ToHandle(poSrcDS) ); } } - return m_apoOverviews[iOverview].poBand; + return m_aoOverviewInfos[iOverview].poBand; } // If not found, external .ovr overviews @@ -1318,7 +1318,7 @@ void VRTRasterBand::SetIsMaskBand() int VRTRasterBand::CloseDependentDatasets() { int ret = FALSE; - for( auto& oOverviewInfo: m_apoOverviews ) + for( auto& oOverviewInfo: m_aoOverviewInfos ) { if( oOverviewInfo.CloseDataset() ) { From 6d8e37ed69a412503d8b980944bca2ab49b9e14c Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 19 Sep 2020 16:50:46 +0200 Subject: [PATCH 048/255] NAS: do not try to write a .gfs file when NAS_GFS_TEMPLATE is specified --- gdal/ogr/ogrsf_frmts/nas/ogrnasdatasource.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/gdal/ogr/ogrsf_frmts/nas/ogrnasdatasource.cpp b/gdal/ogr/ogrsf_frmts/nas/ogrnasdatasource.cpp index 7e01a726f561..bb2f5054b011 100644 --- a/gdal/ogr/ogrsf_frmts/nas/ogrnasdatasource.cpp +++ b/gdal/ogr/ogrsf_frmts/nas/ogrnasdatasource.cpp @@ -110,6 +110,7 @@ int OGRNASDataSource::Open( const char * pszNewName ) pszNASTemplateName ); return FALSE; } + bHaveSchema = true; CPLDebug("NAS", "Schema loaded."); } From 13cfbf880c19e62517c1b7f60c01c156fba1a6a5 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 19 Sep 2020 17:18:46 +0200 Subject: [PATCH 049/255] GDALBuildVRT(): add support for sources such as MEM dataset or non-materialized VRT files --- autotest/utilities/test_gdalbuildvrt_lib.py | 42 ++++- gdal/apps/gdalbuildvrt_lib.cpp | 167 ++++++++++---------- 2 files changed, 125 insertions(+), 84 deletions(-) diff --git a/autotest/utilities/test_gdalbuildvrt_lib.py b/autotest/utilities/test_gdalbuildvrt_lib.py index 7f5316006511..af43f324e101 100755 --- a/autotest/utilities/test_gdalbuildvrt_lib.py +++ b/autotest/utilities/test_gdalbuildvrt_lib.py @@ -29,7 +29,7 @@ # DEALINGS IN THE SOFTWARE. ############################################################################### - +import struct from osgeo import gdal @@ -112,3 +112,43 @@ def test_gdalbuildvrt_lib_te_partial_overlap(): xml = ds.GetMetadata('xml:VRT')[0] assert '' in xml assert '' in xml + + +############################################################################### +# Test BuildVRT() with sources that can't be opened by name + +def test_gdalbuildvrt_lib_mem_sources(): + + def create_sources(): + src1_ds = gdal.GetDriverByName('MEM').Create('i_have_a_name_but_nobody_can_open_me_through_it', 1, 1) + src1_ds.SetGeoTransform([2,1,0,49,0,-1]) + src1_ds.GetRasterBand(1).Fill(100) + + src2_ds = gdal.GetDriverByName('MEM').Create('', 1, 1) + src2_ds.SetGeoTransform([3,1,0,49,0,-1]) + src2_ds.GetRasterBand(1).Fill(200) + + return src1_ds, src2_ds + + def scenario_1(): + src1_ds, src2_ds = create_sources() + vrt_ds = gdal.BuildVRT('', [src1_ds, src2_ds]) + vals = struct.unpack('B' * 2, vrt_ds.ReadRaster()) + assert vals == (100, 200) + + vrt_of_vrt_ds = gdal.BuildVRT('', [vrt_ds]) + vals = struct.unpack('B' * 2, vrt_of_vrt_ds.ReadRaster()) + assert vals == (100, 200) + + # Alternate scenario where the Python objects of sources and intermediate + # VRT are no longer alive when the VRT of VRT is accessed + def scenario_2(): + def get_vrt_of_vrt(): + src1_ds, src2_ds = create_sources() + return gdal.BuildVRT('', [ gdal.BuildVRT('', [src1_ds, src2_ds]) ]) + vrt_of_vrt_ds = get_vrt_of_vrt() + vals = struct.unpack('B' * 2, vrt_of_vrt_ds.ReadRaster()) + assert vals == (100, 200) + + scenario_1() + scenario_2() diff --git a/gdal/apps/gdalbuildvrt_lib.cpp b/gdal/apps/gdalbuildvrt_lib.cpp index fdfb1d8dd48a..8056abe08688 100644 --- a/gdal/apps/gdalbuildvrt_lib.cpp +++ b/gdal/apps/gdalbuildvrt_lib.cpp @@ -200,49 +200,50 @@ static int GetSrcDstWin(DatasetProperty* psDP, class VRTBuilder { /* Input parameters */ - char *pszOutputFilename; - int nInputFiles; - char **ppszInputFilenames; - GDALDatasetH *pahSrcDS; - int nBands; - int *panBandList; - int nMaxBandNo; - ResolutionStrategy resolutionStrategy; - double we_res; - double ns_res; - int bTargetAlignedPixels; - double minX; - double minY; - double maxX; - double maxY; - int bSeparate; - int bAllowProjectionDifference; - int bAddAlpha; - int bHideNoData; - int nSubdataset; - char *pszSrcNoData; - char *pszVRTNoData; - char *pszOutputSRS; - char *pszResampling; - char **papszOpenOptions; + char *pszOutputFilename = nullptr; + int nInputFiles = 0; + char **ppszInputFilenames = nullptr; + int nSrcDSCount = 0; + GDALDatasetH *pahSrcDS = nullptr; + int nBands = 0; + int *panBandList = nullptr; + int nMaxBandNo = 0; + ResolutionStrategy resolutionStrategy = AVERAGE_RESOLUTION; + double we_res = 0; + double ns_res = 0; + int bTargetAlignedPixels = 0; + double minX = 0; + double minY = 0; + double maxX = 0; + double maxY = 0; + int bSeparate = 0; + int bAllowProjectionDifference = 0; + int bAddAlpha = 0; + int bHideNoData = 0; + int nSubdataset = 0; + char *pszSrcNoData = nullptr; + char *pszVRTNoData = nullptr; + char *pszOutputSRS = nullptr; + char *pszResampling = nullptr; + char **papszOpenOptions = nullptr; /* Internal variables */ - char *pszProjectionRef; - BandProperty *pasBandProperties; - int bFirst; - int bHasGeoTransform; - int nRasterXSize; - int nRasterYSize; - DatasetProperty *pasDatasetProperties; - int bUserExtent; - int bAllowSrcNoData; - double *padfSrcNoData; - int nSrcNoDataCount; - int bAllowVRTNoData; - double *padfVRTNoData; - int nVRTNoDataCount; - int bHasRunBuild; - int bHasDatasetMask; + char *pszProjectionRef = nullptr; + BandProperty *pasBandProperties = nullptr; + int bFirst = TRUE; + int bHasGeoTransform = 0; + int nRasterXSize = 0; + int nRasterYSize = 0; + DatasetProperty *pasDatasetProperties = nullptr; + int bUserExtent = 0; + int bAllowSrcNoData = TRUE; + double *padfSrcNoData = nullptr; + int nSrcNoDataCount = 0; + int bAllowVRTNoData = TRUE; + double *padfVRTNoData = nullptr; + int nVRTNoDataCount = 0; + int bHasRunBuild = 0; + int bHasDatasetMask = 0; int AnalyseRaster(GDALDatasetH hDS, DatasetProperty* psDatasetProperties); @@ -294,8 +295,6 @@ VRTBuilder::VRTBuilder(const char* pszOutputFilenameIn, { pszOutputFilename = CPLStrdup(pszOutputFilenameIn); nInputFiles = nInputFilesIn; - pahSrcDS = nullptr; - ppszInputFilenames = nullptr; papszOpenOptions = CSLDuplicate(const_cast(papszOpenOptionsIn)); if( ppszInputFilenamesIn ) @@ -309,6 +308,7 @@ VRTBuilder::VRTBuilder(const char* pszOutputFilenameIn, } else if( pahSrcDSIn ) { + nSrcDSCount = nInputFiles; pahSrcDS = static_cast( CPLMalloc(nInputFiles * sizeof(GDALDatasetH))); memcpy(pahSrcDS, pahSrcDSIn, nInputFiles * sizeof(GDALDatasetH)); @@ -321,7 +321,6 @@ VRTBuilder::VRTBuilder(const char* pszOutputFilenameIn, } nBands = nBandCount; - panBandList = nullptr; if( nBandCount ) { panBandList = static_cast(CPLMalloc(nBands * sizeof(int))); @@ -346,23 +345,6 @@ VRTBuilder::VRTBuilder(const char* pszOutputFilenameIn, pszVRTNoData = (pszVRTNoDataIn) ? CPLStrdup(pszVRTNoDataIn) : nullptr; pszOutputSRS = (pszOutputSRSIn) ? CPLStrdup(pszOutputSRSIn) : nullptr; pszResampling = (pszResamplingIn) ? CPLStrdup(pszResamplingIn) : nullptr; - - bUserExtent = FALSE; - pszProjectionRef = nullptr; - pasBandProperties = nullptr; - bFirst = TRUE; - bHasGeoTransform = FALSE; - nRasterXSize = 0; - nRasterYSize = 0; - pasDatasetProperties = nullptr; - bAllowSrcNoData = TRUE; - padfSrcNoData = nullptr; - nSrcNoDataCount = 0; - bAllowVRTNoData = TRUE; - padfVRTNoData = nullptr; - nVRTNoDataCount = 0; - bHasRunBuild = FALSE; - bHasDatasetMask = FALSE; } /************************************************************************/ @@ -1089,28 +1071,44 @@ void VRTBuilder::CreateVRTNonSeparate(VRTDatasetH hVRTDS) const char* dsFileName = ppszInputFilenames[i]; - GDALProxyPoolDatasetH hProxyDS = - GDALProxyPoolDatasetCreate(dsFileName, - psDatasetProperties->nRasterXSize, - psDatasetProperties->nRasterYSize, - GA_ReadOnly, TRUE, pszProjectionRef, - psDatasetProperties->adfGeoTransform); - reinterpret_cast(hProxyDS)-> - SetOpenOptions( papszOpenOptions ); + GDALDatasetH hSourceDS; + bool bDropRef = false; - for(int j=0;jnBlockXSize, - psDatasetProperties->nBlockYSize); + hSourceDS = pahSrcDS[i]; } - if (bHasDatasetMask && !bAddAlpha) + else { - static_cast(reinterpret_cast(hProxyDS)->GetRasterBand(1))-> - AddSrcMaskBandDescription (GDT_Byte, - psDatasetProperties->nMaskBlockXSize, - psDatasetProperties->nMaskBlockYSize); + bDropRef = true; + GDALProxyPoolDatasetH hProxyDS = + GDALProxyPoolDatasetCreate(dsFileName, + psDatasetProperties->nRasterXSize, + psDatasetProperties->nRasterYSize, + GA_ReadOnly, TRUE, pszProjectionRef, + psDatasetProperties->adfGeoTransform); + reinterpret_cast(hProxyDS)-> + SetOpenOptions( papszOpenOptions ); + + for(int j=0;jnBlockXSize, + psDatasetProperties->nBlockYSize); + } + if (bHasDatasetMask && !bAddAlpha) + { + static_cast(reinterpret_cast(hProxyDS)->GetRasterBand(1))-> + AddSrcMaskBandDescription (GDT_Byte, + psDatasetProperties->nMaskBlockXSize, + psDatasetProperties->nMaskBlockYSize); + } + + hSourceDS = static_cast(hProxyDS); } for(int j=0;jSetResampling(pszResampling); poVRTBand->ConfigureSource( poSimpleSource, - static_cast(GDALGetRasterBand(static_cast(hProxyDS), nSelBand + 1)), + static_cast(GDALGetRasterBand(hSourceDS, nSelBand + 1)), FALSE, dfSrcXOff, dfSrcYOff, dfSrcXSize, dfSrcYSize, @@ -1150,7 +1148,7 @@ void VRTBuilder::CreateVRTNonSeparate(VRTDatasetH hVRTDS) /* Little trick : we use an offset of 255 and a scaling of 0, so that in areas covered */ /* by the source, the value of the alpha band will be 255, otherwise it will be 0 */ static_cast(hVRTBand)->AddComplexSource( - static_cast(GDALGetRasterBand(static_cast(hProxyDS), 1)), + static_cast(GDALGetRasterBand(hSourceDS, 1)), dfSrcXOff, dfSrcYOff, dfSrcXSize, dfSrcYSize, dfDstXOff, dfDstYOff, @@ -1163,7 +1161,7 @@ void VRTBuilder::CreateVRTNonSeparate(VRTDatasetH hVRTDS) if( pszResampling ) poSimpleSource->SetResampling(pszResampling); poMaskVRTBand->ConfigureSource( poSimpleSource, - static_cast(GDALGetRasterBand(static_cast(hProxyDS), 1)), + static_cast(GDALGetRasterBand(hSourceDS, 1)), TRUE, dfSrcXOff, dfSrcYOff, dfSrcXSize, dfSrcYSize, @@ -1173,7 +1171,10 @@ void VRTBuilder::CreateVRTNonSeparate(VRTDatasetH hVRTDS) poMaskVRTBand->AddSource( poSimpleSource ); } - GDALDereferenceDataset(hProxyDS); + if( bDropRef ) + { + GDALDereferenceDataset(hSourceDS); + } } } From c07fe0b9e03cb861d7524f909e59cdbdcf9c0256 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 21 Sep 2020 10:44:37 +0200 Subject: [PATCH 050/255] PCIDSK: fix harmless unsigned-integer-overflow. Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=25822 --- gdal/frmts/pcidsk/sdk/segment/cpcidskads40model.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gdal/frmts/pcidsk/sdk/segment/cpcidskads40model.cpp b/gdal/frmts/pcidsk/sdk/segment/cpcidskads40model.cpp index 3c605b035244..d4dac8bc5323 100644 --- a/gdal/frmts/pcidsk/sdk/segment/cpcidskads40model.cpp +++ b/gdal/frmts/pcidsk/sdk/segment/cpcidskads40model.cpp @@ -80,7 +80,7 @@ void CPCIDSKADS40ModelSegment::Load() return; } - if( data_size - 1024 != 1 * 512 ) + if( data_size != 1024 + 1 * 512 ) { return ThrowPCIDSKException("Wrong data_size in CPCIDSKADS40ModelSegment"); } From 429f09a39b746793a546555ca1de3b01e08cf663 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 21 Sep 2020 15:17:12 +0200 Subject: [PATCH 051/255] MRF: fix Heap-buffer-overflow READ 4. Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=25836. master only --- gdal/frmts/mrf/LERCV1/Lerc1Image.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gdal/frmts/mrf/LERCV1/Lerc1Image.cpp b/gdal/frmts/mrf/LERCV1/Lerc1Image.cpp index 85acead11387..56483d8903ca 100644 --- a/gdal/frmts/mrf/LERCV1/Lerc1Image.cpp +++ b/gdal/frmts/mrf/LERCV1/Lerc1Image.cpp @@ -211,7 +211,7 @@ static bool blockread(Byte** ppByte, size_t& size, std::vector& d) Byte numBits = **ppByte; Byte n = stib67[numBits >> 6]; numBits &= 63; // bits 0-5; - if (numBits >= 32 || n == 0 || size < static_cast(n)) + if (numBits >= 32 || n == 0 || size < 1 + static_cast(n)) return false; *ppByte += 1; size -= 1; @@ -236,7 +236,9 @@ static bool blockread(Byte** ppByte, size_t& size, std::vector& d) int bits = 0; // Available in accumulator, at the high end unsigned int acc = 0; + int countElts = 0; for (unsigned int& val : d) { + countElts ++; if (bits >= numBits) { // Enough bits in accumulator val = acc >> (32 - numBits); acc <<= numBits; From 2442cd5ae08d640944297e1efe56ffcc375ded19 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 21 Sep 2020 15:18:13 +0200 Subject: [PATCH 052/255] MRF: remove useless debugging variables added in previous commit --- gdal/frmts/mrf/LERCV1/Lerc1Image.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/gdal/frmts/mrf/LERCV1/Lerc1Image.cpp b/gdal/frmts/mrf/LERCV1/Lerc1Image.cpp index 56483d8903ca..e536dd85b465 100644 --- a/gdal/frmts/mrf/LERCV1/Lerc1Image.cpp +++ b/gdal/frmts/mrf/LERCV1/Lerc1Image.cpp @@ -236,9 +236,7 @@ static bool blockread(Byte** ppByte, size_t& size, std::vector& d) int bits = 0; // Available in accumulator, at the high end unsigned int acc = 0; - int countElts = 0; for (unsigned int& val : d) { - countElts ++; if (bits >= numBits) { // Enough bits in accumulator val = acc >> (32 - numBits); acc <<= numBits; From c91c85854631084021f9a75fe6797fea2ac071c4 Mon Sep 17 00:00:00 2001 From: Alessandro Pasotti Date: Mon, 21 Sep 2020 15:20:58 +0200 Subject: [PATCH 053/255] Add GDALDataset::AbortSQL (#2953) This adds a method to interrupt running queries launched by ExecuteSQL. --- autotest/ogr/ogr_gpkg.py | 57 ++++++++++++++++++ autotest/ogr/ogr_pg.py | 44 ++++++++++++++ gdal/gcore/gdal.h | 1 + gdal/gcore/gdal_priv.h | 1 + gdal/gcore/gdaldataset.cpp | 57 ++++++++++++++++++ .../ogr/ogrsf_frmts/generic/ogrdatasource.cpp | 1 + gdal/ogr/ogrsf_frmts/pg/ogr_pg.h | 1 + gdal/ogr/ogrsf_frmts/pg/ogrpgdatasource.cpp | 22 +++++++ gdal/ogr/ogrsf_frmts/sqlite/ogr_sqlite.h | 4 +- .../sqlite/ogrsqlitedatasource.cpp | 11 ++++ gdal/swig/include/Dataset.i | 5 ++ gdal/swig/include/ogr.i | 4 ++ gdal/swig/python/extensions/gdal_wrap.cpp | 60 +++++++++++++++++++ gdal/swig/python/extensions/ogr_wrap.cpp | 60 +++++++++++++++++++ gdal/swig/python/osgeo/gdal.py | 5 ++ gdal/swig/python/osgeo/ogr.py | 5 ++ 16 files changed, 337 insertions(+), 1 deletion(-) diff --git a/autotest/ogr/ogr_gpkg.py b/autotest/ogr/ogr_gpkg.py index b82ac423f0bd..1c88c6097fa3 100755 --- a/autotest/ogr/ogr_gpkg.py +++ b/autotest/ogr/ogr_gpkg.py @@ -34,6 +34,8 @@ import struct import sys import pytest +import time +import threading from osgeo import gdal from osgeo import ogr @@ -4146,3 +4148,58 @@ def test_ogr_gpkg_datetime_timezones(): ds = None gdal.Unlink(filename) + + +############################################################################### +# Test AbortSQL + +def test_abort_sql(): + + filename = '/vsimem/test_ogr_gpkg_abort_sql.gpkg' + ds = ogr.GetDriverByName('GPKG').CreateDataSource(filename) + ds.CreateLayer('test') + ds = None + + ds = ogr.Open(filename, update=1) + + def abortAfterDelay(): + print("Aborting SQL...") + assert ds.AbortSQL() == ogr.OGRERR_NONE + + t = threading.Timer(0.5, abortAfterDelay) + t.start() + + start = time.time() + + # Long running query + sql = """ + WITH RECURSIVE r(i) AS ( + VALUES(0) + UNION ALL + SELECT i FROM r + LIMIT 10000000 + ) + SELECT i FROM r WHERE i = 1;""" + + ds.ExecuteSQL(sql) + + end = time.time() + assert int(end - start) < 1 + + # Same test with a GDAL dataset + ds2 = gdal.OpenEx(filename, gdal.OF_VECTOR) + + def abortAfterDelay2(): + print("Aborting SQL...") + assert ds2.AbortSQL() == ogr.OGRERR_NONE + + t = threading.Timer(0.5, abortAfterDelay2) + t.start() + + start = time.time() + + # Long running query + ds2.ExecuteSQL(sql) + + end = time.time() + assert int(end - start) < 1 diff --git a/autotest/ogr/ogr_pg.py b/autotest/ogr/ogr_pg.py index eeae1fef4f9c..207037501c01 100755 --- a/autotest/ogr/ogr_pg.py +++ b/autotest/ogr/ogr_pg.py @@ -30,6 +30,8 @@ import os import sys import shutil +import time +import threading import pytest @@ -4733,6 +4735,48 @@ def test_ogr_pg_table_cleanup(): gdaltest.pg_ds.ExecuteSQL('DROP SCHEMA \"AutoTest-schema\" CASCADE') gdal.PopErrorHandler() +############################################################################### +# Test AbortSQL + +def test_abort_sql(): + + if gdaltest.pg_ds is None: + pytest.skip() + + def abortAfterDelay(): + print("Aborting SQL...") + assert gdaltest.pg_ds.AbortSQL() == ogr.OGRERR_NONE + + t = threading.Timer(0.5, abortAfterDelay) + t.start() + + start = time.time() + + # Long running query + sql = "SELECT pg_sleep(3)" + gdaltest.pg_ds.ExecuteSQL(sql) + + end = time.time() + assert int(end - start) < 1 + + # Same test with a GDAL dataset + ds2 = gdal.OpenEx('PG:' + gdaltest.pg_connection_string, gdal.OF_VECTOR) + + def abortAfterDelay2(): + print("Aborting SQL...") + assert ds2.AbortSQL() == ogr.OGRERR_NONE + + t = threading.Timer(0.5, abortAfterDelay2) + t.start() + + start = time.time() + + # Long running query + ds2.ExecuteSQL(sql) + + end = time.time() + assert int(end - start) < 1 + def test_ogr_pg_cleanup(): diff --git a/gdal/gcore/gdal.h b/gdal/gcore/gdal.h index 6346145a4058..e9247ec34fb1 100644 --- a/gdal/gcore/gdal.h +++ b/gdal/gcore/gdal.h @@ -833,6 +833,7 @@ OGRFeatureH CPL_DLL GDALDatasetGetNextFeature( GDALDatasetH hDS, int CPL_DLL GDALDatasetTestCapability( GDALDatasetH, const char * ); OGRLayerH CPL_DLL GDALDatasetExecuteSQL( GDALDatasetH, const char *, OGRGeometryH, const char * ); +OGRErr CPL_DLL GDALDatasetAbortSQL( GDALDatasetH ); void CPL_DLL GDALDatasetReleaseResultSet( GDALDatasetH, OGRLayerH ); OGRStyleTableH CPL_DLL GDALDatasetGetStyleTable( GDALDatasetH ); void CPL_DLL GDALDatasetSetStyleTableDirectly( GDALDatasetH, OGRStyleTableH ); diff --git a/gdal/gcore/gdal_priv.h b/gdal/gcore/gdal_priv.h index 476ccbb943aa..6c5c25476c2c 100644 --- a/gdal/gcore/gdal_priv.h +++ b/gdal/gcore/gdal_priv.h @@ -791,6 +791,7 @@ class CPL_DLL GDALDataset : public GDALMajorObject OGRGeometry *poSpatialFilter, const char *pszDialect ); virtual void ReleaseResultSet( OGRLayer * poResultsSet ); + virtual OGRErr AbortSQL( ); int GetRefCount() const; int GetSummaryRefCount() const; diff --git a/gdal/gcore/gdaldataset.cpp b/gdal/gcore/gdaldataset.cpp index 0941b3f1d0e2..b743f7a52180 100644 --- a/gdal/gcore/gdaldataset.cpp +++ b/gdal/gcore/gdaldataset.cpp @@ -4578,6 +4578,37 @@ OGRLayerH GDALDatasetExecuteSQL( GDALDatasetH hDS, pszDialect)); } + +/************************************************************************/ +/* GDALDatasetAbortSQL() */ +/************************************************************************/ + +/** + \brief Abort any SQL statement running in the data store. + + This function can be safely called from any thread (pending that the dataset object is still alive). Driver implementations will make sure that it can be called in a thread-safe way. + + This might not be implemented by all drivers. At time of writing, only SQLite, GPKG and PG drivers implement it + + This method is the same as the C++ method GDALDataset::AbortSQL() + + @since GDAL 3.2.0 + + @param hDS the dataset handle. + + @return OGRERR_NONE on success, or OGRERR_UNSUPPORTED_OPERATION if AbortSQL + is not supported for this datasource. . + +*/ + +OGRErr GDALDatasetAbortSQL( GDALDatasetH hDS ) + +{ + VALIDATE_POINTER1(hDS, "GDALDatasetAbortSQL", OGRERR_FAILURE ); + return GDALDataset::FromHandle(hDS)->AbortSQL(); +} + + /************************************************************************/ /* GDALDatasetGetStyleTable() */ /************************************************************************/ @@ -6157,6 +6188,32 @@ GDALDataset::ExecuteSQL( const char *pszStatement, } //! @endcond + +/************************************************************************/ +/* AbortSQL() */ +/************************************************************************/ + +/** + \brief Abort any SQL statement running in the data store. + + This function can be safely called from any thread (pending that the dataset object is still alive). Driver implementations will make sure that it can be called in a thread-safe way. + + This might not be implemented by all drivers. At time of writing, only SQLite, GPKG and PG drivers implement it + + This method is the same as the C method GDALDatasetAbortSQL() + + @since GDAL 3.2.0 + + +*/ + +OGRErr GDALDataset::AbortSQL( ) +{ + CPLError(CE_Failure, CPLE_NotSupported, "AbortSQL is not supported for this driver."); + return OGRERR_UNSUPPORTED_OPERATION; +} + + /************************************************************************/ /* BuildLayerFromSelectInfo() */ /************************************************************************/ diff --git a/gdal/ogr/ogrsf_frmts/generic/ogrdatasource.cpp b/gdal/ogr/ogrsf_frmts/generic/ogrdatasource.cpp index 42365ea3d409..4fe88a2eda08 100644 --- a/gdal/ogr/ogrsf_frmts/generic/ogrdatasource.cpp +++ b/gdal/ogr/ogrsf_frmts/generic/ogrdatasource.cpp @@ -231,6 +231,7 @@ OGRLayerH OGR_DS_ExecuteSQL( OGRDataSourceH hDS, return hLayer; } + /************************************************************************/ /* OGR_DS_ReleaseResultSet() */ /************************************************************************/ diff --git a/gdal/ogr/ogrsf_frmts/pg/ogr_pg.h b/gdal/ogr/ogrsf_frmts/pg/ogr_pg.h index f11ecffd95d2..324c1c4c3ee2 100644 --- a/gdal/ogr/ogrsf_frmts/pg/ogr_pg.h +++ b/gdal/ogr/ogrsf_frmts/pg/ogr_pg.h @@ -564,6 +564,7 @@ class OGRPGDataSource final: public OGRDataSource virtual OGRLayer * ExecuteSQL( const char *pszSQLCommand, OGRGeometry *poSpatialFilter, const char *pszDialect ) override; + virtual OGRErr AbortSQL() override; virtual void ReleaseResultSet( OGRLayer * poLayer ) override; virtual const char* GetMetadataItem(const char* pszKey, diff --git a/gdal/ogr/ogrsf_frmts/pg/ogrpgdatasource.cpp b/gdal/ogr/ogrsf_frmts/pg/ogrpgdatasource.cpp index 97baa34fa046..9f51c4a5c259 100644 --- a/gdal/ogr/ogrsf_frmts/pg/ogrpgdatasource.cpp +++ b/gdal/ogr/ogrsf_frmts/pg/ogrpgdatasource.cpp @@ -3005,6 +3005,28 @@ OGRLayer * OGRPGDataSource::ExecuteSQL( const char *pszSQLCommand, return nullptr; } + +/************************************************************************/ +/* AbortSQL() */ +/************************************************************************/ + + +OGRErr OGRPGDataSource::AbortSQL() +{ + auto cancel = PQgetCancel( hPGConn ) ; + int result; + if ( cancel ) + { + char errbuf[255]; + result = PQcancel( cancel, errbuf, 255 ); + if ( ! result ) + CPLDebug( "PG", "Error canceling the query: %s", errbuf ); + PQfreeCancel( cancel ); + return result ? OGRERR_NONE : OGRERR_FAILURE; + } + return OGRERR_FAILURE; +} + /************************************************************************/ /* ReleaseResultSet() */ /************************************************************************/ diff --git a/gdal/ogr/ogrsf_frmts/sqlite/ogr_sqlite.h b/gdal/ogr/ogrsf_frmts/sqlite/ogr_sqlite.h index 756a7310594a..e59b9465f174 100644 --- a/gdal/ogr/ogrsf_frmts/sqlite/ogr_sqlite.h +++ b/gdal/ogr/ogrsf_frmts/sqlite/ogr_sqlite.h @@ -740,13 +740,15 @@ class OGRSQLiteBaseDataSource CPL_NON_FINAL: public GDALPamDataset virtual std::pair GetLayerWithGetSpatialWhereByName( const char* pszName ) = 0; + virtual OGRErr AbortSQL() override; + virtual OGRErr StartTransaction(int bForce = FALSE) override; virtual OGRErr CommitTransaction() override; virtual OGRErr RollbackTransaction() override; virtual int TestCapability( const char * ) override; - virtual void *GetInternalHandle( const char * ) override; + virtual void *GetInternalHandle( const char * ) override; OGRErr SoftStartTransaction(); OGRErr SoftCommitTransaction(); diff --git a/gdal/ogr/ogrsf_frmts/sqlite/ogrsqlitedatasource.cpp b/gdal/ogr/ogrsf_frmts/sqlite/ogrsqlitedatasource.cpp index 020bf6510933..1dbdd3e3b25a 100644 --- a/gdal/ogr/ogrsf_frmts/sqlite/ogrsqlitedatasource.cpp +++ b/gdal/ogr/ogrsf_frmts/sqlite/ogrsqlitedatasource.cpp @@ -944,6 +944,7 @@ void *OGRSQLiteBaseDataSource::GetInternalHandle( const char * pszKey ) return nullptr; } + /************************************************************************/ /* Create() */ /************************************************************************/ @@ -3781,3 +3782,13 @@ void OGRSQLiteBaseDataSource::SetEnvelopeForSQL(const CPLString& osSQL, { oMapSQLEnvelope[osSQL] = oEnvelope; } + +/************************************************************************/ +/* AbortSQL() */ +/************************************************************************/ + +OGRErr OGRSQLiteBaseDataSource::AbortSQL() +{ + sqlite3_interrupt( hDB ); + return OGRERR_NONE; +} diff --git a/gdal/swig/include/Dataset.i b/gdal/swig/include/Dataset.i index be882401d170..8de0e510a473 100644 --- a/gdal/swig/include/Dataset.i +++ b/gdal/swig/include/Dataset.i @@ -975,6 +975,11 @@ CPLErr AdviseRead( int xoff, int yoff, int xsize, int ysize, #endif /* defined(SWIGPYTHON) || defined(SWIGJAVA) || defined(SWIGPERL) */ + +OGRErr AbortSQL() { + return GDALDatasetAbortSQL(self); +} + #ifndef SWIGJAVA %feature( "kwargs" ) StartTransaction; #endif diff --git a/gdal/swig/include/ogr.i b/gdal/swig/include/ogr.i index 92774ba50dc8..b0689390f8c9 100644 --- a/gdal/swig/include/ogr.i +++ b/gdal/swig/include/ogr.i @@ -853,6 +853,10 @@ public: return layer; } + OGRErr AbortSQL(){ + return GDALDatasetAbortSQL((OGRDataSourceShadow*)self); + } + %apply SWIGTYPE *DISOWN {OGRLayerShadow *layer}; void ReleaseResultSet(OGRLayerShadow *layer){ OGR_DS_ReleaseResultSet(self, layer); diff --git a/gdal/swig/python/extensions/gdal_wrap.cpp b/gdal/swig/python/extensions/gdal_wrap.cpp index d2da5bd5b018..d2ee01dd42df 100644 --- a/gdal/swig/python/extensions/gdal_wrap.cpp +++ b/gdal/swig/python/extensions/gdal_wrap.cpp @@ -5290,6 +5290,9 @@ SWIGINTERN void GDALDatasetShadow_SetStyleTable(GDALDatasetShadow *self,OGRStyle if( table != NULL ) GDALDatasetSetStyleTable(self, (OGRStyleTableH) table); } +SWIGINTERN OGRErr GDALDatasetShadow_AbortSQL(GDALDatasetShadow *self){ + return GDALDatasetAbortSQL(self); +} SWIGINTERN OGRErr GDALDatasetShadow_StartTransaction(GDALDatasetShadow *self,int force=FALSE){ return GDALDatasetStartTransaction(self, force); } @@ -20604,6 +20607,62 @@ SWIGINTERN PyObject *_wrap_Dataset_SetStyleTable(PyObject *SWIGUNUSEDPARM(self), } +SWIGINTERN PyObject *_wrap_Dataset_AbortSQL(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *resultobj = 0; int bLocalUseExceptionsCode = bUseExceptions; + GDALDatasetShadow *arg1 = (GDALDatasetShadow *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject * obj0 = 0 ; + OGRErr result; + + if (!PyArg_ParseTuple(args,(char *)"O:Dataset_AbortSQL",&obj0)) SWIG_fail; + res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_GDALDatasetShadow, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Dataset_AbortSQL" "', argument " "1"" of type '" "GDALDatasetShadow *""'"); + } + arg1 = reinterpret_cast< GDALDatasetShadow * >(argp1); + { + if ( bUseExceptions ) { + ClearErrorState(); + } + { + SWIG_PYTHON_THREAD_BEGIN_ALLOW; + result = (OGRErr)GDALDatasetShadow_AbortSQL(arg1); + SWIG_PYTHON_THREAD_END_ALLOW; + } +#ifndef SED_HACKS + if ( bUseExceptions ) { + CPLErr eclass = CPLGetLastErrorType(); + if ( eclass == CE_Failure || eclass == CE_Fatal ) { + SWIG_exception( SWIG_RuntimeError, CPLGetLastErrorMsg() ); + } + } +#endif + } + { + /* %typemap(out) OGRErr */ + if ( result != 0 && bUseExceptions) { + const char* pszMessage = CPLGetLastErrorMsg(); + if( pszMessage[0] != '\0' ) + PyErr_SetString( PyExc_RuntimeError, pszMessage ); + else + PyErr_SetString( PyExc_RuntimeError, OGRErrMessages(result) ); + SWIG_fail; + } + } + { + /* %typemap(ret) OGRErr */ + if ( ReturnSame(resultobj == Py_None || resultobj == 0) ) { + resultobj = PyInt_FromLong( result ); + } + } + if ( ReturnSame(bLocalUseExceptionsCode) ) { CPLErr eclass = CPLGetLastErrorType(); if ( eclass == CE_Failure || eclass == CE_Fatal ) { Py_XDECREF(resultobj); SWIG_Error( SWIG_RuntimeError, CPLGetLastErrorMsg() ); return NULL; } } + return resultobj; +fail: + return NULL; +} + + SWIGINTERN PyObject *_wrap_Dataset_StartTransaction(PyObject *SWIGUNUSEDPARM(self), PyObject *args, PyObject *kwargs) { PyObject *resultobj = 0; int bLocalUseExceptionsCode = bUseExceptions; GDALDatasetShadow *arg1 = (GDALDatasetShadow *) 0 ; @@ -42645,6 +42704,7 @@ static PyMethodDef SwigMethods[] = { { (char *)"Dataset_ReleaseResultSet", _wrap_Dataset_ReleaseResultSet, METH_VARARGS, (char *)"Dataset_ReleaseResultSet(Dataset self, Layer layer)"}, { (char *)"Dataset_GetStyleTable", _wrap_Dataset_GetStyleTable, METH_VARARGS, (char *)"Dataset_GetStyleTable(Dataset self) -> StyleTable"}, { (char *)"Dataset_SetStyleTable", _wrap_Dataset_SetStyleTable, METH_VARARGS, (char *)"Dataset_SetStyleTable(Dataset self, StyleTable table)"}, + { (char *)"Dataset_AbortSQL", _wrap_Dataset_AbortSQL, METH_VARARGS, (char *)"Dataset_AbortSQL(Dataset self) -> OGRErr"}, { (char *)"Dataset_StartTransaction", (PyCFunction) _wrap_Dataset_StartTransaction, METH_VARARGS | METH_KEYWORDS, (char *)"Dataset_StartTransaction(Dataset self, int force=False) -> OGRErr"}, { (char *)"Dataset_CommitTransaction", _wrap_Dataset_CommitTransaction, METH_VARARGS, (char *)"Dataset_CommitTransaction(Dataset self) -> OGRErr"}, { (char *)"Dataset_RollbackTransaction", _wrap_Dataset_RollbackTransaction, METH_VARARGS, (char *)"Dataset_RollbackTransaction(Dataset self) -> OGRErr"}, diff --git a/gdal/swig/python/extensions/ogr_wrap.cpp b/gdal/swig/python/extensions/ogr_wrap.cpp index f58c7bf45596..d23b46f06040 100644 --- a/gdal/swig/python/extensions/ogr_wrap.cpp +++ b/gdal/swig/python/extensions/ogr_wrap.cpp @@ -4060,6 +4060,9 @@ SWIGINTERN OGRLayerShadow *OGRDataSourceShadow_ExecuteSQL(OGRDataSourceShadow *s dialect); return layer; } +SWIGINTERN OGRErr OGRDataSourceShadow_AbortSQL(OGRDataSourceShadow *self){ + return GDALDatasetAbortSQL((OGRDataSourceShadow*)self); + } SWIGINTERN void OGRDataSourceShadow_ReleaseResultSet(OGRDataSourceShadow *self,OGRLayerShadow *layer){ OGR_DS_ReleaseResultSet(self, layer); } @@ -8230,6 +8233,62 @@ SWIGINTERN PyObject *_wrap_DataSource_ExecuteSQL(PyObject *SWIGUNUSEDPARM(self), } +SWIGINTERN PyObject *_wrap_DataSource_AbortSQL(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *resultobj = 0; int bLocalUseExceptionsCode = bUseExceptions; + OGRDataSourceShadow *arg1 = (OGRDataSourceShadow *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject * obj0 = 0 ; + OGRErr result; + + if (!PyArg_ParseTuple(args,(char *)"O:DataSource_AbortSQL",&obj0)) SWIG_fail; + res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_OGRDataSourceShadow, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "DataSource_AbortSQL" "', argument " "1"" of type '" "OGRDataSourceShadow *""'"); + } + arg1 = reinterpret_cast< OGRDataSourceShadow * >(argp1); + { + if ( bUseExceptions ) { + ClearErrorState(); + } + { + SWIG_PYTHON_THREAD_BEGIN_ALLOW; + result = (OGRErr)OGRDataSourceShadow_AbortSQL(arg1); + SWIG_PYTHON_THREAD_END_ALLOW; + } +#ifndef SED_HACKS + if ( bUseExceptions ) { + CPLErr eclass = CPLGetLastErrorType(); + if ( eclass == CE_Failure || eclass == CE_Fatal ) { + SWIG_exception( SWIG_RuntimeError, CPLGetLastErrorMsg() ); + } + } +#endif + } + { + /* %typemap(out) OGRErr */ + if ( result != 0 && bUseExceptions) { + const char* pszMessage = CPLGetLastErrorMsg(); + if( pszMessage[0] != '\0' ) + PyErr_SetString( PyExc_RuntimeError, pszMessage ); + else + PyErr_SetString( PyExc_RuntimeError, OGRErrMessages(result) ); + SWIG_fail; + } + } + { + /* %typemap(ret) OGRErr */ + if ( ReturnSame(resultobj == Py_None || resultobj == 0) ) { + resultobj = PyInt_FromLong( result ); + } + } + if ( ReturnSame(bLocalUseExceptionsCode) ) { CPLErr eclass = CPLGetLastErrorType(); if ( eclass == CE_Failure || eclass == CE_Fatal ) { Py_XDECREF(resultobj); SWIG_Error( SWIG_RuntimeError, CPLGetLastErrorMsg() ); return NULL; } } + return resultobj; +fail: + return NULL; +} + + SWIGINTERN PyObject *_wrap_DataSource_ReleaseResultSet(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { PyObject *resultobj = 0; int bLocalUseExceptionsCode = bUseExceptions; OGRDataSourceShadow *arg1 = (OGRDataSourceShadow *) 0 ; @@ -29149,6 +29208,7 @@ static PyMethodDef SwigMethods[] = { "an handle to a OGRLayer containing the results of the query.\n" "Deallocate with OGR_DS_ReleaseResultSet(). \n" ""}, + { (char *)"DataSource_AbortSQL", _wrap_DataSource_AbortSQL, METH_VARARGS, (char *)"DataSource_AbortSQL(DataSource self) -> OGRErr"}, { (char *)"DataSource_ReleaseResultSet", _wrap_DataSource_ReleaseResultSet, METH_VARARGS, (char *)"\n" "DataSource_ReleaseResultSet(DataSource self, Layer layer)\n" "\n" diff --git a/gdal/swig/python/osgeo/gdal.py b/gdal/swig/python/osgeo/gdal.py index 3d9482005d91..b9d1513801b5 100644 --- a/gdal/swig/python/osgeo/gdal.py +++ b/gdal/swig/python/osgeo/gdal.py @@ -2426,6 +2426,11 @@ def SetStyleTable(self, *args): return _gdal.Dataset_SetStyleTable(self, *args) + def AbortSQL(self, *args): + """AbortSQL(Dataset self) -> OGRErr""" + return _gdal.Dataset_AbortSQL(self, *args) + + def StartTransaction(self, *args, **kwargs): """StartTransaction(Dataset self, int force=False) -> OGRErr""" return _gdal.Dataset_StartTransaction(self, *args, **kwargs) diff --git a/gdal/swig/python/osgeo/ogr.py b/gdal/swig/python/osgeo/ogr.py index b5e88f49b7e5..2d453e72a96a 100644 --- a/gdal/swig/python/osgeo/ogr.py +++ b/gdal/swig/python/osgeo/ogr.py @@ -802,6 +802,11 @@ def ExecuteSQL(self, *args, **kwargs): return _ogr.DataSource_ExecuteSQL(self, *args, **kwargs) + def AbortSQL(self, *args): + """AbortSQL(DataSource self) -> OGRErr""" + return _ogr.DataSource_AbortSQL(self, *args) + + def ReleaseResultSet(self, *args): """ ReleaseResultSet(DataSource self, Layer layer) From 3ff673b117ce840c84f376fe165c0ad909c568d8 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 21 Sep 2020 11:21:10 +0200 Subject: [PATCH 054/255] gdalbuildvrt_lib.cpp: C++ification. no functional change --- gdal/apps/gdalbuildvrt_lib.cpp | 385 +++++++++++++++------------------ 1 file changed, 178 insertions(+), 207 deletions(-) diff --git a/gdal/apps/gdalbuildvrt_lib.cpp b/gdal/apps/gdalbuildvrt_lib.cpp index 8056abe08688..963e709e77a3 100644 --- a/gdal/apps/gdalbuildvrt_lib.cpp +++ b/gdal/apps/gdalbuildvrt_lib.cpp @@ -40,6 +40,8 @@ #include #include +#include +#include #include "cpl_conv.h" #include "cpl_error.h" @@ -73,38 +75,48 @@ typedef enum USER_RESOLUTION } ResolutionStrategy; -typedef struct +struct DatasetProperty { - int isFileOK; - int nRasterXSize; - int nRasterYSize; + int isFileOK = FALSE; + int nRasterXSize = 0; + int nRasterYSize = 0; double adfGeoTransform[6]; - int nBlockXSize; - int nBlockYSize; - GDALDataType firstBandType; - bool* pabHasNoData; - double* padfNoDataValues; - bool* pabHasOffset; - double* padfOffset; - bool* pabHasScale; - double* padfScale; - int bHasDatasetMask; - int nMaskBlockXSize; - int nMaskBlockYSize; -} DatasetProperty; - -typedef struct + int nBlockXSize = 0; + int nBlockYSize = 0; + GDALDataType firstBandType = GDT_Unknown; + std::vector abHasNoData{}; + std::vector adfNoDataValues{}; + std::vector abHasOffset{}; + std::vector adfOffset{}; + std::vector abHasScale{}; + std::vector adfScale{}; + int bHasDatasetMask = 0; + int nMaskBlockXSize = 0; + int nMaskBlockYSize = 0; + + DatasetProperty() + { + adfGeoTransform[0] = 0; + adfGeoTransform[1] = 0; + adfGeoTransform[2] = 0; + adfGeoTransform[3] = 0; + adfGeoTransform[4] = 0; + adfGeoTransform[5] = 0; + } +}; + +struct BandProperty { - GDALColorInterp colorInterpretation; - GDALDataType dataType; - GDALColorTableH colorTable; - bool bHasNoData; - double noDataValue; - bool bHasOffset; - double dfOffset; - bool bHasScale; - double dfScale; -} BandProperty; + GDALColorInterp colorInterpretation = GCI_Undefined; + GDALDataType dataType = GDT_Unknown; + std::unique_ptr colorTable{}; + bool bHasNoData = false; + double noDataValue = 0; + bool bHasOffset = false; + double dfOffset = 0; + bool bHasScale = false; + double dfScale = 0; +}; /************************************************************************/ /* ArgIsNumeric() */ @@ -229,12 +241,12 @@ class VRTBuilder /* Internal variables */ char *pszProjectionRef = nullptr; - BandProperty *pasBandProperties = nullptr; + std::vector asBandProperties{}; int bFirst = TRUE; int bHasGeoTransform = 0; int nRasterXSize = 0; int nRasterYSize = 0; - DatasetProperty *pasDatasetProperties = nullptr; + std::vector asDatasetProperties{}; int bUserExtent = 0; int bAllowSrcNoData = TRUE; double *padfSrcNoData = nullptr; @@ -368,29 +380,6 @@ VRTBuilder::~VRTBuilder() CPLFree(ppszInputFilenames); CPLFree(pahSrcDS); - if (pasDatasetProperties != nullptr) - { - for(int i=0;i 0 && GDALGetRasterCount(hDS) == 0 ) + GDALDataset* poDS = GDALDataset::FromHandle(hDS); + const char* dsFileName = poDS->GetDescription(); + char** papszMetadata = poDS->GetMetadata( "SUBDATASETS" ); + if( CSLCount(papszMetadata) > 0 && poDS->GetRasterCount() == 0 ) { - pasDatasetProperties = static_cast( - CPLRealloc(pasDatasetProperties, - (nInputFiles + CSLCount(papszMetadata)) * - sizeof(DatasetProperty))); - ppszInputFilenames = static_cast( CPLRealloc(ppszInputFilenames, sizeof(char*) * (nInputFiles + CSLCount(papszMetadata)))); @@ -464,9 +449,10 @@ int VRTBuilder::AnalyseRaster( GDALDatasetH hDS, DatasetProperty* psDatasetPrope { if (EQUALN(*papszMetadata, subdatasetNameKey, strlen(subdatasetNameKey))) { - memset(&pasDatasetProperties[nInputFiles], 0, sizeof(DatasetProperty)); - ppszInputFilenames[nInputFiles++] = + asDatasetProperties.resize(nInputFiles+1); + ppszInputFilenames[nInputFiles] = CPLStrdup(*papszMetadata+strlen(subdatasetNameKey)+1); + nInputFiles++; count++; snprintf(subdatasetNameKey, sizeof(subdatasetNameKey), "SUBDATASET_%d_NAME", count); } @@ -483,16 +469,17 @@ int VRTBuilder::AnalyseRaster( GDALDatasetH hDS, DatasetProperty* psDatasetPrope pszSubdatasetName = CSLFetchNameValue( papszMetadata, subdatasetNameKey ); if ( pszSubdatasetName ) { - memset( &pasDatasetProperties[nInputFiles], 0, sizeof(DatasetProperty) ); - ppszInputFilenames[nInputFiles++] = CPLStrdup( pszSubdatasetName ); + asDatasetProperties.resize(nInputFiles+1); + ppszInputFilenames[nInputFiles] = CPLStrdup( pszSubdatasetName ); + nInputFiles++; } } return FALSE; } - const char* proj = GDALGetProjectionRef(hDS); + const char* proj = poDS->GetProjectionRef(); double* padfGeoTransform = psDatasetProperties->adfGeoTransform; - int bGotGeoTransform = GDALGetGeoTransform(hDS, padfGeoTransform) == CE_None; + int bGotGeoTransform = poDS->GetGeoTransform(padfGeoTransform) == CE_None; if (bSeparate) { if (bFirst) @@ -520,8 +507,8 @@ int VRTBuilder::AnalyseRaster( GDALDatasetH hDS, DatasetProperty* psDatasetPrope return FALSE; } else if (!bHasGeoTransform && - (nRasterXSize != GDALGetRasterXSize(hDS) || - nRasterYSize != GDALGetRasterYSize(hDS))) + (nRasterXSize != poDS->GetRasterXSize() || + nRasterYSize != poDS->GetRasterYSize())) { CPLError(CE_Warning, CPLE_NotSupported, "gdalbuildvrt -separate cannot stack ungeoreferenced images that have not the same dimensions. Skipping %s", @@ -560,12 +547,12 @@ int VRTBuilder::AnalyseRaster( GDALDatasetH hDS, DatasetProperty* psDatasetPrope } } - psDatasetProperties->nRasterXSize = GDALGetRasterXSize(hDS); - psDatasetProperties->nRasterYSize = GDALGetRasterYSize(hDS); + psDatasetProperties->nRasterXSize = poDS->GetRasterXSize(); + psDatasetProperties->nRasterYSize = poDS->GetRasterYSize(); if (bFirst && bSeparate && !bGotGeoTransform) { - nRasterXSize = GDALGetRasterXSize(hDS); - nRasterYSize = GDALGetRasterYSize(hDS); + nRasterXSize = poDS->GetRasterXSize(); + nRasterYSize = poDS->GetRasterYSize(); } double ds_minX = padfGeoTransform[GEOTRSFRM_TOPLEFT_X]; @@ -577,10 +564,6 @@ int VRTBuilder::AnalyseRaster( GDALDatasetH hDS, DatasetProperty* psDatasetPrope GDALGetRasterYSize(hDS) * padfGeoTransform[GEOTRSFRM_NS_RES]; - GDALGetBlockSize(GDALGetRasterBand( hDS, 1 ), - &psDatasetProperties->nBlockXSize, - &psDatasetProperties->nBlockYSize); - int _nBands = GDALGetRasterCount(hDS); //if provided band list @@ -604,62 +587,58 @@ int VRTBuilder::AnalyseRaster( GDALDatasetH hDS, DatasetProperty* psDatasetPrope _nBands = 1; } + GDALRasterBand* poFirstBand = poDS->GetRasterBand(1); + poFirstBand->GetBlockSize( + &psDatasetProperties->nBlockXSize, + &psDatasetProperties->nBlockYSize); + /* For the -separate case */ - psDatasetProperties->firstBandType = GDALGetRasterDataType(GDALGetRasterBand(hDS, 1)); + psDatasetProperties->firstBandType = poFirstBand->GetRasterDataType(); - psDatasetProperties->padfNoDataValues = - static_cast(CPLCalloc(sizeof(double), _nBands)); - psDatasetProperties->pabHasNoData = - static_cast(CPLCalloc(sizeof(bool), _nBands)); + psDatasetProperties->adfNoDataValues.resize(_nBands); + psDatasetProperties->abHasNoData.resize(_nBands); - psDatasetProperties->padfOffset = - static_cast(CPLCalloc(sizeof(double), _nBands)); - psDatasetProperties->pabHasOffset = - static_cast(CPLCalloc(sizeof(bool), _nBands)); + psDatasetProperties->adfOffset.resize(_nBands); + psDatasetProperties->abHasOffset.resize(_nBands); - psDatasetProperties->padfScale= - static_cast(CPLCalloc(sizeof(double), _nBands)); - psDatasetProperties->pabHasScale = - static_cast(CPLCalloc(sizeof(bool), _nBands)); + psDatasetProperties->adfScale.resize(_nBands); + psDatasetProperties->abHasScale.resize(_nBands); - psDatasetProperties->bHasDatasetMask = GDALGetMaskFlags(GDALGetRasterBand(hDS, 1)) == GMF_PER_DATASET; + psDatasetProperties->bHasDatasetMask = poFirstBand->GetMaskFlags() == GMF_PER_DATASET; if (psDatasetProperties->bHasDatasetMask) bHasDatasetMask = TRUE; - GDALGetBlockSize(GDALGetMaskBand(GDALGetRasterBand( hDS, 1 )), + poFirstBand->GetMaskBand()->GetBlockSize( &psDatasetProperties->nMaskBlockXSize, &psDatasetProperties->nMaskBlockYSize); - int j; - for(j=0;j<_nBands;j++) + for(int j=0;j<_nBands;j++) { + GDALRasterBand* poBand = poDS->GetRasterBand(j+1); if (nSrcNoDataCount > 0) { - psDatasetProperties->pabHasNoData[j] = true; + psDatasetProperties->abHasNoData[j] = true; if (j < nSrcNoDataCount) - psDatasetProperties->padfNoDataValues[j] = padfSrcNoData[j]; + psDatasetProperties->adfNoDataValues[j] = padfSrcNoData[j]; else - psDatasetProperties->padfNoDataValues[j] = padfSrcNoData[nSrcNoDataCount - 1]; + psDatasetProperties->adfNoDataValues[j] = padfSrcNoData[nSrcNoDataCount - 1]; } else { int bHasNoData = false; - psDatasetProperties->padfNoDataValues[j] = - GDALGetRasterNoDataValue(GDALGetRasterBand(hDS, j+1), - &bHasNoData); - psDatasetProperties->pabHasNoData[j] = bHasNoData != 0; + psDatasetProperties->adfNoDataValues[j] = + poBand->GetNoDataValue(&bHasNoData); + psDatasetProperties->abHasNoData[j] = bHasNoData != 0; } int bHasOffset = false; - psDatasetProperties->padfOffset[j] = - GDALGetRasterOffset(GDALGetRasterBand(hDS, j+1), &bHasOffset); - psDatasetProperties->pabHasOffset[j] = bHasOffset != 0 && - psDatasetProperties->padfOffset[j] != 0.0; + psDatasetProperties->adfOffset[j] = poBand->GetOffset(&bHasOffset); + psDatasetProperties->abHasOffset[j] = bHasOffset != 0 && + psDatasetProperties->adfOffset[j] != 0.0; int bHasScale = false; - psDatasetProperties->padfScale[j] = - GDALGetRasterScale(GDALGetRasterBand(hDS, j+1), &bHasScale); - psDatasetProperties->pabHasScale[j] = bHasScale != 0 && - psDatasetProperties->padfScale[j] != 1.0; + psDatasetProperties->adfScale[j] = poBand->GetScale(&bHasScale); + psDatasetProperties->abHasScale[j] = bHasScale != 0 && + psDatasetProperties->adfScale[j] != 1.0; } if (bFirst) @@ -680,7 +659,7 @@ int VRTBuilder::AnalyseRaster( GDALDatasetH hDS, DatasetProperty* psDatasetPrope nBands = _nBands; CPLFree(panBandList); panBandList = static_cast(CPLMalloc(nBands * sizeof(int))); - for(j=0;j( - CPLMalloc(nMaxBandNo * sizeof(BandProperty))); - for(j=0;jGetRasterBand(j+1 ); + asBandProperties[j].colorInterpretation = + poBand->GetColorInterpretation(); + asBandProperties[j].dataType = poBand->GetRasterDataType(); + if (asBandProperties[j].colorInterpretation == GCI_PaletteIndex) { - pasBandProperties[j].colorTable = - GDALGetRasterColorTable( hRasterBand ); - if (pasBandProperties[j].colorTable) + auto colorTable = poBand->GetColorTable(); + if (colorTable) { - pasBandProperties[j].colorTable = - GDALCloneColorTable(pasBandProperties[j].colorTable); + asBandProperties[j].colorTable.reset(colorTable->Clone()); } } else - pasBandProperties[j].colorTable = nullptr; + asBandProperties[j].colorTable = nullptr; if (nVRTNoDataCount > 0) { - pasBandProperties[j].bHasNoData = true; + asBandProperties[j].bHasNoData = true; if (j < nVRTNoDataCount) - pasBandProperties[j].noDataValue = padfVRTNoData[j]; + asBandProperties[j].noDataValue = padfVRTNoData[j]; else - pasBandProperties[j].noDataValue = padfVRTNoData[nVRTNoDataCount - 1]; + asBandProperties[j].noDataValue = padfVRTNoData[nVRTNoDataCount - 1]; } else { int bHasNoData = false; - pasBandProperties[j].noDataValue = - GDALGetRasterNoDataValue(hRasterBand, &bHasNoData); - pasBandProperties[j].bHasNoData = bHasNoData != 0; + asBandProperties[j].noDataValue = + poBand->GetNoDataValue(&bHasNoData); + asBandProperties[j].bHasNoData = bHasNoData != 0; } int bHasOffset = false; - pasBandProperties[j].dfOffset = - GDALGetRasterOffset(hRasterBand, &bHasOffset); - pasBandProperties[j].bHasOffset = bHasOffset != 0 && - pasBandProperties[j].dfOffset != 0.0; + asBandProperties[j].dfOffset = poBand->GetOffset(&bHasOffset); + asBandProperties[j].bHasOffset = bHasOffset != 0 && + asBandProperties[j].dfOffset != 0.0; int bHasScale = false; - pasBandProperties[j].dfScale = - GDALGetRasterScale(hRasterBand, &bHasScale); - pasBandProperties[j].bHasScale = bHasScale != 0 && - pasBandProperties[j].dfScale != 1.0; + asBandProperties[j].dfScale = poBand->GetScale(&bHasScale); + asBandProperties[j].bHasScale = bHasScale != 0 && + asBandProperties[j].dfScale != 1.0; } } } @@ -770,43 +744,42 @@ int VRTBuilder::AnalyseRaster( GDALDatasetH hDS, DatasetProperty* psDatasetPrope _nBands, nMaxBandNo, dsFileName); return FALSE; } - for(j=0;jGetRasterBand(j+1 ); + if (asBandProperties[j].colorInterpretation != + poBand->GetColorInterpretation()) { CPLError(CE_Warning, CPLE_NotSupported, "gdalbuildvrt does not support heterogeneous " "band color interpretation: expected %s, got %s. " "Skipping %s", GDALGetColorInterpretationName( - pasBandProperties[j].colorInterpretation), + asBandProperties[j].colorInterpretation), GDALGetColorInterpretationName( - GDALGetRasterColorInterpretation(hRasterBand)), + poBand->GetColorInterpretation()), dsFileName); return FALSE; } - if (pasBandProperties[j].dataType != - GDALGetRasterDataType(hRasterBand)) + if (asBandProperties[j].dataType != poBand->GetRasterDataType()) { CPLError(CE_Warning, CPLE_NotSupported, "gdalbuildvrt does not support heterogeneous " "band data type: expected %s, got %s. " "Skipping %s", GDALGetDataTypeName( - pasBandProperties[j].dataType), + asBandProperties[j].dataType), GDALGetDataTypeName( - GDALGetRasterDataType(hRasterBand)), + poBand->GetRasterDataType()), dsFileName); return FALSE; } - if (pasBandProperties[j].colorTable) + if (asBandProperties[j].colorTable) { - GDALColorTableH colorTable = GDALGetRasterColorTable( hRasterBand ); - int nRefColorEntryCount = GDALGetColorEntryCount(pasBandProperties[j].colorTable); + const GDALColorTable* colorTable = poBand->GetColorTable(); + int nRefColorEntryCount = asBandProperties[j].colorTable->GetColorEntryCount(); if (colorTable == nullptr || - GDALGetColorEntryCount(colorTable) != nRefColorEntryCount) + colorTable->GetColorEntryCount() != nRefColorEntryCount) { CPLError(CE_Warning, CPLE_NotSupported, "gdalbuildvrt does not support rasters with different color tables (different number of color table entries). Skipping %s", @@ -819,8 +792,8 @@ int VRTBuilder::AnalyseRaster( GDALDatasetH hDS, DatasetProperty* psDatasetPrope /* should check that the end result is OK for him. */ for(int i=0;iGetColorEntry(i); + const GDALColorEntry* psEntryRef = asBandProperties[j].colorTable->GetColorEntry(i); if (psEntry->c1 != psEntryRef->c1 || psEntry->c2 != psEntryRef->c2 || psEntry->c3 != psEntryRef->c3 || psEntry->c4 != psEntryRef->c4) { @@ -841,34 +814,34 @@ int VRTBuilder::AnalyseRaster( GDALDatasetH hDS, DatasetProperty* psDatasetPrope } } - if( psDatasetProperties->pabHasOffset[j] != pasBandProperties[j].bHasOffset || - (pasBandProperties[j].bHasOffset && - psDatasetProperties->padfOffset[j] != pasBandProperties[j].dfOffset) ) + if( psDatasetProperties->abHasOffset[j] != asBandProperties[j].bHasOffset || + (asBandProperties[j].bHasOffset && + psDatasetProperties->adfOffset[j] != asBandProperties[j].dfOffset) ) { CPLError(CE_Warning, CPLE_NotSupported, "gdalbuildvrt does not support heterogeneous " "band offset: expected (%d,%f), got (%d,%f). " "Skipping %s", - static_cast(pasBandProperties[j].bHasOffset), - pasBandProperties[j].dfOffset, - static_cast(psDatasetProperties->pabHasOffset[j]), - psDatasetProperties->padfOffset[j], + static_cast(asBandProperties[j].bHasOffset), + asBandProperties[j].dfOffset, + static_cast(psDatasetProperties->abHasOffset[j]), + psDatasetProperties->adfOffset[j], dsFileName); return FALSE; } - if( psDatasetProperties->pabHasScale[j] != pasBandProperties[j].bHasScale || - (pasBandProperties[j].bHasScale && - psDatasetProperties->padfScale[j] != pasBandProperties[j].dfScale) ) + if( psDatasetProperties->abHasScale[j] != asBandProperties[j].bHasScale || + (asBandProperties[j].bHasScale && + psDatasetProperties->adfScale[j] != asBandProperties[j].dfScale) ) { CPLError(CE_Warning, CPLE_NotSupported, "gdalbuildvrt does not support heterogeneous " "band scale: expected (%d,%f), got (%d,%f). " "Skipping %s", - static_cast(pasBandProperties[j].bHasScale), - pasBandProperties[j].dfScale, - static_cast(psDatasetProperties->pabHasScale[j]), - psDatasetProperties->padfScale[j], + static_cast(asBandProperties[j].bHasScale), + asBandProperties[j].dfScale, + static_cast(psDatasetProperties->abHasScale[j]), + psDatasetProperties->adfScale[j], dsFileName); return FALSE; } @@ -922,7 +895,7 @@ void VRTBuilder::CreateVRTSeparate(VRTDatasetH hVRTDS) int iBand = 1; for(int i=0; ppszInputFilenames != nullptr && iisFileOK == FALSE) continue; @@ -972,11 +945,11 @@ void VRTBuilder::CreateVRTSeparate(VRTDatasetH hVRTDS) VRTSourcedRasterBand* poVRTBand = static_cast(hVRTBand); VRTSimpleSource* poSimpleSource; - if (bAllowSrcNoData && psDatasetProperties->pabHasNoData[0]) + if (bAllowSrcNoData && psDatasetProperties->abHasNoData[0]) { - GDALSetRasterNoDataValue(hVRTBand, psDatasetProperties->padfNoDataValues[0]); + GDALSetRasterNoDataValue(hVRTBand, psDatasetProperties->adfNoDataValues[0]); poSimpleSource = new VRTComplexSource(); - poSimpleSource->SetNoDataValue( psDatasetProperties->padfNoDataValues[0] ); + poSimpleSource->SetNoDataValue( psDatasetProperties->adfNoDataValues[0] ); } else poSimpleSource = new VRTSimpleSource(); @@ -990,11 +963,11 @@ void VRTBuilder::CreateVRTSeparate(VRTDatasetH hVRTDS) dfDstXOff, dfDstYOff, dfDstXSize, dfDstYSize ); - if( psDatasetProperties->pabHasOffset[0] ) - poVRTBand->SetOffset( psDatasetProperties->padfOffset[0] ); + if( psDatasetProperties->abHasOffset[0] ) + poVRTBand->SetOffset( psDatasetProperties->adfOffset[0] ); - if( psDatasetProperties->pabHasScale[0] ) - poVRTBand->SetScale( psDatasetProperties->padfScale[0] ); + if( psDatasetProperties->abHasScale[0] ) + poVRTBand->SetScale( psDatasetProperties->adfScale[0] ); poVRTBand->AddSource( poSimpleSource ); @@ -1010,46 +983,45 @@ void VRTBuilder::CreateVRTSeparate(VRTDatasetH hVRTDS) void VRTBuilder::CreateVRTNonSeparate(VRTDatasetH hVRTDS) { + VRTDataset* poVRTDS = reinterpret_cast(hVRTDS); for(int j=0;jAddBand(asBandProperties[nSelBand].dataType); + GDALRasterBand *poBand = poVRTDS->GetRasterBand(j+1); + poBand->SetColorInterpretation(asBandProperties[nSelBand].colorInterpretation); + if (asBandProperties[nSelBand].colorInterpretation == GCI_PaletteIndex) { - GDALSetRasterColorTable(hBand, pasBandProperties[nSelBand].colorTable); + poBand->SetColorTable(asBandProperties[nSelBand].colorTable.get()); } - if (bAllowVRTNoData && pasBandProperties[nSelBand].bHasNoData) - GDALSetRasterNoDataValue(hBand, pasBandProperties[nSelBand].noDataValue); + if (bAllowVRTNoData && asBandProperties[nSelBand].bHasNoData) + poBand->SetNoDataValue(asBandProperties[nSelBand].noDataValue); if ( bHideNoData ) - GDALSetMetadataItem(hBand,"HideNoDataValue","1",nullptr); + poBand->SetMetadataItem("HideNoDataValue","1"); - if( pasBandProperties[nSelBand].bHasOffset ) - GDALSetRasterOffset( hBand, pasBandProperties[nSelBand].dfOffset ); + if( asBandProperties[nSelBand].bHasOffset ) + poBand->SetOffset( asBandProperties[nSelBand].dfOffset ); - if( pasBandProperties[nSelBand].bHasScale ) - GDALSetRasterScale( hBand, pasBandProperties[nSelBand].dfScale ); + if( asBandProperties[nSelBand].bHasScale ) + poBand->SetScale( asBandProperties[nSelBand].dfScale ); } VRTSourcedRasterBand* poMaskVRTBand = nullptr; if (bAddAlpha) { - GDALRasterBandH hBand; - GDALAddBand(hVRTDS, GDT_Byte, nullptr); - hBand = GDALGetRasterBand(hVRTDS, nBands + 1); - GDALSetRasterColorInterpretation(hBand, GCI_AlphaBand); + poVRTDS->AddBand(GDT_Byte); + GDALRasterBand *poBand = poVRTDS->GetRasterBand(nBands + 1); + poBand->SetColorInterpretation(GCI_AlphaBand); } else if (bHasDatasetMask) { - GDALCreateDatasetMaskBand(hVRTDS, GMF_PER_DATASET); - poMaskVRTBand = static_cast(GDALGetMaskBand(GDALGetRasterBand(hVRTDS, 1))); + poVRTDS->CreateMaskBand(GMF_PER_DATASET); + poMaskVRTBand = static_cast(poVRTDS->GetRasterBand(1)->GetMaskBand()); } for( int i = 0; ppszInputFilenames != nullptr && i < nInputFiles; i++ ) { - DatasetProperty* psDatasetProperties = &pasDatasetProperties[i]; + DatasetProperty* psDatasetProperties = &asDatasetProperties[i]; if (psDatasetProperties->isFileOK == FALSE) continue; @@ -1096,7 +1068,7 @@ void VRTBuilder::CreateVRTNonSeparate(VRTDatasetH hVRTDS) for(int j=0;jnBlockXSize, psDatasetProperties->nBlockYSize); } @@ -1114,17 +1086,17 @@ void VRTBuilder::CreateVRTNonSeparate(VRTDatasetH hVRTDS) for(int j=0;j(GDALGetRasterBand(hVRTDS, j + 1)); + static_cast(poVRTDS->GetRasterBand(j + 1)); /* Place the raster band at the right position in the VRT */ int nSelBand = panBandList[j] - 1; VRTSourcedRasterBand* poVRTBand = static_cast(hVRTBand); VRTSimpleSource* poSimpleSource; - if (bAllowSrcNoData && psDatasetProperties->pabHasNoData[nSelBand]) + if (bAllowSrcNoData && psDatasetProperties->abHasNoData[nSelBand]) { poSimpleSource = new VRTComplexSource(); - poSimpleSource->SetNoDataValue( psDatasetProperties->padfNoDataValues[nSelBand] ); + poSimpleSource->SetNoDataValue( psDatasetProperties->adfNoDataValues[nSelBand] ); } else poSimpleSource = new VRTSimpleSource(); @@ -1217,8 +1189,7 @@ GDALDataset* VRTBuilder::Build(GDALProgressFunc pfnProgress, void * pProgressDat we_res = ns_res = 0; } - pasDatasetProperties = static_cast( - CPLCalloc(nInputFiles, sizeof(DatasetProperty))); + asDatasetProperties.resize(nInputFiles); if (pszSrcNoData != nullptr) { @@ -1293,13 +1264,13 @@ GDALDataset* VRTBuilder::Build(GDALProgressFunc pfnProgress, void * pProgressDat GDALOpenEx( ppszInputFilenames[i], GDAL_OF_RASTER | GDAL_OF_VERBOSE_ERROR, nullptr, papszOpenOptions, nullptr ); - pasDatasetProperties[i].isFileOK = FALSE; + asDatasetProperties[i].isFileOK = FALSE; if (hDS) { - if (AnalyseRaster( hDS, &pasDatasetProperties[i] )) + if (AnalyseRaster( hDS, &asDatasetProperties[i] )) { - pasDatasetProperties[i].isFileOK = TRUE; + asDatasetProperties[i].isFileOK = TRUE; nCountValid ++; bFirst = FALSE; } From 0141707f36f13d35ae404b99609069464eb51928 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 21 Sep 2020 14:37:18 +0200 Subject: [PATCH 055/255] VRT: add support for explicit virtual overviews Explicit virtual overviews are overviews that are reported on VRT rasterbands, and rely only on the fact that the source(s) making those bands have themselves overviews, but this is not enforced. Such virtual overviews are generally not needed, unless in some situations, like wrapping a VRT dataset to a lower resolution. The VRTDataset.OverviewList XML element is added to declare such overviews. GDALDataset::BuildOverviews() can also 'build' them when the VRT_VIRTUAL_OVERVIEWS configuration option is set to YES. --- autotest/gdrivers/vrtovr.py | 82 +++++++++++- gdal/data/gdalvrt.xsd | 13 ++ gdal/doc/source/drivers/raster/vrt.rst | 23 +++- gdal/frmts/vrt/vrtdataset.cpp | 166 ++++++++++++++++++++---- gdal/frmts/vrt/vrtdataset.h | 33 +++-- gdal/frmts/vrt/vrtsourcedrasterband.cpp | 24 +++- gdal/frmts/vrt/vrtsources.cpp | 9 +- 7 files changed, 307 insertions(+), 43 deletions(-) diff --git a/autotest/gdrivers/vrtovr.py b/autotest/gdrivers/vrtovr.py index 7b9dd7eff33b..ae65c40799a1 100755 --- a/autotest/gdrivers/vrtovr.py +++ b/autotest/gdrivers/vrtovr.py @@ -3,11 +3,11 @@ # $Id$ # # Project: GDAL/OGR Test Suite -# Purpose: Test overviews in bands in VRT driver +# Purpose: Test overviews support in VRT driver # Author: Even Rouault # ############################################################################### -# Copyright (c) 2010, Even Rouault +# Copyright (c) 2010, 2020, Even Rouault # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -29,7 +29,7 @@ ############################################################################### from osgeo import gdal - +import struct import gdaltest @@ -169,7 +169,81 @@ def test_vrtovr_errors(): with gdaltest.error_handler(): assert not ds.GetRasterBand(1).GetOverview(0) - + +############################################################################### +# Test support for virtual overviews + + +def test_vrtovr_virtual(): + + tif_tmpfilename = '/vsimem/temp.tif' + src_ds = gdal.GetDriverByName('GTiff').Create(tif_tmpfilename, 20, 20, 3) + src_ds.BuildOverviews('NEAR', [2, 4]) + src_ds.GetRasterBand(1).Fill(200) + src_ds.GetRasterBand(2).Fill(100) + src_ds.GetRasterBand(1).GetOverview(0).Fill(100) + src_ds.GetRasterBand(1).GetOverview(1).Fill(50) + src_ds = None + src_ds = gdal.Open(tif_tmpfilename) + + tmpfilename = '/vsimem/temp.vrt' + vrt_ds = gdal.Translate(tmpfilename, src_ds, format='VRT') + assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 0 # we normally don't create implicit overviews on that small datasets + + with gdaltest.config_option('VRT_VIRTUAL_OVERVIEWS', 'YES'): + vrt_ds.BuildOverviews('NEAR', [2, 4, 5, 50]) # level 50 is too big + assert gdal.VSIStatL(tmpfilename + '.ovr') is None + assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 3 + + # Clean overviews + with gdaltest.config_option('VRT_VIRTUAL_OVERVIEWS', 'YES'): + vrt_ds.BuildOverviews('NONE', []) + assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 0 + + # Add in two steps + with gdaltest.config_option('VRT_VIRTUAL_OVERVIEWS', 'YES'): + vrt_ds.BuildOverviews('NEAR', [2, 4]) + vrt_ds.BuildOverviews('NEAR', [5]) + assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 3 + + assert vrt_ds.GetRasterBand(1).GetOverview(0).Checksum() == src_ds.GetRasterBand(1).GetOverview(0).Checksum() + assert vrt_ds.GetRasterBand(1).GetOverview(1).Checksum() == src_ds.GetRasterBand(1).GetOverview(1).Checksum() + assert vrt_ds.ReadRaster(0,0,20,20,10,10) == src_ds.ReadRaster(0,0,20,20,10,10) + assert vrt_ds.GetRasterBand(1).ReadRaster(0,0,20,20,10,10) == src_ds.GetRasterBand(1).ReadRaster(0,0,20,20,10,10) + assert vrt_ds.GetRasterBand(1).ReadRaster(0,0,20,20,5,5) == src_ds.GetRasterBand(1).ReadRaster(0,0,20,20,5,5) + assert struct.unpack('B' * 4 * 4,vrt_ds.GetRasterBand(1).ReadRaster(0,0,20,20,4,4))[0] == 50 + vrt_ds = None + + # Re-open VRT and re-run checks + vrt_ds = gdal.Open(tmpfilename) + assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 3 + assert vrt_ds.GetRasterBand(1).GetOverview(0).Checksum() == src_ds.GetRasterBand(1).GetOverview(0).Checksum() + assert vrt_ds.GetRasterBand(1).GetOverview(1).Checksum() == src_ds.GetRasterBand(1).GetOverview(1).Checksum() + assert vrt_ds.ReadRaster(0,0,20,20,10,10) == src_ds.ReadRaster(0,0,20,20,10,10) + assert vrt_ds.GetRasterBand(1).ReadRaster(0,0,20,20,10,10) == src_ds.GetRasterBand(1).ReadRaster(0,0,20,20,10,10) + assert vrt_ds.GetRasterBand(1).ReadRaster(0,0,20,20,5,5) == src_ds.GetRasterBand(1).ReadRaster(0,0,20,20,5,5) + assert struct.unpack('B' * 4 * 4,vrt_ds.GetRasterBand(1).ReadRaster(0,0,20,20,4,4))[0] == 50 + + gdal.Unlink(tmpfilename) + gdal.Unlink(tif_tmpfilename) + + +############################################################################### +# Test support for virtual overviews, when there are pre-existing implicit +# overviews + + +def test_vrtovr_virtual_with_preexisting_implicit_ovr(): + + src_ds = gdal.GetDriverByName('MEM').Create('', 1000, 1000) + src_ds.BuildOverviews('NEAR', [2]) + vrt_ds = gdal.Translate('', src_ds, format='VRT') + assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 1 + + with gdaltest.config_option('VRT_VIRTUAL_OVERVIEWS', 'YES'): + vrt_ds.BuildOverviews('NEAR', [2, 4]) + assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 2 + ############################################################################### # Cleanup. diff --git a/gdal/data/gdalvrt.xsd b/gdal/data/gdalvrt.xsd index 10ae6d1ceb75..a5cf661aff16 100644 --- a/gdal/data/gdalvrt.xsd +++ b/gdal/data/gdalvrt.xsd @@ -45,6 +45,7 @@ + @@ -53,6 +54,18 @@ + + + + + + + + + + + + diff --git a/gdal/doc/source/drivers/raster/vrt.rst b/gdal/doc/source/drivers/raster/vrt.rst index 43a741826e35..0227802859be 100644 --- a/gdal/doc/source/drivers/raster/vrt.rst +++ b/gdal/doc/source/drivers/raster/vrt.rst @@ -116,6 +116,22 @@ The **dataAxisToSRSAxisMapping** attribute is allowed since GDAL 3.0 to describe +- **OverviewList**: (GDAL >= 3.2.0, not valid for VRTPansharpenedDataset) + This elements contains a list of overview factors, separated by space, to + create "virtual overviews". For example ``2 4``. It can be used so that bands + of the VRT datasets declare overviews. This only makes sense to use if the + sources added in those bands have themselves overviews compatible of the + declared factor. It is generally not needed to use this mechanism, since + downsampling pixel requests on a VRT dataset/band are able to use of the + sources, even when the VRT bands do not declare them. One situation where + explicit overviews are needed at the VRT level is for example warping a VRT + to a lower resolution. + This element can also be used to an existing VRT dataset by running + :cpp:func:`GDALDataset::BuildOverviews` or :program:`gdaladdo` with the + :decl_configoption:`VRT_VIRTUAL_OVERVIEWS` configuration option set to ``YES``. + Virtual overviews have the least priority compared to the **Overview** element + at the **VRTRasterBand** level, or to materialized .vrt.ovr files. + - **VRTRasterBand**: This represents one band of a dataset. @@ -436,7 +452,12 @@ Except if (from top priority to lesser priority) : - The **Overview** element is present in the VRTRasterBand element. See above. - or external .vrt.ovr overviews are built -- (starting with GDAL 2.1) if the VRTRasterBand are made of a single SimpleSource or ComplexSource that has overviews. Those "virtual" overviews will be hidden by external .vrt.ovr overviews that might be built later. +- (starting with GDAL 3.2) explicit virtual overviews, if a **OverviewList** element + is declared in the VRTDataset element (see above). + Those virtual overviews will be hidden by external .vrt.ovr overviews that might be built later. +- (starting with GDAL 2.1) implicit virtual overviews, if the VRTRasterBand are made of + a single SimpleSource or ComplexSource that has overviews. + Those virtual overviews will be hidden by external .vrt.ovr overviews that might be built later. .vrt Descriptions for Raw Files ------------------------------- diff --git a/gdal/frmts/vrt/vrtdataset.cpp b/gdal/frmts/vrt/vrtdataset.cpp index 68785f774d05..4157986cb0f5 100644 --- a/gdal/frmts/vrt/vrtdataset.cpp +++ b/gdal/frmts/vrt/vrtdataset.cpp @@ -49,16 +49,7 @@ CPL_CVSID("$Id$") /* VRTDataset() */ /************************************************************************/ -VRTDataset::VRTDataset( int nXSize, int nYSize ) : - m_bGeoTransformSet(FALSE), - m_nGCPCount(0), - m_pasGCPList(nullptr), - m_bNeedsFlush(FALSE), - m_bWritable(TRUE), - m_pszVRTPath(nullptr), - m_poMaskBand(nullptr), - m_bCompatibleForDatasetIO(-1), - m_papszXMLVRTMetadata(nullptr) +VRTDataset::VRTDataset( int nXSize, int nYSize ) { nRasterXSize = nXSize; nRasterYSize = nYSize; @@ -164,10 +155,10 @@ template void VRTFlushCacheStruct::FlushCache(T& obj) { obj.GDALDataset::FlushCache(); - if( !obj.m_bNeedsFlush || obj.m_bWritable == FALSE) + if( !obj.m_bNeedsFlush || !obj.m_bWritable ) return; - obj.m_bNeedsFlush = FALSE; + obj.m_bNeedsFlush = false; // We don't write to disk if there is no filename. This is a // memory only dataset. @@ -388,6 +379,28 @@ CPLXMLNode *VRTDataset::SerializeToXML( const char *pszVRTPathIn ) } } + /* -------------------------------------------------------------------- */ + /* Overview factors. */ + /* -------------------------------------------------------------------- */ + if( !m_anOverviewFactors.empty() ) + { + CPLString osOverviewList; + for( int nOvFactor: m_anOverviewFactors ) + { + if( !osOverviewList.empty() ) + osOverviewList += " "; + osOverviewList += CPLSPrintf("%d", nOvFactor); + } + CPLXMLNode* psOverviewList = CPLCreateXMLElementAndValue( + psDSTree, "OverviewList", osOverviewList); + if( !m_osOverviewResampling.empty() ) + { + CPLAddXMLAttributeAndValue( psOverviewList, + "resampling", + m_osOverviewResampling ); + } + } + return psDSTree; } @@ -604,6 +617,31 @@ CPLErr VRTDataset::XMLInit( CPLXMLNode *psTree, const char *pszVRTPathIn ) } } +/* -------------------------------------------------------------------- */ +/* Create virtual overviews. */ +/* -------------------------------------------------------------------- */ + const char* pszSubClass = CPLGetXMLValue(psTree, "subClass", ""); + if( EQUAL(pszSubClass, "") ) + { + CPLStringList aosTokens(CSLTokenizeString( + CPLGetXMLValue( psTree, "OverviewList", "" ) )); + m_osOverviewResampling = + CPLGetXMLValue( psTree, "OverviewList.resampling", "" ); + for( int iOverview = 0; iOverview < aosTokens.size(); iOverview++ ) + { + const int nOvFactor = atoi(aosTokens[iOverview]); + if( nOvFactor <= 1 ) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid overview factor"); + return CE_Failure; + } + + AddVirtualOverview(nOvFactor, m_osOverviewResampling.empty() ? + "nearest" : m_osOverviewResampling.c_str()); + } + } + return CE_None; } @@ -649,7 +687,7 @@ CPLErr VRTDataset::SetGCPs( int nGCPCountIn, const GDAL_GCP *pasGCPListIn, m_pasGCPList = GDALDuplicateGCPs( nGCPCountIn, pasGCPListIn ); - m_bNeedsFlush = TRUE; + SetNeedsFlush(); return CE_None; } @@ -668,7 +706,7 @@ CPLErr VRTDataset::SetSpatialRef(const OGRSpatialReference* poSRS) else m_poSRS = nullptr; - m_bNeedsFlush = TRUE; + SetNeedsFlush(); return CE_None; } @@ -683,7 +721,7 @@ CPLErr VRTDataset::SetGeoTransform( double *padfGeoTransformIn ) memcpy( m_adfGeoTransform, padfGeoTransformIn, sizeof(double) * 6 ); m_bGeoTransformSet = TRUE; - m_bNeedsFlush = TRUE; + SetNeedsFlush(); return CE_None; } @@ -873,7 +911,7 @@ GDALDataset *VRTDataset::Open( GDALOpenInfo * poOpenInfo ) OpenXML( pszXML, pszVRTPath, poOpenInfo->eAccess ) ); if( poDS != nullptr ) - poDS->m_bNeedsFlush = FALSE; + poDS->m_bNeedsFlush =false; if( poDS != nullptr ) { @@ -1106,7 +1144,7 @@ GDALDataset *VRTDataset::OpenXML( const char *pszXML, const char *pszVRTPath, CPLErr VRTDataset::AddBand( GDALDataType eType, char **papszOptions ) { - m_bNeedsFlush = TRUE; + SetNeedsFlush(); /* ==================================================================== */ /* Handle a new raw band. */ @@ -1350,7 +1388,7 @@ VRTDataset::Create( const char * pszName, for( int iBand = 0; iBand < nBands; iBand++ ) poDS->AddBand( eType, nullptr ); - poDS->m_bNeedsFlush = TRUE; + poDS->SetNeedsFlush(); poDS->oOvManager.Initialize( poDS, pszName ); @@ -1789,7 +1827,7 @@ CPLErr VRTDataset::IRasterIO( GDALRWFlag eRWFlag, bool bLocalCompatibleForDatasetIO = CPL_TO_BOOL(CheckCompatibleForDatasetIO()); if( bLocalCompatibleForDatasetIO && eRWFlag == GF_Read && - (nBufXSize < nXSize || nBufYSize < nYSize) ) + (nBufXSize < nXSize || nBufYSize < nYSize) && m_apoOverviews.empty() ) { int bTried = FALSE; const CPLErr eErr = TryOverviewRasterIO( eRWFlag, @@ -1801,6 +1839,7 @@ CPLErr VRTDataset::IRasterIO( GDALRWFlag eRWFlag, nBandSpace, psExtraArg, &bTried ); + if( bTried ) { m_nRecursionCounter --; @@ -2062,6 +2101,13 @@ void VRTDataset::BuildVirtualOverviews() return; } + VRTSourcedRasterBand* l_poVRTBand + = cpl::down_cast(papoBands[0]); + VRTSimpleSource* poSource + = cpl::down_cast( l_poVRTBand->papoSources[0] ); + const double dfDstToSrcXRatio = poSource->m_dfDstXSize / poSource->m_dfSrcXSize; + const double dfDstToSrcYRatio = poSource->m_dfDstYSize / poSource->m_dfSrcYSize; + for( int j = 0; j < nOverviews; j++) { auto poOvrBand = poFirstBand->GetOverview(j); @@ -2071,6 +2117,11 @@ void VRTDataset::BuildVirtualOverviews() poOvrBand->GetXSize() ) / poFirstBand->GetXSize(); const double dfYRatio = static_cast( poOvrBand->GetYSize() ) / poFirstBand->GetYSize(); + if( dfXRatio >= dfDstToSrcXRatio || + dfYRatio >= dfDstToSrcYRatio ) + { + continue; + } const int nOvrXSize = static_cast(0.5 + nRasterXSize * dfXRatio); const int nOvrYSize = static_cast(0.5 + nRasterYSize * dfYRatio); if( nOvrXSize < 128 || nOvrYSize < 128 ) @@ -2144,6 +2195,47 @@ void VRTDataset::BuildVirtualOverviews() } } +/************************************************************************/ +/* AddVirtualOverview() */ +/************************************************************************/ + +bool VRTDataset::AddVirtualOverview(int nOvFactor, const char* pszResampling) +{ + if( nRasterXSize / nOvFactor == 0 || + nRasterYSize / nOvFactor == 0 ) + { + return false; + } + + CPLStringList argv; + argv.AddString("-of"); + argv.AddString("VRT"); + argv.AddString("-outsize"); + argv.AddString(CPLSPrintf("%d", nRasterXSize / nOvFactor)); + argv.AddString(CPLSPrintf("%d", nRasterYSize / nOvFactor)); + argv.AddString("-r"); + argv.AddString(pszResampling); + + GDALTranslateOptions* psOptions = GDALTranslateOptionsNew(argv.List(), nullptr); + + // Add a dummy overview so that BuildVirtualOverviews() doesn't trigger + m_apoOverviews.push_back(nullptr); + CPLAssert(m_bCanTakeRef); + m_bCanTakeRef = false; // we don't want hOverviewDS to take a reference on ourselves. + GDALDatasetH hOverviewDS = GDALTranslate("", GDALDataset::ToHandle(this), + psOptions, nullptr); + m_bCanTakeRef = true; + m_apoOverviews.resize(m_apoOverviews.size() - 1); + + GDALTranslateOptionsFree( psOptions ); + if( hOverviewDS == nullptr ) + return false; + + m_anOverviewFactors.push_back(nOvFactor); + m_apoOverviews.push_back(GDALDataset::FromHandle(hOverviewDS)); + return true; +} + /************************************************************************/ /* IBuildOverviews() */ /************************************************************************/ @@ -2157,6 +2249,31 @@ VRTDataset::IBuildOverviews( const char *pszResampling, GDALProgressFunc pfnProgress, void * pProgressData ) { + if( CPLTestBool(CPLGetConfigOption("VRT_VIRTUAL_OVERVIEWS", "NO")) ) + { + SetNeedsFlush(); + if( nOverviews == 0 || + (!m_apoOverviews.empty() && m_anOverviewFactors.empty()) ) + { + m_anOverviewFactors.clear(); + m_apoOverviewsBak.insert(m_apoOverviewsBak.end(), + m_apoOverviews.begin(), + m_apoOverviews.end()); + m_apoOverviews.clear(); + } + m_osOverviewResampling = pszResampling; + for(int i = 0; i < nOverviews; i++ ) + { + if( std::find(m_anOverviewFactors.begin(), + m_anOverviewFactors.end(), + panOverviewList[i]) == m_anOverviewFactors.end() ) + { + AddVirtualOverview(panOverviewList[i], pszResampling); + } + } + return CE_None; + } + if( !oOvManager.IsInitialized() ) { const char* pszDesc = GetDescription(); @@ -2171,8 +2288,10 @@ VRTDataset::IBuildOverviews( const char *pszResampling, // in GDAL API? if( !m_apoOverviews.empty() ) { - m_apoOverviewsBak = m_apoOverviews; - m_apoOverviews.resize(0); + m_apoOverviewsBak.insert(m_apoOverviewsBak.end(), + m_apoOverviews.begin(), + m_apoOverviews.end()); + m_apoOverviews.clear(); } else { @@ -2181,13 +2300,16 @@ VRTDataset::IBuildOverviews( const char *pszResampling, m_apoOverviews.push_back(nullptr); } - return GDALDataset::IBuildOverviews( pszResampling, + CPLErr eErr = GDALDataset::IBuildOverviews( pszResampling, nOverviews, panOverviewList, nListBands, panBandList, pfnProgress, pProgressData ); + + m_apoOverviews.clear(); + return eErr; } /*! @endcond */ diff --git a/gdal/frmts/vrt/vrtdataset.h b/gdal/frmts/vrt/vrtdataset.h index 0562d9219402..ddd2c480d726 100644 --- a/gdal/frmts/vrt/vrtdataset.h +++ b/gdal/frmts/vrt/vrtdataset.h @@ -179,27 +179,33 @@ class CPL_DLL VRTDataset CPL_NON_FINAL: public GDALDataset OGRSpatialReference* m_poSRS = nullptr; - int m_bGeoTransformSet; + int m_bGeoTransformSet = false; double m_adfGeoTransform[6]; - int m_nGCPCount; - GDAL_GCP *m_pasGCPList; + int m_nGCPCount = 0; + GDAL_GCP *m_pasGCPList = nullptr; OGRSpatialReference *m_poGCP_SRS = nullptr; - int m_bNeedsFlush; - int m_bWritable; + bool m_bNeedsFlush = false; + bool m_bWritable = true; + bool m_bCanTakeRef = true; - char *m_pszVRTPath; + char *m_pszVRTPath = nullptr; - VRTRasterBand *m_poMaskBand; + VRTRasterBand *m_poMaskBand = nullptr; - int m_bCompatibleForDatasetIO; + int m_bCompatibleForDatasetIO = -1; int CheckCompatibleForDatasetIO(); void ExpandProxyBands(); + // Virtual (ie not materialized) overviews, created either implicitly + // when it is cheap to do it, or explicitly. std::vector m_apoOverviews{}; std::vector m_apoOverviewsBak{}; - char **m_papszXMLVRTMetadata; + CPLString m_osOverviewResampling{}; + std::vector m_anOverviewFactors{}; + + char **m_papszXMLVRTMetadata = nullptr; std::map m_oMapSharedSources{}; std::shared_ptr m_poRootGroup{}; @@ -209,6 +215,8 @@ class CPL_DLL VRTDataset CPL_NON_FINAL: public GDALDataset VRTRasterBand* InitBand(const char* pszSubclass, int nBand, bool bAllowPansharpened); static GDALDataset *OpenVRTProtocol( const char* pszSpec ); + bool AddVirtualOverview(int nOvFactor, + const char* pszResampling); CPL_DISALLOW_COPY_ASSIGN(VRTDataset) @@ -219,10 +227,10 @@ class CPL_DLL VRTDataset CPL_NON_FINAL: public GDALDataset VRTDataset(int nXSize, int nYSize); virtual ~VRTDataset(); - void SetNeedsFlush() { m_bNeedsFlush = TRUE; } + void SetNeedsFlush() { m_bNeedsFlush = true; } virtual void FlushCache() override; - void SetWritable(int bWritableIn) { m_bWritable = bWritableIn; } + void SetWritable(int bWritableIn) { m_bWritable = CPL_TO_BOOL(bWritableIn); } virtual CPLErr CreateMaskBand( int nFlags ) override; void SetMaskBand(VRTRasterBand* poMaskBand); @@ -879,6 +887,7 @@ class CPL_DLL VRTSimpleSource CPL_NON_FINAL: public VRTSource protected: friend class VRTSourcedRasterBand; + friend class VRTDataset; GDALRasterBand *m_poRasterBand; @@ -906,6 +915,8 @@ class CPL_DLL VRTSimpleSource CPL_NON_FINAL: public VRTSource CPLString m_osSourceFileNameOri{}; int m_nExplicitSharedStatus; // -1 unknown, 0 = unshared, 1 = shared + bool m_bDropRefOnSrcBand; + int NeedMaxValAdjustment() const; public: diff --git a/gdal/frmts/vrt/vrtsourcedrasterband.cpp b/gdal/frmts/vrt/vrtsourcedrasterband.cpp index 46050a7e23f9..9390a9c81804 100644 --- a/gdal/frmts/vrt/vrtsourcedrasterband.cpp +++ b/gdal/frmts/vrt/vrtsourcedrasterband.cpp @@ -167,7 +167,9 @@ CPLErr VRTSourcedRasterBand::IRasterIO( GDALRWFlag eRWFlag, /* Do we have overviews that would be appropriate to satisfy */ /* this request? */ /* ==================================================================== */ - if( (nBufXSize < nXSize || nBufYSize < nYSize) + auto l_poDS = cpl::down_cast(poDS); + if( l_poDS->m_apoOverviews.empty() && + (nBufXSize < nXSize || nBufYSize < nYSize) && GetOverviewCount() > 0 ) { if( OverviewRasterIO( @@ -1117,8 +1119,24 @@ void VRTSourcedRasterBand::ConfigureSource( VRTSimpleSource *poSimpleSource, /* -------------------------------------------------------------------- */ /* If we can get the associated GDALDataset, add a reference to it.*/ /* -------------------------------------------------------------------- */ - if( poSrcBand->GetDataset() != nullptr ) - poSrcBand->GetDataset()->Reference(); + GDALDataset* poSrcBandDataset = poSrcBand->GetDataset(); + if( poSrcBandDataset != nullptr ) + { + VRTDataset* poVRTSrcBandDataset = dynamic_cast(poSrcBandDataset); + if( poVRTSrcBandDataset && !poVRTSrcBandDataset->m_bCanTakeRef ) + { + // Situation triggered by VRTDataset::AddVirtualOverview() + // We create an overview dataset that is a VRT of a reduction of + // ourselves. But we don't want to take a reference on ourselves, + // otherwise this will prevent us to be closed in number of + // circumstances + poSimpleSource->m_bDropRefOnSrcBand = false; + } + else + { + poSrcBandDataset->Reference(); + } + } } /************************************************************************/ diff --git a/gdal/frmts/vrt/vrtsources.cpp b/gdal/frmts/vrt/vrtsources.cpp index 10e0ad1f220d..bf4e30a5736e 100644 --- a/gdal/frmts/vrt/vrtsources.cpp +++ b/gdal/frmts/vrt/vrtsources.cpp @@ -111,7 +111,8 @@ VRTSimpleSource::VRTSimpleSource() : m_dfNoDataValue(VRT_NODATA_UNSET), m_nMaxValue(0), m_bRelativeToVRTOri(-1), - m_nExplicitSharedStatus(-1) + m_nExplicitSharedStatus(-1), + m_bDropRefOnSrcBand(true) {} /************************************************************************/ @@ -134,7 +135,8 @@ VRTSimpleSource::VRTSimpleSource( const VRTSimpleSource* poSrcSource, m_dfNoDataValue(poSrcSource->m_dfNoDataValue), m_nMaxValue(poSrcSource->m_nMaxValue), m_bRelativeToVRTOri(-1), - m_nExplicitSharedStatus(poSrcSource->m_nExplicitSharedStatus) + m_nExplicitSharedStatus(poSrcSource->m_nExplicitSharedStatus), + m_bDropRefOnSrcBand(poSrcSource->m_bDropRefOnSrcBand) {} /************************************************************************/ @@ -144,6 +146,9 @@ VRTSimpleSource::VRTSimpleSource( const VRTSimpleSource* poSrcSource, VRTSimpleSource::~VRTSimpleSource() { + if( !m_bDropRefOnSrcBand ) + return; + if( m_poMaskBandMainBand != nullptr ) { if( m_poMaskBandMainBand->GetDataset() != nullptr ) From 4753e4af3bf1515029b376769db7c53b3466230f Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 21 Sep 2020 14:40:53 +0200 Subject: [PATCH 056/255] gdalbuildvrt: add support for automatically declaring virtual overviews in very restricted situations All sources must have the same resolution, and this must be the one of the VRT dataset itself. --- autotest/utilities/test_gdalbuildvrt_lib.py | 31 ++++++++ gdal/apps/gdalbuildvrt_lib.cpp | 87 +++++++++++++++++++++ 2 files changed, 118 insertions(+) diff --git a/autotest/utilities/test_gdalbuildvrt_lib.py b/autotest/utilities/test_gdalbuildvrt_lib.py index af43f324e101..581738d86b4b 100755 --- a/autotest/utilities/test_gdalbuildvrt_lib.py +++ b/autotest/utilities/test_gdalbuildvrt_lib.py @@ -152,3 +152,34 @@ def get_vrt_of_vrt(): scenario_1() scenario_2() + +############################################################################### +# Test BuildVRT() with virtual overviews + + +def test_gdalbuildvrt_lib_virtual_overviews(): + + src1_ds = gdal.GetDriverByName('MEM').Create('', 1000, 1000) + src1_ds.SetGeoTransform([2,0.001,0,49,0,-0.001]) + src1_ds.BuildOverviews('NEAR', [2, 4, 8]) + + src2_ds = gdal.GetDriverByName('MEM').Create('', 2000, 2000) + src2_ds.SetGeoTransform([3,0.001,0,49,0,-0.001]) + src2_ds.BuildOverviews('NEAR', [2, 4, 16]) + + vrt_ds = gdal.BuildVRT('', [src1_ds, src2_ds]) + assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 2 + + +def test_gdalbuildvrt_lib_virtual_overviews_not_same_res(): + + src1_ds = gdal.GetDriverByName('MEM').Create('', 1000, 1000) + src1_ds.SetGeoTransform([2,0.001,0,49,0,-0.001]) + src1_ds.BuildOverviews('NEAR', [2, 4]) + + src2_ds = gdal.GetDriverByName('MEM').Create('', 500, 500) + src2_ds.SetGeoTransform([3,0.002,0,49,0,-0.002]) + src2_ds.BuildOverviews('NEAR', [2, 4]) + + vrt_ds = gdal.BuildVRT('', [src1_ds, src2_ds]) + assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 0 diff --git a/gdal/apps/gdalbuildvrt_lib.cpp b/gdal/apps/gdalbuildvrt_lib.cpp index 963e709e77a3..02ae01f2ba9e 100644 --- a/gdal/apps/gdalbuildvrt_lib.cpp +++ b/gdal/apps/gdalbuildvrt_lib.cpp @@ -42,6 +42,7 @@ #include #include #include +#include #include "cpl_conv.h" #include "cpl_error.h" @@ -93,6 +94,7 @@ struct DatasetProperty int bHasDatasetMask = 0; int nMaskBlockXSize = 0; int nMaskBlockYSize = 0; + std::vector anOverviewFactors{}; DatasetProperty() { @@ -611,6 +613,33 @@ int VRTBuilder::AnalyseRaster( GDALDatasetH hDS, DatasetProperty* psDatasetPrope &psDatasetProperties->nMaskBlockXSize, &psDatasetProperties->nMaskBlockYSize); + // Collect overview factors. We only handle power-of-two situations for now + const int nOverviews = poFirstBand->GetOverviewCount(); + int nExpectedOvFactor = 2; + for(int j=0; j < nOverviews; j++) + { + GDALRasterBand* poOverview = poFirstBand->GetOverview(j); + if( !poOverview ) + continue; + if( poOverview->GetXSize() < 128 && + poOverview->GetYSize() < 128 ) + { + break; + } + + const int nOvFactor = + GDALComputeOvFactor(poOverview->GetXSize(), + poFirstBand->GetXSize(), + poOverview->GetYSize(), + poFirstBand->GetYSize()); + + if( nOvFactor != nExpectedOvFactor ) + break; + + psDatasetProperties->anOverviewFactors.push_back(nOvFactor); + nExpectedOvFactor *= 2; + } + for(int j=0;j<_nBands;j++) { GDALRasterBand* poBand = poDS->GetRasterBand(j+1); @@ -1019,6 +1048,10 @@ void VRTBuilder::CreateVRTNonSeparate(VRTDatasetH hVRTDS) poMaskVRTBand = static_cast(poVRTDS->GetRasterBand(1)->GetMaskBand()); } + bool bCanCollectOverviewFactors = true; + std::set anOverviewFactorsSet; + std::vector anIdxValidDatasets; + for( int i = 0; ppszInputFilenames != nullptr && i < nInputFiles; i++ ) { DatasetProperty* psDatasetProperties = &asDatasetProperties[i]; @@ -1041,6 +1074,23 @@ void VRTBuilder::CreateVRTNonSeparate(VRTDatasetH hVRTDS) &dfDstXOff, &dfDstYOff, &dfDstXSize, &dfDstYSize) ) continue; + anIdxValidDatasets.push_back(i); + + if( bCanCollectOverviewFactors ) + { + if( std::abs(psDatasetProperties->adfGeoTransform[1] - we_res) > 1e-8 * std::abs(we_res) || + std::abs(psDatasetProperties->adfGeoTransform[5] - ns_res) > 1e-8 * std::abs(ns_res) ) + { + bCanCollectOverviewFactors = false; + anOverviewFactorsSet.clear(); + } + } + if( bCanCollectOverviewFactors ) + { + for( int nOvFactor: psDatasetProperties->anOverviewFactors ) + anOverviewFactorsSet.insert(nOvFactor); + } + const char* dsFileName = ppszInputFilenames[i]; GDALDatasetH hSourceDS; @@ -1148,6 +1198,43 @@ void VRTBuilder::CreateVRTNonSeparate(VRTDatasetH hVRTDS) GDALDereferenceDataset(hSourceDS); } } + + for( int i: anIdxValidDatasets ) + { + const DatasetProperty* psDatasetProperties = &asDatasetProperties[i]; + for( auto oIter = anOverviewFactorsSet.begin(); oIter != anOverviewFactorsSet.end(); ) + { + const int nGlobalOvrFactor = *oIter; + auto oIterNext = oIter; + ++oIterNext; + + if( psDatasetProperties->nRasterXSize / nGlobalOvrFactor < 128 && + psDatasetProperties->nRasterYSize / nGlobalOvrFactor < 128 ) + { + break; + } + if( std::find(psDatasetProperties->anOverviewFactors.begin(), + psDatasetProperties->anOverviewFactors.end(), + nGlobalOvrFactor) == psDatasetProperties->anOverviewFactors.end() ) + { + anOverviewFactorsSet.erase(oIter); + } + + oIter = oIterNext; + } + } + if( !anOverviewFactorsSet.empty() ) + { + std::vector anOverviewFactors; + anOverviewFactors.insert(anOverviewFactors.end(), + anOverviewFactorsSet.begin(), + anOverviewFactorsSet.end()); + CPLConfigOptionSetter oSetter("VRT_VIRTUAL_OVERVIEWS", "YES", false); + poVRTDS->BuildOverviews(pszResampling ? pszResampling : "nearest", + static_cast(anOverviewFactors.size()), + &anOverviewFactors[0], + 0, nullptr, nullptr, nullptr); + } } /************************************************************************/ From 377b8b39f5bc6f05767f34f2177bb9a2be9c2a53 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 21 Sep 2020 15:40:50 +0200 Subject: [PATCH 057/255] ogr_gpkg.py: test_abort_sql: avoid creating new file, and increase number of iterations of the query to be sure it runs well above a one second limit (20s on my machine now) --- autotest/ogr/ogr_gpkg.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/autotest/ogr/ogr_gpkg.py b/autotest/ogr/ogr_gpkg.py index 1c88c6097fa3..3e0c695317b1 100755 --- a/autotest/ogr/ogr_gpkg.py +++ b/autotest/ogr/ogr_gpkg.py @@ -4155,12 +4155,8 @@ def test_ogr_gpkg_datetime_timezones(): def test_abort_sql(): - filename = '/vsimem/test_ogr_gpkg_abort_sql.gpkg' - ds = ogr.GetDriverByName('GPKG').CreateDataSource(filename) - ds.CreateLayer('test') - ds = None - - ds = ogr.Open(filename, update=1) + filename = 'data/gpkg/poly_non_conformant.gpkg' + ds = ogr.Open(filename) def abortAfterDelay(): print("Aborting SQL...") @@ -4177,7 +4173,7 @@ def abortAfterDelay(): VALUES(0) UNION ALL SELECT i FROM r - LIMIT 10000000 + LIMIT 100000000 ) SELECT i FROM r WHERE i = 1;""" From e24af9a4087478b7ee3200182de63a550ef6f3e1 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 21 Sep 2020 17:47:29 +0200 Subject: [PATCH 058/255] Python bindings: remove use of deprecated PyObject_AsReadBuffer() function --- gdal/swig/include/python/typemaps_python.i | 27 ++++--- gdal/swig/python/extensions/gdal_wrap.cpp | 90 ++++++++++++++-------- gdal/swig/python/extensions/ogr_wrap.cpp | 60 ++++++++++----- 3 files changed, 116 insertions(+), 61 deletions(-) diff --git a/gdal/swig/include/python/typemaps_python.i b/gdal/swig/include/python/typemaps_python.i index 4a87d3f40293..2670a0513a09 100644 --- a/gdal/swig/include/python/typemaps_python.i +++ b/gdal/swig/include/python/typemaps_python.i @@ -497,22 +497,24 @@ CreateTupleFromDoubleArray( int *first, unsigned int size ) { } -%typemap(in,numinputs=1) (int nLen, char *pBuf ) (int alloc = 0) +%typemap(in,numinputs=1) (int nLen, char *pBuf ) (int alloc = 0, bool viewIsValid = false, Py_buffer view) { /* %typemap(in,numinputs=1) (int nLen, char *pBuf ) */ { - Py_ssize_t safeLen = 0; - const void *safeBuf = 0; - int res = PyObject_AsReadBuffer($input, &safeBuf, &safeLen); - if (res == 0) { - if( safeLen > INT_MAX ) { + if (PyObject_GetBuffer($input, &view, PyBUF_SIMPLE) == 0) + { + if( view.len > INT_MAX ) { + PyBuffer_Release(&view); SWIG_exception( SWIG_RuntimeError, "too large buffer (>2GB)" ); } - $1 = (int) safeLen; - $2 = ($2_ltype) safeBuf; + viewIsValid = true; + $1 = (int) view.len; + $2 = ($2_ltype) view.buf; goto ok; - } else { - PyErr_Clear(); + } + else + { + PyErr_Clear(); } } %#if PY_VERSION_HEX>=0x03000000 @@ -566,7 +568,10 @@ CreateTupleFromDoubleArray( int *first, unsigned int size ) { %typemap(freearg) (int nLen, char *pBuf ) { /* %typemap(freearg) (int *nLen, char *pBuf ) */ - if( alloc$argnum == SWIG_NEWOBJ ) { + if( viewIsValid$argnum ) { + PyBuffer_Release(&view$argnum); + } + else if( alloc$argnum == SWIG_NEWOBJ ) { delete[] $2; } } diff --git a/gdal/swig/python/extensions/gdal_wrap.cpp b/gdal/swig/python/extensions/gdal_wrap.cpp index d2ee01dd42df..728311e1cb4f 100644 --- a/gdal/swig/python/extensions/gdal_wrap.cpp +++ b/gdal/swig/python/extensions/gdal_wrap.cpp @@ -9097,6 +9097,8 @@ SWIGINTERN PyObject *_wrap_EscapeString(PyObject *SWIGUNUSEDPARM(self), PyObject char *arg2 = (char *) 0 ; int arg3 = (int) CPLES_SQL ; int alloc1 = 0 ; + bool viewIsValid1 = false ; + Py_buffer view1 ; int val3 ; int ecode3 = 0 ; PyObject * obj0 = 0 ; @@ -9110,17 +9112,19 @@ SWIGINTERN PyObject *_wrap_EscapeString(PyObject *SWIGUNUSEDPARM(self), PyObject { /* %typemap(in,numinputs=1) (int nLen, char *pBuf ) */ { - Py_ssize_t safeLen = 0; - const void *safeBuf = 0; - int res = PyObject_AsReadBuffer(obj0, &safeBuf, &safeLen); - if (res == 0) { - if( safeLen > INT_MAX ) { + if (PyObject_GetBuffer(obj0, &view1, PyBUF_SIMPLE) == 0) + { + if( view1.len > INT_MAX ) { + PyBuffer_Release(&view1); SWIG_exception( SWIG_RuntimeError, "too large buffer (>2GB)" ); } - arg1 = (int) safeLen; - arg2 = (char *) safeBuf; + viewIsValid1 = true; + arg1 = (int) view1.len; + arg2 = (char *) view1.buf; goto ok; - } else { + } + else + { PyErr_Clear(); } } @@ -9212,7 +9216,10 @@ SWIGINTERN PyObject *_wrap_EscapeString(PyObject *SWIGUNUSEDPARM(self), PyObject } { /* %typemap(freearg) (int *nLen, char *pBuf ) */ - if (ReturnSame(alloc1) == SWIG_NEWOBJ ) { + if( viewIsValid1 ) { + PyBuffer_Release(&view1); + } + else if (ReturnSame(alloc1) == SWIG_NEWOBJ ) { delete[] arg2; } } @@ -9221,7 +9228,10 @@ SWIGINTERN PyObject *_wrap_EscapeString(PyObject *SWIGUNUSEDPARM(self), PyObject fail: { /* %typemap(freearg) (int *nLen, char *pBuf ) */ - if (ReturnSame(alloc1) == SWIG_NEWOBJ ) { + if( viewIsValid1 ) { + PyBuffer_Release(&view1); + } + else if (ReturnSame(alloc1) == SWIG_NEWOBJ ) { delete[] arg2; } } @@ -10423,6 +10433,8 @@ SWIGINTERN PyObject *_wrap_CPLBinaryToHex(PyObject *SWIGUNUSEDPARM(self), PyObje int arg1 ; GByte *arg2 = (GByte *) 0 ; int alloc1 = 0 ; + bool viewIsValid1 = false ; + Py_buffer view1 ; PyObject * obj0 = 0 ; retStringAndCPLFree *result = 0 ; @@ -10430,17 +10442,19 @@ SWIGINTERN PyObject *_wrap_CPLBinaryToHex(PyObject *SWIGUNUSEDPARM(self), PyObje { /* %typemap(in,numinputs=1) (int nLen, char *pBuf ) */ { - Py_ssize_t safeLen = 0; - const void *safeBuf = 0; - int res = PyObject_AsReadBuffer(obj0, &safeBuf, &safeLen); - if (res == 0) { - if( safeLen > INT_MAX ) { + if (PyObject_GetBuffer(obj0, &view1, PyBUF_SIMPLE) == 0) + { + if( view1.len > INT_MAX ) { + PyBuffer_Release(&view1); SWIG_exception( SWIG_RuntimeError, "too large buffer (>2GB)" ); } - arg1 = (int) safeLen; - arg2 = (GByte *) safeBuf; + viewIsValid1 = true; + arg1 = (int) view1.len; + arg2 = (GByte *) view1.buf; goto ok; - } else { + } + else + { PyErr_Clear(); } } @@ -10525,7 +10539,10 @@ SWIGINTERN PyObject *_wrap_CPLBinaryToHex(PyObject *SWIGUNUSEDPARM(self), PyObje } { /* %typemap(freearg) (int *nLen, char *pBuf ) */ - if (ReturnSame(alloc1) == SWIG_NEWOBJ ) { + if( viewIsValid1 ) { + PyBuffer_Release(&view1); + } + else if (ReturnSame(alloc1) == SWIG_NEWOBJ ) { delete[] arg2; } } @@ -10534,7 +10551,10 @@ SWIGINTERN PyObject *_wrap_CPLBinaryToHex(PyObject *SWIGUNUSEDPARM(self), PyObje fail: { /* %typemap(freearg) (int *nLen, char *pBuf ) */ - if (ReturnSame(alloc1) == SWIG_NEWOBJ ) { + if( viewIsValid1 ) { + PyBuffer_Release(&view1); + } + else if (ReturnSame(alloc1) == SWIG_NEWOBJ ) { delete[] arg2; } } @@ -12858,6 +12878,8 @@ SWIGINTERN PyObject *_wrap_VSIFWriteL(PyObject *SWIGUNUSEDPARM(self), PyObject * int arg4 ; VSILFILE *arg5 = (VSILFILE *) 0 ; int alloc1 = 0 ; + bool viewIsValid1 = false ; + Py_buffer view1 ; int val3 ; int ecode3 = 0 ; int val4 ; @@ -12874,17 +12896,19 @@ SWIGINTERN PyObject *_wrap_VSIFWriteL(PyObject *SWIGUNUSEDPARM(self), PyObject * { /* %typemap(in,numinputs=1) (int nLen, char *pBuf ) */ { - Py_ssize_t safeLen = 0; - const void *safeBuf = 0; - int res = PyObject_AsReadBuffer(obj0, &safeBuf, &safeLen); - if (res == 0) { - if( safeLen > INT_MAX ) { + if (PyObject_GetBuffer(obj0, &view1, PyBUF_SIMPLE) == 0) + { + if( view1.len > INT_MAX ) { + PyBuffer_Release(&view1); SWIG_exception( SWIG_RuntimeError, "too large buffer (>2GB)" ); } - arg1 = (int) safeLen; - arg2 = (char *) safeBuf; + viewIsValid1 = true; + arg1 = (int) view1.len; + arg2 = (char *) view1.buf; goto ok; - } else { + } + else + { PyErr_Clear(); } } @@ -12976,7 +13000,10 @@ SWIGINTERN PyObject *_wrap_VSIFWriteL(PyObject *SWIGUNUSEDPARM(self), PyObject * resultobj = SWIG_From_int(static_cast< int >(result)); { /* %typemap(freearg) (int *nLen, char *pBuf ) */ - if (ReturnSame(alloc1) == SWIG_NEWOBJ ) { + if( viewIsValid1 ) { + PyBuffer_Release(&view1); + } + else if (ReturnSame(alloc1) == SWIG_NEWOBJ ) { delete[] arg2; } } @@ -12985,7 +13012,10 @@ SWIGINTERN PyObject *_wrap_VSIFWriteL(PyObject *SWIGUNUSEDPARM(self), PyObject * fail: { /* %typemap(freearg) (int *nLen, char *pBuf ) */ - if (ReturnSame(alloc1) == SWIG_NEWOBJ ) { + if( viewIsValid1 ) { + PyBuffer_Release(&view1); + } + else if (ReturnSame(alloc1) == SWIG_NEWOBJ ) { delete[] arg2; } } diff --git a/gdal/swig/python/extensions/ogr_wrap.cpp b/gdal/swig/python/extensions/ogr_wrap.cpp index d23b46f06040..39ecc315ddce 100644 --- a/gdal/swig/python/extensions/ogr_wrap.cpp +++ b/gdal/swig/python/extensions/ogr_wrap.cpp @@ -21297,6 +21297,8 @@ SWIGINTERN PyObject *_wrap_CreateGeometryFromWkb(PyObject *SWIGUNUSEDPARM(self), char *arg2 = (char *) 0 ; OSRSpatialReferenceShadow *arg3 = (OSRSpatialReferenceShadow *) NULL ; int alloc1 = 0 ; + bool viewIsValid1 = false ; + Py_buffer view1 ; void *argp3 = 0 ; int res3 = 0 ; PyObject * obj0 = 0 ; @@ -21310,17 +21312,19 @@ SWIGINTERN PyObject *_wrap_CreateGeometryFromWkb(PyObject *SWIGUNUSEDPARM(self), { /* %typemap(in,numinputs=1) (int nLen, char *pBuf ) */ { - Py_ssize_t safeLen = 0; - const void *safeBuf = 0; - int res = PyObject_AsReadBuffer(obj0, &safeBuf, &safeLen); - if (res == 0) { - if( safeLen > INT_MAX ) { + if (PyObject_GetBuffer(obj0, &view1, PyBUF_SIMPLE) == 0) + { + if( view1.len > INT_MAX ) { + PyBuffer_Release(&view1); SWIG_exception( SWIG_RuntimeError, "too large buffer (>2GB)" ); } - arg1 = (int) safeLen; - arg2 = (char *) safeBuf; + viewIsValid1 = true; + arg1 = (int) view1.len; + arg2 = (char *) view1.buf; goto ok; - } else { + } + else + { PyErr_Clear(); } } @@ -21399,7 +21403,10 @@ SWIGINTERN PyObject *_wrap_CreateGeometryFromWkb(PyObject *SWIGUNUSEDPARM(self), resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_OGRGeometryShadow, SWIG_POINTER_OWN | 0 ); { /* %typemap(freearg) (int *nLen, char *pBuf ) */ - if (ReturnSame(alloc1) == SWIG_NEWOBJ ) { + if( viewIsValid1 ) { + PyBuffer_Release(&view1); + } + else if (ReturnSame(alloc1) == SWIG_NEWOBJ ) { delete[] arg2; } } @@ -21408,7 +21415,10 @@ SWIGINTERN PyObject *_wrap_CreateGeometryFromWkb(PyObject *SWIGUNUSEDPARM(self), fail: { /* %typemap(freearg) (int *nLen, char *pBuf ) */ - if (ReturnSame(alloc1) == SWIG_NEWOBJ ) { + if( viewIsValid1 ) { + PyBuffer_Release(&view1); + } + else if (ReturnSame(alloc1) == SWIG_NEWOBJ ) { delete[] arg2; } } @@ -22114,6 +22124,8 @@ SWIGINTERN PyObject *_wrap_new_Geometry(PyObject *SWIGUNUSEDPARM(self), PyObject char *buf2 = 0 ; int alloc2 = 0 ; int alloc3 = 0 ; + bool viewIsValid3 = false ; + Py_buffer view3 ; int res5 ; char *buf5 = 0 ; int alloc5 = 0 ; @@ -22145,17 +22157,19 @@ SWIGINTERN PyObject *_wrap_new_Geometry(PyObject *SWIGUNUSEDPARM(self), PyObject { /* %typemap(in,numinputs=1) (int nLen, char *pBuf ) */ { - Py_ssize_t safeLen = 0; - const void *safeBuf = 0; - int res = PyObject_AsReadBuffer(obj2, &safeBuf, &safeLen); - if (res == 0) { - if( safeLen > INT_MAX ) { + if (PyObject_GetBuffer(obj2, &view3, PyBUF_SIMPLE) == 0) + { + if( view3.len > INT_MAX ) { + PyBuffer_Release(&view3); SWIG_exception( SWIG_RuntimeError, "too large buffer (>2GB)" ); } - arg3 = (int) safeLen; - arg4 = (char *) safeBuf; + viewIsValid3 = true; + arg3 = (int) view3.len; + arg4 = (char *) view3.buf; goto ok; - } else { + } + else + { PyErr_Clear(); } } @@ -22236,7 +22250,10 @@ SWIGINTERN PyObject *_wrap_new_Geometry(PyObject *SWIGUNUSEDPARM(self), PyObject if (alloc2 == SWIG_NEWOBJ) delete[] buf2; { /* %typemap(freearg) (int *nLen, char *pBuf ) */ - if (ReturnSame(alloc3) == SWIG_NEWOBJ ) { + if( viewIsValid3 ) { + PyBuffer_Release(&view3); + } + else if (ReturnSame(alloc3) == SWIG_NEWOBJ ) { delete[] arg4; } } @@ -22247,7 +22264,10 @@ SWIGINTERN PyObject *_wrap_new_Geometry(PyObject *SWIGUNUSEDPARM(self), PyObject if (alloc2 == SWIG_NEWOBJ) delete[] buf2; { /* %typemap(freearg) (int *nLen, char *pBuf ) */ - if (ReturnSame(alloc3) == SWIG_NEWOBJ ) { + if( viewIsValid3 ) { + PyBuffer_Release(&view3); + } + else if (ReturnSame(alloc3) == SWIG_NEWOBJ ) { delete[] arg4; } } From a832da5b936bac8438f9c50e7c20e563fd94c9ff Mon Sep 17 00:00:00 2001 From: Markus Neteler Date: Tue, 22 Sep 2020 13:06:27 +0200 Subject: [PATCH 059/255] GDAL/GRASS driver: simplified to GRASS GIS 7+ only (#2945) This removes the support for GRASS GIS 5 and 6 which are out of maintenance for many years. --- autotest/gdrivers/grass.py | 2 +- gdal/doc/source/drivers/raster/grass.rst | 36 +- gdal/frmts/grass/GNUmakefile | 11 +- .../grass/{grass57dataset.cpp => grass.cpp} | 150 ++--- gdal/frmts/grass/grassdataset.cpp | 593 ------------------ gdal/frmts/grass/pkg/Makefile.in | 4 +- gdal/frmts/grass/pkg/README | 24 +- gdal/ogr/ogrsf_frmts/grass/ogrgrass.h | 4 - .../ogrsf_frmts/grass/ogrgrassdatasource.cpp | 18 +- gdal/ogr/ogrsf_frmts/grass/ogrgrasslayer.cpp | 12 +- 10 files changed, 91 insertions(+), 763 deletions(-) rename gdal/frmts/grass/{grass57dataset.cpp => grass.cpp} (87%) delete mode 100644 gdal/frmts/grass/grassdataset.cpp diff --git a/autotest/gdrivers/grass.py b/autotest/gdrivers/grass.py index 8ea9fa344b2f..3745c03c9f65 100755 --- a/autotest/gdrivers/grass.py +++ b/autotest/gdrivers/grass.py @@ -73,7 +73,7 @@ def test_grass_2(): ret = tst.testOpen(check_prj=srs) if ret != 'success': - gdaltest.post_reason('If that test fails, checks that the GISBASE environment variable point to the root of your GRASS install. For example GIS_BASE=/usr/local/grass-6.4.svn') + gdaltest.post_reason('If that test fails, checks that the GISBASE environment variable point to the root of your GRASS install. For example GIS_BASE=/usr/lib64/grass78') return ret diff --git a/gdal/doc/source/drivers/raster/grass.rst b/gdal/doc/source/drivers/raster/grass.rst index 518f4292512a..05802fb238a5 100644 --- a/gdal/doc/source/drivers/raster/grass.rst +++ b/gdal/doc/source/drivers/raster/grass.rst @@ -8,7 +8,7 @@ GRASS Raster Format .. build_dependencies:: libgrass -GDAL optionally supports reading of existing GRASS raster maps or +GDAL optionally supports reading of existing GRASS GIS raster maps or imagery groups, but not writing or export. The support for GRASS raster format is determined when the library is configured, and requires libgrass to be pre-installed (see Notes below). @@ -18,15 +18,15 @@ GRASS raster maps/imagery groups can be selected in several ways. #. The full path to the ``cellhd`` file can be specified. This is not a relative path, or at least it must contain all the path components within the GRASS database including the database root itself. The - following example opens the raster map "proj_tm" within the GRASS - mapset "PERMANENT" of the GRASS location "proj_tm" in the GRASS - database located at ``/u/data/grassdb``. + following example opens the raster map "elevation" within the GRASS + mapset "PERMANENT" of the GRASS location "myloc" in the GRASS + database located at ``/data/grassdb``. For example: :: - gdalinfo /u/data/grassdb/proj_tm/PERMANENT/cellhd/proj_tm + gdalinfo /data/grassdb/myloc/PERMANENT/cellhd/elevation #. The full path to the directory containing information about an imagery group (or the REF file within it) can be specified to refer @@ -37,13 +37,13 @@ GRASS raster maps/imagery groups can be selected in several ways. :: - gdalinfo /usr2/data/grassdb/imagery/raw/group/testmff/REF - gdalinfo /usr2/data/grassdb/imagery/raw/group/testmff + gdalinfo /data/grassdb/imagery/raw/group/testmff/REF + gdalinfo /data/grassdb/imagery/raw/group/testmff -#. If there is a correct ``.grassrc5`` (GRASS 5), ``.grassrc6`` (GRASS - 6) ``.grass7/rc`` (GRASS 7) setup file in the users home directory - then raster maps or imagery groups may be opened just by the cell - name. This only works for raster maps or imagery groups in the +#. If there is a correct ``.grassrc7/rc`` (GRASS 7) setup file in the + user's home directory then raster maps or imagery groups may be opened + just by the cell or group name. + This only works for raster maps or imagery groups in the current GRASS location and mapset as defined in the GRASS setup file. The following features are supported by the GDAL/GRASS link. @@ -68,16 +68,16 @@ Driver capabilities Notes on driver variations -------------------------- -For GRASS 5.7 Radim Blazek has moved the driver to using the GRASS -shared libraries directly instead of using libgrass. Currently (GDAL -1.2.2 and later) both version of the driver are available and can be -configured using ``--with-libgrass`` for the libgrass variant or -``--with-grass=`` for the new GRASS 5.7+ library based version. The -GRASS 5.7+ driver version is currently not supporting coordinate system +The driver is able to use the GRASS GIS shared libraries directly +instead of using libgrass (not recommended due to potentially circular +dependencies). Currently both versions of the driver are available and +can be configured using ``--with-libgrass`` for the libgrass variant or +``--with-grass=`` for the GRASS GIS library based version. The +GRASS driver version currently does not support coordinate system access, though it is hoped that will be corrected at some point. See Also -------- -- `GRASS GIS home page `__ +- `GRASS GIS home page `__ - `libgrass page `__ diff --git a/gdal/frmts/grass/GNUmakefile b/gdal/frmts/grass/GNUmakefile index 4ec13e930f61..ce4c30a06e9a 100644 --- a/gdal/frmts/grass/GNUmakefile +++ b/gdal/frmts/grass/GNUmakefile @@ -2,13 +2,7 @@ include ../../GDALmake.opt -#OBJ = grassdataset.o - -ifeq ($(GRASS_SETTING),libgrass) -OBJ = grassdataset.o -else -OBJ = grass57dataset.o -endif +OBJ = grass.o CPPFLAGS := -DGRASS_GISBASE=\"$(GRASS_GISBASE)\" $(GRASS_INCLUDE) $(PROJ_INCLUDE) $(CPPFLAGS) @@ -22,8 +16,7 @@ install-obj: $(O_OBJ:.o=.$(OBJ_EXT)) dist: cp -r pkg gdal-grass-$(GDAL_VER) - rm -rf gdal-grass-$(GDAL_VER)/.svn - cp grass57dataset.cpp gdal-grass-$(GDAL_VER) + cp grass.cpp gdal-grass-$(GDAL_VER) cp ../../ogr/ogrsf_frmts/grass/*.cpp gdal-grass-$(GDAL_VER) cp ../../ogr/ogrsf_frmts/grass/*.h gdal-grass-$(GDAL_VER) tar czvf gdal-grass-$(GDAL_VER).tar.gz ./gdal-grass-$(GDAL_VER) diff --git a/gdal/frmts/grass/grass57dataset.cpp b/gdal/frmts/grass/grass.cpp similarity index 87% rename from gdal/frmts/grass/grass57dataset.cpp rename to gdal/frmts/grass/grass.cpp index db76955d7055..9e5204d3a80e 100644 --- a/gdal/frmts/grass/grass57dataset.cpp +++ b/gdal/frmts/grass/grass.cpp @@ -2,14 +2,14 @@ * * Project: GRASS Driver * Purpose: Implement GRASS raster read/write support - * This version is for GRASS 5.7+ and uses GRASS libraries + * This version is for GRASS GIS 7+ and uses GRASS libraries * directly instead of using libgrass. * Author: Frank Warmerdam * Radim Blazek * ****************************************************************************** * Copyright (c) 2000 Frank Warmerdam - * Copyright (c) 2007-2010, Even Rouault + * Copyright (c) 2007-2020, Even Rouault * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -50,49 +50,15 @@ extern "C" { #include #include -#if GRASS_VERSION_MAJOR >= 7 char *GPJ_grass_to_wkt( const struct Key_Value *, const struct Key_Value *, int, int ); -#else -char *GPJ_grass_to_wkt( struct Key_Value *, - struct Key_Value *, - int, int ); -#endif } #define GRASS_MAX_COLORS 100000 // what is the right value CPL_CVSID("$Id$") -#if GRASS_VERSION_MAJOR >= 7 -#define G_get_cellhd Rast_get_cellhd -#define G_get_window Rast_get_window -#define G_set_window Rast_set_window -#define G_raster_map_type Rast_map_type -#define G_read_fp_range Rast_read_fp_range -#define G_get_fp_range_min_max Rast_get_fp_range_min_max -#define G_set_c_null_value Rast_set_c_null_value -#define G_set_f_null_value Rast_set_f_null_value -#define G_set_d_null_value Rast_set_d_null_value -#define G_open_cell_old Rast_open_old -#define G_copy memcpy -#define G_read_colors Rast_read_colors -#define G_get_color_range Rast_get_c_color_range -#define G_colors_count Rast_colors_count -#define G_get_f_color_rule Rast_get_fp_color_rule -#define G_free_colors Rast_free_colors -#define G_close_cell Rast_close -#define G_allocate_c_raster_buf Rast_allocate_c_buf -#define G_get_c_raster_row Rast_get_c_row -#define G_is_c_null_value Rast_is_c_null_value -#define G_get_f_raster_row Rast_get_f_row -#define G_get_d_raster_row Rast_get_d_row -#define G_allocate_f_raster_buf Rast_allocate_f_buf -#define G_allocate_d_raster_buf Rast_allocate_d_buf -#define G__setenv G_setenv_nogisrc -#endif - /************************************************************************/ /* Grass2CPLErrorHook() */ /************************************************************************/ @@ -217,15 +183,15 @@ GRASSRasterBand::GRASSRasterBand( GRASSDataset *poDSIn, int nBandIn, this->pszCellName = G_store ( (char *) pszCellNameIn ); this->pszMapset = G_store ( (char *) pszMapsetIn ); - G_get_cellhd( (char *) pszCellName, (char *) pszMapset, &sCellInfo ); - nGRSType = G_raster_map_type( (char *) pszCellName, (char *) pszMapset ); + Rast_get_cellhd( (char *) pszCellName, (char *) pszMapset, &sCellInfo ); + nGRSType = Rast_map_type( (char *) pszCellName, (char *) pszMapset ); /* -------------------------------------------------------------------- */ /* Get min/max values. */ /* -------------------------------------------------------------------- */ struct FPRange sRange; - if( G_read_fp_range( (char *) pszCellName, (char *) pszMapset, + if( Rast_read_fp_range( (char *) pszCellName, (char *) pszMapset, &sRange ) == -1 ) { bHaveMinMax = FALSE; @@ -233,7 +199,7 @@ GRASSRasterBand::GRASSRasterBand( GRASSDataset *poDSIn, int nBandIn, else { bHaveMinMax = TRUE; - G_get_fp_range_min_max( &sRange, &dfCellMin, &dfCellMax ); + Rast_get_fp_range_min_max( &sRange, &dfCellMin, &dfCellMax ); } /* -------------------------------------------------------------------- */ @@ -268,14 +234,14 @@ GRASSRasterBand::GRASSRasterBand( GRASSDataset *poDSIn, int nBandIn, } else { // maximum is not known or full range is used CELL cval; this->eDataType = GDT_Int32; - G_set_c_null_value ( &cval, 1); + Rast_set_c_null_value ( &cval, 1); dfNoData = (double) cval; } nativeNulls = false; } else { // 3-4 bytes CELL cval; this->eDataType = GDT_Int32; - G_set_c_null_value ( &cval, 1); + Rast_set_c_null_value ( &cval, 1); dfNoData = (double) cval; nativeNulls = true; } @@ -283,7 +249,7 @@ GRASSRasterBand::GRASSRasterBand( GRASSDataset *poDSIn, int nBandIn, else if( nGRSType == FCELL_TYPE ) { FCELL fval; this->eDataType = GDT_Float32; - G_set_f_null_value ( &fval, 1); + Rast_set_f_null_value ( &fval, 1); dfNoData = (double) fval; nativeNulls = true; } @@ -291,7 +257,7 @@ GRASSRasterBand::GRASSRasterBand( GRASSDataset *poDSIn, int nBandIn, { DCELL dval; this->eDataType = GDT_Float64; - G_set_d_null_value ( &dval, 1); + Rast_set_d_null_value ( &dval, 1); dfNoData = (double) dval; nativeNulls = true; } @@ -299,21 +265,21 @@ GRASSRasterBand::GRASSRasterBand( GRASSDataset *poDSIn, int nBandIn, nBlockXSize = poDSIn->nRasterXSize; nBlockYSize = 1; - G_set_window( &(poDSIn->sCellInfo) ); + Rast_set_window ( &(poDSIn->sCellInfo) ); // open the raster only for actual reading hCell = -1; - G_copy((void *) &sOpenWindow, (void *) &(poDSIn->sCellInfo), sizeof(struct Cell_head)); + memcpy((void *) &sOpenWindow, (void *) &(poDSIn->sCellInfo), sizeof(struct Cell_head)); /* -------------------------------------------------------------------- */ /* Do we have a color table? */ /* -------------------------------------------------------------------- */ poCT = NULL; - if( G_read_colors( (char *) pszCellName, (char *) pszMapset, &sGrassColors ) == 1 ) + if( Rast_read_colors( (char *) pszCellName, (char *) pszMapset, &sGrassColors ) == 1 ) { int maxcolor; CELL min, max; - G_get_color_range ( &min, &max, &sGrassColors); + Rast_get_c_color_range ( &min, &max, &sGrassColors); if ( bHaveMinMax ) { if ( max < dfCellMax ) { @@ -340,11 +306,7 @@ GRASSRasterBand::GRASSRasterBand( GRASSDataset *poDSIn, int nBandIn, int nRed, nGreen, nBlue; GDALColorEntry sColor; -#if GRASS_VERSION_MAJOR >= 7 if( Rast_get_c_color( &iColor, &nRed, &nGreen, &nBlue, &sGrassColors ) ) -#else - if( G_get_color( iColor, &nRed, &nGreen, &nBlue, &sGrassColors ) ) -#endif { sColor.c1 = nRed; sColor.c2 = nGreen; @@ -366,7 +328,7 @@ GRASSRasterBand::GRASSRasterBand( GRASSDataset *poDSIn, int nBandIn, /* Create metadata entries for color table rules */ char key[200], value[200]; - int rcount = G_colors_count ( &sGrassColors ); + int rcount = Rast_colors_count ( &sGrassColors ); snprintf ( value, sizeof(value), "%d", rcount ); this->SetMetadataItem( "COLOR_TABLE_RULES_COUNT", value ); @@ -376,7 +338,7 @@ GRASSRasterBand::GRASSRasterBand( GRASSDataset *poDSIn, int nBandIn, DCELL val1, val2; unsigned char r1, g1, b1, r2, g2, b2; - G_get_f_color_rule ( &val1, &r1, &g1, &b1, &val2, &r2, &g2, &b2, &sGrassColors, i ); + Rast_get_fp_color_rule ( &val1, &r1, &g1, &b1, &val2, &r2, &g2, &b2, &sGrassColors, i ); snprintf ( key, sizeof(key), "COLOR_TABLE_RULE_RGB_%d", rcount-i-1 ); snprintf ( value, sizeof(value), "%e %e %d %d %d %d %d %d", val1, val2, r1, g1, b1, r2, g2, b2 ); @@ -396,12 +358,12 @@ GRASSRasterBand::GRASSRasterBand( GRASSDataset *poDSIn, int nBandIn, GRASSRasterBand::~GRASSRasterBand() { if( poCT != NULL ) { - G_free_colors( &sGrassColors ); + Rast_free_colors( &sGrassColors ); delete poCT; } if( hCell >= 0 ) - G_close_cell( hCell ); + Rast_close( hCell ); if ( pszCellName ) G_free ( pszCellName ); @@ -422,17 +384,17 @@ GRASSRasterBand::~GRASSRasterBand() void GRASSRasterBand::SetWindow ( struct Cell_head *sNewWindow ) { if( hCell >= 0 ) { - G_close_cell( hCell ); + Rast_close( hCell ); hCell = -1; } /* Set window */ - G_set_window( sNewWindow ); + Rast_set_window( sNewWindow ); /* Set GRASS env to the current raster, don't open the raster */ - G__setenv( "GISDBASE", ((GRASSDataset *)poDS)->pszGisdbase ); - G__setenv( "LOCATION_NAME", ((GRASSDataset *)poDS)->pszLocation ); - G__setenv( "MAPSET", pszMapset); + G_setenv_nogisrc( "GISDBASE", ((GRASSDataset *)poDS)->pszGisdbase ); + G_setenv_nogisrc( "LOCATION_NAME", ((GRASSDataset *)poDS)->pszLocation ); + G_setenv_nogisrc( "MAPSET", pszMapset); G_reset_mapsets(); G_add_mapset_to_search_path ( pszMapset ); } @@ -455,14 +417,14 @@ CPLErr GRASSRasterBand::ResetReading ( struct Cell_head *sNewWindow ) sNewWindow->rows != sOpenWindow.rows || sNewWindow->cols != sOpenWindow.cols ) { SetWindow ( sNewWindow ); - G_copy((void *) &sOpenWindow, (void *) sNewWindow, sizeof(struct Cell_head)); + memcpy((void *) &sOpenWindow, (void *) sNewWindow, sizeof(struct Cell_head)); } else { /* The windows are identical, check current window */ struct Cell_head sCurrentWindow; - G_get_window ( &sCurrentWindow ); + Rast_get_window ( &sCurrentWindow ); if ( sNewWindow->north != sCurrentWindow.north || sNewWindow->south != sCurrentWindow.south || sNewWindow->east != sCurrentWindow.east || sNewWindow->west != sCurrentWindow.west || @@ -494,19 +456,19 @@ CPLErr GRASSRasterBand::IReadBlock( int /*nBlockXOff*/, int nBlockYOff, } // open for reading if (hCell < 0) { - if ( (hCell = G_open_cell_old((char *) pszCellName, (char *) pszMapset)) < 0 ) { + if ( (hCell = Rast_open_old((char *) pszCellName, (char *) pszMapset)) < 0 ) { CPLError( CE_Failure, CPLE_AppDefined, "GRASS: Cannot open raster '%s'", pszCellName ); return CE_Failure; } } if ( eDataType == GDT_Byte || eDataType == GDT_UInt16 ) { - CELL *cbuf = G_allocate_c_raster_buf(); - G_get_c_raster_row ( hCell, cbuf, nBlockYOff ); + CELL *cbuf = Rast_allocate_c_buf(); + Rast_get_c_row ( hCell, cbuf, nBlockYOff ); /* Reset NULLs */ for ( int col = 0; col < nBlockXSize; col++ ) { - if ( G_is_c_null_value(&(cbuf[col])) ) + if ( Rast_is_c_null_value(&(cbuf[col])) ) cbuf[col] = (CELL) dfNoData; } @@ -518,19 +480,19 @@ CPLErr GRASSRasterBand::IReadBlock( int /*nBlockXOff*/, int nBlockYOff, } else if ( eDataType == GDT_Int32 ) { - G_get_c_raster_row ( hCell, (CELL *) pImage, nBlockYOff ); + Rast_get_c_row ( hCell, (CELL *) pImage, nBlockYOff ); } else if ( eDataType == GDT_Float32 ) { - G_get_f_raster_row ( hCell, (FCELL *) pImage, nBlockYOff ); + Rast_get_f_row ( hCell, (FCELL *) pImage, nBlockYOff ); } else if ( eDataType == GDT_Float64 ) { - G_get_d_raster_row ( hCell, (DCELL *) pImage, nBlockYOff ); + Rast_get_d_row ( hCell, (DCELL *) pImage, nBlockYOff ); } // close to avoid confusion with other GRASS raster bands - G_close_cell( hCell ); + Rast_close( hCell ); hCell = -1; return CE_None; @@ -580,7 +542,7 @@ CPLErr GRASSRasterBand::IRasterIO ( GDALRWFlag eRWFlag, } // open for reading if (hCell < 0) { - if ( (hCell = G_open_cell_old((char *) pszCellName, (char *) pszMapset)) < 0 ) { + if ( (hCell = Rast_open_old((char *) pszCellName, (char *) pszMapset)) < 0 ) { CPLError( CE_Failure, CPLE_AppDefined, "GRASS: Cannot open raster '%s'", pszCellName ); return CE_Failure; } @@ -602,11 +564,11 @@ CPLErr GRASSRasterBand::IRasterIO ( GDALRWFlag eRWFlag, if ( nGRSType == CELL_TYPE && ( !nativeNulls || eBufType != GDT_Int32 || sizeof(CELL) != 4 || nPixelSpace != sizeof(CELL) ) ) { - cbuf = G_allocate_c_raster_buf(); + cbuf = Rast_allocate_c_buf(); } else if( nGRSType == FCELL_TYPE && ( eBufType != GDT_Float32 || nPixelSpace != sizeof(FCELL) ) ) { - fbuf = G_allocate_f_raster_buf(); + fbuf = Rast_allocate_f_buf(); } else if( nGRSType == DCELL_TYPE && ( eBufType != GDT_Float64 || nPixelSpace != sizeof(DCELL) ) ) { - dbuf = G_allocate_d_raster_buf(); + dbuf = Rast_allocate_d_buf(); } else { direct = true; } @@ -616,13 +578,13 @@ CPLErr GRASSRasterBand::IRasterIO ( GDALRWFlag eRWFlag, if ( nGRSType == CELL_TYPE ) { if ( direct ) { - G_get_c_raster_row ( hCell, (CELL *) pnt, row ); + Rast_get_c_row ( hCell, (CELL *) pnt, row ); } else { - G_get_c_raster_row ( hCell, cbuf, row ); + Rast_get_c_row ( hCell, cbuf, row ); /* Reset NULLs */ for ( int col = 0; col < nBufXSize; col++ ) { - if ( G_is_c_null_value(&(cbuf[col])) ) + if ( Rast_is_c_null_value(&(cbuf[col])) ) cbuf[col] = (CELL) dfNoData; } @@ -632,9 +594,9 @@ CPLErr GRASSRasterBand::IRasterIO ( GDALRWFlag eRWFlag, } } else if( nGRSType == FCELL_TYPE ) { if ( direct ) { - G_get_f_raster_row ( hCell, (FCELL *) pnt, row ); + Rast_get_f_row ( hCell, (FCELL *) pnt, row ); } else { - G_get_f_raster_row ( hCell, fbuf, row ); + Rast_get_f_row ( hCell, fbuf, row ); GDALCopyWords ( (void *) fbuf, GDT_Float32, sizeof(FCELL), (void *) pnt, eBufType, nPixelSpace, @@ -642,9 +604,9 @@ CPLErr GRASSRasterBand::IRasterIO ( GDALRWFlag eRWFlag, } } else if( nGRSType == DCELL_TYPE ) { if ( direct ) { - G_get_d_raster_row ( hCell, (DCELL *) pnt, row ); + Rast_get_d_row ( hCell, (DCELL *) pnt, row ); } else { - G_get_d_raster_row ( hCell, dbuf, row ); + Rast_get_d_row ( hCell, dbuf, row ); GDALCopyWords ( (void *) dbuf, GDT_Float64, sizeof(DCELL), (void *) pnt, eBufType, nPixelSpace, @@ -658,7 +620,7 @@ CPLErr GRASSRasterBand::IRasterIO ( GDALRWFlag eRWFlag, if ( dbuf ) G_free ( dbuf ); // close to avoid confusion with other GRASS raster bands - G_close_cell( hCell ); + Rast_close( hCell ); hCell = -1; return CE_None; @@ -866,11 +828,7 @@ bool GRASSDataset::SplitPath( char *path, char **gisdbase, char **location, /* Open() */ /************************************************************************/ -#if (GRASS_VERSION_MAJOR >= 6 && GRASS_VERSION_MINOR >= 3) || GRASS_VERSION_MAJOR >= 7 typedef int (*GrassErrorHandler)(const char *, int); -#else -typedef int (*GrassErrorHandler)(char *, int); -#endif GDALDataset *GRASSDataset::Open( GDALOpenInfo * poOpenInfo ) @@ -935,9 +893,9 @@ GDALDataset *GRASSDataset::Open( GDALOpenInfo * poOpenInfo ) /* Set GRASS variables */ /* -------------------------------------------------------------------- */ - G__setenv( "GISDBASE", pszGisdb ); - G__setenv( "LOCATION_NAME", pszLoc ); - G__setenv( "MAPSET", pszMapset); // group is searched only in current mapset + G_setenv_nogisrc( "GISDBASE", pszGisdb ); + G_setenv_nogisrc( "LOCATION_NAME", pszLoc ); + G_setenv_nogisrc( "MAPSET", pszMapset); // group is searched only in current mapset G_reset_mapsets(); G_add_mapset_to_search_path ( pszMapset ); @@ -995,15 +953,7 @@ GDALDataset *GRASSDataset::Open( GDALOpenInfo * poOpenInfo ) /* Capture some information from the file that is of interest. */ /* -------------------------------------------------------------------- */ -#if GRASS_VERSION_MAJOR >= 7 Rast_get_cellhd( papszCells[0], papszMapsets[0], &(poDS->sCellInfo) ); -#else - if( G_get_cellhd( papszCells[0], papszMapsets[0], &(poDS->sCellInfo) ) != 0 ) { - CPLError( CE_Warning, CPLE_AppDefined, "GRASS: Cannot open raster header"); - delete poDS; - return NULL; - } -#endif poDS->nRasterXSize = poDS->sCellInfo.cols; poDS->nRasterYSize = poDS->sCellInfo.rows; @@ -1068,7 +1018,7 @@ GDALDataset *GRASSDataset::Open( GDALOpenInfo * poOpenInfo ) void GDALRegister_GRASS() { - if( !GDAL_CHECK_VERSION( "GDAL/GRASS57 driver" ) ) + if( !GDAL_CHECK_VERSION( "GDAL/GRASS driver" ) ) return; if( GDALGetDriverByName( "GRASS" ) != NULL ) @@ -1078,7 +1028,7 @@ void GDALRegister_GRASS() poDriver->SetDescription( "GRASS" ); poDriver->SetMetadataItem( GDAL_DCAP_RASTER, "YES" ); - poDriver->SetMetadataItem( GDAL_DMD_LONGNAME, "GRASS Rasters (5.7+)" ); + poDriver->SetMetadataItem( GDAL_DMD_LONGNAME, "GRASS Rasters (7+)" ); poDriver->SetMetadataItem( GDAL_DMD_HELPTOPIC, "drivers/raster/grass.html" ); poDriver->pfnOpen = GRASSDataset::Open; diff --git a/gdal/frmts/grass/grassdataset.cpp b/gdal/frmts/grass/grassdataset.cpp deleted file mode 100644 index 33a0fc7d63a0..000000000000 --- a/gdal/frmts/grass/grassdataset.cpp +++ /dev/null @@ -1,593 +0,0 @@ -/****************************************************************************** - * - * Project: GRASS Driver - * Purpose: Implement GRASS raster read/write support - * Author: Frank Warmerdam, warmerda@home.com - * - ****************************************************************************** - * Copyright (c) 2000, Frank Warmerdam - * Copyright (c) 2007-2009, Even Rouault - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - ****************************************************************************/ - -#include - -#include "cpl_string.h" -#include "gdal_frmts.h" -#include "gdal_priv.h" -#include "ogr_spatialref.h" - -CPL_CVSID("$Id$") - -/************************************************************************/ -/* Grass2CPLErrorHook() */ -/************************************************************************/ - -int Grass2CPLErrorHook( char * pszMessage, int bFatal ) - -{ - if( !bFatal ) - CPLDebug( "libgrass", "%s", pszMessage ); - else - CPLError( CE_Fatal, CPLE_AppDefined, "libgrass: %s", pszMessage ); - - return 0; -} - -/************************************************************************/ -/* ==================================================================== */ -/* GRASSDataset */ -/* ==================================================================== */ -/************************************************************************/ - -class GRASSRasterBand; - -class GRASSDataset final: public GDALDataset -{ - friend class GRASSRasterBand; - - char *pszProjection; - - double adfGeoTransform[6]; - - public: - GRASSDataset(); - ~GRASSDataset() override; - - const char *_GetProjectionRef() override; - const OGRSpatialReference* GetSpatialRef() const override { - return GetSpatialRefFromOldGetProjectionRef(); - } - CPLErr GetGeoTransform( double * ) override; - - static GDALDataset *Open( GDALOpenInfo * ); -}; - -/************************************************************************/ -/* ==================================================================== */ -/* GRASSRasterBand */ -/* ==================================================================== */ -/************************************************************************/ - -class GRASSRasterBand final: public GDALRasterBand -{ - friend class GRASSDataset; - - int hCell; - int nGRSType; - - GDALColorTable *poCT; - - int bHaveMinMax; - double dfCellMin; - double dfCellMax; - - double dfNoData; - - public: - GRASSRasterBand( GRASSDataset *, int, const char *, const char * ); - ~GRASSRasterBand() override; - - CPLErr IReadBlock( int, int, void * ) override; - GDALColorInterp GetColorInterpretation() override; - GDALColorTable *GetColorTable() override; - double GetMinimum( int *pbSuccess = NULL ) override; - double GetMaximum( int *pbSuccess = NULL ) override; - double GetNoDataValue( int *pbSuccess = NULL ) override; -}; - -/************************************************************************/ -/* GRASSRasterBand() */ -/************************************************************************/ - -GRASSRasterBand::GRASSRasterBand( GRASSDataset *poDS, int nBand, - const char * pszMapset, - const char * pszCellName ) - -{ - struct Cell_head sCellInfo; - - this->poDS = poDS; - this->nBand = nBand; - - G_get_cellhd( (char *) pszCellName, (char *) pszMapset, &sCellInfo ); - nGRSType = G_raster_map_type( (char *) pszCellName, (char *) pszMapset ); - -/* -------------------------------------------------------------------- */ -/* Get min/max values. */ -/* -------------------------------------------------------------------- */ - struct FPRange sRange; - - if( G_read_fp_range( (char *) pszCellName, (char *) pszMapset, - &sRange ) == -1 ) - { - bHaveMinMax = FALSE; - } - else - { - bHaveMinMax = TRUE; - G_get_fp_range_min_max( &sRange, &dfCellMin, &dfCellMax ); - } - -/* -------------------------------------------------------------------- */ -/* Setup band type, and preferred nodata value. */ -/* -------------------------------------------------------------------- */ - dfNoData = 0.0; - if( nGRSType == CELL_TYPE && sCellInfo.format == 0 ) - { - if( bHaveMinMax && dfCellMin < 1.0 && dfCellMax > 254.0 ) - { - this->eDataType = GDT_UInt16; - dfNoData = 256.0; - } - else - { - this->eDataType = GDT_Byte; - if( dfCellMax < 255.0 ) - dfNoData = 255.0; - else - dfNoData = 0.0; - } - } - else if( nGRSType == CELL_TYPE && sCellInfo.format == 1 ) - { - this->eDataType = GDT_UInt16; - dfNoData = 65535.0; - } - else if( nGRSType == CELL_TYPE ) - { - this->eDataType = GDT_UInt32; - dfNoData = 65535.0; - } - else if( nGRSType == FCELL_TYPE ) - { - this->eDataType = GDT_Float32; - dfNoData = -12345.0; - } - else if( nGRSType == DCELL_TYPE ) - { - this->eDataType = GDT_Float64; - dfNoData = -12345.0; - } - - nBlockXSize = poDS->nRasterXSize; - nBlockYSize = 1; - - hCell = G_open_cell_old((char *) pszCellName, (char *) pszMapset); - -/* -------------------------------------------------------------------- */ -/* Do we have a color table? */ -/* -------------------------------------------------------------------- */ - struct Colors sGrassColors; - - poCT = NULL; - if( G_read_colors( (char *) pszCellName, (char *) pszMapset, - &sGrassColors ) == 1 ) - { - poCT = new GDALColorTable(); - for( int iColor = 0; iColor < 256; iColor++ ) - { - int nRed, nGreen, nBlue; - GDALColorEntry sColor; - - if( G_get_color( iColor, &nRed, &nGreen, &nBlue, &sGrassColors ) ) - { - sColor.c1 = nRed; - sColor.c2 = nGreen; - sColor.c3 = nBlue; - sColor.c4 = 255; - - poCT->SetColorEntry( iColor, &sColor ); - } - else - { - sColor.c1 = 0; - sColor.c2 = 0; - sColor.c3 = 0; - sColor.c4 = 0; - - poCT->SetColorEntry( iColor, &sColor ); - } - } - - G_free_colors( &sGrassColors ); - } -} - -/************************************************************************/ -/* ~GRASSRasterBand() */ -/************************************************************************/ - -GRASSRasterBand::~GRASSRasterBand() - -{ - if( poCT != NULL ) - delete poCT; - - if( hCell >= 0 ) - G_close_cell( hCell ); -} - -/************************************************************************/ -/* IReadBlock() */ -/* */ -/* We only do "null" testing for floating point values. We */ -/* assume integer values are having the null raster entries set */ -/* to zero which is the "nodata" value for integer layers. */ -/************************************************************************/ - -CPLErr GRASSRasterBand::IReadBlock( int nBlockXOff, int nBlockYOff, - void * pImage ) - -{ - char *pachNullBuf = (char *) CPLMalloc(nBlockXSize); - G_get_null_value_row( hCell, pachNullBuf, nBlockYOff ); - - if( eDataType == GDT_Float32 || eDataType == GDT_Float64 - || eDataType == GDT_UInt32 ) - { - G_get_raster_row( hCell, pImage, nBlockYOff, nGRSType ); - - for( int i = 0; i < nBlockXSize; i++ ) - { - if( pachNullBuf[i] != 0 ) - { - if( eDataType == GDT_UInt32 ) - ((GUInt32 *) pImage)[i] = (GUInt32) dfNoData; - else if( eDataType == GDT_Float32 ) - ((float *) pImage)[i] = dfNoData; - else - ((double *) pImage)[i] = dfNoData; - } - } - } - else - { - GUInt32 *panRow = (GUInt32 *) CPLMalloc(4 * nBlockXSize); - - G_get_raster_row( hCell, panRow, nBlockYOff, nGRSType ); - - for( int i = 0; i < nBlockXSize; i++ ) - { - if( pachNullBuf[i] != 0 ) - panRow[i] = (GUInt32) dfNoData; - } - - GDALCopyWords( panRow, GDT_UInt32, 4, - pImage, eDataType, GDALGetDataTypeSize(eDataType)/8, - nBlockXSize ); - - CPLFree( panRow ); - } - - CPLFree( pachNullBuf ); - - return CE_None; -} - -/************************************************************************/ -/* GetColorInterpretation() */ -/************************************************************************/ - -GDALColorInterp GRASSRasterBand::GetColorInterpretation() - -{ - if( poCT != NULL ) - return GCI_PaletteIndex; - else - return GCI_GrayIndex; -} - -/************************************************************************/ -/* GetColorTable() */ -/************************************************************************/ - -GDALColorTable *GRASSRasterBand::GetColorTable() - -{ - return poCT; -} - -/************************************************************************/ -/* GetMinimum() */ -/************************************************************************/ - -double GRASSRasterBand::GetMinimum( int *pbSuccess ) - -{ - if( pbSuccess ) - *pbSuccess = bHaveMinMax; - - if( bHaveMinMax ) - return dfCellMin; - - else if( eDataType == GDT_Float32 || eDataType == GDT_Float64 ) - return -4294967295.0; - else - return 0; -} - -/************************************************************************/ -/* GetMaximum() */ -/************************************************************************/ - -double GRASSRasterBand::GetMaximum( int *pbSuccess ) - -{ - if( pbSuccess ) - *pbSuccess = bHaveMinMax; - - if( bHaveMinMax ) - return dfCellMax; - - else if( eDataType == GDT_Float32 || eDataType == GDT_Float64 ) - return 4294967295.0; - else if( eDataType == GDT_UInt32 ) - return 4294967295.0; - else if( eDataType == GDT_UInt16 ) - return 65535; - else - return 255; -} - -/************************************************************************/ -/* GetNoDataValue() */ -/************************************************************************/ - -double GRASSRasterBand::GetNoDataValue( int *pbSuccess ) - -{ - if( pbSuccess ) - *pbSuccess = TRUE; - - return dfNoData; -} - -/************************************************************************/ -/* ==================================================================== */ -/* GRASSDataset */ -/* ==================================================================== */ -/************************************************************************/ - -/************************************************************************/ -/* GRASSDataset() */ -/************************************************************************/ - -GRASSDataset::GRASSDataset() - -{ - pszProjection = NULL; - - adfGeoTransform[0] = 0.0; - adfGeoTransform[1] = 1.0; - adfGeoTransform[2] = 0.0; - adfGeoTransform[3] = 0.0; - adfGeoTransform[4] = 0.0; - adfGeoTransform[5] = 1.0; -} - -/************************************************************************/ -/* ~GRASSDataset() */ -/************************************************************************/ - -GRASSDataset::~GRASSDataset() - -{ - CPLFree( pszProjection ); -} - -/************************************************************************/ -/* GetProjectionRef() */ -/************************************************************************/ - -const char *GRASSDataset::_GetProjectionRef() -{ - if( pszProjection == NULL ) - return ""; - else - return pszProjection; -} - -/************************************************************************/ -/* GetGeoTransform() */ -/************************************************************************/ - -CPLErr GRASSDataset::GetGeoTransform( double * padfGeoTransform ) -{ - memcpy( padfGeoTransform, adfGeoTransform, sizeof(double) * 6 ); - - return CE_None; -} - -/************************************************************************/ -/* Open() */ -/************************************************************************/ - -typedef int (*GrassErrorHandler)(); - -GDALDataset *GRASSDataset::Open( GDALOpenInfo * poOpenInfo ) - -{ - static int bDoneGISInit = FALSE; - char *pszMapset = NULL, *pszCell = NULL; - char **papszCells = NULL; - char **papszMapsets = NULL; - - if( !bDoneGISInit ) - { - bDoneGISInit = TRUE; - G_set_error_routine( (GrassErrorHandler) Grass2CPLErrorHook ); - G_gisinit_2( "GDAL", NULL, NULL, NULL ); - } - -/* -------------------------------------------------------------------- */ -/* Check if this is a valid grass cell. */ -/* -------------------------------------------------------------------- */ - if( G_check_cell( poOpenInfo->pszFilename, &pszMapset, &pszCell ) ) - { - papszCells = CSLAddString( papszCells, pszCell ); - papszMapsets = CSLAddString( papszMapsets, pszMapset ); - - G_free( pszMapset ); - G_free( pszCell ); - } - -/* -------------------------------------------------------------------- */ -/* Check if this is a valid GRASS imagery group. */ -/* -------------------------------------------------------------------- */ - else if( I_check_group( poOpenInfo->pszFilename, &pszMapset, &pszCell ) ) - { - struct Ref ref; - - I_init_group_ref( &ref ); - I_get_group_ref( pszCell, &ref ); - - for( int iRef = 0; iRef < ref.nfiles; iRef++ ) - { - papszCells = CSLAddString( papszCells, ref.file[iRef].name ); - papszMapsets = CSLAddString( papszMapsets, ref.file[iRef].mapset ); - } - - I_free_group_ref( &ref ); - - G_free( pszMapset ); - G_free( pszCell ); - } - - else - return NULL; - -/* -------------------------------------------------------------------- */ -/* Create a corresponding GDALDataset. */ -/* -------------------------------------------------------------------- */ - GRASSDataset *poDS = new GRASSDataset(); - - /* notdef: should only allow read access to an existing cell, right? */ - poDS->eAccess = poOpenInfo->eAccess; - -/* -------------------------------------------------------------------- */ -/* Capture some information from the file that is of interest. */ -/* -------------------------------------------------------------------- */ - struct Cell_head sCellInfo; - - if( G_get_cellhd( papszCells[0], papszMapsets[0], &sCellInfo ) != 0 ) - { - /* notdef: report failure. */ - return NULL; - } - - poDS->nRasterXSize = sCellInfo.cols; - poDS->nRasterYSize = sCellInfo.rows; - - G_set_window( &sCellInfo ); - - poDS->adfGeoTransform[0] = sCellInfo.west; - poDS->adfGeoTransform[1] = sCellInfo.ew_res; - poDS->adfGeoTransform[2] = 0.0; - poDS->adfGeoTransform[3] = sCellInfo.north; - poDS->adfGeoTransform[4] = 0.0; - poDS->adfGeoTransform[5] = -1 * sCellInfo.ns_res; - -/* -------------------------------------------------------------------- */ -/* Try to get a projection definition. */ -/* -------------------------------------------------------------------- */ - char *pszProj4 = G_get_cell_as_proj4( papszCells[0], papszMapsets[0] ); - if( pszProj4 != NULL ) - { - OGRSpatialReference oSRS; - - if( oSRS.importFromProj4( pszProj4 ) == OGRERR_NONE ) - { - oSRS.exportToWkt( &(poDS->pszProjection) ); - } - - G_free( pszProj4 ); - } - -/* -------------------------------------------------------------------- */ -/* Create band information objects. */ -/* -------------------------------------------------------------------- */ - for( int iBand = 0; papszCells[iBand] != NULL; iBand++ ) - { - poDS->SetBand( iBand+1, - new GRASSRasterBand( poDS, iBand+1, - papszMapsets[iBand], - papszCells[iBand] ) ); - } - -/* -------------------------------------------------------------------- */ -/* Confirm the requested access is supported. */ -/* -------------------------------------------------------------------- */ - if( poOpenInfo->eAccess == GA_Update ) - { - delete poDS; - CPLError( CE_Failure, CPLE_NotSupported, - "The GRASS driver does not support update access to existing" - " datasets.\n" ); - return NULL; - } - - return poDS; -} - -/************************************************************************/ -/* GDALRegister_GRASS() */ -/************************************************************************/ - -void GDALRegister_GRASS() - -{ - if( !GDAL_CHECK_VERSION( "GDAL/GRASS driver" ) ) - return; - - if( GDALGetDriverByName( "GRASS" ) != NULL ) - return; - - GDALDriver *poDriver = new GDALDriver(); - - poDriver->SetDescription( "GRASS" ); - poDriver->SetMetadataItem( GDAL_DCAP_RASTER, "YES" ); - poDriver->SetMetadataItem( GDAL_DMD_LONGNAME, "GRASS Database Rasters" ); - poDriver->SetMetadataItem( GDAL_DMD_HELPTOPIC, "drivers/raster/grass.html" ); - - poDriver->pfnOpen = GRASSDataset::Open; - - GetGDALDriverManager()->RegisterDriver( poDriver ); -} diff --git a/gdal/frmts/grass/pkg/Makefile.in b/gdal/frmts/grass/pkg/Makefile.in index 568d554d9b81..28683b2dcf55 100644 --- a/gdal/frmts/grass/pkg/Makefile.in +++ b/gdal/frmts/grass/pkg/Makefile.in @@ -33,8 +33,8 @@ distclean: clean rm -fr Makefile config.status config.log autom*.cache -$(GLIBNAME): grass57dataset.o - $(LD_SHARED) $(LDFLAGS) grass57dataset.o $(LIBS) -o $(GLIBNAME) +$(GLIBNAME): grass.o + $(LD_SHARED) $(LDFLAGS) grass.o $(LIBS) -o $(GLIBNAME) $(OLIBNAME): ogrgrassdriver.o ogrgrassdatasource.o ogrgrasslayer.o $(LD_SHARED) $(LDFLAGS) ogrgrassdriver.o ogrgrassdatasource.o ogrgrasslayer.o $(LIBS) -o $(OLIBNAME) diff --git a/gdal/frmts/grass/pkg/README b/gdal/frmts/grass/pkg/README index dfd9f44a8cb5..e897bf378f01 100644 --- a/gdal/frmts/grass/pkg/README +++ b/gdal/frmts/grass/pkg/README @@ -13,14 +13,14 @@ build and install GRASS normally and finally build and install this driver. To build this driver it is necessary for it to find GDAL and GRASS support files. Typically the configure and build process would look something like: -./configure --with-gdal=/usr/local/bin/gdal-config --with-grass=/usr/local/grass-7.0.0 +./configure --with-gdal=/usr/local/bin/gdal-config --with-grass=/usr/local/grass-7.0.1 make sudo make install See also: - http://www.gdal.org/ - http://grass.osgeo.org + https://gdal.org/ + https://grass.osgeo.org --- @@ -31,22 +31,22 @@ FAQs Question: -I am trying to install gdal-grass 1.3.1 on Red hat enterprise linux -advanced server 3.0. I have previously installed gdal 1.3.1 without- -grass, and Grass 6.0.1 with-gdal. I have tried to configure gdal-grass +I am trying to install gdal-grass 3.1.3 on RedHat Enterprise Linux +advanced server 3.0. I have previously installed gdal 3.1.3 --without-grass, +and GRASS 7.0.1 with GDAL. I have tried to configure gdal-grass with: -./configure --with-gdal=/usr/local/gdal/bin/gdal-config --with- -grass=/usr/local/grass-6.0.1 +./configure --with-gdal=/usr/local/bin/gdal-config \ + --with-grass=/usr/local/grass-7.0.1 It seems to find gdal alright, but then balks at the Grass location. The Grass location specified above is indeed the correct location. I have -also tried adding --with-grass=/usr/local/grass-6.0.1/lib, but with no +also tried adding --with-grass=/usr/local/grass-7.0.1/lib, but with no success. My error is: ... checking for G_asprintf in -lgrass_gis ... no -configure: error: --with-grass=/usr/local/grass-6.0.1 requested, but +configure: error: --with-grass=/usr/local/grass-7.0.1 requested, but libraries not found? @@ -55,8 +55,8 @@ Answer: Your problem is likely to be solved by editing /etc/ld.so.conf to include the locations of proj, gdal, grass, and geos. Specifically, the full path to both gdal-config and geos-config, and the full paths -to the library locations of proj (often /usr/local/lib) and grass (/ -usr/local/grass-6.0.1/lib). After editing ld.so.conf, run ldconfig, +to the library locations of proj (often /usr/local/lib) and grass +(/usr/local/grass-7.0.1/lib). After editing ld.so.conf, run ldconfig, and you should be good to go. I ran into this problem this weekend (and posted for help to this diff --git a/gdal/ogr/ogrsf_frmts/grass/ogrgrass.h b/gdal/ogr/ogrsf_frmts/grass/ogrgrass.h index 5dd16e73ef64..26873638f891 100644 --- a/gdal/ogr/ogrsf_frmts/grass/ogrgrass.h +++ b/gdal/ogr/ogrsf_frmts/grass/ogrgrass.h @@ -37,11 +37,7 @@ extern "C" { #include #include #include -#if GRASS_VERSION_MAJOR >= 7 #include -#else - #include -#endif } /************************************************************************/ diff --git a/gdal/ogr/ogrsf_frmts/grass/ogrgrassdatasource.cpp b/gdal/ogr/ogrsf_frmts/grass/ogrgrassdatasource.cpp index d77b14990dc3..e7b480cefcdf 100644 --- a/gdal/ogr/ogrsf_frmts/grass/ogrgrassdatasource.cpp +++ b/gdal/ogr/ogrsf_frmts/grass/ogrgrassdatasource.cpp @@ -6,7 +6,7 @@ * ****************************************************************************** * Copyright (c) 2005, Radim Blazek - * Copyright (c) 2008-2010, Even Rouault + * Copyright (c) 2008-2020, Even Rouault * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -33,9 +33,6 @@ CPL_CVSID("$Id$") -#if GRASS_VERSION_MAJOR >= 7 -#define G__setenv G_setenv_nogisrc -#endif /************************************************************************/ /* Grass2CPLErrorHook() */ @@ -88,11 +85,7 @@ OGRGRASSDataSource::~OGRGRASSDataSource() /* Open() */ /************************************************************************/ -#if (GRASS_VERSION_MAJOR >= 6 && GRASS_VERSION_MINOR >= 3) || GRASS_VERSION_MAJOR >= 7 typedef int (*GrassErrorHandler)(const char *, int); -#else -typedef int (*GrassErrorHandler)(char *, int); -#endif int OGRGRASSDataSource::Open( const char * pszNewName, int /*bUpdate*/, int bTestOpen, int /*bSingleNewFileIn*/ ) @@ -183,18 +176,15 @@ int OGRGRASSDataSource::Open( const char * pszNewName, int /*bUpdate*/, /* -------------------------------------------------------------------- */ /* Set GRASS variables */ /* -------------------------------------------------------------------- */ - G__setenv( "GISDBASE", pszGisdbase ); - G__setenv( "LOCATION_NAME", pszLocation ); - G__setenv( "MAPSET", pszMapset); + G_setenv_nogisrc( "GISDBASE", pszGisdbase ); + G_setenv_nogisrc( "LOCATION_NAME", pszLocation ); + G_setenv_nogisrc( "MAPSET", pszMapset); G_reset_mapsets(); G_add_mapset_to_search_path ( pszMapset ); /* -------------------------------------------------------------------- */ /* Open GRASS vector map */ /* -------------------------------------------------------------------- */ -#if GRASS_VERSION_MAJOR < 7 - Vect_set_fatal_error ( GV_FATAL_PRINT ); // Print error and continue -#endif Vect_set_open_level (2); int level = Vect_open_old ( &map, pszMap, pszMapset); diff --git a/gdal/ogr/ogrsf_frmts/grass/ogrgrasslayer.cpp b/gdal/ogr/ogrsf_frmts/grass/ogrgrasslayer.cpp index 336af720b315..565b70337989 100644 --- a/gdal/ogr/ogrsf_frmts/grass/ogrgrasslayer.cpp +++ b/gdal/ogr/ogrsf_frmts/grass/ogrgrasslayer.cpp @@ -6,7 +6,7 @@ * ****************************************************************************** * Copyright (c) 2005, Radim Blazek - * Copyright (c) 2008-2010, Even Rouault + * Copyright (c) 2008-2020, Even Rouault * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -641,11 +641,7 @@ bool OGRGRASSLayer::SetSpatialMatch() Vect_cidx_get_cat_by_index ( poMap, iLayerIndex, cidx, &cat, &type, &id ); -#if GRASS_VERSION_MAJOR >= 7 - struct bound_box box; -#else - BOUND_BOX box; -#endif + struct bound_box box; switch ( type ) { @@ -1013,11 +1009,7 @@ GIntBig OGRGRASSLayer::GetFeatureCount( int bForce ) /************************************************************************/ OGRErr OGRGRASSLayer::GetExtent (OGREnvelope *psExtent, int /*bForce*/) { -#if GRASS_VERSION_MAJOR >= 7 struct bound_box box; -#else - BOUND_BOX box; -#endif Vect_get_map_box ( poMap, &box ); From 634081b8695f2dd4475df8ccb69569a0f55418b1 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 22 Sep 2020 17:18:06 +0200 Subject: [PATCH 060/255] PCIDSK: fix invalid-enum-value. Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=25719 --- gdal/frmts/pcidsk/sdk/core/pcidsk_pubutils.cpp | 2 +- gdal/frmts/pcidsk/sdk/pcidsk_types.h | 2 +- gdal/frmts/pcidsk/sdk/segment/cpcidsksegment.cpp | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/gdal/frmts/pcidsk/sdk/core/pcidsk_pubutils.cpp b/gdal/frmts/pcidsk/sdk/core/pcidsk_pubutils.cpp index 662456370970..c4f1c7a97346 100644 --- a/gdal/frmts/pcidsk/sdk/core/pcidsk_pubutils.cpp +++ b/gdal/frmts/pcidsk/sdk/core/pcidsk_pubutils.cpp @@ -192,7 +192,7 @@ bool PCIDSK::IsDataTypeComplex(eChanType type) * @return the string for the segment type. */ -std::string PCIDSK::SegmentTypeName( eSegType type ) +std::string PCIDSK::SegmentTypeName( int /* eSegType actually */ type ) { switch( type ) diff --git a/gdal/frmts/pcidsk/sdk/pcidsk_types.h b/gdal/frmts/pcidsk/sdk/pcidsk_types.h index 1f7d26303bbb..a4aa509d8d36 100644 --- a/gdal/frmts/pcidsk/sdk/pcidsk_types.h +++ b/gdal/frmts/pcidsk/sdk/pcidsk_types.h @@ -70,7 +70,7 @@ namespace PCIDSK // Helper functions for working with segments and data types int PCIDSK_DLL DataTypeSize( eChanType ); std::string PCIDSK_DLL DataTypeName( eChanType ); - std::string PCIDSK_DLL SegmentTypeName( eSegType ); + std::string PCIDSK_DLL SegmentTypeName( int /* eSegType actually */ ); eChanType PCIDSK_DLL GetDataTypeFromName(std::string const& type_name); bool PCIDSK_DLL IsDataTypeComplex(eChanType type); diff --git a/gdal/frmts/pcidsk/sdk/segment/cpcidsksegment.cpp b/gdal/frmts/pcidsk/sdk/segment/cpcidsksegment.cpp index d0cb38d35073..de56bb1f5685 100644 --- a/gdal/frmts/pcidsk/sdk/segment/cpcidsksegment.cpp +++ b/gdal/frmts/pcidsk/sdk/segment/cpcidsksegment.cpp @@ -106,7 +106,9 @@ void CPCIDSKSegment::LoadSegmentPointer( const char *segment_pointer ) PCIDSKBuffer segptr( segment_pointer, 32 ); segment_flag = segptr.buffer[0]; - segment_type = (eSegType) (atoi(segptr.Get(1,3))); + const int segment_type_int = atoi(segptr.Get(1,3)); + segment_type = SegmentTypeName(segment_type_int) == "UNKNOWN" ? + SEG_UNKNOWN : static_cast(segment_type_int); data_offset = atouint64(segptr.Get(12,11)); if( data_offset == 0 ) data_offset = 0; // throw exception maybe ? From f532ff9f1bab6c182960cfe47b4594deb3a381ea Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 22 Sep 2020 18:28:30 +0200 Subject: [PATCH 061/255] tif_fax3.h: return error when a buffer overflow occurs. Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=25552 and https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=25849 --- gdal/frmts/gtiff/libtiff/tif_fax3.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gdal/frmts/gtiff/libtiff/tif_fax3.h b/gdal/frmts/gtiff/libtiff/tif_fax3.h index 117619c56fa4..9d765b6b0f72 100644 --- a/gdal/frmts/gtiff/libtiff/tif_fax3.h +++ b/gdal/frmts/gtiff/libtiff/tif_fax3.h @@ -240,6 +240,11 @@ static const char* StateNames[] = { * current row and reset decoding state. */ #define SETVALUE(x) do { \ + if (pa >= thisrun + sp->nruns) { \ + TIFFErrorExt(tif->tif_clientdata, module, "Buffer overflow at line %u of %s %u", \ + sp->line, isTiled(tif) ? "tile" : "strip", isTiled(tif) ? tif->tif_curtile : tif->tif_curstrip); \ + return (-1); \ + } \ *pa++ = RunLength + (x); \ a0 += (x); \ RunLength = 0; \ From bf57a091cd8b6ea71ba5fef163ab4120ce0fce93 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 22 Sep 2020 18:46:55 +0200 Subject: [PATCH 062/255] PDS4 vector: fix potential double free if RenameFileTo() fails, and add asserts to perhaps help tracking https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=25535 --- gdal/frmts/pds/pds4vector.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gdal/frmts/pds/pds4vector.cpp b/gdal/frmts/pds/pds4vector.cpp index 19f774432c42..2fb08e6b884c 100644 --- a/gdal/frmts/pds/pds4vector.cpp +++ b/gdal/frmts/pds/pds4vector.cpp @@ -78,6 +78,7 @@ bool PDS4TableBaseLayer::RenameFileTo(const char* pszNewName) { if( m_fp ) VSIFCloseL(m_fp); + m_fp = nullptr; CPLString osBackup(pszNewName); osBackup += ".bak"; VSIRename(pszNewName, osBackup); @@ -1082,6 +1083,7 @@ static OGRFieldType GetFieldTypeFromPDS4DataType(const char* pszDataType, bool PDS4FixedWidthTable::ReadTableDef(const CPLXMLNode* psTable) { + CPLAssert( m_fp == nullptr ); m_fp = VSIFOpenL(m_osFilename, (m_poDS->GetAccess() == GA_ReadOnly ) ? "rb" : "r+b"); if( !m_fp ) @@ -1432,6 +1434,7 @@ bool PDS4FixedWidthTable::InitializeNewLayer( OGRwkbGeometryType eGType, const char* const* papszOptions) { + CPLAssert( m_fp == nullptr ); m_fp = VSIFOpenL(m_osFilename, "wb+"); if( !m_fp ) { @@ -2065,6 +2068,7 @@ OGRErr PDS4DelimitedTable::CreateField( OGRFieldDefn *poFieldIn, int ) bool PDS4DelimitedTable::ReadTableDef(const CPLXMLNode* psTable) { + CPLAssert( m_fp == nullptr ); m_fp = VSIFOpenL(m_osFilename, (m_poDS->GetAccess() == GA_ReadOnly ) ? "rb" : "r+b"); if( !m_fp ) @@ -2367,6 +2371,7 @@ bool PDS4DelimitedTable::InitializeNewLayer( OGRwkbGeometryType eGType, const char* const* papszOptions) { + CPLAssert( m_fp == nullptr ); m_fp = VSIFOpenL(m_osFilename, "wb+"); if( !m_fp ) { From 68a21c7a8286d5e71e3d417beae106a84b26953c Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 23 Sep 2020 13:59:13 +0200 Subject: [PATCH 063/255] gdalwarpkernel.cpp: avoid unlikely division by zero. CID 1432762 --- gdal/alg/gdalwarpkernel.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/gdal/alg/gdalwarpkernel.cpp b/gdal/alg/gdalwarpkernel.cpp index e93d061b516a..09457c1c705e 100644 --- a/gdal/alg/gdalwarpkernel.cpp +++ b/gdal/alg/gdalwarpkernel.cpp @@ -1142,13 +1142,15 @@ CPLErr GDALWarpKernel::PerformWarp() nCountPointScale ++; } } - - const double dfXScaleFromSampling = dfSumPointXScale / nCountPointScale; + if( nCountPointScale > 0 ) // should always be true + { + const double dfXScaleFromSampling = dfSumPointXScale / nCountPointScale; #if DEBUG_VERBOSE - CPLDebug("WARP", "Correcting dfXScale from %f to %f", - dfXScale, dfXScaleFromSampling); + CPLDebug("WARP", "Correcting dfXScale from %f to %f", + dfXScale, dfXScaleFromSampling); #endif - dfXScale = dfXScaleFromSampling; + dfXScale = dfXScaleFromSampling; + } } } From 1f9a476555fdf5cccdcdc62133e8ddc732f55fc8 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 23 Sep 2020 14:02:25 +0200 Subject: [PATCH 064/255] VSIS3FSHandler::SetFileMetadata(): fix memleak. CID 1432675 --- gdal/port/cpl_vsil_s3.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gdal/port/cpl_vsil_s3.cpp b/gdal/port/cpl_vsil_s3.cpp index 7796bb200890..8872596df4d3 100644 --- a/gdal/port/cpl_vsil_s3.cpp +++ b/gdal/port/cpl_vsil_s3.cpp @@ -2337,7 +2337,9 @@ bool VSIS3FSHandler::SetFileMetadata( const char * pszFilename, CPLFree(pszKey); } - osXML = CPLSerializeXMLTree(psXML); + char* pszXML = CPLSerializeXMLTree(psXML); + osXML = pszXML; + CPLFree(pszXML); CPLDestroyXMLNode(psXML); } From a9ee7ea8116fd33015cdcf75ac88cd74f42f83d2 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 23 Sep 2020 14:05:11 +0200 Subject: [PATCH 065/255] cpl_vsil_s3.cpp: avoid false positive warning about division by zero. CID 1432695 --- gdal/port/cpl_vsil_s3.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gdal/port/cpl_vsil_s3.cpp b/gdal/port/cpl_vsil_s3.cpp index 8872596df4d3..4d646f2f85b2 100644 --- a/gdal/port/cpl_vsil_s3.cpp +++ b/gdal/port/cpl_vsil_s3.cpp @@ -4121,8 +4121,9 @@ bool IVSIS3LikeFSHandler::Sync( const char* pszSource, const char* pszTarget, VSIFReadL(pBuffer, 1, nSizeToRead, fpIn) == nSizeToRead ) { queue->poFS->UpdateHandleFromMap(poS3HandleHelper.get()); - const int nPartNumber = 1 + static_cast( - chunk.nStartOffset / queue->nMaxChunkSize); + const int nPartNumber = 1 + + (queue->nMaxChunkSize == 0 ? 0 /* shouldn't happen */ : + static_cast(chunk.nStartOffset / queue->nMaxChunkSize)); const CPLString osEtag = queue->poFS->UploadPart( osSubTarget, nPartNumber, iter->second.osUploadID, From 445f08141d6744557b1cd658d3989a97db5b8f27 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 23 Sep 2020 14:07:18 +0200 Subject: [PATCH 066/255] Try to address CID 1432689 and 1432691 --- gdal/port/cpl_aws.cpp | 2 ++ gdal/port/cpl_swift.cpp | 1 + 2 files changed, 3 insertions(+) diff --git a/gdal/port/cpl_aws.cpp b/gdal/port/cpl_aws.cpp index 8987ee16fb50..cb2fcfce4c31 100644 --- a/gdal/port/cpl_aws.cpp +++ b/gdal/port/cpl_aws.cpp @@ -694,8 +694,10 @@ bool VSIS3HandleHelper::GetConfigurationFromEC2(CPLString& osSecretAccessKey, CPLString osURLRefreshCredentials; const CPLString osEC2DefaultURL("http://169.254.169.254"); + // coverity[tainted_data] const CPLString osEC2RootURL( CPLGetConfigOption("CPL_AWS_EC2_API_ROOT_URL", osEC2DefaultURL)); + // coverity[tainted_data] const CPLString osECSRelativeURI( CPLGetConfigOption("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "")); CPLString osToken; diff --git a/gdal/port/cpl_swift.cpp b/gdal/port/cpl_swift.cpp index f0d66e8e2200..3bb9d2b9bf01 100644 --- a/gdal/port/cpl_swift.cpp +++ b/gdal/port/cpl_swift.cpp @@ -315,6 +315,7 @@ bool VSISwiftHandleHelper::AuthV3(CPLString& osStorageURL, CPLJSONObject postObject(CreateAuthV3RequestObject()); std::string post = postObject.Format(CPLJSONObject::PrettyFormat::Plain); + // coverity[tainted_data] CPLString osAuthURL = CPLGetConfigOption("OS_AUTH_URL", ""); std::string url = osAuthURL; if( !url.empty() && url.back() != '/' ) From 5b4c61a5f257dffdf7a3efec555a843d89f963e7 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 23 Sep 2020 15:29:20 +0200 Subject: [PATCH 067/255] GTiff: add earlier check to bail out when attempting JPEG compression with paletted image --- autotest/gcore/tiff_write.py | 16 ++++++++++++++++ gdal/frmts/gtiff/geotiff.cpp | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/autotest/gcore/tiff_write.py b/autotest/gcore/tiff_write.py index 099330d22660..67e61f86d029 100755 --- a/autotest/gcore/tiff_write.py +++ b/autotest/gcore/tiff_write.py @@ -7195,6 +7195,22 @@ def test_tiff_write_too_many_tiles(): assert 'File too large regarding tile size' in gdal.GetLastErrorMsg() +############################################################################### +# + + +def test_tiff_write_jpeg_incompatible_of_paletted(): + + md = gdaltest.tiff_drv.GetMetadata() + if md['DMD_CREATIONOPTIONLIST'].find('JPEG') == -1: + pytest.skip() + + src_ds = gdal.Open('data/test_average_palette.tif') + with gdaltest.error_handler(): + assert not gdaltest.tiff_drv.CreateCopy('/vsimem/tmp.tif', src_ds, options = ['COMPRESS=JPEG']) + gdal.Unlink('/vsimem/tmp.tif') + + ############################################################################### diff --git a/gdal/frmts/gtiff/geotiff.cpp b/gdal/frmts/gtiff/geotiff.cpp index e4fe878ec89d..26c77611e4d1 100644 --- a/gdal/frmts/gtiff/geotiff.cpp +++ b/gdal/frmts/gtiff/geotiff.cpp @@ -17455,6 +17455,22 @@ GTiffDataset::CreateCopy( const char * pszFilename, GDALDataset *poSrcDS, "can only be written to 1 band or 2 bands Byte or " "UInt16 GeoTIFF files." ); + if( l_nCompression == COMPRESSION_JPEG ) + { + int l_nPhotometric = 0; + TIFFGetField(l_hTIFF, TIFFTAG_PHOTOMETRIC, &l_nPhotometric); + // Check done in tif_jpeg.c later, but not with a very clear error message + if( l_nPhotometric == PHOTOMETRIC_PALETTE ) + { + CPLError(CE_Failure, CPLE_NotSupported, + "JPEG compression not supported with paletted image"); + XTIFFClose( l_hTIFF ); + VSIUnlink(l_osTmpFilename); + CPL_IGNORE_RET_VAL( VSIFCloseL(l_fpL) ); + return nullptr; + } + } + if( l_nBands == 2 && poSrcDS->GetRasterBand(1)->GetColorTable() != nullptr && (eType == GDT_Byte || eType == GDT_UInt16) ) From 58cbf2754d803f8ecc552e6f74a24d4f9efa142a Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 23 Sep 2020 18:28:01 +0200 Subject: [PATCH 068/255] GTiff: fix previous commit for big-endian platforms --- gdal/frmts/gtiff/geotiff.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gdal/frmts/gtiff/geotiff.cpp b/gdal/frmts/gtiff/geotiff.cpp index 26c77611e4d1..433509cc1c7c 100644 --- a/gdal/frmts/gtiff/geotiff.cpp +++ b/gdal/frmts/gtiff/geotiff.cpp @@ -17457,7 +17457,7 @@ GTiffDataset::CreateCopy( const char * pszFilename, GDALDataset *poSrcDS, if( l_nCompression == COMPRESSION_JPEG ) { - int l_nPhotometric = 0; + uint16 l_nPhotometric = 0; TIFFGetField(l_hTIFF, TIFFTAG_PHOTOMETRIC, &l_nPhotometric); // Check done in tif_jpeg.c later, but not with a very clear error message if( l_nPhotometric == PHOTOMETRIC_PALETTE ) From 9e21b31c4d43ffe6cee869c0754c39b06d9a54d9 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 23 Sep 2020 23:02:00 +0200 Subject: [PATCH 069/255] GRIB: fix retrieval of nodata value for GRIB1 products (GDAL 3.1 regression, fixes #2962) --- autotest/gdrivers/grib.py | 9 ++++++++- gdal/frmts/grib/degrib/degrib/degrib1.cpp | 14 ++++++++------ gdal/frmts/grib/gribdataset.cpp | 20 +++++++++++++------- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/autotest/gdrivers/grib.py b/autotest/gdrivers/grib.py index 07ced92794cc..30905dfe4278 100755 --- a/autotest/gdrivers/grib.py +++ b/autotest/gdrivers/grib.py @@ -65,7 +65,11 @@ def test_grib_1(): def test_grib_2(): tst = gdaltest.GDALTest('GRIB', 'grib/Sample_QuikSCAT.grb', 4, 50714) - return tst.testOpen() + tst.testOpen() + + ds = gdal.Open('data/grib/Sample_QuikSCAT.grb') + assert ds.GetRasterBand(1).GetNoDataValue() == 9999.0 + assert ds.GetRasterBand(1).GetNoDataValue() == 9999.0 # do it again to test correct caching ############################################################################### # This file has different raster sizes for some of the products, which @@ -258,6 +262,9 @@ def test_grib_grib1_read_rotated_pole_lonlat(): assert ds.RasterXSize == 726 and ds.RasterYSize == 550, \ 'Did not get expected dimensions' + assert ds.GetRasterBand(1).GetNoDataValue() is None + assert ds.GetRasterBand(1).GetNoDataValue() is None # do it again to test correct caching + projection = ds.GetProjectionRef() expected_projection_proj_7 = 'GEOGCRS["Coordinate System imported from GRIB file",BASEGEOGCRS["Coordinate System imported from GRIB file",DATUM["unnamed",ELLIPSOID["Sphere",6367470,0,LENGTHUNIT["metre",1,ID["EPSG",9001]]]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433,ID["EPSG",9122]]]],DERIVINGCONVERSION["Pole rotation (GRIB convention)",METHOD["Pole rotation (GRIB convention)"],PARAMETER["Latitude of the southern pole (GRIB convention)",-30,ANGLEUNIT["degree",0.0174532925199433,ID["EPSG",9122]]],PARAMETER["Longitude of the southern pole (GRIB convention)",-15,ANGLEUNIT["degree",0.0174532925199433,ID["EPSG",9122]]],PARAMETER["Axis rotation (GRIB convention)",0,ANGLEUNIT["degree",0.0174532925199433,ID["EPSG",9122]]]],CS[ellipsoidal,2],AXIS["latitude",north,ORDER[1],ANGLEUNIT["degree",0.0174532925199433,ID["EPSG",9122]]],AXIS["longitude",east,ORDER[2],ANGLEUNIT["degree",0.0174532925199433,ID["EPSG",9122]]]]' expected_projection_before_proj_7 = 'PROJCS["unnamed",GEOGCS["Coordinate System imported from GRIB file",DATUM["unnamed",SPHEROID["Sphere",6367470,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]]],PROJECTION["Rotated_pole"],UNIT["metre",1,AUTHORITY["EPSG","9001"]],AXIS["Easting",EAST],AXIS["Northing",NORTH],EXTENSION["PROJ4","+proj=ob_tran +lon_0=-15 +o_proj=longlat +o_lon_p=0 +o_lat_p=30 +a=6367470 +b=6367470 +to_meter=0.0174532925199 +wktext"]]' diff --git a/gdal/frmts/grib/degrib/degrib/degrib1.cpp b/gdal/frmts/grib/degrib/degrib/degrib1.cpp index e2d3bcb5c46c..5a0c930f6076 100644 --- a/gdal/frmts/grib/degrib/degrib/degrib1.cpp +++ b/gdal/frmts/grib/degrib/degrib/degrib1.cpp @@ -1692,9 +1692,6 @@ static int ReadGrib1Sect4 (uChar *bds, uInt4 gribLen, uInt4 *curLoc, meta->gridAttrib.f_miss = 0; } - if( !data ) - return 0; - if (f_bms) { /* #ifdef DEBUG @@ -1716,7 +1713,7 @@ static int ReadGrib1Sect4 (uChar *bds, uInt4 gribLen, uInt4 *curLoc, // cppcheck-suppress nullPointer if (!bitmap[i]) { meta->gridAttrib.numMiss++; - data[newIndex] = UNDEFINED; + if( data ) data[newIndex] = UNDEFINED; } else { if (numBits != 0) { if( ((int)bdsRemainingSize - 1) * 8 + bufLoc < (int)numBits ) @@ -1736,11 +1733,11 @@ static int ReadGrib1Sect4 (uChar *bds, uInt4 gribLen, uInt4 *curLoc, if (meta->gridAttrib.max < d_temp) { meta->gridAttrib.max = d_temp; } - data[newIndex] = d_temp; + if( data ) data[newIndex] = d_temp; } else { /* Assert: d_temp = unitM * refVal / pow (10.0,DSF) + unitB. */ /* Assert: min = unitM * refVal / pow (10.0, DSF) + unitB. */ - data[newIndex] = meta->gridAttrib.min; + if( data ) data[newIndex] = meta->gridAttrib.min; } } } @@ -1757,6 +1754,8 @@ static int ReadGrib1Sect4 (uChar *bds, uInt4 gribLen, uInt4 *curLoc, } if (resetPrim != 0) { meta->gridAttrib.missPri = resetPrim; + } + if (data != NULL && resetPrim != 0) { for (i = 0; i < meta->gds.numPts; i++) { /* Find the destination index. */ if (f_convert) { @@ -1776,6 +1775,9 @@ static int ReadGrib1Sect4 (uChar *bds, uInt4 gribLen, uInt4 *curLoc, } else { + if( !data ) + return 0; + #ifdef DEBUG /* printf ("There is no bitmap?\n"); diff --git a/gdal/frmts/grib/gribdataset.cpp b/gdal/frmts/grib/gribdataset.cpp index b5fd6c878968..b5c6b3b3500c 100644 --- a/gdal/frmts/grib/gribdataset.cpp +++ b/gdal/frmts/grib/gribdataset.cpp @@ -784,17 +784,21 @@ double GRIBRasterBand::GetNoDataValue( int *pbSuccess ) ReadGribData(poGDS->fp, start, subgNum, nullptr, &m_Grib_MetaData); if( m_Grib_MetaData == nullptr ) { - if (pbSuccess) - *pbSuccess = FALSE; - return 0; + m_bHasNoData = false; + m_dfNoData = 0; + if( pbSuccess ) + *pbSuccess = m_bHasNoData; + return m_dfNoData; } } if( m_Grib_MetaData->gridAttrib.f_miss == 0) { + m_bHasNoData = false; + m_dfNoData = 0; if (pbSuccess) - *pbSuccess = FALSE; - return 0; + *pbSuccess = m_bHasNoData; + return m_dfNoData; } if (m_Grib_MetaData->gridAttrib.f_miss == 2) @@ -804,9 +808,11 @@ double GRIBRasterBand::GetNoDataValue( int *pbSuccess ) nBand, m_Grib_MetaData->gridAttrib.missSec); } + m_bHasNoData = true; + m_dfNoData = m_Grib_MetaData->gridAttrib.missPri; if (pbSuccess) - *pbSuccess = TRUE; - return m_Grib_MetaData->gridAttrib.missPri; + *pbSuccess = m_bHasNoData; + return m_dfNoData; } /************************************************************************/ From 1cc2d31e20a9a73ca3d78cd47c6818c6c626388c Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 23 Sep 2020 23:40:42 +0200 Subject: [PATCH 070/255] degrib1.cpp: fix warning introduced in previous commit --- gdal/frmts/grib/degrib/degrib/degrib1.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gdal/frmts/grib/degrib/degrib/degrib1.cpp b/gdal/frmts/grib/degrib/degrib/degrib1.cpp index 5a0c930f6076..b4622ffc50a2 100644 --- a/gdal/frmts/grib/degrib/degrib/degrib1.cpp +++ b/gdal/frmts/grib/degrib/degrib/degrib1.cpp @@ -1755,7 +1755,7 @@ static int ReadGrib1Sect4 (uChar *bds, uInt4 gribLen, uInt4 *curLoc, if (resetPrim != 0) { meta->gridAttrib.missPri = resetPrim; } - if (data != NULL && resetPrim != 0) { + if (data != nullptr && resetPrim != 0) { for (i = 0; i < meta->gds.numPts; i++) { /* Find the destination index. */ if (f_convert) { From 1b265abd1ab5a37f7e14d898ea3ff4bb5a2d0389 Mon Sep 17 00:00:00 2001 From: Adam Williams Date: Thu, 24 Sep 2020 02:20:58 -0600 Subject: [PATCH 071/255] build(gdal/docker): registry/image/tag, multi arch support (#2965) --- gdal/docker/README.md | 38 +++++++++++++++ gdal/docker/alpine-normal/build.sh | 2 +- gdal/docker/alpine-small/build.sh | 2 +- gdal/docker/alpine-ultrasmall/build.sh | 2 +- gdal/docker/ubuntu-full/build.sh | 2 +- gdal/docker/ubuntu-small/build.sh | 2 +- gdal/docker/util.sh | 67 +++++++++++++++++++++----- 7 files changed, 98 insertions(+), 17 deletions(-) diff --git a/gdal/docker/README.md b/gdal/docker/README.md index db4a376466a1..fa1bc6a83f24 100644 --- a/gdal/docker/README.md +++ b/gdal/docker/README.md @@ -95,3 +95,41 @@ Tagged images of recent past releases are available. The last ones (at time of w * osgeo/alpine-normal-3.1.0 * osgeo/ubuntu-small-3.1.0 * osgeo/ubuntu-full-3.1.0 + +## Multi-arch Images + +Each directory contains a `build.sh` shell script that supports building images +for multiple platforms using an experimental feature called [Docker BuildKit](https://docs.docker.com/buildx/working-with-buildx/). + +BuildKit CLI looks like `docker buildx build` vs. `docker build` +and allows images to build not only for the architecture and operating system +that the user invoking the build happens to run, but for others as well. + +There is a small setup process depending on your operating system. Refer to [Preparation toward running Docker on ARM Mac: Building multi-arch images with Docker BuildX](https://medium.com/nttlabs/buildx-multiarch-2c6c2df00ca2). + +#### Example Scenario + +If you're running Docker for MacOS with an Intel CPU +and you wanted to build the `alpine-small` image with support for Raspberry Pi 4, +adding a couple flags when running `alpine-small/build.sh` can greatly simplify this process + +#### Enabling + +Use the two script flags in order to leverage BuildKit: + +| Flag | Description | Arguments | +| ------------- | ------------- | ------------- | +| --with-multi-arch | Will build using the `buildx` plugin | N/A | +| --platform | Which architectures to build | linux/amd64,linux/arm64 | + +**Example** + +`alpine-small/build.sh --with-multi-arch --release --gdal v3.1.3 --proj master --platform linux/arm64,linux/amd64` + +## Custom Image Names + +Override the image and repository by setting the environment variable: `BASE_IMAGE_NAME` + +**Example** + +`BASE_IMAGE_NAME="YOU_DOCKER_USERNAME/gdal" alpine-small/build.sh --release --gdal v3.1.3 --proj master` diff --git a/gdal/docker/alpine-normal/build.sh b/gdal/docker/alpine-normal/build.sh index e2ae69f17071..0aebbfbe7ee7 100755 --- a/gdal/docker/alpine-normal/build.sh +++ b/gdal/docker/alpine-normal/build.sh @@ -19,6 +19,6 @@ esac export SCRIPT_DIR TAG_NAME=$(basename "${SCRIPT_DIR}") -export BASE_IMAGE_NAME=osgeo/gdal:${TAG_NAME} +export BASE_IMAGE_NAME=${BASE_IMAGE_NAME:-osgeo/gdal:${TAG_NAME}} "${SCRIPT_DIR}/../util.sh" "$@" --test-python diff --git a/gdal/docker/alpine-small/build.sh b/gdal/docker/alpine-small/build.sh index 09d0843aa90c..0a40079022e2 100755 --- a/gdal/docker/alpine-small/build.sh +++ b/gdal/docker/alpine-small/build.sh @@ -19,6 +19,6 @@ esac export SCRIPT_DIR TAG_NAME=$(basename "${SCRIPT_DIR}") -export BASE_IMAGE_NAME=osgeo/gdal:${TAG_NAME} +export BASE_IMAGE_NAME=${BASE_IMAGE_NAME:-osgeo/gdal:${TAG_NAME}} "${SCRIPT_DIR}/../util.sh" "$@" diff --git a/gdal/docker/alpine-ultrasmall/build.sh b/gdal/docker/alpine-ultrasmall/build.sh index 09d0843aa90c..0a40079022e2 100755 --- a/gdal/docker/alpine-ultrasmall/build.sh +++ b/gdal/docker/alpine-ultrasmall/build.sh @@ -19,6 +19,6 @@ esac export SCRIPT_DIR TAG_NAME=$(basename "${SCRIPT_DIR}") -export BASE_IMAGE_NAME=osgeo/gdal:${TAG_NAME} +export BASE_IMAGE_NAME=${BASE_IMAGE_NAME:-osgeo/gdal:${TAG_NAME}} "${SCRIPT_DIR}/../util.sh" "$@" diff --git a/gdal/docker/ubuntu-full/build.sh b/gdal/docker/ubuntu-full/build.sh index e2ae69f17071..0aebbfbe7ee7 100755 --- a/gdal/docker/ubuntu-full/build.sh +++ b/gdal/docker/ubuntu-full/build.sh @@ -19,6 +19,6 @@ esac export SCRIPT_DIR TAG_NAME=$(basename "${SCRIPT_DIR}") -export BASE_IMAGE_NAME=osgeo/gdal:${TAG_NAME} +export BASE_IMAGE_NAME=${BASE_IMAGE_NAME:-osgeo/gdal:${TAG_NAME}} "${SCRIPT_DIR}/../util.sh" "$@" --test-python diff --git a/gdal/docker/ubuntu-small/build.sh b/gdal/docker/ubuntu-small/build.sh index e2ae69f17071..0aebbfbe7ee7 100755 --- a/gdal/docker/ubuntu-small/build.sh +++ b/gdal/docker/ubuntu-small/build.sh @@ -19,6 +19,6 @@ esac export SCRIPT_DIR TAG_NAME=$(basename "${SCRIPT_DIR}") -export BASE_IMAGE_NAME=osgeo/gdal:${TAG_NAME} +export BASE_IMAGE_NAME=${BASE_IMAGE_NAME:-osgeo/gdal:${TAG_NAME}} "${SCRIPT_DIR}/../util.sh" "$@" --test-python diff --git a/gdal/docker/util.sh b/gdal/docker/util.sh index 348ce6e661f1..cce5d7ac9c39 100755 --- a/gdal/docker/util.sh +++ b/gdal/docker/util.sh @@ -30,6 +30,8 @@ usage() } RELEASE=no +ARCH_PLATFORMS="linux/amd64" + while (( "$#" )); do case "$1" in @@ -48,6 +50,12 @@ do shift ;; + --platform) + shift + ARCH_PLATFORMS="$1" + shift + ;; + --proj) shift PROJ_VERSION="$1" @@ -70,6 +78,12 @@ do shift ;; + --with-multi-arch) + DOCKER_BUILDKIT=1 + DOCKER_CLI_EXPERIMENTAL=enabled + shift + ;; + --without-debug-symbols) WITH_DEBUG_SYMBOLS=no shift @@ -89,6 +103,11 @@ do esac done +if test "${DOCKER_BUILDKIT}" = "1" && test "${DOCKER_CLI_EXPERIMENTAL}" = "enabled"; then + DOCKER_BUILDX="buildx" + DOCKER_BUILDX_ARGS=("--platform" "${ARCH_PLATFORMS}") +fi + if test "${RELEASE}" = "yes"; then if test "${GDAL_VERSION}" = ""; then echo "--gdal tag must be specified when --release is used." @@ -197,6 +216,15 @@ stop_rsync_host() fi } +build_cmd() +{ + if test "${DOCKER_BUILDX}" = "buildx"; then + echo "${DOCKER_BUILDX}" build "${DOCKER_BUILDX_ARGS[@]}" + else + echo build + fi +} + if test "${BASE_IMAGE_NAME}" = "osgeo/gdal:ubuntu-full"; then PROJ_DATUMGRID_LATEST_LAST_MODIFIED=$(curl -Is https://cdn.proj.org/index.html | grep -i Last-Modified) else @@ -218,7 +246,6 @@ IMAGE_NAME="${BASE_IMAGE_NAME}-${TAG_NAME}" BUILDER_IMAGE_NAME="${IMAGE_NAME}_builder" if test "${RELEASE}" = "yes"; then - BUILD_ARGS=( "--build-arg" "PROJ_DATUMGRID_LATEST_LAST_MODIFIED=${PROJ_DATUMGRID_LATEST_LAST_MODIFIED}" \ "--build-arg" "PROJ_VERSION=${PROJ_VERSION}" \ @@ -227,14 +254,19 @@ if test "${RELEASE}" = "yes"; then "--build-arg" "WITH_DEBUG_SYMBOLS=${WITH_DEBUG_SYMBOLS}" \ ) - docker build "${BUILD_ARGS[@]}" --target builder \ - -t "${BUILDER_IMAGE_NAME}" "${SCRIPT_DIR}" - docker build "${BUILD_ARGS[@]}" -t "${IMAGE_NAME}" "${SCRIPT_DIR}" + docker $(build_cmd) "${BUILD_ARGS[@]}" --target builder -t "${BUILDER_IMAGE_NAME}" "${SCRIPT_DIR}" + docker $(build_cmd) "${BUILD_ARGS[@]}" -t "${IMAGE_NAME}" "${SCRIPT_DIR}" - check_image "${IMAGE_NAME}" + if test "${DOCKER_BUILDX}" != "buildx"; then + check_image "${IMAGE_NAME}" + fi if test "x${PUSH_GDAL_DOCKER_IMAGE}" = "xyes"; then - docker push "${IMAGE_NAME}" + if test "${DOCKER_BUILDX}" = "buildx"; then + docker $(build_cmd) "${BUILD_ARGS[@]}" -t "${IMAGE_NAME}" --push "${SCRIPT_DIR}" + else + docker push "${IMAGE_NAME}" + fi fi else @@ -259,7 +291,7 @@ else if ! docker ps | grep "${RSYNC_DAEMON_CONTAINER}"; then RSYNC_DAEMON_IMAGE=osgeo/gdal:gdal_rsync_daemon docker rmi "${RSYNC_DAEMON_IMAGE}" 2>/dev/null || true - docker build -t "${RSYNC_DAEMON_IMAGE}" - < Date: Thu, 24 Sep 2020 14:51:47 +0200 Subject: [PATCH 072/255] OAPIF: do not list raster or coverage collections --- gdal/ogr/ogrsf_frmts/wfs/ogroapifdriver.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gdal/ogr/ogrsf_frmts/wfs/ogroapifdriver.cpp b/gdal/ogr/ogrsf_frmts/wfs/ogroapifdriver.cpp index b47d369c2f46..13b9a95d08e1 100644 --- a/gdal/ogr/ogrsf_frmts/wfs/ogroapifdriver.cpp +++ b/gdal/ogr/ogrsf_frmts/wfs/ogroapifdriver.cpp @@ -536,6 +536,12 @@ bool OGROAPIFDataset::LoadJSONCollection(const CPLJSONObject& oCollection) { if( oCollection.GetType() != CPLJSONObject::Type::Object ) return false; + + // As used by https://maps.ecere.com/ogcapi/collections?f=json + const auto osLayerDataType = oCollection.GetString("layerDataType"); + if( osLayerDataType == "Raster" || osLayerDataType == "Coverage" ) + return false; + CPLString osName( oCollection.GetString("id") ); #ifndef REMOVE_SUPPORT_FOR_OLD_VERSIONS if( osName.empty() ) From 7c24d0c095ca05eba45ed40003d37e698bbee248 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 24 Sep 2020 14:51:48 +0200 Subject: [PATCH 073/255] GML XSD parser: recognized unsignedLong data type --- gdal/ogr/ogrsf_frmts/gml/parsexsd.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/gdal/ogr/ogrsf_frmts/gml/parsexsd.cpp b/gdal/ogr/ogrsf_frmts/gml/parsexsd.cpp index a53f743681cb..0030a8440067 100644 --- a/gdal/ogr/ogrsf_frmts/gml/parsexsd.cpp +++ b/gdal/ogr/ogrsf_frmts/gml/parsexsd.cpp @@ -117,8 +117,9 @@ bool GetSimpleTypeProperties(CPLXMLNode *psTypeNode, return true; } - else if( EQUAL(pszBase, "long") ) + else if( EQUAL(pszBase, "unsignedLong") ) { + // Optimistically map to signed integer... *pGMLType = GMLPT_Integer64; const char *pszWidth = CPLGetXMLValue(psTypeNode, "restriction.totalDigits.value", "0"); @@ -460,6 +461,11 @@ GMLFeatureClass *GMLParseFeatureType(CPLXMLNode *psSchemaNode, gmlType = GMLPT_Integer; else if (EQUAL(pszStrippedNSType, "long")) gmlType = GMLPT_Integer64; + else if (EQUAL(pszStrippedNSType, "unsignedLong")) + { + // Optimistically map to signed integer + gmlType = GMLPT_Integer64; + } else if (EQUAL(pszStrippedNSType, "short") ) gmlType = GMLPT_Short; else if (EQUAL(pszStrippedNSType, "boolean") ) From b22c0525633697f8dc25afb2b6b848c3ad372686 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 24 Sep 2020 14:51:49 +0200 Subject: [PATCH 074/255] minidriver_tms.cpp: add a TileXMultiplier config element to allow for variable matrix width --- gdal/frmts/wms/minidriver_tms.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gdal/frmts/wms/minidriver_tms.cpp b/gdal/frmts/wms/minidriver_tms.cpp index d3f1a824a6a3..e2f87ca624a0 100644 --- a/gdal/frmts/wms/minidriver_tms.cpp +++ b/gdal/frmts/wms/minidriver_tms.cpp @@ -63,6 +63,9 @@ CPLErr WMSMiniDriver_TMS::Initialize(CPLXMLNode *config, CPL_UNUSED char **papsz const char *format = CPLGetXMLValue(config, "Format", "jpg"); URLSearchAndReplace(&m_base_url, "${format}", "%s", format); + m_nTileXMultiplier = atoi( + CPLGetXMLValue(config, "TileXMultiplier", "1")); + return ret; } @@ -88,7 +91,7 @@ CPLErr WMSMiniDriver_TMS::TiledImageRequest(WMSHTTPRequest &request, // http://tms25.arc.nasa.gov/tile/tile.aspx?T=geocover2000&L=0&X=86&Y=39 url = m_base_url; - URLSearchAndReplace(&url, "${x}", "%d", tiri.m_x); + URLSearchAndReplace(&url, "${x}", "%d", tiri.m_x * m_nTileXMultiplier); URLSearchAndReplace(&url, "${y}", "%d", tms_y); URLSearchAndReplace(&url, "${z}", "%d", tiri.m_level); From 42ff6f24875f8c5459298840613f5b538c815934 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 24 Sep 2020 14:51:51 +0200 Subject: [PATCH 075/255] WMS: add a OGCAPIMaps minidriver --- gdal/frmts/wms/GNUmakefile | 2 +- gdal/frmts/wms/makefile.vc | 2 +- gdal/frmts/wms/minidriver_ogcapimaps.cpp | 67 ++++++++++++++++++++++++ gdal/frmts/wms/minidriver_ogcapimaps.h | 37 +++++++++++++ gdal/frmts/wms/minidriver_tms.h | 2 + gdal/frmts/wms/wmsdriver.cpp | 2 + 6 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 gdal/frmts/wms/minidriver_ogcapimaps.cpp create mode 100644 gdal/frmts/wms/minidriver_ogcapimaps.h diff --git a/gdal/frmts/wms/GNUmakefile b/gdal/frmts/wms/GNUmakefile index c9fe713d9664..bfb96ec48bb3 100644 --- a/gdal/frmts/wms/GNUmakefile +++ b/gdal/frmts/wms/GNUmakefile @@ -7,7 +7,7 @@ OBJ = gdalwmscache.o gdalwmsdataset.o gdalwmsrasterband.o \ minidriver_tileservice.o minidriver_worldwind.o \ minidriver_tms.o minidriver_tiled_wms.o wmsmetadataset.o \ minidriver_virtualearth.o minidriver_arcgis_server.o \ - minidriver_iip.o minidriver_mrf.o + minidriver_iip.o minidriver_mrf.o minidriver_ogcapimaps.o CPPFLAGS := $(CPPFLAGS) -DHAVE_CURL $(CURL_INC) diff --git a/gdal/frmts/wms/makefile.vc b/gdal/frmts/wms/makefile.vc index f52df6500780..e9632693a8dc 100644 --- a/gdal/frmts/wms/makefile.vc +++ b/gdal/frmts/wms/makefile.vc @@ -5,7 +5,7 @@ OBJ = \ minidriver_wms.obj minidriver_tileservice.obj \ minidriver_worldwind.obj minidriver_tms.obj minidriver_tiled_wms.obj \ wmsmetadataset.obj minidriver_virtualearth.obj minidriver_arcgis_server.obj \ - minidriver_iip.obj minidriver_mrf.obj + minidriver_iip.obj minidriver_mrf.obj minidriver_ogcapimaps.obj EXTRAFLAGS = -DHAVE_CURL $(CURL_CFLAGS) $(CURL_INC) diff --git a/gdal/frmts/wms/minidriver_ogcapimaps.cpp b/gdal/frmts/wms/minidriver_ogcapimaps.cpp new file mode 100644 index 000000000000..e2032da7b60e --- /dev/null +++ b/gdal/frmts/wms/minidriver_ogcapimaps.cpp @@ -0,0 +1,67 @@ +/****************************************************************************** + * Project: WMS Client Driver + * Purpose: OGC API maps + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2020, Even Rouault + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#include "wmsdriver.h" +#include "minidriver_ogcapimaps.h" + +CPL_CVSID("$Id$") + +CPLErr WMSMiniDriver_OGCAPIMaps::Initialize(CPLXMLNode *config, CPL_UNUSED char **papszOpenOptions) { + CPLErr ret = CE_None; + + { + const char *base_url = CPLGetXMLValue(config, "ServerURL", ""); + if (base_url[0] != '\0') { + m_base_url = base_url; + } else { + CPLError(CE_Failure, CPLE_AppDefined, "GDALWMS, OGCAPIMaps mini-driver: ServerURL missing."); + ret = CE_Failure; + } + } + + return ret; +} + +CPLErr WMSMiniDriver_OGCAPIMaps::TiledImageRequest(WMSHTTPRequest &request, + const GDALWMSImageRequestInfo &iri, + const GDALWMSTiledImageRequestInfo &) +{ + CPLString &url = request.URL; + + url = m_base_url; + + URLPrepare(url); + url += CPLOPrintf("width=%d&height=%d&bbox=%.18g,%.18g,%.18g,%.18g", + iri.m_sx, + iri.m_sy, + iri.m_x0, + iri.m_y1, + iri.m_x1, + iri.m_y0); + + return CE_None; +} diff --git a/gdal/frmts/wms/minidriver_ogcapimaps.h b/gdal/frmts/wms/minidriver_ogcapimaps.h new file mode 100644 index 000000000000..e567c0276864 --- /dev/null +++ b/gdal/frmts/wms/minidriver_ogcapimaps.h @@ -0,0 +1,37 @@ +/****************************************************************************** + * Project: WMS Client Driver + * Purpose: OGC API maps + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2020, Even Rouault + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +class WMSMiniDriver_OGCAPIMaps : public WMSMiniDriver { +public: + WMSMiniDriver_OGCAPIMaps() = default; + +public: + virtual CPLErr Initialize(CPLXMLNode *config, char **papszOpenOptions) override; + virtual CPLErr TiledImageRequest(WMSHTTPRequest &request, + const GDALWMSImageRequestInfo &iri, + const GDALWMSTiledImageRequestInfo &tiri) override; +}; diff --git a/gdal/frmts/wms/minidriver_tms.h b/gdal/frmts/wms/minidriver_tms.h index 2c4dd4ae3a16..454bc41ef7bc 100644 --- a/gdal/frmts/wms/minidriver_tms.h +++ b/gdal/frmts/wms/minidriver_tms.h @@ -29,6 +29,8 @@ ****************************************************************************/ class WMSMiniDriver_TMS : public WMSMiniDriver { + int m_nTileXMultiplier = 1; + public: WMSMiniDriver_TMS(); virtual ~WMSMiniDriver_TMS(); diff --git a/gdal/frmts/wms/wmsdriver.cpp b/gdal/frmts/wms/wmsdriver.cpp index 140d6a5fcd34..0cd5c5218222 100644 --- a/gdal/frmts/wms/wmsdriver.cpp +++ b/gdal/frmts/wms/wmsdriver.cpp @@ -41,6 +41,7 @@ #include "minidriver_arcgis_server.h" #include "minidriver_iip.h" #include "minidriver_mrf.h" +#include "minidriver_ogcapimaps.h" #include "cpl_json.h" @@ -1057,6 +1058,7 @@ void GDALRegister_WMS() RegisterMinidriver(AGS); RegisterMinidriver(IIP); RegisterMinidriver(MRF); + RegisterMinidriver(OGCAPIMaps); GDALDriver *poDriver = new GDALDriver(); From 9051399a77211bc7690302a84c8e458463da0535 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 24 Sep 2020 14:51:53 +0200 Subject: [PATCH 076/255] MVT: add internal open options to allow specifying georeferencing of a tile --- gdal/ogr/ogrsf_frmts/mvt/ogrmvtdataset.cpp | 36 +++++++++++++++++----- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/gdal/ogr/ogrsf_frmts/mvt/ogrmvtdataset.cpp b/gdal/ogr/ogrsf_frmts/mvt/ogrmvtdataset.cpp index 7603909fa89a..19abf298f28d 100644 --- a/gdal/ogr/ogrsf_frmts/mvt/ogrmvtdataset.cpp +++ b/gdal/ogr/ogrsf_frmts/mvt/ogrmvtdataset.cpp @@ -276,7 +276,8 @@ class OGRMVTDataset final: public GDALDataset GByte *m_pabyData; std::vector> m_apoLayers; bool m_bGeoreferenced = false; - double m_dfTileDim = 0.0; + double m_dfTileDimX = 0.0; + double m_dfTileDimY = 0.0; double m_dfTopX = 0.0; double m_dfTopY = 0.0; CPLString m_osMetadataMemFilename; @@ -822,8 +823,8 @@ void OGRMVTLayer::GetXY(int nX, int nY, double& dfX, double& dfY) { if( m_poDS->m_bGeoreferenced ) { - dfX = m_poDS->m_dfTopX + nX * m_poDS->m_dfTileDim / m_nExtent; - dfY = m_poDS->m_dfTopY - nY * m_poDS->m_dfTileDim / m_nExtent; + dfX = m_poDS->m_dfTopX + nX * m_poDS->m_dfTileDimX / m_nExtent; + dfY = m_poDS->m_dfTopY - nY * m_poDS->m_dfTileDimY / m_nExtent; } else { @@ -1857,7 +1858,8 @@ OGRMVTDataset::~OGRMVTDataset() VSIFree(m_pabyData); if( !m_osMetadataMemFilename.empty() ) VSIUnlink(m_osMetadataMemFilename); - m_poSRS->Release(); + if( m_poSRS ) + m_poSRS->Release(); } /************************************************************************/ @@ -3065,6 +3067,21 @@ GDALDataset *OGRMVTDataset::Open( GDALOpenInfo* poOpenInfo ) CPLString()); } + const char* pszGeorefTopX = CSLFetchNameValue(poOpenInfo->papszOpenOptions, "GEOREF_TOPX"); + const char* pszGeorefTopY = CSLFetchNameValue(poOpenInfo->papszOpenOptions, "GEOREF_TOPY"); + const char* pszGeorefTileDimX = CSLFetchNameValue(poOpenInfo->papszOpenOptions, "GEOREF_TILEDIMX"); + const char* pszGeorefTileDimY = CSLFetchNameValue(poOpenInfo->papszOpenOptions, "GEOREF_TILEDIMY"); + if( pszGeorefTopX && pszGeorefTopY && pszGeorefTileDimX && pszGeorefTileDimY ) + { + poDS->m_bGeoreferenced = true; + poDS->m_dfTileDimX = CPLAtof(pszGeorefTileDimX); + poDS->m_dfTileDimY = CPLAtof(pszGeorefTileDimY); + poDS->m_dfTopX = CPLAtof(pszGeorefTopX); + poDS->m_dfTopY = CPLAtof(pszGeorefTopY); + poDS->m_poSRS->Release(); + poDS->m_poSRS = nullptr; + } + else if( CPLGetValueType(osX) == CPL_VALUE_INTEGER && CPLGetValueType(osY) == CPL_VALUE_INTEGER && CPLGetValueType(osZ) == CPL_VALUE_INTEGER ) @@ -3077,9 +3094,10 @@ GDALDataset *OGRMVTDataset::Open( GDALOpenInfo* poOpenInfo ) nY >= 0 && nY < (1 << nZ) ) { poDS->m_bGeoreferenced = true; - poDS->m_dfTileDim = poDS->m_dfTileDim0 / (1 << nZ); - poDS->m_dfTopX = poDS->m_dfTopXOrigin + nX * poDS->m_dfTileDim; - poDS->m_dfTopY = poDS->m_dfTopYOrigin - nY * poDS->m_dfTileDim; + poDS->m_dfTileDimX = poDS->m_dfTileDim0 / (1 << nZ); + poDS->m_dfTileDimY = poDS->m_dfTileDimX; + poDS->m_dfTopX = poDS->m_dfTopXOrigin + nX * poDS->m_dfTileDimX; + poDS->m_dfTopY = poDS->m_dfTopYOrigin - nY * poDS->m_dfTileDimY; } } @@ -6221,6 +6239,10 @@ void RegisterOGRMVT() "