From dd9e0ac2a0380e9ff69336b9345a7cc4f0bf7637 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 20 Dec 2023 13:11:51 +0100 Subject: [PATCH 01/11] Add raster VRTTI (Virtual Raster Tile Index) driver New installed file: data/gdalvrtti.xsd --- autotest/gdrivers/vrttileindex.py | 2555 ++++++++++++ frmts/drivers.ini | 1 + frmts/gdalallregister.cpp | 1 + frmts/gtiff/gt_jpeg_copy.cpp | 4 +- frmts/vrt/CMakeLists.txt | 2 + frmts/vrt/data/gdalvrtti.xsd | 267 ++ frmts/vrt/vrt_priv.h | 62 + frmts/vrt/vrtdataset.cpp | 2 +- frmts/vrt/vrtdataset.h | 13 +- frmts/vrt/vrtdriver.cpp | 4 +- frmts/vrt/vrtrasterband.cpp | 126 +- frmts/vrt/vrttileindexdataset.cpp | 3623 +++++++++++++++++ gcore/gdal_frmts.h | 1 + gcore/gdal_misc.cpp | 7 +- gcore/gdal_priv.h | 4 +- gcore/gdal_rat.cpp | 4 +- gcore/gdal_rat.h | 2 +- gcore/gdalmultidomainmetadata.cpp | 12 +- .../gpkg/ogrgeopackagedatasource.cpp | 5 +- ogr/ogrsf_frmts/gpkg/ogrgeopackagedriver.cpp | 12 + 20 files changed, 6624 insertions(+), 83 deletions(-) create mode 100755 autotest/gdrivers/vrttileindex.py create mode 100644 frmts/vrt/data/gdalvrtti.xsd create mode 100644 frmts/vrt/vrt_priv.h create mode 100644 frmts/vrt/vrttileindexdataset.cpp diff --git a/autotest/gdrivers/vrttileindex.py b/autotest/gdrivers/vrttileindex.py new file mode 100755 index 000000000000..c4399aa4ae54 --- /dev/null +++ b/autotest/gdrivers/vrttileindex.py @@ -0,0 +1,2555 @@ +#!/usr/bin/env pytest +############################################################################### +# $Id$ +# +# Project: GDAL/OGR Test Suite +# Purpose: Test VRTTileIndexDataset support. +# Author: Even Rouault +# +############################################################################### +# Copyright (c) 2023, Even Rouault +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +############################################################################### + +import math +import os +import shutil +import struct + +import gdaltest +import pytest + +from osgeo import gdal, ogr + +pytestmark = [pytest.mark.require_driver("VRTTI"), pytest.mark.require_driver("GPKG")] + + +def create_basic_tileindex( + index_filename, + src_ds, + location_field_name="location", + sort_field_name=None, + sort_field_type=None, + sort_values=None, +): + if isinstance(src_ds, list): + src_ds_list = src_ds + else: + src_ds_list = [src_ds] + index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) + lyr = index_ds.CreateLayer( + "index", srs=(src_ds_list[0].GetSpatialRef() if src_ds_list else None) + ) + lyr.CreateField(ogr.FieldDefn(location_field_name)) + if sort_values: + lyr.CreateField(ogr.FieldDefn(sort_field_name, sort_field_type)) + lyr.SetMetadataItem("SORT_FIELD", sort_field_name) + for i, src_ds in enumerate(src_ds_list): + f = ogr.Feature(lyr.GetLayerDefn()) + src_gt = src_ds.GetGeoTransform() + minx = src_gt[0] + maxx = minx + src_ds.RasterXSize * src_gt[1] + maxy = src_gt[3] + miny = maxy + src_ds.RasterYSize * src_gt[5] + f[location_field_name] = src_ds.GetDescription() + if sort_values: + f[sort_field_name] = sort_values[i] + f.SetGeometry( + ogr.CreateGeometryFromWkt( + f"POLYGON(({minx} {miny},{minx} {maxy},{maxx} {maxy},{maxx} {miny},{minx} {miny}))" + ) + ) + lyr.CreateFeature(f) + return index_ds, lyr + + +def check_basic( + vrt_ds, src_ds, expected_dt=None, expected_colorinterp=None, expected_md={} +): + assert vrt_ds.RasterXSize == src_ds.RasterXSize + assert vrt_ds.RasterYSize == src_ds.RasterYSize + assert vrt_ds.RasterCount == src_ds.RasterCount + assert vrt_ds.GetGeoTransform() == pytest.approx(src_ds.GetGeoTransform()) + assert vrt_ds.GetSpatialRef().GetAuthorityCode( + None + ) == src_ds.GetSpatialRef().GetAuthorityCode(None) + for iband in range(1, vrt_ds.RasterCount + 1): + vrt_band = vrt_ds.GetRasterBand(iband) + src_band = src_ds.GetRasterBand(iband) + assert vrt_band.DataType == ( + expected_dt if expected_dt is not None else src_band.DataType + ) + assert vrt_band.GetNoDataValue() == src_band.GetNoDataValue() + assert vrt_band.GetColorInterpretation() == ( + expected_colorinterp + if expected_colorinterp is not None + else src_band.GetColorInterpretation() + ) + assert vrt_band.Checksum() == src_band.Checksum() + assert vrt_band.ReadRaster( + 1, 2, 3, 4, buf_type=src_band.DataType + ) == src_band.ReadRaster(1, 2, 3, 4) + + assert vrt_ds.ReadRaster( + 1, 2, 3, 4, buf_type=src_band.DataType + ) == src_ds.ReadRaster(1, 2, 3, 4) + assert vrt_ds.GetMetadata_Dict() == expected_md + + +def test_vrttileindex_no_metadata(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, _ = create_basic_tileindex(index_filename, src_ds) + del index_ds + + vrt_ds = gdal.Open(index_filename) + check_basic(vrt_ds, src_ds) + assert ( + vrt_ds.GetMetadataItem("SCANNED_ONE_FEATURE_AT_OPENING", "__DEBUG__") == "YES" + ) + assert vrt_ds.GetRasterBand(1).GetBlockSize() == [256, 256] + assert vrt_ds.GetRasterBand(1).ReadBlock(0, 0)[0:20] == src_ds.GetRasterBand( + 1 + ).ReadRaster(0, 0, 20, 1) + + assert "byte.tif" in vrt_ds.GetRasterBand(1).GetMetadataItem( + "Pixel_0_0", "LocationInfo" + ) + assert vrt_ds.GetRasterBand(1).GetMetadataItem("foo") is None + assert vrt_ds.GetRasterBand(1).GetMetadataItem("foo", "LocationInfo") is None + assert vrt_ds.GetRasterBand(1).GetMetadataItem("Pixel_", "LocationInfo") is None + assert vrt_ds.GetRasterBand(1).GetMetadataItem("Pixel_0", "LocationInfo") is None + assert vrt_ds.GetRasterBand(1).GetMetadataItem("GeoPixel_", "LocationInfo") is None + assert vrt_ds.GetRasterBand(1).GetMetadataItem("GeoPixel_0", "LocationInfo") is None + + assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 0 + assert vrt_ds.GetRasterBand(1).GetOverview(-1) is None + assert vrt_ds.GetRasterBand(1).GetOverview(0) is None + + +def test_vrttileindex_custom_metadata(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + lyr.SetMetadataItem("BLOCKXSIZE", "2") + lyr.SetMetadataItem("BLOCKYSIZE", "4") + lyr.SetMetadataItem("FOO", "BAR") + del index_ds + + vrt_ds = gdal.Open(index_filename) + check_basic(vrt_ds, src_ds, expected_md={"FOO": "BAR"}) + assert vrt_ds.GetRasterBand(1).GetBlockSize() == [2, 4] + + +def test_vrttileindex_cannot_open_index(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + f = gdal.VSIFOpenL(index_filename, "wb+") + assert f + gdal.VSIFTruncateL(f, 100) + gdal.VSIFCloseL(f) + + with pytest.raises(Exception, match="not recognized as a supported file format"): + gdal.Open(index_filename) + + +def test_vrttileindex_several_layers(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, _ = create_basic_tileindex(index_filename, src_ds) + index_ds.CreateLayer("another_layer") + del index_ds + + with pytest.raises( + Exception, + match="has more than one layer. TILE_INDEX_LAYER metadata item must be defined", + ): + gdal.Open(index_filename) + + with pytest.raises( + Exception, match="has more than one layer. LAYER open option must be defined" + ): + gdal.Open("VRTTI:" + index_filename) + + assert ( + gdal.OpenEx("VRTTI:" + index_filename, open_options=["LAYER=index"]) is not None + ) + + index_ds = ogr.Open(index_filename, update=1) + index_ds.SetMetadataItem("TILE_INDEX_LAYER", "index") + del index_ds + + vrt_ds = gdal.Open(index_filename) + check_basic(vrt_ds, src_ds) + assert ( + vrt_ds.GetMetadataItem("SCANNED_ONE_FEATURE_AT_OPENING", "__DEBUG__") == "YES" + ) + + +def test_vrttileindex_no_metadata_several_layers_wrong_TILE_INDEX_LAYER( + tmp_vsimem, +): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, _ = create_basic_tileindex(index_filename, src_ds) + index_ds.CreateLayer("another_layer") + index_ds.SetMetadataItem("TILE_INDEX_LAYER", "wrong") + del index_ds + + with pytest.raises(Exception, match="Layer wrong does not exist"): + gdal.Open(index_filename) + + +def test_vrttileindex_no_layer(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) + + with pytest.raises(Exception, match="has no vector layer"): + gdal.Open(index_filename, gdal.GA_Update) + + +def test_vrttileindex_no_feature(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) + lyr = index_ds.CreateLayer("index") + lyr.CreateField(ogr.FieldDefn("location")) + del index_ds + + with pytest.raises(Exception, match="metadata items missing"): + gdal.Open(index_filename) + + +def test_vrttileindex_location_wrong_type(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) + lyr = index_ds.CreateLayer("index") + lyr.CreateField(ogr.FieldDefn("location", ogr.OFTInteger)) + del index_ds + + with pytest.raises(Exception, match="Field location is not of type string"): + gdal.Open(index_filename) + + +def test_vrttileindex_wrong_prototype_tile(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) + lyr = index_ds.CreateLayer("index") + lyr.CreateField(ogr.FieldDefn("location")) + f = ogr.Feature(lyr.GetLayerDefn()) + f["location"] = "/i/do/not/exist" + lyr.CreateFeature(f) + del index_ds + + with pytest.raises(Exception): + gdal.Open(index_filename) + + +def test_vrttileindex_prototype_tile_no_gt(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + protods_filename = str(tmp_vsimem / "protods_filename.tif") + ds = gdal.GetDriverByName("GTiff").Create(protods_filename, 1, 1) + del ds + index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) + lyr = index_ds.CreateLayer("index") + lyr.CreateField(ogr.FieldDefn("location")) + f = ogr.Feature(lyr.GetLayerDefn()) + f["location"] = protods_filename + lyr.CreateFeature(f) + del index_ds + + with pytest.raises(Exception, match="Cannot find geotransform"): + gdal.Open(index_filename) + + +def test_vrttileindex_prototype_tile_wrong_gt_3rd_value(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + protods_filename = str(tmp_vsimem / "protods_filename.tif") + ds = gdal.GetDriverByName("GTiff").Create(protods_filename, 1, 1) + del ds + ds = gdal.Open(protods_filename) + ds.SetGeoTransform([0, 0, 1234, 0, 0, -1]) + del ds + index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) + lyr = index_ds.CreateLayer("index") + lyr.CreateField(ogr.FieldDefn("location")) + f = ogr.Feature(lyr.GetLayerDefn()) + f["location"] = protods_filename + lyr.CreateFeature(f) + del index_ds + + with pytest.raises(Exception, match="3rd value of GeoTransform"): + gdal.Open(index_filename) + + +def test_vrttileindex_prototype_tile_wrong_gt_5th_value(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + protods_filename = str(tmp_vsimem / "protods_filename.tif") + ds = gdal.GetDriverByName("GTiff").Create(protods_filename, 1, 1) + del ds + ds = gdal.Open(protods_filename) + ds.SetGeoTransform([0, 0, 0, 0, 1, -1]) + del ds + index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) + lyr = index_ds.CreateLayer("index") + lyr.CreateField(ogr.FieldDefn("location")) + f = ogr.Feature(lyr.GetLayerDefn()) + f["location"] = protods_filename + lyr.CreateFeature(f) + del index_ds + + with pytest.raises(Exception, match="5th value of GeoTransform"): + gdal.Open(index_filename) + + +def test_vrttileindex_prototype_tile_wrong_gt_6th_value(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + protods_filename = str(tmp_vsimem / "protods_filename.tif") + ds = gdal.GetDriverByName("GTiff").Create(protods_filename, 1, 1) + del ds + ds = gdal.Open(protods_filename) + ds.SetGeoTransform([0, 0, 0, 0, 0, 1]) + del ds + index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) + lyr = index_ds.CreateLayer("index") + lyr.CreateField(ogr.FieldDefn("location")) + f = ogr.Feature(lyr.GetLayerDefn()) + f["location"] = protods_filename + lyr.CreateFeature(f) + del index_ds + + with pytest.raises(Exception, match="6th value of GeoTransform"): + gdal.Open(index_filename) + + +def test_vrttileindex_no_extent(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + protods_filename = str(tmp_vsimem / "protods_filename.tif") + ds = gdal.GetDriverByName("GTiff").Create(protods_filename, 1, 1) + ds.SetGeoTransform([0, 1, 0, 0, 0, -1]) + del ds + index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) + lyr = index_ds.CreateLayer("index") + lyr.CreateField(ogr.FieldDefn("location")) + f = ogr.Feature(lyr.GetLayerDefn()) + f["location"] = protods_filename + lyr.CreateFeature(f) + del index_ds + + with pytest.raises(Exception, match="Cannot get layer extent"): + gdal.Open(index_filename) + + +def test_vrttileindex_too_big_x(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + protods_filename = str(tmp_vsimem / "protods_filename.tif") + ds = gdal.GetDriverByName("GTiff").Create(protods_filename, 1, 1) + ds.SetGeoTransform([0, 1e-30, 0, 0, 0, -1]) + del ds + index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) + lyr = index_ds.CreateLayer("index") + lyr.CreateField(ogr.FieldDefn("location")) + f = ogr.Feature(lyr.GetLayerDefn()) + f["location"] = protods_filename + f.SetGeometry(ogr.CreateGeometryFromWkt("POLYGON((0 0,0 1,1 1,1 0,0 0))")) + lyr.CreateFeature(f) + del index_ds + + with pytest.raises(Exception, match="Too small RESX, or wrong layer extent"): + gdal.Open(index_filename) + + +def test_vrttileindex_too_big_y(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + protods_filename = str(tmp_vsimem / "protods_filename.tif") + ds = gdal.GetDriverByName("GTiff").Create(protods_filename, 1, 1) + ds.SetGeoTransform([0, 1, 0, 0, 0, -1e-30]) + del ds + index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) + lyr = index_ds.CreateLayer("index") + lyr.CreateField(ogr.FieldDefn("location")) + f = ogr.Feature(lyr.GetLayerDefn()) + f["location"] = protods_filename + f.SetGeometry(ogr.CreateGeometryFromWkt("POLYGON((0 0,0 1,1 1,1 0,0 0))")) + lyr.CreateFeature(f) + del index_ds + + with pytest.raises(Exception, match="Too small RESY, or wrong layer extent"): + gdal.Open(index_filename) + + +def test_vrttileindex_location_field_missing(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + # Missing LOCATION_FIELD and non-default location field name + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, _ = create_basic_tileindex( + index_filename, src_ds, location_field_name="path" + ) + del index_ds + + with pytest.raises(Exception, match="Cannot find field location"): + gdal.Open(index_filename) + + +def test_vrttileindex_location_field_set(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + # LOCATION_FIELD set + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex( + index_filename, src_ds, location_field_name="path" + ) + lyr.SetMetadataItem("LOCATION_FIELD", "path") + del index_ds + + vrt_ds = gdal.Open(index_filename) + check_basic(vrt_ds, src_ds) + + +@pytest.mark.parametrize("missing_item", ["RESX", "RESY"]) +def test_vrttileindex_resx_resy(tmp_vsimem, missing_item): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + if missing_item != "RESX": + lyr.SetMetadataItem("RESX", "60") + if missing_item != "RESY": + lyr.SetMetadataItem("RESY", "60") + lyr.SetMetadataItem("BAND_COUNT", "1") + lyr.SetMetadataItem("DATA_TYPE", "UInt16") + lyr.SetMetadataItem("COLOR_INTERPRETATION", "Green") + del index_ds + + if missing_item is None: + vrt_ds = gdal.Open(index_filename) + check_basic( + vrt_ds, + src_ds, + expected_dt=gdal.GDT_UInt16, + expected_colorinterp=gdal.GCI_GreenBand, + ) + assert ( + vrt_ds.GetMetadataItem("SCANNED_ONE_FEATURE_AT_OPENING", "__DEBUG__") + == "NO" + ) + else: + with pytest.raises(Exception, match=missing_item): + gdal.Open(index_filename) + + +@pytest.mark.parametrize("missing_item", [None, "XSIZE", "YSIZE", "GEOTRANSFORM"]) +def test_vrttileindex_width_height_geotransform(tmp_vsimem, missing_item): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + gt = src_ds.GetGeoTransform() + if missing_item != "XSIZE": + lyr.SetMetadataItem("XSIZE", "20") + if missing_item != "YSIZE": + lyr.SetMetadataItem("YSIZE", "20") + if missing_item != "GEOTRANSFORM": + lyr.SetMetadataItem("GEOTRANSFORM", ",".join([str(x) for x in gt])) + lyr.SetMetadataItem("BAND_COUNT", "1") + lyr.SetMetadataItem("DATA_TYPE", "Byte") + lyr.SetMetadataItem("COLOR_INTERPRETATION", "Gray") + del index_ds + + if missing_item is None: + vrt_ds = gdal.Open(index_filename) + assert ( + vrt_ds.GetMetadataItem("SCANNED_ONE_FEATURE_AT_OPENING", "__DEBUG__") + == "NO" + ) + check_basic(vrt_ds, src_ds) + else: + with pytest.raises(Exception, match=missing_item): + gdal.Open(index_filename) + + +def test_vrttileindex_wrong_width(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + gt = src_ds.GetGeoTransform() + lyr.SetMetadataItem("XSIZE", "0") + lyr.SetMetadataItem("YSIZE", "20") + lyr.SetMetadataItem("GEOTRANSFORM", ",".join([str(x) for x in gt])) + del index_ds + + with pytest.raises(Exception, match="XSIZE metadata item should be > 0"): + gdal.Open(index_filename) + + +def test_vrttileindex_wrong_height(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + gt = src_ds.GetGeoTransform() + lyr.SetMetadataItem("XSIZE", "20") + lyr.SetMetadataItem("YSIZE", "0") + lyr.SetMetadataItem("GEOTRANSFORM", ",".join([str(x) for x in gt])) + del index_ds + + with pytest.raises(Exception, match="YSIZE metadata item should be > 0"): + gdal.Open(index_filename) + + +def test_vrttileindex_wrong_blockxsize(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + lyr.SetMetadataItem("BLOCKXSIZE", "0") + del index_ds + + with pytest.raises(Exception, match="Invalid BLOCKXSIZE"): + gdal.Open(index_filename) + + +def test_vrttileindex_wrong_blockysize(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + lyr.SetMetadataItem("BLOCKYSIZE", "0") + del index_ds + + with pytest.raises(Exception, match="Invalid BLOCKYSIZE"): + gdal.Open(index_filename) + + +def test_vrttileindex_wrong_blockxsize_blockysize(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + lyr.SetMetadataItem("BLOCKXSIZE", "50000") + lyr.SetMetadataItem("BLOCKYSIZE", "50000") + del index_ds + + with pytest.raises(Exception, match=r"Too big BLOCKXSIZE \* BLOCKYSIZE"): + gdal.Open(index_filename) + + +def test_vrttileindex_wrong_gt(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + lyr.SetMetadataItem("XSIZE", "20") + lyr.SetMetadataItem("YSIZE", "20") + lyr.SetMetadataItem("GEOTRANSFORM", "0,1,0,0,0") + del index_ds + + with pytest.raises( + Exception, + match="GEOTRANSFORM metadata item should be 6 numeric values separated with comma", + ): + gdal.Open(index_filename) + + +def test_vrttileindex_wrong_gt_3rd_term(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + lyr.SetMetadataItem("XSIZE", "20") + lyr.SetMetadataItem("YSIZE", "20") + lyr.SetMetadataItem("GEOTRANSFORM", "0,1,123,0,0,-1") + del index_ds + + with pytest.raises(Exception, match="3rd value of GEOTRANSFORM should be 0"): + gdal.Open(index_filename) + + +def test_vrttileindex_wrong_gt_5th_term(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + lyr.SetMetadataItem("XSIZE", "20") + lyr.SetMetadataItem("YSIZE", "20") + lyr.SetMetadataItem("GEOTRANSFORM", "0,1,0,0,1234,-1") + del index_ds + + with pytest.raises(Exception, match="5th value of GEOTRANSFORM should be 0"): + gdal.Open(index_filename) + + +def test_vrttileindex_wrong_gt_6th_term(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + lyr.SetMetadataItem("XSIZE", "20") + lyr.SetMetadataItem("YSIZE", "20") + lyr.SetMetadataItem("GEOTRANSFORM", "0,1,0,0,0,1") + del index_ds + + with pytest.raises(Exception, match="6th value of GEOTRANSFORM should be < 0"): + gdal.Open(index_filename) + + +@pytest.mark.parametrize("missing_item", [None, "MINX", "MINY", "MAXX", "MAXY"]) +def test_vrttileindex_minx_miny_maxx_maxy(tmp_vsimem, missing_item): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + gt = src_ds.GetGeoTransform() + lyr.SetMetadataItem("RESX", "30") + lyr.SetMetadataItem("RESY", "15") + lyr.SetMetadataItem("BAND_COUNT", "1") + lyr.SetMetadataItem("DATA_TYPE", "Byte") + lyr.SetMetadataItem("COLOR_INTERPRETATION", "Undefined") + if missing_item != "MINX": + lyr.SetMetadataItem("MINX", str(gt[0] + 1 * gt[1])) + if missing_item != "MINY": + lyr.SetMetadataItem("MINY", str(gt[3] + (src_ds.RasterYSize - 4) * gt[5])) + if missing_item != "MAXX": + lyr.SetMetadataItem("MAXX", str(gt[0] + (src_ds.RasterXSize - 3) * gt[1])) + if missing_item != "MAXY": + lyr.SetMetadataItem("MAXY", str(gt[3] + 2 * gt[5])) + lyr.SetMetadataItem("RESAMPLING", "NEAREST") + del index_ds + + if missing_item is None: + vrt_ds = gdal.Open(index_filename) + assert ( + vrt_ds.GetMetadataItem("SCANNED_ONE_FEATURE_AT_OPENING", "__DEBUG__") + == "NO" + ) + assert vrt_ds.RasterXSize == 2 * (src_ds.RasterXSize - 3 - 1) + assert vrt_ds.RasterYSize == 4 * (src_ds.RasterYSize - 4 - 2) + assert vrt_ds.ReadRaster() == src_ds.ReadRaster( + 1, + 2, + vrt_ds.RasterXSize // 2, + vrt_ds.RasterYSize // 4, + buf_xsize=vrt_ds.RasterXSize, + buf_ysize=vrt_ds.RasterYSize, + ) + else: + with pytest.raises(Exception, match=missing_item): + gdal.Open(index_filename) + + +def test_vrttileindex_wrong_resx(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + lyr.SetMetadataItem("RESX", "0") + lyr.SetMetadataItem("RESY", "1") + del index_ds + + with pytest.raises(Exception, match="RESX metadata item should be > 0"): + gdal.Open(index_filename) + + +def test_vrttileindex_wrong_resy(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + lyr.SetMetadataItem("RESX", "1") + lyr.SetMetadataItem("RESY", "0") + del index_ds + + with pytest.raises(Exception, match="RESY metadata item should be > 0"): + gdal.Open(index_filename) + + +def test_vrttileindex_wrong_minx(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + gt = src_ds.GetGeoTransform() + lyr.SetMetadataItem("RESX", "60") + lyr.SetMetadataItem("RESY", "60") + # lyr.SetMetadataItem("MINX", str(gt[0])) + lyr.SetMetadataItem("MINX", str(1e10)) + lyr.SetMetadataItem("MINY", str(gt[3] + src_ds.RasterYSize * gt[5])) + lyr.SetMetadataItem("MAXX", str(gt[0] + src_ds.RasterXSize * gt[1])) + lyr.SetMetadataItem("MAXY", str(gt[3])) + del index_ds + + with pytest.raises(Exception, match="MAXX metadata item should be > MINX"): + gdal.Open(index_filename) + + +def test_vrttileindex_wrong_miny(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + gt = src_ds.GetGeoTransform() + lyr.SetMetadataItem("RESX", "60") + lyr.SetMetadataItem("RESY", "60") + lyr.SetMetadataItem("MINX", str(gt[0])) + # lyr.SetMetadataItem("MINY", str(gt[3] + src_ds.RasterYSize * gt[5])) + lyr.SetMetadataItem("MINY", str(1e10)) + lyr.SetMetadataItem("MAXX", str(gt[0] + src_ds.RasterXSize * gt[1])) + lyr.SetMetadataItem("MAXY", str(gt[3])) + del index_ds + + with pytest.raises(Exception, match="MAXY metadata item should be > MINY"): + gdal.Open(index_filename) + + +def test_vrttileindex_wrong_resx_wrt_min_max_xy(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + gt = src_ds.GetGeoTransform() + lyr.SetMetadataItem("RESX", "1e-10") + lyr.SetMetadataItem("RESY", "60") + lyr.SetMetadataItem("MINX", str(gt[0])) + lyr.SetMetadataItem("MINY", str(gt[3] + src_ds.RasterYSize * gt[5])) + lyr.SetMetadataItem("MAXX", str(gt[0] + src_ds.RasterXSize * gt[1])) + lyr.SetMetadataItem("MAXY", str(gt[3])) + del index_ds + + with pytest.raises(Exception, match="Too small RESX, or wrong layer extent"): + gdal.Open(index_filename) + + +def test_vrttileindex_wrong_resy_wrt_min_max_xy(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + gt = src_ds.GetGeoTransform() + lyr.SetMetadataItem("RESX", "60") + lyr.SetMetadataItem("RESY", "1e-10") + lyr.SetMetadataItem("MINX", str(gt[0])) + lyr.SetMetadataItem("MINY", str(gt[3] + src_ds.RasterYSize * gt[5])) + lyr.SetMetadataItem("MAXX", str(gt[0] + src_ds.RasterXSize * gt[1])) + lyr.SetMetadataItem("MAXY", str(gt[3])) + del index_ds + + with pytest.raises(Exception, match="Too small RESY, or wrong layer extent"): + gdal.Open(index_filename) + + +def test_vrttileindex_invalid_srs(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + lyr.SetMetadataItem("SRS", "invalid") + del index_ds + + with pytest.raises(Exception, match="Invalid SRS"): + gdal.Open(index_filename) + + +def test_vrttileindex_valid_srs(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + lyr.SetMetadataItem("SRS", "EPSG:4267") + del index_ds + + ds = gdal.Open(index_filename) + assert ds.GetSpatialRef().GetAuthorityCode(None) == "4267" + + +def test_vrttileindex_invalid_band_count(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + lyr.SetMetadataItem("BAND_COUNT", "0") + del index_ds + + with pytest.raises(Exception, match="Invalid band count"): + gdal.Open(index_filename) + + +@pytest.mark.parametrize( + "md,error_msg", + [ + ( + {"BAND_COUNT": "2", "COLOR_INTERPRETATION": "Undefined", "NODATA": "None"}, + "Number of data types values found not matching number of bands", + ), + ( + { + "BAND_COUNT": "2", + "COLOR_INTERPRETATION": "Undefined", + "DATA_TYPE": "Byte", + }, + "Number of nodata values found not matching number of bands", + ), + ( + {"BAND_COUNT": "2", "DATA_TYPE": "Byte", "NODATA": "None"}, + "Number of color interpretation values found not matching number of bands", + ), + ], +) +def test_vrttileindex_inconsistent_number_of_values(tmp_vsimem, md, error_msg): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + for item in md: + lyr.SetMetadataItem(item, md[item]) + del index_ds + + with pytest.raises(Exception, match=error_msg): + gdal.Open(index_filename) + + +@pytest.mark.parametrize( + "md,expected_nodata", + [ + ({}, [None]), + ({"NODATA": "none"}, [None]), + ({"NODATA": "1"}, [1]), + ({"NODATA": "1.5"}, [1.5]), + ({"NODATA": "inf"}, [float("inf")]), + ({"NODATA": "-inf"}, [float("-inf")]), + ({"NODATA": "nan"}, [float("nan")]), + ( + { + "BAND_COUNT": "6", + "COLOR_INTERPRETATION": "Undefined", + "DATA_TYPE": "Byte", + "NODATA": "1,2,none,-inf,inf,nan", + }, + [1, 2, None, float("-inf"), float("inf"), float("nan")], + ), + ], +) +def test_vrttileindex_valid_nodata(tmp_vsimem, md, expected_nodata): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + for item in md: + lyr.SetMetadataItem(item, md[item]) + del index_ds + + ds = gdal.Open(index_filename) + for i in range(ds.RasterCount): + got_nd = ds.GetRasterBand(i + 1).GetNoDataValue() + if expected_nodata[i] is None: + assert got_nd is None + elif math.isnan(expected_nodata[i]): + assert math.isnan(got_nd) + else: + assert got_nd == expected_nodata[i] + + +@pytest.mark.parametrize( + "md,error_msg", + [ + ({"NODATA": "invalid"}, "Invalid value for NODATA"), + ({"BAND_COUNT": "2", "NODATA": "0,invalid"}, "Invalid value for NODATA"), + ( + {"BAND_COUNT": "2", "NODATA": "0,0,0"}, + "Number of values in NODATA should be 1 or BAND_COUNT", + ), + ], +) +def test_vrttileindex_invalid_nodata(tmp_vsimem, md, error_msg): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + for item in md: + lyr.SetMetadataItem(item, md[item]) + del index_ds + + with pytest.raises(Exception, match=error_msg): + gdal.Open(index_filename) + + +@pytest.mark.parametrize( + "md,error_msg", + [ + ({"DATA_TYPE": "invalid"}, "Invalid value for DATA_TYPE"), + ( + {"BAND_COUNT": "2", "DATA_TYPE": "byte,invalid"}, + "Invalid value for DATA_TYPE", + ), + ( + {"BAND_COUNT": "2", "DATA_TYPE": "byte,byte,byte"}, + "Number of values in DATA_TYPE should be 1 or BAND_COUNT", + ), + ], +) +def test_vrttileindex_invalid_data_type(tmp_vsimem, md, error_msg): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + for item in md: + lyr.SetMetadataItem(item, md[item]) + del index_ds + + with pytest.raises(Exception, match=error_msg): + gdal.Open(index_filename) + + +@pytest.mark.parametrize( + "md,error_msg", + [ + ({"COLOR_INTERPRETATION": "invalid"}, "Invalid value for COLOR_INTERPRETATION"), + ( + {"BAND_COUNT": "2", "COLOR_INTERPRETATION": "undefined,invalid"}, + "Invalid value for COLOR_INTERPRETATION", + ), + ( + { + "BAND_COUNT": "2", + "COLOR_INTERPRETATION": "undefined,undefined,undefined", + }, + "Number of values in COLOR_INTERPRETATION should be 1 or BAND_COUNT", + ), + ], +) +def test_vrttileindex_invalid_color_interpretation(tmp_vsimem, md, error_msg): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + for item in md: + lyr.SetMetadataItem(item, md[item]) + del index_ds + + with pytest.raises(Exception, match=error_msg): + gdal.Open(index_filename) + + +def test_vrttileindex_no_metadata_rgb(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "small_world.tif")) + index_ds, _ = create_basic_tileindex(index_filename, src_ds) + del index_ds + + vrt_ds = gdal.Open(index_filename) + check_basic(vrt_ds, src_ds) + + +def test_vrttileindex_rgb_left_right(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open("data/small_world.tif") + + left_filename = str(tmp_vsimem / "left.tif") + gdal.Translate(left_filename, src_ds, srcWin=[0, 0, 200, 200]) + + right_filename = str(tmp_vsimem / "right.tif") + gdal.Translate(right_filename, src_ds, srcWin=[200, 0, 200, 200]) + + index_ds, _ = create_basic_tileindex( + index_filename, [gdal.Open(left_filename), gdal.Open(right_filename)] + ) + del index_ds + + vrt_ds = gdal.Open(index_filename) + check_basic(vrt_ds, src_ds) + + index_ds, lyr = create_basic_tileindex( + index_filename, [gdal.Open(left_filename), gdal.Open(right_filename)] + ) + lyr.SetMetadataItem("NODATA", "255") + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.ReadRaster() == src_ds.ReadRaster() + assert vrt_ds.GetMetadataItem("NUMBER_OF_CONTRIBUTING_SOURCES", "__DEBUG__") == "2" + + assert vrt_ds.ReadRaster(199, 100, 2, 1) == src_ds.ReadRaster(199, 100, 2, 1) + assert vrt_ds.GetMetadataItem("NUMBER_OF_CONTRIBUTING_SOURCES", "__DEBUG__") == "2" + + assert vrt_ds.ReadRaster(0, 0, 200, 200) == src_ds.ReadRaster(0, 0, 200, 200) + assert vrt_ds.GetMetadataItem("NUMBER_OF_CONTRIBUTING_SOURCES", "__DEBUG__") == "1" + + assert vrt_ds.ReadRaster(200, 0, 200, 200) == src_ds.ReadRaster(200, 0, 200, 200) + assert vrt_ds.GetMetadataItem("NUMBER_OF_CONTRIBUTING_SOURCES", "__DEBUG__") == "1" + + assert ( + vrt_ds.GetRasterBand(1).GetMetadataItem("Pixel_0_0", "LocationInfo") + == "/vsimem/test_vrttileindex_rgb_left_right/left.tif" + ) + + +def test_vrttileindex_overlapping_sources(tmp_vsimem): + + filename1 = str(tmp_vsimem / "one.tif") + ds = gdal.GetDriverByName("GTiff").Create(filename1, 1, 1) + ds.SetGeoTransform([2, 1, 0, 49, 0, -1]) + ds.GetRasterBand(1).Fill(1) + del ds + + filename2 = str(tmp_vsimem / "two.tif") + ds = gdal.GetDriverByName("GTiff").Create(filename2, 1, 1) + ds.SetGeoTransform([2, 1, 0, 49, 0, -1]) + ds.GetRasterBand(1).Fill(2) + del ds + + # No sorting field: feature with max FID has the priority + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_ds, _ = create_basic_tileindex( + index_filename, [gdal.Open(filename1), gdal.Open(filename2)] + ) + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).Checksum() == 2 + + # Test unsupported sort_field_type = OFTBinary + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + sort_values = [None, None] + index_ds, _ = create_basic_tileindex( + index_filename, + [gdal.Open(filename1), gdal.Open(filename2)], + sort_field_name="z_order", + sort_field_type=ogr.OFTBinary, + sort_values=sort_values, + ) + del index_ds + + with pytest.raises(Exception, match="Unsupported type for field z_order"): + gdal.Open(index_filename) + + # Test non existent SORT_FIELD + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + sort_values = [None, None] + index_ds, lyr = create_basic_tileindex(index_filename, gdal.Open(filename1)) + lyr.SetMetadataItem("SORT_FIELD", "non_existing") + del index_ds + + with pytest.raises(Exception, match="Cannot find field non_existing"): + gdal.Open(index_filename) + + # Test sort_field_type = OFTString + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + sort_values = ["2", "1"] + index_ds, _ = create_basic_tileindex( + index_filename, + [gdal.Open(filename1), gdal.Open(filename2)], + sort_field_name="z_order", + sort_field_type=ogr.OFTString, + sort_values=sort_values, + ) + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).Checksum() == 1, sort_values + + sort_values.reverse() + index_ds, _ = create_basic_tileindex( + index_filename, + [gdal.Open(filename2), gdal.Open(filename1)], + sort_field_name="z_order", + sort_field_type=ogr.OFTString, + sort_values=sort_values, + ) + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).Checksum() == 1, sort_values + + # Test sort_field_type = OFTString + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + sort_values = ["1", "1"] + index_ds, _ = create_basic_tileindex( + index_filename, + [gdal.Open(filename1), gdal.Open(filename2)], + sort_field_name="z_order", + sort_field_type=ogr.OFTString, + sort_values=sort_values, + ) + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).Checksum() == 2, sort_values + + # Test sort_field_type = OFTInteger + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + sort_values = [2, 1] + index_ds, _ = create_basic_tileindex( + index_filename, + [gdal.Open(filename1), gdal.Open(filename2)], + sort_field_name="z_order", + sort_field_type=ogr.OFTInteger, + sort_values=sort_values, + ) + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).Checksum() == 1, sort_values + + sort_values.reverse() + index_ds, _ = create_basic_tileindex( + index_filename, + [gdal.Open(filename2), gdal.Open(filename1)], + sort_field_name="z_order", + sort_field_type=ogr.OFTInteger, + sort_values=sort_values, + ) + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).Checksum() == 1, sort_values + + # Test sort_field_type = OFTInteger64 + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + sort_values = [1234567890123 + 2, 1234567890123 + 1] + index_ds, _ = create_basic_tileindex( + index_filename, + [gdal.Open(filename1), gdal.Open(filename2)], + sort_field_name="z_order", + sort_field_type=ogr.OFTInteger64, + sort_values=sort_values, + ) + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).Checksum() == 1, sort_values + + sort_values.reverse() + index_ds, _ = create_basic_tileindex( + index_filename, + [gdal.Open(filename2), gdal.Open(filename1)], + sort_field_name="z_order", + sort_field_type=ogr.OFTInteger64, + sort_values=sort_values, + ) + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).Checksum() == 1, sort_values + + # Test sort_field_type = OFTReal + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + sort_values = [2.5, 1.5] + index_ds, _ = create_basic_tileindex( + index_filename, + [gdal.Open(filename1), gdal.Open(filename2)], + sort_field_name="z_order", + sort_field_type=ogr.OFTReal, + sort_values=sort_values, + ) + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).Checksum() == 1, sort_values + + sort_values.reverse() + index_ds, _ = create_basic_tileindex( + index_filename, + [gdal.Open(filename2), gdal.Open(filename1)], + sort_field_name="z_order", + sort_field_type=ogr.OFTReal, + sort_values=sort_values, + ) + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).Checksum() == 1, sort_values + + # Test sort_field_type = OFTDate + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + for sort_values in [ + ["2023-01-01", "2022-12-31"], + ["2023-02-01", "2023-01-31"], + ["2023-01-02", "2023-01-01"], + ]: + index_ds, _ = create_basic_tileindex( + index_filename, + [gdal.Open(filename1), gdal.Open(filename2)], + sort_field_name="z_order", + sort_field_type=ogr.OFTDate, + sort_values=sort_values, + ) + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).Checksum() == 1, sort_values + + sort_values.reverse() + index_ds, _ = create_basic_tileindex( + index_filename, + [gdal.Open(filename2), gdal.Open(filename1)], + sort_field_name="z_order", + sort_field_type=ogr.OFTDate, + sort_values=sort_values, + ) + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).Checksum() == 1, sort_values + + # Test sort_field_type = OFTDateTime + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + for sort_values in [ + ["2023-01-01T00:00:00", "2022-12-31T23:59:59"], + ["2023-02-01T00:00:00", "2023-01-31T23:59:59"], + ["2023-01-02T00:00:00", "2023-01-01T23:59:59"], + ["2023-01-01T01:00:00", "2023-01-01T00:59:59"], + ["2023-01-01T00:01:00", "2023-01-01T00:00:59"], + ["2023-01-01T00:00:01", "2023-01-01T00:00:00"], + ]: + index_ds, _ = create_basic_tileindex( + index_filename, + [gdal.Open(filename1), gdal.Open(filename2)], + sort_field_name="z_order", + sort_field_type=ogr.OFTDateTime, + sort_values=sort_values, + ) + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).Checksum() == 1, sort_values + + sort_values.reverse() + index_ds, _ = create_basic_tileindex( + index_filename, + [gdal.Open(filename2), gdal.Open(filename1)], + sort_field_name="z_order", + sort_field_type=ogr.OFTDate, + sort_values=sort_values, + ) + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).Checksum() == 1, sort_values + + # Test SORT_FIELD_ASC=NO + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_ds, lyr = create_basic_tileindex( + index_filename, + [gdal.Open(filename1), gdal.Open(filename2)], + sort_field_name="z_order", + sort_field_type=ogr.OFTInteger, + sort_values=[2, 1], + ) + lyr.SetMetadataItem("SORT_FIELD_ASC", "NO") + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).Checksum() == 2, sort_values + + +def test_vrttileindex_no_source(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_ds, lyr = create_basic_tileindex(index_filename, []) + lyr.SetMetadataItem("XSIZE", "2") + lyr.SetMetadataItem("YSIZE", "3") + lyr.SetMetadataItem("GEOTRANSFORM", "10,1,0,20,0,-1") + lyr.SetMetadataItem("BAND_COUNT", "2") + lyr.SetMetadataItem("NODATA", "255,254") + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.ReadRaster() == (b"\xFF" * 6) + (b"\xFE" * 6) + + assert ( + vrt_ds.GetRasterBand(1).GetMetadataItem("Pixel_0_0", "LocationInfo") + == "" + ) + assert ( + vrt_ds.GetRasterBand(1).GetMetadataItem("Pixel_1_2", "LocationInfo") + == "" + ) + assert vrt_ds.GetRasterBand(1).GetMetadataItem("Pixel_-1_0", "LocationInfo") is None + assert vrt_ds.GetRasterBand(1).GetMetadataItem("Pixel_0_-1", "LocationInfo") is None + assert vrt_ds.GetRasterBand(1).GetMetadataItem("Pixel_1_3", "LocationInfo") is None + assert vrt_ds.GetRasterBand(1).GetMetadataItem("Pixel_2_2", "LocationInfo") is None + + assert ( + vrt_ds.GetRasterBand(1).GetMetadataItem("GeoPixel_10_20", "LocationInfo") + == "" + ) + assert ( + vrt_ds.GetRasterBand(1).GetMetadataItem("GeoPixel_9.99_20", "LocationInfo") + is None + ) + assert ( + vrt_ds.GetRasterBand(1).GetMetadataItem("GeoPixel_10_20.01", "LocationInfo") + is None + ) + + +def test_vrttileindex_invalid_source(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) + lyr = index_ds.CreateLayer("index", geom_type=ogr.wkbPolygon) + lyr.CreateField(ogr.FieldDefn("location")) + + f = ogr.Feature(lyr.GetLayerDefn()) + f.SetGeometry(ogr.CreateGeometryFromWkt("POLYGON((10 20,11 20,11 19,10 19,10 20))")) + # Location not set + lyr.CreateFeature(f) + f = None + + f = ogr.Feature(lyr.GetLayerDefn()) + f.SetGeometry(ogr.CreateGeometryFromWkt("POLYGON((10 20,11 20,11 19,10 19,10 20))")) + # Invalid location + f["location"] = "/i/do/not/exist" + lyr.CreateFeature(f) + f = None + + lyr.SetMetadataItem("XSIZE", "1") + lyr.SetMetadataItem("YSIZE", "1") + lyr.SetMetadataItem("GEOTRANSFORM", "10,1,0,20,0,-1") + lyr.SetMetadataItem("BAND_COUNT", "1") + del index_ds + + vrt_ds = gdal.Open(index_filename) + with pytest.raises(Exception): + vrt_ds.ReadRaster() + + +def test_vrttileindex_source_relative_location(tmp_vsimem): + + tile_filename = str(tmp_vsimem / "tile.tif") + ds = gdal.GetDriverByName("GTiff").Create(tile_filename, 1, 1) + ds.SetGeoTransform([10, 1, 0, 20, 0, -1]) + ds.GetRasterBand(1).Fill(255) + del ds + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) + lyr = index_ds.CreateLayer("index", geom_type=ogr.wkbPolygon) + lyr.CreateField(ogr.FieldDefn("location")) + + f = ogr.Feature(lyr.GetLayerDefn()) + f.SetGeometry(ogr.CreateGeometryFromWkt("POLYGON((10 20,11 20,11 19,10 19,10 20))")) + f["location"] = "tile.tif" + lyr.CreateFeature(f) + f = None + + lyr.SetMetadataItem("XSIZE", "1") + lyr.SetMetadataItem("YSIZE", "1") + lyr.SetMetadataItem("GEOTRANSFORM", "10,1,0,20,0,-1") + lyr.SetMetadataItem("BAND_COUNT", "1") + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.ReadRaster() == b"\xFF" + + +def test_vrttileindex_source_lacks_bands(tmp_vsimem): + + tile_filename = str(tmp_vsimem / "tile.tif") + ds = gdal.GetDriverByName("GTiff").Create(tile_filename, 1, 1) + ds.SetGeoTransform([10, 1, 0, 20, 0, -1]) + del ds + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) + lyr = index_ds.CreateLayer("index", geom_type=ogr.wkbPolygon) + lyr.CreateField(ogr.FieldDefn("location")) + + f = ogr.Feature(lyr.GetLayerDefn()) + f.SetGeometry(ogr.CreateGeometryFromWkt("POLYGON((10 20,11 20,11 19,10 19,10 20))")) + f["location"] = tile_filename + lyr.CreateFeature(f) + f = None + + lyr.SetMetadataItem("XSIZE", "1") + lyr.SetMetadataItem("YSIZE", "1") + lyr.SetMetadataItem("GEOTRANSFORM", "10,1,0,20,0,-1") + lyr.SetMetadataItem("BAND_COUNT", "2") + del index_ds + + vrt_ds = gdal.Open(index_filename) + with pytest.raises(Exception, match="has not enough bands"): + vrt_ds.ReadRaster() + + +def test_vrttileindex_source_lacks_bands_and_relative_location(tmp_vsimem): + + tile_filename = str(tmp_vsimem / "tile.tif") + ds = gdal.GetDriverByName("GTiff").Create(tile_filename, 1, 1) + ds.SetGeoTransform([10, 1, 0, 20, 0, -1]) + del ds + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) + lyr = index_ds.CreateLayer("index", geom_type=ogr.wkbPolygon) + lyr.CreateField(ogr.FieldDefn("location")) + + f = ogr.Feature(lyr.GetLayerDefn()) + f.SetGeometry(ogr.CreateGeometryFromWkt("POLYGON((10 20,11 20,11 19,10 19,10 20))")) + f["location"] = "tile.tif" + lyr.CreateFeature(f) + f = None + + lyr.SetMetadataItem("XSIZE", "1") + lyr.SetMetadataItem("YSIZE", "1") + lyr.SetMetadataItem("GEOTRANSFORM", "10,1,0,20,0,-1") + lyr.SetMetadataItem("BAND_COUNT", "2") + del index_ds + + vrt_ds = gdal.Open(index_filename) + with pytest.raises(Exception, match="has not enough bands"): + vrt_ds.ReadRaster() + + +@pytest.mark.require_driver("netCDF") +def test_vrttileindex_source_netcdf_subdataset_absolute(tmp_path): + + index_filename = str(tmp_path / "index.vrt.gpkg") + src_ds = gdal.Open( + 'netCDF:"' + os.path.join(os.getcwd(), "data", "netcdf", "byte.nc") + '":Band1' + ) + index_ds, _ = create_basic_tileindex(index_filename, [src_ds]) + del index_ds + + assert gdal.Open(index_filename) is not None + + +@pytest.mark.require_driver("netCDF") +def test_vrttileindex_source_netcdf_subdataset_relative(tmp_path): + + tmp_netcdf_filename = str(tmp_path / "byte.nc") + shutil.copy("data/netcdf/byte.nc", tmp_netcdf_filename) + index_filename = str(tmp_path / "index.vrt.gpkg") + cwd = os.getcwd() + try: + os.chdir(tmp_path) + src_ds = gdal.Open('netCDF:"byte.nc":Band1') + finally: + os.chdir(cwd) + index_ds, _ = create_basic_tileindex(index_filename, [src_ds]) + del index_ds + + assert gdal.Open(index_filename) is not None + + +def test_vrttileindex_single_source_nodata_same_as_vrt(tmp_vsimem): + + filename1 = str(tmp_vsimem / "one.tif") + ds = gdal.GetDriverByName("GTiff").Create(filename1, 3, 1, 3) + ds.SetGeoTransform([2, 1, 0, 49, 0, -1]) + ds.GetRasterBand(1).WriteRaster(0, 0, 3, 1, b"\x01\x02\x01") + ds.GetRasterBand(2).WriteRaster(0, 0, 3, 1, b"\x01\x03\x01") + ds.GetRasterBand(3).WriteRaster(0, 0, 3, 1, b"\x01\x04\x01") + ds.GetRasterBand(1).SetNoDataValue(1) + ds.GetRasterBand(2).SetNoDataValue(1) + ds.GetRasterBand(3).SetNoDataValue(1) + del ds + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_ds, lyr = create_basic_tileindex(index_filename, gdal.Open(filename1)) + lyr.SetMetadataItem("NODATA", "1") + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.ReadRaster() == b"\x01\x02\x01" + b"\x01\x03\x01" + b"\x01\x04\x01" + assert ( + vrt_ds.GetRasterBand(1).GetMetadataItem("Pixel_0_0", "LocationInfo") + == "/vsimem/test_vrttileindex_single_source_nodata_same_as_vrt/one.tif" + ) + + +def test_vrttileindex_overlapping_sources_nodata(tmp_vsimem): + + filename1 = str(tmp_vsimem / "one.tif") + ds = gdal.GetDriverByName("GTiff").Create(filename1, 3, 1) + ds.SetGeoTransform([2, 1, 0, 49, 0, -1]) + ds.GetRasterBand(1).WriteRaster(0, 0, 3, 1, b"\x01\x02\x01") + ds.GetRasterBand(1).SetNoDataValue(1) + del ds + + filename2 = str(tmp_vsimem / "two.tif") + ds = gdal.GetDriverByName("GTiff").Create(filename2, 3, 1) + ds.SetGeoTransform([2, 1, 0, 49, 0, -1]) + ds.GetRasterBand(1).WriteRaster(0, 0, 3, 1, b"\x04\x03\x03") + ds.GetRasterBand(1).SetNoDataValue(3) + del ds + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + sort_values = [1, 2] + index_ds, lyr = create_basic_tileindex( + index_filename, + [gdal.Open(filename1), gdal.Open(filename2)], + sort_field_name="z_order", + sort_field_type=ogr.OFTInteger, + sort_values=sort_values, + ) + lyr.SetMetadataItem("NODATA", "255") + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).ReadRaster() == b"\x04\x02\xFF" + + assert ( + vrt_ds.GetRasterBand(1).GetMetadataItem("Pixel_0_0", "LocationInfo") + == "/vsimem/test_vrttileindex_overlapping_sources_nodata/one.tif/vsimem/test_vrttileindex_overlapping_sources_nodata/two.tif" + ) + + +def test_vrttileindex_on_the_fly_rgb_color_table_expansion(tmp_vsimem): + + tile_filename = str(tmp_vsimem / "color_table.tif") + tile_ds = gdal.GetDriverByName("GTiff").Create(tile_filename, 1, 1) + tile_ds.SetGeoTransform([2, 1, 0, 49, 0, -1]) + ct = gdal.ColorTable() + ct.SetColorEntry(0, (1, 2, 3, 255)) + assert tile_ds.GetRasterBand(1).SetRasterColorTable(ct) == gdal.CE_None + tile_ds = None + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_ds, lyr = create_basic_tileindex(index_filename, gdal.Open(tile_filename)) + lyr.SetMetadataItem("BAND_COUNT", "3") + lyr.SetMetadataItem("COLOR_INTERPRETATION", "Red,Green,Blue") + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).ReadRaster() == b"\x01" + assert vrt_ds.GetRasterBand(2).ReadRaster() == b"\x02" + assert vrt_ds.GetRasterBand(3).ReadRaster() == b"\x03" + + +def test_vrttileindex_on_the_fly_rgba_color_table_expansion(tmp_vsimem): + + tile_filename = str(tmp_vsimem / "color_table.tif") + tile_ds = gdal.GetDriverByName("GTiff").Create(tile_filename, 1, 1) + tile_ds.SetGeoTransform([2, 1, 0, 49, 0, -1]) + ct = gdal.ColorTable() + ct.SetColorEntry(0, (1, 2, 3, 255)) + assert tile_ds.GetRasterBand(1).SetRasterColorTable(ct) == gdal.CE_None + tile_ds = None + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_ds, lyr = create_basic_tileindex(index_filename, gdal.Open(tile_filename)) + lyr.SetMetadataItem("BAND_COUNT", "4") + lyr.SetMetadataItem("COLOR_INTERPRETATION", "Red,Green,Blue,Alpha") + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).ReadRaster() == b"\x01" + assert vrt_ds.GetRasterBand(2).ReadRaster() == b"\x02" + assert vrt_ds.GetRasterBand(3).ReadRaster() == b"\x03" + assert vrt_ds.GetRasterBand(4).ReadRaster() == b"\xFF" + + +def test_vrttileindex_on_the_fly_warping(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + warped_ds = gdal.Warp("", src_ds, format="VRT", dstSRS="EPSG:4267") + + index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) + lyr = index_ds.CreateLayer("index", srs=warped_ds.GetSpatialRef()) + lyr.CreateField(ogr.FieldDefn("location")) + f = ogr.Feature(lyr.GetLayerDefn()) + warped_gt = warped_ds.GetGeoTransform() + minx = warped_gt[0] + maxx = minx + warped_ds.RasterXSize * warped_gt[1] + maxy = warped_gt[3] + miny = maxy + warped_ds.RasterYSize * warped_gt[5] + f["location"] = src_ds.GetDescription() + f.SetGeometry( + ogr.CreateGeometryFromWkt( + f"POLYGON(({minx} {miny},{minx} {maxy},{maxx} {maxy},{maxx} {miny},{minx} {miny}))" + ) + ) + lyr.CreateFeature(f) + lyr.SetMetadataItem("RESAMPLING", "CUBIC") + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).Checksum() == 4772 + + # Check that we add transparency to the warped source + index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) + lyr = index_ds.CreateLayer("index", srs=warped_ds.GetSpatialRef()) + lyr.CreateField(ogr.FieldDefn("location")) + f = ogr.Feature(lyr.GetLayerDefn()) + minx -= 5 * warped_gt[1] + maxx += 5 * warped_gt[1] + maxy -= 5 * warped_gt[5] + miny += 5 * warped_gt[5] + f["location"] = src_ds.GetDescription() + f.SetGeometry( + ogr.CreateGeometryFromWkt( + f"POLYGON(({minx} {miny},{minx} {maxy},{maxx} {maxy},{maxx} {miny},{minx} {miny}))" + ) + ) + lyr.CreateFeature(f) + lyr.SetMetadataItem("NODATA", "254") + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).ReadRaster(0, 0, 1, 1) == b"\xFE" + + +def test_vrttileindex_single_source_alpha_no_dest_nodata(tmp_vsimem): + + filename1 = str(tmp_vsimem / "one.tif") + ds = gdal.GetDriverByName("GTiff").Create(filename1, 2, 1, 2) + ds.SetGeoTransform([2, 1, 0, 49, 0, -1]) + ds.GetRasterBand(2).SetColorInterpretation(gdal.GCI_AlphaBand) + ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, b"\x01\x02") + ds.GetRasterBand(2).WriteRaster(0, 0, 2, 1, b"\xFF\x00") + del ds + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_ds, _ = create_basic_tileindex(index_filename, gdal.Open(filename1)) + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.RasterCount == 2 + assert ( + vrt_ds.GetRasterBand(1).GetMaskFlags() == gdal.GMF_PER_DATASET | gdal.GMF_ALPHA + ) + assert vrt_ds.GetRasterBand(1).GetMaskBand().GetBand() == 2 + assert vrt_ds.GetRasterBand(1).ReadRaster() == b"\x01\x02" + assert vrt_ds.GetRasterBand(2).ReadRaster() == b"\xFF\x00" + + +def test_vrttileindex_overlapping_opaque_sources(tmp_vsimem): + + filename1 = str(tmp_vsimem / "one.tif") + ds = gdal.GetDriverByName("GTiff").Create(filename1, 1, 1, 1) + ds.SetGeoTransform([2, 1, 0, 49, 0, -1]) + ds.GetRasterBand(1).WriteRaster(0, 0, 1, 1, b"\x01") + del ds + + filename2 = str(tmp_vsimem / "two.tif") + ds = gdal.GetDriverByName("GTiff").Create(filename2, 1, 1, 1) + ds.SetGeoTransform([2, 1, 0, 49, 0, -1]) + ds.GetRasterBand(1).WriteRaster(0, 0, 1, 1, b"\x02") + del ds + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + sort_values = [1, 2] + index_ds, _ = create_basic_tileindex( + index_filename, + [gdal.Open(filename1), gdal.Open(filename2)], + sort_field_name="z_order", + sort_field_type=ogr.OFTInteger, + sort_values=sort_values, + ) + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).ReadRaster() == b"\x02" + assert vrt_ds.GetMetadataItem("NUMBER_OF_CONTRIBUTING_SOURCES", "__DEBUG__") == "1" + + assert ( + vrt_ds.GetRasterBand(1).GetMetadataItem("Pixel_0_0", "LocationInfo") + == "/vsimem/test_vrttileindex_overlapping_opaque_sources/two.tif" + ) + + +def test_vrttileindex_overlapping_sources_alpha_2x1(tmp_vsimem): + + filename1 = str(tmp_vsimem / "one.tif") + ds = gdal.GetDriverByName("GTiff").Create(filename1, 2, 1, 2) + ds.SetGeoTransform([2, 1, 0, 49, 0, -1]) + ds.GetRasterBand(2).SetColorInterpretation(gdal.GCI_AlphaBand) + ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, b"\x01\x02") + ds.GetRasterBand(2).WriteRaster(0, 0, 2, 1, b"\xFF\x00") + del ds + + filename2 = str(tmp_vsimem / "two.tif") + ds = gdal.GetDriverByName("GTiff").Create(filename2, 2, 1, 2) + ds.SetGeoTransform([2, 1, 0, 49, 0, -1]) + ds.GetRasterBand(2).SetColorInterpretation(gdal.GCI_AlphaBand) + ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, b"\x03\x04") + ds.GetRasterBand(2).WriteRaster(0, 0, 2, 1, b"\x00\xFE") + del ds + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + sort_values = [1, 2] + index_ds, _ = create_basic_tileindex( + index_filename, + [gdal.Open(filename1), gdal.Open(filename2)], + sort_field_name="z_order", + sort_field_type=ogr.OFTInteger, + sort_values=sort_values, + ) + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.RasterCount == 2 + assert ( + vrt_ds.GetRasterBand(1).GetMaskFlags() == gdal.GMF_PER_DATASET | gdal.GMF_ALPHA + ) + assert vrt_ds.GetRasterBand(1).GetMaskBand().GetBand() == 2 + assert vrt_ds.GetRasterBand(1).ReadRaster() == b"\x01\x04" + assert vrt_ds.GetRasterBand(2).ReadRaster() == b"\xFF\xFE" + + assert struct.unpack( + "H" * 2, vrt_ds.GetRasterBand(1).ReadRaster(buf_type=gdal.GDT_UInt16) + ) == (1, 4) + + assert vrt_ds.GetRasterBand(1).ReadRaster(0, 0, 1, 1) == b"\x01" + assert vrt_ds.GetRasterBand(1).ReadRaster(1, 0, 1, 1) == b"\x04" + assert vrt_ds.GetRasterBand(2).ReadRaster(0, 0, 1, 1) == b"\xFF" + assert vrt_ds.GetRasterBand(2).ReadRaster(1, 0, 1, 1) == b"\xFE" + + assert vrt_ds.ReadRaster() == b"\x01\x04\xFF\xFE" + assert struct.unpack("H" * 4, vrt_ds.ReadRaster(buf_type=gdal.GDT_UInt16)) == ( + 1, + 4, + 255, + 254, + ) + + +def test_vrttileindex_overlapping_sources_alpha_1x2(tmp_vsimem): + + filename1 = str(tmp_vsimem / "one.tif") + ds = gdal.GetDriverByName("GTiff").Create(filename1, 1, 2, 2) + ds.SetGeoTransform([2, 1, 0, 49, 0, -1]) + ds.GetRasterBand(2).SetColorInterpretation(gdal.GCI_AlphaBand) + ds.GetRasterBand(1).WriteRaster(0, 0, 1, 2, b"\x01\x02") + ds.GetRasterBand(2).WriteRaster(0, 0, 1, 2, b"\xFF\x00") + del ds + + filename2 = str(tmp_vsimem / "two.tif") + ds = gdal.GetDriverByName("GTiff").Create(filename2, 1, 2, 2) + ds.SetGeoTransform([2, 1, 0, 49, 0, -1]) + ds.GetRasterBand(2).SetColorInterpretation(gdal.GCI_AlphaBand) + ds.GetRasterBand(1).WriteRaster(0, 0, 1, 2, b"\x03\x04") + ds.GetRasterBand(2).WriteRaster(0, 0, 1, 2, b"\x00\xFE") + del ds + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + sort_values = [1, 2] + index_ds, _ = create_basic_tileindex( + index_filename, + [gdal.Open(filename1), gdal.Open(filename2)], + sort_field_name="z_order", + sort_field_type=ogr.OFTInteger, + sort_values=sort_values, + ) + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.RasterCount == 2 + assert vrt_ds.GetRasterBand(1).ReadRaster() == b"\x01\x04" + assert vrt_ds.GetRasterBand(2).ReadRaster() == b"\xFF\xFE" + + assert struct.unpack( + "H" * 2, vrt_ds.GetRasterBand(1).ReadRaster(buf_type=gdal.GDT_UInt16) + ) == (1, 4) + + assert vrt_ds.GetRasterBand(1).ReadRaster(0, 0, 1, 1) == b"\x01" + assert vrt_ds.GetRasterBand(1).ReadRaster(0, 1, 1, 1) == b"\x04" + assert vrt_ds.GetRasterBand(2).ReadRaster(0, 0, 1, 1) == b"\xFF" + assert vrt_ds.GetRasterBand(2).ReadRaster(0, 1, 1, 1) == b"\xFE" + + assert vrt_ds.ReadRaster() == b"\x01\x04\xFF\xFE" + assert struct.unpack("H" * 4, vrt_ds.ReadRaster(buf_type=gdal.GDT_UInt16)) == ( + 1, + 4, + 255, + 254, + ) + + +def test_vrttileindex_overlapping_sources_alpha_sse2_optim(tmp_vsimem): + + filename1 = str(tmp_vsimem / "one.tif") + ds = gdal.GetDriverByName("GTiff").Create(filename1, 17, 1, 2) + ds.SetGeoTransform([2, 1, 0, 49, 0, -1]) + ds.GetRasterBand(2).SetColorInterpretation(gdal.GCI_AlphaBand) + ds.GetRasterBand(1).WriteRaster( + 0, + 0, + 17, + 1, + b"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F\x00\x01", + ) + ds.GetRasterBand(2).WriteRaster( + 0, + 0, + 17, + 1, + b"\xFF\x00\xFF\x00\xFF\x00\xFF\x00\xFF\x00\xFF\x00\xFF\x00\xFF\x00\xFF", + ) + del ds + + filename2 = str(tmp_vsimem / "two.tif") + ds = gdal.GetDriverByName("GTiff").Create(filename2, 17, 1, 2) + ds.SetGeoTransform([2, 1, 0, 49, 0, -1]) + ds.GetRasterBand(2).SetColorInterpretation(gdal.GCI_AlphaBand) + ds.GetRasterBand(1).WriteRaster( + 0, + 0, + 17, + 1, + b"\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F\x01\x02\x03\x04\x05\x06\x07\x08", + ) + ds.GetRasterBand(2).WriteRaster( + 0, + 0, + 17, + 1, + b"\x00\xFE\x00\xFE\x00\xFE\x00\xFE\x00\xFE\x00\xFE\x00\xFE\x00\xFE\x00", + ) + del ds + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + sort_values = [1, 2] + index_ds, _ = create_basic_tileindex( + index_filename, + [gdal.Open(filename1), gdal.Open(filename2)], + sort_field_name="z_order", + sort_field_type=ogr.OFTInteger, + sort_values=sort_values, + ) + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.RasterCount == 2 + assert ( + vrt_ds.GetRasterBand(1).ReadRaster() + == b"\x01\x08\x03\x0A\x05\x0C\x07\x0E\x09\x01\x0B\x03\x0D\x05\x0F\x07\x01" + ) + assert ( + vrt_ds.GetRasterBand(2).ReadRaster() + == b"\xFF\xFE\xFF\xFE\xFF\xFE\xFF\xFE\xFF\xFE\xFF\xFE\xFF\xFE\xFF\xFE\xFF" + ) + + +def test_vrttileindex_mix_rgb_rgba(tmp_vsimem): + + filename1 = str(tmp_vsimem / "rgba.tif") + ds = gdal.GetDriverByName("GTiff").Create(filename1, 2, 1, 4) + ds.SetGeoTransform([2, 1, 0, 49, 0, -1]) + ds.GetRasterBand(4).SetColorInterpretation(gdal.GCI_AlphaBand) + ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, b"\x01\x02") + ds.GetRasterBand(2).WriteRaster(0, 0, 2, 1, b"\x03\x04") + ds.GetRasterBand(3).WriteRaster(0, 0, 2, 1, b"\x05\x06") + ds.GetRasterBand(4).WriteRaster(0, 0, 2, 1, b"\xFE\x00") + del ds + + filename2 = str(tmp_vsimem / "rgb.tif") + ds = gdal.GetDriverByName("GTiff").Create(filename2, 2, 1, 3) + ds.SetGeoTransform([2, 1, 0, 49, 0, -1]) + ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, b"\x07\x08") + ds.GetRasterBand(2).WriteRaster(0, 0, 2, 1, b"\x0A\x0B") + ds.GetRasterBand(3).WriteRaster(0, 0, 2, 1, b"\x0C\x0D") + del ds + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + sort_values = [1, 2] + index_ds, _ = create_basic_tileindex( + index_filename, + [gdal.Open(filename1), gdal.Open(filename2)], + sort_field_name="z_order", + sort_field_type=ogr.OFTInteger, + sort_values=sort_values, + ) + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.RasterCount == 4 + assert vrt_ds.GetRasterBand(1).ReadRaster() == b"\x07\x08" + assert vrt_ds.GetRasterBand(2).ReadRaster() == b"\x0A\x0B" + assert vrt_ds.GetRasterBand(3).ReadRaster() == b"\x0C\x0D" + assert vrt_ds.GetRasterBand(4).ReadRaster() == b"\xFF\xFF" + assert vrt_ds.ReadRaster() == b"\x07\x08" + b"\x0A\x0B" + b"\x0C\x0D" + b"\xFF\xFF" + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + sort_values = [2, 1] + index_ds, _ = create_basic_tileindex( + index_filename, + [gdal.Open(filename1), gdal.Open(filename2)], + sort_field_name="z_order", + sort_field_type=ogr.OFTInteger, + sort_values=sort_values, + ) + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.RasterCount == 4 + assert vrt_ds.GetRasterBand(1).ReadRaster() == b"\x01\x08" + assert vrt_ds.GetRasterBand(2).ReadRaster() == b"\x03\x0B" + assert vrt_ds.GetRasterBand(3).ReadRaster() == b"\x05\x0D" + assert vrt_ds.GetRasterBand(4).ReadRaster() == b"\xFE\xFF" + assert vrt_ds.ReadRaster() == b"\x01\x08" + b"\x03\x0B" + b"\x05\x0D" + b"\xFE\xFF" + + +def test_vrttileindex_overlapping_sources_mask_band(tmp_vsimem): + + filename1 = str(tmp_vsimem / "one.tif") + ds = gdal.GetDriverByName("GTiff").Create(filename1, 2, 1) + ds.SetGeoTransform([2, 1, 0, 49, 0, -1]) + ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, b"\x01\x02") + ds.CreateMaskBand(gdal.GMF_PER_DATASET) + ds.GetRasterBand(1).GetMaskBand().WriteRaster(0, 0, 2, 1, b"\xFF\x00") + del ds + + filename2 = str(tmp_vsimem / "two.tif") + ds = gdal.GetDriverByName("GTiff").Create(filename2, 2, 1) + ds.SetGeoTransform([2, 1, 0, 49, 0, -1]) + ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, b"\x03\x04") + ds.CreateMaskBand(gdal.GMF_PER_DATASET) + ds.GetRasterBand(1).GetMaskBand().WriteRaster(0, 0, 2, 1, b"\x00\xFE") + del ds + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + sort_values = [1, 2] + index_ds, _ = create_basic_tileindex( + index_filename, + [gdal.Open(filename1), gdal.Open(filename2)], + sort_field_name="z_order", + sort_field_type=ogr.OFTInteger, + sort_values=sort_values, + ) + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.RasterCount == 1 + assert vrt_ds.GetRasterBand(1).GetMaskFlags() == gdal.GMF_PER_DATASET + assert vrt_ds.GetRasterBand(1).ReadRaster() == b"\x01\x04" + assert vrt_ds.GetRasterBand(1).GetMaskBand().ReadRaster() == b"\xFF\xFE" + + # Test the mask band of the mask band... + assert vrt_ds.GetRasterBand(1).GetMaskBand().GetMaskFlags() == gdal.GMF_ALL_VALID + assert ( + vrt_ds.GetRasterBand(1).GetMaskBand().GetMaskBand().ReadRaster() == b"\xFF\xFF" + ) + + assert struct.unpack( + "H" * 2, vrt_ds.GetRasterBand(1).ReadRaster(buf_type=gdal.GDT_UInt16) + ) == (1, 4) + + assert struct.unpack( + "H" * 2, + vrt_ds.GetRasterBand(1).GetMaskBand().ReadRaster(buf_type=gdal.GDT_UInt16), + ) == (255, 254) + + +def test_vrttileindex_mask_band_explicit(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + lyr.SetMetadataItem("MASK_BAND", "YES") + del index_ds + + vrt_ds = gdal.Open(index_filename) + check_basic(vrt_ds, src_ds) + assert vrt_ds.RasterCount == 1 + assert vrt_ds.GetRasterBand(1).GetMaskFlags() == gdal.GMF_PER_DATASET + assert vrt_ds.GetRasterBand(1).ReadRaster() == src_ds.GetRasterBand(1).ReadRaster() + assert vrt_ds.GetRasterBand(1).GetMaskBand().ReadRaster() == b"\xFF" * (20 * 20) + + +def test_vrttileindex_flushcache(tmp_vsimem): + + filename1 = str(tmp_vsimem / "one.tif") + ds = gdal.GetDriverByName("GTiff").Create(filename1, 1, 1, 1) + ds.SetGeoTransform([2, 1, 0, 49, 0, -1]) + ds.GetRasterBand(1).WriteRaster(0, 0, 1, 1, b"\x01") + del ds + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_ds, _ = create_basic_tileindex( + index_filename, + gdal.Open(filename1), + ) + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).ReadRaster() == b"\x01" + + # Modify source + src_ds = gdal.Open(filename1, gdal.GA_Update) + src_ds.GetRasterBand(1).WriteRaster(0, 0, 1, 1, b"\x02") + src_ds = None + + # We still access a previously cached value + assert vrt_ds.GetRasterBand(1).ReadRaster() == b"\x01" + + # Now flush the VRT cache and we should get the updated value + vrt_ds.FlushCache() + + assert vrt_ds.GetRasterBand(1).ReadRaster() == b"\x02" + + +def test_vrttileindex_ovr_factor(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + lyr.SetMetadataItem("MASK_BAND", "YES") + lyr.SetMetadataItem("OVERVIEW_1_FACTOR", "2") + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 1 + assert vrt_ds.GetRasterBand(1).GetOverview(-1) is None + assert vrt_ds.GetRasterBand(1).GetOverview(1) is None + ovr = vrt_ds.GetRasterBand(1).GetOverview(0) + assert ovr + assert ovr.XSize == 10 + assert ovr.YSize == 10 + assert ovr.ReadRaster() == src_ds.ReadRaster(buf_xsize=10, buf_ysize=10) + + mask_band = vrt_ds.GetRasterBand(1).GetMaskBand() + assert mask_band.GetOverviewCount() == 1 + assert mask_band.GetOverview(-1) is None + assert mask_band.GetOverview(1) is None + ovr = mask_band.GetOverview(0) + assert ovr + assert ovr.XSize == 10 + assert ovr.YSize == 10 + assert ovr.ReadRaster() == src_ds.GetRasterBand(1).GetMaskBand().ReadRaster( + buf_xsize=10, buf_ysize=10 + ) + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).ReadRaster( + buf_xsize=10, buf_ysize=10 + ) == src_ds.ReadRaster(buf_xsize=10, buf_ysize=10) + + +def test_vrttileindex_ovr_factor_invalid(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + lyr.SetMetadataItem("OVERVIEW_1_FACTOR", "0.5") + del index_ds + + vrt_ds = gdal.Open(index_filename) + with pytest.raises(Exception, match="Wrong overview factor"): + assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 0 + + +def test_vrttileindex_ovr_ds_name(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + lyr.SetMetadataItem("OVERVIEW_1_DATASET", "/i/do/not/exist") + del index_ds + + vrt_ds = gdal.Open(index_filename) + with pytest.raises(Exception): + assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 0 + + +def test_vrttileindex_ovr_lyr_name(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + lyr.SetMetadataItem("OVERVIEW_1_LAYER", "non_existing") + del index_ds + + vrt_ds = gdal.Open(index_filename) + with pytest.raises(Exception, match="Layer non_existing does not exist"): + assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 0 + + +def test_vrttileindex_external_ovr(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, _ = create_basic_tileindex(index_filename, src_ds) + del index_ds + + vrt_ds = gdal.Open(index_filename) + vrt_ds.BuildOverviews("CUBIC", [2]) + vrt_ds = None + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).ReadRaster( + buf_xsize=10, buf_ysize=10 + ) == src_ds.ReadRaster(buf_xsize=10, buf_ysize=10, resample_alg=gdal.GRIORA_Cubic) + + +def test_vrttileindex_dataset_metadata(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + lyr.SetMetadataItem("FOO", "BAR") + lyr.SetMetadataItem("RESX", "59.999") + lyr.SetMetadataItem("RESY", "59.999") + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetMetadata_Dict() == {"FOO": "BAR"} + vrt_ds.SetMetadataItem("BAR", "BAZ") + del vrt_ds + + assert gdal.VSIStatL(index_filename + ".aux.xml") + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetGeoTransform()[1] == 59.999 + assert vrt_ds.GetMetadata_Dict() == { + "FOO": "BAR", + "BAR": "BAZ", + } + del vrt_ds + + gdal.Unlink(index_filename + ".aux.xml") + vrt_ds = gdal.Open(index_filename, gdal.GA_Update) + vrt_ds.SetMetadataItem("FOO", "BAR2") + md = vrt_ds.GetMetadata_Dict() + md["BAR"] = "BAZ2" + vrt_ds.SetMetadata(md) + del vrt_ds + + assert gdal.VSIStatL(index_filename + ".aux.xml") is None + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetGeoTransform()[1] == 59.999 + assert vrt_ds.GetMetadata_Dict() == { + "FOO": "BAR2", + "BAR": "BAZ2", + } + del vrt_ds + + +def test_vrttileindex_band_metadata(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + lyr.SetMetadataItem("BAND_1_FOO", "BAR") + lyr.SetMetadataItem("BAND_1_OFFSET", "2") + lyr.SetMetadataItem("BAND_1_SCALE", "3") + lyr.SetMetadataItem("BAND_1_UNITTYPE", "dn") + lyr.SetMetadataItem("BAND_0_FOO", "BAR0") + lyr.SetMetadataItem("BAND_2_FOO", "BAR2") + del index_ds + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).GetOffset() == 2 + assert vrt_ds.GetRasterBand(1).GetScale() == 3 + assert vrt_ds.GetRasterBand(1).GetUnitType() == "dn" + assert vrt_ds.GetRasterBand(1).GetMetadata_Dict() == {"FOO": "BAR"} + vrt_ds.GetRasterBand(1).ComputeStatistics(False) + vrt_ds.GetRasterBand(1).SetMetadataItem("BAR", "BAZ") + del vrt_ds + + assert gdal.VSIStatL(index_filename + ".aux.xml") + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).GetOffset() == 2 + assert vrt_ds.GetRasterBand(1).GetScale() == 3 + assert vrt_ds.GetRasterBand(1).GetUnitType() == "dn" + assert vrt_ds.GetRasterBand(1).GetMetadataDomainList() == ["", "LocationInfo"] + assert vrt_ds.GetRasterBand(1).GetMetadata_Dict() == { + "FOO": "BAR", + "BAR": "BAZ", + "STATISTICS_MAXIMUM": "255", + "STATISTICS_MEAN": "126.765", + "STATISTICS_MINIMUM": "74", + "STATISTICS_STDDEV": "22.928470838676", + "STATISTICS_VALID_PERCENT": "100", + } + del vrt_ds + + gdal.Unlink(index_filename + ".aux.xml") + vrt_ds = gdal.Open(index_filename, gdal.GA_Update) + vrt_ds.GetRasterBand(1).ComputeStatistics(False) + vrt_ds.GetRasterBand(1).SetMetadataItem("BAR", "BAZ") + del vrt_ds + + assert gdal.VSIStatL(index_filename + ".aux.xml") is None + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).GetOffset() == 2 + assert vrt_ds.GetRasterBand(1).GetScale() == 3 + assert vrt_ds.GetRasterBand(1).GetUnitType() == "dn" + assert vrt_ds.GetRasterBand(1).GetMetadata_Dict() == { + "FOO": "BAR", + "BAR": "BAZ", + "STATISTICS_MAXIMUM": "255", + "STATISTICS_MEAN": "126.765", + "STATISTICS_MINIMUM": "74", + "STATISTICS_STDDEV": "22.928470838676", + "STATISTICS_VALID_PERCENT": "100", + } + del vrt_ds + + vrt_ds = gdal.Open(index_filename, gdal.GA_Update) + md = vrt_ds.GetMetadata_Dict() + md["BAR"] = "BAZ2" + vrt_ds.SetMetadata(md) + md = vrt_ds.GetRasterBand(1).GetMetadata_Dict() + md["BAR"] = "BAZ3" + vrt_ds.GetRasterBand(1).SetMetadata(md) + del vrt_ds + + assert gdal.VSIStatL(index_filename + ".aux.xml") is None + + vrt_ds = gdal.Open(index_filename) + assert vrt_ds.GetRasterBand(1).GetOffset() == 2 + assert vrt_ds.GetRasterBand(1).GetScale() == 3 + assert vrt_ds.GetRasterBand(1).GetUnitType() == "dn" + assert vrt_ds.GetRasterBand(1).GetMetadata_Dict() == { + "FOO": "BAR", + "BAR": "BAZ3", + "STATISTICS_MAXIMUM": "255", + "STATISTICS_MEAN": "126.765", + "STATISTICS_MINIMUM": "74", + "STATISTICS_STDDEV": "22.928470838676", + "STATISTICS_VALID_PERCENT": "100", + } + assert vrt_ds.GetMetadata_Dict() == { + "BAR": "BAZ2", + } + + +def test_vrttileindex_connection_prefix(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, _ = create_basic_tileindex(index_filename, src_ds) + del index_ds + + vrt_ds = gdal.Open(f"VRTTI:{index_filename}") + check_basic(vrt_ds, src_ds) + del vrt_ds + + +def test_vrttileindex_xml(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, _ = create_basic_tileindex(index_filename, src_ds) + del index_ds + + xml_filename = str(tmp_vsimem / "index.xml") + xml_content = f""" + {index_filename} +""" + + vrt_ds = gdal.Open(xml_content) + check_basic(vrt_ds, src_ds) + del vrt_ds + + with gdaltest.tempfile(xml_filename, xml_content): + vrt_ds = gdal.Open(xml_filename) + check_basic(vrt_ds, src_ds) + del vrt_ds + + vrt_ds = gdal.Open(xml_filename, gdal.GA_Update) + vrt_ds.SetMetadata({"FOO": "BAR"}) + vrt_ds.SetMetadataItem("BAR", "BAZ") + vrt_ds.GetRasterBand(1).SetMetadata({"xFOO": "BAR"}) + vrt_ds.GetRasterBand(1).SetMetadataItem("xBAR", "BAZ") + del vrt_ds + + assert gdal.VSIStatL(xml_filename + ".aux.xml") is None + + vrt_ds = gdal.Open(xml_filename) + assert vrt_ds.GetMetadata_Dict() == {"FOO": "BAR", "BAR": "BAZ"} + assert vrt_ds.GetRasterBand(1).GetMetadata_Dict() == { + "xFOO": "BAR", + "xBAR": "BAZ", + } + del vrt_ds + + xml_content = f""" + {index_filename} + invalid +""" + with pytest.raises(Exception, match="failed to prepare SQL"): + gdal.Open(xml_content) + + xml_content = f""" + {index_filename} + 60 + 60 + location + true + + my band + 2 + 3 + 4 + dn + Gray + + cat + + +""" + + vrt_ds = gdal.Open(xml_content) + band = vrt_ds.GetRasterBand(1) + assert band.GetDescription() == "my band" + assert band.DataType == gdal.GDT_UInt16 + assert band.GetOffset() == 2 + assert band.GetScale() == 3 + assert band.GetNoDataValue() == 4 + assert band.GetUnitType() == "dn" + assert band.GetColorInterpretation() == gdal.GCI_GrayIndex + assert band.GetColorTable() is not None + assert band.GetCategoryNames() == ["cat"] + assert band.GetDefaultRAT() is not None + del vrt_ds + + xml_content = f""" + {index_filename} + 60 + 60 + +""" + with pytest.raises(Exception, match="band attribute missing on Band element"): + gdal.Open(xml_content) + + xml_content = f""" + {index_filename} + 60 + 60 + +""" + with pytest.raises(Exception, match="Invalid band number"): + gdal.Open(xml_content) + + xml_content = f""" + {index_filename} + 60 + 60 + +""" + with pytest.raises(Exception, match="Invalid band number: found 2, expected 1"): + gdal.Open(xml_content) + + xml_content = f""" + {index_filename} + 60 + 60 + 2 + +""" + with pytest.raises( + Exception, match="Inconsistent BandCount with actual number of Band elements" + ): + gdal.Open(xml_content) + + xml_content = f""" + {index_filename} + + 2 + +""" + vrt_ds = gdal.Open(xml_content) + assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 1 + assert vrt_ds.GetRasterBand(1).GetOverview(0).XSize == 10 + del vrt_ds + + xml_content = f""" + {index_filename} + + {index_filename} + +""" + vrt_ds = gdal.Open(xml_content) + assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 1 + assert vrt_ds.GetRasterBand(1).GetOverview(0).XSize == 20 + del vrt_ds + + xml_content = f""" + {index_filename} + + index + +""" + vrt_ds = gdal.Open(xml_content) + assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 1 + assert vrt_ds.GetRasterBand(1).GetOverview(0).XSize == 20 + del vrt_ds + + xml_content = f""" + {index_filename} + + index + + 2 + + +""" + vrt_ds = gdal.Open(xml_content) + assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 1 + assert vrt_ds.GetRasterBand(1).GetOverview(0).XSize == 10 + del vrt_ds + + xml_content = """""" + with gdaltest.tempfile(xml_filename, xml_content): + with pytest.raises( + Exception, + match="Missing VRTTileIndexDataset root element", + ): + gdal.Open(xml_filename) + + xml_content = """""" + with pytest.raises( + Exception, + match="Missing IndexDataset element", + ): + gdal.Open(xml_content) + + xml_content = """ + i_do_not_exist +""" + with pytest.raises( + Exception, + match="i_do_not_exist", + ): + gdal.Open(xml_content) + + xml_content = f""" + {index_filename} + i_do_not_exist +""" + with pytest.raises( + Exception, + match="i_do_not_exist", + ): + gdal.Open(xml_content) + + xml_content = f""" + {index_filename} + + +""" + with pytest.raises( + Exception, + match="At least one of Dataset, Layer or Factor element must be present as an Overview child", + ): + gdal.Open(xml_content) + + xml_content = f""" + {index_filename} + + i_do_not_exist + +""" + vrt_ds = gdal.Open(xml_content) + with pytest.raises(Exception, match="i_do_not_exist"): + assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 0 + + xml_content = f""" + {index_filename} + + i_do_not_exist + +""" + vrt_ds = gdal.Open(xml_content) + with pytest.raises(Exception, match="i_do_not_exist"): + assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 0 + + +def test_vrttileindex_open_options(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, _ = create_basic_tileindex(index_filename, src_ds) + del index_ds + + vrt_ds = gdal.OpenEx(index_filename, open_options=["RESX=30", "RESY=30"]) + assert vrt_ds.GetGeoTransform() == pytest.approx( + (440720.0, 30.0, 0.0, 3751320.0, 0.0, -30.0) + ) diff --git a/frmts/drivers.ini b/frmts/drivers.ini index ccc77679b30c..06e69303af0c 100644 --- a/frmts/drivers.ini +++ b/frmts/drivers.ini @@ -12,6 +12,7 @@ [order] VRT +VRTTI Derived GTiff COG diff --git a/frmts/gdalallregister.cpp b/frmts/gdalallregister.cpp index a69a5f8f5986..4510ebf10827 100644 --- a/frmts/gdalallregister.cpp +++ b/frmts/gdalallregister.cpp @@ -323,6 +323,7 @@ void CPL_STDCALL GDALAllRegister() #ifdef FRMT_vrt GDALRegister_VRT(); + GDALRegister_VRTTI(); GDALRegister_Derived(); #endif diff --git a/frmts/gtiff/gt_jpeg_copy.cpp b/frmts/gtiff/gt_jpeg_copy.cpp index 7cdbc182820b..7995860094e5 100644 --- a/frmts/gtiff/gt_jpeg_copy.cpp +++ b/frmts/gtiff/gt_jpeg_copy.cpp @@ -50,10 +50,8 @@ static GDALDataset *GetUnderlyingDataset(GDALDataset *poSrcDS) { // Test if we can directly copy original JPEG content if available. - if (poSrcDS->GetDriver() != nullptr && - poSrcDS->GetDriver() == GDALGetDriverByName("VRT")) + if (auto poVRTDS = dynamic_cast(poSrcDS)) { - VRTDataset *poVRTDS = cpl::down_cast(poSrcDS); poSrcDS = poVRTDS->GetSingleSimpleSource(); } diff --git a/frmts/vrt/CMakeLists.txt b/frmts/vrt/CMakeLists.txt index 136a79ba524f..4f35a8baca61 100644 --- a/frmts/vrt/CMakeLists.txt +++ b/frmts/vrt/CMakeLists.txt @@ -15,12 +15,14 @@ add_gdal_driver( pixelfunctions.cpp vrtpansharpened.cpp vrtmultidim.cpp + vrttileindexdataset.cpp STRONG_CXX_WFLAGS) gdal_standard_includes(gdal_vrt) target_include_directories(gdal_vrt PRIVATE ${GDAL_RASTER_FORMAT_SOURCE_DIR}/raw) set(GDAL_DATA_FILES ${CMAKE_CURRENT_SOURCE_DIR}/data/gdalvrt.xsd + ${CMAKE_CURRENT_SOURCE_DIR}/data/gdalvrtti.xsd ) set_property( TARGET ${GDAL_LIB_TARGET_NAME} diff --git a/frmts/vrt/data/gdalvrtti.xsd b/frmts/vrt/data/gdalvrtti.xsd new file mode 100644 index 000000000000..4435e61428db --- /dev/null +++ b/frmts/vrt/data/gdalvrtti.xsd @@ -0,0 +1,267 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/frmts/vrt/vrt_priv.h b/frmts/vrt/vrt_priv.h new file mode 100644 index 000000000000..c7ca081608e1 --- /dev/null +++ b/frmts/vrt/vrt_priv.h @@ -0,0 +1,62 @@ +/****************************************************************************** + * Project: Virtual GDAL Datasets + * Purpose: Declaration of VRT private stuff + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2023, Even Rouault + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#ifndef VRT_PRIV_H_INCLUDED +#define VRT_PRIV_H_INCLUDED + +#ifndef DOXYGEN_SKIP + +#include "cpl_port.h" +#include "gdal_priv.h" + +#include +#include + +struct CPL_DLL VRTTISourceDesc +{ + std::string osFilename{}; + int nDstXOff = 0; + int nDstYOff = 0; + int nDstXSize = 0; + int nDstYSize = 0; +}; + +class VRTTileIndexDataset; + +VRTTileIndexDataset CPL_DLL *GDALDatasetCastToVRTTIDataset(GDALDataset *poDS); + +std::vector CPL_DLL +VRTTIGetSourcesMoreRecentThan(VRTTileIndexDataset *poDS, int64_t mTime); + +CPLStringList VRTParseCategoryNames(const CPLXMLNode *psCategoryNames); + +std::unique_ptr +VRTParseColorTable(const CPLXMLNode *psColorTable); + +#endif + +#endif // VRT_PRIV_H_INCLUDED diff --git a/frmts/vrt/vrtdataset.cpp b/frmts/vrt/vrtdataset.cpp index c10472cc7997..69e65fa68ae6 100644 --- a/frmts/vrt/vrtdataset.cpp +++ b/frmts/vrt/vrtdataset.cpp @@ -1553,7 +1553,7 @@ GDALDataset *VRTDataset::OpenVRTProtocol(const char *pszSpec) poSrcDS->ReleaseRef(); - auto poDS = cpl::down_cast(GDALDataset::FromHandle(hRet)); + auto poDS = dynamic_cast(GDALDataset::FromHandle(hRet)); if (poDS) { if (bPatchSourceFilename) diff --git a/frmts/vrt/vrtdataset.h b/frmts/vrt/vrtdataset.h index 4fecac8c107d..d893e7361fdb 100644 --- a/frmts/vrt/vrtdataset.h +++ b/frmts/vrt/vrtdataset.h @@ -46,11 +46,10 @@ #include #include -int VRTApplyMetadata(CPLXMLNode *, GDALMajorObject *); -CPLXMLNode *VRTSerializeMetadata(GDALMajorObject *); CPLErr GDALRegisterDefaultPixelFunc(); CPLString VRTSerializeNoData(double dfVal, GDALDataType eDataType, int nPrecision); + #if 0 int VRTWarpedOverviewTransform( void *pTransformArg, int bDstToSrc, int nPointCount, @@ -541,7 +540,7 @@ class CPL_DLL VRTRasterBand CPL_NON_FINAL : public GDALRasterBand GDALColorInterp m_eColorInterp = GCI_Undefined; char *m_pszUnitType = nullptr; - char **m_papszCategoryNames = nullptr; + CPLStringList m_aosCategoryNames{}; double m_dfOffset = 0.0; double m_dfScale = 1.0; @@ -1025,6 +1024,8 @@ class CPL_DLL VRTSimpleSource CPL_NON_FINAL : public VRTSource protected: friend class VRTSourcedRasterBand; friend class VRTDataset; + friend class VRTTileIndexDataset; + friend class VRTTileIndexBand; int m_nBand = 0; bool m_bGetMaskBand = false; @@ -1057,6 +1058,12 @@ class CPL_DLL VRTSimpleSource CPL_NON_FINAL : public VRTSource return m_poRasterBand; } + void SetRasterBand(GDALRasterBand *poBand, bool bDropRef) + { + m_poRasterBand = poBand; + m_bDropRefOnSrcBand = bDropRef; + } + virtual bool ValidateOpenedBand(GDALRasterBand * /*poBand*/) const { return true; diff --git a/frmts/vrt/vrtdriver.cpp b/frmts/vrt/vrtdriver.cpp index 3047d20d6e35..73a17a1565c1 100644 --- a/frmts/vrt/vrtdriver.cpp +++ b/frmts/vrt/vrtdriver.cpp @@ -193,8 +193,7 @@ static GDALDataset *VRTCreateCopy(const char *pszFilename, GDALDataset *poSrcDS, /* it to disk as a special case to avoid extra layers of */ /* indirection. */ /* -------------------------------------------------------------------- */ - if (poSrcDS->GetDriver() != nullptr && - EQUAL(poSrcDS->GetDriver()->GetDescription(), "VRT")) + if (auto poSrcVRTDS = dynamic_cast(poSrcDS)) { /* -------------------------------------------------------------------- @@ -203,7 +202,6 @@ static GDALDataset *VRTCreateCopy(const char *pszFilename, GDALDataset *poSrcDS, /* -------------------------------------------------------------------- */ char *pszVRTPath = CPLStrdup(CPLGetPath(pszFilename)); - auto poSrcVRTDS = cpl::down_cast(poSrcDS); poSrcVRTDS->UnsetPreservedRelativeFilenames(); CPLXMLNode *psDSTree = poSrcVRTDS->SerializeToXML(pszVRTPath); diff --git a/frmts/vrt/vrtrasterband.cpp b/frmts/vrt/vrtrasterband.cpp index 3540ac4881d9..ffdc2b75ec10 100644 --- a/frmts/vrt/vrtrasterband.cpp +++ b/frmts/vrt/vrtrasterband.cpp @@ -47,6 +47,7 @@ #include "cpl_progress.h" #include "cpl_string.h" #include "cpl_vsi.h" +#include "vrt_priv.h" /*! @cond Doxygen_Suppress */ @@ -93,7 +94,6 @@ VRTRasterBand::~VRTRasterBand() { CPLFree(m_pszUnitType); - CSLDestroy(m_papszCategoryNames); if (m_psSavedHistograms != nullptr) CPLDestroyXMLNode(m_psSavedHistograms); @@ -267,7 +267,7 @@ CPLErr VRTRasterBand::SetScale(double dfNewScale) char **VRTRasterBand::GetCategoryNames() { - return m_papszCategoryNames; + return m_aosCategoryNames.List(); } /************************************************************************/ @@ -279,12 +279,65 @@ CPLErr VRTRasterBand::SetCategoryNames(char **papszNewNames) { static_cast(poDS)->SetNeedsFlush(); - CSLDestroy(m_papszCategoryNames); - m_papszCategoryNames = CSLDuplicate(papszNewNames); + m_aosCategoryNames = CSLDuplicate(papszNewNames); return CE_None; } +/************************************************************************/ +/* VRTParseCategoryNames() */ +/************************************************************************/ + +CPLStringList VRTParseCategoryNames(const CPLXMLNode *psCategoryNames) +{ + CPLStringList oCategoryNames; + + for (const CPLXMLNode *psEntry = psCategoryNames->psChild; + psEntry != nullptr; psEntry = psEntry->psNext) + { + if (psEntry->eType != CXT_Element || + !EQUAL(psEntry->pszValue, "Category") || + (psEntry->psChild != nullptr && + psEntry->psChild->eType != CXT_Text)) + continue; + + oCategoryNames.AddString((psEntry->psChild) ? psEntry->psChild->pszValue + : ""); + } + + return oCategoryNames; +} + +/************************************************************************/ +/* VRTParseColorTable() */ +/************************************************************************/ + +std::unique_ptr +VRTParseColorTable(const CPLXMLNode *psColorTable) +{ + auto poColorTable = std::make_unique(); + int iEntry = 0; + + for (const CPLXMLNode *psEntry = psColorTable->psChild; psEntry != nullptr; + psEntry = psEntry->psNext) + { + if (psEntry->eType != CXT_Element || !EQUAL(psEntry->pszValue, "Entry")) + { + continue; + } + + const GDALColorEntry sCEntry = { + static_cast(atoi(CPLGetXMLValue(psEntry, "c1", "0"))), + static_cast(atoi(CPLGetXMLValue(psEntry, "c2", "0"))), + static_cast(atoi(CPLGetXMLValue(psEntry, "c3", "0"))), + static_cast(atoi(CPLGetXMLValue(psEntry, "c4", "255")))}; + + poColorTable->SetColorEntry(iEntry++, &sCEntry); + } + + return poColorTable; +} + /************************************************************************/ /* XMLInit() */ /************************************************************************/ @@ -399,66 +452,29 @@ VRTRasterBand::XMLInit(CPLXMLNode *psTree, const char *pszVRTPath, /* -------------------------------------------------------------------- */ /* Category names. */ /* -------------------------------------------------------------------- */ - if (CPLGetXMLNode(psTree, "CategoryNames") != nullptr) + if (const CPLXMLNode *psCategoryNames = + CPLGetXMLNode(psTree, "CategoryNames")) { - CSLDestroy(m_papszCategoryNames); - m_papszCategoryNames = nullptr; - - CPLStringList oCategoryNames; - - for (CPLXMLNode *psEntry = - CPLGetXMLNode(psTree, "CategoryNames")->psChild; - psEntry != nullptr; psEntry = psEntry->psNext) - { - if (psEntry->eType != CXT_Element || - !EQUAL(psEntry->pszValue, "Category") || - (psEntry->psChild != nullptr && - psEntry->psChild->eType != CXT_Text)) - continue; - - oCategoryNames.AddString( - (psEntry->psChild) ? psEntry->psChild->pszValue : ""); - } - - m_papszCategoryNames = oCategoryNames.StealList(); + m_aosCategoryNames = VRTParseCategoryNames(psCategoryNames); } /* -------------------------------------------------------------------- */ /* Collect a color table. */ /* -------------------------------------------------------------------- */ - if (CPLGetXMLNode(psTree, "ColorTable") != nullptr) + if (const CPLXMLNode *psColorTable = CPLGetXMLNode(psTree, "ColorTable")) { - GDALColorTable oTable; - int iEntry = 0; - - for (CPLXMLNode *psEntry = CPLGetXMLNode(psTree, "ColorTable")->psChild; - psEntry != nullptr; psEntry = psEntry->psNext) - { - if (psEntry->eType != CXT_Element || - !EQUAL(psEntry->pszValue, "Entry")) - { - continue; - } - - const GDALColorEntry sCEntry = { - static_cast(atoi(CPLGetXMLValue(psEntry, "c1", "0"))), - static_cast(atoi(CPLGetXMLValue(psEntry, "c2", "0"))), - static_cast(atoi(CPLGetXMLValue(psEntry, "c3", "0"))), - static_cast(atoi(CPLGetXMLValue(psEntry, "c4", "255")))}; - - oTable.SetColorEntry(iEntry++, &sCEntry); - } - - SetColorTable(&oTable); + auto poColorTable = VRTParseColorTable(psColorTable); + if (poColorTable) + SetColorTable(poColorTable.get()); } /* -------------------------------------------------------------------- */ /* Raster Attribute Table */ /* -------------------------------------------------------------------- */ - CPLXMLNode *psRAT = CPLGetXMLNode(psTree, "GDALRasterAttributeTable"); - if (psRAT != nullptr) + if (const CPLXMLNode *psRAT = + CPLGetXMLNode(psTree, "GDALRasterAttributeTable")) { - m_poRAT.reset(new GDALDefaultRasterAttributeTable()); + m_poRAT = std::make_unique(); m_poRAT->XMLInit(psRAT, ""); } @@ -708,16 +724,16 @@ CPLXMLNode *VRTRasterBand::SerializeToXML(const char *pszVRTPath) /* -------------------------------------------------------------------- */ /* Category names. */ /* -------------------------------------------------------------------- */ - if (m_papszCategoryNames != nullptr) + if (!m_aosCategoryNames.empty()) { CPLXMLNode *psCT_XML = CPLCreateXMLNode(psTree, CXT_Element, "CategoryNames"); CPLXMLNode *psLastChild = nullptr; - for (int iEntry = 0; m_papszCategoryNames[iEntry] != nullptr; iEntry++) + for (const char *pszName : m_aosCategoryNames) { - CPLXMLNode *psNode = CPLCreateXMLElementAndValue( - nullptr, "Category", m_papszCategoryNames[iEntry]); + CPLXMLNode *psNode = + CPLCreateXMLElementAndValue(nullptr, "Category", pszName); if (psLastChild == nullptr) psCT_XML->psChild = psNode; else diff --git a/frmts/vrt/vrttileindexdataset.cpp b/frmts/vrt/vrttileindexdataset.cpp new file mode 100644 index 000000000000..45d8131592df --- /dev/null +++ b/frmts/vrt/vrttileindexdataset.cpp @@ -0,0 +1,3623 @@ +/****************************************************************************** + * + * Project: Virtual GDAL Datasets + * Purpose: Tile index based VRT + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2023, Even Rouault + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +/*! @cond Doxygen_Suppress */ + +#include +#include +#include +#include +#include +#include +#include + +#include "cpl_port.h" +#include "cpl_mem_cache.h" +#include "cpl_minixml.h" +#include "vrtdataset.h" +#include "vrt_priv.h" +#include "ogrsf_frmts.h" +#include "gdal_proxy.h" +#include "gdal_utils.h" + +#if defined(__SSE2__) || defined(_M_X64) +#define USE_SSE2_OPTIM +#include +// MSVC doesn't define __SSE4_1__, but if -arch:AVX2 is enabled, we do have SSE4.1 +#if defined(__SSE4_1__) || defined(__AVX2__) +#define USE_SSE41_OPTIM +#include +#endif +#endif + +constexpr int GT_TOPLEFT_X = 0; +constexpr int GT_WE_RES = 1; +constexpr int GT_ROTATION_PARAM1 = 2; +constexpr int GT_TOPLEFT_Y = 3; +constexpr int GT_ROTATION_PARAM2 = 4; +constexpr int GT_NS_RES = 5; + +constexpr const char *VRTTI_PREFIX = "VRTTI:"; + +constexpr const char *MD_DS_TILE_INDEX_LAYER = "TILE_INDEX_LAYER"; + +constexpr const char *MD_RESX = "RESX"; +constexpr const char *MD_RESY = "RESY"; +constexpr const char *MD_BAND_COUNT = "BAND_COUNT"; +constexpr const char *MD_DATA_TYPE = "DATA_TYPE"; +constexpr const char *MD_NODATA = "NODATA"; +constexpr const char *MD_MINX = "MINX"; +constexpr const char *MD_MINY = "MINY"; +constexpr const char *MD_MAXX = "MAXX"; +constexpr const char *MD_MAXY = "MAXY"; +constexpr const char *MD_GEOTRANSFORM = "GEOTRANSFORM"; +constexpr const char *MD_XSIZE = "XSIZE"; +constexpr const char *MD_YSIZE = "YSIZE"; +constexpr const char *MD_COLOR_INTERPRETATION = "COLOR_INTERPRETATION"; +constexpr const char *MD_SRS = "SRS"; +constexpr const char *MD_LOCATION_FIELD = "LOCATION_FIELD"; +constexpr const char *MD_SORT_FIELD = "SORT_FIELD"; +constexpr const char *MD_SORT_FIELD_ASC = "SORT_FIELD_ASC"; +constexpr const char *MD_BLOCKXSIZE = "BLOCKXSIZE"; +constexpr const char *MD_BLOCKYSIZE = "BLOCKYSIZE"; +constexpr const char *MD_MASK_BAND = "MASK_BAND"; +constexpr const char *MD_RESAMPLING = "RESAMPLING"; + +constexpr const char *const apszTIOptions[] = {MD_RESX, + MD_RESY, + MD_BAND_COUNT, + MD_DATA_TYPE, + MD_NODATA, + MD_MINX, + MD_MINY, + MD_MAXX, + MD_MAXY, + MD_GEOTRANSFORM, + MD_XSIZE, + MD_YSIZE, + MD_COLOR_INTERPRETATION, + MD_SRS, + MD_LOCATION_FIELD, + MD_SORT_FIELD, + MD_SORT_FIELD_ASC, + MD_BLOCKXSIZE, + MD_BLOCKYSIZE, + MD_MASK_BAND, + MD_RESAMPLING}; + +constexpr const char *const MD_BAND_OFFSET = "OFFSET"; +constexpr const char *const MD_BAND_SCALE = "SCALE"; +constexpr const char *const MD_BAND_UNITTYPE = "UNITTYPE"; +constexpr const char *const apszReservedBandItems[] = { + MD_BAND_OFFSET, MD_BAND_SCALE, MD_BAND_UNITTYPE}; + +constexpr const char *VRTTI_XML_BANDCOUNT = "BandCount"; +constexpr const char *VRTTI_XML_DATATYPE = "DataType"; +constexpr const char *VRTTI_XML_NODATAVALUE = "NoDataValue"; +constexpr const char *VRTTI_XML_COLORINTERP = "ColorInterp"; +constexpr const char *VRTTI_XML_LOCATIONFIELD = "LocationField"; +constexpr const char *VRTTI_XML_SORTFIELD = "SortField"; +constexpr const char *VRTTI_XML_SORTFIELDASC = "SortFieldAsc"; +constexpr const char *VRTTI_XML_MASKBAND = "MaskBand"; +constexpr const char *VRTTI_XML_OVERVIEW_ELEMENT = "Overview"; +constexpr const char *VRTTI_XML_OVERVIEW_DATASET = "Dataset"; +constexpr const char *VRTTI_XML_OVERVIEW_LAYER = "Layer"; +constexpr const char *VRTTI_XML_OVERVIEW_FACTOR = "Factor"; + +constexpr const char *VRTTI_XML_BAND_ELEMENT = "Band"; +constexpr const char *VRTTI_XML_BAND_NUMBER = "band"; +constexpr const char *VRTTI_XML_BAND_DATATYPE = "dataType"; +constexpr const char *VRTTI_XML_BAND_DESCRIPTION = "Description"; +constexpr const char *VRTTI_XML_BAND_OFFSET = "Offset"; +constexpr const char *VRTTI_XML_BAND_SCALE = "Scale"; +constexpr const char *VRTTI_XML_BAND_NODATAVALUE = "NoDataValue"; +constexpr const char *VRTTI_XML_BAND_UNITTYPE = "UnitType"; +constexpr const char *VRTTI_XML_BAND_COLORINTERP = "ColorInterp"; +constexpr const char *VRTTI_XML_CATEGORYNAMES = "CategoryNames"; +constexpr const char *VRTTI_XML_COLORTABLE = "ColorTable"; +constexpr const char *VRTTI_XML_RAT = "GDALRasterAttributeTable"; + +/************************************************************************/ +/* ENDS_WITH_CI() */ +/************************************************************************/ + +static inline bool ENDS_WITH_CI(const char *a, const char *b) +{ + return strlen(a) >= strlen(b) && EQUAL(a + strlen(a) - strlen(b), b); +} + +/************************************************************************/ +/* VRTTileIndexDataset */ +/************************************************************************/ + +class VRTTileIndexBand; +class VRTTileIndexDataset final : public GDALPamDataset +{ + friend class VRTTileIndexBand; + + CPLXMLTreeCloser m_psXMLTree{nullptr}; + bool m_bXMLUpdatable = false; + bool m_bXMLModified = false; + const std::string m_osUniqueHandle; + std::unique_ptr m_poVectorDS{}; + OGRLayer *m_poLayer = nullptr; + std::array m_adfGeoTransform{0, 0, 0, 0, 0, 0}; + int m_nLocationFieldIndex = -1; + OGRSpatialReference m_oSRS{}; + lru11::Cache> m_oMapSharedSources{ + 500}; + std::unique_ptr m_poMaskBand{}; + bool m_bSameDataType = true; + bool m_bSameNoData = true; + double m_dfLastMinXFilter = std::numeric_limits::quiet_NaN(); + double m_dfLastMinYFilter = std::numeric_limits::quiet_NaN(); + double m_dfLastMaxXFilter = std::numeric_limits::quiet_NaN(); + double m_dfLastMaxYFilter = std::numeric_limits::quiet_NaN(); + int m_nSortFieldIndex = -1; + bool m_bSortFieldAsc = true; + std::string m_osResampling = "near"; + GDALRIOResampleAlg m_eResampling = GRIORA_NearestNeighbour; + std::string m_osWKT{}; + bool m_bScannedOneFeatureAtOpening = false; + std::vector> + m_aoOverviewDescriptor{}; + std::vector> m_apoOverviews{}; + VRTSource::WorkingState m_oWorkingState{}; + + struct SourceDesc + { + std::string osName{}; + std::shared_ptr poDS{}; + std::unique_ptr poSource{}; + std::unique_ptr poFeature{}; + std::vector abyMask{}; + bool bCoversWholeAOI = false; + bool bHasNoData = false; + bool bSameNoData = false; + double dfSameNoData = 0; + GDALRasterBand *poMaskBand = nullptr; + }; + std::vector m_aoSourceDesc{}; + + bool GetSourceDesc(const std::string &osTileName, SourceDesc &oSourceDesc); + bool CollectSources(double dfXOff, double dfYOff, double dfXSize, + double dfYSize); + void SortSourceDesc(); + bool NeedInitBuffer(int nBandCount, const int *panBandMap) const; + void InitBuffer(void *pData, int nBufXSize, int nBufYSize, + GDALDataType eBufType, int nBandCount, + const int *panBandMap, GSpacing nPixelSpace, + GSpacing nLineSpace, GSpacing nBandSpace) const; + + bool TileIndexSupportsEditingLayerMetadata() const; + + CPL_DISALLOW_COPY_ASSIGN(VRTTileIndexDataset) + + public: + VRTTileIndexDataset(); + ~VRTTileIndexDataset() override; + + bool Open(GDALOpenInfo *poOpenInfo); + + CPLErr FlushCache(bool bAtClosing) override; + + CPLErr GetGeoTransform(double *padfGeoTransform) override; + const OGRSpatialReference *GetSpatialRef() const override; + + CPLErr IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, + int nYSize, void *pData, int nBufXSize, int nBufYSize, + GDALDataType eBufType, int nBandCount, int *panBandMap, + GSpacing nPixelSpace, GSpacing nLineSpace, + GSpacing nBandSpace, + GDALRasterIOExtraArg *psExtraArg) override; + + const char *GetMetadataItem(const char *pszName, + const char *pszDomain) override; + CPLErr SetMetadataItem(const char *pszName, const char *pszValue, + const char *pszDomain) override; + CPLErr SetMetadata(char **papszMD, const char *pszDomain) override; + + void LoadOverviews(); + + std::vector GetSourcesMoreRecentThan(int64_t mTime); +}; + +/************************************************************************/ +/* VRTTileIndexBand */ +/************************************************************************/ + +class VRTTileIndexBand final : public GDALPamRasterBand +{ + friend class VRTTileIndexDataset; + + VRTTileIndexDataset *m_poDS = nullptr; + bool m_bNoDataValueSet = false; + double m_dfNoDataValue = 0; + GDALColorInterp m_eColorInterp = GCI_Undefined; + std::string m_osLastLocationInfo{}; + double m_dfScale = std::numeric_limits::quiet_NaN(); + double m_dfOffset = std::numeric_limits::quiet_NaN(); + std::string m_osUnit{}; + CPLStringList m_aosCategoryNames{}; + std::unique_ptr m_poColorTable{}; + std::unique_ptr m_poRAT{}; + + CPL_DISALLOW_COPY_ASSIGN(VRTTileIndexBand) + + public: + VRTTileIndexBand(VRTTileIndexDataset *poDSIn, int nBandIn, GDALDataType eDT, + int nBlockXSizeIn, int nBlockYSizeIn); + + double GetNoDataValue(int *pbHasNoData) override + { + if (pbHasNoData) + *pbHasNoData = m_bNoDataValueSet; + return m_dfNoDataValue; + } + + GDALColorInterp GetColorInterpretation() override + { + return m_eColorInterp; + } + + CPLErr IReadBlock(int nBlockXOff, int nBlockYOff, void *pData) override; + + CPLErr IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, + int nYSize, void *pData, int nBufXSize, int nBufYSize, + GDALDataType eBufType, GSpacing nPixelSpace, + GSpacing nLineSpace, + GDALRasterIOExtraArg *psExtraArg) override; + + int GetMaskFlags() override + { + if (m_poDS->m_poMaskBand && m_poDS->m_poMaskBand.get() != this) + return GMF_PER_DATASET; + return GDALPamRasterBand::GetMaskFlags(); + } + + GDALRasterBand *GetMaskBand() override + { + if (m_poDS->m_poMaskBand && m_poDS->m_poMaskBand.get() != this) + return m_poDS->m_poMaskBand.get(); + return GDALPamRasterBand::GetMaskBand(); + } + + double GetOffset(int *pbHasValue) override + { + int bHasValue = FALSE; + double dfVal = GDALPamRasterBand::GetOffset(&bHasValue); + if (bHasValue) + { + if (pbHasValue) + *pbHasValue = true; + return dfVal; + } + if (pbHasValue) + *pbHasValue = !std::isnan(m_dfOffset); + return std::isnan(m_dfOffset) ? 0.0 : m_dfOffset; + } + + double GetScale(int *pbHasValue) override + { + int bHasValue = FALSE; + double dfVal = GDALPamRasterBand::GetScale(&bHasValue); + if (bHasValue) + { + if (pbHasValue) + *pbHasValue = true; + return dfVal; + } + if (pbHasValue) + *pbHasValue = !std::isnan(m_dfScale); + return std::isnan(m_dfScale) ? 1.0 : m_dfScale; + } + + const char *GetUnitType() override + { + const char *pszVal = GDALPamRasterBand::GetUnitType(); + if (pszVal && *pszVal) + return pszVal; + return m_osUnit.c_str(); + } + + char **GetCategoryNames() override + { + return m_aosCategoryNames.List(); + } + + GDALColorTable *GetColorTable() override + { + return m_poColorTable.get(); + } + + GDALRasterAttributeTable *GetDefaultRAT() override + { + return m_poRAT.get(); + } + + int GetOverviewCount() override; + GDALRasterBand *GetOverview(int iOvr) override; + + char **GetMetadataDomainList() override; + const char *GetMetadataItem(const char *pszName, + const char *pszDomain) override; + CPLErr SetMetadataItem(const char *pszName, const char *pszValue, + const char *pszDomain) override; + CPLErr SetMetadata(char **papszMD, const char *pszDomain) override; +}; + +/************************************************************************/ +/* IsSameNaNAware() */ +/************************************************************************/ + +static inline bool IsSameNaNAware(double a, double b) +{ + return a == b || (std::isnan(a) && std::isnan(b)); +} + +/************************************************************************/ +/* VRTTileIndexDataset() */ +/************************************************************************/ + +VRTTileIndexDataset::VRTTileIndexDataset() + : m_osUniqueHandle(CPLSPrintf("%p", this)) +{ +} + +/************************************************************************/ +/* GetAbsoluteFileName() */ +/************************************************************************/ + +static std::string GetAbsoluteFileName(const char *pszTileName, + const char *pszVRTName) +{ + if (CPLIsFilenameRelative(pszTileName) && + !STARTS_WITH(pszTileName, "GetPathComponent().empty()) + { + const std::string osPath(oSubDSInfo->GetPathComponent()); + const std::string osRet = + CPLIsFilenameRelative(osPath.c_str()) + ? oSubDSInfo->ModifyPathComponent( + CPLProjectRelativeFilename(CPLGetPath(pszVRTName), + osPath.c_str())) + : std::string(pszTileName); + GDALDestroySubdatasetInfo(oSubDSInfo); + return osRet; + } + + const std::string osRelativeMadeAbsolute = + CPLProjectRelativeFilename(CPLGetPath(pszVRTName), pszTileName); + VSIStatBufL sStat; + if (VSIStatL(osRelativeMadeAbsolute.c_str(), &sStat) == 0) + return osRelativeMadeAbsolute; + } + return pszTileName; +} + +/************************************************************************/ +/* Open() */ +/************************************************************************/ + +bool VRTTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) +{ + eAccess = poOpenInfo->eAccess; + + CPLXMLNode *psRoot = nullptr; + const char *pszIndexDataset = poOpenInfo->pszFilename; + + if (STARTS_WITH(poOpenInfo->pszFilename, VRTTI_PREFIX)) + { + pszIndexDataset = poOpenInfo->pszFilename + strlen(VRTTI_PREFIX); + } + else if (STARTS_WITH(poOpenInfo->pszFilename, "pszFilename)); + if (m_psXMLTree == nullptr) + return false; + } + else if (strstr(reinterpret_cast(poOpenInfo->pabyHeader), + "pszFilename)); + if (m_psXMLTree == nullptr) + return false; + m_bXMLUpdatable = (poOpenInfo->eAccess == GA_Update); + } + + if (m_psXMLTree) + { + psRoot = CPLGetXMLNode(m_psXMLTree.get(), "=VRTTileIndexDataset"); + if (psRoot == nullptr) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Missing VRTTileIndexDataset root element."); + return false; + } + + pszIndexDataset = CPLGetXMLValue(psRoot, "IndexDataset", nullptr); + if (!pszIndexDataset) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Missing IndexDataset element."); + return false; + } + } + + m_poVectorDS.reset(GDALDataset::Open( + pszIndexDataset, + GDAL_OF_VECTOR | GDAL_OF_VERBOSE_ERROR | + ((poOpenInfo->nOpenFlags & GDAL_OF_UPDATE) ? GDAL_OF_UPDATE + : GDAL_OF_READONLY))); + if (!m_poVectorDS) + return false; + + if (m_poVectorDS->GetLayerCount() == 0) + { + CPLError(CE_Failure, CPLE_AppDefined, "%s has no vector layer", + poOpenInfo->pszFilename); + return false; + } + + double dfOvrFactor = 1.0; + if (const char *pszFactor = + CSLFetchNameValue(poOpenInfo->papszOpenOptions, "FACTOR")) + { + dfOvrFactor = CPLAtof(pszFactor); + if (!(dfOvrFactor > 1.0)) + { + CPLError(CE_Failure, CPLE_AppDefined, "Wrong overview factor"); + return false; + } + } + + const char *pszLayerName; + + if ((pszLayerName = CSLFetchNameValue(poOpenInfo->papszOpenOptions, + "LAYER")) != nullptr) + { + m_poLayer = m_poVectorDS->GetLayerByName(pszLayerName); + if (!m_poLayer) + { + CPLError(CE_Failure, CPLE_AppDefined, "Layer %s does not exist", + pszLayerName); + return false; + } + } + else if (psRoot && (pszLayerName = CPLGetXMLValue(psRoot, "IndexLayer", + nullptr)) != nullptr) + { + m_poLayer = m_poVectorDS->GetLayerByName(pszLayerName); + if (!m_poLayer) + { + CPLError(CE_Failure, CPLE_AppDefined, "Layer %s does not exist", + pszLayerName); + return false; + } + } + else if (!psRoot && (pszLayerName = m_poVectorDS->GetMetadataItem( + MD_DS_TILE_INDEX_LAYER)) != nullptr) + { + m_poLayer = m_poVectorDS->GetLayerByName(pszLayerName); + if (!m_poLayer) + { + CPLError(CE_Failure, CPLE_AppDefined, "Layer %s does not exist", + pszLayerName); + return false; + } + } + else if (m_poVectorDS->GetLayerCount() == 1) + { + m_poLayer = m_poVectorDS->GetLayer(0); + if (!m_poLayer) + { + CPLError(CE_Failure, CPLE_AppDefined, "Cannot open layer 0"); + return false; + } + } + else + { + if (STARTS_WITH(poOpenInfo->pszFilename, VRTTI_PREFIX)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "%s has more than one layer. LAYER open option " + "must be defined to specify which one to " + "use as the tile index", + pszIndexDataset); + } + else if (psRoot) + { + CPLError(CE_Failure, CPLE_AppDefined, + "%s has more than one layer. IndexLayer element must be " + "defined to specify which one to " + "use as the tile index", + pszIndexDataset); + } + else + { + CPLError(CE_Failure, CPLE_AppDefined, + "%s has more than one layer. %s " + "metadata item must be defined to specify which one to " + "use as the tile index", + pszIndexDataset, MD_DS_TILE_INDEX_LAYER); + } + return false; + } + + const auto GetOption = [poOpenInfo, psRoot, this](const char *pszItem) + { + if (psRoot) + { + const char *pszVal = CPLGetXMLValue(psRoot, pszItem, nullptr); + if (pszVal) + return pszVal; + + if (EQUAL(pszItem, MD_BAND_COUNT)) + pszItem = VRTTI_XML_BANDCOUNT; + else if (EQUAL(pszItem, MD_DATA_TYPE)) + pszItem = VRTTI_XML_DATATYPE; + else if (EQUAL(pszItem, MD_NODATA)) + pszItem = VRTTI_XML_NODATAVALUE; + else if (EQUAL(pszItem, MD_COLOR_INTERPRETATION)) + pszItem = VRTTI_XML_COLORINTERP; + else if (EQUAL(pszItem, MD_LOCATION_FIELD)) + pszItem = VRTTI_XML_LOCATIONFIELD; + else if (EQUAL(pszItem, MD_SORT_FIELD)) + pszItem = VRTTI_XML_SORTFIELD; + else if (EQUAL(pszItem, MD_SORT_FIELD_ASC)) + pszItem = VRTTI_XML_SORTFIELDASC; + else if (EQUAL(pszItem, MD_MASK_BAND)) + pszItem = VRTTI_XML_MASKBAND; + pszVal = CPLGetXMLValue(psRoot, pszItem, nullptr); + if (pszVal) + return pszVal; + } + + const char *pszVal = + CSLFetchNameValue(poOpenInfo->papszOpenOptions, pszItem); + if (pszVal) + return pszVal; + return m_poLayer->GetMetadataItem(pszItem); + }; + + const char *pszFilter = GetOption("Filter"); + if (pszFilter) + { + if (m_poLayer->SetAttributeFilter(pszFilter) != OGRERR_NONE) + return false; + } + + const char *pszLocationFieldName = GetOption(MD_LOCATION_FIELD); + if (!pszLocationFieldName) + { + constexpr const char *DEFAULT_LOCATION_FIELD_NAME = "location"; + pszLocationFieldName = DEFAULT_LOCATION_FIELD_NAME; + } + auto poLayerDefn = m_poLayer->GetLayerDefn(); + m_nLocationFieldIndex = poLayerDefn->GetFieldIndex(pszLocationFieldName); + if (m_nLocationFieldIndex < 0) + { + CPLError(CE_Failure, CPLE_AppDefined, "Cannot find field %s", + pszLocationFieldName); + return false; + } + if (poLayerDefn->GetFieldDefn(m_nLocationFieldIndex)->GetType() != + OFTString) + { + CPLError(CE_Failure, CPLE_AppDefined, "Field %s is not of type string", + pszLocationFieldName); + return false; + } + + const char *pszSortFieldName = GetOption(MD_SORT_FIELD); + if (pszSortFieldName) + { + m_nSortFieldIndex = poLayerDefn->GetFieldIndex(pszSortFieldName); + if (m_nSortFieldIndex < 0) + { + CPLError(CE_Failure, CPLE_AppDefined, "Cannot find field %s", + pszSortFieldName); + return false; + } + + const auto eFieldType = + poLayerDefn->GetFieldDefn(m_nSortFieldIndex)->GetType(); + if (eFieldType != OFTString && eFieldType != OFTInteger && + eFieldType != OFTInteger64 && eFieldType != OFTReal && + eFieldType != OFTDate && eFieldType != OFTDateTime) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Unsupported type for field %s", pszSortFieldName); + return false; + } + + const char *pszSortFieldAsc = GetOption(MD_SORT_FIELD_ASC); + if (pszSortFieldAsc) + { + m_bSortFieldAsc = CPLTestBool(pszSortFieldAsc); + } + } + + const char *pszResX = GetOption(MD_RESX); + const char *pszResY = GetOption(MD_RESY); + if (pszResX && !pszResY) + { + CPLError(CE_Failure, CPLE_AppDefined, + "%s metadata item defined, but not %s", MD_RESX, MD_RESY); + return false; + } + if (!pszResX && pszResY) + { + CPLError(CE_Failure, CPLE_AppDefined, + "%s metadata item defined, but not %s", MD_RESY, MD_RESX); + return false; + } + + const char *pszResampling = GetOption(MD_RESAMPLING); + if (pszResampling) + { + const auto nErrorCountBefore = CPLGetErrorCounter(); + m_eResampling = GDALRasterIOGetResampleAlg(pszResampling); + if (nErrorCountBefore != CPLGetErrorCounter()) + { + return false; + } + m_osResampling = pszResampling; + } + + const char *pszMinX = GetOption(MD_MINX); + const char *pszMinY = GetOption(MD_MINY); + const char *pszMaxX = GetOption(MD_MAXX); + const char *pszMaxY = GetOption(MD_MAXY); + const int nCountMinMaxXY = (pszMinX ? 1 : 0) + (pszMinY ? 1 : 0) + + (pszMaxX ? 1 : 0) + (pszMaxY ? 1 : 0); + if (nCountMinMaxXY != 0 && nCountMinMaxXY != 4) + { + CPLError(CE_Failure, CPLE_AppDefined, + "None or all of %s, %s, %s and %s should be specified", + MD_MINX, MD_MINY, MD_MAXX, MD_MAXY); + return false; + } + + const char *pszXSize = GetOption(MD_XSIZE); + const char *pszYSize = GetOption(MD_YSIZE); + const char *pszGeoTransform = GetOption(MD_GEOTRANSFORM); + const int nCountXSizeYSizeGT = + (pszXSize ? 1 : 0) + (pszYSize ? 1 : 0) + (pszGeoTransform ? 1 : 0); + if (nCountXSizeYSizeGT != 0 && nCountXSizeYSizeGT != 3) + { + CPLError(CE_Failure, CPLE_AppDefined, + "None or all of %s, %s, %s should be specified", MD_XSIZE, + MD_YSIZE, MD_GEOTRANSFORM); + return false; + } + + const char *pszDataType = GetOption(MD_DATA_TYPE); + const char *pszColorInterp = GetOption(MD_COLOR_INTERPRETATION); + int nBandCount = 0; + std::vector aeDataTypes; + std::vector> aNoData; + std::vector aeColorInterp; + + const char *pszSRS = GetOption(MD_SRS); + m_oSRS.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); + if (pszSRS) + { + if (m_oSRS.SetFromUserInput( + pszSRS, + OGRSpatialReference::SET_FROM_USER_INPUT_LIMITATIONS_get()) != + OGRERR_NONE) + { + CPLError(CE_Failure, CPLE_AppDefined, "Invalid %s", MD_SRS); + return false; + } + } + else + { + const auto poSRS = m_poLayer->GetSpatialRef(); + // Ignore GPKG "Undefined geographic SRS" and "Undefined Cartesian SRS" + if (poSRS && !STARTS_WITH(poSRS->GetName(), "Undefined ")) + m_oSRS = *poSRS; + } + + std::vector apoXMLNodeBands; + if (psRoot) + { + int nExpectedBandNumber = 1; + for (const CPLXMLNode *psIter = psRoot->psChild; psIter; + psIter = psIter->psNext) + { + if (psIter->eType == CXT_Element && + strcmp(psIter->pszValue, VRTTI_XML_BAND_ELEMENT) == 0) + { + const char *pszBand = + CPLGetXMLValue(psIter, VRTTI_XML_BAND_NUMBER, nullptr); + if (!pszBand) + { + CPLError(CE_Failure, CPLE_AppDefined, + "%s attribute missing on %s element", + VRTTI_XML_BAND_NUMBER, VRTTI_XML_BAND_ELEMENT); + return false; + } + const int nBand = atoi(pszBand); + if (nBand <= 0) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid band number"); + return false; + } + if (nBand != nExpectedBandNumber) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid band number: found %d, expected %d", + nBand, nExpectedBandNumber); + return false; + } + apoXMLNodeBands.push_back(psIter); + ++nExpectedBandNumber; + } + } + } + + const char *pszBandCount = GetOption(MD_BAND_COUNT); + if (pszBandCount) + nBandCount = atoi(pszBandCount); + + if (!apoXMLNodeBands.empty()) + { + if (!pszBandCount) + nBandCount = static_cast(apoXMLNodeBands.size()); + else if (nBandCount != static_cast(apoXMLNodeBands.size())) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Inconsistent %s with actual number of %s elements", + VRTTI_XML_BANDCOUNT, VRTTI_XML_BAND_ELEMENT); + return false; + } + } + + bool bHasMaskBand = false; + if ((!pszBandCount && apoXMLNodeBands.empty()) || + (!(pszResX && pszResY) && nCountXSizeYSizeGT == 0)) + { + CPLDebug("VRT", "Inspecting one feature due to missing metadata items"); + m_bScannedOneFeatureAtOpening = true; + + auto poFeature = + std::unique_ptr(m_poLayer->GetNextFeature()); + if (!poFeature || + !poFeature->IsFieldSetAndNotNull(m_nLocationFieldIndex)) + { + CPLError( + CE_Failure, CPLE_AppDefined, + "BAND_COUNT(+DATA_TYPE+COLOR_INTERPRETATION)+ (RESX+RESY or " + "XSIZE+YSIZE+GEOTRANSFORM) metadata items " + "missing"); + return false; + } + + const char *pszTileName = + poFeature->GetFieldAsString(m_nLocationFieldIndex); + const std::string osTileName( + GetAbsoluteFileName(pszTileName, poOpenInfo->pszFilename)); + pszTileName = osTileName.c_str(); + + struct Releaser + { + void operator()(GDALDataset *poDS) + { + if (poDS) + poDS->Release(); + } + }; + + auto poTileDS = std::shared_ptr( + GDALDataset::Open(pszTileName, + GDAL_OF_RASTER | GDAL_OF_VERBOSE_ERROR), + Releaser()); + if (!poTileDS) + { + return false; + } + + // do palette -> RGB(A) expansion + if (poTileDS->GetRasterCount() == 1 && + (nBandCount == 3 || nBandCount == 4) && + poTileDS->GetRasterBand(1)->GetColorTable() != nullptr) + { + + CPLStringList aosOptions; + aosOptions.AddString("-of"); + aosOptions.AddString("VRT"); + + aosOptions.AddString("-expand"); + aosOptions.AddString(nBandCount == 3 ? "rgb" : "rgba"); + + GDALTranslateOptions *psOptions = + GDALTranslateOptionsNew(aosOptions.List(), nullptr); + int bUsageError = false; + auto poRGBDS = std::unique_ptr(GDALDataset::FromHandle( + GDALTranslate("", GDALDataset::ToHandle(poTileDS.get()), + psOptions, &bUsageError))); + GDALTranslateOptionsFree(psOptions); + if (!poRGBDS) + { + return false; + } + + poTileDS.reset(poRGBDS.release()); + } + + const int nTileBandCount = poTileDS->GetRasterCount(); + for (int i = 0; i < nTileBandCount; ++i) + { + auto poTileBand = poTileDS->GetRasterBand(i + 1); + aeDataTypes.push_back(poTileBand->GetRasterDataType()); + int bHasNoData = FALSE; + const double dfNoData = poTileBand->GetNoDataValue(&bHasNoData); + aNoData.emplace_back(CPL_TO_BOOL(bHasNoData), dfNoData); + aeColorInterp.push_back(poTileBand->GetColorInterpretation()); + + if (poTileBand->GetMaskFlags() == GMF_PER_DATASET) + bHasMaskBand = true; + } + if (!pszBandCount && nBandCount == 0) + nBandCount = nTileBandCount; + + auto poTileSRS = poTileDS->GetSpatialRef(); + if (!m_oSRS.IsEmpty() && poTileSRS && !m_oSRS.IsSame(poTileSRS)) + { + CPLStringList aosOptions; + aosOptions.AddString("-of"); + aosOptions.AddString("VRT"); + + char *pszWKT = nullptr; + const char *const apszWKTOptions[] = {"FORMAT=WKT2_2019", nullptr}; + m_oSRS.exportToWkt(&pszWKT, apszWKTOptions); + if (pszWKT) + m_osWKT = pszWKT; + CPLFree(pszWKT); + + if (m_osWKT.empty()) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Cannot export VRT SRS to WKT2"); + return false; + } + + aosOptions.AddString("-t_srs"); + aosOptions.AddString(m_osWKT.c_str()); + + GDALWarpAppOptions *psWarpOptions = + GDALWarpAppOptionsNew(aosOptions.List(), nullptr); + GDALDatasetH ahSrcDS[] = {GDALDataset::ToHandle(poTileDS.get())}; + int bUsageError = false; + auto poWarpDS = + std::unique_ptr(GDALDataset::FromHandle(GDALWarp( + "", nullptr, 1, ahSrcDS, psWarpOptions, &bUsageError))); + GDALWarpAppOptionsFree(psWarpOptions); + if (!poWarpDS) + { + return false; + } + + poTileDS.reset(poWarpDS.release()); + poTileSRS = poTileDS->GetSpatialRef(); + CPL_IGNORE_RET_VAL(poTileSRS); + } + + double adfGeoTransformTile[6]; + if (poTileDS->GetGeoTransform(adfGeoTransformTile) != CE_None) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Cannot find geotransform on %s", pszTileName); + return false; + } + if (!(adfGeoTransformTile[GT_ROTATION_PARAM1] == 0)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "3rd value of GeoTransform of %s should be 0", + pszTileName); + return false; + } + if (!(adfGeoTransformTile[GT_ROTATION_PARAM2] == 0)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "5th value of GeoTransform of %s should be 0", + pszTileName); + return false; + } + if (!(adfGeoTransformTile[GT_NS_RES] < 0)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "6th value of GeoTransform of %s should be < 0", + pszTileName); + return false; + } + + const double dfResX = adfGeoTransformTile[GT_WE_RES]; + const double dfResY = -adfGeoTransformTile[GT_NS_RES]; + + OGREnvelope sEnvelope; + if (m_poLayer->GetExtent(&sEnvelope, /* bForce = */ false) == + OGRERR_FAILURE) + { + if (m_poLayer->GetExtent(&sEnvelope, /* bForce = */ true) == + OGRERR_FAILURE) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Cannot get layer extent"); + return false; + } + CPLError( + CE_Warning, CPLE_AppDefined, + "Could get layer extent, but using a potentially slow method"); + } + + const double dfXSize = (sEnvelope.MaxX - sEnvelope.MinX) / dfResX; + if (!(dfXSize >= 0 && dfXSize < INT_MAX)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Too small %s, or wrong layer extent", MD_RESX); + return false; + } + + const double dfYSize = (sEnvelope.MaxY - sEnvelope.MinY) / dfResY; + if (!(dfYSize >= 0 && dfYSize < INT_MAX)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Too small %s, or wrong layer extent", MD_RESY); + return false; + } + + m_adfGeoTransform[GT_TOPLEFT_X] = sEnvelope.MinX; + m_adfGeoTransform[GT_WE_RES] = dfResX; + m_adfGeoTransform[GT_ROTATION_PARAM1] = 0; + m_adfGeoTransform[GT_TOPLEFT_Y] = sEnvelope.MaxY; + m_adfGeoTransform[GT_ROTATION_PARAM2] = 0; + m_adfGeoTransform[GT_NS_RES] = -dfResY; + nRasterXSize = static_cast(std::ceil(dfXSize)); + nRasterYSize = static_cast(std::ceil(dfYSize)); + } + + if (pszXSize && pszYSize && pszGeoTransform) + { + const int nXSize = atoi(pszXSize); + if (nXSize <= 0) + { + CPLError(CE_Failure, CPLE_AppDefined, + "%s metadata item should be > 0", MD_XSIZE); + return false; + } + + const int nYSize = atoi(pszYSize); + if (nYSize <= 0) + { + CPLError(CE_Failure, CPLE_AppDefined, + "%s metadata item should be > 0", MD_YSIZE); + return false; + } + + const CPLStringList aosTokens( + CSLTokenizeString2(pszGeoTransform, ",", 0)); + if (aosTokens.size() != 6) + { + CPLError(CE_Failure, CPLE_AppDefined, + "%s metadata item should be 6 numeric values " + "separated with comma", + MD_GEOTRANSFORM); + return false; + } + for (int i = 0; i < 6; ++i) + { + m_adfGeoTransform[i] = CPLAtof(aosTokens[i]); + } + if (!(m_adfGeoTransform[GT_ROTATION_PARAM1] == 0)) + { + CPLError(CE_Failure, CPLE_AppDefined, "3rd value of %s should be 0", + MD_GEOTRANSFORM); + return false; + } + if (!(m_adfGeoTransform[GT_ROTATION_PARAM2] == 0)) + { + CPLError(CE_Failure, CPLE_AppDefined, "5th value of %s should be 0", + MD_GEOTRANSFORM); + return false; + } + if (!(m_adfGeoTransform[GT_NS_RES] < 0)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "6th value of %s should be < 0", MD_GEOTRANSFORM); + return false; + } + + nRasterXSize = nXSize; + nRasterYSize = nYSize; + } + else if (pszResX && pszResY) + { + const double dfResX = CPLAtof(pszResX); + if (!(dfResX > 0)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "RESX metadata item should be > 0"); + return false; + } + const double dfResY = CPLAtof(pszResY); + if (!(dfResY > 0)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "RESY metadata item should be > 0"); + return false; + } + + OGREnvelope sEnvelope; + + if (nCountMinMaxXY == 4) + { + if (pszXSize || pszYSize || pszGeoTransform) + { + CPLError(CE_Warning, CPLE_AppDefined, + "Ignoring %s, %s and %s when %s, " + "%s, %s and %s are specified", + MD_XSIZE, MD_YSIZE, MD_GEOTRANSFORM, MD_MINX, MD_MINY, + MD_MAXX, MD_MAXY); + } + const double dfMinX = CPLAtof(pszMinX); + const double dfMinY = CPLAtof(pszMinY); + const double dfMaxX = CPLAtof(pszMaxX); + const double dfMaxY = CPLAtof(pszMaxY); + if (!(dfMaxX > dfMinX)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "%s metadata item should be > %s", MD_MAXX, MD_MINX); + return false; + } + if (!(dfMaxY > dfMinY)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "%s metadata item should be > %s", MD_MAXY, MD_MINY); + return false; + } + sEnvelope.MinX = dfMinX; + sEnvelope.MinY = dfMinY; + sEnvelope.MaxX = dfMaxX; + sEnvelope.MaxY = dfMaxY; + } + else if (m_poLayer->GetExtent(&sEnvelope, /* bForce = */ false) == + OGRERR_FAILURE) + { + if (m_poLayer->GetExtent(&sEnvelope, /* bForce = */ true) == + OGRERR_FAILURE) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Cannot get layer extent"); + return false; + } + CPLError( + CE_Warning, CPLE_AppDefined, + "Could get layer extent, but using a potentially slow method"); + } + + const double dfXSize = (sEnvelope.MaxX - sEnvelope.MinX) / dfResX; + if (!(dfXSize >= 0 && dfXSize < INT_MAX)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Too small %s, or wrong layer extent", MD_RESX); + return false; + } + + const double dfYSize = (sEnvelope.MaxY - sEnvelope.MinY) / dfResY; + if (!(dfYSize >= 0 && dfYSize < INT_MAX)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Too small %s, or wrong layer extent", MD_RESY); + return false; + } + + m_adfGeoTransform[GT_TOPLEFT_X] = sEnvelope.MinX; + m_adfGeoTransform[GT_WE_RES] = dfResX; + m_adfGeoTransform[GT_ROTATION_PARAM1] = 0; + m_adfGeoTransform[GT_TOPLEFT_Y] = sEnvelope.MaxY; + m_adfGeoTransform[GT_ROTATION_PARAM2] = 0; + m_adfGeoTransform[GT_NS_RES] = -dfResY; + nRasterXSize = static_cast(std::ceil(dfXSize)); + nRasterYSize = static_cast(std::ceil(dfYSize)); + } + + if (nBandCount == 0 && !pszBandCount) + { + CPLError(CE_Failure, CPLE_AppDefined, "%s metadata item missing", + MD_BAND_COUNT); + return false; + } + + if (!GDALCheckBandCount(nBandCount, false)) + return false; + + if (aeDataTypes.empty() && !pszDataType) + { + aeDataTypes.resize(nBandCount, GDT_Byte); + } + else if (pszDataType) + { + aeDataTypes.clear(); + const CPLStringList aosTokens(CSLTokenizeString2(pszDataType, ", ", 0)); + if (aosTokens.size() == 1) + { + const auto eDataType = GDALGetDataTypeByName(aosTokens[0]); + if (eDataType == GDT_Unknown) + { + CPLError(CE_Failure, CPLE_AppDefined, "Invalid value for %s", + MD_DATA_TYPE); + return false; + } + aeDataTypes.resize(nBandCount, eDataType); + } + else if (aosTokens.size() == nBandCount) + { + for (int i = 0; i < nBandCount; ++i) + { + const auto eDataType = GDALGetDataTypeByName(aosTokens[i]); + if (eDataType == GDT_Unknown) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid value for %s", MD_DATA_TYPE); + return false; + } + aeDataTypes.push_back(eDataType); + } + } + else + { + CPLError(CE_Failure, CPLE_AppDefined, + "Number of values in %s should be 1 or %s", MD_DATA_TYPE, + MD_BAND_COUNT); + return false; + } + } + + const char *pszNoData = GetOption(MD_NODATA); + if (pszNoData) + { + const auto IsValidNoDataStr = [](const char *pszStr) + { + return EQUAL(pszStr, "inf") || EQUAL(pszStr, "-inf") || + EQUAL(pszStr, "nan") || + CPLGetValueType(pszStr) != CPL_VALUE_STRING; + }; + + aNoData.clear(); + const CPLStringList aosTokens(CSLTokenizeString2(pszNoData, ", ", 0)); + if (aosTokens.size() == 1) + { + if (!EQUAL(aosTokens[0], "NONE")) + { + if (!IsValidNoDataStr(aosTokens[0])) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid value for %s", MD_NODATA); + return false; + } + aNoData.resize(nBandCount, + std::pair(true, CPLAtof(aosTokens[0]))); + } + } + else if (aosTokens.size() == nBandCount) + { + for (int i = 0; i < nBandCount; ++i) + { + if (EQUAL(aosTokens[i], "NONE")) + { + aNoData.emplace_back(false, 0); + } + else if (IsValidNoDataStr(aosTokens[i])) + { + aNoData.emplace_back(true, CPLAtof(aosTokens[i])); + } + else + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid value for %s", MD_NODATA); + return false; + } + } + } + else + { + CPLError(CE_Failure, CPLE_AppDefined, + "Number of values in %s should be 1 or %s", MD_NODATA, + MD_BAND_COUNT); + return false; + } + } + + if (pszColorInterp) + { + aeColorInterp.clear(); + const CPLStringList aosTokens( + CSLTokenizeString2(pszColorInterp, ", ", 0)); + if (aosTokens.size() == 1) + { + const auto eInterp = GDALGetColorInterpretationByName(aosTokens[0]); + if (eInterp == GCI_Undefined && + !EQUAL(aosTokens[0], + GDALGetColorInterpretationName(GCI_Undefined))) + { + CPLError(CE_Failure, CPLE_AppDefined, "Invalid value for %s", + MD_COLOR_INTERPRETATION); + return false; + } + aeColorInterp.resize(nBandCount, eInterp); + } + else if (aosTokens.size() == nBandCount) + { + for (int i = 0; i < nBandCount; ++i) + { + const auto eInterp = + GDALGetColorInterpretationByName(aosTokens[i]); + if (eInterp == GCI_Undefined && + !EQUAL(aosTokens[i], + GDALGetColorInterpretationName(GCI_Undefined))) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid value for %s", MD_COLOR_INTERPRETATION); + return false; + } + aeColorInterp.emplace_back(eInterp); + } + } + else + { + CPLError(CE_Failure, CPLE_AppDefined, + "Number of values in %s should be 1 or " + "%s", + MD_COLOR_INTERPRETATION, MD_BAND_COUNT); + return false; + } + } + + /* -------------------------------------------------------------------- */ + /* Create bands. */ + /* -------------------------------------------------------------------- */ + if (aeDataTypes.size() != static_cast(nBandCount)) + { + CPLError( + CE_Failure, CPLE_AppDefined, + "Number of data types values found not matching number of bands"); + return false; + } + if (!aNoData.empty() && aNoData.size() != static_cast(nBandCount)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Number of nodata values found not matching number of bands"); + return false; + } + if (!aeColorInterp.empty() && + aeColorInterp.size() != static_cast(nBandCount)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Number of color interpretation values found not matching " + "number of bands"); + return false; + } + + int nBlockXSize = 256; + const char *pszBlockXSize = GetOption(MD_BLOCKXSIZE); + if (pszBlockXSize) + { + nBlockXSize = atoi(pszBlockXSize); + if (nBlockXSize <= 0) + { + CPLError(CE_Failure, CPLE_AppDefined, "Invalid %s", MD_BLOCKXSIZE); + return false; + } + } + + int nBlockYSize = 256; + const char *pszBlockYSize = GetOption(MD_BLOCKYSIZE); + if (pszBlockYSize) + { + nBlockYSize = atoi(pszBlockYSize); + if (nBlockYSize <= 0) + { + CPLError(CE_Failure, CPLE_AppDefined, "Invalid %s", MD_BLOCKYSIZE); + return false; + } + } + + if (nBlockXSize > INT_MAX / nBlockYSize) + { + CPLError(CE_Failure, CPLE_AppDefined, "Too big %s * %s", MD_BLOCKXSIZE, + MD_BLOCKYSIZE); + return false; + } + + if (dfOvrFactor > 1.0) + { + m_adfGeoTransform[GT_WE_RES] *= dfOvrFactor; + m_adfGeoTransform[GT_NS_RES] *= dfOvrFactor; + nRasterXSize = static_cast(std::ceil(nRasterXSize / dfOvrFactor)); + nRasterYSize = static_cast(std::ceil(nRasterYSize / dfOvrFactor)); + } + + VRTTileIndexBand *poFirstBand = nullptr; + for (int i = 0; i < nBandCount; ++i) + { + GDALDataType eDataType = aeDataTypes[i]; + if (!apoXMLNodeBands.empty()) + { + const char *pszVal = CPLGetXMLValue( + apoXMLNodeBands[i], VRTTI_XML_BAND_DATATYPE, nullptr); + if (pszVal) + { + eDataType = GDALGetDataTypeByName(pszVal); + if (eDataType == GDT_Unknown) + return false; + } + } + auto poBandUniquePtr = std::make_unique( + this, i + 1, eDataType, nBlockXSize, nBlockYSize); + auto poBand = poBandUniquePtr.get(); + SetBand(i + 1, poBandUniquePtr.release()); + if (!poFirstBand) + poFirstBand = poBand; + if (poBand->GetRasterDataType() != poFirstBand->GetRasterDataType()) + { + m_bSameDataType = false; + } + + if (!apoXMLNodeBands.empty()) + { + const char *pszVal = CPLGetXMLValue( + apoXMLNodeBands[i], VRTTI_XML_BAND_DESCRIPTION, nullptr); + if (pszVal) + { + poBand->GDALRasterBand::SetDescription(pszVal); + } + } + + if (!aNoData.empty() && aNoData[i].first) + { + poBand->m_bNoDataValueSet = true; + poBand->m_dfNoDataValue = aNoData[i].second; + } + if (!apoXMLNodeBands.empty()) + { + const char *pszVal = CPLGetXMLValue( + apoXMLNodeBands[i], VRTTI_XML_BAND_NODATAVALUE, nullptr); + if (pszVal) + { + poBand->m_bNoDataValueSet = true; + poBand->m_dfNoDataValue = CPLAtof(pszVal); + } + } + if (poBand->m_bNoDataValueSet != poFirstBand->m_bNoDataValueSet || + !IsSameNaNAware(poBand->m_dfNoDataValue, + poFirstBand->m_dfNoDataValue)) + { + m_bSameNoData = false; + } + + if (!aeColorInterp.empty()) + { + poBand->m_eColorInterp = aeColorInterp[i]; + } + if (!apoXMLNodeBands.empty()) + { + const char *pszVal = CPLGetXMLValue( + apoXMLNodeBands[i], VRTTI_XML_BAND_COLORINTERP, nullptr); + if (pszVal) + { + poBand->m_eColorInterp = + GDALGetColorInterpretationByName(pszVal); + } + } + + if (const char *pszScale = + GetOption(CPLSPrintf("BAND_%d_%s", i + 1, MD_BAND_SCALE))) + { + poBand->m_dfScale = CPLAtof(pszScale); + } + if (!apoXMLNodeBands.empty()) + { + const char *pszVal = CPLGetXMLValue(apoXMLNodeBands[i], + VRTTI_XML_BAND_SCALE, nullptr); + if (pszVal) + { + poBand->m_dfScale = CPLAtof(pszVal); + } + } + + if (const char *pszOffset = + GetOption(CPLSPrintf("BAND_%d_%s", i + 1, MD_BAND_OFFSET))) + { + poBand->m_dfOffset = CPLAtof(pszOffset); + } + if (!apoXMLNodeBands.empty()) + { + const char *pszVal = CPLGetXMLValue(apoXMLNodeBands[i], + VRTTI_XML_BAND_OFFSET, nullptr); + if (pszVal) + { + poBand->m_dfOffset = CPLAtof(pszVal); + } + } + + if (const char *pszUnit = + GetOption(CPLSPrintf("BAND_%d_%s", i + 1, MD_BAND_UNITTYPE))) + { + poBand->m_osUnit = pszUnit; + } + if (!apoXMLNodeBands.empty()) + { + const char *pszVal = CPLGetXMLValue( + apoXMLNodeBands[i], VRTTI_XML_BAND_UNITTYPE, nullptr); + if (pszVal) + { + poBand->m_osUnit = pszVal; + } + } + + if (!apoXMLNodeBands.empty()) + { + const CPLXMLNode *psBandNode = apoXMLNodeBands[i]; + poBand->oMDMD.XMLInit(psBandNode, TRUE); + + if (const CPLXMLNode *psCategoryNames = + CPLGetXMLNode(psBandNode, VRTTI_XML_CATEGORYNAMES)) + { + poBand->m_aosCategoryNames = + VRTParseCategoryNames(psCategoryNames); + } + + if (const CPLXMLNode *psColorTable = + CPLGetXMLNode(psBandNode, VRTTI_XML_COLORTABLE)) + { + poBand->m_poColorTable = VRTParseColorTable(psColorTable); + } + + if (const CPLXMLNode *psRAT = + CPLGetXMLNode(psBandNode, VRTTI_XML_RAT)) + { + poBand->m_poRAT = + std::make_unique(); + poBand->m_poRAT->XMLInit(psRAT, ""); + } + } + } + + const char *pszMaskBand = GetOption(MD_MASK_BAND); + if (pszMaskBand) + bHasMaskBand = CPLTestBool(pszMaskBand); + if (bHasMaskBand) + { + m_poMaskBand = std::make_unique( + this, 0, GDT_Byte, nBlockXSize, nBlockYSize); + } + + if (dfOvrFactor == 1.0) + { + if (psRoot) + { + for (const CPLXMLNode *psIter = psRoot->psChild; psIter; + psIter = psIter->psNext) + { + if (psIter->eType == CXT_Element && + strcmp(psIter->pszValue, VRTTI_XML_OVERVIEW_ELEMENT) == 0) + { + const char *pszDataset = CPLGetXMLValue( + psIter, VRTTI_XML_OVERVIEW_DATASET, nullptr); + const char *pszLayer = CPLGetXMLValue( + psIter, VRTTI_XML_OVERVIEW_LAYER, nullptr); + const char *pszFactor = CPLGetXMLValue( + psIter, VRTTI_XML_OVERVIEW_FACTOR, nullptr); + if (!pszDataset && !pszLayer && !pszFactor) + { + CPLError(CE_Failure, CPLE_AppDefined, + "At least one of %s, %s or %s element " + "must be present as an %s child", + VRTTI_XML_OVERVIEW_DATASET, + VRTTI_XML_OVERVIEW_LAYER, + VRTTI_XML_OVERVIEW_FACTOR, + VRTTI_XML_OVERVIEW_ELEMENT); + return false; + } + m_aoOverviewDescriptor.emplace_back( + std::string(pszDataset ? pszDataset : ""), + CPLStringList( + GDALDeserializeOpenOptionsFromXML(psIter)), + std::string(pszLayer ? pszLayer : ""), + pszFactor ? CPLAtof(pszFactor) : 0.0); + } + } + } + else + { + for (int iOvr = 1;; ++iOvr) + { + const char *pszOvrDSName = + GetOption(CPLSPrintf("OVERVIEW_%d_DATASET", iOvr)); + const char *pszOpenOptions = + GetOption(CPLSPrintf("OVERVIEW_%d_OPEN_OPTIONS", iOvr)); + const char *pszOvrLayer = + GetOption(CPLSPrintf("OVERVIEW_%d_LAYER", iOvr)); + const char *pszOvrFactor = + GetOption(CPLSPrintf("OVERVIEW_%d_FACTOR", iOvr)); + if (!pszOvrDSName && !pszOvrLayer && !pszOvrFactor) + break; + m_aoOverviewDescriptor.emplace_back( + std::string(pszOvrDSName ? pszOvrDSName : ""), + pszOpenOptions ? CPLStringList(CSLTokenizeString2( + pszOpenOptions, ",", 0)) + : CPLStringList(), + std::string(pszOvrLayer ? pszOvrLayer : ""), + pszOvrFactor ? CPLAtof(pszOvrFactor) : 0.0); + } + } + } + + if (psRoot) + { + oMDMD.XMLInit(psRoot, TRUE); + } + else + { + // Set on the dataset all metadata items from the index layer which are + // not "reserved" keywords. + char **papszLayerMD = m_poLayer->GetMetadata(); + for (CSLConstList papszIter = papszLayerMD; papszIter && *papszIter; + ++papszIter) + { + if (STARTS_WITH_CI(*papszIter, "OVERVIEW_")) + { + continue; + } + + bool bIsVRTItem = false; + for (const char *pszTest : apszTIOptions) + { + if (STARTS_WITH(*papszIter, pszTest) && + (*papszIter)[strlen(pszTest)] == '=') + { + bIsVRTItem = true; + break; + } + } + if (!bIsVRTItem) + { + if (STARTS_WITH_CI(*papszIter, "BAND_")) + { + const int nBandNr = atoi(*papszIter + strlen("BAND_")); + const char *pszNextUnderscore = + strchr(*papszIter + strlen("BAND_"), '_'); + if (pszNextUnderscore && nBandNr >= 1 && nBandNr <= nBands) + { + char *pszKey = nullptr; + const char *pszValue = + CPLParseNameValue(pszNextUnderscore + 1, &pszKey); + if (pszKey && pszValue) + { + bool bIsReservedBandItem = false; + for (const char *pszItem : apszReservedBandItems) + { + if (EQUAL(pszKey, pszItem)) + { + bIsReservedBandItem = true; + break; + } + } + if (!bIsReservedBandItem) + { + GetRasterBand(nBandNr) + ->GDALRasterBand::SetMetadataItem(pszKey, + pszValue); + } + } + CPLFree(pszKey); + } + } + else + { + char *pszKey = nullptr; + const char *pszValue = + CPLParseNameValue(*papszIter, &pszKey); + if (pszKey && pszValue) + { + GDALDataset::SetMetadataItem(pszKey, pszValue); + } + CPLFree(pszKey); + } + } + } + } + + if (nBandCount > 1 && !GetMetadata("IMAGE_STRUCTURE")) + { + GDALDataset::SetMetadataItem("INTERLEAVE", "PIXEL", "IMAGE_STRUCTURE"); + } + + /* -------------------------------------------------------------------- */ + /* Initialize any PAM information. */ + /* -------------------------------------------------------------------- */ + SetDescription(poOpenInfo->pszFilename); + TryLoadXML(); + + /* -------------------------------------------------------------------- */ + /* Check for overviews. */ + /* -------------------------------------------------------------------- */ + oOvManager.Initialize(this, poOpenInfo->pszFilename); + + return true; +} + +/************************************************************************/ +/* GetMetadataItem() */ +/************************************************************************/ + +const char *VRTTileIndexDataset::GetMetadataItem(const char *pszName, + const char *pszDomain) +{ + if (pszName && pszDomain && EQUAL(pszDomain, "__DEBUG__")) + { + if (EQUAL(pszName, "SCANNED_ONE_FEATURE_AT_OPENING")) + { + return m_bScannedOneFeatureAtOpening ? "YES" : "NO"; + } + else if (EQUAL(pszName, "NUMBER_OF_CONTRIBUTING_SOURCES")) + { + return CPLSPrintf("%d", static_cast(m_aoSourceDesc.size())); + } + } + return GDALPamDataset::GetMetadataItem(pszName, pszDomain); +} + +/************************************************************************/ +/* TileIndexSupportsEditingLayerMetadata() */ +/************************************************************************/ + +bool VRTTileIndexDataset::TileIndexSupportsEditingLayerMetadata() const +{ + return eAccess == GA_Update && m_poVectorDS->GetDriver() && + EQUAL(m_poVectorDS->GetDriver()->GetDescription(), "GPKG"); +} + +/************************************************************************/ +/* SetMetadataItem() */ +/************************************************************************/ + +CPLErr VRTTileIndexDataset::SetMetadataItem(const char *pszName, + const char *pszValue, + const char *pszDomain) +{ + if (m_bXMLUpdatable) + { + m_bXMLModified = true; + return GDALDataset::SetMetadataItem(pszName, pszValue, pszDomain); + } + else if (TileIndexSupportsEditingLayerMetadata()) + { + m_poLayer->SetMetadataItem(pszName, pszValue, pszDomain); + return GDALDataset::SetMetadataItem(pszName, pszValue, pszDomain); + } + else + { + return GDALPamDataset::SetMetadataItem(pszName, pszValue, pszDomain); + } +} + +/************************************************************************/ +/* SetMetadata() */ +/************************************************************************/ + +CPLErr VRTTileIndexDataset::SetMetadata(char **papszMD, const char *pszDomain) +{ + if (m_bXMLUpdatable) + { + m_bXMLModified = true; + return GDALDataset::SetMetadata(papszMD, pszDomain); + } + else if (TileIndexSupportsEditingLayerMetadata()) + { + if (!pszDomain || pszDomain[0] == 0) + { + CPLStringList aosMD(CSLDuplicate(papszMD)); + + // Reinject dataset reserved items + for (const char *pszItem : apszTIOptions) + { + if (!aosMD.FetchNameValue(pszItem)) + { + const char *pszValue = m_poLayer->GetMetadataItem(pszItem); + if (pszValue) + { + aosMD.SetNameValue(pszItem, pszValue); + } + } + } + + // Reinject band metadata + char **papszExistingLayerMD = m_poLayer->GetMetadata(); + for (int i = 0; papszExistingLayerMD && papszExistingLayerMD[i]; + ++i) + { + if (STARTS_WITH_CI(papszExistingLayerMD[i], "BAND_")) + { + aosMD.AddString(papszExistingLayerMD[i]); + } + } + + m_poLayer->SetMetadata(aosMD.List(), pszDomain); + } + else + { + m_poLayer->SetMetadata(papszMD, pszDomain); + } + return GDALDataset::SetMetadata(papszMD, pszDomain); + } + else + { + return GDALPamDataset::SetMetadata(papszMD, pszDomain); + } +} + +/************************************************************************/ +/* VRTTileIndexDatasetIdentify() */ +/************************************************************************/ + +static int VRTTileIndexDatasetIdentify(GDALOpenInfo *poOpenInfo) +{ + if (STARTS_WITH(poOpenInfo->pszFilename, VRTTI_PREFIX)) + return true; + + if (STARTS_WITH(poOpenInfo->pszFilename, "nHeaderBytes > 0 && + (poOpenInfo->nOpenFlags & GDAL_OF_RASTER) != 0 && + (strstr(reinterpret_cast(poOpenInfo->pabyHeader), + "pszFilename, ".vrt.gpkg") || + ENDS_WITH_CI(poOpenInfo->pszFilename, ".vrt.fgb") || + ENDS_WITH_CI(poOpenInfo->pszFilename, ".vrt.parquet")); +} + +/************************************************************************/ +/* VRTTileIndexDatasetOpen() */ +/************************************************************************/ + +static GDALDataset *VRTTileIndexDatasetOpen(GDALOpenInfo *poOpenInfo) +{ + if (!VRTTileIndexDatasetIdentify(poOpenInfo)) + return nullptr; + auto poDS = std::make_unique(); + if (!poDS->Open(poOpenInfo)) + return nullptr; + return poDS.release(); +} + +/************************************************************************/ +/* ~VRTTileIndexDataset() */ +/************************************************************************/ + +VRTTileIndexDataset::~VRTTileIndexDataset() +{ + VRTTileIndexDataset::FlushCache(true); +} + +/************************************************************************/ +/* FlushCache() */ +/************************************************************************/ + +CPLErr VRTTileIndexDataset::FlushCache(bool bAtClosing) +{ + CPLErr eErr = CE_None; + if (bAtClosing && m_bXMLModified) + { + CPLXMLNode *psRoot = + CPLGetXMLNode(m_psXMLTree.get(), "=VRTTileIndexDataset"); + + // Suppress existing dataset metadata + while (true) + { + CPLXMLNode *psExistingMetadata = CPLGetXMLNode(psRoot, "Metadata"); + if (!psExistingMetadata) + break; + CPLRemoveXMLChild(psRoot, psExistingMetadata); + } + + // Serialize new dataset metadata + if (CPLXMLNode *psMD = oMDMD.Serialize()) + CPLAddXMLChild(psRoot, psMD); + + // Update existing band metadata + if (CPLGetXMLNode(psRoot, VRTTI_XML_BAND_ELEMENT)) + { + for (CPLXMLNode *psIter = psRoot->psChild; psIter; + psIter = psIter->psNext) + { + if (psIter->eType == CXT_Element && + strcmp(psIter->pszValue, VRTTI_XML_BAND_ELEMENT)) + { + const char *pszBand = + CPLGetXMLValue(psIter, VRTTI_XML_BAND_NUMBER, nullptr); + if (pszBand) + { + const int nBand = atoi(pszBand); + if (nBand >= 1 && nBand <= nBands) + { + while (true) + { + CPLXMLNode *psExistingMetadata = + CPLGetXMLNode(psIter, "Metadata"); + if (!psExistingMetadata) + break; + CPLRemoveXMLChild(psIter, psExistingMetadata); + } + + auto poBand = cpl::down_cast( + papoBands[nBand - 1]); + if (CPLXMLNode *psMD = poBand->oMDMD.Serialize()) + CPLAddXMLChild(psIter, psMD); + } + } + } + } + } + else + { + // Create new band objects if they have metadata + std::vector aoBandXML; + bool bHasBandMD = false; + for (int i = 1; i <= nBands; ++i) + { + auto poBand = + cpl::down_cast(papoBands[i - 1]); + auto psMD = poBand->oMDMD.Serialize(); + if (psMD) + bHasBandMD = true; + aoBandXML.emplace_back(CPLXMLTreeCloser(psMD)); + } + if (bHasBandMD) + { + for (int i = 1; i <= nBands; ++i) + { + auto poBand = + cpl::down_cast(papoBands[i - 1]); + + CPLXMLNode *psBand = CPLCreateXMLNode( + psRoot, CXT_Element, VRTTI_XML_BAND_ELEMENT); + CPLAddXMLAttributeAndValue(psBand, VRTTI_XML_BAND_NUMBER, + CPLSPrintf("%d", i)); + CPLAddXMLAttributeAndValue( + psBand, VRTTI_XML_BAND_DATATYPE, + GDALGetDataTypeName(poBand->GetRasterDataType())); + + const char *pszDescription = poBand->GetDescription(); + if (pszDescription && pszDescription[0]) + CPLSetXMLValue(psBand, VRTTI_XML_BAND_DESCRIPTION, + pszDescription); + + const auto eColorInterp = poBand->GetColorInterpretation(); + if (eColorInterp != GCI_Undefined) + CPLSetXMLValue( + psBand, VRTTI_XML_BAND_COLORINTERP, + GDALGetColorInterpretationName(eColorInterp)); + + if (!std::isnan(poBand->m_dfOffset)) + CPLSetXMLValue(psBand, VRTTI_XML_BAND_OFFSET, + CPLSPrintf("%.16g", poBand->m_dfOffset)); + + if (!std::isnan(poBand->m_dfScale)) + CPLSetXMLValue(psBand, VRTTI_XML_BAND_SCALE, + CPLSPrintf("%.16g", poBand->m_dfScale)); + + if (!poBand->m_osUnit.empty()) + CPLSetXMLValue(psBand, VRTTI_XML_BAND_UNITTYPE, + poBand->m_osUnit.c_str()); + + if (poBand->m_bNoDataValueSet) + { + CPLSetXMLValue( + psBand, VRTTI_XML_BAND_NODATAVALUE, + VRTSerializeNoData(poBand->m_dfNoDataValue, + poBand->GetRasterDataType(), 18) + .c_str()); + } + if (aoBandXML[i - 1]) + { + CPLAddXMLChild(psBand, aoBandXML[i - 1].release()); + } + } + } + } + + if (!CPLSerializeXMLTreeToFile(m_psXMLTree.get(), GetDescription())) + eErr = CE_Failure; + } + + m_oMapSharedSources.clear(); + m_dfLastMinXFilter = std::numeric_limits::quiet_NaN(); + m_dfLastMinYFilter = std::numeric_limits::quiet_NaN(); + m_dfLastMaxXFilter = std::numeric_limits::quiet_NaN(); + m_dfLastMaxYFilter = std::numeric_limits::quiet_NaN(); + m_aoSourceDesc.clear(); + if (GDALPamDataset::FlushCache(bAtClosing) != CE_None) + eErr = CE_Failure; + return eErr; +} + +/************************************************************************/ +/* LoadOverviews() */ +/************************************************************************/ + +void VRTTileIndexDataset::LoadOverviews() +{ + if (m_apoOverviews.empty() && !m_aoOverviewDescriptor.empty()) + { + for (const auto &[osDSName, aosOpenOptions, osLyrName, dfFactor] : + m_aoOverviewDescriptor) + { + CPLStringList aosNewOpenOptions(aosOpenOptions); + if (dfFactor != 0) + { + aosNewOpenOptions.SetNameValue("@FACTOR", + CPLSPrintf("%.18g", dfFactor)); + } + if (!osLyrName.empty()) + { + aosNewOpenOptions.SetNameValue("@LAYER", osLyrName.c_str()); + } + + std::unique_ptr poOvrDS(GDALDataset::Open( + !osDSName.empty() ? osDSName.c_str() : GetDescription(), + GDAL_OF_RASTER | GDAL_OF_VERBOSE_ERROR, nullptr, + aosNewOpenOptions.List(), nullptr)); + if (poOvrDS) + { + if (poOvrDS->GetRasterCount() == GetRasterCount()) + { + m_apoOverviews.emplace_back(std::move(poOvrDS)); + } + else + { + CPLError(CE_Warning, CPLE_AppDefined, + "%s has not the same number of bands as %s", + poOvrDS->GetDescription(), GetDescription()); + } + } + } + } +} + +/************************************************************************/ +/* GetOverviewCount() */ +/************************************************************************/ + +int VRTTileIndexBand::GetOverviewCount() +{ + const int nPAMOverviews = GDALPamRasterBand::GetOverviewCount(); + if (nPAMOverviews) + return nPAMOverviews; + + m_poDS->LoadOverviews(); + return static_cast(m_poDS->m_apoOverviews.size()); +} + +/************************************************************************/ +/* GetOverview() */ +/************************************************************************/ + +GDALRasterBand *VRTTileIndexBand::GetOverview(int iOvr) +{ + if (iOvr < 0 || iOvr >= GetOverviewCount()) + return nullptr; + + const int nPAMOverviews = GDALPamRasterBand::GetOverviewCount(); + if (nPAMOverviews) + return GDALPamRasterBand::GetOverview(iOvr); + + if (nBand == 0) + { + auto poBand = m_poDS->m_apoOverviews[iOvr]->GetRasterBand(1); + if (!poBand) + return nullptr; + return poBand->GetMaskBand(); + } + else + { + return m_poDS->m_apoOverviews[iOvr]->GetRasterBand(nBand); + } +} + +/************************************************************************/ +/* GetGeoTransform() */ +/************************************************************************/ + +CPLErr VRTTileIndexDataset::GetGeoTransform(double *padfGeoTransform) +{ + memcpy(padfGeoTransform, m_adfGeoTransform.data(), 6 * sizeof(double)); + return CE_None; +} + +/************************************************************************/ +/* GetSpatialRef() */ +/************************************************************************/ + +const OGRSpatialReference *VRTTileIndexDataset::GetSpatialRef() const +{ + return m_oSRS.IsEmpty() ? nullptr : &m_oSRS; +} + +/************************************************************************/ +/* VRTTileIndexBand() */ +/************************************************************************/ + +VRTTileIndexBand::VRTTileIndexBand(VRTTileIndexDataset *poDSIn, int nBandIn, + GDALDataType eDT, int nBlockXSizeIn, + int nBlockYSizeIn) +{ + m_poDS = poDSIn; + nBand = nBandIn; + eDataType = eDT; + nRasterXSize = poDSIn->GetRasterXSize(); + nRasterYSize = poDSIn->GetRasterYSize(); + nBlockXSize = nBlockXSizeIn; + nBlockYSize = nBlockYSizeIn; +} + +/************************************************************************/ +/* IReadBlock() */ +/************************************************************************/ + +CPLErr VRTTileIndexBand::IReadBlock(int nBlockXOff, int nBlockYOff, + void *pImage) + +{ + const int nPixelSize = GDALGetDataTypeSizeBytes(eDataType); + + int nReadXSize = nBlockXSize; + int nReadYSize = nBlockYSize; + GetActualBlockSize(nBlockXOff, nBlockYOff, &nReadXSize, &nReadYSize); + + GDALRasterIOExtraArg sExtraArg; + INIT_RASTERIO_EXTRA_ARG(sExtraArg); + + return IRasterIO(GF_Read, nBlockXOff * nBlockXSize, + nBlockYOff * nBlockYSize, nReadXSize, nReadYSize, pImage, + nReadXSize, nReadYSize, eDataType, nPixelSize, + nPixelSize * nBlockXSize, &sExtraArg); +} + +/************************************************************************/ +/* IRasterIO() */ +/************************************************************************/ + +CPLErr VRTTileIndexBand::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, + int nXSize, int nYSize, void *pData, + int nBufXSize, int nBufYSize, + GDALDataType eBufType, GSpacing nPixelSpace, + GSpacing nLineSpace, + GDALRasterIOExtraArg *psExtraArg) +{ + int anBand[] = {nBand}; + + return m_poDS->IRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize, pData, + nBufXSize, nBufYSize, eBufType, 1, anBand, + nPixelSpace, nLineSpace, 0, psExtraArg); +} + +/************************************************************************/ +/* GetMetadataDomainList() */ +/************************************************************************/ + +char **VRTTileIndexBand::GetMetadataDomainList() +{ + return CSLAddString(GDALRasterBand::GetMetadataDomainList(), + "LocationInfo"); +} + +/************************************************************************/ +/* GetMetadataItem() */ +/************************************************************************/ + +const char *VRTTileIndexBand::GetMetadataItem(const char *pszName, + const char *pszDomain) + +{ + /* ==================================================================== */ + /* LocationInfo handling. */ + /* ==================================================================== */ + if (pszDomain != nullptr && EQUAL(pszDomain, "LocationInfo") && + (STARTS_WITH_CI(pszName, "Pixel_") || + STARTS_WITH_CI(pszName, "GeoPixel_"))) + { + // What pixel are we aiming at? + int iPixel = 0; + int iLine = 0; + + if (STARTS_WITH_CI(pszName, "Pixel_")) + { + pszName += strlen("Pixel_"); + iPixel = atoi(pszName); + const char *const pszUnderscore = strchr(pszName, '_'); + if (!pszUnderscore) + return nullptr; + iLine = atoi(pszUnderscore + 1); + } + else if (STARTS_WITH_CI(pszName, "GeoPixel_")) + { + pszName += strlen("GeoPixel_"); + const double dfGeoX = CPLAtof(pszName); + const char *const pszUnderscore = strchr(pszName, '_'); + if (!pszUnderscore) + return nullptr; + const double dfGeoY = CPLAtof(pszUnderscore + 1); + + double adfInvGeoTransform[6] = {0.0}; + if (!GDALInvGeoTransform(m_poDS->m_adfGeoTransform.data(), + adfInvGeoTransform)) + return nullptr; + + iPixel = static_cast(floor(adfInvGeoTransform[0] + + adfInvGeoTransform[1] * dfGeoX + + adfInvGeoTransform[2] * dfGeoY)); + iLine = static_cast(floor(adfInvGeoTransform[3] + + adfInvGeoTransform[4] * dfGeoX + + adfInvGeoTransform[5] * dfGeoY)); + } + else + { + return nullptr; + } + + if (iPixel < 0 || iLine < 0 || iPixel >= GetXSize() || + iLine >= GetYSize()) + return nullptr; + + if (!m_poDS->CollectSources(iPixel, iLine, 1, 1)) + return nullptr; + + // Format into XML. + m_osLastLocationInfo = ""; + + if (!m_poDS->m_aoSourceDesc.empty()) + { + const auto AddSource = + [&](const VRTTileIndexDataset::SourceDesc &oSourceDesc) + { + m_osLastLocationInfo += ""; + char *const pszXMLEscaped = + CPLEscapeString(oSourceDesc.osName.c_str(), -1, CPLES_XML); + m_osLastLocationInfo += pszXMLEscaped; + CPLFree(pszXMLEscaped); + m_osLastLocationInfo += ""; + }; + + const int anBand[] = {nBand}; + if (!m_poDS->NeedInitBuffer(1, anBand)) + { + AddSource(m_poDS->m_aoSourceDesc.back()); + } + else + { + for (const auto &oSourceDesc : m_poDS->m_aoSourceDesc) + { + if (oSourceDesc.poDS) + AddSource(oSourceDesc); + } + } + } + + m_osLastLocationInfo += ""; + + return m_osLastLocationInfo.c_str(); + } + + return GDALPamRasterBand::GetMetadataItem(pszName, pszDomain); +} + +/************************************************************************/ +/* SetMetadataItem() */ +/************************************************************************/ + +CPLErr VRTTileIndexBand::SetMetadataItem(const char *pszName, + const char *pszValue, + const char *pszDomain) +{ + if (nBand > 0 && m_poDS->m_bXMLUpdatable) + { + m_poDS->m_bXMLModified = true; + return GDALRasterBand::SetMetadataItem(pszName, pszValue, pszDomain); + } + else if (nBand > 0 && m_poDS->TileIndexSupportsEditingLayerMetadata()) + { + m_poDS->m_poLayer->SetMetadataItem( + CPLSPrintf("BAND_%d_%s", nBand, pszName), pszValue, pszDomain); + return GDALRasterBand::SetMetadataItem(pszName, pszValue, pszDomain); + } + else + { + return GDALPamRasterBand::SetMetadataItem(pszName, pszValue, pszDomain); + } +} + +/************************************************************************/ +/* SetMetadata() */ +/************************************************************************/ + +CPLErr VRTTileIndexBand::SetMetadata(char **papszMD, const char *pszDomain) +{ + if (nBand > 0 && m_poDS->m_bXMLUpdatable) + { + m_poDS->m_bXMLModified = true; + return GDALRasterBand::SetMetadata(papszMD, pszDomain); + } + else if (nBand > 0 && m_poDS->TileIndexSupportsEditingLayerMetadata()) + { + CPLStringList aosMD; + + if (!pszDomain || pszDomain[0] == 0) + { + // Reinject dataset metadata + char **papszLayerMD = m_poDS->m_poLayer->GetMetadata(pszDomain); + for (const char *const *papszIter = papszLayerMD; + papszIter && *papszIter; ++papszIter) + { + if (!STARTS_WITH(*papszIter, "BAND_") || + STARTS_WITH(*papszIter, MD_BAND_COUNT)) + aosMD.AddString(*papszIter); + } + } + + for (int i = 0; papszMD && papszMD[i]; ++i) + { + aosMD.AddString(CPLSPrintf("BAND_%d_%s", nBand, papszMD[i])); + } + + if (!pszDomain || pszDomain[0] == 0) + { + for (const char *pszItem : apszReservedBandItems) + { + const char *pszKey = CPLSPrintf("BAND_%d_%s", nBand, pszItem); + if (!aosMD.FetchNameValue(pszKey)) + { + if (const char *pszVal = + m_poDS->m_poLayer->GetMetadataItem(pszKey)) + { + aosMD.SetNameValue(pszKey, pszVal); + } + } + } + } + + m_poDS->m_poLayer->SetMetadata(aosMD.List(), pszDomain); + return GDALRasterBand::SetMetadata(papszMD, pszDomain); + } + else + { + return GDALPamRasterBand::SetMetadata(papszMD, pszDomain); + } +} + +/************************************************************************/ +/* GetSrcDstWin() */ +/************************************************************************/ + +static bool GetSrcDstWin(const double adfTileGT[6], int nTileXSize, + int nTileYSize, const double adfVRTGT[6], + int nVRTXSize, int nVRTYSize, double *pdfSrcXOff, + double *pdfSrcYOff, double *pdfSrcXSize, + double *pdfSrcYSize, double *pdfDstXOff, + double *pdfDstYOff, double *pdfDstXSize, + double *pdfDstYSize) +{ + const double minX = adfVRTGT[GT_TOPLEFT_X]; + const double we_res = adfVRTGT[GT_WE_RES]; + const double maxX = minX + nVRTXSize * we_res; + const double maxY = adfVRTGT[GT_TOPLEFT_Y]; + const double ns_res = adfVRTGT[GT_NS_RES]; + const double minY = maxY + nVRTYSize * ns_res; + + /* Check that the destination bounding box intersects the source bounding + * box */ + if (adfTileGT[GT_TOPLEFT_X] + nTileXSize * adfTileGT[GT_WE_RES] <= minX) + return false; + if (adfTileGT[GT_TOPLEFT_X] >= maxX) + return false; + if (adfTileGT[GT_TOPLEFT_Y] + nTileYSize * adfTileGT[GT_NS_RES] >= maxY) + return false; + if (adfTileGT[GT_TOPLEFT_Y] <= minY) + return false; + + if (adfTileGT[GT_TOPLEFT_X] < minX) + { + *pdfSrcXOff = (minX - adfTileGT[GT_TOPLEFT_X]) / adfTileGT[GT_WE_RES]; + *pdfDstXOff = 0.0; + } + else + { + *pdfSrcXOff = 0.0; + *pdfDstXOff = ((adfTileGT[GT_TOPLEFT_X] - minX) / we_res); + } + if (maxY < adfTileGT[GT_TOPLEFT_Y]) + { + *pdfSrcYOff = (adfTileGT[GT_TOPLEFT_Y] - maxY) / -adfTileGT[GT_NS_RES]; + *pdfDstYOff = 0.0; + } + else + { + *pdfSrcYOff = 0.0; + *pdfDstYOff = ((maxY - adfTileGT[GT_TOPLEFT_Y]) / -ns_res); + } + + *pdfSrcXSize = nTileXSize; + *pdfSrcYSize = nTileYSize; + if (*pdfSrcXOff > 0) + *pdfSrcXSize -= *pdfSrcXOff; + if (*pdfSrcYOff > 0) + *pdfSrcYSize -= *pdfSrcYOff; + + const double dfSrcToDstXSize = adfTileGT[GT_WE_RES] / we_res; + *pdfDstXSize = *pdfSrcXSize * dfSrcToDstXSize; + const double dfSrcToDstYSize = adfTileGT[GT_NS_RES] / ns_res; + *pdfDstYSize = *pdfSrcYSize * dfSrcToDstYSize; + + if (*pdfDstXOff + *pdfDstXSize > nVRTXSize) + { + *pdfDstXSize = nVRTXSize - *pdfDstXOff; + *pdfSrcXSize = *pdfDstXSize / dfSrcToDstXSize; + } + + if (*pdfDstYOff + *pdfDstYSize > nVRTYSize) + { + *pdfDstYSize = nVRTYSize - *pdfDstYOff; + *pdfSrcYSize = *pdfDstYSize / dfSrcToDstYSize; + } + + return *pdfSrcXSize > 0 && *pdfDstXSize > 0 && *pdfSrcYSize > 0 && + *pdfDstYSize > 0; +} + +/************************************************************************/ +/* GDALDatasetCastToVRTTIDataset() */ +/************************************************************************/ + +VRTTileIndexDataset *GDALDatasetCastToVRTTIDataset(GDALDataset *poDS) +{ + return dynamic_cast(poDS); +} + +/************************************************************************/ +/* VRTTIGetSourcesMoreRecentThan() */ +/************************************************************************/ + +std::vector +VRTTIGetSourcesMoreRecentThan(VRTTileIndexDataset *poDS, int64_t mTime) +{ + return poDS->GetSourcesMoreRecentThan(mTime); +} + +/************************************************************************/ +/* GetSourcesMoreRecentThan() */ +/************************************************************************/ + +std::vector +VRTTileIndexDataset::GetSourcesMoreRecentThan(int64_t mTime) +{ + std::vector oRes; + + m_poLayer->SetSpatialFilter(nullptr); + for (auto &&poFeature : m_poLayer) + { + if (!poFeature->IsFieldSetAndNotNull(m_nLocationFieldIndex)) + { + continue; + } + + auto poGeom = poFeature->GetGeometryRef(); + if (!poGeom || poGeom->IsEmpty()) + continue; + + OGREnvelope sEnvelope; + poGeom->getEnvelope(&sEnvelope); + + double dfXOff = (sEnvelope.MinX - m_adfGeoTransform[GT_TOPLEFT_X]) / + m_adfGeoTransform[GT_WE_RES]; + if (dfXOff >= nRasterXSize) + continue; + + double dfYOff = (sEnvelope.MaxY - m_adfGeoTransform[GT_TOPLEFT_Y]) / + m_adfGeoTransform[GT_NS_RES]; + if (dfYOff >= nRasterYSize) + continue; + + double dfXSize = + (sEnvelope.MaxX - sEnvelope.MinX) / m_adfGeoTransform[GT_WE_RES]; + if (dfXOff < 0) + { + dfXSize += dfXOff; + dfXOff = 0; + if (dfXSize <= 0) + continue; + } + + double dfYSize = (sEnvelope.MaxY - sEnvelope.MinY) / + std::fabs(m_adfGeoTransform[GT_NS_RES]); + if (dfYOff < 0) + { + dfYSize += dfYOff; + dfYOff = 0; + if (dfYSize <= 0) + continue; + } + + const char *pszTileName = + poFeature->GetFieldAsString(m_nLocationFieldIndex); + const std::string osTileName( + GetAbsoluteFileName(pszTileName, GetDescription())); + VSIStatBufL sStatSource; + if (VSIStatL(osTileName.c_str(), &sStatSource) != 0 || + sStatSource.st_mtime <= mTime) + { + continue; + } + + constexpr double EPS = 1e-8; + VRTTISourceDesc oSourceDesc; + oSourceDesc.osFilename = osTileName; + oSourceDesc.nDstXOff = static_cast(dfXOff + EPS); + oSourceDesc.nDstYOff = static_cast(dfYOff + EPS); + oSourceDesc.nDstXSize = static_cast(dfXSize + 0.5); + oSourceDesc.nDstYSize = static_cast(dfYSize + 0.5); + oRes.emplace_back(std::move(oSourceDesc)); + } + + return oRes; +} + +/************************************************************************/ +/* GetSourceDesc() */ +/************************************************************************/ + +bool VRTTileIndexDataset::GetSourceDesc(const std::string &osTileName, + SourceDesc &oSourceDesc) +{ + std::shared_ptr poTileDS; + if (!m_oMapSharedSources.tryGet(osTileName, poTileDS)) + { + struct Releaser + { + void operator()(GDALDataset *poDS) + { + if (poDS) + poDS->Release(); + } + }; + + poTileDS = std::shared_ptr( + GDALProxyPoolDataset::Create( + osTileName.c_str(), nullptr, GA_ReadOnly, + /* bShared = */ true, m_osUniqueHandle.c_str()), + Releaser()); + if (!poTileDS || poTileDS->GetRasterCount() == 0) + { + return false; + } + + // do palette -> RGB(A) expansion + if (poTileDS->GetRasterCount() == 1 && (nBands == 3 || nBands == 4) && + poTileDS->GetRasterBand(1)->GetColorTable() != nullptr) + { + + CPLStringList aosOptions; + aosOptions.AddString("-of"); + aosOptions.AddString("VRT"); + + aosOptions.AddString("-expand"); + aosOptions.AddString(nBands == 3 ? "rgb" : "rgba"); + + GDALTranslateOptions *psOptions = + GDALTranslateOptionsNew(aosOptions.List(), nullptr); + int bUsageError = false; + auto poRGBDS = std::unique_ptr(GDALDataset::FromHandle( + GDALTranslate("", GDALDataset::ToHandle(poTileDS.get()), + psOptions, &bUsageError))); + GDALTranslateOptionsFree(psOptions); + if (!poRGBDS) + { + return false; + } + + poTileDS.reset(poRGBDS.release()); + } + + const OGRSpatialReference *poTileSRS; + if (!m_oSRS.IsEmpty() && + (poTileSRS = poTileDS->GetSpatialRef()) != nullptr && + !m_oSRS.IsSame(poTileSRS)) + { + CPLDebug("VRT", + "Tile %s has not the same SRS as the VRT. " + "Proceed to on-the-fly warping", + osTileName.c_str()); + + CPLStringList aosOptions; + aosOptions.AddString("-of"); + aosOptions.AddString("VRT"); + + if ((poTileDS->GetRasterBand(1)->GetColorTable() == nullptr && + poTileDS->GetRasterBand(1)->GetCategoryNames() == nullptr) || + m_eResampling == GRIORA_Mode) + { + aosOptions.AddString("-r"); + aosOptions.AddString(m_osResampling.c_str()); + } + + if (m_osWKT.empty()) + { + char *pszWKT = nullptr; + const char *const apszWKTOptions[] = {"FORMAT=WKT2_2019", + nullptr}; + m_oSRS.exportToWkt(&pszWKT, apszWKTOptions); + if (pszWKT) + m_osWKT = pszWKT; + CPLFree(pszWKT); + } + if (m_osWKT.empty()) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Cannot export VRT SRS to WKT2"); + return false; + } + + aosOptions.AddString("-t_srs"); + aosOptions.AddString(m_osWKT.c_str()); + + // First pass to get the extent of the tile in the + // target VRT SRS + GDALWarpAppOptions *psWarpOptions = + GDALWarpAppOptionsNew(aosOptions.List(), nullptr); + GDALDatasetH ahSrcDS[] = {GDALDataset::ToHandle(poTileDS.get())}; + int bUsageError = false; + auto poWarpDS = + std::unique_ptr(GDALDataset::FromHandle(GDALWarp( + "", nullptr, 1, ahSrcDS, psWarpOptions, &bUsageError))); + GDALWarpAppOptionsFree(psWarpOptions); + if (!poWarpDS) + { + return false; + } + + // Second pass to create a warped source VRT whose + // extent is aligned on the one of the target VRT + double adfWarpDSGeoTransform[6]; + const auto eErr = poWarpDS->GetGeoTransform(adfWarpDSGeoTransform); + CPL_IGNORE_RET_VAL(eErr); + CPLAssert(eErr == CE_None); + const double dfVRTMinX = m_adfGeoTransform[GT_TOPLEFT_X]; + const double dfVRTResX = m_adfGeoTransform[GT_WE_RES]; + const double dfVRTMaxY = m_adfGeoTransform[GT_TOPLEFT_Y]; + const double dfVRTResYAbs = -m_adfGeoTransform[GT_NS_RES]; + const double dfWarpMinX = + std::floor((adfWarpDSGeoTransform[GT_TOPLEFT_X] - dfVRTMinX) / + dfVRTResX) * + dfVRTResX + + dfVRTMinX; + const double dfWarpMaxX = + std::ceil((adfWarpDSGeoTransform[GT_TOPLEFT_X] + + adfWarpDSGeoTransform[GT_WE_RES] * + poWarpDS->GetRasterXSize() - + dfVRTMinX) / + dfVRTResX) * + dfVRTResX + + dfVRTMinX; + const double dfWarpMaxY = + dfVRTMaxY - + std::floor((dfVRTMaxY - adfWarpDSGeoTransform[GT_TOPLEFT_Y]) / + dfVRTResYAbs) * + dfVRTResYAbs; + const double dfWarpMinY = + dfVRTMaxY - + std::ceil((dfVRTMaxY - (adfWarpDSGeoTransform[GT_TOPLEFT_Y] + + adfWarpDSGeoTransform[GT_NS_RES] * + poWarpDS->GetRasterYSize())) / + dfVRTResYAbs) * + dfVRTResYAbs; + + aosOptions.AddString("-te"); + aosOptions.AddString(CPLSPrintf("%.18g", dfWarpMinX)); + aosOptions.AddString(CPLSPrintf("%.18g", dfWarpMinY)); + aosOptions.AddString(CPLSPrintf("%.18g", dfWarpMaxX)); + aosOptions.AddString(CPLSPrintf("%.18g", dfWarpMaxY)); + + aosOptions.AddString("-tr"); + aosOptions.AddString(CPLSPrintf("%.18g", dfVRTResX)); + aosOptions.AddString(CPLSPrintf("%.18g", dfVRTResYAbs)); + + aosOptions.AddString("-dstalpha"); + + psWarpOptions = GDALWarpAppOptionsNew(aosOptions.List(), nullptr); + poWarpDS.reset(GDALDataset::FromHandle(GDALWarp( + "", nullptr, 1, ahSrcDS, psWarpOptions, &bUsageError))); + GDALWarpAppOptionsFree(psWarpOptions); + if (!poWarpDS) + { + return false; + } + + poTileDS.reset(poWarpDS.release()); + } + + m_oMapSharedSources.insert(osTileName, poTileDS); + } + + double adfGeoTransformTile[6]; + if (poTileDS->GetGeoTransform(adfGeoTransformTile) != CE_None) + { + CPLError(CE_Failure, CPLE_AppDefined, "%s lacks geotransform", + osTileName.c_str()); + return false; + } + + bool bHasNoData = false; + bool bSameNoData = true; + double dfNoDataValue = 0; + GDALRasterBand *poMaskBand = nullptr; + const int nBandCount = poTileDS->GetRasterCount(); + for (int iBand = 0; iBand < nBandCount; ++iBand) + { + auto poTileBand = poTileDS->GetRasterBand(iBand + 1); + int bThisBandHasNoData = false; + const double dfThisBandNoDataValue = + poTileBand->GetNoDataValue(&bThisBandHasNoData); + if (bThisBandHasNoData) + { + bHasNoData = true; + dfNoDataValue = dfThisBandNoDataValue; + } + if (iBand > 0 && + (static_cast(bThisBandHasNoData) != + static_cast(bHasNoData) || + (bHasNoData && + !IsSameNaNAware(dfNoDataValue, dfThisBandNoDataValue)))) + { + bSameNoData = false; + } + + if (poTileBand->GetMaskFlags() == GMF_PER_DATASET) + poMaskBand = poTileBand->GetMaskBand(); + else if (poTileBand->GetColorInterpretation() == GCI_AlphaBand) + poMaskBand = poTileBand; + } + + std::unique_ptr poSource; + if (!bHasNoData) + { + poSource = std::make_unique(); + } + else + { + auto poComplexSource = std::make_unique(); + poComplexSource->SetNoDataValue(dfNoDataValue); + poSource = std::move(poComplexSource); + } + + if (!GetSrcDstWin(adfGeoTransformTile, poTileDS->GetRasterXSize(), + poTileDS->GetRasterYSize(), m_adfGeoTransform.data(), + GetRasterXSize(), GetRasterYSize(), + &poSource->m_dfSrcXOff, &poSource->m_dfSrcYOff, + &poSource->m_dfSrcXSize, &poSource->m_dfSrcYSize, + &poSource->m_dfDstXOff, &poSource->m_dfDstYOff, + &poSource->m_dfDstXSize, &poSource->m_dfDstYSize)) + { + // Should not happen on a consistent tile index + CPLDebug("VRT", "Tile %s does not actually intersect area of interest", + osTileName.c_str()); + return false; + } + + oSourceDesc.osName = osTileName; + oSourceDesc.poDS = std::move(poTileDS); + oSourceDesc.poSource = std::move(poSource); + oSourceDesc.bHasNoData = bHasNoData; + oSourceDesc.bSameNoData = bSameNoData; + if (bSameNoData) + oSourceDesc.dfSameNoData = dfNoDataValue; + oSourceDesc.poMaskBand = poMaskBand; + return true; +} + +/************************************************************************/ +/* CollectSources() */ +/************************************************************************/ + +bool VRTTileIndexDataset::CollectSources(double dfXOff, double dfYOff, + double dfXSize, double dfYSize) +{ + const double dfMinX = + m_adfGeoTransform[GT_TOPLEFT_X] + dfXOff * m_adfGeoTransform[GT_WE_RES]; + const double dfMaxX = dfMinX + dfXSize * m_adfGeoTransform[GT_WE_RES]; + const double dfMaxY = + m_adfGeoTransform[GT_TOPLEFT_Y] + dfYOff * m_adfGeoTransform[GT_NS_RES]; + const double dfMinY = dfMaxY + dfYSize * m_adfGeoTransform[GT_NS_RES]; + + if (dfMinX == m_dfLastMinXFilter && dfMinY == m_dfLastMinYFilter && + dfMaxX == m_dfLastMaxXFilter && dfMaxY == m_dfLastMaxYFilter) + { + return true; + } + + m_dfLastMinXFilter = dfMinX; + m_dfLastMinYFilter = dfMinY; + m_dfLastMaxXFilter = dfMaxX; + m_dfLastMaxYFilter = dfMaxY; + + m_poLayer->SetSpatialFilterRect(dfMinX, dfMinY, dfMaxX, dfMaxY); + m_poLayer->ResetReading(); + + m_aoSourceDesc.clear(); + while (true) + { + auto poFeature = + std::unique_ptr(m_poLayer->GetNextFeature()); + if (!poFeature) + break; + if (!poFeature->IsFieldSetAndNotNull(m_nLocationFieldIndex)) + { + continue; + } + + SourceDesc oSourceDesc; + oSourceDesc.poFeature = std::move(poFeature); + m_aoSourceDesc.emplace_back(std::move(oSourceDesc)); + + if (m_aoSourceDesc.size() > 10 * 1000 * 1000) + { + // Safety belt... + CPLError(CE_Failure, CPLE_AppDefined, + "More than 10 million contributing sources to a " + "single RasterIO() request is not supported"); + return false; + } + } + + if (m_aoSourceDesc.size() > 1) + { + SortSourceDesc(); + } + + // Try to find the last (most prioritary) fully opaque source covering + // the whole AOI. We only need to start rendering from it. + size_t i = m_aoSourceDesc.size(); + while (i > 0) + { + --i; + auto &poFeature = m_aoSourceDesc[i].poFeature; + const char *pszTileName = + poFeature->GetFieldAsString(m_nLocationFieldIndex); + const std::string osTileName( + GetAbsoluteFileName(pszTileName, GetDescription())); + + SourceDesc oSourceDesc; + if (!GetSourceDesc(osTileName, oSourceDesc)) + continue; + + const auto &poSource = oSourceDesc.poSource; + if (dfXOff >= poSource->m_dfDstXOff + poSource->m_dfDstXSize || + dfYOff >= poSource->m_dfDstYOff + poSource->m_dfDstYSize || + poSource->m_dfDstXOff >= dfXOff + dfXSize || + poSource->m_dfDstYOff >= dfYOff + dfYSize) + { + // Can happen as some spatial filters select slightly more features + // than strictly needed. + continue; + } + + const bool bCoversWholeAOI = + (poSource->m_dfDstXOff <= dfXOff && + poSource->m_dfDstYOff <= dfYOff && + poSource->m_dfDstXOff + poSource->m_dfDstXSize >= + dfXOff + dfXSize && + poSource->m_dfDstYOff + poSource->m_dfDstYSize >= + dfYOff + dfYSize); + oSourceDesc.bCoversWholeAOI = bCoversWholeAOI; + + m_aoSourceDesc[i] = std::move(oSourceDesc); + + if (m_aoSourceDesc[i].bCoversWholeAOI && + !m_aoSourceDesc[i].bHasNoData && !m_aoSourceDesc[i].poMaskBand) + { + break; + } + } + + if (i > 0) + { + // Remove sources that will not be rendered + m_aoSourceDesc.erase(m_aoSourceDesc.begin(), + m_aoSourceDesc.begin() + i); + } + + // Truncate the array when its last elements have no dataset + i = m_aoSourceDesc.size(); + while (i > 0) + { + --i; + if (!m_aoSourceDesc[i].poDS) + { + m_aoSourceDesc.resize(i); + break; + } + } + + return true; +} + +/************************************************************************/ +/* SortSourceDesc() */ +/************************************************************************/ + +void VRTTileIndexDataset::SortSourceDesc() +{ + const auto eFieldType = m_nSortFieldIndex >= 0 + ? m_poLayer->GetLayerDefn() + ->GetFieldDefn(m_nSortFieldIndex) + ->GetType() + : OFTMaxType; + std::sort( + m_aoSourceDesc.begin(), m_aoSourceDesc.end(), + [this, eFieldType](const SourceDesc &a, const SourceDesc &b) + { + const auto &poFeatureA = (m_bSortFieldAsc ? a : b).poFeature; + const auto &poFeatureB = (m_bSortFieldAsc ? b : a).poFeature; + if (m_nSortFieldIndex >= 0 && + poFeatureA->IsFieldSetAndNotNull(m_nSortFieldIndex) && + poFeatureB->IsFieldSetAndNotNull(m_nSortFieldIndex)) + { + if (eFieldType == OFTString) + { + const int nCmp = + strcmp(poFeatureA->GetFieldAsString(m_nSortFieldIndex), + poFeatureB->GetFieldAsString(m_nSortFieldIndex)); + if (nCmp < 0) + return true; + if (nCmp > 0) + return false; + } + else if (eFieldType == OFTInteger || eFieldType == OFTInteger64) + { + const auto nA = + poFeatureA->GetFieldAsInteger64(m_nSortFieldIndex); + const auto nB = + poFeatureB->GetFieldAsInteger64(m_nSortFieldIndex); + if (nA < nB) + return true; + if (nA > nB) + return false; + } + else if (eFieldType == OFTReal) + { + const auto dfA = + poFeatureA->GetFieldAsDouble(m_nSortFieldIndex); + const auto dfB = + poFeatureB->GetFieldAsDouble(m_nSortFieldIndex); + if (dfA < dfB) + return true; + if (dfA > dfB) + return false; + } + else if (eFieldType == OFTDate || eFieldType == OFTDateTime) + { + const auto poFieldA = + poFeatureA->GetRawFieldRef(m_nSortFieldIndex); + const auto poFieldB = + poFeatureB->GetRawFieldRef(m_nSortFieldIndex); + +#define COMPARE_DATE_COMPONENT(comp) \ + do \ + { \ + if (poFieldA->Date.comp < poFieldB->Date.comp) \ + return true; \ + if (poFieldA->Date.comp > poFieldB->Date.comp) \ + return false; \ + } while (0) + + COMPARE_DATE_COMPONENT(Year); + COMPARE_DATE_COMPONENT(Month); + COMPARE_DATE_COMPONENT(Day); + COMPARE_DATE_COMPONENT(Hour); + COMPARE_DATE_COMPONENT(Minute); + COMPARE_DATE_COMPONENT(Second); + } + else + { + CPLAssert(false); + } + } + return poFeatureA->GetFID() < poFeatureB->GetFID(); + }); +} + +/************************************************************************/ +/* CompositeSrcWithMaskIntoDest() */ +/************************************************************************/ + +static void +CompositeSrcWithMaskIntoDest(const int nOutXSize, const int nOutYSize, + const GDALDataType eBufType, + const int nBufTypeSize, const GSpacing nPixelSpace, + const GSpacing nLineSpace, const GByte *pabySrc, + const GByte *const pabyMask, GByte *const pabyDest) +{ + size_t iMaskIdx = 0; + if (eBufType == GDT_Byte) + { + // Optimization for byte case + for (int iY = 0; iY < nOutYSize; iY++) + { + GByte *pabyDestLine = + pabyDest + static_cast(iY * nLineSpace); + int iX = 0; +#ifdef USE_SSE2_OPTIM + if (nPixelSpace == 1) + { + // SSE2 version up to 6 times faster than portable version + const auto xmm_zero = _mm_setzero_si128(); + constexpr int SIZEOF_REG = static_cast(sizeof(xmm_zero)); + for (; iX + SIZEOF_REG <= nOutXSize; iX += SIZEOF_REG) + { + auto xmm_mask = _mm_loadu_si128( + reinterpret_cast<__m128i const *>(pabyMask + iMaskIdx)); + const auto xmm_src = _mm_loadu_si128( + reinterpret_cast<__m128i const *>(pabySrc)); + auto xmm_dst = _mm_loadu_si128( + reinterpret_cast<__m128i const *>(pabyDestLine)); +#ifdef USE_SSE41_OPTIM + xmm_dst = _mm_blendv_epi8(xmm_dst, xmm_src, xmm_mask); +#else + // mask[i] = 0 becomes 255, and mask[i] != 0 becomes 0 + xmm_mask = _mm_cmpeq_epi8(xmm_mask, xmm_zero); + // dst_data[i] = (mask[i] & dst_data[i]) | + // (~mask[i] & src_data[i]) + // That is: + // dst_data[i] = dst_data[i] when mask[i] = 255 + // dst_data[i] = src_data[i] when mask[i] = 0 + xmm_dst = _mm_or_si128(_mm_and_si128(xmm_mask, xmm_dst), + _mm_andnot_si128(xmm_mask, xmm_src)); +#endif + _mm_storeu_si128(reinterpret_cast<__m128i *>(pabyDestLine), + xmm_dst); + pabyDestLine += SIZEOF_REG; + pabySrc += SIZEOF_REG; + iMaskIdx += SIZEOF_REG; + } + } +#endif + for (; iX < nOutXSize; iX++) + { + if (pabyMask[iMaskIdx]) + { + *pabyDestLine = *pabySrc; + } + pabyDestLine += static_cast(nPixelSpace); + pabySrc++; + iMaskIdx++; + } + } + } + else + { + for (int iY = 0; iY < nOutYSize; iY++) + { + GByte *pabyDestLine = + pabyDest + static_cast(iY * nLineSpace); + for (int iX = 0; iX < nOutXSize; iX++) + { + if (pabyMask[iMaskIdx]) + { + memcpy(pabyDestLine, pabySrc, nBufTypeSize); + } + pabyDestLine += static_cast(nPixelSpace); + pabySrc += nBufTypeSize; + iMaskIdx++; + } + } + } +} + +/************************************************************************/ +/* NeedInitBuffer() */ +/************************************************************************/ + +// Must be called after CollectSources() +bool VRTTileIndexDataset::NeedInitBuffer(int nBandCount, + const int *panBandMap) const +{ + bool bNeedInitBuffer = true; + // If the last source (that is the most prioritary one) covers at least + // the window of interest and is fully opaque, then we don't need to + // initialize the buffer, and can directly render that source. + int bHasNoData = false; + if (!m_aoSourceDesc.empty() && m_aoSourceDesc.back().bCoversWholeAOI && + (!m_aoSourceDesc.back().bHasNoData || + // Also, if there's a single source and that the VRT bands and the + // source bands have the same nodata value, we can skip initialization. + (m_aoSourceDesc.size() == 1 && m_aoSourceDesc.back().bSameNoData && + m_bSameNoData && m_bSameDataType && + IsSameNaNAware(papoBands[0]->GetNoDataValue(&bHasNoData), + m_aoSourceDesc.back().dfSameNoData) && + bHasNoData)) && + (!m_aoSourceDesc.back().poMaskBand || + // Also, if there's a single source that has a mask band, and the VRT + // bands have no-nodata or a 0-nodata value, we can skip + // initialization. + (m_aoSourceDesc.size() == 1 && m_bSameDataType && + !(nBandCount == 1 && panBandMap[0] == 0) && m_bSameNoData && + papoBands[0]->GetNoDataValue(&bHasNoData) == 0))) + { + bNeedInitBuffer = false; + } + return bNeedInitBuffer; +} + +/************************************************************************/ +/* InitBuffer() */ +/************************************************************************/ + +void VRTTileIndexDataset::InitBuffer(void *pData, int nBufXSize, int nBufYSize, + GDALDataType eBufType, int nBandCount, + const int *panBandMap, + GSpacing nPixelSpace, GSpacing nLineSpace, + GSpacing nBandSpace) const +{ + const int nBufTypeSize = GDALGetDataTypeSizeBytes(eBufType); + if (m_bSameNoData && nBandCount > 1 && + ((nPixelSpace == nBufTypeSize && + nLineSpace == nBufXSize * nPixelSpace && + nBandSpace == nBufYSize * nLineSpace) || + (nBandSpace == nBufTypeSize && + nPixelSpace == nBandCount * nBandSpace && + nLineSpace == nBufXSize * nPixelSpace))) + { + const int nBandNr = panBandMap[0]; + auto poVRTBand = + nBandNr == 0 + ? m_poMaskBand.get() + : cpl::down_cast(papoBands[nBandNr - 1]); + const double dfNoData = poVRTBand->m_dfNoDataValue; + if (dfNoData == 0.0) + { + memset(pData, 0, + static_cast(nBufXSize) * nBufYSize * nBandCount * + nBufTypeSize); + } + else + { + GDALCopyWords64( + &dfNoData, GDT_Float64, 0, pData, eBufType, nBufTypeSize, + static_cast(nBufXSize) * nBufYSize * nBandCount); + } + } + else + { + for (int i = 0; i < nBandCount; ++i) + { + const int nBandNr = panBandMap[i]; + auto poVRTBand = nBandNr == 0 ? m_poMaskBand.get() + : cpl::down_cast( + papoBands[nBandNr - 1]); + GByte *pabyBandData = static_cast(pData) + i * nBandSpace; + if (nPixelSpace == nBufTypeSize && + poVRTBand->m_dfNoDataValue == 0.0) + { + if (nLineSpace == nBufXSize * nPixelSpace) + { + memset(pabyBandData, 0, + static_cast(nBufYSize * nLineSpace)); + } + else + { + for (int iLine = 0; iLine < nBufYSize; iLine++) + { + memset(static_cast(pabyBandData) + + static_cast(iLine) * nLineSpace, + 0, static_cast(nBufXSize * nPixelSpace)); + } + } + } + else + { + double dfWriteValue = poVRTBand->m_dfNoDataValue; + + for (int iLine = 0; iLine < nBufYSize; iLine++) + { + GDALCopyWords(&dfWriteValue, GDT_Float64, 0, + static_cast(pabyBandData) + + static_cast(nLineSpace) * iLine, + eBufType, static_cast(nPixelSpace), + nBufXSize); + } + } + } + } +} + +/************************************************************************/ +/* IRasterIO() */ +/************************************************************************/ + +CPLErr VRTTileIndexDataset::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, + int nXSize, int nYSize, void *pData, + int nBufXSize, int nBufYSize, + GDALDataType eBufType, int nBandCount, + int *panBandMap, GSpacing nPixelSpace, + GSpacing nLineSpace, GSpacing nBandSpace, + GDALRasterIOExtraArg *psExtraArg) +{ + if (eRWFlag != GF_Read) + return CE_Failure; + + if (nBufXSize < nXSize && nBufYSize < nYSize && AreOverviewsEnabled()) + { + int bTried = FALSE; + const CPLErr eErr = TryOverviewRasterIO( + eRWFlag, nXOff, nYOff, nXSize, nYSize, pData, nBufXSize, nBufYSize, + eBufType, nBandCount, panBandMap, nPixelSpace, nLineSpace, + nBandSpace, psExtraArg, &bTried); + if (bTried) + return eErr; + } + + double dfXOff = nXOff; + double dfYOff = nYOff; + double dfXSize = nXSize; + double dfYSize = nYSize; + if (psExtraArg->bFloatingPointWindowValidity) + { + dfXOff = psExtraArg->dfXOff; + dfYOff = psExtraArg->dfYOff; + dfXSize = psExtraArg->dfXSize; + dfYSize = psExtraArg->dfYSize; + } + + if (!CollectSources(dfXOff, dfYOff, dfXSize, dfYSize)) + { + return CE_Failure; + } + + // We might be called with nBandCount == 1 && panBandMap[0] == 0 + // to mean m_poMaskBand + int nBandNrMax = 0; + for (int i = 0; i < nBandCount; ++i) + { + const int nBandNr = panBandMap[i]; + nBandNrMax = std::max(nBandNrMax, nBandNr); + } + + const bool bNeedInitBuffer = NeedInitBuffer(nBandCount, panBandMap); + + const auto RenderSource = [=](SourceDesc &oSourceDesc) + { + auto &poTileDS = oSourceDesc.poDS; + auto &poSource = oSourceDesc.poSource; + auto poComplexSource = dynamic_cast(poSource.get()); + CPLErr eErr = CE_None; + + if (poTileDS->GetRasterCount() + 1 == nBandNrMax && + GetRasterBand(nBandNrMax)->GetColorInterpretation() == + GCI_AlphaBand && + GetRasterBand(nBandNrMax)->GetRasterDataType() == GDT_Byte) + { + // Special case when there's typically a mix of RGB and RGBA source + // datasets and we read a RGB one. + for (int iBand = 0; iBand < nBandCount && eErr == CE_None; ++iBand) + { + const int nBandNr = panBandMap[iBand]; + if (nBandNr == nBandNrMax) + { + // The window we will actually request from the source raster band. + double dfReqXOff = 0.0; + double dfReqYOff = 0.0; + double dfReqXSize = 0.0; + double dfReqYSize = 0.0; + int nReqXOff = 0; + int nReqYOff = 0; + int nReqXSize = 0; + int nReqYSize = 0; + + // The window we will actual set _within_ the pData buffer. + int nOutXOff = 0; + int nOutYOff = 0; + int nOutXSize = 0; + int nOutYSize = 0; + + bool bError = false; + + auto poTileBand = poTileDS->GetRasterBand(1); + poSource->SetRasterBand(poTileBand, false); + if (poSource->GetSrcDstWindow( + dfXOff, dfYOff, dfXSize, dfYSize, nBufXSize, + nBufYSize, &dfReqXOff, &dfReqYOff, &dfReqXSize, + &dfReqYSize, &nReqXOff, &nReqYOff, &nReqXSize, + &nReqYSize, &nOutXOff, &nOutYOff, &nOutXSize, + &nOutYSize, bError)) + { + GByte *pabyOut = + static_cast(pData) + + static_cast(iBand * nBandSpace + + nOutXOff * nPixelSpace + + nOutYOff * nLineSpace); + + GByte n255 = 255; + for (int iY = 0; iY < nOutYSize; iY++) + { + GDALCopyWords(&n255, GDT_Byte, 0, + pabyOut + static_cast( + iY * nLineSpace), + eBufType, + static_cast(nPixelSpace), + nOutXSize); + } + } + } + else + { + auto poTileBand = poTileDS->GetRasterBand(nBandNr); + if (poComplexSource) + { + int bHasNoData = false; + const double dfNoDataValue = + poTileBand->GetNoDataValue(&bHasNoData); + poComplexSource->SetNoDataValue( + bHasNoData ? dfNoDataValue : VRT_NODATA_UNSET); + } + poSource->SetRasterBand(poTileBand, false); + + GDALRasterIOExtraArg sExtraArg; + INIT_RASTERIO_EXTRA_ARG(sExtraArg); + if (psExtraArg->eResampleAlg != GRIORA_NearestNeighbour) + sExtraArg.eResampleAlg = psExtraArg->eResampleAlg; + else + sExtraArg.eResampleAlg = m_eResampling; + + GByte *pabyBandData = + static_cast(pData) + iBand * nBandSpace; + eErr = poSource->RasterIO( + poTileBand->GetRasterDataType(), nXOff, nYOff, nXSize, + nYSize, pabyBandData, nBufXSize, nBufYSize, eBufType, + nPixelSpace, nLineSpace, &sExtraArg, m_oWorkingState); + } + } + return eErr; + } + else if (poTileDS->GetRasterCount() < nBandNrMax) + { + CPLError(CE_Failure, CPLE_AppDefined, "%s has not enough bands.", + oSourceDesc.osName.c_str()); + return CE_Failure; + } + + if ((oSourceDesc.poMaskBand && bNeedInitBuffer) || nBandNrMax == 0) + { + // The window we will actually request from the source raster band. + double dfReqXOff = 0.0; + double dfReqYOff = 0.0; + double dfReqXSize = 0.0; + double dfReqYSize = 0.0; + int nReqXOff = 0; + int nReqYOff = 0; + int nReqXSize = 0; + int nReqYSize = 0; + + // The window we will actual set _within_ the pData buffer. + int nOutXOff = 0; + int nOutYOff = 0; + int nOutXSize = 0; + int nOutYSize = 0; + + bool bError = false; + + auto poFirstTileBand = poTileDS->GetRasterBand(1); + poSource->SetRasterBand(poFirstTileBand, false); + if (poSource->GetSrcDstWindow( + dfXOff, dfYOff, dfXSize, dfYSize, nBufXSize, nBufYSize, + &dfReqXOff, &dfReqYOff, &dfReqXSize, &dfReqYSize, &nReqXOff, + &nReqYOff, &nReqXSize, &nReqYSize, &nOutXOff, &nOutYOff, + &nOutXSize, &nOutYSize, bError)) + { + int iMaskBandIdx = -1; + if (eBufType == GDT_Byte && nBandNrMax == 0) + { + // when called from m_poMaskBand + iMaskBandIdx = 0; + } + else if (oSourceDesc.poMaskBand) + { + // If we request a Byte buffer and the mask band is actually + // one of the queried bands of this request, we can save + // requesting it separately. + const int nMaskBandNr = oSourceDesc.poMaskBand->GetBand(); + if (eBufType == GDT_Byte && nMaskBandNr >= 1 && + nMaskBandNr <= poTileDS->GetRasterCount() && + poTileDS->GetRasterBand(nMaskBandNr) == + oSourceDesc.poMaskBand) + { + for (int iBand = 0; iBand < nBandCount; ++iBand) + { + if (panBandMap[iBand] == nMaskBandNr) + { + iMaskBandIdx = iBand; + break; + } + } + } + } + + GDALRasterIOExtraArg sExtraArg; + INIT_RASTERIO_EXTRA_ARG(sExtraArg); + if (psExtraArg->eResampleAlg != GRIORA_NearestNeighbour) + sExtraArg.eResampleAlg = psExtraArg->eResampleAlg; + else + sExtraArg.eResampleAlg = m_eResampling; + sExtraArg.bFloatingPointWindowValidity = TRUE; + sExtraArg.dfXOff = dfReqXOff; + sExtraArg.dfYOff = dfReqYOff; + sExtraArg.dfXSize = dfReqXSize; + sExtraArg.dfYSize = dfReqYSize; + + if (iMaskBandIdx < 0 && oSourceDesc.abyMask.empty() && + oSourceDesc.poMaskBand) + { + // Fetch the mask band + try + { + oSourceDesc.abyMask.resize( + static_cast(nOutXSize) * nOutYSize); + } + catch (const std::bad_alloc &) + { + CPLError(CE_Failure, CPLE_OutOfMemory, + "Cannot allocate working buffer for mask"); + return CE_Failure; + } + + if (oSourceDesc.poMaskBand->RasterIO( + GF_Read, nReqXOff, nReqYOff, nReqXSize, nReqYSize, + oSourceDesc.abyMask.data(), nOutXSize, nOutYSize, + GDT_Byte, 0, 0, &sExtraArg) != CE_None) + { + oSourceDesc.abyMask.clear(); + return CE_Failure; + } + } + + // Allocate a temporary contiguous buffer to receive pixel data + const int nBufTypeSize = GDALGetDataTypeSizeBytes(eBufType); + const size_t nWorkBufferBandSize = + static_cast(nOutXSize) * nOutYSize * nBufTypeSize; + std::vector abyWorkBuffer; + try + { + abyWorkBuffer.resize(nBandCount * nWorkBufferBandSize); + } + catch (const std::bad_alloc &) + { + CPLError(CE_Failure, CPLE_OutOfMemory, + "Cannot allocate working buffer"); + return CE_Failure; + } + + const GByte *const pabyMask = + iMaskBandIdx >= 0 ? abyWorkBuffer.data() + + iMaskBandIdx * nWorkBufferBandSize + : oSourceDesc.abyMask.data(); + + if (nBandNrMax == 0) + { + // Special case when called from m_poMaskBand + if (poTileDS->GetRasterBand(1)->GetMaskBand()->RasterIO( + GF_Read, nReqXOff, nReqYOff, nReqXSize, nReqYSize, + abyWorkBuffer.data(), nOutXSize, nOutYSize, + eBufType, 0, 0, &sExtraArg) != CE_None) + { + return CE_Failure; + } + } + else if (poTileDS->RasterIO( + GF_Read, nReqXOff, nReqYOff, nReqXSize, nReqYSize, + abyWorkBuffer.data(), nOutXSize, nOutYSize, + eBufType, nBandCount, panBandMap, 0, 0, 0, + &sExtraArg) != CE_None) + { + return CE_Failure; + } + + // Compose the temporary contiguous buffer into the target + // buffer, taking into account the mask + GByte *pabyOut = + static_cast(pData) + + static_cast(nOutXOff * nPixelSpace + + nOutYOff * nLineSpace); + + for (int iBand = 0; iBand < nBandCount && eErr == CE_None; + ++iBand) + { + GByte *pabyDestBand = + pabyOut + static_cast(iBand * nBandSpace); + const GByte *pabySrc = + abyWorkBuffer.data() + iBand * nWorkBufferBandSize; + + CompositeSrcWithMaskIntoDest(nOutXSize, nOutYSize, eBufType, + nBufTypeSize, nPixelSpace, + nLineSpace, pabySrc, pabyMask, + pabyDestBand); + } + } + } + else if (m_bSameDataType && !bNeedInitBuffer && oSourceDesc.bHasNoData) + { + // We create a non-VRTComplexSource SimpleSource copy of poSource + // to be able to call DatasetRasterIO() + VRTSimpleSource oSimpleSource(poSource.get(), 1.0, 1.0); + + GDALRasterIOExtraArg sExtraArg; + INIT_RASTERIO_EXTRA_ARG(sExtraArg); + if (psExtraArg->eResampleAlg != GRIORA_NearestNeighbour) + sExtraArg.eResampleAlg = psExtraArg->eResampleAlg; + else + sExtraArg.eResampleAlg = m_eResampling; + + auto poTileBand = poTileDS->GetRasterBand(panBandMap[0]); + oSimpleSource.SetRasterBand(poTileBand, false); + eErr = oSimpleSource.DatasetRasterIO( + papoBands[0]->GetRasterDataType(), nXOff, nYOff, nXSize, nYSize, + pData, nBufXSize, nBufYSize, eBufType, nBandCount, panBandMap, + nPixelSpace, nLineSpace, nBandSpace, &sExtraArg); + } + else if (m_bSameDataType && !poComplexSource) + { + auto poTileBand = poTileDS->GetRasterBand(panBandMap[0]); + poSource->SetRasterBand(poTileBand, false); + + GDALRasterIOExtraArg sExtraArg; + INIT_RASTERIO_EXTRA_ARG(sExtraArg); + if (poTileBand->GetColorTable()) + sExtraArg.eResampleAlg = GRIORA_NearestNeighbour; + else if (psExtraArg->eResampleAlg != GRIORA_NearestNeighbour) + sExtraArg.eResampleAlg = psExtraArg->eResampleAlg; + else + sExtraArg.eResampleAlg = m_eResampling; + + eErr = poSource->DatasetRasterIO( + papoBands[0]->GetRasterDataType(), nXOff, nYOff, nXSize, nYSize, + pData, nBufXSize, nBufYSize, eBufType, nBandCount, panBandMap, + nPixelSpace, nLineSpace, nBandSpace, &sExtraArg); + } + else + { + for (int i = 0; i < nBandCount && eErr == CE_None; ++i) + { + const int nBandNr = panBandMap[i]; + GByte *pabyBandData = + static_cast(pData) + i * nBandSpace; + auto poTileBand = poTileDS->GetRasterBand(nBandNr); + if (poComplexSource) + { + int bHasNoData = false; + const double dfNoDataValue = + poTileBand->GetNoDataValue(&bHasNoData); + poComplexSource->SetNoDataValue( + bHasNoData ? dfNoDataValue : VRT_NODATA_UNSET); + } + poSource->SetRasterBand(poTileBand, false); + + GDALRasterIOExtraArg sExtraArg; + INIT_RASTERIO_EXTRA_ARG(sExtraArg); + if (poTileBand->GetColorTable()) + sExtraArg.eResampleAlg = GRIORA_NearestNeighbour; + else if (psExtraArg->eResampleAlg != GRIORA_NearestNeighbour) + sExtraArg.eResampleAlg = psExtraArg->eResampleAlg; + else + sExtraArg.eResampleAlg = m_eResampling; + + eErr = poSource->RasterIO( + papoBands[nBandNr - 1]->GetRasterDataType(), nXOff, nYOff, + nXSize, nYSize, pabyBandData, nBufXSize, nBufYSize, + eBufType, nPixelSpace, nLineSpace, &sExtraArg, + m_oWorkingState); + } + } + return eErr; + }; + + if (!bNeedInitBuffer) + { + return RenderSource(m_aoSourceDesc.back()); + } + else + { + InitBuffer(pData, nBufXSize, nBufYSize, eBufType, nBandCount, + panBandMap, nPixelSpace, nLineSpace, nBandSpace); + + // Now render from bottom of the stack to top. + for (auto &oSourceDesc : m_aoSourceDesc) + { + if (oSourceDesc.poDS && RenderSource(oSourceDesc) != CE_None) + return CE_Failure; + } + + return CE_None; + } +} + +/************************************************************************/ +/* GDALRegister_VRTTI() */ +/************************************************************************/ + +void GDALRegister_VRTTI() +{ + if (GDALGetDriverByName("VRTTI") != nullptr) + return; + + auto poDriver = std::make_unique(); + + poDriver->SetDescription("VRTTI"); + poDriver->SetMetadataItem(GDAL_DCAP_RASTER, "YES"); + poDriver->SetMetadataItem(GDAL_DMD_LONGNAME, "Virtual Raster Tile Index"); + poDriver->SetMetadataItem(GDAL_DMD_EXTENSIONS, "vrt.gpkg vrt.fgb vrtti"); + poDriver->SetMetadataItem(GDAL_DMD_CONNECTION_PREFIX, VRTTI_PREFIX); + poDriver->SetMetadataItem(GDAL_DMD_HELPTOPIC, "drivers/raster/vrtti.html"); + + poDriver->pfnOpen = VRTTileIndexDatasetOpen; + poDriver->pfnIdentify = VRTTileIndexDatasetIdentify; + + poDriver->SetMetadataItem(GDAL_DCAP_VIRTUALIO, "YES"); + + poDriver->SetMetadataItem(GDAL_DMD_OPENOPTIONLIST, + "" + " "); + + GetGDALDriverManager()->RegisterDriver(poDriver.release()); +} + +/*! @endcond */ diff --git a/gcore/gdal_frmts.h b/gcore/gdal_frmts.h index c514dc7e7f3b..66117ba6edad 100644 --- a/gcore/gdal_frmts.h +++ b/gcore/gdal_frmts.h @@ -84,6 +84,7 @@ void CPL_DLL GDALRegister_ECW_JP2ECW(); void CPL_DLL GDALRegister_FujiBAS(void); void CPL_DLL GDALRegister_FIT(void); void CPL_DLL GDALRegister_VRT(void); +void CPL_DLL GDALRegister_VRTTI(void); void CPL_DLL GDALRegister_USGSDEM(void); void CPL_DLL GDALRegister_FAST(void); void CPL_DLL GDALRegister_HDF4(void); diff --git a/gcore/gdal_misc.cpp b/gcore/gdal_misc.cpp index a1c9dfd5a19f..c711581d219e 100644 --- a/gcore/gdal_misc.cpp +++ b/gcore/gdal_misc.cpp @@ -4259,13 +4259,14 @@ void GDALSerializeOpenOptionsToXML(CPLXMLNode *psParentNode, /* GDALDeserializeOpenOptionsFromXML() */ /************************************************************************/ -char **GDALDeserializeOpenOptionsFromXML(CPLXMLNode *psParentNode) +char **GDALDeserializeOpenOptionsFromXML(const CPLXMLNode *psParentNode) { char **papszOpenOptions = nullptr; - CPLXMLNode *psOpenOptions = CPLGetXMLNode(psParentNode, "OpenOptions"); + const CPLXMLNode *psOpenOptions = + CPLGetXMLNode(psParentNode, "OpenOptions"); if (psOpenOptions != nullptr) { - CPLXMLNode *psOOI; + const CPLXMLNode *psOOI; for (psOOI = psOpenOptions->psChild; psOOI != nullptr; psOOI = psOOI->psNext) { diff --git a/gcore/gdal_priv.h b/gcore/gdal_priv.h index 36558a79283d..64bc2c4d8d75 100644 --- a/gcore/gdal_priv.h +++ b/gcore/gdal_priv.h @@ -106,7 +106,7 @@ class CPL_DLL GDALMultiDomainMetadata GDALMultiDomainMetadata(); ~GDALMultiDomainMetadata(); - int XMLInit(CPLXMLNode *psMetadata, int bMerge); + int XMLInit(const CPLXMLNode *psMetadata, int bMerge); CPLXMLNode *Serialize(); char **GetDomainList() @@ -4006,7 +4006,7 @@ void GDALDeserializeGCPListFromXML(CPLXMLNode *psGCPList, void GDALSerializeOpenOptionsToXML(CPLXMLNode *psParentNode, char **papszOpenOptions); -char **GDALDeserializeOpenOptionsFromXML(CPLXMLNode *psParentNode); +char **GDALDeserializeOpenOptionsFromXML(const CPLXMLNode *psParentNode); int GDALCanFileAcceptSidecarFile(const char *pszFilename); diff --git a/gcore/gdal_rat.cpp b/gcore/gdal_rat.cpp index af8edab87e48..a9e8a5cdaaa5 100644 --- a/gcore/gdal_rat.cpp +++ b/gcore/gdal_rat.cpp @@ -831,7 +831,7 @@ void *GDALRasterAttributeTable::SerializeJSON() const * @param psTree XML tree * @return error code. */ -CPLErr GDALRasterAttributeTable::XMLInit(CPLXMLNode *psTree, +CPLErr GDALRasterAttributeTable::XMLInit(const CPLXMLNode *psTree, const char * /*pszVRTPath*/) { @@ -884,7 +884,7 @@ CPLErr GDALRasterAttributeTable::XMLInit(CPLXMLNode *psTree, /* -------------------------------------------------------------------- */ /* Row data. */ /* -------------------------------------------------------------------- */ - for (CPLXMLNode *psChild = psTree->psChild; psChild != nullptr; + for (const CPLXMLNode *psChild = psTree->psChild; psChild != nullptr; psChild = psChild->psNext) { if (psChild->eType == CXT_Element && EQUAL(psChild->pszValue, "Row")) diff --git a/gcore/gdal_rat.h b/gcore/gdal_rat.h index fa3187a8d5a5..b8ca5bc0bb35 100644 --- a/gcore/gdal_rat.h +++ b/gcore/gdal_rat.h @@ -281,7 +281,7 @@ class CPL_DLL GDALRasterAttributeTable */ virtual CPLXMLNode *Serialize() const; virtual void *SerializeJSON() const; - virtual CPLErr XMLInit(CPLXMLNode *, const char *); + virtual CPLErr XMLInit(const CPLXMLNode *, const char *); virtual CPLErr InitializeFromColorTable(const GDALColorTable *); virtual GDALColorTable *TranslateToColorTable(int nEntryCount = -1); diff --git a/gcore/gdalmultidomainmetadata.cpp b/gcore/gdalmultidomainmetadata.cpp index 0bb7ee5aef3f..62d8da6fc34e 100644 --- a/gcore/gdalmultidomainmetadata.cpp +++ b/gcore/gdalmultidomainmetadata.cpp @@ -191,9 +191,9 @@ CPLErr GDALMultiDomainMetadata::SetMetadataItem(const char *pszName, /* elements. */ /************************************************************************/ -int GDALMultiDomainMetadata::XMLInit(CPLXMLNode *psTree, int /* bMerge */) +int GDALMultiDomainMetadata::XMLInit(const CPLXMLNode *psTree, int /* bMerge */) { - CPLXMLNode *psMetadata = nullptr; + const CPLXMLNode *psMetadata = nullptr; /* ==================================================================== */ /* Process all elements, each for one domain. */ @@ -226,7 +226,7 @@ int GDALMultiDomainMetadata::XMLInit(CPLXMLNode *psTree, int /* bMerge */) if (EQUAL(pszFormat, "xml")) { // Find first non-attribute child of current element. - CPLXMLNode *psSubDoc = psMetadata->psChild; + const CPLXMLNode *psSubDoc = psMetadata->psChild; while (psSubDoc != nullptr && psSubDoc->eType == CXT_Attribute) psSubDoc = psSubDoc->psNext; @@ -244,7 +244,7 @@ int GDALMultiDomainMetadata::XMLInit(CPLXMLNode *psTree, int /* bMerge */) else if (EQUAL(pszFormat, "json")) { // Find first text child of current element. - CPLXMLNode *psSubDoc = psMetadata->psChild; + const CPLXMLNode *psSubDoc = psMetadata->psChild; while (psSubDoc != nullptr && psSubDoc->eType != CXT_Text) psSubDoc = psSubDoc->psNext; if (psSubDoc) @@ -262,8 +262,8 @@ int GDALMultiDomainMetadata::XMLInit(CPLXMLNode *psTree, int /* bMerge */) */ else { - for (CPLXMLNode *psMDI = psMetadata->psChild; psMDI != nullptr; - psMDI = psMDI->psNext) + for (const CPLXMLNode *psMDI = psMetadata->psChild; + psMDI != nullptr; psMDI = psMDI->psNext) { if (!EQUAL(psMDI->pszValue, "MDI") || psMDI->eType != CXT_Element || psMDI->psChild == nullptr || diff --git a/ogr/ogrsf_frmts/gpkg/ogrgeopackagedatasource.cpp b/ogr/ogrsf_frmts/gpkg/ogrgeopackagedatasource.cpp index ec438fb63d20..2f9ad21a969c 100644 --- a/ogr/ogrsf_frmts/gpkg/ogrgeopackagedatasource.cpp +++ b/ogr/ogrsf_frmts/gpkg/ogrgeopackagedatasource.cpp @@ -5904,11 +5904,8 @@ bool GDALGeoPackageDataset::HasGriddedCoverageAncillaryTable() static GDALDataset *GetUnderlyingDataset(GDALDataset *poSrcDS) { - if (EQUAL(poSrcDS->GetDescription(), "") && - poSrcDS->GetDriver() != nullptr && - poSrcDS->GetDriver() == GDALGetDriverByName("VRT")) + if (auto poVRTDS = dynamic_cast(poSrcDS)) { - VRTDataset *poVRTDS = cpl::down_cast(poSrcDS); auto poTmpDS = poVRTDS->GetSingleSimpleSource(); if (poTmpDS) return poTmpDS; diff --git a/ogr/ogrsf_frmts/gpkg/ogrgeopackagedriver.cpp b/ogr/ogrsf_frmts/gpkg/ogrgeopackagedriver.cpp index 8a4ab342886d..6e4c438978fb 100644 --- a/ogr/ogrsf_frmts/gpkg/ogrgeopackagedriver.cpp +++ b/ogr/ogrsf_frmts/gpkg/ogrgeopackagedriver.cpp @@ -35,6 +35,11 @@ // g++ -g -Wall -fPIC -shared -o ogr_geopackage.so -Iport -Igcore -Iogr // -Iogr/ogrsf_frmts -Iogr/ogrsf_frmts/gpkg ogr/ogrsf_frmts/gpkg/*.c* -L. -lgdal +static inline bool ENDS_WITH_CI(const char *a, const char *b) +{ + return strlen(a) >= strlen(b) && EQUAL(a + strlen(a) - strlen(b), b); +} + /************************************************************************/ /* OGRGeoPackageDriverIdentify() */ /************************************************************************/ @@ -88,6 +93,13 @@ static int OGRGeoPackageDriverIdentify(GDALOpenInfo *poOpenInfo, return FALSE; } + if ((poOpenInfo->nOpenFlags & GDAL_OF_RASTER) != 0 && + ENDS_WITH_CI(poOpenInfo->pszFilename, ".vrt.gpkg")) + { + // Handled by VRT driver + return FALSE; + } + /* Requirement 3: File name has to end in "gpkg" */ /* http://opengis.github.io/geopackage/#_file_extension_name */ /* But be tolerant, if the GPKG application id is found, because some */ From abd623fcab693e54b109994db79f6e855fbec59c Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 20 Dec 2023 13:11:57 +0100 Subject: [PATCH 02/11] Add documentation for VRTTI driver --- doc/source/drivers/raster/index.rst | 1 + doc/source/drivers/raster/vrt.rst | 3 + doc/source/drivers/raster/vrtti.rst | 422 ++++++++++++++++++++++++++++ 3 files changed, 426 insertions(+) create mode 100644 doc/source/drivers/raster/vrtti.rst diff --git a/doc/source/drivers/raster/index.rst b/doc/source/drivers/raster/index.rst index 976e117f508c..426311e960dc 100644 --- a/doc/source/drivers/raster/index.rst +++ b/doc/source/drivers/raster/index.rst @@ -179,6 +179,7 @@ Raster drivers usgsdem vicar vrt + vrtti wcs webp wms diff --git a/doc/source/drivers/raster/vrt.rst b/doc/source/drivers/raster/vrt.rst index b50724db595c..d46d47e0b21d 100644 --- a/doc/source/drivers/raster/vrt.rst +++ b/doc/source/drivers/raster/vrt.rst @@ -20,6 +20,9 @@ extension .vrt. The VRT format can also describe :ref:`gdal_vrttut_warped` and :ref:`gdal_vrttut_pansharpen` +For mosaic with a very large number of tiles (tens of thousands or mores), +the :ref:`VRTTI ` driver may be used starting with GDAL 3.9. + An example of a simple .vrt file referring to a 512x512 dataset with one band loaded from :file:`utm.tif` might look like this: diff --git a/doc/source/drivers/raster/vrtti.rst b/doc/source/drivers/raster/vrtti.rst new file mode 100644 index 000000000000..99fb6eb3476d --- /dev/null +++ b/doc/source/drivers/raster/vrtti.rst @@ -0,0 +1,422 @@ +.. _raster.vrtti: + +================================================================================ +VRTTI -- GDAL Virtual Raster Tile Index +================================================================================ + +.. versionadded:: 3.9 + +.. shortname:: VRTTI + +.. built_in_by_default:: + +Introduction +------------ + +The VRTTI driver is a driver that allows to handle catalogs with a large +number of raster files (called "tiles" in the rest of this document, even if a +regular tiling is not required by the driver), and build a virtual mosaic from +them. Each tile may be in any GDAL supported raster format, and be a file +stored on a regular filesystem, or any GDAL supported virtual filesystem (for +raster drivers that support such files). + +This driver offers similar functionality as the :ref:`VRT ` +driver with the following main differences: + +* The tiles are listed as features of any GDAL supported vector format. Use of + formats with efficient spatial filtering is recommended, such as + :ref:`GeoPackage `, :ref:`FlatGeoBuf ` or + :ref:`PostGIS `. The VRTTI driver can thus use a larger number of + tiles than the VRT driver (hundreds of thousands or more), provided the + underlying vector format is efficient. + +* The tiles may have different SRS. The VRTTI driver is capable of on-the-fly + reprojection + +* The VRTTI driver offers control on the order in which tiles are composited, + when they overlap (z-order) + +* The VRTTI driver honours the mask/alpha band when compositing together + overlapping tiles. + +* Contrary to the VRT driver, the VRTTI driver does not enable to alter + characteristics of referenced tiles, such as their georeferencing, nodata value, + etc. If such behavior is desired, the tiles must be for example wrapped + individually in a VRT file before being referenced in the VRTTI index. + +Connection strings +------------------ + +The VRTTI driver accepts different types of connection strings: + +* a vector file in GeoPackage format with a ``.vrt.gpkg`` extension, or in + FlatGeoBuf format with a ``.vrt.fgb`` extension, meeting the minimum requirements + for a VRTTI compatible tile index, detailed later. + + For example: ``tileindex.vrt.gpkg`` + +* any vector file in a GDAL supported format, with its filename (or connection + string prefixed with ``VRTTI:`` + + For example: ``VRTTI:tileindex.shp`` or ``VRTTI:PG:database=my_db schema=tileindex`` + +* a XML file, following the below VRTTI XML format, generally with the + recommended ``.vrtti`` extension, referencing a vector file. Using such + XML file may be more practical for tile indexes not stored in a file, or + if some additional metadata must be defined at the dataset or band level of + the virtual mosaic. + + For example: ``tileindex.vrtti`` + +Tile index requirements +----------------------- + +The minimum requirements for a VRTTI compatible tile index is to be a +vector format supported by GDAL, with a geometry column storing polygons with +the extent of the tiles, and an attribute field of type string, storing the +path to each tile. The default name for this attribute field is ``location``. +If relative filenames are stored in the tile index, they are considered to +be relative to the path of the tile index. + +In addition, for formats that can store layer metadata (GeoPackage, FlatGeoBuf, +PostGIS, ...), the following layer metadata items may be set: + +* ``RESX=`` and ``RESY=``: resolution along X and Y axis, + in SRS units / pixel. + + Setting those metadata items is recommended, otherwise + the driver will try to open one of the tiles referenced in the tile index, + and use its resolution as the resolution for the mosaic. + +* ``BAND_COUNT=``: number of bands of the virtual mosaic. The tiles + stored in an index should generally have the same number of bands. + + Setting that metadata item is recommended, otherwise + the driver will try to open one of the tiles referenced in the tile index, and + use it as the number of bands for the mosaic. + + A mix of tiles with N and N+1 bands is allowed, provided that the color + interpretation of the (N+1)th band is alpha. The N+1 value must be written + as the band count in that situation. + + If tiles contains a single band with a color table, and that the color table + may differ among tiles, BAND_COUNT should be set to 3 (resp. 4) to perform + expansion to Red, Green, Blue components (resp. Red, Green, Blue, Alpha). + If the color table is identical between the tiles, and it is desired to + preserve it, the VRRTI XML file format may be used to include the ColorTable + element. + + +* ``DATA_TYPE=``: data type of the tiles of the tile index + ``Byte``, ``Int8``, ``UInt16``, + ``Int16``, ``UInt32``, ``Int32``, ``UInt64``, ``Int64``, ``Float32``, ``Float64``, ``CInt16``, + ``CInt32``, ``CFloat32`` or ``CFloat64`` + + Setting that metadata item is recommended, otherwise + the driver will try to open one of the tiles referenced in the tile index, and + use it as the data type for the mosaic. + +* ``NODATA=[,``, ``MINY=``, ``MAXX=`` and ``MAXY=``: + defines the extent of the virtual mosaic. + + For vector formats that have efficient retrieval of the layer extent, setting + those items is not needed. + +* ``GEOTRANSFORM=,,,,,``: defines the GeoTransform. + Used together with ``XSIZE`` and ``YSIZE``, this is an alternate way of + defining the extent and resolution os the virtual mosaic. + + It is not necessary to define this item if ``RESX=`` and ``RESY`` are set + (potentially accompanied with ``MINX``, ``MINY``, ``MAXX`` and ``MAXY``) + +* ``XSIZE=``, ``YSIZE=``: size of the virtual mosaic in pixel. + +* ``COLOR_INTERPRETATION=[,``: defines the SRS of the virtual mosaic, using any value + supported by the :cpp:func:`OGRSpatialReference::SetFromUserInput` call, which + includes EPSG Projected, Geographic or Compound CRS (i.e. EPSG:4296), a + well known text (WKT) CRS definition, PROJ.4 declarations, etc. + + It is not necessary to define this element if the virtual mosaic SRS is + recorded as the SRS of the vector layer of the tile index. + +* ``LOCATION_FIELD=``: name of the field where the tile location is + stored. Defaults to ``location``. + +* ``SORT_FIELD=``: name of a field to use to control the order in which + tiles are composited, when they overlap (z-order). That field may be of + type String, Integer, Integer64, Date or DateTime. By default, the higher the + value in that field, the last the corresponding tile will be rendered in the + virtual mosaic (unless SORT_FIELD_ASC=NO is set) + +* ``SORT_FIELD_ASC=YES|NO``: whether the values in SORT_FIELD should be sorted + in ascendent or descent order. Defaults to YES (ascendent) + +* ``BLOCKXSIZE=`` and ``BLOCKYSIZE=``: Block size of bands of the + virtual mosaic. Defaults to 256x256. + +* ``MASK_BAND=YES|NO``: whether a dataset-level mask band should be exposed. + Defaults to NO. + +* ``RESAMPLING=``: Resampling method to use for on-the-fly reprojection, + or rendering of tiles whose origin coordinates are not at an offset multiple + of the resolution of the mosaic compared to the origin of the mosaic. In that + later case, RESAMPLING is only taken into account when requesting pixels with + the default nearest resampling mode. + + Possible values: ``nearest``, ``cubic``, ``cubicspline``, ``lanczos``, ``average``, ``rms``, ``gauss`` + + Defaults to ``nearest`` + +* ``BAND__OFFSET=`` where number is an integer index starting at 1. + + Additive offset to apply to the raw numbers of the band. + +* ``BAND__SCALE=`` where number is an integer index starting at 1. + + Multiplicative factor to apply to the raw numbers of the band. + +* ``BAND__UNITTYPE=`` where number is an integer index starting at 1. + + Unit of the band. + +* ``OVERVIEW__DATASET=`` where idx is an integer index starting at 0. + + Name of the dataset to use as the first overview level. This may be a + raster dataset (for example a GeoTIFF file, or another VRTTI dataset). + This may also be a vector dataset with a VRTTI compatible layer, potentially + specified with ``OVERVIEW__LAYER``. + +* ``OVERVIEW__OPEN_OPTIONS=[,key2=value2]...`` where idx is an integer index starting at 0. + + Open options(s) to use to open ``OVERVIEW__DATASET``. + +* ``OVERVIEW__LAYER=`` where idx is an integer index starting at 0. + + Name of the vector layer to use as the first overview level, assuming + ``OVERVIEW__DATASET`` points to a vector dataset. ``OVERVIEW__DATASET`` + may also not be specified, in which case the vector dataset of the full + resolution virtual mosaic is used. + +* ``OVERVIEW__FACTOR=`` where idx is an integer index starting at 0. + + Sub-sampling factor, strictly greater than 1. If ``OVERVIEW__DATASET`` + and ``OVERVIEW__LAYER`` are not specified, then all tiles of the full + resolution virtual mosaic are used, with the specified sub-sampling factor + (it is recommended, but not required, that those tiles do have a corresponding overview). + ``OVERVIEW__DATASET`` and/or ``OVERVIEW__LAYER`` may also be + specified to point to another tile index. + +All overviews *must* have exactly the same extent as the full resolution +virtual mosaic. The VRTTI driver does not check that, and if that condition is +not met, subsampled pixel request will lead to incorrect result. + +In addition to those layer metadata items, the dataset-level metadata item +``TILE_INDEX_LAYER`` may be set to indicate, for dataset with multiple layers, +which one should be used as the tile index layer. + +VRTTI XML format +---------------- + +A `XML schema of the GDAL VRTTI format `_ +is available. + +The following artificial example contains all potential elements and attributes. +A number of them have similar name and same semantics as layer metadata items +mentioned in the previous section. + +.. code-block:: xml + + + PG:dbname=my_db + my_layer + pub_date >= '2023/12/01' + pub_date + true + EPSG:4326 + 60 + 60 + 0 + 1 + 2 + 3 + 2,1,0,49,0,-1 + 2048 + 1024 + 256 + 256 + Cubic + 1 + + + + + + my band + 2 + 3 + 4 + dn + Gray + + + + + cat + + + + BAR + + + BAR + + + + + BAR + + + BAR + + + + + 2 + + + + 4 + + + + some.tif + + + + other.vrt.gpkg + other_layer + + 0 + 1 + 2 + 3 + + + + + + +At the VRTTileIndexDataset level, the elements specific to VRTTI XML are: + +* ``Filter``: value of a SQL WHERE clause, used to select a subset of the + features of the index. + +* ``BlockXSize`` / ``BlockYSize``: dimension of the block size of bands. + Defaults to 256x256 + +* ``Metadata``: defines dataset-level metadata. You can refer to the + documentation of the :ref:`VRT ` driver for its syntax. + +At the Band level, the elements specific to VRTTI XML are: Description, +Offset, Scale, UnitType, ColorTable, CategoryNames, GDALRasterAttributeTable, +Metadata. +You can refer to the documentation of the :ref:`VRT ` driver for +their syntax and semantics. + + +How to build a VRTTI comptatible index ? +---------------------------------------- + +The :ref:`gdaltindex` program may be used to generate both a vector tile index, +and optionally a wrapping .vrtti XML file. + +A VRTTI comptatible index may also be created by any programmatic means, provided +the above format specifications are met. + + +Open options +------------ + +The following open options are available. Most of them can be +also defined as layer metadata items or in the .vrtti XML file + + +- .. oo:: LAYER + :choices: + + For dataset with multiple layers, indicates which one should be used as + the tile index layer. + Same role as the TILE_INDEX_LAYER dataset level metadata item + + +- .. oo:: LOCATION_FIELD + :choices: + :default: location + + Name of the field where the tile location is stored. + + +- .. oo:: SORT_FIELD + :choices: + + Name of a field to use to control the order in which + tiles are composited, when they overlap (z-order). That field may be of + type String, Integer, Integer64, Date or DateTime. By default, the higher the + value in that field, the last the corresponding tile will be rendered in the + virtual mosaic (unless SORT_FIELD_ASC=NO is set) + +- .. oo:: SORT_FIELD_ASC + :choices: YES, NO + :default: YES + + Whether the values in SORT_FIELD should be sorted in ascendent or descent order + +- .. oo:: FILTER + :choices: + + Value of a SQL WHERE clause, used to select a subset of the features of the index. + +- .. oo:: RESX + :choices: + + Resolution along X axis in SRS units / pixel. + +- .. oo:: RESY + :choices: + + Resolution along Y axis in SRS units / pixel. + +- .. oo:: MINX + :choices: + + Minimum X value for the virtual mosaic extent + +- .. oo:: MINY + :choices: + + Minimum Y value for the virtual mosaic extent + +- .. oo:: MAXX + :choices: + + Maximum X value for the virtual mosaic extent + +- .. oo:: MAXY + :choices: + + Maximum Y value for the virtual mosaic extent From d28bfd7f4f519d25bffa456a3bad80b569c86585 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 20 Dec 2023 13:11:59 +0100 Subject: [PATCH 03/11] gdaladdo: make --partial-refresh-from-source-timestamp work on VRTTI datasets --- apps/gdaladdo.cpp | 162 ++++++++++++++-------------- autotest/utilities/test_gdaladdo.py | 78 +++++++++++++- 2 files changed, 161 insertions(+), 79 deletions(-) diff --git a/apps/gdaladdo.cpp b/apps/gdaladdo.cpp index b99ce37014dc..0fdd7c44cd25 100644 --- a/apps/gdaladdo.cpp +++ b/apps/gdaladdo.cpp @@ -32,6 +32,7 @@ #include "gdal_priv.h" #include "commonutils.h" #include "vrtdataset.h" +#include "vrt_priv.h" #include @@ -275,15 +276,6 @@ static bool PartialRefreshFromSourceTimestamp( bool bMinSizeSpecified, int nMinSize, GDALProgressFunc pfnProgress, void *pProgressArg) { - auto poVRTDS = dynamic_cast(poDS); - if (!poVRTDS) - { - CPLError(CE_Failure, CPLE_AppDefined, - "--partial-refresh-from-source-timestamp only works on a VRT " - "dataset"); - return false; - } - std::vector anOvrIndices; if (!GetOvrIndices(poDS, nLevelCount, panLevels, bMinSizeSpecified, nMinSize, anOvrIndices)) @@ -304,82 +296,96 @@ static bool PartialRefreshFromSourceTimestamp( return false; } - auto poVRTBand = - dynamic_cast(poDS->GetRasterBand(1)); - if (!poVRTBand) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Band is not a VRTSourcedRasterBand"); - return false; - } - - struct Region - { - std::string osFileName; - int nXOff; - int nYOff; - int nXSize; - int nYSize; - }; - std::vector regions; + std::vector regions; double dfTotalPixels = 0; - for (int i = 0; i < poVRTBand->nSources; ++i) + + if (dynamic_cast(poDS)) { - auto poSource = - dynamic_cast(poVRTBand->papoSources[i]); - if (poSource) + auto poVRTBand = + dynamic_cast(poDS->GetRasterBand(1)); + if (!poVRTBand) { - VSIStatBufL sStatSource; - if (VSIStatL(poSource->GetSourceDatasetName().c_str(), - &sStatSource) == 0) + CPLError(CE_Failure, CPLE_AppDefined, + "Band is not a VRTSourcedRasterBand"); + return false; + } + + for (int i = 0; i < poVRTBand->nSources; ++i) + { + auto poSource = + dynamic_cast(poVRTBand->papoSources[i]); + if (poSource) { - if (sStatSource.st_mtime > sStatVRTOvr.st_mtime) + VSIStatBufL sStatSource; + if (VSIStatL(poSource->GetSourceDatasetName().c_str(), + &sStatSource) == 0) { - double dfXOff, dfYOff, dfXSize, dfYSize; - poSource->GetDstWindow(dfXOff, dfYOff, dfXSize, dfYSize); - constexpr double EPS = 1e-8; - int nXOff = static_cast(dfXOff + EPS); - int nYOff = static_cast(dfYOff + EPS); - int nXSize = static_cast(dfXSize + 0.5); - int nYSize = static_cast(dfYSize + 0.5); - if (nXOff > poDS->GetRasterXSize() || - nYOff > poDS->GetRasterYSize() || nXSize <= 0 || - nYSize <= 0) + if (sStatSource.st_mtime > sStatVRTOvr.st_mtime) { - continue; - } - if (nXOff < 0) - { - nXSize += nXOff; - nXOff = 0; - } - if (nXOff > poDS->GetRasterXSize() - nXSize) - { - nXSize = poDS->GetRasterXSize() - nXOff; - } - if (nYOff < 0) - { - nYSize += nYOff; - nYOff = 0; - } - if (nYOff > poDS->GetRasterYSize() - nYSize) - { - nYSize = poDS->GetRasterYSize() - nYOff; - } + double dfXOff, dfYOff, dfXSize, dfYSize; + poSource->GetDstWindow(dfXOff, dfYOff, dfXSize, + dfYSize); + constexpr double EPS = 1e-8; + int nXOff = static_cast(dfXOff + EPS); + int nYOff = static_cast(dfYOff + EPS); + int nXSize = static_cast(dfXSize + 0.5); + int nYSize = static_cast(dfYSize + 0.5); + if (nXOff > poDS->GetRasterXSize() || + nYOff > poDS->GetRasterYSize() || nXSize <= 0 || + nYSize <= 0) + { + continue; + } + if (nXOff < 0) + { + nXSize += nXOff; + nXOff = 0; + } + if (nXOff > poDS->GetRasterXSize() - nXSize) + { + nXSize = poDS->GetRasterXSize() - nXOff; + } + if (nYOff < 0) + { + nYSize += nYOff; + nYOff = 0; + } + if (nYOff > poDS->GetRasterYSize() - nYSize) + { + nYSize = poDS->GetRasterYSize() - nYOff; + } - dfTotalPixels += static_cast(nXSize) * nYSize; - Region region; - region.osFileName = poSource->GetSourceDatasetName(); - region.nXOff = nXOff; - region.nYOff = nYOff; - region.nXSize = nXSize; - region.nYSize = nYSize; - regions.push_back(std::move(region)); + dfTotalPixels += static_cast(nXSize) * nYSize; + VRTTISourceDesc region; + region.osFilename = poSource->GetSourceDatasetName(); + region.nDstXOff = nXOff; + region.nDstYOff = nYOff; + region.nDstXSize = nXSize; + region.nDstYSize = nYSize; + regions.push_back(std::move(region)); + } } } } } + else if (auto poVRTTIDS = GDALDatasetCastToVRTTIDataset(poDS)) + { + regions = + VRTTIGetSourcesMoreRecentThan(poVRTTIDS, sStatVRTOvr.st_mtime); + for (const auto ®ion : regions) + { + dfTotalPixels += + static_cast(region.nDstXSize) * region.nDstYSize; + } + } + else + { + CPLError(CE_Failure, CPLE_AppDefined, + "--partial-refresh-from-source-timestamp only works on a VRT " + "dataset"); + return false; + } if (!regions.empty()) { @@ -389,22 +395,22 @@ static bool PartialRefreshFromSourceTimestamp( if (pfnProgress == GDALDummyProgress) { CPLDebug("GDAL", "Refresh from source %s", - region.osFileName.c_str()); + region.osFilename.c_str()); } else { - printf("Refresh from source %s.\n", region.osFileName.c_str()); + printf("Refresh from source %s.\n", region.osFilename.c_str()); } double dfNextCurPixels = dfCurPixels + - static_cast(region.nXSize) * region.nYSize; + static_cast(region.nDstXSize) * region.nDstYSize; void *pScaledProgress = GDALCreateScaledProgress( dfCurPixels / dfTotalPixels, dfNextCurPixels / dfTotalPixels, pfnProgress, pProgressArg); bool bRet = PartialRefresh( poDS, anOvrIndices, nBandCount, panBandList, pszResampling, - region.nXOff, region.nYOff, region.nXSize, region.nYSize, - GDALScaledProgress, pScaledProgress); + region.nDstXOff, region.nDstYOff, region.nDstXSize, + region.nDstYSize, GDALScaledProgress, pScaledProgress); GDALDestroyScaledProgress(pScaledProgress); if (!bRet) return false; diff --git a/autotest/utilities/test_gdaladdo.py b/autotest/utilities/test_gdaladdo.py index 9f056dea1091..a36af5ef62ba 100755 --- a/autotest/utilities/test_gdaladdo.py +++ b/autotest/utilities/test_gdaladdo.py @@ -38,7 +38,7 @@ import test_cli_utilities from gcore import tiff_ovr -from osgeo import gdal +from osgeo import gdal, ogr pytestmark = pytest.mark.skipif( test_cli_utilities.get_gdaladdo_path() is None, reason="gdaladdo not available" @@ -375,3 +375,79 @@ def test_gdaladdo_reuse_previous_resampling_and_levels( assert ds.GetRasterBand(1).GetOverview(0).GetMetadataItem("RESAMPLING") == "CUBIC" assert ds.GetRasterBand(1).GetOverview(0).Checksum() == 1059 ds = None + + +############################################################################### +# Test --partial-refresh-from-source-timestamp with VRTTI dataset + + +@pytest.mark.require_driver("GPKG") +def test_gdaladdo_partial_refresh_from_source_timestamp_vrtti(gdaladdo_path, tmp_path): + + left_tif = str(tmp_path / "left.tif") + right_tif = str(tmp_path / "right.tif") + + gdal.Translate(left_tif, "../gcore/data/byte.tif", options="-srcwin 0 0 10 20") + gdal.Translate(right_tif, "../gcore/data/byte.tif", options="-srcwin 10 0 10 20") + + source_ds = [gdal.Open(left_tif), gdal.Open(right_tif)] + tmp_vrt = str(tmp_path / "test.vrt.gpkg") + index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(tmp_vrt) + lyr = index_ds.CreateLayer("index", srs=source_ds[0].GetSpatialRef()) + lyr.CreateField(ogr.FieldDefn("location")) + for i, src_ds in enumerate(source_ds): + f = ogr.Feature(lyr.GetLayerDefn()) + src_gt = src_ds.GetGeoTransform() + minx = src_gt[0] + maxx = minx + src_ds.RasterXSize * src_gt[1] + maxy = src_gt[3] + miny = maxy + src_ds.RasterYSize * src_gt[5] + f["location"] = src_ds.GetDescription() + f.SetGeometry( + ogr.CreateGeometryFromWkt( + f"POLYGON(({minx} {miny},{minx} {maxy},{maxx} {maxy},{maxx} {miny},{minx} {miny}))" + ) + ) + lyr.CreateFeature(f) + index_ds.Close() + del source_ds + + gdaltest.runexternal(f"{gdaladdo_path} -r bilinear {tmp_vrt} 2") + + ds = gdal.Open(tmp_vrt, gdal.GA_Update) + ovr_data_ori = array.array("B", ds.GetRasterBand(1).GetOverview(0).ReadRaster()) + ds = None + + ds = gdal.Open(left_tif, gdal.GA_Update) + ds.GetRasterBand(1).Fill(0) + ds = None + + # Make sure timestamp of left.tif is before tmp.vrt.ovr + timestamp = int(os.stat(tmp_vrt + ".ovr").st_mtime) - 10 + os.utime(left_tif, times=(timestamp, timestamp)) + + ds = gdal.Open(right_tif, gdal.GA_Update) + ds.GetRasterBand(1).Fill(0) + ds = None + + # Make sure timestamp of right.tif is after tmp.vrt.ovr + timestamp = int(os.stat(tmp_vrt + ".ovr").st_mtime) + 10 + os.utime(right_tif, times=(timestamp, timestamp)) + + out, err = gdaltest.runexternal_out_and_err( + f"{gdaladdo_path} -r bilinear --partial-refresh-from-source-timestamp {tmp_vrt}" + ) + assert "ERROR" not in err, (out, err) + + ds = gdal.Open(tmp_vrt) + ovr_band = ds.GetRasterBand(1).GetOverview(0) + ovr_data_refreshed = array.array("B", ovr_band.ReadRaster()) + # Test that data is zero only in the refreshed area, and unchanged + # in other areas + for j in range(10): + for i in range(5): + idx = (j) * ovr_band.XSize + (i + 5) + assert ovr_data_refreshed[idx] == 0 + ovr_data_refreshed[idx] = ovr_data_ori[idx] + assert ovr_data_refreshed == ovr_data_ori + ds = None From 60535b9e5470ccd6fdfee5108574b970aa162be6 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 20 Dec 2023 13:12:01 +0100 Subject: [PATCH 04/11] Make gdaltindex a C callable function: GDALTileIndex() --- apps/CMakeLists.txt | 5 +- apps/gdal_utils.h | 17 + apps/gdal_utils_priv.h | 8 + apps/gdaltindex.cpp | 746 -------------------- apps/gdaltindex_bin.cpp | 158 +++++ apps/gdaltindex_lib.cpp | 788 ++++++++++++++++++++++ apps/gdalwarp_lib.cpp | 7 +- apps/ogr2ogr_lib.cpp | 7 +- autotest/utilities/test_gdaltindex.py | 127 ---- autotest/utilities/test_gdaltindex_lib.py | 246 +++++++ doc/source/programs/gdaltindex.rst | 7 +- swig/include/gdal.i | 71 ++ swig/include/python/gdal_python.i | 96 +++ 13 files changed, 1403 insertions(+), 880 deletions(-) delete mode 100644 apps/gdaltindex.cpp create mode 100644 apps/gdaltindex_bin.cpp create mode 100644 apps/gdaltindex_lib.cpp create mode 100644 autotest/utilities/test_gdaltindex_lib.py diff --git a/apps/CMakeLists.txt b/apps/CMakeLists.txt index 70e106058c5d..b4258090d99a 100644 --- a/apps/CMakeLists.txt +++ b/apps/CMakeLists.txt @@ -19,7 +19,8 @@ add_library( nearblack_lib_floodfill.cpp gdal_footprint_lib.cpp gdalmdiminfo_lib.cpp - gdalmdimtranslate_lib.cpp) + gdalmdimtranslate_lib.cpp + gdaltindex_lib.cpp) add_dependencies(appslib generate_gdal_version_h) target_sources(${GDAL_LIB_TARGET_NAME} PRIVATE $) gdal_standard_includes(appslib) @@ -55,7 +56,7 @@ if (BUILD_APPS) add_executable(gdalmanage gdalmanage.cpp) add_executable(gdalsrsinfo gdalsrsinfo.cpp) target_link_libraries(gdalsrsinfo PRIVATE PROJ::proj) - add_executable(gdaltindex gdaltindex.cpp) + add_executable(gdaltindex gdal_utils_priv.h gdaltindex_bin.cpp) add_executable(gdal_rasterize gdal_rasterize_bin.cpp) add_executable(gdaldem gdaldem_bin.cpp) add_executable(gdaltransform gdaltransform.cpp) diff --git a/apps/gdal_utils.h b/apps/gdal_utils.h index 6d48aedbfa2f..a7a5b6846c77 100644 --- a/apps/gdal_utils.h +++ b/apps/gdal_utils.h @@ -307,6 +307,23 @@ void CPL_DLL GDALVectorInfoOptionsFree(GDALVectorInfoOptions *psOptions); char CPL_DLL *GDALVectorInfo(GDALDatasetH hDataset, const GDALVectorInfoOptions *psOptions); +/*! Options for GDALTileIndex(). Opaque type */ +typedef struct GDALTileIndexOptions GDALTileIndexOptions; + +/** Opaque type */ +typedef struct GDALTileIndexOptionsForBinary GDALTileIndexOptionsForBinary; + +GDALTileIndexOptions CPL_DLL * +GDALTileIndexOptionsNew(char **papszArgv, + GDALTileIndexOptionsForBinary *psOptionsForBinary); + +void CPL_DLL GDALTileIndexOptionsFree(GDALTileIndexOptions *psOptions); + +GDALDatasetH CPL_DLL GDALTileIndex(const char *pszDest, int nSrcCount, + const char *const *papszSrcDSNames, + const GDALTileIndexOptions *psOptions, + int *pbUsageError); + CPL_C_END #endif /* GDAL_UTILS_H_INCLUDED */ diff --git a/apps/gdal_utils_priv.h b/apps/gdal_utils_priv.h index 2a705cdbc2df..b34f7a3cd4e7 100644 --- a/apps/gdal_utils_priv.h +++ b/apps/gdal_utils_priv.h @@ -222,6 +222,14 @@ struct GDALFootprintOptionsForBinary std::string osDestLayerName{}; }; +struct GDALTileIndexOptionsForBinary +{ + CPLStringList aosSrcFiles{}; + bool bDestSpecified = false; + std::string osDest{}; + bool bQuiet = false; +}; + #endif /* #ifndef DOXYGEN_SKIP */ #endif /* GDAL_UTILS_PRIV_H_INCLUDED */ diff --git a/apps/gdaltindex.cpp b/apps/gdaltindex.cpp deleted file mode 100644 index 36bb77c5b1f2..000000000000 --- a/apps/gdaltindex.cpp +++ /dev/null @@ -1,746 +0,0 @@ -/****************************************************************************** - * - * Project: MapServer - * Purpose: Commandline App to build tile index for raster files. - * Author: Frank Warmerdam, warmerdam@pobox.com - * - ****************************************************************************** - * Copyright (c) 2001, Frank Warmerdam, DM Solutions Group Inc - * Copyright (c) 2007-2013, Even Rouault - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - ****************************************************************************/ - -#include "cpl_port.h" -#include "cpl_conv.h" -#include "cpl_string.h" -#include "gdal_version.h" -#include "gdal.h" -#include "ogr_api.h" -#include "ogr_srs_api.h" -#include "commonutils.h" - -#include - -/************************************************************************/ -/* Usage() */ -/************************************************************************/ - -static void Usage(bool bIsError, const char *pszErrorMsg) - -{ - fprintf( - bIsError ? stderr : stdout, "%s", - "Usage: gdaltindex [--help] [--help-general]\n" - " [-f ] [-tileindex ] " - "[-write_absolute_path] \n" - " [-skip_different_projection] [-t_srs ]\n" - " [-src_srs_name field_name] [-src_srs_format " - "{AUTO|WKT|EPSG|PROJ}]\n" - " [-lyr_name ] " - "[]...\n" - "\n" - "e.g.\n" - " % gdaltindex doq_index.shp doq/*.tif\n" - "\n" - "NOTES:\n" - " o The index will be created if it doesn't already exist.\n" - " o The default tile index field is 'location'.\n" - " o Raster filenames will be put in the file exactly as they are " - "specified\n" - " on the commandline unless the option -write_absolute_path is " - "used.\n" - " o If -skip_different_projection is specified, only files with same " - "projection ref\n" - " as files already inserted in the tileindex will be inserted " - "(unless t_srs is specified).\n" - " o If -t_srs is specified, geometries of input files will be " - "transformed to the desired\n" - " target coordinate reference system.\n" - " Note that using this option generates files that are NOT " - "compatible with MapServer < 6.4.\n" - " o Simple rectangular polygons are generated in the same coordinate " - "reference system\n" - " as the rasters, or in target reference system if the -t_srs " - "option is used.\n"); - - if (pszErrorMsg != nullptr) - fprintf(stderr, "\nFAILURE: %s\n", pszErrorMsg); - - exit(bIsError ? 1 : 0); -} - -/************************************************************************/ -/* main() */ -/************************************************************************/ - -#define CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(nExtraArg) \ - do \ - { \ - if (iArg + nExtraArg >= argc) \ - Usage(true, CPLSPrintf("%s option requires %d argument(s)", \ - argv[iArg], nExtraArg)); \ - } while (false) - -typedef enum -{ - FORMAT_AUTO, - FORMAT_WKT, - FORMAT_EPSG, - FORMAT_PROJ -} SrcSRSFormat; - -MAIN_START(argc, argv) -{ - // Check that we are running against at least GDAL 1.4. - // Note to developers: if we use newer API, please change the requirement. - if (atoi(GDALVersionInfo("VERSION_NUM")) < 1400) - { - fprintf(stderr, - "At least, GDAL >= 1.4.0 is required for this version of %s, " - "which was compiled against GDAL %s\n", - argv[0], GDAL_RELEASE_NAME); - exit(1); - } - - GDALAllRegister(); - OGRRegisterAll(); - - argc = GDALGeneralCmdLineProcessor(argc, &argv, 0); - if (argc < 1) - exit(-argc); - - /* -------------------------------------------------------------------- */ - /* Get commandline arguments other than the GDAL raster filenames. */ - /* -------------------------------------------------------------------- */ - const char *pszIndexLayerName = nullptr; - const char *index_filename = nullptr; - const char *tile_index = "location"; - const char *pszDriverName = nullptr; - size_t nMaxFieldSize = 254; - bool write_absolute_path = false; - char *current_path = nullptr; - bool skip_different_projection = false; - const char *pszTargetSRS = ""; - bool bSetTargetSRS = false; - const char *pszSrcSRSName = nullptr; - int i_SrcSRSName = -1; - bool bSrcSRSFormatSpecified = false; - SrcSRSFormat eSrcSRSFormat = FORMAT_AUTO; - - int iArg = 1; // Used after for. - for (; iArg < argc; iArg++) - { - if (EQUAL(argv[iArg], "--utility_version")) - { - printf("%s was compiled against GDAL %s and is running against " - "GDAL %s\n", - argv[0], GDAL_RELEASE_NAME, GDALVersionInfo("RELEASE_NAME")); - CSLDestroy(argv); - return 0; - } - else if (EQUAL(argv[iArg], "--help")) - Usage(false, nullptr); - else if ((strcmp(argv[iArg], "-f") == 0 || - strcmp(argv[iArg], "-of") == 0)) - { - CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); - pszDriverName = argv[++iArg]; - } - else if (strcmp(argv[iArg], "-lyr_name") == 0) - { - CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); - pszIndexLayerName = argv[++iArg]; - } - else if (strcmp(argv[iArg], "-tileindex") == 0) - { - CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); - tile_index = argv[++iArg]; - } - else if (strcmp(argv[iArg], "-t_srs") == 0) - { - CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); - pszTargetSRS = argv[++iArg]; - bSetTargetSRS = true; - } - else if (strcmp(argv[iArg], "-write_absolute_path") == 0) - { - write_absolute_path = true; - } - else if (strcmp(argv[iArg], "-skip_different_projection") == 0) - { - skip_different_projection = true; - } - else if (strcmp(argv[iArg], "-src_srs_name") == 0) - { - CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); - pszSrcSRSName = argv[++iArg]; - } - else if (strcmp(argv[iArg], "-src_srs_format") == 0) - { - const char *pszFormat; - bSrcSRSFormatSpecified = true; - CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); - pszFormat = argv[++iArg]; - if (EQUAL(pszFormat, "AUTO")) - eSrcSRSFormat = FORMAT_AUTO; - else if (EQUAL(pszFormat, "WKT")) - eSrcSRSFormat = FORMAT_WKT; - else if (EQUAL(pszFormat, "EPSG")) - eSrcSRSFormat = FORMAT_EPSG; - else if (EQUAL(pszFormat, "PROJ")) - eSrcSRSFormat = FORMAT_PROJ; - } - else if (argv[iArg][0] == '-') - Usage(true, CPLSPrintf("Unknown option name '%s'", argv[iArg])); - else if (index_filename == nullptr) - { - index_filename = argv[iArg]; - iArg++; - break; - } - } - - if (index_filename == nullptr) - Usage(true, "No index filename specified."); - if (iArg == argc) - Usage(true, "No file to index specified."); - if (bSrcSRSFormatSpecified && pszSrcSRSName == nullptr) - Usage(true, "-src_srs_name must be specified when -src_srs_format is " - "specified."); - - /* -------------------------------------------------------------------- */ - /* Create and validate target SRS if given. */ - /* -------------------------------------------------------------------- */ - OGRSpatialReferenceH hTargetSRS = nullptr; - if (bSetTargetSRS) - { - if (skip_different_projection) - { - fprintf(stderr, - "Warning : -skip_different_projection does not apply " - "when -t_srs is requested.\n"); - } - hTargetSRS = OSRNewSpatialReference(""); - OSRSetAxisMappingStrategy(hTargetSRS, OAMS_TRADITIONAL_GIS_ORDER); - // coverity[tainted_data] - if (OSRSetFromUserInput(hTargetSRS, pszTargetSRS) != CE_None) - { - OSRDestroySpatialReference(hTargetSRS); - fprintf(stderr, "Invalid target SRS `%s'.\n", pszTargetSRS); - exit(1); - } - } - - /* -------------------------------------------------------------------- */ - /* Open or create the target datasource */ - /* -------------------------------------------------------------------- */ - GDALDatasetH hTileIndexDS = - GDALOpenEx(index_filename, GDAL_OF_VECTOR | GDAL_OF_UPDATE, nullptr, - nullptr, nullptr); - OGRLayerH hLayer = nullptr; - CPLString osFormat; - if (hTileIndexDS != nullptr) - { - GDALDriverH hDriver = GDALGetDatasetDriver(hTileIndexDS); - if (hDriver) - osFormat = GDALGetDriverShortName(hDriver); - - if (GDALDatasetGetLayerCount(hTileIndexDS) == 1) - { - hLayer = GDALDatasetGetLayer(hTileIndexDS, 0); - } - else - { - if (pszIndexLayerName == nullptr) - { - printf("-lyr_name must be specified.\n"); - exit(1); - } - CPLPushErrorHandler(CPLQuietErrorHandler); - hLayer = GDALDatasetGetLayerByName(hTileIndexDS, pszIndexLayerName); - CPLPopErrorHandler(); - } - } - else - { - printf("Creating new index file...\n"); - if (pszDriverName == nullptr) - { - std::vector aoDrivers = - GetOutputDriversFor(index_filename, GDAL_OF_VECTOR); - if (aoDrivers.empty()) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Cannot guess driver for %s", index_filename); - exit(10); - } - else - { - if (aoDrivers.size() > 1) - { - CPLError(CE_Warning, CPLE_AppDefined, - "Several drivers matching %s extension. Using %s", - CPLGetExtension(index_filename), - aoDrivers[0].c_str()); - } - osFormat = aoDrivers[0]; - } - } - else - { - osFormat = pszDriverName; - } - if (!EQUAL(osFormat, "ESRI Shapefile")) - nMaxFieldSize = 0; - - GDALDriverH hDriver = GDALGetDriverByName(osFormat.c_str()); - if (hDriver == nullptr) - { - printf("%s driver not available.\n", osFormat.c_str()); - exit(1); - } - - hTileIndexDS = - GDALCreate(hDriver, index_filename, 0, 0, 0, GDT_Unknown, nullptr); - } - - if (hTileIndexDS != nullptr && hLayer == nullptr) - { - OGRSpatialReferenceH hSpatialRef = nullptr; - char *pszLayerName = nullptr; - if (pszIndexLayerName == nullptr) - { - VSIStatBuf sStat; - if (EQUAL(osFormat, "ESRI Shapefile") || - VSIStat(index_filename, &sStat) == 0) - { - pszLayerName = CPLStrdup(CPLGetBasename(index_filename)); - } - else - { - printf("-lyr_name must be specified.\n"); - exit(1); - } - } - else - { - pszLayerName = CPLStrdup(pszIndexLayerName); - } - - /* get spatial reference for output file from target SRS (if set) */ - /* or from first input file */ - if (bSetTargetSRS) - { - hSpatialRef = OSRClone(hTargetSRS); - } - else - { - GDALDatasetH hDS = GDALOpen(argv[iArg], GA_ReadOnly); - if (hDS) - { - const char *pszWKT = GDALGetProjectionRef(hDS); - if (pszWKT != nullptr && pszWKT[0] != '\0') - { - hSpatialRef = OSRNewSpatialReference(pszWKT); - OSRSetAxisMappingStrategy(hSpatialRef, - OAMS_TRADITIONAL_GIS_ORDER); - } - GDALClose(hDS); - } - } - - hLayer = GDALDatasetCreateLayer(hTileIndexDS, pszLayerName, hSpatialRef, - wkbPolygon, nullptr); - CPLFree(pszLayerName); - if (hSpatialRef) - OSRRelease(hSpatialRef); - - if (hLayer) - { - OGRFieldDefnH hFieldDefn = OGR_Fld_Create(tile_index, OFTString); - if (nMaxFieldSize) - OGR_Fld_SetWidth(hFieldDefn, static_cast(nMaxFieldSize)); - OGR_L_CreateField(hLayer, hFieldDefn, TRUE); - OGR_Fld_Destroy(hFieldDefn); - if (pszSrcSRSName != nullptr) - { - hFieldDefn = OGR_Fld_Create(pszSrcSRSName, OFTString); - if (nMaxFieldSize) - OGR_Fld_SetWidth(hFieldDefn, - static_cast(nMaxFieldSize)); - OGR_L_CreateField(hLayer, hFieldDefn, TRUE); - OGR_Fld_Destroy(hFieldDefn); - } - } - } - - if (hTileIndexDS == nullptr || hLayer == nullptr) - { - fprintf(stderr, "Unable to open/create shapefile `%s'.\n", - index_filename); - exit(2); - } - - OGRFeatureDefnH hFDefn = OGR_L_GetLayerDefn(hLayer); - - const int ti_field = OGR_FD_GetFieldIndex(hFDefn, tile_index); - if (ti_field < 0) - { - fprintf(stderr, "Unable to find field `%s' in file `%s'.\n", tile_index, - index_filename); - exit(2); - } - - if (pszSrcSRSName != nullptr) - i_SrcSRSName = OGR_FD_GetFieldIndex(hFDefn, pszSrcSRSName); - - // Load in memory existing file names in SHP. - int nExistingFiles = static_cast(OGR_L_GetFeatureCount(hLayer, FALSE)); - if (nExistingFiles < 0) - nExistingFiles = 0; - - char **existingFilesTab = nullptr; - bool alreadyExistingProjectionRefValid = false; - char *alreadyExistingProjectionRef = nullptr; - if (nExistingFiles > 0) - { - OGRFeatureH hFeature = nullptr; - existingFilesTab = - static_cast(CPLMalloc(nExistingFiles * sizeof(char *))); - for (int i = 0; i < nExistingFiles; i++) - { - hFeature = OGR_L_GetNextFeature(hLayer); - existingFilesTab[i] = - CPLStrdup(OGR_F_GetFieldAsString(hFeature, ti_field)); - if (i == 0) - { - GDALDatasetH hDS = GDALOpen(existingFilesTab[i], GA_ReadOnly); - if (hDS) - { - alreadyExistingProjectionRefValid = true; - alreadyExistingProjectionRef = - CPLStrdup(GDALGetProjectionRef(hDS)); - GDALClose(hDS); - } - } - OGR_F_Destroy(hFeature); - } - } - - if (write_absolute_path) - { - current_path = CPLGetCurrentDir(); - if (current_path == nullptr) - { - fprintf(stderr, - "This system does not support the CPLGetCurrentDir call. " - "The option -write_absolute_path will have no effect\n"); - write_absolute_path = FALSE; - } - } - - int nRetCode = 0; - - /* -------------------------------------------------------------------- */ - /* loop over GDAL files, processing. */ - /* -------------------------------------------------------------------- */ - for (; nRetCode == 0 && iArg < argc; iArg++) - { - char *fileNameToWrite = nullptr; - VSIStatBuf sStatBuf; - - // Make sure it is a file before building absolute path name. - if (write_absolute_path && CPLIsFilenameRelative(argv[iArg]) && - VSIStat(argv[iArg], &sStatBuf) == 0) - { - fileNameToWrite = - CPLStrdup(CPLProjectRelativeFilename(current_path, argv[iArg])); - } - else - { - fileNameToWrite = CPLStrdup(argv[iArg]); - } - - // Checks that file is not already in tileindex. - { - int i = 0; // Used after for. - for (; i < nExistingFiles; i++) - { - if (EQUAL(fileNameToWrite, existingFilesTab[i])) - { - fprintf(stderr, - "File %s is already in tileindex. Skipping it.\n", - fileNameToWrite); - break; - } - } - if (i != nExistingFiles) - { - CPLFree(fileNameToWrite); - continue; - } - } - - GDALDatasetH hDS = GDALOpen(argv[iArg], GA_ReadOnly); - if (hDS == nullptr) - { - fprintf(stderr, "Unable to open %s, skipping.\n", argv[iArg]); - CPLFree(fileNameToWrite); - continue; - } - - double adfGeoTransform[6] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; - GDALGetGeoTransform(hDS, adfGeoTransform); - if (adfGeoTransform[0] == 0.0 && adfGeoTransform[1] == 1.0 && - adfGeoTransform[3] == 0.0 && std::abs(adfGeoTransform[5]) == 1.0) - { - fprintf(stderr, - "It appears no georeferencing is available for\n" - "`%s', skipping.\n", - argv[iArg]); - GDALClose(hDS); - CPLFree(fileNameToWrite); - continue; - } - - const char *projectionRef = GDALGetProjectionRef(hDS); - - // If not set target srs, test that the current file uses same - // projection as others. - if (!bSetTargetSRS) - { - if (alreadyExistingProjectionRefValid) - { - int projectionRefNotNull, alreadyExistingProjectionRefNotNull; - projectionRefNotNull = projectionRef && projectionRef[0]; - alreadyExistingProjectionRefNotNull = - alreadyExistingProjectionRef && - alreadyExistingProjectionRef[0]; - if ((projectionRefNotNull && - alreadyExistingProjectionRefNotNull && - EQUAL(projectionRef, alreadyExistingProjectionRef) == 0) || - (projectionRefNotNull != - alreadyExistingProjectionRefNotNull)) - { - fprintf( - stderr, - "Warning : %s is not using the same projection system " - "as other files in the tileindex.\n" - "This may cause problems when using it in MapServer " - "for example.\n" - "Use -t_srs option to set target projection system " - "(not supported by MapServer).\n" - "%s\n", - argv[iArg], - skip_different_projection ? "Skipping this file." : ""); - if (skip_different_projection) - { - CPLFree(fileNameToWrite); - GDALClose(hDS); - continue; - } - } - } - else - { - alreadyExistingProjectionRefValid = true; - alreadyExistingProjectionRef = CPLStrdup(projectionRef); - } - } - - const int nXSize = GDALGetRasterXSize(hDS); - const int nYSize = GDALGetRasterYSize(hDS); - - double adfX[5] = {0.0, 0.0, 0.0, 0.0, 0.0}; - double adfY[5] = {0.0, 0.0, 0.0, 0.0, 0.0}; - adfX[0] = adfGeoTransform[0] + 0 * adfGeoTransform[1] + - 0 * adfGeoTransform[2]; - adfY[0] = adfGeoTransform[3] + 0 * adfGeoTransform[4] + - 0 * adfGeoTransform[5]; - - adfX[1] = adfGeoTransform[0] + nXSize * adfGeoTransform[1] + - 0 * adfGeoTransform[2]; - adfY[1] = adfGeoTransform[3] + nXSize * adfGeoTransform[4] + - 0 * adfGeoTransform[5]; - - adfX[2] = adfGeoTransform[0] + nXSize * adfGeoTransform[1] + - nYSize * adfGeoTransform[2]; - adfY[2] = adfGeoTransform[3] + nXSize * adfGeoTransform[4] + - nYSize * adfGeoTransform[5]; - - adfX[3] = adfGeoTransform[0] + 0 * adfGeoTransform[1] + - nYSize * adfGeoTransform[2]; - adfY[3] = adfGeoTransform[3] + 0 * adfGeoTransform[4] + - nYSize * adfGeoTransform[5]; - - adfX[4] = adfGeoTransform[0] + 0 * adfGeoTransform[1] + - 0 * adfGeoTransform[2]; - adfY[4] = adfGeoTransform[3] + 0 * adfGeoTransform[4] + - 0 * adfGeoTransform[5]; - - OGRSpatialReferenceH hSourceSRS = nullptr; - if ((bSetTargetSRS || i_SrcSRSName >= 0) && projectionRef != nullptr && - projectionRef[0] != '\0') - { - hSourceSRS = OSRNewSpatialReference(projectionRef); - OSRSetAxisMappingStrategy(hSourceSRS, OAMS_TRADITIONAL_GIS_ORDER); - } - - // If set target srs, do the forward transformation of all points. - if (bSetTargetSRS && projectionRef != nullptr && - projectionRef[0] != '\0') - { - OGRCoordinateTransformationH hCT = nullptr; - if (hSourceSRS && !OSRIsSame(hSourceSRS, hTargetSRS)) - { - hCT = OCTNewCoordinateTransformation(hSourceSRS, hTargetSRS); - if (hCT == nullptr || - !OCTTransform(hCT, 5, adfX, adfY, nullptr)) - { - fprintf(stderr, - "Warning : unable to transform points from source " - "SRS `%s' to target SRS `%s'\n" - "for file `%s' - file skipped\n", - projectionRef, pszTargetSRS, fileNameToWrite); - if (hCT) - OCTDestroyCoordinateTransformation(hCT); - OSRDestroySpatialReference(hSourceSRS); - continue; - } - OCTDestroyCoordinateTransformation(hCT); - } - } - - OGRFeatureH hFeature = OGR_F_Create(OGR_L_GetLayerDefn(hLayer)); - OGR_F_SetFieldString(hFeature, ti_field, fileNameToWrite); - - if (i_SrcSRSName >= 0 && hSourceSRS != nullptr) - { - const char *pszAuthorityCode = - OSRGetAuthorityCode(hSourceSRS, nullptr); - const char *pszAuthorityName = - OSRGetAuthorityName(hSourceSRS, nullptr); - if (eSrcSRSFormat == FORMAT_AUTO) - { - if (pszAuthorityName != nullptr && pszAuthorityCode != nullptr) - { - OGR_F_SetFieldString(hFeature, i_SrcSRSName, - CPLSPrintf("%s:%s", pszAuthorityName, - pszAuthorityCode)); - } - else if (nMaxFieldSize == 0 || - strlen(projectionRef) <= nMaxFieldSize) - { - OGR_F_SetFieldString(hFeature, i_SrcSRSName, projectionRef); - } - else - { - char *pszProj4 = nullptr; - if (OSRExportToProj4(hSourceSRS, &pszProj4) == OGRERR_NONE) - { - OGR_F_SetFieldString(hFeature, i_SrcSRSName, pszProj4); - CPLFree(pszProj4); - } - else - { - OGR_F_SetFieldString(hFeature, i_SrcSRSName, - projectionRef); - } - } - } - else if (eSrcSRSFormat == FORMAT_WKT) - { - if (nMaxFieldSize == 0 || - strlen(projectionRef) <= nMaxFieldSize) - { - OGR_F_SetFieldString(hFeature, i_SrcSRSName, projectionRef); - } - else - { - fprintf(stderr, - "Cannot write WKT for file %s as it is too long!\n", - fileNameToWrite); - } - } - else if (eSrcSRSFormat == FORMAT_PROJ) - { - char *pszProj4 = nullptr; - if (OSRExportToProj4(hSourceSRS, &pszProj4) == OGRERR_NONE) - { - OGR_F_SetFieldString(hFeature, i_SrcSRSName, pszProj4); - CPLFree(pszProj4); - } - } - else if (eSrcSRSFormat == FORMAT_EPSG) - { - if (pszAuthorityName != nullptr && pszAuthorityCode != nullptr) - OGR_F_SetFieldString(hFeature, i_SrcSRSName, - CPLSPrintf("%s:%s", pszAuthorityName, - pszAuthorityCode)); - } - } - if (hSourceSRS) - OSRDestroySpatialReference(hSourceSRS); - - OGRGeometryH hPoly = OGR_G_CreateGeometry(wkbPolygon); - OGRGeometryH hRing = OGR_G_CreateGeometry(wkbLinearRing); - for (int k = 0; k < 5; k++) - OGR_G_SetPoint_2D(hRing, k, adfX[k], adfY[k]); - OGR_G_AddGeometryDirectly(hPoly, hRing); - OGR_F_SetGeometryDirectly(hFeature, hPoly); - - if (OGR_L_CreateFeature(hLayer, hFeature) != OGRERR_NONE) - { - printf("Failed to create feature in shapefile.\n"); - nRetCode = 1; - } - - OGR_F_Destroy(hFeature); - - CPLFree(fileNameToWrite); - - GDALClose(hDS); - } - - CPLFree(current_path); - - if (nExistingFiles) - { - for (int i = 0; i < nExistingFiles; i++) - { - CPLFree(existingFilesTab[i]); - } - CPLFree(existingFilesTab); - } - CPLFree(alreadyExistingProjectionRef); - - if (hTargetSRS) - OSRDestroySpatialReference(hTargetSRS); - - if (GDALClose(hTileIndexDS) != CE_None) - nRetCode = 1; - - GDALDestroyDriverManager(); - OGRCleanupAll(); - CSLDestroy(argv); - - exit(nRetCode); -} -MAIN_END diff --git a/apps/gdaltindex_bin.cpp b/apps/gdaltindex_bin.cpp new file mode 100644 index 000000000000..1e8e21af2058 --- /dev/null +++ b/apps/gdaltindex_bin.cpp @@ -0,0 +1,158 @@ +/****************************************************************************** + * + * Project: MapServer + * Purpose: Commandline App to build tile index for raster files. + * Author: Frank Warmerdam, warmerdam@pobox.com + * + ****************************************************************************** + * Copyright (c) 2001, Frank Warmerdam, DM Solutions Group Inc + * Copyright (c) 2007-2013, Even Rouault + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#include "cpl_string.h" +#include "cpl_error.h" +#include "commonutils.h" +#include "gdal_version.h" +#include "gdal_utils_priv.h" + +/************************************************************************/ +/* Usage() */ +/************************************************************************/ + +static void Usage(bool bIsError, const char *pszErrorMsg) + +{ + fprintf( + bIsError ? stderr : stdout, "%s", + "Usage: gdaltindex [--help] [--help-general]\n" + " [-f ] [-tileindex ] " + "[-write_absolute_path] \n" + " [-skip_different_projection] [-t_srs ]\n" + " [-src_srs_name field_name] [-src_srs_format " + "{AUTO|WKT|EPSG|PROJ}]\n" + " [-lyr_name ] " + "[]...\n" + "\n" + "e.g.\n" + " % gdaltindex doq_index.shp doq/*.tif\n" + "\n" + "NOTES:\n" + " o The index will be created if it doesn't already exist.\n" + " o The default tile index field is 'location'.\n" + " o Raster filenames will be put in the file exactly as they are " + "specified\n" + " on the commandline unless the option -write_absolute_path is " + "used.\n" + " o If -skip_different_projection is specified, only files with same " + "projection ref\n" + " as files already inserted in the tileindex will be inserted " + "(unless t_srs is specified).\n" + " o If -t_srs is specified, geometries of input files will be " + "transformed to the desired\n" + " target coordinate reference system.\n" + " Note that using this option generates files that are NOT " + "compatible with MapServer < 6.4.\n" + " o Simple rectangular polygons are generated in the same coordinate " + "reference system\n" + " as the rasters, or in target reference system if the -t_srs " + "option is used.\n"); + + if (pszErrorMsg != nullptr) + fprintf(stderr, "\nFAILURE: %s\n", pszErrorMsg); + + exit(bIsError ? 1 : 0); +} + +/************************************************************************/ +/* main() */ +/************************************************************************/ + +MAIN_START(argc, argv) +{ + EarlySetConfigOptions(argc, argv); + + /* -------------------------------------------------------------------- */ + /* Register standard GDAL drivers, and process generic GDAL */ + /* command options. */ + /* -------------------------------------------------------------------- */ + GDALAllRegister(); + argc = GDALGeneralCmdLineProcessor(argc, &argv, 0); + if (argc < 1) + exit(-argc); + + for (int i = 0; argv != nullptr && argv[i] != nullptr; i++) + { + if (EQUAL(argv[i], "--utility_version")) + { + printf("%s was compiled against GDAL %s and is running against " + "GDAL %s\n", + argv[0], GDAL_RELEASE_NAME, GDALVersionInfo("RELEASE_NAME")); + CSLDestroy(argv); + return 0; + } + else if (EQUAL(argv[i], "--help")) + { + Usage(false, nullptr); + } + } + + auto psOptionsForBinary = std::make_unique(); + /* coverity[tainted_data] */ + GDALTileIndexOptions *psOptions = + GDALTileIndexOptionsNew(argv + 1, psOptionsForBinary.get()); + CSLDestroy(argv); + + if (psOptions == nullptr) + { + Usage(true, nullptr); + } + + if (!psOptionsForBinary->bDestSpecified) + Usage(true, "No index filename specified."); + + int bUsageError = FALSE; + GDALDatasetH hOutDS = GDALTileIndex(psOptionsForBinary->osDest.c_str(), + psOptionsForBinary->aosSrcFiles.size(), + psOptionsForBinary->aosSrcFiles.List(), + psOptions, &bUsageError); + if (bUsageError) + Usage(true, nullptr); + int nRetCode = (hOutDS) ? 0 : 1; + + GDALTileIndexOptionsFree(psOptions); + + CPLErrorReset(); + // The flush to disk is only done at that stage, so check if any error has + // happened + if (GDALClose(hOutDS) != CE_None) + nRetCode = 1; + if (CPLGetLastErrorType() != CE_None) + nRetCode = 1; + + GDALDumpOpenDatasets(stderr); + + GDALDestroyDriverManager(); + + OGRCleanupAll(); + + return nRetCode; +} +MAIN_END diff --git a/apps/gdaltindex_lib.cpp b/apps/gdaltindex_lib.cpp new file mode 100644 index 000000000000..df8f37b7c38b --- /dev/null +++ b/apps/gdaltindex_lib.cpp @@ -0,0 +1,788 @@ +/****************************************************************************** + * + * Project: MapServer + * Purpose: Commandline App to build tile index for raster files. + * Author: Frank Warmerdam, warmerdam@pobox.com + * + ****************************************************************************** + * Copyright (c) 2001, Frank Warmerdam, DM Solutions Group Inc + * Copyright (c) 2007-2023, Even Rouault + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#include "cpl_port.h" +#include "cpl_conv.h" +#include "cpl_string.h" +#include "gdal_utils.h" +#include "gdal_priv.h" +#include "gdal_utils_priv.h" +#include "ogr_api.h" +#include "ogrsf_frmts.h" +#include "ogr_spatialref.h" +#include "commonutils.h" + +#include +#include + +typedef enum +{ + FORMAT_AUTO, + FORMAT_WKT, + FORMAT_EPSG, + FORMAT_PROJ +} SrcSRSFormat; + +/************************************************************************/ +/* GDALTileIndexOptions */ +/************************************************************************/ + +struct GDALTileIndexOptions +{ + std::string osFormat{}; + std::string osIndexLayerName{}; + std::string osLocationField = "location"; + std::string osTargetSRS{}; + bool bWriteAbsolutePath = false; + bool bSkipDifferentProjection = false; + std::string osSrcSRSFieldName{}; + SrcSRSFormat eSrcSRSFormat = FORMAT_AUTO; +}; + +/************************************************************************/ +/* GDALTileIndex() */ +/************************************************************************/ + +/* clang-format off */ +/** + * Build a tile index from a list of datasets. + * + * This is the equivalent of the + * gdaltindex utility. + * + * GDALTileIndexOptions* must be allocated and freed with + * GDALTileIndexOptionsNew() and GDALTileIndexOptionsFree() respectively. + * + * @param pszDest the destination dataset path. + * @param nSrcCount the number of input datasets. + * @param papszSrcDSNames the list of input dataset names + * @param psOptionsIn the options struct returned by GDALTileIndexOptionsNew() or + * NULL. + * @param pbUsageError pointer to a integer output variable to store if any + * usage error has occurred. + * @return the output dataset (new dataset that must be closed using + * GDALClose()) or NULL in case of error. + * + * @since GDAL3.9 + */ +/* clang-format on */ + +GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, + const char *const *papszSrcDSNames, + const GDALTileIndexOptions *psOptionsIn, + int *pbUsageError) +{ + if (nSrcCount == 0) + { + CPLError(CE_Failure, CPLE_AppDefined, "No input dataset specified."); + + if (pbUsageError) + *pbUsageError = TRUE; + return nullptr; + } + + auto psOptions = psOptionsIn + ? std::make_unique(*psOptionsIn) + : std::make_unique(); + + /* -------------------------------------------------------------------- */ + /* Create and validate target SRS if given. */ + /* -------------------------------------------------------------------- */ + OGRSpatialReference oTargetSRS; + if (!psOptions->osTargetSRS.empty()) + { + if (psOptions->bSkipDifferentProjection) + { + CPLError(CE_Warning, CPLE_AppDefined, + "-skip_different_projections does not apply " + "when -t_srs is requested."); + } + oTargetSRS.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); + // coverity[tainted_data] + oTargetSRS.SetFromUserInput(psOptions->osTargetSRS.c_str()); + } + + /* -------------------------------------------------------------------- */ + /* Open or create the target datasource */ + /* -------------------------------------------------------------------- */ + auto poTileIndexDS = std::unique_ptr(GDALDataset::Open( + pszDest, GDAL_OF_VECTOR | GDAL_OF_UPDATE, nullptr, nullptr, nullptr)); + OGRLayer *poLayer = nullptr; + std::string osFormat; + int nMaxFieldSize = 254; + bool bExistingLayer = false; + + if (poTileIndexDS != nullptr) + { + auto poDriver = poTileIndexDS->GetDriver(); + if (poDriver) + osFormat = poDriver->GetDescription(); + + if (poTileIndexDS->GetLayerCount() == 1) + { + poLayer = poTileIndexDS->GetLayer(0); + } + else + { + if (psOptions->osIndexLayerName.empty()) + { + CPLError(CE_Failure, CPLE_AppDefined, + "-lyr_name must be specified."); + if (pbUsageError) + *pbUsageError = true; + return nullptr; + } + CPLPushErrorHandler(CPLQuietErrorHandler); + poLayer = poTileIndexDS->GetLayerByName( + psOptions->osIndexLayerName.c_str()); + CPLPopErrorHandler(); + } + } + else + { + if (psOptions->osFormat.empty()) + { + std::vector aoDrivers = + GetOutputDriversFor(pszDest, GDAL_OF_VECTOR); + if (aoDrivers.empty()) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Cannot guess driver for %s", pszDest); + return nullptr; + } + else + { + if (aoDrivers.size() > 1) + { + CPLError(CE_Warning, CPLE_AppDefined, + "Several drivers matching %s extension. Using %s", + CPLGetExtension(pszDest), aoDrivers[0].c_str()); + } + osFormat = aoDrivers[0]; + } + } + else + { + osFormat = psOptions->osFormat; + } + if (!EQUAL(osFormat.c_str(), "ESRI Shapefile")) + nMaxFieldSize = 0; + + auto poDriver = + GetGDALDriverManager()->GetDriverByName(osFormat.c_str()); + if (poDriver == nullptr) + { + CPLError(CE_Warning, CPLE_AppDefined, "%s driver not available.", + osFormat.c_str()); + return nullptr; + } + + poTileIndexDS.reset( + poDriver->Create(pszDest, 0, 0, 0, GDT_Unknown, nullptr)); + if (!poTileIndexDS) + return nullptr; + } + + if (poLayer) + { + bExistingLayer = true; + } + else + { + std::string osLayerName; + if (psOptions->osIndexLayerName.empty()) + { + VSIStatBuf sStat; + if (EQUAL(osFormat.c_str(), "ESRI Shapefile") || + VSIStat(pszDest, &sStat) == 0) + { + osLayerName = CPLGetBasename(pszDest); + } + else + { + CPLError(CE_Failure, CPLE_AppDefined, + "-lyr_name must be specified."); + if (pbUsageError) + *pbUsageError = true; + return nullptr; + } + } + else + { + osLayerName = psOptions->osIndexLayerName; + } + + /* get spatial reference for output file from target SRS (if set) */ + /* or from first input file */ + OGRSpatialReference oSRS; + if (!oTargetSRS.IsEmpty()) + { + oSRS = oTargetSRS; + } + else + { + auto poSrcDS = std::unique_ptr(GDALDataset::Open( + papszSrcDSNames[0], GDAL_OF_RASTER | GDAL_OF_VERBOSE_ERROR, + nullptr, nullptr, nullptr)); + if (!poSrcDS) + return nullptr; + + auto poSrcSRS = poSrcDS->GetSpatialRef(); + if (poSrcSRS) + oSRS = *poSrcSRS; + } + + poLayer = poTileIndexDS->CreateLayer(osLayerName.c_str(), + oSRS.IsEmpty() ? nullptr : &oSRS, + wkbPolygon, nullptr); + if (!poLayer) + return nullptr; + + OGRFieldDefn oLocationField(psOptions->osLocationField.c_str(), + OFTString); + oLocationField.SetWidth(nMaxFieldSize); + if (poLayer->CreateField(&oLocationField) != OGRERR_NONE) + return nullptr; + + if (!psOptions->osSrcSRSFieldName.empty()) + { + OGRFieldDefn oSrcSRSField(psOptions->osSrcSRSFieldName.c_str(), + OFTString); + oSrcSRSField.SetWidth(nMaxFieldSize); + if (poLayer->CreateField(&oSrcSRSField) != OGRERR_NONE) + return nullptr; + } + } + + auto poLayerDefn = poLayer->GetLayerDefn(); + const int ti_field = + poLayerDefn->GetFieldIndex(psOptions->osLocationField.c_str()); + if (ti_field < 0) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Unable to find field `%s' in file `%s'.", + psOptions->osLocationField.c_str(), pszDest); + return nullptr; + } + + int i_SrcSRSName = -1; + if (!psOptions->osSrcSRSFieldName.empty()) + { + i_SrcSRSName = + poLayerDefn->GetFieldIndex(psOptions->osSrcSRSFieldName.c_str()); + if (i_SrcSRSName < 0) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Unable to find field `%s' in file `%s'.", + psOptions->osSrcSRSFieldName.c_str(), pszDest); + return nullptr; + } + } + + // Load in memory existing file names in tile index. + std::set oSetExistingFiles; + OGRSpatialReference oAlreadyExistingSRS; + if (bExistingLayer) + { + for (auto &&poFeature : poLayer) + { + if (poFeature->IsFieldSetAndNotNull(ti_field)) + { + if (oSetExistingFiles.empty()) + { + auto poSrcDS = + std::unique_ptr(GDALDataset::Open( + poFeature->GetFieldAsString(ti_field), + GDAL_OF_RASTER, nullptr, nullptr, nullptr)); + if (poSrcDS) + { + auto poSrcSRS = poSrcDS->GetSpatialRef(); + if (poSrcSRS) + oAlreadyExistingSRS = *poSrcSRS; + } + } + oSetExistingFiles.insert(poFeature->GetFieldAsString(ti_field)); + } + } + } + + std::string osCurrentPath; + if (psOptions->bWriteAbsolutePath) + { + char *pszCurrentPath = CPLGetCurrentDir(); + if (pszCurrentPath) + { + osCurrentPath = pszCurrentPath; + } + else + { + CPLError(CE_Warning, CPLE_AppDefined, + "This system does not support the CPLGetCurrentDir call. " + "The option -bWriteAbsolutePath will have no effect."); + } + CPLFree(pszCurrentPath); + } + + /* -------------------------------------------------------------------- */ + /* loop over GDAL files, processing. */ + /* -------------------------------------------------------------------- */ + for (int iSrc = 0; iSrc < nSrcCount; ++iSrc) + { + std::string osFileNameToWrite; + VSIStatBuf sStatBuf; + + // Make sure it is a file before building absolute path name. + if (!osCurrentPath.empty() && + CPLIsFilenameRelative(papszSrcDSNames[iSrc]) && + VSIStat(papszSrcDSNames[iSrc], &sStatBuf) == 0) + { + osFileNameToWrite = CPLProjectRelativeFilename( + osCurrentPath.c_str(), papszSrcDSNames[iSrc]); + } + else + { + osFileNameToWrite = papszSrcDSNames[iSrc]; + } + + // Checks that file is not already in tileindex. + if (oSetExistingFiles.find(osFileNameToWrite) != + oSetExistingFiles.end()) + { + CPLError(CE_Warning, CPLE_AppDefined, + "File %s is already in tileindex. Skipping it.", + osFileNameToWrite.c_str()); + continue; + } + + auto poSrcDS = std::unique_ptr(GDALDataset::Open( + papszSrcDSNames[iSrc], GDAL_OF_RASTER | GDAL_OF_VERBOSE_ERROR, + nullptr, nullptr, nullptr)); + if (poSrcDS == nullptr) + { + CPLError(CE_Warning, CPLE_AppDefined, + "Unable to open %s, skipping.", papszSrcDSNames[iSrc]); + continue; + } + + double adfGeoTransform[6] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; + if (poSrcDS->GetGeoTransform(adfGeoTransform) != CE_None) + { + CPLError(CE_Warning, CPLE_AppDefined, + "It appears no georeferencing is available for\n" + "`%s', skipping.", + papszSrcDSNames[iSrc]); + continue; + } + + auto poSrcSRS = poSrcDS->GetSpatialRef(); + // If not set target srs, test that the current file uses same + // projection as others. + if (oTargetSRS.IsEmpty()) + { + if (!oAlreadyExistingSRS.IsEmpty()) + { + if (poSrcSRS == nullptr || + !poSrcSRS->IsSame(&oAlreadyExistingSRS)) + { + CPLError( + CE_Warning, CPLE_AppDefined, + "%s is not using the same projection system " + "as other files in the tileindex.\n" + "This may cause problems when using it in MapServer " + "for example.\n" + "Use -t_srs option to set target projection system " + "(not supported by MapServer). %s", + papszSrcDSNames[iSrc], + psOptions->bSkipDifferentProjection + ? "Skipping this file." + : ""); + if (psOptions->bSkipDifferentProjection) + { + continue; + } + } + } + else + { + if (poSrcSRS) + oAlreadyExistingSRS = *poSrcSRS; + } + } + + const int nXSize = poSrcDS->GetRasterXSize(); + const int nYSize = poSrcDS->GetRasterYSize(); + + double adfX[5] = {0.0, 0.0, 0.0, 0.0, 0.0}; + double adfY[5] = {0.0, 0.0, 0.0, 0.0, 0.0}; + adfX[0] = adfGeoTransform[0] + 0 * adfGeoTransform[1] + + 0 * adfGeoTransform[2]; + adfY[0] = adfGeoTransform[3] + 0 * adfGeoTransform[4] + + 0 * adfGeoTransform[5]; + + adfX[1] = adfGeoTransform[0] + nXSize * adfGeoTransform[1] + + 0 * adfGeoTransform[2]; + adfY[1] = adfGeoTransform[3] + nXSize * adfGeoTransform[4] + + 0 * adfGeoTransform[5]; + + adfX[2] = adfGeoTransform[0] + nXSize * adfGeoTransform[1] + + nYSize * adfGeoTransform[2]; + adfY[2] = adfGeoTransform[3] + nXSize * adfGeoTransform[4] + + nYSize * adfGeoTransform[5]; + + adfX[3] = adfGeoTransform[0] + 0 * adfGeoTransform[1] + + nYSize * adfGeoTransform[2]; + adfY[3] = adfGeoTransform[3] + 0 * adfGeoTransform[4] + + nYSize * adfGeoTransform[5]; + + adfX[4] = adfGeoTransform[0] + 0 * adfGeoTransform[1] + + 0 * adfGeoTransform[2]; + adfY[4] = adfGeoTransform[3] + 0 * adfGeoTransform[4] + + 0 * adfGeoTransform[5]; + + // If set target srs, do the forward transformation of all points. + if (!oTargetSRS.IsEmpty() && poSrcSRS) + { + if (!poSrcSRS->IsSame(&oTargetSRS)) + { + auto poCT = std::unique_ptr( + OGRCreateCoordinateTransformation(poSrcSRS, &oTargetSRS)); + if (!poCT || !poCT->Transform(5, adfX, adfY, nullptr)) + { + CPLError(CE_Warning, CPLE_AppDefined, + "unable to transform points from source " + "SRS `%s' to target SRS `%s' for file `%s' - file " + "skipped", + poSrcDS->GetProjectionRef(), + psOptions->osTargetSRS.c_str(), + osFileNameToWrite.c_str()); + continue; + } + } + } + + auto poFeature = std::make_unique(poLayerDefn); + poFeature->SetField(ti_field, osFileNameToWrite.c_str()); + + if (i_SrcSRSName >= 0 && poSrcSRS) + { + const char *pszAuthorityCode = poSrcSRS->GetAuthorityCode(nullptr); + const char *pszAuthorityName = poSrcSRS->GetAuthorityName(nullptr); + if (psOptions->eSrcSRSFormat == FORMAT_AUTO) + { + if (pszAuthorityName != nullptr && pszAuthorityCode != nullptr) + { + poFeature->SetField(i_SrcSRSName, + CPLSPrintf("%s:%s", pszAuthorityName, + pszAuthorityCode)); + } + else if (nMaxFieldSize == 0 || + strlen(poSrcDS->GetProjectionRef()) <= + static_cast(nMaxFieldSize)) + { + poFeature->SetField(i_SrcSRSName, + poSrcDS->GetProjectionRef()); + } + else + { + char *pszProj4 = nullptr; + if (poSrcSRS->exportToProj4(&pszProj4) == OGRERR_NONE) + { + poFeature->SetField(i_SrcSRSName, pszProj4); + CPLFree(pszProj4); + } + else + { + poFeature->SetField(i_SrcSRSName, + poSrcDS->GetProjectionRef()); + } + } + } + else if (psOptions->eSrcSRSFormat == FORMAT_WKT) + { + if (nMaxFieldSize == 0 || + strlen(poSrcDS->GetProjectionRef()) <= + static_cast(nMaxFieldSize)) + { + poFeature->SetField(i_SrcSRSName, + poSrcDS->GetProjectionRef()); + } + else + { + CPLError(CE_Warning, CPLE_AppDefined, + "Cannot write WKT for file %s as it is too long!", + osFileNameToWrite.c_str()); + } + } + else if (psOptions->eSrcSRSFormat == FORMAT_PROJ) + { + char *pszProj4 = nullptr; + if (poSrcSRS->exportToProj4(&pszProj4) == OGRERR_NONE) + { + poFeature->SetField(i_SrcSRSName, pszProj4); + CPLFree(pszProj4); + } + } + else if (psOptions->eSrcSRSFormat == FORMAT_EPSG) + { + if (pszAuthorityName != nullptr && pszAuthorityCode != nullptr) + poFeature->SetField(i_SrcSRSName, + CPLSPrintf("%s:%s", pszAuthorityName, + pszAuthorityCode)); + } + } + + auto poPoly = std::make_unique(); + auto poRing = std::make_unique(); + for (int k = 0; k < 5; k++) + poRing->addPoint(adfX[k], adfY[k]); + poPoly->addRingDirectly(poRing.release()); + poFeature->SetGeometryDirectly(poPoly.release()); + + if (poLayer->CreateFeature(poFeature.get()) != OGRERR_NONE) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Failed to create feature in tile index."); + return nullptr; + } + } + + return GDALDataset::ToHandle(poTileIndexDS.release()); +} + +/************************************************************************/ +/* CHECK_HAS_ENOUGH_ADDITIONAL_ARGS() */ +/************************************************************************/ + +#ifndef CheckHasEnoughAdditionalArgs_defined +#define CheckHasEnoughAdditionalArgs_defined +static bool CheckHasEnoughAdditionalArgs(CSLConstList papszArgv, int i, + int nExtraArg, int nArgc) +{ + if (i + nExtraArg >= nArgc) + { + CPLError(CE_Failure, CPLE_IllegalArg, + "%s option requires %d argument%s", papszArgv[i], nExtraArg, + nExtraArg == 1 ? "" : "s"); + return false; + } + return true; +} +#endif + +#define CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(nExtraArg) \ + if (!CheckHasEnoughAdditionalArgs(papszArgv, iArg, nExtraArg, argc)) \ + { \ + return nullptr; \ + } + +/************************************************************************/ +/* SanitizeSRS */ +/************************************************************************/ + +static char *SanitizeSRS(const char *pszUserInput) + +{ + OGRSpatialReferenceH hSRS; + char *pszResult = nullptr; + + CPLErrorReset(); + + hSRS = OSRNewSpatialReference(nullptr); + if (OSRSetFromUserInput(hSRS, pszUserInput) == OGRERR_NONE) + OSRExportToWkt(hSRS, &pszResult); + else + { + CPLError(CE_Failure, CPLE_AppDefined, "Translating SRS failed:\n%s", + pszUserInput); + } + + OSRDestroySpatialReference(hSRS); + + return pszResult; +} + +/************************************************************************/ +/* GDALTileIndexOptionsNew() */ +/************************************************************************/ + +/** + * Allocates a GDALTileIndexOptions struct. + * + * @param papszArgv NULL terminated list of options (potentially including + * filename and open options too), or NULL. The accepted options are the ones of + * the gdaltindex utility. + * @param psOptionsForBinary (output) may be NULL (and should generally be + * NULL), otherwise (gdaltindex_bin.cpp use case) must be allocated with + * GDALTileIndexOptionsForBinaryNew() prior to this function. Will be filled + * with potentially present filename, open options,... + * @return pointer to the allocated GDALTileIndexOptions struct. Must be freed + * with GDALTileIndexOptionsFree(). + * + * @since GDAL 3.9 + */ + +GDALTileIndexOptions * +GDALTileIndexOptionsNew(char **papszArgv, + GDALTileIndexOptionsForBinary *psOptionsForBinary) +{ + auto psOptions = std::make_unique(); + + bool bSrcSRSFormatSpecified = false; + + /* -------------------------------------------------------------------- */ + /* Parse arguments. */ + /* -------------------------------------------------------------------- */ + int argc = CSLCount(papszArgv); + for (int iArg = 0; papszArgv != nullptr && iArg < argc; iArg++) + { + if ((strcmp(papszArgv[iArg], "-f") == 0 || + strcmp(papszArgv[iArg], "-of") == 0)) + { + CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); + psOptions->osFormat = papszArgv[++iArg]; + } + else if (strcmp(papszArgv[iArg], "-lyr_name") == 0) + { + CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); + psOptions->osIndexLayerName = papszArgv[++iArg]; + } + else if (strcmp(papszArgv[iArg], "-tileindex") == 0) + { + CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); + psOptions->osLocationField = papszArgv[++iArg]; + } + else if (strcmp(papszArgv[iArg], "-t_srs") == 0) + { + CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); + char *pszSRS = SanitizeSRS(papszArgv[++iArg]); + if (pszSRS == nullptr) + { + return nullptr; + } + psOptions->osTargetSRS = pszSRS; + CPLFree(pszSRS); + } + else if (strcmp(papszArgv[iArg], "-write_absolute_path") == 0) + { + psOptions->bWriteAbsolutePath = true; + } + else if (strcmp(papszArgv[iArg], "-skip_different_projection") == 0) + { + psOptions->bSkipDifferentProjection = true; + } + else if (strcmp(papszArgv[iArg], "-src_srs_name") == 0) + { + CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); + psOptions->osSrcSRSFieldName = papszArgv[++iArg]; + } + else if (strcmp(papszArgv[iArg], "-src_srs_format") == 0) + { + CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); + const char *pszFormat = papszArgv[++iArg]; + bSrcSRSFormatSpecified = true; + if (EQUAL(pszFormat, "AUTO")) + psOptions->eSrcSRSFormat = FORMAT_AUTO; + else if (EQUAL(pszFormat, "WKT")) + psOptions->eSrcSRSFormat = FORMAT_WKT; + else if (EQUAL(pszFormat, "EPSG")) + psOptions->eSrcSRSFormat = FORMAT_EPSG; + else if (EQUAL(pszFormat, "PROJ")) + psOptions->eSrcSRSFormat = FORMAT_PROJ; + else + { + CPLError(CE_Failure, CPLE_IllegalArg, + "Unhandled value for -src_srs_format"); + return nullptr; + } + } + else if (EQUAL(papszArgv[iArg], "-q") || + EQUAL(papszArgv[iArg], "-quiet")) + { + if (psOptionsForBinary) + { + psOptionsForBinary->bQuiet = true; + } + } + else if (papszArgv[iArg][0] == '-') + { + CPLError(CE_Failure, CPLE_NotSupported, "Unknown option name '%s'", + papszArgv[iArg]); + return nullptr; + } + else + { + if (psOptionsForBinary) + { + if (!psOptionsForBinary->bDestSpecified) + { + psOptionsForBinary->bDestSpecified = true; + psOptionsForBinary->osDest = papszArgv[iArg]; + } + else + { + psOptionsForBinary->aosSrcFiles.AddString(papszArgv[iArg]); + } + } + else + { + CPLError(CE_Failure, CPLE_NotSupported, "Unexpected argument"); + return nullptr; + } + } + } + + if (bSrcSRSFormatSpecified && psOptions->osTargetSRS.empty()) + { + CPLError(CE_Failure, CPLE_NotSupported, + "-src_srs_name must be specified when -src_srs_format is " + "specified."); + return nullptr; + } + + return psOptions.release(); +} + +/************************************************************************/ +/* GDALTileIndexOptionsFree() */ +/************************************************************************/ + +/** + * Frees the GDALTileIndexOptions struct. + * + * @param psOptions the options struct for GDALTileIndex(). + * + * @since GDAL 3.9 + */ + +void GDALTileIndexOptionsFree(GDALTileIndexOptions *psOptions) +{ + delete psOptions; +} + +#undef CHECK_HAS_ENOUGH_ADDITIONAL_ARGS diff --git a/apps/gdalwarp_lib.cpp b/apps/gdalwarp_lib.cpp index a99712970c8f..df7e3c4d26c9 100644 --- a/apps/gdalwarp_lib.cpp +++ b/apps/gdalwarp_lib.cpp @@ -5269,7 +5269,8 @@ static bool IsValidSRS(const char *pszUserInput) /* GDALWarpAppOptionsNew() */ /************************************************************************/ -#ifndef CHECK_HAS_ENOUGH_ADDITIONAL_ARGS +#ifndef CheckHasEnoughAdditionalArgs_defined +#define CheckHasEnoughAdditionalArgs_defined static bool CheckHasEnoughAdditionalArgs(CSLConstList papszArgv, int i, int nExtraArg, int nArgc) { @@ -5282,13 +5283,13 @@ static bool CheckHasEnoughAdditionalArgs(CSLConstList papszArgv, int i, } return true; } +#endif #define CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(nExtraArg) \ if (!CheckHasEnoughAdditionalArgs(papszArgv, i, nExtraArg, nArgc)) \ { \ return nullptr; \ } -#endif /** * Allocates a GDALWarpAppOptions struct. @@ -5942,3 +5943,5 @@ void GDALWarpAppOptionsSetWarpOption(GDALWarpAppOptions *psOptions, { psOptions->aosWarpOptions.SetNameValue(pszKey, pszValue); } + +#undef CHECK_HAS_ENOUGH_ADDITIONAL_ARGS diff --git a/apps/ogr2ogr_lib.cpp b/apps/ogr2ogr_lib.cpp index 87c7945ba834..a8f8a6471478 100644 --- a/apps/ogr2ogr_lib.cpp +++ b/apps/ogr2ogr_lib.cpp @@ -6443,7 +6443,8 @@ LayerTranslator::GetSrcClipGeom(const OGRSpatialReference *poGeomSRS) /* CHECK_HAS_ENOUGH_ADDITIONAL_ARGS() */ /************************************************************************/ -#ifndef CHECK_HAS_ENOUGH_ADDITIONAL_ARGS +#ifndef CheckHasEnoughAdditionalArgs_defined +#define CheckHasEnoughAdditionalArgs_defined static bool CheckHasEnoughAdditionalArgs(CSLConstList papszArgv, int i, int nExtraArg, int nArgc) { @@ -6456,13 +6457,13 @@ static bool CheckHasEnoughAdditionalArgs(CSLConstList papszArgv, int i, } return true; } +#endif #define CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(nExtraArg) \ if (!CheckHasEnoughAdditionalArgs(papszArgv, i, nExtraArg, nArgc)) \ { \ return nullptr; \ } -#endif /************************************************************************/ /* GDALVectorTranslateOptionsNew() */ @@ -7369,3 +7370,5 @@ void GDALVectorTranslateOptionsSetProgress( if (pfnProgress == GDALTermProgress) psOptions->bQuiet = false; } + +#undef CHECK_HAS_ENOUGH_ADDITIONAL_ARGS diff --git a/autotest/utilities/test_gdaltindex.py b/autotest/utilities/test_gdaltindex.py index 3597cdfbf619..97fb4a3b5073 100755 --- a/autotest/utilities/test_gdaltindex.py +++ b/autotest/utilities/test_gdaltindex.py @@ -100,113 +100,6 @@ def four_tile_index(gdaltindex_path, four_tiles, tmp_path): return f"{tmp_path}/tileindex.shp" -def test_gdaltindex_1(gdaltindex_path, four_tile_index): - - ds = ogr.Open(four_tile_index) - assert ds.GetLayer(0).GetFeatureCount() == 4 - - tileindex_wkt = ds.GetLayer(0).GetSpatialRef().ExportToWkt() - assert "WGS_1984" in tileindex_wkt - - expected_wkts = [ - "POLYGON ((49 2,50 2,50 1,49 1,49 2))", - "POLYGON ((49 3,50 3,50 2,49 2,49 3))", - "POLYGON ((48 2,49 2,49 1,48 1,48 2))", - "POLYGON ((48 3,49 3,49 2,48 2,48 3))", - ] - - for i, feat in enumerate(ds.GetLayer(0)): - assert ( - feat.GetGeometryRef().ExportToWkt() == expected_wkts[i] - ), "i=%d, wkt=%s" % (i, feat.GetGeometryRef().ExportToWkt()) - - -############################################################################### -# Try adding the same rasters again - - -def test_gdaltindex_2(gdaltindex_path, four_tiles, four_tile_index, tmp_path): - - (_, ret_stderr) = gdaltest.runexternal_out_and_err( - f"{gdaltindex_path} {four_tile_index} {four_tiles[0]} {four_tiles[1]} {four_tiles[2]} {four_tiles[3]}" - ) - - assert "gdaltindex1.tif is already in tileindex. Skipping it." in ret_stderr - assert "gdaltindex2.tif is already in tileindex. Skipping it." in ret_stderr - assert "gdaltindex3.tif is already in tileindex. Skipping it." in ret_stderr - assert "gdaltindex4.tif is already in tileindex. Skipping it." in ret_stderr - - ds = ogr.Open(four_tile_index) - assert ds.GetLayer(0).GetFeatureCount() == 4 - - -############################################################################### -# Try adding a raster in another projection with -skip_different_projection -# 5th tile should NOT be inserted - - -def test_gdaltindex_3(gdaltindex_path, tmp_path, four_tile_index): - - drv = gdal.GetDriverByName("GTiff") - wkt = """GEOGCS["WGS 72", - DATUM["WGS_1972", - SPHEROID["WGS 72",6378135,298.26], - TOWGS84[0,0,4.5,0,0,0.554,0.2263]], - PRIMEM["Greenwich",0], - UNIT["degree",0.0174532925199433]]""" - - ds = drv.Create(f"{tmp_path}/gdaltindex5.tif", 10, 10, 1) - ds.SetProjection(wkt) - ds.SetGeoTransform([47, 0.1, 0, 2, 0, -0.1]) - ds = None - - (_, ret_stderr) = gdaltest.runexternal_out_and_err( - f"{gdaltindex_path} -skip_different_projection {four_tile_index} {tmp_path}/gdaltindex5.tif" - ) - - assert ( - "gdaltindex5.tif is not using the same projection system as other files in the tileindex." - in ret_stderr - ) - assert ( - "Use -t_srs option to set target projection system (not supported by MapServer)." - in ret_stderr - ) - - ds = ogr.Open(four_tile_index) - assert ds.GetLayer(0).GetFeatureCount() == 4 - - -############################################################################### -# Try adding a raster in another projection with -t_srs -# 5th tile should be inserted, will not be if there is a srs transformation error - - -def test_gdaltindex_4(gdaltindex_path, tmp_path, four_tile_index): - - drv = gdal.GetDriverByName("GTiff") - wkt = """GEOGCS["WGS 72", - DATUM["WGS_1972", - SPHEROID["WGS 72",6378135,298.26], - TOWGS84[0,0,4.5,0,0,0.554,0.2263]], - PRIMEM["Greenwich",0], - UNIT["degree",0.0174532925199433]]""" - - ds = drv.Create(f"{tmp_path}/gdaltindex5.tif", 10, 10, 1) - ds.SetProjection(wkt) - ds.SetGeoTransform([47, 0.1, 0, 2, 0, -0.1]) - ds = None - - gdaltest.runexternal_out_and_err( - f"{gdaltindex_path} -t_srs EPSG:4326 {four_tile_index} {tmp_path}/gdaltindex5.tif" - ) - - ds = ogr.Open(four_tile_index) - assert ds.GetLayer(0).GetFeatureCount() == 5, ( - "got %d features, expecting 5" % ds.GetLayer(0).GetFeatureCount() - ) - - ############################################################################### # Test -src_srs_name, -src_srs_format options @@ -257,23 +150,3 @@ def test_gdaltindex_5(gdaltindex_path, tmp_path, four_tiles, src_srs_format): pytest.fail() else: assert feat.GetField("src_srs") == "EPSG:4322" - - -############################################################################### -# Test -f, -lyr_name - - -@pytest.mark.parametrize("option", ["", "-lyr_name tileindex"]) -def test_gdaltindex_6(gdaltindex_path, tmp_path, four_tiles, option): - - index_mif = str(tmp_path / "test_gdaltindex6.mif") - - gdaltest.runexternal_out_and_err( - f'{gdaltindex_path} -f "MapInfo File" {option} {index_mif} {four_tiles[0]}' - ) - ds = ogr.Open(index_mif) - lyr = ds.GetLayer(0) - assert lyr.GetFeatureCount() == 1, ( - "got %d features, expecting 1" % lyr.GetFeatureCount() - ) - ds = None diff --git a/autotest/utilities/test_gdaltindex_lib.py b/autotest/utilities/test_gdaltindex_lib.py new file mode 100644 index 000000000000..1bda614b77f6 --- /dev/null +++ b/autotest/utilities/test_gdaltindex_lib.py @@ -0,0 +1,246 @@ +#!/usr/bin/env pytest +# -*- coding: utf-8 -*- +############################################################################### +# $Id$ +# +# Project: GDAL/OGR Test Suite +# Purpose: Library version of gdaltindex testing +# Author: Even Rouault +# +############################################################################### +# Copyright (c) 2023, Even Rouault +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +############################################################################### + +import os + +import gdaltest +import pytest + +from osgeo import gdal, ogr + +############################################################################### +# Simple test + + +@pytest.fixture(scope="module") +def four_tiles(tmp_path_factory): + + drv = gdal.GetDriverByName("GTiff") + wkt = 'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],TOWGS84[0,0,0,0,0,0,0],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9108"]],AUTHORITY["EPSG","4326"]]' + + dirname = tmp_path_factory.mktemp("test_gdaltindex") + fnames = [f"{dirname}/gdaltindex{i}.tif" for i in (1, 2, 3, 4)] + + ds = drv.Create(fnames[0], 10, 10, 1) + ds.SetProjection(wkt) + ds.SetGeoTransform([49, 0.1, 0, 2, 0, -0.1]) + ds = None + + ds = drv.Create(fnames[1], 10, 10, 1) + ds.SetProjection(wkt) + ds.SetGeoTransform([49, 0.1, 0, 3, 0, -0.1]) + ds = None + + ds = drv.Create(fnames[2], 10, 10, 1) + ds.SetProjection(wkt) + ds.SetGeoTransform([48, 0.1, 0, 2, 0, -0.1]) + ds = None + + ds = drv.Create(fnames[3], 10, 10, 1) + ds.SetProjection(wkt) + ds.SetGeoTransform([48, 0.1, 0, 3, 0, -0.1]) + ds = None + + return fnames + + +@pytest.fixture() +def four_tile_index(four_tiles, tmp_path): + + gdal.TileIndex(f"{tmp_path}/tileindex.shp", [four_tiles[0], four_tiles[1]]) + gdal.TileIndex(f"{tmp_path}/tileindex.shp", [four_tiles[2], four_tiles[3]]) + return f"{tmp_path}/tileindex.shp" + + +def test_gdaltindex_lib_basic(four_tile_index): + + ds = ogr.Open(four_tile_index) + assert ds.GetLayer(0).GetFeatureCount() == 4 + + tileindex_wkt = ds.GetLayer(0).GetSpatialRef().ExportToWkt() + assert "WGS_1984" in tileindex_wkt + + expected_wkts = [ + "POLYGON ((49 2,50 2,50 1,49 1,49 2))", + "POLYGON ((49 3,50 3,50 2,49 2,49 3))", + "POLYGON ((48 2,49 2,49 1,48 1,48 2))", + "POLYGON ((48 3,49 3,49 2,48 2,48 3))", + ] + + for i, feat in enumerate(ds.GetLayer(0)): + assert ( + feat.GetGeometryRef().ExportToWkt() == expected_wkts[i] + ), "i=%d, wkt=%s" % (i, feat.GetGeometryRef().ExportToWkt()) + + +############################################################################### +# Try adding the same rasters again + + +def test_gdaltindex_lib_already_existing_rasters(four_tiles, four_tile_index, tmp_path): + class GdalErrorHandler(object): + def __init__(self): + self.warnings = [] + + def handler(self, err_level, err_no, err_msg): + if err_level == gdal.CE_Warning: + self.warnings.append(err_msg) + + err_handler = GdalErrorHandler() + with gdaltest.error_handler(err_handler.handler): + ds = gdal.TileIndex(four_tile_index, four_tiles) + del ds + + assert len(err_handler.warnings) == 4 + assert ( + "gdaltindex1.tif is already in tileindex. Skipping it." + in err_handler.warnings[0] + ) + assert ( + "gdaltindex2.tif is already in tileindex. Skipping it." + in err_handler.warnings[1] + ) + assert ( + "gdaltindex3.tif is already in tileindex. Skipping it." + in err_handler.warnings[2] + ) + assert ( + "gdaltindex4.tif is already in tileindex. Skipping it." + in err_handler.warnings[3] + ) + + ds = ogr.Open(four_tile_index) + assert ds.GetLayer(0).GetFeatureCount() == 4 + + +############################################################################### +# Try adding a raster in another projection with -skip_different_projection +# 5th tile should NOT be inserted + + +def test_gdaltindex_skipDifferentProjection(tmp_path, four_tile_index): + + drv = gdal.GetDriverByName("GTiff") + wkt = """GEOGCS["WGS 72", + DATUM["WGS_1972", + SPHEROID["WGS 72",6378135,298.26], + TOWGS84[0,0,4.5,0,0,0.554,0.2263]], + PRIMEM["Greenwich",0], + UNIT["degree",0.0174532925199433]]""" + + ds = drv.Create(f"{tmp_path}/gdaltindex5.tif", 10, 10, 1) + ds.SetProjection(wkt) + ds.SetGeoTransform([47, 0.1, 0, 2, 0, -0.1]) + ds = None + + class GdalErrorHandler(object): + def __init__(self): + self.warning = None + + def handler(self, err_level, err_no, err_msg): + if err_level == gdal.CE_Warning: + self.warning = err_msg + + err_handler = GdalErrorHandler() + with gdaltest.error_handler(err_handler.handler): + gdal.TileIndex( + four_tile_index, + [f"{tmp_path}/gdaltindex5.tif"], + skipDifferentProjection=True, + ) + assert ( + "gdaltindex5.tif is not using the same projection system as other files in the tileindex" + in err_handler.warning + ) + + ds = ogr.Open(four_tile_index) + assert ds.GetLayer(0).GetFeatureCount() == 4 + + +############################################################################### +# Try adding a raster in another projection with -t_srs +# 5th tile should be inserted, will not be if there is a srs transformation error + + +def test_gdaltindex_lib_outputSRS_writeAbsoluePath(tmp_path, four_tile_index): + + drv = gdal.GetDriverByName("GTiff") + wkt = """GEOGCS["WGS 72", + DATUM["WGS_1972", + SPHEROID["WGS 72",6378135,298.26], + TOWGS84[0,0,4.5,0,0,0.554,0.2263]], + PRIMEM["Greenwich",0], + UNIT["degree",0.0174532925199433]]""" + + ds = drv.Create(f"{tmp_path}/gdaltindex5.tif", 10, 10, 1) + ds.SetProjection(wkt) + ds.SetGeoTransform([47, 0.1, 0, 2, 0, -0.1]) + ds = None + + saved_dir = os.getcwd() + try: + os.chdir(tmp_path) + gdal.TileIndex( + four_tile_index, + ["gdaltindex5.tif"], + outputSRS="EPSG:4326", + writeAbsolutePath=True, + ) + finally: + os.chdir(saved_dir) + + ds = ogr.Open(four_tile_index) + lyr = ds.GetLayer(0) + assert lyr.GetFeatureCount() == 5, ( + "got %d features, expecting 5" % ds.GetLayer(0).GetFeatureCount() + ) + filename = lyr.GetFeature(4).GetField("location") + assert filename.endswith("gdaltindex5.tif") + assert filename != "gdaltindex5.tif" + + +############################################################################### +# Test -f, -lyr_name + + +def test_gdaltindex_lib_format_layerName(tmp_path, four_tiles): + + index_mif = str(tmp_path / "test_gdaltindex6.mif") + + gdal.TileIndex( + index_mif, [four_tiles[0]], format="MapInfo File", layerName="tileindex" + ) + ds = ogr.Open(index_mif) + lyr = ds.GetLayer(0) + assert lyr.GetFeatureCount() == 1, ( + "got %d features, expecting 1" % lyr.GetFeatureCount() + ) + ds = None diff --git a/doc/source/programs/gdaltindex.rst b/doc/source/programs/gdaltindex.rst index cbf7ff169382..84b49ecdeb56 100644 --- a/doc/source/programs/gdaltindex.rst +++ b/doc/source/programs/gdaltindex.rst @@ -29,7 +29,7 @@ an attribute containing the filename, and a polygon geometry outlining the raster. This output is suitable for use with `MapServer `__ as a raster tileindex. -.. program:: ogrtindex +.. program:: gdaltindex .. include:: options/help_and_help_general.rst @@ -120,6 +120,11 @@ Examples gdaltindex doq_index.shp --optfile my_list.txt +C API +----- + +This utility is also callable from C with :cpp:func:`GDALTileIndex`. + See also -------- diff --git a/swig/include/gdal.i b/swig/include/gdal.i index 9427597c885d..2b69b930f7ce 100644 --- a/swig/include/gdal.i +++ b/swig/include/gdal.i @@ -2028,6 +2028,77 @@ GDALDatasetShadow* wrapper_GDALBuildVRT_names( const char* dest, %} %clear char** source_filenames; +//************************************************************************ +// gdal.TileIndex() +//************************************************************************ + +#ifdef SWIGJAVA +%rename (TileIndexOptions) GDALTileIndexOptions; +#endif +struct GDALTileIndexOptions { +%extend { + GDALTileIndexOptions(char** options) { + return GDALTileIndexOptionsNew(options, NULL); + } + + ~GDALTileIndexOptions() { + GDALTileIndexOptionsFree( self ); + } +} +}; + +#ifdef SWIGPYTHON +%rename (TileIndexInternalNames) wrapper_TileIndex_names; +#elif defined(SWIGJAVA) +%rename (TileIndex) wrapper_TileIndex_names; +#endif +%newobject wrapper_TileIndex_names; + +%apply (char **options) {char** source_filenames}; +%inline %{ +GDALDatasetShadow* wrapper_TileIndex_names( const char* dest, + char ** source_filenames, + GDALTileIndexOptions* options, + GDALProgressFunc callback=NULL, + void* callback_data=NULL) +{ + int usageError; /* ignored */ +#if 0 + bool bFreeOptions = false; + if( callback ) + { + if( options == NULL ) + { + bFreeOptions = true; + options = GDALTileIndexOptionsNew(NULL, NULL); + } + GDALTileIndexOptionsSetProgress(options, callback, callback_data); + } +#endif + +#ifdef SWIGPYTHON + std::vector aoErrors; + if( GetUseExceptions() ) + { + PushStackingErrorHandler(&aoErrors); + } +#endif + GDALDatasetH hDSRet = GDALTileIndex(dest, CSLCount(source_filenames), source_filenames, options, &usageError); +#if 0 + if( bFreeOptions ) + GDALTileIndexOptionsFree(options); +#endif +#ifdef SWIGPYTHON + if( GetUseExceptions() ) + { + PopStackingErrorHandler(&aoErrors, hDSRet != NULL); + } +#endif + return hDSRet; +} +%} +%clear char** source_filenames; + //************************************************************************ // gdal.MultiDimTranslate() //************************************************************************ diff --git a/swig/include/python/gdal_python.i b/swig/include/python/gdal_python.i index 091b41ca4e6f..17c54436888d 100644 --- a/swig/include/python/gdal_python.i +++ b/swig/include/python/gdal_python.i @@ -3806,6 +3806,102 @@ def BuildVRT(destName, srcDSOrSrcDSTab, **kwargs): return BuildVRTInternalNames(destName, srcDSNamesTab, opts, callback, callback_data) +def TileIndexOptions(options=None, + format=None, + layerName=None, + locationFieldName="location", + outputSRS=None, + writeAbsolutePath=None, + skipDifferentProjection=None): + """Create a TileIndexOptions() object that can be passed to gdal.TileIndex() + + Parameters + ---------- + options: + can be be an array of strings, a string or let empty and filled from other keywords. + format: + output format ("ESRI Shapefile", "GPKG", etc...) + layerName: + output layer name + locationFieldName: + Specifies the name of the field in the resulting vector dataset where the path of the input dataset will be stored. The default field name is "location". Can be set to None to disable creation of such field. + outputSRS: + assigned output SRS + writeAbsolutePath: + Enables writing the absolute path of the input dataset. By default, the filename is written in the location field exactly as the dataset name. + skipDifferentProjection: + Whether to skip sources that have a different SRS + """ + + # Only used for tests + return_option_list = options == '__RETURN_OPTION_LIST__' + if return_option_list: + options = [] + else: + options = [] if options is None else options + + if isinstance(options, str): + new_options = ParseCommandLine(options) + else: + new_options = options + if format: + new_options += ['-f', format] + if layerName is not None: + new_options += ['-lyr_name', layerName] + if locationFieldName is not None: + new_options += ['-tileindex', locationFieldName] + if outputSRS is not None: + new_options += ['-t_srs', str(outputSRS)] + if writeAbsolutePath: + new_options += ['-write_absolute_path'] + if skipDifferentProjection: + new_options += ['-skip_different_projection'] + + if return_option_list: + return new_options + + callback = None + callback_data = None + return (GDALTileIndexOptions(new_options), callback, callback_data) + +def TileIndex(destName, srcFilenames, **kwargs): + """Build a tileindex from a list of datasets. + + Parameters + ---------- + destName: + Output dataset name. + srcFilenames: + An array of filenames. + kwargs: + options: return of gdal.TileIndexOptions(), string or array of strings, + other keywords arguments of gdal.TileIndexOptions(). + If options is provided as a gdal.TileIndexOptions() object, + other keywords are ignored. + """ + + _WarnIfUserHasNotSpecifiedIfUsingExceptions() + + if 'options' not in kwargs or isinstance(kwargs['options'], (list, str)): + (opts, callback, callback_data) = TileIndexOptions(**kwargs) + else: + (opts, callback, callback_data) = kwargs['options'] + + srcDSNamesTab = [] + + import os + + if isinstance(srcFilenames, (str, os.PathLike)): + srcDSNamesTab = [str(srcFilenames)] + elif isinstance(srcFilenames, list): + for elt in srcFilenames: + srcDSNamesTab.append(str(elt)) + else: + raise Exception("Unexpected type for srcFilenames") + + return TileIndexInternalNames(destName, srcDSNamesTab, opts, callback, callback_data) + + def MultiDimTranslateOptions(options=None, format=None, creationOptions=None, arraySpecs=None, groupSpecs=None, subsetSpecs=None, scaleAxesSpecs=None, callback=None, callback_data=None): From 54053db2eb5668776acffeb9efa617c2a6b8e03d Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 20 Dec 2023 13:12:04 +0100 Subject: [PATCH 05/11] gdaltindex: add -overwrite, -vrtti_filename, -tr, -te, -ot, -bandcount, -nodata, -colorinterp, -mask, -mo options --- apps/gdaltindex_bin.cpp | 12 +- apps/gdaltindex_lib.cpp | 290 +++++++++++++++++++++- autotest/utilities/test_gdaltindex_lib.py | 111 +++++++++ doc/source/programs/gdalbuildvrt.rst | 7 + doc/source/programs/gdaltindex.rst | 150 ++++++++++- swig/include/python/gdal_python.i | 67 ++++- 6 files changed, 628 insertions(+), 9 deletions(-) diff --git a/apps/gdaltindex_bin.cpp b/apps/gdaltindex_bin.cpp index 1e8e21af2058..b2e8d29860cd 100644 --- a/apps/gdaltindex_bin.cpp +++ b/apps/gdaltindex_bin.cpp @@ -43,13 +43,21 @@ static void Usage(bool bIsError, const char *pszErrorMsg) fprintf( bIsError ? stderr : stdout, "%s", "Usage: gdaltindex [--help] [--help-general]\n" + " [-overwrite]\n" " [-f ] [-tileindex ] " "[-write_absolute_path] \n" " [-skip_different_projection] [-t_srs ]\n" " [-src_srs_name field_name] [-src_srs_format " "{AUTO|WKT|EPSG|PROJ}]\n" - " [-lyr_name ] " - "[]...\n" + " [-lyr_name ]\n" + " [-vrtti_filename ]\n" + " [-tr ] [-te " + "]\n" + " [-ot ] [-bandcount ] [-nodata " + "[,...]]\n" + " [-colorinterp [,...]] [-mask]\n" + " [-mo =]...\n" + " []...\n" "\n" "e.g.\n" " % gdaltindex doq_index.shp doq/*.tif\n" diff --git a/apps/gdaltindex_lib.cpp b/apps/gdaltindex_lib.cpp index df8f37b7c38b..34f0adf9aa5f 100644 --- a/apps/gdaltindex_lib.cpp +++ b/apps/gdaltindex_lib.cpp @@ -29,6 +29,7 @@ #include "cpl_port.h" #include "cpl_conv.h" +#include "cpl_minixml.h" #include "cpl_string.h" #include "gdal_utils.h" #include "gdal_priv.h" @@ -38,7 +39,9 @@ #include "ogr_spatialref.h" #include "commonutils.h" +#include #include +#include #include typedef enum @@ -55,6 +58,7 @@ typedef enum struct GDALTileIndexOptions { + bool bOverwrite = false; std::string osFormat{}; std::string osIndexLayerName{}; std::string osLocationField = "location"; @@ -63,6 +67,19 @@ struct GDALTileIndexOptions bool bSkipDifferentProjection = false; std::string osSrcSRSFieldName{}; SrcSRSFormat eSrcSRSFormat = FORMAT_AUTO; + double xres = std::numeric_limits::quiet_NaN(); + double yres = std::numeric_limits::quiet_NaN(); + double xmin = std::numeric_limits::quiet_NaN(); + double ymin = std::numeric_limits::quiet_NaN(); + double xmax = std::numeric_limits::quiet_NaN(); + double ymax = std::numeric_limits::quiet_NaN(); + std::string osBandCount{}; + std::string osNodata{}; + std::string osColorInterp{}; + std::string osDataType{}; + bool bMaskBand = false; + std::vector aosMetadata{}; + std::string osVRTTIFilename{}; }; /************************************************************************/ @@ -131,6 +148,18 @@ GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, /* -------------------------------------------------------------------- */ /* Open or create the target datasource */ /* -------------------------------------------------------------------- */ + + if (psOptions->bOverwrite) + { + CPLPushErrorHandler(CPLQuietErrorHandler); + auto hDriver = GDALIdentifyDriver(pszDest, nullptr); + if (hDriver) + GDALDeleteDataset(hDriver, pszDest); + else + VSIUnlink(pszDest); + CPLPopErrorHandler(); + } + auto poTileIndexDS = std::unique_ptr(GDALDataset::Open( pszDest, GDAL_OF_VECTOR | GDAL_OF_UPDATE, nullptr, nullptr, nullptr)); OGRLayer *poLayer = nullptr; @@ -235,6 +264,20 @@ GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, } else { + if (psOptions->bOverwrite) + { + for (int i = 0; i < poTileIndexDS->GetLayerCount(); ++i) + { + auto poExistingLayer = poTileIndexDS->GetLayer(i); + if (poExistingLayer && poExistingLayer->GetName() == + psOptions->osIndexLayerName) + { + poTileIndexDS->DeleteLayer(i); + break; + } + } + } + osLayerName = psOptions->osIndexLayerName; } @@ -280,6 +323,178 @@ GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, } } + if (!psOptions->osVRTTIFilename.empty()) + { + if (!psOptions->aosMetadata.empty()) + { + CPLError(CE_Failure, CPLE_NotSupported, + "-mo is not supported when -vrtti_filename is used"); + return nullptr; + } + CPLXMLNode *psRoot = + CPLCreateXMLNode(nullptr, CXT_Element, "VRTTileIndexDataset"); + CPLCreateXMLElementAndValue(psRoot, "IndexDataset", pszDest); + CPLCreateXMLElementAndValue(psRoot, "IndexLayer", poLayer->GetName()); + CPLCreateXMLElementAndValue(psRoot, "LocationField", + psOptions->osLocationField.c_str()); + if (!std::isnan(psOptions->xres)) + { + CPLCreateXMLElementAndValue(psRoot, "ResX", + CPLSPrintf("%.18g", psOptions->xres)); + CPLCreateXMLElementAndValue(psRoot, "ResY", + CPLSPrintf("%.18g", psOptions->yres)); + } + if (!std::isnan(psOptions->xmin)) + { + CPLCreateXMLElementAndValue(psRoot, "MinX", + CPLSPrintf("%.18g", psOptions->xmin)); + CPLCreateXMLElementAndValue(psRoot, "MinY", + CPLSPrintf("%.18g", psOptions->ymin)); + CPLCreateXMLElementAndValue(psRoot, "MaxX", + CPLSPrintf("%.18g", psOptions->xmax)); + CPLCreateXMLElementAndValue(psRoot, "MaxY", + CPLSPrintf("%.18g", psOptions->ymax)); + } + + int nBandCount = 0; + if (!psOptions->osBandCount.empty()) + { + nBandCount = atoi(psOptions->osBandCount.c_str()); + } + else + { + if (!psOptions->osDataType.empty()) + { + nBandCount = std::max( + nBandCount, + CPLStringList(CSLTokenizeString2( + psOptions->osDataType.c_str(), ", ", 0)) + .size()); + } + if (!psOptions->osNodata.empty()) + { + nBandCount = std::max( + nBandCount, + CPLStringList(CSLTokenizeString2( + psOptions->osNodata.c_str(), ", ", 0)) + .size()); + } + if (!psOptions->osColorInterp.empty()) + { + nBandCount = + std::max(nBandCount, + CPLStringList( + CSLTokenizeString2( + psOptions->osColorInterp.c_str(), ", ", 0)) + .size()); + } + } + + for (int i = 0; i < nBandCount; ++i) + { + auto psBand = CPLCreateXMLNode(psRoot, CXT_Element, "Band"); + CPLAddXMLAttributeAndValue(psBand, "band", CPLSPrintf("%d", i + 1)); + if (!psOptions->osDataType.empty()) + { + const CPLStringList aosTokens( + CSLTokenizeString2(psOptions->osDataType.c_str(), ", ", 0)); + if (aosTokens.size() == 1) + CPLAddXMLAttributeAndValue(psBand, "dataType", + aosTokens[0]); + else if (i < aosTokens.size()) + CPLAddXMLAttributeAndValue(psBand, "dataType", + aosTokens[i]); + } + if (!psOptions->osNodata.empty()) + { + const CPLStringList aosTokens( + CSLTokenizeString2(psOptions->osNodata.c_str(), ", ", 0)); + if (aosTokens.size() == 1) + CPLCreateXMLElementAndValue(psBand, "NoDataValue", + aosTokens[0]); + else if (i < aosTokens.size()) + CPLCreateXMLElementAndValue(psBand, "NoDataValue", + aosTokens[i]); + } + if (!psOptions->osColorInterp.empty()) + { + const CPLStringList aosTokens(CSLTokenizeString2( + psOptions->osColorInterp.c_str(), ", ", 0)); + if (aosTokens.size() == 1) + CPLCreateXMLElementAndValue(psBand, "ColorInterp", + aosTokens[0]); + else if (i < aosTokens.size()) + CPLCreateXMLElementAndValue(psBand, "ColorInterp", + aosTokens[i]); + } + } + + if (psOptions->bMaskBand) + { + CPLCreateXMLElementAndValue(psRoot, "MaskBand", "true"); + } + int res = CPLSerializeXMLTreeToFile(psRoot, + psOptions->osVRTTIFilename.c_str()); + CPLDestroyXMLNode(psRoot); + if (!res) + return nullptr; + } + else + { + poLayer->SetMetadataItem("LOCATION_FIELD", + psOptions->osLocationField.c_str()); + if (!std::isnan(psOptions->xres)) + { + poLayer->SetMetadataItem("RESX", + CPLSPrintf("%.18g", psOptions->xres)); + poLayer->SetMetadataItem("RESY", + CPLSPrintf("%.18g", psOptions->yres)); + } + if (!std::isnan(psOptions->xmin)) + { + poLayer->SetMetadataItem("MINX", + CPLSPrintf("%.18g", psOptions->xmin)); + poLayer->SetMetadataItem("MINY", + CPLSPrintf("%.18g", psOptions->ymin)); + poLayer->SetMetadataItem("MAXX", + CPLSPrintf("%.18g", psOptions->xmax)); + poLayer->SetMetadataItem("MAXY", + CPLSPrintf("%.18g", psOptions->ymax)); + } + if (!psOptions->osBandCount.empty()) + { + poLayer->SetMetadataItem("BAND_COUNT", + psOptions->osBandCount.c_str()); + } + if (!psOptions->osDataType.empty()) + { + poLayer->SetMetadataItem("DATA_TYPE", + psOptions->osDataType.c_str()); + } + if (!psOptions->osNodata.empty()) + { + poLayer->SetMetadataItem("NODATA", psOptions->osNodata.c_str()); + } + if (!psOptions->osColorInterp.empty()) + { + poLayer->SetMetadataItem("COLOR_INTERPRETATION", + psOptions->osColorInterp.c_str()); + } + if (psOptions->bMaskBand) + { + poLayer->SetMetadataItem("MASK_BAND", "YES"); + } + for (const auto &osNameValue : psOptions->aosMetadata) + { + char *pszKey = nullptr; + const char *pszValue = + CPLParseNameValue(osNameValue.c_str(), &pszKey); + if (pszKey && pszValue) + poLayer->SetMetadataItem(pszKey, pszValue); + CPLFree(pszKey); + } + } + auto poLayerDefn = poLayer->GetLayerDefn(); const int ti_field = poLayerDefn->GetFieldIndex(psOptions->osLocationField.c_str()); @@ -349,6 +564,13 @@ GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, CPLFree(pszCurrentPath); } + const bool bIsVRTTIContext = + !std::isnan(psOptions->xres) || !std::isnan(psOptions->xmin) || + !psOptions->osBandCount.empty() || !psOptions->osNodata.empty() || + !psOptions->osColorInterp.empty() || !psOptions->osDataType.empty() || + psOptions->bMaskBand || !psOptions->aosMetadata.empty() || + !psOptions->osVRTTIFilename.empty(); + /* -------------------------------------------------------------------- */ /* loop over GDAL files, processing. */ /* -------------------------------------------------------------------- */ @@ -416,8 +638,7 @@ GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, "as other files in the tileindex.\n" "This may cause problems when using it in MapServer " "for example.\n" - "Use -t_srs option to set target projection system " - "(not supported by MapServer). %s", + "Use -t_srs option to set target projection system. %s", papszSrcDSNames[iSrc], psOptions->bSkipDifferentProjection ? "Skipping this file." @@ -485,6 +706,19 @@ GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, } } } + else if (bIsVRTTIContext && !oAlreadyExistingSRS.IsEmpty() && + (poSrcSRS == nullptr || + !poSrcSRS->IsSame(&oAlreadyExistingSRS))) + { + CPLError( + CE_Failure, CPLE_AppDefined, + "%s is not using the same projection system " + "as other files in the tileindex. This is not compatible of " + "VRTTI use. Use -t_srs option to reproject tile extents " + "to a common SRS.", + papszSrcDSNames[iSrc]); + return nullptr; + } auto poFeature = std::make_unique(poLayerDefn); poFeature->SetField(ti_field, osFileNameToWrite.c_str()); @@ -729,6 +963,58 @@ GDALTileIndexOptionsNew(char **papszArgv, psOptionsForBinary->bQuiet = true; } } + else if (EQUAL(papszArgv[iArg], "-tr")) + { + CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(2); + psOptions->xres = CPLAtofM(papszArgv[++iArg]); + psOptions->yres = CPLAtofM(papszArgv[++iArg]); + } + else if (EQUAL(papszArgv[iArg], "-te")) + { + CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(4); + psOptions->xmin = CPLAtofM(papszArgv[++iArg]); + psOptions->ymin = CPLAtofM(papszArgv[++iArg]); + psOptions->xmax = CPLAtofM(papszArgv[++iArg]); + psOptions->ymax = CPLAtofM(papszArgv[++iArg]); + } + else if (EQUAL(papszArgv[iArg], "-ot")) + { + CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); + psOptions->osDataType = papszArgv[++iArg]; + } + else if (EQUAL(papszArgv[iArg], "-mo")) + { + CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); + psOptions->aosMetadata.push_back(papszArgv[++iArg]); + } + else if (EQUAL(papszArgv[iArg], "-bandcount")) + { + CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); + psOptions->osBandCount = papszArgv[++iArg]; + } + else if (EQUAL(papszArgv[iArg], "-nodata")) + { + CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); + psOptions->osNodata = papszArgv[++iArg]; + } + else if (EQUAL(papszArgv[iArg], "-colorinterp")) + { + CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); + psOptions->osColorInterp = papszArgv[++iArg]; + } + else if (EQUAL(papszArgv[iArg], "-mask")) + { + psOptions->bMaskBand = true; + } + else if (EQUAL(papszArgv[iArg], "-vrtti_filename")) + { + CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); + psOptions->osVRTTIFilename = papszArgv[++iArg]; + } + else if (EQUAL(papszArgv[iArg], "-overwrite")) + { + psOptions->bOverwrite = true; + } else if (papszArgv[iArg][0] == '-') { CPLError(CE_Failure, CPLE_NotSupported, "Unknown option name '%s'", diff --git a/autotest/utilities/test_gdaltindex_lib.py b/autotest/utilities/test_gdaltindex_lib.py index 1bda614b77f6..092f462622af 100644 --- a/autotest/utilities/test_gdaltindex_lib.py +++ b/autotest/utilities/test_gdaltindex_lib.py @@ -244,3 +244,114 @@ def test_gdaltindex_lib_format_layerName(tmp_path, four_tiles): "got %d features, expecting 1" % lyr.GetFeatureCount() ) ds = None + + +############################################################################### +# Test -overwrite + + +def test_gdaltindex_lib_overwrite(tmp_path, four_tiles): + + index_filename = str(tmp_path / "test_gdaltindex_lib_overwrite.shp") + + gdal.TileIndex(index_filename, [four_tiles[0]]) + gdal.TileIndex(index_filename, [four_tiles[1], four_tiles[2]], overwrite=True) + ds = ogr.Open(index_filename) + lyr = ds.GetLayer(0) + assert lyr.GetFeatureCount() == 2 + + +############################################################################### +# Test VRTTI related options + + +@pytest.mark.require_driver("GPKG") +@pytest.mark.require_driver("VRTTI") +def test_gdaltindex_lib_vrtti_non_xml(tmp_path, four_tiles): + + index_filename = str(tmp_path / "test_gdaltindex_lib_vrtti_non_xml.vrt.gpkg") + + gdal.TileIndex( + index_filename, + four_tiles, + layerName="tileindex", + xRes=60, + yRes=60, + outputBounds=[0, 1, 2, 3], + bandCount=1, + noData=0, + colorInterpretation="gray", + mask=True, + metadataOptions={"foo": "bar"}, + ) + + ds = ogr.Open(index_filename) + lyr = ds.GetLayer(0) + assert lyr.GetMetadataItem("RESX") == "60" + assert lyr.GetMetadataItem("RESY") == "60" + assert lyr.GetMetadataItem("MINX") == "0" + assert lyr.GetMetadataItem("MINY") == "1" + assert lyr.GetMetadataItem("MAXX") == "2" + assert lyr.GetMetadataItem("MAXY") == "3" + assert lyr.GetMetadataItem("BAND_COUNT") == "1" + assert lyr.GetMetadataItem("NODATA") == "0" + assert lyr.GetMetadataItem("COLOR_INTERPRETATION") == "gray" + assert lyr.GetMetadataItem("MASK_BAND") == "YES" + assert lyr.GetMetadataItem("foo") == "bar" + del ds + + ds = gdal.Open(index_filename) + assert ds.GetGeoTransform() == (0.0, 60.0, 0.0, 3.0, 0.0, -60.0) + assert ds.RasterCount == 1 + assert ds.GetRasterBand(1).GetNoDataValue() == 0 + assert ds.GetRasterBand(1).GetColorInterpretation() == gdal.GCI_GrayIndex + assert ds.GetRasterBand(1).GetMaskFlags() == gdal.GMF_PER_DATASET + del ds + + +############################################################################### +# Test VRTTI related options + + +@pytest.mark.require_driver("GPKG") +@pytest.mark.require_driver("VRTTI") +def test_gdaltindex_lib_vrtti_xml(tmp_path, four_tiles): + + index_filename = str(tmp_path / "test_gdaltindex_lib_vrtti_non_xml.vrt.gpkg") + vrtti_filename = str(tmp_path / "test_gdaltindex_lib_vrtti_non_xml.vrtti") + + gdal.TileIndex( + index_filename, + four_tiles, + layerName="tileindex", + vrttiFilename=vrtti_filename, + xRes=60, + yRes=60, + outputBounds=[0, 1, 2, 3], + noData=[0], + colorInterpretation=["gray"], + mask=True, + ) + + xml = open(vrtti_filename, "rb").read().decode("UTF-8") + assert "test_gdaltindex_lib_vrtti_non_xml.vrt.gpkg" in xml + assert "tileindex" in xml + assert "location" in xml + assert "60" in xml + assert "60" in xml + assert "0" in xml + assert "1" in xml + assert "2" in xml + assert "3" in xml + assert ( + """\n 0\n gray\n """ + in xml + ) + assert "true" in xml + + ds = gdal.Open(vrtti_filename) + assert ds.GetGeoTransform() == (0.0, 60.0, 0.0, 3.0, 0.0, -60.0) + assert ds.RasterCount == 1 + assert ds.GetRasterBand(1).GetNoDataValue() == 0 + assert ds.GetRasterBand(1).GetColorInterpretation() == gdal.GCI_GrayIndex + assert ds.GetRasterBand(1).GetMaskFlags() == gdal.GMF_PER_DATASET diff --git a/doc/source/programs/gdalbuildvrt.rst b/doc/source/programs/gdalbuildvrt.rst index 6bb2e7ac8021..48afa0bc909a 100644 --- a/doc/source/programs/gdalbuildvrt.rst +++ b/doc/source/programs/gdalbuildvrt.rst @@ -40,6 +40,13 @@ of the command line, or put in a text file (one filename per line) for very long or it can be a MapServer tileindex (see :ref:`gdaltindex` utility). In the later case, all entries in the tile index will be added to the VRT. +.. note:: + + Starting with GDAL 3.9, for virtual mosaic with a very large number of tiles + (typically hundreds of thousands of tiles, or more), it is advised to use the + :ref:`gdaltindex` utility to generate a tile index compatible of the + :ref:`VRTTI ` driver. + With -separate, each files goes into a separate band in the VRT dataset. Otherwise, the files are considered as tiles of a larger mosaic and the VRT file has as many bands as one of the input files. diff --git a/doc/source/programs/gdaltindex.rst b/doc/source/programs/gdaltindex.rst index 84b49ecdeb56..1ef95c10ec8b 100644 --- a/doc/source/programs/gdaltindex.rst +++ b/doc/source/programs/gdaltindex.rst @@ -16,10 +16,17 @@ Synopsis .. code-block:: gdaltindex [--help] [--help-general] + [-overwrite] [-f ] [-tileindex ] [-write_absolute_path] [-skip_different_projection] [-t_srs ] [-src_srs_name ] [-src_srs_format {AUTO|WKT|EPSG|PROJ}] - [-lyr_name ] []... + [-lyr_name ] + [-vrtti_filename ] + [-tr ] [-te ] + [-ot ] [-bandcount ] [-nodata [,...]] + [-colorinterp [,...]] [-mask] + [-mo =]... + []... Description ----------- @@ -27,12 +34,18 @@ Description This program creates an OGR-supported dataset with a record for each input raster file, an attribute containing the filename, and a polygon geometry outlining the raster. This output is suitable for use with `MapServer `__ as a raster -tileindex. +tileindex, or as input for the :ref:`VRTTI ` driver. .. program:: gdaltindex .. include:: options/help_and_help_general.rst +.. option:: -overwrite + + .. versionadded:: 3.9 + + Overwrite the tile index if it already exists. + .. option:: -f The OGR format of the output tile index file. Starting with @@ -72,6 +85,7 @@ tileindex. The format in which the SRS of each tile must be written. Types can be ``AUTO``, ``WKT``, ``EPSG``, ``PROJ``. + This option should be used together with :option:`-src_srs_format`. .. option:: -lyr_name @@ -89,6 +103,133 @@ tileindex. Wildcards my also be used. Stores the file locations in the same style as specified here, unless :option:`-write_absolute_path` option is also used. + +Options specific to use by the GDAL VRTTI driver +------------------------------------------------ + +gdaltindex can be used to generate a tile index suitable for use by the +:ref:`VRTTI ` driver. There are two possibilities: + +- either use directly a vector tile index generated by gdaltindex as the input + of the VRTTI driver + +- or generate a small XML .vrtti wrapper file, for easier use with non-file-based + formats such as databases, or for vector formats that do not support setting + layer metadata items. + +Formats that support layer metadata are for example GeoPackage (``-f GPKG``), +FlatGeoBuf (``-f FlatGeoBuf``) or PostGIS (``-f PG``) + +Setting :option:`-tr` and :option:`-ot` is recommended to avoid the VRTTI +driver to have to deduce them by opening the first tile in the index. If the tiles +have nodata or mask band, :option:`-nodata` and :option:`-mask` should also +be set. + +In a VRTTI context, the extent of all tiles referenced in the tile index must +be expressed in a single SRS. Consequently, if input tiles may have different +SRS, either :option:`-t_srs` or :option:`-skip_different_projection` should be +specified. + + +.. option:: -vrtti_filename + + .. versionadded:: 3.9 + + Filename of the XML Virtual Tile Index file to generate, that can be used + as an input for the GDAL VRTTI / Virtual Raster Tile Index driver. + This can be useful when writing the tile index in a vector format that + does not support writing layer metadata items. + +.. option:: -tr + + .. versionadded:: 3.9 + + Target resolution in SRS unit per pixel. + + Written in the XML Virtutual Tile Index if :option:`-vrtti_filename` + is specified, or as ``RESX`` and ``RESY`` layer metadata items for formats that + support layer metadata. + +.. option:: -te + + .. versionadded:: 3.9 + + Target extent in SRS unit. + + Written in the XML Virtutual Tile Index if :option:`-vrtti_filename` + is specified, or as ``MINX``, ``MINY``, ``MAXX`` and ``MAXY`` layer metadata + items for formats that support layer metadata. + +.. option:: -ot + + .. versionadded:: 3.9 + + Data type of the tiles of the tile index: ``Byte``, ``Int8``, ``UInt16``, + ``Int16``, ``UInt32``, ``Int32``, ``UInt64``, ``Int64``, ``Float32``, ``Float64``, ``CInt16``, + ``CInt32``, ``CFloat32`` or ``CFloat64`` + + Written in the XML Virtutual Tile Index if :option:`-vrtti_filename` + is specified, or as ``DATA_TYPE`` layer metadata item for formats that + support layer metadata. + +.. option:: -bandcount + + .. versionadded:: 3.9 + + Number of bands of the tiles of the tile index. + + Written in the XML Virtutual Tile Index if :option:`-vrtti_filename` + is specified, or as ``BAND_COUNT`` layer metadata item for formats that + support layer metadata. + + A mix of tiles with N and N+1 bands is allowed, provided that the color + interpretation of the (N+1)th band is alpha. The N+1 value must be written + as the band count in that situation. + + If :option:`-nodata` or :option:`-colorinterp` are specified and have multiple + values, the band count is also inferred from that number. + +.. option:: -nodata [,...] + + .. versionadded:: 3.9 + + Nodata value of the tiles of the tile index. + + Written in the XML Virtutual Tile Index if :option:`-vrtti_filename` + is specified, or as ``NODATA`` layer metadata item for formats that + support layer metadata. + +.. option:: -colorinterp [,...] + + .. versionadded:: 3.9 + + Color interpretation of of the tiles of the tile index: + ``red``, ``green``, ``blue``, ``alpha``, ``gray``, ``undefined``. + + Written in the XML Virtutual Tile Index if :option:`-vrtti_filename` + is specified, or as ``COLOR_INTERPRETATION`` layer metadata item for formats that + support layer metadata. + +.. option:: -mask + + .. versionadded:: 3.9 + + Whether tiles in the tile index have a mask band. + + Written in the XML Virtutual Tile Index if :option:`-vrtti_filename` + is specified, or as ``MASK_BAND`` layer metadata item for formats that + support layer metadata. + +.. option:: -mo = + + .. versionadded:: 3.9 + + Write an arbitrary layer metadata item, for formats that support layer + metadata. + This option may be repeated. + + .. note:: This option cannot be used together :option:`-vrtti_filename` + Examples -------- @@ -114,11 +255,12 @@ Examples gdaltindex -t_srs EPSG:4326 -src_srs_name src_srs tile_index_mixed_srs.shp *.tif -- Make a tile index from files listed in a text file : +- Make a tile index from files listed in a text file, with metadata suitable + for use by the GDAL VRTTI / Virtual Raster Tile Index driver. :: - gdaltindex doq_index.shp --optfile my_list.txt + gdaltindex tile_index.vrt.gpkg -datatype Byte -tr 60 60 -colorinterp Red,Green,Blue --optfile my_list.txt C API ----- diff --git a/swig/include/python/gdal_python.i b/swig/include/python/gdal_python.i index 17c54436888d..2440790735a3 100644 --- a/swig/include/python/gdal_python.i +++ b/swig/include/python/gdal_python.i @@ -3807,18 +3807,30 @@ def BuildVRT(destName, srcDSOrSrcDSTab, **kwargs): def TileIndexOptions(options=None, + overwrite=None, format=None, layerName=None, locationFieldName="location", outputSRS=None, writeAbsolutePath=None, - skipDifferentProjection=None): + skipDifferentProjection=None, + vrttiFilename=None, + xRes=None, + yRes=None, + outputBounds=None, + colorInterpretation=None, + noData=None, + bandCount=None, + mask=None, + metadataOptions=None): """Create a TileIndexOptions() object that can be passed to gdal.TileIndex() Parameters ---------- options: can be be an array of strings, a string or let empty and filled from other keywords. + overwrite: + Whether to overwrite the existing tile index format: output format ("ESRI Shapefile", "GPKG", etc...) layerName: @@ -3831,6 +3843,24 @@ def TileIndexOptions(options=None, Enables writing the absolute path of the input dataset. By default, the filename is written in the location field exactly as the dataset name. skipDifferentProjection: Whether to skip sources that have a different SRS + vrttiFilename: + Filename of the XML Virtual Tile Index file + xRes: + output horizontal resolution + yRes: + output vertical resolution + outputBounds: + output bounds as [minx, miny, maxx, maxy] + colorInterpretation: + tile color interpretation, as a single value or a list, of the following values: "red", "green", "blue", "alpha", "grey", "undefined" + noData: + tile nodata value, as a single value or a list + bandCount: + number of band of tiles in the index + mask: + whether tiles have a band mask + metadataOptions: + list or dict of metadata options """ # Only used for tests @@ -3844,6 +3874,8 @@ def TileIndexOptions(options=None, new_options = ParseCommandLine(options) else: new_options = options + if overwrite: + new_options += ['-overwrite'] if format: new_options += ['-f', format] if layerName is not None: @@ -3856,6 +3888,39 @@ def TileIndexOptions(options=None, new_options += ['-write_absolute_path'] if skipDifferentProjection: new_options += ['-skip_different_projection'] + if vrttiFilename is not None: + new_options += ['-vrtti_filename', vrttiFilename] + if xRes is not None and yRes is not None: + new_options += ['-tr', _strHighPrec(xRes), _strHighPrec(yRes)] + elif xRes is not None: + raise Exception("yRes should also be specified") + elif yRes is not None: + raise Exception("xRes should also be specified") + if outputBounds is not None: + new_options += ['-te', _strHighPrec(outputBounds[0]), _strHighPrec(outputBounds[1]), _strHighPrec(outputBounds[2]), _strHighPrec(outputBounds[3])] + if colorInterpretation is not None: + if isinstance(noData, list): + new_options += ['-colorinterp', ','.join(colorInterpretation)] + else: + new_options += ['-colorinterp', colorInterpretation] + if noData is not None: + if isinstance(noData, list): + new_options += ['-nodata', ','.join([_strHighPrec(x) for x in noData])] + else: + new_options += ['-nodata', _strHighPrec(noData)] + if bandCount is not None: + new_options += ['-bandcount', str(bandCount)] + if mask: + new_options += ['-mask'] + if metadataOptions is not None: + if isinstance(metadataOptions, str): + new_options += ['-mo', metadataOptions] + elif isinstance(metadataOptions, dict): + for k, v in metadataOptions.items(): + new_options += ['-mo', f'{k}={v}'] + else: + for opt in metadataOptions: + new_options += ['-mo', opt] if return_option_list: return new_options From 18e6cbe2516a723ff1c7420bf69ba5a9a33f909f Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 20 Dec 2023 16:14:30 +0100 Subject: [PATCH 06/11] gdaltindex: add -recursive, -filename_filter, -min_pixel_size, -max_pixel_size, -fetch_md --- apps/gdaltindex_bin.cpp | 8 +- apps/gdaltindex_lib.cpp | 352 +++++++++++++++++++++- autotest/utilities/test_gdaltindex_lib.py | 99 ++++++ doc/source/programs/gdaltindex.rst | 65 +++- swig/include/python/gdal_python.i | 37 ++- 5 files changed, 543 insertions(+), 18 deletions(-) diff --git a/apps/gdaltindex_bin.cpp b/apps/gdaltindex_bin.cpp index b2e8d29860cd..f5f8352c9400 100644 --- a/apps/gdaltindex_bin.cpp +++ b/apps/gdaltindex_bin.cpp @@ -43,7 +43,9 @@ static void Usage(bool bIsError, const char *pszErrorMsg) fprintf( bIsError ? stderr : stdout, "%s", "Usage: gdaltindex [--help] [--help-general]\n" - " [-overwrite]\n" + " [-overwrite] [-recursive] [-filename_filter " + "]...\n" + " [-min_pixel_size ] [-max_pixel_size ]\n" " [-f ] [-tileindex ] " "[-write_absolute_path] \n" " [-skip_different_projection] [-t_srs ]\n" @@ -57,7 +59,9 @@ static void Usage(bool bIsError, const char *pszErrorMsg) "[,...]]\n" " [-colorinterp [,...]] [-mask]\n" " [-mo =]...\n" - " []...\n" + " [-fetch_md " + "]...\n" + " []...\n" "\n" "e.g.\n" " % gdaltindex doq_index.shp doq/*.tif\n" diff --git a/apps/gdaltindex_lib.cpp b/apps/gdaltindex_lib.cpp index 34f0adf9aa5f..255c62c6a547 100644 --- a/apps/gdaltindex_lib.cpp +++ b/apps/gdaltindex_lib.cpp @@ -39,6 +39,8 @@ #include "ogr_spatialref.h" #include "commonutils.h" +#include + #include #include #include @@ -52,6 +54,17 @@ typedef enum FORMAT_PROJ } SrcSRSFormat; +/************************************************************************/ +/* GDALTileIndexRasterMetadata */ +/************************************************************************/ + +struct GDALTileIndexRasterMetadata +{ + OGRFieldType eType = OFTString; + std::string osFieldName{}; + std::string osRasterItemName{}; +}; + /************************************************************************/ /* GDALTileIndexOptions */ /************************************************************************/ @@ -80,6 +93,186 @@ struct GDALTileIndexOptions bool bMaskBand = false; std::vector aosMetadata{}; std::string osVRTTIFilename{}; + bool bRecursive = false; + double dfMinPixelSize = std::numeric_limits::quiet_NaN(); + double dfMaxPixelSize = std::numeric_limits::quiet_NaN(); + std::vector aoFetchMD{}; + std::set oSetFilenameFilters{}; +}; + +/************************************************************************/ +/* PatternMatch() */ +/************************************************************************/ + +static bool PatternMatch(const char *input, const char *pattern) + +{ + while (*input != '\0') + { + if (*pattern == '\0') + return false; + + else if (*pattern == '?') + { + pattern++; + if (static_cast(*input) > 127) + { + // Continuation bytes of such characters are of the form + // 10xxxxxx (0x80), whereas single-byte are 0xxxxxxx + // and the start of a multi-byte is 11xxxxxx + do + { + input++; + } while (static_cast(*input) > 127); + } + else + { + input++; + } + } + else if (*pattern == '*') + { + if (pattern[1] == '\0') + return true; + + // Try eating varying amounts of the input till we get a positive. + for (int eat = 0; input[eat] != '\0'; eat++) + { + if (PatternMatch(input + eat, pattern + 1)) + return true; + } + + return false; + } + else + { + if (tolower(*pattern) != tolower(*input)) + { + return false; + } + else + { + input++; + pattern++; + } + } + } + + if (*pattern != '\0' && strcmp(pattern, "*") != 0) + return false; + else + return true; +} + +/************************************************************************/ +/* GDALTileIndexTileIterator */ +/************************************************************************/ + +struct GDALTileIndexTileIterator +{ + const GDALTileIndexOptions *psOptions = nullptr; + int nSrcCount = 0; + const char *const *papszSrcDSNames = nullptr; + std::string osCurDir{}; + int iCurSrc = 0; + VSIDIR *psDir = nullptr; + + GDALTileIndexTileIterator(const GDALTileIndexOptions *psOptionsIn, + int nSrcCountIn, + const char *const *papszSrcDSNamesIn) + : psOptions(psOptionsIn), nSrcCount(nSrcCountIn), + papszSrcDSNames(papszSrcDSNamesIn) + { + } + + void reset() + { + if (psDir) + VSICloseDir(psDir); + psDir = nullptr; + iCurSrc = 0; + } + + std::string next() + { + while (true) + { + if (!psDir) + { + if (iCurSrc == nSrcCount) + { + break; + } + + VSIStatBufL sStatBuf; + const std::string osCurName = papszSrcDSNames[iCurSrc++]; + if (VSIStatL(osCurName.c_str(), &sStatBuf) == 0 && + VSI_ISDIR(sStatBuf.st_mode)) + { + auto poSrcDS = std::unique_ptr( + GDALDataset::Open(osCurName.c_str(), GDAL_OF_RASTER, + nullptr, nullptr, nullptr)); + if (poSrcDS) + return osCurName; + + osCurDir = osCurName; + psDir = VSIOpenDir( + osCurDir.c_str(), + /*nDepth=*/psOptions->bRecursive ? -1 : 0, nullptr); + if (!psDir) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Cannot open directory %s", osCurDir.c_str()); + return std::string(); + } + } + else + { + return osCurName; + } + } + + auto psEntry = VSIGetNextDirEntry(psDir); + if (!psEntry) + { + VSICloseDir(psDir); + psDir = nullptr; + continue; + } + + if (!psOptions->oSetFilenameFilters.empty()) + { + bool bMatchFound = false; + const std::string osFilenameOnly = + CPLGetFilename(psEntry->pszName); + for (const auto &osFilter : psOptions->oSetFilenameFilters) + { + if (PatternMatch(osFilenameOnly.c_str(), osFilter.c_str())) + { + bMatchFound = true; + break; + } + } + if (!bMatchFound) + continue; + } + + const std::string osFilename = + CPLFormFilename(osCurDir.c_str(), psEntry->pszName, nullptr); + if (VSI_ISDIR(psEntry->nMode)) + { + auto poSrcDS = std::unique_ptr( + GDALDataset::Open(osFilename.c_str(), GDAL_OF_RASTER, + nullptr, nullptr, nullptr)); + if (poSrcDS) + return osFilename; + continue; + } + + return osFilename; + } + return std::string(); + } }; /************************************************************************/ @@ -128,6 +321,9 @@ GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, ? std::make_unique(*psOptionsIn) : std::make_unique(); + GDALTileIndexTileIterator oGDALTileIndexTileIterator( + psOptions.get(), nSrcCount, papszSrcDSNames); + /* -------------------------------------------------------------------- */ /* Create and validate target SRS if given. */ /* -------------------------------------------------------------------- */ @@ -290,8 +486,15 @@ GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, } else { + std::string osFilename = oGDALTileIndexTileIterator.next(); + if (osFilename.empty()) + { + CPLError(CE_Failure, CPLE_AppDefined, "Cannot find any tile"); + return nullptr; + } + oGDALTileIndexTileIterator.reset(); auto poSrcDS = std::unique_ptr(GDALDataset::Open( - papszSrcDSNames[0], GDAL_OF_RASTER | GDAL_OF_VERBOSE_ERROR, + osFilename.c_str(), GDAL_OF_RASTER | GDAL_OF_VERBOSE_ERROR, nullptr, nullptr, nullptr)); if (!poSrcDS) return nullptr; @@ -323,6 +526,18 @@ GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, } } + auto poLayerDefn = poLayer->GetLayerDefn(); + + for (const auto &oFetchMD : psOptions->aoFetchMD) + { + if (poLayerDefn->GetFieldIndex(oFetchMD.osFieldName.c_str()) < 0) + { + OGRFieldDefn oField(oFetchMD.osFieldName.c_str(), oFetchMD.eType); + if (poLayer->CreateField(&oField) != OGRERR_NONE) + return nullptr; + } + } + if (!psOptions->osVRTTIFilename.empty()) { if (!psOptions->aosMetadata.empty()) @@ -495,7 +710,6 @@ GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, } } - auto poLayerDefn = poLayer->GetLayerDefn(); const int ti_field = poLayerDefn->GetFieldIndex(psOptions->osLocationField.c_str()); if (ti_field < 0) @@ -574,22 +788,26 @@ GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, /* -------------------------------------------------------------------- */ /* loop over GDAL files, processing. */ /* -------------------------------------------------------------------- */ - for (int iSrc = 0; iSrc < nSrcCount; ++iSrc) + while (true) { + const std::string osSrcFilename = oGDALTileIndexTileIterator.next(); + if (osSrcFilename.empty()) + break; + std::string osFileNameToWrite; VSIStatBuf sStatBuf; // Make sure it is a file before building absolute path name. if (!osCurrentPath.empty() && - CPLIsFilenameRelative(papszSrcDSNames[iSrc]) && - VSIStat(papszSrcDSNames[iSrc], &sStatBuf) == 0) + CPLIsFilenameRelative(osSrcFilename.c_str()) && + VSIStat(osSrcFilename.c_str(), &sStatBuf) == 0) { osFileNameToWrite = CPLProjectRelativeFilename( - osCurrentPath.c_str(), papszSrcDSNames[iSrc]); + osCurrentPath.c_str(), osSrcFilename.c_str()); } else { - osFileNameToWrite = papszSrcDSNames[iSrc]; + osFileNameToWrite = osSrcFilename.c_str(); } // Checks that file is not already in tileindex. @@ -603,12 +821,12 @@ GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, } auto poSrcDS = std::unique_ptr(GDALDataset::Open( - papszSrcDSNames[iSrc], GDAL_OF_RASTER | GDAL_OF_VERBOSE_ERROR, + osSrcFilename.c_str(), GDAL_OF_RASTER | GDAL_OF_VERBOSE_ERROR, nullptr, nullptr, nullptr)); if (poSrcDS == nullptr) { CPLError(CE_Warning, CPLE_AppDefined, - "Unable to open %s, skipping.", papszSrcDSNames[iSrc]); + "Unable to open %s, skipping.", osSrcFilename.c_str()); continue; } @@ -618,7 +836,7 @@ GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, CPLError(CE_Warning, CPLE_AppDefined, "It appears no georeferencing is available for\n" "`%s', skipping.", - papszSrcDSNames[iSrc]); + osSrcFilename.c_str()); continue; } @@ -639,7 +857,7 @@ GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, "This may cause problems when using it in MapServer " "for example.\n" "Use -t_srs option to set target projection system. %s", - papszSrcDSNames[iSrc], + osSrcFilename.c_str(), psOptions->bSkipDifferentProjection ? "Skipping this file." : ""); @@ -658,6 +876,13 @@ GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, const int nXSize = poSrcDS->GetRasterXSize(); const int nYSize = poSrcDS->GetRasterYSize(); + if (nXSize == 0 || nYSize == 0) + { + CPLError(CE_Warning, CPLE_AppDefined, + "%s has 0 width or height. Skipping", + osSrcFilename.c_str()); + continue; + } double adfX[5] = {0.0, 0.0, 0.0, 0.0, 0.0}; double adfY[5] = {0.0, 0.0, 0.0, 0.0, 0.0}; @@ -716,10 +941,37 @@ GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, "as other files in the tileindex. This is not compatible of " "VRTTI use. Use -t_srs option to reproject tile extents " "to a common SRS.", - papszSrcDSNames[iSrc]); + osSrcFilename.c_str()); return nullptr; } + const double dfMinX = + std::min(std::min(adfX[0], adfX[1]), std::min(adfX[2], adfX[3])); + const double dfMinY = + std::min(std::min(adfY[0], adfY[1]), std::min(adfY[2], adfY[3])); + const double dfMaxX = + std::max(std::max(adfX[0], adfX[1]), std::max(adfX[2], adfX[3])); + const double dfMaxY = + std::max(std::max(adfY[0], adfY[1]), std::max(adfY[2], adfY[3])); + const double dfRes = + (dfMaxX - dfMinX) * (dfMaxY - dfMinY) / nXSize / nYSize; + if (!std::isnan(psOptions->dfMinPixelSize) && + dfRes < psOptions->dfMinPixelSize) + { + CPLError(CE_Warning, CPLE_AppDefined, + "%s has %f as pixel size (< %f). Skipping", + osSrcFilename.c_str(), dfRes, psOptions->dfMinPixelSize); + continue; + } + if (!std::isnan(psOptions->dfMaxPixelSize) && + dfRes > psOptions->dfMaxPixelSize) + { + CPLError(CE_Warning, CPLE_AppDefined, + "%s has %f as pixel size (> %f). Skipping", + osSrcFilename.c_str(), dfRes, psOptions->dfMaxPixelSize); + continue; + } + auto poFeature = std::make_unique(poLayerDefn); poFeature->SetField(ti_field, osFileNameToWrite.c_str()); @@ -791,6 +1043,36 @@ GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, } } + for (const auto &oFetchMD : psOptions->aoFetchMD) + { + if (EQUAL(oFetchMD.osRasterItemName.c_str(), "{PIXEL_SIZE}")) + { + poFeature->SetField(oFetchMD.osFieldName.c_str(), dfRes); + continue; + } + + const char *pszMD = + poSrcDS->GetMetadataItem(oFetchMD.osRasterItemName.c_str()); + if (pszMD) + { + if (EQUAL(oFetchMD.osRasterItemName.c_str(), + "TIFFTAG_DATETIME")) + { + int nYear, nMonth, nDay, nHour, nMin, nSec; + if (sscanf(pszMD, "%04d:%02d:%02d %02d:%02d:%02d", &nYear, + &nMonth, &nDay, &nHour, &nMin, &nSec) == 6) + { + poFeature->SetField( + oFetchMD.osFieldName.c_str(), + CPLSPrintf("%04d/%02d/%02d %02d:%02d:%02d", nYear, + nMonth, nDay, nHour, nMin, nSec)); + continue; + } + } + poFeature->SetField(oFetchMD.osFieldName.c_str(), pszMD); + } + } + auto poPoly = std::make_unique(); auto poRing = std::make_unique(); for (int k = 0; k < 5; k++) @@ -1015,6 +1297,52 @@ GDALTileIndexOptionsNew(char **papszArgv, { psOptions->bOverwrite = true; } + else if (EQUAL(papszArgv[iArg], "-recursive")) + { + psOptions->bRecursive = true; + } + else if (EQUAL(papszArgv[iArg], "-min_pixel_size")) + { + CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); + psOptions->dfMinPixelSize = CPLAtofM(papszArgv[++iArg]); + } + else if (EQUAL(papszArgv[iArg], "-max_pixel_size")) + { + CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); + psOptions->dfMaxPixelSize = CPLAtofM(papszArgv[++iArg]); + } + else if (EQUAL(papszArgv[iArg], "-filename_filter")) + { + CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); + psOptions->oSetFilenameFilters.insert(papszArgv[++iArg]); + } + else if (EQUAL(papszArgv[iArg], "-fetch_md")) + { + CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(3); + GDALTileIndexRasterMetadata md; + md.osRasterItemName = papszArgv[++iArg]; + md.osFieldName = papszArgv[++iArg]; + const char *pszType = papszArgv[++iArg]; + if (EQUAL(pszType, "String")) + md.eType = OFTString; + else if (EQUAL(pszType, "Integer")) + md.eType = OFTInteger; + else if (EQUAL(pszType, "Integer64")) + md.eType = OFTInteger64; + else if (EQUAL(pszType, "Real")) + md.eType = OFTReal; + else if (EQUAL(pszType, "Date")) + md.eType = OFTDate; + else if (EQUAL(pszType, "DateTime")) + md.eType = OFTDateTime; + else + { + CPLError(CE_Failure, CPLE_NotSupported, "Unknown type '%s'", + pszType); + return nullptr; + } + psOptions->aoFetchMD.emplace_back(std::move(md)); + } else if (papszArgv[iArg][0] == '-') { CPLError(CE_Failure, CPLE_NotSupported, "Unknown option name '%s'", diff --git a/autotest/utilities/test_gdaltindex_lib.py b/autotest/utilities/test_gdaltindex_lib.py index 092f462622af..fedd2a778d3f 100644 --- a/autotest/utilities/test_gdaltindex_lib.py +++ b/autotest/utilities/test_gdaltindex_lib.py @@ -50,6 +50,8 @@ def four_tiles(tmp_path_factory): fnames = [f"{dirname}/gdaltindex{i}.tif" for i in (1, 2, 3, 4)] ds = drv.Create(fnames[0], 10, 10, 1) + ds.SetMetadataItem("foo", "bar") + ds.SetMetadataItem("TIFFTAG_DATETIME", "2023:12:20 16:10:00") ds.SetProjection(wkt) ds.SetGeoTransform([49, 0.1, 0, 2, 0, -0.1]) ds = None @@ -355,3 +357,100 @@ def test_gdaltindex_lib_vrtti_xml(tmp_path, four_tiles): assert ds.GetRasterBand(1).GetNoDataValue() == 0 assert ds.GetRasterBand(1).GetColorInterpretation() == gdal.GCI_GrayIndex assert ds.GetRasterBand(1).GetMaskFlags() == gdal.GMF_PER_DATASET + + +############################################################################### +# Test directory exploration and filtering + + +def test_gdaltindex_lib_directory(tmp_path, four_tiles): + + index_filename = str(tmp_path / "test_gdaltindex_lib_overwrite.shp") + + gdal.TileIndex( + index_filename, + [os.path.dirname(four_tiles[0])], + recursive=True, + filenameFilter="*.?if", + minPixelSize=0, + maxPixelSize=1, + ) + + ds = ogr.Open(index_filename) + lyr = ds.GetLayer(0) + assert lyr.GetFeatureCount() == 4 + del ds + + with gdal.quiet_errors(): + gdal.TileIndex( + index_filename, + [os.path.dirname(four_tiles[0])], + minPixelSize=10, + overwrite=True, + ) + + ds = ogr.Open(index_filename) + lyr = ds.GetLayer(0) + assert lyr.GetFeatureCount() == 0 + del ds + + with gdal.quiet_errors(): + gdal.TileIndex( + index_filename, + [os.path.dirname(four_tiles[0])], + maxPixelSize=1e-3, + overwrite=True, + ) + + ds = ogr.Open(index_filename) + lyr = ds.GetLayer(0) + assert lyr.GetFeatureCount() == 0 + del ds + + with pytest.raises(Exception, match="Cannot find any tile"): + gdal.TileIndex( + index_filename, + [os.path.dirname(four_tiles[0])], + filenameFilter="*.xyz", + overwrite=True, + ) + + +############################################################################### +# Test -fetchMD + + +@pytest.mark.require_driver("GPKG") +def test_gdaltindex_lib_fetch_md(tmp_path, four_tiles): + + index_filename = str(tmp_path / "test_gdaltindex_lib_fetch_md.gpkg") + + gdal.TileIndex( + index_filename, + four_tiles[0], + fetchMD=[ + ("foo", "foo_field", "String"), + ("{PIXEL_SIZE}", "pixel_size", "Real"), + ("TIFFTAG_DATETIME", "dt", "DateTime"), + ], + ) + + ds = ogr.Open(index_filename) + lyr = ds.GetLayer(0) + f = lyr.GetNextFeature() + assert f["foo_field"] == "bar" + assert f["dt"] == "2023/12/20 16:10:00" + assert f["pixel_size"] == pytest.approx(0.01) + del ds + + gdal.TileIndex( + index_filename, + four_tiles[0], + fetchMD=("foo", "foo_field", "String"), + overwrite=True, + ) + + ds = ogr.Open(index_filename) + lyr = ds.GetLayer(0) + f = lyr.GetNextFeature() + assert f["foo_field"] == "bar" diff --git a/doc/source/programs/gdaltindex.rst b/doc/source/programs/gdaltindex.rst index 1ef95c10ec8b..cb7fb1afdcea 100644 --- a/doc/source/programs/gdaltindex.rst +++ b/doc/source/programs/gdaltindex.rst @@ -16,7 +16,8 @@ Synopsis .. code-block:: gdaltindex [--help] [--help-general] - [-overwrite] + [-overwrite] [-recursive] [-filename_filter ]... + [-min_pixel_size ] [-max_pixel_size ] [-f ] [-tileindex ] [-write_absolute_path] [-skip_different_projection] [-t_srs ] [-src_srs_name ] [-src_srs_format {AUTO|WKT|EPSG|PROJ}] @@ -26,7 +27,8 @@ Synopsis [-ot ] [-bandcount ] [-nodata [,...]] [-colorinterp [,...]] [-mask] [-mo =]... - []... + [-fetch_md ]... + []... Description ----------- @@ -46,6 +48,41 @@ tileindex, or as input for the :ref:`VRTTI ` driver. Overwrite the tile index if it already exists. +.. option:: -recursive + + .. versionadded:: 3.9 + + Whether directories specified in should be explored recursively. + +.. option:: -filename_filter + + .. versionadded:: 3.9 + + Pattern that the filenames contained in directories pointed by + should follow. + '*' is a wildcard character that matches any number of any characters + including none. '?' is a wildcard character that matches a single character. + Comparisons are done in a case insensitive way. + Several filters may be specified + + For example :``-filename_filter "*.tif" -filename_filter "*.tiff"`` + +.. option:: -min_pixel_size + + .. versionadded:: 3.9 + + Minimum pixel size that a raster should have to be selected. The pixel size + is evaluated after reprojection of its extent to the target SRS defined + by :option:`-t_srs`. + +.. option:: -max_pixel_size + + .. versionadded:: 3.9 + + Maximum pixel size that a raster should have to be selected. The pixel size + is evaluated after reprojection of its extent to the target SRS defined + by :option:`-t_srs`. + .. option:: -f The OGR format of the output tile index file. Starting with @@ -97,12 +134,15 @@ tileindex, or as input for the :ref:`VRTTI ` driver. be created if it doesn't already exist, otherwise it will append to the existing dataset. -.. option:: +.. option:: The input GDAL raster files, can be multiple files separated by spaces. Wildcards my also be used. Stores the file locations in the same style as specified here, unless :option:`-write_absolute_path` option is also used. + Starting with GDAL 3.9, this can also be a directory name. :option:`-recursive` + might also be used to recursve down to sub-directories. + Options specific to use by the GDAL VRTTI driver ------------------------------------------------ @@ -230,6 +270,25 @@ specified. .. note:: This option cannot be used together :option:`-vrtti_filename` +.. option:: -fetch_md + + .. versionadded:: 3.9 + + Fetch a metadata item from the raster tile and write it as a field in the + tile index. + + should be the name of the raster metadata item. + ``{PIXEL_SIZE}`` may be used as a special name to indicate the pixel size. + + should be the name of the field to create in the tile index. + + should be the name of the type to create. + One of ``String``, ``Integer``, ``Integer64``, ``Real``, ``Date``, ``DateTime`` + + This option may be repeated. + + For example: ``-fetch_md TIFFTAG_DATETIME creation_date DateTime`` + Examples -------- diff --git a/swig/include/python/gdal_python.i b/swig/include/python/gdal_python.i index 2440790735a3..c35c072f116b 100644 --- a/swig/include/python/gdal_python.i +++ b/swig/include/python/gdal_python.i @@ -3808,6 +3808,10 @@ def BuildVRT(destName, srcDSOrSrcDSTab, **kwargs): def TileIndexOptions(options=None, overwrite=None, + recursive=None, + filenameFilter=None, + minPixelSize=None, + maxPixelSize=None, format=None, layerName=None, locationFieldName="location", @@ -3822,7 +3826,8 @@ def TileIndexOptions(options=None, noData=None, bandCount=None, mask=None, - metadataOptions=None): + metadataOptions=None, + fetchMD=None): """Create a TileIndexOptions() object that can be passed to gdal.TileIndex() Parameters @@ -3831,6 +3836,14 @@ def TileIndexOptions(options=None, can be be an array of strings, a string or let empty and filled from other keywords. overwrite: Whether to overwrite the existing tile index + recursive: + Whether directories specified in source filenames should be explored recursively + filenameFilter: + Pattern that the filenames contained in directories pointed by should follow. '*' and '?' wildcard can be used. String or list of strings. + minPixelSize: + Minimum pixel size that a raster should have to be selected. + maxPixelSize: + Maximum pixel size that a raster should have to be selected. format: output format ("ESRI Shapefile", "GPKG", etc...) layerName: @@ -3861,6 +3874,10 @@ def TileIndexOptions(options=None, whether tiles have a band mask metadataOptions: list or dict of metadata options + fetchMD: + Fetch a metadata item from the raster tile and write it as a field in the + tile index. + Tuple (raster metadata item name, target field name, target field type), or list of such tuples, with target field type in "String", "Integer", "Integer64", "Real", "Date", "DateTime"; """ # Only used for tests @@ -3876,6 +3893,18 @@ def TileIndexOptions(options=None, new_options = options if overwrite: new_options += ['-overwrite'] + if recursive: + new_options += ['-recursive'] + if filenameFilter is not None: + if isinstance(filenameFilter, list): + for filter in filenameFilter: + new_options += ['-filename_filter', filter] + else: + new_options += ['-filename_filter', filenameFilter] + if minPixelSize is not None: + new_options += ['-min_pixel_size', _strHighPrec(minPixelSize)] + if maxPixelSize is not None: + new_options += ['-max_pixel_size', _strHighPrec(maxPixelSize)] if format: new_options += ['-f', format] if layerName is not None: @@ -3921,6 +3950,12 @@ def TileIndexOptions(options=None, else: for opt in metadataOptions: new_options += ['-mo', opt] + if fetchMD is not None: + if isinstance(fetchMD, list): + for mdItemName, fieldName, fieldType in fetchMD: + new_options += ['-fetch_md', mdItemName, fieldName, fieldType] + else: + new_options += ['-fetch_md', fetchMD[0], fetchMD[1], fetchMD[2]] if return_option_list: return new_options From 02a4291409f191ad2480e8d9998b8891facc2273 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 20 Dec 2023 17:18:51 +0100 Subject: [PATCH 07/11] Add a slow test for VRTTI --- autotest/slow_tests/raster.py | 68 +++++++++++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 2 deletions(-) diff --git a/autotest/slow_tests/raster.py b/autotest/slow_tests/raster.py index 95eb89b0f50c..210d730e8976 100644 --- a/autotest/slow_tests/raster.py +++ b/autotest/slow_tests/raster.py @@ -35,10 +35,11 @@ import gdaltest import pytest -from osgeo import gdal +from osgeo import gdal, ogr, osr + +pytestmark = pytest.mark.slow -@pytest.mark.slow() def test_translate_vrt_with_complex_source(tmp_path): metatile_filename = str(tmp_path / "metatile.tif") @@ -157,3 +158,66 @@ def test_translate_vrt_with_complex_source(tmp_path): assert ( out_ds.ReadRaster(xoff, yoff, metatile_width, metatile_height) == tile_data ) + + +@pytest.mark.require_driver("GPKG") +def test_translate_vrtti(tmp_path): + + tile_filename = str(tmp_path / "tile.tif") + tile_width = 1 + tile_height = 1 + tile_ds = gdal.GetDriverByName("GTiff").Create( + tile_filename, tile_width, tile_height, 3 + ) + srs = osr.SpatialReference() + srs.ImportFromEPSG(4326) + tile_ds.SetSpatialRef(srs) + tile_ds.GetRasterBand(1).Fill(1) + tile_ds.GetRasterBand(2).Fill(2) + tile_ds.GetRasterBand(3).Fill(3) + del tile_ds + + print("Creating tile index...") + tileindex = str(tmp_path / "tileindex.vrt.gpkg") + ds = ogr.GetDriverByName("GPKG").CreateDataSource(tileindex) + lyr = ds.CreateLayer("index", geom_type=ogr.wkbPolygon, srs=srs) + lyr.CreateField(ogr.FieldDefn("location", ogr.OFTString)) + lyr.SetMetadataItem("RESX", "0.001") + lyr.SetMetadataItem("RESY", "0.001") + lyr.SetMetadataItem("BAND_COUNT", "3") + lyr.SetMetadataItem("DATA_TYPE", "Byte") + lyr.StartTransaction() + tile_y_count = 200 + tile_x_count = 200 + resx = 0.001 + resy = 0.001 + for j in range(tile_y_count): + for i in range(tile_x_count): + minx = i * resx + miny = j * resy + maxx = minx + resx + maxy = miny + resy + f = ogr.Feature(lyr.GetLayerDefn()) + f.SetField( + "location", f"vrt://{tile_filename}?a_ullr={minx},{maxy},{maxx},{miny}" + ) + f.SetGeometry( + ogr.CreateGeometryFromWkt( + f"POLYGON(({minx} {miny},{minx} {maxy},{maxx} {maxy},{maxx} {miny},{minx} {miny}))" + ) + ) + lyr.CreateFeature(f) + lyr.CommitTransaction() + del ds + print("... done") + + print("Reading tile index...") + ds = gdal.Open(tileindex) + assert ds.RasterXSize == tile_x_count + assert ds.RasterYSize == tile_y_count + assert ds.GetGeoTransform() == pytest.approx( + (0.0, resx, 0.0, resy * tile_y_count, 0.0, -resy) + ) + assert ds.ReadRaster() == b"\x01" * (tile_y_count * tile_x_count) + b"\x02" * ( + tile_y_count * tile_x_count + ) + b"\x03" * (tile_y_count * tile_x_count) From 82ab1daabd020d5a9ed0a1e830ecae43c7e92c49 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 5 Jan 2024 19:43:02 +0100 Subject: [PATCH 08/11] VRTTI: also support setting the metadata as a XML attached to the xml:VRTTI metadata domain of a vector layer --- autotest/gdrivers/vrttileindex.py | 43 +++++++++++++++++++++++++++++ doc/source/drivers/raster/vrtti.rst | 7 ++++- frmts/vrt/data/gdalvrtti.xsd | 4 ++- frmts/vrt/vrttileindexdataset.cpp | 20 ++++++++++++++ 4 files changed, 72 insertions(+), 2 deletions(-) diff --git a/autotest/gdrivers/vrttileindex.py b/autotest/gdrivers/vrttileindex.py index c4399aa4ae54..6d6ccb0687a7 100755 --- a/autotest/gdrivers/vrttileindex.py +++ b/autotest/gdrivers/vrttileindex.py @@ -2553,3 +2553,46 @@ def test_vrttileindex_open_options(tmp_vsimem): assert vrt_ds.GetGeoTransform() == pytest.approx( (440720.0, 30.0, 0.0, 3751320.0, 0.0, -30.0) ) + + +def test_vrttileindex_xml_vrtti_embedded(tmp_vsimem): + + index_filename = str(tmp_vsimem / "index.vrt.gpkg") + + src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) + index_ds, lyr = create_basic_tileindex(index_filename, src_ds) + + xml_content = """ + 60 + 60 + location + true + + my band + 2 + 3 + 4 + dn + Gray + + cat + + +""" + + lyr.SetMetadata([xml_content], "xml:VRTTI") + del index_ds + + vrt_ds = gdal.Open(index_filename) + band = vrt_ds.GetRasterBand(1) + assert band.GetDescription() == "my band" + assert band.DataType == gdal.GDT_UInt16 + assert band.GetOffset() == 2 + assert band.GetScale() == 3 + assert band.GetNoDataValue() == 4 + assert band.GetUnitType() == "dn" + assert band.GetColorInterpretation() == gdal.GCI_GrayIndex + assert band.GetColorTable() is not None + assert band.GetCategoryNames() == ["cat"] + assert band.GetDefaultRAT() is not None + del vrt_ds diff --git a/doc/source/drivers/raster/vrtti.rst b/doc/source/drivers/raster/vrtti.rst index 99fb6eb3476d..313fa4cee448 100644 --- a/doc/source/drivers/raster/vrtti.rst +++ b/doc/source/drivers/raster/vrtti.rst @@ -223,6 +223,11 @@ In addition to those layer metadata items, the dataset-level metadata item ``TILE_INDEX_LAYER`` may be set to indicate, for dataset with multiple layers, which one should be used as the tile index layer. +Alternatively to setting those metadata items individually, the corresponding +information can be grouped together in a VRTTI XML document, attached in the +``xml:VRTTI`` metadata domain of the layer (for drivers that support alternate +metadata domains such as GeoPackage) + VRTTI XML format ---------------- @@ -236,7 +241,7 @@ mentioned in the previous section. .. code-block:: xml - PG:dbname=my_db + PG:dbname=my_db my_layer pub_date >= '2023/12/01' pub_date diff --git a/frmts/vrt/data/gdalvrtti.xsd b/frmts/vrt/data/gdalvrtti.xsd index 4435e61428db..e8d314021b3f 100644 --- a/frmts/vrt/data/gdalvrtti.xsd +++ b/frmts/vrt/data/gdalvrtti.xsd @@ -34,7 +34,9 @@ - + + + diff --git a/frmts/vrt/vrttileindexdataset.cpp b/frmts/vrt/vrttileindexdataset.cpp index 45d8131592df..82380458c0b1 100644 --- a/frmts/vrt/vrttileindexdataset.cpp +++ b/frmts/vrt/vrttileindexdataset.cpp @@ -572,6 +572,26 @@ bool VRTTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) return false; } + // Try to get the metadata from an embedded xml:VRTTI domain + if (!m_psXMLTree) + { + char **papszMD = m_poLayer->GetMetadata("xml:VRTTI"); + if (papszMD && papszMD[0]) + { + m_psXMLTree.reset(CPLParseXMLString(papszMD[0])); + if (m_psXMLTree == nullptr) + return false; + + psRoot = CPLGetXMLNode(m_psXMLTree.get(), "=VRTTileIndexDataset"); + if (psRoot == nullptr) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Missing VRTTileIndexDataset root element."); + return false; + } + } + } + const auto GetOption = [poOpenInfo, psRoot, this](const char *pszItem) { if (psRoot) From ce9b1a70d6d8dfd9c457c1e1b3c373df9293eeaa Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 9 Jan 2024 14:48:02 +0100 Subject: [PATCH 09/11] Rename VRTTI driver to GTI --- apps/gdaladdo.cpp | 9 +- apps/gdaltindex_bin.cpp | 2 +- apps/gdaltindex_lib.cpp | 24 +- autotest/gdrivers/{vrttileindex.py => gti.py} | 434 +++++++++--------- autotest/slow_tests/raster.py | 2 +- autotest/utilities/test_gdaladdo.py | 6 +- autotest/utilities/test_gdaltindex_lib.py | 26 +- .../drivers/raster/{vrtti.rst => gti.rst} | 78 ++-- doc/source/drivers/raster/index.rst | 2 +- doc/source/drivers/raster/vrt.rst | 2 +- doc/source/programs/gdalbuildvrt.rst | 2 +- doc/source/programs/gdaltindex.rst | 40 +- frmts/drivers.ini | 2 +- frmts/gdalallregister.cpp | 2 +- frmts/vrt/CMakeLists.txt | 4 +- .../data/{gdalvrtti.xsd => gdaltileindex.xsd} | 8 +- ...exdataset.cpp => gdaltileindexdataset.cpp} | 405 ++++++++-------- frmts/vrt/vrt_priv.h | 10 +- frmts/vrt/vrtdataset.h | 4 +- gcore/gdal_frmts.h | 2 +- ogr/ogrsf_frmts/gpkg/ogrgeopackagedriver.cpp | 4 +- swig/include/python/gdal_python.i | 10 +- 22 files changed, 538 insertions(+), 540 deletions(-) rename autotest/gdrivers/{vrttileindex.py => gti.py} (86%) rename doc/source/drivers/raster/{vrtti.rst => gti.rst} (86%) rename frmts/vrt/data/{gdalvrtti.xsd => gdaltileindex.xsd} (98%) rename frmts/vrt/{vrttileindexdataset.cpp => gdaltileindexdataset.cpp} (91%) diff --git a/apps/gdaladdo.cpp b/apps/gdaladdo.cpp index 0fdd7c44cd25..c29a35df8c3d 100644 --- a/apps/gdaladdo.cpp +++ b/apps/gdaladdo.cpp @@ -296,7 +296,7 @@ static bool PartialRefreshFromSourceTimestamp( return false; } - std::vector regions; + std::vector regions; double dfTotalPixels = 0; @@ -357,7 +357,7 @@ static bool PartialRefreshFromSourceTimestamp( } dfTotalPixels += static_cast(nXSize) * nYSize; - VRTTISourceDesc region; + GTISourceDesc region; region.osFilename = poSource->GetSourceDatasetName(); region.nDstXOff = nXOff; region.nDstYOff = nYOff; @@ -369,10 +369,9 @@ static bool PartialRefreshFromSourceTimestamp( } } } - else if (auto poVRTTIDS = GDALDatasetCastToVRTTIDataset(poDS)) + else if (auto poGTIDS = GDALDatasetCastToGTIDataset(poDS)) { - regions = - VRTTIGetSourcesMoreRecentThan(poVRTTIDS, sStatVRTOvr.st_mtime); + regions = GTIGetSourcesMoreRecentThan(poGTIDS, sStatVRTOvr.st_mtime); for (const auto ®ion : regions) { dfTotalPixels += diff --git a/apps/gdaltindex_bin.cpp b/apps/gdaltindex_bin.cpp index f5f8352c9400..dd1f56814fb3 100644 --- a/apps/gdaltindex_bin.cpp +++ b/apps/gdaltindex_bin.cpp @@ -52,7 +52,7 @@ static void Usage(bool bIsError, const char *pszErrorMsg) " [-src_srs_name field_name] [-src_srs_format " "{AUTO|WKT|EPSG|PROJ}]\n" " [-lyr_name ]\n" - " [-vrtti_filename ]\n" + " [-gti_filename ]\n" " [-tr ] [-te " "]\n" " [-ot ] [-bandcount ] [-nodata " diff --git a/apps/gdaltindex_lib.cpp b/apps/gdaltindex_lib.cpp index 255c62c6a547..f5c58fc7488f 100644 --- a/apps/gdaltindex_lib.cpp +++ b/apps/gdaltindex_lib.cpp @@ -92,7 +92,7 @@ struct GDALTileIndexOptions std::string osDataType{}; bool bMaskBand = false; std::vector aosMetadata{}; - std::string osVRTTIFilename{}; + std::string osGTIFilename{}; bool bRecursive = false; double dfMinPixelSize = std::numeric_limits::quiet_NaN(); double dfMaxPixelSize = std::numeric_limits::quiet_NaN(); @@ -538,16 +538,16 @@ GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, } } - if (!psOptions->osVRTTIFilename.empty()) + if (!psOptions->osGTIFilename.empty()) { if (!psOptions->aosMetadata.empty()) { CPLError(CE_Failure, CPLE_NotSupported, - "-mo is not supported when -vrtti_filename is used"); + "-mo is not supported when -gti_filename is used"); return nullptr; } CPLXMLNode *psRoot = - CPLCreateXMLNode(nullptr, CXT_Element, "VRTTileIndexDataset"); + CPLCreateXMLNode(nullptr, CXT_Element, "GDALTileIndexDataset"); CPLCreateXMLElementAndValue(psRoot, "IndexDataset", pszDest); CPLCreateXMLElementAndValue(psRoot, "IndexLayer", poLayer->GetName()); CPLCreateXMLElementAndValue(psRoot, "LocationField", @@ -648,8 +648,8 @@ GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, { CPLCreateXMLElementAndValue(psRoot, "MaskBand", "true"); } - int res = CPLSerializeXMLTreeToFile(psRoot, - psOptions->osVRTTIFilename.c_str()); + int res = + CPLSerializeXMLTreeToFile(psRoot, psOptions->osGTIFilename.c_str()); CPLDestroyXMLNode(psRoot); if (!res) return nullptr; @@ -778,12 +778,12 @@ GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, CPLFree(pszCurrentPath); } - const bool bIsVRTTIContext = + const bool bIsGTIContext = !std::isnan(psOptions->xres) || !std::isnan(psOptions->xmin) || !psOptions->osBandCount.empty() || !psOptions->osNodata.empty() || !psOptions->osColorInterp.empty() || !psOptions->osDataType.empty() || psOptions->bMaskBand || !psOptions->aosMetadata.empty() || - !psOptions->osVRTTIFilename.empty(); + !psOptions->osGTIFilename.empty(); /* -------------------------------------------------------------------- */ /* loop over GDAL files, processing. */ @@ -931,7 +931,7 @@ GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, } } } - else if (bIsVRTTIContext && !oAlreadyExistingSRS.IsEmpty() && + else if (bIsGTIContext && !oAlreadyExistingSRS.IsEmpty() && (poSrcSRS == nullptr || !poSrcSRS->IsSame(&oAlreadyExistingSRS))) { @@ -939,7 +939,7 @@ GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, CE_Failure, CPLE_AppDefined, "%s is not using the same projection system " "as other files in the tileindex. This is not compatible of " - "VRTTI use. Use -t_srs option to reproject tile extents " + "GTI use. Use -t_srs option to reproject tile extents " "to a common SRS.", osSrcFilename.c_str()); return nullptr; @@ -1288,10 +1288,10 @@ GDALTileIndexOptionsNew(char **papszArgv, { psOptions->bMaskBand = true; } - else if (EQUAL(papszArgv[iArg], "-vrtti_filename")) + else if (EQUAL(papszArgv[iArg], "-gti_filename")) { CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); - psOptions->osVRTTIFilename = papszArgv[++iArg]; + psOptions->osGTIFilename = papszArgv[++iArg]; } else if (EQUAL(papszArgv[iArg], "-overwrite")) { diff --git a/autotest/gdrivers/vrttileindex.py b/autotest/gdrivers/gti.py similarity index 86% rename from autotest/gdrivers/vrttileindex.py rename to autotest/gdrivers/gti.py index 6d6ccb0687a7..20f928fdce5c 100755 --- a/autotest/gdrivers/vrttileindex.py +++ b/autotest/gdrivers/gti.py @@ -3,7 +3,7 @@ # $Id$ # # Project: GDAL/OGR Test Suite -# Purpose: Test VRTTileIndexDataset support. +# Purpose: Test GDALTileIndexDataset support. # Author: Even Rouault # ############################################################################### @@ -38,7 +38,7 @@ from osgeo import gdal, ogr -pytestmark = [pytest.mark.require_driver("VRTTI"), pytest.mark.require_driver("GPKG")] +pytestmark = [pytest.mark.require_driver("GTI"), pytest.mark.require_driver("GPKG")] def create_basic_tileindex( @@ -113,9 +113,9 @@ def check_basic( assert vrt_ds.GetMetadata_Dict() == expected_md -def test_vrttileindex_no_metadata(tmp_vsimem): +def test_gti_no_metadata(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, _ = create_basic_tileindex(index_filename, src_ds) @@ -146,9 +146,9 @@ def test_vrttileindex_no_metadata(tmp_vsimem): assert vrt_ds.GetRasterBand(1).GetOverview(0) is None -def test_vrttileindex_custom_metadata(tmp_vsimem): +def test_gti_custom_metadata(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -162,9 +162,9 @@ def test_vrttileindex_custom_metadata(tmp_vsimem): assert vrt_ds.GetRasterBand(1).GetBlockSize() == [2, 4] -def test_vrttileindex_cannot_open_index(tmp_vsimem): +def test_gti_cannot_open_index(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") f = gdal.VSIFOpenL(index_filename, "wb+") assert f gdal.VSIFTruncateL(f, 100) @@ -174,9 +174,9 @@ def test_vrttileindex_cannot_open_index(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_several_layers(tmp_vsimem): +def test_gti_several_layers(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, _ = create_basic_tileindex(index_filename, src_ds) @@ -192,10 +192,10 @@ def test_vrttileindex_several_layers(tmp_vsimem): with pytest.raises( Exception, match="has more than one layer. LAYER open option must be defined" ): - gdal.Open("VRTTI:" + index_filename) + gdal.Open("GTI:" + index_filename) assert ( - gdal.OpenEx("VRTTI:" + index_filename, open_options=["LAYER=index"]) is not None + gdal.OpenEx("GTI:" + index_filename, open_options=["LAYER=index"]) is not None ) index_ds = ogr.Open(index_filename, update=1) @@ -209,11 +209,11 @@ def test_vrttileindex_several_layers(tmp_vsimem): ) -def test_vrttileindex_no_metadata_several_layers_wrong_TILE_INDEX_LAYER( +def test_gti_no_metadata_several_layers_wrong_TILE_INDEX_LAYER( tmp_vsimem, ): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, _ = create_basic_tileindex(index_filename, src_ds) @@ -225,18 +225,18 @@ def test_vrttileindex_no_metadata_several_layers_wrong_TILE_INDEX_LAYER( gdal.Open(index_filename) -def test_vrttileindex_no_layer(tmp_vsimem): +def test_gti_no_layer(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) with pytest.raises(Exception, match="has no vector layer"): gdal.Open(index_filename, gdal.GA_Update) -def test_vrttileindex_no_feature(tmp_vsimem): +def test_gti_no_feature(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) lyr = index_ds.CreateLayer("index") lyr.CreateField(ogr.FieldDefn("location")) @@ -246,9 +246,9 @@ def test_vrttileindex_no_feature(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_location_wrong_type(tmp_vsimem): +def test_gti_location_wrong_type(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) lyr = index_ds.CreateLayer("index") lyr.CreateField(ogr.FieldDefn("location", ogr.OFTInteger)) @@ -258,9 +258,9 @@ def test_vrttileindex_location_wrong_type(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_wrong_prototype_tile(tmp_vsimem): +def test_gti_wrong_prototype_tile(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) lyr = index_ds.CreateLayer("index") lyr.CreateField(ogr.FieldDefn("location")) @@ -273,9 +273,9 @@ def test_vrttileindex_wrong_prototype_tile(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_prototype_tile_no_gt(tmp_vsimem): +def test_gti_prototype_tile_no_gt(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") protods_filename = str(tmp_vsimem / "protods_filename.tif") ds = gdal.GetDriverByName("GTiff").Create(protods_filename, 1, 1) del ds @@ -291,9 +291,9 @@ def test_vrttileindex_prototype_tile_no_gt(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_prototype_tile_wrong_gt_3rd_value(tmp_vsimem): +def test_gti_prototype_tile_wrong_gt_3rd_value(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") protods_filename = str(tmp_vsimem / "protods_filename.tif") ds = gdal.GetDriverByName("GTiff").Create(protods_filename, 1, 1) del ds @@ -312,9 +312,9 @@ def test_vrttileindex_prototype_tile_wrong_gt_3rd_value(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_prototype_tile_wrong_gt_5th_value(tmp_vsimem): +def test_gti_prototype_tile_wrong_gt_5th_value(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") protods_filename = str(tmp_vsimem / "protods_filename.tif") ds = gdal.GetDriverByName("GTiff").Create(protods_filename, 1, 1) del ds @@ -333,9 +333,9 @@ def test_vrttileindex_prototype_tile_wrong_gt_5th_value(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_prototype_tile_wrong_gt_6th_value(tmp_vsimem): +def test_gti_prototype_tile_wrong_gt_6th_value(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") protods_filename = str(tmp_vsimem / "protods_filename.tif") ds = gdal.GetDriverByName("GTiff").Create(protods_filename, 1, 1) del ds @@ -354,9 +354,9 @@ def test_vrttileindex_prototype_tile_wrong_gt_6th_value(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_no_extent(tmp_vsimem): +def test_gti_no_extent(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") protods_filename = str(tmp_vsimem / "protods_filename.tif") ds = gdal.GetDriverByName("GTiff").Create(protods_filename, 1, 1) ds.SetGeoTransform([0, 1, 0, 0, 0, -1]) @@ -373,9 +373,9 @@ def test_vrttileindex_no_extent(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_too_big_x(tmp_vsimem): +def test_gti_too_big_x(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") protods_filename = str(tmp_vsimem / "protods_filename.tif") ds = gdal.GetDriverByName("GTiff").Create(protods_filename, 1, 1) ds.SetGeoTransform([0, 1e-30, 0, 0, 0, -1]) @@ -393,9 +393,9 @@ def test_vrttileindex_too_big_x(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_too_big_y(tmp_vsimem): +def test_gti_too_big_y(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") protods_filename = str(tmp_vsimem / "protods_filename.tif") ds = gdal.GetDriverByName("GTiff").Create(protods_filename, 1, 1) ds.SetGeoTransform([0, 1, 0, 0, 0, -1e-30]) @@ -413,9 +413,9 @@ def test_vrttileindex_too_big_y(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_location_field_missing(tmp_vsimem): +def test_gti_location_field_missing(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") # Missing LOCATION_FIELD and non-default location field name src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) @@ -428,9 +428,9 @@ def test_vrttileindex_location_field_missing(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_location_field_set(tmp_vsimem): +def test_gti_location_field_set(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") # LOCATION_FIELD set src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) @@ -445,9 +445,9 @@ def test_vrttileindex_location_field_set(tmp_vsimem): @pytest.mark.parametrize("missing_item", ["RESX", "RESY"]) -def test_vrttileindex_resx_resy(tmp_vsimem, missing_item): +def test_gti_resx_resy(tmp_vsimem, missing_item): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -478,9 +478,9 @@ def test_vrttileindex_resx_resy(tmp_vsimem, missing_item): @pytest.mark.parametrize("missing_item", [None, "XSIZE", "YSIZE", "GEOTRANSFORM"]) -def test_vrttileindex_width_height_geotransform(tmp_vsimem, missing_item): +def test_gti_width_height_geotransform(tmp_vsimem, missing_item): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -508,9 +508,9 @@ def test_vrttileindex_width_height_geotransform(tmp_vsimem, missing_item): gdal.Open(index_filename) -def test_vrttileindex_wrong_width(tmp_vsimem): +def test_gti_wrong_width(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -524,9 +524,9 @@ def test_vrttileindex_wrong_width(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_wrong_height(tmp_vsimem): +def test_gti_wrong_height(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -540,9 +540,9 @@ def test_vrttileindex_wrong_height(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_wrong_blockxsize(tmp_vsimem): +def test_gti_wrong_blockxsize(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -553,9 +553,9 @@ def test_vrttileindex_wrong_blockxsize(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_wrong_blockysize(tmp_vsimem): +def test_gti_wrong_blockysize(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -566,9 +566,9 @@ def test_vrttileindex_wrong_blockysize(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_wrong_blockxsize_blockysize(tmp_vsimem): +def test_gti_wrong_blockxsize_blockysize(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -580,9 +580,9 @@ def test_vrttileindex_wrong_blockxsize_blockysize(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_wrong_gt(tmp_vsimem): +def test_gti_wrong_gt(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -598,9 +598,9 @@ def test_vrttileindex_wrong_gt(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_wrong_gt_3rd_term(tmp_vsimem): +def test_gti_wrong_gt_3rd_term(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -613,9 +613,9 @@ def test_vrttileindex_wrong_gt_3rd_term(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_wrong_gt_5th_term(tmp_vsimem): +def test_gti_wrong_gt_5th_term(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -628,9 +628,9 @@ def test_vrttileindex_wrong_gt_5th_term(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_wrong_gt_6th_term(tmp_vsimem): +def test_gti_wrong_gt_6th_term(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -644,9 +644,9 @@ def test_vrttileindex_wrong_gt_6th_term(tmp_vsimem): @pytest.mark.parametrize("missing_item", [None, "MINX", "MINY", "MAXX", "MAXY"]) -def test_vrttileindex_minx_miny_maxx_maxy(tmp_vsimem, missing_item): +def test_gti_minx_miny_maxx_maxy(tmp_vsimem, missing_item): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -688,9 +688,9 @@ def test_vrttileindex_minx_miny_maxx_maxy(tmp_vsimem, missing_item): gdal.Open(index_filename) -def test_vrttileindex_wrong_resx(tmp_vsimem): +def test_gti_wrong_resx(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -702,9 +702,9 @@ def test_vrttileindex_wrong_resx(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_wrong_resy(tmp_vsimem): +def test_gti_wrong_resy(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -716,9 +716,9 @@ def test_vrttileindex_wrong_resy(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_wrong_minx(tmp_vsimem): +def test_gti_wrong_minx(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -736,9 +736,9 @@ def test_vrttileindex_wrong_minx(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_wrong_miny(tmp_vsimem): +def test_gti_wrong_miny(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -756,9 +756,9 @@ def test_vrttileindex_wrong_miny(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_wrong_resx_wrt_min_max_xy(tmp_vsimem): +def test_gti_wrong_resx_wrt_min_max_xy(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -775,9 +775,9 @@ def test_vrttileindex_wrong_resx_wrt_min_max_xy(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_wrong_resy_wrt_min_max_xy(tmp_vsimem): +def test_gti_wrong_resy_wrt_min_max_xy(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -794,9 +794,9 @@ def test_vrttileindex_wrong_resy_wrt_min_max_xy(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_invalid_srs(tmp_vsimem): +def test_gti_invalid_srs(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -807,9 +807,9 @@ def test_vrttileindex_invalid_srs(tmp_vsimem): gdal.Open(index_filename) -def test_vrttileindex_valid_srs(tmp_vsimem): +def test_gti_valid_srs(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -820,9 +820,9 @@ def test_vrttileindex_valid_srs(tmp_vsimem): assert ds.GetSpatialRef().GetAuthorityCode(None) == "4267" -def test_vrttileindex_invalid_band_count(tmp_vsimem): +def test_gti_invalid_band_count(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -854,9 +854,9 @@ def test_vrttileindex_invalid_band_count(tmp_vsimem): ), ], ) -def test_vrttileindex_inconsistent_number_of_values(tmp_vsimem, md, error_msg): +def test_gti_inconsistent_number_of_values(tmp_vsimem, md, error_msg): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -889,9 +889,9 @@ def test_vrttileindex_inconsistent_number_of_values(tmp_vsimem, md, error_msg): ), ], ) -def test_vrttileindex_valid_nodata(tmp_vsimem, md, expected_nodata): +def test_gti_valid_nodata(tmp_vsimem, md, expected_nodata): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -921,9 +921,9 @@ def test_vrttileindex_valid_nodata(tmp_vsimem, md, expected_nodata): ), ], ) -def test_vrttileindex_invalid_nodata(tmp_vsimem, md, error_msg): +def test_gti_invalid_nodata(tmp_vsimem, md, error_msg): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -949,9 +949,9 @@ def test_vrttileindex_invalid_nodata(tmp_vsimem, md, error_msg): ), ], ) -def test_vrttileindex_invalid_data_type(tmp_vsimem, md, error_msg): +def test_gti_invalid_data_type(tmp_vsimem, md, error_msg): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -980,9 +980,9 @@ def test_vrttileindex_invalid_data_type(tmp_vsimem, md, error_msg): ), ], ) -def test_vrttileindex_invalid_color_interpretation(tmp_vsimem, md, error_msg): +def test_gti_invalid_color_interpretation(tmp_vsimem, md, error_msg): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -994,9 +994,9 @@ def test_vrttileindex_invalid_color_interpretation(tmp_vsimem, md, error_msg): gdal.Open(index_filename) -def test_vrttileindex_no_metadata_rgb(tmp_vsimem): +def test_gti_no_metadata_rgb(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "small_world.tif")) index_ds, _ = create_basic_tileindex(index_filename, src_ds) @@ -1006,9 +1006,9 @@ def test_vrttileindex_no_metadata_rgb(tmp_vsimem): check_basic(vrt_ds, src_ds) -def test_vrttileindex_rgb_left_right(tmp_vsimem): +def test_gti_rgb_left_right(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open("data/small_world.tif") @@ -1047,11 +1047,11 @@ def test_vrttileindex_rgb_left_right(tmp_vsimem): assert ( vrt_ds.GetRasterBand(1).GetMetadataItem("Pixel_0_0", "LocationInfo") - == "/vsimem/test_vrttileindex_rgb_left_right/left.tif" + == "/vsimem/test_gti_rgb_left_right/left.tif" ) -def test_vrttileindex_overlapping_sources(tmp_vsimem): +def test_gti_overlapping_sources(tmp_vsimem): filename1 = str(tmp_vsimem / "one.tif") ds = gdal.GetDriverByName("GTiff").Create(filename1, 1, 1) @@ -1066,7 +1066,7 @@ def test_vrttileindex_overlapping_sources(tmp_vsimem): del ds # No sorting field: feature with max FID has the priority - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") index_ds, _ = create_basic_tileindex( index_filename, [gdal.Open(filename1), gdal.Open(filename2)] ) @@ -1076,7 +1076,7 @@ def test_vrttileindex_overlapping_sources(tmp_vsimem): assert vrt_ds.GetRasterBand(1).Checksum() == 2 # Test unsupported sort_field_type = OFTBinary - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") sort_values = [None, None] index_ds, _ = create_basic_tileindex( index_filename, @@ -1091,7 +1091,7 @@ def test_vrttileindex_overlapping_sources(tmp_vsimem): gdal.Open(index_filename) # Test non existent SORT_FIELD - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") sort_values = [None, None] index_ds, lyr = create_basic_tileindex(index_filename, gdal.Open(filename1)) lyr.SetMetadataItem("SORT_FIELD", "non_existing") @@ -1101,7 +1101,7 @@ def test_vrttileindex_overlapping_sources(tmp_vsimem): gdal.Open(index_filename) # Test sort_field_type = OFTString - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") sort_values = ["2", "1"] index_ds, _ = create_basic_tileindex( index_filename, @@ -1129,7 +1129,7 @@ def test_vrttileindex_overlapping_sources(tmp_vsimem): assert vrt_ds.GetRasterBand(1).Checksum() == 1, sort_values # Test sort_field_type = OFTString - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") sort_values = ["1", "1"] index_ds, _ = create_basic_tileindex( index_filename, @@ -1144,7 +1144,7 @@ def test_vrttileindex_overlapping_sources(tmp_vsimem): assert vrt_ds.GetRasterBand(1).Checksum() == 2, sort_values # Test sort_field_type = OFTInteger - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") sort_values = [2, 1] index_ds, _ = create_basic_tileindex( index_filename, @@ -1172,7 +1172,7 @@ def test_vrttileindex_overlapping_sources(tmp_vsimem): assert vrt_ds.GetRasterBand(1).Checksum() == 1, sort_values # Test sort_field_type = OFTInteger64 - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") sort_values = [1234567890123 + 2, 1234567890123 + 1] index_ds, _ = create_basic_tileindex( index_filename, @@ -1200,7 +1200,7 @@ def test_vrttileindex_overlapping_sources(tmp_vsimem): assert vrt_ds.GetRasterBand(1).Checksum() == 1, sort_values # Test sort_field_type = OFTReal - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") sort_values = [2.5, 1.5] index_ds, _ = create_basic_tileindex( index_filename, @@ -1228,7 +1228,7 @@ def test_vrttileindex_overlapping_sources(tmp_vsimem): assert vrt_ds.GetRasterBand(1).Checksum() == 1, sort_values # Test sort_field_type = OFTDate - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") for sort_values in [ ["2023-01-01", "2022-12-31"], ["2023-02-01", "2023-01-31"], @@ -1260,7 +1260,7 @@ def test_vrttileindex_overlapping_sources(tmp_vsimem): assert vrt_ds.GetRasterBand(1).Checksum() == 1, sort_values # Test sort_field_type = OFTDateTime - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") for sort_values in [ ["2023-01-01T00:00:00", "2022-12-31T23:59:59"], ["2023-02-01T00:00:00", "2023-01-31T23:59:59"], @@ -1295,7 +1295,7 @@ def test_vrttileindex_overlapping_sources(tmp_vsimem): assert vrt_ds.GetRasterBand(1).Checksum() == 1, sort_values # Test SORT_FIELD_ASC=NO - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") index_ds, lyr = create_basic_tileindex( index_filename, [gdal.Open(filename1), gdal.Open(filename2)], @@ -1310,9 +1310,9 @@ def test_vrttileindex_overlapping_sources(tmp_vsimem): assert vrt_ds.GetRasterBand(1).Checksum() == 2, sort_values -def test_vrttileindex_no_source(tmp_vsimem): +def test_gti_no_source(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") index_ds, lyr = create_basic_tileindex(index_filename, []) lyr.SetMetadataItem("XSIZE", "2") lyr.SetMetadataItem("YSIZE", "3") @@ -1351,9 +1351,9 @@ def test_vrttileindex_no_source(tmp_vsimem): ) -def test_vrttileindex_invalid_source(tmp_vsimem): +def test_gti_invalid_source(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) lyr = index_ds.CreateLayer("index", geom_type=ogr.wkbPolygon) lyr.CreateField(ogr.FieldDefn("location")) @@ -1382,7 +1382,7 @@ def test_vrttileindex_invalid_source(tmp_vsimem): vrt_ds.ReadRaster() -def test_vrttileindex_source_relative_location(tmp_vsimem): +def test_gti_source_relative_location(tmp_vsimem): tile_filename = str(tmp_vsimem / "tile.tif") ds = gdal.GetDriverByName("GTiff").Create(tile_filename, 1, 1) @@ -1390,7 +1390,7 @@ def test_vrttileindex_source_relative_location(tmp_vsimem): ds.GetRasterBand(1).Fill(255) del ds - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) lyr = index_ds.CreateLayer("index", geom_type=ogr.wkbPolygon) lyr.CreateField(ogr.FieldDefn("location")) @@ -1411,14 +1411,14 @@ def test_vrttileindex_source_relative_location(tmp_vsimem): assert vrt_ds.ReadRaster() == b"\xFF" -def test_vrttileindex_source_lacks_bands(tmp_vsimem): +def test_gti_source_lacks_bands(tmp_vsimem): tile_filename = str(tmp_vsimem / "tile.tif") ds = gdal.GetDriverByName("GTiff").Create(tile_filename, 1, 1) ds.SetGeoTransform([10, 1, 0, 20, 0, -1]) del ds - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) lyr = index_ds.CreateLayer("index", geom_type=ogr.wkbPolygon) lyr.CreateField(ogr.FieldDefn("location")) @@ -1440,14 +1440,14 @@ def test_vrttileindex_source_lacks_bands(tmp_vsimem): vrt_ds.ReadRaster() -def test_vrttileindex_source_lacks_bands_and_relative_location(tmp_vsimem): +def test_gti_source_lacks_bands_and_relative_location(tmp_vsimem): tile_filename = str(tmp_vsimem / "tile.tif") ds = gdal.GetDriverByName("GTiff").Create(tile_filename, 1, 1) ds.SetGeoTransform([10, 1, 0, 20, 0, -1]) del ds - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(index_filename) lyr = index_ds.CreateLayer("index", geom_type=ogr.wkbPolygon) lyr.CreateField(ogr.FieldDefn("location")) @@ -1470,9 +1470,9 @@ def test_vrttileindex_source_lacks_bands_and_relative_location(tmp_vsimem): @pytest.mark.require_driver("netCDF") -def test_vrttileindex_source_netcdf_subdataset_absolute(tmp_path): +def test_gti_source_netcdf_subdataset_absolute(tmp_path): - index_filename = str(tmp_path / "index.vrt.gpkg") + index_filename = str(tmp_path / "index.gti.gpkg") src_ds = gdal.Open( 'netCDF:"' + os.path.join(os.getcwd(), "data", "netcdf", "byte.nc") + '":Band1' ) @@ -1483,11 +1483,11 @@ def test_vrttileindex_source_netcdf_subdataset_absolute(tmp_path): @pytest.mark.require_driver("netCDF") -def test_vrttileindex_source_netcdf_subdataset_relative(tmp_path): +def test_gti_source_netcdf_subdataset_relative(tmp_path): tmp_netcdf_filename = str(tmp_path / "byte.nc") shutil.copy("data/netcdf/byte.nc", tmp_netcdf_filename) - index_filename = str(tmp_path / "index.vrt.gpkg") + index_filename = str(tmp_path / "index.gti.gpkg") cwd = os.getcwd() try: os.chdir(tmp_path) @@ -1500,7 +1500,7 @@ def test_vrttileindex_source_netcdf_subdataset_relative(tmp_path): assert gdal.Open(index_filename) is not None -def test_vrttileindex_single_source_nodata_same_as_vrt(tmp_vsimem): +def test_gti_single_source_nodata_same_as_vrt(tmp_vsimem): filename1 = str(tmp_vsimem / "one.tif") ds = gdal.GetDriverByName("GTiff").Create(filename1, 3, 1, 3) @@ -1513,7 +1513,7 @@ def test_vrttileindex_single_source_nodata_same_as_vrt(tmp_vsimem): ds.GetRasterBand(3).SetNoDataValue(1) del ds - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") index_ds, lyr = create_basic_tileindex(index_filename, gdal.Open(filename1)) lyr.SetMetadataItem("NODATA", "1") del index_ds @@ -1522,11 +1522,11 @@ def test_vrttileindex_single_source_nodata_same_as_vrt(tmp_vsimem): assert vrt_ds.ReadRaster() == b"\x01\x02\x01" + b"\x01\x03\x01" + b"\x01\x04\x01" assert ( vrt_ds.GetRasterBand(1).GetMetadataItem("Pixel_0_0", "LocationInfo") - == "/vsimem/test_vrttileindex_single_source_nodata_same_as_vrt/one.tif" + == "/vsimem/test_gti_single_source_nodata_same_as_vrt/one.tif" ) -def test_vrttileindex_overlapping_sources_nodata(tmp_vsimem): +def test_gti_overlapping_sources_nodata(tmp_vsimem): filename1 = str(tmp_vsimem / "one.tif") ds = gdal.GetDriverByName("GTiff").Create(filename1, 3, 1) @@ -1542,7 +1542,7 @@ def test_vrttileindex_overlapping_sources_nodata(tmp_vsimem): ds.GetRasterBand(1).SetNoDataValue(3) del ds - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") sort_values = [1, 2] index_ds, lyr = create_basic_tileindex( index_filename, @@ -1559,11 +1559,11 @@ def test_vrttileindex_overlapping_sources_nodata(tmp_vsimem): assert ( vrt_ds.GetRasterBand(1).GetMetadataItem("Pixel_0_0", "LocationInfo") - == "/vsimem/test_vrttileindex_overlapping_sources_nodata/one.tif/vsimem/test_vrttileindex_overlapping_sources_nodata/two.tif" + == "/vsimem/test_gti_overlapping_sources_nodata/one.tif/vsimem/test_gti_overlapping_sources_nodata/two.tif" ) -def test_vrttileindex_on_the_fly_rgb_color_table_expansion(tmp_vsimem): +def test_gti_on_the_fly_rgb_color_table_expansion(tmp_vsimem): tile_filename = str(tmp_vsimem / "color_table.tif") tile_ds = gdal.GetDriverByName("GTiff").Create(tile_filename, 1, 1) @@ -1573,7 +1573,7 @@ def test_vrttileindex_on_the_fly_rgb_color_table_expansion(tmp_vsimem): assert tile_ds.GetRasterBand(1).SetRasterColorTable(ct) == gdal.CE_None tile_ds = None - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") index_ds, lyr = create_basic_tileindex(index_filename, gdal.Open(tile_filename)) lyr.SetMetadataItem("BAND_COUNT", "3") lyr.SetMetadataItem("COLOR_INTERPRETATION", "Red,Green,Blue") @@ -1585,7 +1585,7 @@ def test_vrttileindex_on_the_fly_rgb_color_table_expansion(tmp_vsimem): assert vrt_ds.GetRasterBand(3).ReadRaster() == b"\x03" -def test_vrttileindex_on_the_fly_rgba_color_table_expansion(tmp_vsimem): +def test_gti_on_the_fly_rgba_color_table_expansion(tmp_vsimem): tile_filename = str(tmp_vsimem / "color_table.tif") tile_ds = gdal.GetDriverByName("GTiff").Create(tile_filename, 1, 1) @@ -1595,7 +1595,7 @@ def test_vrttileindex_on_the_fly_rgba_color_table_expansion(tmp_vsimem): assert tile_ds.GetRasterBand(1).SetRasterColorTable(ct) == gdal.CE_None tile_ds = None - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") index_ds, lyr = create_basic_tileindex(index_filename, gdal.Open(tile_filename)) lyr.SetMetadataItem("BAND_COUNT", "4") lyr.SetMetadataItem("COLOR_INTERPRETATION", "Red,Green,Blue,Alpha") @@ -1608,9 +1608,9 @@ def test_vrttileindex_on_the_fly_rgba_color_table_expansion(tmp_vsimem): assert vrt_ds.GetRasterBand(4).ReadRaster() == b"\xFF" -def test_vrttileindex_on_the_fly_warping(tmp_vsimem): +def test_gti_on_the_fly_warping(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) warped_ds = gdal.Warp("", src_ds, format="VRT", dstSRS="EPSG:4267") @@ -1660,7 +1660,7 @@ def test_vrttileindex_on_the_fly_warping(tmp_vsimem): assert vrt_ds.GetRasterBand(1).ReadRaster(0, 0, 1, 1) == b"\xFE" -def test_vrttileindex_single_source_alpha_no_dest_nodata(tmp_vsimem): +def test_gti_single_source_alpha_no_dest_nodata(tmp_vsimem): filename1 = str(tmp_vsimem / "one.tif") ds = gdal.GetDriverByName("GTiff").Create(filename1, 2, 1, 2) @@ -1670,7 +1670,7 @@ def test_vrttileindex_single_source_alpha_no_dest_nodata(tmp_vsimem): ds.GetRasterBand(2).WriteRaster(0, 0, 2, 1, b"\xFF\x00") del ds - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") index_ds, _ = create_basic_tileindex(index_filename, gdal.Open(filename1)) del index_ds @@ -1684,7 +1684,7 @@ def test_vrttileindex_single_source_alpha_no_dest_nodata(tmp_vsimem): assert vrt_ds.GetRasterBand(2).ReadRaster() == b"\xFF\x00" -def test_vrttileindex_overlapping_opaque_sources(tmp_vsimem): +def test_gti_overlapping_opaque_sources(tmp_vsimem): filename1 = str(tmp_vsimem / "one.tif") ds = gdal.GetDriverByName("GTiff").Create(filename1, 1, 1, 1) @@ -1698,7 +1698,7 @@ def test_vrttileindex_overlapping_opaque_sources(tmp_vsimem): ds.GetRasterBand(1).WriteRaster(0, 0, 1, 1, b"\x02") del ds - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") sort_values = [1, 2] index_ds, _ = create_basic_tileindex( index_filename, @@ -1715,11 +1715,11 @@ def test_vrttileindex_overlapping_opaque_sources(tmp_vsimem): assert ( vrt_ds.GetRasterBand(1).GetMetadataItem("Pixel_0_0", "LocationInfo") - == "/vsimem/test_vrttileindex_overlapping_opaque_sources/two.tif" + == "/vsimem/test_gti_overlapping_opaque_sources/two.tif" ) -def test_vrttileindex_overlapping_sources_alpha_2x1(tmp_vsimem): +def test_gti_overlapping_sources_alpha_2x1(tmp_vsimem): filename1 = str(tmp_vsimem / "one.tif") ds = gdal.GetDriverByName("GTiff").Create(filename1, 2, 1, 2) @@ -1737,7 +1737,7 @@ def test_vrttileindex_overlapping_sources_alpha_2x1(tmp_vsimem): ds.GetRasterBand(2).WriteRaster(0, 0, 2, 1, b"\x00\xFE") del ds - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") sort_values = [1, 2] index_ds, _ = create_basic_tileindex( index_filename, @@ -1775,7 +1775,7 @@ def test_vrttileindex_overlapping_sources_alpha_2x1(tmp_vsimem): ) -def test_vrttileindex_overlapping_sources_alpha_1x2(tmp_vsimem): +def test_gti_overlapping_sources_alpha_1x2(tmp_vsimem): filename1 = str(tmp_vsimem / "one.tif") ds = gdal.GetDriverByName("GTiff").Create(filename1, 1, 2, 2) @@ -1793,7 +1793,7 @@ def test_vrttileindex_overlapping_sources_alpha_1x2(tmp_vsimem): ds.GetRasterBand(2).WriteRaster(0, 0, 1, 2, b"\x00\xFE") del ds - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") sort_values = [1, 2] index_ds, _ = create_basic_tileindex( index_filename, @@ -1827,7 +1827,7 @@ def test_vrttileindex_overlapping_sources_alpha_1x2(tmp_vsimem): ) -def test_vrttileindex_overlapping_sources_alpha_sse2_optim(tmp_vsimem): +def test_gti_overlapping_sources_alpha_sse2_optim(tmp_vsimem): filename1 = str(tmp_vsimem / "one.tif") ds = gdal.GetDriverByName("GTiff").Create(filename1, 17, 1, 2) @@ -1869,7 +1869,7 @@ def test_vrttileindex_overlapping_sources_alpha_sse2_optim(tmp_vsimem): ) del ds - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") sort_values = [1, 2] index_ds, _ = create_basic_tileindex( index_filename, @@ -1892,7 +1892,7 @@ def test_vrttileindex_overlapping_sources_alpha_sse2_optim(tmp_vsimem): ) -def test_vrttileindex_mix_rgb_rgba(tmp_vsimem): +def test_gti_mix_rgb_rgba(tmp_vsimem): filename1 = str(tmp_vsimem / "rgba.tif") ds = gdal.GetDriverByName("GTiff").Create(filename1, 2, 1, 4) @@ -1912,7 +1912,7 @@ def test_vrttileindex_mix_rgb_rgba(tmp_vsimem): ds.GetRasterBand(3).WriteRaster(0, 0, 2, 1, b"\x0C\x0D") del ds - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") sort_values = [1, 2] index_ds, _ = create_basic_tileindex( index_filename, @@ -1931,7 +1931,7 @@ def test_vrttileindex_mix_rgb_rgba(tmp_vsimem): assert vrt_ds.GetRasterBand(4).ReadRaster() == b"\xFF\xFF" assert vrt_ds.ReadRaster() == b"\x07\x08" + b"\x0A\x0B" + b"\x0C\x0D" + b"\xFF\xFF" - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") sort_values = [2, 1] index_ds, _ = create_basic_tileindex( index_filename, @@ -1951,7 +1951,7 @@ def test_vrttileindex_mix_rgb_rgba(tmp_vsimem): assert vrt_ds.ReadRaster() == b"\x01\x08" + b"\x03\x0B" + b"\x05\x0D" + b"\xFE\xFF" -def test_vrttileindex_overlapping_sources_mask_band(tmp_vsimem): +def test_gti_overlapping_sources_mask_band(tmp_vsimem): filename1 = str(tmp_vsimem / "one.tif") ds = gdal.GetDriverByName("GTiff").Create(filename1, 2, 1) @@ -1969,7 +1969,7 @@ def test_vrttileindex_overlapping_sources_mask_band(tmp_vsimem): ds.GetRasterBand(1).GetMaskBand().WriteRaster(0, 0, 2, 1, b"\x00\xFE") del ds - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") sort_values = [1, 2] index_ds, _ = create_basic_tileindex( index_filename, @@ -2002,9 +2002,9 @@ def test_vrttileindex_overlapping_sources_mask_band(tmp_vsimem): ) == (255, 254) -def test_vrttileindex_mask_band_explicit(tmp_vsimem): +def test_gti_mask_band_explicit(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -2019,7 +2019,7 @@ def test_vrttileindex_mask_band_explicit(tmp_vsimem): assert vrt_ds.GetRasterBand(1).GetMaskBand().ReadRaster() == b"\xFF" * (20 * 20) -def test_vrttileindex_flushcache(tmp_vsimem): +def test_gti_flushcache(tmp_vsimem): filename1 = str(tmp_vsimem / "one.tif") ds = gdal.GetDriverByName("GTiff").Create(filename1, 1, 1, 1) @@ -2027,7 +2027,7 @@ def test_vrttileindex_flushcache(tmp_vsimem): ds.GetRasterBand(1).WriteRaster(0, 0, 1, 1, b"\x01") del ds - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") index_ds, _ = create_basic_tileindex( index_filename, gdal.Open(filename1), @@ -2051,9 +2051,9 @@ def test_vrttileindex_flushcache(tmp_vsimem): assert vrt_ds.GetRasterBand(1).ReadRaster() == b"\x02" -def test_vrttileindex_ovr_factor(tmp_vsimem): +def test_gti_ovr_factor(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -2089,9 +2089,9 @@ def test_vrttileindex_ovr_factor(tmp_vsimem): ) == src_ds.ReadRaster(buf_xsize=10, buf_ysize=10) -def test_vrttileindex_ovr_factor_invalid(tmp_vsimem): +def test_gti_ovr_factor_invalid(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -2103,9 +2103,9 @@ def test_vrttileindex_ovr_factor_invalid(tmp_vsimem): assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 0 -def test_vrttileindex_ovr_ds_name(tmp_vsimem): +def test_gti_ovr_ds_name(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -2117,9 +2117,9 @@ def test_vrttileindex_ovr_ds_name(tmp_vsimem): assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 0 -def test_vrttileindex_ovr_lyr_name(tmp_vsimem): +def test_gti_ovr_lyr_name(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -2131,9 +2131,9 @@ def test_vrttileindex_ovr_lyr_name(tmp_vsimem): assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 0 -def test_vrttileindex_external_ovr(tmp_vsimem): +def test_gti_external_ovr(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, _ = create_basic_tileindex(index_filename, src_ds) @@ -2149,9 +2149,9 @@ def test_vrttileindex_external_ovr(tmp_vsimem): ) == src_ds.ReadRaster(buf_xsize=10, buf_ysize=10, resample_alg=gdal.GRIORA_Cubic) -def test_vrttileindex_dataset_metadata(tmp_vsimem): +def test_gti_dataset_metadata(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -2192,9 +2192,9 @@ def test_vrttileindex_dataset_metadata(tmp_vsimem): del vrt_ds -def test_vrttileindex_band_metadata(tmp_vsimem): +def test_gti_band_metadata(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -2283,31 +2283,31 @@ def test_vrttileindex_band_metadata(tmp_vsimem): } -def test_vrttileindex_connection_prefix(tmp_vsimem): +def test_gti_connection_prefix(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, _ = create_basic_tileindex(index_filename, src_ds) del index_ds - vrt_ds = gdal.Open(f"VRTTI:{index_filename}") + vrt_ds = gdal.Open(f"GTI:{index_filename}") check_basic(vrt_ds, src_ds) del vrt_ds -def test_vrttileindex_xml(tmp_vsimem): +def test_gti_xml(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, _ = create_basic_tileindex(index_filename, src_ds) del index_ds xml_filename = str(tmp_vsimem / "index.xml") - xml_content = f""" + xml_content = f""" {index_filename} -""" +""" vrt_ds = gdal.Open(xml_content) check_basic(vrt_ds, src_ds) @@ -2335,14 +2335,14 @@ def test_vrttileindex_xml(tmp_vsimem): } del vrt_ds - xml_content = f""" + xml_content = f""" {index_filename} invalid -""" +""" with pytest.raises(Exception, match="failed to prepare SQL"): gdal.Open(xml_content) - xml_content = f""" + xml_content = f""" {index_filename} 60 60 @@ -2359,7 +2359,7 @@ def test_vrttileindex_xml(tmp_vsimem): cat -""" +""" vrt_ds = gdal.Open(xml_content) band = vrt_ds.GetRasterBand(1) @@ -2375,79 +2375,79 @@ def test_vrttileindex_xml(tmp_vsimem): assert band.GetDefaultRAT() is not None del vrt_ds - xml_content = f""" + xml_content = f""" {index_filename} 60 60 -""" +""" with pytest.raises(Exception, match="band attribute missing on Band element"): gdal.Open(xml_content) - xml_content = f""" + xml_content = f""" {index_filename} 60 60 -""" +""" with pytest.raises(Exception, match="Invalid band number"): gdal.Open(xml_content) - xml_content = f""" + xml_content = f""" {index_filename} 60 60 -""" +""" with pytest.raises(Exception, match="Invalid band number: found 2, expected 1"): gdal.Open(xml_content) - xml_content = f""" + xml_content = f""" {index_filename} 60 60 2 -""" +""" with pytest.raises( Exception, match="Inconsistent BandCount with actual number of Band elements" ): gdal.Open(xml_content) - xml_content = f""" + xml_content = f""" {index_filename} 2 -""" +""" vrt_ds = gdal.Open(xml_content) assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 1 assert vrt_ds.GetRasterBand(1).GetOverview(0).XSize == 10 del vrt_ds - xml_content = f""" + xml_content = f""" {index_filename} {index_filename} -""" +""" vrt_ds = gdal.Open(xml_content) assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 1 assert vrt_ds.GetRasterBand(1).GetOverview(0).XSize == 20 del vrt_ds - xml_content = f""" + xml_content = f""" {index_filename} index -""" +""" vrt_ds = gdal.Open(xml_content) assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 1 assert vrt_ds.GetRasterBand(1).GetOverview(0).XSize == 20 del vrt_ds - xml_content = f""" + xml_content = f""" {index_filename} index @@ -2455,13 +2455,13 @@ def test_vrttileindex_xml(tmp_vsimem): 2 -""" +""" vrt_ds = gdal.Open(xml_content) assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 1 assert vrt_ds.GetRasterBand(1).GetOverview(0).XSize == 10 del vrt_ds - xml_content = """""" + xml_content = """""" with gdaltest.tempfile(xml_filename, xml_content): with pytest.raises( Exception, - match="Missing VRTTileIndexDataset root element", + match="Missing GDALTileIndexDataset root element", ): gdal.Open(xml_filename) - xml_content = """""" + xml_content = """""" with pytest.raises( Exception, match="Missing IndexDataset element", ): gdal.Open(xml_content) - xml_content = """ + xml_content = """ i_do_not_exist -""" +""" with pytest.raises( Exception, match="i_do_not_exist", ): gdal.Open(xml_content) - xml_content = f""" + xml_content = f""" {index_filename} i_do_not_exist -""" +""" with pytest.raises( Exception, match="i_do_not_exist", ): gdal.Open(xml_content) - xml_content = f""" + xml_content = f""" {index_filename} -""" +""" with pytest.raises( Exception, match="At least one of Dataset, Layer or Factor element must be present as an Overview child", ): gdal.Open(xml_content) - xml_content = f""" + xml_content = f""" {index_filename} i_do_not_exist -""" +""" vrt_ds = gdal.Open(xml_content) with pytest.raises(Exception, match="i_do_not_exist"): assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 0 - xml_content = f""" + xml_content = f""" {index_filename} i_do_not_exist -""" +""" vrt_ds = gdal.Open(xml_content) with pytest.raises(Exception, match="i_do_not_exist"): assert vrt_ds.GetRasterBand(1).GetOverviewCount() == 0 -def test_vrttileindex_open_options(tmp_vsimem): +def test_gti_open_options(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, _ = create_basic_tileindex(index_filename, src_ds) @@ -2555,14 +2555,14 @@ def test_vrttileindex_open_options(tmp_vsimem): ) -def test_vrttileindex_xml_vrtti_embedded(tmp_vsimem): +def test_gti_xml_vrtti_embedded(tmp_vsimem): - index_filename = str(tmp_vsimem / "index.vrt.gpkg") + index_filename = str(tmp_vsimem / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) - xml_content = """ + xml_content = """ 60 60 location @@ -2578,9 +2578,9 @@ def test_vrttileindex_xml_vrtti_embedded(tmp_vsimem): cat -""" +""" - lyr.SetMetadata([xml_content], "xml:VRTTI") + lyr.SetMetadata([xml_content], "xml:GTI") del index_ds vrt_ds = gdal.Open(index_filename) diff --git a/autotest/slow_tests/raster.py b/autotest/slow_tests/raster.py index 210d730e8976..c96198b55f33 100644 --- a/autotest/slow_tests/raster.py +++ b/autotest/slow_tests/raster.py @@ -178,7 +178,7 @@ def test_translate_vrtti(tmp_path): del tile_ds print("Creating tile index...") - tileindex = str(tmp_path / "tileindex.vrt.gpkg") + tileindex = str(tmp_path / "tileindex.gti.gpkg") ds = ogr.GetDriverByName("GPKG").CreateDataSource(tileindex) lyr = ds.CreateLayer("index", geom_type=ogr.wkbPolygon, srs=srs) lyr.CreateField(ogr.FieldDefn("location", ogr.OFTString)) diff --git a/autotest/utilities/test_gdaladdo.py b/autotest/utilities/test_gdaladdo.py index a36af5ef62ba..29f23085d3a8 100755 --- a/autotest/utilities/test_gdaladdo.py +++ b/autotest/utilities/test_gdaladdo.py @@ -378,11 +378,11 @@ def test_gdaladdo_reuse_previous_resampling_and_levels( ############################################################################### -# Test --partial-refresh-from-source-timestamp with VRTTI dataset +# Test --partial-refresh-from-source-timestamp with GTI dataset @pytest.mark.require_driver("GPKG") -def test_gdaladdo_partial_refresh_from_source_timestamp_vrtti(gdaladdo_path, tmp_path): +def test_gdaladdo_partial_refresh_from_source_timestamp_gti(gdaladdo_path, tmp_path): left_tif = str(tmp_path / "left.tif") right_tif = str(tmp_path / "right.tif") @@ -391,7 +391,7 @@ def test_gdaladdo_partial_refresh_from_source_timestamp_vrtti(gdaladdo_path, tmp gdal.Translate(right_tif, "../gcore/data/byte.tif", options="-srcwin 10 0 10 20") source_ds = [gdal.Open(left_tif), gdal.Open(right_tif)] - tmp_vrt = str(tmp_path / "test.vrt.gpkg") + tmp_vrt = str(tmp_path / "test.gti.gpkg") index_ds = ogr.GetDriverByName("GPKG").CreateDataSource(tmp_vrt) lyr = index_ds.CreateLayer("index", srs=source_ds[0].GetSpatialRef()) lyr.CreateField(ogr.FieldDefn("location")) diff --git a/autotest/utilities/test_gdaltindex_lib.py b/autotest/utilities/test_gdaltindex_lib.py index fedd2a778d3f..ee3d59525b4b 100644 --- a/autotest/utilities/test_gdaltindex_lib.py +++ b/autotest/utilities/test_gdaltindex_lib.py @@ -264,14 +264,14 @@ def test_gdaltindex_lib_overwrite(tmp_path, four_tiles): ############################################################################### -# Test VRTTI related options +# Test GTI related options @pytest.mark.require_driver("GPKG") -@pytest.mark.require_driver("VRTTI") -def test_gdaltindex_lib_vrtti_non_xml(tmp_path, four_tiles): +@pytest.mark.require_driver("GTI") +def test_gdaltindex_lib_gti_non_xml(tmp_path, four_tiles): - index_filename = str(tmp_path / "test_gdaltindex_lib_vrtti_non_xml.vrt.gpkg") + index_filename = str(tmp_path / "test_gdaltindex_lib_gti_non_xml.gti.gpkg") gdal.TileIndex( index_filename, @@ -312,21 +312,21 @@ def test_gdaltindex_lib_vrtti_non_xml(tmp_path, four_tiles): ############################################################################### -# Test VRTTI related options +# Test GTI related options @pytest.mark.require_driver("GPKG") -@pytest.mark.require_driver("VRTTI") -def test_gdaltindex_lib_vrtti_xml(tmp_path, four_tiles): +@pytest.mark.require_driver("GTI") +def test_gdaltindex_lib_gti_xml(tmp_path, four_tiles): - index_filename = str(tmp_path / "test_gdaltindex_lib_vrtti_non_xml.vrt.gpkg") - vrtti_filename = str(tmp_path / "test_gdaltindex_lib_vrtti_non_xml.vrtti") + index_filename = str(tmp_path / "test_gdaltindex_lib_gti_non_xml.gti.gpkg") + gti_filename = str(tmp_path / "test_gdaltindex_lib_gti_non_xml.gti") gdal.TileIndex( index_filename, four_tiles, layerName="tileindex", - vrttiFilename=vrtti_filename, + gtiFilename=gti_filename, xRes=60, yRes=60, outputBounds=[0, 1, 2, 3], @@ -335,8 +335,8 @@ def test_gdaltindex_lib_vrtti_xml(tmp_path, four_tiles): mask=True, ) - xml = open(vrtti_filename, "rb").read().decode("UTF-8") - assert "test_gdaltindex_lib_vrtti_non_xml.vrt.gpkg" in xml + xml = open(gti_filename, "rb").read().decode("UTF-8") + assert "test_gdaltindex_lib_gti_non_xml.gti.gpkg" in xml assert "tileindex" in xml assert "location" in xml assert "60" in xml @@ -351,7 +351,7 @@ def test_gdaltindex_lib_vrtti_xml(tmp_path, four_tiles): ) assert "true" in xml - ds = gdal.Open(vrtti_filename) + ds = gdal.Open(gti_filename) assert ds.GetGeoTransform() == (0.0, 60.0, 0.0, 3.0, 0.0, -60.0) assert ds.RasterCount == 1 assert ds.GetRasterBand(1).GetNoDataValue() == 0 diff --git a/doc/source/drivers/raster/vrtti.rst b/doc/source/drivers/raster/gti.rst similarity index 86% rename from doc/source/drivers/raster/vrtti.rst rename to doc/source/drivers/raster/gti.rst index 313fa4cee448..e389918e07c8 100644 --- a/doc/source/drivers/raster/vrtti.rst +++ b/doc/source/drivers/raster/gti.rst @@ -1,19 +1,19 @@ -.. _raster.vrtti: +.. _raster.gti: ================================================================================ -VRTTI -- GDAL Virtual Raster Tile Index +GTI -- GDAL Raster Tile Index ================================================================================ .. versionadded:: 3.9 -.. shortname:: VRTTI +.. shortname:: GTI .. built_in_by_default:: Introduction ------------ -The VRTTI driver is a driver that allows to handle catalogs with a large +The GTI driver is a driver that allows to handle catalogs with a large number of raster files (called "tiles" in the rest of this document, even if a regular tiling is not required by the driver), and build a virtual mosaic from them. Each tile may be in any GDAL supported raster format, and be a file @@ -26,52 +26,52 @@ driver with the following main differences: * The tiles are listed as features of any GDAL supported vector format. Use of formats with efficient spatial filtering is recommended, such as :ref:`GeoPackage `, :ref:`FlatGeoBuf ` or - :ref:`PostGIS `. The VRTTI driver can thus use a larger number of + :ref:`PostGIS `. The GTI driver can thus use a larger number of tiles than the VRT driver (hundreds of thousands or more), provided the underlying vector format is efficient. -* The tiles may have different SRS. The VRTTI driver is capable of on-the-fly +* The tiles may have different SRS. The GTI driver is capable of on-the-fly reprojection -* The VRTTI driver offers control on the order in which tiles are composited, +* The GTI driver offers control on the order in which tiles are composited, when they overlap (z-order) -* The VRTTI driver honours the mask/alpha band when compositing together +* The GTI driver honours the mask/alpha band when compositing together overlapping tiles. -* Contrary to the VRT driver, the VRTTI driver does not enable to alter +* Contrary to the VRT driver, the GTI driver does not enable to alter characteristics of referenced tiles, such as their georeferencing, nodata value, etc. If such behavior is desired, the tiles must be for example wrapped - individually in a VRT file before being referenced in the VRTTI index. + individually in a VRT file before being referenced in the GTI index. Connection strings ------------------ -The VRTTI driver accepts different types of connection strings: +The GTI driver accepts different types of connection strings: -* a vector file in GeoPackage format with a ``.vrt.gpkg`` extension, or in - FlatGeoBuf format with a ``.vrt.fgb`` extension, meeting the minimum requirements - for a VRTTI compatible tile index, detailed later. +* a vector file in GeoPackage format with a ``.gti.gpkg`` extension, or in + FlatGeoBuf format with a ``.gti.fgb`` extension, meeting the minimum requirements + for a GTI compatible tile index, detailed later. - For example: ``tileindex.vrt.gpkg`` + For example: ``tileindex.gti.gpkg`` * any vector file in a GDAL supported format, with its filename (or connection - string prefixed with ``VRTTI:`` + string prefixed with ``GTI:`` - For example: ``VRTTI:tileindex.shp`` or ``VRTTI:PG:database=my_db schema=tileindex`` + For example: ``GTI:tileindex.shp`` or ``GTI:PG:database=my_db schema=tileindex`` -* a XML file, following the below VRTTI XML format, generally with the - recommended ``.vrtti`` extension, referencing a vector file. Using such +* a XML file, following the below GTI XML format, generally with the + recommended ``.gti`` extension, referencing a vector file. Using such XML file may be more practical for tile indexes not stored in a file, or if some additional metadata must be defined at the dataset or band level of the virtual mosaic. - For example: ``tileindex.vrtti`` + For example: ``tileindex.gti`` Tile index requirements ----------------------- -The minimum requirements for a VRTTI compatible tile index is to be a +The minimum requirements for a GTI compatible tile index is to be a vector format supported by GDAL, with a geometry column storing polygons with the extent of the tiles, and an attribute field of type string, storing the path to each tile. The default name for this attribute field is ``location``. @@ -191,8 +191,8 @@ PostGIS, ...), the following layer metadata items may be set: * ``OVERVIEW__DATASET=`` where idx is an integer index starting at 0. Name of the dataset to use as the first overview level. This may be a - raster dataset (for example a GeoTIFF file, or another VRTTI dataset). - This may also be a vector dataset with a VRTTI compatible layer, potentially + raster dataset (for example a GeoTIFF file, or another GTI dataset). + This may also be a vector dataset with a GTI compatible layer, potentially specified with ``OVERVIEW__LAYER``. * ``OVERVIEW__OPEN_OPTIONS=[,key2=value2]...`` where idx is an integer index starting at 0. @@ -216,7 +216,7 @@ PostGIS, ...), the following layer metadata items may be set: specified to point to another tile index. All overviews *must* have exactly the same extent as the full resolution -virtual mosaic. The VRTTI driver does not check that, and if that condition is +virtual mosaic. The GTI driver does not check that, and if that condition is not met, subsampled pixel request will lead to incorrect result. In addition to those layer metadata items, the dataset-level metadata item @@ -224,14 +224,14 @@ In addition to those layer metadata items, the dataset-level metadata item which one should be used as the tile index layer. Alternatively to setting those metadata items individually, the corresponding -information can be grouped together in a VRTTI XML document, attached in the -``xml:VRTTI`` metadata domain of the layer (for drivers that support alternate +information can be grouped together in a GTI XML document, attached in the +``xml:GTI`` metadata domain of the layer (for drivers that support alternate metadata domains such as GeoPackage) -VRTTI XML format +GTI XML format ---------------- -A `XML schema of the GDAL VRTTI format `_ +A `XML schema of the GDAL GTI format `_ is available. The following artificial example contains all potential elements and attributes. @@ -240,8 +240,8 @@ mentioned in the previous section. .. code-block:: xml - - PG:dbname=my_db + + PG:dbname=my_db my_layer pub_date >= '2023/12/01' pub_date @@ -312,8 +312,8 @@ mentioned in the previous section. some.tif - - other.vrt.gpkg + + other.gti.gpkg other_layer 0 @@ -323,10 +323,10 @@ mentioned in the previous section. - + -At the VRTTileIndexDataset level, the elements specific to VRTTI XML are: +At the GDALTileIndexDataset level, the elements specific to GTI XML are: * ``Filter``: value of a SQL WHERE clause, used to select a subset of the features of the index. @@ -337,20 +337,20 @@ At the VRTTileIndexDataset level, the elements specific to VRTTI XML are: * ``Metadata``: defines dataset-level metadata. You can refer to the documentation of the :ref:`VRT ` driver for its syntax. -At the Band level, the elements specific to VRTTI XML are: Description, +At the Band level, the elements specific to GTI XML are: Description, Offset, Scale, UnitType, ColorTable, CategoryNames, GDALRasterAttributeTable, Metadata. You can refer to the documentation of the :ref:`VRT ` driver for their syntax and semantics. -How to build a VRTTI comptatible index ? +How to build a GTI comptatible index ? ---------------------------------------- The :ref:`gdaltindex` program may be used to generate both a vector tile index, -and optionally a wrapping .vrtti XML file. +and optionally a wrapping .gti XML file. -A VRTTI comptatible index may also be created by any programmatic means, provided +A GTI comptatible index may also be created by any programmatic means, provided the above format specifications are met. @@ -358,7 +358,7 @@ Open options ------------ The following open options are available. Most of them can be -also defined as layer metadata items or in the .vrtti XML file +also defined as layer metadata items or in the .gti XML file - .. oo:: LAYER diff --git a/doc/source/drivers/raster/index.rst b/doc/source/drivers/raster/index.rst index 426311e960dc..e9230e34d44e 100644 --- a/doc/source/drivers/raster/index.rst +++ b/doc/source/drivers/raster/index.rst @@ -81,6 +81,7 @@ Raster drivers gsbg gsc gta + gti gtiff gxf hdf4 @@ -179,7 +180,6 @@ Raster drivers usgsdem vicar vrt - vrtti wcs webp wms diff --git a/doc/source/drivers/raster/vrt.rst b/doc/source/drivers/raster/vrt.rst index d46d47e0b21d..8b3333304e81 100644 --- a/doc/source/drivers/raster/vrt.rst +++ b/doc/source/drivers/raster/vrt.rst @@ -21,7 +21,7 @@ The VRT format can also describe :ref:`gdal_vrttut_warped` and :ref:`gdal_vrttut_pansharpen` For mosaic with a very large number of tiles (tens of thousands or mores), -the :ref:`VRTTI ` driver may be used starting with GDAL 3.9. +the :ref:`GTI ` driver may be used starting with GDAL 3.9. An example of a simple .vrt file referring to a 512x512 dataset with one band loaded from :file:`utm.tif` might look like this: diff --git a/doc/source/programs/gdalbuildvrt.rst b/doc/source/programs/gdalbuildvrt.rst index 48afa0bc909a..d008a5eb9ab5 100644 --- a/doc/source/programs/gdalbuildvrt.rst +++ b/doc/source/programs/gdalbuildvrt.rst @@ -45,7 +45,7 @@ entries in the tile index will be added to the VRT. Starting with GDAL 3.9, for virtual mosaic with a very large number of tiles (typically hundreds of thousands of tiles, or more), it is advised to use the :ref:`gdaltindex` utility to generate a tile index compatible of the - :ref:`VRTTI ` driver. + :ref:`GTI ` driver. With -separate, each files goes into a separate band in the VRT dataset. Otherwise, the files are considered as tiles of a larger mosaic and the VRT file has as many bands as one diff --git a/doc/source/programs/gdaltindex.rst b/doc/source/programs/gdaltindex.rst index cb7fb1afdcea..a5707a16852c 100644 --- a/doc/source/programs/gdaltindex.rst +++ b/doc/source/programs/gdaltindex.rst @@ -22,7 +22,7 @@ Synopsis [-skip_different_projection] [-t_srs ] [-src_srs_name ] [-src_srs_format {AUTO|WKT|EPSG|PROJ}] [-lyr_name ] - [-vrtti_filename ] + [-gti_filename ] [-tr ] [-te ] [-ot ] [-bandcount ] [-nodata [,...]] [-colorinterp [,...]] [-mask] @@ -36,7 +36,7 @@ Description This program creates an OGR-supported dataset with a record for each input raster file, an attribute containing the filename, and a polygon geometry outlining the raster. This output is suitable for use with `MapServer `__ as a raster -tileindex, or as input for the :ref:`VRTTI ` driver. +tileindex, or as input for the :ref:`GTI ` driver. .. program:: gdaltindex @@ -144,39 +144,39 @@ tileindex, or as input for the :ref:`VRTTI ` driver. might also be used to recursve down to sub-directories. -Options specific to use by the GDAL VRTTI driver +Options specific to use by the GDAL GTI driver ------------------------------------------------ gdaltindex can be used to generate a tile index suitable for use by the -:ref:`VRTTI ` driver. There are two possibilities: +:ref:`GTI ` driver. There are two possibilities: - either use directly a vector tile index generated by gdaltindex as the input - of the VRTTI driver + of the GTI driver -- or generate a small XML .vrtti wrapper file, for easier use with non-file-based +- or generate a small XML .gti wrapper file, for easier use with non-file-based formats such as databases, or for vector formats that do not support setting layer metadata items. Formats that support layer metadata are for example GeoPackage (``-f GPKG``), FlatGeoBuf (``-f FlatGeoBuf``) or PostGIS (``-f PG``) -Setting :option:`-tr` and :option:`-ot` is recommended to avoid the VRTTI +Setting :option:`-tr` and :option:`-ot` is recommended to avoid the GTI driver to have to deduce them by opening the first tile in the index. If the tiles have nodata or mask band, :option:`-nodata` and :option:`-mask` should also be set. -In a VRTTI context, the extent of all tiles referenced in the tile index must +In a GTI context, the extent of all tiles referenced in the tile index must be expressed in a single SRS. Consequently, if input tiles may have different SRS, either :option:`-t_srs` or :option:`-skip_different_projection` should be specified. -.. option:: -vrtti_filename +.. option:: -gti_filename .. versionadded:: 3.9 Filename of the XML Virtual Tile Index file to generate, that can be used - as an input for the GDAL VRTTI / Virtual Raster Tile Index driver. + as an input for the GDAL GTI / Virtual Raster Tile Index driver. This can be useful when writing the tile index in a vector format that does not support writing layer metadata items. @@ -186,7 +186,7 @@ specified. Target resolution in SRS unit per pixel. - Written in the XML Virtutual Tile Index if :option:`-vrtti_filename` + Written in the XML Virtutual Tile Index if :option:`-gti_filename` is specified, or as ``RESX`` and ``RESY`` layer metadata items for formats that support layer metadata. @@ -196,7 +196,7 @@ specified. Target extent in SRS unit. - Written in the XML Virtutual Tile Index if :option:`-vrtti_filename` + Written in the XML Virtutual Tile Index if :option:`-gti_filename` is specified, or as ``MINX``, ``MINY``, ``MAXX`` and ``MAXY`` layer metadata items for formats that support layer metadata. @@ -208,7 +208,7 @@ specified. ``Int16``, ``UInt32``, ``Int32``, ``UInt64``, ``Int64``, ``Float32``, ``Float64``, ``CInt16``, ``CInt32``, ``CFloat32`` or ``CFloat64`` - Written in the XML Virtutual Tile Index if :option:`-vrtti_filename` + Written in the XML Virtutual Tile Index if :option:`-gti_filename` is specified, or as ``DATA_TYPE`` layer metadata item for formats that support layer metadata. @@ -218,7 +218,7 @@ specified. Number of bands of the tiles of the tile index. - Written in the XML Virtutual Tile Index if :option:`-vrtti_filename` + Written in the XML Virtutual Tile Index if :option:`-gti_filename` is specified, or as ``BAND_COUNT`` layer metadata item for formats that support layer metadata. @@ -235,7 +235,7 @@ specified. Nodata value of the tiles of the tile index. - Written in the XML Virtutual Tile Index if :option:`-vrtti_filename` + Written in the XML Virtutual Tile Index if :option:`-gti_filename` is specified, or as ``NODATA`` layer metadata item for formats that support layer metadata. @@ -246,7 +246,7 @@ specified. Color interpretation of of the tiles of the tile index: ``red``, ``green``, ``blue``, ``alpha``, ``gray``, ``undefined``. - Written in the XML Virtutual Tile Index if :option:`-vrtti_filename` + Written in the XML Virtutual Tile Index if :option:`-gti_filename` is specified, or as ``COLOR_INTERPRETATION`` layer metadata item for formats that support layer metadata. @@ -256,7 +256,7 @@ specified. Whether tiles in the tile index have a mask band. - Written in the XML Virtutual Tile Index if :option:`-vrtti_filename` + Written in the XML Virtutual Tile Index if :option:`-gti_filename` is specified, or as ``MASK_BAND`` layer metadata item for formats that support layer metadata. @@ -268,7 +268,7 @@ specified. metadata. This option may be repeated. - .. note:: This option cannot be used together :option:`-vrtti_filename` + .. note:: This option cannot be used together :option:`-gti_filename` .. option:: -fetch_md @@ -315,11 +315,11 @@ Examples gdaltindex -t_srs EPSG:4326 -src_srs_name src_srs tile_index_mixed_srs.shp *.tif - Make a tile index from files listed in a text file, with metadata suitable - for use by the GDAL VRTTI / Virtual Raster Tile Index driver. + for use by the GDAL GTI / Virtual Raster Tile Index driver. :: - gdaltindex tile_index.vrt.gpkg -datatype Byte -tr 60 60 -colorinterp Red,Green,Blue --optfile my_list.txt + gdaltindex tile_index.gti.gpkg -datatype Byte -tr 60 60 -colorinterp Red,Green,Blue --optfile my_list.txt C API ----- diff --git a/frmts/drivers.ini b/frmts/drivers.ini index 06e69303af0c..01cce7154ea0 100644 --- a/frmts/drivers.ini +++ b/frmts/drivers.ini @@ -12,7 +12,7 @@ [order] VRT -VRTTI +GTI Derived GTiff COG diff --git a/frmts/gdalallregister.cpp b/frmts/gdalallregister.cpp index 4510ebf10827..0e1b73b85fc2 100644 --- a/frmts/gdalallregister.cpp +++ b/frmts/gdalallregister.cpp @@ -323,7 +323,7 @@ void CPL_STDCALL GDALAllRegister() #ifdef FRMT_vrt GDALRegister_VRT(); - GDALRegister_VRTTI(); + GDALRegister_GTI(); GDALRegister_Derived(); #endif diff --git a/frmts/vrt/CMakeLists.txt b/frmts/vrt/CMakeLists.txt index 4f35a8baca61..ba770561f569 100644 --- a/frmts/vrt/CMakeLists.txt +++ b/frmts/vrt/CMakeLists.txt @@ -15,14 +15,14 @@ add_gdal_driver( pixelfunctions.cpp vrtpansharpened.cpp vrtmultidim.cpp - vrttileindexdataset.cpp + gdaltileindexdataset.cpp STRONG_CXX_WFLAGS) gdal_standard_includes(gdal_vrt) target_include_directories(gdal_vrt PRIVATE ${GDAL_RASTER_FORMAT_SOURCE_DIR}/raw) set(GDAL_DATA_FILES ${CMAKE_CURRENT_SOURCE_DIR}/data/gdalvrt.xsd - ${CMAKE_CURRENT_SOURCE_DIR}/data/gdalvrtti.xsd + ${CMAKE_CURRENT_SOURCE_DIR}/data/gdaltileindex.xsd ) set_property( TARGET ${GDAL_LIB_TARGET_NAME} diff --git a/frmts/vrt/data/gdalvrtti.xsd b/frmts/vrt/data/gdaltileindex.xsd similarity index 98% rename from frmts/vrt/data/gdalvrtti.xsd rename to frmts/vrt/data/gdaltileindex.xsd index e8d314021b3f..7e0d38a35599 100644 --- a/frmts/vrt/data/gdalvrtti.xsd +++ b/frmts/vrt/data/gdaltileindex.xsd @@ -4,7 +4,7 @@ * $Id$ * * Project: GDAL/OGR - * Purpose: XML Schema for GDAL VRTTI files. + * Purpose: XML Schema for GDAL GTI files. * Author: Even Rouault, * ********************************************************************** @@ -30,12 +30,12 @@ ****************************************************************************/ --> - + - - + + diff --git a/frmts/vrt/vrttileindexdataset.cpp b/frmts/vrt/gdaltileindexdataset.cpp similarity index 91% rename from frmts/vrt/vrttileindexdataset.cpp rename to frmts/vrt/gdaltileindexdataset.cpp index 82380458c0b1..e80365c34ab6 100644 --- a/frmts/vrt/vrttileindexdataset.cpp +++ b/frmts/vrt/gdaltileindexdataset.cpp @@ -62,7 +62,7 @@ constexpr int GT_TOPLEFT_Y = 3; constexpr int GT_ROTATION_PARAM2 = 4; constexpr int GT_NS_RES = 5; -constexpr const char *VRTTI_PREFIX = "VRTTI:"; +constexpr const char *GTI_PREFIX = "GTI:"; constexpr const char *MD_DS_TILE_INDEX_LAYER = "TILE_INDEX_LAYER"; @@ -116,31 +116,31 @@ constexpr const char *const MD_BAND_UNITTYPE = "UNITTYPE"; constexpr const char *const apszReservedBandItems[] = { MD_BAND_OFFSET, MD_BAND_SCALE, MD_BAND_UNITTYPE}; -constexpr const char *VRTTI_XML_BANDCOUNT = "BandCount"; -constexpr const char *VRTTI_XML_DATATYPE = "DataType"; -constexpr const char *VRTTI_XML_NODATAVALUE = "NoDataValue"; -constexpr const char *VRTTI_XML_COLORINTERP = "ColorInterp"; -constexpr const char *VRTTI_XML_LOCATIONFIELD = "LocationField"; -constexpr const char *VRTTI_XML_SORTFIELD = "SortField"; -constexpr const char *VRTTI_XML_SORTFIELDASC = "SortFieldAsc"; -constexpr const char *VRTTI_XML_MASKBAND = "MaskBand"; -constexpr const char *VRTTI_XML_OVERVIEW_ELEMENT = "Overview"; -constexpr const char *VRTTI_XML_OVERVIEW_DATASET = "Dataset"; -constexpr const char *VRTTI_XML_OVERVIEW_LAYER = "Layer"; -constexpr const char *VRTTI_XML_OVERVIEW_FACTOR = "Factor"; - -constexpr const char *VRTTI_XML_BAND_ELEMENT = "Band"; -constexpr const char *VRTTI_XML_BAND_NUMBER = "band"; -constexpr const char *VRTTI_XML_BAND_DATATYPE = "dataType"; -constexpr const char *VRTTI_XML_BAND_DESCRIPTION = "Description"; -constexpr const char *VRTTI_XML_BAND_OFFSET = "Offset"; -constexpr const char *VRTTI_XML_BAND_SCALE = "Scale"; -constexpr const char *VRTTI_XML_BAND_NODATAVALUE = "NoDataValue"; -constexpr const char *VRTTI_XML_BAND_UNITTYPE = "UnitType"; -constexpr const char *VRTTI_XML_BAND_COLORINTERP = "ColorInterp"; -constexpr const char *VRTTI_XML_CATEGORYNAMES = "CategoryNames"; -constexpr const char *VRTTI_XML_COLORTABLE = "ColorTable"; -constexpr const char *VRTTI_XML_RAT = "GDALRasterAttributeTable"; +constexpr const char *GTI_XML_BANDCOUNT = "BandCount"; +constexpr const char *GTI_XML_DATATYPE = "DataType"; +constexpr const char *GTI_XML_NODATAVALUE = "NoDataValue"; +constexpr const char *GTI_XML_COLORINTERP = "ColorInterp"; +constexpr const char *GTI_XML_LOCATIONFIELD = "LocationField"; +constexpr const char *GTI_XML_SORTFIELD = "SortField"; +constexpr const char *GTI_XML_SORTFIELDASC = "SortFieldAsc"; +constexpr const char *GTI_XML_MASKBAND = "MaskBand"; +constexpr const char *GTI_XML_OVERVIEW_ELEMENT = "Overview"; +constexpr const char *GTI_XML_OVERVIEW_DATASET = "Dataset"; +constexpr const char *GTI_XML_OVERVIEW_LAYER = "Layer"; +constexpr const char *GTI_XML_OVERVIEW_FACTOR = "Factor"; + +constexpr const char *GTI_XML_BAND_ELEMENT = "Band"; +constexpr const char *GTI_XML_BAND_NUMBER = "band"; +constexpr const char *GTI_XML_BAND_DATATYPE = "dataType"; +constexpr const char *GTI_XML_BAND_DESCRIPTION = "Description"; +constexpr const char *GTI_XML_BAND_OFFSET = "Offset"; +constexpr const char *GTI_XML_BAND_SCALE = "Scale"; +constexpr const char *GTI_XML_BAND_NODATAVALUE = "NoDataValue"; +constexpr const char *GTI_XML_BAND_UNITTYPE = "UnitType"; +constexpr const char *GTI_XML_BAND_COLORINTERP = "ColorInterp"; +constexpr const char *GTI_XML_CATEGORYNAMES = "CategoryNames"; +constexpr const char *GTI_XML_COLORTABLE = "ColorTable"; +constexpr const char *GTI_XML_RAT = "GDALRasterAttributeTable"; /************************************************************************/ /* ENDS_WITH_CI() */ @@ -152,13 +152,13 @@ static inline bool ENDS_WITH_CI(const char *a, const char *b) } /************************************************************************/ -/* VRTTileIndexDataset */ +/* GDALTileIndexDataset */ /************************************************************************/ -class VRTTileIndexBand; -class VRTTileIndexDataset final : public GDALPamDataset +class GDALTileIndexBand; +class GDALTileIndexDataset final : public GDALPamDataset { - friend class VRTTileIndexBand; + friend class GDALTileIndexBand; CPLXMLTreeCloser m_psXMLTree{nullptr}; bool m_bXMLUpdatable = false; @@ -171,7 +171,7 @@ class VRTTileIndexDataset final : public GDALPamDataset OGRSpatialReference m_oSRS{}; lru11::Cache> m_oMapSharedSources{ 500}; - std::unique_ptr m_poMaskBand{}; + std::unique_ptr m_poMaskBand{}; bool m_bSameDataType = true; bool m_bSameNoData = true; double m_dfLastMinXFilter = std::numeric_limits::quiet_NaN(); @@ -216,11 +216,11 @@ class VRTTileIndexDataset final : public GDALPamDataset bool TileIndexSupportsEditingLayerMetadata() const; - CPL_DISALLOW_COPY_ASSIGN(VRTTileIndexDataset) + CPL_DISALLOW_COPY_ASSIGN(GDALTileIndexDataset) public: - VRTTileIndexDataset(); - ~VRTTileIndexDataset() override; + GDALTileIndexDataset(); + ~GDALTileIndexDataset() override; bool Open(GDALOpenInfo *poOpenInfo); @@ -244,18 +244,18 @@ class VRTTileIndexDataset final : public GDALPamDataset void LoadOverviews(); - std::vector GetSourcesMoreRecentThan(int64_t mTime); + std::vector GetSourcesMoreRecentThan(int64_t mTime); }; /************************************************************************/ -/* VRTTileIndexBand */ +/* GDALTileIndexBand */ /************************************************************************/ -class VRTTileIndexBand final : public GDALPamRasterBand +class GDALTileIndexBand final : public GDALPamRasterBand { - friend class VRTTileIndexDataset; + friend class GDALTileIndexDataset; - VRTTileIndexDataset *m_poDS = nullptr; + GDALTileIndexDataset *m_poDS = nullptr; bool m_bNoDataValueSet = false; double m_dfNoDataValue = 0; GDALColorInterp m_eColorInterp = GCI_Undefined; @@ -267,11 +267,11 @@ class VRTTileIndexBand final : public GDALPamRasterBand std::unique_ptr m_poColorTable{}; std::unique_ptr m_poRAT{}; - CPL_DISALLOW_COPY_ASSIGN(VRTTileIndexBand) + CPL_DISALLOW_COPY_ASSIGN(GDALTileIndexBand) public: - VRTTileIndexBand(VRTTileIndexDataset *poDSIn, int nBandIn, GDALDataType eDT, - int nBlockXSizeIn, int nBlockYSizeIn); + GDALTileIndexBand(GDALTileIndexDataset *poDSIn, int nBandIn, + GDALDataType eDT, int nBlockXSizeIn, int nBlockYSizeIn); double GetNoDataValue(int *pbHasNoData) override { @@ -381,10 +381,10 @@ static inline bool IsSameNaNAware(double a, double b) } /************************************************************************/ -/* VRTTileIndexDataset() */ +/* GDALTileIndexDataset() */ /************************************************************************/ -VRTTileIndexDataset::VRTTileIndexDataset() +GDALTileIndexDataset::GDALTileIndexDataset() : m_osUniqueHandle(CPLSPrintf("%p", this)) { } @@ -398,7 +398,7 @@ static std::string GetAbsoluteFileName(const char *pszTileName, { if (CPLIsFilenameRelative(pszTileName) && !STARTS_WITH(pszTileName, "GetPathComponent().empty()) @@ -427,25 +427,25 @@ static std::string GetAbsoluteFileName(const char *pszTileName, /* Open() */ /************************************************************************/ -bool VRTTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) +bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) { eAccess = poOpenInfo->eAccess; CPLXMLNode *psRoot = nullptr; const char *pszIndexDataset = poOpenInfo->pszFilename; - if (STARTS_WITH(poOpenInfo->pszFilename, VRTTI_PREFIX)) + if (STARTS_WITH(poOpenInfo->pszFilename, GTI_PREFIX)) { - pszIndexDataset = poOpenInfo->pszFilename + strlen(VRTTI_PREFIX); + pszIndexDataset = poOpenInfo->pszFilename + strlen(GTI_PREFIX); } - else if (STARTS_WITH(poOpenInfo->pszFilename, "pszFilename, "pszFilename)); if (m_psXMLTree == nullptr) return false; } else if (strstr(reinterpret_cast(poOpenInfo->pabyHeader), - "pszFilename)); if (m_psXMLTree == nullptr) @@ -455,11 +455,11 @@ bool VRTTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) if (m_psXMLTree) { - psRoot = CPLGetXMLNode(m_psXMLTree.get(), "=VRTTileIndexDataset"); + psRoot = CPLGetXMLNode(m_psXMLTree.get(), "=GDALTileIndexDataset"); if (psRoot == nullptr) { CPLError(CE_Failure, CPLE_AppDefined, - "Missing VRTTileIndexDataset root element."); + "Missing GDALTileIndexDataset root element."); return false; } @@ -545,7 +545,7 @@ bool VRTTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) } else { - if (STARTS_WITH(poOpenInfo->pszFilename, VRTTI_PREFIX)) + if (STARTS_WITH(poOpenInfo->pszFilename, GTI_PREFIX)) { CPLError(CE_Failure, CPLE_AppDefined, "%s has more than one layer. LAYER open option " @@ -572,21 +572,21 @@ bool VRTTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) return false; } - // Try to get the metadata from an embedded xml:VRTTI domain + // Try to get the metadata from an embedded xml:GTI domain if (!m_psXMLTree) { - char **papszMD = m_poLayer->GetMetadata("xml:VRTTI"); + char **papszMD = m_poLayer->GetMetadata("xml:GTI"); if (papszMD && papszMD[0]) { m_psXMLTree.reset(CPLParseXMLString(papszMD[0])); if (m_psXMLTree == nullptr) return false; - psRoot = CPLGetXMLNode(m_psXMLTree.get(), "=VRTTileIndexDataset"); + psRoot = CPLGetXMLNode(m_psXMLTree.get(), "=GDALTileIndexDataset"); if (psRoot == nullptr) { CPLError(CE_Failure, CPLE_AppDefined, - "Missing VRTTileIndexDataset root element."); + "Missing GDALTileIndexDataset root element."); return false; } } @@ -601,21 +601,21 @@ bool VRTTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) return pszVal; if (EQUAL(pszItem, MD_BAND_COUNT)) - pszItem = VRTTI_XML_BANDCOUNT; + pszItem = GTI_XML_BANDCOUNT; else if (EQUAL(pszItem, MD_DATA_TYPE)) - pszItem = VRTTI_XML_DATATYPE; + pszItem = GTI_XML_DATATYPE; else if (EQUAL(pszItem, MD_NODATA)) - pszItem = VRTTI_XML_NODATAVALUE; + pszItem = GTI_XML_NODATAVALUE; else if (EQUAL(pszItem, MD_COLOR_INTERPRETATION)) - pszItem = VRTTI_XML_COLORINTERP; + pszItem = GTI_XML_COLORINTERP; else if (EQUAL(pszItem, MD_LOCATION_FIELD)) - pszItem = VRTTI_XML_LOCATIONFIELD; + pszItem = GTI_XML_LOCATIONFIELD; else if (EQUAL(pszItem, MD_SORT_FIELD)) - pszItem = VRTTI_XML_SORTFIELD; + pszItem = GTI_XML_SORTFIELD; else if (EQUAL(pszItem, MD_SORT_FIELD_ASC)) - pszItem = VRTTI_XML_SORTFIELDASC; + pszItem = GTI_XML_SORTFIELDASC; else if (EQUAL(pszItem, MD_MASK_BAND)) - pszItem = VRTTI_XML_MASKBAND; + pszItem = GTI_XML_MASKBAND; pszVal = CPLGetXMLValue(psRoot, pszItem, nullptr); if (pszVal) return pszVal; @@ -776,15 +776,15 @@ bool VRTTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) psIter = psIter->psNext) { if (psIter->eType == CXT_Element && - strcmp(psIter->pszValue, VRTTI_XML_BAND_ELEMENT) == 0) + strcmp(psIter->pszValue, GTI_XML_BAND_ELEMENT) == 0) { const char *pszBand = - CPLGetXMLValue(psIter, VRTTI_XML_BAND_NUMBER, nullptr); + CPLGetXMLValue(psIter, GTI_XML_BAND_NUMBER, nullptr); if (!pszBand) { CPLError(CE_Failure, CPLE_AppDefined, "%s attribute missing on %s element", - VRTTI_XML_BAND_NUMBER, VRTTI_XML_BAND_ELEMENT); + GTI_XML_BAND_NUMBER, GTI_XML_BAND_ELEMENT); return false; } const int nBand = atoi(pszBand); @@ -819,7 +819,7 @@ bool VRTTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) { CPLError(CE_Failure, CPLE_AppDefined, "Inconsistent %s with actual number of %s elements", - VRTTI_XML_BANDCOUNT, VRTTI_XML_BAND_ELEMENT); + GTI_XML_BANDCOUNT, GTI_XML_BAND_ELEMENT); return false; } } @@ -1389,14 +1389,14 @@ bool VRTTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) nRasterYSize = static_cast(std::ceil(nRasterYSize / dfOvrFactor)); } - VRTTileIndexBand *poFirstBand = nullptr; + GDALTileIndexBand *poFirstBand = nullptr; for (int i = 0; i < nBandCount; ++i) { GDALDataType eDataType = aeDataTypes[i]; if (!apoXMLNodeBands.empty()) { - const char *pszVal = CPLGetXMLValue( - apoXMLNodeBands[i], VRTTI_XML_BAND_DATATYPE, nullptr); + const char *pszVal = CPLGetXMLValue(apoXMLNodeBands[i], + GTI_XML_BAND_DATATYPE, nullptr); if (pszVal) { eDataType = GDALGetDataTypeByName(pszVal); @@ -1404,7 +1404,7 @@ bool VRTTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) return false; } } - auto poBandUniquePtr = std::make_unique( + auto poBandUniquePtr = std::make_unique( this, i + 1, eDataType, nBlockXSize, nBlockYSize); auto poBand = poBandUniquePtr.get(); SetBand(i + 1, poBandUniquePtr.release()); @@ -1418,7 +1418,7 @@ bool VRTTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) if (!apoXMLNodeBands.empty()) { const char *pszVal = CPLGetXMLValue( - apoXMLNodeBands[i], VRTTI_XML_BAND_DESCRIPTION, nullptr); + apoXMLNodeBands[i], GTI_XML_BAND_DESCRIPTION, nullptr); if (pszVal) { poBand->GDALRasterBand::SetDescription(pszVal); @@ -1433,7 +1433,7 @@ bool VRTTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) if (!apoXMLNodeBands.empty()) { const char *pszVal = CPLGetXMLValue( - apoXMLNodeBands[i], VRTTI_XML_BAND_NODATAVALUE, nullptr); + apoXMLNodeBands[i], GTI_XML_BAND_NODATAVALUE, nullptr); if (pszVal) { poBand->m_bNoDataValueSet = true; @@ -1454,7 +1454,7 @@ bool VRTTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) if (!apoXMLNodeBands.empty()) { const char *pszVal = CPLGetXMLValue( - apoXMLNodeBands[i], VRTTI_XML_BAND_COLORINTERP, nullptr); + apoXMLNodeBands[i], GTI_XML_BAND_COLORINTERP, nullptr); if (pszVal) { poBand->m_eColorInterp = @@ -1469,8 +1469,8 @@ bool VRTTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) } if (!apoXMLNodeBands.empty()) { - const char *pszVal = CPLGetXMLValue(apoXMLNodeBands[i], - VRTTI_XML_BAND_SCALE, nullptr); + const char *pszVal = + CPLGetXMLValue(apoXMLNodeBands[i], GTI_XML_BAND_SCALE, nullptr); if (pszVal) { poBand->m_dfScale = CPLAtof(pszVal); @@ -1485,7 +1485,7 @@ bool VRTTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) if (!apoXMLNodeBands.empty()) { const char *pszVal = CPLGetXMLValue(apoXMLNodeBands[i], - VRTTI_XML_BAND_OFFSET, nullptr); + GTI_XML_BAND_OFFSET, nullptr); if (pszVal) { poBand->m_dfOffset = CPLAtof(pszVal); @@ -1499,8 +1499,8 @@ bool VRTTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) } if (!apoXMLNodeBands.empty()) { - const char *pszVal = CPLGetXMLValue( - apoXMLNodeBands[i], VRTTI_XML_BAND_UNITTYPE, nullptr); + const char *pszVal = CPLGetXMLValue(apoXMLNodeBands[i], + GTI_XML_BAND_UNITTYPE, nullptr); if (pszVal) { poBand->m_osUnit = pszVal; @@ -1513,20 +1513,20 @@ bool VRTTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) poBand->oMDMD.XMLInit(psBandNode, TRUE); if (const CPLXMLNode *psCategoryNames = - CPLGetXMLNode(psBandNode, VRTTI_XML_CATEGORYNAMES)) + CPLGetXMLNode(psBandNode, GTI_XML_CATEGORYNAMES)) { poBand->m_aosCategoryNames = VRTParseCategoryNames(psCategoryNames); } if (const CPLXMLNode *psColorTable = - CPLGetXMLNode(psBandNode, VRTTI_XML_COLORTABLE)) + CPLGetXMLNode(psBandNode, GTI_XML_COLORTABLE)) { poBand->m_poColorTable = VRTParseColorTable(psColorTable); } if (const CPLXMLNode *psRAT = - CPLGetXMLNode(psBandNode, VRTTI_XML_RAT)) + CPLGetXMLNode(psBandNode, GTI_XML_RAT)) { poBand->m_poRAT = std::make_unique(); @@ -1540,7 +1540,7 @@ bool VRTTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) bHasMaskBand = CPLTestBool(pszMaskBand); if (bHasMaskBand) { - m_poMaskBand = std::make_unique( + m_poMaskBand = std::make_unique( this, 0, GDT_Byte, nBlockXSize, nBlockYSize); } @@ -1552,23 +1552,22 @@ bool VRTTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) psIter = psIter->psNext) { if (psIter->eType == CXT_Element && - strcmp(psIter->pszValue, VRTTI_XML_OVERVIEW_ELEMENT) == 0) + strcmp(psIter->pszValue, GTI_XML_OVERVIEW_ELEMENT) == 0) { const char *pszDataset = CPLGetXMLValue( - psIter, VRTTI_XML_OVERVIEW_DATASET, nullptr); - const char *pszLayer = CPLGetXMLValue( - psIter, VRTTI_XML_OVERVIEW_LAYER, nullptr); + psIter, GTI_XML_OVERVIEW_DATASET, nullptr); + const char *pszLayer = + CPLGetXMLValue(psIter, GTI_XML_OVERVIEW_LAYER, nullptr); const char *pszFactor = CPLGetXMLValue( - psIter, VRTTI_XML_OVERVIEW_FACTOR, nullptr); + psIter, GTI_XML_OVERVIEW_FACTOR, nullptr); if (!pszDataset && !pszLayer && !pszFactor) { - CPLError(CE_Failure, CPLE_AppDefined, - "At least one of %s, %s or %s element " - "must be present as an %s child", - VRTTI_XML_OVERVIEW_DATASET, - VRTTI_XML_OVERVIEW_LAYER, - VRTTI_XML_OVERVIEW_FACTOR, - VRTTI_XML_OVERVIEW_ELEMENT); + CPLError( + CE_Failure, CPLE_AppDefined, + "At least one of %s, %s or %s element " + "must be present as an %s child", + GTI_XML_OVERVIEW_DATASET, GTI_XML_OVERVIEW_LAYER, + GTI_XML_OVERVIEW_FACTOR, GTI_XML_OVERVIEW_ELEMENT); return false; } m_aoOverviewDescriptor.emplace_back( @@ -1703,8 +1702,8 @@ bool VRTTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) /* GetMetadataItem() */ /************************************************************************/ -const char *VRTTileIndexDataset::GetMetadataItem(const char *pszName, - const char *pszDomain) +const char *GDALTileIndexDataset::GetMetadataItem(const char *pszName, + const char *pszDomain) { if (pszName && pszDomain && EQUAL(pszDomain, "__DEBUG__")) { @@ -1724,7 +1723,7 @@ const char *VRTTileIndexDataset::GetMetadataItem(const char *pszName, /* TileIndexSupportsEditingLayerMetadata() */ /************************************************************************/ -bool VRTTileIndexDataset::TileIndexSupportsEditingLayerMetadata() const +bool GDALTileIndexDataset::TileIndexSupportsEditingLayerMetadata() const { return eAccess == GA_Update && m_poVectorDS->GetDriver() && EQUAL(m_poVectorDS->GetDriver()->GetDescription(), "GPKG"); @@ -1734,9 +1733,9 @@ bool VRTTileIndexDataset::TileIndexSupportsEditingLayerMetadata() const /* SetMetadataItem() */ /************************************************************************/ -CPLErr VRTTileIndexDataset::SetMetadataItem(const char *pszName, - const char *pszValue, - const char *pszDomain) +CPLErr GDALTileIndexDataset::SetMetadataItem(const char *pszName, + const char *pszValue, + const char *pszDomain) { if (m_bXMLUpdatable) { @@ -1758,7 +1757,7 @@ CPLErr VRTTileIndexDataset::SetMetadataItem(const char *pszName, /* SetMetadata() */ /************************************************************************/ -CPLErr VRTTileIndexDataset::SetMetadata(char **papszMD, const char *pszDomain) +CPLErr GDALTileIndexDataset::SetMetadata(char **papszMD, const char *pszDomain) { if (m_bXMLUpdatable) { @@ -1810,60 +1809,60 @@ CPLErr VRTTileIndexDataset::SetMetadata(char **papszMD, const char *pszDomain) } /************************************************************************/ -/* VRTTileIndexDatasetIdentify() */ +/* GDALTileIndexDatasetIdentify() */ /************************************************************************/ -static int VRTTileIndexDatasetIdentify(GDALOpenInfo *poOpenInfo) +static int GDALTileIndexDatasetIdentify(GDALOpenInfo *poOpenInfo) { - if (STARTS_WITH(poOpenInfo->pszFilename, VRTTI_PREFIX)) + if (STARTS_WITH(poOpenInfo->pszFilename, GTI_PREFIX)) return true; - if (STARTS_WITH(poOpenInfo->pszFilename, "pszFilename, "nHeaderBytes > 0 && (poOpenInfo->nOpenFlags & GDAL_OF_RASTER) != 0 && (strstr(reinterpret_cast(poOpenInfo->pabyHeader), - "pszFilename, ".vrt.gpkg") || - ENDS_WITH_CI(poOpenInfo->pszFilename, ".vrt.fgb") || - ENDS_WITH_CI(poOpenInfo->pszFilename, ".vrt.parquet")); + "pszFilename, ".gti.gpkg") || + ENDS_WITH_CI(poOpenInfo->pszFilename, ".gti.fgb") || + ENDS_WITH_CI(poOpenInfo->pszFilename, ".gti.parquet")); } /************************************************************************/ -/* VRTTileIndexDatasetOpen() */ +/* GDALTileIndexDatasetOpen() */ /************************************************************************/ -static GDALDataset *VRTTileIndexDatasetOpen(GDALOpenInfo *poOpenInfo) +static GDALDataset *GDALTileIndexDatasetOpen(GDALOpenInfo *poOpenInfo) { - if (!VRTTileIndexDatasetIdentify(poOpenInfo)) + if (!GDALTileIndexDatasetIdentify(poOpenInfo)) return nullptr; - auto poDS = std::make_unique(); + auto poDS = std::make_unique(); if (!poDS->Open(poOpenInfo)) return nullptr; return poDS.release(); } /************************************************************************/ -/* ~VRTTileIndexDataset() */ +/* ~GDALTileIndexDataset() */ /************************************************************************/ -VRTTileIndexDataset::~VRTTileIndexDataset() +GDALTileIndexDataset::~GDALTileIndexDataset() { - VRTTileIndexDataset::FlushCache(true); + GDALTileIndexDataset::FlushCache(true); } /************************************************************************/ /* FlushCache() */ /************************************************************************/ -CPLErr VRTTileIndexDataset::FlushCache(bool bAtClosing) +CPLErr GDALTileIndexDataset::FlushCache(bool bAtClosing) { CPLErr eErr = CE_None; if (bAtClosing && m_bXMLModified) { CPLXMLNode *psRoot = - CPLGetXMLNode(m_psXMLTree.get(), "=VRTTileIndexDataset"); + CPLGetXMLNode(m_psXMLTree.get(), "=GDALTileIndexDataset"); // Suppress existing dataset metadata while (true) @@ -1879,16 +1878,16 @@ CPLErr VRTTileIndexDataset::FlushCache(bool bAtClosing) CPLAddXMLChild(psRoot, psMD); // Update existing band metadata - if (CPLGetXMLNode(psRoot, VRTTI_XML_BAND_ELEMENT)) + if (CPLGetXMLNode(psRoot, GTI_XML_BAND_ELEMENT)) { for (CPLXMLNode *psIter = psRoot->psChild; psIter; psIter = psIter->psNext) { if (psIter->eType == CXT_Element && - strcmp(psIter->pszValue, VRTTI_XML_BAND_ELEMENT)) + strcmp(psIter->pszValue, GTI_XML_BAND_ELEMENT)) { const char *pszBand = - CPLGetXMLValue(psIter, VRTTI_XML_BAND_NUMBER, nullptr); + CPLGetXMLValue(psIter, GTI_XML_BAND_NUMBER, nullptr); if (pszBand) { const int nBand = atoi(pszBand); @@ -1903,7 +1902,7 @@ CPLErr VRTTileIndexDataset::FlushCache(bool bAtClosing) CPLRemoveXMLChild(psIter, psExistingMetadata); } - auto poBand = cpl::down_cast( + auto poBand = cpl::down_cast( papoBands[nBand - 1]); if (CPLXMLNode *psMD = poBand->oMDMD.Serialize()) CPLAddXMLChild(psIter, psMD); @@ -1920,7 +1919,7 @@ CPLErr VRTTileIndexDataset::FlushCache(bool bAtClosing) for (int i = 1; i <= nBands; ++i) { auto poBand = - cpl::down_cast(papoBands[i - 1]); + cpl::down_cast(papoBands[i - 1]); auto psMD = poBand->oMDMD.Serialize(); if (psMD) bHasBandMD = true; @@ -1931,43 +1930,43 @@ CPLErr VRTTileIndexDataset::FlushCache(bool bAtClosing) for (int i = 1; i <= nBands; ++i) { auto poBand = - cpl::down_cast(papoBands[i - 1]); + cpl::down_cast(papoBands[i - 1]); - CPLXMLNode *psBand = CPLCreateXMLNode( - psRoot, CXT_Element, VRTTI_XML_BAND_ELEMENT); - CPLAddXMLAttributeAndValue(psBand, VRTTI_XML_BAND_NUMBER, + CPLXMLNode *psBand = CPLCreateXMLNode(psRoot, CXT_Element, + GTI_XML_BAND_ELEMENT); + CPLAddXMLAttributeAndValue(psBand, GTI_XML_BAND_NUMBER, CPLSPrintf("%d", i)); CPLAddXMLAttributeAndValue( - psBand, VRTTI_XML_BAND_DATATYPE, + psBand, GTI_XML_BAND_DATATYPE, GDALGetDataTypeName(poBand->GetRasterDataType())); const char *pszDescription = poBand->GetDescription(); if (pszDescription && pszDescription[0]) - CPLSetXMLValue(psBand, VRTTI_XML_BAND_DESCRIPTION, + CPLSetXMLValue(psBand, GTI_XML_BAND_DESCRIPTION, pszDescription); const auto eColorInterp = poBand->GetColorInterpretation(); if (eColorInterp != GCI_Undefined) CPLSetXMLValue( - psBand, VRTTI_XML_BAND_COLORINTERP, + psBand, GTI_XML_BAND_COLORINTERP, GDALGetColorInterpretationName(eColorInterp)); if (!std::isnan(poBand->m_dfOffset)) - CPLSetXMLValue(psBand, VRTTI_XML_BAND_OFFSET, + CPLSetXMLValue(psBand, GTI_XML_BAND_OFFSET, CPLSPrintf("%.16g", poBand->m_dfOffset)); if (!std::isnan(poBand->m_dfScale)) - CPLSetXMLValue(psBand, VRTTI_XML_BAND_SCALE, + CPLSetXMLValue(psBand, GTI_XML_BAND_SCALE, CPLSPrintf("%.16g", poBand->m_dfScale)); if (!poBand->m_osUnit.empty()) - CPLSetXMLValue(psBand, VRTTI_XML_BAND_UNITTYPE, + CPLSetXMLValue(psBand, GTI_XML_BAND_UNITTYPE, poBand->m_osUnit.c_str()); if (poBand->m_bNoDataValueSet) { CPLSetXMLValue( - psBand, VRTTI_XML_BAND_NODATAVALUE, + psBand, GTI_XML_BAND_NODATAVALUE, VRTSerializeNoData(poBand->m_dfNoDataValue, poBand->GetRasterDataType(), 18) .c_str()); @@ -1999,7 +1998,7 @@ CPLErr VRTTileIndexDataset::FlushCache(bool bAtClosing) /* LoadOverviews() */ /************************************************************************/ -void VRTTileIndexDataset::LoadOverviews() +void GDALTileIndexDataset::LoadOverviews() { if (m_apoOverviews.empty() && !m_aoOverviewDescriptor.empty()) { @@ -2042,7 +2041,7 @@ void VRTTileIndexDataset::LoadOverviews() /* GetOverviewCount() */ /************************************************************************/ -int VRTTileIndexBand::GetOverviewCount() +int GDALTileIndexBand::GetOverviewCount() { const int nPAMOverviews = GDALPamRasterBand::GetOverviewCount(); if (nPAMOverviews) @@ -2056,7 +2055,7 @@ int VRTTileIndexBand::GetOverviewCount() /* GetOverview() */ /************************************************************************/ -GDALRasterBand *VRTTileIndexBand::GetOverview(int iOvr) +GDALRasterBand *GDALTileIndexBand::GetOverview(int iOvr) { if (iOvr < 0 || iOvr >= GetOverviewCount()) return nullptr; @@ -2082,7 +2081,7 @@ GDALRasterBand *VRTTileIndexBand::GetOverview(int iOvr) /* GetGeoTransform() */ /************************************************************************/ -CPLErr VRTTileIndexDataset::GetGeoTransform(double *padfGeoTransform) +CPLErr GDALTileIndexDataset::GetGeoTransform(double *padfGeoTransform) { memcpy(padfGeoTransform, m_adfGeoTransform.data(), 6 * sizeof(double)); return CE_None; @@ -2092,18 +2091,18 @@ CPLErr VRTTileIndexDataset::GetGeoTransform(double *padfGeoTransform) /* GetSpatialRef() */ /************************************************************************/ -const OGRSpatialReference *VRTTileIndexDataset::GetSpatialRef() const +const OGRSpatialReference *GDALTileIndexDataset::GetSpatialRef() const { return m_oSRS.IsEmpty() ? nullptr : &m_oSRS; } /************************************************************************/ -/* VRTTileIndexBand() */ +/* GDALTileIndexBand() */ /************************************************************************/ -VRTTileIndexBand::VRTTileIndexBand(VRTTileIndexDataset *poDSIn, int nBandIn, - GDALDataType eDT, int nBlockXSizeIn, - int nBlockYSizeIn) +GDALTileIndexBand::GDALTileIndexBand(GDALTileIndexDataset *poDSIn, int nBandIn, + GDALDataType eDT, int nBlockXSizeIn, + int nBlockYSizeIn) { m_poDS = poDSIn; nBand = nBandIn; @@ -2118,8 +2117,8 @@ VRTTileIndexBand::VRTTileIndexBand(VRTTileIndexDataset *poDSIn, int nBandIn, /* IReadBlock() */ /************************************************************************/ -CPLErr VRTTileIndexBand::IReadBlock(int nBlockXOff, int nBlockYOff, - void *pImage) +CPLErr GDALTileIndexBand::IReadBlock(int nBlockXOff, int nBlockYOff, + void *pImage) { const int nPixelSize = GDALGetDataTypeSizeBytes(eDataType); @@ -2141,12 +2140,12 @@ CPLErr VRTTileIndexBand::IReadBlock(int nBlockXOff, int nBlockYOff, /* IRasterIO() */ /************************************************************************/ -CPLErr VRTTileIndexBand::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, - int nXSize, int nYSize, void *pData, - int nBufXSize, int nBufYSize, - GDALDataType eBufType, GSpacing nPixelSpace, - GSpacing nLineSpace, - GDALRasterIOExtraArg *psExtraArg) +CPLErr GDALTileIndexBand::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, + int nXSize, int nYSize, void *pData, + int nBufXSize, int nBufYSize, + GDALDataType eBufType, GSpacing nPixelSpace, + GSpacing nLineSpace, + GDALRasterIOExtraArg *psExtraArg) { int anBand[] = {nBand}; @@ -2159,7 +2158,7 @@ CPLErr VRTTileIndexBand::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, /* GetMetadataDomainList() */ /************************************************************************/ -char **VRTTileIndexBand::GetMetadataDomainList() +char **GDALTileIndexBand::GetMetadataDomainList() { return CSLAddString(GDALRasterBand::GetMetadataDomainList(), "LocationInfo"); @@ -2169,8 +2168,8 @@ char **VRTTileIndexBand::GetMetadataDomainList() /* GetMetadataItem() */ /************************************************************************/ -const char *VRTTileIndexBand::GetMetadataItem(const char *pszName, - const char *pszDomain) +const char *GDALTileIndexBand::GetMetadataItem(const char *pszName, + const char *pszDomain) { /* ==================================================================== */ @@ -2232,7 +2231,7 @@ const char *VRTTileIndexBand::GetMetadataItem(const char *pszName, if (!m_poDS->m_aoSourceDesc.empty()) { const auto AddSource = - [&](const VRTTileIndexDataset::SourceDesc &oSourceDesc) + [&](const GDALTileIndexDataset::SourceDesc &oSourceDesc) { m_osLastLocationInfo += ""; char *const pszXMLEscaped = @@ -2269,9 +2268,9 @@ const char *VRTTileIndexBand::GetMetadataItem(const char *pszName, /* SetMetadataItem() */ /************************************************************************/ -CPLErr VRTTileIndexBand::SetMetadataItem(const char *pszName, - const char *pszValue, - const char *pszDomain) +CPLErr GDALTileIndexBand::SetMetadataItem(const char *pszName, + const char *pszValue, + const char *pszDomain) { if (nBand > 0 && m_poDS->m_bXMLUpdatable) { @@ -2294,7 +2293,7 @@ CPLErr VRTTileIndexBand::SetMetadataItem(const char *pszName, /* SetMetadata() */ /************************************************************************/ -CPLErr VRTTileIndexBand::SetMetadata(char **papszMD, const char *pszDomain) +CPLErr GDALTileIndexBand::SetMetadata(char **papszMD, const char *pszDomain) { if (nBand > 0 && m_poDS->m_bXMLUpdatable) { @@ -2428,20 +2427,20 @@ static bool GetSrcDstWin(const double adfTileGT[6], int nTileXSize, } /************************************************************************/ -/* GDALDatasetCastToVRTTIDataset() */ +/* GDALDatasetCastToGTIDataset() */ /************************************************************************/ -VRTTileIndexDataset *GDALDatasetCastToVRTTIDataset(GDALDataset *poDS) +GDALTileIndexDataset *GDALDatasetCastToGTIDataset(GDALDataset *poDS) { - return dynamic_cast(poDS); + return dynamic_cast(poDS); } /************************************************************************/ -/* VRTTIGetSourcesMoreRecentThan() */ +/* GTIGetSourcesMoreRecentThan() */ /************************************************************************/ -std::vector -VRTTIGetSourcesMoreRecentThan(VRTTileIndexDataset *poDS, int64_t mTime) +std::vector +GTIGetSourcesMoreRecentThan(GDALTileIndexDataset *poDS, int64_t mTime) { return poDS->GetSourcesMoreRecentThan(mTime); } @@ -2450,10 +2449,10 @@ VRTTIGetSourcesMoreRecentThan(VRTTileIndexDataset *poDS, int64_t mTime) /* GetSourcesMoreRecentThan() */ /************************************************************************/ -std::vector -VRTTileIndexDataset::GetSourcesMoreRecentThan(int64_t mTime) +std::vector +GDALTileIndexDataset::GetSourcesMoreRecentThan(int64_t mTime) { - std::vector oRes; + std::vector oRes; m_poLayer->SetSpatialFilter(nullptr); for (auto &&poFeature : m_poLayer) @@ -2512,7 +2511,7 @@ VRTTileIndexDataset::GetSourcesMoreRecentThan(int64_t mTime) } constexpr double EPS = 1e-8; - VRTTISourceDesc oSourceDesc; + GTISourceDesc oSourceDesc; oSourceDesc.osFilename = osTileName; oSourceDesc.nDstXOff = static_cast(dfXOff + EPS); oSourceDesc.nDstYOff = static_cast(dfYOff + EPS); @@ -2528,8 +2527,8 @@ VRTTileIndexDataset::GetSourcesMoreRecentThan(int64_t mTime) /* GetSourceDesc() */ /************************************************************************/ -bool VRTTileIndexDataset::GetSourceDesc(const std::string &osTileName, - SourceDesc &oSourceDesc) +bool GDALTileIndexDataset::GetSourceDesc(const std::string &osTileName, + SourceDesc &oSourceDesc) { std::shared_ptr poTileDS; if (!m_oMapSharedSources.tryGet(osTileName, poTileDS)) @@ -2780,8 +2779,8 @@ bool VRTTileIndexDataset::GetSourceDesc(const std::string &osTileName, /* CollectSources() */ /************************************************************************/ -bool VRTTileIndexDataset::CollectSources(double dfXOff, double dfYOff, - double dfXSize, double dfYSize) +bool GDALTileIndexDataset::CollectSources(double dfXOff, double dfYOff, + double dfXSize, double dfYSize) { const double dfMinX = m_adfGeoTransform[GT_TOPLEFT_X] + dfXOff * m_adfGeoTransform[GT_WE_RES]; @@ -2906,7 +2905,7 @@ bool VRTTileIndexDataset::CollectSources(double dfXOff, double dfYOff, /* SortSourceDesc() */ /************************************************************************/ -void VRTTileIndexDataset::SortSourceDesc() +void GDALTileIndexDataset::SortSourceDesc() { const auto eFieldType = m_nSortFieldIndex >= 0 ? m_poLayer->GetLayerDefn() @@ -3079,8 +3078,8 @@ CompositeSrcWithMaskIntoDest(const int nOutXSize, const int nOutYSize, /************************************************************************/ // Must be called after CollectSources() -bool VRTTileIndexDataset::NeedInitBuffer(int nBandCount, - const int *panBandMap) const +bool GDALTileIndexDataset::NeedInitBuffer(int nBandCount, + const int *panBandMap) const { bool bNeedInitBuffer = true; // If the last source (that is the most prioritary one) covers at least @@ -3113,11 +3112,11 @@ bool VRTTileIndexDataset::NeedInitBuffer(int nBandCount, /* InitBuffer() */ /************************************************************************/ -void VRTTileIndexDataset::InitBuffer(void *pData, int nBufXSize, int nBufYSize, - GDALDataType eBufType, int nBandCount, - const int *panBandMap, - GSpacing nPixelSpace, GSpacing nLineSpace, - GSpacing nBandSpace) const +void GDALTileIndexDataset::InitBuffer(void *pData, int nBufXSize, int nBufYSize, + GDALDataType eBufType, int nBandCount, + const int *panBandMap, + GSpacing nPixelSpace, GSpacing nLineSpace, + GSpacing nBandSpace) const { const int nBufTypeSize = GDALGetDataTypeSizeBytes(eBufType); if (m_bSameNoData && nBandCount > 1 && @@ -3132,7 +3131,7 @@ void VRTTileIndexDataset::InitBuffer(void *pData, int nBufXSize, int nBufYSize, auto poVRTBand = nBandNr == 0 ? m_poMaskBand.get() - : cpl::down_cast(papoBands[nBandNr - 1]); + : cpl::down_cast(papoBands[nBandNr - 1]); const double dfNoData = poVRTBand->m_dfNoDataValue; if (dfNoData == 0.0) { @@ -3153,7 +3152,7 @@ void VRTTileIndexDataset::InitBuffer(void *pData, int nBufXSize, int nBufYSize, { const int nBandNr = panBandMap[i]; auto poVRTBand = nBandNr == 0 ? m_poMaskBand.get() - : cpl::down_cast( + : cpl::down_cast( papoBands[nBandNr - 1]); GByte *pabyBandData = static_cast(pData) + i * nBandSpace; if (nPixelSpace == nBufTypeSize && @@ -3195,13 +3194,13 @@ void VRTTileIndexDataset::InitBuffer(void *pData, int nBufXSize, int nBufYSize, /* IRasterIO() */ /************************************************************************/ -CPLErr VRTTileIndexDataset::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, - int nXSize, int nYSize, void *pData, - int nBufXSize, int nBufYSize, - GDALDataType eBufType, int nBandCount, - int *panBandMap, GSpacing nPixelSpace, - GSpacing nLineSpace, GSpacing nBandSpace, - GDALRasterIOExtraArg *psExtraArg) +CPLErr GDALTileIndexDataset::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, + int nXSize, int nYSize, void *pData, + int nBufXSize, int nBufYSize, + GDALDataType eBufType, int nBandCount, + int *panBandMap, GSpacing nPixelSpace, + GSpacing nLineSpace, GSpacing nBandSpace, + GDALRasterIOExtraArg *psExtraArg) { if (eRWFlag != GF_Read) return CE_Failure; @@ -3600,25 +3599,25 @@ CPLErr VRTTileIndexDataset::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, } /************************************************************************/ -/* GDALRegister_VRTTI() */ +/* GDALRegister_GTI() */ /************************************************************************/ -void GDALRegister_VRTTI() +void GDALRegister_GTI() { - if (GDALGetDriverByName("VRTTI") != nullptr) + if (GDALGetDriverByName("GTI") != nullptr) return; auto poDriver = std::make_unique(); - poDriver->SetDescription("VRTTI"); + poDriver->SetDescription("GTI"); poDriver->SetMetadataItem(GDAL_DCAP_RASTER, "YES"); - poDriver->SetMetadataItem(GDAL_DMD_LONGNAME, "Virtual Raster Tile Index"); - poDriver->SetMetadataItem(GDAL_DMD_EXTENSIONS, "vrt.gpkg vrt.fgb vrtti"); - poDriver->SetMetadataItem(GDAL_DMD_CONNECTION_PREFIX, VRTTI_PREFIX); - poDriver->SetMetadataItem(GDAL_DMD_HELPTOPIC, "drivers/raster/vrtti.html"); + poDriver->SetMetadataItem(GDAL_DMD_LONGNAME, "GDAL Raster Tile Index"); + poDriver->SetMetadataItem(GDAL_DMD_EXTENSIONS, "gti.gpkg gti.fgb gti"); + poDriver->SetMetadataItem(GDAL_DMD_CONNECTION_PREFIX, GTI_PREFIX); + poDriver->SetMetadataItem(GDAL_DMD_HELPTOPIC, "drivers/raster/gti.html"); - poDriver->pfnOpen = VRTTileIndexDatasetOpen; - poDriver->pfnIdentify = VRTTileIndexDatasetIdentify; + poDriver->pfnOpen = GDALTileIndexDatasetOpen; + poDriver->pfnIdentify = GDALTileIndexDatasetIdentify; poDriver->SetMetadataItem(GDAL_DCAP_VIRTUALIO, "YES"); diff --git a/frmts/vrt/vrt_priv.h b/frmts/vrt/vrt_priv.h index c7ca081608e1..ca2af63383fb 100644 --- a/frmts/vrt/vrt_priv.h +++ b/frmts/vrt/vrt_priv.h @@ -36,7 +36,7 @@ #include #include -struct CPL_DLL VRTTISourceDesc +struct CPL_DLL GTISourceDesc { std::string osFilename{}; int nDstXOff = 0; @@ -45,12 +45,12 @@ struct CPL_DLL VRTTISourceDesc int nDstYSize = 0; }; -class VRTTileIndexDataset; +class GDALTileIndexDataset; -VRTTileIndexDataset CPL_DLL *GDALDatasetCastToVRTTIDataset(GDALDataset *poDS); +GDALTileIndexDataset CPL_DLL *GDALDatasetCastToGTIDataset(GDALDataset *poDS); -std::vector CPL_DLL -VRTTIGetSourcesMoreRecentThan(VRTTileIndexDataset *poDS, int64_t mTime); +std::vector CPL_DLL +GTIGetSourcesMoreRecentThan(GDALTileIndexDataset *poDS, int64_t mTime); CPLStringList VRTParseCategoryNames(const CPLXMLNode *psCategoryNames); diff --git a/frmts/vrt/vrtdataset.h b/frmts/vrt/vrtdataset.h index d893e7361fdb..59dc1ecf91af 100644 --- a/frmts/vrt/vrtdataset.h +++ b/frmts/vrt/vrtdataset.h @@ -1024,8 +1024,8 @@ class CPL_DLL VRTSimpleSource CPL_NON_FINAL : public VRTSource protected: friend class VRTSourcedRasterBand; friend class VRTDataset; - friend class VRTTileIndexDataset; - friend class VRTTileIndexBand; + friend class GDALTileIndexDataset; + friend class GDALTileIndexBand; int m_nBand = 0; bool m_bGetMaskBand = false; diff --git a/gcore/gdal_frmts.h b/gcore/gdal_frmts.h index 66117ba6edad..129845bbe006 100644 --- a/gcore/gdal_frmts.h +++ b/gcore/gdal_frmts.h @@ -84,7 +84,7 @@ void CPL_DLL GDALRegister_ECW_JP2ECW(); void CPL_DLL GDALRegister_FujiBAS(void); void CPL_DLL GDALRegister_FIT(void); void CPL_DLL GDALRegister_VRT(void); -void CPL_DLL GDALRegister_VRTTI(void); +void CPL_DLL GDALRegister_GTI(void); void CPL_DLL GDALRegister_USGSDEM(void); void CPL_DLL GDALRegister_FAST(void); void CPL_DLL GDALRegister_HDF4(void); diff --git a/ogr/ogrsf_frmts/gpkg/ogrgeopackagedriver.cpp b/ogr/ogrsf_frmts/gpkg/ogrgeopackagedriver.cpp index 6e4c438978fb..3c43b5bf383b 100644 --- a/ogr/ogrsf_frmts/gpkg/ogrgeopackagedriver.cpp +++ b/ogr/ogrsf_frmts/gpkg/ogrgeopackagedriver.cpp @@ -94,9 +94,9 @@ static int OGRGeoPackageDriverIdentify(GDALOpenInfo *poOpenInfo, } if ((poOpenInfo->nOpenFlags & GDAL_OF_RASTER) != 0 && - ENDS_WITH_CI(poOpenInfo->pszFilename, ".vrt.gpkg")) + ENDS_WITH_CI(poOpenInfo->pszFilename, ".gti.gpkg")) { - // Handled by VRT driver + // Handled by GTI driver return FALSE; } diff --git a/swig/include/python/gdal_python.i b/swig/include/python/gdal_python.i index c35c072f116b..adf5aace5b78 100644 --- a/swig/include/python/gdal_python.i +++ b/swig/include/python/gdal_python.i @@ -3818,7 +3818,7 @@ def TileIndexOptions(options=None, outputSRS=None, writeAbsolutePath=None, skipDifferentProjection=None, - vrttiFilename=None, + gtiFilename=None, xRes=None, yRes=None, outputBounds=None, @@ -3856,8 +3856,8 @@ def TileIndexOptions(options=None, Enables writing the absolute path of the input dataset. By default, the filename is written in the location field exactly as the dataset name. skipDifferentProjection: Whether to skip sources that have a different SRS - vrttiFilename: - Filename of the XML Virtual Tile Index file + gtiFilename: + Filename of the GDAL XML Tile Index file xRes: output horizontal resolution yRes: @@ -3917,8 +3917,8 @@ def TileIndexOptions(options=None, new_options += ['-write_absolute_path'] if skipDifferentProjection: new_options += ['-skip_different_projection'] - if vrttiFilename is not None: - new_options += ['-vrtti_filename', vrttiFilename] + if gtiFilename is not None: + new_options += ['-gti_filename', gtiFilename] if xRes is not None and yRes is not None: new_options += ['-tr', _strHighPrec(xRes), _strHighPrec(yRes)] elif xRes is not None: From e24cb91b1095061042c3dfe66f9e9ae9a446af21 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 17 Jan 2024 17:45:15 +0100 Subject: [PATCH 10/11] GTI: take into account comments of code review --- apps/gdaltindex_lib.cpp | 2 +- autotest/gdrivers/gti.py | 27 +- autotest/utilities/test_gdaltindex_lib.py | 2 + doc/source/programs/gdalbuildvrt.rst | 6 +- doc/source/programs/gdaltindex.rst | 27 +- frmts/vrt/gdaltileindexdataset.cpp | 425 +++++++++++++--------- gcore/gdal_priv.h | 11 + swig/include/python/gdal_python.i | 4 +- 8 files changed, 310 insertions(+), 194 deletions(-) diff --git a/apps/gdaltindex_lib.cpp b/apps/gdaltindex_lib.cpp index f5c58fc7488f..2f30395bb587 100644 --- a/apps/gdaltindex_lib.cpp +++ b/apps/gdaltindex_lib.cpp @@ -1249,7 +1249,7 @@ GDALTileIndexOptionsNew(char **papszArgv, { CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(2); psOptions->xres = CPLAtofM(papszArgv[++iArg]); - psOptions->yres = CPLAtofM(papszArgv[++iArg]); + psOptions->yres = std::fabs(CPLAtofM(papszArgv[++iArg])); } else if (EQUAL(papszArgv[iArg], "-te")) { diff --git a/autotest/gdrivers/gti.py b/autotest/gdrivers/gti.py index 20f928fdce5c..d6757d96e231 100755 --- a/autotest/gdrivers/gti.py +++ b/autotest/gdrivers/gti.py @@ -269,6 +269,7 @@ def test_gti_wrong_prototype_tile(tmp_vsimem): lyr.CreateFeature(f) del index_ds + # no match, because error message is operating system dependent with pytest.raises(Exception): gdal.Open(index_filename) @@ -520,7 +521,7 @@ def test_gti_wrong_width(tmp_vsimem): lyr.SetMetadataItem("GEOTRANSFORM", ",".join([str(x) for x in gt])) del index_ds - with pytest.raises(Exception, match="XSIZE metadata item should be > 0"): + with pytest.raises(Exception, match="XSIZE metadata item must be > 0"): gdal.Open(index_filename) @@ -536,7 +537,7 @@ def test_gti_wrong_height(tmp_vsimem): lyr.SetMetadataItem("GEOTRANSFORM", ",".join([str(x) for x in gt])) del index_ds - with pytest.raises(Exception, match="YSIZE metadata item should be > 0"): + with pytest.raises(Exception, match="YSIZE metadata item must be > 0"): gdal.Open(index_filename) @@ -593,7 +594,7 @@ def test_gti_wrong_gt(tmp_vsimem): with pytest.raises( Exception, - match="GEOTRANSFORM metadata item should be 6 numeric values separated with comma", + match="GEOTRANSFORM metadata item must be 6 numeric values separated with comma", ): gdal.Open(index_filename) @@ -609,7 +610,7 @@ def test_gti_wrong_gt_3rd_term(tmp_vsimem): lyr.SetMetadataItem("GEOTRANSFORM", "0,1,123,0,0,-1") del index_ds - with pytest.raises(Exception, match="3rd value of GEOTRANSFORM should be 0"): + with pytest.raises(Exception, match="3rd value of GEOTRANSFORM must be 0"): gdal.Open(index_filename) @@ -624,7 +625,7 @@ def test_gti_wrong_gt_5th_term(tmp_vsimem): lyr.SetMetadataItem("GEOTRANSFORM", "0,1,0,0,1234,-1") del index_ds - with pytest.raises(Exception, match="5th value of GEOTRANSFORM should be 0"): + with pytest.raises(Exception, match="5th value of GEOTRANSFORM must be 0"): gdal.Open(index_filename) @@ -639,7 +640,7 @@ def test_gti_wrong_gt_6th_term(tmp_vsimem): lyr.SetMetadataItem("GEOTRANSFORM", "0,1,0,0,0,1") del index_ds - with pytest.raises(Exception, match="6th value of GEOTRANSFORM should be < 0"): + with pytest.raises(Exception, match="6th value of GEOTRANSFORM must be < 0"): gdal.Open(index_filename) @@ -698,7 +699,7 @@ def test_gti_wrong_resx(tmp_vsimem): lyr.SetMetadataItem("RESY", "1") del index_ds - with pytest.raises(Exception, match="RESX metadata item should be > 0"): + with pytest.raises(Exception, match="RESX metadata item must be > 0"): gdal.Open(index_filename) @@ -712,7 +713,7 @@ def test_gti_wrong_resy(tmp_vsimem): lyr.SetMetadataItem("RESY", "0") del index_ds - with pytest.raises(Exception, match="RESY metadata item should be > 0"): + with pytest.raises(Exception, match="RESY metadata item must be > 0"): gdal.Open(index_filename) @@ -732,7 +733,7 @@ def test_gti_wrong_minx(tmp_vsimem): lyr.SetMetadataItem("MAXY", str(gt[3])) del index_ds - with pytest.raises(Exception, match="MAXX metadata item should be > MINX"): + with pytest.raises(Exception, match="MAXX metadata item must be > MINX"): gdal.Open(index_filename) @@ -752,7 +753,7 @@ def test_gti_wrong_miny(tmp_vsimem): lyr.SetMetadataItem("MAXY", str(gt[3])) del index_ds - with pytest.raises(Exception, match="MAXY metadata item should be > MINY"): + with pytest.raises(Exception, match="MAXY metadata item must be > MINY"): gdal.Open(index_filename) @@ -917,7 +918,7 @@ def test_gti_valid_nodata(tmp_vsimem, md, expected_nodata): ({"BAND_COUNT": "2", "NODATA": "0,invalid"}, "Invalid value for NODATA"), ( {"BAND_COUNT": "2", "NODATA": "0,0,0"}, - "Number of values in NODATA should be 1 or BAND_COUNT", + "Number of values in NODATA must be 1 or BAND_COUNT", ), ], ) @@ -945,7 +946,7 @@ def test_gti_invalid_nodata(tmp_vsimem, md, error_msg): ), ( {"BAND_COUNT": "2", "DATA_TYPE": "byte,byte,byte"}, - "Number of values in DATA_TYPE should be 1 or BAND_COUNT", + "Number of values in DATA_TYPE must be 1 or BAND_COUNT", ), ], ) @@ -976,7 +977,7 @@ def test_gti_invalid_data_type(tmp_vsimem, md, error_msg): "BAND_COUNT": "2", "COLOR_INTERPRETATION": "undefined,undefined,undefined", }, - "Number of values in COLOR_INTERPRETATION should be 1 or BAND_COUNT", + "Number of values in COLOR_INTERPRETATION must be 1 or BAND_COUNT", ), ], ) diff --git a/autotest/utilities/test_gdaltindex_lib.py b/autotest/utilities/test_gdaltindex_lib.py index ee3d59525b4b..5edd1027b92f 100644 --- a/autotest/utilities/test_gdaltindex_lib.py +++ b/autotest/utilities/test_gdaltindex_lib.py @@ -225,6 +225,7 @@ def test_gdaltindex_lib_outputSRS_writeAbsoluePath(tmp_path, four_tile_index): "got %d features, expecting 5" % ds.GetLayer(0).GetFeatureCount() ) filename = lyr.GetFeature(4).GetField("location") + # Check that path is absolute assert filename.endswith("gdaltindex5.tif") assert filename != "gdaltindex5.tif" @@ -382,6 +383,7 @@ def test_gdaltindex_lib_directory(tmp_path, four_tiles): del ds with gdal.quiet_errors(): + # triggers warnings: "file XXXX has 0.010000 as pixel size (< 10.000000). Skipping" gdal.TileIndex( index_filename, [os.path.dirname(four_tiles[0])], diff --git a/doc/source/programs/gdalbuildvrt.rst b/doc/source/programs/gdalbuildvrt.rst index d008a5eb9ab5..253e1bcc462a 100644 --- a/doc/source/programs/gdalbuildvrt.rst +++ b/doc/source/programs/gdalbuildvrt.rst @@ -42,13 +42,13 @@ entries in the tile index will be added to the VRT. .. note:: - Starting with GDAL 3.9, for virtual mosaic with a very large number of tiles - (typically hundreds of thousands of tiles, or more), it is advised to use the + Starting with GDAL 3.9, for virtual mosaic with a very large number of source rasters + (typically hundreds of thousands of source rasters, or more), it is advised to use the :ref:`gdaltindex` utility to generate a tile index compatible of the :ref:`GTI ` driver. With -separate, each files goes into a separate band in the VRT dataset. Otherwise, -the files are considered as tiles of a larger mosaic and the VRT file has as many bands as one +the files are considered as source rasters of a larger mosaic and the VRT file has as many bands as one of the input files. If one GDAL dataset is made of several subdatasets and has 0 raster bands, diff --git a/doc/source/programs/gdaltindex.rst b/doc/source/programs/gdaltindex.rst index a5707a16852c..f54cf6d28e73 100644 --- a/doc/source/programs/gdaltindex.rst +++ b/doc/source/programs/gdaltindex.rst @@ -63,7 +63,7 @@ tileindex, or as input for the :ref:`GTI ` driver. '*' is a wildcard character that matches any number of any characters including none. '?' is a wildcard character that matches a single character. Comparisons are done in a case insensitive way. - Several filters may be specified + Several filters may be specified. For example :``-filename_filter "*.tif" -filename_filter "*.tiff"`` @@ -71,7 +71,8 @@ tileindex, or as input for the :ref:`GTI ` driver. .. versionadded:: 3.9 - Minimum pixel size that a raster should have to be selected. The pixel size + Minimum pixel size in term of geospatial extent per pixel (resolution) that + a raster should have to be selected. The pixel size is evaluated after reprojection of its extent to the target SRS defined by :option:`-t_srs`. @@ -79,7 +80,8 @@ tileindex, or as input for the :ref:`GTI ` driver. .. versionadded:: 3.9 - Maximum pixel size that a raster should have to be selected. The pixel size + Maximum pixel size in term of geospatial extent per pixel (resolution) that + a raster should have to be selected. The pixel size is evaluated after reprojection of its extent to the target SRS defined by :option:`-t_srs`. @@ -141,7 +143,10 @@ tileindex, or as input for the :ref:`GTI ` driver. specified here, unless :option:`-write_absolute_path` option is also used. Starting with GDAL 3.9, this can also be a directory name. :option:`-recursive` - might also be used to recursve down to sub-directories. + can also be used to recurse down to sub-directories. + + It is also possible to use the generic option ``--optfile filelist.txt`` + to specify a list of source files. Options specific to use by the GDAL GTI driver @@ -186,7 +191,7 @@ specified. Target resolution in SRS unit per pixel. - Written in the XML Virtutual Tile Index if :option:`-gti_filename` + Written in the XML Virtual Tile Index if :option:`-gti_filename` is specified, or as ``RESX`` and ``RESY`` layer metadata items for formats that support layer metadata. @@ -196,7 +201,7 @@ specified. Target extent in SRS unit. - Written in the XML Virtutual Tile Index if :option:`-gti_filename` + Written in the XML Virtual Tile Index if :option:`-gti_filename` is specified, or as ``MINX``, ``MINY``, ``MAXX`` and ``MAXY`` layer metadata items for formats that support layer metadata. @@ -208,7 +213,7 @@ specified. ``Int16``, ``UInt32``, ``Int32``, ``UInt64``, ``Int64``, ``Float32``, ``Float64``, ``CInt16``, ``CInt32``, ``CFloat32`` or ``CFloat64`` - Written in the XML Virtutual Tile Index if :option:`-gti_filename` + Written in the XML Virtual Tile Index if :option:`-gti_filename` is specified, or as ``DATA_TYPE`` layer metadata item for formats that support layer metadata. @@ -218,7 +223,7 @@ specified. Number of bands of the tiles of the tile index. - Written in the XML Virtutual Tile Index if :option:`-gti_filename` + Written in the XML Virtual Tile Index if :option:`-gti_filename` is specified, or as ``BAND_COUNT`` layer metadata item for formats that support layer metadata. @@ -235,7 +240,7 @@ specified. Nodata value of the tiles of the tile index. - Written in the XML Virtutual Tile Index if :option:`-gti_filename` + Written in the XML Virtual Tile Index if :option:`-gti_filename` is specified, or as ``NODATA`` layer metadata item for formats that support layer metadata. @@ -246,7 +251,7 @@ specified. Color interpretation of of the tiles of the tile index: ``red``, ``green``, ``blue``, ``alpha``, ``gray``, ``undefined``. - Written in the XML Virtutual Tile Index if :option:`-gti_filename` + Written in the XML Virtual Tile Index if :option:`-gti_filename` is specified, or as ``COLOR_INTERPRETATION`` layer metadata item for formats that support layer metadata. @@ -256,7 +261,7 @@ specified. Whether tiles in the tile index have a mask band. - Written in the XML Virtutual Tile Index if :option:`-gti_filename` + Written in the XML Virtual Tile Index if :option:`-gti_filename` is specified, or as ``MASK_BAND`` layer metadata item for formats that support layer metadata. diff --git a/frmts/vrt/gdaltileindexdataset.cpp b/frmts/vrt/gdaltileindexdataset.cpp index e80365c34ab6..fbe6fe62b8e4 100644 --- a/frmts/vrt/gdaltileindexdataset.cpp +++ b/frmts/vrt/gdaltileindexdataset.cpp @@ -55,6 +55,7 @@ #endif #endif +// Semantincs of indices of a GeoTransform (double[6]) matrix constexpr int GT_TOPLEFT_X = 0; constexpr int GT_WE_RES = 1; constexpr int GT_ROTATION_PARAM1 = 2; @@ -83,8 +84,8 @@ constexpr const char *MD_SRS = "SRS"; constexpr const char *MD_LOCATION_FIELD = "LOCATION_FIELD"; constexpr const char *MD_SORT_FIELD = "SORT_FIELD"; constexpr const char *MD_SORT_FIELD_ASC = "SORT_FIELD_ASC"; -constexpr const char *MD_BLOCKXSIZE = "BLOCKXSIZE"; -constexpr const char *MD_BLOCKYSIZE = "BLOCKYSIZE"; +constexpr const char *MD_BLOCK_X_SIZE = "BLOCKXSIZE"; +constexpr const char *MD_BLOCK_Y_SIZE = "BLOCKYSIZE"; constexpr const char *MD_MASK_BAND = "MASK_BAND"; constexpr const char *MD_RESAMPLING = "RESAMPLING"; @@ -105,8 +106,8 @@ constexpr const char *const apszTIOptions[] = {MD_RESX, MD_LOCATION_FIELD, MD_SORT_FIELD, MD_SORT_FIELD_ASC, - MD_BLOCKXSIZE, - MD_BLOCKYSIZE, + MD_BLOCK_X_SIZE, + MD_BLOCK_Y_SIZE, MD_MASK_BAND, MD_RESAMPLING}; @@ -158,93 +159,183 @@ static inline bool ENDS_WITH_CI(const char *a, const char *b) class GDALTileIndexBand; class GDALTileIndexDataset final : public GDALPamDataset { + public: + GDALTileIndexDataset(); + ~GDALTileIndexDataset() override; + + bool Open(GDALOpenInfo *poOpenInfo); + + CPLErr FlushCache(bool bAtClosing) override; + + CPLErr GetGeoTransform(double *padfGeoTransform) override; + const OGRSpatialReference *GetSpatialRef() const override; + + CPLErr IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, + int nYSize, void *pData, int nBufXSize, int nBufYSize, + GDALDataType eBufType, int nBandCount, int *panBandMap, + GSpacing nPixelSpace, GSpacing nLineSpace, + GSpacing nBandSpace, + GDALRasterIOExtraArg *psExtraArg) override; + + const char *GetMetadataItem(const char *pszName, + const char *pszDomain) override; + CPLErr SetMetadataItem(const char *pszName, const char *pszValue, + const char *pszDomain) override; + CPLErr SetMetadata(char **papszMD, const char *pszDomain) override; + + void LoadOverviews(); + + std::vector GetSourcesMoreRecentThan(int64_t mTime); + + private: friend class GDALTileIndexBand; + //! Optional GTI XML CPLXMLTreeCloser m_psXMLTree{nullptr}; + + //! Wehther the GTI XML might be modified (by SetMetadata/SetMetadataItem) bool m_bXMLUpdatable = false; + + //! Whether the GTI XML has been modified (by SetMetadata/SetMetadataItem) bool m_bXMLModified = false; + + //! Unique string (without the process) for this tile index. Passed to + //! GDALProxyPoolDataset to ensure that sources are unique for a given owner const std::string m_osUniqueHandle; + + //! Vector dataset with the sources std::unique_ptr m_poVectorDS{}; + + //! Vector layer with the sources OGRLayer *m_poLayer = nullptr; + + //! Geotransform matrix of the tile index std::array m_adfGeoTransform{0, 0, 0, 0, 0, 0}; + + //! Index of the "location" (or alternate name given by user) field + //! (within m_poLayer->GetLayerDefn()), that contain source dataset names. int m_nLocationFieldIndex = -1; + + //! SRS of the tile index. OGRSpatialReference m_oSRS{}; + + //! Cache from dataset name to dataset handle. + //! Note that the dataset objects are ultimately GDALProxyPoolDataset, + //! and that the GDALProxyPoolDataset limits the number of simultaneously + //! opened real datasets (controlled by GDAL_MAX_DATASET_POOL_SIZE). Hence 500 is not too big. lru11::Cache> m_oMapSharedSources{ 500}; + + //! Mask band (e.g. for JPEG compressed + mask band) std::unique_ptr m_poMaskBand{}; + + //! Whether all bands of the tile index have the same data type. bool m_bSameDataType = true; + + //! Whether all bands of the tile index have the same nodata value. bool m_bSameNoData = true; + + //! Minimum X of the current pixel request, in georeferenced units. double m_dfLastMinXFilter = std::numeric_limits::quiet_NaN(); + + //! Minimum Y of the current pixel request, in georeferenced units. double m_dfLastMinYFilter = std::numeric_limits::quiet_NaN(); + + //! Maximum X of the current pixel request, in georeferenced units. double m_dfLastMaxXFilter = std::numeric_limits::quiet_NaN(); + + //! Maximum Y of the current pixel request, in georeferenced units. double m_dfLastMaxYFilter = std::numeric_limits::quiet_NaN(); + + //! Index of the field (within m_poLayer->GetLayerDefn()) used to sort, or -1 if none. int m_nSortFieldIndex = -1; + + //! Whether sorting must be ascending (true) or descending (false). bool m_bSortFieldAsc = true; + + //! Resampling method by default for warping or when a source has not + //! the same resolution as the tile index. std::string m_osResampling = "near"; GDALRIOResampleAlg m_eResampling = GRIORA_NearestNeighbour; + + //! WKT2 representation of the tile index SRS (if needed, typically for on-the-fly warping). std::string m_osWKT{}; + + //! Whether we had to open of the sources at tile index opening. bool m_bScannedOneFeatureAtOpening = false; + + //! Array of overview descriptors. + //! Each descriptor is a tuple (dataset_name, concatenated_open_options, layer_name, overview_factor). std::vector> m_aoOverviewDescriptor{}; + + //! Array of overview datasets. std::vector> m_apoOverviews{}; + + //! Cache of buffers used by VRTComplexSource to avoid memory reallocation. VRTSource::WorkingState m_oWorkingState{}; + //! Structure describing one of the source raster in the tile index. struct SourceDesc { + //! Source dataset name. std::string osName{}; + + //! Source dataset handle. std::shared_ptr poDS{}; + + //! VRTSimpleSource or VRTComplexSource for the source. std::unique_ptr poSource{}; + + //! OGRFeature corresponding to the source in the tile index. std::unique_ptr poFeature{}; + + //! Work buffer containing the value of the mask band for the current pixel query. std::vector abyMask{}; + + //! Whether the source covers the whole area of interest of the current pixel query. bool bCoversWholeAOI = false; + + //! Whether the source has a nodata value at least in one of its band. bool bHasNoData = false; + + //! Whether all bands of the source have the same nodata value. bool bSameNoData = false; + + //! Nodata value of all bands (when bSameNoData == true). double dfSameNoData = 0; + + //! Mask band of the source. GDALRasterBand *poMaskBand = nullptr; }; + //! Array of sources participating to the current pixel query. std::vector m_aoSourceDesc{}; + //! From a source dataset name, return its SourceDesc description structure. bool GetSourceDesc(const std::string &osTileName, SourceDesc &oSourceDesc); + + //! Collect sources corresponding to the georeferenced window of interest, + //! and store them in m_aoSourceDesc[]. bool CollectSources(double dfXOff, double dfYOff, double dfXSize, double dfYSize); + + //! Sort sources according to m_nSortFieldIndex. void SortSourceDesc(); + + //! Whether the output buffer needs to be nodata initialized, or if + //! sources are fully covering it. bool NeedInitBuffer(int nBandCount, const int *panBandMap) const; + + //! Nodata initialze the output buffer. void InitBuffer(void *pData, int nBufXSize, int nBufYSize, GDALDataType eBufType, int nBandCount, const int *panBandMap, GSpacing nPixelSpace, GSpacing nLineSpace, GSpacing nBandSpace) const; + //! Whether m_poVectorDS supports SetMetadata()/SetMetadataItem() bool TileIndexSupportsEditingLayerMetadata() const; CPL_DISALLOW_COPY_ASSIGN(GDALTileIndexDataset) - - public: - GDALTileIndexDataset(); - ~GDALTileIndexDataset() override; - - bool Open(GDALOpenInfo *poOpenInfo); - - CPLErr FlushCache(bool bAtClosing) override; - - CPLErr GetGeoTransform(double *padfGeoTransform) override; - const OGRSpatialReference *GetSpatialRef() const override; - - CPLErr IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, - int nYSize, void *pData, int nBufXSize, int nBufYSize, - GDALDataType eBufType, int nBandCount, int *panBandMap, - GSpacing nPixelSpace, GSpacing nLineSpace, - GSpacing nBandSpace, - GDALRasterIOExtraArg *psExtraArg) override; - - const char *GetMetadataItem(const char *pszName, - const char *pszDomain) override; - CPLErr SetMetadataItem(const char *pszName, const char *pszValue, - const char *pszDomain) override; - CPLErr SetMetadata(char **papszMD, const char *pszDomain) override; - - void LoadOverviews(); - - std::vector GetSourcesMoreRecentThan(int64_t mTime); }; /************************************************************************/ @@ -253,22 +344,6 @@ class GDALTileIndexDataset final : public GDALPamDataset class GDALTileIndexBand final : public GDALPamRasterBand { - friend class GDALTileIndexDataset; - - GDALTileIndexDataset *m_poDS = nullptr; - bool m_bNoDataValueSet = false; - double m_dfNoDataValue = 0; - GDALColorInterp m_eColorInterp = GCI_Undefined; - std::string m_osLastLocationInfo{}; - double m_dfScale = std::numeric_limits::quiet_NaN(); - double m_dfOffset = std::numeric_limits::quiet_NaN(); - std::string m_osUnit{}; - CPLStringList m_aosCategoryNames{}; - std::unique_ptr m_poColorTable{}; - std::unique_ptr m_poRAT{}; - - CPL_DISALLOW_COPY_ASSIGN(GDALTileIndexBand) - public: GDALTileIndexBand(GDALTileIndexDataset *poDSIn, int nBandIn, GDALDataType eDT, int nBlockXSizeIn, int nBlockYSizeIn); @@ -369,6 +444,44 @@ class GDALTileIndexBand final : public GDALPamRasterBand CPLErr SetMetadataItem(const char *pszName, const char *pszValue, const char *pszDomain) override; CPLErr SetMetadata(char **papszMD, const char *pszDomain) override; + + private: + friend class GDALTileIndexDataset; + + //! Dataset that owns this band. + GDALTileIndexDataset *m_poDS = nullptr; + + //! Whether a nodata value is set to this band. + bool m_bNoDataValueSet = false; + + //! Nodata value. + double m_dfNoDataValue = 0; + + //! Color interpretation. + GDALColorInterp m_eColorInterp = GCI_Undefined; + + //! Cached value for GetMetadataItem("Pixel_X_Y", "LocationInfo"). + std::string m_osLastLocationInfo{}; + + //! Scale value (returned by GetScale()) + double m_dfScale = std::numeric_limits::quiet_NaN(); + + //! Offset value (returned by GetOffset()) + double m_dfOffset = std::numeric_limits::quiet_NaN(); + + //! Unit type (returned by GetUnitType()). + std::string m_osUnit{}; + + //! Category names (returned by GetCategoryNames()). + CPLStringList m_aosCategoryNames{}; + + //! Color table (returned by GetColorTable()). + std::unique_ptr m_poColorTable{}; + + //! Raster attribute table (returned by GetDefaultRAT()). + std::unique_ptr m_poRAT{}; + + CPL_DISALLOW_COPY_ASSIGN(GDALTileIndexBand) }; /************************************************************************/ @@ -423,6 +536,44 @@ static std::string GetAbsoluteFileName(const char *pszTileName, return pszTileName; } +/************************************************************************/ +/* GTIDoPaletteExpansionIfNeeded() */ +/************************************************************************/ + +//! Do palette -> RGB(A) expansion +static bool +GTIDoPaletteExpansionIfNeeded(std::shared_ptr &poTileDS, + int nBandCount) +{ + if (poTileDS->GetRasterCount() == 1 && + (nBandCount == 3 || nBandCount == 4) && + poTileDS->GetRasterBand(1)->GetColorTable() != nullptr) + { + + CPLStringList aosOptions; + aosOptions.AddString("-of"); + aosOptions.AddString("VRT"); + + aosOptions.AddString("-expand"); + aosOptions.AddString(nBandCount == 3 ? "rgb" : "rgba"); + + GDALTranslateOptions *psOptions = + GDALTranslateOptionsNew(aosOptions.List(), nullptr); + int bUsageError = false; + auto poRGBDS = std::unique_ptr(GDALDataset::FromHandle( + GDALTranslate("", GDALDataset::ToHandle(poTileDS.get()), psOptions, + &bUsageError))); + GDALTranslateOptionsFree(psOptions); + if (!poRGBDS) + { + return false; + } + + poTileDS.reset(poRGBDS.release()); + } + return true; +} + /************************************************************************/ /* Open() */ /************************************************************************/ @@ -440,6 +591,7 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) } else if (STARTS_WITH(poOpenInfo->pszFilename, "pszFilename)); if (m_psXMLTree == nullptr) return false; @@ -447,6 +599,7 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) else if (strstr(reinterpret_cast(poOpenInfo->pabyHeader), "pszFilename)); if (m_psXMLTree == nullptr) return false; @@ -592,11 +745,21 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) } } + // Get the value of an option. + // The order of lookup is the following one (first to last): + // - open options + // - XML file + // - Layer metadata items. const auto GetOption = [poOpenInfo, psRoot, this](const char *pszItem) { + const char *pszVal = + CSLFetchNameValue(poOpenInfo->papszOpenOptions, pszItem); + if (pszVal) + return pszVal; + if (psRoot) { - const char *pszVal = CPLGetXMLValue(psRoot, pszItem, nullptr); + pszVal = CPLGetXMLValue(psRoot, pszItem, nullptr); if (pszVal) return pszVal; @@ -621,10 +784,6 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) return pszVal; } - const char *pszVal = - CSLFetchNameValue(poOpenInfo->papszOpenOptions, pszItem); - if (pszVal) - return pszVal; return m_poLayer->GetMetadataItem(pszItem); }; @@ -722,8 +881,8 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) if (nCountMinMaxXY != 0 && nCountMinMaxXY != 4) { CPLError(CE_Failure, CPLE_AppDefined, - "None or all of %s, %s, %s and %s should be specified", - MD_MINX, MD_MINY, MD_MAXX, MD_MAXY); + "None or all of %s, %s, %s and %s must be specified", MD_MINX, + MD_MINY, MD_MAXX, MD_MAXY); return false; } @@ -735,7 +894,7 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) if (nCountXSizeYSizeGT != 0 && nCountXSizeYSizeGT != 3) { CPLError(CE_Failure, CPLE_AppDefined, - "None or all of %s, %s, %s should be specified", MD_XSIZE, + "None or all of %s, %s, %s must be specified", MD_XSIZE, MD_YSIZE, MD_GEOTRANSFORM); return false; } @@ -850,51 +1009,18 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) GetAbsoluteFileName(pszTileName, poOpenInfo->pszFilename)); pszTileName = osTileName.c_str(); - struct Releaser - { - void operator()(GDALDataset *poDS) - { - if (poDS) - poDS->Release(); - } - }; - auto poTileDS = std::shared_ptr( GDALDataset::Open(pszTileName, GDAL_OF_RASTER | GDAL_OF_VERBOSE_ERROR), - Releaser()); + GDALDatasetUniquePtrReleaser()); if (!poTileDS) { return false; } - // do palette -> RGB(A) expansion - if (poTileDS->GetRasterCount() == 1 && - (nBandCount == 3 || nBandCount == 4) && - poTileDS->GetRasterBand(1)->GetColorTable() != nullptr) - { - - CPLStringList aosOptions; - aosOptions.AddString("-of"); - aosOptions.AddString("VRT"); - - aosOptions.AddString("-expand"); - aosOptions.AddString(nBandCount == 3 ? "rgb" : "rgba"); - - GDALTranslateOptions *psOptions = - GDALTranslateOptionsNew(aosOptions.List(), nullptr); - int bUsageError = false; - auto poRGBDS = std::unique_ptr(GDALDataset::FromHandle( - GDALTranslate("", GDALDataset::ToHandle(poTileDS.get()), - psOptions, &bUsageError))); - GDALTranslateOptionsFree(psOptions); - if (!poRGBDS) - { - return false; - } - - poTileDS.reset(poRGBDS.release()); - } + // do palette -> RGB(A) expansion if needed + if (!GTIDoPaletteExpansionIfNeeded(poTileDS, nBandCount)) + return false; const int nTileBandCount = poTileDS->GetRasterCount(); for (int i = 0; i < nTileBandCount; ++i) @@ -964,21 +1090,19 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) if (!(adfGeoTransformTile[GT_ROTATION_PARAM1] == 0)) { CPLError(CE_Failure, CPLE_AppDefined, - "3rd value of GeoTransform of %s should be 0", - pszTileName); + "3rd value of GeoTransform of %s must be 0", pszTileName); return false; } if (!(adfGeoTransformTile[GT_ROTATION_PARAM2] == 0)) { CPLError(CE_Failure, CPLE_AppDefined, - "5th value of GeoTransform of %s should be 0", - pszTileName); + "5th value of GeoTransform of %s must be 0", pszTileName); return false; } if (!(adfGeoTransformTile[GT_NS_RES] < 0)) { CPLError(CE_Failure, CPLE_AppDefined, - "6th value of GeoTransform of %s should be < 0", + "6th value of GeoTransform of %s must be < 0", pszTileName); return false; } @@ -997,9 +1121,8 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) "Cannot get layer extent"); return false; } - CPLError( - CE_Warning, CPLE_AppDefined, - "Could get layer extent, but using a potentially slow method"); + CPLError(CE_Warning, CPLE_AppDefined, + "Could get layer extent, but using a slower method"); } const double dfXSize = (sEnvelope.MaxX - sEnvelope.MinX) / dfResX; @@ -1034,7 +1157,7 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) if (nXSize <= 0) { CPLError(CE_Failure, CPLE_AppDefined, - "%s metadata item should be > 0", MD_XSIZE); + "%s metadata item must be > 0", MD_XSIZE); return false; } @@ -1042,7 +1165,7 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) if (nYSize <= 0) { CPLError(CE_Failure, CPLE_AppDefined, - "%s metadata item should be > 0", MD_YSIZE); + "%s metadata item must be > 0", MD_YSIZE); return false; } @@ -1051,7 +1174,7 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) if (aosTokens.size() != 6) { CPLError(CE_Failure, CPLE_AppDefined, - "%s metadata item should be 6 numeric values " + "%s metadata item must be 6 numeric values " "separated with comma", MD_GEOTRANSFORM); return false; @@ -1062,20 +1185,20 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) } if (!(m_adfGeoTransform[GT_ROTATION_PARAM1] == 0)) { - CPLError(CE_Failure, CPLE_AppDefined, "3rd value of %s should be 0", + CPLError(CE_Failure, CPLE_AppDefined, "3rd value of %s must be 0", MD_GEOTRANSFORM); return false; } if (!(m_adfGeoTransform[GT_ROTATION_PARAM2] == 0)) { - CPLError(CE_Failure, CPLE_AppDefined, "5th value of %s should be 0", + CPLError(CE_Failure, CPLE_AppDefined, "5th value of %s must be 0", MD_GEOTRANSFORM); return false; } if (!(m_adfGeoTransform[GT_NS_RES] < 0)) { - CPLError(CE_Failure, CPLE_AppDefined, - "6th value of %s should be < 0", MD_GEOTRANSFORM); + CPLError(CE_Failure, CPLE_AppDefined, "6th value of %s must be < 0", + MD_GEOTRANSFORM); return false; } @@ -1088,14 +1211,14 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) if (!(dfResX > 0)) { CPLError(CE_Failure, CPLE_AppDefined, - "RESX metadata item should be > 0"); + "RESX metadata item must be > 0"); return false; } const double dfResY = CPLAtof(pszResY); if (!(dfResY > 0)) { CPLError(CE_Failure, CPLE_AppDefined, - "RESY metadata item should be > 0"); + "RESY metadata item must be > 0"); return false; } @@ -1118,13 +1241,13 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) if (!(dfMaxX > dfMinX)) { CPLError(CE_Failure, CPLE_AppDefined, - "%s metadata item should be > %s", MD_MAXX, MD_MINX); + "%s metadata item must be > %s", MD_MAXX, MD_MINX); return false; } if (!(dfMaxY > dfMinY)) { CPLError(CE_Failure, CPLE_AppDefined, - "%s metadata item should be > %s", MD_MAXY, MD_MINY); + "%s metadata item must be > %s", MD_MAXY, MD_MINY); return false; } sEnvelope.MinX = dfMinX; @@ -1142,9 +1265,8 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) "Cannot get layer extent"); return false; } - CPLError( - CE_Warning, CPLE_AppDefined, - "Could get layer extent, but using a potentially slow method"); + CPLError(CE_Warning, CPLE_AppDefined, + "Could get layer extent, but using a slower method"); } const double dfXSize = (sEnvelope.MaxX - sEnvelope.MinX) / dfResX; @@ -1219,7 +1341,7 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) else { CPLError(CE_Failure, CPLE_AppDefined, - "Number of values in %s should be 1 or %s", MD_DATA_TYPE, + "Number of values in %s must be 1 or %s", MD_DATA_TYPE, MD_BAND_COUNT); return false; } @@ -1230,9 +1352,11 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) { const auto IsValidNoDataStr = [](const char *pszStr) { - return EQUAL(pszStr, "inf") || EQUAL(pszStr, "-inf") || - EQUAL(pszStr, "nan") || - CPLGetValueType(pszStr) != CPL_VALUE_STRING; + if (EQUAL(pszStr, "inf") || EQUAL(pszStr, "-inf") || + EQUAL(pszStr, "nan")) + return true; + const auto eType = CPLGetValueType(pszStr); + return eType == CPL_VALUE_INTEGER || eType == CPL_VALUE_REAL; }; aNoData.clear(); @@ -1274,7 +1398,7 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) else { CPLError(CE_Failure, CPLE_AppDefined, - "Number of values in %s should be 1 or %s", MD_NODATA, + "Number of values in %s must be 1 or %s", MD_NODATA, MD_BAND_COUNT); return false; } @@ -1318,7 +1442,7 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) else { CPLError(CE_Failure, CPLE_AppDefined, - "Number of values in %s should be 1 or " + "Number of values in %s must be 1 or " "%s", MD_COLOR_INTERPRETATION, MD_BAND_COUNT); return false; @@ -1351,33 +1475,35 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) } int nBlockXSize = 256; - const char *pszBlockXSize = GetOption(MD_BLOCKXSIZE); + const char *pszBlockXSize = GetOption(MD_BLOCK_X_SIZE); if (pszBlockXSize) { nBlockXSize = atoi(pszBlockXSize); if (nBlockXSize <= 0) { - CPLError(CE_Failure, CPLE_AppDefined, "Invalid %s", MD_BLOCKXSIZE); + CPLError(CE_Failure, CPLE_AppDefined, "Invalid %s", + MD_BLOCK_X_SIZE); return false; } } int nBlockYSize = 256; - const char *pszBlockYSize = GetOption(MD_BLOCKYSIZE); + const char *pszBlockYSize = GetOption(MD_BLOCK_Y_SIZE); if (pszBlockYSize) { nBlockYSize = atoi(pszBlockYSize); if (nBlockYSize <= 0) { - CPLError(CE_Failure, CPLE_AppDefined, "Invalid %s", MD_BLOCKYSIZE); + CPLError(CE_Failure, CPLE_AppDefined, "Invalid %s", + MD_BLOCK_Y_SIZE); return false; } } if (nBlockXSize > INT_MAX / nBlockYSize) { - CPLError(CE_Failure, CPLE_AppDefined, "Too big %s * %s", MD_BLOCKXSIZE, - MD_BLOCKYSIZE); + CPLError(CE_Failure, CPLE_AppDefined, "Too big %s * %s", + MD_BLOCK_X_SIZE, MD_BLOCK_Y_SIZE); return false; } @@ -1983,6 +2109,9 @@ CPLErr GDALTileIndexDataset::FlushCache(bool bAtClosing) eErr = CE_Failure; } + // We also clear the cache of opened sources, in case the user would + // change the content of a source and would want the GTI dataset to see + // the refreshed content. m_oMapSharedSources.clear(); m_dfLastMinXFilter = std::numeric_limits::quiet_NaN(); m_dfLastMinYFilter = std::numeric_limits::quiet_NaN(); @@ -2533,51 +2662,19 @@ bool GDALTileIndexDataset::GetSourceDesc(const std::string &osTileName, std::shared_ptr poTileDS; if (!m_oMapSharedSources.tryGet(osTileName, poTileDS)) { - struct Releaser - { - void operator()(GDALDataset *poDS) - { - if (poDS) - poDS->Release(); - } - }; - poTileDS = std::shared_ptr( GDALProxyPoolDataset::Create( osTileName.c_str(), nullptr, GA_ReadOnly, /* bShared = */ true, m_osUniqueHandle.c_str()), - Releaser()); + GDALDatasetUniquePtrReleaser()); if (!poTileDS || poTileDS->GetRasterCount() == 0) { return false; } - // do palette -> RGB(A) expansion - if (poTileDS->GetRasterCount() == 1 && (nBands == 3 || nBands == 4) && - poTileDS->GetRasterBand(1)->GetColorTable() != nullptr) - { - - CPLStringList aosOptions; - aosOptions.AddString("-of"); - aosOptions.AddString("VRT"); - - aosOptions.AddString("-expand"); - aosOptions.AddString(nBands == 3 ? "rgb" : "rgba"); - - GDALTranslateOptions *psOptions = - GDALTranslateOptionsNew(aosOptions.List(), nullptr); - int bUsageError = false; - auto poRGBDS = std::unique_ptr(GDALDataset::FromHandle( - GDALTranslate("", GDALDataset::ToHandle(poTileDS.get()), - psOptions, &bUsageError))); - GDALTranslateOptionsFree(psOptions); - if (!poRGBDS) - { - return false; - } - - poTileDS.reset(poRGBDS.release()); - } + // do palette -> RGB(A) expansion if needed + if (!GTIDoPaletteExpansionIfNeeded(poTileDS, nBands)) + return false; const OGRSpatialReference *poTileSRS; if (!m_oSRS.IsEmpty() && @@ -3296,7 +3393,7 @@ CPLErr GDALTileIndexDataset::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, nOutXOff * nPixelSpace + nOutYOff * nLineSpace); - GByte n255 = 255; + constexpr GByte n255 = 255; for (int iY = 0; iY < nOutYSize; iY++) { GDALCopyWords(&n255, GDT_Byte, 0, diff --git a/gcore/gdal_priv.h b/gcore/gdal_priv.h index 64bc2c4d8d75..7ede63088578 100644 --- a/gcore/gdal_priv.h +++ b/gcore/gdal_priv.h @@ -964,6 +964,17 @@ struct CPL_DLL GDALDatasetUniquePtrDeleter }; //! @endcond +//! @cond Doxygen_Suppress +struct CPL_DLL GDALDatasetUniquePtrReleaser +{ + void operator()(GDALDataset *poDataset) const + { + if (poDataset) + poDataset->Release(); + } +}; +//! @endcond + /** Unique pointer type for GDALDataset. * Appropriate for use on datasets open in non-shared mode and onto which * reference counter has not been manually modified. diff --git a/swig/include/python/gdal_python.i b/swig/include/python/gdal_python.i index adf5aace5b78..f50e54f50ad7 100644 --- a/swig/include/python/gdal_python.i +++ b/swig/include/python/gdal_python.i @@ -3841,9 +3841,9 @@ def TileIndexOptions(options=None, filenameFilter: Pattern that the filenames contained in directories pointed by should follow. '*' and '?' wildcard can be used. String or list of strings. minPixelSize: - Minimum pixel size that a raster should have to be selected. + Minimum pixel size in term of geospatial extent per pixel (resolution) that a raster should have to be selected. maxPixelSize: - Maximum pixel size that a raster should have to be selected. + Maximum pixel size in term of geospatial extent per pixel (resolution) that a raster should have to be selected. format: output format ("ESRI Shapefile", "GPKG", etc...) layerName: From c624f2bacb9e310ce1e04122a63d4eda8e4bed83 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 22 Jan 2024 20:09:11 +0100 Subject: [PATCH 11/11] GTI / GPKG: make them both be able to handle .gti.gpkg files --- autotest/gdrivers/gpkg.py | 13 +++++ autotest/gdrivers/gti.py | 4 +- frmts/vrt/gdaltileindexdataset.cpp | 60 ++++++++++++++++---- ogr/ogrsf_frmts/gpkg/ogrgeopackagedriver.cpp | 17 +++--- 4 files changed, 74 insertions(+), 20 deletions(-) diff --git a/autotest/gdrivers/gpkg.py b/autotest/gdrivers/gpkg.py index 9510ba486616..c4bbe1edc280 100755 --- a/autotest/gdrivers/gpkg.py +++ b/autotest/gdrivers/gpkg.py @@ -4266,3 +4266,16 @@ def test_gpkg_sql_gdal_get_layer_pixel_value(): assert f[0] is None gdal.Unlink(filename) + + +############################################################################### +# Test that we can write and open a .gti.gpkg file + + +def test_gpkg_gti_gpkg_ext(tmp_vsimem): + + filename = str(tmp_vsimem / "test_gpkg_gti_gpkg_ext.gti.gpkg") + gdal.Translate(filename, "data/byte.tif", format="GPKG") + ds = gdal.Open(filename) + assert ds.GetDriver().ShortName == "GPKG" + assert ds.GetRasterBand(1).Checksum() == 4672 diff --git a/autotest/gdrivers/gti.py b/autotest/gdrivers/gti.py index d6757d96e231..19bec2c0ab57 100755 --- a/autotest/gdrivers/gti.py +++ b/autotest/gdrivers/gti.py @@ -170,7 +170,9 @@ def test_gti_cannot_open_index(tmp_vsimem): gdal.VSIFTruncateL(f, 100) gdal.VSIFCloseL(f) - with pytest.raises(Exception, match="not recognized as a supported file format"): + with pytest.raises( + Exception, match="not recognized as being in a supported file format" + ): gdal.Open(index_filename) diff --git a/frmts/vrt/gdaltileindexdataset.cpp b/frmts/vrt/gdaltileindexdataset.cpp index fbe6fe62b8e4..ce3881b46124 100644 --- a/frmts/vrt/gdaltileindexdataset.cpp +++ b/frmts/vrt/gdaltileindexdataset.cpp @@ -596,7 +596,8 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) if (m_psXMLTree == nullptr) return false; } - else if (strstr(reinterpret_cast(poOpenInfo->pabyHeader), + else if (poOpenInfo->nHeaderBytes > 0 && + strstr(reinterpret_cast(poOpenInfo->pabyHeader), "nOpenFlags & GDAL_OF_UPDATE) ? GDAL_OF_UPDATE - : GDAL_OF_READONLY))); - if (!m_poVectorDS) - return false; + if (ENDS_WITH_CI(pszIndexDataset, ".gti.gpkg") && + poOpenInfo->nHeaderBytes >= 100 && + STARTS_WITH(reinterpret_cast(poOpenInfo->pabyHeader), + "SQLite format 3")) + { + const char *const apszAllowedDrivers[] = {"GPKG", nullptr}; + m_poVectorDS.reset(GDALDataset::Open( + std::string("GPKG:\"").append(pszIndexDataset).append("\"").c_str(), + GDAL_OF_VECTOR | GDAL_OF_RASTER | GDAL_OF_VERBOSE_ERROR | + ((poOpenInfo->nOpenFlags & GDAL_OF_UPDATE) ? GDAL_OF_UPDATE + : GDAL_OF_READONLY), + apszAllowedDrivers)); + if (!m_poVectorDS) + { + return false; + } + if (m_poVectorDS->GetLayerCount() == 0 && + (m_poVectorDS->GetRasterCount() != 0 || + m_poVectorDS->GetMetadata("SUBDATASETS") != nullptr)) + { + return false; + } + } + else + { + m_poVectorDS.reset(GDALDataset::Open( + pszIndexDataset, GDAL_OF_VECTOR | GDAL_OF_VERBOSE_ERROR | + ((poOpenInfo->nOpenFlags & GDAL_OF_UPDATE) + ? GDAL_OF_UPDATE + : GDAL_OF_READONLY))); + if (!m_poVectorDS) + { + return false; + } + } if (m_poVectorDS->GetLayerCount() == 0) { @@ -1935,7 +1964,7 @@ CPLErr GDALTileIndexDataset::SetMetadata(char **papszMD, const char *pszDomain) } /************************************************************************/ -/* GDALTileIndexDatasetIdentify() */ +/* GDALTileIndexDatasetIdentify() */ /************************************************************************/ static int GDALTileIndexDatasetIdentify(GDALOpenInfo *poOpenInfo) @@ -1946,11 +1975,20 @@ static int GDALTileIndexDatasetIdentify(GDALOpenInfo *poOpenInfo) if (STARTS_WITH(poOpenInfo->pszFilename, "nHeaderBytes >= 100 && + STARTS_WITH(reinterpret_cast(poOpenInfo->pabyHeader), + "SQLite format 3") && + ENDS_WITH_CI(poOpenInfo->pszFilename, ".gti.gpkg") && + !STARTS_WITH(poOpenInfo->pszFilename, "GPKG:")) + { + // Most likely handled by GTI driver, but we can't be sure + return GDAL_IDENTIFY_UNKNOWN; + } + return poOpenInfo->nHeaderBytes > 0 && (poOpenInfo->nOpenFlags & GDAL_OF_RASTER) != 0 && (strstr(reinterpret_cast(poOpenInfo->pabyHeader), "pszFilename, ".gti.gpkg") || ENDS_WITH_CI(poOpenInfo->pszFilename, ".gti.fgb") || ENDS_WITH_CI(poOpenInfo->pszFilename, ".gti.parquet")); } @@ -1961,7 +1999,7 @@ static int GDALTileIndexDatasetIdentify(GDALOpenInfo *poOpenInfo) static GDALDataset *GDALTileIndexDatasetOpen(GDALOpenInfo *poOpenInfo) { - if (!GDALTileIndexDatasetIdentify(poOpenInfo)) + if (GDALTileIndexDatasetIdentify(poOpenInfo) == GDAL_IDENTIFY_FALSE) return nullptr; auto poDS = std::make_unique(); if (!poDS->Open(poOpenInfo)) diff --git a/ogr/ogrsf_frmts/gpkg/ogrgeopackagedriver.cpp b/ogr/ogrsf_frmts/gpkg/ogrgeopackagedriver.cpp index 3c43b5bf383b..2deb5c99906b 100644 --- a/ogr/ogrsf_frmts/gpkg/ogrgeopackagedriver.cpp +++ b/ogr/ogrsf_frmts/gpkg/ogrgeopackagedriver.cpp @@ -93,13 +93,6 @@ static int OGRGeoPackageDriverIdentify(GDALOpenInfo *poOpenInfo, return FALSE; } - if ((poOpenInfo->nOpenFlags & GDAL_OF_RASTER) != 0 && - ENDS_WITH_CI(poOpenInfo->pszFilename, ".gti.gpkg")) - { - // Handled by GTI driver - return FALSE; - } - /* Requirement 3: File name has to end in "gpkg" */ /* http://opengis.github.io/geopackage/#_file_extension_name */ /* But be tolerant, if the GPKG application id is found, because some */ @@ -251,6 +244,13 @@ static int OGRGeoPackageDriverIdentify(GDALOpenInfo *poOpenInfo, } } + if ((poOpenInfo->nOpenFlags & GDAL_OF_RASTER) != 0 && + ENDS_WITH_CI(poOpenInfo->pszFilename, ".gti.gpkg")) + { + // Most likely handled by GTI driver, but we can't be sure + return GDAL_IDENTIFY_UNKNOWN; + } + return TRUE; } @@ -343,7 +343,8 @@ OGRGeoPackageDriverGetSubdatasetInfo(const char *pszFileName) static GDALDataset *OGRGeoPackageDriverOpen(GDALOpenInfo *poOpenInfo) { std::string osFilenameInGpkgZip; - if (!OGRGeoPackageDriverIdentify(poOpenInfo, osFilenameInGpkgZip, true)) + if (OGRGeoPackageDriverIdentify(poOpenInfo, osFilenameInGpkgZip, true) == + GDAL_IDENTIFY_FALSE) return nullptr; GDALGeoPackageDataset *poDS = new GDALGeoPackageDataset();