Merge pull request #2 from OSGeo/master

Refresh from OSGeo

.github/workflows/code_checks.yml

@@ -13,7 +13,8 @@ jobs:
 
       - name: Install Requirements
         run: |
-          sudo apt install -y cppcheck libsqlite3-dev ccache
+          sudo apt update
+          sudo apt install -y cppcheck libsqlite3-dev ccache sqlite3
 
       - name: Cache PROJ build
         uses: actions/cache@v2
@@ -46,7 +47,8 @@ jobs:
 
       - name: Install Requirements
         run: |
-          sudo apt install -y cppcheck libsqlite3-dev ccache
+          sudo apt update
+          sudo apt install -y cppcheck libsqlite3-dev ccache sqlite3
 
       - name: Cache PROJ build
         uses: actions/cache@v2
@@ -88,6 +90,9 @@ jobs:
       - name: Detect suspicous char digit zero
         run: ./gdal/scripts/detect_suspicious_char_digit_zero.sh
 
+      - name: Shellcheck
+        run: shellcheck -e SC2086,SC2046 $(find gdal -name '*.sh' -a -not -name ltmain.sh)
+
   flake8:
     runs-on: ubuntu-latest
     steps:

.github/workflows/ubuntu_20.04.yml

@@ -0,0 +1,39 @@
+name: Ubuntu 20.04 build
+
+on: [push, pull_request]
+
+jobs:
+
+  ubuntu_20_04_build:
+    runs-on: ubuntu-latest
+    #if: "!contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]')"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Cache deps
+        uses: actions/cache@v2
+        id: cache-deps
+        with:
+          path: /tmp/docker
+          key: ${{ runner.os }}-cache-ubuntu-20.04-deps-${{ hashFiles('.github/workflows/ubuntu_20.04/build-deps.sh') }}
+
+      - name: Build dependencies
+        if: steps.cache-deps.outputs.cache-hit != 'true'
+        run: |
+            docker build -f .github/workflows/ubuntu_20.04/Dockerfile.deps -t gdal-deps-ubuntu-20.04 .github/workflows/ubuntu_20.04
+            mkdir /tmp/docker
+            docker image save -o /tmp/docker/gdal-deps-ubuntu-20.04.tar gdal-deps-ubuntu-20.04
+
+      - name: Restore dependencies
+        if: steps.cache-deps.outputs.cache-hit == 'true'
+        run: |
+            docker image load -i /tmp/docker/gdal-deps-ubuntu-20.04.tar
+            docker images
+
+      - name: Build
+        run: docker build -f .github/workflows/ubuntu_20.04/Dockerfile -t gdal .
+
+      - name: Run tests
+        # --security-opt seccomp=unconfined, so that the userfaulfd syscall is available
+        run: docker run --security-opt seccomp=unconfined --rm gdal sh -c "cd /build/autotest && pip3 install -U -r requirements.txt && pytest"

.github/workflows/ubuntu_20.04/Dockerfile

@@ -0,0 +1,8 @@
+ARG BASE_IMAGE=gdal-deps-ubuntu-20.04
+
+FROM $BASE_IMAGE as builder
+
+COPY . /build
+COPY .github/workflows/ubuntu_20.04/build.sh /build
+
+RUN /build/build.sh

.github/workflows/ubuntu_20.04/Dockerfile.deps

@@ -0,0 +1,7 @@
+ARG BASE_IMAGE=ubuntu:20.04
+
+FROM $BASE_IMAGE as builder
+
+COPY build-deps.sh /
+
+RUN /build-deps.sh

.github/workflows/ubuntu_20.04/build-deps.sh

@@ -0,0 +1,54 @@
+#!/bin/sh
+
+set -eu
+
+apt-get update -y
+DEBIAN_FRONTEND=noninteractive apt-get install -y --fix-missing --no-install-recommends \
+    software-properties-common build-essential ca-certificates \
+    git make cmake wget unzip libtool automake \
+    zlib1g-dev libsqlite3-dev pkg-config libcurl4-gnutls-dev \
+    libproj-dev libtiff5-dev \
+    libcharls-dev libopenjp2-7-dev libcairo2-dev \
+    python3-dev python3-numpy python3-pip \
+    libpng-dev libjpeg-dev libgif-dev liblzma-dev libgeos-dev \
+    curl libxml2-dev libexpat-dev libxerces-c-dev \
+    libnetcdf-dev libpoppler-dev libpoppler-private-dev \
+    libspatialite-dev swig libhdf4-alt-dev libhdf5-serial-dev \
+    libfreexl-dev unixodbc-dev libwebp-dev libepsilon-dev \
+    liblcms2-2 libpcre3-dev libcrypto++-dev libdap-dev libfyba-dev \
+    libkml-dev libmysqlclient-dev libogdi-dev \
+    libcfitsio-dev openjdk-8-jdk libzstd-dev \
+    libpq-dev libssl-dev libboost-dev \
+    autoconf automake bash-completion libarmadillo-dev \
+    libopenexr-dev libheif-dev
+
+# Build likbkea
+KEA_VERSION=1.4.13
+wget -q https://github.com/ubarsc/kealib/archive/kealib-${KEA_VERSION}.zip \
+    && unzip -q kealib-${KEA_VERSION}.zip \
+    && rm -f kealib-${KEA_VERSION}.zip \
+    && cd kealib-kealib-${KEA_VERSION} \
+    && cmake . -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_INSTALL_PREFIX=/usr -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial \
+        -DHDF5_LIB_PATH=/usr/lib/x86_64-linux-gnu/hdf5/serial -DLIBKEA_WITH_GDAL=OFF \
+    && make -j$(nproc) \
+    && make install \
+    && cd .. \
+    && rm -rf kealib-kealib-${KEA_VERSION}
+
+# Build tiledb
+TILEDB_VERSION=2.0.2
+mkdir tiledb \
+    && wget -q https://github.com/TileDB-Inc/TileDB/archive/${TILEDB_VERSION}.tar.gz -O - \
+        | tar xz -C tiledb --strip-components=1 \
+    && cd tiledb \
+    && mkdir build_cmake \
+    && cd build_cmake \
+    && ../bootstrap --prefix=/usr \
+    && make -j$(nproc) \
+    && make install-tiledb \
+    && cd ../.. \
+    && rm -f tiledeb
+
+# Workaround bug in ogdi packaging
+ln -s /usr/lib/ogdi/libvrf.so /usr/lib

.github/workflows/ubuntu_20.04/build.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+
+set -eu
+
+cd /build
+
+cd gdal
+
+CXXFLAGS="-std=c++17 -O1" ./configure --prefix=/usr \
+    --without-libtool \
+    --with-hide-internal-symbols \
+    --with-jpeg12 \
+    --with-python \
+    --with-poppler \
+    --with-spatialite \
+    --with-mysql \
+    --with-liblzma \
+    --with-webp \
+    --with-epsilon \
+    --with-hdf5 \
+    --with-dods-root=/usr \
+    --with-sosi \
+    --with-libtiff=internal --with-rename-internal-libtiff-symbols \
+    --with-geotiff=internal --with-rename-internal-libgeotiff-symbols \
+    --with-kea=/usr/bin/kea-config \
+    --with-tiledb \
+    --with-crypto
+
+make "-j$(nproc)" USER_DEFS=-Werror
+(cd apps; make test_ogrsf  USER_DEFS=-Werror)
+make install "-j$(nproc)"
+ldconfig
+
+cd ../autotest/cpp
+make "-j$(nproc)" 

.github/workflows/windows_build.yml

@@ -26,7 +26,7 @@ jobs:
         platform: ${{ matrix.platform }}
         PROJ_BRANCH: "${{ matrix.PROJ_BRANCH }}"
         APPVEYOR: true # to skip some tests
-        PYTHON_VERSION: "3.7.7"
+        PYTHON_VERSION: "3.7.8"
 
     steps:
 

.gitignore

@@ -25,5 +25,6 @@ __pycache__/
 CMakeLists.txt
 *.cmake
 
+# NuGet package manager storage
 gdal/packages*
 gdal/share

autotest/alg/contour.py

@@ -28,7 +28,7 @@
 # DEALINGS IN THE SOFTWARE.
 ###############################################################################
 
-import array
+import struct
 import os
 
 
@@ -67,19 +67,19 @@ def test_contour_1():
     ds.SetProjection(wkt)
     ds.SetGeoTransform([1, precision, 0, 50, 0, -precision])
 
-    raw_data = array.array('h', [10 for i in range(int(size / 2))]).tostring()
+    raw_data = struct.pack('h', 10) * int(size / 2)
     for i in range(int(size / 2)):
         ds.WriteRaster(int(size / 4), i + int(size / 4), int(size / 2), 1, raw_data,
                        buf_type=gdal.GDT_Int16,
                        band_list=[1])
 
-    raw_data = array.array('h', [20 for i in range(int(size / 2))]).tostring()
+    raw_data = struct.pack('h', 20) * int(size / 2)
     for i in range(int(size / 4)):
         ds.WriteRaster(int(size / 4) + int(size / 8), i + int(size / 4) + int(size / 8), int(size / 4), 1, raw_data,
                        buf_type=gdal.GDT_Int16,
                        band_list=[1])
 
-    raw_data = array.array('h', [25 for i in range(int(size / 4))]).tostring()
+    raw_data = struct.pack('h', 25) * int(size / 4)
     for i in range(int(size / 8)):
         ds.WriteRaster(int(size / 4) + int(size / 8) + int(size / 16), i + int(size / 4) + int(size / 8) + int(size / 16), int(size / 8), 1, raw_data,
                        buf_type=gdal.GDT_Int16,

autotest/alg/data/3by3_average_with_srcoffset.vrt

@@ -0,0 +1,38 @@
+<VRTDataset rasterXSize="1" rasterYSize="1" subClass="VRTWarpedDataset">
+  <GeoTransform>  3.3333333333333331e-01,  3.3333333333333331e-01,  0.0000000000000000e+00,  6.6666666666666663e-01,  0.0000000000000000e+00, -3.3333333333333331e-01</GeoTransform>
+  <VRTRasterBand dataType="Float64" band="1" subClass="VRTWarpedRasterBand">
+    <NoDataValue>-1.7e+308</NoDataValue>
+    <ColorInterp>Gray</ColorInterp>
+  </VRTRasterBand>
+  <BlockXSize>1</BlockXSize>
+  <BlockYSize>1</BlockYSize>
+  <GDALWarpOptions>
+    <WarpMemoryLimit>6.71089e+07</WarpMemoryLimit>
+    <ResampleAlg>Average</ResampleAlg>
+    <WorkingDataType>Float64</WorkingDataType>
+    <Option name="INIT_DEST">NO_DATA</Option>
+    <Option name="ERROR_OUT_IF_EMPTY_SOURCE_WINDOW">FALSE</Option>
+    <SourceDataset relativeToVRT="1">4by4.tif</SourceDataset>
+    <Transformer>
+      <ApproxTransformer>
+        <MaxError>0.125</MaxError>
+        <BaseTransformer>
+          <GenImgProjTransformer>
+            <SrcGeoTransform>0,0.25,0,1,0,-0.25</SrcGeoTransform>
+            <SrcInvGeoTransform>-0,4,0,4,0,-4</SrcInvGeoTransform>
+            <DstGeoTransform>0.333333333333333315,0.333333333333333315,0,0.66666666666666663,0,-0.333333333333333315</DstGeoTransform>
+            <DstInvGeoTransform>-1,3,0,2,0,-3</DstInvGeoTransform>
+          </GenImgProjTransformer>
+        </BaseTransformer>
+      </ApproxTransformer>
+    </Transformer>
+    <BandList>
+      <BandMapping src="1" dst="1">
+        <SrcNoDataReal>-1.7e+308</SrcNoDataReal>
+        <SrcNoDataImag>0</SrcNoDataImag>
+        <DstNoDataReal>-1.7e+308</DstNoDataReal>
+        <DstNoDataImag>0</DstNoDataImag>
+      </BandMapping>
+    </BandList>
+  </GDALWarpOptions>
+</VRTDataset>

autotest/alg/warp.py

@@ -1332,6 +1332,16 @@ def test_warp_weighted_average():
 
     assert maxdiff <= 1, 'Image too different from reference'
 
+###############################################################################
+# test weighted average, with src offset (fix for #2665)
+
+
+def test_warp_weighted_average_with_srcoffset():
+
+    ds = gdal.Open('data/3by3_average_with_srcoffset.vrt')
+    val = struct.unpack('d', ds.ReadRaster(0, 0, 1, 1))[0]
+    assert val == pytest.approx(8.5, abs=1e-5)
+
 ###############################################################################
 # test sum
 

autotest/cpp/test_osr_set_proj_search_paths.cpp

@@ -61,6 +61,30 @@ static void func2(void*)
     OSRDestroySpatialReference(hSRS);
 }
 
+static void func3(void*)
+{
+    OGRSpatialReferenceH hSRS = OSRNewSpatialReference(nullptr);
+    if( OSRImportFromEPSG(hSRS, 32631) != OGRERR_NONE )
+    {
+        fprintf(stderr, "failure not expected (3)\n");
+        exit(1);
+    }
+
+    // Test cleanup effect
+    OSRCleanup();
+
+    for(int epsg = 32601; epsg <= 32661; epsg++ )
+    {
+        if( OSRImportFromEPSG(hSRS, epsg) != OGRERR_NONE ||
+            OSRImportFromEPSG(hSRS, epsg+100) != OGRERR_NONE )
+        {
+            fprintf(stderr, "failure not expected (4)\n");
+            exit(1);
+        }
+    }
+    OSRDestroySpatialReference(hSRS);
+}
+
 int main()
 {
     auto tokens = OSRGetPROJSearchPaths();
@@ -88,5 +112,16 @@ int main()
     CSLDestroy(tokens);
     OSRCleanup();
 
+    // Test fix for #2744
+    CPLJoinableThread* ahThreads[4];
+    for( int i = 0; i< 4; i++ )
+    {
+        ahThreads[i] = CPLCreateJoinableThread(func3, nullptr);
+    }
+    for( int i = 0; i< 4; i++ )
+    {
+        CPLJoinThread(ahThreads[i]);
+    }
+
     return 0;
 }