Merge pull request #4 from OSGeo/master

refresh
OSGeo · Oct 25, 2020 · fd6ead2 · fd6ead2
2 parents a1b2b0c + cba2f92
commit fd6ead2
Show file tree

Hide file tree

Showing 535 changed files with 25,087 additions and 12,053 deletions.
diff --git a/.github/workflows/code_checks.yml b/.github/workflows/code_checks.yml
@@ -111,3 +111,19 @@ jobs:
             $FLAKE8 autotest
             $FLAKE8 gdal/swig/python/scripts
             $FLAKE8 gdal/swig/python/samples
+
+  doxygen:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install Requirements
+        run: |
+            sudo apt install doxygen texlive-latex-base
+
+      - name: Run doxygen
+        run: |
+            cd gdal/doc
+            make .doxygen_up_to_date >log.txt 2>&1
+            if grep -i warning log.txt | grep -v -e russian -e brazilian; then echo "Doxygen warnings found" && cat log.txt && /bin/false; else echo "No Doxygen warnings found"; fi
diff --git a/.github/workflows/ubuntu_20.04/build-deps.sh b/.github/workflows/ubuntu_20.04/build-deps.sh
@@ -21,6 +21,7 @@ DEBIAN_FRONTEND=noninteractive apt-get install -y --fix-missing --no-install-rec
     libpq-dev libssl-dev libboost-dev \
     autoconf automake bash-completion libarmadillo-dev \
     libopenexr-dev libheif-dev \
+    libdeflate-dev \
     mono-mcs libmono-system-drawing4.0-cil
 
 # Build likbkea

diff --git a/.github/workflows/validate_xml.yml b/.github/workflows/validate_xml.yml
@@ -0,0 +1,22 @@
+name: Validate XML
+
+on: [push, pull_request]
+
+jobs:
+  validate_xml:
+    runs-on: ubuntu-18.04
+    if: "!contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]')"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install requirements
+        run: |
+          sudo apt update
+          sudo apt install -y libxml2-utils
+
+      - name: Run xmllint checks
+        run: |
+          xmllint --schema ./gdal/data/nitf_spec.xsd ./gdal/data/nitf_spec.xml --noout
+          xmllint --schema ./gdal/data/vdv452.xsd ./gdal/data/vdv452.xml --noout
+          xmllint --schema ./gdal/data/gmlasconf.xsd ./gdal/data/gmlasconf.xml --noout
diff --git a/.github/workflows/windows_build.yml b/.github/workflows/windows_build.yml
@@ -17,6 +17,9 @@ jobs:
             MSVC_VER: 1920
             platform: x86
             PROJ_BRANCH: "7.0"
+            LIBWEBP_URL: "https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-1.0.3-windows-x86.zip"
+            LIBZSTD_URL: "https://github.com/facebook/zstd/releases/download/v1.4.5/zstd-v1.4.5-win32.zip"
+            LIBDEFLATE_URL: "https://github.com/ebiggers/libdeflate/releases/download/v1.6/libdeflate-1.6-windows-i686-bin.zip"
 
     env:
         VS_VERSION: ${{ matrix.VS_VERSION }}
@@ -25,6 +28,9 @@ jobs:
         MSVC_VER: ${{ matrix.MSVC_VER }}
         platform: ${{ matrix.platform }}
         PROJ_BRANCH: "${{ matrix.PROJ_BRANCH }}"
+        LIBWEBP_URL: "${{ matrix.LIBWEBP_URL }}"
+        LIBZSTD_URL: "${{ matrix.LIBZSTD_URL }}"
+        LIBDEFLATE_URL: "${{ matrix.LIBDEFLATE_URL }}"
         APPVEYOR: true # to skip some tests
         PYTHON_VERSION: "3.7.9"
 
@@ -99,10 +105,21 @@ jobs:
             if(-Not (Test-Path -Path downloads)) { mkdir downloads }
             cd downloads
             if(-Not (Test-Path -Path $env:SDK_ZIP )) { Invoke-WebRequest "$env:SDK_URL" -OutFile "$env:SDK_ZIP" }
+            $env:LIBZSTD_ZIP="libzstd.zip"
+            if(-Not (Test-Path -Path $env:LIBZSTD_ZIP -PathType Leaf)) { Invoke-WebRequest "$env:LIBZSTD_URL" -OutFile "$env:LIBZSTD_ZIP" }
+            $env:LIBWEBP_ZIP="libwebp.zip"
+            if(-Not (Test-Path -Path $env:LIBWEBP_ZIP -PathType Leaf)) { Invoke-WebRequest "$env:LIBWEBP_URL" -OutFile "$env:LIBWEBP_ZIP" }
+            $env:LIBDEFLATE_ZIP="libdeflate.zip"
+            if(-Not (Test-Path -Path $env:LIBDEFLATE_ZIP -PathType Leaf)) { Invoke-WebRequest "$env:LIBDEFLATE_URL" -OutFile "$env:LIBDEFLATE_ZIP" }
             cd ..
             mkdir sdk
             cd sdk
             exec { 7z x ..\downloads\$env:SDK_ZIP }
+            cd $env:SDK
+            #exec { 7z x ..\..\downloads\$env:LIBZSTD_ZIP }
+            exec { 7z x ..\..\downloads\$env:LIBWEBP_ZIP }
+            exec { 7z x -y ..\..\downloads\$env:LIBDEFLATE_ZIP }
+            cd ..
             $env:SDK_PREFIX="$env:GITHUB_WORKSPACE\sdk\$env:SDK"
             $env:SDK_INC="$env:SDK_PREFIX\include"
             $env:SDK_LIB="$env:SDK_PREFIX\lib"
@@ -151,6 +168,14 @@ jobs:
             $env:NMAKE_LOCAL+="NETCDF_HAS_NC4 = yes`n"
             $env:NMAKE_LOCAL+="PROJ_INCLUDE=-I$env:GITHUB_WORKSPACE\install-proj\include`n"
             $env:NMAKE_LOCAL+="PROJ_LIBRARY=$env:GITHUB_WORKSPACE\install-proj\lib\proj.lib`n"
+            # For some reason libzstd.dll and libdeflate.dll conflicts at runtime
+            #$env:NMAKE_LOCAL+="ZSTD_CFLAGS = -I`$(SDK_PREFIX)\include`n"
+            #$env:NMAKE_LOCAL+="ZSTD_LIBS = `$(SDK_PREFIX)\dll\libzstd.dll.a`n"
+            $env:NMAKE_LOCAL+="WEBP_ENABLED=YES`n"
+            $env:NMAKE_LOCAL+="WEBP_CFLAGS = -I`$(SDK_PREFIX)\libwebp-1.0.3-windows-x86\include`n"
+            $env:NMAKE_LOCAL+="WEBP_LIBS = /NODEFAULTLIB:libcmt.lib `$(SDK_PREFIX)\libwebp-1.0.3-windows-x86\lib\libwebp.lib`n"
+            $env:NMAKE_LOCAL+="LIBDEFLATE_CFLAGS = -I`$(SDK_PREFIX)`n"
+            $env:NMAKE_LOCAL+="LIBDEFLATE_LIB = `$(SDK_PREFIX)\libdeflate.lib`n"
             $env:NMAKE_LOCAL | Set-Content "$env:GITHUB_WORKSPACE\gdal\nmake.local"
             #
             cd $env:GITHUB_WORKSPACE
@@ -168,7 +193,7 @@ jobs:
             cmake -G $env:VS_VERSION -A $env:CMAKE_ARCHITECTURE .. $env:CMAKE_INSTALL_PREFIX -DPROJ_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -DBUILD_LIBPROJ_SHARED=ON -DCMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake -DBUILD_SHARED_LIBS=ON -DENABLE_CURL=OFF -DENABLE_TIFF=OFF -DBUILD_PROJSYNC=OFF
             exec { cmake --build . --config Release --target install }
             cd $env:GITHUB_WORKSPACE\install-proj\share\proj
-            Invoke-WebRequest "https://download.osgeo.org/proj/proj-datumgrid-1.8.zip" -OutFile "proj-datumgrid-1.8.zip"
+            Invoke-WebRequest "http://download.osgeo.org/proj/proj-datumgrid-1.8.zip" -OutFile "proj-datumgrid-1.8.zip"
             7z x proj-datumgrid-1.8.zip -aoa
             #
             cd $env:GITHUB_WORKSPACE\gdal
@@ -199,7 +224,7 @@ jobs:
             }
             $env:SDK_PREFIX="$env:GITHUB_WORKSPACE\sdk\$env:SDK"
             $env:SDK_BIN="$env:SDK_PREFIX\bin"
-            $env:PATH="$env:GITHUB_WORKSPACE\gdal;$env:GITHUB_WORKSPACE\gdal\apps;$env:SDK_BIN;$env:PATH"
+            $env:PATH="$env:GITHUB_WORKSPACE\gdal;$env:GITHUB_WORKSPACE\gdal\apps;$env:SDK_PREFIX\dll;$env:SDK_PREFIX;$env:SDK_BIN;$env:PATH"
             $env:GDAL_DATA="$env:GITHUB_WORKSPACE\gdal\data"
             $env:DO_NOT_FAIL_ON_RECODE_ERRORS="YES"
             # The ca-bundle.crt file which we could point to is invalid in the current SDK

diff --git a/appveyor.yml b/appveyor.yml
@@ -70,7 +70,7 @@ install:
   - git clone https://github.com/microsoft/vcpkg
   - cd vcpkg
 # Workaround https://github.com/microsoft/vcpkg/issues/11666
-  - if "%VS_VER%" == "2015" git checkout a64dc07690bc8806e717e190f62eb58e198b599c
+#  - if "%VS_VER%" == "2015" git checkout a64dc07690bc8806e717e190f62eb58e198b599c
   - bootstrap-vcpkg.bat
   - set PATH=%CD%;%PATH%
   - cd ..
@@ -167,7 +167,7 @@ install:
       exec { cmake -G $env:VS_VERSION -A $env:CMAKE_ARCHITECTURE .. $env:CMAKE_INSTALL_PREFIX -DPROJ_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -DBUILD_LIBPROJ_SHARED=ON -DCMAKE_TOOLCHAIN_FILE=c:/projects/gdal/vcpkg/scripts/buildsystems/vcpkg.cmake -DBUILD_SHARED_LIBS=ON -DENABLE_CURL=OFF -DENABLE_TIFF=OFF -DBUILD_PROJSYNC=OFF }
       exec { cmake --build . --config Release --target install }
       cd $env:APPVEYOR_BUILD_FOLDER\install-proj\share\proj
-      appveyor DownloadFile https://download.osgeo.org/proj/proj-datumgrid-1.8.zip
+      appveyor DownloadFile http://download.osgeo.org/proj/proj-datumgrid-1.8.zip
       exec { 7z x proj-datumgrid-1.8.zip -aoa }
       #
       cd $env:APPVEYOR_BUILD_FOLDER\gdal

diff --git a/autotest/conftest.py b/autotest/conftest.py
@@ -19,7 +19,8 @@
 # So we skip searching them during test collection.
 collect_ignore = ["kml_generate_test_files.py",
                   "gdrivers/netcdf_cfchecks.py",
-                  "gdrivers/generate_bag.py"]
+                  "gdrivers/generate_bag.py",
+                  "gdrivers/generate_fits.py"]
 
 # we set ECW to not resolve projection and datum strings to get 3.x behavior.
 gdal.SetConfigOption("ECW_DO_NOT_RESOLVE_DATUM_PROJECTION", "YES")

diff --git a/autotest/cpp/Makefile b/autotest/cpp/Makefile
@@ -14,7 +14,7 @@ test check: all
 	make quick_test
 	./testperfcopywords
 
-quick_test: gdal_unit_test testcopywords testclosedondestroydm testthreadcond testvirtualmem testblockcache testblockcachewrite testblockcachelimits testmultithreadedwriting testdestroy bug1488
+quick_test: gdal_unit_test testcopywords testclosedondestroydm testthreadcond testvirtualmem testblockcache testblockcachewrite testblockcachelimits testmultithreadedwriting testdestroy test_osr_set_proj_search_paths bug1488 proj_with_fork
 	./gdal_unit_test
 	./testcopywords
 	./testclosedondestroydm

diff --git a/autotest/cpp/bug1488.cpp b/autotest/cpp/bug1488.cpp
@@ -51,9 +51,9 @@ static int CPL_STDCALL myProgress(double, const char *, void *)
 
 static void CPL_STDCALL myErrorHandler(CPLErr, CPLErrorNum, const char* msg);
 
-static void CPL_STDCALL myErrorHandler(CPLErr eErrClass, CPLErrorNum, const char* msg)
+static void CPL_STDCALL myErrorHandler(CPLErr, CPLErrorNum errorNum, const char* msg)
 {
-    if( eErrClass != CPLE_UserInterrupt && strstr(msg, "User terminated") == nullptr )
+    if( errorNum != CPLE_UserInterrupt && strstr(msg, "User terminated") == nullptr )
     {
         fprintf(stderr, "An error occurred: %s\n", msg);
         fprintf(stderr, "Likely a threading issue !\n");

diff --git a/autotest/cpp/test_cpl.cpp b/autotest/cpp/test_cpl.cpp
@@ -2890,4 +2890,160 @@ namespace tut
         }
     }
 
+    // Test CPLHTTPFetch
+    template<>
+    template<>
+    void object::test<41>()
+    {
+#ifdef HAVE_CURL        
+        CPLStringList oOptions;
+        oOptions.AddNameVlue("FORM_ITEM_COUNT", "5");
+        oOptions.AddNameVlue("FORM_KEY_0", "qqq");
+        oOptions.AddNameVlue("FORM_VALUE_0", "www");
+        CPLHTTPResult *pResult = CPLHTTPFetch("http://example.com", oOptions);
+        ensure_equals(pResult->nStatus, 34);
+        CPLHTTPDestroyResult(pResult);
+        pResult = nullptr;
+        oOptions.Clear();
+
+        oOptions.AddNameVlue("FORM_FILE_PATH", "not_existed");
+        pResult = CPLHTTPFetch("http://example.com", oOptions);
+        ensure_equals(pResult->nStatus, 34);
+        CPLHTTPDestroyResult(pResult);
+
+#endif // HAVE_CURL        
+    }    
+
+    // Test CPLHTTPPushFetchCallback
+    template<>
+    template<>
+    void object::test<42>()
+    {
+        struct myCbkUserDataStruct
+        {
+            CPLString osURL{};
+            CSLConstList papszOptions = nullptr;
+            GDALProgressFunc pfnProgress = nullptr;
+            void *pProgressArg = nullptr;
+            CPLHTTPFetchWriteFunc pfnWrite = nullptr;
+            void *pWriteArg = nullptr;
+        };
+
+        const auto myCbk = [](const char *pszURL,
+                              CSLConstList papszOptions,
+                              GDALProgressFunc pfnProgress,
+                              void *pProgressArg,
+                              CPLHTTPFetchWriteFunc pfnWrite,
+                              void *pWriteArg,
+                              void* pUserData )
+        {
+            myCbkUserDataStruct* pCbkUserData = static_cast<myCbkUserDataStruct*>(pUserData);
+            pCbkUserData->osURL = pszURL;
+            pCbkUserData->papszOptions = papszOptions;
+            pCbkUserData->pfnProgress = pfnProgress;
+            pCbkUserData->pProgressArg = pProgressArg;
+            pCbkUserData->pfnWrite = pfnWrite;
+            pCbkUserData->pWriteArg = pWriteArg;
+            auto psResult = static_cast<CPLHTTPResult*>(CPLCalloc(sizeof(CPLHTTPResult), 1));
+            psResult->nStatus = 123;
+            return psResult;
+        };
+
+        myCbkUserDataStruct userData;
+        ensure( CPLHTTPPushFetchCallback(myCbk, &userData) );
+
+        int progressArg = 0;
+        const auto myWriteCbk = [](void *, size_t, size_t, void *) -> size_t { return 0; };
+        int writeCbkArg = 00;
+
+        CPLStringList aosOptions;
+        GDALProgressFunc pfnProgress = GDALTermProgress;
+        CPLHTTPFetchWriteFunc pfnWriteCbk = myWriteCbk;
+        CPLHTTPResult* pResult = CPLHTTPFetchEx("http://example.com",
+                                                aosOptions.List(),
+                                                pfnProgress,
+                                                &progressArg,
+                                                pfnWriteCbk,
+                                                &writeCbkArg);
+        ensure(pResult != nullptr);
+        ensure_equals(pResult->nStatus, 123);
+        CPLHTTPDestroyResult(pResult);
+
+        ensure( CPLHTTPPopFetchCallback() );
+        CPLPushErrorHandler(CPLQuietErrorHandler);
+        ensure( !CPLHTTPPopFetchCallback() );
+        CPLPopErrorHandler();
+
+        ensure_equals( userData.osURL, std::string("http://example.com") );
+        ensure_equals( userData.papszOptions, aosOptions.List() );
+        ensure_equals( userData.pfnProgress, pfnProgress );
+        ensure_equals( userData.pProgressArg, &progressArg );
+        ensure_equals( userData.pfnWrite, pfnWriteCbk );
+        ensure_equals( userData.pWriteArg, &writeCbkArg );
+    }
+
+    // Test CPLHTTPSetFetchCallback
+    template<>
+    template<>
+    void object::test<43>()
+    {
+        struct myCbkUserDataStruct
+        {
+            CPLString osURL{};
+            CSLConstList papszOptions = nullptr;
+            GDALProgressFunc pfnProgress = nullptr;
+            void *pProgressArg = nullptr;
+            CPLHTTPFetchWriteFunc pfnWrite = nullptr;
+            void *pWriteArg = nullptr;
+        };
+
+        const auto myCbk2 = [](const char *pszURL,
+                              CSLConstList papszOptions,
+                              GDALProgressFunc pfnProgress,
+                              void *pProgressArg,
+                              CPLHTTPFetchWriteFunc pfnWrite,
+                              void *pWriteArg,
+                              void* pUserData )
+        {
+            myCbkUserDataStruct* pCbkUserData = static_cast<myCbkUserDataStruct*>(pUserData);
+            pCbkUserData->osURL = pszURL;
+            pCbkUserData->papszOptions = papszOptions;
+            pCbkUserData->pfnProgress = pfnProgress;
+            pCbkUserData->pProgressArg = pProgressArg;
+            pCbkUserData->pfnWrite = pfnWrite;
+            pCbkUserData->pWriteArg = pWriteArg;
+            auto psResult = static_cast<CPLHTTPResult*>(CPLCalloc(sizeof(CPLHTTPResult), 1));
+            psResult->nStatus = 124;
+            return psResult;
+        };
+        myCbkUserDataStruct userData2;
+        CPLHTTPSetFetchCallback(myCbk2, &userData2);
+
+        int progressArg = 0;
+        const auto myWriteCbk = [](void *, size_t, size_t, void *) -> size_t { return 0; };
+        int writeCbkArg = 00;
+
+        CPLStringList aosOptions;
+        GDALProgressFunc pfnProgress = GDALTermProgress;
+        CPLHTTPFetchWriteFunc pfnWriteCbk = myWriteCbk;
+        CPLHTTPResult* pResult = CPLHTTPFetchEx("http://example.com",
+                                                aosOptions.List(),
+                                                pfnProgress,
+                                                &progressArg,
+                                                pfnWriteCbk,
+                                                &writeCbkArg);
+        ensure(pResult != nullptr);
+        ensure_equals(pResult->nStatus, 124);
+        CPLHTTPDestroyResult(pResult);
+
+        CPLHTTPSetFetchCallback(nullptr, nullptr);
+
+        ensure_equals( userData2.osURL, std::string("http://example.com") );
+        ensure_equals( userData2.papszOptions, aosOptions.List() );
+        ensure_equals( userData2.pfnProgress, pfnProgress );
+        ensure_equals( userData2.pProgressArg, &progressArg );
+        ensure_equals( userData2.pfnWrite, pfnWriteCbk );
+        ensure_equals( userData2.pWriteArg, &writeCbkArg );
+    }
+
 } // namespace tut
diff --git a/autotest/cpp/test_gdal.cpp b/autotest/cpp/test_gdal.cpp
@@ -1361,6 +1361,8 @@ namespace tut
         // TODO investigate what fails exactly
         if( EQUAL(CPLGetConfigOption("GITHUB_WORKFLOW", ""), "MacOS build") )
             return;
+        if( EQUAL(CPLGetConfigOption("APPVEYOR_BUILD_WORKER_IMAGE", ""), "Visual Studio 2015") )
+            return;
 
         {
             auto l = gdal::TileMatrixSet::listPredefinedTileMatrixSets();

diff --git a/autotest/gcore/basic_test.py b/autotest/gcore/basic_test.py
@@ -506,7 +506,7 @@ def test_basic_test_16():
     with gdaltest.error_handler():
         gdal.OpenEx('/vsimem/temp.tif', gdal.OF_UPDATE, open_options=['@NUM_THREADS=INVALID'])
     gdal.Unlink('/vsimem/temp.tif')
-    assert gdal.GetLastErrorMsg() == 'Invalid value for NUM_THREADS: INVALID'
+    assert 'Invalid value for NUM_THREADS: INVALID' in gdal.GetLastErrorMsg()
 
 ###############################################################################
 # Test mix of gdal/ogr.UseExceptions()/DontUseExceptions()