Merge PR #376 to solve issue #375 - GFS filter for BRK ETL

#375 add GFS filter for BRK ETL and some general Win updates
nlextract · Mar 27, 2024 · ef42cd9 · ef42cd9
2 parents 13ad9b5 + dea2a7f
commit ef42cd9
Show file tree

Hide file tree

Showing 6 changed files with 23 additions and 99 deletions.
diff --git a/brk/etl/README.md b/brk/etl/README.md
@@ -1,7 +1,7 @@
 # NLExtract - BRK - DKK
 BRK inlezen met [Stetl ETL framework](https://stetl.org).
 
-door: Just van den Broecke en Frank Steggink
+door: Just van den Broecke en Frank Steggink - update 2024
 
 Deze map bevat de ETL configuratie en commando om via [Stetl](http://stetl.org)
 de BRK (Digitale Kadastrale Kaart, BRK-DKK) vanuit de gezipte
@@ -23,9 +23,10 @@ Meer over Stetl: https://stetl.org
 
 ## Downloaden GML
 
-Met het hulpscript [download-brk.sh <doelmap>](download-brk.sh) kan de BRK-DKK eerst gedownload worden naar het doelbestand `dkk-gml-nl-nohist.zip`.
+Met het hulpscript [download-brk.sh](download-brk.sh) kan de BRK-DKK 
+eerst gedownload worden naar het doelbestand `dkk-gml-nl-nohist.zip`.
 
-Onder Windows: [download-brk.cmd <doelmap>](download-brk.cmd)   NEEDS FIXING!!!
+Onder Windows: [download-brk.cmd <doelmap>](download-brk.cmd)   naar `doelmap/dkk-gml-nl-nohist.zip`
 
 NB, soms zijn de gedownloade files 0 bytes. Oorzaak is vreemd HTTPS probleem bij PDOK vermoedelijk. Dit is
 ondervangen door het downloaden met wget in een loop uit te voeren en vervolgens met unzip de inhoud

diff --git a/brk/etl/cached-download.sh b/brk/etl/cached-download.sh
diff --git a/brk/etl/conf/etl-brk.cfg b/brk/etl/conf/etl-brk.cfg
@@ -15,7 +15,7 @@
 
 [etl]
 chains = input_sql_pre|schema_name_filter|output_postgres,
-         input_zip_file|extract_zip_file|handle_orl|handle_pand|output_ogr2ogr,
+         input_zip_file|extract_zip_file|handle_orl|handle_pand|prepare_gfs|output_ogr2ogr,
          input_sql_post|schema_name_filter|output_postgres
 
 # Alternative chains for testing
@@ -100,6 +100,11 @@ keep_parent_feature = True
 child_feature_alt_tag_name = BuildingPart_nummeraanduiding
 parent_feature_geom_name = imgeo:geometrie2dGrondvlak
 
+# Prepare the generic GFS file to optimize the loading speed
+[prepare_gfs]
+class = stetlcomponents.gfspreparationfilter.GfsPreparationFilter
+input_gfs = {gfs_template}
+
 # Below alternative outputs are meant for testing
 # The ogr2ogr command-line, may use any output here, as long as
 # the input is a GML file.

diff --git a/brk/etl/download-brk.cmd b/brk/etl/download-brk.cmd
@@ -7,37 +7,24 @@ if "%1"=="" (
     goto :eof
 )
 
-:: ID's van 32x32 km gebieden om de BRK te downloaden. Let op, de ID's mogen geen voorloopnullen bevatten.
-set blocks=39,44,45,47,48,50,51,54,55,56,57,58,59,60,61,62,63,74,75,96,97,98,99,104,105,106,107,110,111,144,145,147,148,149,150,151,153,155,156,157,158,159,180,181,183,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,224,225,226,227,228,229,230
-
-:: Basis URL
-set base_url="https://www.pdok.nl/download/service/extract.zip?extractname=kadastralekaartv3^&extractset=gml^&excludedtypes=undefined^&history=false^&tiles=%%7B%%22layers%%22%%3A%%5B%%7B%%22aggregateLevel%%22%%3A4%%2C%%22codes%%22%%3A%%5B{block}%%5D%%7D%%5D%%7D"
 
 :: Download
 set doel_dir=%1
 
-:: Ga door alle blokken
-:download
-for /f "tokens=1,* delims=," %%i in ("%blocks%") do set "block=%%i" &set "blocks=%%j"
-
-set target_file="%doel_dir%\brk_%block%.zip"
-set target_url=%base_url:{block}=!block!%
+:: Download URL
+set download_url="https://api.pdok.nl/kadaster/kadastralekaart/download/v5_0/full/predefined/kadastralekaart-gml-nl-nohist.zip"
+set target_file="%doel_dir%\dkk-gml-nl-nohist.zip"
 
 :download_inner
 :: Haal bestand op
-echo Downloading blok %block% ...
+echo Downloading BRK-DKK van %download_url% naar %target_file% ...
 del /f "%target_file%" >nul 2>&1
-wget -O "%target_file%" --no-check-certificate %target_url%
+wget -O "%target_file%" --no-check-certificate %download_url%
 
 :: Controleer of het ZIP-bestand geopend kan worden
 unzip -l %target_file%
 if errorlevel 1 goto download_inner
 
-:: Bestand is gedownload, ga door met het volgende bestand
-echo Download blok %block% OK!
-echo.
-if not "%blocks%"=="" goto download
-
 :: Het downloaden is gereed
 echo Klaar, bestanden in %doel_dir%!
 

diff --git a/brk/etl/etl.cmd b/brk/etl/etl.cmd
@@ -14,11 +14,14 @@ if "%STETL_HOME%"=="" (
     set STETL_HOME=../../externals/stetl
 )
 
+set NLX_HOME=../..
+set BGT_HOME=../../bgt/etl
+
 :: Nodig voor imports
 if "%PYTHONPATH%"=="" (
-    set PYTHONPATH=%STETL_HOME%
+    set PYTHONPATH=%BGT_HOME%;%NLX_HOME%;%STETL_HOME%
 ) else (
-    set PYTHONPATH=%STETL_HOME%;%PYTHONPATH%
+    set PYTHONPATH==%BGT_HOME%;%NLX_HOME%;%STETL_HOME%;%PYTHONPATH%
 )
 
 :: Default argumenten/opties

diff --git a/brk/etl/etl.sh b/brk/etl/etl.sh
@@ -12,19 +12,18 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 pushd $DIR >/dev/null
 
 NLX_HOME=../..
+BGT_HOME=../../bgt/etl
 
 # Gebruik Stetl meegeleverd met NLExtract (kan in theorie ook Stetl via pip install stetl zijn)
 if [ -z "$STETL_HOME" ]; then
   STETL_HOME=../../externals/stetl
 fi
 
-BGT_HOME=../../bgt/etl
-
 # Nodig voor imports
 if [ -z "$PYTHONPATH" ]; then
-  export PYTHONPATH=$STETL_HOME:${BGT_HOME}
+  export PYTHONPATH=$STETL_HOME:$BGT_HOME:$NLX_HOME:.
 else
-  export PYTHONPATH=$STETL_HOME:${BGT_HOME}:$PYTHONPATH
+  export PYTHONPATH=$STETL_HOME:$BGT_HOME:$NLX_HOME:.:$PYTHONPATH
 fi
 
 # Default arguments/options